From afe6d5008ee7c88d8f4f0b3cdecd5676fec5790f Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 11 Mar 2026 12:40:06 +1100 Subject: [PATCH 001/116] One big edit. --- README.md | 5 + docs/README.md | 6 +- .../for_exotic_py_candidate_inits_maker.py | 7 +- exotic/api/colab.py | 5 +- exotic/api/elca.py | 39 +- exotic/api/ephemeris.py | 41 +- exotic/api/filters.py | 28 + exotic/api/joint_fitter.py | 17 +- exotic/api/ld.py | 12 +- exotic/api/nbody.py | 14 +- exotic/api/nea.py | 93 +- exotic/api/nested_linear_fitter.py | 39 +- exotic/api/output_aavso.py | 12 +- exotic/api/plate_solution.py | 230 +- exotic/api/rv_fitter.py | 17 +- exotic/api/ultranest_utils.py | 265 ++ exotic/exotic.py | 2673 ++++++++++++++--- exotic/exotic_gui.py | 96 +- exotic/inputs.py | 279 +- exotic/output_files.py | 70 +- exotic/plots.py | 174 +- exotic/version.py | 2 +- inits.json | 180 +- requirements.txt | 1 + tests/test_centroid_wcs.py | 245 ++ tests/test_exotic_proper_motion.py | 344 +++ tests/test_inputs.py | 397 +++ tests/test_ld.py | 35 + tests/test_nea_nextastro_fallback.py | 68 + tests/test_nextastro_astrometry.py | 171 ++ tests/test_nextastro_variability.py | 109 + tests/test_output_files.py | 166 + tests/test_plots.py | 48 + tests/test_ultranest_utils.py | 253 ++ 34 files changed, 5479 insertions(+), 662 deletions(-) create mode 100644 exotic/api/ultranest_utils.py create mode 100644 tests/test_centroid_wcs.py create mode 100644 tests/test_exotic_proper_motion.py create mode 100644 tests/test_inputs.py create mode 100644 tests/test_nea_nextastro_fallback.py create mode 100644 tests/test_nextastro_astrometry.py create mode 100644 tests/test_nextastro_variability.py create mode 100644 tests/test_output_files.py create mode 100644 tests/test_plots.py create mode 100644 tests/test_ultranest_utils.py diff --git a/README.md b/README.md index 49a6c7f2..db41351f 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "AAVSO Observer Code (blank if none)": "RTZ", "Secondary Observer Codes (blank if none)": "", + "Observatory Full Title": "", "Observation date": "17-December-2017", "Obs. Latitude": "+32.41638889", @@ -159,6 +160,10 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, + "Fast Aperture Mask (y/n)": true, + "Use target-driven comp selection rather than comp-driven comp selection": "n", + "require_comp_star": "y", + "Pixel Scale (Ex: 5.21 arcsecs/pixel)": null, "Exposure Time (s)": 60.0 diff --git a/docs/README.md b/docs/README.md index 732c61ac..ebe26a15 100644 --- a/docs/README.md +++ b/docs/README.md @@ -109,6 +109,7 @@ The scatter in the residuals of the lightcurve fit is: 0.5414 % - If you do not have any of these calibrations, enter `null` - AAVSO Observer Code - if you do not have one, leave as N/A - Secondary Observer Codes - the AAVSO observer codes of anyone who helped out with your observations; if you do not have one, leave as N/A + - Observatory Full Title - optional full observatory name; if provided, EXOTIC writes it to the AAVSO header as `OBSNAME` - Observation date - the date of your observation in DAY-MONTH-YEAR format - Obs. Latitude - the latitude of your observations, where North is denoted with a + and South is denoted with a - - Obs. Longitude - the longitude of your observations, where East is denoted with a + and West is denoted with a - @@ -167,6 +168,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "AAVSO Observer Code (N/A if none)": "RTZ", "Secondary Observer Codes (N/A if none)": "N/A", + "Observatory Full Title": "", "Observation date": "December 17, 2017", "Obs. Latitude": "+31.68", @@ -211,7 +213,9 @@ Get EXOTIC up and running faster with a json file. Please see the included file "optional_info": { "Pixel Scale (Ex: 5.21 arcsecs/pixel)": null, "Filter Minimum Wavelength (nm)": null, - "Filter Maximum Wavelength (nm)": null + "Filter Maximum Wavelength (nm)": null, + "Fast Aperture Mask (y/n)": true, + "require_comp_star": "y" } } ``` diff --git a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py index fd29bcbb..0fed82ad 100644 --- a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py +++ b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py @@ -538,6 +538,7 @@ def create_inits_file(parameters, file_name): "Directory of Biases": parameters.get("Directory of Biases", None), "AAVSO Observer Code (N/A if none)": parameters.get("AAVSO Observer Code (N/A if none)", "N/A"), "Secondary Observer Codes (N/A if none)": parameters.get("Secondary Observer Codes (N/A if none)", "N/A"), + "Observatory Full Title": parameters.get("Observatory Full Title", ""), "Observation date": parameters.get("Observation date", None), "Obs. Latitude": parameters.get("Obs. Latitude", None), "Obs. Longitude": parameters.get("Obs. Longitude", None), @@ -547,14 +548,14 @@ def create_inits_file(parameters, file_name): "Filter Name (aavso.org/filters)": parameters.get("Filter Name (aavso.org/filters)", None), "Observing Notes": parameters.get("Observing Notes", "N/A"), "Plate Solution? (y/n)": parameters.get("Plate Solution? (y/n)", None), - "Align Images? (y/n)": parameters.get("Align Images? (y/n)", None), "Target Star X & Y Pixel": parameters.get("Target Star X & Y Pixel", None), "Comparison Star(s) X & Y Pixel": parameters.get("Comparison Star(s) X & Y Pixel", None) }, "optional_info": { "Pixel Scale (Ex: 5.21 arcsecs/pixel)": parameters.get("Pixel Scale (Ex: 5.21 arcsecs/pixel)", None), "Filter Minimum Wavelength (nm)": parameters.get("Filter Minimum Wavelength (nm)", None), - "Filter Maximum Wavelength (nm)": parameters.get("Filter Maximum Wavelength (nm)", None) + "Filter Maximum Wavelength (nm)": parameters.get("Filter Maximum Wavelength (nm)", None), + "require_comp_star": parameters.get("require_comp_star", "y") } } # Update the filename to include the planet name @@ -669,7 +670,6 @@ def create_inits_file(parameters, file_name): filter_name = input("Enter filter name: ").strip() observing_notes = input("Enter observing notes (or 'N/A' if none): ").strip() plate_solution = input("Plate solution? (y/n): ").strip() - align_images = input("Align images? (y/n): ").strip() target_star_xy = [int(coord) for coord in input("Enter target star X & Y Pixel (comma separated): ").strip().split(',')] comparison_stars_xy = [ [int(coord) for coord in star.strip().split(',')] @@ -695,7 +695,6 @@ def create_inits_file(parameters, file_name): stored_parameters["Filter Name (aavso.org/filters)"] = filter_name stored_parameters["Observing Notes"] = observing_notes stored_parameters["Plate Solution? (y/n)"] = plate_solution - stored_parameters["Align Images? (y/n)"] = align_images stored_parameters["Target Star X & Y Pixel"] = target_star_xy stored_parameters["Comparison Star(s) X & Y Pixel"] = comparison_stars_xy stored_parameters["Pixel Scale (Ex: 5.21 arcsecs/pixel)"] = pixel_scale diff --git a/exotic/api/colab.py b/exotic/api/colab.py index c7662d0f..b8e23fac 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -359,6 +359,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "AAVSO Observer Code (N/A if none)": "%s", "Secondary Observer Codes (N/A if none)": "%s", + "Observatory Full Title": "", "Observation date": "%s", "Obs. Latitude": "%s", @@ -381,7 +382,9 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "optional_info": { "Pixel Scale (Ex: 5.21 arcsecs/pixel)": null, "Filter Minimum Wavelength (nm)": %s, - "Filter Maximum Wavelength (nm)": %s + "Filter Maximum Wavelength (nm)": %s, + "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "require_comp_star": "y" } } """ % (planetary_params, image_dir, output_dir, flats_dir, darks_dir, biases_dir, diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 093f9cd7..a75b135c 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -42,6 +42,7 @@ from astropy.time import Time import copy from itertools import cycle +import bottleneck as bn import matplotlib.pyplot as plt import numpy as np from pylightcurve.models.exoplanet_lc import transit as pytransit @@ -61,6 +62,11 @@ except ImportError: from .plotting import corner +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + def weightedflux(flux, gw, nearest): return np.sum(flux[nearest] * gw, axis=-1) @@ -127,9 +133,9 @@ def time_bin(time, flux, dt=1. / (60 * 24)): for i in range(bins): mask = (time >= (min(time) + i * dt)) & (time < (min(time) + (i + 1) * dt)) if mask.sum() > 0: - bflux[i] = np.nanmean(flux[mask]) - btime[i] = np.nanmean(time[mask]) - bstds[i] = np.nanstd(flux[mask]) / (mask.sum() ** 0.5) + bflux[i] = bn.nanmean(flux[mask]) + btime[i] = bn.nanmean(time[mask]) + bstds[i] = bn.nanstd(flux[mask]) / (mask.sum() ** 0.5) zmask = (bflux == 0) | (btime == 0) | np.isnan(bflux) | np.isnan(btime) return btime[~zmask], bflux[~zmask], bstds[~zmask] @@ -139,7 +145,7 @@ def binner(arr, n, err=''): if len(err) == 0: ecks = np.pad(arr.astype(float), (0, ((n - arr.size % n) % n)), mode='constant', constant_values=np.NaN).reshape(-1, n) - arr = np.nanmean(ecks, axis=1) + arr = bn.nanmean(ecks, axis=1) return arr else: ecks = np.pad(arr.astype(float), (0, ((n - arr.size % n) % n)), mode='constant', @@ -148,8 +154,8 @@ def binner(arr, n, err=''): -1, n) weights = 1. / (why ** 2.) # Calculate the weighted average - arr = np.nansum(ecks * weights, axis=1) / np.nansum(weights, axis=1) - err = np.array([np.sqrt(1. / np.nansum(1. / (np.array(i) ** 2.))) for i in why]) + arr = bn.nansum(ecks * weights, axis=1) / bn.nansum(weights, axis=1) + err = np.array([np.sqrt(1. / bn.nansum(1. / (np.array(i) ** 2.))) for i in why]) return arr, err @@ -308,11 +314,11 @@ def prior_transform(upars): self.ns_type = 'ultranest' test = ReactiveNestedSampler(freekeys, loglike, prior_transform) - noop = lambda *args, **kwargs: None - if self.verbose is True: - self.results = test.run(max_ncalls=int(self.max_ncalls)) - else: - self.results = test.run(max_ncalls=int(self.max_ncalls), show_status=False, viz_callback=noop) + self.results = run_reactive_sampler( + test, + run_kwargs={"max_ncalls": int(self.max_ncalls)}, + verbose=self.verbose, + ) for i, key in enumerate(freekeys): self.parameters[key] = self.results['maximum_likelihood']['point'][i] @@ -679,11 +685,12 @@ def loglike(pars): #clean_name = self.lc_data[n].get('name', n).replace(' ','_').replace('(','').replace(')','').replace('[','').replace(']','').replace('-','_').split('-')[0] freekeys.append(f"local_{k}_{n}") - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=1e6, show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=1e6, show_status=False, viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(1e6)}, + verbose=self.verbose, + ) self.quantiles = {} self.errors = {} diff --git a/exotic/api/ephemeris.py b/exotic/api/ephemeris.py index d75bb476..312e2114 100644 --- a/exotic/api/ephemeris.py +++ b/exotic/api/ephemeris.py @@ -56,6 +56,11 @@ except ImportError: from .plotting import corner +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + class ephemeris_fitter(object): @@ -111,17 +116,12 @@ def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - # estimate slope and intercept - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=False, - viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(4e5), "min_num_live_points": 420}, + verbose=self.verbose, + ) # alloc data for best fit + error self.errors = {} self.quantiles = {} @@ -705,17 +705,12 @@ def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - # estimate slope and intercept - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=False, - viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(4e5), "min_num_live_points": 420}, + verbose=self.verbose, + ) # alloc data for best fit + error self.errors = {} self.quantiles = {} @@ -1146,4 +1141,4 @@ def plot_triangle(self): for key in nlf.parameters: print(f"Parameter {key} = {nlf.parameters[key]:.2e} +- {nlf.errors[key]:.2e}") - # TODO BIC Values \ No newline at end of file + # TODO BIC Values diff --git a/exotic/api/filters.py b/exotic/api/filters.py index eaeceb75..0976aa18 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -45,6 +45,7 @@ # MObs Clear Filter; Source(s): Martin Fowler "MObs CV": {"name": "CV", "fwhm": ("350.0", "850.0")}, + "ClearV": {"name": "CV", "fwhm": ("350.0", "1000.0")}, # Astrodon CBB; Source(s): George Silvis; https://astrodon.com/products/astrodon-exo-planet-filter/ "Astrodon ExoPlanet-BB": {"name": "CBB", "fwhm": ("500.0", "1000.0")}, @@ -77,6 +78,33 @@ "Clear with blue-blocking": "Astrodon ExoPlanet-BB", "Exop": "Astrodon ExoPlanet-BB", + + # additional short aliases found in FILTER column values + "bu": "Johnson U", + "bb": "Johnson B", + "pb": "Johnson B", + "bv": "Johnson V", + "pg": "Johnson V", + "br": "Johnson R", + "pr": "Johnson R", + "bi": "Johnson I", + "up": "Sloan u", + "gp": "Sloan g", + "rp": "Sloan r", + "ip": "Sloan i", + "zp": "Sloan z", + "su": "Stromgren u", + "sv": "Stromgren v", + "sb": "Stromgren b", + "sy": "Stromgren y", + "hb": "Stromgren Hbw", + "zs": "PanSTARRS z-short", + "clearV": "ClearV", + "clear": "ClearV", + "lum": "ClearV", + "w": "ClearV", + "pl": "ClearV", + "exo": "Astrodon ExoPlanet-BB" } # standard filters w/o precisely defined FWHM values diff --git a/exotic/api/joint_fitter.py b/exotic/api/joint_fitter.py index 035be933..7b30f804 100644 --- a/exotic/api/joint_fitter.py +++ b/exotic/api/joint_fitter.py @@ -56,6 +56,11 @@ except ImportError: from .elca import glc_fitter, lc_fitter +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + AU = const.au.to(u.m).value Mjup = const.M_jup.to(u.kg).value Msun = const.M_sun.to(u.kg).value @@ -421,10 +426,12 @@ def loglike(pars): for k in lfreekeys[n]: freekeys.append(f"local_{n}_{k}") - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=2e5) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=2e5, show_status=self.verbose, viz_callback=self.verbose) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(2e5)}, + verbose=self.verbose, + ) try: self.parameters = deepcopy(self.lc_data[0]['priors']) @@ -886,4 +893,4 @@ def plot_oc_eclipses(self): ax.set_ylabel("Residuals [min]",fontsize=14) ax.grid(True, ls='--') plt.tight_layout() - return fig \ No newline at end of file + return fig diff --git a/exotic/api/ld.py b/exotic/api/ld.py index 5e939f1e..9696ed59 100644 --- a/exotic/api/ld.py +++ b/exotic/api/ld.py @@ -65,7 +65,8 @@ class LimbDarkening: # lookup table: fwhm_lookup references filters irrespective of spacing and punctuation # 1 - combine optimized str lookups in lookup table fwhm_lookup = {k.strip().replace(' ', '').lower(): k for k in fwhm.keys()} - fwhm_lookup.update({k.strip().replace(' ', '').lower(): v for k, v in fwhm_alias.items()}) + for k, v in fwhm_alias.items(): + fwhm_lookup.setdefault(k.strip().replace(' ', '').lower(), v) # 2 - ignore punctuation in lookup table fwhm_lookup = {re.sub(ld_re_punct_p, '', k): v for k, v in fwhm_lookup.items()} # lookup set: filter_desc_nonspecific_lookup_set references descriptions that do not represent a specific filter @@ -143,13 +144,20 @@ def check_standard(self, filter_: dict = None, loose: bool = False, loose_len: i if k == 'name' and filter_[k]: # format 'name' (if exists) to uppercase, no spaces filter_[k] = filter_[k].upper().replace(' ', '') if filter_['filter']: # make matcher by removing spaces, remove punctuation and lowercase + if filter_['filter'] in LimbDarkening.fwhm_alias: + filter_['filter'] = LimbDarkening.fwhm_alias[filter_['filter']] filter_matcher = filter_['filter'].lower().replace(' ', '') filter_matcher = re.sub(ld_re_punct_p, '', filter_matcher) # names that do not represent a specific filter combined into one tuple filter_names_nonspecific = set(LimbDarkening.fwhm_names_nonspecific.keys()) filter_names_nonspecific.update(LimbDarkening.filter_names_undefined) + # prefer explicit all-uppercase abbreviations (e.g. 'SU') before loose lookup aliases + if (filter_['filter'] and filter_['filter'] == filter_['filter'].upper() and + filter_['filter'].strip() not in filter_names_nonspecific): + filter_alias = next((f for f in LimbDarkening.fwhm.values() + if filter_['filter'].strip() == f['name'].strip().upper()), None) # identify defined filters via optimized lookup table - if (filter_matcher and filter_matcher in LimbDarkening.fwhm_lookup and + if (not filter_alias and filter_matcher and filter_matcher in LimbDarkening.fwhm_lookup and filter_matcher not in LimbDarkening.filter_desc_nonspecific_lookup_set): filter_['filter'] = LimbDarkening.fwhm_lookup[filter_matcher] # sets to actual filter reference key for f in LimbDarkening.fwhm.values(): diff --git a/exotic/api/nbody.py b/exotic/api/nbody.py index 068cf0cd..915970a9 100644 --- a/exotic/api/nbody.py +++ b/exotic/api/nbody.py @@ -47,6 +47,7 @@ import matplotlib.pyplot as plt import rebound from exotic.api.plotting import corner +from exotic.api.ultranest_utils import run_reactive_sampler from ultranest import ReactiveNestedSampler from astropy.io import fits from astropy import units as u @@ -598,11 +599,12 @@ def loglike(pars): def prior_transform(upars): return (boundarray[:,0] + bounddiff*upars) - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=1e5) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=1e5, show_status=self.verbose, -viz_callback=self.verbose) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(1e5)}, + verbose=self.verbose, + ) self.errors = {} self.quantiles = {} @@ -752,4 +754,4 @@ def prior_transform(upars): nfit = nbody_fitter(data, nbody_prior, nbody_bounds) # print(nfit.parameters) - # print(nfit.errors) \ No newline at end of file + # print(nfit.errors) diff --git a/exotic/api/nea.py b/exotic/api/nea.py index dbafcf1b..a5203288 100644 --- a/exotic/api/nea.py +++ b/exotic/api/nea.py @@ -47,7 +47,7 @@ import requests import time import urllib.parse -from tenacity import retry, retry_if_exception_type, stop_after_attempt, \ +from tenacity import RetryError, retry, retry_if_exception_type, stop_after_attempt, \ wait_exponential # constants @@ -114,7 +114,14 @@ def planet_info(self, fancy=False): return json.dumps(flabels, indent=4) else: - self.planet, candidate = self._new_scrape(filename="eaConf.json") + try: + self.planet, candidate = self._new_scrape(filename="eaConf.json") + except (RetryError, requests.exceptions.RequestException, ConnectionError): + if not self._load_params_from_nextastro_cache(): + raise + candidate = False + print(f"Successfully found {self.planet} in NextAstro cached NASA Exoplanet Archive parameters!") + return self.planet, candidate, self.pl_dict if not candidate: with open("eaConf.json", "r") as confirmed: @@ -126,6 +133,88 @@ def planet_info(self, fancy=False): return self.planet, candidate, self.pl_dict + @staticmethod + def _extract_value_and_errors(payload): + if not isinstance(payload, dict): + return payload, None, None + + value = payload.get('value') + err_plus = payload.get('errPlus') + err_minus = payload.get('errMinus') + return value, err_plus, err_minus + + @staticmethod + def _negative_error(value): + if value is None: + return None + return -abs(value) + + def _load_params_from_nextastro_cache(self): + if not self.planet: + return False + + endpoint = "https://archive.nextastro.org/api/exoplanet_params" + response = requests.get( + endpoint, + params={'name': self.planet}, + timeout=self.requests_timeout + ) + response.raise_for_status() + payload = response.json() + params = payload.get('params') if isinstance(payload, dict) else None + + if not isinstance(params, dict): + return False + + period, period_ep, period_em = self._extract_value_and_errors(params.get('orbitalPeriodDays')) + midt, midt_ep, midt_em = self._extract_value_and_errors(params.get('midTransitTimeDays')) + rprs, rprs_ep, rprs_em = self._extract_value_and_errors(params.get('rpOverRs')) + ars, ars_ep, ars_em = self._extract_value_and_errors(params.get('aOverRs')) + incl, incl_ep, incl_em = self._extract_value_and_errors(params.get('inclinationDeg')) + teff, teff_ep, teff_em = self._extract_value_and_errors(params.get('starTeffK')) + feh, feh_ep, feh_em = self._extract_value_and_errors(params.get('starFeh')) + logg, logg_ep, logg_em = self._extract_value_and_errors(params.get('starLogg')) + + mapped_data = { + 'pl_name': params.get('name', self.planet), + 'hostname': params.get('hostStarName'), + 'ra': params.get('raDeg'), + 'dec': params.get('decDeg'), + 'pl_orbper': period, + 'pl_orbpererr1': period_ep, + 'pl_orbpererr2': self._negative_error(period_em), + 'pl_tranmid': midt, + 'pl_tranmiderr1': midt_ep, + 'pl_tranmiderr2': self._negative_error(midt_em), + 'pl_ratror': rprs, + 'pl_ratrorerr1': rprs_ep, + 'pl_ratrorerr2': self._negative_error(rprs_em), + 'pl_ratdor': ars, + 'pl_ratdorerr1': ars_ep, + 'pl_ratdorerr2': self._negative_error(ars_em), + 'pl_orbincl': incl, + 'pl_orbinclerr1': incl_ep, + 'pl_orbinclerr2': self._negative_error(incl_em), + 'pl_orbeccen': params.get('eccentricity'), + 'pl_orblper': params.get('argPeriastronDeg'), + 'st_teff': teff, + 'st_tefferr1': teff_ep, + 'st_tefferr2': self._negative_error(teff_em), + 'st_met': feh, + 'st_meterr1': feh_ep, + 'st_meterr2': self._negative_error(feh_em), + 'st_logg': logg, + 'st_loggerr1': logg_ep, + 'st_loggerr2': self._negative_error(logg_em), + 'sy_dist': None, + 'sy_pmra': None, + 'sy_pmdec': None, + } + + self.planet = mapped_data['pl_name'] + self._get_params(mapped_data) + return True + @staticmethod def dataframe_to_jsonfile(dataframe, filename): jsondata = json.loads(dataframe.to_json(orient='table', index=False)) diff --git a/exotic/api/nested_linear_fitter.py b/exotic/api/nested_linear_fitter.py index 677d8b1a..2eeb6abb 100644 --- a/exotic/api/nested_linear_fitter.py +++ b/exotic/api/nested_linear_fitter.py @@ -55,6 +55,11 @@ except ImportError: from .plotting import corner +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + class linear_fitter(object): def __init__(self, data, dataerr, bounds=None, prior=None, labels=None, verbose=True): @@ -104,17 +109,12 @@ def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - # estimate slope and intercept - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=False, - viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(4e5), "min_num_live_points": 420}, + verbose=self.verbose, + ) # alloc data for best fit + error self.errors = {} self.quantiles = {} @@ -677,17 +677,12 @@ def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - # estimate slope and intercept - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=False, - viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(4e5), "min_num_live_points": 420}, + verbose=self.verbose, + ) # alloc data for best fit + error self.errors = {} self.quantiles = {} diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 9e237056..91b5ff31 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -270,8 +270,16 @@ def aavso_dicts(planet_dict, fit, i_dict, durs, ld0, ld1, ld2, ld3): filter_type = { 'name': "I", - 'fwhm': [{'value': 600, 'units': "nm"}, - {'value': 1000, 'units': "nm"}], + 'filter_width': { + 'left_side_wavelength': { + 'value': 600, + 'units': "nm" + }, + 'right_side_wavelength': { + 'value': 1000, + 'units': "nm" + } + }, } results = { diff --git a/exotic/api/plate_solution.py b/exotic/api/plate_solution.py index 68f6be5b..a3f87dc5 100644 --- a/exotic/api/plate_solution.py +++ b/exotic/api/plate_solution.py @@ -35,10 +35,14 @@ # EXOplanet Transit Interpretation Code (EXOTIC) # # NOTE: See companion file version.py for version info. # ########################################################################### # -from astropy.io.fits import PrimaryHDU, getdata, getheader +from astropy.io.fits import Header, PrimaryHDU, getdata, getheader +from astropy.stats import sigma_clipped_stats from json import dumps from pathlib import Path +import numpy as np +from photutils.detection import DAOStarFinder import requests +import time from tenacity import retry, retry_if_exception_type, retry_if_result, \ stop_after_attempt, wait_exponential @@ -46,6 +50,10 @@ _R_MAX_STOPS = 10 _R_MAX_SECS = 37 _RQ_TIMEOUT = 16.0 +_NEXTASTRO_MAX_SOURCES = 200 +_NEXTASTRO_STATUS_MAX_POLLS = 60 +_NEXTASTRO_STATUS_POLL_SEC = 2 +_NEXTASTRO_IN_PROGRESS_STATUSES = {'queued', 'running'} def is_false(value): @@ -59,13 +67,19 @@ def result_if_max_retry_count(retry_state): class PlateSolution: def __init__(self, file=None, directory=None, api_key=None, - api_url='http://nova.astrometry.net/api/'): + api_url='http://nova.astrometry.net/api/', ra=None, dec=None, + pixel_scale=None, radius=2.0, scale_err=25.0): if api_key is None: api_key = {'apikey': 'vfsyxlmdxfryhprq'} self.api_url = api_url self.api_key = api_key self.file = file self.directory = directory + self.ra = ra + self.dec = dec + self.pixel_scale = pixel_scale + self.radius = radius + self.scale_err = scale_err def plate_solution(self): session = self._login() @@ -109,11 +123,29 @@ def _login(self): retry=(retry_if_result(is_false) | retry_if_exception_type(requests.exceptions.RequestException)), retry_error_callback=result_if_max_retry_count) def _upload(self, session): - files = {'file': open(self.file, 'rb')} - headers = {'request-json': dumps({"session": session}), 'allow_commercial_use': 'n', + request_payload = {"session": session} + + if self.ra is not None and self.dec is not None: + request_payload.update({ + "center_ra": float(self.ra), + "center_dec": float(self.dec), + "radius": float(self.radius) + }) + + if self.pixel_scale not in (None, ""): + request_payload.update({ + "scale_units": "arcsecperpix", + "scale_type": "ev", + "scale_est": float(self.pixel_scale), + "scale_err": float(self.scale_err) + }) + + headers = {'request-json': dumps(request_payload), 'allow_commercial_use': 'n', 'allow_modifications': 'n', 'publicly_visible': 'n'} - r = requests.post(self.api_url + 'upload', files=files, data=headers, timeout=_RQ_TIMEOUT) + with open(self.file, 'rb') as image_file: + files = {'file': image_file} + r = requests.post(self.api_url + 'upload', files=files, data=headers, timeout=_RQ_TIMEOUT) if r.json()['status'] == 'success': return r.json()['subid'] @@ -143,7 +175,189 @@ def _job_status(self, job_url, wcs_file, download_url): return False @staticmethod - def fail(error_type): - print("WARNING: After multiple attempts, EXOTIC could not retrieve a plate solution from nova.astrometry.net" - f" due to {error_type}. EXOTIC will continue reducing data without a plate solution.") + def fail(error_type, service_name='nova.astrometry.net'): + print("WARNING: After multiple attempts, EXOTIC could not retrieve a plate solution from " + f"{service_name} due to {error_type}. EXOTIC will continue reducing data without a plate solution.") + return False + + +class NextAstroPlateSolution: + + def __init__(self, file=None, directory=None, api_url='https://astrometry.nextastro.org/', ra=None, dec=None, + pixel_scale=None): + self.api_url = api_url.rstrip('/') + self.file = file + self.directory = directory + self.ra = ra + self.dec = dec + self.pixel_scale = pixel_scale + + def plate_solution(self): + print(f"Using NextAstro astrometry server at {self.api_url} for plate solving.") + source_list = self._generate_source_list() + if not source_list: + return PlateSolution.fail('Source extraction for NextAstro astrometry server', + service_name=f'NextAstro ({self.api_url})') + + request_id = self._submit_solve_request(source_list) + if not request_id: + return PlateSolution.fail('NextAstro solve submission', service_name=f'NextAstro ({self.api_url})') + + wcs_header = self._poll_for_solution(request_id) + if not wcs_header: + return PlateSolution.fail('NextAstro solve status', service_name=f'NextAstro ({self.api_url})') + + wcs_file = Path(self.directory) / "temp" / "wcs.fits" + hdu = PrimaryHDU(data=getdata(filename=self.file), header=wcs_header) + hdu.writeto(wcs_file, overwrite=True) + print("WCS file creation successful.") + return wcs_file + + def _generate_source_list(self): + image_data = np.asarray(getdata(filename=self.file), dtype=float) + if image_data.ndim > 2: + image_data = image_data.squeeze() + + median, _, std = sigma_clipped_stats(image_data, sigma=3.0) + if std <= 0: + std = float(np.nanstd(image_data)) + if std <= 0: + return None + + finder = DAOStarFinder(fwhm=3.0, threshold=3.5 * std) + sources = finder(image_data - median) + + if sources is not None and len(sources) > 0: + bright_sources = self._limit_to_brightest_sources( + x_coords=sources['xcentroid'], + y_coords=sources['ycentroid'], + fluxes=sources['flux'] + ) + if bright_sources is None: + return None + return { + "x": bright_sources["x"], + "y": bright_sources["y"], + "flux": bright_sources["flux"], + "pixel_indexing": "0-based" + } + + return self._fallback_source_list(image_data, median, std) + + + def _fallback_source_list(self, image_data, median, std): + threshold = median + 3.5 * std + candidate_indices = np.argwhere(image_data > threshold) + if candidate_indices.size == 0: + return None + + candidate_fluxes = image_data[candidate_indices[:, 0], candidate_indices[:, 1]] + bright_sources = self._limit_to_brightest_sources( + x_coords=candidate_indices[:, 1], + y_coords=candidate_indices[:, 0], + fluxes=candidate_fluxes + ) + if bright_sources is None: + return None + + return { + "x": bright_sources["x"], + "y": bright_sources["y"], + "flux": bright_sources["flux"], + "pixel_indexing": "0-based" + } + + @staticmethod + def _limit_to_brightest_sources(x_coords, y_coords, fluxes): + fluxes = np.asarray(fluxes, dtype=float) + x_coords = np.asarray(x_coords, dtype=float) + y_coords = np.asarray(y_coords, dtype=float) + + finite_flux_mask = np.isfinite(fluxes) + if not np.any(finite_flux_mask): + return None + + fluxes = fluxes[finite_flux_mask] + x_coords = x_coords[finite_flux_mask] + y_coords = y_coords[finite_flux_mask] + + sorted_indices = np.argsort(fluxes)[::-1][:_NEXTASTRO_MAX_SOURCES] + + return { + "x": x_coords[sorted_indices].tolist(), + "y": y_coords[sorted_indices].tolist(), + "flux": fluxes[sorted_indices].tolist() + } + + def _submit_solve_request(self, source_list): + image_data = getdata(filename=self.file) + payload = { + "sources": source_list, + "image": { + "width": int(image_data.shape[-1]), + "height": int(image_data.shape[-2]), + }, + "options": { + "timeout_sec": 120, + "max_sources": _NEXTASTRO_MAX_SOURCES + } + } + + hints = self._extract_astrometry_hints() + if hints is not None: + payload["hints"] = hints + + print(f"[NextAstro] Solve request payload: {payload}") + response = requests.post(f"{self.api_url}/solve", json=payload, timeout=_RQ_TIMEOUT) + print(f"[NextAstro] Solve response: {response.json()}") + if response.status_code >= 400: + return False + + response_json = response.json() + if response_json.get('status') in {'queued', 'running'}: + return response_json.get('request_id') + return False + + def _extract_astrometry_hints(self): + hints = {} + + if self.ra is not None and self.dec is not None: + hints.update({"ra_deg": self.ra, "dec_deg": self.dec}) + + if self.pixel_scale not in (None, ""): + hints.update({"scale_arcsec_per_pix": float(self.pixel_scale), "scale_tolerance_frac": 0.25}) + + if not hints: + return None + + return hints + + def _poll_for_solution(self, request_id): + latest_status = None + for _ in range(_NEXTASTRO_STATUS_MAX_POLLS): + response = requests.get(f"{self.api_url}/status/{request_id}", timeout=_RQ_TIMEOUT) + if response.status_code >= 400: + print(f"[NextAstro] Status response (HTTP {response.status_code}): {response.json()}") + return False + + response_json = response.json() + status = str(response_json.get('status', '')).lower() + latest_status = response_json.get('status') + if status == 'solved': + print(f"[NextAstro] Status response (solved): {response_json}") + header_dict = response_json.get('solution', {}).get('wcs_header') + if isinstance(header_dict, dict): + return Header(header_dict) + return False + if status == 'failed': + print(f"[NextAstro] Status response (failed): {response_json}") + return False + + if status not in _NEXTASTRO_IN_PROGRESS_STATUSES: + print(f"[NextAstro] Status response (unexpected): {response_json}") + return False + + time.sleep(_NEXTASTRO_STATUS_POLL_SEC) + + print(f"[NextAstro] Polling timed out waiting for terminal status; latest status={latest_status!r}") return False diff --git a/exotic/api/rv_fitter.py b/exotic/api/rv_fitter.py index af4fa742..fd6850e2 100644 --- a/exotic/api/rv_fitter.py +++ b/exotic/api/rv_fitter.py @@ -50,6 +50,11 @@ except ImportError: from .elca import lc_fitter +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + Mjup = const.M_jup.to(u.kg).value Msun = const.M_sun.to(u.kg).value @@ -261,10 +266,12 @@ def loglike(pars): for k in lfreekeys[n]: freekeys.append(f"local_{n}_{k}") - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=6e5) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=6e5, show_status=self.verbose, viz_callback=self.verbose) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(6e5)}, + verbose=self.verbose, + ) self.parameters = {} self.quantiles = {} @@ -593,4 +600,4 @@ def plot_orbit(self): return fig - \ No newline at end of file + diff --git a/exotic/api/ultranest_utils.py b/exotic/api/ultranest_utils.py new file mode 100644 index 00000000..3947de3a --- /dev/null +++ b/exotic/api/ultranest_utils.py @@ -0,0 +1,265 @@ +import logging +import math +import os +import sys +import time +from contextlib import contextmanager + + +_TRUTHY = {"1", "true", "yes", "on"} +DEFAULT_PROGRESS_INTERVAL_SECONDS = 10.0 + + +def _is_enabled(value): + return str(value).strip().lower() in _TRUTHY + + +def supports_ultranest_live_status(stream=None): + """Return True only when rich UltraNest status is explicitly enabled.""" + if _is_enabled(os.environ.get("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "")): + return False + if not _is_enabled(os.environ.get("EXOTIC_ULTRANEST_RICH_PROGRESS", "")): + return False + + if stream is None: + stream = sys.stdout + if stream is None: + return False + + isatty = getattr(stream, "isatty", None) + if not callable(isatty) or not isatty(): + return False + + if _is_enabled(os.environ.get("CI", "")): + return False + + term = str(os.environ.get("TERM", "")).strip().lower() + if term == "dumb": + return False + + return True + + +def supports_ultranest_simple_status(): + """Return True only when simple UltraNest status is explicitly enabled.""" + return _is_enabled(os.environ.get("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "")) + + +def _progress_mode(verbose, stream=None): + if not verbose: + return "silent" + if supports_ultranest_live_status(stream=stream): + return "rich" + if supports_ultranest_simple_status(): + return "simple" + return "simple" + + +@contextmanager +def _mute_ultranest_logging(sampler): + muted = [] + seen = set() + + sampler_logger = getattr(sampler, "logger", None) + if isinstance(sampler_logger, logging.Logger): + muted.append(sampler_logger) + seen.add(id(sampler_logger)) + + for logger in logging.root.manager.loggerDict.values(): + if not isinstance(logger, logging.Logger): + continue + if not logger.name.startswith("ultranest"): + continue + if id(logger) in seen: + continue + muted.append(logger) + seen.add(id(logger)) + + states = [(logger, logger.disabled) for logger in muted] + try: + for logger in muted: + logger.disabled = True + yield + finally: + for logger, disabled in states: + logger.disabled = disabled + + +def _read_float(mapping, *keys): + for key in keys: + if key not in mapping: + continue + try: + return float(mapping[key]) + except (TypeError, ValueError): + continue + return None + + +def _read_int(mapping, *keys): + for key in keys: + if key not in mapping: + continue + try: + return int(mapping[key]) + except (TypeError, ValueError): + continue + return None + + +def _extract_info(args, kwargs): + info = kwargs.get("info") + if isinstance(info, dict): + return info + for item in reversed(args): + if isinstance(item, dict): + return item + return {} + + +class _UltraNestSimpleProgress: + def __init__(self, stream=None, interval_seconds=DEFAULT_PROGRESS_INTERVAL_SECONDS, bar_width=28): + self.stream = stream if stream is not None else sys.stdout + self.interval_seconds = max(float(interval_seconds), 0.0) + self.bar_width = max(int(bar_width), 8) + self.start_time = time.monotonic() + self.last_emit = self.start_time - self.interval_seconds + self.iteration = None + self.evaluations = None + self.progress = 0.0 + + def _write(self, message): + if self.stream is None: + return + self.stream.write(message + "\n") + self.stream.flush() + + @staticmethod + def _progress_from_info(info): + progress = _read_float(info, "progress", "fraction_done") + if progress is not None: + if progress > 1.0: + progress /= 100.0 + return min(max(progress, 0.0), 1.0) + + remainder = _read_float(info, "remainder_fraction", "remaining_fraction", "frac_remain") + if remainder is not None: + if remainder > 1.0: + remainder /= 100.0 + return min(max(1.0 - remainder, 0.0), 1.0) + + logz = _read_float(info, "logz") + logz_remain = _read_float(info, "logz_remain", "logzremain") + if logz is None or logz_remain is None or not math.isfinite(logz_remain): + return None + if not math.isfinite(logz): + return 0.0 + + delta = logz_remain - logz + if delta >= 50: + return 0.0 + if delta <= -50: + return 1.0 + return 1.0 / (1.0 + math.exp(delta)) + + def _elapsed(self): + total = max(int(time.monotonic() - self.start_time), 0) + hours, rem = divmod(total, 3600) + minutes, seconds = divmod(rem, 60) + if hours: + return f"{hours:d}:{minutes:02d}:{seconds:02d}" + return f"{minutes:02d}:{seconds:02d}" + + def _line(self, done=False): + fraction = 1.0 if done else min(max(self.progress, 0.0), 1.0) + filled = int(round(fraction * self.bar_width)) + bar = "#" * filled + "-" * (self.bar_width - filled) + pct = f"{100.0 * fraction:6.2f}%" + iteration = "?" if self.iteration is None else str(self.iteration) + evaluations = "?" if self.evaluations is None else str(self.evaluations) + state = "done" if done else "running" + return ( + f"[ultranest] {state} {pct} [{bar}] " + f"it={iteration} evals={evaluations} elapsed={self._elapsed()}" + ) + + def start(self): + interval_label = f"{self.interval_seconds:g}s" + self._write( + "[ultranest] Using simple progress updates " + f"({interval_label} heartbeat, set EXOTIC_ULTRANEST_RICH_PROGRESS=1 for native status)." + ) + + def update(self, *args, **kwargs): + info = _extract_info(args, kwargs) + if not info: + return + + progress = self._progress_from_info(info) + if progress is not None: + self.progress = progress + + iteration = _read_int(info, "it", "iteration") + if iteration is not None: + self.iteration = iteration + + evaluations = _read_int(info, "ncall", "ncalls", "evals") + if evaluations is not None: + self.evaluations = evaluations + + def maybe_emit(self, force=False): + now = time.monotonic() + if not force and (now - self.last_emit) < self.interval_seconds: + return + self.last_emit = now + self._write(self._line(done=False)) + + def finish(self): + self._write(self._line(done=True)) + + +def run_reactive_sampler( + sampler, + run_kwargs=None, + verbose=True, + stream=None, + interval_seconds=DEFAULT_PROGRESS_INTERVAL_SECONDS, +): + """ + Run an UltraNest sampler with simple text progress by default. + + Set EXOTIC_ULTRANEST_RICH_PROGRESS=1 for UltraNest's native status output. + Use verbose=False to silence all progress updates. + """ + kwargs = {} if run_kwargs is None else dict(run_kwargs) + mode = _progress_mode(verbose=verbose, stream=stream) + + if mode == "silent": + kwargs["show_status"] = False + kwargs["viz_callback"] = False + with _mute_ultranest_logging(sampler): + return sampler.run(**kwargs) + + if mode == "rich": + kwargs.setdefault("show_status", True) + return sampler.run(**kwargs) + + progress = _UltraNestSimpleProgress(stream=stream, interval_seconds=interval_seconds) + upstream_callback = kwargs.get("viz_callback") + + def callback(*args, **callback_kwargs): + progress.update(*args, **callback_kwargs) + progress.maybe_emit(force=False) + if callable(upstream_callback): + upstream_callback(*args, **callback_kwargs) + + kwargs["show_status"] = False + kwargs["viz_callback"] = callback + + progress.start() + try: + with _mute_ultranest_logging(sampler): + result = sampler.run(**kwargs) + finally: + progress.finish() + return result diff --git a/exotic/exotic.py b/exotic/exotic.py index f7001f9d..b0f8cfc9 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -57,8 +57,11 @@ # standard imports import argparse +import json import hashlib -from time import sleep +import os +from concurrent.futures import ProcessPoolExecutor, as_completed +from time import sleep, perf_counter # Image alignment import import astroalign as aa aa.PIXEL_TOL = 1 @@ -74,7 +77,6 @@ from astroquery.gaia import Gaia # UTC to BJD converter import from barycorrpy.utc_tdb import JDUTC_to_BJDTDB -import copy # julian conversion imports import dateutil.parser as dup import imreg_dft as ird @@ -84,6 +86,7 @@ from logging.handlers import TimedRotatingFileHandler from matplotlib.animation import FuncAnimation # Pyplot imports +import bottleneck as bn import matplotlib.pyplot as plt import numpy as np # photometry @@ -93,10 +96,9 @@ import requests # scipy imports from scipy.optimize import least_squares -from scipy.stats import mode from scipy.signal import savgol_filter -from scipy.ndimage import binary_erosion -from skimage.util import view_as_windows +from scipy.ndimage import binary_erosion, gaussian_filter +from skimage.registration import phase_cross_correlation from skimage.transform import SimilarityTransform from skimage.color import rgb2gray # error handling for scraper @@ -117,27 +119,29 @@ except ImportError: # package import from api.ld import LimbDarkening, ld_re_punct_p try: # plate solution - from .api.plate_solution import PlateSolution + from .api.plate_solution import NextAstroPlateSolution, PlateSolution except ImportError: # package import - from api.plate_solution import PlateSolution + from api.plate_solution import NextAstroPlateSolution, PlateSolution try: # nea from .api.nea import NASAExoplanetArchive except ImportError: # package import from api.nea import NASAExoplanetArchive try: # output files - from output_files import OutputFiles, AIDOutputFiles + from output_files import OutputFiles, AIDOutputFiles, save_comp_star_calibration_summary except ImportError: # package import - from .output_files import OutputFiles, AIDOutputFiles + from .output_files import OutputFiles, AIDOutputFiles, save_comp_star_calibration_summary try: from plate_status import PlateStatus except ImportError: from .plate_status import PlateStatus try: # plots from plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ - plot_stellar_variability, plot_variable_residuals + plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ + plot_comp_star_calibration_series, plot_comp_star_suitability except ImportError: # package import from .plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ - plot_stellar_variability, plot_variable_residuals + plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ + plot_comp_star_calibration_series, plot_comp_star_suitability try: # tools from utils import round_to_2, user_input except ImportError: # package import @@ -158,6 +162,8 @@ # logging -- https://docs.python.org/3/library/logging.html log = logging.getLogger(__name__) +_mid_transit_warning_reported = False +RELATIVE_FLUX_MAX = 2.0 def log_info(string, warn=False, error=False): if error: @@ -169,6 +175,100 @@ def log_info(string, warn=False, error=False): log.debug(string) return True + +def log_mid_transit_range_warning_once(array_times, tmid_prior): + global _mid_transit_warning_reported + if _mid_transit_warning_reported: + return + + _mid_transit_warning_reported = True + # Keep this warning in plain black text and show it once per run. + log_info("\nWarning:") + log_info(" Estimated mid-transit time is not within the observations") + log_info(" Check Period & Mid-transit time in inits.json. Make sure the uncertainties are not 0 or Nan.") + log_info(f" obs start:{array_times.min()}") + log_info(f" obs end:{array_times.max()}") + log_info(f" tmid prior:{tmid_prior}\n") + + +def relative_flux_filter_mask(relative_flux, max_relative_flux=RELATIVE_FLUX_MAX): + relative_flux = np.asarray(relative_flux, dtype=float) + return np.isfinite(relative_flux) & np.less_equal(relative_flux, max_relative_flux) + + +def is_fast_aperture_mask_enabled(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'fast', 'center', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'exact', 'off'): + return False + + log_info("Warning: Invalid 'Fast Aperture Mask (y/n)' value; using fast mode.", warn=True) + return True + + +def is_comp_star_required(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off'): + return False + + log_info("Warning: Invalid 'require_comp_star' value; requiring a comparison star.", warn=True) + return True + + +def is_target_driven_comp_selection_enabled(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off'): + return False + + log_info("Warning: Invalid target-driven comparison selection value; using comp-driven selection.", warn=True) + return False + + +def should_ignore_header_wcs(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'Ignore WCS in Header and Do Manual Alignment? (y/n)' value; " + "using header WCS when available.", warn=True) + return False + + # Initialze plate status log plateStatus = PlateStatus(log_info) @@ -179,7 +279,10 @@ def sigma_clip(ogdata, sigma=3, dt=21, po=2): mdata = savgol_filter(ogdata[~nanmask], window_length=dt, polyorder=po) # mdata = median_filter(ogdata[~nanmask], dt) res = ogdata[~nanmask] - mdata - std = np.nanmedian([np.nanstd(np.random.choice(res, 25)) for i in range(100)]) + # Vectorized bootstrap estimate avoids Python-loop overhead in tight runs. + sample_size = min(25, res.size) + bootstrap_samples = np.random.choice(res, size=(100, sample_size), replace=True) + std = bn.nanmedian(bn.nanstd(bootstrap_samples, axis=1)) # std = np.nanstd(res) # biased from large outliers sigmask = np.abs(res) > sigma * std nanmask[~nanmask] = sigmask @@ -187,6 +290,116 @@ def sigma_clip(ogdata, sigma=3, dt=21, po=2): return nanmask +def robust_scatter(data): + values = np.asarray(data, dtype=float) + finite = values[np.isfinite(values)] + if finite.size < 2: + return np.nan + + center = bn.nanmedian(finite) + mad = bn.nanmedian(np.abs(finite - center)) + if np.isfinite(mad) and mad > 0: + return 1.4826 * mad + + scatter = bn.nanstd(finite) + if np.isfinite(scatter) and scatter > 0: + return scatter + + return np.nan + + +def phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max_iters=3): + values = np.asarray(values, dtype=float) + phase = np.asarray(phase, dtype=float) + nanmask = ~np.isfinite(values) | ~np.isfinite(phase) + valid_indices = np.flatnonzero(~nanmask) + + if valid_indices.size < max(min_points, 3): + return nanmask + + phase_valid = phase[valid_indices] + min_phase = np.nanmin(phase_valid) + max_phase = np.nanmax(phase_valid) + if not np.isfinite(min_phase) or not np.isfinite(max_phase) or min_phase == max_phase: + return nanmask + + bin_count = max(1, int(bins)) + edges = np.linspace(min_phase, max_phase, bin_count + 1) + bin_ids = np.searchsorted(edges[1:-1], phase_valid, side='right') + keep_mask = np.ones(valid_indices.size, dtype=bool) + values_valid = values[valid_indices] + + for bin_id in range(bin_count): + local_positions = np.flatnonzero(bin_ids == bin_id) + if local_positions.size < min_points: + continue + + local_keep = np.ones(local_positions.size, dtype=bool) + for _ in range(max_iters): + candidate_values = values_valid[local_positions][local_keep] + if candidate_values.size < min_points: + break + + center = bn.nanmedian(candidate_values) + scatter = robust_scatter(candidate_values) + if not np.isfinite(scatter) or scatter <= 0: + break + + within_limits = np.abs(candidate_values - center) <= sigma * scatter + if np.all(within_limits): + break + + local_keep[np.flatnonzero(local_keep)[~within_limits]] = False + + keep_mask[local_positions] &= local_keep + + nanmask[valid_indices] = ~keep_mask + return nanmask + + +def apply_lightcurve_mask(lightcurve, mask, sort_index=None): + if lightcurve is None: + return + + mask = np.asarray(mask, dtype=bool) + target_length = mask.shape[0] + if sort_index is not None: + sort_index = np.asarray(sort_index) + target_length = sort_index.shape[0] + + array_attrs = ( + 'time', + 'data', + 'airmass', + 'transit', + 'jd_times', + 'phase', + 'residuals', + 'model', + 'detrended', + 'detrendederr', + 'dataerr', + 'airmass_model', + 'wf', + ) + + for attr in array_attrs: + if not hasattr(lightcurve, attr): + continue + + values = getattr(lightcurve, attr) + if values is None: + continue + + array_values = np.asarray(values) + if array_values.ndim == 0 or array_values.shape[0] != target_length: + continue + + if sort_index is not None: + array_values = array_values[sort_index] + setattr(lightcurve, attr, array_values[mask]) + + def exp_offset(hdr, time_unit, exp): """Returns exposure offset (in days) of more than 0 if headers reveals the time was estimated at the start of the exposure rather than the middle @@ -518,9 +731,22 @@ def get_planetary_parameters(candplanetbool, userpdict, pdict=None): def radec_hours_to_degree(ra, dec): while True: try: - ra = ra.replace(' ', '').replace(':', ' ') - dec = dec.replace(' ', '').replace(':', ' ') - c = SkyCoord(ra + ' ' + dec, unit=(u.hourangle, u.deg)) + ra_value = str(ra).strip() + dec_value = str(dec).strip() + + # Accept either sexagesimal RA strings (HH:MM:SS) or decimal RA degrees. + # A decimal-like value with no separators should be treated as degrees. + ra_unit = u.hourangle if any(sep in ra_value for sep in (':', ' ')) else u.deg + + # Declination can be provided as either sexagesimal or decimal degrees. + dec_unit = u.deg + if any(sep in dec_value for sep in (':', ' ')): + dec_value = dec_value.replace(':', ' ') + + if ra_unit is u.hourangle: + ra_value = ra_value.replace(':', ' ') + + c = SkyCoord(ra=ra_value, dec=dec_value, unit=(ra_unit, dec_unit)) return c.ra.degree, c.dec.degree except ValueError: log_info("Error: The format entered for Right Ascension and/or Declination is not correct, " @@ -603,27 +829,32 @@ def nonlinear_ld(ld, info_dict): } ld.check_fwhm(observed_filter) - if check_all_standard_filters(ld, observed_filter): - pass - elif observed_filter['wl_min'] and observed_filter['wl_max']: - custom_range(ld, observed_filter) - ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) - else: - opt = user_input("\nWould you like EXOTIC to calculate your limb darkening parameters " - "with uncertainties? (y/n):", type_=str, values=['y', 'n']) - - if opt == 'y': - opt = user_input("Please enter 1 to use a standard filter or 2 for a customized filter:", - type_=int, values=[1, 2]) - if opt == 1: - observed_filter['filter'] = None - standard_filter(ld, observed_filter) - elif opt == 2: - custom_range(ld, observed_filter) - ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) + if not check_all_standard_filters(ld, observed_filter): + if observed_filter['wl_min'] and observed_filter['wl_max']: + custom_range(ld, observed_filter) + ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) else: - user_entered_ld(ld, observed_filter) - user_entered = True + opt = info_dict.get('ld_uncertainties') + + if isinstance(opt, str): + opt = opt.lower().strip() + + if opt not in ('y', 'n'): + opt = user_input("\nWould you like EXOTIC to calculate your limb darkening parameters " + "with uncertainties? (y/n):", type_=str, values=['y', 'n']) + + if opt == 'y': + opt = user_input("Please enter 1 to use a standard filter or 2 for a customized filter:", + type_=int, values=[1, 2]) + if opt == 1: + observed_filter['filter'] = None + standard_filter(ld, observed_filter) + elif opt == 2: + custom_range(ld, observed_filter) + ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) + else: + user_entered_ld(ld, observed_filter) + user_entered = True if not user_entered: ld.calculate_ld() @@ -668,11 +899,18 @@ def corruption_check(files): plateStatus.fitsFormatError(e) return valid_files -def check_wcs(fits_file, save_directory, plate_opt, rt=False): +def check_wcs(fits_file, save_directory, plate_opt, rt=False, use_nextastro_astrometry=False, + ra=None, dec=None, pixel_scale=None, ignore_header_wcs=False): wcs_file = None if plate_opt == 'y' and not rt: - wcs_file = get_wcs(fits_file, save_directory) + wcs_file = get_wcs(fits_file, save_directory, use_nextastro_astrometry=use_nextastro_astrometry, ra=ra, dec=dec, pixel_scale=pixel_scale) + if ignore_header_wcs: + if wcs_file: + log_info("Ignoring FITS header WCS for alignment and using the legacy image-to-image alignment path.") + else: + log_info("Ignoring FITS header WCS and using the legacy image-to-image alignment path.") + return wcs_file if not wcs_file: if search_wcs(fits_file).is_celestial: log_info("Your FITS files have WCS (World Coordinate System) information in their headers. " @@ -685,19 +923,97 @@ def check_wcs(fits_file, save_directory, plate_opt, rt=False): def search_wcs(file): + header = get_first_image_header(file) + return search_wcs_from_header(header) + + +def search_wcs_from_header(header): with warnings.catch_warnings(): warnings.simplefilter('ignore', category=FITSFixedWarning) - header = fits.getheader(filename=file) return WCS(header) # return WCS(fits.open(file)[('SCI', 1)].header) -def get_wcs(file, directory=""): +def get_first_image_header(file_name): + extension = 0 + header = fits.getheader(filename=file_name, ext=extension) + while header.get('NAXIS', 0) == 0: + extension += 1 + header = fits.getheader(filename=file_name, ext=extension) + return header + + +def evaluate_celestial_wcs_coverage(inputfiles): + missing_wcs_files = [] + for file_name in inputfiles: + try: + image_header = get_first_image_header(file_name) + if not search_wcs_from_header(image_header).is_celestial: + missing_wcs_files.append(str(file_name)) + except Exception: + missing_wcs_files.append(str(file_name)) + + total_files = len(inputfiles) + all_have_celestial_wcs = total_files > 0 and len(missing_wcs_files) == 0 + return all_have_celestial_wcs, missing_wcs_files + + +def should_use_multiprocess_transform_precompute(inputfiles, requested_processes, ignore_header_wcs=False): + if requested_processes is None or requested_processes <= 0: + return False + + if ignore_header_wcs: + log_info("Header WCS ignore override enabled. Keeping multiprocessing transformation precompute.") + return True + + all_have_celestial_wcs, missing_wcs_files = evaluate_celestial_wcs_coverage(inputfiles) + if all_have_celestial_wcs: + log_info("All input FITS files have celestial WCS in their headers. " + "Skipping multiprocessing transformation precompute.") + return False + + total_files = len(inputfiles) + missing_count = len(missing_wcs_files) + log_info(f"WCS precheck: {total_files - missing_count}/{total_files} files have celestial WCS. " + "Keeping multiprocessing transformation precompute for fallback alignment.") + if missing_count > 0: + preview = ", ".join(missing_wcs_files[:3]) + remainder = missing_count - 3 + if remainder > 0: + preview = f"{preview}, ... (+{remainder} more)" + log.debug(f"Files without usable celestial WCS: {preview}") + + return True + + +def get_wcs(file, directory="", use_nextastro_astrometry=False, ra=None, dec=None, pixel_scale=None): + astrometry_service = 'NextAstro astrometry server (https://astrometry.nextastro.org/)' if use_nextastro_astrometry else 'nova.astrometry.net' log_info("\nGetting the plate solution for your imaging file to translate pixel coordinates on the sky. " + f"\nUsing astrometry service: {astrometry_service}." "\nPlease wait....") + + if use_nextastro_astrometry: + print("Contacting NextAstro Astrometry Server") + wcs_file = NextAstroPlateSolution(file=file, directory=directory, ra=ra, dec=dec, pixel_scale=pixel_scale).plate_solution() + if wcs_file: + return wcs_file + + log_info("NextAstro astrometry server did not return a solution; falling back to nova.astrometry.net.") + print("Communication with nova.astrometry.net") + return PlateSolution(file=file, directory=directory, ra=ra, dec=dec, + pixel_scale=pixel_scale).plate_solution() + animate_toggle(True) - wcs_obj = PlateSolution(file=file, directory=directory) - wcs_file = wcs_obj.plate_solution() + wcs_file = PlateSolution(file=file, directory=directory, ra=ra, dec=dec, + pixel_scale=pixel_scale).plate_solution() + if wcs_file: + animate_toggle() + return wcs_file + + log_info("nova.astrometry.net did not return a solution; falling back to NextAstro astrometry server.") + print("Contacting NextAstro Astrometry Server") + wcs_file = NextAstroPlateSolution(file=file, directory=directory, ra=ra, dec=dec, + pixel_scale=pixel_scale).plate_solution() animate_toggle() return wcs_file @@ -716,32 +1032,90 @@ def deg_to_pix(exp_ra, exp_dec, ra_list, dec_list): return np.unravel_index(dist.argmin(), dist.shape) -def check_target_pixel_wcs(input_x_pixel, input_y_pixel, info_dict, ra_list, dec_list, image_data, obs_time): +def project_target_pixel_wcs(exp_ra, exp_dec, ra_list, dec_list, wcs_header=None): + if wcs_header is not None: + try: + x_pixel, y_pixel = WCS(wcs_header).all_world2pix(exp_ra, exp_dec, 0) + x_pixel = float(np.asarray(x_pixel).reshape(-1)[0]) + y_pixel = float(np.asarray(y_pixel).reshape(-1)[0]) + if np.isfinite(x_pixel) and np.isfinite(y_pixel): + return x_pixel, y_pixel + except Exception as exc: + log.debug(f"Direct WCS pixel projection failed; falling back to grid search: {exc}") + + calculated_y_pixel, calculated_x_pixel = deg_to_pix(exp_ra, exp_dec, ra_list, dec_list) + return float(calculated_x_pixel), float(calculated_y_pixel) + + +def pixel_within_image(x_pixel, y_pixel, image_shape, margin=0.0): + height, width = image_shape[:2] + return ( + np.isfinite(x_pixel) + and np.isfinite(y_pixel) + and margin <= x_pixel < (width - margin) + and margin <= y_pixel < (height - margin) + ) + + +def any_projected_coord_out_of_frame(coords, image_shape): + for x_pixel, y_pixel in np.asarray(coords, dtype=float): + if not pixel_within_image(x_pixel, y_pixel, image_shape): + return True + return False + + +def check_target_pixel_wcs(input_x_pixel, input_y_pixel, info_dict, ra_list, dec_list, image_data, obs_time, + non_interactive_run=False, wcs_header=None): """ Verify the provided pixel coordinates match the target's right ascension and declination. """ updated_ra, updated_dec = update_coordinates_with_proper_motion(info_dict, obs_time) - calculated_y_pixel, calculated_x_pixel = deg_to_pix(updated_ra, updated_dec, ra_list, dec_list) + calculated_x_pixel, calculated_y_pixel = project_target_pixel_wcs( + updated_ra, updated_dec, ra_list, dec_list, wcs_header=wcs_header + ) + + if not pixel_within_image(calculated_x_pixel, calculated_y_pixel, image_data.shape): + log_info("Warning: WCS-derived target pixel coordinates fall outside the image; " + "keeping the input target coordinates.", warn=True) + return input_x_pixel, input_y_pixel + + centroid_margin = 7.5 + if not pixel_within_image(calculated_x_pixel, calculated_y_pixel, image_data.shape, margin=centroid_margin): + log_info("Warning: WCS-derived target pixel coordinates are too close to the image edge for " + "centroid fitting; keeping the input target coordinates.", warn=True) + return input_x_pixel, input_y_pixel centroid_x, centroid_y, sigma_x, sigma_y = get_psf_parameters(image_data, calculated_x_pixel, calculated_y_pixel) return check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, - calculated_x_pixel, calculated_y_pixel) + calculated_x_pixel, calculated_y_pixel, non_interactive_run=non_interactive_run) def get_psf_parameters(image_data, x_pixel, y_pixel): - psf_data = fit_centroid(image_data, [x_pixel, y_pixel], 0) + try: + psf_data = fit_centroid(image_data, [x_pixel, y_pixel], 0) + except Exception as exc: + log.debug(f"Centroid fit failed while validating WCS target coordinates: {exc}") + return np.nan, np.nan, np.nan, np.nan return psf_data[0], psf_data[1], psf_data[3], psf_data[4] def check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, - calculated_x_pixel, calculated_y_pixel): + calculated_x_pixel, calculated_y_pixel, non_interactive_run=False): while True: try: validate_pixel_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y) return input_x_pixel, input_y_pixel except ValueError: + if non_interactive_run: + if np.isfinite(centroid_x) and np.isfinite(centroid_y): + log_info("Proceeding with WCS-derived centroided target coordinates due to " + "--non-interactive-run.", warn=True) + return centroid_x, centroid_y + log_info("Proceeding with WCS-derived target pixel coordinates due to " + "--non-interactive-run (centroid unavailable).", warn=True) + return calculated_x_pixel, calculated_y_pixel new_x_pixel, new_y_pixel = prompt_user_for_coordinates(input_x_pixel, input_y_pixel, calculated_x_pixel, calculated_y_pixel) if new_x_pixel == input_x_pixel and new_y_pixel == input_y_pixel: @@ -848,9 +1222,11 @@ def vsx_auid(ra, dec, radius=0.01, maglimit=14): @retry(stop=stop_after_delay(30)) def vsx_variable(ra, dec, radius=0.01, maglimit=14): + default_vsx_error = None try: url = f"https://www.aavso.org/vsx/index.php?view=api.list&ra={ra}&dec={dec}&radius={radius}&tomag={maglimit}&format=json" - result = requests.get(url) + result = requests.get(url, timeout=30) + result.raise_for_status() var = result.json()['VSXObjects']['VSXObject'][0]['Category'] if var.lower() == "variable": @@ -861,6 +1237,16 @@ def vsx_variable(ra, dec, radius=0.01, maglimit=14): f"and will be removed from reduction.", warn=True) return True return False + except Exception as err: + default_vsx_error = err + + try: + log_info(f"\nDefault VSX request failed ({default_vsx_error}); falling back to NextAstro VSX server.", warn=True) + fallback_result = nextastro_variability_test([(ra, dec)]) + is_variable = bool(fallback_result[0]) + if is_variable: + log_info("\nNextAstro VSX fallback flagged this star as variable and it will be removed from reduction.", warn=True) + return is_variable except Exception: return False @@ -871,7 +1257,50 @@ def build_comp_ra_dec(ra_wcs, dec_wcs, comp_stars): dec_wcs[int(comp_star[1])][int(comp_star[0])]]) return comp_ra_dec -def check_for_variable_stars(ra_wcs, dec_wcs, comp_stars): + +@retry(stop=stop_after_delay(30)) +def nextastro_variability_test(comp_ra_dec): + api_url = 'https://photometry.nextastro.org/variability_test' + + payload = [{'ra': float(ra), 'dec': float(dec)} for ra, dec in comp_ra_dec] + log_info(f"NextAstro variability request JSON: {json.dumps(payload)}") + result = requests.post(api_url, json=payload, timeout=30) + if result.status_code != 200: + raise RuntimeError(f"NextAstro variability server returned HTTP {result.status_code}.") + + body = result.json() + log_info(f"NextAstro variability response JSON: {json.dumps(body)}") + if not isinstance(body, list) or len(body) != len(payload): + raise RuntimeError("NextAstro variability server returned an unexpected response format.") + + variability_flags = [] + for index, star in enumerate(body): + is_in_vsx = int(star.get('is_in_vsx', 0)) + if is_in_vsx not in [0, 1]: + raise RuntimeError(f"Unexpected is_in_vsx value ({is_in_vsx}) for star index {index}.") + variability_flags.append(bool(is_in_vsx)) + + return variability_flags + + +def check_for_variable_stars(ra_wcs, dec_wcs, comp_stars, use_nextastro_variability_server=False): + if use_nextastro_variability_server and comp_stars: + try: + log_info("\nChecking for variability using NextAstro variability server.") + comp_ra_dec = build_comp_ra_dec(ra_wcs, dec_wcs, comp_stars) + variability_flags = nextastro_variability_test(comp_ra_dec) + + for i, (comp_star, is_variable) in enumerate(zip(comp_stars[:], variability_flags)): + log_info(f"\nChecking for variability in Comparison Star #{i + 1}:" + f"\n\tPixel X: {comp_star[0]} Pixel Y: {comp_star[1]}" + f"\n\tNextAstro flagged variable: {is_variable}") + if is_variable: + comp_stars.remove(comp_star) + return + except Exception as e: + log_info(f"\nWarning: NextAstro variability server check failed ({e}). " + "Falling back to individual VSX variability checks.", warn=True) + for i, comp_star in enumerate(comp_stars[:]): ra = ra_wcs[int(comp_star[1])][int(comp_star[0])] dec = dec_wcs[int(comp_star[1])][int(comp_star[0])] @@ -1048,58 +1477,369 @@ def check_comp_star_exists(user_stars, vsp_star, tol=10): return False, vsp_star + +TRANSFORM_TIMING_STAGES = [ + 'astroalign_direct', + 'fft_translation', + 'astroalign_filtered', + 'astroalign_mask', + 'imreg_dft', +] + +_TRANSFORM_TIMING_STATS = { + stage: {'count': 0, 'success': 0, 'total_s': 0.0} for stage in TRANSFORM_TIMING_STAGES +} +_TRANSFORM_TIMING_STATS['mask_loops_skipped'] = 0 + +PHOTOMETRY_TIMING_STAGES = ['fit_centroid', 'aperPhot'] +_PHOTOMETRY_TIMING_STATS = { + stage: {'count': 0, 'total_s': 0.0} for stage in PHOTOMETRY_TIMING_STAGES +} + + +def reset_transform_timing_stats(): + for stage in TRANSFORM_TIMING_STAGES: + _TRANSFORM_TIMING_STATS[stage] = {'count': 0, 'success': 0, 'total_s': 0.0} + _TRANSFORM_TIMING_STATS['mask_loops_skipped'] = 0 + + +def reset_photometry_timing_stats(): + for stage in PHOTOMETRY_TIMING_STAGES: + _PHOTOMETRY_TIMING_STATS[stage] = {'count': 0, 'total_s': 0.0} + + +def _record_transform_stage_timing(stage, elapsed_s, success): + stage_stats = _TRANSFORM_TIMING_STATS[stage] + stage_stats['count'] += 1 + stage_stats['total_s'] += elapsed_s + if success: + stage_stats['success'] += 1 + + +def _record_photometry_stage_timing(stage, elapsed_s): + stage_stats = _PHOTOMETRY_TIMING_STATS[stage] + stage_stats['count'] += 1 + stage_stats['total_s'] += elapsed_s + + +def log_transform_timing_stats(prefix='Transformation timing summary'): + logged_any = False + lines = [] + + for stage in TRANSFORM_TIMING_STAGES: + stage_stats = _TRANSFORM_TIMING_STATS[stage] + if stage_stats['count'] == 0: + continue + + avg_ms = 1000.0 * stage_stats['total_s'] / stage_stats['count'] + lines.append( + f"{stage}: calls={stage_stats['count']}, success={stage_stats['success']}, " + f"avg_ms={avg_ms:.2f}, total_s={stage_stats['total_s']:.2f}" + ) + logged_any = True + + if _TRANSFORM_TIMING_STATS['mask_loops_skipped']: + lines.append(f"astroalign_mask_loops_skipped={_TRANSFORM_TIMING_STATS['mask_loops_skipped']}") + logged_any = True + + if logged_any: + log_info(f"{prefix}: " + " | ".join(lines)) + + +def log_photometry_timing_stats(prefix='Photometry timing summary'): + logged_any = False + lines = [] + + for stage in PHOTOMETRY_TIMING_STAGES: + stage_stats = _PHOTOMETRY_TIMING_STATS[stage] + if stage_stats['count'] == 0: + continue + + avg_ms = 1000.0 * stage_stats['total_s'] / stage_stats['count'] + lines.append( + f"{stage}: calls={stage_stats['count']}, avg_ms={avg_ms:.2f}, total_s={stage_stats['total_s']:.2f}" + ) + logged_any = True + + if logged_any: + log_info(f"{prefix}: " + " | ".join(lines)) + + +def log_reduction_timing_overview(prefix='Reduction timing overview'): + transform_total_s = sum(_TRANSFORM_TIMING_STATS[stage]['total_s'] for stage in TRANSFORM_TIMING_STAGES) + photometry_total_s = sum(_PHOTOMETRY_TIMING_STATS[stage]['total_s'] for stage in PHOTOMETRY_TIMING_STAGES) + combined_total_s = transform_total_s + photometry_total_s + if combined_total_s <= 0: + return + + dominant_bucket = 'transform' + if photometry_total_s > transform_total_s: + dominant_bucket = 'photometry' + + transform_pct = 100.0 * transform_total_s / combined_total_s + photometry_pct = 100.0 * photometry_total_s / combined_total_s + log_info( + f"{prefix}: transform_total_s={transform_total_s:.2f} ({transform_pct:.1f}%), " + f"photometry_total_s={photometry_total_s:.2f} ({photometry_pct:.1f}%), " + f"dominant={dominant_bucket}" + ) + + # Aligns imaging data from .fits file to easily track the host and comparison star's positions -def transformation(image_data, file_name, roi=1): +def transformation(image_data, file_name, roi=1, report_failure=True, reference_image=None): + start_time = perf_counter() + + if report_failure: + plateStatus.setCurrentFilename(file_name) + # crop image to ROI - height = image_data.shape[1] - width = image_data.shape[2] - roix = slice(int(width * (0.5 - roi / 2)), int(width * (0.5 + roi / 2))) - roiy = slice(int(height * (0.5 - roi / 2)), int(height * (0.5 + roi / 2))) + if reference_image is None: + current_image = image_data[0] + reference_image = image_data[1] + else: + current_image = image_data + + reference_cache = _get_reference_transform_cache(reference_image, roi) + roix = reference_cache['roix'] + roiy = reference_cache['roiy'] + roi_reference = reference_cache['roi_reference'] + roi_current = current_image[roiy, roix] + + if roi_reference.shape != roi_current.shape or roi_reference.size == 0: + log.debug(f"Warning: Following image failed pre-alignment checks in {perf_counter() - start_time:.2f}s - {file_name}") + if report_failure: + plateStatus.alignmentError() + return SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + fft_tform = None + + # Fast FFT translation estimate before more expensive fallback stages. + # Most cadence images are dominated by small translations, so this stage + # can often solve alignment without invoking significantly slower + # feature-matching methods. + stage_start = perf_counter() + try: + shift, error, _ = phase_cross_correlation(roi_current, roi_reference, upsample_factor=4) + if np.all(np.isfinite(shift)) and np.isfinite(error): + max_shift = max(abs(shift[0]), abs(shift[1])) + if max_shift <= max(roi_current.shape): + fft_tform = SimilarityTransform(scale=1, rotation=0, translation=[-shift[1], -shift[0]]) + fft_high_confidence = error <= 0.1 and max_shift <= max(roi_current.shape) * 0.25 + _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, True) + if fft_high_confidence: + log.debug(f"Transformation solved via high-confidence FFT in {perf_counter() - start_time:.2f}s for {file_name}") + return fft_tform + else: + _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, False) + else: + _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, False) + except Exception: + _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, False) # Find transformation from .FITS files and catch exceptions if not able to. + stage_start = perf_counter() try: - results = aa.find_transform(image_data[1][roiy, roix], image_data[0][roiy, roix]) + results = aa.find_transform(roi_reference, roi_current) + _record_transform_stage_timing('astroalign_direct', perf_counter() - stage_start, True) + log.debug(f"Transformation solved via astroalign direct pass in {perf_counter() - start_time:.2f}s for {file_name}") return results[0] except Exception: - ws = 5 - # smooth image and try to align again - windows = view_as_windows(image_data[0], (ws,ws), step=1) - medimg = np.median(windows, axis=(2,3)) + _record_transform_stage_timing('astroalign_direct', perf_counter() - stage_start, False) - windows = view_as_windows(image_data[1], (ws,ws), step=1) - medimg1 = np.median(windows, axis=(2,3)) + # One cheap filtered pass to suppress noise and retry astroalign. + filtered_current = gaussian_filter(roi_current, sigma=1.0) + filtered_reference = reference_cache['filtered_reference'] - try: - results = aa.find_transform(medimg1[roiy, roix], medimg[roiy, roix]) - return results[0] - except Exception: - pass + stage_start = perf_counter() + try: + results = aa.find_transform(filtered_reference, filtered_current) + _record_transform_stage_timing('astroalign_filtered', perf_counter() - stage_start, True) + log.debug(f"Transformation solved via filtered astroalign in {perf_counter() - start_time:.2f}s for {file_name}") + return results[0] + except Exception: + _record_transform_stage_timing('astroalign_filtered', perf_counter() - stage_start, False) - for p in [99, 98, 95, 90]: - for it in [2, 1, 0]: + for p in [99, 98, 95, 90]: + base_mask1 = reference_cache['reference_masks'][p] + p_cur = np.percentile(roi_current, p) + base_mask0 = roi_current > p_cur - # create binary mask to align image - mask1 = image_data[1][roiy, roix] > np.percentile(image_data[1][roiy, roix], p) - mask1 = binary_erosion(mask1, iterations=it) + for it in [2, 1, 0]: + # create binary mask to align image + mask1 = base_mask1 + mask0 = base_mask0 - mask0 = image_data[0][roiy, roix] > np.percentile(image_data[0][roiy, roix], p) + if it > 0: + mask1 = binary_erosion(mask1, iterations=it) mask0 = binary_erosion(mask0, iterations=it) - try: - results = aa.find_transform(mask1, mask0) - return results[0] - except Exception: - try: - result1 = ird.similarity(image_data[1][roiy, roix], image_data[0][roiy, roix], numiter=3) - return SimilarityTransform(scale=result1['scale'], rotation=np.radians(result1['angle']), - translation=[-1 * result1['tvec'][1], -1 * result1['tvec'][0]]) - except Exception: - pass + stage_start = perf_counter() + try: + results = aa.find_transform(mask1, mask0) + _record_transform_stage_timing('astroalign_mask', perf_counter() - stage_start, True) + log.debug(f"Transformation solved via mask astroalign (p={p}, erode={it}) in {perf_counter() - start_time:.2f}s for {file_name}") + return results[0] + except Exception: + _record_transform_stage_timing('astroalign_mask', perf_counter() - stage_start, False) + + stage_start = perf_counter() + try: + result1 = ird.similarity(roi_reference, roi_current, numiter=3) + _record_transform_stage_timing('imreg_dft', perf_counter() - stage_start, True) + log.debug(f"Transformation solved via imreg_dft fallback in {perf_counter() - start_time:.2f}s for {file_name}") + return SimilarityTransform(scale=result1['scale'], rotation=np.radians(result1['angle']), + translation=[-1 * result1['tvec'][1], -1 * result1['tvec'][0]]) + except Exception: + _record_transform_stage_timing('imreg_dft', perf_counter() - stage_start, False) + + if fft_tform is not None: + log.debug(f"Transformation fell back to FFT translation in {perf_counter() - start_time:.2f}s for {file_name}") + return fft_tform - log.debug(f"Warning: Following image failed to align - {file_name}") - plateStatus.alignmentError() + log.debug(f"Warning: Following image failed to align in {perf_counter() - start_time:.2f}s - {file_name}") + if report_failure: + plateStatus.alignmentError() return SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) +def load_image_data(file_name): + hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + image_data = hdul[extension].data + hdul.close() + return image_data + + +def transformation_task(i, file_name, reference_file): + if i == 0: + return i, SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + image_data = load_image_data(file_name) + reference_image = load_image_data(reference_file) + # Multiprocess pre-computation should not emit plate-status warnings; the + # serial reduction path decides whether the fallback transform is needed. + return i, transformation(image_data, file_name, report_failure=False, reference_image=reference_image) + + +_TRANSFORM_REFERENCE_IMAGE = None +_TRANSFORM_REFERENCE_CACHE = None + + +def _build_reference_transform_cache(reference_image, roi): + height = reference_image.shape[0] + width = reference_image.shape[1] + roix = slice(int(width * (0.5 - roi / 2)), int(width * (0.5 + roi / 2))) + roiy = slice(int(height * (0.5 - roi / 2)), int(height * (0.5 + roi / 2))) + + roi_reference = reference_image[roiy, roix] + + cache = { + 'ref_id': id(reference_image), + 'shape': reference_image.shape, + 'roi': roi, + 'roix': roix, + 'roiy': roiy, + 'roi_reference': roi_reference, + 'filtered_reference': gaussian_filter(roi_reference, sigma=1.0), + } + + reference_masks = {} + for p in [99, 98, 95, 90]: + p_ref = np.percentile(roi_reference, p) + reference_masks[p] = roi_reference > p_ref + cache['reference_masks'] = reference_masks + + return cache + + +def _get_reference_transform_cache(reference_image, roi): + global _TRANSFORM_REFERENCE_CACHE + + if (_TRANSFORM_REFERENCE_CACHE is None + or _TRANSFORM_REFERENCE_CACHE['ref_id'] != id(reference_image) + or _TRANSFORM_REFERENCE_CACHE['shape'] != reference_image.shape + or _TRANSFORM_REFERENCE_CACHE['roi'] != roi): + _TRANSFORM_REFERENCE_CACHE = _build_reference_transform_cache(reference_image, roi) + + return _TRANSFORM_REFERENCE_CACHE + + +def _transformation_pool_initializer(reference_file): + global _TRANSFORM_REFERENCE_IMAGE, _TRANSFORM_REFERENCE_CACHE + _TRANSFORM_REFERENCE_IMAGE = load_image_data(reference_file) + _TRANSFORM_REFERENCE_CACHE = None + + +def transformation_task_with_cached_reference(i, file_name): + image_data = load_image_data(file_name) + return i, transformation(image_data, file_name, report_failure=False, reference_image=_TRANSFORM_REFERENCE_IMAGE) + + +MAX_MULTIPROCESS_TRANSFORM_WORKERS = 8 + +# Automatic aperture-grid tuning constants (in PSF sigma units) +APERTURE_SIGMA_MIN = 1.5 +APERTURE_SIGMA_MAX = 6.0 +ANNULUS_SIGMA_MIN = 6.0 +ANNULUS_SIGMA_MAX = 15.0 +APERTURE_AUTOTUNE_COARSE_APER_POINTS = 5 +APERTURE_AUTOTUNE_COARSE_ANNULUS_POINTS = 4 +APERTURE_AUTOTUNE_REFINED_APER_POINTS = 6 +APERTURE_AUTOTUNE_REFINED_ANNULUS_POINTS = 6 +APERTURE_AUTOTUNE_APER_HALF_WIDTH_SIGMA = 0.9 +APERTURE_AUTOTUNE_ANNULUS_HALF_WIDTH_SIGMA = 2.0 +APERTURE_AUTOTUNE_MIN_FRAMES = 8 +APERTURE_AUTOTUNE_MAX_FRAMES = 12 + +# Refit full PSF moments periodically; use a faster moment estimator for most frames. +CENTROID_FULL_FIT_CADENCE = 6 + + +def build_multiprocess_transformations(inputfiles, max_processes): + reference_file = str(inputfiles[0]) + max_workers = min(max_processes, os.cpu_count() or 1, len(inputfiles), MAX_MULTIPROCESS_TRANSFORM_WORKERS) + transforms = {} + total_jobs = len(inputfiles) + + log_info( + "Using multiprocessing for transformations " + f"with {max_workers} worker(s) across {total_jobs} image(s)." + ) + + transforms[0] = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + with ProcessPoolExecutor(max_workers=max_workers, initializer=_transformation_pool_initializer, + initargs=(reference_file,)) as executor: + futures = [executor.submit(transformation_task_with_cached_reference, i, str(file_name)) + for i, file_name in enumerate(inputfiles) if i != 0] + + completed = 1 + for future in as_completed(futures): + i, tform = future.result() + transforms[i] = tform + completed += 1 + + if completed == total_jobs or completed % 10 == 0: + log_info(f"Multiprocessing transformations progress: {completed}/{total_jobs}") + + return transforms + + +def log_finding_transformation_progress(i, total_jobs, file_name, use_multiprocess_progress): + if use_multiprocess_progress: + completed = i + 1 + if completed == total_jobs or completed % 10 == 0: + log_info(f"Multiprocessing finding transformations progress: {completed}/{total_jobs}") + return + + sys.stdout.write(f"Finding transformation {i + 1} of {total_jobs} : {file_name}\n") + log.debug(f"Finding transformation {i + 1} of {total_jobs} : {file_name}\n") + sys.stdout.flush() + def get_img_scale(hdr, wcs_file, pixel_init): if wcs_file: @@ -1153,13 +1893,26 @@ def update_coordinates_with_proper_motion(info_dict, time_obs): 'pm_dec': 'Proper Motion DEC (mas/yr)' } - missing_values = [parameter_names[key] for key in ['dist', 'pm_ra', 'pm_dec'] if info_dict.get(key, 0.0) == 0.0] + numeric_values = {} + missing_values = [] + + for key in ['dist', 'pm_ra', 'pm_dec']: + raw_value = info_dict.get(key, 0.0) + + try: + parsed_value = float(raw_value) + except (TypeError, ValueError): + parsed_value = 0.0 + + numeric_values[key] = parsed_value + if parsed_value == 0.0: + missing_values.append(parameter_names[key]) if missing_values: missing_values = ", ".join(missing_values) log_info("Warning: Cannot account for proper motion due to missing values in: " - f"\n{missing_values}. Please re-run and fill in values in the initialization file to account " - f"for proper motion", warn=True) + f"\n{missing_values}. If you find your target or comparisons are not detected well, please " + f"re-run and fill in values in the initialization file to account for proper motion", warn=True) return info_dict['ra'], info_dict['dec'] else: @@ -1169,9 +1922,9 @@ def update_coordinates_with_proper_motion(info_dict, time_obs): coord = SkyCoord( ra=info_dict['ra'] * u.deg, dec=info_dict['dec'] * u.deg, - distance=info_dict['dist'] * u.pc, - pm_ra_cosdec=info_dict['pm_ra'] * u.mas / u.yr, - pm_dec=info_dict['pm_dec'] * u.mas / u.yr, + distance=numeric_values['dist'] * u.pc, + pm_ra_cosdec=numeric_values['pm_ra'] * u.mas / u.yr, + pm_dec=numeric_values['pm_dec'] * u.mas / u.yr, frame="icrs", obstime=time_j2000 ) @@ -1202,72 +1955,215 @@ def mesh_box(pos, box, maxx=0, maxy=0): return xv.astype(int), yv.astype(int) -# Method fits a 2D gaussian function that matches the star_psf to the star image and returns its pixel coordinates -def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weightedcenter=True): - # get sub field in image - xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) - subarray = data[yv, xv] - try: - init = [np.nanmax(subarray) - np.nanmin(subarray), 1, 1, 0, np.nanmin(subarray)] - except ValueError as ve: - # Handle null subfield - cannot solve - plateStatus.outOfFrameWarning(starIndex) - log.debug(f"Warning: empty subfield for fit_centroid at {np.round(pos, 2)}") +def should_use_fast_centroid(frame_index): + return frame_index % CENTROID_FULL_FIT_CADENCE != 0 + + +def _fit_centroid_moments(subarray, xv, yv, pos, box): + background = bn.nanmedian(subarray) + weights = subarray - background + weights = np.where(np.isfinite(weights) & (weights > 0), weights, 0.0) + wsum = np.sum(weights) + + if not np.isfinite(wsum) or wsum <= 0: + floor = np.nanmin(subarray) + weights = subarray - floor + weights = np.where(np.isfinite(weights) & (weights > 0), weights, 0.0) + wsum = np.sum(weights) + + if not np.isfinite(wsum) or wsum <= 0: return np.empty(7) * np.nan - # compute flux weighted centroid in x and y - wx = np.sum(xv[0]*subarray.sum(0))/subarray.sum(0).sum() - wy = np.sum(yv[:,0]*subarray.sum(1))/subarray.sum(1).sum() - # lower bound: [xc, yc, amp, sigx, sigy, rotation, bg] - lo = [pos[0] - box * 0.5, pos[1] - box * 0.5, 0, 0.5, 0.5, -np.pi / 4, np.nanmin(subarray) - 1] - up = [pos[0] + box * 0.5, pos[1] + box * 0.5, 1e7, 20, 20, np.pi / 4, np.nanmax(subarray) + 1] + wx = float(np.sum(xv * weights) / wsum) + wy = float(np.sum(yv * weights) / wsum) - def fcn2min(pars): - model = psf_function(xv, yv, *pars) - return (subarray - model).flatten() + # Keep centroid near the expected star location in crowded fields. + wx = float(np.clip(wx, pos[0] - box * 0.5, pos[0] + box * 0.5)) + wy = float(np.clip(wy, pos[1] - box * 0.5, pos[1] + box * 0.5)) + dx = xv - wx + dy = yv - wy + var_x = float(np.sum(weights * dx * dx) / wsum) + var_y = float(np.sum(weights * dy * dy) / wsum) + cov_xy = float(np.sum(weights * dx * dy) / wsum) + + sigx = float(np.clip(np.sqrt(max(var_x, 0.25)), 0.5, 20.0)) + sigy = float(np.clip(np.sqrt(max(var_y, 0.25)), 0.5, 20.0)) + rot = float(0.5 * np.arctan2(2.0 * cov_xy, var_x - var_y)) if np.isfinite(cov_xy) else 0.0 + amp = float(max(np.nanmax(subarray) - background, 0.0)) + + return np.array([wx, wy, amp, sigx, sigy, rot, float(background)], dtype=float) + + +def _has_usable_centroid_signal(subarray, amplitude, min_snr=5.0): + if not np.isfinite(amplitude) or amplitude <= 0: + return False + + scatter = float(bn.nanstd(subarray)) + if not np.isfinite(scatter) or scatter <= 0: + return True + + return amplitude >= (min_snr * scatter) + + +def _nan_psf_result(): + return np.full(7, np.nan, dtype=float) + + +def fit_centroid_or_warn_out_of_frame(data, pos, starIndex, **kwargs): + if not pixel_within_image(pos[0], pos[1], data.shape): + plateStatus.outOfFrameWarning(starIndex) + return _nan_psf_result() + return fit_centroid(data, pos, starIndex, **kwargs) + + +def fractional_flux_change_within_limit(current_amplitude, previous_amplitude, limit=0.5): + if (not np.isfinite(current_amplitude) + or not np.isfinite(previous_amplitude) + or previous_amplitude == 0): + return False + + return np.abs((current_amplitude - previous_amplitude) / previous_amplitude) <= limit + + +def centroid_offset_matches_reference(psf_a, psf_b, expected_dx, expected_dy, tolerance=1): + x_values = [psf_a[0], psf_b[0]] + y_values = [psf_a[1], psf_b[1]] + if not np.all(np.isfinite(x_values + y_values)): + return False + + return ( + expected_dx - tolerance <= abs(int(psf_a[0]) - int(psf_b[0])) <= expected_dx + tolerance + and expected_dy - tolerance <= abs(int(psf_a[1]) - int(psf_b[1])) <= expected_dy + tolerance + ) + + +# Method fits a 2D gaussian function that matches the star_psf to the star image and returns its pixel coordinates +def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weightedcenter=True, fast_mode=False): + stage_start = perf_counter() + # get sub field in image try: - res = least_squares(fcn2min, x0=[*pos, *init], bounds=[lo, up], jac='3-point', xtol=None, method='trf') - except: - # Report low flux warning - plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) - log.debug(f"Warning: Measured flux amplitude is really low---are you sure there is a star at {np.round(pos, 2)}?") + xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + try: + init = [np.nanmax(subarray) - np.nanmin(subarray), 1, 1, 0, np.nanmin(subarray)] + except ValueError as ve: + # Handle null subfield - cannot solve + plateStatus.outOfFrameWarning(starIndex) + log.debug(f"Warning: empty subfield for fit_centroid at {np.round(pos, 2)}") + return _nan_psf_result() + + moment_fit = _fit_centroid_moments(subarray, xv, yv, pos, box) + if np.isfinite(moment_fit[0]): + wx, wy = moment_fit[0], moment_fit[1] + init = [moment_fit[2], moment_fit[3], moment_fit[4], moment_fit[5], moment_fit[6]] + if fast_mode: + return moment_fit + else: + # compute flux weighted centroid in x and y + wx = np.sum(xv[0] * subarray.sum(0)) / subarray.sum(0).sum() + wy = np.sum(yv[:, 0] * subarray.sum(1)) / subarray.sum(1).sum() + + # lower bound: [xc, yc, amp, sigx, sigy, rotation, bg] + lo = [pos[0] - box * 0.5, pos[1] - box * 0.5, 0, 0.5, 0.5, -np.pi / 4, np.nanmin(subarray) - 1] + up = [pos[0] + box * 0.5, pos[1] + box * 0.5, 1e7, 20, 20, np.pi / 4, np.nanmax(subarray) + 1] + x0 = np.array([*pos, *init], dtype=float) + lo_arr = np.array(lo, dtype=float) + up_arr = np.array(up, dtype=float) + if np.all(np.isfinite(x0)): + x0 = np.clip(x0, lo_arr + 1e-6, up_arr - 1e-6) + has_usable_signal = _has_usable_centroid_signal(subarray, init[0]) + + def fcn2min(pars): + model = psf_function(xv, yv, *pars) + return (subarray - model).flatten() - res = least_squares(fcn2min, x0=[*pos, *init], jac='3-point', xtol=None, method='lm') + try: + res = least_squares(fcn2min, x0=x0, bounds=[lo, up], jac='2-point', xtol=None, method='trf') + except Exception as exc: + if has_usable_signal and np.isfinite(moment_fit[0]): + log.debug(f"Centroid PSF fit failed at {np.round(pos, 2)}; using moment centroid instead: {exc}") + return moment_fit + + if not has_usable_signal: + plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) + log.debug(f"Warning: Measured flux amplitude is really low---are you sure there is a star at {np.round(pos, 2)}?") + else: + log.debug(f"Centroid PSF fit failed at {np.round(pos, 2)}; attempting LM fallback: {exc}") + + try: + res = least_squares(fcn2min, x0=x0, jac='2-point', xtol=1e-12, method='lm') + except Exception as lm_exc: + log.debug(f"Centroid LM fallback failed at {np.round(pos, 2)}: {lm_exc}") + return _nan_psf_result() + + # override psf fit results with weighted centroid + if weightedcenter: + res.x[0] = wx + res.x[1] = wy + + return res.x + finally: + _record_photometry_stage_timing('fit_centroid', perf_counter() - stage_start) + + +def sigma_clipped_nanmedian(data, sigma=3.0, max_iters=3): + clipped = np.array(data, dtype=float, copy=True) + if clipped.size == 0: + return np.nan, np.nan + + clipped[~np.isfinite(clipped)] = np.nan + nan_count_prev = np.count_nonzero(np.isnan(clipped)) + + for _ in range(max_iters): + center = bn.nanmedian(clipped) + scatter = bn.nanstd(clipped) + if not np.isfinite(center): + return np.nan, np.nan + if not np.isfinite(scatter) or scatter <= 0: + break - # override psf fit results with weighted centroid - if weightedcenter: - res.x[0] = wx - res.x[1] = wy + clipped[np.abs(clipped - center) > sigma * scatter] = np.nan + nan_count = np.count_nonzero(np.isnan(clipped)) + if nan_count == nan_count_prev: + break + nan_count_prev = nan_count - return res.x + return bn.nanmedian(clipped), bn.nanstd(clipped) # Method calculates the flux of the star (uses the skybg_phot method to do background sub) -def aperPhot(data, starIndex, xc, yc, r=5, dr=5): - # Check for invalid coordinates - if np.isnan(xc) or np.isnan(yc): - return 0, 0 - - # Calculate background if dr > 0 - if dr > 0: - bgflux, sigmabg, Nbg = skybg_phot(data, starIndex, xc, yc, r + 2, dr) - else: - bgflux, sigmabg, Nbg = 0, 0, 0 - - # Create aperture and mask - aperture = CircularAperture(positions=[(xc, yc)], r=r) - mask = aperture.to_mask(method='exact')[0] - data_cutout = mask.cutout(data) - - # Check if aperture is valid - if data_cutout is None: - # Aperture is partially or fully outside the image - return 0, bgflux # Return zero flux but valid background - - # Calculate and return aperture sum - aperture_sum = (mask.data * (data_cutout - bgflux)).sum() - return aperture_sum, bgflux +def aperPhot(data, starIndex, xc, yc, r=5, dr=5, fast_mode=True): + stage_start = perf_counter() + try: + # Check for invalid coordinates + if np.isnan(xc) or np.isnan(yc): + return 0, 0 + + # Calculate background if dr > 0 + if dr > 0: + bgflux, sigmabg, Nbg = skybg_phot(data, starIndex, xc, yc, r + 2, dr) + if not np.isfinite(bgflux): + return np.nan, bgflux + else: + bgflux, sigmabg, Nbg = 0, 0, 0 + + # Create aperture and mask + aperture = CircularAperture(positions=[(xc, yc)], r=r) + mask_method = 'center' if fast_mode else 'exact' + mask = aperture.to_mask(method=mask_method)[0] + data_cutout = mask.cutout(data) + + # Check if aperture is valid + if data_cutout is None: + # Aperture is partially or fully outside the image + return 0, bgflux # Return zero flux but valid background + + # Calculate and return aperture sum + aperture_sum = (mask.data * (data_cutout - bgflux)).sum() + return aperture_sum, bgflux + finally: + _record_photometry_stage_timing('aperPhot', perf_counter() - stage_start) def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False): @@ -1275,22 +2171,36 @@ def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False): # the box will not extend beyond the borders of the image image_height, image_width = data.shape xv, yv = mesh_box([xc, yc], np.round(r + dr), maxx=image_width, maxy=image_height) - rv = ((xv - xc) ** 2 + (yv - yc) ** 2) ** 0.5 - mask = (rv > r) & (rv < (r + dr)) + if xv.size == 0 or yv.size == 0: + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: empty sky background box for {xc:.1f}, {yc:.1f}." + f"\nCheck if star is present or close to border.") + return np.nan, np.nan, 0 + + r_inner2 = float(r) ** 2 + r_outer2 = float(r + dr) ** 2 + rv2 = (xv - xc) ** 2 + (yv - yc) ** 2 + mask = (rv2 > r_inner2) & (rv2 < r_outer2) + if not np.any(mask): + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: empty sky background annulus for {xc:.1f}, {yc:.1f}." + f"\nCheck if star is present or close to border.") + return np.nan, np.nan, 0 + + annulus_pixels = np.asarray(data[yv, xv][mask], dtype=float) + if annulus_pixels.size == 0: + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: no valid sky background pixels for {xc:.1f}, {yc:.1f}." + f"\nCheck if star is present or close to border.") + return np.nan, np.nan, 0 + try: - cutoff = np.nanpercentile(data[yv, xv][mask], ptol) - except IndexError: + cutoff = np.nanpercentile(annulus_pixels, ptol) + except (IndexError, ValueError): plateStatus.skyBackgroundWarning(starIndex, xc, yc) log.debug(f"Warning: IndexError, problem computing sky bg for {xc:.1f}, {yc:.1f}." f"\nCheck if star is present or close to border.") - - # create pixel wise mask on entire image - x = np.arange(data.shape[1]) - y = np.arange(data.shape[0]) - xv, yv = np.meshgrid(x, y) - rv = ((xv - xc) ** 2 + (yv - yc) ** 2) ** 0.5 - mask = (rv > r) & (rv < (r + dr)) - cutoff = np.nanpercentile(data[yv, xv][mask], ptol) + return np.nan, np.nan, 0 dat = np.array(data[yv, xv], dtype=float) dat[dat > cutoff] = np.nan # ignore pixels brighter than percentile @@ -1301,8 +2211,8 @@ def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False): nanmask = np.nan * np.zeros(mask.shape) nanmask[mask] = 1 bgsky = data[yv, xv] * nanmask - cmode = mode(dat.flatten(), nan_policy='omit', keepdims=True).mode[0] - amode = mode(bgsky.flatten(), nan_policy='omit', keepdims=True).mode[0] + cmed, _ = sigma_clipped_nanmedian(dat.flatten(), sigma=3.0, max_iters=3) + amed, _ = sigma_clipped_nanmedian(bgsky.flatten(), sigma=3.0, max_iters=3) fig, ax = plt.subplots(2, 2, figsize=(9, 9)) im = ax[0, 0].imshow(data[yv, xv], vmin=minb, vmax=maxb, cmap='inferno') @@ -1312,9 +2222,9 @@ def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False): cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') - ax[1, 0].hist(bgsky.flatten(), label=f'Sky Annulus ({np.nanmedian(bgsky):.1f}, {amode:.1f})', + ax[1, 0].hist(bgsky.flatten(), label=f'Sky Annulus ({np.nanmedian(bgsky):.1f}, {amed:.1f})', alpha=0.5, bins=np.arange(minb, maxb)) - ax[1, 0].hist(dat.flatten(), label=f'Clipped ({np.nanmedian(dat):.1f}, {cmode:.1f})', alpha=0.5, + ax[1, 0].hist(dat.flatten(), label=f'Clipped ({np.nanmedian(dat):.1f}, {cmed:.1f})', alpha=0.5, bins=np.arange(minb, maxb)) ax[1, 0].legend(loc='best') ax[1, 0].set_title("Sky Background") @@ -1327,7 +2237,9 @@ def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False): ax[0, 1].set_title("Sky Annulus") plt.tight_layout() plt.show() - return mode(dat.flatten(), nan_policy='omit', keepdims=True).mode[0], np.nanstd(dat.flatten()), np.sum(mask) + dat_flat = dat.ravel() + sky_median, sky_sigma = sigma_clipped_nanmedian(dat_flat, sigma=3.0, max_iters=3) + return sky_median, sky_sigma, np.sum(mask) def process_dark_frames(dark_files): """Process dark frames and return the master dark.""" @@ -1536,8 +2448,9 @@ def save_comp_ra_dec(wcs_file, ra_file, dec_file, comp_coords): return comp_star -def realTimeReduce(i, target_name, p_dict, info_dict, ax): +def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astrometry=False, multiprocess_transformations=None): timeList, airMassList, exptimes, norm_flux = [], [], [], [] + ignore_header_wcs = should_ignore_header_wcs(info_dict.get('ignore_header_wcs')) plateStatus.initializeFilenames(info_dict['images']) inputfiles = corruption_check(info_dict['images']) @@ -1556,11 +2469,22 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax): si = np.argsort(times) inputfiles = np.array(inputfiles)[si] + + use_multiprocess_transform_precompute = should_use_multiprocess_transform_precompute( + inputfiles, multiprocess_transformations, ignore_header_wcs=ignore_header_wcs + ) + fallback_transforms = {} + if use_multiprocess_transform_precompute: + fallback_transforms = build_multiprocess_transformations(inputfiles, multiprocess_transformations) + exotic_UIprevTPX = info_dict['tar_coords'][0] exotic_UIprevTPY = info_dict['tar_coords'][1] plateStatus.setCurrentFilename(inputfiles[0]) - wcs_file = check_wcs(inputfiles[0], info_dict['save'], info_dict['plate_opt'], rt=True) + wcs_file = check_wcs(inputfiles[0], info_dict['save'], info_dict['plate_opt'], rt=True, + use_nextastro_astrometry=use_nextastro_astrometry, + ra=p_dict.get('ra'), dec=p_dict.get('dec'), pixel_scale=info_dict.get('pixel_scale'), + ignore_header_wcs=ignore_header_wcs) comp_star = info_dict['comp_stars'] tar_radec, comp_radec = None, [] @@ -1576,12 +2500,17 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax): comp_radec.append((ra, dec)) + target_and_comp_radec = None + if tar_radec is not None and comp_radec: + target_and_comp_radec = np.array([tar_radec, comp_radec[0]], dtype=float) + first_image = fits.getdata(inputfiles[0]) targ_sig_xy = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0)[3:5] # aperture size in stdev (sigma) of PSF aper = 3 * max(targ_sig_xy) annulus = 10 + fast_aperture_mask = is_fast_aperture_mask_enabled(info_dict.get('fast_aperture_mask')) # alloc psf fitting param psf_data = { @@ -1594,10 +2523,13 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax): } # open files, calibrate, align, photometry + reset_transform_timing_stats() + reset_photometry_timing_stats() for i, fileName in enumerate(inputfiles): plateStatus.setCurrentFilename(fileName) hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + frame_fast_centroid = should_use_fast_centroid(i) extension = 0 image_header = hdul[extension].header @@ -1615,50 +2547,106 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax): if i == 0: firstImage = np.copy(imageData) - sys.stdout.write(f"Finding transformation {i + 1} of {len(inputfiles)} : {fileName}\n") - log.debug(f"Finding transformation {i + 1} of {len(inputfiles)} : {fileName}\n") - sys.stdout.flush() - - try: - wcs_hdr = search_wcs(fileName) - if not wcs_hdr.is_celestial: - raise Exception - - if i == 0: - tx, ty = exotic_UIprevTPX, exotic_UIprevTPY - else: - pix_coords = wcs_hdr.world_to_pixel_values(tar_radec[0], tar_radec[1]) - tx, ty = pix_coords[0].take(0), pix_coords[1].take(0) - - psf_data['target'][i] = fit_centroid(imageData, [tx, ty], 0) + log_finding_transformation_progress( + i, + len(inputfiles), + fileName, + use_multiprocess_transform_precompute, + ) - if i != 0 and np.abs((psf_data['target'][i][2] - psf_data['target'][i - 1][2]) - / psf_data['target'][i - 1][2]) > 0.5: - raise Exception + use_wcs_alignment = False + if not ignore_header_wcs: + try: + wcs_hdr = search_wcs_from_header(image_header) + use_wcs_alignment = wcs_hdr.is_celestial + except Exception: + use_wcs_alignment = False - pix_coords = wcs_hdr.world_to_pixel_values(comp_radec[0][0], comp_radec[0][1]) - cx, cy = pix_coords[0].take(0), pix_coords[1].take(0) - psf_data['comp'][i] = fit_centroid(imageData, [cx, cy], 1) + if use_wcs_alignment: + try: + if i == 0: + tx, ty = exotic_UIprevTPX, exotic_UIprevTPY + cx, cy = comp_star + else: + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + tx, ty = pix_x[0], pix_y[0] + cx, cy = pix_x[1], pix_y[1] + + projected_coords = np.array([[tx, ty], [cx, cy]], dtype=float) + projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) + + psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + [tx, ty], + 0, + fast_mode=frame_fast_centroid, + ) + psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + [cx, cy], + 1, + fast_mode=frame_fast_centroid, + ) + + target_flux_change_ok = True + comp_valid = True + if projected_off_frame: + use_wcs_alignment = True + elif i != 0: + target_flux_change_ok = fractional_flux_change_within_limit( + psf_data['target'][i][2], + psf_data['target'][i - 1][2], + ) + comp_valid = ( + centroid_offset_matches_reference( + psf_data['comp'][i], + psf_data['target'][i], + tar_comp_dist['comp'][0], + tar_comp_dist['comp'][1], + ) + and fractional_flux_change_within_limit( + psf_data['comp'][i][2], + psf_data['comp'][i - 1][2], + ) + ) + use_wcs_alignment = target_flux_change_ok and comp_valid + else: + tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) + tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) + use_wcs_alignment = True + except Exception: + use_wcs_alignment = False - if i != 0: - if not (tar_comp_dist['comp'][0] - 1 <= abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][i][0])) <= tar_comp_dist['comp'][0] + 1 and - tar_comp_dist['comp'][1] - 1 <= abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][i][1])) <= tar_comp_dist['comp'][1] + 1) or \ - np.abs((psf_data['comp'][i][2] - psf_data['comp'][i - 1][2]) / psf_data['comp'][i - 1][2]) > 0.5: - raise Exception - else: - tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) - except Exception: + if not use_wcs_alignment: if i == 0: tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) else: - tform = transformation(np.array([imageData, firstImage]), fileName) + tform = fallback_transforms[i] if i in fallback_transforms else transformation(imageData, fileName, reference_image=firstImage) - tx, ty = tform([exotic_UIprevTPX, exotic_UIprevTPY])[0] - psf_data['target'][i] = fit_centroid(imageData, [tx, ty], 0) + transformed_coords = np.asarray( + tform(np.array([[exotic_UIprevTPX, exotic_UIprevTPY], comp_star], dtype=float)), + dtype=float, + ) + tx, ty = transformed_coords[0] + psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + [tx, ty], + 0, + fast_mode=frame_fast_centroid, + ) - cx, cy = tform(comp_star)[0] - psf_data['comp'][i] = fit_centroid(imageData, [cx, cy], 1) + cx, cy = transformed_coords[1] + psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + [cx, cy], + 1, + fast_mode=frame_fast_centroid, + ) if i == 0: tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) @@ -1670,8 +2658,10 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax): aper *= sigma annulus *= sigma - tFlux = aperPhot(imageData, 0, psf_data['target'][i, 0], psf_data['target'][i, 1], aper, annulus)[0] - cFlux = aperPhot(imageData, 1, psf_data['comp'][i, 0], psf_data['comp'][i, 1], aper, annulus)[0] + tFlux = aperPhot(imageData, 0, psf_data['target'][i, 0], psf_data['target'][i, 1], aper, annulus, + fast_mode=fast_aperture_mask)[0] + cFlux = aperPhot(imageData, 1, psf_data['comp'][i, 0], psf_data['comp'][i, 1], aper, annulus, + fast_mode=fast_aperture_mask)[0] norm_flux.append(tFlux / cFlux) # close file + delete from memory @@ -1680,6 +2670,10 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax): # Replaced each loop, so clean up del imageData + log_transform_timing_stats('Transformation timing summary (real-time reduce)') + log_photometry_timing_stats('Photometry timing summary (real-time reduce)') + log_reduction_timing_overview('Reduction timing overview (real-time reduce)') + ax.clear() ax.set_title(target_name) ax.set_ylabel('Normalized Flux') @@ -1687,26 +2681,52 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax): ax.plot(timeList, norm_flux, 'bo') -def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None): +def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, + allow_mid_transit_range_warning=True): # remove outliers si = np.argsort(times) - dt = np.mean(np.diff(np.sort(times))) + times_sorted = times[si] + tflux_sorted = tFlux[si] + cflux_sorted = cFlux[si] + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio_sorted = np.divide(tflux_sorted, cflux_sorted) + + has_reference_flux = not np.allclose(cflux_sorted, 1.0) + if has_reference_flux: + relative_flux_mask = relative_flux_filter_mask(flux_ratio_sorted) + times_sorted = times_sorted[relative_flux_mask] + tflux_sorted = tflux_sorted[relative_flux_mask] + cflux_sorted = cflux_sorted[relative_flux_mask] + flux_ratio_sorted = flux_ratio_sorted[relative_flux_mask] + jd_times_sorted = jd_times[si][relative_flux_mask] + airmass_sorted = airmass[si][relative_flux_mask] + else: + jd_times_sorted = jd_times[si] + airmass_sorted = airmass[si] + + if len(times_sorted) <= 1: + log_info('No data left after filtering', warn=True) + return None, None, None + + dt = np.mean(np.diff(times_sorted)) ndt = int(25. / 24. / 60. / dt) * 2 + 1 - if ndt > len(times): - ndt = int(len(times)/4) * 2 + 1 - filtered_data = sigma_clip((tFlux / cFlux)[si], sigma=3, dt=max(5,ndt)) - arrayFinalFlux = (tFlux / cFlux)[si][~filtered_data] - f1 = tFlux[si][~filtered_data] + if ndt > len(times_sorted): + ndt = int(len(times_sorted)/4) * 2 + 1 + filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt)) + valid_mask = ~filtered_data + + arrayFinalFlux = flux_ratio_sorted[valid_mask] + f1 = tflux_sorted[valid_mask] sigf1 = f1 ** 0.5 - f2 = cFlux[si][~filtered_data] + f2 = cflux_sorted[valid_mask] sigf2 = f2 ** 0.5 if np.sum(cFlux) == len(cFlux): arrayNormUnc = sigf1 else: arrayNormUnc = np.sqrt((sigf1 / f2) ** 2 + (sigf2 * f1 / f2 ** 2) ** 2) - arrayTimes = times[si][~filtered_data] - arrayJDTimes = jd_times[si][~filtered_data] - arrayAirmass = airmass[si][~filtered_data] + arrayTimes = times_sorted[valid_mask] + arrayJDTimes = jd_times_sorted[valid_mask] + arrayAirmass = airmass_sorted[valid_mask] # remove nans nanmask = np.isnan(arrayFinalFlux) | np.isnan(arrayNormUnc) | np.isnan(arrayTimes) | np.isnan( @@ -1723,6 +2743,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None): arrayTimes = arrayTimes[~nanmask] arrayJDTimes = arrayJDTimes[~nanmask] arrayAirmass = arrayAirmass[~nanmask] + f1 = f1[~nanmask] + f2 = f2[~nanmask] # -----LM LIGHTCURVE FIT-------------------------------------- @@ -1750,13 +2772,11 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None): if lower < prior['tmid'] - 0.25 * prior['per']: lower = prior['tmid'] - 0.25 * prior['per'] - if np.floor(arrayPhases).max() - np.floor(arrayPhases).min() == 0: - log_info("\nWarning:", warn=True) - log_info(" Estimated mid-transit time is not within the observations", warn=True) - log_info(" Check Period & Mid-transit time in inits.json. Make sure the uncertainties are not 0 or Nan.", warn=True) - log_info(f" obs start:{arrayTimes.min()}", warn=True) - log_info(f" obs end:{arrayTimes.max()}", warn=True) - log_info(f" tmid prior:{prior['tmid']}\n", warn=True) + if ( + allow_mid_transit_range_warning + and np.floor(arrayPhases).max() - np.floor(arrayPhases).min() == 0 + ): + log_mid_transit_range_warning_once(arrayTimes, prior['tmid']) mybounds = { 'rprs': [0, prior['rprs'] * 1.25], @@ -1780,9 +2800,447 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None): mode='lm' ) + if ( + myfit is not None + and hasattr(myfit, 'residuals') + and hasattr(myfit, 'phase') + and np.shape(myfit.residuals) == np.shape(arrayTimes) + and np.shape(myfit.phase) == np.shape(arrayTimes) + ): + phase_clip_mask = phase_bin_sigma_clip(myfit.residuals, myfit.phase, sigma=3, bins=10) + min_required_points = max(len(mybounds) + 1, 5) + if np.count_nonzero(~phase_clip_mask) >= min_required_points and np.any(phase_clip_mask): + arrayFinalFlux = arrayFinalFlux[~phase_clip_mask] + arrayNormUnc = arrayNormUnc[~phase_clip_mask] + arrayTimes = arrayTimes[~phase_clip_mask] + arrayJDTimes = arrayJDTimes[~phase_clip_mask] + arrayAirmass = arrayAirmass[~phase_clip_mask] + f1 = f1[~phase_clip_mask] + f2 = f2[~phase_clip_mask] + + myfit = lc_fitter( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + arrayAirmass, + prior, + mybounds, + jd_times=arrayJDTimes, + mode='lm' + ) + return myfit, f1, f2 +def cheap_lightcurve_prescore(tFlux, cFlux, airmass): + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio = np.divide(tFlux, cFlux) + + finite_mask = np.isfinite(flux_ratio) & np.isfinite(airmass) & (flux_ratio > 0) + if not np.allclose(cFlux, 1.0): + finite_mask &= relative_flux_filter_mask(flux_ratio) + if np.count_nonzero(finite_mask) < 5: + return np.inf + + x_vals = airmass[finite_mask] + y_vals = flux_ratio[finite_mask] + + if np.ptp(x_vals) == 0: + detrended = y_vals / bn.nanmedian(y_vals) + else: + slope, intercept = np.polyfit(x_vals, y_vals, 1) + trend = slope * x_vals + intercept + with np.errstate(divide='ignore', invalid='ignore'): + detrended = np.divide(y_vals, trend) + + return bn.nanstd(detrended) + + +def evaluate_lightcurve_candidate(task): + times, tflux, cflux, airmass, ld, p_dict, jd_times = task + myfit, tflux_fit, cflux_fit = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + allow_mid_transit_range_warning=False, + ) + if myfit is None: + return None, tflux_fit, cflux_fit + + res_std = myfit.residuals.std() / np.median(myfit.data) + return { + 'myfit': myfit, + 'res_std': res_std, + }, tflux_fit, cflux_fit + + +def normalize_flux_series(flux_values): + flux_values = np.asarray(flux_values, dtype=float) + normalized = np.full(flux_values.shape, np.nan, dtype=float) + finite_mask = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(finite_mask) < 5: + return normalized + + flux_median = bn.nanmedian(flux_values[finite_mask]) + if not np.isfinite(flux_median) or flux_median <= 0: + return normalized + + normalized[finite_mask] = flux_values[finite_mask] / flux_median + return normalized + + +def normalized_ratio_series(numerator_flux, denominator_flux): + numerator_flux = np.asarray(numerator_flux, dtype=float) + denominator_flux = np.asarray(denominator_flux, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + ratio = np.divide(numerator_flux, denominator_flux) + ratio[~np.isfinite(ratio)] = np.nan + ratio[ratio <= 0] = np.nan + ratio[ratio > RELATIVE_FLUX_MAX] = np.nan + return ratio + + +def build_normalized_comp_ensemble(normalized_flux_map, exclude_key): + ensemble_members = [flux for key, flux in normalized_flux_map.items() if key != exclude_key] + if not ensemble_members: + return None + + ensemble_stack = np.vstack(ensemble_members) + valid_mask = np.any(np.isfinite(ensemble_stack), axis=0) + if not np.any(valid_mask): + return None + + ensemble = np.full(ensemble_stack.shape[1], np.nan, dtype=float) + ensemble[valid_mask] = np.nanmedian(ensemble_stack[:, valid_mask], axis=0) + return ensemble + + +def comparison_star_stability_summary(comp_flux_map, airmass): + if not comp_flux_map: + return { + 'pairwise_matrix': np.empty((0, 0), dtype=float), + 'comp_summaries': [], + 'field_score': np.inf, + 'best_comp_index': None, + 'best_comp_score': np.inf, + } + + comp_keys = list(comp_flux_map.keys()) + normalized_flux_map = {key: normalize_flux_series(comp_flux_map[key]) for key in comp_keys} + pairwise_matrix = np.full((len(comp_keys), len(comp_keys)), np.nan, dtype=float) + comp_summaries = [] + + for i, key in enumerate(comp_keys): + normalized_flux = normalized_flux_map[key] + self_score = cheap_lightcurve_prescore(normalized_flux, np.ones(normalized_flux.shape[0]), airmass) + pairwise_scores = [] + pairwise_series = {} + + for j, other_key in enumerate(comp_keys): + if i == j: + continue + other_flux = normalized_flux_map[other_key] + score = cheap_lightcurve_prescore(normalized_flux, other_flux, airmass) + pairwise_matrix[i, j] = score + pairwise_series[f"vs {j + 1}"] = normalized_ratio_series(normalized_flux, other_flux) + if np.isfinite(score): + pairwise_scores.append(float(score)) + + ensemble_flux = build_normalized_comp_ensemble(normalized_flux_map, key) + ensemble_score = np.inf + ensemble_ratio_series = np.full(normalized_flux.shape, np.nan, dtype=float) + if ensemble_flux is not None: + ensemble_score = cheap_lightcurve_prescore(normalized_flux, ensemble_flux, airmass) + ensemble_ratio_series = normalized_ratio_series(normalized_flux, ensemble_flux) + + if pairwise_scores: + pairwise_median = float(np.nanmedian(pairwise_scores)) + pairwise_max = float(np.nanmax(pairwise_scores)) + pairwise_upper = float(np.nanpercentile(pairwise_scores, 75)) + else: + pairwise_median = np.inf + pairwise_max = np.inf + pairwise_upper = np.inf + + aggregate_inputs = [score for score in (ensemble_score, pairwise_upper) if np.isfinite(score)] + aggregate_score = max(aggregate_inputs) if aggregate_inputs else self_score + + comp_summaries.append({ + 'comp_index': i, + 'key': key, + 'label': f"Comp {i + 1}", + 'pairwise_median_score': pairwise_median, + 'pairwise_max_score': pairwise_max, + 'ensemble_score': float(ensemble_score) if np.isfinite(ensemble_score) else np.inf, + 'self_score': float(self_score) if np.isfinite(self_score) else np.inf, + 'aggregate_score': float(aggregate_score) if np.isfinite(aggregate_score) else np.inf, + 'valid_pair_count': len(pairwise_scores), + 'pairwise_ratio_series': pairwise_series, + 'ensemble_ratio_series': ensemble_ratio_series, + }) + + finite_comp_scores = [summary['aggregate_score'] for summary in comp_summaries if np.isfinite(summary['aggregate_score'])] + field_score = float(np.nanmedian(finite_comp_scores)) if finite_comp_scores else np.inf + best_comp_index = None + best_comp_score = np.inf + for summary in comp_summaries: + if summary['aggregate_score'] < best_comp_score: + best_comp_score = summary['aggregate_score'] + best_comp_index = summary['comp_index'] + + return { + 'pairwise_matrix': pairwise_matrix, + 'comp_summaries': comp_summaries, + 'field_score': field_score, + 'best_comp_index': best_comp_index, + 'best_comp_score': best_comp_score, + } + + +def comparison_field_sort_key(summary): + return ( + np.inf if summary.get('field_score') is None else summary['field_score'], + np.inf if summary.get('best_comp_score') is None else summary['best_comp_score'], + ) + + +def initialize_aperture_data_store(frame_count, aperture_count, annulus_count, comp_star_count): + aper_shape = (frame_count, aperture_count, annulus_count) + aper_data = { + 'target': np.full(aper_shape, np.nan, dtype=float), + 'target_bg': np.full(aper_shape, np.nan, dtype=float), + } + + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + aper_data[ckey] = np.full(aper_shape, np.nan, dtype=float) + aper_data[f"{ckey}_bg"] = np.full(aper_shape, np.nan, dtype=float) + + return aper_data + + +def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=True): + flux_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) + bg_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) + + if np.isnan(xc) or np.isnan(yc): + return flux_grid, bg_grid + + mask_method = 'center' if fast_mode else 'exact' + + for a_idx, aperture_radius in enumerate(apertures): + aperture = CircularAperture(positions=[(xc, yc)], r=float(aperture_radius)) + mask = aperture.to_mask(method=mask_method)[0] + data_cutout = mask.cutout(data) + + mask_area = None + raw_aperture_sum = None + if data_cutout is not None: + mask_area = np.sum(mask.data) + raw_aperture_sum = (mask.data * data_cutout).sum() + + for an_idx, annulus_width in enumerate(annuli): + stage_start = perf_counter() + try: + if annulus_width > 0: + bgflux, _, _ = skybg_phot(data, star_index, xc, yc, float(aperture_radius) + 2, float(annulus_width)) + else: + bgflux = 0 + + bg_grid[a_idx, an_idx] = bgflux + + if data_cutout is None: + flux_grid[a_idx, an_idx] = 0 + else: + flux_grid[a_idx, an_idx] = raw_aperture_sum - bgflux * mask_area + finally: + _record_photometry_stage_timing('aperPhot', perf_counter() - stage_start) + + return flux_grid, bg_grid + + +def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_star_count, aper_data, apertures, annuli, + fast_aperture_mask): + target_flux, target_bg = compute_star_aperture_grid( + image_data, + 0, + psf_data['target'][frame_index, 0], + psf_data['target'][frame_index, 1], + apertures, + annuli, + fast_mode=fast_aperture_mask, + ) + aper_data['target'][frame_index] = target_flux + aper_data['target_bg'][frame_index] = target_bg + + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + comp_flux, comp_bg = compute_star_aperture_grid( + image_data, + comp_idx + 1, + psf_data[ckey][frame_index, 0], + psf_data[ckey][frame_index, 1], + apertures, + annuli, + fast_mode=fast_aperture_mask, + ) + aper_data[ckey][frame_index] = comp_flux + aper_data[f"{ckey}_bg"][frame_index] = comp_bg + + +def load_calibrated_reduction_image(file_name, generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult): + hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + image_data = hdul[extension].data + hdul.close() + + image_data = apply_cals(image_data, generalDark, generalBias, generalFlat, 1) + image_data = demosaic_img(image_data, demosaic_fmt, demosaic_out, demosaic_mult, 1) + return image_data + + +def _refined_sigma_grid(center, lower_bound, upper_bound, half_width, points): + low = max(lower_bound, center - half_width) + high = min(upper_bound, center + half_width) + if high <= low: + low, high = lower_bound, upper_bound + return np.linspace(low, high, points) + + +def auto_tune_aperture_sigma_grid(coarse_apertures_sigma, coarse_annuli_sigma, coarse_aper_data, comp_star_count, + subset_airmass, require_comp_star=True): + best_candidate = None + best_score = np.inf + + for a_idx, aperture_sigma in enumerate(coarse_apertures_sigma): + for an_idx, annulus_sigma in enumerate(coarse_annuli_sigma): + comp_flux_map = { + f"comp{comp_idx + 1}": coarse_aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx] + for comp_idx in range(comp_star_count) + } + field_summary = comparison_star_stability_summary(comp_flux_map, subset_airmass) + field_score = field_summary['field_score'] + if np.isfinite(field_score) and comparison_field_sort_key(field_summary) < (best_score, np.inf): + best_score = field_score + best_candidate = { + 'aper_sigma': float(aperture_sigma), + 'annulus_sigma': float(annulus_sigma), + 'comp_index': field_summary['best_comp_index'], + } + + if best_candidate is None: + center_aper_sigma = float(np.median(coarse_apertures_sigma)) + center_annulus_sigma = float(np.median(coarse_annuli_sigma)) + fallback_comp_index = None + if comp_star_count > 0: + fallback_comp_index = 0 + elif require_comp_star: + fallback_comp_index = None + best_candidate = { + 'aper_sigma': center_aper_sigma, + 'annulus_sigma': center_annulus_sigma, + 'comp_index': fallback_comp_index, + } + else: + center_aper_sigma = best_candidate['aper_sigma'] + center_annulus_sigma = best_candidate['annulus_sigma'] + + refined_apertures_sigma = _refined_sigma_grid( + center_aper_sigma, + APERTURE_SIGMA_MIN, + APERTURE_SIGMA_MAX, + APERTURE_AUTOTUNE_APER_HALF_WIDTH_SIGMA, + APERTURE_AUTOTUNE_REFINED_APER_POINTS, + ) + refined_annuli_sigma = _refined_sigma_grid( + center_annulus_sigma, + ANNULUS_SIGMA_MIN, + ANNULUS_SIGMA_MAX, + APERTURE_AUTOTUNE_ANNULUS_HALF_WIDTH_SIGMA, + APERTURE_AUTOTUNE_REFINED_ANNULUS_POINTS, + ) + + return refined_apertures_sigma, refined_annuli_sigma, best_candidate, best_score + + +def comparison_method_label(candidate): + if candidate.get('method') == 'psf': + return "PSF photometry" + return f"Aperture photometry (aper={candidate['aper']:.2f}px, annulus={candidate['annulus']:.2f}px)" + + +def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, airmass, comp_stars, sigma): + candidate_summaries = [] + comp_star_count = len(comp_stars) + + if comp_star_count == 0: + return None + + psf_flux_map = { + f"comp{comp_idx + 1}": 2 * np.pi * psf_data[f"comp{comp_idx + 1}"][:, 2] + * psf_data[f"comp{comp_idx + 1}"][:, 3] + * psf_data[f"comp{comp_idx + 1}"][:, 4] + for comp_idx in range(comp_star_count) + } + psf_summary = comparison_star_stability_summary(psf_flux_map, airmass) + psf_summary.update({ + 'method': 'psf', + 'a': None, + 'an': None, + 'aper': 0.0, + 'annulus': float(15 * sigma), + }) + candidate_summaries.append(psf_summary) + + for a_idx, aperture in enumerate(apers): + for an_idx, annulus in enumerate(annuli): + comp_flux_map = { + f"comp{comp_idx + 1}": aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx] + for comp_idx in range(comp_star_count) + } + candidate_summary = comparison_star_stability_summary(comp_flux_map, airmass) + candidate_summary.update({ + 'method': 'aperture', + 'a': a_idx, + 'an': an_idx, + 'aper': float(aperture), + 'annulus': float(annulus), + }) + candidate_summaries.append(candidate_summary) + + finite_candidates = [ + candidate for candidate in candidate_summaries + if np.isfinite(candidate['field_score']) and candidate['best_comp_index'] is not None + ] + if not finite_candidates: + return None + + finite_candidates.sort(key=comparison_field_sort_key) + best_candidate = finite_candidates[0] + best_comp_index = best_candidate['best_comp_index'] + method_label = comparison_method_label(best_candidate) + comp_summaries = [] + for summary in best_candidate['comp_summaries']: + comp_summary = dict(summary) + comp_summary['position'] = comp_stars[comp_summary['comp_index']] + comp_summary['selected'] = comp_summary['comp_index'] == best_comp_index + comp_summaries.append(comp_summary) + + best_candidate['comp_summaries'] = comp_summaries + best_candidate['method_label'] = method_label + return best_candidate + + def parse_args(): parser = argparse.ArgumentParser(description="Using a JSON initialization file to bypass user inputs for EXOTIC.") parser.add_argument('-rt', '--realtime', @@ -1814,12 +3272,38 @@ def parse_args(): "Can be used as an additional argument with -rt (--realtime), -red (--reduce), " "-pre (--prereduced), and -phot (--photometry)." "Do not combine with the -ov, --override argument.") + parser.add_argument('--use-nextastro-astrometry', + action='store_true', + help="Use NextAstro's astrometry service (https://astrometry.nextastro.org/) instead of nova.astrometry.net for plate solving.") + parser.add_argument('--use-nextastro-variability-server', + action='store_true', + help="Use NextAstro's variability server for a batch comparison-star variability check. " + "If the service returns an error, EXOTIC falls back to individual VSX checks.") + parser.add_argument('--non-interactive-run', + action='store_true', + help="Run without interactive prompts for target pixel-coordinate mismatch checks. " + "If a mismatch is detected, EXOTIC logs a warning and proceeds with the " + "user-provided coordinates.") + parser.add_argument('--multiprocess-transformations', + type=int, + default=None, + help="Use multiprocessing when finding image transformations. " + "Provide an integer number of processes to use.") + parser.add_argument('--multiprocess-lightcurve-fits', + type=int, + default=None, + help="Use multiprocessing while evaluating candidate lightcurve fits. " + "Provide an integer number of processes to use.") return parser.parse_args() def main(): # command line args args = parse_args() + if args.multiprocess_transformations is not None and args.multiprocess_transformations < 1: + raise ValueError("--multiprocess-transformations requires an integer greater than 0.") + if args.multiprocess_lightcurve_fits is not None and args.multiprocess_lightcurve_fits < 1: + raise ValueError("--multiprocess-lightcurve-fits requires an integer greater than 0.") log.debug("*************************") log.debug("EXOTIC reduction log file") @@ -1893,7 +3377,12 @@ def main(): ax.set_ylabel('Normalized Flux') ax.set_xlabel('Time (JD)') - anim = FuncAnimation(fig, realTimeReduce, fargs=(userpDict['pName'], userpDict, exotic_infoDict, ax), interval=15000) + anim = FuncAnimation( + fig, + realTimeReduce, + fargs=(userpDict['pName'], userpDict, exotic_infoDict, ax, args.use_nextastro_astrometry, args.multiprocess_transformations), + interval=15000 + ) plt.show() # ----USER INPUTS---------------------------------------------------------- @@ -2088,7 +3577,16 @@ def main(): jd_times = jd_times[inc:] plateStatus.setCurrentFilename(inputfiles[0]) - wcs_file = check_wcs(inputfiles[0], exotic_infoDict['save'], exotic_infoDict['plate_opt']) + # For astrometry hints, prioritize coordinates explicitly provided by the user + # (from inits.json / CLI) over values scraped from NASA Exoplanet Archive. + hint_ra = userpDict.get('ra', pDict.get('ra')) + hint_dec = userpDict.get('dec', pDict.get('dec')) + ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) + + wcs_file = check_wcs(inputfiles[0], exotic_infoDict['save'], exotic_infoDict['plate_opt'], + use_nextastro_astrometry=args.use_nextastro_astrometry, + ra=hint_ra, dec=hint_dec, pixel_scale=exotic_infoDict.get('pixel_scale'), + ignore_header_wcs=ignore_header_wcs) img_scale_str, img_scale = get_img_scale(header, wcs_file, exotic_infoDict['pixel_scale']) plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) ra_dec_tar, ra_dec_wcs = None, [] @@ -2102,13 +3600,16 @@ def main(): exotic_UIprevTPX, exotic_UIprevTPY = check_target_pixel_wcs(exotic_UIprevTPX, exotic_UIprevTPY, pDict, ra_wcs, dec_wcs, fits.getdata(inputfiles[0]), - jd_times[0]) + jd_times[0], + non_interactive_run=args.non_interactive_run, + wcs_header=wcs_header) ra_dec_tar = (ra_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], dec_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) auid = vsx_auid(ra_dec_tar[0], ra_dec_tar[1]) - check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) + check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], + use_nextastro_variability_server=args.use_nextastro_variability_server) if exotic_infoDict['aavso_comp'] == 'y': vsp_comp_stars, chart_id = vsp_query(wcs_file,[header['NAXIS1'], header['NAXIS2']], @@ -2121,41 +3622,82 @@ def main(): log_info("\nThere are no comparison stars left as all of them were indicated as variable stars." "\nPlease reenter new comparison star coordinates.") exotic_infoDict['comp_stars'] = comparison_star_coords(exotic_infoDict['comp_stars'], False) - check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) + check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], + use_nextastro_variability_server=args.use_nextastro_variability_server) # Build RA/Dec for comp after list is finalized (avoid off by one issues, etc ra_dec_wcs = build_comp_ra_dec(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) - # aperture sizes in stdev (sigma) of PSF - apers = np.linspace(1.5, 6, 20) - annuli = np.linspace(6, 15, 19) - # alloc psf fitting param psf_data = { # x-cent, y-cent, amplitude, sigma-x, sigma-y, rotation, offset 'target': np.zeros((len(inputfiles), 7)), # PSF fit } - aper_data = { - 'target': np.zeros((len(inputfiles), len(apers), len(annuli))), - 'target_bg': np.zeros((len(inputfiles), len(apers), len(annuli))) - } tar_comp_dist = {} vsp_num = [] + comp_star_count = len(exotic_infoDict['comp_stars']) + require_comp_star = is_comp_star_required(exotic_infoDict.get('require_comp_star', 'y')) + target_driven_comp_selection = is_target_driven_comp_selection_enabled( + exotic_infoDict.get('target_driven_comp_selection', 'n') + ) for i, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{i + 1}" if coord in vsp_list: vsp_num.append(i) psf_data[ckey] = np.zeros((len(inputfiles), 7)) - aper_data[ckey] = np.zeros((len(inputfiles), len(apers), len(annuli))) - aper_data[f"{ckey}_bg"] = np.zeros((len(inputfiles), len(apers), len(annuli))) tar_comp_dist[ckey] = np.zeros(2) + coarse_tune_frames = min(len(inputfiles), APERTURE_AUTOTUNE_MAX_FRAMES) + if len(inputfiles) >= APERTURE_AUTOTUNE_MIN_FRAMES: + coarse_tune_frames = max(APERTURE_AUTOTUNE_MIN_FRAMES, coarse_tune_frames) + coarse_apertures_sigma = np.linspace(APERTURE_SIGMA_MIN, APERTURE_SIGMA_MAX, APERTURE_AUTOTUNE_COARSE_APER_POINTS) + coarse_annuli_sigma = np.linspace(ANNULUS_SIGMA_MIN, ANNULUS_SIGMA_MAX, APERTURE_AUTOTUNE_COARSE_ANNULUS_POINTS) + log_info( + "Automatic aperture tuning enabled: " + f"coarse_grid={len(coarse_apertures_sigma)}x{len(coarse_annuli_sigma)}, " + f"coarse_frames={coarse_tune_frames}." + ) + + sigma = None + coarse_apertures = None + coarse_annuli = None + apers = None + annuli = None + aperture_grid_tuned = False + coarse_aper_data = initialize_aperture_data_store( + coarse_tune_frames, + len(coarse_apertures_sigma), + len(coarse_annuli_sigma), + comp_star_count, + ) + aper_data = None + coarse_frame_cache = [None] * coarse_tune_frames + + use_multiprocess_transform_precompute = should_use_multiprocess_transform_precompute( + inputfiles, args.multiprocess_transformations, ignore_header_wcs=ignore_header_wcs + ) + fallback_transforms = {} + if use_multiprocess_transform_precompute: + fallback_transforms = build_multiprocess_transformations(inputfiles, args.multiprocess_transformations) + + target_and_comp_radec = None + if ra_dec_tar is not None and ra_dec_wcs: + target_and_comp_radec = np.array([ra_dec_tar, *ra_dec_wcs], dtype=float) + target_and_comp_pixels = np.array( + [[exotic_UIprevTPX, exotic_UIprevTPY], *exotic_infoDict['comp_stars']], + dtype=float, + ) + fast_aperture_mask = is_fast_aperture_mask_enabled(exotic_infoDict.get('fast_aperture_mask')) + # open files, calibrate, align, photometry + reset_transform_timing_stats() + reset_photometry_timing_stats() for i, fileName in enumerate(inputfiles): plateStatus.setCurrentFilename(fileName) hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + frame_fast_centroid = should_use_fast_centroid(i) extension = 0 image_header = hdul[extension].header @@ -2179,58 +3721,119 @@ def main(): if i == 0: firstImage = np.copy(imageData) - sys.stdout.write(f"Finding transformation {i + 1} of {len(inputfiles)} : {fileName}\n") - log.debug(f"Finding transformation {i + 1} of {len(inputfiles)} : {fileName}\n") - sys.stdout.flush() + log_finding_transformation_progress( + i, + len(inputfiles), + fileName, + use_multiprocess_transform_precompute, + ) - try: - wcs_hdr = search_wcs(fileName) - if not wcs_hdr.is_celestial: - raise Exception - - if i == 0: - tx, ty = exotic_UIprevTPX, exotic_UIprevTPY - else: - pix_coords = wcs_hdr.world_to_pixel_values(ra_dec_tar[0], ra_dec_tar[1]) - tx, ty = pix_coords[0].take(0), pix_coords[1].take(0) + use_wcs_alignment = False + if not ignore_header_wcs: + try: + wcs_hdr = search_wcs_from_header(image_header) + use_wcs_alignment = wcs_hdr.is_celestial + except Exception: + use_wcs_alignment = False - psf_data['target'][i] = fit_centroid(imageData, [tx, ty], 0) + if use_wcs_alignment: + try: + pix_x = pix_y = None + if target_and_comp_radec is not None: + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) - # TODO: Add check for flux on target/comp stars relative to others in the field - # in case of cloudy data, large changes, etc. - if i != 0 and np.abs((psf_data['target'][i][2] - psf_data['target'][i - 1][2]) - / psf_data['target'][i - 1][2]) > 0.5: - raise Exception + if i == 0: + tx, ty = exotic_UIprevTPX, exotic_UIprevTPY + else: + tx, ty = pix_x[0], pix_y[0] + + projected_coords = np.array( + [[tx, ty], *np.column_stack((pix_x[1:], pix_y[1:]))] if pix_x is not None else [[tx, ty]], + dtype=float, + ) + projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) + + psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + [tx, ty], + 0, + fast_mode=frame_fast_centroid, + ) + + # TODO: Add check for flux on target/comp stars relative to others in the field + # in case of cloudy data, large changes, etc. + target_flux_change_ok = True + if not projected_off_frame and i != 0: + target_flux_change_ok = fractional_flux_change_within_limit( + psf_data['target'][i][2], + psf_data['target'][i - 1][2], + ) + + comp_valid = True + for j in range(len(exotic_infoDict['comp_stars'])): + ckey = f"comp{j + 1}" - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" + cx, cy = pix_x[j + 1], pix_y[j + 1] + psf_data[ckey][i] = fit_centroid_or_warn_out_of_frame( + imageData, + [cx, cy], + j + 1, + fast_mode=frame_fast_centroid, + ) + + if projected_off_frame: + continue + if i != 0: + comp_valid = comp_valid and ( + centroid_offset_matches_reference( + psf_data[ckey][i], + psf_data['target'][i], + tar_comp_dist[ckey][0], + tar_comp_dist[ckey][1], + ) + and fractional_flux_change_within_limit( + psf_data[ckey][i][2], + psf_data[ckey][i - 1][2], + ) + ) + else: + tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) + tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) - pix_coords = wcs_hdr.world_to_pixel_values(ra_dec_wcs[j][0], ra_dec_wcs[j][1]) - cx, cy = pix_coords[0].take(0), pix_coords[1].take(0) - psf_data[ckey][i] = fit_centroid(imageData, [cx, cy], j+1) + use_wcs_alignment = projected_off_frame or (target_flux_change_ok and comp_valid) + except Exception: + use_wcs_alignment = False - if i != 0: - if not (tar_comp_dist[ckey][0] - 1 <= abs(int(psf_data[ckey][i][0]) - int(psf_data['target'][i][0])) <= tar_comp_dist[ckey][0] + 1 and - tar_comp_dist[ckey][1] - 1 <= abs(int(psf_data[ckey][i][1]) - int(psf_data['target'][i][1])) <= tar_comp_dist[ckey][1] + 1) or \ - np.abs((psf_data[ckey][i][2] - psf_data[ckey][i - 1][2]) / psf_data[ckey][i - 1][2]) > 0.5: - raise Exception - else: - tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) - except Exception: + if not use_wcs_alignment: if i == 0: tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) else: - tform = transformation(np.array([imageData, firstImage]), fileName) + tform = fallback_transforms[i] if i in fallback_transforms else transformation(imageData, fileName, reference_image=firstImage) - tx, ty = tform([exotic_UIprevTPX, exotic_UIprevTPY])[0] - psf_data['target'][i] = fit_centroid(imageData, [tx, ty], 0) + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + tx, ty = transformed_coords[0] + psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + [tx, ty], + 0, + fast_mode=frame_fast_centroid, + ) for j, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{j + 1}" - cx, cy = tform(coord)[0] - psf_data[ckey][i] = fit_centroid(imageData, [cx, cy], j+1) + cx, cy = transformed_coords[j + 1] + psf_data[ckey][i] = fit_centroid_or_warn_out_of_frame( + imageData, + [cx, cy], + j + 1, + fast_mode=frame_fast_centroid, + ) if i == 0: tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) @@ -2239,35 +3842,109 @@ def main(): # aperture photometry if i == 0: sigma = float((psf_data['target'][0][3] + psf_data['target'][0][4]) * 0.5) - apers *= sigma - annuli *= sigma - - for a, aper in enumerate(apers): - for an, annulus in enumerate(annuli): - if not np.isnan(psf_data['target'][i, 0]): - aper_data["target"][i][a][an], aper_data["target_bg"][i][a][an] = aperPhot(imageData, 0, - psf_data['target'][i, 0], - psf_data['target'][i, 1], - aper, annulus) - else: - aper_data["target"][i][a][an] = np.nan - aper_data["target_bg"][i][a][an] = np.nan - # loop through comp stars - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - if not np.isnan(psf_data[ckey][i][0]): - aper_data[ckey][i][a][an], \ - aper_data[f"{ckey}_bg"][i][a][an] = aperPhot(imageData, j + 1, psf_data[ckey][i, 0], - psf_data[ckey][i, 1], aper, annulus) - else: - aper_data[ckey][i][a][an] = np.nan - aper_data[f"{ckey}_bg"][i][a][an] = np.nan + if not np.isfinite(sigma) or sigma <= 0: + log_info("Warning: Initial PSF sigma is invalid; using sigma=1.0 for automatic aperture tuning.", warn=True) + sigma = 1.0 + coarse_apertures = coarse_apertures_sigma * sigma + coarse_annuli = coarse_annuli_sigma * sigma + + if i < coarse_tune_frames: + coarse_frame_cache[i] = np.array(imageData, copy=True) + populate_aperture_data_for_frame( + imageData, + i, + psf_data, + comp_star_count, + coarse_aper_data, + coarse_apertures, + coarse_annuli, + fast_aperture_mask, + ) + + if i == coarse_tune_frames - 1: + subset_airmass = np.asarray(airMassList[:coarse_tune_frames], dtype=float) + refined_apertures_sigma, refined_annuli_sigma, best_coarse_candidate, best_coarse_score = auto_tune_aperture_sigma_grid( + coarse_apertures_sigma, + coarse_annuli_sigma, + coarse_aper_data, + comp_star_count, + subset_airmass, + require_comp_star=require_comp_star, + ) + apers = refined_apertures_sigma * sigma + annuli = refined_annuli_sigma * sigma + aper_data = initialize_aperture_data_store(len(inputfiles), len(apers), len(annuli), comp_star_count) + aperture_grid_tuned = True + + best_comp_label = "none" + if best_coarse_candidate['comp_index'] is not None: + best_comp_label = str(best_coarse_candidate['comp_index'] + 1) + score_text = "n/a" if not np.isfinite(best_coarse_score) else f"{best_coarse_score:.5f}" + log_info( + "Auto-tuned aperture grid: " + f"coarse_best=(aper={best_coarse_candidate['aper_sigma']:.2f} sigma, " + f"annulus={best_coarse_candidate['annulus_sigma']:.2f} sigma, comp={best_comp_label}, score={score_text}), " + f"refined_grid={len(refined_apertures_sigma)}x{len(refined_annuli_sigma)}." + ) + + log_info(f"Backfilling refined aperture photometry for the first {coarse_tune_frames} frame(s).") + for backfill_idx in range(coarse_tune_frames): + backfill_image = coarse_frame_cache[backfill_idx] + loaded_from_disk = False + if backfill_image is None: + backfill_image = load_calibrated_reduction_image( + inputfiles[backfill_idx], + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ) + loaded_from_disk = True + try: + populate_aperture_data_for_frame( + backfill_image, + backfill_idx, + psf_data, + comp_star_count, + aper_data, + apers, + annuli, + fast_aperture_mask, + ) + finally: + if loaded_from_disk: + del backfill_image + coarse_frame_cache[backfill_idx] = None + else: + if not aperture_grid_tuned: + # Defensive fallback for unexpected control flow. + apers = coarse_apertures + annuli = coarse_annuli + aper_data = initialize_aperture_data_store(len(inputfiles), len(apers), len(annuli), comp_star_count) + aperture_grid_tuned = True + + populate_aperture_data_for_frame( + imageData, + i, + psf_data, + comp_star_count, + aper_data, + apers, + annuli, + fast_aperture_mask, + ) # close file + delete from memory hdul.close() del hdul del imageData + log_transform_timing_stats('Transformation timing summary (full reduction)') + log_photometry_timing_stats('Photometry timing summary (full reduction)') + log_reduction_timing_overview('Reduction timing overview (full reduction)') + # filter bad images badmask = np.isnan(psf_data["target"][:, 0]) | (psf_data["target"][:, 0] == 0) | (aper_data["target"][:, 0, 0] == 0) | np.isnan( aper_data["target"][:, 0, 0]) @@ -2323,70 +4000,169 @@ def main(): 'comp_star_coords': None, 'min_std': 100000, 'min_aperture': None, - 'min_annulus': None + 'min_annulus': None, + 'calibration_field_score': np.inf, + 'selection_basis': 'target_fit', } - # loop over comp stars - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" + comparison_calibration = None + if target_driven_comp_selection: + log_info("\nUsing target-driven comparison-star selection per optional_info setting.") + else: + comparison_calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + apers, + annuli, + airmass, + exotic_infoDict['comp_stars'], + sigma, + ) + + if comparison_calibration is not None: + log_info("\nCalibrating comparison stars before target fitting. Please wait.") + log_info(f"Comparison-star field method: {comparison_calibration['method_label']}") + log_info(f"Comparison-star field score: {comparison_calibration['field_score'] * 100.0:.4f}%") + for summary in comparison_calibration['comp_summaries']: + aggregate_text = "n/a" if not np.isfinite(summary['aggregate_score']) else f"{summary['aggregate_score'] * 100.0:.4f}%" + ensemble_text = "n/a" if not np.isfinite(summary['ensemble_score']) else f"{summary['ensemble_score'] * 100.0:.4f}%" + pairwise_text = "n/a" if not np.isfinite(summary['pairwise_median_score']) else f"{summary['pairwise_median_score'] * 100.0:.4f}%" + selected_label = " [selected]" if summary['selected'] else "" + log_info( + f" {summary['label']}{selected_label}: suitability={aggregate_text}, " + f"ensemble={ensemble_text}, pairwise_median={pairwise_text}, " + f"valid_pairs={summary['valid_pair_count']}" + ) - cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] - myfit, tFlux1, cFlux1 = fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times) + try: + plot_comp_star_pairwise_matrix( + comparison_calibration['pairwise_matrix'], + comparison_calibration['best_comp_index'], + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + ) + plot_comp_star_calibration_series( + times, + comparison_calibration['comp_summaries'], + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + ) + plot_comp_star_suitability( + comparison_calibration['comp_summaries'], + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + ) + save_comp_star_calibration_summary( + exotic_infoDict['save'], + pDict['pName'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + comparison_calibration['field_score'], + comparison_calibration['comp_summaries'], + comparison_calibration['best_comp_index'], + ) + except Exception as e: + log_info(f"Warning: Could not save comparison-star calibration outputs ({e}).", warn=True) + + selected_comp_index = comparison_calibration['best_comp_index'] + selected_ckey = f"comp{selected_comp_index + 1}" + selected_comp_coords = exotic_infoDict['comp_stars'][selected_comp_index] + selected_min_aperture = 0 if comparison_calibration['method'] == 'psf' else comparison_calibration['aper'] + selected_min_annulus = comparison_calibration['annulus'] + + if comparison_calibration['method'] == 'psf': + selected_target_flux = tFlux + selected_comp_flux = ( + 2 * np.pi * psf_data[selected_ckey][:, 2] * psf_data[selected_ckey][:, 3] * psf_data[selected_ckey][:, 4] + ) + else: + best_a = comparison_calibration['a'] + best_an = comparison_calibration['an'] + selected_target_flux = aper_data['target'][:, best_a, best_an] + selected_comp_flux = aper_data[selected_ckey][:, best_a, best_an] + myfit, tFlux1, cFlux1 = fit_lightcurve(times, selected_target_flux, selected_comp_flux, airmass, ld, pDict, jd_times) if myfit is not None: - for k in myfit.bounds.keys(): - log.debug(f" {k}: {myfit.parameters[k]:.6f}") - - log.debug("The Residual Standard Deviation is: " - f"{round(100 * myfit.residuals.std() / np.median(myfit.data), 6)}%") - log.debug(f"The Mean Squared Error is: {round(np.sum(myfit.residuals ** 2), 6)}\n") - res_std = myfit.residuals.std() / np.median(myfit.data) - - if photometry_info['min_std'] > res_std and myfit is not None: - photometry_info.update(best_fit_lc=copy.deepcopy(myfit), - comp_star_num=j + 1, comp_star_coords=exotic_infoDict['comp_stars'][j], - min_std=res_std, min_aperture=0, min_annulus=15 * sigma) + photometry_info.update(best_fit_lc=myfit, + comp_star_num=selected_comp_index + 1, + comp_star_coords=selected_comp_coords, + min_std=res_std, + min_aperture=selected_min_aperture, + min_annulus=selected_min_annulus, + calibration_field_score=comparison_calibration['field_score'], + selection_basis='comparison_field') flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], - x_ref=psf_data[ckey][:, 0], y_ref=psf_data[ckey][:, 1]) - - if j in vsp_num: - ref_flux[j] = { - 'myfit': copy.deepcopy(myfit), - 'pos': exotic_infoDict['comp_stars'][j] - } - - log_info("\nComputing best comparison star, aperture, and sky annulus. Please wait.") + x_ref=psf_data[selected_ckey][:, 0], y_ref=psf_data[selected_ckey][:, 1]) + + if selected_comp_index in vsp_num: + ref_flux[selected_comp_index] = { + 'myfit': myfit, + 'pos': exotic_infoDict['comp_stars'][selected_comp_index] + } + + if vsp_num: + if comparison_calibration['method'] == 'psf': + for j in vsp_num: + ckey = f"comp{j + 1}" + cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] + vsp_fit, _, _ = fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times) + ref_flux[j] = { + 'myfit': vsp_fit, + 'pos': exotic_infoDict['comp_stars'][j] + } + else: + best_a = comparison_calibration['a'] + best_an = comparison_calibration['an'] + best_target_flux = aper_data['target'][:, best_a, best_an] + for j in vsp_num: + ckey = f"comp{j + 1}" + aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) + cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] + vsp_fit, _, _ = fit_lightcurve(times[aper_mask], best_target_flux[aper_mask], cFlux, + airmass[aper_mask], ld, pDict, jd_times[aper_mask]) + ref_flux[j] = { + 'myfit': vsp_fit, + 'pos': exotic_infoDict['comp_stars'][j] + } + else: + log_info("Warning: Comparison-star calibration selected a photometry setup that failed target fitting." + " Falling back to target-driven photometry selection.", warn=True) - # Aperture Photometry - for a, aper in enumerate(apers): - for an, annulus in enumerate(annuli): - tFlux = aper_data['target'][:, a, an] - ref_flux_opt, ref_flux_opt2, backtrack = False, False, True - temp_ref_flux = {i: None for i in vsp_num} + if photometry_info['best_fit_lc'] is None: + # Legacy fallback when comparison-star-only calibration cannot determine a usable setup. + for j in range(len(exotic_infoDict['comp_stars'])): + ckey = f"comp{j + 1}" - # fit without a comparison star - myfit, tFlux1, cFlux1 = fit_lightcurve(times, tFlux, np.ones(tFlux.shape[0]), airmass, ld, pDict, jd_times) + cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] + myfit, tFlux1, cFlux1 = fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times) + res_std = np.inf if myfit is not None: for k in myfit.bounds.keys(): log.debug(f" {k}: {myfit.parameters[k]:.6f}") log.debug("The Residual Standard Deviation is: " - f"{round(100 * myfit.residuals.std() / np.median(myfit.data), 6)}%") + f"{round(100 * myfit.residuals.std() / np.median(myfit.data), 6)}%") log.debug(f"The Mean Squared Error is: {round(np.sum(myfit.residuals ** 2), 6)}\n") res_std = myfit.residuals.std() / np.median(myfit.data) - if photometry_info['min_std'] > res_std and myfit is not None: - ref_flux_opt = True - photometry_info.update(best_fit_lc=copy.deepcopy(myfit), - comp_star_num=None, comp_star_coords=None, - min_std=res_std, min_aperture=-aper, min_annulus=annulus) + if photometry_info['min_std'] > res_std and myfit is not None: + photometry_info.update(best_fit_lc=myfit, + comp_star_num=j + 1, comp_star_coords=exotic_infoDict['comp_stars'][j], + min_std=res_std, min_aperture=0, min_annulus=15 * sigma, + selection_basis='target_fit') flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) @@ -2394,69 +4170,150 @@ def main(): centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], x_ref=psf_data[ckey][:, 0], y_ref=psf_data[ckey][:, 1]) - # try to fit data with comp star - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - aper_mask = np.isfinite(aper_data[ckey][:, a, an]) - cFlux = aper_data[ckey][aper_mask][:, a, an] + if j in vsp_num: + ref_flux[j] = { + 'myfit': myfit, + 'pos': exotic_infoDict['comp_stars'][j] + } - myfit, tFlux1, cFlux1 = fit_lightcurve(times[aper_mask], tFlux[aper_mask], cFlux, airmass[aper_mask], ld, pDict, jd_times[aper_mask]) + log_info("\nComputing best comparison star, aperture, and sky annulus from the target lightcurve. Please wait.") - if myfit is not None: - if j in vsp_num: - temp_ref_flux[j] = { - 'myfit': copy.deepcopy(myfit), - 'pos': exotic_infoDict['comp_stars'][j] - } + candidate_jobs = [] + for a, aper in enumerate(apers): + for an, annulus in enumerate(annuli): + target_flux = aper_data['target'][:, a, an] + + if not require_comp_star: + candidate_jobs.append({ + 'a': a, + 'an': an, + 'aper': aper, + 'annulus': annulus, + 'comp_index': None, + 'ckey': None, + 'mask': np.ones(target_flux.shape[0], dtype=bool), + 'prescore': cheap_lightcurve_prescore(target_flux, np.ones(target_flux.shape[0]), airmass), + }) - for k in myfit.bounds.keys(): - log.debug(f" {k}: {myfit.parameters[k]:.6f}") + for j in range(len(exotic_infoDict['comp_stars'])): + ckey = f"comp{j + 1}" + comp_series = aper_data[ckey][:, a, an] + aper_mask = np.isfinite(comp_series) + comp_flux = comp_series[aper_mask] + candidate_jobs.append({ + 'a': a, + 'an': an, + 'aper': aper, + 'annulus': annulus, + 'comp_index': j, + 'ckey': ckey, + 'mask': aper_mask, + 'prescore': cheap_lightcurve_prescore(target_flux[aper_mask], comp_flux, airmass[aper_mask]), + }) + + finite_candidates = [c for c in candidate_jobs if np.isfinite(c['prescore'])] + if finite_candidates: + finite_candidates.sort(key=lambda candidate: candidate['prescore']) + shortlist_count = max(50, int(0.35 * len(finite_candidates))) + shortlist = finite_candidates[:min(len(finite_candidates), shortlist_count)] + else: + shortlist = [] - log.debug("The Residual Standard Deviation is: " - f"{round(100 * myfit.residuals.std() / np.median(myfit.data), 6)}%") - log.debug(f"The Mean Squared Error is: {round(np.sum(myfit.residuals ** 2), 6)}\n") + if not shortlist: + shortlist = candidate_jobs - res_std = myfit.residuals.std() / np.median(myfit.data) - if photometry_info['min_std'] > res_std and myfit is not None: # If the standard deviation is less than the previous min - ref_flux_opt2 = True + fit_tasks = [] + for candidate in shortlist: + candidate_mask = candidate['mask'] + target_flux = aper_data['target'][:, candidate['a'], candidate['an']][candidate_mask] + if candidate['comp_index'] is None: + comp_flux = np.ones(target_flux.shape[0]) + else: + comp_flux = aper_data[candidate['ckey']][:, candidate['a'], candidate['an']][candidate_mask] + + fit_tasks.append(( + times[candidate_mask], + target_flux, + comp_flux, + airmass[candidate_mask], + ld, + pDict, + jd_times[candidate_mask], + )) + + fit_results = [] + if args.multiprocess_lightcurve_fits is not None and args.multiprocess_lightcurve_fits > 0: + log_info(f"Using multiprocessing for candidate lightcurve fits ({args.multiprocess_lightcurve_fits} processes).") + with ProcessPoolExecutor(max_workers=args.multiprocess_lightcurve_fits) as executor: + fit_results = list(executor.map(evaluate_lightcurve_candidate, fit_tasks)) + else: + fit_results = [evaluate_lightcurve_candidate(task) for task in fit_tasks] - photometry_info.update(best_fit_lc=copy.deepcopy(myfit), - comp_star_num=j + 1, - comp_star_coords=exotic_infoDict['comp_stars'][j], - min_std=res_std, min_aperture=aper, min_annulus=annulus) + best_candidate = None + for candidate, result in zip(shortlist, fit_results): + fit_meta, tFlux1, cFlux1 = result + if fit_meta is None: + continue - flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, - flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) + myfit = fit_meta['myfit'] + res_std = fit_meta['res_std'] - centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], - x_ref=psf_data[ckey][:, 0], y_ref=psf_data[ckey][:, 1]) + if photometry_info['min_std'] > res_std: + best_candidate = candidate + photometry_info.update(best_fit_lc=myfit, + comp_star_num=(None if candidate['comp_index'] is None else candidate['comp_index'] + 1), + comp_star_coords=(None if candidate['comp_index'] is None else exotic_infoDict['comp_stars'][candidate['comp_index']]), + min_std=res_std, + min_aperture=(-candidate['aper'] if candidate['comp_index'] is None else candidate['aper']), + min_annulus=candidate['annulus'], + selection_basis='target_fit') - if ref_flux_opt or ref_flux_opt2: - if j in vsp_num: - ref_flux[j] = { - 'myfit': copy.deepcopy(myfit), - 'pos': exotic_infoDict['comp_stars'][j] - } + flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, + flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) + + x_ref_data = psf_data['target'][:, 0] + y_ref_data = psf_data['target'][:, 1] + if candidate['ckey'] is not None: + x_ref_data = psf_data[candidate['ckey']][:, 0] + y_ref_data = psf_data[candidate['ckey']][:, 1] + + centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], + x_ref=x_ref_data, y_ref=y_ref_data) - if backtrack: - for i, value in enumerate(temp_ref_flux.values()): - if value is not None and i != j: - ref_flux[i] = value - backtrack = False + if best_candidate is not None and vsp_num: + best_a = best_candidate['a'] + best_an = best_candidate['an'] + best_target_flux = aper_data['target'][:, best_a, best_an] + for j in vsp_num: + ckey = f"comp{j + 1}" + aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) + cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] + vsp_fit, _, _ = fit_lightcurve(times[aper_mask], best_target_flux[aper_mask], cFlux, + airmass[aper_mask], ld, pDict, jd_times[aper_mask]) + ref_flux[j] = { + 'myfit': vsp_fit, + 'pos': exotic_infoDict['comp_stars'][j] + } + + if require_comp_star and photometry_info['comp_star_num'] is None: + log_info("Error: require_comp_star is enabled, but no valid comparison star could be selected.", error=True) + return log_info("\n\n*********************************************") + if np.isfinite(photometry_info['calibration_field_score']): + log_info(f"Comparison-Star Field Score: {round(photometry_info['calibration_field_score'] * 100, 4)}%") if photometry_info['min_aperture'] == 0: # psf log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") - log_info(f"Minimum Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") + log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") log_info("Optimal Method: PSF photometry") elif photometry_info['min_aperture'] < 0: # no comp star log_info("Best Comparison Star: None") - log_info(f"Minimum Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") + log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") log_info(f"Optimal Aperture: {abs(np.round(photometry_info['min_aperture'], 2))}") log_info(f"Optimal Annulus: {np.round(photometry_info['min_annulus'], 2)}") else: log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") - log_info(f"Minimum Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") + log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") log_info(f"Optimal Aperture: {np.round(photometry_info['min_aperture'], 2)}") log_info(f"Optimal Annulus: {np.round(photometry_info['min_annulus'], 2)}") log_info("*********************************************\n") @@ -2475,7 +4332,14 @@ def main(): si = np.argsort(best_fit_lc.time) dt = np.mean(np.diff(np.sort(best_fit_lc.time))) ndt = int(30. / 24. / 60. / dt) * 2 + 1 # ~30 minutes - gi = ~sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt) # good indexs + time_clip_mask = sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt) + phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + if hasattr(best_fit_lc, 'residuals') and hasattr(best_fit_lc, 'phase'): + phase_clip_mask = phase_bin_sigma_clip(best_fit_lc.residuals[si], best_fit_lc.phase[si], sigma=3, bins=10) + gi = ~(time_clip_mask | phase_clip_mask) # good indexs + phase_clip_removed = np.count_nonzero(phase_clip_mask & ~time_clip_mask) + if phase_clip_removed: + log_info(f"Removed {phase_clip_removed} phase-binned residual outlier(s) before final fit.") # Calculate the proper timeseries uncertainties from the residuals of the out-of-transit data OOT = (best_fit_lc.transit == 1) # find out-of-transit portion of the lightcurve @@ -2495,11 +4359,7 @@ def main(): log_info("Error: No valid photometry data found.", error=True) return - best_fit_lc.time = best_fit_lc.time[si][gi] - best_fit_lc.data = best_fit_lc.data[si][gi] - best_fit_lc.airmass = best_fit_lc.airmass[si][gi] - best_fit_lc.transit = best_fit_lc.transit[si][gi] - best_fit_lc.jd_times = best_fit_lc.jd_times[si][gi] + apply_lightcurve_mask(best_fit_lc, gi, sort_index=si) goodTimes = best_fit_lc.time goodFluxes = goodFluxes[si][gi] @@ -2516,6 +4376,28 @@ def main(): flux_unc_tar=flux_values['flux_unc_tar'][si][gi], flux_unc_ref=flux_values['flux_unc_ref'][si][gi]) + relative_flux_mask = relative_flux_filter_mask(goodFluxes) + if np.count_nonzero(relative_flux_mask) == 0: + log_info("Error: No valid photometry data found after removing relative flux values above 2.", error=True) + return + + apply_lightcurve_mask(best_fit_lc, relative_flux_mask) + + goodTimes = goodTimes[relative_flux_mask] + goodFluxes = goodFluxes[relative_flux_mask] + goodNormUnc = goodNormUnc[relative_flux_mask] + goodAirmasses = goodAirmasses[relative_flux_mask] + + centroid_positions.update(x_targ=centroid_positions['x_targ'][relative_flux_mask], + y_targ=centroid_positions['y_targ'][relative_flux_mask], + x_ref=centroid_positions['x_ref'][relative_flux_mask], + y_ref=centroid_positions['y_ref'][relative_flux_mask]) + + flux_values.update(flux_tar=flux_values['flux_tar'][relative_flux_mask], + flux_ref=flux_values['flux_ref'][relative_flux_mask], + flux_unc_tar=flux_values['flux_unc_tar'][relative_flux_mask], + flux_unc_ref=flux_values['flux_unc_ref'][relative_flux_mask]) + if photometry_info['min_aperture'] == 0: opt_method = "PSF" @@ -2591,6 +4473,14 @@ def main(): goodAirmasses = np.array(goodAirmasses) if exotic_infoDict['file_time'] != 'BJD_TDB': + missing_location = [ + label for key, label in (('long', 'longitude'), ('lat', 'latitude'), ('elev', 'elevation')) + if exotic_infoDict.get(key) is None + ] + if missing_location: + log_info("Error: Longitude, latitude, and elevation are required to convert " + f"pre-reduced {exotic_infoDict['file_time']} timestamps to BJD_TDB.", error=True) + return time_offset = 2400000.5 if exotic_infoDict['file_time'] == 'MJD_UTC' else 0.0 goodTimes = convert_jd_to_bjd([time_ + time_offset for time_ in goodTimes], pDict, exotic_infoDict) @@ -2598,6 +4488,16 @@ def main(): print("check flux convert") goodFluxes, goodNormUnc = flux_conversion(goodFluxes, goodNormUnc, exotic_infoDict['file_units']) + relative_flux_mask = relative_flux_filter_mask(goodFluxes) + if np.count_nonzero(relative_flux_mask) == 0: + log_info("Error: No valid photometry data found after removing relative flux values above 2.", error=True) + return + + goodTimes = goodTimes[relative_flux_mask] + goodFluxes = goodFluxes[relative_flux_mask] + goodNormUnc = goodNormUnc[relative_flux_mask] + goodAirmasses = goodAirmasses[relative_flux_mask] + # for k in myfit.bounds.keys(): # print(f"{myfit.parameters[k]:.6f} +- {myfit.errors[k]}") @@ -2678,7 +4578,8 @@ def main(): if fitsortext == 1: plot_obs_stats(myfit, exotic_infoDict['comp_stars'], psf_data, si, gi, pDict['pName'], - exotic_infoDict['save'], exotic_infoDict['date']) + exotic_infoDict['save'], exotic_infoDict['date'], + relative_flux_mask=relative_flux_mask) ####################################################################### # print final extracted planetary parameters @@ -2694,6 +4595,8 @@ def main(): log_info(f" Airmass coefficient 2: {round_to_2(myfit.parameters['a2'], myfit.errors['a2'])} +/- {round_to_2(myfit.errors['a2'])}") log_info(f" Residual scatter: {round_to_2(100. * np.std(myfit.residuals / np.median(myfit.data)))} %") if fitsortext == 1: + if np.isfinite(photometry_info.get('calibration_field_score', np.inf)): + log_info(f" Comparison-Star Field Score: {round_to_2(100. * photometry_info['calibration_field_score'])} %") if photometry_info['min_aperture'] >= 0: log_info(f" Best Comparison Star: #{bestCompStar} - {comp_coords}") else: diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 54172eda..85da2aab 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -77,6 +77,11 @@ except ImportError: # package import from version import __version__ +try: + from .inputs import parse_aavso_comp_star +except ImportError: + from inputs import parse_aavso_comp_star + animate_toggle() @@ -363,6 +368,9 @@ def save_input(): "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", + "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", + "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." } @@ -377,6 +385,11 @@ def save_input(): new_inits['planetary_parameters'] = { "Planet Name": input_data['pName'], } + new_inits['optional_info'] = { + "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "Use target-driven comp selection rather than comp-driven comp selection": "n", + "require_comp_star": "y" + } now = datetime.now() dt_string = now.strftime("%d_%m_%Y__%H_%M_%S") @@ -498,41 +511,19 @@ def save_input(): exp_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 - comp_star_label = tk.Label(root, text="Comparison Star used in Photometry (leave blank if none):", - justify=tk.LEFT) - comp_star_label.grid(row=i, column=j, sticky=tk.W, pady=2) - i += 1 - - comp_star_ra_label = tk.Label(root, text="Comparison Star RA", justify=tk.LEFT) - comp_star_ra_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - comp_star_ra_label.grid(row=i, column=j, sticky=tk.W, pady=2) - comp_star_ra_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) - i += 1 - - comp_star_dec_label = tk.Label(root, text="Comparison Star DEC", justify=tk.LEFT) - comp_star_dec_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - comp_star_dec_label.grid(row=i, column=j, sticky=tk.W, pady=2) - comp_star_dec_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) - i += 1 - - comp_star_x_label = tk.Label(root, text="Comparison Star X Pixel Coordinate", justify=tk.LEFT) - comp_star_x_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - comp_star_x_label.grid(row=i, column=j, sticky=tk.W, pady=2) - comp_star_x_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) - i += 1 - - comp_star_y_label = tk.Label(root, text="Comparison Star Y Pixel Coordinate", justify=tk.LEFT) - comp_star_y_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - comp_star_y_label.grid(row=i, column=j, sticky=tk.W, pady=2) - comp_star_y_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) + comp_star_note = tk.Label( + root, + text="Comparison star metadata will be loaded from an AAVSO #COMP_STAR-XC header when available.", + justify=tk.LEFT + ) + comp_star_note.grid(row=i, column=j, columnspan=2, sticky=tk.W, pady=2) i += 1 def save_input(): input_data['file_time'] = pretime_entry.get() input_data['file_units'] = preunit_entry.get() input_data['exp'] = float(exp_entry.get()) - input_data['phot_comp_star'] = {'ra': comp_star_ra_entry.get(), 'dec': comp_star_dec_entry.get(), - 'x': comp_star_x_entry.get(), 'y': comp_star_y_entry.get()} + input_data['phot_comp_star'] = parse_aavso_comp_star(prered_file.file_path) root.destroy() # Button for closing @@ -689,23 +680,33 @@ def save_input(): i += 1 # # # "Obs. Latitude": "+32.41638889", - lat_label = tk.Label(root, text="Obs. Latitude (+ = North; - = South; e.g. +32.41)", justify=tk.LEFT) + lat_label_text = "Obs. Latitude (+ = North; - = South; e.g. +32.41)" + if fitsortext.get() == 2: + lat_label_text += " [optional for pre-reduced runs]" + lat_label = tk.Label(root, text=lat_label_text, justify=tk.LEFT) lat_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) lat_label.grid(row=i, column=j, sticky=tk.W, pady=2) lat_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 # # # "Obs. Longitude": "-110.73444444", - long_label = tk.Label(root, text="Obs. Longitude (+ = East; - = West; e.g. -110.74) ", justify=tk.LEFT) + long_label_text = "Obs. Longitude (+ = East; - = West; e.g. -110.74)" + if fitsortext.get() == 2: + long_label_text += " [optional for pre-reduced runs]" + long_label = tk.Label(root, text=long_label_text, justify=tk.LEFT) long_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) long_label.grid(row=i, column=j, sticky=tk.W, pady=2) long_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 # # # "Obs. Elevation (meters)": 2616, - elevation_label = tk.Label(root, text="Obs. Elevation [meters]", justify=tk.LEFT) + elevation_label_text = "Obs. Elevation [meters]" + if fitsortext.get() == 2: + elevation_label_text += " [optional for pre-reduced runs]" + elevation_label = tk.Label(root, text=elevation_label_text, justify=tk.LEFT) elevation_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - elevation_entry.insert(tk.END, "0") + if fitsortext.get() == 1: + elevation_entry.insert(tk.END, "0") elevation_label.grid(row=i, column=j, sticky=tk.W, pady=2) elevation_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -837,9 +838,13 @@ def save_input(): input_data['obscode'] = obscode_entry.get() input_data['secondobscode'] = secondobscode_entry.get() input_data['obsdate'] = obsdate_entry.get() - input_data['lat'] = lat_entry.get() - input_data['long'] = long_entry.get() - input_data['elevation'] = float(elevation_entry.get()) + input_data['lat'] = lat_entry.get().strip() + input_data['long'] = long_entry.get().strip() + elevation_value = elevation_entry.get().strip() + if fitsortext.get() == 1 or elevation_value: + input_data['elevation'] = float(elevation_value) + else: + input_data['elevation'] = None input_data['pixscale'] = pixscale_entry.get() if fitsortext.get() == 1: input_data['comppos'] = str(list(ast.literal_eval(comppos_entry.get()))) @@ -1387,6 +1392,9 @@ def save_input(): "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", + "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", + "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." } @@ -1404,6 +1412,7 @@ def save_input(): "AAVSO Observer Code (blank if none)": input_data['obscode'], "Secondary Observer Codes (blank if none)": input_data['secondobscode'], + "Observatory Full Title": "", "Observation date": input_data['obsdate'], "Obs. Latitude": input_data['lat'], @@ -1447,7 +1456,11 @@ def save_input(): new_inits['optional_info'] = { "Filter Minimum Wavelength (nm)": input_data.get('filtermin', null), - "Filter Maximum Wavelength (nm)": input_data.get('filtermax', null) + "Filter Maximum Wavelength (nm)": input_data.get('filtermax', null), + "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "Use target-driven comp selection rather than comp-driven comp selection": "n", + "require_comp_star": "y" } if 'pixscale' not in input_data.keys(): @@ -1465,11 +1478,12 @@ def save_input(): "AAVSO Observer Code (blank if none)": input_data['obscode'], "Secondary Observer Codes (blank if none)": input_data['secondobscode'], + "Observatory Full Title": "", "Observation date": input_data['obsdate'], "Obs. Latitude": input_data['lat'], "Obs. Longitude": input_data['long'], - "Obs. Elevation (meters)": float(input_data.get('elevation', 0)), + "Obs. Elevation (meters; Note: leave blank if unknown)": input_data.get('elevation'), "Camera Type (CCD or DSLR)": input_data['cameratype'], "Pixel Binning": input_data['pixbin'], "Filter Name (aavso.org/filters)": input_data['obsfilter'], @@ -1487,7 +1501,11 @@ def save_input(): "Pre-reduced File Time Format (BJD_TDB, JD_UTC, MJD_UTC)": input_data['file_time'], "Pre-reduced File Units of Flux (flux, magnitude, millimagnitude)": input_data['file_units'], "Comparison Star used in Photometry (blank if none)": input_data['phot_comp_star'], - "Exposure Time (s)": input_data['exp'] + "Exposure Time (s)": input_data['exp'], + "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "Use target-driven comp selection rather than comp-driven comp selection": "n", + "require_comp_star": "y" } if planetparams.get() in ["manual", "nea"]: diff --git a/exotic/inputs.py b/exotic/inputs.py index d4ad1a4c..7907090d 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -3,6 +3,7 @@ import json from pathlib import Path from astropy.io import fits +from astropy.time import Time import re try: @@ -25,6 +26,22 @@ consoleHandler.setLevel(logging.INFO) log.addHandler(consoleHandler) +PHOT_COMP_STAR_KEYS = ("ra", "dec", "x", "y") +AAVSO_OBSDATE_HEADER_KEYS = ('OBSDATE',) +AAVSO_LOCATION_HEADER_KEYS = { + 'lat': ('OBSLAT', 'LATITUDE', 'OBS_LATITUDE', 'LAT'), + 'long': ('OBSLON', 'OBSLONG', 'LONGITUDE', 'OBS_LONGITUDE', 'LONG'), + 'elev': ('OBSELEV', 'OBSALT', 'ELEVATION', 'ALTITUDE', 'HEIGHT'), +} + + +def is_blank_value(value): + if value is None: + return True + if isinstance(value, str): + return value.strip().lower() in ('', 'n/a', 'na', 'null', 'none') + return False + class Inputs: @@ -32,12 +49,14 @@ def __init__(self, init_opt): self.init_opt = init_opt self.info_dict = { 'images': None, 'save': None, 'flats': None, 'darks': None, 'biases': None, - 'aavso_num': None, 'second_obs': None, 'date': None, 'lat': None, 'long': None, + 'aavso_num': None, 'second_obs': None, 'obs_name': '', 'date': None, 'lat': None, 'long': None, 'elev': None, 'camera': None, 'pixel_bin': None, 'filter': None, 'notes': None, 'plate_opt': None, 'aavso_comp': None, 'tar_coords': None, 'comp_stars': None, 'prered_file': None, 'file_units': None, 'file_time': None, 'phot_comp_star': None, 'wl_min': None, 'wl_max': None, 'pixel_scale': None, 'exposure': None, - 'random_seed': None, "demosaic_fmt": None, "demosaic_out": None + 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, + 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', + 'target_driven_comp_selection': 'n' } self.params = { 'images': imaging_files, 'save': save_directory, 'aavso_num': obs_code, 'second_obs': second_obs_code, @@ -81,12 +100,18 @@ def complete_red(self, planet): return self.info_dict, planet def prereduced(self, planet): - rem_list = ['images', 'plate_opt', 'tar_coords', 'comp_stars'] + rem_list = ['images', 'plate_opt', 'aavso_comp', 'tar_coords', 'comp_stars'] [self.params.pop(key) for key in rem_list] + self.info_dict['aavso_comp'] = 'n' self.params.update({'exposure': exposure, 'file_units': data_file_units, 'file_time': data_file_time, 'phot_comp_star': phot_comp_star}) self.info_dict['prered_file'] = prereduced_file(self.info_dict['prered_file']) + aavso_location = parse_aavso_location(self.info_dict['prered_file']) + + for key in ('lat', 'long', 'elev'): + if is_blank_value(self.info_dict.get(key)) and aavso_location[key] is not None: + self.info_dict[key] = aavso_location[key] if not planet: planet = planet_name(planet) @@ -94,10 +119,24 @@ def prereduced(self, planet): for key, value in list(self.params.items()): if key == 'elev': self.info_dict[key] = self.params[key](self.info_dict[key], self.info_dict['lat'], - self.info_dict['long']) + self.info_dict['long'], required=False) + elif key == 'lat': + self.info_dict[key] = self.params[key](self.info_dict[key], required=False) + elif key == 'long': + self.info_dict[key] = self.params[key](self.info_dict[key], required=False) + elif key == 'phot_comp_star': + self.info_dict[key] = self.params[key](self.info_dict[key], self.info_dict['prered_file']) + elif key == 'date': + continue else: self.info_dict[key] = self.params[key](self.info_dict[key]) + self.info_dict['date'] = prereduced_obs_date( + self.info_dict.get('date'), + self.info_dict['prered_file'], + self.info_dict.get('file_time'), + ) + return self.info_dict, planet def real_time(self, planet): @@ -158,9 +197,15 @@ def comp_params(self, init_file, planet_dict): 'demosaic_fmt': 'Demosaic Format', 'demosaic_out': 'Demosaic Output', 'aavso_num': ('AAVSO Observer Code (N/A if none)', 'AAVSO Observer Code (blank if none)'), 'second_obs': ('Secondary Observer Codes (N/A if none)', 'Secondary Observer Codes (blank if none)'), + 'obs_name': 'Observatory Full Title', 'date': 'Observation date', 'lat': 'Obs. Latitude', 'long': 'Obs. Longitude', 'elev': ('Obs. Elevation (meters)', 'Obs. Elevation (meters; Note: leave blank if unknown)'), - 'camera': 'Camera Type (CCD or DSLR)', + 'camera': ( + 'Camera Type (CCD or DSLR)', + 'Camera Type', + 'Camera Type (e.g., CCD or DSLR)', + 'Camera Type (e.g., CCD or DSLR; Note: if you are using a CMOS, please enter CCD here and then note your actual camera type in "Observing Notes")' + ), 'pixel_bin': 'Pixel Binning', 'filter': 'Filter Name (aavso.org/filters)', 'notes': 'Observing Notes', 'plate_opt': 'Plate Solution? (y/n)', 'aavso_comp': 'Add Comparison Stars from AAVSO? (y/n)', @@ -191,9 +236,26 @@ def comp_params(self, init_file, planet_dict): opt_info = { 'prered_file': 'Pre-reduced File:', 'file_time': 'Pre-reduced File Time Format (BJD_TDB, JD_UTC, MJD_UTC)', 'file_units': 'Pre-reduced File Units of Flux (flux, magnitude, millimagnitude)', - 'phot_comp_star': "Comparison Star used in Photometry (leave blank if none)", + 'phot_comp_star': ( + "Comparison Star used in Photometry (leave blank if none)", + "Comparison Star used in Photometry (blank if none)" + ), 'wl_min': 'Filter Minimum Wavelength (nm)', 'wl_max': 'Filter Maximum Wavelength (nm)', - 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)'), + 'ld_uncertainties': 'Calculate Limb Darkening Coefficients with Uncertainties? (y/n)', + 'fast_aperture_mask': ('Fast Aperture Mask (y/n)', 'Use Fast Aperture Mask (y/n)'), + 'require_comp_star': ('require_comp_star', 'Require Comparison Star? (y/n)'), + 'target_driven_comp_selection': ( + 'Use target-driven comp selection rather than comp-driven comp selection', + 'target_driven_comp_selection', + ), + 'ignore_header_wcs': ( + 'Ignore WCS in Header and Do Manual Alignment? (y/n)', + 'Ignore WCS in Header and Do Manual Alignment', + 'Ignore WCS in header and do manual alignment', + 'ignore_header_wcs', + ), + 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)', + 'Pixel Scale (arsec/pixel)'), 'exposure': 'Exposure Time (s)', 'random_seed': 'Random Seed' } @@ -361,17 +423,29 @@ def obs_date(date): return date -def latitude(lat, hdr=None): +def normalize_obs_date(date): + if is_blank_value(date): + return None + + date = str(date).strip() + if '/' in date: + date = date.replace('/', '-') + return date + + +def latitude(lat, hdr=None, required=True): while True: - if not lat: + if is_blank_value(lat): if hdr: lat = find(hdr, ['LATITUDE', 'LAT', 'SITELAT']) if lat: return lat + if not required: + return None lat = user_input("Enter the latitude (in degrees) of where you observed. " "(Don't forget the sign where North is '+' and South is '-')! " "(Example: -32.12): ", type_=str) - lat = lat.strip() + lat = str(lat).strip() if lat[0] == '+' or lat[0] == '-': # Convert to float if latitude in decimal. If latitude is in +/-HH:MM:SS format, convert to a float. @@ -391,17 +465,19 @@ def latitude(lat, hdr=None): lat = None -def longitude(long, hdr=None): +def longitude(long, hdr=None, required=True): while True: - if not long: + if is_blank_value(long): if hdr: long = find(hdr, ['LONGITUD', 'LONG', 'LONGITUDE', 'SITELONG']) if long: return long + if not required: + return None long = user_input("Enter the longitude (in degrees) of where you observed. " "(Don't forget the sign where East is '+' and West is '-')! " "(Example: +152.51): ", type_=str) - long = long.strip() + long = str(long).strip() if long[0] == '+' or long[0] == '-': # Convert to float if longitude in decimal. If longitude is in +/-HH:MM:SS format, convert to a float. @@ -421,21 +497,29 @@ def longitude(long, hdr=None): long = None -def elevation(elev, lat, long, hdr=None): +def elevation(elev, lat, long, hdr=None, required=True): while True: try: - elev = typecast_check(type_=float, val=elev) - if not elev: + if is_blank_value(elev): + elev = None + else: + elev = typecast_check(type_=float, val=elev) + if elev is False: + raise ValueError + + if elev is None: if hdr: elev = find(hdr, ['HEIGHT', 'ELEVATION', 'ELE', 'EL', 'OBSGEO-H', 'ALT-OBS', 'SITEELEV']) - if elev: - return int(elev) + if not is_blank_value(elev): + return float(elev) + if not required: + return None log_info("\nEXOTIC is retrieving elevation based on entered " "latitude and longitude from Open Elevation.") animate_toggle(True) elev = open_elevation(lat, long) animate_toggle() - if not elev: + if elev is False: log_info("\nWarning: EXOTIC could not retrieve elevation.", warn=True) elev = user_input("Enter the elevation (in meters) of where you observed: ", type_=float) return elev @@ -445,16 +529,9 @@ def elevation(elev, lat, long, hdr=None): def camera(c_type): - while True: - if not c_type: - c_type = user_input("\nPlease enter the camera type (e.g., CCD or DSLR;\n" - "Note: if you are using a CMOS, please enter CCD here and\n" - "then note your actual camera type in \"Observing Notes\"): ", type_=str) - c_type = c_type.strip().upper() - if c_type not in ["CCD", "DSLR"]: - c_type = None - else: - return c_type + if isinstance(c_type, str) and "DSLR" in c_type.strip().upper(): + return "DSLR" + return "CCD" def pixel_bin(pix_bin): @@ -566,19 +643,141 @@ def prereduced_file(file): file = None -def phot_comp_star(comp_star): +def blank_phot_comp_star(): + return {key: '' for key in PHOT_COMP_STAR_KEYS} + + +def normalize_phot_comp_star(comp_star): + normalized_comp_star = blank_phot_comp_star() + if not isinstance(comp_star, dict): - comp_star_opt = user_input("Was a Comparison Star used during Photometry? (y/n): ", - type_=str, values=['y', 'n']) - - comp_star = { - 'ra': user_input("\nEnter Comparison Star RA: ", type_=str) if comp_star_opt == 'y' else '', - 'dec': user_input("Enter Comparison Star DEC: ", type_=str) if comp_star_opt == 'y' else '', - 'x': user_input("\nEnter Comparison Star X Pixel Coordinate: ", type_=str) if comp_star_opt == 'y' else '', - 'y': user_input("Enter Comparison Star Y Pixel Coordinate: ", type_=str) if comp_star_opt == 'y' else '' - } + return normalized_comp_star + + for key in PHOT_COMP_STAR_KEYS: + value = comp_star.get(key, '') + if value is None: + continue + + value = str(value).strip() + normalized_comp_star[key] = '' if value.lower() in ('null', 'none') else value + + return normalized_comp_star + + +def parse_aavso_metadata(prereduced_file_path): + if not prereduced_file_path: + return {} + + try: + with Path(prereduced_file_path).open('r', encoding='utf-8') as file: + for line in file: + metadata_line = line.strip() + if not metadata_line: + continue + if not metadata_line.startswith('#'): + break + if '=' not in metadata_line: + continue + + key, value = metadata_line[1:].split('=', 1) + yield key.strip().upper(), value.strip() + except (FileNotFoundError, OSError, TypeError): + return + + +def first_aavso_metadata_value(metadata, aliases): + for key in aliases: + value = metadata.get(key) + if not is_blank_value(value): + return value + return None + + +def parse_aavso_location(prereduced_file_path): + metadata = dict(parse_aavso_metadata(prereduced_file_path) or []) + return { + key: first_aavso_metadata_value(metadata, aliases) + for key, aliases in AAVSO_LOCATION_HEADER_KEYS.items() + } + + +def parse_aavso_obsdate(prereduced_file_path): + metadata = dict(parse_aavso_metadata(prereduced_file_path) or []) + return normalize_obs_date(first_aavso_metadata_value(metadata, AAVSO_OBSDATE_HEADER_KEYS)) + + +def parse_aavso_comp_star(prereduced_file_path): + metadata = dict(parse_aavso_metadata(prereduced_file_path) or []) + comp_star_json = metadata.get('COMP_STAR-XC') + if is_blank_value(comp_star_json): + return blank_phot_comp_star() + + try: + return normalize_phot_comp_star(json.loads(comp_star_json)) + except (TypeError, json.JSONDecodeError): + return blank_phot_comp_star() + + +def phot_comp_star(comp_star, prereduced_file_path=None): + if isinstance(comp_star, dict): + return normalize_phot_comp_star(comp_star) + return parse_aavso_comp_star(prereduced_file_path) + + +def first_prereduced_timestamp(prereduced_file_path): + if not prereduced_file_path: + return None + + try: + with Path(prereduced_file_path).open('r', encoding='utf-8') as file: + for line in file: + data_line = line.strip() + if not data_line or data_line.startswith('#'): + continue + + first_column = re.split(r'[\s,]+', data_line, maxsplit=1)[0] + try: + return float(first_column) + except ValueError: + continue + except (FileNotFoundError, OSError, TypeError): + return None + + return None + + +def obs_date_from_first_prereduced_entry(prereduced_file_path, time_format): + first_timestamp = first_prereduced_timestamp(prereduced_file_path) + if first_timestamp is None: + return None + + try: + if time_format == 'MJD_UTC': + return Time(first_timestamp, format='mjd', scale='utc').to_value('iso', subfmt='date') + if time_format == 'BJD_TDB': + return Time(first_timestamp, format='jd', scale='tdb').to_value('iso', subfmt='date') + if time_format == 'JD_UTC': + return Time(first_timestamp, format='jd', scale='utc').to_value('iso', subfmt='date') + except (TypeError, ValueError): + return None + + return None + + +def prereduced_obs_date(date, prereduced_file_path=None, time_format=None): + aavso_obsdate = parse_aavso_obsdate(prereduced_file_path) + if aavso_obsdate is not None: + return aavso_obsdate + + derived_obsdate = obs_date_from_first_prereduced_entry(prereduced_file_path, time_format) + if derived_obsdate is not None: + return derived_obsdate + + normalized_date = normalize_obs_date(date) + if normalized_date is not None: + return normalized_date - return comp_star + return "" def data_file_time(time_format): diff --git a/exotic/output_files.py b/exotic/output_files.py index 87e97681..07d6269d 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -78,6 +78,8 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): priors_dict, filter_dict, results_dict = aavso_dicts(self.p_dict, self.fit, self.i_dict, self.durs, ld0, ld1, ld2, ld3) + obs_name = format_aavso_header_value(self.i_dict.get('obs_name')) + obs_name_header = f"#OBSNAME={obs_name}\n" if obs_name else "" params_file = self.dir / f"AAVSO_{self.p_dict['pName']}_{self.i_dict['date']}.txt" @@ -88,11 +90,16 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): f"#SOFTWARE=EXOTIC v{__version__}\n" # fixed "#DELIM=,\n" # fixed "#DATE_TYPE=BJD_TDB\n" # fixed + f"#OBSDATE={format_aavso_header_value(self.i_dict.get('date'))}\n" + f"{obs_name_header}" f"#OBSTYPE={self.i_dict['camera']}\n" f"#STAR_NAME={self.p_dict['sName']}\n" # code yields f"#EXOPLANET_NAME={self.p_dict['pName']}\n" # code yields f"#BINNING={self.i_dict['pixel_bin']}\n" # user input f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI + f"#OBSLAT={format_aavso_header_value(self.i_dict.get('lat'))}\n" + f"#OBSLON={format_aavso_header_value(self.i_dict.get('long'))}\n" + f"#OBSELEV={format_aavso_header_value(self.i_dict.get('elev'))}\n" f"#COMP_STAR-XC={dumps(comp_star)}\n" f"#NOTES={self.i_dict['notes']}\n" "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" # fixed @@ -157,7 +164,11 @@ def aavso(self): f"#SOFTWARE=EXOTIC v{__version__}\n" # fixed "#DELIM=,\n" # fixed "#DATE=JD\n" # fixed - f"#OBSTYPE={self.i_dict['camera']}\n") + f"#OBSDATE={format_aavso_header_value(self.i_dict.get('date'))}\n" + f"#OBSTYPE={self.i_dict['camera']}\n" + f"#OBSLAT={format_aavso_header_value(self.i_dict.get('lat'))}\n" + f"#OBSLON={format_aavso_header_value(self.i_dict.get('long'))}\n" + f"#OBSELEV={format_aavso_header_value(self.i_dict.get('elev'))}\n") f.write( "# EXOTIC is developed by Exoplanet Watch (exoplanets.nasa.gov/exoplanet-watch/), a citizen science " "project managed by NASA's Jet Propulsion Laboratory on behalf of NASA's Universe of Learning. " @@ -218,8 +229,16 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): filter_type = { 'name': info_dict['filter'], 'desc': info_dict['filter_desc'], - 'fwhm': [{'value': str(info_dict['wl_min']) if info_dict['wl_min'] else info_dict['wl_min'], 'units': "nm"}, - {'value': str(info_dict['wl_max']) if info_dict['wl_max'] else info_dict['wl_max'], 'units': "nm"}], + 'filter_width': { + 'left_side_wavelength': { + 'value': str(info_dict['wl_min']) if info_dict['wl_min'] else info_dict['wl_min'], + 'units': "nm" + }, + 'right_side_wavelength': { + 'value': str(info_dict['wl_max']) if info_dict['wl_max'] else info_dict['wl_max'], + 'units': "nm" + } + }, } results = { @@ -252,3 +271,48 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): } return priors, filter_type, results + + +def format_aavso_header_value(value): + if value is None: + return "" + if isinstance(value, str): + stripped = value.strip() + return "" if stripped.lower() in ('', 'n/a', 'na', 'null', 'none') else stripped + return str(value) + + +def save_comp_star_calibration_summary(save_dir, target_name, date, method_label, field_score, + comp_summaries, best_comp_index): + temp_dir = Path(save_dir) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + summary_file = temp_dir / f"CompStarCalibrationSummary_{target_name}_{date}.csv" + + with summary_file.open('w') as handle: + handle.write(f"# Comparison-star calibration summary for {target_name}\n") + handle.write(f"# Method,{method_label}\n") + if field_score is not None and field_score == field_score: + handle.write(f"# Field suitability score,{field_score}\n") + else: + handle.write("# Field suitability score,\n") + handle.write(f"# Selected comparison star,{'' if best_comp_index is None else best_comp_index + 1}\n") + handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,ensemble_score,pairwise_median_score," + "pairwise_max_score,self_score,valid_pair_count\n") + + for summary in comp_summaries: + position = summary.get('position') or [None, None] + values = [ + summary.get('label', ''), + position[0], + position[1], + str(bool(summary.get('selected'))).lower(), + summary.get('aggregate_score'), + summary.get('ensemble_score'), + summary.get('pairwise_median_score'), + summary.get('pairwise_max_score'), + summary.get('self_score'), + summary.get('valid_pair_count'), + ] + handle.write(",".join("" if value is None else str(value) for value in values) + "\n") + + return summary_file diff --git a/exotic/plots.py b/exotic/plots.py index 387b8311..92b632d2 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -164,6 +164,126 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, f.write(f"{round(ti, 8)},{round(fi, 7)},{round(erri, 6)},{round(ami, 2)}\n") +def plot_comp_star_pairwise_matrix(pairwise_matrix, best_comp_index, targ_name, save, date, method_label): + matrix = np.asarray(pairwise_matrix, dtype=float) + if matrix.size == 0: + return + + temp_dir = Path(save) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + + fig, ax = plt.subplots(figsize=(max(6, matrix.shape[0] * 1.3), max(5, matrix.shape[0] * 1.1))) + plot_matrix = np.ma.masked_invalid(matrix * 100.0) + im = ax.imshow(plot_matrix, origin='upper', cmap='viridis') + fig.colorbar(im, ax=ax, label="Residual Scatter [%]") + + labels = [f"Comp {index + 1}" for index in range(matrix.shape[0])] + ax.set_xticks(np.arange(matrix.shape[0])) + ax.set_yticks(np.arange(matrix.shape[0])) + ax.set_xticklabels(labels, rotation=45, ha='right') + ax.set_yticklabels(labels) + ax.set_title(f"{targ_name} Comparison-Star Pairwise Scatter\n{method_label}") + + for row in range(matrix.shape[0]): + for col in range(matrix.shape[1]): + value = matrix[row, col] + if np.isfinite(value): + ax.text(col, row, f"{value * 100.0:.3f}", ha='center', va='center', color='white', fontsize=8) + + if best_comp_index is not None and 0 <= best_comp_index < matrix.shape[0]: + ax.add_patch(plt.Rectangle((best_comp_index - 0.5, best_comp_index - 0.5), 1, 1, + fill=False, edgecolor='tomato', linewidth=2.5)) + + ax.set_xlabel("Reference Comparison Star") + ax.set_ylabel("Candidate Comparison Star") + fig.tight_layout() + fig.savefig(temp_dir / f"CompStarPairwiseScatter_{targ_name}_{date}.png", bbox_inches="tight") + fig.savefig(temp_dir / f"CompStarPairwiseScatter_{targ_name}_{date}.pdf", bbox_inches="tight") + plt.close(fig) + + +def plot_comp_star_calibration_series(times, comp_summaries, targ_name, save, date, method_label): + if not comp_summaries: + return + + times = np.asarray(times, dtype=float) + temp_dir = Path(save) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + colors = plt.cm.tab10(np.linspace(0.0, 1.0, 10)) + + fig_height = max(3.2, 2.4 * len(comp_summaries)) + fig, axes = plt.subplots(len(comp_summaries), 1, figsize=(12, fig_height), sharex=True) + if len(comp_summaries) == 1: + axes = [axes] + + for axis, summary in zip(axes, comp_summaries): + axis.axhline(1.0, color='lightgray', lw=1.0, zorder=1) + pairwise_series = summary.get('pairwise_ratio_series', {}) + for color_index, (other_label, ratio_series) in enumerate(pairwise_series.items()): + ratio_series = np.asarray(ratio_series, dtype=float) + valid = np.isfinite(times) & np.isfinite(ratio_series) + if np.any(valid): + axis.plot(times[valid], ratio_series[valid], color=colors[color_index % len(colors)], + alpha=0.55, lw=1.0, label=other_label) + + ensemble_ratio = np.asarray(summary.get('ensemble_ratio_series'), dtype=float) + ensemble_valid = np.isfinite(times) & np.isfinite(ensemble_ratio) + if np.any(ensemble_valid): + axis.plot(times[ensemble_valid], ensemble_ratio[ensemble_valid], color='black', lw=1.8, + label='Ensemble') + + selected_text = " selected" if summary.get('selected') else "" + aggregate = summary.get('aggregate_score', np.nan) + aggregate_text = "n/a" if not np.isfinite(aggregate) else f"{aggregate * 100.0:.3f}%" + axis.set_ylabel("Norm Ratio") + axis.set_title(f"{summary['label']}{selected_text} | suitability={aggregate_text}", loc='left', fontsize=10) + axis.grid(alpha=0.2) + axis.legend(ncol=4, fontsize=8, loc='upper right') + + axes[-1].set_xlabel("Time [BJD_TDB]") + fig.suptitle(f"{targ_name} Comparison-Star Calibration Curves\n{method_label}", y=1.01) + fig.tight_layout() + fig.savefig(temp_dir / f"CompStarCalibrationCurves_{targ_name}_{date}.png", bbox_inches="tight") + fig.savefig(temp_dir / f"CompStarCalibrationCurves_{targ_name}_{date}.pdf", bbox_inches="tight") + plt.close(fig) + + +def plot_comp_star_suitability(comp_summaries, targ_name, save, date, method_label): + if not comp_summaries: + return + + temp_dir = Path(save) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + + labels = [summary['label'] for summary in comp_summaries] + positions = np.arange(len(labels)) + aggregate = np.array([summary.get('aggregate_score', np.nan) for summary in comp_summaries], dtype=float) * 100.0 + ensemble = np.array([summary.get('ensemble_score', np.nan) for summary in comp_summaries], dtype=float) * 100.0 + pairwise = np.array([summary.get('pairwise_median_score', np.nan) for summary in comp_summaries], dtype=float) * 100.0 + + fig, ax = plt.subplots(figsize=(max(7, 1.5 * len(labels)), 5)) + width = 0.25 + ax.bar(positions - width, aggregate, width=width, label='Suitability') + ax.bar(positions, ensemble, width=width, label='Vs ensemble') + ax.bar(positions + width, pairwise, width=width, label='Pairwise median') + + for position, summary in zip(positions, comp_summaries): + if summary.get('selected'): + ax.text(position - width, aggregate[position] if np.isfinite(aggregate[position]) else 0.0, 'selected', + rotation=90, va='bottom', ha='center', fontsize=8, color='tomato') + + ax.set_xticks(positions) + ax.set_xticklabels(labels) + ax.set_ylabel("Residual Scatter [%]") + ax.set_title(f"{targ_name} Comparison-Star Suitability Summary\n{method_label}") + ax.legend() + ax.grid(axis='y', alpha=0.25) + fig.tight_layout() + fig.savefig(temp_dir / f"CompStarSuitability_{targ_name}_{date}.png", bbox_inches="tight") + fig.savefig(temp_dir / f"CompStarSuitability_{targ_name}_{date}.pdf", bbox_inches="tight") + plt.close(fig) + + def plot_variable_residuals(save): plt.title("Stellar Variability Residuals") plt.ylabel("Residuals (flux)") @@ -185,7 +305,31 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): # Observation statistics from PSF data -def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date): +def _select_psf_rows(psf_rows, sort_index=None, sigma_mask=None, relative_flux_mask=None): + rows = np.asarray(psf_rows) + + if sort_index is not None: + rows = rows[np.asarray(sort_index)] + + if sigma_mask is not None: + sigma_mask = np.asarray(sigma_mask) + if sigma_mask.dtype == bool and rows.shape[0] == sigma_mask.shape[0]: + rows = rows[sigma_mask] + + if relative_flux_mask is not None: + relative_flux_mask = np.asarray(relative_flux_mask) + if relative_flux_mask.dtype == bool and rows.shape[0] == relative_flux_mask.shape[0]: + rows = rows[relative_flux_mask] + + return rows + + +def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relative_flux_mask=None): + fit_time = np.asarray(fit.time) + fit_airmass = np.asarray(fit.airmass) + temp_dir = Path(save) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + for i in range(len(comp_stars) + 1): if i == 0: title, key = target_name, "target" @@ -195,29 +339,41 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date): fig, axs = plt.subplots(3, 2, figsize=(12, 10)) fig.suptitle(f"Observing Statistics - {title} - {date}") + star_stats = _select_psf_rows(psf[key], sort_index=si, sigma_mask=gi, + relative_flux_mask=relative_flux_mask) + + plot_len = min(fit_time.shape[0], fit_airmass.shape[0], star_stats.shape[0]) + if plot_len == 0: + plt.close(fig) + continue + + time_data = fit_time[:plot_len] + airmass_data = fit_airmass[:plot_len] + star_stats = star_stats[:plot_len] + axs[0, 0].set(xlabel="Time [BJD_TDB]", ylabel="X-Centroid [px]") - axs[0, 0].plot(fit.time, psf[key][si, 0][gi], 'k.') + axs[0, 0].plot(time_data, star_stats[:, 0], 'k.') axs[0, 1].set(xlabel="Time [BJD_TDB]", ylabel="Y-Centroid [px]") - axs[0, 1].plot(fit.time, psf[key][si, 1][gi], 'k.') + axs[0, 1].plot(time_data, star_stats[:, 1], 'k.') axs[1, 0].set(xlabel="Time [BJD_TDB]", ylabel="Seeing [px]") - axs[1, 0].plot(fit.time, 2.355 * 0.5 * (psf[key][si, 3][gi] + psf[key][si, 4][gi]), 'k.') + axs[1, 0].plot(time_data, 2.355 * 0.5 * (star_stats[:, 3] + star_stats[:, 4]), 'k.') axs[1, 1].set(xlabel="Time [BJD_TDB]", ylabel="Airmass") - axs[1, 1].plot(fit.time, fit.airmass, 'k.') + axs[1, 1].plot(time_data, airmass_data, 'k.') axs[2, 0].set(xlabel="Time [BJD_TDB]", ylabel="Amplitude [ADU]") - axs[2, 0].plot(fit.time, psf[key][si, 2][gi], 'k.') + axs[2, 0].plot(time_data, star_stats[:, 2], 'k.') axs[2, 1].set(xlabel="Time [BJD_TDB]", ylabel="Background [ADU]") - axs[2, 1].plot(fit.time, psf[key][si, 6][gi], 'k.') + axs[2, 1].plot(time_data, star_stats[:, 6], 'k.') plt.tight_layout() try: - fig.savefig(Path(save) / "temp" / f"Observing_Statistics_{key}_{date}.png", bbox_inches="tight") - fig.savefig(Path(save) / "temp" / f"Observing_Statistics_{key}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / f"Observing_Statistics_{key}_{date}.png", bbox_inches="tight") + fig.savefig(temp_dir / f"Observing_Statistics_{key}_{date}.pdf", bbox_inches="tight") except Exception: pass plt.close() diff --git a/exotic/version.py b/exotic/version.py index 376d9ccc..26a6c390 100644 --- a/exotic/version.py +++ b/exotic/version.py @@ -1 +1 @@ -__version__ = '4.3.1' +__version__ = '4.4.0' diff --git a/inits.json b/inits.json index 49d057f9..222a747a 100644 --- a/inits.json +++ b/inits.json @@ -1,97 +1,101 @@ { "inits_guide": { - "Title": "EXOTIC's Initialization File", - "Comment": "Please answer all the following requirements below by following the format of the given", - "Comment1": "sample dataset HAT-P-32 b. Edit this file as needed to match the data wanting to be reduced.", - "Comment2": "Do not delete areas where there are quotation marks, commas, and brackets.", - "Comment3": "The inits_guide dictionary (these lines of text) does not have to be edited", - "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", - "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", - "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", - "Comparison Star(s) Guide": "Up to 10 comparison stars can be added following the format given below.", - "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", - "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", - "Camera Type (1)": "If you are using a CMOS, please enter CCD in 'Camera Type (CCD or DSLR)' and then note", - "Camera Type (2)": "your actual camera type under 'Observing Notes'.", - "Plate Solution": "For your image to be given a plate solution, type y.", - "Plate Solution Disclaimer": "One of your imaging files will be publicly viewable on nova.astrometry.net.", - "Standard Filter": "To use EXOTIC standard filters, type only the filter name.", - "Custom Filter": "To use a custom filter, enter in the FWHM in optional_info.", - "Target Star RA": "Must be in HH:MM:SS sexagesimal format.", - "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", - "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", - "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", - "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", - "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." + "Title": "EXOTIC's Initialization File", + "Comment": "Please answer all the following requirements below by following the format of the given", + "Comment1": "sample dataset HAT-P-32 b. Edit this file as needed to match the data wanting to be reduced.", + "Comment2": "Do not delete areas where there are quotation marks, commas, and brackets.", + "Comment3": "The inits_guide dictionary (these lines of text) does not have to be edited", + "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", + "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", + "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", + "Comparison Star(s) Guide": "Up to 10 comparison stars can be added following the format given below.", + "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", + "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", + "Camera Type (1)": "If you are using a CMOS, please enter CCD in 'Camera Type (CCD or DSLR)' and then note", + "Camera Type (2)": "your actual camera type under 'Observing Notes'.", + "Observatory Full Title": "Optional full observatory name. If present, EXOTIC writes it to AAVSO output as OBSNAME.", + "Plate Solution": "For your image to be given a plate solution, type y.", + "Plate Solution Disclaimer": "One of your imaging files will be publicly viewable on nova.astrometry.net.", + "Standard Filter": "To use EXOTIC standard filters, type only the filter name.", + "Custom Filter": "To use a custom filter, enter in the FWHM in optional_info.", + "Target Star RA": "Must be in HH:MM:SS sexagesimal format.", + "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", + "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", + "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", + "Fast Aperture Mask": "Default true/fast mode for quicker aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to false to opt out and use exact masks.", + "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", + "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", + "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", + "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." }, "user_info": { - "Directory with FITS files": "/Users/rzellem/Documents/EXOTIC/sample-data/HatP32Dec202017", - "Directory to Save Plots": "/Users/rzellem/Documents/EXOTIC/sample-data/", - "Directory of Flats": null, - "Directory of Darks": null, - "Directory of Biases": null, - - "AAVSO Observer Code (blank if none)": "RTZ", - "Secondary Observer Codes (blank if none)": "", - - "Observation date": "17-December-2017", - "Obs. Latitude": "+32.41638889", - "Obs. Longitude": "-110.73444444", - "Obs. Elevation (meters)": 2616, - "Camera Type (CCD or DSLR)": "CCD", - "Pixel Binning": "1x1", - "Filter Name (aavso.org/filters)": "CV", - "Observing Notes": "Weather, seeing was nice.", - - "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "y", - - "Target Star X & Y Pixel": "[424, 286]", - "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", - - "Demosaic Format": null, - "Demosaic Output": null + "Directory with FITS files": "/Users/rzellem/Documents/EXOTIC/sample-data/HatP32Dec202017", + "Directory to Save Plots": "/Users/rzellem/Documents/EXOTIC/sample-data/", + "Directory of Flats": null, + "Directory of Darks": null, + "Directory of Biases": null, + "AAVSO Observer Code (blank if none)": "RTZ", + "Secondary Observer Codes (blank if none)": "", + "Observatory Full Title": "", + "Observation date": "17-December-2017", + "Obs. Latitude": "+32.41638889", + "Obs. Longitude": "-110.73444444", + "Obs. Elevation (meters)": 2616, + "Camera Type (CCD or DSLR)": "CCD", + "Pixel Binning": "1x1", + "Filter Name (aavso.org/filters)": "CV", + "Observing Notes": "Weather, seeing was nice.", + "Plate Solution? (y/n)": "y", + "Add Comparison Stars from AAVSO? (y/n)": "y", + "Target Star X & Y Pixel": "[424, 286]", + "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", + "Demosaic Format": null, + "Demosaic Output": null }, "planetary_parameters": { - "Target Star RA": "02:04:10", - "Target Star Dec": "+46:41:23", - "Planet Name": "HAT-P-32 b", - "Host Star Name": "HAT-P-32", - "Orbital Period (days)": 2.1500082, - "Orbital Period Uncertainty": 1.3e-07, - "Published Mid-Transit Time (BJD-UTC)": 2455867.402743, - "Mid-Transit Time Uncertainty": 4.9e-05, - "Ratio of Planet to Stellar Radius (Rp/Rs)": 0.14886235252742716, - "Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty": 0.0005539487393037134, - "Ratio of Distance to Stellar Radius (a/Rs)": 5.344, - "Ratio of Distance to Stellar Radius (a/Rs) Uncertainty": 0.039496835316262996, - "Orbital Inclination (deg)": 88.98, - "Orbital Inclination (deg) Uncertainty": 0.7602631123499285, - "Orbital Eccentricity (0 if null)": 0.159, - "Argument of Periastron (deg)": 50, - "Star Effective Temperature (K)": 6001.0, - "Star Effective Temperature (+) Uncertainty": 88.0, - "Star Effective Temperature (-) Uncertainty": -88.0, - "Star Metallicity ([FE/H])": -0.16, - "Star Metallicity (+) Uncertainty": 0.08, - "Star Metallicity (-) Uncertainty": -0.08, - "Star Surface Gravity (log(g))": 4.22, - "Star Surface Gravity (+) Uncertainty": 0.04, - "Star Surface Gravity (-) Uncertainty": -0.04, - "Star Distance (pc)": 289.21, - "Star Proper Motion RA (mas/yr)": -9.82, - "Star Proper Motion DEC (mas/yr)": 3.48 + "Target Star RA": "02:04:10", + "Target Star Dec": "+46:41:23", + "Planet Name": "HAT-P-32 b", + "Host Star Name": "HAT-P-32", + "Orbital Period (days)": 2.1500082, + "Orbital Period Uncertainty": 1.3e-07, + "Published Mid-Transit Time (BJD-UTC)": 2455867.402743, + "Mid-Transit Time Uncertainty": 4.9e-05, + "Ratio of Planet to Stellar Radius (Rp/Rs)": 0.14886235252742716, + "Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty": 0.0005539487393037134, + "Ratio of Distance to Stellar Radius (a/Rs)": 5.344, + "Ratio of Distance to Stellar Radius (a/Rs) Uncertainty": 0.039496835316262996, + "Orbital Inclination (deg)": 88.98, + "Orbital Inclination (deg) Uncertainty": 0.7602631123499285, + "Orbital Eccentricity (0 if null)": 0.159, + "Argument of Periastron (deg)": 50, + "Star Effective Temperature (K)": 6001.0, + "Star Effective Temperature (+) Uncertainty": 88.0, + "Star Effective Temperature (-) Uncertainty": -88.0, + "Star Metallicity ([FE/H])": -0.16, + "Star Metallicity (+) Uncertainty": 0.08, + "Star Metallicity (-) Uncertainty": -0.08, + "Star Surface Gravity (log(g))": 4.22, + "Star Surface Gravity (+) Uncertainty": 0.04, + "Star Surface Gravity (-) Uncertainty": -0.04, + "Star Distance (pc)": 289.21, + "Star Proper Motion RA (mas/yr)": -9.82, + "Star Proper Motion DEC (mas/yr)": 3.48 }, "optional_info": { - "Pre-reduced File:": "/sample-data/NormalizedFlux_HAT-P-32 b_December 17, 2017.txt", - "Pre-reduced File Time Format (BJD_TDB, JD_UTC, MJD_UTC)": "BJD_TDB", - "Pre-reduced File Units of Flux (flux, magnitude, millimagnitude)": "flux", - - "Filter Minimum Wavelength (nm)": null, - "Filter Maximum Wavelength (nm)": null, - - "Image Scale (Ex: 5.21 arcsecs/pixel)": null, - - "Exposure Time (s)": 60.0 + "Pre-reduced File:": "/sample-data/NormalizedFlux_HAT-P-32 b_December 17, 2017.txt", + "Pre-reduced File Time Format (BJD_TDB, JD_UTC, MJD_UTC)": "BJD_TDB", + "Pre-reduced File Units of Flux (flux, magnitude, millimagnitude)": "flux", + "Filter Minimum Wavelength (nm)": null, + "Filter Maximum Wavelength (nm)": null, + "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "Fast Aperture Mask (y/n)": true, + "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "Use target-driven comp selection rather than comp-driven comp selection": "n", + "require_comp_star": "y", + "Image Scale (Ex: 5.21 arcsecs/pixel)": null, + "Pixel Scale (arsec/pixel)": null, + "Exposure Time (s)": 60.0 } -} \ No newline at end of file +} diff --git a/requirements.txt b/requirements.txt index 6b8f125f..fb0af820 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ astroalign~=2.6.0 astropy~=6.1 astroquery~=0.4.7 barycorrpy~=0.4.4 +bottleneck~=1.4.2 colour_demosaicing~=0.2.6 dynesty~=1.2.3;platform_system=='Windows' holoviews~=1.19.1 diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py new file mode 100644 index 00000000..a82acce5 --- /dev/null +++ b/tests/test_centroid_wcs.py @@ -0,0 +1,245 @@ +import sys +import types +import importlib.util + +import numpy as np +from astropy.wcs import WCS + + +def _module_available(name): + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + +def _install_stub_module(name, **attrs): + module = types.ModuleType(name) + for attr, value in attrs.items(): + setattr(module, attr, value) + sys.modules[name] = module + return module + + +class _DummyDAOStarFinder: + def __init__(self, *args, **kwargs): + pass + + def __call__(self, *args, **kwargs): + return None + + +if not _module_available("astroalign"): + _install_stub_module("astroalign", PIXEL_TOL=1) +if not _module_available("astroquery"): + _install_stub_module("astroquery") +if not _module_available("astroquery.simbad"): + _install_stub_module("astroquery.simbad", Simbad=object) +if not _module_available("astroquery.gaia"): + _install_stub_module("astroquery.gaia", Gaia=object) +if not _module_available("barycorrpy"): + _install_stub_module("barycorrpy") +if not _module_available("barycorrpy.utc_tdb"): + _install_stub_module("barycorrpy.utc_tdb", JDUTC_to_BJDTDB=lambda *args, **kwargs: None) +if not _module_available("imreg_dft"): + _install_stub_module("imreg_dft") +if not _module_available("pyvo"): + _install_stub_module("pyvo") +if not _module_available("photutils"): + _install_stub_module("photutils") +if not _module_available("photutils.aperture"): + _install_stub_module("photutils.aperture", CircularAperture=object) +if not _module_available("photutils.detection"): + _install_stub_module("photutils.detection", DAOStarFinder=_DummyDAOStarFinder) +if not _module_available("colour_demosaicing"): + _install_stub_module("colour_demosaicing", demosaicing_CFA_Bayer_bilinear=lambda *args, **kwargs: None) +if not _module_available("ldtk"): + fake_ldtk = _install_stub_module("ldtk") + fake_ldtk.LDPSet = type("LDPSet", (), {}) + fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) +if not _module_available("ldtk.ldmodel"): + _install_stub_module( + "ldtk.ldmodel", + LinearModel=type("LinearModel", (), {}), + QuadraticModel=type("QuadraticModel", (), {}), + NonlinearModel=type("NonlinearModel", (), {}), + ) +if not _module_available("lmfit"): + _install_stub_module("lmfit") + +_install_stub_module( + "exotic.api.elca", + lc_fitter=lambda *args, **kwargs: None, + binner=lambda *args, **kwargs: None, + transit=lambda *args, **kwargs: None, + get_phase=lambda *args, **kwargs: None, +) + +from exotic import exotic as exotic_module + + +def _gaussian_image(shape=(80, 80), center=(40.0, 35.0), amplitude=5000.0, sigma=2.0, background=100.0): + y, x = np.indices(shape, dtype=float) + cx, cy = center + image = background + amplitude * np.exp(-((x - cx) ** 2 + (y - cy) ** 2) / (2.0 * sigma ** 2)) + return image + + +def test_fit_centroid_uses_moment_fallback_when_psf_fit_fails(monkeypatch): + image = _gaussian_image() + low_flux_warnings = [] + + def fail_least_squares(*args, **kwargs): + raise ValueError("Residuals are not finite in the initial point.") + + monkeypatch.setattr(exotic_module, "least_squares", fail_least_squares) + monkeypatch.setattr( + exotic_module.plateStatus, + "lowFluxAmplitudeWarning", + lambda star_index, xc, yc: low_flux_warnings.append((star_index, xc, yc)), + ) + + result = exotic_module.fit_centroid(image, [40.0, 35.0], 0) + + assert np.isfinite(result[0]) + assert np.isfinite(result[1]) + assert abs(result[0] - 40.0) < 1.0 + assert abs(result[1] - 35.0) < 1.0 + assert low_flux_warnings == [] + + +def test_fit_centroid_or_warn_out_of_frame_skips_centroid_fit(monkeypatch): + image = np.zeros((40, 50), dtype=float) + out_of_frame_warnings = [] + + monkeypatch.setattr( + exotic_module, + "fit_centroid", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("fit_centroid should be skipped")), + ) + monkeypatch.setattr( + exotic_module.plateStatus, + "outOfFrameWarning", + lambda star_index: out_of_frame_warnings.append(star_index), + ) + + result = exotic_module.fit_centroid_or_warn_out_of_frame(image, [75.0, 20.0], 1) + + assert np.all(np.isnan(result)) + assert out_of_frame_warnings == [1] + + +def test_skybg_phot_returns_nan_when_annulus_box_is_empty(monkeypatch): + image = np.zeros((20, 20), dtype=float) + sky_warnings = [] + + monkeypatch.setattr( + exotic_module, + "mesh_box", + lambda *args, **kwargs: ( + np.empty((0, 0), dtype=int), + np.empty((0, 0), dtype=int), + ), + ) + monkeypatch.setattr( + exotic_module.plateStatus, + "skyBackgroundWarning", + lambda star_index, xc, yc: sky_warnings.append((star_index, xc, yc)), + ) + + bgflux, sigmabg, nbg = exotic_module.skybg_phot(image, 0, 30.0, 30.0) + + assert np.isnan(bgflux) + assert np.isnan(sigmabg) + assert nbg == 0 + assert sky_warnings == [(0, 30.0, 30.0)] + + +def test_check_target_pixel_wcs_keeps_input_coords_when_wcs_target_is_off_frame(monkeypatch): + image = np.zeros((100, 120), dtype=float) + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [210.0, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + header = wcs.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 120 + header["NAXIS2"] = 100 + + ra_list, dec_list = exotic_module.get_ra_dec(header) + + monkeypatch.setattr( + exotic_module, + "update_coordinates_with_proper_motion", + lambda info_dict, obs_time: (212.0, 54.0), + ) + monkeypatch.setattr( + exotic_module, + "get_psf_parameters", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("centroiding should be skipped")), + ) + + x_pixel, y_pixel = exotic_module.check_target_pixel_wcs( + 25.0, + 30.0, + {"ra": 210.0, "dec": 54.0, "dist": 0.0, "pm_ra": 0.0, "pm_dec": 0.0}, + ra_list, + dec_list, + image, + 2461100.5, + non_interactive_run=True, + wcs_header=header, + ) + + assert x_pixel == 25.0 + assert y_pixel == 30.0 + + +def test_should_ignore_header_wcs_defaults_to_false(): + assert exotic_module.should_ignore_header_wcs(None) is False + assert exotic_module.should_ignore_header_wcs("n") is False + assert exotic_module.should_ignore_header_wcs("y") is True + + +def test_check_wcs_ignores_header_wcs_when_override_enabled(monkeypatch): + monkeypatch.setattr( + exotic_module, + "search_wcs", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("header WCS should be ignored")), + ) + + wcs_file = exotic_module.check_wcs( + "frame.fits", + ".", + "n", + ignore_header_wcs=True, + ) + + assert wcs_file is None + + +def test_check_wcs_keeps_plate_solution_when_override_enabled(monkeypatch): + monkeypatch.setattr(exotic_module, "get_wcs", lambda *_args, **_kwargs: "solved_wcs.fits") + monkeypatch.setattr( + exotic_module, + "search_wcs", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("header WCS should not be consulted")), + ) + + wcs_file = exotic_module.check_wcs( + "frame.fits", + ".", + "y", + ignore_header_wcs=True, + ) + + assert wcs_file == "solved_wcs.fits" + + +def test_should_use_multiprocess_transform_precompute_respects_header_wcs_override(): + assert exotic_module.should_use_multiprocess_transform_precompute( + ["frame1.fits", "frame2.fits"], + requested_processes=2, + ignore_header_wcs=True, + ) is True diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py new file mode 100644 index 00000000..91404566 --- /dev/null +++ b/tests/test_exotic_proper_motion.py @@ -0,0 +1,344 @@ +import importlib.util +import sys +import types +import numpy as np + + +def _module_available(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + +def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: + if not _module_available(name): + sys.modules.setdefault(name, module) + + +fake_barycorrpy = types.ModuleType("barycorrpy") +fake_utc_tdb = types.ModuleType("barycorrpy.utc_tdb") +fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None +fake_astroalign = types.ModuleType("astroalign") +fake_astroalign.PIXEL_TOL = 1 +fake_astroquery = types.ModuleType("astroquery") +fake_astroquery_simbad = types.ModuleType("astroquery.simbad") +fake_astroquery_simbad.Simbad = type("Simbad", (), {}) +fake_astroquery_gaia = types.ModuleType("astroquery.gaia") +fake_astroquery_gaia.Gaia = type("Gaia", (), {}) +fake_imreg_dft = types.ModuleType("imreg_dft") +fake_colour_demosaicing = types.ModuleType("colour_demosaicing") +fake_colour_demosaicing.demosaicing_CFA_Bayer_bilinear = lambda *args, **kwargs: None +fake_photutils = types.ModuleType("photutils") +fake_photutils_aperture = types.ModuleType("photutils.aperture") +fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_detection = types.ModuleType("photutils.detection") +fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) +fake_ldtk = types.ModuleType("ldtk") +fake_ldtk.LDPSet = type("LDPSet", (), {}) +fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) +fake_ldtk_ldmodel = types.ModuleType("ldtk.ldmodel") +fake_ldtk_ldmodel.LinearModel = type("LinearModel", (), {}) +fake_ldtk_ldmodel.QuadraticModel = type("QuadraticModel", (), {}) +fake_ldtk_ldmodel.NonlinearModel = type("NonlinearModel", (), {}) +fake_lmfit = types.ModuleType("lmfit") +fake_pylightcurve = types.ModuleType("pylightcurve") +fake_pylightcurve_models = types.ModuleType("pylightcurve.models") +fake_pylightcurve_exoplanet = types.ModuleType("pylightcurve.models.exoplanet_lc") +fake_pylightcurve_exoplanet.transit = lambda *args, **kwargs: None +fake_pyvo = types.ModuleType("pyvo") +fake_ultranest = types.ModuleType("ultranest") +fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) +fake_elca = types.ModuleType("exotic.api.elca") +fake_elca.lc_fitter = lambda *args, **kwargs: None +fake_elca.binner = lambda *args, **kwargs: None +fake_elca.transit = lambda *args, **kwargs: None +fake_elca.get_phase = lambda *args, **kwargs: None +fake_ld = types.ModuleType("exotic.api.ld") +fake_ld.LimbDarkening = type("LimbDarkening", (), {}) +fake_ld.ld_re_punct_p = lambda *args, **kwargs: None + +_set_stub_if_missing("astroalign", fake_astroalign) +_set_stub_if_missing("astroquery", fake_astroquery) +_set_stub_if_missing("astroquery.simbad", fake_astroquery_simbad) +_set_stub_if_missing("astroquery.gaia", fake_astroquery_gaia) +_set_stub_if_missing("imreg_dft", fake_imreg_dft) +_set_stub_if_missing("colour_demosaicing", fake_colour_demosaicing) +_set_stub_if_missing("photutils", fake_photutils) +_set_stub_if_missing("photutils.aperture", fake_photutils_aperture) +_set_stub_if_missing("photutils.detection", fake_photutils_detection) +_set_stub_if_missing("ldtk", fake_ldtk) +_set_stub_if_missing("ldtk.ldmodel", fake_ldtk_ldmodel) +_set_stub_if_missing("lmfit", fake_lmfit) +_set_stub_if_missing("pylightcurve", fake_pylightcurve) +_set_stub_if_missing("pylightcurve.models", fake_pylightcurve_models) +_set_stub_if_missing("pylightcurve.models.exoplanet_lc", fake_pylightcurve_exoplanet) +_set_stub_if_missing("pyvo", fake_pyvo) +_set_stub_if_missing("ultranest", fake_ultranest) +_set_stub_if_missing("barycorrpy", fake_barycorrpy) +_set_stub_if_missing("barycorrpy.utc_tdb", fake_utc_tdb) +sys.modules.setdefault("exotic.api.elca", fake_elca) +sys.modules.setdefault("exotic.api.ld", fake_ld) + +from exotic.exotic import ( + auto_tune_aperture_sigma_grid, + check_coordinates, + cheap_lightcurve_prescore, + comparison_star_stability_summary, + fit_lightcurve, + is_comp_star_required, + is_target_driven_comp_selection_enabled, + phase_bin_sigma_clip, + update_coordinates_with_proper_motion, +) + + +def test_update_coordinates_handles_non_numeric_proper_motion_values(): + info = { + "ra": 10.0, + "dec": 20.0, + "dist": "", + "pm_ra": "nan-value", + "pm_dec": None, + } + + updated_ra, updated_dec = update_coordinates_with_proper_motion(info, 2459945.5) + + assert updated_ra == info["ra"] + assert updated_dec == info["dec"] + + +def test_update_coordinates_accepts_numeric_strings(): + info = { + "ra": 10.0, + "dec": 20.0, + "dist": "100", + "pm_ra": "10.5", + "pm_dec": "-5.25", + } + + updated_ra, updated_dec = update_coordinates_with_proper_motion(info, 2459945.5) + + assert isinstance(updated_ra, float) + assert isinstance(updated_dec, float) + + +def test_check_coordinates_non_interactive_prefers_wcs_centroid(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=5, + input_y_pixel=5, + centroid_x=100.25, + centroid_y=200.75, + sigma_x=1.0, + sigma_y=1.0, + calculated_x_pixel=100, + calculated_y_pixel=201, + non_interactive_run=True, + ) + + assert x_pixel == 100.25 + assert y_pixel == 200.75 + + +def test_check_coordinates_non_interactive_uses_wcs_pixel_when_centroid_is_nan(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=5, + input_y_pixel=5, + centroid_x=float("nan"), + centroid_y=float("nan"), + sigma_x=1.0, + sigma_y=1.0, + calculated_x_pixel=100, + calculated_y_pixel=201, + non_interactive_run=True, + ) + + assert x_pixel == 100 + assert y_pixel == 201 + + +def test_is_comp_star_required_parses_values(): + assert is_comp_star_required(None) is True + assert is_comp_star_required("y") is True + assert is_comp_star_required("n") is False + + +def test_is_target_driven_comp_selection_enabled_parses_values(): + assert is_target_driven_comp_selection_enabled(None) is False + assert is_target_driven_comp_selection_enabled("y") is True + assert is_target_driven_comp_selection_enabled("n") is False + + +def test_auto_tune_aperture_grid_uses_comparison_field_consistency(): + coarse_apertures_sigma = np.array([2.0, 3.0]) + coarse_annuli_sigma = np.array([8.0]) + coarse_aper_data = { + "target": np.array([[[10.0]], [[11.0]], [[12.0]], [[13.0]], [[14.0]], [[15.0]]]), + "comp1": np.array([[[5.0], [5.0]], [[5.0], [5.0]], [[5.0], [5.0]], [[5.0], [5.0]], [[5.0], [5.0]], [[5.0], [5.0]]]), + "comp2": np.array([[[7.5], [7.5]], [[7.5], [7.5]], [[7.5], [12.0]], [[7.5], [7.5]], [[7.5], [7.5]], [[7.5], [7.5]]]), + } + subset_airmass = np.arange(1.0, 7.0) + + _, _, best_candidate, _ = auto_tune_aperture_sigma_grid( + coarse_apertures_sigma, + coarse_annuli_sigma, + coarse_aper_data, + comp_star_count=2, + subset_airmass=subset_airmass, + require_comp_star=True, + ) + + assert best_candidate["aper_sigma"] == 2.0 + assert best_candidate["comp_index"] in (0, 1) + + +def test_comparison_star_stability_summary_penalizes_variable_candidates(): + airmass = np.linspace(1.0, 1.5, 6) + summary = comparison_star_stability_summary( + { + "comp1": np.array([100.0, 101.0, 100.5, 101.5, 100.8, 101.2]), + "comp2": np.array([80.0, 80.8, 80.4, 81.0, 80.6, 80.9]), + "comp3": np.array([60.0, 60.4, 84.0, 60.6, 60.5, 60.3]), + }, + airmass, + ) + + assert np.isfinite(summary["field_score"]) + assert summary["best_comp_index"] in (0, 1) + assert summary["comp_summaries"][2]["aggregate_score"] > summary["comp_summaries"][0]["aggregate_score"] + + +def test_cheap_lightcurve_prescore_ignores_relative_flux_above_two(): + tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) + cflux = np.full(tflux.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, tflux.shape[0]) + + score = cheap_lightcurve_prescore(tflux, cflux, airmass) + + assert np.isclose(score, 0.0) + + +def test_cheap_lightcurve_prescore_keeps_target_only_mode_unfiltered(): + tflux = np.array([10.0, 11.0, 12.0, 13.0, 14.0, 15.0]) + cflux = np.ones(tflux.shape[0]) + airmass = np.linspace(1.0, 1.5, tflux.shape[0]) + + score = cheap_lightcurve_prescore(tflux, cflux, airmass) + + assert np.isfinite(score) + + +def test_phase_bin_sigma_clip_flags_local_phase_outlier(): + phase_centers = np.linspace(-0.045, 0.045, 10) + phase = np.concatenate([center + np.linspace(-1e-4, 1e-4, 5) for center in phase_centers]) + base_profile = np.array([-0.002, -0.001, 0.0, 0.001, 0.002]) + values = np.concatenate([1.0 + base_profile for _ in phase_centers]) + values[27] = 1.15 + + mask = phase_bin_sigma_clip(values, phase, sigma=3, bins=10) + + assert mask.sum() == 1 + assert mask[27] + + +def test_fit_lightcurve_removes_relative_flux_above_two_before_fit(monkeypatch): + captured = {} + + def fake_lc_fitter(times, fluxes, flux_unc, airmass, prior, bounds, jd_times=None, mode=None): + captured["times"] = np.array(times) + captured["fluxes"] = np.array(fluxes) + captured["flux_unc"] = np.array(flux_unc) + captured["airmass"] = np.array(airmass) + captured["jd_times"] = np.array(jd_times) + captured["mode"] = mode + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) + cflux = np.full(tflux.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, tflux.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, fit_tflux, fit_cflux = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is not None + assert captured["mode"] == "lm" + assert len(captured["fluxes"]) == 5 + assert np.all(captured["fluxes"] <= 2.0) + assert np.allclose(captured["fluxes"], 1.0) + assert np.allclose(fit_tflux, 2.0) + assert np.allclose(fit_cflux, 2.0) + + +def test_fit_lightcurve_refits_after_phase_binned_clip(monkeypatch): + captured_calls = [] + + def fake_lc_fitter(times, fluxes, flux_unc, airmass, prior, bounds, jd_times=None, mode=None): + call_index = len(captured_calls) + captured_calls.append({ + "times": np.array(times), + "fluxes": np.array(fluxes), + "flux_unc": np.array(flux_unc), + "airmass": np.array(airmass), + "jd_times": np.array(jd_times), + "mode": mode, + }) + if call_index == 0: + return types.SimpleNamespace( + residuals=np.zeros(len(times)), + phase=np.linspace(-0.05, 0.05, len(times)), + ) + return types.SimpleNamespace() + + def fake_phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max_iters=3): + mask = np.zeros(len(values), dtype=bool) + mask[-1] = True + return mask + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr("exotic.exotic.phase_bin_sigma_clip", fake_phase_bin_sigma_clip) + + times = np.linspace(0.0, 0.08, 8) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, fit_tflux, fit_cflux = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is not None + assert len(captured_calls) == 2 + assert captured_calls[0]["mode"] == "lm" + assert captured_calls[1]["mode"] == "lm" + assert len(captured_calls[0]["times"]) == 8 + assert len(captured_calls[1]["times"]) == 7 + assert len(fit_tflux) == 7 + assert len(fit_cflux) == 7 diff --git a/tests/test_inputs.py b/tests/test_inputs.py new file mode 100644 index 00000000..81e972d4 --- /dev/null +++ b/tests/test_inputs.py @@ -0,0 +1,397 @@ +import json + +from exotic.inputs import Inputs, camera + + +def test_camera_accepts_cmos_as_ccd_without_prompt(): + assert camera("CMOS") == "CCD" + + +def test_camera_defaults_to_ccd_when_missing_or_unrecognized(): + assert camera(None) == "CCD" + assert camera("") == "CCD" + assert camera("mirrorless") == "CCD" + + +def test_camera_keeps_dslr_as_dslr(): + assert camera("DSLR") == "DSLR" + assert camera("canon dslr") == "DSLR" + + +def test_comp_params_accepts_verbose_camera_key(tmp_path): + init_data = { + "user_info": { + "Camera Type (e.g., CCD or DSLR; Note: if you are using a CMOS, please enter CCD here and then note your actual camera type in \"Observing Notes\")": "CCD" + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["camera"] == "CCD" + + +def test_comp_params_defaults_require_comp_star_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["require_comp_star"] == "y" + + +def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["ignore_header_wcs"] == "n" + + +def test_comp_params_reads_observatory_full_title_from_user_info(tmp_path): + init_data = { + "user_info": {"Observatory Full Title": "Whipple Observatory"}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["obs_name"] == "Whipple Observatory" + + +def test_comp_params_reads_require_comp_star_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"require_comp_star": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["require_comp_star"] == "n" + + +def test_comp_params_reads_ignore_header_wcs_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["ignore_header_wcs"] == "y" + + +def test_prereduced_mode_forces_aavso_comp_to_no(tmp_path): + pre_reduced_file = tmp_path / "prereduced.txt" + pre_reduced_file.write_text("time flux uncertainty\n") + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": {"ra": "", "dec": "", "x": "", "y": ""}, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["aavso_comp"] == "n" + + +def test_prereduced_allows_blank_observatory_location_for_bjd_tdb(tmp_path): + pre_reduced_file = tmp_path / "prereduced.txt" + pre_reduced_file.write_text("time flux uncertainty\n") + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "", + "long": "", + "elev": "", + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["lat"] is None + assert info_dict["long"] is None + assert info_dict["elev"] is None + + +def test_comp_params_accepts_blank_if_none_phot_comp_star_key(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "Comparison Star used in Photometry (blank if none)": { + "ra": "", + "dec": "", + "x": "493", + "y": "202", + } + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["phot_comp_star"] == {"ra": "", "dec": "", "x": "493", "y": "202"} + + +def test_prereduced_uses_aavso_comp_star_metadata_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#COMP_STAR-XC={\"ra\": null, \"dec\": null, \"x\": \"493\", \"y\": \"202\"}\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["phot_comp_star"] == {"ra": "", "dec": "", "x": "493", "y": "202"} + + +def test_prereduced_uses_aavso_observatory_metadata_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#OBSLAT=+32.41638889\n" + "#OBSLON=-110.73444444\n" + "#OBSELEV=2616\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "", + "long": "", + "elev": "", + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["lat"] == 32.41638889 + assert info_dict["long"] == -110.73444444 + assert info_dict["elev"] == 2616.0 + + +def test_prereduced_uses_aavso_obsdate_metadata_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#OBSDATE=2026-03-08\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["date"] == "2026-03-08" + + +def test_prereduced_prefers_aavso_obsdate_metadata_over_init_date(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#OBSDATE=2026-03-08\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "1999-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["date"] == "2026-03-08" + + +def test_prereduced_derives_obsdate_from_first_data_row_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "prereduced.txt" + pre_reduced_file.write_text( + "time,flux,uncertainty\n" + "2458849.5,0.979108,0.0386426\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "JD_UTC", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["date"] == "2020-01-01" + + +def test_prereduced_leaves_phot_comp_star_blank_when_missing_from_aavso_metadata(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["phot_comp_star"] == {"ra": "", "dec": "", "x": "", "y": ""} diff --git a/tests/test_ld.py b/tests/test_ld.py index 794b9c94..db37447c 100644 --- a/tests/test_ld.py +++ b/tests/test_ld.py @@ -213,3 +213,38 @@ def test_invalid_fwhm_range_2() -> None: ld_obj = LimbDarkening(stellar_params) assert ld_obj.check_fwhm(observed_filter) == False + + +def test_additional_standard_filter_aliases_in_filter_column() -> None: + alias_cases = [ + ("bu", "Johnson U", "U", "333.8", "398.8"), + ("pb", "Johnson B", "B", "391.6", "480.6"), + ("pg", "Johnson V", "V", "502.8", "586.8"), + ("pr", "Johnson R", "RJ", "590.0", "810.0"), + ("bi", "Johnson I", "IJ", "780.0", "1020.0"), + ("up", "Sloan u", "SU", "321.8", "386.8"), + ("gp", "Sloan g", "SG", "402.5", "551.5"), + ("rp", "Sloan r", "SR", "553.1", "693.1"), + ("ip", "Sloan i", "SI", "697.5", "827.5"), + ("zp", "Sloan z", "SZ", "841.2", "978.2"), + ("su", "Stromgren u", "STU", "336.3", "367.7"), + ("sv", "Stromgren v", "STV", "401.5", "418.5"), + ("sb", "Stromgren b", "STB", "459.55", "478.05"), + ("sy", "Stromgren y", "STY", "536.7", "559.3"), + ("hb", "Stromgren Hbw", "STHBW", "481.5", "496.5"), + ("zs", "PanSTARRS z-short", "ZS", "826.0", "920.0"), + ("clearV", "MObs CV", "CV", "350.0", "850.0"), + ("w", "MObs CV", "CV", "350.0", "850.0"), + ("pl", "MObs CV", "CV", "350.0", "850.0"), + ("exo", "Astrodon ExoPlanet-BB", "CBB", "500.0", "1000.0"), + ] + + for alias, expected_filter, expected_name, expected_min, expected_max in alias_cases: + observed_filter = {'filter': alias, 'name': None, 'wl_min': None, 'wl_max': None} + setting_filter_values(observed_filter) + assert observed_filter == { + 'filter': expected_filter, + 'name': expected_name, + 'wl_min': expected_min, + 'wl_max': expected_max, + } diff --git a/tests/test_nea_nextastro_fallback.py b/tests/test_nea_nextastro_fallback.py new file mode 100644 index 00000000..1d05d854 --- /dev/null +++ b/tests/test_nea_nextastro_fallback.py @@ -0,0 +1,68 @@ +import requests + +from exotic.api.nea import NASAExoplanetArchive + + +class DummyResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +def test_planet_info_uses_nextastro_fallback_when_nasa_archive_unavailable(monkeypatch): + nea = NASAExoplanetArchive('WASP-12 b') + + def raise_direct_failure(*args, **kwargs): + raise requests.exceptions.RequestException('ipac unavailable') + + monkeypatch.setattr(nea, '_new_scrape', raise_direct_failure) + + payload = { + 'params': { + 'name': 'WASP-12 b', + 'hostStarName': 'WASP-12', + 'raDeg': 180.0, + 'decDeg': 29.0, + 'orbitalPeriodDays': {'value': 1.09, 'errPlus': 0.001, 'errMinus': 0.001}, + 'midTransitTimeDays': {'value': 2450000.5, 'errPlus': 0.0002, 'errMinus': 0.0003}, + 'rpOverRs': {'value': 0.12, 'errPlus': 0.004, 'errMinus': 0.003}, + 'aOverRs': {'value': 3.0, 'errPlus': 0.2, 'errMinus': 0.1}, + 'inclinationDeg': {'value': 83.5, 'errPlus': 0.8, 'errMinus': 0.7}, + 'eccentricity': 0.0, + 'argPeriastronDeg': 0.0, + 'starTeffK': {'value': 6300.0, 'errPlus': 50.0, 'errMinus': 40.0}, + 'starFeh': {'value': 0.2, 'errPlus': 0.03, 'errMinus': 0.02}, + 'starLogg': {'value': 4.1, 'errPlus': 0.05, 'errMinus': 0.04}, + 'source': {'table': 'pscomppars', 'localCache': True}, + } + } + + called = {} + + def fake_get(url, params, timeout): + called['url'] = url + called['params'] = params + called['timeout'] = timeout + return DummyResponse(payload) + + monkeypatch.setattr(requests, 'get', fake_get) + + planet_name, candidate, pl_dict = nea.planet_info() + + assert planet_name == 'WASP-12 b' + assert candidate is False + assert called['url'] == 'https://archive.nextastro.org/api/exoplanet_params' + assert called['params'] == {'name': 'WASP-12 b'} + assert pl_dict['pName'] == 'WASP-12 b' + assert pl_dict['sName'] == 'WASP-12' + assert pl_dict['ra'] == 180.0 + assert pl_dict['dec'] == 29.0 + assert pl_dict['pPer'] == 1.09 + assert pl_dict['midT'] == 2450000.5 + assert pl_dict['rprs'] == 0.12 + assert pl_dict['aRs'] == 3.0 diff --git a/tests/test_nextastro_astrometry.py b/tests/test_nextastro_astrometry.py new file mode 100644 index 00000000..9b367dcc --- /dev/null +++ b/tests/test_nextastro_astrometry.py @@ -0,0 +1,171 @@ +from pathlib import Path + +import numpy as np +from astropy.io.fits import getheader, writeto + +from exotic.api.plate_solution import NextAstroPlateSolution, PlateSolution + + +class DummyResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + +def _create_test_fits(tmp_path: Path) -> Path: + image = np.zeros((100, 120), dtype=float) + image[30, 25] = 10000.0 + image[70, 80] = 8000.0 + image[50, 60] = 7000.0 + fits_path = tmp_path / "image.fits" + writeto(fits_path, image, overwrite=True) + return fits_path + + +def test_generate_source_list(tmp_path): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + source_list = solver._generate_source_list() + + assert source_list is not None + assert source_list["pixel_indexing"] == "0-based" + assert len(source_list["x"]) > 0 + assert len(source_list["x"]) == len(source_list["y"]) == len(source_list["flux"]) + + +def test_plate_solution_writes_wcs_file(tmp_path, monkeypatch): + fits_path = _create_test_fits(tmp_path) + (tmp_path / "temp").mkdir() + + def fake_post(url, json, timeout): + assert url.endswith('/solve') + assert json['image'] == {'width': 120, 'height': 100} + assert json['hints']['ra_deg'] == 210.8023 + assert json['hints']['dec_deg'] == 54.3489 + assert json['hints']['scale_arcsec_per_pix'] == 1.23 + assert json['hints']['scale_tolerance_frac'] == 0.25 + return DummyResponse({'status': 'queued', 'request_id': 'abc123'}) + + def fake_get(url, timeout): + assert url.endswith('/status/abc123') + return DummyResponse({ + 'status': 'solved', + 'solution': { + 'wcs_header': { + 'SIMPLE': True, + 'BITPIX': -64, + 'NAXIS': 2, + 'NAXIS1': 120, + 'NAXIS2': 100, + 'CTYPE1': 'RA---TAN', + 'CTYPE2': 'DEC--TAN', + 'CRVAL1': 210.8, + 'CRVAL2': 54.3, + 'CRPIX1': 60.0, + 'CRPIX2': 50.0, + 'CD1_1': -0.00028, + 'CD1_2': 0.0, + 'CD2_1': 0.0, + 'CD2_2': 0.00028, + } + } + }) + + monkeypatch.setattr('exotic.api.plate_solution.requests.post', fake_post) + monkeypatch.setattr('exotic.api.plate_solution.requests.get', fake_get) + monkeypatch.setattr('exotic.api.plate_solution.time.sleep', lambda _: None) + + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path, ra=210.8023, dec=54.3489, pixel_scale=1.23) + wcs_file = solver.plate_solution() + + assert wcs_file == tmp_path / 'temp' / 'wcs.fits' + header = getheader(wcs_file) + assert header['CTYPE1'] == 'RA---TAN' + assert header['CTYPE2'] == 'DEC--TAN' + + +def test_extract_astrometry_hints_with_scale_only(tmp_path): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path, pixel_scale=2.0) + + hints = solver._extract_astrometry_hints() + + assert hints == {'scale_arcsec_per_pix': 2.0, 'scale_tolerance_frac': 0.25} + + +def test_poll_for_solution_accepts_case_insensitive_running_status(tmp_path, monkeypatch): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + responses = iter([ + DummyResponse({'status': 'RUNNING'}), + DummyResponse({ + 'status': 'solved', + 'solution': { + 'wcs_header': { + 'SIMPLE': True, + 'BITPIX': -64, + 'NAXIS': 2, + 'NAXIS1': 120, + 'NAXIS2': 100, + 'CTYPE1': 'RA---TAN', + 'CTYPE2': 'DEC--TAN', + } + } + }) + ]) + + monkeypatch.setattr('exotic.api.plate_solution.requests.get', lambda url, timeout: next(responses)) + monkeypatch.setattr('exotic.api.plate_solution.time.sleep', lambda _: None) + + header = solver._poll_for_solution('abc123') + + assert header is not False + assert header['CTYPE1'] == 'RA---TAN' + + +def test_poll_for_solution_logs_unexpected_status(tmp_path, monkeypatch, capsys): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + monkeypatch.setattr('exotic.api.plate_solution.requests.get', + lambda url, timeout: DummyResponse({'status': 'processing'})) + + header = solver._poll_for_solution('abc123') + + assert header is False + output = capsys.readouterr().out + assert "Status response (unexpected)" in output + assert "'status': 'processing'" in output + + +def test_nova_upload_includes_astrometry_hints(tmp_path, monkeypatch): + fits_path = _create_test_fits(tmp_path) + + captured_payload = {} + + def fake_post(url, files, data, timeout): + captured_payload['url'] = url + captured_payload['request_json'] = data['request-json'] + return DummyResponse({'status': 'success', 'subid': 42}) + + monkeypatch.setattr('exotic.api.plate_solution.requests.post', fake_post) + + solver = PlateSolution(file=fits_path, directory=tmp_path, ra=150.123, dec=-2.456, + pixel_scale=1.5, radius=1.2, scale_err=30) + sub_id = solver._upload(session='session-id') + + assert sub_id == 42 + assert captured_payload['url'].endswith('/upload') + assert '"session": "session-id"' in captured_payload['request_json'] + assert '"center_ra": 150.123' in captured_payload['request_json'] + assert '"center_dec": -2.456' in captured_payload['request_json'] + assert '"radius": 1.2' in captured_payload['request_json'] + assert '"scale_units": "arcsecperpix"' in captured_payload['request_json'] + assert '"scale_type": "ev"' in captured_payload['request_json'] + assert '"scale_est": 1.5' in captured_payload['request_json'] + assert '"scale_err": 30.0' in captured_payload['request_json'] diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py new file mode 100644 index 00000000..231f16f2 --- /dev/null +++ b/tests/test_nextastro_variability.py @@ -0,0 +1,109 @@ +import sys +import types + +import numpy as np + +fake_barycorrpy = types.ModuleType('barycorrpy') +fake_utc_tdb = types.ModuleType('barycorrpy.utc_tdb') +fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None +fake_barycorrpy.utc_tdb = fake_utc_tdb +sys.modules.setdefault('barycorrpy', fake_barycorrpy) +sys.modules.setdefault('barycorrpy.utc_tdb', fake_utc_tdb) + +from exotic import exotic as exotic_module + + +class DummyResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + +def test_nextastro_variability_logs_json_request_and_response(monkeypatch): + captured = {} + logged = [] + + def fake_post(url, json, timeout): + captured['url'] = url + captured['json'] = json + captured['timeout'] = timeout + return DummyResponse([ + {'is_in_vsx': 0}, + {'is_in_vsx': 1}, + ]) + + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + variability_flags = exotic_module.nextastro_variability_test([(10.1, -11.2), (22.3, -33.4)]) + + assert captured['url'].endswith('/variability_test') + assert captured['timeout'] == 30 + assert captured['json'] == [{'ra': 10.1, 'dec': -11.2}, {'ra': 22.3, 'dec': -33.4}] + assert variability_flags == [False, True] + assert any('NextAstro variability request JSON:' in message for message in logged) + assert any('NextAstro variability response JSON:' in message for message in logged) + + +def test_check_for_variable_stars_uses_nextastro_flags_to_filter(monkeypatch): + logged = [] + + ra_wcs = np.array([[100.1, 100.2], [100.3, 100.4]]) + dec_wcs = np.array([[-10.1, -10.2], [-10.3, -10.4]]) + comp_stars = [[0, 0], [1, 1]] + + monkeypatch.setattr(exotic_module, 'nextastro_variability_test', lambda payload: [False, True]) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + exotic_module.check_for_variable_stars( + ra_wcs, dec_wcs, comp_stars, use_nextastro_variability_server=True + ) + + assert comp_stars == [[0, 0]] + assert any('NextAstro flagged variable: False' in message for message in logged) + assert any('NextAstro flagged variable: True' in message for message in logged) + + +def test_get_wcs_falls_back_to_nextastro_when_nova_fails(monkeypatch): + service_calls = [] + + class DummyPlateSolution: + def __init__(self, **kwargs): + pass + + def plate_solution(self): + service_calls.append('nova') + return False + + class DummyNextAstroSolution: + def __init__(self, **kwargs): + pass + + def plate_solution(self): + service_calls.append('nextastro') + return 'nextastro-wcs' + + monkeypatch.setattr(exotic_module, 'PlateSolution', DummyPlateSolution) + monkeypatch.setattr(exotic_module, 'NextAstroPlateSolution', DummyNextAstroSolution) + monkeypatch.setattr(exotic_module, 'animate_toggle', lambda *args, **kwargs: None) + + solved_wcs = exotic_module.get_wcs('frame.fits', directory='.') + + assert solved_wcs == 'nextastro-wcs' + assert service_calls == ['nova', 'nextastro'] + + +def test_vsx_variable_falls_back_to_nextastro(monkeypatch): + class DummyFailedResponse: + def raise_for_status(self): + raise RuntimeError('VSX unavailable') + + monkeypatch.setattr(exotic_module.requests, 'get', lambda *args, **kwargs: DummyFailedResponse()) + monkeypatch.setattr(exotic_module, 'nextastro_variability_test', lambda payload: [True]) + + is_variable = exotic_module.vsx_variable(ra=10.0, dec=20.0) + + assert is_variable is True diff --git a/tests/test_output_files.py b/tests/test_output_files.py new file mode 100644 index 00000000..3d14b511 --- /dev/null +++ b/tests/test_output_files.py @@ -0,0 +1,166 @@ +from exotic.output_files import OutputFiles, save_comp_star_calibration_summary + + +class DummyFit: + def __init__(self): + self.parameters = { + "tmid": 2450000.123456, + "rprs": 0.1234, + "inc": 88.5, + "a1": 1.0, + "a2": 0.0, + } + self.errors = { + "tmid": 0.0001, + "rprs": 0.001, + "inc": 0.2, + "a1": 0.1, + "a2": 0.1, + } + self.time = [2450000.123456] + self.data = [1.0] + self.dataerr = [0.01] + self.airmass_model = [1.0] + + +def test_aavso_output_includes_observatory_location_headers(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + "pPer": 2.1500082, + "pPerUnc": 1.3e-07, + "rprs": 0.1488623525, + "rprsUnc": 0.0005539487, + "aRs": 5.344, + "aRsUnc": 0.03949, + "inc": 88.98, + "incUnc": 0.7602, + "ecc": 0.159, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "second_obs": "", + "obs_name": "Whipple Observatory", + "camera": "CCD", + "pixel_bin": "1x1", + "exposure": 60.0, + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + "notes": "na", + "filter": "CV", + "filter_desc": "Clear with V zero-point", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( + {"ra": "", "dec": "", "x": "493", "y": "202"}, + [1.0], + (0.1, 0.01), + (0.2, 0.01), + (0.3, 0.01), + (0.4, 0.01), + None, + ) + + output_file = tmp_path / "AAVSO_HAT-P-32 b_2020-01-01.txt" + output_text = output_file.read_text(encoding="utf-8") + + assert "#OBSDATE=2020-01-01" in output_text + assert "#OBSNAME=Whipple Observatory" in output_text + assert "#OBSLAT=+32.41638889" in output_text + assert "#OBSLON=-110.73444444" in output_text + assert "#OBSELEV=2616" in output_text + + +def test_aavso_output_omits_obsname_header_when_blank(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + "pPer": 2.1500082, + "pPerUnc": 1.3e-07, + "rprs": 0.1488623525, + "rprsUnc": 0.0005539487, + "aRs": 5.344, + "aRsUnc": 0.03949, + "inc": 88.98, + "incUnc": 0.7602, + "ecc": 0.159, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "second_obs": "", + "obs_name": "", + "camera": "CCD", + "pixel_bin": "1x1", + "exposure": 60.0, + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + "notes": "na", + "filter": "CV", + "filter_desc": "Clear with V zero-point", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( + {"ra": "", "dec": "", "x": "493", "y": "202"}, + [1.0], + (0.1, 0.01), + (0.2, 0.01), + (0.3, 0.01), + (0.4, 0.01), + None, + ) + + output_file = tmp_path / "AAVSO_HAT-P-32 b_2020-01-01.txt" + output_text = output_file.read_text(encoding="utf-8") + + assert "#OBSNAME=" not in output_text + + +def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): + summary_path = save_comp_star_calibration_summary( + tmp_path, + "HAT-P-32 b", + "2026-03-09", + "PSF photometry", + 0.0012, + [ + { + "label": "Comp 1", + "position": [101, 202], + "selected": True, + "aggregate_score": 0.0012, + "ensemble_score": 0.0010, + "pairwise_median_score": 0.0011, + "pairwise_max_score": 0.0014, + "self_score": 0.0009, + "valid_pair_count": 2, + }, + { + "label": "Comp 2", + "position": [303, 404], + "selected": False, + "aggregate_score": 0.0031, + "ensemble_score": 0.0028, + "pairwise_median_score": 0.0030, + "pairwise_max_score": 0.0035, + "self_score": 0.0012, + "valid_pair_count": 2, + }, + ], + 0, + ) + + text = summary_path.read_text() + assert "# Selected comparison star,1" in text + assert "Comp 1,101,202,true" in text diff --git a/tests/test_plots.py b/tests/test_plots.py new file mode 100644 index 00000000..1b2d17bd --- /dev/null +++ b/tests/test_plots.py @@ -0,0 +1,48 @@ +import matplotlib +matplotlib.use("Agg") + +import numpy as np +from matplotlib.axes import Axes + +from exotic.plots import plot_obs_stats + + +class DummyFit: + def __init__(self): + self.time = np.array([1.0, 2.0, 3.0]) + self.airmass = np.array([1.1, 1.2, 1.3]) + + +def test_plot_obs_stats_applies_relative_flux_mask(tmp_path, monkeypatch): + fit = DummyFit() + psf_rows = np.arange(35, dtype=float).reshape(5, 7) + psf = {"target": psf_rows} + si = np.array([2, 0, 4, 1, 3]) + gi = np.array([True, False, True, True, True]) + relative_flux_mask = np.array([True, False, True, True]) + captured = [] + + original_plot = Axes.plot + + def spy_plot(self, x, y, *args, **kwargs): + captured.append((np.asarray(x), np.asarray(y))) + return original_plot(self, x, y, *args, **kwargs) + + monkeypatch.setattr(Axes, "plot", spy_plot) + + plot_obs_stats( + fit, + [], + psf, + si, + gi, + "Target", + str(tmp_path), + "2026-03-09", + relative_flux_mask=relative_flux_mask, + ) + + assert captured + np.testing.assert_array_equal(captured[0][0], fit.time) + np.testing.assert_array_equal(captured[0][1], np.array([14.0, 7.0, 21.0])) + assert (tmp_path / "temp" / "Observing_Statistics_target_2026-03-09.png").exists() diff --git a/tests/test_ultranest_utils.py b/tests/test_ultranest_utils.py new file mode 100644 index 00000000..5d4d11d6 --- /dev/null +++ b/tests/test_ultranest_utils.py @@ -0,0 +1,253 @@ +import io +import logging + +from exotic.api.ultranest_utils import run_reactive_sampler +from exotic.api.ultranest_utils import supports_ultranest_live_status + + +class _FakeStream(io.StringIO): + def __init__(self, tty): + super().__init__() + self._tty = tty + + def isatty(self): + return self._tty + + +def _reset_ultranest_env(monkeypatch): + monkeypatch.delenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_RICH_PROGRESS", raising=False) + monkeypatch.delenv("CI", raising=False) + + +def test_supports_ultranest_live_status_overrides(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("TERM", "xterm-256color") + stream = _FakeStream(tty=True) + + monkeypatch.setenv("EXOTIC_ULTRANEST_RICH_PROGRESS", "1") + assert supports_ultranest_live_status(stream=stream) is True + + monkeypatch.setenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "1") + assert supports_ultranest_live_status(stream=stream) is False + + +def test_run_reactive_sampler_compat_mode_emits_progress(monkeypatch): + _reset_ultranest_env(monkeypatch) + stream = _FakeStream(tty=False) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + callback = kwargs["viz_callback"] + callback(None, {"it": 120, "ncall": 540, "logz": -151.9, "logz_remain": -150.0}) + callback(None, {"it": 360, "ncall": 907, "logz": -139.2, "logz_remain": -141.0}) + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=stream, + interval_seconds=0.0, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert callable(sampler.kwargs["viz_callback"]) + output = stream.getvalue() + assert "Using simple progress updates" in output + assert "[ultranest] running" in output + assert "[ultranest] done 100.00%" in output + + +def test_run_reactive_sampler_silent_mode(monkeypatch): + _reset_ultranest_env(monkeypatch) + stream = _FakeStream(tty=False) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=False, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert sampler.kwargs["viz_callback"] is False + assert stream.getvalue() == "" + + +def test_run_reactive_sampler_tty_defaults_to_simple_status(monkeypatch): + _reset_ultranest_env(monkeypatch) + stream = _FakeStream(tty=True) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert callable(sampler.kwargs["viz_callback"]) + output = stream.getvalue() + assert "Using simple progress updates" in output + assert "10s heartbeat" in output + + +def test_run_reactive_sampler_plain_override_uses_simple_status(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "1") + stream = _FakeStream(tty=True) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert callable(sampler.kwargs["viz_callback"]) + output = stream.getvalue() + assert "Using simple progress updates" in output + assert "10s heartbeat" in output + + +def test_run_reactive_sampler_rich_override_uses_native_status(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_RICH_PROGRESS", "1") + monkeypatch.setenv("TERM", "xterm-256color") + stream = _FakeStream(tty=True) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is True + assert "viz_callback" not in sampler.kwargs + assert stream.getvalue() == "" + + +def test_run_reactive_sampler_silent_mode_mutes_ultranest_logger(monkeypatch): + _reset_ultranest_env(monkeypatch) + progress_stream = _FakeStream(tty=False) + log_stream = io.StringIO() + + class FakeSampler: + def __init__(self): + self.kwargs = None + self.logger = logging.getLogger("ultranest.tests.silent_mode") + self.logger.handlers = [] + self.logger.propagate = False + handler = logging.StreamHandler(log_stream) + handler.setLevel(logging.INFO) + self.logger.addHandler(handler) + self.logger.setLevel(logging.INFO) + + def run(self, **kwargs): + self.kwargs = kwargs + self.logger.info("native ultranest noise") + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=False, + stream=progress_stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert sampler.kwargs["viz_callback"] is False + assert progress_stream.getvalue() == "" + assert log_stream.getvalue() == "" + + +def test_run_reactive_sampler_plain_mode_mutes_ultranest_logger(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "1") + progress_stream = _FakeStream(tty=False) + log_stream = io.StringIO() + + class FakeSampler: + def __init__(self): + self.kwargs = None + self.logger = logging.getLogger("ultranest.tests.plain_mode") + self.logger.handlers = [] + self.logger.propagate = False + handler = logging.StreamHandler(log_stream) + handler.setLevel(logging.INFO) + self.logger.addHandler(handler) + self.logger.setLevel(logging.INFO) + + def run(self, **kwargs): + self.kwargs = kwargs + callback = kwargs["viz_callback"] + self.logger.info("native ultranest noise") + callback(None, {"it": 25, "ncall": 40, "logz": -10.0, "logz_remain": -9.5}) + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=progress_stream, + interval_seconds=0.0, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert callable(sampler.kwargs["viz_callback"]) + assert "Using simple progress updates" in progress_stream.getvalue() + assert "native ultranest noise" not in progress_stream.getvalue() + assert log_stream.getvalue() == "" From d69e32862de77bcc00ef2825ef04f0b5045e6c1a Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Wed, 11 Mar 2026 16:39:07 +1100 Subject: [PATCH 002/116] quality edits from kalee --- exotic/api/output_aavso.py | 18 ++ exotic/exotic.py | 52 +++- exotic/exotic_gui.py | 148 ++++++++-- exotic/inputs.py | 441 ++++++++++++++++++++++++++-- exotic/output_files.py | 9 + tests/test_inputs.py | 213 +++++++++++++- tests/test_nextastro_variability.py | 121 ++++++++ tests/test_output_files.py | 12 + 8 files changed, 953 insertions(+), 61 deletions(-) diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 91b5ff31..8b945b04 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -113,6 +113,12 @@ def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): priors_dict, filter_dict, results_dict = aavso_dicts(self.p_dict, self.fit, self.i_dict, self.durs, ld0, ld1, ld2, ld3) + gaia_dist = "" if self.p_dict.get('dist') is None else str(self.p_dict.get('dist')) + gaia_pmra = "" if self.p_dict.get('pm_ra') is None else str(self.p_dict.get('pm_ra')) + gaia_pmdec = "" if self.p_dict.get('pm_dec') is None else str(self.p_dict.get('pm_dec')) + gaia_dist_header = f"#GAIADIST={gaia_dist}\n" if gaia_dist else "" + gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" + gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" # compute 32 character hash of the results_dict hash_object = hashlib.sha256(dumps(results_dict).encode()) @@ -134,6 +140,9 @@ def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): f"#EXOPLANET_NAME={self.p_dict['pl_name']}\n" # code yields f"#BINNING=1x1\n" # uhhh i just put One. f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI + f"{gaia_dist_header}" + f"{gaia_pmra_header}" + f"{gaia_pmdec_header}" f"#COMP_STAR-XC=null\n" f"#NOTES=TESS Data\n" "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" # fixed @@ -176,6 +185,12 @@ def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): priors_dict, filter_dict, results_dict = aavso_dicts(self.p_dict, self.fit, self.i_dict, self.durs, ld0, ld1, ld2, ld3) + gaia_dist = "" if self.p_dict.get('dist') is None else str(self.p_dict.get('dist')) + gaia_pmra = "" if self.p_dict.get('pm_ra') is None else str(self.p_dict.get('pm_ra')) + gaia_pmdec = "" if self.p_dict.get('pm_dec') is None else str(self.p_dict.get('pm_dec')) + gaia_dist_header = f"#GAIADIST={gaia_dist}\n" if gaia_dist else "" + gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" + gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" params_file = self.dir / f"TESS_{tmidstr}_{self.p_dict['pl_name']}_lightcurve.csv" @@ -191,6 +206,9 @@ def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): f"#EXOPLANET_NAME={self.p_dict['pl_name']}\n" # code yields f"#BINNING=1x1\n" # uhhh i just put One. f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI + f"{gaia_dist_header}" + f"{gaia_pmra_header}" + f"{gaia_pmdec_header}" f"#COMP_STAR-XC=null\n" f"#NOTES=TESS Data\n" "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" # fixed diff --git a/exotic/exotic.py b/exotic/exotic.py index b0f8cfc9..28e3386b 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -169,7 +169,7 @@ def log_info(string, warn=False, error=False): if error: print(f"\033[31m {string}\033[0m") elif warn: - print(f"\033[33m {string}\033[0m") + print(f"\033[34m {string}\033[0m") else: print(string) log.debug(string) @@ -1213,13 +1213,36 @@ def query_variable_star_apis(ra, dec): def vsx_auid(ra, dec, radius=0.01, maglimit=14): try: url = f"https://www.aavso.org/vsx/index.php?view=api.list&ra={ra}&dec={dec}&radius={radius}&tomag={maglimit}&format=json" - result = requests.get(url) - return result.json()['VSXObjects']['VSXObject'][0]['AUID'] + result = requests.get(url, timeout=30) + result.raise_for_status() + vsx_objects = extract_vsx_objects(result.json()) + if not vsx_objects: + return False + return vsx_objects[0].get('AUID', False) or False except Exception: log.info("\nThe target star does not have an AUID.") return False +def extract_vsx_objects(payload): + if not isinstance(payload, dict): + return [] + + vsx_objects = payload.get('VSXObjects', []) + if isinstance(vsx_objects, dict): + vsx_object = vsx_objects.get('VSXObject', []) + if isinstance(vsx_object, dict): + return [vsx_object] + if isinstance(vsx_object, list): + return vsx_object + return [] + + if isinstance(vsx_objects, list): + return vsx_objects + + return [] + + @retry(stop=stop_after_delay(30)) def vsx_variable(ra, dec, radius=0.01, maglimit=14): default_vsx_error = None @@ -1227,12 +1250,17 @@ def vsx_variable(ra, dec, radius=0.01, maglimit=14): url = f"https://www.aavso.org/vsx/index.php?view=api.list&ra={ra}&dec={dec}&radius={radius}&tomag={maglimit}&format=json" result = requests.get(url, timeout=30) result.raise_for_status() - var = result.json()['VSXObjects']['VSXObject'][0]['Category'] + vsx_objects = extract_vsx_objects(result.json()) + if not vsx_objects: + return False + + first_vsx_object = vsx_objects[0] + var = first_vsx_object.get('Category', '') - if var.lower() == "variable": - vname = result.json()['VSXObjects']['VSXObject'][0]['Name'] - vdec = result.json()['VSXObjects']['VSXObject'][0]['Declination2000'] - vra = result.json()['VSXObjects']['VSXObject'][0]['RA2000'] + if isinstance(var, str) and var.lower() == "variable": + vname = first_vsx_object.get('Name') + vdec = first_vsx_object.get('Declination2000') + vra = first_vsx_object.get('RA2000') log_info(f"\nVSX variable check found {vname} at RA {vra}, DEC {vdec}\n" f"and will be removed from reduction.", warn=True) return True @@ -2705,7 +2733,6 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, airmass_sorted = airmass[si] if len(times_sorted) <= 1: - log_info('No data left after filtering', warn=True) return None, None, None dt = np.mean(np.diff(times_sorted)) @@ -2735,7 +2762,6 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, arrayAirmass) if np.sum(~nanmask) <= 1: - log_info('No data left after filtering', warn=True) return None, None, None else: arrayFinalFlux = arrayFinalFlux[~nanmask] @@ -3425,6 +3451,12 @@ def main(): exotic_infoDict, userpDict['pName'] = inputs_obj.complete_red(userpDict['pName']) else: exotic_infoDict, userpDict['pName'] = inputs_obj.prereduced(userpDict['pName']) + for motion_key in ('dist', 'pm_ra', 'pm_dec'): + current_motion_value = userpDict.get(motion_key) + if current_motion_value is None or (isinstance(current_motion_value, str) and not current_motion_value.strip()): + header_motion_value = exotic_infoDict.get(motion_key) + if header_motion_value is not None: + userpDict[motion_key] = header_motion_value # Make a temp directory of helpful files Path(Path(exotic_infoDict['save']) / "temp").mkdir(exist_ok=True) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 85da2aab..ac03c612 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -48,6 +48,7 @@ import os import platform import python_version +import re import subprocess import sys @@ -78,9 +79,9 @@ from version import __version__ try: - from .inputs import parse_aavso_comp_star + from .inputs import parse_aavso_comp_star, parse_aavso_prereduced_overrides except ImportError: - from inputs import parse_aavso_comp_star + from inputs import parse_aavso_comp_star, parse_aavso_prereduced_overrides animate_toggle() @@ -127,6 +128,49 @@ def file_path(self): return self.filePath.get() +def stringify_prefill(value): + if value is None: + return "" + return str(value) + + +def normalize_filter_option_lookup(value): + if value is None: + return None + return re.sub(r'[\W_]+', '', str(value).strip().lower()) + + +def preselected_filter_option(prefill, choices): + if not isinstance(prefill, dict): + return choices[0] + + filter_desc = prefill.get('filter_desc') + filter_value = prefill.get('filter') + if filter_desc in photometric_filters: + return filter_desc + if filter_value in photometric_filters: + return filter_value + + normalized_candidates = { + normalize_filter_option_lookup(filter_desc), + normalize_filter_option_lookup(filter_value), + } + normalized_candidates.discard(None) + + for option in choices: + if option == "N/A": + continue + option_metadata = photometric_filters.get(option, {}) + if normalize_filter_option_lookup(option) in normalized_candidates: + return option + if normalize_filter_option_lookup(option_metadata.get('name')) in normalized_candidates: + return option + + if prefill.get('wl_min') and prefill.get('wl_max'): + return "N/A" + return choices[0] + + def main(): try: python_version.check(min=(3, 8, 0), max=(4, 0, 0)) @@ -263,7 +307,10 @@ def main(): # "Planet Name": "HAT-P-32 b", planet_label = tk.Label(root, text="Planet Name", justify=tk.LEFT) planet_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - planet_entry.insert(tk.END, "HAT-P-32 b") + planet_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('planet')) or "HAT-P-32 b", + ) planet_label.grid(row=i, column=j, sticky=tk.W, pady=2) planet_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -513,17 +560,27 @@ def save_input(): comp_star_note = tk.Label( root, - text="Comparison star metadata will be loaded from an AAVSO #COMP_STAR-XC header when available.", + text="Leave these blank to load time, units, exposure, filter, and comparison-star metadata from an AAVSO header when available.", justify=tk.LEFT ) comp_star_note.grid(row=i, column=j, columnspan=2, sticky=tk.W, pady=2) i += 1 def save_input(): - input_data['file_time'] = pretime_entry.get() - input_data['file_units'] = preunit_entry.get() - input_data['exp'] = float(exp_entry.get()) + aavso_prefill = parse_aavso_prereduced_overrides(prered_file.file_path) + exposure_text = exp_entry.get().strip() + + input_data['aavso_prefill'] = aavso_prefill + input_data['file_time'] = pretime_entry.get().strip() or stringify_prefill(aavso_prefill.get('file_time')) + input_data['file_units'] = preunit_entry.get().strip() or stringify_prefill(aavso_prefill.get('file_units')) + input_data['exp'] = float(exposure_text) if exposure_text else aavso_prefill.get('exposure') input_data['phot_comp_star'] = parse_aavso_comp_star(prered_file.file_path) + input_data['filtermin'] = aavso_prefill.get('wl_min') + input_data['filtermax'] = aavso_prefill.get('wl_max') + input_data['obs_name'] = stringify_prefill(aavso_prefill.get('obs_name')) + input_data['dist'] = aavso_prefill.get('dist') + input_data['pm_ra'] = aavso_prefill.get('pm_ra') + input_data['pm_dec'] = aavso_prefill.get('pm_dec') root.destroy() # Button for closing @@ -583,6 +640,7 @@ def save_input(): # Set up rows + columns i = 1 j = 0 + aavso_prefill = input_data.get('aavso_prefill', {}) if fitsortext.get() == 2 else {} folderPath = tk.StringVar() @@ -658,7 +716,7 @@ def save_input(): # # "AAVSO Observer Code (blank if none)": "RTZ", obscode_label = tk.Label(root, text="AAVSO Observer Code (leave blank if none)", justify=tk.LEFT) obscode_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - obscode_entry.insert(tk.END, "") + obscode_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('aavso_num'))) obscode_label.grid(row=i, column=j, sticky=tk.W, pady=2) obscode_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -667,7 +725,7 @@ def save_input(): # # "Secondary Observer Codes (blank if none)": "", secondobscode_label = tk.Label(root, text="Secondary Observer Codes (leave blank if none)", justify=tk.LEFT) secondobscode_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - secondobscode_entry.insert(tk.END, "") + secondobscode_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('second_obs'))) secondobscode_label.grid(row=i, column=j, sticky=tk.W, pady=2) secondobscode_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -675,6 +733,7 @@ def save_input(): # # "Observation date": "17-December-2017", obsdate_label = tk.Label(root, text="Observation date (e.g. DAY-MONTH-YEAR)", justify=tk.LEFT) obsdate_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + obsdate_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('date'))) obsdate_label.grid(row=i, column=j, sticky=tk.W, pady=2) obsdate_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -685,6 +744,7 @@ def save_input(): lat_label_text += " [optional for pre-reduced runs]" lat_label = tk.Label(root, text=lat_label_text, justify=tk.LEFT) lat_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + lat_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('lat'))) lat_label.grid(row=i, column=j, sticky=tk.W, pady=2) lat_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -695,6 +755,7 @@ def save_input(): long_label_text += " [optional for pre-reduced runs]" long_label = tk.Label(root, text=long_label_text, justify=tk.LEFT) long_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + long_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('long'))) long_label.grid(row=i, column=j, sticky=tk.W, pady=2) long_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -707,6 +768,8 @@ def save_input(): elevation_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) if fitsortext.get() == 1: elevation_entry.insert(tk.END, "0") + elif aavso_prefill.get('elev') is not None: + elevation_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('elev'))) elevation_label.grid(row=i, column=j, sticky=tk.W, pady=2) elevation_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -718,6 +781,7 @@ def save_input(): "then note your actual camera type under \"Observing Notes\" below)", justify=tk.LEFT) cameratype_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + cameratype_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('camera'))) cameratype_label.grid(row=i, column=j, sticky=tk.W, pady=2) cameratype_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -725,6 +789,7 @@ def save_input(): # # "Pixel Binning": "1x1", pixbin_label = tk.Label(root, text="Pixel Binning (e.g 1x1)", justify=tk.LEFT) pixbin_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + pixbin_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('pixel_bin'))) pixbin_label.grid(row=i, column=j, sticky=tk.W, pady=2) pixbin_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -748,7 +813,7 @@ def save_input(): choices = [item for item in photometric_filters.keys()] + ["N/A"] choices = sorted(set(choices)) # sort and list unique values filteroptions = tk.StringVar(root) - filteroptions.set(choices[0]) # default value + filteroptions.set(preselected_filter_option(aavso_prefill, choices)) l3 = tk.Label(root, text='Filter (use N/A for custom)', justify=tk.LEFT) l3.grid(row=i, column=j, sticky=tk.W, pady=2) @@ -769,6 +834,7 @@ def save_input(): # "Observing Notes": "Weather, seeing was nice.", obsnotes_label = tk.Label(root, text="Observing Notes", justify=tk.LEFT) obsnotes_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + obsnotes_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('notes'))) obsnotes_label.grid(row=i, column=j, sticky=tk.W, pady=2) obsnotes_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -828,23 +894,24 @@ def save_input(): # root.mainloop() def save_input(): - input_data['obsnotes'] = obsnotes_entry.get() + input_data['obsnotes'] = obsnotes_entry.get().strip() or stringify_prefill(aavso_prefill.get('notes')) if filteroptions.get() == "N/A": - input_data['obsfilter'] = "N/A" + input_data['obsfilter'] = stringify_prefill(aavso_prefill.get('filter')) or "N/A" else: input_data['obsfilter'] = photometric_filters[filteroptions.get()]["name"] - input_data['pixbin'] = pixbin_entry.get() - input_data['cameratype'] = cameratype_entry.get() - input_data['obscode'] = obscode_entry.get() - input_data['secondobscode'] = secondobscode_entry.get() - input_data['obsdate'] = obsdate_entry.get() - input_data['lat'] = lat_entry.get().strip() - input_data['long'] = long_entry.get().strip() + input_data['pixbin'] = pixbin_entry.get().strip() or stringify_prefill(aavso_prefill.get('pixel_bin')) + input_data['cameratype'] = cameratype_entry.get().strip() or stringify_prefill(aavso_prefill.get('camera')) + input_data['obscode'] = obscode_entry.get().strip() or stringify_prefill(aavso_prefill.get('aavso_num')) + input_data['secondobscode'] = secondobscode_entry.get().strip() or stringify_prefill(aavso_prefill.get('second_obs')) + input_data['obsdate'] = obsdate_entry.get().strip() or stringify_prefill(aavso_prefill.get('date')) + input_data['lat'] = lat_entry.get().strip() or stringify_prefill(aavso_prefill.get('lat')) + input_data['long'] = long_entry.get().strip() or stringify_prefill(aavso_prefill.get('long')) elevation_value = elevation_entry.get().strip() if fitsortext.get() == 1 or elevation_value: input_data['elevation'] = float(elevation_value) else: - input_data['elevation'] = None + input_data['elevation'] = aavso_prefill.get('elev') + input_data['obs_name'] = stringify_prefill(aavso_prefill.get('obs_name')) input_data['pixscale'] = pixscale_entry.get() if fitsortext.get() == 1: input_data['comppos'] = str(list(ast.literal_eval(comppos_entry.get()))) @@ -872,7 +939,7 @@ def save_input(): pass try: - if filteroptions.get() == "N/A": + if filteroptions.get() == "N/A" and (input_data.get('filtermin') is None or input_data.get('filtermax') is None): root=tk.Tk() root.protocol("WM_DELETE_WINDOW", exit) root.title(f"EXOTIC v{__version__}") @@ -891,6 +958,8 @@ def save_input(): # "Filter Minimum Wavelength (nm)": null, filtermin_label = tk.Label(root, text="Filter Minimum Wavelength (nm)", justify=tk.LEFT) filtermin_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + if input_data.get('filtermin') is not None: + filtermin_entry.insert(tk.END, stringify_prefill(input_data.get('filtermin'))) filtermin_label.grid(row=i, column=j, sticky=tk.W, pady=2) filtermin_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -898,13 +967,17 @@ def save_input(): # "Filter Maximum Wavelength (nm)": null filtermax_label = tk.Label(root, text="Filter Maximum Wavelength (nm)", justify=tk.LEFT) filtermax_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + if input_data.get('filtermax') is not None: + filtermax_entry.insert(tk.END, stringify_prefill(input_data.get('filtermax'))) filtermax_label.grid(row=i, column=j, sticky=tk.W, pady=2) filtermax_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 def save_input(): - input_data['filtermax'] = float(filtermax_entry.get()) - input_data['filtermin'] = float(filtermin_entry.get()) + filtermax_value = filtermax_entry.get().strip() + filtermin_value = filtermin_entry.get().strip() + input_data['filtermax'] = float(filtermax_value) if filtermax_value else input_data.get('filtermax') + input_data['filtermin'] = float(filtermin_value) if filtermin_value else input_data.get('filtermin') root.destroy() # Button for closing @@ -975,7 +1048,10 @@ def save_input(): # "Planet Name": "HAT-P-32 b", planet_label = tk.Label(root, text="Planet Name", justify=tk.LEFT) planet_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - planet_entry.insert(tk.END, "HAT-P-32 b") + planet_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('planet')) or "HAT-P-32 b", + ) planet_label.grid(row=i, column=j, sticky=tk.W, pady=2) planet_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -983,7 +1059,10 @@ def save_input(): # "Host Star Name": "HAT-P-32", star_label = tk.Label(root, text="Host Star Name", justify=tk.LEFT) star_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - star_entry.insert(tk.END, "HAT-P-32") + star_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('host_star')) or "HAT-P-32", + ) star_label.grid(row=i, column=j, sticky=tk.W, pady=2) star_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -1232,7 +1311,10 @@ def save_input(): # "Planet Name": "HAT-P-32 b", planet_label = tk.Label(root, text="Planet Name", justify=tk.LEFT) planet_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - planet_entry.insert(tk.END, "HAT-P-32 b") + planet_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('planet')) or "HAT-P-32 b", + ) planet_label.grid(row=i, column=j, sticky=tk.W, pady=2) planet_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -1240,7 +1322,10 @@ def save_input(): # "Host Star Name": "HAT-P-32", star_label = tk.Label(root, text="Host Star Name", justify=tk.LEFT) star_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - star_entry.insert(tk.END, "HAT-P-32") + star_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('host_star')) or "HAT-P-32", + ) star_label.grid(row=i, column=j, sticky=tk.W, pady=2) star_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -1412,7 +1497,7 @@ def save_input(): "AAVSO Observer Code (blank if none)": input_data['obscode'], "Secondary Observer Codes (blank if none)": input_data['secondobscode'], - "Observatory Full Title": "", + "Observatory Full Title": input_data.get('obs_name', ""), "Observation date": input_data['obsdate'], "Obs. Latitude": input_data['lat'], @@ -1478,7 +1563,7 @@ def save_input(): "AAVSO Observer Code (blank if none)": input_data['obscode'], "Secondary Observer Codes (blank if none)": input_data['secondobscode'], - "Observatory Full Title": "", + "Observatory Full Title": input_data.get('obs_name', ""), "Observation date": input_data['obsdate'], "Obs. Latitude": input_data['lat'], @@ -1534,7 +1619,10 @@ def save_input(): "Star Metallicity (-) Uncertainty": float(input_data['metUncNeg']), "Star Surface Gravity (log(g))": float(input_data['logg']), "Star Surface Gravity (+) Uncertainty": float(input_data['loggUncPos']), - "Star Surface Gravity (-) Uncertainty": float(input_data['loggUncNeg']) + "Star Surface Gravity (-) Uncertainty": float(input_data['loggUncNeg']), + "Star Distance (pc)": null if input_data.get('dist') in (None, "") else float(input_data['dist']), + "Star Proper Motion RA (mas/yr)": null if input_data.get('pm_ra') in (None, "") else float(input_data['pm_ra']), + "Star Proper Motion DEC (mas/yr)": null if input_data.get('pm_dec') in (None, "") else float(input_data['pm_dec']) } elif planetparams.get() == "inits": diff --git a/exotic/inputs.py b/exotic/inputs.py index 7907090d..d0dd85c2 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -1,9 +1,13 @@ import logging import sys import json +import math from pathlib import Path +import requests from astropy.io import fits from astropy.time import Time +from astropy.coordinates import SkyCoord +import astropy.units as u import re try: @@ -16,6 +20,10 @@ from animate import animate_toggle except ImportError: from .animate import animate_toggle +try: + from api.filters import fwhm as photometric_filters, fwhm_alias as photometric_filter_aliases +except ImportError: + from .api.filters import fwhm as photometric_filters, fwhm_alias as photometric_filter_aliases log = logging.getLogger(__name__) @@ -33,6 +41,43 @@ 'long': ('OBSLON', 'OBSLONG', 'LONGITUDE', 'OBS_LONGITUDE', 'LONG'), 'elev': ('OBSELEV', 'OBSALT', 'ELEVATION', 'ALTITUDE', 'HEIGHT'), } +AAVSO_FILTER_HEADER_KEYS = ('FILTER',) +AAVSO_FILTER_XC_HEADER_KEYS = ('FILTER-XC',) +AAVSO_TEXT_HEADER_KEYS = { + 'aavso_num': ('OBSCODE',), + 'second_obs': ('SECONDARY_OBSCODES',), + 'obs_name': ('OBSNAME',), + 'camera': ('OBSTYPE',), + 'pixel_bin': ('BINNING',), + 'notes': ('NOTES',), + 'planet': ('EXOPLANET_NAME',), + 'host_star': ('STAR_NAME',), +} +AAVSO_GAIA_HEADER_KEYS = { + 'dist': ('GAIADIST',), + 'pm_ra': ('GAIAPMRA',), + 'pm_dec': ('GAIAPMDEC', 'GAIADEC'), +} +AAVSO_EXPOSURE_HEADER_KEYS = ('EXPOSURE_TIME', 'EXPTIME', 'EXPOSURE', 'EXP') +AAVSO_TIME_FORMAT_HEADER_KEYS = ('DATE_TYPE',) +AAVSO_MEASUREMENT_TYPE_HEADER_KEYS = ('MEASUREMENT_TYPE',) +AAVSO_ALLOWED_FILE_TIME_FORMATS = {'BJD_TDB', 'JD_UTC', 'MJD_UTC'} +AAVSO_WAVELENGTH_UNIT_FACTORS_TO_NM = { + 'a': 0.1, + 'angstrom': 0.1, + 'angstroms': 0.1, + 'nm': 1.0, + 'nanometer': 1.0, + 'nanometers': 1.0, + 'um': 1000.0, + 'micron': 1000.0, + 'microns': 1000.0, + 'micrometer': 1000.0, + 'micrometers': 1000.0, + 'mum': 1000.0, +} +NEXTASTRO_GAIA_DISTPM_ENDPOINT = 'https://archive.nextastro.org/single_star_gaia_distpm' +NEXTASTRO_REQUEST_TIMEOUT = 30 def is_blank_value(value): @@ -43,6 +88,111 @@ def is_blank_value(value): return False +def normalize_aavso_filter_lookup_key(value): + if is_blank_value(value): + return None + return re.sub(r'[\W_]+', '', str(value).strip().lower()) + + +def coerce_finite_float(value): + if is_blank_value(value): + return None + + try: + numeric_value = float(str(value).strip()) + except (TypeError, ValueError): + return None + + if not math.isfinite(numeric_value): + return None + + return numeric_value + + +def radec_to_decimal_degrees(ra, dec): + if is_blank_value(ra) or is_blank_value(dec): + return None, None + + ra_value = str(ra).strip() + dec_value = str(dec).strip() + ra_unit = u.hourangle if any(separator in ra_value for separator in (':', ' ')) else u.deg + + if ra_unit is u.hourangle: + ra_value = ra_value.replace(':', ' ') + if any(separator in dec_value for separator in (':', ' ')): + dec_value = dec_value.replace(':', ' ') + + try: + coords = SkyCoord(ra=ra_value, dec=dec_value, unit=(ra_unit, u.deg)) + except ValueError: + return None, None + + if not math.isfinite(coords.ra.degree) or not math.isfinite(coords.dec.degree): + return None, None + + return coords.ra.degree, coords.dec.degree + + +def fetch_nextastro_gaia_distpm(ra_deg, dec_deg): + response = requests.get( + NEXTASTRO_GAIA_DISTPM_ENDPOINT, + params={'ra': ra_deg, 'dec': dec_deg}, + timeout=NEXTASTRO_REQUEST_TIMEOUT, + ) + response.raise_for_status() + + payload = response.json() + gaia = payload.get('gaia') if isinstance(payload, dict) else None + if not isinstance(gaia, dict): + return {} + + return { + 'dist': coerce_finite_float(gaia.get('distance_pc')), + 'pm_ra': coerce_finite_float(gaia.get('pmra_mas_per_year')), + 'pm_dec': coerce_finite_float(gaia.get('pmdec_mas_per_year')), + } + + +def populate_missing_gaia_astrometry(planet_dict): + missing_keys = [key for key in ('dist', 'pm_ra', 'pm_dec') if is_blank_value(planet_dict.get(key))] + if not missing_keys: + return planet_dict + + ra_deg, dec_deg = radec_to_decimal_degrees(planet_dict.get('ra'), planet_dict.get('dec')) + if ra_deg is None or dec_deg is None: + return planet_dict + + try: + gaia_values = fetch_nextastro_gaia_distpm(ra_deg, dec_deg) + except requests.exceptions.RequestException as exc: + log_info(f"\nWarning: NextAstro Gaia astrometry lookup failed ({exc}); continuing without missing Gaia values.", + warn=True) + return planet_dict + + filled_keys = [] + for key in missing_keys: + if gaia_values.get(key) is None: + continue + planet_dict[key] = gaia_values[key] + filled_keys.append(key) + + if filled_keys: + log_info("\nRetrieved missing Gaia distance/proper motion from NextAstro archive lookup.") + + return planet_dict + + +AAVSO_FILTER_LOOKUP = {} +for filter_desc, filter_metadata in photometric_filters.items(): + AAVSO_FILTER_LOOKUP[normalize_aavso_filter_lookup_key(filter_desc)] = filter_metadata + AAVSO_FILTER_LOOKUP[normalize_aavso_filter_lookup_key(filter_metadata.get('name'))] = filter_metadata + +for alias, canonical in photometric_filter_aliases.items(): + filter_metadata = photometric_filters.get(canonical) + if filter_metadata is not None: + AAVSO_FILTER_LOOKUP[normalize_aavso_filter_lookup_key(alias)] = filter_metadata + + class Inputs: def __init__(self, init_opt): @@ -54,6 +204,7 @@ def __init__(self, init_opt): 'plate_opt': None, 'aavso_comp': None, 'tar_coords': None, 'comp_stars': None, 'prered_file': None, 'file_units': None, 'file_time': None, 'phot_comp_star': None, 'wl_min': None, 'wl_max': None, 'pixel_scale': None, 'exposure': None, + 'dist': None, 'pm_ra': None, 'pm_dec': None, 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', 'target_driven_comp_selection': 'n' @@ -107,12 +258,18 @@ def prereduced(self, planet): self.params.update({'exposure': exposure, 'file_units': data_file_units, 'file_time': data_file_time, 'phot_comp_star': phot_comp_star}) self.info_dict['prered_file'] = prereduced_file(self.info_dict['prered_file']) - aavso_location = parse_aavso_location(self.info_dict['prered_file']) - - for key in ('lat', 'long', 'elev'): - if is_blank_value(self.info_dict.get(key)) and aavso_location[key] is not None: - self.info_dict[key] = aavso_location[key] - + aavso_overrides = parse_aavso_prereduced_overrides(self.info_dict['prered_file']) + + for key in ( + 'aavso_num', 'second_obs', 'obs_name', 'lat', 'long', 'elev', 'camera', 'pixel_bin', + 'filter', 'notes', 'wl_min', 'wl_max', 'exposure', 'file_time', 'file_units', + 'dist', 'pm_ra', 'pm_dec' + ): + if is_blank_value(self.info_dict.get(key)) and aavso_overrides.get(key) is not None: + self.info_dict[key] = aavso_overrides[key] + + if not planet and not is_blank_value(aavso_overrides.get('planet')): + planet = aavso_overrides['planet'] if not planet: planet = planet_name(planet) @@ -262,7 +419,8 @@ def comp_params(self, init_file, planet_dict): self.info_dict = init_params(user_info, self.info_dict, data['user_info']) self.info_dict = init_params(opt_info, self.info_dict, data['optional_info']) - return init_params(planet_params, planet_dict, data['planetary_parameters']) + planet_dict = init_params(planet_params, planet_dict, data['planetary_parameters']) + return populate_missing_gaia_astrometry(planet_dict) def check_imaging_files(directory, img_type): @@ -428,6 +586,8 @@ def normalize_obs_date(date): return None date = str(date).strip() + if re.fullmatch(r'\d{8}', date): + date = f"{date[0:4]}-{date[4:6]}-{date[6:8]}" if '/' in date: date = date.replace('/', '-') return date @@ -664,6 +824,10 @@ def normalize_phot_comp_star(comp_star): return normalized_comp_star +def read_aavso_metadata(prereduced_file_path): + return dict(parse_aavso_metadata(prereduced_file_path) or []) + + def parse_aavso_metadata(prereduced_file_path): if not prereduced_file_path: return {} @@ -693,29 +857,266 @@ def first_aavso_metadata_value(metadata, aliases): return None -def parse_aavso_location(prereduced_file_path): - metadata = dict(parse_aavso_metadata(prereduced_file_path) or []) +def first_aavso_metadata_text(metadata, aliases, allow_blank=False): + for key in aliases: + if key not in metadata: + continue + + value = metadata.get(key) + if value is None: + return '' if allow_blank else None + + value = str(value).strip() + if allow_blank: + return '' if value.lower() in ('null', 'none') else value + if not is_blank_value(value): + return value + return None + + +def normalize_aavso_code(value): + if value is None: + return None + value = str(value).strip() + return '' if value.lower() in ('', 'n/a', 'na', 'null', 'none') else value + + +def normalize_aavso_blankable_text(value): + if value is None: + return None + value = str(value).strip() + return '' if value.lower() in ('null', 'none') else value + + +def normalize_aavso_coordinate_text(value): + if is_blank_value(value): + return None + + value = str(value).strip() + if value[0] in ('+', '-'): + return value + + try: + numeric_value = float(value) + except ValueError: + return value + + if numeric_value >= 0: + return f"+{value}" + return value + + +def format_aavso_numeric_string(value): + value = float(value) + if value.is_integer(): + return f"{value:.1f}" + return str(value) + + +def convert_aavso_wavelength_to_nm(value, units='nm'): + if is_blank_value(value): + return None + + units_key = 'nm' if units is None else str(units).strip().lower() + units_key = units_key.replace('µ', 'u').replace('μ', 'u') + factor = AAVSO_WAVELENGTH_UNIT_FACTORS_TO_NM.get(units_key) + if factor is None: + return None + + try: + return format_aavso_numeric_string(float(str(value).strip()) * factor) + except ValueError: + return None + + +def parse_aavso_json(value): + if is_blank_value(value): + return None + + try: + return json.loads(value) + except (TypeError, json.JSONDecodeError): + return None + + +def parse_aavso_comp_star_from_metadata(metadata): + comp_star_json = metadata.get('COMP_STAR-XC') + comp_star = parse_aavso_json(comp_star_json) + if comp_star is None: + return blank_phot_comp_star() + return normalize_phot_comp_star(comp_star) + + +def lookup_aavso_filter_metadata(*candidates): + for candidate in candidates: + lookup_key = normalize_aavso_filter_lookup_key(candidate) + if lookup_key and lookup_key in AAVSO_FILTER_LOOKUP: + return AAVSO_FILTER_LOOKUP[lookup_key] + return None + + +def parse_aavso_filter_xc_fwhm(filter_metadata): + if not isinstance(filter_metadata, dict): + return None, None + + fwhm = filter_metadata.get('fwhm') + if isinstance(fwhm, dict): + values = [ + convert_aavso_wavelength_to_nm(fwhm.get('min'), fwhm.get('units', 'nm')), + convert_aavso_wavelength_to_nm(fwhm.get('max'), fwhm.get('units', 'nm')), + ] + elif isinstance(fwhm, (list, tuple)): + values = [] + for item in fwhm[:2]: + if isinstance(item, dict): + values.append(convert_aavso_wavelength_to_nm(item.get('value'), item.get('units', 'nm'))) + else: + values.append(convert_aavso_wavelength_to_nm(item)) + else: + values = [] + + values = [value for value in values if value is not None] + if len(values) < 2: + return None, None + + values = sorted(values[:2], key=float) + return values[0], values[1] + + +def parse_aavso_filter_metadata_from_metadata(metadata): + filter_value = first_aavso_metadata_text(metadata, AAVSO_FILTER_HEADER_KEYS) + filter_xc = parse_aavso_json(first_aavso_metadata_text(metadata, AAVSO_FILTER_XC_HEADER_KEYS)) + + parsed_filter = { + 'filter': filter_value, + 'filter_desc': None, + 'wl_min': None, + 'wl_max': None, + } + + if isinstance(filter_xc, dict): + filter_name = normalize_aavso_blankable_text(filter_xc.get('name')) + filter_desc = normalize_aavso_blankable_text(filter_xc.get('desc')) + if is_blank_value(parsed_filter['filter']): + parsed_filter['filter'] = filter_name or filter_desc + if filter_desc: + parsed_filter['filter_desc'] = filter_desc + + wl_min, wl_max = parse_aavso_filter_xc_fwhm(filter_xc) + if wl_min is not None and wl_max is not None: + parsed_filter['wl_min'] = wl_min + parsed_filter['wl_max'] = wl_max + + filter_record = lookup_aavso_filter_metadata( + parsed_filter['filter'], + parsed_filter['filter_desc'], + ) + if filter_record is not None: + if is_blank_value(parsed_filter['filter']): + parsed_filter['filter'] = filter_record.get('name') or filter_record.get('desc') + if is_blank_value(parsed_filter['filter_desc']): + parsed_filter['filter_desc'] = filter_record.get('desc') + if parsed_filter['wl_min'] is None: + parsed_filter['wl_min'] = filter_record['fwhm'][0] + if parsed_filter['wl_max'] is None: + parsed_filter['wl_max'] = filter_record['fwhm'][1] + + return parsed_filter + + +def parse_aavso_time_format_from_metadata(metadata): + value = first_aavso_metadata_text(metadata, AAVSO_TIME_FORMAT_HEADER_KEYS) + if is_blank_value(value): + return None + + normalized = value.upper().strip().replace('-', '_').replace(' ', '_') + if normalized in AAVSO_ALLOWED_FILE_TIME_FORMATS: + return normalized + if normalized == 'BJD': + return 'BJD_TDB' + if normalized == 'JD': + return 'JD_UTC' + if normalized == 'MJD': + return 'MJD_UTC' + return None + + +def parse_aavso_measurement_units_from_metadata(metadata): + value = first_aavso_metadata_text(metadata, AAVSO_MEASUREMENT_TYPE_HEADER_KEYS) + if is_blank_value(value): + return None + + normalized = re.sub(r'[\W_]+', '', value.lower()) + if 'millimag' in normalized or normalized == 'mmag': + return 'millimagnitude' + if 'flux' in normalized: + return 'flux' + if 'mag' in normalized: + return 'magnitude' + return None + + +def parse_aavso_exposure_from_metadata(metadata): + value = first_aavso_metadata_text(metadata, AAVSO_EXPOSURE_HEADER_KEYS) + if is_blank_value(value): + return None + + try: + return float(str(value).strip()) + except ValueError: + return None + + +def parse_aavso_prereduced_overrides(prereduced_file_path): + metadata = read_aavso_metadata(prereduced_file_path) + filter_metadata = parse_aavso_filter_metadata_from_metadata(metadata) + return { + 'aavso_num': normalize_aavso_code(first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['aavso_num'], allow_blank=True)), + 'second_obs': normalize_aavso_code(first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['second_obs'], allow_blank=True)), + 'obs_name': normalize_aavso_blankable_text(first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['obs_name'], allow_blank=True)), + 'date': normalize_obs_date(first_aavso_metadata_value(metadata, AAVSO_OBSDATE_HEADER_KEYS)), + 'lat': normalize_aavso_coordinate_text(first_aavso_metadata_value(metadata, AAVSO_LOCATION_HEADER_KEYS['lat'])), + 'long': normalize_aavso_coordinate_text(first_aavso_metadata_value(metadata, AAVSO_LOCATION_HEADER_KEYS['long'])), + 'elev': first_aavso_metadata_value(metadata, AAVSO_LOCATION_HEADER_KEYS['elev']), + 'camera': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['camera']), + 'pixel_bin': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['pixel_bin']), + 'filter': filter_metadata['filter'], + 'filter_desc': filter_metadata['filter_desc'], + 'wl_min': filter_metadata['wl_min'], + 'wl_max': filter_metadata['wl_max'], + 'notes': normalize_aavso_blankable_text(first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['notes'], allow_blank=True)), + 'file_time': parse_aavso_time_format_from_metadata(metadata), + 'file_units': parse_aavso_measurement_units_from_metadata(metadata), + 'exposure': parse_aavso_exposure_from_metadata(metadata), + 'dist': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['dist']), + 'pm_ra': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['pm_ra']), + 'pm_dec': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['pm_dec']), + 'phot_comp_star': parse_aavso_comp_star_from_metadata(metadata), + 'planet': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['planet']), + 'host_star': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['host_star']), + } + + +def parse_aavso_location(prereduced_file_path): + metadata = read_aavso_metadata(prereduced_file_path) + parsed_location = { key: first_aavso_metadata_value(metadata, aliases) for key, aliases in AAVSO_LOCATION_HEADER_KEYS.items() } + parsed_location['lat'] = normalize_aavso_coordinate_text(parsed_location['lat']) + parsed_location['long'] = normalize_aavso_coordinate_text(parsed_location['long']) + return parsed_location def parse_aavso_obsdate(prereduced_file_path): - metadata = dict(parse_aavso_metadata(prereduced_file_path) or []) + metadata = read_aavso_metadata(prereduced_file_path) return normalize_obs_date(first_aavso_metadata_value(metadata, AAVSO_OBSDATE_HEADER_KEYS)) def parse_aavso_comp_star(prereduced_file_path): - metadata = dict(parse_aavso_metadata(prereduced_file_path) or []) - comp_star_json = metadata.get('COMP_STAR-XC') - if is_blank_value(comp_star_json): - return blank_phot_comp_star() - - try: - return normalize_phot_comp_star(json.loads(comp_star_json)) - except (TypeError, json.JSONDecodeError): - return blank_phot_comp_star() + metadata = read_aavso_metadata(prereduced_file_path) + return parse_aavso_comp_star_from_metadata(metadata) def phot_comp_star(comp_star, prereduced_file_path=None): @@ -819,7 +1220,7 @@ def log_info(string, warn=False, error=False): if error: print(f"\033[91m {string}\033[00m") elif warn: - print(f"\033[93m {string}\033[00m") + print(f"\033[34m {string}\033[00m") else: print(string) log.debug(string) diff --git a/exotic/output_files.py b/exotic/output_files.py index 07d6269d..1c3a786b 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -80,6 +80,12 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): ld0, ld1, ld2, ld3) obs_name = format_aavso_header_value(self.i_dict.get('obs_name')) obs_name_header = f"#OBSNAME={obs_name}\n" if obs_name else "" + gaia_dist = format_aavso_header_value(self.p_dict.get('dist')) + gaia_pmra = format_aavso_header_value(self.p_dict.get('pm_ra')) + gaia_pmdec = format_aavso_header_value(self.p_dict.get('pm_dec')) + gaia_dist_header = f"#GAIADIST={gaia_dist}\n" if gaia_dist else "" + gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" + gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" params_file = self.dir / f"AAVSO_{self.p_dict['pName']}_{self.i_dict['date']}.txt" @@ -100,6 +106,9 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): f"#OBSLAT={format_aavso_header_value(self.i_dict.get('lat'))}\n" f"#OBSLON={format_aavso_header_value(self.i_dict.get('long'))}\n" f"#OBSELEV={format_aavso_header_value(self.i_dict.get('elev'))}\n" + f"{gaia_dist_header}" + f"{gaia_pmra_header}" + f"{gaia_pmdec_header}" f"#COMP_STAR-XC={dumps(comp_star)}\n" f"#NOTES={self.i_dict['notes']}\n" "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" # fixed diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 81e972d4..ec804cf4 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -1,6 +1,9 @@ import json +import requests +import pytest -from exotic.inputs import Inputs, camera +import exotic.inputs as inputs_module +from exotic.inputs import Inputs, camera, parse_aavso_prereduced_overrides def test_camera_accepts_cmos_as_ccd_without_prompt(): @@ -110,6 +113,123 @@ def test_comp_params_reads_ignore_header_wcs_from_optional_info(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "y" +class DummyResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +def test_comp_params_fetches_missing_gaia_astrometry_from_nextastro(tmp_path, monkeypatch): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": { + "Target Star RA": "01:02:03", + "Target Star Dec": "+04:05:06", + "Star Distance (pc)": None, + "Star Proper Motion RA (mas/yr)": "", + "Star Proper Motion DEC (mas/yr)": "null", + }, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + called = {} + + def fake_get(url, params, timeout): + called["url"] = url + called["params"] = params + called["timeout"] = timeout + return DummyResponse({ + "gaia": { + "distance_pc": 200.0, + "pmra_mas_per_year": 10.0, + "pmdec_mas_per_year": -20.0, + } + }) + + monkeypatch.setattr(inputs_module.requests, "get", fake_get) + + inputs = Inputs(init_opt="y") + planet_dict = inputs.comp_params(init_file, {}) + + assert called["url"] == "https://archive.nextastro.org/single_star_gaia_distpm" + assert called["timeout"] == 30 + assert called["params"]["ra"] == pytest.approx(15.5125) + assert called["params"]["dec"] == pytest.approx(4.085) + assert planet_dict["dist"] == 200.0 + assert planet_dict["pm_ra"] == 10.0 + assert planet_dict["pm_dec"] == -20.0 + + +def test_comp_params_only_backfills_missing_gaia_fields(tmp_path, monkeypatch): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": { + "Target Star RA": 123.4501, + "Target Star Dec": -12.3402, + "Star Distance (pc)": 111.0, + "Star Proper Motion RA (mas/yr)": None, + "Star Proper Motion DEC (mas/yr)": "", + }, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + monkeypatch.setattr( + inputs_module.requests, + "get", + lambda url, params, timeout: DummyResponse({ + "gaia": { + "distance_pc": 222.0, + "pmra_mas_per_year": 8.5, + "pmdec_mas_per_year": -4.25, + } + }), + ) + + inputs = Inputs(init_opt="y") + planet_dict = inputs.comp_params(init_file, {}) + + assert planet_dict["dist"] == 111.0 + assert planet_dict["pm_ra"] == 8.5 + assert planet_dict["pm_dec"] == -4.25 + + +def test_comp_params_continues_when_nextastro_gaia_lookup_fails(tmp_path, monkeypatch): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": { + "Target Star RA": 123.4501, + "Target Star Dec": -12.3402, + "Star Distance (pc)": None, + "Star Proper Motion RA (mas/yr)": "", + "Star Proper Motion DEC (mas/yr)": None, + }, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + def raise_request_exception(*args, **kwargs): + raise requests.exceptions.RequestException("service unavailable") + + monkeypatch.setattr(inputs_module.requests, "get", raise_request_exception) + + inputs = Inputs(init_opt="y") + planet_dict = inputs.comp_params(init_file, {}) + + assert planet_dict["dist"] is None + assert planet_dict["pm_ra"] == "" + assert planet_dict["pm_dec"] is None + + def test_prereduced_mode_forces_aavso_comp_to_no(tmp_path): pre_reduced_file = tmp_path / "prereduced.txt" pre_reduced_file.write_text("time flux uncertainty\n") @@ -298,6 +418,97 @@ def test_prereduced_uses_aavso_obsdate_metadata_without_prompt(tmp_path): assert info_dict["date"] == "2026-03-08" +def test_prereduced_uses_aavso_filter_and_observing_metadata_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#OBSCODE=\n" + "#SECONDARY_OBSCODES=\n" + "#OBSNAME=Backyard Dome\n" + "#OBSDATE=20260303\n" + "#OBSTYPE=CCD\n" + "#BINNING=1x1\n" + "#EXPOSURE_TIME=30.0\n" + "#OBSLAT=35.554298\n" + "#OBSLON=-105.870197\n" + "#OBSELEV=2194.0\n" + "#GAIADIST=512.4\n" + "#GAIAPMRA=13.25\n" + "#GAIAPMDEC=-7.5\n" + "#NOTES=na\n" + "#DATE_TYPE=BJD_TDB\n" + "#MEASUREMENT_TYPE=Rnflux\n" + "#EXOPLANET_NAME=TOI-1259 A b\n" + "#FILTER=CBB\n" + "#FILTER-XC={\"name\": \"CBB\", \"desc\": \"Astrodon ExoPlanet-BB\", \"fwhm\": [{\"value\": \"500.0\", \"units\": \"nm\"}, {\"value\": \"1000.0\", \"units\": \"nm\"}]}\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": None, + "second_obs": None, + "date": "", + "lat": "", + "long": "", + "elev": "", + "camera": None, + "pixel_bin": None, + "filter": None, + "notes": None, + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": None, + "file_units": None, + "file_time": None, + "phot_comp_star": None, + "wl_min": None, + "wl_max": None, + }) + + info_dict, planet = inputs.prereduced(None) + + assert planet == "TOI-1259 A b" + assert info_dict["aavso_num"] == "" + assert info_dict["second_obs"] == "" + assert info_dict["obs_name"] == "Backyard Dome" + assert info_dict["date"] == "2026-03-03" + assert info_dict["lat"] == 35.554298 + assert info_dict["long"] == -105.870197 + assert info_dict["elev"] == 2194.0 + assert info_dict["camera"] == "CCD" + assert info_dict["pixel_bin"] == "1x1" + assert info_dict["notes"] == "na" + assert info_dict["file_time"] == "BJD_TDB" + assert info_dict["file_units"] == "flux" + assert info_dict["exposure"] == 30.0 + assert info_dict["dist"] == "512.4" + assert info_dict["pm_ra"] == "13.25" + assert info_dict["pm_dec"] == "-7.5" + assert info_dict["filter"] == "CBB" + assert info_dict["wl_min"] == "500.0" + assert info_dict["wl_max"] == "1000.0" + + +def test_parse_aavso_prereduced_overrides_uses_known_filter_lookup_when_filter_xc_missing(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=CBB\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "CBB" + assert overrides["filter_desc"] == "Astrodon ExoPlanet-BB" + assert overrides["wl_min"] == "500.0" + assert overrides["wl_max"] == "1000.0" + + def test_prereduced_prefers_aavso_obsdate_metadata_over_init_date(tmp_path): pre_reduced_file = tmp_path / "aavso_prereduced.txt" pre_reduced_file.write_text( diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 231f16f2..bb63388b 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -1,3 +1,4 @@ +import importlib.util import sys import types @@ -7,8 +8,80 @@ fake_utc_tdb = types.ModuleType('barycorrpy.utc_tdb') fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None fake_barycorrpy.utc_tdb = fake_utc_tdb + + +def _module_available(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + +def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: + if not _module_available(name): + sys.modules.setdefault(name, module) + + +fake_astroalign = types.ModuleType("astroalign") +fake_astroalign.PIXEL_TOL = 1 +fake_astroquery = types.ModuleType("astroquery") +fake_astroquery_simbad = types.ModuleType("astroquery.simbad") +fake_astroquery_simbad.Simbad = type("Simbad", (), {}) +fake_astroquery_gaia = types.ModuleType("astroquery.gaia") +fake_astroquery_gaia.Gaia = type("Gaia", (), {}) +fake_imreg_dft = types.ModuleType("imreg_dft") +fake_colour_demosaicing = types.ModuleType("colour_demosaicing") +fake_colour_demosaicing.demosaicing_CFA_Bayer_bilinear = lambda *args, **kwargs: None +fake_photutils = types.ModuleType("photutils") +fake_photutils_aperture = types.ModuleType("photutils.aperture") +fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_detection = types.ModuleType("photutils.detection") +fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) +fake_ldtk = types.ModuleType("ldtk") +fake_ldtk.LDPSet = type("LDPSet", (), {}) +fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) +fake_ldtk_ldmodel = types.ModuleType("ldtk.ldmodel") +fake_ldtk_ldmodel.LinearModel = type("LinearModel", (), {}) +fake_ldtk_ldmodel.QuadraticModel = type("QuadraticModel", (), {}) +fake_ldtk_ldmodel.NonlinearModel = type("NonlinearModel", (), {}) +fake_lmfit = types.ModuleType("lmfit") +fake_pylightcurve = types.ModuleType("pylightcurve") +fake_pylightcurve_models = types.ModuleType("pylightcurve.models") +fake_pylightcurve_exoplanet = types.ModuleType("pylightcurve.models.exoplanet_lc") +fake_pylightcurve_exoplanet.transit = lambda *args, **kwargs: None +fake_pyvo = types.ModuleType("pyvo") +fake_ultranest = types.ModuleType("ultranest") +fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) +fake_elca = types.ModuleType("exotic.api.elca") +fake_elca.lc_fitter = lambda *args, **kwargs: None +fake_elca.binner = lambda *args, **kwargs: None +fake_elca.transit = lambda *args, **kwargs: None +fake_elca.get_phase = lambda *args, **kwargs: None +fake_ld = types.ModuleType("exotic.api.ld") +fake_ld.LimbDarkening = type("LimbDarkening", (), {}) +fake_ld.ld_re_punct_p = lambda *args, **kwargs: None + +_set_stub_if_missing("astroalign", fake_astroalign) +_set_stub_if_missing("astroquery", fake_astroquery) +_set_stub_if_missing("astroquery.simbad", fake_astroquery_simbad) +_set_stub_if_missing("astroquery.gaia", fake_astroquery_gaia) +_set_stub_if_missing("imreg_dft", fake_imreg_dft) +_set_stub_if_missing("colour_demosaicing", fake_colour_demosaicing) +_set_stub_if_missing("photutils", fake_photutils) +_set_stub_if_missing("photutils.aperture", fake_photutils_aperture) +_set_stub_if_missing("photutils.detection", fake_photutils_detection) +_set_stub_if_missing("ldtk", fake_ldtk) +_set_stub_if_missing("ldtk.ldmodel", fake_ldtk_ldmodel) +_set_stub_if_missing("lmfit", fake_lmfit) +_set_stub_if_missing("pylightcurve", fake_pylightcurve) +_set_stub_if_missing("pylightcurve.models", fake_pylightcurve_models) +_set_stub_if_missing("pylightcurve.models.exoplanet_lc", fake_pylightcurve_exoplanet) +_set_stub_if_missing("pyvo", fake_pyvo) +_set_stub_if_missing("ultranest", fake_ultranest) sys.modules.setdefault('barycorrpy', fake_barycorrpy) sys.modules.setdefault('barycorrpy.utc_tdb', fake_utc_tdb) +sys.modules.setdefault("exotic.api.elca", fake_elca) +sys.modules.setdefault("exotic.api.ld", fake_ld) from exotic import exotic as exotic_module @@ -21,6 +94,10 @@ def __init__(self, payload, status_code=200): def json(self): return self._payload + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + def test_nextastro_variability_logs_json_request_and_response(monkeypatch): captured = {} @@ -107,3 +184,47 @@ def raise_for_status(self): is_variable = exotic_module.vsx_variable(ra=10.0, dec=20.0) assert is_variable is True + + +def test_vsx_variable_handles_empty_default_response_without_fallback(monkeypatch): + nextastro_called = {'value': False} + + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda *args, **kwargs: DummyResponse({'VSXObjects': []}), + ) + monkeypatch.setattr( + exotic_module, + 'nextastro_variability_test', + lambda payload: nextastro_called.__setitem__('value', True), + ) + + is_variable = exotic_module.vsx_variable(ra=10.0, dec=20.0) + + assert is_variable is False + assert nextastro_called['value'] is False + + +def test_vsx_variable_parses_default_vsx_object_list(monkeypatch): + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda *args, **kwargs: DummyResponse({ + 'VSXObjects': { + 'VSXObject': [ + { + 'Name': 'alf Ori', + 'RA2000': '88.79292', + 'Declination2000': '7.40706', + 'Category': 'Variable', + } + ] + } + }), + ) + monkeypatch.setattr(exotic_module, 'nextastro_variability_test', lambda payload: [False]) + + is_variable = exotic_module.vsx_variable(ra=88.79292, dec=7.40706) + + assert is_variable is True diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 3d14b511..b57e31f2 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -37,6 +37,9 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): "inc": 88.98, "incUnc": 0.7602, "ecc": 0.159, + "dist": 245.7, + "pm_ra": 14.25, + "pm_dec": -9.5, } i_dict = { "save": str(tmp_path), @@ -75,6 +78,9 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): assert "#OBSLAT=+32.41638889" in output_text assert "#OBSLON=-110.73444444" in output_text assert "#OBSELEV=2616" in output_text + assert "#GAIADIST=245.7" in output_text + assert "#GAIAPMRA=14.25" in output_text + assert "#GAIAPMDEC=-9.5" in output_text def test_aavso_output_omits_obsname_header_when_blank(tmp_path): @@ -91,6 +97,9 @@ def test_aavso_output_omits_obsname_header_when_blank(tmp_path): "inc": 88.98, "incUnc": 0.7602, "ecc": 0.159, + "dist": None, + "pm_ra": None, + "pm_dec": None, } i_dict = { "save": str(tmp_path), @@ -125,6 +134,9 @@ def test_aavso_output_omits_obsname_header_when_blank(tmp_path): output_text = output_file.read_text(encoding="utf-8") assert "#OBSNAME=" not in output_text + assert "#GAIADIST=" not in output_text + assert "#GAIAPMRA=" not in output_text + assert "#GAIAPMDEC=" not in output_text def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): From 65025feac9eebe1b3a86b8e17b43acdd1d9a2715 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Wed, 11 Mar 2026 17:32:46 +1100 Subject: [PATCH 003/116] remove dead and loud imports --- exotic/api/colab.py | 12 +- exotic/api/elca.py | 11 +- exotic/api/joint_fitter.py | 25 ++- exotic/api/nbody.py | 1 - exotic/api/output_aavso.py | 2 - exotic/api/plotting.py | 17 +-- exotic/api/rv_fitter.py | 1 - exotic/exotic.py | 99 ++++++------ tests/test_centroid_wcs.py | 29 ++++ tests/test_lazy_pylightcurve_imports.py | 192 ++++++++++++++++++++++++ 10 files changed, 295 insertions(+), 94 deletions(-) create mode 100644 tests/test_lazy_pylightcurve_imports.py diff --git a/exotic/api/colab.py b/exotic/api/colab.py index b8e23fac..3b7a683f 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -53,27 +53,21 @@ ######################################################### from astropy.io import fits from astropy.time import Time -from barycorrpy import utc_tdb # import bokeh.io # from bokeh.io import output_notebook -from bokeh.palettes import Viridis256 -from bokeh.plotting import figure, output_file, show -from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LinearColorMapper, LogColorMapper, \ +from bokeh.plotting import figure, show +from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LogColorMapper, \ LogTicker, PanTool, ResetTool, WheelZoomTool # import copy -from io import BytesIO from IPython.display import display, HTML # from IPython.display import Image # from ipywidgets import widgets, HBox import json import numpy as np import os -from pprint import pprint import re -from scipy.ndimage import label -from skimage.transform import rescale, resize, downscale_local_mean +from skimage.transform import downscale_local_mean # import subprocess -import time def display_image(filename): diff --git a/exotic/api/elca.py b/exotic/api/elca.py index a75b135c..c7fc9c22 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -40,12 +40,14 @@ # Fit an exoplanet transit model to time series data. # ########################################################################### # from astropy.time import Time +import builtins import copy +from contextlib import redirect_stderr, redirect_stdout +import io from itertools import cycle import bottleneck as bn import matplotlib.pyplot as plt import numpy as np -from pylightcurve.models.exoplanet_lc import transit as pytransit from scipy import spatial from scipy.optimize import least_squares from scipy.signal import savgol_filter @@ -67,6 +69,13 @@ except ImportError: from .ultranest_utils import run_reactive_sampler +if not getattr(builtins, "_EXOTIC_IMPORTING_MODULES_PRINTED", False): + print("Importing modules. Please wait.......") + builtins._EXOTIC_IMPORTING_MODULES_PRINTED = True + +with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + from pylightcurve.models.exoplanet_lc import transit as pytransit + def weightedflux(flux, gw, nearest): return np.sum(flux[nearest] * gw, axis=-1) diff --git a/exotic/api/joint_fitter.py b/exotic/api/joint_fitter.py index 7b30f804..4aadd3e4 100644 --- a/exotic/api/joint_fitter.py +++ b/exotic/api/joint_fitter.py @@ -37,11 +37,13 @@ # ########################################################################### # from astropy import constants as const from astropy import units as u +import builtins from copy import deepcopy +from contextlib import redirect_stderr, redirect_stdout +import io from itertools import cycle import matplotlib.pyplot as plt import numpy as np -from pylightcurve.models.exoplanet_lc import eclipse_mid_time, transit_flux_drop from scipy import stats try: from ultranest import ReactiveNestedSampler @@ -61,6 +63,13 @@ except ImportError: from .ultranest_utils import run_reactive_sampler +if not getattr(builtins, "_EXOTIC_IMPORTING_MODULES_PRINTED", False): + print("Importing modules. Please wait.......") + builtins._EXOTIC_IMPORTING_MODULES_PRINTED = True + +with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + from pylightcurve.models.exoplanet_lc import eclipse_mid_time, transit as _pylightcurve_transit + AU = const.au.to(u.m).value Mjup = const.M_jup.to(u.kg).value Msun = const.M_sun.to(u.kg).value @@ -116,15 +125,8 @@ def planet_orbit(period, sma_over_rs, eccentricity, inclination, periastron, mid def pytransit(limb_darkening_coefficients, rp_over_rs, period, sma_over_rs, eccentricity, inclination, periastron, mid_time, time_array, method='claret', precision=3): - - position_vector = planet_orbit(period, sma_over_rs, eccentricity, inclination, periastron, mid_time, time_array) - - projected_distance = np.where( - position_vector[0] < 0, 1.0 + 5.0 * rp_over_rs, - np.sqrt(position_vector[1] * position_vector[1] + position_vector[2] * position_vector[2])) - - return transit_flux_drop(limb_darkening_coefficients, rp_over_rs, projected_distance, - method=method, precision=precision) + return _pylightcurve_transit(limb_darkening_coefficients, rp_over_rs, period, sma_over_rs, eccentricity, + inclination, periastron, mid_time, time_array, method=method, precision=precision) def transit(times, values): model = pytransit([values['u0'], values['u1'], values['u2'], values['u3']], @@ -133,9 +135,6 @@ def transit(times, values): values['tmid'], times, method='claret', precision=3) return model -from pylightcurve.models.exoplanet_lc import transit as pytransit -from pylightcurve.models.exoplanet_lc import eclipse_mid_time - def eclipse(times, values): tme = eclipse_mid_time(values['per'], values['ars'], values['ecc'], values['inc'], values['omega'], values['tmid']) model = pytransit([0,0,0,0], diff --git a/exotic/api/nbody.py b/exotic/api/nbody.py index 915970a9..e4d0784a 100644 --- a/exotic/api/nbody.py +++ b/exotic/api/nbody.py @@ -46,7 +46,6 @@ import numpy as np import matplotlib.pyplot as plt import rebound -from exotic.api.plotting import corner from exotic.api.ultranest_utils import run_reactive_sampler from ultranest import ReactiveNestedSampler from astropy.io import fits diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 8b945b04..09afac50 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -38,10 +38,8 @@ import hashlib from json import dump, dumps from numpy import mean, median, std -import os from pathlib import Path import re -from tkinter import NONE try: from .utils import round_to_2 diff --git a/exotic/api/plotting.py b/exotic/api/plotting.py index 35f324ab..a4525ec9 100644 --- a/exotic/api/plotting.py +++ b/exotic/api/plotting.py @@ -35,16 +35,13 @@ # EXOplanet Transit Interpretation Code (EXOTIC) # # NOTE: See companion file version.py for version info. # ########################################################################### # -from astropy.io import fits -# from astroscrappy import detect_cosmics -from bokeh.io import output_notebook -from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LinearColorMapper, LogColorMapper, \ - LogTicker, PanTool, ResetTool, WheelZoomTool -from bokeh.palettes import Viridis256 -from bokeh.plotting import figure, output_file, show -from io import BytesIO -import json -import logging +from astropy.io import fits +# from astroscrappy import detect_cosmics +from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LogColorMapper, \ + LogTicker, PanTool, ResetTool, WheelZoomTool +from bokeh.plotting import figure, output_file, show +import json +import logging import matplotlib.pyplot as plt from matplotlib.ticker import MaxNLocator, NullLocator, ScalarFormatter import numpy as np diff --git a/exotic/api/rv_fitter.py b/exotic/api/rv_fitter.py index fd6850e2..92e8b058 100644 --- a/exotic/api/rv_fitter.py +++ b/exotic/api/rv_fitter.py @@ -43,7 +43,6 @@ import matplotlib.pyplot as plt import numpy as np from ultranest import ReactiveNestedSampler -from scipy.optimize import least_squares try: from elca import lc_fitter diff --git a/exotic/exotic.py b/exotic/exotic.py index 28e3386b..c12866cd 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -73,15 +73,12 @@ from astropy.time import Time from astropy.visualization import astropy_mpl_style from astropy.wcs import WCS, FITSFixedWarning -from astroquery.simbad import Simbad -from astroquery.gaia import Gaia # UTC to BJD converter import from barycorrpy.utc_tdb import JDUTC_to_BJDTDB # julian conversion imports import dateutil.parser as dup import imreg_dft as ird from pathlib import Path -import pyvo as vo import logging from logging.handlers import TimedRotatingFileHandler from matplotlib.animation import FuncAnimation @@ -91,7 +88,6 @@ import numpy as np # photometry from photutils.aperture import CircularAperture -import pandas as pd import re import requests # scipy imports @@ -100,16 +96,15 @@ from scipy.ndimage import binary_erosion, gaussian_filter from skimage.registration import phase_cross_correlation from skimage.transform import SimilarityTransform -from skimage.color import rgb2gray # error handling for scraper from tenacity import retry, stop_after_delay # color, color_demosaicing from colour_demosaicing import demosaicing_CFA_Bayer_bilinear # ########## EXOTIC imports ########## try: # light curve numerics - from .api.elca import lc_fitter, binner, transit, get_phase + from .api.elca import lc_fitter, transit, get_phase except ImportError: # package import - from api.elca import lc_fitter, binner, transit, get_phase + from api.elca import lc_fitter, transit, get_phase try: # output files from inputs import Inputs, comparison_star_coords except ImportError: # package import @@ -1169,27 +1164,6 @@ def query_variable_star_apis(ra, dec): # Convert comparison star coordinates from pixel to WCS sample = SkyCoord(ra * u.deg, dec * u.deg, frame='fk5') return vsx_variable(sample.ra.deg, sample.dec.deg) - # radius = u.Quantity(20.0, u.arcsec) - # # Query GAIA first to check for variability using the phot_variable_flag trait - # gaia_result = gaia_query(sample, radius) - # if not gaia_result: - # log_info("Warning: Your comparison star cannot be resolved in the Gaia star database; " - # "EXOTIC cannot check if it is variable or not. " - # "\nEXOTIC will still include this star in the reduction. " - # "\nPlease proceed with caution as we cannot check for stellar variability.\n", warn=True) - # else: - # # Individually go through the phot_variable_flag indicator for each star to see if variable or not - # variableFlagList = gaia_result.columns["phot_variable_flag"] - # constantCounter = 0 - # for currFlag in variableFlagList: - # if currFlag == "VARIABLE": - # return True - # elif currFlag == "NOT_AVAILABLE": - # continue - # elif currFlag == "CONSTANT": - # constantCounter += 1 - # if constantCounter == len(variableFlagList): - # return False # # # Query SIMBAD and search identifier result table to determine if comparison star is variable in any form # # This is a secondary check if GAIA query returns inconclusive results @@ -1338,25 +1312,6 @@ def check_for_variable_stars(ra_wcs, dec_wcs, comp_stars, use_nextastro_variabil if query_variable_star_apis(ra, dec): comp_stars.remove(comp_star) - -@retry(stop=stop_after_delay(30)) -def gaia_query(sample, radius): - try: - gaia_query = Gaia.cone_search(sample, radius) - return gaia_query.get_results() - except Exception: - return False - - -@retry(stop=stop_after_delay(30)) -def simbad_query(sample): - try: - simbad_result = Simbad.query_region(sample, radius=20 * u.arcsec) - return simbad_result['MAIN_ID'][0].decode("utf-8") - except Exception: - return False - - # Apply calibrations if applicable def apply_cals(image_data, gen_dark, gen_bias, gen_flat, i): if gen_dark is not None and gen_dark.size != 0: @@ -1613,9 +1568,14 @@ def log_reduction_timing_overview(prefix='Reduction timing overview'): ) +def _display_filename(file_name): + return str(file_name).replace("\\", "/").rsplit("/", 1)[-1] + + # Aligns imaging data from .fits file to easily track the host and comparison star's positions def transformation(image_data, file_name, roi=1, report_failure=True, reference_image=None): start_time = perf_counter() + display_file_name = _display_filename(file_name) if report_failure: plateStatus.setCurrentFilename(file_name) @@ -1634,7 +1594,10 @@ def transformation(image_data, file_name, roi=1, report_failure=True, reference_ roi_current = current_image[roiy, roix] if roi_reference.shape != roi_current.shape or roi_reference.size == 0: - log.debug(f"Warning: Following image failed pre-alignment checks in {perf_counter() - start_time:.2f}s - {file_name}") + log.debug( + f"Warning: Following image failed pre-alignment checks in " + f"{perf_counter() - start_time:.2f}s - {display_file_name}" + ) if report_failure: plateStatus.alignmentError() return SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) @@ -1655,7 +1618,10 @@ def transformation(image_data, file_name, roi=1, report_failure=True, reference_ fft_high_confidence = error <= 0.1 and max_shift <= max(roi_current.shape) * 0.25 _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, True) if fft_high_confidence: - log.debug(f"Transformation solved via high-confidence FFT in {perf_counter() - start_time:.2f}s for {file_name}") + log.debug( + f"Transformation solved via high-confidence FFT in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) return fft_tform else: _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, False) @@ -1669,7 +1635,10 @@ def transformation(image_data, file_name, roi=1, report_failure=True, reference_ try: results = aa.find_transform(roi_reference, roi_current) _record_transform_stage_timing('astroalign_direct', perf_counter() - stage_start, True) - log.debug(f"Transformation solved via astroalign direct pass in {perf_counter() - start_time:.2f}s for {file_name}") + log.debug( + f"Transformation solved via astroalign direct pass in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) return results[0] except Exception: _record_transform_stage_timing('astroalign_direct', perf_counter() - stage_start, False) @@ -1682,7 +1651,10 @@ def transformation(image_data, file_name, roi=1, report_failure=True, reference_ try: results = aa.find_transform(filtered_reference, filtered_current) _record_transform_stage_timing('astroalign_filtered', perf_counter() - stage_start, True) - log.debug(f"Transformation solved via filtered astroalign in {perf_counter() - start_time:.2f}s for {file_name}") + log.debug( + f"Transformation solved via filtered astroalign in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) return results[0] except Exception: _record_transform_stage_timing('astroalign_filtered', perf_counter() - stage_start, False) @@ -1705,7 +1677,10 @@ def transformation(image_data, file_name, roi=1, report_failure=True, reference_ try: results = aa.find_transform(mask1, mask0) _record_transform_stage_timing('astroalign_mask', perf_counter() - stage_start, True) - log.debug(f"Transformation solved via mask astroalign (p={p}, erode={it}) in {perf_counter() - start_time:.2f}s for {file_name}") + log.debug( + f"Transformation solved via mask astroalign (p={p}, erode={it}) in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) return results[0] except Exception: _record_transform_stage_timing('astroalign_mask', perf_counter() - stage_start, False) @@ -1714,17 +1689,26 @@ def transformation(image_data, file_name, roi=1, report_failure=True, reference_ try: result1 = ird.similarity(roi_reference, roi_current, numiter=3) _record_transform_stage_timing('imreg_dft', perf_counter() - stage_start, True) - log.debug(f"Transformation solved via imreg_dft fallback in {perf_counter() - start_time:.2f}s for {file_name}") + log.debug( + f"Transformation solved via imreg_dft fallback in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) return SimilarityTransform(scale=result1['scale'], rotation=np.radians(result1['angle']), translation=[-1 * result1['tvec'][1], -1 * result1['tvec'][0]]) except Exception: _record_transform_stage_timing('imreg_dft', perf_counter() - stage_start, False) if fft_tform is not None: - log.debug(f"Transformation fell back to FFT translation in {perf_counter() - start_time:.2f}s for {file_name}") + log.debug( + f"Transformation fell back to FFT translation in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) return fft_tform - log.debug(f"Warning: Following image failed to align in {perf_counter() - start_time:.2f}s - {file_name}") + log.debug( + f"Warning: Following image failed to align in " + f"{perf_counter() - start_time:.2f}s - {display_file_name}" + ) if report_failure: plateStatus.alignmentError() return SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) @@ -1864,8 +1848,9 @@ def log_finding_transformation_progress(i, total_jobs, file_name, use_multiproce log_info(f"Multiprocessing finding transformations progress: {completed}/{total_jobs}") return - sys.stdout.write(f"Finding transformation {i + 1} of {total_jobs} : {file_name}\n") - log.debug(f"Finding transformation {i + 1} of {total_jobs} : {file_name}\n") + display_file_name = _display_filename(file_name) + sys.stdout.write(f"Finding transformation {i + 1} of {total_jobs} : {display_file_name}\n") + log.debug(f"Finding transformation {i + 1} of {total_jobs} : {display_file_name}\n") sys.stdout.flush() diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index a82acce5..fe4ccd9a 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -1,3 +1,4 @@ +import io import sys import types import importlib.util @@ -202,6 +203,34 @@ def test_should_ignore_header_wcs_defaults_to_false(): assert exotic_module.should_ignore_header_wcs("y") is True +def test_display_filename_returns_basename_for_unix_and_windows_paths(): + assert ( + exotic_module._display_filename( + "/content/drive/MyDrive/0.Exoplanets/2.Transits/run/frame_001.fits.fz" + ) + == "frame_001.fits.fz" + ) + assert exotic_module._display_filename(r"C:\data\run\frame_002.fits.fz") == "frame_002.fits.fz" + + +def test_log_finding_transformation_progress_prints_basename(monkeypatch): + stdout = io.StringIO() + debug_messages = [] + + monkeypatch.setattr(exotic_module.sys, "stdout", stdout) + monkeypatch.setattr(exotic_module.log, "debug", lambda message: debug_messages.append(message)) + + exotic_module.log_finding_transformation_progress( + 144, + 220, + "/content/drive/MyDrive/0.Exoplanets/2.Transits/run/frame_145.fits.fz", + False, + ) + + assert stdout.getvalue() == "Finding transformation 145 of 220 : frame_145.fits.fz\n" + assert debug_messages == ["Finding transformation 145 of 220 : frame_145.fits.fz\n"] + + def test_check_wcs_ignores_header_wcs_when_override_enabled(monkeypatch): monkeypatch.setattr( exotic_module, diff --git a/tests/test_lazy_pylightcurve_imports.py b/tests/test_lazy_pylightcurve_imports.py new file mode 100644 index 00000000..9602eb2f --- /dev/null +++ b/tests/test_lazy_pylightcurve_imports.py @@ -0,0 +1,192 @@ +import subprocess +import sys +import textwrap +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_imports_eagerly_load_pylightcurve_without_noise(): + script = textwrap.dedent( + """ + import tempfile + import sys + import types + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + package_dir = root / "pylightcurve" + model_dir = package_dir / "models" + model_dir.mkdir(parents=True) + + (package_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "exoplanet_lc.py").write_text( + "import sys\\n" + "print('LOUD-STDOUT')\\n" + "print('LOUD-STDERR', file=sys.stderr)\\n" + "def transit(*args, **kwargs): return 'stub-transit'\\n" + "def eclipse_mid_time(*args, **kwargs): return 0.0\\n", + encoding="utf-8", + ) + + sys.path.insert(0, str(root)) + + for name in list(sys.modules): + if name == "exotic.api.elca" or name == "exotic.api.joint_fitter" or name.startswith("pylightcurve"): + sys.modules.pop(name) + + fake_ultranest = types.ModuleType("ultranest") + fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) + sys.modules["ultranest"] = fake_ultranest + fake_plotting = types.ModuleType("plotting") + fake_plotting.corner = lambda *args, **kwargs: None + sys.modules["plotting"] = fake_plotting + sys.modules["exotic.api.plotting"] = fake_plotting + fake_ultranest_utils = types.ModuleType("ultranest_utils") + fake_ultranest_utils.run_reactive_sampler = lambda *args, **kwargs: None + sys.modules["ultranest_utils"] = fake_ultranest_utils + sys.modules["exotic.api.ultranest_utils"] = fake_ultranest_utils + + import exotic.api.elca as elca + import exotic.api.joint_fitter as joint_fitter + + assert "pylightcurve.models.exoplanet_lc" in sys.modules + + minimal_values = { + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "rprs": 0.1, + "per": 1.0, + "ars": 10.0, + "ecc": 0.0, + "inc": 89.0, + "omega": 90.0, + "tmid": 0.0, + } + + assert elca.transit([0.0], minimal_values) == "stub-transit" + assert joint_fitter.pytransit([0.0, 0.0, 0.0, 0.0], 0.1, 1.0, 10.0, 0.0, 89.0, 90.0, 0.0, [0.0]) == "stub-transit" + print("imports-ok") + """ + ) + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr or result.stdout + assert "imports-ok" in result.stdout + assert result.stdout.count("Importing modules. Please wait.......") == 1 + assert "LOUD-STDOUT" not in result.stdout + assert "LOUD-STDERR" not in result.stderr + + +def test_import_exotic_avoids_unused_astroquery_modules(): + script = textwrap.dedent( + """ + import sys + import tempfile + import types + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + astroquery_dir = root / "astroquery" + astroquery_dir.mkdir(parents=True) + (astroquery_dir / "__init__.py").write_text("", encoding="utf-8") + (astroquery_dir / "simbad.py").write_text( + "import sys\\n" + "print('LOUD-SIMBAD-STDOUT')\\n" + "print('LOUD-SIMBAD-STDERR', file=sys.stderr)\\n" + "class Simbad:\\n pass\\n", + encoding="utf-8", + ) + (astroquery_dir / "gaia.py").write_text( + "import sys\\n" + "print('LOUD-GAIA-STDOUT')\\n" + "print('LOUD-GAIA-STDERR', file=sys.stderr)\\n" + "class Gaia:\\n pass\\n", + encoding="utf-8", + ) + sys.path.insert(0, str(root)) + + fake_barycorrpy = types.ModuleType("barycorrpy") + fake_utc_tdb = types.ModuleType("barycorrpy.utc_tdb") + fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None + fake_astroalign = types.ModuleType("astroalign") + fake_astroalign.PIXEL_TOL = 1 + fake_imreg_dft = types.ModuleType("imreg_dft") + fake_colour_demosaicing = types.ModuleType("colour_demosaicing") + fake_colour_demosaicing.demosaicing_CFA_Bayer_bilinear = lambda *args, **kwargs: None + fake_photutils = types.ModuleType("photutils") + fake_photutils_aperture = types.ModuleType("photutils.aperture") + fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) + fake_photutils_detection = types.ModuleType("photutils.detection") + fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) + fake_ldtk = types.ModuleType("ldtk") + fake_ldtk.LDPSet = type("LDPSet", (), {}) + fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) + fake_ldtk_ldmodel = types.ModuleType("ldtk.ldmodel") + fake_ldtk_ldmodel.LinearModel = type("LinearModel", (), {}) + fake_ldtk_ldmodel.QuadraticModel = type("QuadraticModel", (), {}) + fake_ldtk_ldmodel.NonlinearModel = type("NonlinearModel", (), {}) + fake_lmfit = types.ModuleType("lmfit") + fake_pyvo = types.ModuleType("pyvo") + fake_ultranest = types.ModuleType("ultranest") + fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) + fake_elca = types.ModuleType("exotic.api.elca") + fake_elca.lc_fitter = lambda *args, **kwargs: None + fake_elca.binner = lambda *args, **kwargs: None + fake_elca.transit = lambda *args, **kwargs: None + fake_elca.get_phase = lambda *args, **kwargs: None + fake_ld = types.ModuleType("exotic.api.ld") + fake_ld.LimbDarkening = type("LimbDarkening", (), {}) + fake_ld.ld_re_punct_p = lambda *args, **kwargs: None + + sys.modules.setdefault("astroalign", fake_astroalign) + sys.modules.setdefault("barycorrpy", fake_barycorrpy) + sys.modules.setdefault("barycorrpy.utc_tdb", fake_utc_tdb) + sys.modules.setdefault("imreg_dft", fake_imreg_dft) + sys.modules.setdefault("colour_demosaicing", fake_colour_demosaicing) + sys.modules.setdefault("photutils", fake_photutils) + sys.modules.setdefault("photutils.aperture", fake_photutils_aperture) + sys.modules.setdefault("photutils.detection", fake_photutils_detection) + sys.modules.setdefault("ldtk", fake_ldtk) + sys.modules.setdefault("ldtk.ldmodel", fake_ldtk_ldmodel) + sys.modules.setdefault("lmfit", fake_lmfit) + sys.modules.setdefault("pyvo", fake_pyvo) + sys.modules.setdefault("ultranest", fake_ultranest) + sys.modules.setdefault("exotic.api.elca", fake_elca) + sys.modules.setdefault("exotic.api.ld", fake_ld) + + import exotic.exotic + + assert "astroquery.gaia" not in sys.modules + assert "astroquery.simbad" not in sys.modules + print("exotic-import-ok") + """ + ) + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr or result.stdout + assert "exotic-import-ok" in result.stdout + assert "LOUD-SIMBAD-STDOUT" not in result.stdout + assert "LOUD-SIMBAD-STDERR" not in result.stderr + assert "LOUD-GAIA-STDOUT" not in result.stdout + assert "LOUD-GAIA-STDERR" not in result.stderr From f7d2ca106afe18f5de40a058e6e6d1a08267cfbb Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 15 Mar 2026 09:30:22 +1100 Subject: [PATCH 004/116] filter tweaks --- exotic/api/filters.py | 20 +++++++++++++++---- exotic/api/plotting.py | 14 ++++++------- exotic/exotic.py | 11 +++++++++- tests/test_centroid_wcs.py | 9 +++++++++ tests/test_inputs.py | 17 ++++++++++++++++ tests/test_ld.py | 41 +++++++++++++++++++++++++++++++++++--- 6 files changed, 97 insertions(+), 15 deletions(-) diff --git a/exotic/api/filters.py b/exotic/api/filters.py index 0976aa18..49164cae 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -9,6 +9,11 @@ "Johnson R": {"name": "RJ", "fwhm": ("590.0", "810.0")}, "Johnson I": {"name": "IJ", "fwhm": ("780.0", "1020.0")}, + # Photographic + "Photographic B": {"name": "PB", "fwhm": ("391.6", "480.6")}, + "Photographic G": {"name": "PG", "fwhm": ("502.8", "586.8")}, + "Photographic R": {"name": "PR", "fwhm": ("590.0", "810.0")}, + # Cousins "Cousins R": {"name": "R", "fwhm": ("561.7", "719.7")}, "Cousins I": {"name": "I", "fwhm": ("721.0", "875.0")}, @@ -82,11 +87,11 @@ # additional short aliases found in FILTER column values "bu": "Johnson U", "bb": "Johnson B", - "pb": "Johnson B", + "pb": "Photographic B", "bv": "Johnson V", - "pg": "Johnson V", + "pg": "Photographic G", "br": "Johnson R", - "pr": "Johnson R", + "pr": "Photographic R", "bi": "Johnson I", "up": "Sloan u", "gp": "Sloan g", @@ -104,7 +109,14 @@ "lum": "ClearV", "w": "ClearV", "pl": "ClearV", - "exo": "Astrodon ExoPlanet-BB" + "exo": "Astrodon ExoPlanet-BB", + + # OSC split-channel aliases + "b1": "Photographic B", + "g1": "Photographic G", + "g2": "Photographic G", + "r1": "Photographic R", + "r2": "Photographic R", } # standard filters w/o precisely defined FWHM values diff --git a/exotic/api/plotting.py b/exotic/api/plotting.py index a4525ec9..e0ba287a 100644 --- a/exotic/api/plotting.py +++ b/exotic/api/plotting.py @@ -35,13 +35,13 @@ # EXOplanet Transit Interpretation Code (EXOTIC) # # NOTE: See companion file version.py for version info. # ########################################################################### # -from astropy.io import fits -# from astroscrappy import detect_cosmics -from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LogColorMapper, \ - LogTicker, PanTool, ResetTool, WheelZoomTool -from bokeh.plotting import figure, output_file, show -import json -import logging +from astropy.io import fits +# from astroscrappy import detect_cosmics +from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LogColorMapper, \ + LogTicker, PanTool, ResetTool, WheelZoomTool +from bokeh.plotting import figure, output_file, show +import json +import logging import matplotlib.pyplot as plt from matplotlib.ticker import MaxNLocator, NullLocator, ScalarFormatter import numpy as np diff --git a/exotic/exotic.py b/exotic/exotic.py index c12866cd..e33b5650 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -171,6 +171,14 @@ def log_info(string, warn=False, error=False): return True +def should_log_plate_solution_path(wcs_file): + if not wcs_file: + return False + + normalized_path = os.fspath(wcs_file).replace("\\", "/") + return normalized_path != "/tmp" and not normalized_path.startswith("/tmp/") + + def log_mid_transit_range_warning_once(array_times, tmid_prior): global _mid_transit_warning_reported if _mid_transit_warning_reported: @@ -3610,7 +3618,8 @@ def main(): chart_id, vsp_comp_stars, vsp_list = None, None, [] if wcs_file: - log_info(f"\nHere is the path to your plate solution: {wcs_file}") + if should_log_plate_solution_path(wcs_file): + log_info(f"\nHere is the path to your plate solution: {wcs_file}") wcs_header = fits.getheader(filename=wcs_file) ra_wcs, dec_wcs = get_ra_dec(wcs_header) diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index fe4ccd9a..21c086a1 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -266,6 +266,15 @@ def test_check_wcs_keeps_plate_solution_when_override_enabled(monkeypatch): assert wcs_file == "solved_wcs.fits" +def test_should_log_plate_solution_path_suppresses_posix_tmp_paths(): + assert exotic_module.should_log_plate_solution_path("/tmp/tmp42fmzrv8/temp/wcs.fits") is False + + +def test_should_log_plate_solution_path_keeps_non_tmp_paths(): + assert exotic_module.should_log_plate_solution_path("/data/session/temp/wcs.fits") is True + assert exotic_module.should_log_plate_solution_path("/tmp_backup/temp/wcs.fits") is True + + def test_should_use_multiprocess_transform_precompute_respects_header_wcs_override(): assert exotic_module.should_use_multiprocess_transform_precompute( ["frame1.fits", "frame2.fits"], diff --git a/tests/test_inputs.py b/tests/test_inputs.py index ec804cf4..a447919f 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -509,6 +509,23 @@ def test_parse_aavso_prereduced_overrides_uses_known_filter_lookup_when_filter_x assert overrides["wl_max"] == "1000.0" +def test_parse_aavso_prereduced_overrides_uses_osc_split_filter_alias_lookup(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=G2\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "G2" + assert overrides["filter_desc"] == "Photographic G" + assert overrides["wl_min"] == "502.8" + assert overrides["wl_max"] == "586.8" + + def test_prereduced_prefers_aavso_obsdate_metadata_over_init_date(tmp_path): pre_reduced_file = tmp_path / "aavso_prereduced.txt" pre_reduced_file.write_text( diff --git a/tests/test_ld.py b/tests/test_ld.py index db37447c..c0cc8eb0 100644 --- a/tests/test_ld.py +++ b/tests/test_ld.py @@ -215,12 +215,27 @@ def test_invalid_fwhm_range_2() -> None: assert ld_obj.check_fwhm(observed_filter) == False +def test_photographic_filter_aliases_in_filter_column() -> None: + alias_cases = [ + ("pb", "Photographic B", "PB", "391.6", "480.6"), + ("pg", "Photographic G", "PG", "502.8", "586.8"), + ("pr", "Photographic R", "PR", "590.0", "810.0"), + ] + + for alias, expected_filter, expected_name, expected_min, expected_max in alias_cases: + observed_filter = {'filter': alias, 'name': None, 'wl_min': None, 'wl_max': None} + setting_filter_values(observed_filter) + assert observed_filter == { + 'filter': expected_filter, + 'name': expected_name, + 'wl_min': expected_min, + 'wl_max': expected_max, + } + + def test_additional_standard_filter_aliases_in_filter_column() -> None: alias_cases = [ ("bu", "Johnson U", "U", "333.8", "398.8"), - ("pb", "Johnson B", "B", "391.6", "480.6"), - ("pg", "Johnson V", "V", "502.8", "586.8"), - ("pr", "Johnson R", "RJ", "590.0", "810.0"), ("bi", "Johnson I", "IJ", "780.0", "1020.0"), ("up", "Sloan u", "SU", "321.8", "386.8"), ("gp", "Sloan g", "SG", "402.5", "551.5"), @@ -248,3 +263,23 @@ def test_additional_standard_filter_aliases_in_filter_column() -> None: 'wl_min': expected_min, 'wl_max': expected_max, } + + +def test_osc_split_filter_aliases_in_filter_column() -> None: + alias_cases = [ + ("B1", "Photographic B", "PB", "391.6", "480.6"), + ("G1", "Photographic G", "PG", "502.8", "586.8"), + ("G2", "Photographic G", "PG", "502.8", "586.8"), + ("R1", "Photographic R", "PR", "590.0", "810.0"), + ("R2", "Photographic R", "PR", "590.0", "810.0"), + ] + + for alias, expected_filter, expected_name, expected_min, expected_max in alias_cases: + observed_filter = {'filter': alias, 'name': None, 'wl_min': None, 'wl_max': None} + setting_filter_values(observed_filter) + assert observed_filter == { + 'filter': expected_filter, + 'name': expected_name, + 'wl_min': expected_min, + 'wl_max': expected_max, + } From 705e2ea915a31d91653a881337f171650294d7f4 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Fri, 20 Mar 2026 15:54:03 +1100 Subject: [PATCH 005/116] Too big a commit because I forgot to push it. --- .../single_transit/transit_fit_example.py | 19 +- .../for_exotic_py_candidate_inits_maker.py | 1 + exotic/api/elca.py | 232 +++++++++++++++--- exotic/api/nea.py | 23 ++ exotic/api/plotting.py | 14 +- exotic/api/ultranest_utils.py | 4 +- exotic/exotic.py | 157 ++++++++++-- exotic/exotic_gui.py | 5 + exotic/inputs.py | 27 +- exotic/output_files.py | 86 +++++-- inits.json | 2 + tests/test_elca_baseline.py | 136 ++++++++++ tests/test_exotic_proper_motion.py | 168 +++++++++++++ tests/test_inputs.py | 78 ++++++ tests/test_nea_nextastro_fallback.py | 31 +++ tests/test_output_files.py | 81 ++++++ 16 files changed, 980 insertions(+), 84 deletions(-) create mode 100644 tests/test_elca_baseline.py diff --git a/examples/single_transit/transit_fit_example.py b/examples/single_transit/transit_fit_example.py index 63c1caf8..97e9a548 100644 --- a/examples/single_transit/transit_fit_example.py +++ b/examples/single_transit/transit_fit_example.py @@ -12,8 +12,8 @@ 'ecc': 0.5, # Eccentricity 'omega': 120, # Arg of periastron 'tmid': 0.75, # Time of mid transit [day], - 'a1': 50, # Airmass coefficients - 'a2': 0., # trend = a1 * np.exp(a2 * airmass) + 'a0': 50, # Baseline flux normalization + 'a2': 0., # trend = a0 * np.exp(a2 * airmass) 'T*':5000, 'FE/H': 0, @@ -36,8 +36,8 @@ airmass = np.zeros(time.shape[0]) # GENERATE NOISY DATA - data = transit(time, prior)*prior['a1']*np.exp(prior['a2']*airmass) - data += np.random.normal(0, prior['a1']*250e-6, len(time)) + data = transit(time, prior)*prior['a0']*np.exp(prior['a2']*airmass) + data += np.random.normal(0, prior['a0']*250e-6, len(time)) dataerr = np.random.normal(300e-6, 50e-6, len(time)) + np.random.normal(300e-6, 50e-6, len(time)) # add optimization bounds for free parameters only @@ -45,14 +45,15 @@ 'rprs': [0, 0.1], 'tmid': [prior['tmid']-0.01, prior['tmid']+0.01], 'inc': [87,90], + #'a0': [0.95 * prior['a0'], 1.05 * prior['a0']], # optional explicit baseline offset #'a2': [0, 0.3] # uncomment if you want to fit for airmass - # a2 is used for individual airmass detrending using: a1*exp(airmass*a2) - # a1 is solved for automatically using mean(data/model) and does not need + # a2 is used for individual airmass detrending using: a0*exp(airmass*a2) + # a0 is optional. If omitted, the normalization is solved analytically. + # a1 is kept as a legacy alias for the resolved normalization. # to be included as a free parameter. A monte carlo process is used after # fitting to derive uncertainties on it. It acts like a normalization factor. - # never list 'a1' in bounds, it is perfectly correlated to exp(a2*airmass) - # and is solved for during the fit + # never list both 'a0' and 'a1' in bounds because they are the same scale term } # call the fitting routine @@ -98,4 +99,4 @@ test_ld(ld_obj, filter_info) ld = [ld_obj.ld0[0], ld_obj.ld1[0], ld_obj.ld2[0], ld_obj.ld3[0]] prior['u0'],prior['u1'],prior['u2'],prior['u3'] = ld - """ \ No newline at end of file + """ diff --git a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py index 0fed82ad..3e9f9654 100644 --- a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py +++ b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py @@ -555,6 +555,7 @@ def create_inits_file(parameters, file_name): "Pixel Scale (Ex: 5.21 arcsecs/pixel)": parameters.get("Pixel Scale (Ex: 5.21 arcsecs/pixel)", None), "Filter Minimum Wavelength (nm)": parameters.get("Filter Minimum Wavelength (nm)", None), "Filter Maximum Wavelength (nm)": parameters.get("Filter Maximum Wavelength (nm)", None), + "disable vertical flux normalization": parameters.get("disable vertical flux normalization", False), "require_comp_star": parameters.get("require_comp_star", "y") } } diff --git a/exotic/api/elca.py b/exotic/api/elca.py index c7fc9c22..2f22ae14 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -110,11 +110,112 @@ def get_phase(times, per, tmid): return (times - tmid + 0.25 * per) / per % 1 - 0.25 -def mc_a1(m_a2, sig_a2, transit, airmass, data, n=10000): +def fallback_flux_baseline(): + return 1.0 + + +def has_explicit_flux_baseline(bounds): + return any(key in bounds for key in ('a0', 'a1')) + + +def get_flux_baseline(values, fallback=1.0): + if 'a0' in values: + return values['a0'] + if 'a1' in values: + return values['a1'] + return fallback + + +def solve_flux_baseline(model, data, dataerr=None): + model = np.asarray(model, dtype=float) + data = np.asarray(data, dtype=float) + weights = np.ones(model.shape, dtype=float) + + if dataerr is not None: + dataerr = np.asarray(dataerr, dtype=float) + weights = np.zeros(model.shape, dtype=float) + valid_err = np.isfinite(dataerr) & (dataerr > 0) + weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) + + mask = np.isfinite(model) & np.isfinite(data) & (model != 0) + if dataerr is not None: + mask &= np.isfinite(weights) & (weights > 0) + + if not np.any(mask): + return fallback_flux_baseline() + + masked_model = model[mask] + masked_data = data[mask] + masked_weights = weights[mask] + denom = np.sum(masked_weights * masked_model ** 2) + + if not np.isfinite(denom) or denom <= 0: + ratio = masked_data / masked_model + ratio = ratio[np.isfinite(ratio)] + if ratio.size == 0: + return fallback_flux_baseline() + baseline = np.nanmedian(ratio) + return baseline if np.isfinite(baseline) else fallback_flux_baseline() + + baseline = np.sum(masked_weights * masked_data * masked_model) / denom + return baseline if np.isfinite(baseline) else fallback_flux_baseline() + + +def solve_flux_baseline_uncertainty(model, dataerr): + if dataerr is None: + return 0.0 + model = np.asarray(model, dtype=float) + dataerr = np.asarray(dataerr, dtype=float) + mask = np.isfinite(model) & np.isfinite(dataerr) & (dataerr > 0) + if not np.any(mask): + return 0.0 + denom = np.sum((model[mask] / dataerr[mask]) ** 2) + if not np.isfinite(denom) or denom <= 0: + return 0.0 + return (1.0 / denom) ** 0.5 + + +def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000): + n = int(n) a2 = np.random.normal(m_a2, sig_a2, n) - model = transit * np.exp(np.repeat(np.expand_dims(a2, 0), airmass.shape[0], 0).T * airmass) - detrend = data / model - return np.mean(np.median(detrend, 0)), np.std(np.median(detrend, 0)) + model = transit * np.exp(np.outer(a2, airmass)) + weights = np.ones(transit.shape[0], dtype=float) + + if dataerr is not None: + dataerr = np.asarray(dataerr, dtype=float) + weights = np.zeros(transit.shape[0], dtype=float) + valid_err = np.isfinite(dataerr) & (dataerr > 0) + weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) + + mask = np.isfinite(data) & np.isfinite(transit) + if dataerr is not None: + mask &= np.isfinite(weights) & (weights > 0) + + if not np.any(mask): + return fallback_flux_baseline(), 0.0 + + masked_model = model[:, mask] + masked_data = np.asarray(data, dtype=float)[mask] + masked_weights = weights[mask] + + numer = np.sum(masked_weights * masked_data * masked_model, axis=1) + denom = np.sum(masked_weights * masked_model ** 2, axis=1) + valid = np.isfinite(numer) & np.isfinite(denom) & (denom > 0) + + if not np.any(valid): + best_model = transit * np.exp(m_a2 * airmass) + baseline = solve_flux_baseline(best_model, data, dataerr) + return baseline, solve_flux_baseline_uncertainty(best_model, dataerr) + + baselines = numer[valid] / denom[valid] + baseline = float(np.nanmean(baselines)) + baseline_unc = float(np.nanstd(baselines)) + + if baseline_unc == 0.0: + best_model = transit * np.exp(m_a2 * airmass) + baseline_unc = solve_flux_baseline_uncertainty(best_model, dataerr) + + return baseline, baseline_unc def round_to_2(*args): @@ -188,9 +289,27 @@ def __init__(self, time, data, dataerr, airmass, prior, bounds, neighbors=200, m elif self.mode == "ns": self.fit_nested() + def _validate_flux_baseline_keys(self): + free_flux_keys = [key for key in self.bounds if key in ('a0', 'a1')] + if len(free_flux_keys) > 1: + raise ValueError("Use only one of 'a0' or 'a1' as a free baseline parameter.") + + def _has_free_flux_baseline(self): + return has_explicit_flux_baseline(self.bounds) + + def _set_flux_baseline(self, value, error=0.0): + self.parameters['a0'] = value + self.errors['a0'] = error + self.parameters['a1'] = value + self.errors['a1'] = error + + def _build_systematics_model(self, values): + return get_flux_baseline(values) * np.exp(values.get('a2', 0) * self.airmass) + def fit_LM(self): freekeys = list(self.bounds.keys()) boundarray = np.array([self.bounds[k] for k in freekeys]) + self._validate_flux_baseline_keys() # trim data around predicted transit/eclipse time if np.ndim(self.airmass) == 2: @@ -210,7 +329,11 @@ def lc2min_airmass(pars): for i in range(len(pars)): self.prior[freekeys[i]] = pars[i] model = transit(self.time, self.prior) - model *= self.prior['a1'] * np.exp(self.prior['a2'] * self.airmass) + model *= np.exp(self.prior.get('a2', 0) * self.airmass) + if self._has_free_flux_baseline(): + model *= get_flux_baseline(self.prior) + else: + model *= solve_flux_baseline(model, self.data, self.dataerr) return ((self.data - model) / self.dataerr) ** 2 try: @@ -252,9 +375,21 @@ def create_fit_variables(self): self.time_upsample = np.linspace(min(self.time), max(self.time), 1000) self.transit_upsample = transit(self.time_upsample, self.parameters) self.phase_upsample = get_phase(self.time_upsample, self.parameters['per'], self.parameters['tmid']) - if self.mode == "ns": - self.parameters['a1'], self.errors['a1'] = mc_a1(self.parameters.get('a2', 0), self.errors.get('a2', 1e-6), - self.transit, self.airmass, self.data) + if np.ndim(self.airmass) != 2: + if self.mode == "ns" and not self._has_free_flux_baseline(): + flux_scale, flux_scale_err = mc_a1( + self.parameters.get('a2', 0), + self.errors.get('a2', 1e-6), + self.transit, + self.airmass, + self.data, + self.dataerr, + ) + else: + systematics = self.transit * np.exp(self.parameters.get('a2', 0) * self.airmass) + flux_scale = solve_flux_baseline(systematics, self.data, self.dataerr) + flux_scale_err = self.errors.get('a0', self.errors.get('a1', solve_flux_baseline_uncertainty(systematics, self.dataerr))) + self._set_flux_baseline(flux_scale, flux_scale_err) if np.ndim(self.airmass) == 2: detrended = self.data / self.transit self.wf = weightedflux(detrended, self.gw, self.nearest) @@ -262,7 +397,7 @@ def create_fit_variables(self): self.detrended = self.data / self.wf self.detrendederr = self.dataerr / self.wf else: - self.airmass_model = self.parameters['a1'] * np.exp(self.parameters.get('a2', 0) * self.airmass) + self.airmass_model = self._build_systematics_model(self.parameters) self.model = self.transit * self.airmass_model self.detrended = self.data / self.airmass_model self.detrendederr = self.dataerr / self.airmass_model @@ -299,6 +434,7 @@ def fit_nested(self): freekeys = list(self.bounds.keys()) boundarray = np.array([self.bounds[k] for k in freekeys]) bounddiff = np.diff(boundarray, 1).reshape(-1) + self._validate_flux_baseline_keys() # alloc data for best fit + error self.errors = {} @@ -310,9 +446,11 @@ def loglike(pars): for i in range(len(pars)): self.prior[freekeys[i]] = pars[i] model = transit(self.time, self.prior) - model *= np.exp(self.prior['a2'] * self.airmass) - detrend = self.data / model # used to estimate a1 - model *= np.median(detrend) + model *= np.exp(self.prior.get('a2', 0) * self.airmass) + if self._has_free_flux_baseline(): + model *= get_flux_baseline(self.prior) + else: + model *= solve_flux_baseline(model, self.data, self.dataerr) return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) def prior_transform(upars): @@ -384,9 +522,20 @@ def prior_transform(upars): chis = [] for i in range(len(tests)): lightcurve = transit(self.time, tests[i]) - tests[i]['a1'] = mc_a1(tests[i].get('a2', 0), self.errors.get('a2', 1e-6), - lightcurve, self.airmass, self.data)[0] - airmass = tests[i]['a1'] * np.exp(tests[i].get('a2', 0) * self.airmass) + if self._has_free_flux_baseline(): + flux_scale = get_flux_baseline(tests[i]) + else: + flux_scale = mc_a1( + tests[i].get('a2', 0), + self.errors.get('a2', 1e-6), + lightcurve, + self.airmass, + self.data, + self.dataerr, + )[0] + tests[i]['a0'] = flux_scale + tests[i]['a1'] = flux_scale + airmass = flux_scale * np.exp(tests[i].get('a2', 0) * self.airmass) residuals = self.data - (lightcurve * airmass) chis.append(np.sum(residuals ** 2)) @@ -420,6 +569,12 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) ) lclabel = lclabel1 + "\n" + lclabel2 + if 'a0' in self.parameters: + lclabel3 = r"$a_0$ = %s $\pm$ %s" % ( + str(round_to_2(self.parameters['a0'], self.errors.get('a0', 0))), + str(round_to_2(self.errors.get('a0', 0))) + ) + lclabel += "\n" + lclabel3 if zoom: axs[0].set_ylim([1 - 1.25 * self.parameters['rprs'] ** 2, 1 + 0.5 * self.parameters['rprs'] ** 2]) @@ -510,7 +665,7 @@ def plot_triangle(self): self.parameters[key] + 5 * self.errors[key] ]) - if key == 'a2' or key == 'a1': + if key in ('a0', 'a1', 'a2'): continue mask3 = mask3 & \ @@ -678,9 +833,11 @@ def loglike(pars): # compute model model = transit(self.lc_data[i]['time'], self.lc_data[i]['priors']) - model *= np.exp(self.lc_data[i]['priors']['a2']*self.lc_data[i]['airmass']) - detrend = self.lc_data[i]['flux']/model - model *= np.mean(detrend) + model *= np.exp(self.lc_data[i]['priors'].get('a2', 0) * self.lc_data[i]['airmass']) + if has_explicit_flux_baseline(self.global_bounds) or has_explicit_flux_baseline(self.local_bounds[i]): + model *= get_flux_baseline(self.lc_data[i]['priors']) + else: + model *= solve_flux_baseline(model, self.lc_data[i]['flux'], self.lc_data[i]['ferr']) # add to chi2 chi2 += np.sum( ((self.lc_data[i]['flux']-model)/self.lc_data[i]['ferr'])**2 ) @@ -750,13 +907,21 @@ def loglike(pars): local_rprs.append(self.lc_data[n]['priors'][k]) local_rprs_err.append(self.lc_data[n]['errors'][k]) - # solve for a1 + # solve for the local baseline flux scale model = transit(self.lc_data[n]['time'], self.lc_data[n]['priors']) - airmass = np.exp(self.lc_data[n]['airmass']*self.lc_data[n]['priors']['a2']) - detrend = self.lc_data[n]['flux']/(model*airmass) - self.lc_data[n]['priors']['a1'] = np.mean(detrend) - self.lc_data[n]['residuals'] = self.lc_data[n]['flux'] - model*airmass*self.lc_data[n]['priors']['a1'] - self.lc_data[n]['detrend'] = self.lc_data[n]['flux']/(airmass*self.lc_data[n]['priors']['a1']) + airmass = np.exp(self.lc_data[n]['airmass'] * self.lc_data[n]['priors'].get('a2', 0)) + if has_explicit_flux_baseline(self.global_bounds) or has_explicit_flux_baseline(self.local_bounds[n]): + flux_scale = get_flux_baseline(self.lc_data[n]['priors']) + flux_scale_err = self.lc_data[n]['errors'].get('a0', self.lc_data[n]['errors'].get('a1', 0)) + else: + flux_scale = solve_flux_baseline(model * airmass, self.lc_data[n]['flux'], self.lc_data[n]['ferr']) + flux_scale_err = solve_flux_baseline_uncertainty(model * airmass, self.lc_data[n]['ferr']) + self.lc_data[n]['priors']['a0'] = flux_scale + self.lc_data[n]['priors']['a1'] = flux_scale + self.lc_data[n]['errors']['a0'] = flux_scale_err + self.lc_data[n]['errors']['a1'] = flux_scale_err + self.lc_data[n]['residuals'] = self.lc_data[n]['flux'] - model * airmass * flux_scale + self.lc_data[n]['detrend'] = self.lc_data[n]['flux'] / (airmass * flux_scale) # phase self.lc_data[n]['phase'] = get_phase(self.lc_data[n]['time'], self.lc_data[n]['priors']['per'], self.lc_data[n]['priors']['tmid']) @@ -799,8 +964,8 @@ def plot_bestfits(self): nmarker = next(markers) model = transit(self.lc_data[i]['time'], self.lc_data[i]['priors']) - airmass = np.exp(self.lc_data[i]['airmass']*self.lc_data[i]['priors']['a2']) - detrend = self.lc_data[i]['flux']/(model*airmass) + airmass = np.exp(self.lc_data[i]['airmass'] * self.lc_data[i]['priors'].get('a2', 0)) + detrend = self.lc_data[i]['flux'] / (model * airmass) if ax.ndim == 1: ax[i].axis('on') @@ -1076,8 +1241,8 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): 'ecc': 0.5, # Eccentricity 'omega': 120, # Arg of periastron 'tmid': 0.75, # Time of mid transit [day], - 'a1': 50, # Airmass coefficients - 'a2': 0., # trend = a1 * np.exp(a2 * airmass) + 'a0': 50, # Baseline flux normalization + 'a2': 0., # trend = a0 * np.exp(a2 * airmass) 'teff': 5000, 'tefferr': 50, @@ -1104,8 +1269,8 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): airmass = np.zeros(time.shape[0]) # GENERATE NOISY DATA - data = transit(time, prior) * prior['a1'] * np.exp(prior['a2'] * airmass) - data += np.random.normal(0, prior['a1'] * 250e-6, len(time)) + data = transit(time, prior) * prior['a0'] * np.exp(prior['a2'] * airmass) + data += np.random.normal(0, prior['a0'] * 250e-6, len(time)) dataerr = np.random.normal(300e-6, 50e-6, len(time)) + np.random.normal(300e-6, 50e-6, len(time)) # add bounds for free parameters only @@ -1113,9 +1278,10 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): 'rprs': [0, 0.1], 'tmid': [prior['tmid'] - 0.01, prior['tmid'] + 0.01], 'ars': [13, 15], + # 'a0': [0.95 * prior['a0'], 1.05 * prior['a0']], # optional explicit baseline offset # 'a2': [0, 0.3] # uncomment if you want to fit for airmass - # never list 'a1' in bounds, it is perfectly correlated to exp(a2*airmass) - # and is solved for during the fit + # if a0 is omitted, the normalization is solved analytically during the fit + # never list both 'a0' and 'a1' in bounds because they are the same scale term } myfit = lc_fitter(time, data, dataerr, airmass, prior, mybounds, mode='ns') diff --git a/exotic/api/nea.py b/exotic/api/nea.py index a5203288..5ba69ede 100644 --- a/exotic/api/nea.py +++ b/exotic/api/nea.py @@ -149,6 +149,23 @@ def _negative_error(value): return None return -abs(value) + @staticmethod + def _candidate_name_reason(name): + if not isinstance(name, str): + return None + + normalized = name.strip().upper() + if not normalized: + return None + + if normalized.startswith('TIC'): + return "the name starts with 'TIC'" + + if re.search(r'\.\d{2,}$', normalized): + return "the name ends with a decimal suffix" + + return None + def _load_params_from_nextastro_cache(self): if not self.planet: return False @@ -336,6 +353,12 @@ def _new_scrape(self, filename="eaConf.json"): extra = self._tap_query(uri_ipac_base, uri_ipac_query) if len(default) == 0: + candidate_reason = self._candidate_name_reason(self.planet) + if candidate_reason: + print(f"Cannot find target ({self.planet}) in NASA Exoplanet Archive." + f"\nAssuming {self.planet} is a planet candidate because {candidate_reason}.") + return self.planet, True + self.planet = input(f"Cannot find target ({self.planet}) in NASA Exoplanet Archive." f"\nPlease go to https://exoplanetarchive.ipac.caltech.edu to check naming and" "\nre-enter the planet's name or type 'candidate' if this is a planet candidate: ") diff --git a/exotic/api/plotting.py b/exotic/api/plotting.py index e0ba287a..a4525ec9 100644 --- a/exotic/api/plotting.py +++ b/exotic/api/plotting.py @@ -35,13 +35,13 @@ # EXOplanet Transit Interpretation Code (EXOTIC) # # NOTE: See companion file version.py for version info. # ########################################################################### # -from astropy.io import fits -# from astroscrappy import detect_cosmics -from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LogColorMapper, \ - LogTicker, PanTool, ResetTool, WheelZoomTool -from bokeh.plotting import figure, output_file, show -import json -import logging +from astropy.io import fits +# from astroscrappy import detect_cosmics +from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LogColorMapper, \ + LogTicker, PanTool, ResetTool, WheelZoomTool +from bokeh.plotting import figure, output_file, show +import json +import logging import matplotlib.pyplot as plt from matplotlib.ticker import MaxNLocator, NullLocator, ScalarFormatter import numpy as np diff --git a/exotic/api/ultranest_utils.py b/exotic/api/ultranest_utils.py index 3947de3a..75df5b61 100644 --- a/exotic/api/ultranest_utils.py +++ b/exotic/api/ultranest_utils.py @@ -184,10 +184,8 @@ def _line(self, done=False): ) def start(self): - interval_label = f"{self.interval_seconds:g}s" self._write( - "[ultranest] Using simple progress updates " - f"({interval_label} heartbeat, set EXOTIC_ULTRANEST_RICH_PROGRESS=1 for native status)." + "[ultranest] Using simple progress updates." ) def update(self, *args, **kwargs): diff --git a/exotic/exotic.py b/exotic/exotic.py index e33b5650..5c88f5d3 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -159,6 +159,41 @@ log = logging.getLogger(__name__) _mid_transit_warning_reported = False RELATIVE_FLUX_MAX = 2.0 +AIRMASS_FLAT_RANGE_THRESHOLD = 0.05 + + +def airmass_span(airmass): + values = np.asarray(airmass, dtype=float) + finite = values[np.isfinite(values)] + if finite.size == 0: + return np.nan + return float(np.nanmax(finite) - np.nanmin(finite)) + + +def should_skip_airmass_fit(airmass, max_span=AIRMASS_FLAT_RANGE_THRESHOLD): + span = airmass_span(airmass) + return np.isfinite(span) and span <= max_span + + +def annotate_airmass_fit(fit, airmass, skipped, max_span=AIRMASS_FLAT_RANGE_THRESHOLD, note=None): + if fit is None: + return + + span = airmass_span(airmass) + fit.airmass_span = span + fit.airmass_fit_threshold = max_span + fit.airmass_fit_skipped = bool(skipped) + fit.airmass_correction_note = None + if fit.airmass_fit_skipped: + if note: + fit.airmass_correction_note = note + elif np.isfinite(span): + fit.airmass_correction_note = ( + f"Skipped (airmass span {span:.4f} <= {max_span:.2f}); no airmass correction applied." + ) + else: + fit.airmass_correction_note = "Skipped; no airmass correction applied." + def log_info(string, warn=False, error=False): if error: @@ -272,6 +307,37 @@ def should_ignore_header_wcs(config_value): return False +def is_vertical_flux_normalization_disabled(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'disable vertical flux normalization' value; using default enabled normalization.", warn=True) + return False + + +def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled): + finite_flux = np.asarray(flux_values, dtype=float) + finite_flux = finite_flux[np.isfinite(finite_flux) & (finite_flux > 0)] + baseline_guess = 1.0 if finite_flux.size == 0 else float(np.nanmedian(finite_flux)) + baseline_guess = float(np.clip(baseline_guess, 0.95, 1.05)) + + prior['a0'] = baseline_guess + prior['a1'] = baseline_guess + + if not disabled: + bounds['a0'] = [0.95, 1.05] + + # Initialze plate status log plateStatus = PlateStatus(log_info) @@ -2703,7 +2769,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, - allow_mid_transit_range_warning=True): + allow_mid_transit_range_warning=True, disable_vertical_flux_normalization=False): # remove outliers si = np.argsort(times) times_sorted = times[si] @@ -2765,6 +2831,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, f1 = f1[~nanmask] f2 = f2[~nanmask] + skip_airmass_fit = should_skip_airmass_fit(arrayAirmass) + # -----LM LIGHTCURVE FIT-------------------------------------- prior = { @@ -2776,7 +2844,6 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, 'ecc': pDict['ecc'], # Eccentricity 'omega': pDict['omega'], # Arg of periastron 'tmid': pDict['midT'], # time of mid transit [day] - 'a1': arrayFinalFlux.mean(), # max() - arrayFinalFlux.min(), #mid Flux 'a2': 0, # Flux lower bound } @@ -2801,9 +2868,15 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, 'rprs': [0, prior['rprs'] * 1.25], 'tmid': [lower, upper], 'inc': [prior['inc'] - 5, min(90, prior['inc'] + 5)], - 'a1': [0.5 * min(arrayFinalFlux), 2 * max(arrayFinalFlux)], - 'a2': [-1, 1] } + apply_vertical_flux_normalization_bound( + prior, + mybounds, + arrayFinalFlux, + disable_vertical_flux_normalization, + ) + if not skip_airmass_fit: + mybounds['a2'] = [-1, 1] if np.isnan(arrayTimes).any() or np.isnan(arrayFinalFlux).any() or np.isnan(arrayNormUnc).any(): log_info("\nWarning: NANs in time, flux or error", warn=True) @@ -2818,6 +2891,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, jd_times=arrayJDTimes, mode='lm' ) + annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) if ( myfit is not None @@ -2847,6 +2921,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, jd_times=arrayJDTimes, mode='lm' ) + annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) return myfit, f1, f2 @@ -2864,7 +2939,7 @@ def cheap_lightcurve_prescore(tFlux, cFlux, airmass): x_vals = airmass[finite_mask] y_vals = flux_ratio[finite_mask] - if np.ptp(x_vals) == 0: + if should_skip_airmass_fit(x_vals): detrended = y_vals / bn.nanmedian(y_vals) else: slope, intercept = np.polyfit(x_vals, y_vals, 1) @@ -2876,7 +2951,7 @@ def cheap_lightcurve_prescore(tFlux, cFlux, airmass): def evaluate_lightcurve_candidate(task): - times, tflux, cflux, airmass, ld, p_dict, jd_times = task + times, tflux, cflux, airmass, ld, p_dict, jd_times, disable_vertical_flux_normalization = task myfit, tflux_fit, cflux_fit = fit_lightcurve( times, tflux, @@ -2886,6 +2961,7 @@ def evaluate_lightcurve_candidate(task): p_dict, jd_times, allow_mid_transit_range_warning=False, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, ) if myfit is None: return None, tflux_fit, cflux_fit @@ -3450,6 +3526,9 @@ def main(): header_motion_value = exotic_infoDict.get(motion_key) if header_motion_value is not None: userpDict[motion_key] = header_motion_value + disable_vertical_flux_normalization = is_vertical_flux_normalization_disabled( + exotic_infoDict.get('disable_vertical_flux_normalization', False) + ) # Make a temp directory of helpful files Path(Path(exotic_infoDict['save']) / "temp").mkdir(exist_ok=True) @@ -4113,7 +4192,10 @@ def main(): selected_target_flux = aper_data['target'][:, best_a, best_an] selected_comp_flux = aper_data[selected_ckey][:, best_a, best_an] - myfit, tFlux1, cFlux1 = fit_lightcurve(times, selected_target_flux, selected_comp_flux, airmass, ld, pDict, jd_times) + myfit, tFlux1, cFlux1 = fit_lightcurve( + times, selected_target_flux, selected_comp_flux, airmass, ld, pDict, jd_times, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + ) if myfit is not None: res_std = myfit.residuals.std() / np.median(myfit.data) photometry_info.update(best_fit_lc=myfit, @@ -4142,7 +4224,10 @@ def main(): for j in vsp_num: ckey = f"comp{j + 1}" cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] - vsp_fit, _, _ = fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times) + vsp_fit, _, _ = fit_lightcurve( + times, tFlux, cFlux, airmass, ld, pDict, jd_times, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + ) ref_flux[j] = { 'myfit': vsp_fit, 'pos': exotic_infoDict['comp_stars'][j] @@ -4155,8 +4240,11 @@ def main(): ckey = f"comp{j + 1}" aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] - vsp_fit, _, _ = fit_lightcurve(times[aper_mask], best_target_flux[aper_mask], cFlux, - airmass[aper_mask], ld, pDict, jd_times[aper_mask]) + vsp_fit, _, _ = fit_lightcurve( + times[aper_mask], best_target_flux[aper_mask], cFlux, + airmass[aper_mask], ld, pDict, jd_times[aper_mask], + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + ) ref_flux[j] = { 'myfit': vsp_fit, 'pos': exotic_infoDict['comp_stars'][j] @@ -4171,7 +4259,10 @@ def main(): ckey = f"comp{j + 1}" cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] - myfit, tFlux1, cFlux1 = fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times) + myfit, tFlux1, cFlux1 = fit_lightcurve( + times, tFlux, cFlux, airmass, ld, pDict, jd_times, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + ) res_std = np.inf if myfit is not None: @@ -4265,6 +4356,7 @@ def main(): ld, pDict, jd_times[candidate_mask], + disable_vertical_flux_normalization, )) fit_results = [] @@ -4314,8 +4406,11 @@ def main(): ckey = f"comp{j + 1}" aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] - vsp_fit, _, _ = fit_lightcurve(times[aper_mask], best_target_flux[aper_mask], cFlux, - airmass[aper_mask], ld, pDict, jd_times[aper_mask]) + vsp_fit, _, _ = fit_lightcurve( + times[aper_mask], best_target_flux[aper_mask], cFlux, + airmass[aper_mask], ld, pDict, jd_times[aper_mask], + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + ) ref_flux[j] = { 'myfit': vsp_fit, 'pos': exotic_infoDict['comp_stars'][j] @@ -4569,12 +4664,38 @@ def main(): log_info(f" end:{np.max(goodTimes)}", error=True) log_info(f"prior:{prior['tmid']}", error=True) + final_airmass_span = airmass_span(goodAirmasses) + airmass_skip_note = None + skip_final_airmass_fit = bool(exotic_infoDict.get('airmass_already_corrected')) + if skip_final_airmass_fit: + airmass_skip_note = ( + "Skipped (input AAVSO file already reports AIRMASS, AIRMASS CORRECTION FUNCTION); " + "no airmass correction applied." + ) + log_info( + "Input AAVSO file reports AIRMASS, AIRMASS CORRECTION FUNCTION; " + "skipping airmass fitting and applying no airmass correction." + ) + elif should_skip_airmass_fit(goodAirmasses): + skip_final_airmass_fit = True + log_info( + f"Airmass span {final_airmass_span:.4f} <= {AIRMASS_FLAT_RANGE_THRESHOLD:.2f}; " + "skipping airmass fitting and applying no airmass correction." + ) + mybounds = { 'rprs': [0, prior['rprs'] * 1.25], 'tmid': [lower, upper], 'inc': [prior['inc'] - 5, min(90, prior['inc'] + 5)], - 'a2': [-3, 3], } + apply_vertical_flux_normalization_bound( + prior, + mybounds, + goodFluxes, + disable_vertical_flux_normalization, + ) + if not skip_final_airmass_fit: + mybounds['a2'] = [-3, 3] if np.isnan(goodFluxes).all(): log_info("Error: No valid photometry data found.", error=True) @@ -4582,6 +4703,7 @@ def main(): # final light curve fit myfit = lc_fitter(goodTimes, goodFluxes, goodNormUnc, goodAirmasses, prior, mybounds, mode='ns') + annotate_airmass_fit(myfit, goodAirmasses, skip_final_airmass_fit, note=airmass_skip_note) # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) @@ -4617,8 +4739,11 @@ def main(): log_info(f" Radius Ratio (Planet/Star) [Rp/R*]: {round_to_2(myfit.parameters['rprs'], myfit.errors['rprs'])} +/- {round_to_2(myfit.errors['rprs'])}") log_info(f" Transit depth [(Rp/R*)^2]: {round_to_2(100. * (myfit.parameters['rprs'] ** 2.))} +/- {round_to_2(100. * 2. * myfit.parameters['rprs'] * myfit.errors['rprs'])} [%]") log_info(f" Orbital Inclination [inc]: {round_to_2(myfit.parameters['inc'], myfit.errors['inc'])} +/- {round_to_2(myfit.errors['inc'])}") - log_info(f" Airmass coefficient 1: {round_to_2(myfit.parameters['a1'], myfit.errors['a1'])} +/- {round_to_2(myfit.errors['a1'])}") - log_info(f" Airmass coefficient 2: {round_to_2(myfit.parameters['a2'], myfit.errors['a2'])} +/- {round_to_2(myfit.errors['a2'])}") + if getattr(myfit, 'airmass_fit_skipped', False): + log_info(f" Airmass correction: {myfit.airmass_correction_note}") + else: + log_info(f" Airmass coefficient 1: {round_to_2(myfit.parameters['a1'], myfit.errors['a1'])} +/- {round_to_2(myfit.errors['a1'])}") + log_info(f" Airmass coefficient 2: {round_to_2(myfit.parameters['a2'], myfit.errors['a2'])} +/- {round_to_2(myfit.errors['a2'])}") log_info(f" Residual scatter: {round_to_2(100. * np.std(myfit.residuals / np.median(myfit.data)))} %") if fitsortext == 1: if np.isfinite(photometry_info.get('calibration_field_score', np.inf)): diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index ac03c612..676f5014 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -416,6 +416,7 @@ def save_input(): "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -434,6 +435,7 @@ def save_input(): } new_inits['optional_info'] = { "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "disable vertical flux normalization": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } @@ -1478,6 +1480,7 @@ def save_input(): "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -1544,6 +1547,7 @@ def save_input(): "Filter Maximum Wavelength (nm)": input_data.get('filtermax', null), "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "disable vertical flux normalization": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } @@ -1589,6 +1593,7 @@ def save_input(): "Exposure Time (s)": input_data['exp'], "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "disable vertical flux normalization": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } diff --git a/exotic/inputs.py b/exotic/inputs.py index d0dd85c2..f0f03503 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -61,6 +61,7 @@ AAVSO_EXPOSURE_HEADER_KEYS = ('EXPOSURE_TIME', 'EXPTIME', 'EXPOSURE', 'EXP') AAVSO_TIME_FORMAT_HEADER_KEYS = ('DATE_TYPE',) AAVSO_MEASUREMENT_TYPE_HEADER_KEYS = ('MEASUREMENT_TYPE',) +AAVSO_DETREND_PARAMETER_HEADER_KEYS = ('DETREND_PARAMETERS',) AAVSO_ALLOWED_FILE_TIME_FORMATS = {'BJD_TDB', 'JD_UTC', 'MJD_UTC'} AAVSO_WAVELENGTH_UNIT_FACTORS_TO_NM = { 'a': 0.1, @@ -204,10 +205,10 @@ def __init__(self, init_opt): 'plate_opt': None, 'aavso_comp': None, 'tar_coords': None, 'comp_stars': None, 'prered_file': None, 'file_units': None, 'file_time': None, 'phot_comp_star': None, 'wl_min': None, 'wl_max': None, 'pixel_scale': None, 'exposure': None, - 'dist': None, 'pm_ra': None, 'pm_dec': None, + 'dist': None, 'pm_ra': None, 'pm_dec': None, 'airmass_already_corrected': False, 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', - 'target_driven_comp_selection': 'n' + 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False } self.params = { 'images': imaging_files, 'save': save_directory, 'aavso_num': obs_code, 'second_obs': second_obs_code, @@ -267,6 +268,7 @@ def prereduced(self, planet): ): if is_blank_value(self.info_dict.get(key)) and aavso_overrides.get(key) is not None: self.info_dict[key] = aavso_overrides[key] + self.info_dict['airmass_already_corrected'] = bool(aavso_overrides.get('airmass_already_corrected')) if not planet and not is_blank_value(aavso_overrides.get('planet')): planet = aavso_overrides['planet'] @@ -411,6 +413,10 @@ def comp_params(self, init_file, planet_dict): 'Ignore WCS in header and do manual alignment', 'ignore_header_wcs', ), + 'disable_vertical_flux_normalization': ( + 'disable vertical flux normalization', + 'Disable vertical flux normalization', + ), 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (arsec/pixel)'), 'exposure': 'Exposure Time (s)', @@ -1067,6 +1073,22 @@ def parse_aavso_exposure_from_metadata(metadata): return None +def parse_aavso_airmass_detrend_from_metadata(metadata): + value = first_aavso_metadata_text(metadata, AAVSO_DETREND_PARAMETER_HEADER_KEYS, allow_blank=True) + if is_blank_value(value): + return False + + detrend_parameters = { + re.sub(r'\s+', ' ', item.strip()).upper() + for item in re.split(r'[;,]', value) + if item.strip() + } + return ( + 'AIRMASS' in detrend_parameters + and 'AIRMASS CORRECTION FUNCTION' in detrend_parameters + ) + + def parse_aavso_prereduced_overrides(prereduced_file_path): metadata = read_aavso_metadata(prereduced_file_path) filter_metadata = parse_aavso_filter_metadata_from_metadata(metadata) @@ -1092,6 +1114,7 @@ def parse_aavso_prereduced_overrides(prereduced_file_path): 'dist': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['dist']), 'pm_ra': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['pm_ra']), 'pm_dec': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['pm_dec']), + 'airmass_already_corrected': parse_aavso_airmass_detrend_from_metadata(metadata), 'phot_comp_star': parse_aavso_comp_star_from_metadata(metadata), 'planet': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['planet']), 'host_star': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['host_star']), diff --git a/exotic/output_files.py b/exotic/output_files.py index 1c3a786b..1e710abc 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1,6 +1,7 @@ from json import dump, dumps from numpy import mean, median, std from pathlib import Path +import numpy as np try: from utils import round_to_2 @@ -16,6 +17,42 @@ from .plate_status import PlateStatus +def aavso_airmass_results(fit): + if getattr(fit, 'airmass_fit_skipped', False): + return ( + ('Am1', '0', '0'), + ('Am2', '0', '0'), + ) + + if 'a0' in fit.parameters: + first_result = ( + 'A0', + str(round_to_2(fit.parameters['a0'], fit.errors['a0'])), + str(round_to_2(fit.errors['a0'])), + ) + else: + first_result = ( + 'Am1', + str(round_to_2(fit.parameters['a1'], fit.errors['a1'])), + str(round_to_2(fit.errors['a1'])), + ) + + return ( + first_result, + ( + 'Am2', + str(round_to_2(fit.parameters.get('a2', 0), fit.errors.get('a2', 0))), + str(round_to_2(fit.errors.get('a2', 0))), + ), + ) + + +def aavso_detrend_model(fit): + if getattr(fit, 'airmass_fit_skipped', False): + return np.ones(len(fit.time), dtype=float) + return np.asarray(fit.airmass_model, dtype=float) + + class OutputFiles: def __init__(self, fit, p_dict, i_dict, durs): self.fit = fit @@ -47,13 +84,30 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor "Transit depth (Rp/Rs)^2": f"{round_to_2(100. * (self.fit.parameters['rprs'] ** 2.))} +/- " f"{round_to_2(100. * 2. * self.fit.parameters['rprs'] * self.fit.errors['rprs'])} [%]", "Orbital Inclination (inc)": f"{round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- " - f"{round_to_2(self.fit.errors['inc'])} ", - "Airmass coefficient 1 (a1)": f"{round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- " - f"{round_to_2(self.fit.errors['a1'])}", - "Airmass coefficient 2 (a2)": f"{round_to_2(self.fit.parameters['a2'], self.fit.errors['a2'])} +/- " - f"{round_to_2(self.fit.errors['a2'])}", + f"{round_to_2(self.fit.errors['inc'])} ", "Scatter in the residuals of the lightcurve fit is": f"{round_to_2(100. * std(self.fit.residuals / median(self.fit.data)))} %", } + if getattr(self.fit, 'airmass_fit_skipped', False): + params_num["Airmass correction"] = getattr( + self.fit, + 'airmass_correction_note', + "Skipped; no airmass correction applied.", + ) + else: + if 'a0' in self.fit.parameters: + params_num["Baseline flux (a0)"] = ( + f"{round_to_2(self.fit.parameters['a0'], self.fit.errors['a0'])} +/- " + f"{round_to_2(self.fit.errors['a0'])}" + ) + else: + params_num["Flux normalization (a1)"] = ( + f"{round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- " + f"{round_to_2(self.fit.errors['a1'])}" + ) + params_num["Airmass coefficient 2 (a2)"] = ( + f"{round_to_2(self.fit.parameters['a2'], self.fit.errors['a2'])} +/- " + f"{round_to_2(self.fit.errors['a2'])}" + ) if vsp_params: params_num["Variable Reference Star"] = f"AAVSO Label: {vsp_params[0]['cname']}, " + \ @@ -78,6 +132,8 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): priors_dict, filter_dict, results_dict = aavso_dicts(self.p_dict, self.fit, self.i_dict, self.durs, ld0, ld1, ld2, ld3) + aavso_airmass_terms = aavso_airmass_results(self.fit) + detrend_model = aavso_detrend_model(self.fit) obs_name = format_aavso_header_value(self.i_dict.get('obs_name')) obs_name_header = f"#OBSNAME={obs_name}\n" if obs_name else "" gaia_dist = format_aavso_header_value(self.p_dict.get('dist')) @@ -128,8 +184,8 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): f"#RESULTS=Tc={round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- {round_to_2(self.fit.errors['tmid'])}" f",Rp/R*={round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- {round_to_2(self.fit.errors['rprs'])}" f",inc={round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- {round_to_2(self.fit.errors['inc'])}" - f",Am1={round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- {round_to_2(self.fit.errors['a1'])}" - f",Am2={round_to_2(self.fit.parameters['a2'], self.fit.errors['a2'])} +/- {round_to_2(self.fit.errors['a2'])}\n" + f",{aavso_airmass_terms[0][0]}={aavso_airmass_terms[0][1]} +/- {aavso_airmass_terms[0][2]}" + f",{aavso_airmass_terms[1][0]}={aavso_airmass_terms[1][1]} +/- {aavso_airmass_terms[1][2]}\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields if epw_md5: @@ -150,7 +206,7 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): # f"{round(self.fit.airmass_model[aavsoC] / self.fit.parameters['a1'], 7)}\n") f.write(f"{round(self.fit.time[aavsoC], 8)},{round(self.fit.data[aavsoC], 7)}," f"{round(self.fit.dataerr[aavsoC], 7)},{round(airmasses[aavsoC], 7)}," - f"{round(self.fit.airmass_model[aavsoC], 7)}\n") + f"{round(detrend_model[aavsoC], 7)}\n") def plate_status(self, plate_status: PlateStatus): plate_status_file = self.dir / "temp" / f"PlateStatus_{self.p_dict['pName']}_{self.i_dict['date']}.csv" plate_status.writePlateStatus(plate_status_file) @@ -194,6 +250,7 @@ def aavso(self): def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): + aavso_airmass_terms = aavso_airmass_results(fit) priors = { 'Period': { 'value': str(round_to_2(planet_dict['pPer'], planet_dict['pPerUnc'])), @@ -264,13 +321,9 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): 'value': str(round_to_2(fit.parameters['inc'], fit.errors['inc'])), 'uncertainty': str(round_to_2(fit.errors['inc'])), }, - 'Am1': { - 'value': str(round_to_2(fit.parameters['a1'], fit.errors['a1'])), - 'uncertainty': str(round_to_2(fit.errors['a1'])) - }, 'Am2': { - 'value': str(round_to_2(fit.parameters['a2'], fit.errors['a2'])), - 'uncertainty': str(round_to_2(fit.errors['a2'])) + 'value': aavso_airmass_terms[1][1], + 'uncertainty': aavso_airmass_terms[1][2] }, 'Duration': { 'value': str(round_to_2(mean(durs))), @@ -279,6 +332,11 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): } } + results[aavso_airmass_terms[0][0]] = { + 'value': aavso_airmass_terms[0][1], + 'uncertainty': aavso_airmass_terms[0][2] + } + return priors, filter_type, results diff --git a/inits.json b/inits.json index 222a747a..d4b18b1f 100644 --- a/inits.json +++ b/inits.json @@ -24,6 +24,7 @@ "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Fast Aperture Mask": "Default true/fast mode for quicker aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to false to opt out and use exact masks.", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -92,6 +93,7 @@ "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Fast Aperture Mask (y/n)": true, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "disable vertical flux normalization": false, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y", "Image Scale (Ex: 5.21 arcsecs/pixel)": null, diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py new file mode 100644 index 00000000..8040ac46 --- /dev/null +++ b/tests/test_elca_baseline.py @@ -0,0 +1,136 @@ +import importlib +import sys +import types + +import numpy as np +import pytest + + +def load_elca_with_stubs(monkeypatch, tmp_path): + root = tmp_path / "stubdeps" + package_dir = root / "pylightcurve" + model_dir = package_dir / "models" + model_dir.mkdir(parents=True) + + (package_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "exoplanet_lc.py").write_text( + "import numpy as np\n" + "def transit(ld, rprs, per, ars, ecc, inc, omega, tmid, times, method=None, precision=None):\n" + " times = np.asarray(times, dtype=float)\n" + " return 1.0 - (rprs ** 2) * np.exp(-0.5 * ((times - tmid) / 0.01) ** 2)\n", + encoding="utf-8", + ) + + monkeypatch.syspath_prepend(str(root)) + + for name in list(sys.modules): + if name == "exotic.api.elca" or name.startswith("pylightcurve"): + sys.modules.pop(name, None) + + fake_ultranest = types.ModuleType("ultranest") + fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) + fake_plotting = types.ModuleType("plotting") + fake_plotting.corner = lambda *args, **kwargs: None + fake_ultranest_utils = types.ModuleType("ultranest_utils") + fake_ultranest_utils.run_reactive_sampler = lambda *args, **kwargs: None + + monkeypatch.setitem(sys.modules, "ultranest", fake_ultranest) + monkeypatch.setitem(sys.modules, "plotting", fake_plotting) + monkeypatch.setitem(sys.modules, "exotic.api.plotting", fake_plotting) + monkeypatch.setitem(sys.modules, "ultranest_utils", fake_ultranest_utils) + monkeypatch.setitem(sys.modules, "exotic.api.ultranest_utils", fake_ultranest_utils) + + import exotic.api.elca as elca + + return importlib.reload(elca) + + +def make_prior(): + return { + "rprs": 0.1, + "ars": 12.0, + "per": 3.0, + "inc": 89.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "ecc": 0.0, + "omega": 90.0, + "tmid": 0.0, + "a0": 1.0, + "a2": 0.0, + } + + +def test_lc_fitter_recovers_explicit_a0_baseline(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 301) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.98 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + oot_mask = np.abs(fit.time - fit.parameters["tmid"]) > 0.02 + assert fit.parameters["a0"] == pytest.approx(0.98, abs=1e-4) + assert fit.parameters["a1"] == pytest.approx(0.98, abs=1e-4) + assert np.median(fit.detrended[oot_mask]) == pytest.approx(1.0, abs=5e-4) + + +def test_lc_fitter_auto_solves_baseline_when_a0_is_not_free(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 301) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 1.03 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005]}, + mode="lm", + verbose=False, + ) + + oot_mask = np.abs(fit.time - fit.parameters["tmid"]) > 0.02 + assert fit.parameters["a0"] == pytest.approx(1.03, abs=1e-4) + assert fit.parameters["a1"] == pytest.approx(1.03, abs=1e-4) + assert np.median(fit.detrended[oot_mask]) == pytest.approx(1.0, abs=5e-4) + + +def test_lc_fitter_rejects_redundant_a0_and_a1_bounds(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + prior["a1"] = 1.0 + time = np.linspace(-0.03, 0.03, 31) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + with pytest.raises(ValueError, match="Use only one of 'a0' or 'a1'"): + elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior, + {"a0": [0.95, 1.05], "a1": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 91404566..23081b0e 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -2,6 +2,7 @@ import sys import types import numpy as np +import pytest def _module_available(name: str) -> bool: @@ -89,6 +90,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: is_comp_star_required, is_target_driven_comp_selection_enabled, phase_bin_sigma_clip, + should_skip_airmass_fit, update_coordinates_with_proper_motion, ) @@ -228,6 +230,12 @@ def test_cheap_lightcurve_prescore_keeps_target_only_mode_unfiltered(): assert np.isfinite(score) +def test_should_skip_airmass_fit_when_airmass_span_is_small(): + airmass = np.array([1.10, 1.12, 1.14, 1.15]) + + assert should_skip_airmass_fit(airmass) + + def test_phase_bin_sigma_clip_flags_local_phase_outlier(): phase_centers = np.linspace(-0.045, 0.045, 10) phase = np.concatenate([center + np.linspace(-1e-4, 1e-4, 5) for center in phase_centers]) @@ -342,3 +350,163 @@ def fake_phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max assert len(captured_calls[1]["times"]) == 7 assert len(fit_tflux) == 7 assert len(fit_cflux) == 7 + + +def test_fit_lightcurve_skips_airmass_term_when_airmass_span_is_small(monkeypatch): + captured = {} + + def fake_lc_fitter(times, fluxes, flux_unc, airmass, prior, bounds, jd_times=None, mode=None): + captured["bounds"] = dict(bounds) + captured["airmass"] = np.array(airmass) + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.array([1.10, 1.11, 1.12, 1.13, 1.14, 1.15]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is not None + assert "a2" not in captured["bounds"] + assert myfit.airmass_fit_skipped is True + + +def _run_main_until_vertical_flux_bound(monkeypatch, tmp_path, disable_vertical_flux_normalization=Ellipsis): + import exotic.exotic as exotic_module + + class BoundReached(Exception): + pass + + prered_file = tmp_path / "prereduced.csv" + prered_file.write_text( + "\n".join( + [ + "2450000.00,1.00,0.01,1.10", + "2450000.10,1.01,0.01,1.12", + "2450000.20,0.99,0.01,1.14", + "2450000.30,1.00,0.01,1.16", + "2450000.40,1.02,0.01,1.18", + "2450000.50,1.01,0.01,1.20", + ] + ) + ) + + user_pdict = { + "ra": 10.0, + "dec": 20.0, + "pName": "Test Planet b", + "sName": "Test Star", + "pPer": 1.0, + "pPerUnc": 0.001, + "midT": 2450000.25, + "midTUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.01, + "aRs": 15.0, + "aRsUnc": 0.1, + "inc": 89.0, + "incUnc": 0.1, + "omega": 0.0, + "ecc": 0.0, + "teff": 5500.0, + "teffUncPos": 100.0, + "teffUncNeg": 100.0, + "met": 0.0, + "metUncPos": 0.1, + "metUncNeg": 0.1, + "logg": 4.4, + "loggUncPos": 0.1, + "loggUncNeg": 0.1, + "dist": 100.0, + "pm_ra": 0.0, + "pm_dec": 0.0, + } + exotic_info = { + "save": tmp_path, + "prered_file": prered_file, + "file_time": "BJD_TDB", + "file_units": "flux", + "airmass_already_corrected": False, + "random_seed": 123, + "date": "2026-03-19", + } + if disable_vertical_flux_normalization is not Ellipsis: + exotic_info["disable_vertical_flux_normalization"] = disable_vertical_flux_normalization + + args = types.SimpleNamespace( + multiprocess_transformations=None, + multiprocess_lightcurve_fits=None, + realtime=None, + reduce=None, + prereduced=str(tmp_path / "inits.json"), + photometry=None, + override=True, + nasaexoarch=False, + non_interactive_run=True, + use_nextastro_astrometry=False, + use_nextastro_variability_server=False, + ) + + class FakeInputs: + def __init__(self, init_opt): + self.init_opt = init_opt + + def search_init(self, init_path, planet_dict): + return init_path, dict(user_pdict) + + def prereduced(self, planet): + return dict(exotic_info), planet or user_pdict["pName"] + + captured = {} + + monkeypatch.setattr(exotic_module, "parse_args", lambda: args) + monkeypatch.setattr(exotic_module, "Inputs", FakeInputs) + monkeypatch.setattr( + exotic_module, + "get_ld_values", + lambda *_args, **_kwargs: ([0.1, 0.1, 0.1, 0.1], [0.1], [0.1], [0.1], [0.1]), + ) + + def fake_apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled): + captured["disabled"] = disabled + raise BoundReached() + + monkeypatch.setattr( + exotic_module, + "apply_vertical_flux_normalization_bound", + fake_apply_vertical_flux_normalization_bound, + ) + + with pytest.raises(BoundReached): + exotic_module.main() + + return captured["disabled"] + + +def test_main_prereduced_defaults_vertical_flux_normalization_to_enabled(monkeypatch, tmp_path): + disabled = _run_main_until_vertical_flux_bound(monkeypatch, tmp_path) + + assert disabled is False + + +def test_main_prereduced_respects_disable_vertical_flux_normalization_option(monkeypatch, tmp_path): + disabled = _run_main_until_vertical_flux_bound(monkeypatch, tmp_path, disable_vertical_flux_normalization=True) + + assert disabled is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py index a447919f..3982c018 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -68,6 +68,21 @@ def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "n" +def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["disable_vertical_flux_normalization"] is False + + def test_comp_params_reads_observatory_full_title_from_user_info(tmp_path): init_data = { "user_info": {"Observatory Full Title": "Whipple Observatory"}, @@ -113,6 +128,21 @@ def test_comp_params_reads_ignore_header_wcs_from_optional_info(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "y" +def test_comp_params_reads_disable_vertical_flux_normalization_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"disable vertical flux normalization": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["disable_vertical_flux_normalization"] is True + + class DummyResponse: def __init__(self, payload): self._payload = payload @@ -526,6 +556,20 @@ def test_parse_aavso_prereduced_overrides_uses_osc_split_filter_alias_lookup(tmp assert overrides["wl_max"] == "586.8" +def test_parse_aavso_prereduced_overrides_marks_airmass_as_already_corrected(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" + "#DATE,DIFF,ERR,DETREND_1,DETREND_2\n" + "2461102.76092732,0.979108,0.0386426,1.3811172,0.998\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["airmass_already_corrected"] is True + + def test_prereduced_prefers_aavso_obsdate_metadata_over_init_date(tmp_path): pre_reduced_file = tmp_path / "aavso_prereduced.txt" pre_reduced_file.write_text( @@ -623,3 +667,37 @@ def test_prereduced_leaves_phot_comp_star_blank_when_missing_from_aavso_metadata info_dict, _ = inputs.prereduced("HAT-P-32 b") assert info_dict["phot_comp_star"] == {"ra": "", "dec": "", "x": "", "y": ""} + + +def test_prereduced_carries_aavso_airmass_corrected_flag(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" + "#DATE,DIFF,ERR,DETREND_1,DETREND_2\n" + "2461102.76092732,0.979108,0.0386426,1.3811172,0.998\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["airmass_already_corrected"] is True diff --git a/tests/test_nea_nextastro_fallback.py b/tests/test_nea_nextastro_fallback.py index 1d05d854..8c44c50e 100644 --- a/tests/test_nea_nextastro_fallback.py +++ b/tests/test_nea_nextastro_fallback.py @@ -1,3 +1,5 @@ +import pandas +import pytest import requests from exotic.api.nea import NASAExoplanetArchive @@ -66,3 +68,32 @@ def fake_get(url, params, timeout): assert pl_dict['midT'] == 2450000.5 assert pl_dict['rprs'] == 0.12 assert pl_dict['aRs'] == 3.0 + + +@pytest.mark.parametrize( + ("planet_name", "reason"), + [ + ("TOI-3889.01", "the name ends with a decimal suffix"), + ("TIC 123456789", "the name starts with 'TIC'"), + ], +) +def test_new_scrape_auto_marks_candidate_like_names_without_prompt(monkeypatch, tmp_path, capsys, + planet_name, reason): + nea = NASAExoplanetArchive(planet_name) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(nea, 'planet_names', lambda filename="pl_names.json": None) + monkeypatch.setattr(nea, '_tap_query', lambda *args, **kwargs: pandas.DataFrame()) + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("interactive prompt should not run for candidate-like targets"), + ) + + resolved_name, candidate = nea._new_scrape() + + assert resolved_name == planet_name + assert candidate is True + + output = capsys.readouterr().out + assert f"Cannot find target ({planet_name}) in NASA Exoplanet Archive." in output + assert f"Assuming {planet_name} is a planet candidate because {reason}." in output diff --git a/tests/test_output_files.py b/tests/test_output_files.py index b57e31f2..74f0eb96 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -20,6 +20,7 @@ def __init__(self): self.time = [2450000.123456] self.data = [1.0] self.dataerr = [0.01] + self.residuals = 0.01 self.airmass_model = [1.0] @@ -176,3 +177,83 @@ def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): text = summary_path.read_text() assert "# Selected comparison star,1" in text assert "Comp 1,101,202,true" in text + + +def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): + fit = DummyFit() + fit.airmass_fit_skipped = True + fit.airmass_correction_note = "Skipped (airmass span 0.0400 <= 0.05); no airmass correction applied." + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_text = output_file.read_text(encoding="utf-8") + + assert "Airmass correction" in output_text + assert "no airmass correction applied" in output_text + assert "Airmass coefficient 1 (a1)" not in output_text + + +def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_path): + fit = DummyFit() + fit.airmass_fit_skipped = True + fit.airmass_correction_note = "Skipped (input AAVSO file already reports AIRMASS, AIRMASS CORRECTION FUNCTION); no airmass correction applied." + + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + "pPer": 2.1500082, + "pPerUnc": 1.3e-07, + "rprs": 0.1488623525, + "rprsUnc": 0.0005539487, + "aRs": 5.344, + "aRsUnc": 0.03949, + "inc": 88.98, + "incUnc": 0.7602, + "ecc": 0.159, + "dist": None, + "pm_ra": None, + "pm_dec": None, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "second_obs": "", + "obs_name": "", + "camera": "CCD", + "pixel_bin": "1x1", + "exposure": 60.0, + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + "notes": "na", + "filter": "CV", + "filter_desc": "Clear with V zero-point", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( + {"ra": "", "dec": "", "x": "493", "y": "202"}, + [1.0], + (0.1, 0.01), + (0.2, 0.01), + (0.3, 0.01), + (0.4, 0.01), + None, + ) + + output_file = tmp_path / "AAVSO_HAT-P-32 b_2020-01-01.txt" + output_text = output_file.read_text(encoding="utf-8") + + assert "Am1=0 +/- 0" in output_text + assert "Am2=0 +/- 0" in output_text + assert output_text.strip().endswith("1.0") From 5ba97427cd219d6bb58f9939fdccc032f35e4b62 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 29 Mar 2026 07:57:21 +1100 Subject: [PATCH 006/116] Handle NextAstroPlateSolution JSON --- exotic/api/plate_solution.py | 36 ++++++++++++++++---- tests/test_nextastro_astrometry.py | 53 +++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/exotic/api/plate_solution.py b/exotic/api/plate_solution.py index a3f87dc5..36df92a9 100644 --- a/exotic/api/plate_solution.py +++ b/exotic/api/plate_solution.py @@ -289,6 +289,28 @@ def _limit_to_brightest_sources(x_coords, y_coords, fluxes): "flux": fluxes[sorted_indices].tolist() } + @staticmethod + def _response_body_preview(response, max_chars=240): + body = getattr(response, 'text', None) + if body is None: + content = getattr(response, 'content', b'') + body = content.decode(errors='replace') if isinstance(content, bytes) else str(content) + + body = " ".join(str(body).split()) + if not body: + return "" + if len(body) > max_chars: + return body[:max_chars - 3] + "..." + return body + + def _decode_response_json(self, response, context): + try: + return response.json() + except ValueError: + print(f"[NextAstro] {context} returned non-JSON response " + f"(HTTP {response.status_code}): {self._response_body_preview(response)}") + return None + def _submit_solve_request(self, source_list): image_data = getdata(filename=self.file) payload = { @@ -309,11 +331,11 @@ def _submit_solve_request(self, source_list): print(f"[NextAstro] Solve request payload: {payload}") response = requests.post(f"{self.api_url}/solve", json=payload, timeout=_RQ_TIMEOUT) - print(f"[NextAstro] Solve response: {response.json()}") - if response.status_code >= 400: + response_json = self._decode_response_json(response, 'Solve response') + if response_json is not None: + print(f"[NextAstro] Solve response: {response_json}") + if response.status_code >= 400 or response_json is None: return False - - response_json = response.json() if response_json.get('status') in {'queued', 'running'}: return response_json.get('request_id') return False @@ -336,11 +358,13 @@ def _poll_for_solution(self, request_id): latest_status = None for _ in range(_NEXTASTRO_STATUS_MAX_POLLS): response = requests.get(f"{self.api_url}/status/{request_id}", timeout=_RQ_TIMEOUT) + response_json = self._decode_response_json(response, 'Status response') + if response_json is None: + return False if response.status_code >= 400: - print(f"[NextAstro] Status response (HTTP {response.status_code}): {response.json()}") + print(f"[NextAstro] Status response (HTTP {response.status_code}): {response_json}") return False - response_json = response.json() status = str(response_json.get('status', '')).lower() latest_status = response_json.get('status') if status == 'solved': diff --git a/tests/test_nextastro_astrometry.py b/tests/test_nextastro_astrometry.py index 9b367dcc..14638c96 100644 --- a/tests/test_nextastro_astrometry.py +++ b/tests/test_nextastro_astrometry.py @@ -7,11 +7,15 @@ class DummyResponse: - def __init__(self, payload, status_code=200): + def __init__(self, payload=None, status_code=200, text=None, json_error=None): self._payload = payload self.status_code = status_code + self.text = text if text is not None else ("" if payload is None else str(payload)) + self._json_error = json_error def json(self): + if self._json_error is not None: + raise self._json_error return self._payload @@ -143,6 +147,53 @@ def test_poll_for_solution_logs_unexpected_status(tmp_path, monkeypatch, capsys) assert "'status': 'processing'" in output +def test_submit_solve_request_handles_non_json_response(tmp_path, monkeypatch, capsys): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + monkeypatch.setattr( + 'exotic.api.plate_solution.requests.post', + lambda url, json, timeout: DummyResponse( + status_code=502, + text='bad gateway', + json_error=ValueError('not json') + ) + ) + + request_id = solver._submit_solve_request({ + 'x': [25.0], + 'y': [30.0], + 'flux': [10000.0], + 'pixel_indexing': '0-based' + }) + + assert request_id is False + output = capsys.readouterr().out + assert "Solve response returned non-JSON response" in output + assert "bad gateway" in output.lower() + + +def test_poll_for_solution_handles_non_json_response(tmp_path, monkeypatch, capsys): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + monkeypatch.setattr( + 'exotic.api.plate_solution.requests.get', + lambda url, timeout: DummyResponse( + status_code=200, + text='', + json_error=ValueError('not json') + ) + ) + + header = solver._poll_for_solution('abc123') + + assert header is False + output = capsys.readouterr().out + assert "Status response returned non-JSON response" in output + assert "" in output + + def test_nova_upload_includes_astrometry_hints(tmp_path, monkeypatch): fits_path = _create_test_fits(tmp_path) From 3ca22404307ba0c591c83f8972a78acb39a2d5ab Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 29 Mar 2026 08:30:03 +1100 Subject: [PATCH 007/116] Add NextAstro 502 messaging --- exotic/api/plate_solution.py | 70 ++++++++++++++++++++--------- exotic/exotic.py | 51 +++++++++++++++++---- tests/test_nextastro_astrometry.py | 6 ++- tests/test_nextastro_variability.py | 69 ++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 33 deletions(-) diff --git a/exotic/api/plate_solution.py b/exotic/api/plate_solution.py index 36df92a9..237403ec 100644 --- a/exotic/api/plate_solution.py +++ b/exotic/api/plate_solution.py @@ -68,7 +68,7 @@ class PlateSolution: def __init__(self, file=None, directory=None, api_key=None, api_url='http://nova.astrometry.net/api/', ra=None, dec=None, - pixel_scale=None, radius=2.0, scale_err=25.0): + pixel_scale=None, radius=2.0, scale_err=25.0, suppress_fail_warning=False): if api_key is None: api_key = {'apikey': 'vfsyxlmdxfryhprq'} self.api_url = api_url @@ -80,27 +80,30 @@ def __init__(self, file=None, directory=None, api_key=None, self.pixel_scale = pixel_scale self.radius = radius self.scale_err = scale_err + self.suppress_fail_warning = suppress_fail_warning + self.last_error_type = None def plate_solution(self): + self.last_error_type = None session = self._login() if not session: - return PlateSolution.fail('Login') + return self._fail('Login') sub_id = self._upload(session) if not sub_id: - return PlateSolution.fail('Upload') + return self._fail('Upload') sub_url = self._get_url(f"submissions/{sub_id}") job_id = self._sub_status(sub_url) if not job_id: - return PlateSolution.fail('Submission ID') + return self._fail('Submission ID') job_url = self._get_url(f"jobs/{job_id}") download_url = self.api_url.replace("/api/", f"/wcs_file/{job_id}/") wcs_file = Path(self.directory) / "temp" / "wcs.fits" wcs_file = self._job_status(job_url, wcs_file, download_url) if not wcs_file: - return PlateSolution.fail('Job Status') + return self._fail('Job Status') else: print("WCS file creation successful.") return wcs_file @@ -108,6 +111,12 @@ def plate_solution(self): def _get_url(self, service): return self.api_url + service + def _fail(self, error_type, service_name='nova.astrometry.net'): + self.last_error_type = error_type + if self.suppress_fail_warning: + return False + return PlateSolution.fail(error_type, service_name=service_name) + @retry(stop=stop_after_attempt(_R_MAX_STOPS_LOW), wait=wait_exponential(multiplier=1, min=4, max=_R_MAX_SECS), retry=(retry_if_result(is_false) | retry_if_exception_type(requests.exceptions.RequestException)), retry_error_callback=result_if_max_retry_count) @@ -184,35 +193,49 @@ def fail(error_type, service_name='nova.astrometry.net'): class NextAstroPlateSolution: def __init__(self, file=None, directory=None, api_url='https://astrometry.nextastro.org/', ra=None, dec=None, - pixel_scale=None): + pixel_scale=None, suppress_fail_warning=False): self.api_url = api_url.rstrip('/') self.file = file self.directory = directory self.ra = ra self.dec = dec self.pixel_scale = pixel_scale + self.suppress_fail_warning = suppress_fail_warning + self.last_error_type = None + self.last_http_status = None def plate_solution(self): - print(f"Using NextAstro astrometry server at {self.api_url} for plate solving.") + self.last_error_type = None + self.last_http_status = None + self._emit_debug(f"Using NextAstro astrometry server at {self.api_url} for plate solving.") source_list = self._generate_source_list() if not source_list: - return PlateSolution.fail('Source extraction for NextAstro astrometry server', - service_name=f'NextAstro ({self.api_url})') + return self._fail('Source extraction for NextAstro astrometry server') request_id = self._submit_solve_request(source_list) if not request_id: - return PlateSolution.fail('NextAstro solve submission', service_name=f'NextAstro ({self.api_url})') + return self._fail('NextAstro solve submission') wcs_header = self._poll_for_solution(request_id) if not wcs_header: - return PlateSolution.fail('NextAstro solve status', service_name=f'NextAstro ({self.api_url})') + return self._fail('NextAstro solve status') wcs_file = Path(self.directory) / "temp" / "wcs.fits" hdu = PrimaryHDU(data=getdata(filename=self.file), header=wcs_header) hdu.writeto(wcs_file, overwrite=True) - print("WCS file creation successful.") + self._emit_debug("WCS file creation successful.") return wcs_file + def _emit_debug(self, message): + if not self.suppress_fail_warning: + print(message) + + def _fail(self, error_type): + self.last_error_type = error_type + if self.suppress_fail_warning: + return False + return PlateSolution.fail(error_type, service_name=f'NextAstro ({self.api_url})') + def _generate_source_list(self): image_data = np.asarray(getdata(filename=self.file), dtype=float) if image_data.ndim > 2: @@ -304,11 +327,13 @@ def _response_body_preview(response, max_chars=240): return body def _decode_response_json(self, response, context): + self.last_http_status = getattr(response, 'status_code', None) try: return response.json() except ValueError: - print(f"[NextAstro] {context} returned non-JSON response " - f"(HTTP {response.status_code}): {self._response_body_preview(response)}") + if response.status_code != 502: + self._emit_debug(f"[NextAstro] {context} returned non-JSON response " + f"(HTTP {response.status_code}): {self._response_body_preview(response)}") return None def _submit_solve_request(self, source_list): @@ -329,11 +354,11 @@ def _submit_solve_request(self, source_list): if hints is not None: payload["hints"] = hints - print(f"[NextAstro] Solve request payload: {payload}") + self._emit_debug(f"[NextAstro] Solve request payload: {payload}") response = requests.post(f"{self.api_url}/solve", json=payload, timeout=_RQ_TIMEOUT) response_json = self._decode_response_json(response, 'Solve response') - if response_json is not None: - print(f"[NextAstro] Solve response: {response_json}") + if response_json is not None and response.status_code != 502: + self._emit_debug(f"[NextAstro] Solve response: {response_json}") if response.status_code >= 400 or response_json is None: return False if response_json.get('status') in {'queued', 'running'}: @@ -362,26 +387,27 @@ def _poll_for_solution(self, request_id): if response_json is None: return False if response.status_code >= 400: - print(f"[NextAstro] Status response (HTTP {response.status_code}): {response_json}") + if response.status_code != 502: + self._emit_debug(f"[NextAstro] Status response (HTTP {response.status_code}): {response_json}") return False status = str(response_json.get('status', '')).lower() latest_status = response_json.get('status') if status == 'solved': - print(f"[NextAstro] Status response (solved): {response_json}") + self._emit_debug(f"[NextAstro] Status response (solved): {response_json}") header_dict = response_json.get('solution', {}).get('wcs_header') if isinstance(header_dict, dict): return Header(header_dict) return False if status == 'failed': - print(f"[NextAstro] Status response (failed): {response_json}") + self._emit_debug(f"[NextAstro] Status response (failed): {response_json}") return False if status not in _NEXTASTRO_IN_PROGRESS_STATUSES: - print(f"[NextAstro] Status response (unexpected): {response_json}") + self._emit_debug(f"[NextAstro] Status response (unexpected): {response_json}") return False time.sleep(_NEXTASTRO_STATUS_POLL_SEC) - print(f"[NextAstro] Polling timed out waiting for terminal status; latest status={latest_status!r}") + self._emit_debug(f"[NextAstro] Polling timed out waiting for terminal status; latest status={latest_status!r}") return False diff --git a/exotic/exotic.py b/exotic/exotic.py index 5c88f5d3..c8adc895 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -1063,28 +1063,61 @@ def get_wcs(file, directory="", use_nextastro_astrometry=False, ra=None, dec=Non if use_nextastro_astrometry: print("Contacting NextAstro Astrometry Server") - wcs_file = NextAstroPlateSolution(file=file, directory=directory, ra=ra, dec=dec, pixel_scale=pixel_scale).plate_solution() + nextastro_solver = NextAstroPlateSolution( + file=file, + directory=directory, + ra=ra, + dec=dec, + pixel_scale=pixel_scale, + suppress_fail_warning=True + ) + wcs_file = nextastro_solver.plate_solution() if wcs_file: return wcs_file - log_info("NextAstro astrometry server did not return a solution; falling back to nova.astrometry.net.") + nextastro_bad_gateway = nextastro_solver.last_http_status == 502 + if nextastro_bad_gateway: + log_info("NextAstro Server not responding. Will try nova.astrometry.net") + else: + log_info("NextAstro astrometry server did not return a solution; falling back to nova.astrometry.net.") print("Communication with nova.astrometry.net") - return PlateSolution(file=file, directory=directory, ra=ra, dec=dec, - pixel_scale=pixel_scale).plate_solution() + nova_solver = PlateSolution(file=file, directory=directory, ra=ra, dec=dec, + pixel_scale=pixel_scale, suppress_fail_warning=True) + wcs_file = nova_solver.plate_solution() + if wcs_file: + return wcs_file + if nextastro_bad_gateway: + log_info("NextAstro Server not responding. Both astrometry methods trialed, pushing forward without astrometry solution") + return False + return PlateSolution.fail(nova_solver.last_error_type or 'plate solution lookup') animate_toggle(True) - wcs_file = PlateSolution(file=file, directory=directory, ra=ra, dec=dec, - pixel_scale=pixel_scale).plate_solution() + nova_solver = PlateSolution(file=file, directory=directory, ra=ra, dec=dec, + pixel_scale=pixel_scale, suppress_fail_warning=True) + wcs_file = nova_solver.plate_solution() if wcs_file: animate_toggle() return wcs_file log_info("nova.astrometry.net did not return a solution; falling back to NextAstro astrometry server.") print("Contacting NextAstro Astrometry Server") - wcs_file = NextAstroPlateSolution(file=file, directory=directory, ra=ra, dec=dec, - pixel_scale=pixel_scale).plate_solution() + nextastro_solver = NextAstroPlateSolution( + file=file, + directory=directory, + ra=ra, + dec=dec, + pixel_scale=pixel_scale, + suppress_fail_warning=True + ) + wcs_file = nextastro_solver.plate_solution() animate_toggle() - return wcs_file + if wcs_file: + return wcs_file + if nextastro_solver.last_http_status == 502: + log_info("NextAstro Server not responding. Both astrometry methods trialed, pushing forward without astrometry solution") + return False + return PlateSolution.fail(nextastro_solver.last_error_type or 'plate solution lookup', + service_name=f'NextAstro ({nextastro_solver.api_url})') # Getting the right ascension and declination for every pixel in imaging file if there is a plate solution diff --git a/tests/test_nextastro_astrometry.py b/tests/test_nextastro_astrometry.py index 14638c96..a9308276 100644 --- a/tests/test_nextastro_astrometry.py +++ b/tests/test_nextastro_astrometry.py @@ -169,8 +169,10 @@ def test_submit_solve_request_handles_non_json_response(tmp_path, monkeypatch, c assert request_id is False output = capsys.readouterr().out - assert "Solve response returned non-JSON response" in output - assert "bad gateway" in output.lower() + assert "Solve request payload" in output + assert "Solve response returned non-JSON response" not in output + assert "bad gateway" not in output.lower() + assert solver.last_http_status == 502 def test_poll_for_solution_handles_non_json_response(tmp_path, monkeypatch, capsys): diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index bb63388b..ef504346 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -173,6 +173,75 @@ def plate_solution(self): assert service_calls == ['nova', 'nextastro'] +def test_get_wcs_logs_nextastro_bad_gateway_before_nova_fallback(monkeypatch): + logged = [] + service_calls = [] + + class DummyPlateSolution: + def __init__(self, **kwargs): + self.last_error_type = None + + def plate_solution(self): + service_calls.append('nova') + return 'nova-wcs' + + class DummyNextAstroSolution: + def __init__(self, **kwargs): + self.last_http_status = 502 + self.last_error_type = 'NextAstro solve submission' + self.api_url = 'https://astrometry.nextastro.org' + + def plate_solution(self): + service_calls.append('nextastro') + return False + + monkeypatch.setattr(exotic_module, 'PlateSolution', DummyPlateSolution) + monkeypatch.setattr(exotic_module, 'NextAstroPlateSolution', DummyNextAstroSolution) + monkeypatch.setattr(exotic_module, 'animate_toggle', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + solved_wcs = exotic_module.get_wcs('frame.fits', directory='.', use_nextastro_astrometry=True) + + assert solved_wcs == 'nova-wcs' + assert service_calls == ['nextastro', 'nova'] + assert any(message == 'NextAstro Server not responding. Will try nova.astrometry.net' for message in logged) + + +def test_get_wcs_logs_nextastro_bad_gateway_after_both_methods_fail(monkeypatch): + logged = [] + service_calls = [] + + class DummyPlateSolution: + def __init__(self, **kwargs): + self.last_error_type = 'Upload' + + def plate_solution(self): + service_calls.append('nova') + return False + + class DummyNextAstroSolution: + def __init__(self, **kwargs): + self.last_http_status = 502 + self.last_error_type = 'NextAstro solve submission' + self.api_url = 'https://astrometry.nextastro.org' + + def plate_solution(self): + service_calls.append('nextastro') + return False + + monkeypatch.setattr(exotic_module, 'PlateSolution', DummyPlateSolution) + monkeypatch.setattr(exotic_module, 'NextAstroPlateSolution', DummyNextAstroSolution) + monkeypatch.setattr(exotic_module, 'animate_toggle', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + solved_wcs = exotic_module.get_wcs('frame.fits', directory='.') + + assert solved_wcs is False + assert service_calls == ['nova', 'nextastro'] + assert any(message == 'NextAstro Server not responding. Both astrometry methods trialed, pushing forward without astrometry solution' + for message in logged) + + def test_vsx_variable_falls_back_to_nextastro(monkeypatch): class DummyFailedResponse: def raise_for_status(self): From b87dd77eee4ad6b66c9028d5b61942f707563f44 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 30 Mar 2026 09:57:04 +1100 Subject: [PATCH 008/116] Add adaptive aperture scaling based on per-frame PSF width --- exotic/api/colab.py | 1 + exotic/exotic.py | 204 +++++++++++++++++++++++++---- exotic/exotic_gui.py | 5 + exotic/inputs.py | 8 +- inits.json | 2 + tests/test_exotic_proper_motion.py | 34 +++++ tests/test_inputs.py | 30 +++++ 7 files changed, 256 insertions(+), 28 deletions(-) diff --git a/exotic/api/colab.py b/exotic/api/colab.py index 3b7a683f..e7fcc3dc 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -378,6 +378,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Filter Minimum Wavelength (nm)": %s, "Filter Maximum Wavelength (nm)": %s, "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "use_adaptive_apertures": false, "require_comp_star": "y" } } diff --git a/exotic/exotic.py b/exotic/exotic.py index c8adc895..6492ebc5 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -288,6 +288,24 @@ def is_target_driven_comp_selection_enabled(config_value): return False +def is_adaptive_aperture_mode_enabled(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'use_adaptive_apertures' value; using fixed apertures.", warn=True) + return False + + def should_ignore_header_wcs(config_value): if config_value is None: return False @@ -338,6 +356,59 @@ def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled bounds['a0'] = [0.95, 1.05] +def psf_sigma_from_fit(psf_row, fallback_sigma=np.nan): + try: + sigx = float(psf_row[3]) + sigy = float(psf_row[4]) + sigma = 0.5 * (sigx + sigy) + except (IndexError, TypeError, ValueError): + sigma = np.nan + + if np.isfinite(sigma) and sigma > 0: + return float(sigma) + + if np.isfinite(fallback_sigma) and fallback_sigma > 0: + return float(fallback_sigma) + + return np.nan + + +def representative_psf_sigma(psf_rows, fallback_sigma=np.nan): + try: + sigmas = np.asarray(psf_rows[:, 3], dtype=float) + np.asarray(psf_rows[:, 4], dtype=float) + except (IndexError, TypeError, ValueError): + sigmas = np.array([], dtype=float) + + if sigmas.size: + sigmas *= 0.5 + sigmas[~np.isfinite(sigmas) | (sigmas <= 0)] = np.nan + center, _ = sigma_clipped_nanmedian(sigmas) + if np.isfinite(center) and center > 0: + return float(center) + + if np.isfinite(fallback_sigma) and fallback_sigma > 0: + return float(fallback_sigma) + + return np.nan + + +def resolve_frame_aperture_radii(apertures, annuli, adaptive_apertures=False, frame_sigma=np.nan, + fallback_sigma=np.nan): + aperture_values = np.asarray(apertures, dtype=float) + annulus_values = np.asarray(annuli, dtype=float) + + if not adaptive_apertures: + return aperture_values, annulus_values + + sigma_to_use = float(frame_sigma) if np.isfinite(frame_sigma) and frame_sigma > 0 else np.nan + if (not np.isfinite(sigma_to_use) or sigma_to_use <= 0) and np.isfinite(fallback_sigma) and fallback_sigma > 0: + sigma_to_use = float(fallback_sigma) + if not np.isfinite(sigma_to_use) or sigma_to_use <= 0: + sigma_to_use = 1.0 + + return aperture_values * sigma_to_use, annulus_values * sigma_to_use + + # Initialze plate status log plateStatus = PlateStatus(log_info) @@ -2627,10 +2698,16 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet first_image = fits.getdata(inputfiles[0]) targ_sig_xy = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0)[3:5] - # aperture size in stdev (sigma) of PSF - aper = 3 * max(targ_sig_xy) - annulus = 10 + # aperture and annulus scale factors in PSF sigma units + aper_sigma = 3 * max(targ_sig_xy) + annulus_sigma = 10 fast_aperture_mask = is_fast_aperture_mask_enabled(info_dict.get('fast_aperture_mask')) + use_adaptive_apertures = is_adaptive_aperture_mode_enabled(info_dict.get('use_adaptive_apertures')) + aper = np.nan + annulus = np.nan + sigma = np.nan + if use_adaptive_apertures: + log_info("Adaptive aperture scaling enabled for realtime photometry.") # alloc psf fitting param psf_data = { @@ -2773,10 +2850,33 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) # aperture photometry + frame_sigma = psf_sigma_from_fit(psf_data['target'][i], fallback_sigma=sigma) if i == 0: - sigma = float((psf_data['target'][0][3] + psf_data['target'][0][4]) * 0.5) - aper *= sigma - annulus *= sigma + sigma = frame_sigma + if not np.isfinite(sigma) or sigma <= 0: + log_info("Warning: Initial PSF sigma is invalid; using sigma=1.0 for aperture photometry.", warn=True) + sigma = 1.0 + + if use_adaptive_apertures: + frame_aper, frame_annulus = resolve_frame_aperture_radii( + [aper_sigma], + [annulus_sigma], + adaptive_apertures=True, + frame_sigma=frame_sigma, + fallback_sigma=sigma, + ) + aper = float(frame_aper[0]) + annulus = float(frame_annulus[0]) + elif i == 0: + aper, annulus = resolve_frame_aperture_radii( + [aper_sigma], + [annulus_sigma], + adaptive_apertures=True, + frame_sigma=sigma, + fallback_sigma=sigma, + ) + aper = float(aper[0]) + annulus = float(annulus[0]) tFlux = aperPhot(imageData, 0, psf_data['target'][i, 0], psf_data['target'][i, 1], aper, annulus, fast_mode=fast_aperture_mask)[0] @@ -3192,14 +3292,23 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_star_count, aper_data, apertures, annuli, - fast_aperture_mask): + fast_aperture_mask, adaptive_apertures=False, fallback_sigma=np.nan): + frame_sigma = psf_sigma_from_fit(psf_data['target'][frame_index], fallback_sigma=fallback_sigma) + frame_apertures, frame_annuli = resolve_frame_aperture_radii( + apertures, + annuli, + adaptive_apertures=adaptive_apertures, + frame_sigma=frame_sigma, + fallback_sigma=fallback_sigma, + ) + target_flux, target_bg = compute_star_aperture_grid( image_data, 0, psf_data['target'][frame_index, 0], psf_data['target'][frame_index, 1], - apertures, - annuli, + frame_apertures, + frame_annuli, fast_mode=fast_aperture_mask, ) aper_data['target'][frame_index] = target_flux @@ -3212,8 +3321,8 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta comp_idx + 1, psf_data[ckey][frame_index, 0], psf_data[ckey][frame_index, 1], - apertures, - annuli, + frame_apertures, + frame_annuli, fast_mode=fast_aperture_mask, ) aper_data[ckey][frame_index] = comp_flux @@ -3778,6 +3887,9 @@ def main(): target_driven_comp_selection = is_target_driven_comp_selection_enabled( exotic_infoDict.get('target_driven_comp_selection', 'n') ) + use_adaptive_apertures = is_adaptive_aperture_mode_enabled( + exotic_infoDict.get('use_adaptive_apertures', False) + ) for i, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{i + 1}" @@ -3798,8 +3910,10 @@ def main(): ) sigma = None - coarse_apertures = None - coarse_annuli = None + coarse_aperture_values = None + coarse_annulus_values = None + aperture_values = None + annulus_values = None apers = None annuli = None aperture_grid_tuned = False @@ -3827,6 +3941,8 @@ def main(): dtype=float, ) fast_aperture_mask = is_fast_aperture_mask_enabled(exotic_infoDict.get('fast_aperture_mask')) + if use_adaptive_apertures: + log_info("Adaptive aperture scaling enabled: evaluating aperture candidates in PSF sigma units per frame.") # open files, calibrate, align, photometry reset_transform_timing_stats() @@ -3979,12 +4095,16 @@ def main(): # aperture photometry if i == 0: - sigma = float((psf_data['target'][0][3] + psf_data['target'][0][4]) * 0.5) + sigma = psf_sigma_from_fit(psf_data['target'][0]) if not np.isfinite(sigma) or sigma <= 0: log_info("Warning: Initial PSF sigma is invalid; using sigma=1.0 for automatic aperture tuning.", warn=True) sigma = 1.0 - coarse_apertures = coarse_apertures_sigma * sigma - coarse_annuli = coarse_annuli_sigma * sigma + if use_adaptive_apertures: + coarse_aperture_values = coarse_apertures_sigma + coarse_annulus_values = coarse_annuli_sigma + else: + coarse_aperture_values = coarse_apertures_sigma * sigma + coarse_annulus_values = coarse_annuli_sigma * sigma if i < coarse_tune_frames: coarse_frame_cache[i] = np.array(imageData, copy=True) @@ -3994,9 +4114,11 @@ def main(): psf_data, comp_star_count, coarse_aper_data, - coarse_apertures, - coarse_annuli, + coarse_aperture_values, + coarse_annulus_values, fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, ) if i == coarse_tune_frames - 1: @@ -4009,6 +4131,12 @@ def main(): subset_airmass, require_comp_star=require_comp_star, ) + if use_adaptive_apertures: + aperture_values = refined_apertures_sigma + annulus_values = refined_annuli_sigma + else: + aperture_values = refined_apertures_sigma * sigma + annulus_values = refined_annuli_sigma * sigma apers = refined_apertures_sigma * sigma annuli = refined_annuli_sigma * sigma aper_data = initialize_aperture_data_store(len(inputfiles), len(apers), len(annuli), comp_star_count) @@ -4047,9 +4175,11 @@ def main(): psf_data, comp_star_count, aper_data, - apers, - annuli, + aperture_values, + annulus_values, fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, ) finally: if loaded_from_disk: @@ -4058,8 +4188,14 @@ def main(): else: if not aperture_grid_tuned: # Defensive fallback for unexpected control flow. - apers = coarse_apertures - annuli = coarse_annuli + aperture_values = coarse_aperture_values + annulus_values = coarse_annulus_values + if use_adaptive_apertures: + apers = coarse_apertures_sigma * sigma + annuli = coarse_annuli_sigma * sigma + else: + apers = coarse_aperture_values + annuli = coarse_annulus_values aper_data = initialize_aperture_data_store(len(inputfiles), len(apers), len(annuli), comp_star_count) aperture_grid_tuned = True @@ -4069,9 +4205,11 @@ def main(): psf_data, comp_star_count, aper_data, - apers, - annuli, + aperture_values, + annulus_values, fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, ) # close file + delete from memory @@ -4103,6 +4241,18 @@ def main(): aper_data[ckey] = aper_data[ckey][goodmask] aper_data[f"{ckey}_bg"] = aper_data[f"{ckey}_bg"][goodmask] + sigma_display = representative_psf_sigma(psf_data['target'], fallback_sigma=sigma) + if not np.isfinite(sigma_display) or sigma_display <= 0: + sigma_display = 1.0 + + if aperture_values is not None and annulus_values is not None: + if use_adaptive_apertures: + apers = np.asarray(aperture_values, dtype=float) * sigma_display + annuli = np.asarray(annulus_values, dtype=float) * sigma_display + else: + apers = np.asarray(aperture_values, dtype=float) + annuli = np.asarray(annulus_values, dtype=float) + exotic_infoDict['exposure'] = exp_time_med(exptimes) # save PSF data to disk using savetxt @@ -4154,7 +4304,7 @@ def main(): annuli, airmass, exotic_infoDict['comp_stars'], - sigma, + sigma_display, ) if comparison_calibration is not None: @@ -4311,7 +4461,7 @@ def main(): if photometry_info['min_std'] > res_std and myfit is not None: photometry_info.update(best_fit_lc=myfit, comp_star_num=j + 1, comp_star_coords=exotic_infoDict['comp_stars'][j], - min_std=res_std, min_aperture=0, min_annulus=15 * sigma, + min_std=res_std, min_aperture=0, min_annulus=15 * sigma_display, selection_basis='target_fit') flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, @@ -4564,7 +4714,7 @@ def main(): min_aper_fov = float(photometry_info['min_aperture']) min_annulus_fov = float(photometry_info['min_annulus']) - plot_fov(photometry_info['min_aperture'], photometry_info['min_annulus'], sigma, + plot_fov(photometry_info['min_aperture'], photometry_info['min_annulus'], sigma_display, centroid_positions['x_targ'][0], centroid_positions['y_targ'][0], centroid_positions['x_ref'][0], centroid_positions['y_ref'][0], firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 676f5014..13870bdd 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -417,6 +417,7 @@ def save_input(): "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -436,6 +437,7 @@ def save_input(): new_inits['optional_info'] = { "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "disable vertical flux normalization": False, + "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } @@ -1481,6 +1483,7 @@ def save_input(): "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -1548,6 +1551,7 @@ def save_input(): "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "disable vertical flux normalization": False, + "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } @@ -1594,6 +1598,7 @@ def save_input(): "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "disable vertical flux normalization": False, + "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } diff --git a/exotic/inputs.py b/exotic/inputs.py index f0f03503..dbf85749 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -208,7 +208,8 @@ def __init__(self, init_opt): 'dist': None, 'pm_ra': None, 'pm_dec': None, 'airmass_already_corrected': False, 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', - 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False + 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, + 'use_adaptive_apertures': False } self.params = { 'images': imaging_files, 'save': save_directory, 'aavso_num': obs_code, 'second_obs': second_obs_code, @@ -417,6 +418,11 @@ def comp_params(self, init_file, planet_dict): 'disable vertical flux normalization', 'Disable vertical flux normalization', ), + 'use_adaptive_apertures': ( + 'use_adaptive_apertures', + 'Use Adaptive Apertures? (y/n)', + 'Use Adaptive Apertures (y/n)', + ), 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (arsec/pixel)'), 'exposure': 'Exposure Time (s)', diff --git a/inits.json b/inits.json index d4b18b1f..2150daec 100644 --- a/inits.json +++ b/inits.json @@ -25,6 +25,7 @@ "Fast Aperture Mask": "Default true/fast mode for quicker aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to false to opt out and use exact masks.", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -94,6 +95,7 @@ "Fast Aperture Mask (y/n)": true, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "disable vertical flux normalization": false, + "use_adaptive_apertures": false, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y", "Image Scale (Ex: 5.21 arcsecs/pixel)": null, diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 23081b0e..d5194d76 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -87,9 +87,12 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: cheap_lightcurve_prescore, comparison_star_stability_summary, fit_lightcurve, + is_adaptive_aperture_mode_enabled, is_comp_star_required, is_target_driven_comp_selection_enabled, phase_bin_sigma_clip, + representative_psf_sigma, + resolve_frame_aperture_radii, should_skip_airmass_fit, update_coordinates_with_proper_motion, ) @@ -171,6 +174,37 @@ def test_is_target_driven_comp_selection_enabled_parses_values(): assert is_target_driven_comp_selection_enabled("n") is False +def test_is_adaptive_aperture_mode_enabled_parses_values(): + assert is_adaptive_aperture_mode_enabled(None) is False + assert is_adaptive_aperture_mode_enabled("y") is True + assert is_adaptive_aperture_mode_enabled("n") is False + assert is_adaptive_aperture_mode_enabled(True) is True + + +def test_resolve_frame_aperture_radii_scales_sigma_grid(): + apertures, annuli = resolve_frame_aperture_radii( + np.array([2.0, 3.0]), + np.array([8.0, 10.0]), + adaptive_apertures=True, + frame_sigma=1.5, + fallback_sigma=1.0, + ) + + assert np.allclose(apertures, np.array([3.0, 4.5])) + assert np.allclose(annuli, np.array([12.0, 15.0])) + + +def test_representative_psf_sigma_uses_valid_frames_and_fallback(): + psf_rows = np.array([ + [0.0, 0.0, 1.0, 2.0, 2.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 2.2, 1.8, 0.0, 0.0], + [0.0, 0.0, 1.0, np.nan, np.nan, 0.0, 0.0], + ]) + + assert np.isclose(representative_psf_sigma(psf_rows, fallback_sigma=1.0), 2.0) + assert np.isclose(representative_psf_sigma(np.full((0, 7), np.nan), fallback_sigma=1.25), 1.25) + + def test_auto_tune_aperture_grid_uses_comparison_field_consistency(): coarse_apertures_sigma = np.array([2.0, 3.0]) coarse_annuli_sigma = np.array([8.0]) diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 3982c018..460c03c1 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -83,6 +83,21 @@ def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_p assert inputs.info_dict["disable_vertical_flux_normalization"] is False +def test_comp_params_defaults_use_adaptive_apertures_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_adaptive_apertures"] is False + + def test_comp_params_reads_observatory_full_title_from_user_info(tmp_path): init_data = { "user_info": {"Observatory Full Title": "Whipple Observatory"}, @@ -143,6 +158,21 @@ def test_comp_params_reads_disable_vertical_flux_normalization_from_optional_inf assert inputs.info_dict["disable_vertical_flux_normalization"] is True +def test_comp_params_reads_use_adaptive_apertures_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_adaptive_apertures": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_adaptive_apertures"] is True + + class DummyResponse: def __init__(self, payload): self._payload = payload From 136bb104958513e8fbea1ff3bc6f32dd9fccf4c2 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 30 Mar 2026 10:35:01 +1100 Subject: [PATCH 009/116] Prefer existing header WCS over external plate solving --- exotic/exotic.py | 18 +++++++++++------- tests/test_centroid_wcs.py | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 6492ebc5..0aa64b77 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -1043,6 +1043,17 @@ def check_wcs(fits_file, save_directory, plate_opt, rt=False, use_nextastro_astr ra=None, dec=None, pixel_scale=None, ignore_header_wcs=False): wcs_file = None + if not ignore_header_wcs and search_wcs(fits_file).is_celestial: + if plate_opt == 'y' and not rt: + log_info("Your FITS files already have WCS (World Coordinate System) information in their headers. " + "EXOTIC will use the existing header WCS and skip external plate solving.") + else: + log_info("Your FITS files have WCS (World Coordinate System) information in their headers. " + "EXOTIC will proceed to use these. " + "NOTE: If you do not trust your WCS coordinates, " + "please restart EXOTIC after enabling plate solutions via astrometry.net.") + return fits_file + if plate_opt == 'y' and not rt: wcs_file = get_wcs(fits_file, save_directory, use_nextastro_astrometry=use_nextastro_astrometry, ra=ra, dec=dec, pixel_scale=pixel_scale) if ignore_header_wcs: @@ -1051,13 +1062,6 @@ def check_wcs(fits_file, save_directory, plate_opt, rt=False, use_nextastro_astr else: log_info("Ignoring FITS header WCS and using the legacy image-to-image alignment path.") return wcs_file - if not wcs_file: - if search_wcs(fits_file).is_celestial: - log_info("Your FITS files have WCS (World Coordinate System) information in their headers. " - "EXOTIC will proceed to use these. " - "NOTE: If you do not trust your WCS coordinates, " - "please restart EXOTIC after enabling plate solutions via astrometry.net.") - wcs_file = fits_file return wcs_file diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 21c086a1..29992364 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -266,6 +266,29 @@ def test_check_wcs_keeps_plate_solution_when_override_enabled(monkeypatch): assert wcs_file == "solved_wcs.fits" +def test_check_wcs_prefers_header_wcs_over_plate_solution(monkeypatch): + monkeypatch.setattr( + exotic_module, + "search_wcs", + lambda *_args, **_kwargs: types.SimpleNamespace(is_celestial=True), + ) + monkeypatch.setattr( + exotic_module, + "get_wcs", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("plate solution should be skipped when header WCS exists") + ), + ) + + wcs_file = exotic_module.check_wcs( + "frame.fits", + ".", + "y", + ) + + assert wcs_file == "frame.fits" + + def test_should_log_plate_solution_path_suppresses_posix_tmp_paths(): assert exotic_module.should_log_plate_solution_path("/tmp/tmp42fmzrv8/temp/wcs.fits") is False From a0d55824386123ff8af8a70bdfc9850fdcff77c8 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 30 Mar 2026 11:38:05 +1100 Subject: [PATCH 010/116] Report adaptive aperture statistics and add diagnostics --- exotic/exotic.py | 182 ++++++++++++++++++++++++++--- exotic/output_files.py | 23 +++- exotic/plots.py | 80 +++++++++++++ tests/test_exotic_proper_motion.py | 21 ++++ tests/test_output_files.py | 40 +++++++ tests/test_plots.py | 20 +++- 6 files changed, 349 insertions(+), 17 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 0aa64b77..29e3638c 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -132,11 +132,11 @@ try: # plots from plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ - plot_comp_star_calibration_series, plot_comp_star_suitability + plot_comp_star_calibration_series, plot_comp_star_suitability, plot_adaptive_aperture_diagnostics except ImportError: # package import from .plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ - plot_comp_star_calibration_series, plot_comp_star_suitability + plot_comp_star_calibration_series, plot_comp_star_suitability, plot_adaptive_aperture_diagnostics try: # tools from utils import round_to_2, user_input except ImportError: # package import @@ -392,6 +392,88 @@ def representative_psf_sigma(psf_rows, fallback_sigma=np.nan): return np.nan +def summarize_adaptive_aperture_usage(psf_rows, aperture_scale, annulus_scale, fallback_sigma=np.nan): + try: + aperture_scale = float(aperture_scale) + annulus_scale = float(annulus_scale) + except (TypeError, ValueError): + return None + + if not np.isfinite(aperture_scale) or not np.isfinite(annulus_scale): + return None + + rows = np.asarray(psf_rows) + if rows.ndim != 2 or rows.shape[0] == 0: + return None + + frame_sigma = np.array( + [psf_sigma_from_fit(row, fallback_sigma=fallback_sigma) for row in rows], + dtype=float, + ) + frame_sigma[~np.isfinite(frame_sigma) | (frame_sigma <= 0)] = np.nan + + aperture_series = aperture_scale * frame_sigma + annulus_series = annulus_scale * frame_sigma + fwhm_series = 2.355 * frame_sigma + + if not np.any(np.isfinite(aperture_series)) or not np.any(np.isfinite(annulus_series)): + return None + + return { + 'aperture_sigma': aperture_scale, + 'annulus_sigma': annulus_scale, + 'frame_sigma': frame_sigma, + 'fwhm_series': fwhm_series, + 'aperture_series': aperture_series, + 'annulus_series': annulus_series, + 'aperture_median': float(np.nanmedian(aperture_series)), + 'aperture_std': float(np.nanstd(aperture_series)), + 'aperture_min': float(np.nanmin(aperture_series)), + 'aperture_max': float(np.nanmax(aperture_series)), + 'annulus_median': float(np.nanmedian(annulus_series)), + 'annulus_std': float(np.nanstd(annulus_series)), + 'annulus_min': float(np.nanmin(annulus_series)), + 'annulus_max': float(np.nanmax(annulus_series)), + } + + +def update_photometry_adaptive_summary(photometry_info, use_adaptive_apertures, aperture_values, annulus_values, + psf_rows, fallback_sigma=np.nan): + photometry_info['adaptive_summary'] = None + + if (not use_adaptive_apertures) or photometry_info.get('min_aperture') in (None, 0): + return None + + a_idx = photometry_info.get('aperture_index') + an_idx = photometry_info.get('annulus_index') + if a_idx is None or an_idx is None or aperture_values is None or annulus_values is None: + return None + + aperture_grid = np.asarray(aperture_values, dtype=float) + annulus_grid = np.asarray(annulus_values, dtype=float) + if a_idx >= aperture_grid.size or an_idx >= annulus_grid.size: + return None + + photometry_info['adaptive_summary'] = summarize_adaptive_aperture_usage( + psf_rows, + aperture_grid[a_idx], + annulus_grid[an_idx], + fallback_sigma=fallback_sigma, + ) + return photometry_info['adaptive_summary'] + + +def reported_photometry_aperture_radii(photometry_info): + adaptive_summary = photometry_info.get('adaptive_summary') + if adaptive_summary is None: + return photometry_info.get('min_aperture'), photometry_info.get('min_annulus') + + aperture = adaptive_summary['aperture_median'] + if photometry_info.get('min_aperture') is not None and photometry_info['min_aperture'] < 0: + aperture = -aperture + return aperture, adaptive_summary['annulus_median'] + + def resolve_frame_aperture_radii(apertures, annuli, adaptive_apertures=False, frame_sigma=np.nan, fallback_sigma=np.nan): aperture_values = np.asarray(apertures, dtype=float) @@ -4293,6 +4375,9 @@ def main(): 'min_std': 100000, 'min_aperture': None, 'min_annulus': None, + 'aperture_index': None, + 'annulus_index': None, + 'adaptive_summary': None, 'calibration_field_score': np.inf, 'selection_basis': 'target_fit', } @@ -4367,6 +4452,8 @@ def main(): selected_comp_coords = exotic_infoDict['comp_stars'][selected_comp_index] selected_min_aperture = 0 if comparison_calibration['method'] == 'psf' else comparison_calibration['aper'] selected_min_annulus = comparison_calibration['annulus'] + selected_a = None if comparison_calibration['method'] == 'psf' else comparison_calibration['a'] + selected_an = None if comparison_calibration['method'] == 'psf' else comparison_calibration['an'] if comparison_calibration['method'] == 'psf': selected_target_flux = tFlux @@ -4391,6 +4478,8 @@ def main(): min_std=res_std, min_aperture=selected_min_aperture, min_annulus=selected_min_annulus, + aperture_index=selected_a, + annulus_index=selected_an, calibration_field_score=comparison_calibration['field_score'], selection_basis='comparison_field') @@ -4466,6 +4555,7 @@ def main(): photometry_info.update(best_fit_lc=myfit, comp_star_num=j + 1, comp_star_coords=exotic_infoDict['comp_stars'][j], min_std=res_std, min_aperture=0, min_annulus=15 * sigma_display, + aperture_index=None, annulus_index=None, selection_basis='target_fit') flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, @@ -4571,6 +4661,8 @@ def main(): min_std=res_std, min_aperture=(-candidate['aper'] if candidate['comp_index'] is None else candidate['aper']), min_annulus=candidate['annulus'], + aperture_index=candidate['a'], + annulus_index=candidate['an'], selection_basis='target_fit') flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, @@ -4603,6 +4695,15 @@ def main(): 'pos': exotic_infoDict['comp_stars'][j] } + update_photometry_adaptive_summary( + photometry_info, + use_adaptive_apertures, + aperture_values, + annulus_values, + psf_data['target'], + fallback_sigma=sigma, + ) + if require_comp_star and photometry_info['comp_star_num'] is None: log_info("Error: require_comp_star is enabled, but no valid comparison star could be selected.", error=True) return @@ -4610,6 +4711,8 @@ def main(): log_info("\n\n*********************************************") if np.isfinite(photometry_info['calibration_field_score']): log_info(f"Comparison-Star Field Score: {round(photometry_info['calibration_field_score'] * 100, 4)}%") + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] == 0: # psf log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") @@ -4617,13 +4720,29 @@ def main(): elif photometry_info['min_aperture'] < 0: # no comp star log_info("Best Comparison Star: None") log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") - log_info(f"Optimal Aperture: {abs(np.round(photometry_info['min_aperture'], 2))}") - log_info(f"Optimal Annulus: {np.round(photometry_info['min_annulus'], 2)}") + if adaptive_summary is not None: + log_info(f"Optimal Aperture: {abs(display_aperture):.2f} +/- {adaptive_summary['aperture_std']:.2f} px") + log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") + log_info(f"Adaptive Aperture Scale: {adaptive_summary['aperture_sigma']:.2f} sigma") + log_info(f"Adaptive Annulus Scale: {adaptive_summary['annulus_sigma']:.2f} sigma") + log_info(f"Aperture Range: {adaptive_summary['aperture_min']:.2f} to {adaptive_summary['aperture_max']:.2f} px") + log_info(f"Annulus Range: {adaptive_summary['annulus_min']:.2f} to {adaptive_summary['annulus_max']:.2f} px") + else: + log_info(f"Optimal Aperture: {abs(np.round(display_aperture, 2))}") + log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") else: log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") - log_info(f"Optimal Aperture: {np.round(photometry_info['min_aperture'], 2)}") - log_info(f"Optimal Annulus: {np.round(photometry_info['min_annulus'], 2)}") + if adaptive_summary is not None: + log_info(f"Optimal Aperture: {display_aperture:.2f} +/- {adaptive_summary['aperture_std']:.2f} px") + log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") + log_info(f"Adaptive Aperture Scale: {adaptive_summary['aperture_sigma']:.2f} sigma") + log_info(f"Adaptive Annulus Scale: {adaptive_summary['annulus_sigma']:.2f} sigma") + log_info(f"Aperture Range: {adaptive_summary['aperture_min']:.2f} to {adaptive_summary['aperture_max']:.2f} px") + log_info(f"Annulus Range: {adaptive_summary['annulus_min']:.2f} to {adaptive_summary['annulus_max']:.2f} px") + else: + log_info(f"Optimal Aperture: {np.round(display_aperture, 2)}") + log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") log_info("*********************************************\n") best_fit_lc = photometry_info['best_fit_lc'] @@ -4706,6 +4825,16 @@ def main(): flux_unc_tar=flux_values['flux_unc_tar'][relative_flux_mask], flux_unc_ref=flux_values['flux_unc_ref'][relative_flux_mask]) + update_photometry_adaptive_summary( + photometry_info, + use_adaptive_apertures, + aperture_values, + annulus_values, + psf_data['target'][si][gi][relative_flux_mask], + fallback_sigma=sigma, + ) + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + if photometry_info['min_aperture'] == 0: opt_method = "PSF" @@ -4715,10 +4844,10 @@ def main(): min_annulus_fov = float(15 * stdev_fov.mean()) else: opt_method = "Aperture" - min_aper_fov = float(photometry_info['min_aperture']) - min_annulus_fov = float(photometry_info['min_annulus']) + min_aper_fov = float(display_aperture) + min_annulus_fov = float(display_annulus) - plot_fov(photometry_info['min_aperture'], photometry_info['min_annulus'], sigma_display, + plot_fov(display_aperture, display_annulus, sigma_display, centroid_positions['x_targ'][0], centroid_positions['y_targ'][0], centroid_positions['x_ref'][0], centroid_positions['y_ref'][0], firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov) @@ -4732,6 +4861,21 @@ def main(): goodFluxes, goodNormUnc, goodAirmasses, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + adaptive_summary = photometry_info.get('adaptive_summary') + if adaptive_summary is not None: + plot_adaptive_aperture_diagnostics( + goodTimes, + adaptive_summary['aperture_series'], + adaptive_summary['annulus_series'], + adaptive_summary['fwhm_series'], + goodAirmasses, + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + adaptive_summary['aperture_sigma'], + adaptive_summary['annulus_sigma'], + ) + # TODO: convert the exoplanet archive mid transit time to bjd - need to take into account observatory location listed in Exoplanet Archive # tMidtoC = astropy.time.Time(timeMidTransit, format='jd', scale='utc') # forPhaseResult = JDUTC_to_BJDTDB(tMidtoC, ra=raDeg, dec=decDeg, lat=lati, longi=longit, alt=2000) @@ -4935,6 +5079,8 @@ def main(): if fitsortext == 1: if np.isfinite(photometry_info.get('calibration_field_score', np.inf)): log_info(f" Comparison-Star Field Score: {round_to_2(100. * photometry_info['calibration_field_score'])} %") + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] >= 0: log_info(f" Best Comparison Star: #{bestCompStar} - {comp_coords}") else: @@ -4942,8 +5088,16 @@ def main(): if photometry_info['min_aperture'] == 0: log_info(" Optimal Method: PSF photometry") else: - log_info(f" Optimal Aperture: {abs(np.round(photometry_info['min_aperture'], 2))}") - log_info(f" Optimal Annulus: {np.round(photometry_info['min_annulus'], 2)}") + if adaptive_summary is not None: + log_info(f" Optimal Aperture: {abs(display_aperture):.2f} +/- {adaptive_summary['aperture_std']:.2f} px") + log_info(f" Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") + log_info(f" Adaptive Aperture Scale: {adaptive_summary['aperture_sigma']:.2f} sigma") + log_info(f" Adaptive Annulus Scale: {adaptive_summary['annulus_sigma']:.2f} sigma") + log_info(f" Aperture Range: {adaptive_summary['aperture_min']:.2f} to {adaptive_summary['aperture_max']:.2f} px") + log_info(f" Annulus Range: {adaptive_summary['annulus_min']:.2f} to {adaptive_summary['annulus_max']:.2f} px") + else: + log_info(f" Optimal Aperture: {abs(np.round(display_aperture, 2))}") + log_info(f" Optimal Annulus: {np.round(display_annulus, 2)}") log_info(f" Transit Duration [day]: {round_to_2(np.mean(durs), np.std(durs))} +/- {round_to_2(np.std(durs))}") log_info("*********************************************************") @@ -4967,10 +5121,12 @@ def main(): log_info(f"\nError: Could not create FinalLightCurve.csv. {error_txt}\n\t{e}", error=True) try: if fitsortext == 1: + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) output_files.final_planetary_params(phot_opt=True, vsp_params=vsp_params, comp_star=bestCompStar, comp_coords=comp_coords, - min_aper=np.round(photometry_info['min_aperture'], 2), - min_annul=np.round(photometry_info['min_annulus'], 2)) + min_aper=np.round(display_aperture, 2), + min_annul=np.round(display_annulus, 2), + adaptive_summary=photometry_info.get('adaptive_summary')) else: output_files.final_planetary_params(phot_opt=False, vsp_params=vsp_params) except Exception as e: diff --git a/exotic/output_files.py b/exotic/output_files.py index 1e710abc..cc23d3b7 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -73,7 +73,8 @@ def final_lightcurve(self, phase): self.fit.transit, self.fit.airmass_model): f.write(f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}\n") - def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, min_annul=None): + def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, + min_annul=None, adaptive_summary=None): params_file = self.dir / "temp" / f"FinalParams_{self.p_dict['pName']}_{self.i_dict['date']}.json" params_num = { @@ -118,8 +119,24 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor if min_aper == 0: phot_ext["Optimal Method"] = "PSF photometry" else: - phot_ext["Optimal Aperture"] = f"{abs(min_aper)}" - phot_ext["Optimal Annulus"] = f"{min_annul}" + if adaptive_summary: + phot_ext["Adaptive Aperture Scale"] = f"{adaptive_summary['aperture_sigma']:.2f} sigma" + phot_ext["Adaptive Annulus Scale"] = f"{adaptive_summary['annulus_sigma']:.2f} sigma" + phot_ext["Optimal Aperture"] = ( + f"{adaptive_summary['aperture_median']:.2f} +/- {adaptive_summary['aperture_std']:.2f} px" + ) + phot_ext["Aperture Range"] = ( + f"{adaptive_summary['aperture_min']:.2f} to {adaptive_summary['aperture_max']:.2f} px" + ) + phot_ext["Optimal Annulus"] = ( + f"{adaptive_summary['annulus_median']:.2f} +/- {adaptive_summary['annulus_std']:.2f} px" + ) + phot_ext["Annulus Range"] = ( + f"{adaptive_summary['annulus_min']:.2f} to {adaptive_summary['annulus_max']:.2f} px" + ) + else: + phot_ext["Optimal Aperture"] = f"{abs(min_aper)}" + phot_ext["Optimal Annulus"] = f"{min_annul}" params_num.update(phot_ext) params_num["Transit Duration (day)"] = (f"{round_to_2(mean(self.durs), std(self.durs))} +/- " diff --git a/exotic/plots.py b/exotic/plots.py index 92b632d2..a7ffb82a 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -284,6 +284,86 @@ def plot_comp_star_suitability(comp_summaries, targ_name, save, date, method_lab plt.close(fig) +def plot_adaptive_aperture_diagnostics(times, aperture_series, annulus_series, fwhm_series, airmass, + targ_name, save, date, aperture_sigma, annulus_sigma): + times = np.asarray(times, dtype=float) + aperture_series = np.asarray(aperture_series, dtype=float) + annulus_series = np.asarray(annulus_series, dtype=float) + fwhm_series = np.asarray(fwhm_series, dtype=float) + airmass = np.asarray(airmass, dtype=float) + + plot_len = min(times.size, aperture_series.size, annulus_series.size, fwhm_series.size, airmass.size) + if plot_len == 0: + return + + times = times[:plot_len] + aperture_series = aperture_series[:plot_len] + annulus_series = annulus_series[:plot_len] + fwhm_series = fwhm_series[:plot_len] + airmass = airmass[:plot_len] + + valid_time = np.isfinite(times) + valid_aperture = np.isfinite(aperture_series) + valid_annulus = np.isfinite(annulus_series) + valid_fwhm = np.isfinite(fwhm_series) + valid_airmass = np.isfinite(airmass) + + temp_dir = Path(save) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + + fig, axes = plt.subplots(2, 2, figsize=(12, 8.5)) + fig.suptitle( + f"{targ_name} Adaptive Aperture Diagnostics\n" + f"aper={aperture_sigma:.2f} sigma, annulus={annulus_sigma:.2f} sigma" + ) + + time_mask = valid_time & valid_aperture + time_zero = np.nanmin(times[time_mask]) if np.any(time_mask) else 0.0 + axes[0, 0].set_title("Aperture Radius vs Time") + axes[0, 0].set_xlabel(f"Time [BJD_TDB-{time_zero:.5f}]") + axes[0, 0].set_ylabel("Aperture Radius [px]") + if np.any(time_mask): + axes[0, 0].plot(times[time_mask] - time_zero, aperture_series[time_mask], color='tab:blue', + marker='o', ms=3, lw=1.1) + axes[0, 0].grid(alpha=0.25) + + annulus_mask = valid_time & valid_annulus + axes[0, 1].set_title("Annulus Radius vs Time") + axes[0, 1].set_xlabel(f"Time [BJD_TDB-{time_zero:.5f}]") + axes[0, 1].set_ylabel("Annulus Radius [px]") + if np.any(annulus_mask): + axes[0, 1].plot(times[annulus_mask] - time_zero, annulus_series[annulus_mask], color='tab:orange', + marker='o', ms=3, lw=1.1) + axes[0, 1].grid(alpha=0.25) + + fwhm_mask = valid_aperture & valid_fwhm + axes[1, 0].set_title("Aperture Radius vs Target FWHM") + axes[1, 0].set_xlabel("Target PSF FWHM [px]") + axes[1, 0].set_ylabel("Aperture Radius [px]") + if np.any(fwhm_mask): + axes[1, 0].scatter(fwhm_series[fwhm_mask], aperture_series[fwhm_mask], color='tab:green', s=18, alpha=0.8) + order = np.argsort(fwhm_series[fwhm_mask]) + axes[1, 0].plot(fwhm_series[fwhm_mask][order], aperture_series[fwhm_mask][order], color='tab:green', + alpha=0.35, lw=1.0) + axes[1, 0].grid(alpha=0.25) + + airmass_mask = valid_aperture & valid_airmass + axes[1, 1].set_title("Aperture Radius vs Airmass") + axes[1, 1].set_xlabel("Airmass") + axes[1, 1].set_ylabel("Aperture Radius [px]") + if np.any(airmass_mask): + axes[1, 1].scatter(airmass[airmass_mask], aperture_series[airmass_mask], color='tab:red', s=18, alpha=0.8) + order = np.argsort(airmass[airmass_mask]) + axes[1, 1].plot(airmass[airmass_mask][order], aperture_series[airmass_mask][order], color='tab:red', + alpha=0.35, lw=1.0) + axes[1, 1].grid(alpha=0.25) + + fig.tight_layout() + fig.savefig(temp_dir / f"AdaptiveApertureDiagnostics_{targ_name}_{date}.png", bbox_inches="tight") + fig.savefig(temp_dir / f"AdaptiveApertureDiagnostics_{targ_name}_{date}.pdf", bbox_inches="tight") + plt.close(fig) + + def plot_variable_residuals(save): plt.title("Stellar Variability Residuals") plt.ylabel("Residuals (flux)") diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index d5194d76..8b53f881 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -93,6 +93,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: phase_bin_sigma_clip, representative_psf_sigma, resolve_frame_aperture_radii, + summarize_adaptive_aperture_usage, should_skip_airmass_fit, update_coordinates_with_proper_motion, ) @@ -205,6 +206,26 @@ def test_representative_psf_sigma_uses_valid_frames_and_fallback(): assert np.isclose(representative_psf_sigma(np.full((0, 7), np.nan), fallback_sigma=1.25), 1.25) +def test_summarize_adaptive_aperture_usage_reports_frame_scaled_stats(): + psf_rows = np.array([ + [0.0, 0.0, 1.0, 2.0, 2.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 3.0, 3.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 4.0, 4.0, 0.0, 0.0], + ]) + + summary = summarize_adaptive_aperture_usage(psf_rows, aperture_scale=2.5, annulus_scale=9.0, fallback_sigma=1.0) + + np.testing.assert_allclose(summary["aperture_series"], np.array([5.0, 7.5, 10.0])) + np.testing.assert_allclose(summary["annulus_series"], np.array([18.0, 27.0, 36.0])) + np.testing.assert_allclose(summary["fwhm_series"], np.array([4.71, 7.065, 9.42])) + assert np.isclose(summary["aperture_median"], 7.5) + assert np.isclose(summary["aperture_std"], np.std([5.0, 7.5, 10.0])) + assert np.isclose(summary["aperture_min"], 5.0) + assert np.isclose(summary["aperture_max"], 10.0) + assert summary["aperture_sigma"] == 2.5 + assert summary["annulus_sigma"] == 9.0 + + def test_auto_tune_aperture_grid_uses_comparison_field_consistency(): coarse_apertures_sigma = np.array([2.0, 3.0]) coarse_annuli_sigma = np.array([8.0]) diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 74f0eb96..017d10e5 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -201,6 +201,46 @@ def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): assert "Airmass coefficient 1 (a1)" not in output_text +def test_final_planetary_params_reports_adaptive_aperture_summary(tmp_path): + fit = DummyFit() + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + adaptive_summary = { + "aperture_sigma": 2.62, + "annulus_sigma": 9.00, + "aperture_median": 7.98, + "aperture_std": 0.41, + "aperture_min": 7.12, + "aperture_max": 8.76, + "annulus_median": 27.43, + "annulus_std": 1.39, + "annulus_min": 25.11, + "annulus_max": 30.08, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=True, + vsp_params=[], + comp_star=9, + comp_coords=[1446.0, 2399.0], + min_aper=7.98, + min_annul=27.43, + adaptive_summary=adaptive_summary, + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_text = output_file.read_text(encoding="utf-8") + + assert "Adaptive Aperture Scale" in output_text + assert "2.62 sigma" in output_text + assert "Optimal Aperture" in output_text + assert "7.98 +/- 0.41 px" in output_text + assert "Aperture Range" in output_text + assert "7.12 to 8.76 px" in output_text + + def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_path): fit = DummyFit() fit.airmass_fit_skipped = True diff --git a/tests/test_plots.py b/tests/test_plots.py index 1b2d17bd..1bd1e5c6 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -4,7 +4,7 @@ import numpy as np from matplotlib.axes import Axes -from exotic.plots import plot_obs_stats +from exotic.plots import plot_adaptive_aperture_diagnostics, plot_obs_stats class DummyFit: @@ -46,3 +46,21 @@ def spy_plot(self, x, y, *args, **kwargs): np.testing.assert_array_equal(captured[0][0], fit.time) np.testing.assert_array_equal(captured[0][1], np.array([14.0, 7.0, 21.0])) assert (tmp_path / "temp" / "Observing_Statistics_target_2026-03-09.png").exists() + + +def test_plot_adaptive_aperture_diagnostics_writes_outputs(tmp_path): + plot_adaptive_aperture_diagnostics( + times=np.array([1.0, 2.0, 3.0]), + aperture_series=np.array([7.5, 8.0, 8.5]), + annulus_series=np.array([25.0, 26.0, 27.0]), + fwhm_series=np.array([3.0, 3.2, 3.4]), + airmass=np.array([1.1, 1.2, 1.3]), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + aperture_sigma=2.5, + annulus_sigma=9.0, + ) + + assert (tmp_path / "temp" / "AdaptiveApertureDiagnostics_Target_2026-03-09.png").exists() + assert (tmp_path / "temp" / "AdaptiveApertureDiagnostics_Target_2026-03-09.pdf").exists() From 2942a4f26bbedfdb73a49d2e6709f1f59ddeab01 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 30 Mar 2026 11:51:46 +1100 Subject: [PATCH 011/116] Handle WCS headers from image extensions --- exotic/exotic.py | 41 +++++++++++++++++++++++-------- tests/test_centroid_wcs.py | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 10 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 29e3638c..91dfffae 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -1278,10 +1278,30 @@ def get_wcs(file, directory="", use_nextastro_astrometry=False, ra=None, dec=Non # Getting the right ascension and declination for every pixel in imaging file if there is a plate solution -def get_ra_dec(header): +def _resolve_wcs_image_dimensions(header, image_shape=None): + width = header.get('NAXIS1', header.get('ZNAXIS1')) + height = header.get('NAXIS2', header.get('ZNAXIS2')) + if width is not None and height is not None: + return int(width), int(height) + + if image_shape is not None and len(image_shape) >= 2: + return int(image_shape[-1]), int(image_shape[-2]) + wcs_header = WCS(header) - xaxis = np.arange(header['NAXIS1']) - yaxis = np.arange(header['NAXIS2']) + if wcs_header.pixel_shape is not None and len(wcs_header.pixel_shape) >= 2: + return int(wcs_header.pixel_shape[0]), int(wcs_header.pixel_shape[1]) + + if wcs_header.array_shape is not None and len(wcs_header.array_shape) >= 2: + return int(wcs_header.array_shape[1]), int(wcs_header.array_shape[0]) + + raise KeyError("Keyword 'NAXIS1' not found.") + + +def get_ra_dec(header, image_shape=None): + wcs_header = WCS(header) + width, height = _resolve_wcs_image_dimensions(header, image_shape=image_shape) + xaxis = np.arange(width) + yaxis = np.arange(height) x, y = np.meshgrid(xaxis, yaxis) return wcs_header.all_pix2world(x, y, 1) @@ -2120,7 +2140,7 @@ def log_finding_transformation_progress(i, total_jobs, file_name, use_multiproce def get_img_scale(hdr, wcs_file, pixel_init): if wcs_file: - wcs_hdr = fits.getheader(wcs_file) + wcs_hdr = get_first_image_header(wcs_file) astrometry_scale = [key.value.split(' ') for key in wcs_hdr._cards if 'scale:' in str(key.value)] if astrometry_scale: @@ -2764,11 +2784,12 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet ignore_header_wcs=ignore_header_wcs) comp_star = info_dict['comp_stars'] tar_radec, comp_radec = None, [] + first_image = fits.getdata(inputfiles[0]) if wcs_file: - wcs_header = fits.getheader(filename=wcs_file) + wcs_header = get_first_image_header(wcs_file) - ra_file, dec_file = get_ra_dec(wcs_header) + ra_file, dec_file = get_ra_dec(wcs_header, image_shape=first_image.shape) tar_radec = (ra_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], dec_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) @@ -2781,7 +2802,6 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet if tar_radec is not None and comp_radec: target_and_comp_radec = np.array([tar_radec, comp_radec[0]], dtype=float) - first_image = fits.getdata(inputfiles[0]) targ_sig_xy = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0)[3:5] # aperture and annulus scale factors in PSF sigma units @@ -3927,12 +3947,13 @@ def main(): if wcs_file: if should_log_plate_solution_path(wcs_file): log_info(f"\nHere is the path to your plate solution: {wcs_file}") - wcs_header = fits.getheader(filename=wcs_file) - ra_wcs, dec_wcs = get_ra_dec(wcs_header) + reference_image = fits.getdata(inputfiles[0]) + wcs_header = get_first_image_header(wcs_file) + ra_wcs, dec_wcs = get_ra_dec(wcs_header, image_shape=reference_image.shape) exotic_UIprevTPX, exotic_UIprevTPY = check_target_pixel_wcs(exotic_UIprevTPX, exotic_UIprevTPY, pDict, ra_wcs, dec_wcs, - fits.getdata(inputfiles[0]), + reference_image, jd_times[0], non_interactive_run=args.non_interactive_run, wcs_header=wcs_header) diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 29992364..a8c2dac8 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -4,6 +4,7 @@ import importlib.util import numpy as np +from astropy.io import fits from astropy.wcs import WCS @@ -86,6 +87,22 @@ def _gaussian_image(shape=(80, 80), center=(40.0, 35.0), amplitude=5000.0, sigma return image +def _write_extension_wcs_fits(tmp_path, shape=(100, 120)): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [shape[1] / 2.0, shape[0] / 2.0] + wcs.wcs.crval = [210.0, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + + path = tmp_path / "extension_wcs.fits" + hdul = fits.HDUList([ + fits.PrimaryHDU(), + fits.ImageHDU(data=np.zeros(shape, dtype=float), header=wcs.to_header(), name="SCI"), + ]) + hdul.writeto(path, overwrite=True) + return path + + def test_fit_centroid_uses_moment_fallback_when_psf_fit_fails(monkeypatch): image = _gaussian_image() low_flux_warnings = [] @@ -197,6 +214,38 @@ def test_check_target_pixel_wcs_keeps_input_coords_when_wcs_target_is_off_frame( assert y_pixel == 30.0 +def test_get_ra_dec_uses_image_shape_when_header_lacks_naxis(): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [210.0, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + + ra_list, dec_list = exotic_module.get_ra_dec(wcs.to_header(), image_shape=(100, 120)) + + assert ra_list.shape == (100, 120) + assert dec_list.shape == (100, 120) + + +def test_get_first_image_header_skips_empty_primary_hdu(tmp_path): + wcs_path = _write_extension_wcs_fits(tmp_path) + + header = exotic_module.get_first_image_header(wcs_path) + + assert header["NAXIS1"] == 120 + assert header["NAXIS2"] == 100 + assert header["CTYPE1"] == "RA---TAN" + + +def test_get_img_scale_uses_first_image_extension_header(tmp_path): + wcs_path = _write_extension_wcs_fits(tmp_path) + + img_scale_str, img_scale = exotic_module.get_img_scale(fits.Header(), wcs_path, None) + + assert img_scale_str == "Image scale in arcsec/pixel: 36.0" + assert img_scale == 36.0 + + def test_should_ignore_header_wcs_defaults_to_false(): assert exotic_module.should_ignore_header_wcs(None) is False assert exotic_module.should_ignore_header_wcs("n") is False From 54913e381f4f33849bde9e0018980ed9e406b470 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 30 Mar 2026 12:42:55 +1100 Subject: [PATCH 012/116] Fix up some wcs bugs. Also have some changable values to reject a handful of bad wcs frames if the majority of the dataset already has wcs. --- exotic/api/colab.py | 1 + exotic/exotic.py | 120 +++++++++++++++++++++++++++++++++---- exotic/exotic_gui.py | 5 ++ exotic/inputs.py | 6 +- inits.json | 2 + tests/test_centroid_wcs.py | 54 +++++++++++++++++ tests/test_inputs.py | 30 ++++++++++ 7 files changed, 206 insertions(+), 12 deletions(-) diff --git a/exotic/api/colab.py b/exotic/api/colab.py index e7fcc3dc..814d90d0 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -378,6 +378,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Filter Minimum Wavelength (nm)": %s, "Filter Maximum Wavelength (nm)": %s, "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "bad_wcs_threshold_percent": 3.0, "use_adaptive_apertures": false, "require_comp_star": "y" } diff --git a/exotic/exotic.py b/exotic/exotic.py index 91dfffae..db5fa574 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -325,6 +325,36 @@ def should_ignore_header_wcs(config_value): return False +def get_bad_wcs_threshold_fraction(config_value): + default_fraction = SPARSE_MISSING_WCS_DROP_THRESHOLD + default_percent = default_fraction * 100.0 + if config_value is None: + return default_fraction + + if isinstance(config_value, str): + normalized = config_value.strip() + if normalized == "": + return default_fraction + if normalized.endswith('%'): + normalized = normalized[:-1].strip() + else: + normalized = config_value + + try: + threshold_percent = float(normalized) + except (TypeError, ValueError): + log_info(f"Warning: Invalid 'bad_wcs_threshold_percent' value; using default {default_percent:g}%.", + warn=True) + return default_fraction + + if not np.isfinite(threshold_percent) or threshold_percent < 0 or threshold_percent > 100: + log_info(f"Warning: Invalid 'bad_wcs_threshold_percent' value; using default {default_percent:g}%.", + warn=True) + return default_fraction + + return threshold_percent / 100.0 + + def is_vertical_flux_normalization_disabled(config_value): if config_value is None: return False @@ -1169,21 +1199,72 @@ def get_first_image_header(file_name): return header -def evaluate_celestial_wcs_coverage(inputfiles): +def collect_celestial_wcs_coverage(inputfiles): + has_celestial_wcs = [] missing_wcs_files = [] for file_name in inputfiles: + file_has_celestial_wcs = False try: image_header = get_first_image_header(file_name) - if not search_wcs_from_header(image_header).is_celestial: - missing_wcs_files.append(str(file_name)) + file_has_celestial_wcs = search_wcs_from_header(image_header).is_celestial except Exception: + file_has_celestial_wcs = False + + has_celestial_wcs.append(file_has_celestial_wcs) + if not file_has_celestial_wcs: missing_wcs_files.append(str(file_name)) + return np.array(has_celestial_wcs, dtype=bool), missing_wcs_files + + +def evaluate_celestial_wcs_coverage(inputfiles): + has_celestial_wcs, missing_wcs_files = collect_celestial_wcs_coverage(inputfiles) total_files = len(inputfiles) - all_have_celestial_wcs = total_files > 0 and len(missing_wcs_files) == 0 + all_have_celestial_wcs = total_files > 0 and bool(has_celestial_wcs.all()) return all_have_celestial_wcs, missing_wcs_files +def log_missing_celestial_wcs_preview(missing_wcs_files): + if not missing_wcs_files: + return + + preview = ", ".join(missing_wcs_files[:3]) + remainder = len(missing_wcs_files) - 3 + if remainder > 0: + preview = f"{preview}, ... (+{remainder} more)" + log.debug(f"Files without usable celestial WCS: {preview}") + + +def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_missing_fraction=None): + inputfiles = np.array(inputfiles) + keep_mask = np.ones(len(inputfiles), dtype=bool) + if ignore_header_wcs or len(inputfiles) == 0: + return inputfiles, keep_mask, [] + + if max_missing_fraction is None: + max_missing_fraction = SPARSE_MISSING_WCS_DROP_THRESHOLD + + keep_mask, missing_wcs_files = collect_celestial_wcs_coverage(inputfiles) + missing_count = len(missing_wcs_files) + total_files = len(inputfiles) + if missing_count == 0: + return inputfiles, keep_mask, [] + + missing_fraction = missing_count / total_files + if missing_count < total_files and missing_fraction < max_missing_fraction: + retained_files = inputfiles[keep_mask] + threshold_percent = max_missing_fraction * 100.0 + log_info( + f"WCS precheck: {len(retained_files)}/{total_files} files have celestial WCS. " + f"Dropping {missing_count} file(s) without celestial WCS because they are below the " + f"{threshold_percent:g}% threshold." + ) + log_missing_celestial_wcs_preview(missing_wcs_files) + return retained_files, keep_mask, missing_wcs_files + + return inputfiles, np.ones(total_files, dtype=bool), [] + + def should_use_multiprocess_transform_precompute(inputfiles, requested_processes, ignore_header_wcs=False): if requested_processes is None or requested_processes <= 0: return False @@ -1202,12 +1283,7 @@ def should_use_multiprocess_transform_precompute(inputfiles, requested_processes missing_count = len(missing_wcs_files) log_info(f"WCS precheck: {total_files - missing_count}/{total_files} files have celestial WCS. " "Keeping multiprocessing transformation precompute for fallback alignment.") - if missing_count > 0: - preview = ", ".join(missing_wcs_files[:3]) - remainder = missing_count - 3 - if remainder > 0: - preview = f"{preview}, ... (+{remainder} more)" - log.debug(f"Files without usable celestial WCS: {preview}") + log_missing_celestial_wcs_preview(missing_wcs_files) return True @@ -2076,6 +2152,7 @@ def transformation_task_with_cached_reference(i, file_name): MAX_MULTIPROCESS_TRANSFORM_WORKERS = 8 +SPARSE_MISSING_WCS_DROP_THRESHOLD = 0.03 # Automatic aperture-grid tuning constants (in PSF sigma units) APERTURE_SIGMA_MIN = 1.5 @@ -2748,6 +2825,7 @@ def save_comp_ra_dec(wcs_file, ra_file, dec_file, comp_coords): def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astrometry=False, multiprocess_transformations=None): timeList, airMassList, exptimes, norm_flux = [], [], [], [] ignore_header_wcs = should_ignore_header_wcs(info_dict.get('ignore_header_wcs')) + bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction(info_dict.get('bad_wcs_threshold_percent')) plateStatus.initializeFilenames(info_dict['images']) inputfiles = corruption_check(info_dict['images']) @@ -2766,6 +2844,13 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet si = np.argsort(times) inputfiles = np.array(inputfiles)[si] + inputfiles, _, dropped_wcs_files = filter_sparse_missing_wcs_frames( + inputfiles, + ignore_header_wcs=ignore_header_wcs, + max_missing_fraction=bad_wcs_threshold_fraction, + ) + if dropped_wcs_files: + plateStatus.initializeFilenames(list(inputfiles)) use_multiprocess_transform_precompute = should_use_multiprocess_transform_precompute( inputfiles, multiprocess_transformations, ignore_header_wcs=ignore_header_wcs @@ -3902,6 +3987,19 @@ def main(): times = np.array(times)[si] jd_times = np.array(jd_times)[si] inputfiles = np.array(inputfiles)[si] + ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) + bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction( + exotic_infoDict.get('bad_wcs_threshold_percent') + ) + inputfiles, wcs_keep_mask, dropped_wcs_files = filter_sparse_missing_wcs_frames( + inputfiles, + ignore_header_wcs=ignore_header_wcs, + max_missing_fraction=bad_wcs_threshold_fraction, + ) + if dropped_wcs_files: + times = times[wcs_keep_mask] + jd_times = jd_times[wcs_keep_mask] + plateStatus.initializeFilenames(list(inputfiles)) exotic_UIprevTPX = exotic_infoDict['tar_coords'][0] exotic_UIprevTPY = exotic_infoDict['tar_coords'][1] @@ -3928,12 +4026,12 @@ def main(): times = times[inc:] jd_times = jd_times[inc:] plateStatus.setCurrentFilename(inputfiles[0]) + header = get_first_image_header(inputfiles[0]) # For astrometry hints, prioritize coordinates explicitly provided by the user # (from inits.json / CLI) over values scraped from NASA Exoplanet Archive. hint_ra = userpDict.get('ra', pDict.get('ra')) hint_dec = userpDict.get('dec', pDict.get('dec')) - ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) wcs_file = check_wcs(inputfiles[0], exotic_infoDict['save'], exotic_infoDict['plate_opt'], use_nextastro_astrometry=args.use_nextastro_astrometry, diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 13870bdd..a8891d97 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -416,6 +416,7 @@ def save_input(): "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -436,6 +437,7 @@ def save_input(): } new_inits['optional_info'] = { "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1482,6 +1484,7 @@ def save_input(): "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -1550,6 +1553,7 @@ def save_input(): "Filter Maximum Wavelength (nm)": input_data.get('filtermax', null), "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1597,6 +1601,7 @@ def save_input(): "Exposure Time (s)": input_data['exp'], "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/exotic/inputs.py b/exotic/inputs.py index dbf85749..9bdb6290 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -209,7 +209,7 @@ def __init__(self, init_opt): 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, - 'use_adaptive_apertures': False + 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0 } self.params = { 'images': imaging_files, 'save': save_directory, 'aavso_num': obs_code, 'second_obs': second_obs_code, @@ -423,6 +423,10 @@ def comp_params(self, init_file, planet_dict): 'Use Adaptive Apertures? (y/n)', 'Use Adaptive Apertures (y/n)', ), + 'bad_wcs_threshold_percent': ( + 'bad_wcs_threshold_percent', + 'Bad WCS Threshold Percent', + ), 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (arsec/pixel)'), 'exposure': 'Exposure Time (s)', diff --git a/inits.json b/inits.json index 2150daec..bf0b12f4 100644 --- a/inits.json +++ b/inits.json @@ -24,6 +24,7 @@ "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Fast Aperture Mask": "Default true/fast mode for quicker aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to false to opt out and use exact masks.", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", @@ -94,6 +95,7 @@ "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Fast Aperture Mask (y/n)": true, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": false, "use_adaptive_apertures": false, "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index a8c2dac8..1f724d1a 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -4,6 +4,7 @@ import importlib.util import numpy as np +import pytest from astropy.io import fits from astropy.wcs import WCS @@ -252,6 +253,23 @@ def test_should_ignore_header_wcs_defaults_to_false(): assert exotic_module.should_ignore_header_wcs("y") is True +def test_get_bad_wcs_threshold_fraction_defaults_to_three_percent(): + assert exotic_module.get_bad_wcs_threshold_fraction(None) == pytest.approx(0.03) + assert exotic_module.get_bad_wcs_threshold_fraction("") == pytest.approx(0.03) + + +def test_get_bad_wcs_threshold_fraction_reads_numeric_percent_values(): + assert exotic_module.get_bad_wcs_threshold_fraction(5.5) == pytest.approx(0.055) + assert exotic_module.get_bad_wcs_threshold_fraction("7.25") == pytest.approx(0.0725) + assert exotic_module.get_bad_wcs_threshold_fraction("4%") == pytest.approx(0.04) + + +def test_get_bad_wcs_threshold_fraction_falls_back_for_invalid_values(): + assert exotic_module.get_bad_wcs_threshold_fraction("not-a-number") == pytest.approx(0.03) + assert exotic_module.get_bad_wcs_threshold_fraction(-1) == pytest.approx(0.03) + assert exotic_module.get_bad_wcs_threshold_fraction(101) == pytest.approx(0.03) + + def test_display_filename_returns_basename_for_unix_and_windows_paths(): assert ( exotic_module._display_filename( @@ -353,3 +371,39 @@ def test_should_use_multiprocess_transform_precompute_respects_header_wcs_overri requested_processes=2, ignore_header_wcs=True, ) is True + + +def test_filter_sparse_missing_wcs_frames_drops_files_below_three_percent(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(34)] + missing_frame = frames[7] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: str(file_name)) + monkeypatch.setattr( + exotic_module, + "search_wcs_from_header", + lambda header: types.SimpleNamespace(is_celestial=header != missing_frame), + ) + + filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames(frames) + + assert filtered.tolist() == [frame for frame in frames if frame != missing_frame] + assert keep_mask.tolist() == [frame != missing_frame for frame in frames] + assert dropped == [missing_frame] + + +def test_filter_sparse_missing_wcs_frames_keeps_files_at_three_percent_or_higher(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(33)] + missing_frame = frames[5] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: str(file_name)) + monkeypatch.setattr( + exotic_module, + "search_wcs_from_header", + lambda header: types.SimpleNamespace(is_celestial=header != missing_frame), + ) + + filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames(frames) + + assert filtered.tolist() == frames + assert keep_mask.tolist() == [True] * len(frames) + assert dropped == [] diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 460c03c1..df951879 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -68,6 +68,21 @@ def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "n" +def test_comp_params_defaults_bad_wcs_threshold_percent_to_three(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["bad_wcs_threshold_percent"] == 3.0 + + def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_path): init_data = { "user_info": {}, @@ -143,6 +158,21 @@ def test_comp_params_reads_ignore_header_wcs_from_optional_info(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "y" +def test_comp_params_reads_bad_wcs_threshold_percent_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"bad_wcs_threshold_percent": 5.5}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["bad_wcs_threshold_percent"] == 5.5 + + def test_comp_params_reads_disable_vertical_flux_normalization_from_optional_info(tmp_path): init_data = { "user_info": {}, From a5a880c2485cbc3125c7e40b254d27fc8fb83575 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 30 Mar 2026 16:10:16 +1100 Subject: [PATCH 013/116] Fix adaptive aperture sigma scaling and clip radius outliers --- exotic/exotic.py | 85 ++++++++++++++++++++++++++++-- tests/test_exotic_proper_motion.py | 19 +++++++ 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index db5fa574..d91ba8bd 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -542,6 +542,62 @@ def sigma_clip(ogdata, sigma=3, dt=21, po=2): return nanmask +def adaptive_aperture_outlier_mask(aperture_series, annulus_series=None, sigma=4.5, window=15, polyorder=2): + aperture_series = np.asarray(aperture_series, dtype=float) + combined_mask = _adaptive_series_outlier_mask( + aperture_series, + sigma=sigma, + window=window, + polyorder=polyorder, + ) + + if annulus_series is None: + return combined_mask + + annulus_series = np.asarray(annulus_series, dtype=float) + annulus_mask = _adaptive_series_outlier_mask( + annulus_series, + sigma=sigma, + window=window, + polyorder=polyorder, + ) + return combined_mask | annulus_mask + + +def _adaptive_series_outlier_mask(series, sigma=4.5, window=15, polyorder=2): + values = np.asarray(series, dtype=float) + nanmask = ~np.isfinite(values) + valid_indices = np.flatnonzero(~nanmask) + if valid_indices.size < max(polyorder + 3, 7): + return nanmask + + valid_values = values[valid_indices] + window_length = min(int(window), valid_values.size) + if window_length % 2 == 0: + window_length -= 1 + + if window_length >= polyorder + 2: + trend = savgol_filter(valid_values, window_length=window_length, polyorder=polyorder, mode='interp') + residuals = valid_values - trend + scatter = robust_scatter(residuals) + center = trend + else: + scatter = np.nan + center = np.full(valid_values.shape, bn.nanmedian(valid_values)) + + if not np.isfinite(scatter) or scatter <= 0: + center = np.full(valid_values.shape, bn.nanmedian(valid_values)) + residuals = valid_values - center + scatter = robust_scatter(residuals) + if not np.isfinite(scatter) or scatter <= 0: + return nanmask + + local_mask = np.abs(valid_values - center) > sigma * scatter + outlier_mask = nanmask.copy() + outlier_mask[valid_indices] = local_mask + return outlier_mask + + def robust_scatter(data): values = np.asarray(data, dtype=float) finite = values[np.isfinite(values)] @@ -2333,6 +2389,10 @@ def should_use_fast_centroid(frame_index): return frame_index % CENTROID_FULL_FIT_CADENCE != 0 +def should_use_fast_target_centroid(frame_index, adaptive_apertures=False): + return should_use_fast_centroid(frame_index) and not adaptive_apertures + + def _fit_centroid_moments(subarray, xv, yv, pos, box): background = bn.nanmedian(subarray) weights = subarray - background @@ -2918,6 +2978,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) frame_fast_centroid = should_use_fast_centroid(i) + target_fast_centroid = should_use_fast_target_centroid(i, adaptive_apertures=use_adaptive_apertures) extension = 0 image_header = hdul[extension].header @@ -2972,7 +3033,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet imageData, [tx, ty], 0, - fast_mode=frame_fast_centroid, + fast_mode=target_fast_centroid, ) psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( imageData, @@ -3025,7 +3086,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet imageData, [tx, ty], 0, - fast_mode=frame_fast_centroid, + fast_mode=target_fast_centroid, ) cx, cy = transformed_coords[1] @@ -4157,6 +4218,10 @@ def main(): hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) frame_fast_centroid = should_use_fast_centroid(i) + target_fast_centroid = should_use_fast_target_centroid( + i, + adaptive_apertures=use_adaptive_apertures, + ) extension = 0 image_header = hdul[extension].header @@ -4221,7 +4286,7 @@ def main(): imageData, [tx, ty], 0, - fast_mode=frame_fast_centroid, + fast_mode=target_fast_centroid, ) # TODO: Add check for flux on target/comp stars relative to others in the field @@ -4280,7 +4345,7 @@ def main(): imageData, [tx, ty], 0, - fast_mode=frame_fast_centroid, + fast_mode=target_fast_centroid, ) for j, coord in enumerate(exotic_infoDict['comp_stars']): @@ -4882,10 +4947,20 @@ def main(): phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) if hasattr(best_fit_lc, 'residuals') and hasattr(best_fit_lc, 'phase'): phase_clip_mask = phase_bin_sigma_clip(best_fit_lc.residuals[si], best_fit_lc.phase[si], sigma=3, bins=10) - gi = ~(time_clip_mask | phase_clip_mask) # good indexs + adaptive_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + if use_adaptive_apertures and adaptive_summary is not None: + sorted_apertures = np.asarray(adaptive_summary['aperture_series'], dtype=float)[si] + sorted_annuli = np.asarray(adaptive_summary['annulus_series'], dtype=float)[si] + retained_mask = adaptive_aperture_outlier_mask(sorted_apertures[~time_clip_mask & ~phase_clip_mask], + sorted_annuli[~time_clip_mask & ~phase_clip_mask]) + adaptive_clip_mask[~time_clip_mask & ~phase_clip_mask] = retained_mask + gi = ~(time_clip_mask | phase_clip_mask | adaptive_clip_mask) # good indexs phase_clip_removed = np.count_nonzero(phase_clip_mask & ~time_clip_mask) if phase_clip_removed: log_info(f"Removed {phase_clip_removed} phase-binned residual outlier(s) before final fit.") + adaptive_clip_removed = np.count_nonzero(adaptive_clip_mask) + if adaptive_clip_removed: + log_info(f"Removed {adaptive_clip_removed} adaptive-aperture radius outlier(s) before final fit.") # Calculate the proper timeseries uncertainties from the residuals of the out-of-transit data OOT = (best_fit_lc.transit == 1) # find out-of-transit portion of the lightcurve diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 8b53f881..ae3951d4 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -82,6 +82,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: sys.modules.setdefault("exotic.api.ld", fake_ld) from exotic.exotic import ( + adaptive_aperture_outlier_mask, auto_tune_aperture_sigma_grid, check_coordinates, cheap_lightcurve_prescore, @@ -95,6 +96,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: resolve_frame_aperture_radii, summarize_adaptive_aperture_usage, should_skip_airmass_fit, + should_use_fast_target_centroid, update_coordinates_with_proper_motion, ) @@ -182,6 +184,12 @@ def test_is_adaptive_aperture_mode_enabled_parses_values(): assert is_adaptive_aperture_mode_enabled(True) is True +def test_should_use_fast_target_centroid_disables_fast_sigma_path_for_adaptive_runs(): + assert should_use_fast_target_centroid(1, adaptive_apertures=False) is True + assert should_use_fast_target_centroid(6, adaptive_apertures=False) is False + assert should_use_fast_target_centroid(1, adaptive_apertures=True) is False + + def test_resolve_frame_aperture_radii_scales_sigma_grid(): apertures, annuli = resolve_frame_aperture_radii( np.array([2.0, 3.0]), @@ -226,6 +234,17 @@ def test_summarize_adaptive_aperture_usage_reports_frame_scaled_stats(): assert summary["annulus_sigma"] == 9.0 +def test_adaptive_aperture_outlier_mask_rejects_isolated_spike_but_keeps_repeated_lower_mode(): + aperture_series = np.array([10.0, 10.1, 9.4, 10.0, 9.4, 10.1, 10.0, 15.2, 10.1, 9.4, 10.0, 10.1]) + annulus_series = aperture_series * 3.0 + + mask = adaptive_aperture_outlier_mask(aperture_series, annulus_series) + + expected = np.zeros_like(aperture_series, dtype=bool) + expected[7] = True + np.testing.assert_array_equal(mask, expected) + + def test_auto_tune_aperture_grid_uses_comparison_field_consistency(): coarse_apertures_sigma = np.array([2.0, 3.0]) coarse_annuli_sigma = np.array([8.0]) From 039dc0338aa8c76c5c7a306ed357a14f2cec20b3 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 30 Mar 2026 16:50:21 +1100 Subject: [PATCH 014/116] Report WCS source using only the filename --- exotic/exotic.py | 6 +++++- tests/test_centroid_wcs.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index d91ba8bd..31b8d838 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -1988,6 +1988,10 @@ def _display_filename(file_name): return str(file_name).replace("\\", "/").rsplit("/", 1)[-1] +def format_plate_solution_reference(wcs_file): + return f"Here is the filename where we got the WCS from: {_display_filename(wcs_file)}" + + # Aligns imaging data from .fits file to easily track the host and comparison star's positions def transformation(image_data, file_name, roi=1, report_failure=True, reference_image=None): start_time = perf_counter() @@ -4105,7 +4109,7 @@ def main(): if wcs_file: if should_log_plate_solution_path(wcs_file): - log_info(f"\nHere is the path to your plate solution: {wcs_file}") + log_info(f"\n{format_plate_solution_reference(wcs_file)}") reference_image = fits.getdata(inputfiles[0]) wcs_header = get_first_image_header(wcs_file) ra_wcs, dec_wcs = get_ra_dec(wcs_header, image_shape=reference_image.shape) diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 1f724d1a..422ec154 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -280,6 +280,17 @@ def test_display_filename_returns_basename_for_unix_and_windows_paths(): assert exotic_module._display_filename(r"C:\data\run\frame_002.fits.fz") == "frame_002.fits.fz" +def test_format_plate_solution_reference_uses_basename_only(): + assert ( + exotic_module.format_plate_solution_reference( + "/mnt/md0/ftp/user_data/psyfitz/DATA_INBOX/Z.good.TOI 2969 b_2026-03-05_ECO1/" + "NxAst-TOI2969b_rp_2461105d05262731_20260305_1a016_30_eco1.fits.fz" + ) + == "Here is the filename where we got the WCS from: " + "NxAst-TOI2969b_rp_2461105d05262731_20260305_1a016_30_eco1.fits.fz" + ) + + def test_log_finding_transformation_progress_prints_basename(monkeypatch): stdout = io.StringIO() debug_messages = [] From c090daf1e1aa3084e8a995127525229ac03601fb Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 11 Apr 2026 11:07:06 +1000 Subject: [PATCH 015/116] Add impact parameter fit option and centered geometry triangle plot --- README.md | 6 + docs/README.md | 5 + exotic/api/colab.py | 2 + exotic/api/elca.py | 549 +++++++++--- exotic/exotic.py | 1271 ++++++++++++++++++++++++++-- exotic/exotic_gui.py | 10 + exotic/inputs.py | 33 +- exotic/output_files.py | 7 +- exotic/plots.py | 130 ++- inits.json | 12 + tests/test_centroid_wcs.py | 11 + tests/test_elca_baseline.py | 242 ++++++ tests/test_exotic_proper_motion.py | 557 +++++++++++- tests/test_inputs.py | 195 +++++ tests/test_plots.py | 100 ++- 15 files changed, 2888 insertions(+), 242 deletions(-) diff --git a/README.md b/README.md index db41351f..89a75c91 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,12 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Maximum Wavelength (nm)": null, "Fast Aperture Mask (y/n)": true, + "use_psf_photometry": "y", + "use_aperture_photometry": "y", + "skip_low_comparison_coverage_rejection": "n", + "fit_lightcurve_to_every_comparison_candidate": "n", + "detrend_on_outoftransit_baseline": true, + "use_impactparameter_rather_than_inclination_to_fit": "y", "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y", diff --git a/docs/README.md b/docs/README.md index ebe26a15..a1aaf38a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -215,6 +215,11 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, "Fast Aperture Mask (y/n)": true, + "use_psf_photometry": "y", + "use_aperture_photometry": "y", + "detrend_on_outoftransit_baseline": true, + "use_impactparameter_rather_than_inclination_to_fit": "y", + "skip_low_comparison_coverage_rejection": "n", "require_comp_star": "y" } } diff --git a/exotic/api/colab.py b/exotic/api/colab.py index 814d90d0..b745d9c1 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -379,6 +379,8 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Filter Maximum Wavelength (nm)": %s, "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "bad_wcs_threshold_percent": 3.0, + "detrend_on_outoftransit_baseline": true, + "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": false, "require_comp_star": "y" } diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 2f22ae14..48051d94 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -106,6 +106,27 @@ def transit(times, values): return model +def impact_parameter_scale(values): + ecc = values.get('ecc', 0.0) + omega = np.deg2rad(values.get('omega', 0.0)) + denom = 1.0 + ecc * np.sin(omega) + if np.any(np.isclose(denom, 0.0)): + denom = np.where(np.isclose(denom, 0.0), np.finfo(float).eps, denom) + return values['ars'] * (1.0 - ecc ** 2) / denom + + +def impact_parameter_from_inclination(values, inclination): + return impact_parameter_scale(values) * np.cos(np.deg2rad(inclination)) + + +def inclination_from_impact_parameter(values, impact_parameter): + scale = impact_parameter_scale(values) + if np.any(np.isclose(scale, 0.0)): + scale = np.where(np.isclose(scale, 0.0), np.finfo(float).eps, scale) + cosi = np.clip(np.asarray(impact_parameter, dtype=float) / scale, -1.0, 1.0) + return np.rad2deg(np.arccos(cosi)) + + def get_phase(times, per, tmid): return (times - tmid + 0.25 * per) / per % 1 - 0.25 @@ -271,7 +292,20 @@ def binner(arr, n, err=''): class lc_fitter(object): - def __init__(self, time, data, dataerr, airmass, prior, bounds, neighbors=200, mode='ns', jd_times=None, verbose=True): + def __init__( + self, + time, + data, + dataerr, + airmass, + prior, + bounds, + neighbors=200, + mode='ns', + jd_times=None, + verbose=True, + use_impactparameter_rather_than_inclination_to_fit=True, + ): self.time = time self.data = data self.dataerr = dataerr @@ -283,7 +317,13 @@ def __init__(self, time, data, dataerr, airmass, prior, bounds, neighbors=200, m self.jd_times = jd_times self.mode = mode self.neighbors = neighbors + self.use_impactparameter_rather_than_inclination_to_fit = use_impactparameter_rather_than_inclination_to_fit self.results = None + self.sampled_keys = list(bounds.keys()) + self.sample_bounds = copy.deepcopy(bounds) + self.sample_parameters = {} + self.sample_errors = {} + self.sample_quantiles = {} if self.mode == "lm": self.fit_LM() elif self.mode == "ns": @@ -306,6 +346,229 @@ def _set_flux_baseline(self, value, error=0.0): def _build_systematics_model(self, values): return get_flux_baseline(values) * np.exp(values.get('a2', 0) * self.airmass) + def _uses_internal_impact_parameter(self): + return ( + self.use_impactparameter_rather_than_inclination_to_fit + and self.mode == "ns" + and 'inc' in self.bounds + and 'b' not in self.bounds + ) + + def _get_sampled_keys(self, bound_keys=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + if not self._uses_internal_impact_parameter(): + return bound_keys + return ['b' if key == 'inc' else key for key in bound_keys] + + def _get_sample_bounds(self, bound_keys=None, values=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + sampled_keys = self._get_sampled_keys(bound_keys) + values = self.prior if values is None else values + sample_bounds = {} + for key, sampled_key in zip(bound_keys, sampled_keys): + if key == 'inc' and sampled_key == 'b': + inc_lower, inc_upper = self.bounds[key] + lower = float(np.min(impact_parameter_from_inclination(values, np.array([inc_lower, inc_upper])))) + upper = float(np.max(impact_parameter_from_inclination(values, np.array([inc_lower, inc_upper])))) + sample_bounds[sampled_key] = [lower, upper] + else: + sample_bounds[sampled_key] = list(self.bounds[key]) + return sample_bounds + + def _sample_point_from_unit_cube(self, upars, bound_keys=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + boundarray = np.array([self.bounds[k] for k in bound_keys], dtype=float) + physical = copy.deepcopy(self.prior) + sample_point = np.zeros(len(bound_keys), dtype=float) + + for i, key in enumerate(bound_keys): + if key == 'inc' and self._uses_internal_impact_parameter(): + continue + physical[key] = boundarray[i, 0] + (boundarray[i, 1] - boundarray[i, 0]) * upars[i] + + for i, key in enumerate(bound_keys): + if key == 'inc' and self._uses_internal_impact_parameter(): + inc = boundarray[i, 0] + (boundarray[i, 1] - boundarray[i, 0]) * upars[i] + sample_point[i] = impact_parameter_from_inclination(physical, inc) + else: + sample_point[i] = physical[key] + + return sample_point + + def _physical_values_from_sample_point(self, sample_point, bound_keys=None, sampled_keys=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + sampled_keys = self._get_sampled_keys(bound_keys) if sampled_keys is None else list(sampled_keys) + physical = copy.deepcopy(self.prior) + + for value, bound_key, sampled_key in zip(sample_point, bound_keys, sampled_keys): + if sampled_key == 'b' and bound_key == 'inc': + continue + physical[bound_key] = value + + for value, bound_key, sampled_key in zip(sample_point, bound_keys, sampled_keys): + if sampled_key == 'b' and bound_key == 'inc': + physical['b'] = value + physical['inc'] = float(inclination_from_impact_parameter(physical, value)) + + return physical + + def _summarize_derived_parameter(self, samples, point_estimate): + samples = np.asarray(samples, dtype=float) + center = float(point_estimate) + std = float(np.nanstd(samples)) + lower = float(np.nanpercentile(samples, 16)) + upper = float(np.nanpercentile(samples, 84)) + return center, std, [lower - center, upper - center] + + def _get_plot_range(self, key): + sample_parameters = getattr(self, 'sample_parameters', {}) + sample_errors = getattr(self, 'sample_errors', {}) + sample_bounds = getattr(self, 'sample_bounds', self.bounds) + center = sample_parameters[key] if key in sample_parameters else self.parameters[key] + error = sample_errors[key] if key in sample_errors else self.errors[key] + lower = center - 5 * error + upper = center + 5 * error + + if key in sample_bounds: + bound_lower, bound_upper = sample_bounds[key] + lower = max(lower, bound_lower) + upper = min(upper, bound_upper) + + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + if key in sample_bounds: + bound_lower, bound_upper = sample_bounds[key] + return [bound_lower, bound_upper] + + pad = error if np.isfinite(error) and error > 0 else max(abs(center) * 1e-6, 1e-6) + return [center - pad, center + pad] + + return [lower, upper] + + def _get_triangle_plot_samples(self): + if self.ns_type == 'ultranest': + points = np.asarray(self.results['weighted_samples']['points'], dtype=float) + logl = np.asarray(self.results['weighted_samples']['logl'], dtype=float) + return points, logl + + points = np.asarray(self.results.samples, dtype=float) + weights = np.exp(self.results.logwt - self.results.logz[-1]) + index_samples = resample_equal(np.arange(points.shape[0], dtype=float)[:, None], weights) + index_samples = np.clip(np.rint(index_samples[:, 0]).astype(int), 0, points.shape[0] - 1) + return points[index_samples], np.asarray(self.results.logl, dtype=float)[index_samples] + + def _get_triangle_plot_display_spec(self, sampled_keys, sample_parameters, sample_errors, sample_points): + if 'b' in sampled_keys: + key = 'b' + label = 'Distance from fitted b (mirrored)' + elif 'inc' in sampled_keys: + key = 'inc' + label = 'Distance from fitted Inc. [deg] (mirrored)' + else: + return None + + geometry_index = sampled_keys.index(key) + center = float(sample_parameters.get(key, self.parameters.get(key, 0.0))) + magnitude_samples = np.abs(np.asarray(sample_points[:, geometry_index], dtype=float) - center) + error = float(sample_errors.get(key, np.nanstd(magnitude_samples))) + if not np.isfinite(error) or error <= 0: + error = float(np.nanstd(magnitude_samples)) + plot_lower, plot_upper = self._get_plot_range(key) + max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) + if not np.isfinite(max_distance) or max_distance <= 0: + max_distance = float(np.nanmax(magnitude_samples)) + if not np.isfinite(max_distance) or max_distance <= 0: + max_distance = max(abs(center) * 1e-6, 1e-6) + return { + 'key': key, + 'index': geometry_index, + 'label': label, + 'mask_center': 0.0, + 'mask_error': error, + 'magnitude_samples': magnitude_samples, + 'range': [-max_distance, max_distance], + } + + def _get_triangle_plot_payload(self): + sampled_keys = getattr(self, 'sampled_keys', list(self.bounds.keys())) + sample_parameters = getattr(self, 'sample_parameters', self.parameters) + sample_errors = getattr(self, 'sample_errors', self.errors) + sample_points, sample_logl = self._get_triangle_plot_samples() + display_spec = self._get_triangle_plot_display_spec(sampled_keys, sample_parameters, sample_errors, sample_points) + + display_points = np.array(sample_points, copy=True) + display_logl = np.array(sample_logl, copy=True) + mask_values = np.array(sample_points, copy=True) + + if display_spec is not None: + geometry_index = display_spec['index'] + positive_points = np.array(sample_points, copy=True) + negative_points = np.array(sample_points, copy=True) + positive_points[:, geometry_index] = display_spec['magnitude_samples'] + negative_points[:, geometry_index] = -display_spec['magnitude_samples'] + display_points = np.vstack([positive_points, negative_points]) + display_logl = np.concatenate([sample_logl, sample_logl]) + mask_values = np.array(display_points, copy=True) + + flabels = { + 'rprs': r'R$_{p}$/R$_{s}$', + 'per': r'Period [day]', + 'tmid': r'T$_{mid}$', + 'ars': r'a/R$_{s}$', + 'inc': r'Inc. [deg]', + 'b': r'Impact parameter', + 'u1': r'u$_1$', + 'fpfs': r'F$_{p}$/F$_{s}$', + 'omega': r'$\omega$ [deg]', + 'mplanet': r'M$_{p}$ [M$_{\oplus}$]', + 'mstar': r'M$_{s}$ [M$_{\odot}$]', + 'ecc': r'$e$', + 'c0': r'$c_0$', + 'c1': r'$c_1$', + 'c2': r'$c_2$', + 'c3': r'$c_3$', + 'c4': r'$c_4$', + 'a0': r'$a_0$', + 'a1': r'$a_1$', + 'a2': r'$a_2$' + } + + labels = [] + titles = [] + ranges = [] + mask_centers = [] + mask_errors = [] + + for key in sampled_keys: + center = sample_parameters.get(key, self.parameters.get(key, 0.0)) + error = sample_errors.get(key, self.errors.get(key, 0.0)) + label = flabels.get(key, key) + title = f"{center:.5f} +- {error:.5f}" + plot_range = self._get_plot_range(key) + + if display_spec is not None and key == display_spec['key']: + label = display_spec['label'] + plot_range = display_spec['range'] + center = display_spec['mask_center'] + error = display_spec['mask_error'] + + labels.append(label) + titles.append(title) + ranges.append(plot_range) + mask_centers.append(center) + mask_errors.append(error) + + return { + 'sampled_keys': sampled_keys, + 'display_points': display_points, + 'display_logl': display_logl, + 'mask_values': mask_values, + 'labels': labels, + 'titles': titles, + 'ranges': ranges, + 'mask_centers': mask_centers, + 'mask_errors': mask_errors, + } + def fit_LM(self): freekeys = list(self.bounds.keys()) boundarray = np.array([self.bounds[k] for k in freekeys]) @@ -362,10 +625,18 @@ def lc2min_airmass(pars): self.parameters = copy.deepcopy(self.prior) self.errors = {} + self.quantiles = {} for i, k in enumerate(freekeys): self.parameters[k] = res.x[i] self.errors[k] = 0 + self.quantiles[k] = [0, 0] + + self.sampled_keys = list(freekeys) + self.sample_bounds = copy.deepcopy(self.bounds) + self.sample_parameters = {k: self.parameters[k] for k in self.sampled_keys} + self.sample_errors = {k: self.errors[k] for k in self.sampled_keys} + self.sample_quantiles = {k: self.quantiles[k] for k in self.sampled_keys} self.create_fit_variables() @@ -376,7 +647,10 @@ def create_fit_variables(self): self.transit_upsample = transit(self.time_upsample, self.parameters) self.phase_upsample = get_phase(self.time_upsample, self.parameters['per'], self.parameters['tmid']) if np.ndim(self.airmass) != 2: - if self.mode == "ns" and not self._has_free_flux_baseline(): + if self._has_free_flux_baseline(): + flux_scale = get_flux_baseline(self.parameters) + flux_scale_err = self.errors.get('a0', self.errors.get('a1', 0.0)) + elif self.mode == "ns": flux_scale, flux_scale_err = mc_a1( self.parameters.get('a2', 0), self.errors.get('a2', 1e-6), @@ -431,35 +705,44 @@ def create_fit_variables(self): self.duration_expected = newdur def fit_nested(self): - freekeys = list(self.bounds.keys()) - boundarray = np.array([self.bounds[k] for k in freekeys]) - bounddiff = np.diff(boundarray, 1).reshape(-1) + bound_keys = list(self.bounds.keys()) + sampled_keys = self._get_sampled_keys(bound_keys) self._validate_flux_baseline_keys() + self.sampled_keys = list(sampled_keys) + self.sample_bounds = self._get_sample_bounds(bound_keys, self.prior) + + if len(set(self.sampled_keys)) != len(self.sampled_keys): + raise ValueError("Free-parameter labels must be unique after internal parameter transforms.") # alloc data for best fit + error + self.sample_parameters = {} + self.sample_errors = {} + self.sample_quantiles = {} self.errors = {} self.quantiles = {} self.parameters = copy.deepcopy(self.prior) + def physical_from_sample_point(sample_point): + return self._physical_values_from_sample_point(sample_point, bound_keys, sampled_keys) + def loglike(pars): # chi-squared - for i in range(len(pars)): - self.prior[freekeys[i]] = pars[i] - model = transit(self.time, self.prior) - model *= np.exp(self.prior.get('a2', 0) * self.airmass) + physical = physical_from_sample_point(pars) + model = transit(self.time, physical) + model *= np.exp(physical.get('a2', 0) * self.airmass) if self._has_free_flux_baseline(): - model *= get_flux_baseline(self.prior) + model *= get_flux_baseline(physical) else: model *= solve_flux_baseline(model, self.data, self.dataerr) return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume - return boundarray[:, 0] + bounddiff * upars + return self._sample_point_from_unit_cube(upars, bound_keys) try: self.ns_type = 'ultranest' - test = ReactiveNestedSampler(freekeys, loglike, prior_transform) + test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform) self.results = run_reactive_sampler( test, @@ -467,21 +750,43 @@ def prior_transform(upars): verbose=self.verbose, ) - for i, key in enumerate(freekeys): - self.parameters[key] = self.results['maximum_likelihood']['point'][i] - self.errors[key] = self.results['posterior']['stdev'][i] - self.quantiles[key] = [ + ml_point = self.results['maximum_likelihood']['point'] + self.sample_bounds = self._get_sample_bounds(bound_keys, physical_from_sample_point(ml_point)) + + for i, key in enumerate(sampled_keys): + self.sample_parameters[key] = ml_point[i] + self.sample_errors[key] = self.results['posterior']['stdev'][i] + self.sample_quantiles[key] = [ self.results['posterior']['errlo'][i], self.results['posterior']['errup'][i]] + + physical_ml = physical_from_sample_point(ml_point) + self.parameters.update(physical_ml) + + for bound_key, sampled_key in zip(bound_keys, sampled_keys): + if bound_key == 'inc' and sampled_key == 'b': + continue + self.errors[bound_key] = self.sample_errors[sampled_key] + self.quantiles[bound_key] = self.sample_quantiles[sampled_key] + + if 'inc' in bound_keys and 'b' in sampled_keys: + inc_samples = np.array([ + physical_from_sample_point(point)['inc'] + for point in self.results['weighted_samples']['points'] + ]) + center, std, quantiles = self._summarize_derived_parameter(inc_samples, physical_ml['inc']) + self.parameters['inc'] = center + self.errors['inc'] = std + self.quantiles['inc'] = quantiles except NameError: self.ns_type = 'dynesty' - dsampler = dynesty.DynamicNestedSampler(loglike, prior_transform, ndim=len(freekeys), + dsampler = dynesty.DynamicNestedSampler(loglike, prior_transform, ndim=len(sampled_keys), bound='multi', sample='unif') dsampler.run_nested(maxcall=int(1e5), dlogz_init=0.05, maxbatch=10, nlive_batch=100, print_progress=self.verbose) self.results = dsampler.results - tests = [copy.deepcopy(self.prior) for i in range(5)] + tests = [np.zeros(len(sampled_keys), dtype=float) for _ in range(5)] # Derive kernel density estimate for best fit weights = np.exp(self.results.logwt - self.results.logz[-1]) @@ -495,52 +800,91 @@ def prior_transform(upars): # errors + final values mean, cov = dynesty.utils.mean_and_cov(self.results.samples, weights) mean2, cov2 = dynesty.utils.mean_and_cov(self.results.samples, self.weights) - for i in range(len(freekeys)): - self.errors[freekeys[i]] = cov[i, i] ** 0.5 - tests[0][freekeys[i]] = mean[i] - tests[1][freekeys[i]] = mean2[i] + for i in range(len(sampled_keys)): + self.sample_errors[sampled_keys[i]] = cov[i, i] ** 0.5 + tests[0][i] = mean[i] + tests[1][i] = mean2[i] counts, bins = np.histogram(samples[:, i], bins=100, weights=weights) mi = np.argmax(counts) - tests[4][freekeys[i]] = bins[mi] + 0.5 * np.mean(np.diff(bins)) + tests[4][i] = bins[mi] + 0.5 * np.mean(np.diff(bins)) # finds median and +- 2sigma, will vary from mode if non-gaussian - self.quantiles[freekeys[i]] = dynesty.utils.quantile(self.results.samples[:, i], [0.025, 0.5, 0.975], - weights=weights) - tests[2][freekeys[i]] = self.quantiles[freekeys[i]][1] + self.sample_quantiles[sampled_keys[i]] = dynesty.utils.quantile( + self.results.samples[:, i], + [0.025, 0.5, 0.975], + weights=weights, + ) + tests[2][i] = self.sample_quantiles[sampled_keys[i]][1] # find minimum near weighted mean - mask = (samples[:, 0] < self.parameters[freekeys[0]] + 2 * self.errors[freekeys[0]]) & ( - samples[:, 0] > self.parameters[freekeys[0]] - 2 * self.errors[freekeys[0]]) + mask = (samples[:, 0] < mean[0] + 2 * self.sample_errors[sampled_keys[0]]) & ( + samples[:, 0] > mean[0] - 2 * self.sample_errors[sampled_keys[0]]) bi = np.argmin(self.weights[mask]) - for i in range(len(freekeys)): - tests[3][freekeys[i]] = samples[mask][bi, i] + for i in range(len(sampled_keys)): + tests[3][i] = samples[mask][bi, i] # tests[4][freekeys[i]] = np.average(samples[mask][:, i], weights=self.weights[mask], axis=0) # find best fit from chi2 minimization chis = [] + physical_tests = [] for i in range(len(tests)): - lightcurve = transit(self.time, tests[i]) + test_values = physical_from_sample_point(tests[i]) + lightcurve = transit(self.time, test_values) if self._has_free_flux_baseline(): - flux_scale = get_flux_baseline(tests[i]) + flux_scale = get_flux_baseline(test_values) else: flux_scale = mc_a1( - tests[i].get('a2', 0), + test_values.get('a2', 0), self.errors.get('a2', 1e-6), lightcurve, self.airmass, self.data, self.dataerr, )[0] - tests[i]['a0'] = flux_scale - tests[i]['a1'] = flux_scale - airmass = flux_scale * np.exp(tests[i].get('a2', 0) * self.airmass) + test_values['a0'] = flux_scale + test_values['a1'] = flux_scale + airmass = flux_scale * np.exp(test_values.get('a2', 0) * self.airmass) residuals = self.data - (lightcurve * airmass) chis.append(np.sum(residuals ** 2)) + physical_tests.append(test_values) mi = np.argmin(chis) - self.parameters = copy.deepcopy(tests[mi]) + self.parameters = copy.deepcopy(physical_tests[mi]) + self.sample_bounds = self._get_sample_bounds(bound_keys, self.parameters) + self.sample_parameters = {key: tests[mi][i] for i, key in enumerate(sampled_keys)} + + for bound_key, sampled_key in zip(bound_keys, sampled_keys): + if bound_key == 'inc' and sampled_key == 'b': + continue + self.errors[bound_key] = self.sample_errors[sampled_key] + self.quantiles[bound_key] = self.sample_quantiles[sampled_key] + + if 'inc' in bound_keys and 'b' in sampled_keys: + inc_samples = np.array([ + physical_from_sample_point(point)['inc'] + for point in samples + ]) + center, std, quantiles = self._summarize_derived_parameter(inc_samples, self.parameters['inc']) + self.parameters['inc'] = center + self.errors['inc'] = std + self.quantiles['inc'] = quantiles + else: + for key in sampled_keys: + self.sample_errors.setdefault(key, 0.0) + self.sample_quantiles.setdefault(key, [0, 0, 0]) + + if not self.sample_parameters: + self.sample_parameters = { + key: self.parameters.get(key, self.sample_parameters.get(key)) + for key in self.sampled_keys + } + for bound_key, sampled_key in zip(bound_keys, sampled_keys): + if sampled_key not in self.sample_errors and bound_key in self.errors: + self.sample_errors[sampled_key] = self.errors[bound_key] + if sampled_key not in self.sample_quantiles and bound_key in self.quantiles: + self.sample_quantiles[sampled_key] = self.quantiles[bound_key] # final model self.create_fit_variables() @@ -629,88 +973,55 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) return f, axs def plot_triangle(self): - if self.ns_type == 'ultranest': - ranges = [] - mask1 = np.ones(len(self.results['weighted_samples']['logl']), dtype=bool) - mask2 = np.ones(len(self.results['weighted_samples']['logl']), dtype=bool) - mask3 = np.ones(len(self.results['weighted_samples']['logl']), dtype=bool) - titles = [] - labels = [] - flabels = { - 'rprs': r'R$_{p}$/R$_{s}$', - 'per': r'Period [day]', - 'tmid': r'T$_{mid}$', - 'ars': r'a/R$_{s}$', - 'inc': r'Inc. [deg]', - 'u1': r'u$_1$', - 'fpfs': r'F$_{p}$/F$_{s}$', - 'omega': r'$\omega$ [deg]', - 'mplanet': r'M$_{p}$ [M$_{\oplus}$]', - 'mstar': r'M$_{s}$ [M$_{\odot}$]', - 'ecc': r'$e$', - 'c0': r'$c_0$', - 'c1': r'$c_1$', - 'c2': r'$c_2$', - 'c3': r'$c_3$', - 'c4': r'$c_4$', - 'a0': r'$a_0$', - 'a1': r'$a_1$', - 'a2': r'$a_2$' - } - for i, key in enumerate(self.quantiles): - labels.append(flabels.get(key, key)) - titles.append(f"{self.parameters[key]:.5f} +- {self.errors[key]:.5f}") - ranges.append([ - self.parameters[key] - 5 * self.errors[key], - self.parameters[key] + 5 * self.errors[key] - ]) - - if key in ('a0', 'a1', 'a2'): - continue - - mask3 = mask3 & \ - (self.results['weighted_samples']['points'][:, i] > (self.parameters[key] - 3 * self.errors[key])) & \ - (self.results['weighted_samples']['points'][:, i] < (self.parameters[key] + 3 * self.errors[key])) - - mask1 = mask1 & \ - (self.results['weighted_samples']['points'][:, i] > (self.parameters[key] - self.errors[key])) & \ - (self.results['weighted_samples']['points'][:, i] < (self.parameters[key] + self.errors[key])) - - mask2 = mask2 & \ - (self.results['weighted_samples']['points'][:, i] > (self.parameters[key] - 2 * self.errors[key])) & \ - (self.results['weighted_samples']['points'][:, i] < (self.parameters[key] + 2 * self.errors[key])) - - chi2 = self.results['weighted_samples']['logl'] * -2 - fig = corner(self.results['weighted_samples']['points'], - labels=labels, - bins=int(np.sqrt(self.results['samples'].shape[0])), - range=ranges, - # quantiles=(0.1, 0.84), - plot_contours=True, - levels=[np.percentile(chi2[mask1], 95), np.percentile(chi2[mask2], 95), - np.percentile(chi2[mask3], 95)], - plot_density=False, - titles=titles, - data_kwargs={ - 'c': chi2, - 'vmin': np.percentile(chi2[mask3], 1), - 'vmax': np.percentile(chi2[mask3], 95), - 'cmap': 'viridis' - }, - label_kwargs={ - 'labelpad': 15, - }, - hist_kwargs={ - 'color': 'black', - } - ) - else: - fig, axs = dynesty.plotting.cornerplot(self.results, labels=list(self.bounds.keys()), - quantiles_2d=[0.4, 0.85], - smooth=0.015, show_titles=True, use_math_text=True, title_fmt='.2e', - hist2d_kwargs={ 'fill_contours': False}) - dynesty.plotting.cornerpoints(self.results, labels=list(self.bounds.keys()), - fig=[fig, axs[1:, :-1]], plot_kwargs={'alpha': 0.1, 'zorder': 1, }) + payload = self._get_triangle_plot_payload() + chi2 = payload['display_logl'] * -2 + mask1 = np.ones(len(chi2), dtype=bool) + mask2 = np.ones(len(chi2), dtype=bool) + mask3 = np.ones(len(chi2), dtype=bool) + + for i, key in enumerate(payload['sampled_keys']): + if key in ('a0', 'a1', 'a2'): + continue + + center = payload['mask_centers'][i] + error = payload['mask_errors'][i] + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + continue + + values = payload['mask_values'][:, i] + mask3 = mask3 & (values > (center - 3 * error)) & (values < (center + 3 * error)) + mask1 = mask1 & (values > (center - error)) & (values < (center + error)) + mask2 = mask2 & (values > (center - 2 * error)) & (values < (center + 2 * error)) + + if not np.any(mask1): + mask1 = np.ones(len(chi2), dtype=bool) + if not np.any(mask2): + mask2 = np.ones(len(chi2), dtype=bool) + if not np.any(mask3): + mask3 = np.ones(len(chi2), dtype=bool) + + fig = corner(payload['display_points'], + labels=payload['labels'], + bins=int(np.sqrt(payload['display_points'].shape[0])), + range=payload['ranges'], + plot_contours=True, + levels=[np.percentile(chi2[mask1], 95), np.percentile(chi2[mask2], 95), + np.percentile(chi2[mask3], 95)], + plot_density=False, + titles=payload['titles'], + data_kwargs={ + 'c': chi2, + 'vmin': np.percentile(chi2[mask3], 1), + 'vmax': np.percentile(chi2[mask3], 95), + 'cmap': 'viridis' + }, + label_kwargs={ + 'labelpad': 15, + }, + hist_kwargs={ + 'color': 'black', + } + ) return fig # simultaneously fit multiple data sets with global and local parameters diff --git a/exotic/exotic.py b/exotic/exotic.py index 31b8d838..16873797 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -132,11 +132,15 @@ try: # plots from plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ - plot_comp_star_calibration_series, plot_comp_star_suitability, plot_adaptive_aperture_diagnostics + plot_comp_star_calibration_series, plot_individual_comp_star_calibration_series, \ + plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ + plot_adaptive_aperture_diagnostics except ImportError: # package import from .plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ - plot_comp_star_calibration_series, plot_comp_star_suitability, plot_adaptive_aperture_diagnostics + plot_comp_star_calibration_series, plot_individual_comp_star_calibration_series, \ + plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ + plot_adaptive_aperture_diagnostics try: # tools from utils import round_to_2, user_input except ImportError: # package import @@ -160,6 +164,12 @@ _mid_transit_warning_reported = False RELATIVE_FLUX_MAX = 2.0 AIRMASS_FLAT_RANGE_THRESHOLD = 0.05 +LIGHTCURVE_MIN_VALID_POINTS = 5 +COMPARISON_STAR_MIN_COVERAGE_FRACTION = 0.8 +COMPARISON_STAR_MIN_VALID_FRAMES = 5 +COMPARISON_STAR_COVERAGE_SIGMA = 3.0 +COMPARISON_STAR_COVERAGE_MAX_ITERS = 10 +OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 def airmass_span(airmass): @@ -195,6 +205,26 @@ def annotate_airmass_fit(fit, airmass, skipped, max_span=AIRMASS_FLAT_RANGE_THRE fit.airmass_correction_note = "Skipped; no airmass correction applied." +def annotate_out_of_transit_baseline_detrending( + fit, + applied, + note=None, + slope=None, + intercept=None, + pre_points=0, + post_points=0, +): + if fit is None: + return + + fit.oot_baseline_detrending_applied = bool(applied) + fit.oot_baseline_detrending_note = note + fit.oot_baseline_slope = slope + fit.oot_baseline_intercept = intercept + fit.oot_baseline_pre_points = int(pre_points) if pre_points is not None else 0 + fit.oot_baseline_post_points = int(post_points) if post_points is not None else 0 + + def log_info(string, warn=False, error=False): if error: print(f"\033[31m {string}\033[0m") @@ -234,6 +264,11 @@ def relative_flux_filter_mask(relative_flux, max_relative_flux=RELATIVE_FLUX_MAX return np.isfinite(relative_flux) & np.less_equal(relative_flux, max_relative_flux) +def valid_comparison_frame_mask(flux_values): + flux_values = np.asarray(flux_values, dtype=float) + return np.isfinite(flux_values) & (flux_values > 0) + + def is_fast_aperture_mask_enabled(config_value): if config_value is None: return True @@ -288,6 +323,80 @@ def is_target_driven_comp_selection_enabled(config_value): return False +def should_skip_low_comparison_coverage_rejection(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'skip_low_comparison_coverage_rejection' value; keeping coverage rejection enabled.", + warn=True) + return False + + +def should_fit_lightcurve_to_every_comparison_candidate(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'fit_lightcurve_to_every_comparison_candidate' value; defaulting to disabled.", + warn=True) + return False + + +def should_use_psf_photometry(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'use_psf_photometry' value; keeping PSF photometry enabled.", warn=True) + return True + + +def should_use_aperture_photometry(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'use_aperture_photometry' value; keeping aperture photometry enabled.", warn=True) + return True + + def is_adaptive_aperture_mode_enabled(config_value): if config_value is None: return False @@ -373,6 +482,49 @@ def is_vertical_flux_normalization_disabled(config_value): return False +def is_out_of_transit_baseline_detrending_enabled(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'detrend_on_outoftransit_baseline' value; using default enabled setting.", + warn=True, + ) + return True + + +def should_use_impactparameter_rather_than_inclination_to_fit(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_impactparameter_rather_than_inclination_to_fit' value; " + "using impact parameter for nested fitting.", + warn=True, + ) + return True + + def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled): finite_flux = np.asarray(flux_values, dtype=float) finite_flux = finite_flux[np.isfinite(finite_flux) & (finite_flux > 0)] @@ -386,6 +538,224 @@ def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled bounds['a0'] = [0.95, 1.05] +def detrend_flux_on_out_of_transit_baseline( + times, + flux_values, + flux_errors, + fit, + depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + transit_model = np.asarray(getattr(fit, 'transit', []), dtype=float) + + if transit_model.shape != flux_values.shape: + return { + 'applied': False, + 'note': 'initial fit did not provide a transit model aligned with the light curve.', + } + + valid = ( + np.isfinite(times) + & np.isfinite(flux_values) + & (flux_values > 0) + & np.isfinite(transit_model) + ) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) + else: + flux_errors = np.ones_like(flux_values, dtype=float) + + if np.count_nonzero(valid) < 3: + return { + 'applied': False, + 'note': 'not enough finite flux points remain to fit an out-of-transit baseline.', + } + + depth = np.clip(1.0 - transit_model, 0.0, None) + max_depth = np.nanmax(depth[valid]) + if not np.isfinite(max_depth) or max_depth <= 0: + return { + 'applied': False, + 'note': 'initial fit did not produce a measurable transit depth for baseline isolation.', + } + + threshold = max(1e-6, depth_fraction * max_depth) + in_transit = valid & (depth > threshold) + if not np.any(in_transit): + return { + 'applied': False, + 'note': 'could not isolate ingress and egress from the initial fit.', + } + + ingress_time = float(np.nanmin(times[in_transit])) + egress_time = float(np.nanmax(times[in_transit])) + oot_mask = valid & ((times < ingress_time) | (times > egress_time)) + + mid_transit = float(getattr(fit, 'parameters', {}).get('tmid', np.nanmedian(times[valid]))) + pre_mask = oot_mask & (times < mid_transit) + post_mask = oot_mask & (times > mid_transit) + pre_points = int(np.count_nonzero(pre_mask)) + post_points = int(np.count_nonzero(post_mask)) + + if pre_points == 0 or post_points == 0: + return { + 'applied': False, + 'note': 'need out-of-transit coverage on both sides of transit to fit a linear baseline.', + 'pre_points': pre_points, + 'post_points': post_points, + } + + x = times[oot_mask] - mid_transit + if np.allclose(x, x[0]): + return { + 'applied': False, + 'note': 'out-of-transit timestamps do not span enough time to fit a line.', + 'pre_points': pre_points, + 'post_points': post_points, + } + + design = np.column_stack((np.ones_like(x), x)) + oot_errors = flux_errors[oot_mask] + weights = np.ones_like(x, dtype=float) + valid_weights = np.isfinite(oot_errors) & (oot_errors > 0) + if np.any(valid_weights): + weights = np.zeros_like(x, dtype=float) + weights[valid_weights] = 1.0 / (oot_errors[valid_weights] ** 2) + if not np.any(weights > 0): + weights = np.ones_like(x, dtype=float) + + sqrt_weights = np.sqrt(weights) + try: + coeffs, _, _, _ = np.linalg.lstsq(design * sqrt_weights[:, None], flux_values[oot_mask] * sqrt_weights, rcond=None) + except np.linalg.LinAlgError: + return { + 'applied': False, + 'note': 'linear out-of-transit baseline fit failed.', + 'pre_points': pre_points, + 'post_points': post_points, + } + + intercept, slope = coeffs + baseline = intercept + slope * (times - mid_transit) + if not np.all(np.isfinite(baseline)) or np.any(baseline <= 0): + return { + 'applied': False, + 'note': 'linear baseline prediction became non-physical for part of the light curve.', + 'pre_points': pre_points, + 'post_points': post_points, + } + + return { + 'applied': True, + 'note': ( + f"Applied weighted linear out-of-transit baseline detrending using " + f"{pre_points} pre-ingress and {post_points} post-egress points." + ), + 'flux': flux_values / baseline, + 'unc': flux_errors / baseline, + 'baseline': baseline, + 'slope': float(slope), + 'intercept': float(intercept), + 'pre_points': pre_points, + 'post_points': post_points, + 'ingress_time': ingress_time, + 'egress_time': egress_time, + } + + +def fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + skip_airmass_fit=False, + airmass_skip_note=None, + disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, + use_impactparameter_rather_than_inclination_to_fit=True, +): + fit = lc_fitter( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=jd_times, + mode='ns', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + ) + annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) + + if not detrend_on_outoftransit_baseline: + annotate_out_of_transit_baseline_detrending( + fit, + False, + note="Disabled; using the direct nested-sampling fit.", + ) + return fit, flux_values, flux_errors + + detrend_result = detrend_flux_on_out_of_transit_baseline(times, flux_values, flux_errors, fit) + if not detrend_result.get('applied'): + note = f"Skipped; {detrend_result.get('note', 'unable to fit an out-of-transit baseline.')}" + log_info(f"Optional out-of-transit baseline detrending skipped: {detrend_result.get('note', 'unknown reason')}") + annotate_out_of_transit_baseline_detrending( + fit, + False, + note=note, + pre_points=detrend_result.get('pre_points', 0), + post_points=detrend_result.get('post_points', 0), + ) + return fit, flux_values, flux_errors + + log_info("Applying optional out-of-transit linear baseline detrending and refitting final light curve.") + log_info(detrend_result['note']) + + refit_prior = dict(prior) + for key in ('rprs', 'tmid', 'inc', 'a2'): + if key in refit_prior and key in fit.parameters: + refit_prior[key] = fit.parameters[key] + + refit_bounds = { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + } + apply_vertical_flux_normalization_bound( + refit_prior, + refit_bounds, + detrend_result['flux'], + disable_vertical_flux_normalization, + ) + + refit = lc_fitter( + times, + detrend_result['flux'], + detrend_result['unc'], + airmass, + refit_prior, + refit_bounds, + jd_times=jd_times, + mode='ns', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + ) + annotate_airmass_fit(refit, airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_out_of_transit_baseline_detrending( + refit, + True, + note=detrend_result['note'], + slope=detrend_result['slope'], + intercept=detrend_result['intercept'], + pre_points=detrend_result['pre_points'], + post_points=detrend_result['post_points'], + ) + return refit, detrend_result['flux'], detrend_result['unc'] + + def psf_sigma_from_fit(psf_row, fallback_sigma=np.nan): try: sigx = float(psf_row[3]) @@ -504,6 +874,32 @@ def reported_photometry_aperture_radii(photometry_info): return aperture, adaptive_summary['annulus_median'] +def build_observing_background_series(psf_data, aper_data, photometry_info, comp_star_count): + use_aperture_background = photometry_info.get('min_aperture') != 0 + a_idx = photometry_info.get('aperture_index') + an_idx = photometry_info.get('annulus_index') + + if use_aperture_background and aper_data is not None and a_idx is not None and an_idx is not None: + background_series = { + 'target': np.asarray(aper_data['target_bg'][:, a_idx, an_idx], dtype=float), + } + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + bg_key = f"{ckey}_bg" + if bg_key in aper_data: + background_series[ckey] = np.asarray(aper_data[bg_key][:, a_idx, an_idx], dtype=float) + return background_series + + background_series = { + 'target': np.asarray(psf_data['target'][:, 6], dtype=float), + } + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + if ckey in psf_data: + background_series[ckey] = np.asarray(psf_data[ckey][:, 6], dtype=float) + return background_series + + def resolve_frame_aperture_radii(apertures, annuli, adaptive_apertures=False, frame_sigma=np.nan, fallback_sigma=np.nan): aperture_values = np.asarray(apertures, dtype=float) @@ -2496,7 +2892,15 @@ def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weight wx, wy = moment_fit[0], moment_fit[1] init = [moment_fit[2], moment_fit[3], moment_fit[4], moment_fit[5], moment_fit[6]] if fast_mode: - return moment_fit + if _has_usable_centroid_signal(subarray, init[0]): + return moment_fit + + plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) + log.debug( + f"Warning: Measured fast centroid amplitude is really low---" + f"are you sure there is a star at {np.round(pos, 2)}?" + ) + return _nan_psf_result() else: # compute flux weighted centroid in x and y wx = np.sum(xv[0] * subarray.sum(0)) / subarray.sum(0).sum() @@ -2539,6 +2943,8 @@ def fcn2min(pars): if weightedcenter: res.x[0] = wx res.x[1] = wy + if np.isfinite(moment_fit[6]): + res.x[6] = moment_fit[6] return res.x finally: @@ -3158,7 +3564,9 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, - allow_mid_transit_range_warning=True, disable_vertical_flux_normalization=False): + allow_mid_transit_range_warning=True, disable_vertical_flux_normalization=False, + final_fit_mode='lm', + use_impactparameter_rather_than_inclination_to_fit=True): # remove outliers si = np.argsort(times) times_sorted = times[si] @@ -3267,6 +3675,9 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, if not skip_airmass_fit: mybounds['a2'] = [-1, 1] + if arrayTimes.shape[0] < LIGHTCURVE_MIN_VALID_POINTS: + return None, None, None + if np.isnan(arrayTimes).any() or np.isnan(arrayFinalFlux).any() or np.isnan(arrayNormUnc).any(): log_info("\nWarning: NANs in time, flux or error", warn=True) @@ -3278,7 +3689,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, prior, mybounds, jd_times=arrayJDTimes, - mode='lm' + mode='lm', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) @@ -3308,13 +3720,146 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, prior, mybounds, jd_times=arrayJDTimes, - mode='lm' + mode='lm', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) + if final_fit_mode == 'ns' and myfit is not None: + myfit = lc_fitter( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + arrayAirmass, + prior, + mybounds, + jd_times=arrayJDTimes, + mode='ns', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + ) + annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) + return myfit, f1, f2 +def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass): + times = np.asarray(times, dtype=float) + tflux = np.asarray(tflux, dtype=float) + cflux = np.asarray(cflux, dtype=float) + airmass = np.asarray(airmass, dtype=float) + + diagnostics = { + 'input_point_count': int(times.shape[0]), + 'has_reference_flux': False, + 'relative_flux_point_count': 0, + 'sigma_clip_point_count': 0, + 'usable_point_count': 0, + 'failed_stage': None, + 'failure_reason': None, + } + + if diagnostics['input_point_count'] <= 1: + diagnostics.update({ + 'relative_flux_point_count': diagnostics['input_point_count'], + 'failed_stage': 'coverage', + 'failure_reason': ( + f"only {diagnostics['input_point_count']} frame(s) remained after masking invalid " + "comparison flux; need at least 2 to fit." + ), + }) + return diagnostics + + si = np.argsort(times) + times_sorted = times[si] + tflux_sorted = tflux[si] + cflux_sorted = cflux[si] + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio_sorted = np.divide(tflux_sorted, cflux_sorted) + + has_reference_flux = not np.allclose(cflux_sorted, 1.0) + diagnostics['has_reference_flux'] = bool(has_reference_flux) + diagnostics['relative_flux_point_count'] = int(times_sorted.shape[0]) + + if has_reference_flux: + relative_flux_mask = relative_flux_filter_mask(flux_ratio_sorted) + diagnostics['relative_flux_point_count'] = int(np.count_nonzero(relative_flux_mask)) + times_sorted = times_sorted[relative_flux_mask] + tflux_sorted = tflux_sorted[relative_flux_mask] + cflux_sorted = cflux_sorted[relative_flux_mask] + flux_ratio_sorted = flux_ratio_sorted[relative_flux_mask] + airmass_sorted = airmass[si][relative_flux_mask] + if diagnostics['relative_flux_point_count'] <= 1: + diagnostics.update({ + 'failed_stage': 'relative_flux_filter', + 'failure_reason': ( + "relative-flux filtering left " + f"{diagnostics['relative_flux_point_count']} usable point(s); invalid, non-finite, " + "or >2x target/reference ratios were rejected." + ), + }) + return diagnostics + else: + airmass_sorted = airmass[si] + + dt = np.mean(np.diff(times_sorted)) + if np.isfinite(dt) and dt > 0: + ndt = int(25. / 24. / 60. / dt) * 2 + 1 + else: + ndt = 5 + if ndt > len(times_sorted): + ndt = int(len(times_sorted) / 4) * 2 + 1 + filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt)) + valid_mask = ~filtered_data + diagnostics['sigma_clip_point_count'] = int(np.count_nonzero(valid_mask)) + if diagnostics['sigma_clip_point_count'] <= 1: + diagnostics.update({ + 'failed_stage': 'sigma_clip', + 'failure_reason': ( + "sigma clipping left " + f"{diagnostics['sigma_clip_point_count']} usable point(s); not enough data remained " + "for a lightcurve fit." + ), + }) + return diagnostics + + arrayFinalFlux = flux_ratio_sorted[valid_mask] + f1 = tflux_sorted[valid_mask] + sigf1 = f1 ** 0.5 + f2 = cflux_sorted[valid_mask] + sigf2 = f2 ** 0.5 + if np.sum(cflux) == len(cflux): + arrayNormUnc = sigf1 + else: + arrayNormUnc = np.sqrt((sigf1 / f2) ** 2 + (sigf2 * f1 / f2 ** 2) ** 2) + arrayTimes = times_sorted[valid_mask] + arrayAirmass = airmass_sorted[valid_mask] + + nanmask = np.isnan(arrayFinalFlux) | np.isnan(arrayNormUnc) | np.isnan(arrayTimes) | np.isnan(arrayAirmass) + nanmask = nanmask | np.less_equal(arrayFinalFlux, 0) | np.less_equal(arrayNormUnc, 0) + nanmask = nanmask | np.isinf(arrayFinalFlux) | np.isinf(arrayNormUnc) | np.isinf(arrayTimes) | np.isinf( + arrayAirmass + ) + diagnostics['usable_point_count'] = int(np.count_nonzero(~nanmask)) + if diagnostics['usable_point_count'] <= 1: + diagnostics.update({ + 'failed_stage': 'nan_filter', + 'failure_reason': ( + "filtering non-finite or non-positive flux/uncertainty values left " + f"{diagnostics['usable_point_count']} usable point(s); need at least 2." + ), + }) + elif diagnostics['usable_point_count'] < LIGHTCURVE_MIN_VALID_POINTS: + diagnostics.update({ + 'failed_stage': 'minimum_points', + 'failure_reason': ( + f"only {diagnostics['usable_point_count']} usable point(s) remained after filtering; " + f"need at least {LIGHTCURVE_MIN_VALID_POINTS} for a lightcurve fit." + ), + }) + + return diagnostics + + def cheap_lightcurve_prescore(tFlux, cFlux, airmass): with np.errstate(divide='ignore', invalid='ignore'): flux_ratio = np.divide(tFlux, cFlux) @@ -3340,7 +3885,17 @@ def cheap_lightcurve_prescore(tFlux, cFlux, airmass): def evaluate_lightcurve_candidate(task): - times, tflux, cflux, airmass, ld, p_dict, jd_times, disable_vertical_flux_normalization = task + ( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + ) = task myfit, tflux_fit, cflux_fit = fit_lightcurve( times, tflux, @@ -3351,6 +3906,7 @@ def evaluate_lightcurve_candidate(task): jd_times, allow_mid_transit_range_warning=False, disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) if myfit is None: return None, tflux_fit, cflux_fit @@ -3362,6 +3918,293 @@ def evaluate_lightcurve_candidate(task): }, tflux_fit, cflux_fit +def selected_photometry_method_label(photometry_info): + min_aperture = photometry_info.get('min_aperture') + min_annulus = photometry_info.get('min_annulus') + + if min_aperture == 0: + return "PSF photometry" + if min_aperture is None: + return "Photometry" + + aper_text = abs(float(min_aperture)) + if min_annulus is None or not np.isfinite(min_annulus): + return f"Aperture photometry (aper={aper_text:.2f}px)" + return f"Aperture photometry (aper={aper_text:.2f}px, annulus={float(min_annulus):.2f}px)" + + +def format_comp_star_position(position): + if position is None: + return "x=n/a, y=n/a" + + try: + x_pos, y_pos = position + return f"x={float(x_pos):.1f}, y={float(y_pos):.1f}" + except (TypeError, ValueError): + return f"coords={position}" + + +def comparison_calibration_selection_reason(summary, best_comp_score): + if summary.get('selected'): + return "selected: lowest suitability score among coverage-qualified comparison stars for this method" + + if summary.get('coverage_rejected'): + return ( + "not selected: low coverage " + f"({summary['coverage_count']} < {summary['coverage_min_required_count']} valid frames)" + ) + + aggregate_score = summary.get('aggregate_score', np.inf) + if not np.isfinite(aggregate_score): + return "not selected: no usable ensemble or pairwise calibration score" + + if np.isfinite(best_comp_score): + score_gap = aggregate_score - best_comp_score + if np.isfinite(score_gap) and score_gap > 0: + return ( + "not selected: suitability score was " + f"{score_gap * 100.0:.4f}% above the selected comparison star" + ) + + return "not selected: another comparison star ranked better for this photometry method" + + +def comparison_candidate_fit_selection_reason(summary, photometry_info): + if summary.get('failure_reason'): + return summary['failure_reason'] + + selection_basis = photometry_info.get('selection_basis', 'target_fit') + selected_comp_num = photometry_info.get('comp_star_num') + selected_res_std = photometry_info.get('min_std', np.inf) + candidate_res_std = summary.get('res_std', np.inf) + + if summary.get('selected'): + if selection_basis == 'comparison_field': + return "selected: comparison-field calibration ranked this star best for the chosen photometry method" + return "selected: lowest target-fit residual scatter in the chosen search" + + if selection_basis == 'comparison_field': + if selected_comp_num is None: + return "not selected: comparison-field calibration chose a different candidate" + return f"not selected: comparison-field calibration chose Comp {selected_comp_num}" + + if np.isfinite(candidate_res_std) and np.isfinite(selected_res_std): + if candidate_res_std > selected_res_std + 1e-12: + return ( + "not selected: residual scatter was " + f"{candidate_res_std * 100.0:.4f}% vs {selected_res_std * 100.0:.4f}% for the selected fit" + ) + if candidate_res_std < selected_res_std - 1e-12: + return ( + "not selected: this post-selection diagnostic fit looks better than the selected fit; " + "the earlier target-fit search did not choose it" + ) + + if selected_comp_num is None: + return "not selected: another candidate remained preferred in the target-fit search" + return f"not selected: Comp {selected_comp_num} remained preferred in the target-fit search" + + +def format_fit_parameter_with_uncertainty(value, error=None, scale=1.0, suffix=""): + if value is None or not np.isfinite(value): + return "n/a" + + scaled_value = float(value) * scale + if error is None or not np.isfinite(error) or error < 0: + return f"{round_to_2(scaled_value)}{suffix}" + + scaled_error = float(error) * abs(scale) + return f"{round_to_2(scaled_value, scaled_error)} +/- {round_to_2(scaled_error)}{suffix}" + + +def summarize_lightcurve_fit_parameters(fit): + if fit is None or not hasattr(fit, 'parameters'): + return None + + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + fit_method = getattr(fit, 'ns_type', 'lm') + summary_parts = [ + f"fit_method={fit_method}", + f"Tmid={format_fit_parameter_with_uncertainty(parameters.get('tmid'), errors.get('tmid'))}", + f"Rp/R*={format_fit_parameter_with_uncertainty(parameters.get('rprs'), errors.get('rprs'))}", + ] + + rprs = parameters.get('rprs') + rprs_err = errors.get('rprs') + depth = None if rprs is None else 100.0 * float(rprs) ** 2 + depth_err = None + if rprs is not None and rprs_err is not None and np.isfinite(rprs) and np.isfinite(rprs_err): + depth_err = 200.0 * float(rprs) * float(rprs_err) + summary_parts.append(f"depth={format_fit_parameter_with_uncertainty(depth, depth_err, suffix='%')}") + summary_parts.append(f"inc={format_fit_parameter_with_uncertainty(parameters.get('inc'), errors.get('inc'))}") + + if getattr(fit, 'airmass_fit_skipped', False): + summary_parts.append("airmass=skipped") + else: + baseline_key = 'a0' if 'a0' in parameters else 'a1' + summary_parts.append( + f"{baseline_key}={format_fit_parameter_with_uncertainty(parameters.get(baseline_key), errors.get(baseline_key))}" + ) + summary_parts.append( + f"a2={format_fit_parameter_with_uncertainty(parameters.get('a2'), errors.get('a2'))}" + ) + + return ", ".join(summary_parts) + + +def log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_info): + if not candidate_fit_summaries: + return + + selection_basis = photometry_info.get('selection_basis', 'target_fit').replace('_', '-') + log_info("\nComparison-star lightcurve fit diagnostics:") + log_info(f"Selection basis: {selection_basis}") + + for summary in candidate_fit_summaries: + selected_label = " [selected]" if summary.get('selected') else "" + position_text = format_comp_star_position(summary.get('position')) + diagnostics = summary.get('fit_diagnostics') or {} + usable_point_count = diagnostics.get('usable_point_count', 0) + coverage_median = summary.get('coverage_reference_count', np.nan) + coverage_text = ( + f"{summary['coverage_count']}/{summary.get('coverage_min_required_count', 0)} valid frame(s)" + ) + if np.isfinite(coverage_median): + coverage_text += f", peer_median={coverage_median:.1f}" + residual_text = "n/a" + if summary.get('fit') is not None and np.isfinite(summary.get('res_std', np.inf)): + residual_text = f"{summary['res_std'] * 100.0:.4f}%" + reason_text = comparison_candidate_fit_selection_reason(summary, photometry_info) + log_info( + f" {summary['label']}{selected_label} ({position_text}): " + f"coverage={coverage_text}, " + f"usable_after_filters={usable_point_count}, fit_points={summary['fit_point_count']}, " + f"residual_scatter={residual_text}, reason={reason_text}" + ) + parameter_summary = summary.get('parameter_summary') + if parameter_summary: + log_info(f" parameters: {parameter_summary}") + + +def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p_dict, comp_stars, + psf_data, aper_data, photometry_info, + disable_vertical_flux_normalization=False, + skip_low_comparison_coverage_rejection=False, + use_impactparameter_rather_than_inclination_to_fit=True): + if photometry_info.get('best_fit_lc') is None or not comp_stars: + return [] + + use_psf_photometry = photometry_info.get('min_aperture') == 0 + if use_psf_photometry: + target_flux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + comp_flux_map = { + f"comp{comp_index + 1}": 2 * np.pi * psf_data[f"comp{comp_index + 1}"][:, 2] + * psf_data[f"comp{comp_index + 1}"][:, 3] + * psf_data[f"comp{comp_index + 1}"][:, 4] + for comp_index in range(len(comp_stars)) + } + else: + aperture_index = photometry_info.get('aperture_index') + annulus_index = photometry_info.get('annulus_index') + if aperture_index is None or annulus_index is None: + return [] + target_flux = aper_data['target'][:, aperture_index, annulus_index] + comp_flux_map = { + f"comp{comp_index + 1}": aper_data[f"comp{comp_index + 1}"][:, aperture_index, annulus_index] + for comp_index in range(len(comp_stars)) + } + + candidate_fit_summaries = [] + selected_comp_star_num = photometry_info.get('comp_star_num') + coverage_summary = comparison_star_coverage_summary( + comp_flux_map, + skip_rejection=skip_low_comparison_coverage_rejection, + ) + + for comp_index, position in enumerate(comp_stars): + label = f"Comp {comp_index + 1}" + ckey = f"comp{comp_index + 1}" + comp_flux_series = comp_flux_map[ckey] + + fit_mask = valid_comparison_frame_mask(comp_flux_series) + coverage_count = coverage_summary[ckey]['coverage_count'] + coverage_reference_count = coverage_summary[ckey]['coverage_reference_count'] + coverage_min_required_count = coverage_summary[ckey]['coverage_min_required_count'] + coverage_rejected = coverage_summary[ckey]['coverage_rejected'] + fit_result, target_fit_flux, comp_fit_flux = None, None, None + fit_diagnostics = { + 'input_point_count': int(times.shape[0]), + 'has_reference_flux': True, + 'relative_flux_point_count': 0, + 'sigma_clip_point_count': 0, + 'usable_point_count': 0, + 'failed_stage': 'coverage', + 'failure_reason': ( + f"only {coverage_count} frame(s) had finite positive comparison flux; need at least 2 to fit." + ), + } + if coverage_rejected: + fit_diagnostics['failure_reason'] = ( + "comparison candidate rejected after iterative low-coverage clipping " + f"({coverage_count} < {coverage_min_required_count} valid frame(s); " + f"peer median={coverage_reference_count:.1f})." + ) + elif coverage_count > 1: + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + target_flux[fit_mask], + comp_flux_series[fit_mask], + airmass[fit_mask], + ) + if not coverage_rejected and coverage_count > 1 and fit_diagnostics['failure_reason'] is None: + fit_result, target_fit_flux, comp_fit_flux = fit_lightcurve( + times[fit_mask], + target_flux[fit_mask], + comp_flux_series[fit_mask], + airmass[fit_mask], + ld, + p_dict, + jd_times[fit_mask], + allow_mid_transit_range_warning=False, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + final_fit_mode='ns', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + ) + if fit_result is None: + fit_diagnostics.update({ + 'failed_stage': 'nested_fit', + 'failure_reason': "the nested lightcurve fitter did not converge to a usable solution.", + }) + + res_std = np.inf + fit_point_count = 0 if target_fit_flux is None else int(len(target_fit_flux)) + if fit_result is not None and hasattr(fit_result, 'residuals') and hasattr(fit_result, 'data'): + with np.errstate(divide='ignore', invalid='ignore'): + res_std = float(np.std(fit_result.residuals / np.median(fit_result.data))) + parameter_summary = summarize_lightcurve_fit_parameters(fit_result) + + candidate_fit_summaries.append({ + 'comp_index': comp_index, + 'label': label, + 'position': position, + 'selected': selected_comp_star_num == comp_index + 1, + 'fit': fit_result, + 'res_std': res_std, + 'coverage_count': coverage_count, + 'coverage_reference_count': coverage_reference_count, + 'coverage_min_required_count': coverage_min_required_count, + 'coverage_rejected': coverage_rejected, + 'fit_point_count': fit_point_count, + 'fit_diagnostics': fit_diagnostics, + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'fit_method': None if fit_result is None else getattr(fit_result, 'ns_type', 'lm'), + 'parameter_summary': parameter_summary, + }) + + return candidate_fit_summaries + + def normalize_flux_series(flux_values): flux_values = np.asarray(flux_values, dtype=float) normalized = np.full(flux_values.shape, np.nan, dtype=float) @@ -3403,7 +4246,63 @@ def build_normalized_comp_ensemble(normalized_flux_map, exclude_key): return ensemble -def comparison_star_stability_summary(comp_flux_map, airmass): +def comparison_star_coverage_summary(comp_flux_map, + min_fraction=COMPARISON_STAR_MIN_COVERAGE_FRACTION, + min_points=COMPARISON_STAR_MIN_VALID_FRAMES, + skip_rejection=False): + comp_keys = list(comp_flux_map.keys()) + if not comp_keys: + return {} + + coverage_counts = { + key: int(np.count_nonzero(valid_comparison_frame_mask(comp_flux_map[key]))) + for key in comp_keys + } + total_frame_count = max(np.asarray(comp_flux_map[key]).shape[0] for key in comp_keys) + effective_min_points = int(min_points) if total_frame_count >= int(min_points) else 0 + active_keys = list(comp_keys) + coverage_reference_count = float(np.nanmedian([coverage_counts[key] for key in active_keys])) + coverage_min_required_count = max(effective_min_points, 0) + coverage_scatter = np.nan + + for _ in range(COMPARISON_STAR_COVERAGE_MAX_ITERS): + active_counts = np.asarray([coverage_counts[key] for key in active_keys], dtype=float) + if active_counts.size == 0: + break + + coverage_reference_count = float(np.nanmedian(active_counts)) + coverage_scatter = robust_scatter(active_counts) + threshold_candidates = [ + effective_min_points, + int(np.ceil(float(min_fraction) * coverage_reference_count)), + ] + if np.isfinite(coverage_scatter) and coverage_scatter > 0: + threshold_candidates.append( + int(np.ceil(coverage_reference_count - COMPARISON_STAR_COVERAGE_SIGMA * coverage_scatter)) + ) + coverage_min_required_count = max(threshold_candidates) + + kept_keys = [key for key in active_keys if coverage_counts[key] >= coverage_min_required_count] + if len(kept_keys) == len(active_keys): + break + active_keys = kept_keys + + coverage_summary = {} + active_key_set = set(active_keys) + for key in comp_keys: + coverage_summary[key] = { + 'coverage_count': coverage_counts[key], + 'coverage_reference_count': coverage_reference_count, + 'coverage_median_count': coverage_reference_count, + 'coverage_scatter': coverage_scatter, + 'coverage_min_required_count': coverage_min_required_count, + 'coverage_rejected': False if skip_rejection else key not in active_key_set, + } + + return coverage_summary + + +def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_rejection=False): if not comp_flux_map: return { 'pairwise_matrix': np.empty((0, 0), dtype=float), @@ -3415,6 +4314,14 @@ def comparison_star_stability_summary(comp_flux_map, airmass): comp_keys = list(comp_flux_map.keys()) normalized_flux_map = {key: normalize_flux_series(comp_flux_map[key]) for key in comp_keys} + coverage_summary = comparison_star_coverage_summary( + comp_flux_map, + skip_rejection=skip_low_coverage_rejection, + ) + eligible_keys = { + key for key in comp_keys + if not coverage_summary[key]['coverage_rejected'] + } pairwise_matrix = np.full((len(comp_keys), len(comp_keys)), np.nan, dtype=float) comp_summaries = [] @@ -3425,7 +4332,7 @@ def comparison_star_stability_summary(comp_flux_map, airmass): pairwise_series = {} for j, other_key in enumerate(comp_keys): - if i == j: + if i == j or other_key not in eligible_keys: continue other_flux = normalized_flux_map[other_key] score = cheap_lightcurve_prescore(normalized_flux, other_flux, airmass) @@ -3434,7 +4341,8 @@ def comparison_star_stability_summary(comp_flux_map, airmass): if np.isfinite(score): pairwise_scores.append(float(score)) - ensemble_flux = build_normalized_comp_ensemble(normalized_flux_map, key) + eligible_flux_map = {eligible_key: normalized_flux_map[eligible_key] for eligible_key in eligible_keys} + ensemble_flux = build_normalized_comp_ensemble(eligible_flux_map, key) ensemble_score = np.inf ensemble_ratio_series = np.full(normalized_flux.shape, np.nan, dtype=float) if ensemble_flux is not None: @@ -3452,6 +4360,8 @@ def comparison_star_stability_summary(comp_flux_map, airmass): aggregate_inputs = [score for score in (ensemble_score, pairwise_upper) if np.isfinite(score)] aggregate_score = max(aggregate_inputs) if aggregate_inputs else self_score + if coverage_summary[key]['coverage_rejected']: + aggregate_score = np.inf comp_summaries.append({ 'comp_index': i, @@ -3465,6 +4375,10 @@ def comparison_star_stability_summary(comp_flux_map, airmass): 'valid_pair_count': len(pairwise_scores), 'pairwise_ratio_series': pairwise_series, 'ensemble_ratio_series': ensemble_ratio_series, + 'coverage_count': coverage_summary[key]['coverage_count'], + 'coverage_reference_count': coverage_summary[key]['coverage_reference_count'], + 'coverage_min_required_count': coverage_summary[key]['coverage_min_required_count'], + 'coverage_rejected': coverage_summary[key]['coverage_rejected'], }) finite_comp_scores = [summary['aggregate_score'] for summary in comp_summaries if np.isfinite(summary['aggregate_score'])] @@ -3611,7 +4525,8 @@ def _refined_sigma_grid(center, lower_bound, upper_bound, half_width, points): def auto_tune_aperture_sigma_grid(coarse_apertures_sigma, coarse_annuli_sigma, coarse_aper_data, comp_star_count, - subset_airmass, require_comp_star=True): + subset_airmass, require_comp_star=True, + skip_low_comparison_coverage_rejection=False): best_candidate = None best_score = np.inf @@ -3621,7 +4536,11 @@ def auto_tune_aperture_sigma_grid(coarse_apertures_sigma, coarse_annuli_sigma, c f"comp{comp_idx + 1}": coarse_aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx] for comp_idx in range(comp_star_count) } - field_summary = comparison_star_stability_summary(comp_flux_map, subset_airmass) + field_summary = comparison_star_stability_summary( + comp_flux_map, + subset_airmass, + skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, + ) field_score = field_summary['field_score'] if np.isfinite(field_score) and comparison_field_sort_key(field_summary) < (best_score, np.inf): best_score = field_score @@ -3672,44 +4591,57 @@ def comparison_method_label(candidate): return f"Aperture photometry (aper={candidate['aper']:.2f}px, annulus={candidate['annulus']:.2f}px)" -def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, airmass, comp_stars, sigma): +def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, airmass, comp_stars, sigma, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=True): candidate_summaries = [] comp_star_count = len(comp_stars) if comp_star_count == 0: return None - psf_flux_map = { - f"comp{comp_idx + 1}": 2 * np.pi * psf_data[f"comp{comp_idx + 1}"][:, 2] - * psf_data[f"comp{comp_idx + 1}"][:, 3] - * psf_data[f"comp{comp_idx + 1}"][:, 4] - for comp_idx in range(comp_star_count) - } - psf_summary = comparison_star_stability_summary(psf_flux_map, airmass) - psf_summary.update({ - 'method': 'psf', - 'a': None, - 'an': None, - 'aper': 0.0, - 'annulus': float(15 * sigma), - }) - candidate_summaries.append(psf_summary) - - for a_idx, aperture in enumerate(apers): - for an_idx, annulus in enumerate(annuli): - comp_flux_map = { - f"comp{comp_idx + 1}": aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx] - for comp_idx in range(comp_star_count) - } - candidate_summary = comparison_star_stability_summary(comp_flux_map, airmass) - candidate_summary.update({ - 'method': 'aperture', - 'a': a_idx, - 'an': an_idx, - 'aper': float(aperture), - 'annulus': float(annulus), - }) - candidate_summaries.append(candidate_summary) + if use_psf_photometry: + psf_flux_map = { + f"comp{comp_idx + 1}": 2 * np.pi * psf_data[f"comp{comp_idx + 1}"][:, 2] + * psf_data[f"comp{comp_idx + 1}"][:, 3] + * psf_data[f"comp{comp_idx + 1}"][:, 4] + for comp_idx in range(comp_star_count) + } + psf_summary = comparison_star_stability_summary( + psf_flux_map, + airmass, + skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, + ) + psf_summary.update({ + 'method': 'psf', + 'a': None, + 'an': None, + 'aper': 0.0, + 'annulus': float(15 * sigma), + }) + candidate_summaries.append(psf_summary) + + if use_aperture_photometry and aper_data is not None and apers is not None and annuli is not None: + for a_idx, aperture in enumerate(apers): + for an_idx, annulus in enumerate(annuli): + comp_flux_map = { + f"comp{comp_idx + 1}": aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx] + for comp_idx in range(comp_star_count) + } + candidate_summary = comparison_star_stability_summary( + comp_flux_map, + airmass, + skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, + ) + candidate_summary.update({ + 'method': 'aperture', + 'a': a_idx, + 'an': an_idx, + 'aper': float(aperture), + 'annulus': float(annulus), + }) + candidate_summaries.append(candidate_summary) finite_candidates = [ candidate for candidate in candidate_summaries @@ -3727,6 +4659,10 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, comp_summary = dict(summary) comp_summary['position'] = comp_stars[comp_summary['comp_index']] comp_summary['selected'] = comp_summary['comp_index'] == best_comp_index + comp_summary['selection_reason'] = comparison_calibration_selection_reason( + comp_summary, + best_candidate['best_comp_score'], + ) comp_summaries.append(comp_summary) best_candidate['comp_summaries'] = comp_summaries @@ -3927,6 +4863,14 @@ def main(): disable_vertical_flux_normalization = is_vertical_flux_normalization_disabled( exotic_infoDict.get('disable_vertical_flux_normalization', False) ) + detrend_on_outoftransit_baseline = is_out_of_transit_baseline_detrending_enabled( + exotic_infoDict.get('detrend_on_outoftransit_baseline', True) + ) + use_impactparameter_rather_than_inclination_to_fit = ( + should_use_impactparameter_rather_than_inclination_to_fit( + exotic_infoDict.get('use_impactparameter_rather_than_inclination_to_fit', 'y') + ) + ) # Make a temp directory of helpful files Path(Path(exotic_infoDict['save']) / "temp").mkdir(exist_ok=True) @@ -4157,9 +5101,30 @@ def main(): target_driven_comp_selection = is_target_driven_comp_selection_enabled( exotic_infoDict.get('target_driven_comp_selection', 'n') ) + skip_low_comp_coverage_rejection = should_skip_low_comparison_coverage_rejection( + exotic_infoDict.get('skip_low_comparison_coverage_rejection', 'n') + ) + if skip_low_comp_coverage_rejection: + log_info("Skipping low-coverage comparison-star rejection per optional_info setting.") + fit_every_comparison_candidate = should_fit_lightcurve_to_every_comparison_candidate( + exotic_infoDict.get('fit_lightcurve_to_every_comparison_candidate', 'n') + ) + use_psf_photometry = should_use_psf_photometry( + exotic_infoDict.get('use_psf_photometry', 'y') + ) + use_aperture_photometry = should_use_aperture_photometry( + exotic_infoDict.get('use_aperture_photometry', 'y') + ) use_adaptive_apertures = is_adaptive_aperture_mode_enabled( exotic_infoDict.get('use_adaptive_apertures', False) ) + if not use_psf_photometry and not use_aperture_photometry: + log_info("Error: both PSF and aperture photometry are disabled in optional_info.", error=True) + return + if not use_psf_photometry: + log_info("PSF photometry disabled per optional_info setting.") + if not use_aperture_photometry: + log_info("Aperture photometry disabled per optional_info setting.") for i, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{i + 1}" @@ -4168,18 +5133,30 @@ def main(): psf_data[ckey] = np.zeros((len(inputfiles), 7)) tar_comp_dist[ckey] = np.zeros(2) - coarse_tune_frames = min(len(inputfiles), APERTURE_AUTOTUNE_MAX_FRAMES) - if len(inputfiles) >= APERTURE_AUTOTUNE_MIN_FRAMES: - coarse_tune_frames = max(APERTURE_AUTOTUNE_MIN_FRAMES, coarse_tune_frames) - coarse_apertures_sigma = np.linspace(APERTURE_SIGMA_MIN, APERTURE_SIGMA_MAX, APERTURE_AUTOTUNE_COARSE_APER_POINTS) - coarse_annuli_sigma = np.linspace(ANNULUS_SIGMA_MIN, ANNULUS_SIGMA_MAX, APERTURE_AUTOTUNE_COARSE_ANNULUS_POINTS) - log_info( - "Automatic aperture tuning enabled: " - f"coarse_grid={len(coarse_apertures_sigma)}x{len(coarse_annuli_sigma)}, " - f"coarse_frames={coarse_tune_frames}." - ) + coarse_tune_frames = 0 + coarse_apertures_sigma = None + coarse_annuli_sigma = None + if use_aperture_photometry: + coarse_tune_frames = min(len(inputfiles), APERTURE_AUTOTUNE_MAX_FRAMES) + if len(inputfiles) >= APERTURE_AUTOTUNE_MIN_FRAMES: + coarse_tune_frames = max(APERTURE_AUTOTUNE_MIN_FRAMES, coarse_tune_frames) + coarse_apertures_sigma = np.linspace( + APERTURE_SIGMA_MIN, + APERTURE_SIGMA_MAX, + APERTURE_AUTOTUNE_COARSE_APER_POINTS, + ) + coarse_annuli_sigma = np.linspace( + ANNULUS_SIGMA_MIN, + ANNULUS_SIGMA_MAX, + APERTURE_AUTOTUNE_COARSE_ANNULUS_POINTS, + ) + log_info( + "Automatic aperture tuning enabled: " + f"coarse_grid={len(coarse_apertures_sigma)}x{len(coarse_annuli_sigma)}, " + f"coarse_frames={coarse_tune_frames}." + ) - sigma = None + sigma = np.nan coarse_aperture_values = None coarse_annulus_values = None aperture_values = None @@ -4187,14 +5164,16 @@ def main(): apers = None annuli = None aperture_grid_tuned = False - coarse_aper_data = initialize_aperture_data_store( - coarse_tune_frames, - len(coarse_apertures_sigma), - len(coarse_annuli_sigma), - comp_star_count, - ) + coarse_aper_data = None + if use_aperture_photometry: + coarse_aper_data = initialize_aperture_data_store( + coarse_tune_frames, + len(coarse_apertures_sigma), + len(coarse_annuli_sigma), + comp_star_count, + ) aper_data = None - coarse_frame_cache = [None] * coarse_tune_frames + coarse_frame_cache = [None] * coarse_tune_frames if use_aperture_photometry else [] use_multiprocess_transform_precompute = should_use_multiprocess_transform_precompute( inputfiles, args.multiprocess_transformations, ignore_header_wcs=ignore_header_wcs @@ -4211,7 +5190,7 @@ def main(): dtype=float, ) fast_aperture_mask = is_fast_aperture_mask_enabled(exotic_infoDict.get('fast_aperture_mask')) - if use_adaptive_apertures: + if use_aperture_photometry and use_adaptive_apertures: log_info("Adaptive aperture scaling enabled: evaluating aperture candidates in PSF sigma units per frame.") # open files, calibrate, align, photometry @@ -4221,11 +5200,16 @@ def main(): plateStatus.setCurrentFilename(fileName) hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) - frame_fast_centroid = should_use_fast_centroid(i) - target_fast_centroid = should_use_fast_target_centroid( - i, - adaptive_apertures=use_adaptive_apertures, - ) + if use_psf_photometry: + # Keep PSF photometry on one consistent measurement path for reduction frames. + frame_fast_centroid = False + target_fast_centroid = False + else: + frame_fast_centroid = should_use_fast_centroid(i) + target_fast_centroid = should_use_fast_target_centroid( + i, + adaptive_apertures=use_adaptive_apertures, + ) extension = 0 image_header = hdul[extension].header @@ -4368,7 +5352,7 @@ def main(): tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) # aperture photometry - if i == 0: + if use_aperture_photometry and i == 0: sigma = psf_sigma_from_fit(psf_data['target'][0]) if not np.isfinite(sigma) or sigma <= 0: log_info("Warning: Initial PSF sigma is invalid; using sigma=1.0 for automatic aperture tuning.", warn=True) @@ -4380,7 +5364,7 @@ def main(): coarse_aperture_values = coarse_apertures_sigma * sigma coarse_annulus_values = coarse_annuli_sigma * sigma - if i < coarse_tune_frames: + if use_aperture_photometry and i < coarse_tune_frames: coarse_frame_cache[i] = np.array(imageData, copy=True) populate_aperture_data_for_frame( imageData, @@ -4404,6 +5388,7 @@ def main(): comp_star_count, subset_airmass, require_comp_star=require_comp_star, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, ) if use_adaptive_apertures: aperture_values = refined_apertures_sigma @@ -4459,7 +5444,7 @@ def main(): if loaded_from_disk: del backfill_image coarse_frame_cache[backfill_idx] = None - else: + elif use_aperture_photometry: if not aperture_grid_tuned: # Defensive fallback for unexpected control flow. aperture_values = coarse_aperture_values @@ -4496,8 +5481,9 @@ def main(): log_reduction_timing_overview('Reduction timing overview (full reduction)') # filter bad images - badmask = np.isnan(psf_data["target"][:, 0]) | (psf_data["target"][:, 0] == 0) | (aper_data["target"][:, 0, 0] == 0) | np.isnan( - aper_data["target"][:, 0, 0]) + badmask = np.isnan(psf_data["target"][:, 0]) | (psf_data["target"][:, 0] == 0) + if aper_data is not None: + badmask = badmask | (aper_data["target"][:, 0, 0] == 0) | np.isnan(aper_data["target"][:, 0, 0]) goodmask = ~badmask if np.sum(goodmask) == 0: log_info("No images to fit...check reference image for alignment (first image of sequence)") @@ -4507,13 +5493,15 @@ def main(): jd_times = jd_times[goodmask] airmass = np.array(airMassList)[goodmask] psf_data["target"] = psf_data["target"][goodmask] - aper_data["target"] = aper_data["target"][goodmask] - aper_data["target_bg"] = aper_data["target_bg"][goodmask] + if aper_data is not None: + aper_data["target"] = aper_data["target"][goodmask] + aper_data["target_bg"] = aper_data["target_bg"][goodmask] for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" psf_data[ckey] = psf_data[ckey][goodmask] - aper_data[ckey] = aper_data[ckey][goodmask] - aper_data[f"{ckey}_bg"] = aper_data[f"{ckey}_bg"][goodmask] + if aper_data is not None: + aper_data[ckey] = aper_data[ckey][goodmask] + aper_data[f"{ckey}_bg"] = aper_data[f"{ckey}_bg"][goodmask] sigma_display = representative_psf_sigma(psf_data['target'], fallback_sigma=sigma) if not np.isfinite(sigma_display) or sigma_display <= 0: @@ -4582,6 +5570,9 @@ def main(): airmass, exotic_infoDict['comp_stars'], sigma_display, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=use_psf_photometry, + use_aperture_photometry=use_aperture_photometry, ) if comparison_calibration is not None: @@ -4593,10 +5584,18 @@ def main(): ensemble_text = "n/a" if not np.isfinite(summary['ensemble_score']) else f"{summary['ensemble_score'] * 100.0:.4f}%" pairwise_text = "n/a" if not np.isfinite(summary['pairwise_median_score']) else f"{summary['pairwise_median_score'] * 100.0:.4f}%" selected_label = " [selected]" if summary['selected'] else "" + position_text = format_comp_star_position(summary['position']) + coverage_text = ( + f"coverage={summary['coverage_count']}/{summary['coverage_min_required_count']} " + f"(peer_median={summary['coverage_reference_count']:.1f})" + ) + if summary['coverage_rejected']: + coverage_text += " [rejected: low coverage]" log_info( - f" {summary['label']}{selected_label}: suitability={aggregate_text}, " + f" {summary['label']}{selected_label} ({position_text}): suitability={aggregate_text}, " f"ensemble={ensemble_text}, pairwise_median={pairwise_text}, " - f"valid_pairs={summary['valid_pair_count']}" + f"valid_pairs={summary['valid_pair_count']}, {coverage_text}, " + f"reason={summary['selection_reason']}" ) try: @@ -4616,6 +5615,14 @@ def main(): exotic_infoDict['date'], comparison_calibration['method_label'], ) + plot_individual_comp_star_calibration_series( + times, + comparison_calibration['comp_summaries'], + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + ) plot_comp_star_suitability( comparison_calibration['comp_summaries'], pDict['pName'], @@ -4657,6 +5664,8 @@ def main(): myfit, tFlux1, cFlux1 = fit_lightcurve( times, selected_target_flux, selected_comp_flux, airmass, ld, pDict, jd_times, disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, ) if myfit is not None: res_std = myfit.residuals.std() / np.median(myfit.data) @@ -4691,6 +5700,8 @@ def main(): vsp_fit, _, _ = fit_lightcurve( times, tFlux, cFlux, airmass, ld, pDict, jd_times, disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -4708,6 +5719,8 @@ def main(): times[aper_mask], best_target_flux[aper_mask], cFlux, airmass[aper_mask], ld, pDict, jd_times[aper_mask], disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -4717,15 +5730,29 @@ def main(): log_info("Warning: Comparison-star calibration selected a photometry setup that failed target fitting." " Falling back to target-driven photometry selection.", warn=True) - if photometry_info['best_fit_lc'] is None: + if photometry_info['best_fit_lc'] is None and use_psf_photometry: # Legacy fallback when comparison-star-only calibration cannot determine a usable setup. + psf_comp_flux_map = { + f"comp{j + 1}": 2 * np.pi * psf_data[f"comp{j + 1}"][:, 2] + * psf_data[f"comp{j + 1}"][:, 3] + * psf_data[f"comp{j + 1}"][:, 4] + for j in range(len(exotic_infoDict['comp_stars'])) + } + psf_comp_coverage = comparison_star_coverage_summary( + psf_comp_flux_map, + skip_rejection=skip_low_comp_coverage_rejection, + ) for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" + if psf_comp_coverage[ckey]['coverage_rejected']: + continue - cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] + cFlux = psf_comp_flux_map[ckey] myfit, tFlux1, cFlux1 = fit_lightcurve( times, tFlux, cFlux, airmass, ld, pDict, jd_times, disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, ) res_std = np.inf @@ -4758,12 +5785,21 @@ def main(): 'pos': exotic_infoDict['comp_stars'][j] } + if photometry_info['best_fit_lc'] is None and use_aperture_photometry: log_info("\nComputing best comparison star, aperture, and sky annulus from the target lightcurve. Please wait.") candidate_jobs = [] for a, aper in enumerate(apers): for an, annulus in enumerate(annuli): target_flux = aper_data['target'][:, a, an] + aperture_comp_flux_map = { + f"comp{j + 1}": aper_data[f"comp{j + 1}"][:, a, an] + for j in range(len(exotic_infoDict['comp_stars'])) + } + aperture_comp_coverage = comparison_star_coverage_summary( + aperture_comp_flux_map, + skip_rejection=skip_low_comp_coverage_rejection, + ) if not require_comp_star: candidate_jobs.append({ @@ -4779,8 +5815,10 @@ def main(): for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" - comp_series = aper_data[ckey][:, a, an] - aper_mask = np.isfinite(comp_series) + if aperture_comp_coverage[ckey]['coverage_rejected']: + continue + comp_series = aperture_comp_flux_map[ckey] + aper_mask = valid_comparison_frame_mask(comp_series) comp_flux = comp_series[aper_mask] candidate_jobs.append({ 'a': a, @@ -4822,6 +5860,7 @@ def main(): pDict, jd_times[candidate_mask], disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, )) fit_results = [] @@ -4877,6 +5916,8 @@ def main(): times[aper_mask], best_target_flux[aper_mask], cFlux, airmass[aper_mask], ld, pDict, jd_times[aper_mask], disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -4899,6 +5940,7 @@ def main(): log_info("\n\n*********************************************") if np.isfinite(photometry_info['calibration_field_score']): log_info(f"Comparison-Star Field Score: {round(photometry_info['calibration_field_score'] * 100, 4)}%") + selected_method_label = selected_photometry_method_label(photometry_info) display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] == 0: # psf @@ -4937,6 +5979,44 @@ def main(): bestCompStar = photometry_info['comp_star_num'] comp_coords = photometry_info['comp_star_coords'] + if fit_every_comparison_candidate and exotic_infoDict['comp_stars']: + candidate_fit_summaries = fit_lightcurve_to_every_comparison_candidate( + times, + jd_times, + airmass, + ld, + pDict, + exotic_infoDict['comp_stars'], + psf_data, + aper_data, + photometry_info, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + ) + saved_candidate_fit_count = sum(1 for summary in candidate_fit_summaries if summary['fit'] is not None) + failed_candidate_fit_count = len(candidate_fit_summaries) - saved_candidate_fit_count + if candidate_fit_summaries: + log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_info) + try: + plot_comp_star_candidate_lightcurve_fits( + candidate_fit_summaries, + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + selected_method_label, + ) + log_info( + f"Saved {saved_candidate_fit_count} comparison-candidate lightcurve fit plot(s) to temp/." + ) + if failed_candidate_fit_count: + log_info( + f"Skipped {failed_candidate_fit_count} comparison candidate(s) that did not yield a usable lightcurve fit." + ) + except Exception as e: + log_info(f"Warning: Could not save comparison-candidate lightcurve plots ({e}).", warn=True) + # save psf_data to disk for best comparison star if bestCompStar: np.savetxt(Path(exotic_infoDict['save']) / "temp" / "psf_data_comp.txt", psf_data[f"comp{bestCompStar}"], @@ -5231,8 +6311,20 @@ def main(): return # final light curve fit - myfit = lc_fitter(goodTimes, goodFluxes, goodNormUnc, goodAirmasses, prior, mybounds, mode='ns') - annotate_airmass_fit(myfit, goodAirmasses, skip_final_airmass_fit, note=airmass_skip_note) + myfit, goodFluxes, goodNormUnc = fit_final_lightcurve_with_oot_baseline_detrending( + goodTimes, + goodFluxes, + goodNormUnc, + goodAirmasses, + prior, + mybounds, + skip_airmass_fit=skip_final_airmass_fit, + airmass_skip_note=airmass_skip_note, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + ) # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) @@ -5254,9 +6346,16 @@ def main(): plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) if fitsortext == 1: + observing_background_series = build_observing_background_series( + psf_data, + aper_data, + photometry_info, + len(exotic_infoDict['comp_stars']), + ) plot_obs_stats(myfit, exotic_infoDict['comp_stars'], psf_data, si, gi, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], - relative_flux_mask=relative_flux_mask) + relative_flux_mask=relative_flux_mask, + background_series=observing_background_series) ####################################################################### # print final extracted planetary parameters diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index a8891d97..46608762 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -418,6 +418,8 @@ def save_input(): "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", @@ -439,6 +441,8 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, + "detrend_on_outoftransit_baseline": True, + "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" @@ -1486,6 +1490,8 @@ def save_input(): "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", @@ -1555,6 +1561,8 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, + "detrend_on_outoftransit_baseline": True, + "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" @@ -1603,6 +1611,8 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, + "detrend_on_outoftransit_baseline": True, + "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" diff --git a/exotic/inputs.py b/exotic/inputs.py index 9bdb6290..1845a688 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -209,7 +209,12 @@ def __init__(self, init_opt): 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, - 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0 + 'detrend_on_outoftransit_baseline': True, + 'use_impactparameter_rather_than_inclination_to_fit': 'y', + 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', + 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, + 'skip_low_comparison_coverage_rejection': 'n', + 'fit_lightcurve_to_every_comparison_candidate': 'n', } self.params = { 'images': imaging_files, 'save': save_directory, 'aavso_num': obs_code, 'second_obs': second_obs_code, @@ -418,11 +423,37 @@ def comp_params(self, init_file, planet_dict): 'disable vertical flux normalization', 'Disable vertical flux normalization', ), + 'detrend_on_outoftransit_baseline': ( + 'detrend_on_outoftransit_baseline', + 'Detrend on Out-of-Transit Baseline', + 'detrend_on_out_of_transit_baseline', + ), + 'use_impactparameter_rather_than_inclination_to_fit': ( + 'use_impactparameter_rather_than_inclination_to_fit', + 'Use impact parameter rather than inclination to fit? (y/n)', + 'Use Impact Parameter Rather Than Inclination To Fit? (y/n)', + ), + 'use_psf_photometry': ( + 'use_psf_photometry', + 'Use PSF Photometry? (y/n)', + ), + 'use_aperture_photometry': ( + 'use_aperture_photometry', + 'Use Aperture Photometry? (y/n)', + ), 'use_adaptive_apertures': ( 'use_adaptive_apertures', 'Use Adaptive Apertures? (y/n)', 'Use Adaptive Apertures (y/n)', ), + 'skip_low_comparison_coverage_rejection': ( + 'skip_low_comparison_coverage_rejection', + 'Skip Low Comparison Coverage Rejection? (y/n)', + ), + 'fit_lightcurve_to_every_comparison_candidate': ( + 'fit_lightcurve_to_every_comparison_candidate', + 'Fit Lightcurve to Every Comparison Candidate? (y/n)', + ), 'bad_wcs_threshold_percent': ( 'bad_wcs_threshold_percent', 'Bad WCS Threshold Percent', diff --git a/exotic/output_files.py b/exotic/output_files.py index cc23d3b7..9d7a9274 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -381,7 +381,8 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label handle.write("# Field suitability score,\n") handle.write(f"# Selected comparison star,{'' if best_comp_index is None else best_comp_index + 1}\n") handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,ensemble_score,pairwise_median_score," - "pairwise_max_score,self_score,valid_pair_count\n") + "pairwise_max_score,self_score,valid_pair_count,coverage_count,coverage_peer_median," + "coverage_min_required,coverage_rejected\n") for summary in comp_summaries: position = summary.get('position') or [None, None] @@ -396,6 +397,10 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label summary.get('pairwise_max_score'), summary.get('self_score'), summary.get('valid_pair_count'), + summary.get('coverage_count'), + summary.get('coverage_reference_count'), + summary.get('coverage_min_required_count'), + summary.get('coverage_rejected'), ] handle.write(",".join("" if value is None else str(value) for value in values) + "\n") diff --git a/exotic/plots.py b/exotic/plots.py index a7ffb82a..1fcc6c4c 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -217,28 +217,7 @@ def plot_comp_star_calibration_series(times, comp_summaries, targ_name, save, da axes = [axes] for axis, summary in zip(axes, comp_summaries): - axis.axhline(1.0, color='lightgray', lw=1.0, zorder=1) - pairwise_series = summary.get('pairwise_ratio_series', {}) - for color_index, (other_label, ratio_series) in enumerate(pairwise_series.items()): - ratio_series = np.asarray(ratio_series, dtype=float) - valid = np.isfinite(times) & np.isfinite(ratio_series) - if np.any(valid): - axis.plot(times[valid], ratio_series[valid], color=colors[color_index % len(colors)], - alpha=0.55, lw=1.0, label=other_label) - - ensemble_ratio = np.asarray(summary.get('ensemble_ratio_series'), dtype=float) - ensemble_valid = np.isfinite(times) & np.isfinite(ensemble_ratio) - if np.any(ensemble_valid): - axis.plot(times[ensemble_valid], ensemble_ratio[ensemble_valid], color='black', lw=1.8, - label='Ensemble') - - selected_text = " selected" if summary.get('selected') else "" - aggregate = summary.get('aggregate_score', np.nan) - aggregate_text = "n/a" if not np.isfinite(aggregate) else f"{aggregate * 100.0:.3f}%" - axis.set_ylabel("Norm Ratio") - axis.set_title(f"{summary['label']}{selected_text} | suitability={aggregate_text}", loc='left', fontsize=10) - axis.grid(alpha=0.2) - axis.legend(ncol=4, fontsize=8, loc='upper right') + _draw_comp_star_calibration_axis(axis, times, summary, colors) axes[-1].set_xlabel("Time [BJD_TDB]") fig.suptitle(f"{targ_name} Comparison-Star Calibration Curves\n{method_label}", y=1.01) @@ -248,6 +227,79 @@ def plot_comp_star_calibration_series(times, comp_summaries, targ_name, save, da plt.close(fig) +def plot_individual_comp_star_calibration_series(times, comp_summaries, targ_name, save, date, method_label): + if not comp_summaries: + return + + times = np.asarray(times, dtype=float) + temp_dir = Path(save) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + colors = plt.cm.tab10(np.linspace(0.0, 1.0, 10)) + + for summary in comp_summaries: + fig, axis = plt.subplots(figsize=(12, 4)) + _draw_comp_star_calibration_axis(axis, times, summary, colors) + axis.set_xlabel("Time [BJD_TDB]") + fig.suptitle(f"{targ_name} {summary['label']} Calibration Curves\n{method_label}") + fig.tight_layout() + label_slug = summary['label'].replace(" ", "") + fig.savefig(temp_dir / f"CompStarCalibrationCurve_{label_slug}_{targ_name}_{date}.png", bbox_inches="tight") + fig.savefig(temp_dir / f"CompStarCalibrationCurve_{label_slug}_{targ_name}_{date}.pdf", bbox_inches="tight") + plt.close(fig) + + +def plot_comp_star_candidate_lightcurve_fits(candidate_fit_summaries, targ_name, save, date, method_label): + if not candidate_fit_summaries: + return + + temp_dir = Path(save) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + + for summary in candidate_fit_summaries: + fit = summary.get('fit') + if fit is None: + continue + + fig, (ax_lc, ax_res) = fit.plot_bestfit(phase=False) + selected_text = " selected" if summary.get('selected') else "" + res_std = summary.get('res_std', np.nan) + res_std_text = "n/a" if not np.isfinite(res_std) else f"{res_std * 100.0:.3f}%" + ax_lc.set_title(f"{targ_name} vs {summary['label']}{selected_text}\n{method_label} | scatter={res_std_text}") + ax_res.set_title("") + + label_slug = summary['label'].replace(" ", "") + fig.savefig(temp_dir / f"CompStarLightCurveFit_{label_slug}_{targ_name}_{date}.png", bbox_inches="tight") + fig.savefig(temp_dir / f"CompStarLightCurveFit_{label_slug}_{targ_name}_{date}.pdf", bbox_inches="tight") + plt.close(fig) + + +def _draw_comp_star_calibration_axis(axis, times, summary, colors): + axis.axhline(1.0, color='lightgray', lw=1.0, zorder=1) + pairwise_series = summary.get('pairwise_ratio_series', {}) + for color_index, (other_label, ratio_series) in enumerate(pairwise_series.items()): + ratio_series = np.asarray(ratio_series, dtype=float) + valid = np.isfinite(times) & np.isfinite(ratio_series) + if np.any(valid): + axis.plot(times[valid], ratio_series[valid], color=colors[color_index % len(colors)], + alpha=0.55, lw=1.0, label=other_label) + + ensemble_ratio = np.asarray(summary.get('ensemble_ratio_series'), dtype=float) + ensemble_valid = np.isfinite(times) & np.isfinite(ensemble_ratio) + if np.any(ensemble_valid): + axis.plot(times[ensemble_valid], ensemble_ratio[ensemble_valid], color='black', lw=1.8, + label='Ensemble') + + selected_text = " selected" if summary.get('selected') else "" + aggregate = summary.get('aggregate_score', np.nan) + aggregate_text = "n/a" if not np.isfinite(aggregate) else f"{aggregate * 100.0:.3f}%" + axis.set_ylabel("Norm Ratio") + axis.set_title(f"{summary['label']}{selected_text} | suitability={aggregate_text}", loc='left', fontsize=10) + axis.grid(alpha=0.2) + handles, labels = axis.get_legend_handles_labels() + if handles and labels: + axis.legend(ncol=4, fontsize=8, loc='upper right') + + def plot_comp_star_suitability(comp_summaries, targ_name, save, date, method_label): if not comp_summaries: return @@ -384,9 +436,9 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): plt.close() -# Observation statistics from PSF data -def _select_psf_rows(psf_rows, sort_index=None, sigma_mask=None, relative_flux_mask=None): - rows = np.asarray(psf_rows) +# Observation statistics series selection +def _select_plot_rows(rows, sort_index=None, sigma_mask=None, relative_flux_mask=None): + rows = np.asarray(rows) if sort_index is not None: rows = rows[np.asarray(sort_index)] @@ -404,7 +456,8 @@ def _select_psf_rows(psf_rows, sort_index=None, sigma_mask=None, relative_flux_m return rows -def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relative_flux_mask=None): +def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relative_flux_mask=None, + background_series=None): fit_time = np.asarray(fit.time) fit_airmass = np.asarray(fit.airmass) temp_dir = Path(save) / "temp" @@ -419,10 +472,21 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relati fig, axs = plt.subplots(3, 2, figsize=(12, 10)) fig.suptitle(f"Observing Statistics - {title} - {date}") - star_stats = _select_psf_rows(psf[key], sort_index=si, sigma_mask=gi, - relative_flux_mask=relative_flux_mask) - - plot_len = min(fit_time.shape[0], fit_airmass.shape[0], star_stats.shape[0]) + star_stats = _select_plot_rows(psf[key], sort_index=si, sigma_mask=gi, + relative_flux_mask=relative_flux_mask) + background_data = None + if background_series is not None and key in background_series: + background_data = _select_plot_rows( + background_series[key], + sort_index=si, + sigma_mask=gi, + relative_flux_mask=relative_flux_mask, + ) + + plot_len_inputs = [fit_time.shape[0], fit_airmass.shape[0], star_stats.shape[0]] + if background_data is not None: + plot_len_inputs.append(background_data.shape[0]) + plot_len = min(plot_len_inputs) if plot_len == 0: plt.close(fig) continue @@ -430,6 +494,10 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relati time_data = fit_time[:plot_len] airmass_data = fit_airmass[:plot_len] star_stats = star_stats[:plot_len] + if background_data is None: + background_data = star_stats[:, 6] + else: + background_data = np.asarray(background_data)[:plot_len] axs[0, 0].set(xlabel="Time [BJD_TDB]", ylabel="X-Centroid [px]") axs[0, 0].plot(time_data, star_stats[:, 0], 'k.') @@ -447,7 +515,7 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relati axs[2, 0].plot(time_data, star_stats[:, 2], 'k.') axs[2, 1].set(xlabel="Time [BJD_TDB]", ylabel="Background [ADU]") - axs[2, 1].plot(time_data, star_stats[:, 6], 'k.') + axs[2, 1].plot(time_data, background_data, 'k.') plt.tight_layout() diff --git a/inits.json b/inits.json index bf0b12f4..aaf6f40b 100644 --- a/inits.json +++ b/inits.json @@ -26,7 +26,13 @@ "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", + "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", + "Skip Low Comparison Coverage Rejection": "Set optional_info 'skip_low_comparison_coverage_rejection' to y to disable EXOTIC's default rejection of comparison stars that are valid in far fewer frames than the rest of the comparison-star field. Default n.", + "Fit Lightcurve to Every Comparison Candidate": "Set optional_info 'fit_lightcurve_to_every_comparison_candidate' to y to save one target lightcurve fit plot per comparison star into temp/ using the selected photometry setup. Default n.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -97,7 +103,13 @@ "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": false, + "detrend_on_outoftransit_baseline": true, + "use_impactparameter_rather_than_inclination_to_fit": "y", + "use_psf_photometry": "y", + "use_aperture_photometry": "y", "use_adaptive_apertures": false, + "skip_low_comparison_coverage_rejection": "n", + "fit_lightcurve_to_every_comparison_candidate": "n", "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y", "Image Scale (Ex: 5.21 arcsecs/pixel)": null, diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 422ec154..e5ab9c70 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -127,6 +127,17 @@ def fail_least_squares(*args, **kwargs): assert low_flux_warnings == [] +def test_fit_centroid_reports_consistent_background_between_fast_and_full_modes(): + image = _gaussian_image(center=(40.3, 35.7), amplitude=5000.0, sigma=2.0, background=123.4) + + fast_result = exotic_module.fit_centroid(image, [40.0, 36.0], 0, fast_mode=True) + full_result = exotic_module.fit_centroid(image, [40.0, 36.0], 0, fast_mode=False) + + assert np.isfinite(fast_result[6]) + assert np.isfinite(full_result[6]) + assert full_result[6] == pytest.approx(fast_result[6], abs=1e-8) + + def test_fit_centroid_or_warn_out_of_frame_skips_centroid_fit(monkeypatch): image = np.zeros((40, 50), dtype=float) out_of_frame_warnings = [] diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 8040ac46..0b5fead1 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -134,3 +134,245 @@ def test_lc_fitter_rejects_redundant_a0_and_a1_bounds(monkeypatch, tmp_path): mode="lm", verbose=False, ) + + +def test_create_fit_variables_preserves_explicit_baseline_in_nested_mode(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + sampled = make_prior() + sampled["rprs"] = 0.09 + sampled["a0"] = 0.98 + sampled["a1"] = 0.98 + + truth = make_prior() + truth["rprs"] = 0.12 + + fit.time = np.linspace(-0.03, 0.03, 301) + fit.data = elca.transit(fit.time, truth) + fit.dataerr = np.full_like(fit.time, 1e-3) + fit.airmass = np.zeros_like(fit.time) + fit.prior = sampled.copy() + fit.bounds = {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]} + fit.mode = "ns" + fit.parameters = sampled.copy() + fit.errors = {"rprs": 1e-3, "tmid": 1e-4, "a0": 2e-3} + + fit.create_fit_variables() + + assert fit.parameters["a0"] == pytest.approx(0.98, abs=1e-9) + assert fit.parameters["a1"] == pytest.approx(0.98, abs=1e-9) + + +def test_plot_triangle_clips_ranges_to_parameter_bounds(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + captured = {} + + def fake_corner(*args, **kwargs): + captured["points"] = args[0] + captured["labels"] = kwargs["labels"] + captured["range"] = kwargs["range"] + return "figure" + + monkeypatch.setattr(elca, "corner", fake_corner) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.125], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.quantiles = {"rprs": [], "inc": [], "a0": []} + fit.parameters = {"rprs": 0.10, "inc": 88.42, "a0": 0.94962} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} + + points = np.array( + [ + [0.099, 88.30, 0.9501], + [0.101, 88.55, 0.9502], + [0.102, 88.10, 0.9515], + [0.098, 88.70, 0.9520], + [0.100, 88.40, 0.9508], + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.array([-5.0, -4.0, -4.5, -5.5, -4.2]), + }, + "samples": points.copy(), + } + + fig = fit.plot_triangle() + + assert fig == "figure" + assert captured["labels"][1] == "Distance from fitted Inc. [deg] (mirrored)" + assert captured["range"][0][0] == pytest.approx(0.05) + assert captured["range"][0][1] == pytest.approx(0.125) + expected_inc_distance_limit = np.max(np.abs(np.array([84.67, 90.0]) - fit.parameters["inc"])) + assert captured["range"][1][0] == pytest.approx(-expected_inc_distance_limit) + assert captured["range"][1][1] == pytest.approx(expected_inc_distance_limit) + assert captured["range"][2][0] == pytest.approx(0.95) + assert captured["range"][2][1] == pytest.approx(0.96932) + assert captured["points"].shape == (10, 3) + expected_inc_distance = np.abs(points[:, 1] - fit.parameters["inc"]) + np.testing.assert_allclose(captured["points"][:5, 1], expected_inc_distance) + np.testing.assert_allclose(captured["points"][5:, 1], -expected_inc_distance) + + +def test_internal_impact_parameter_transform_round_trips_inclination(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "inc": [87.0, 90.0], + "tmid": [-0.005, 0.005], + } + + sample_point = fit._sample_point_from_unit_cube(np.array([0.25, 0.4, 0.75])) + physical = fit._physical_values_from_sample_point(sample_point) + expected_inc = 87.0 + 0.4 * (90.0 - 87.0) + + assert fit._get_sampled_keys() == ["rprs", "b", "tmid"] + assert physical["inc"] == pytest.approx(expected_inc) + assert physical["b"] == pytest.approx(sample_point[1]) + + +def test_nested_fit_can_keep_inclination_parameterization_when_requested(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = False + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "inc": [87.0, 90.0], + "tmid": [-0.005, 0.005], + } + + assert fit._get_sampled_keys() == ["rprs", "inc", "tmid"] + + +def test_nested_fit_reports_inclination_from_internal_impact_parameter(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 101) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + data[0] += 5e-5 + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + b_ml = float(elca.impact_parameter_from_inclination(prior, 88.8)) + sample_points = np.array( + [ + [0.100, b_ml - 0.02, 0.0000], + [0.101, b_ml - 0.01, 0.0002], + [0.099, b_ml + 0.01, -0.0001], + [0.100, b_ml + 0.02, 0.0001], + ] + ) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr( + elca, + "run_reactive_sampler", + lambda *args, **kwargs: { + "maximum_likelihood": {"point": np.array([0.100, b_ml, 0.0])}, + "posterior": { + "stdev": np.array([0.005, 0.02, 0.0005]), + "errlo": np.array([-0.005, -0.02, -0.0005]), + "errup": np.array([0.005, 0.02, 0.0005]), + }, + "weighted_samples": { + "points": sample_points, + "logl": np.array([-4.0, -3.0, -3.2, -3.8]), + }, + "samples": sample_points.copy(), + }, + ) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "inc": [87.0, 90.0], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + ) + + assert fit.sampled_keys == ["rprs", "b", "tmid"] + assert fit.sample_parameters["b"] == pytest.approx(b_ml) + assert fit.parameters["inc"] == pytest.approx(88.8, abs=1e-6) + assert fit.errors["inc"] > 0 + + +def test_plot_triangle_uses_mirrored_distance_from_fitted_impact_parameter_axis(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + captured = {} + + def fake_corner(*args, **kwargs): + captured["points"] = args[0] + captured["labels"] = kwargs["labels"] + captured["range"] = kwargs["range"] + return "figure" + + monkeypatch.setattr(elca, "corner", fake_corner) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.125], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.sampled_keys = ["rprs", "b", "a0"] + fit.sample_bounds = { + "rprs": [0.0, 0.125], + "b": [0.0, 1.25434156], + "a0": [0.95, 1.05], + } + fit.sample_parameters = {"rprs": 0.10, "b": 0.314, "a0": 0.94962} + fit.sample_errors = {"rprs": 0.01, "b": 0.05, "a0": 0.00394} + fit.parameters = {"rprs": 0.10, "inc": 88.5, "a0": 0.94962} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} + + points = np.array( + [ + [0.099, 0.300, 0.9501], + [0.101, 0.330, 0.9502], + [0.102, 0.290, 0.9515], + [0.098, 0.360, 0.9520], + [0.100, 0.314, 0.9508], + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.array([-5.0, -4.0, -4.5, -5.5, -4.2]), + }, + "samples": points.copy(), + } + + fig = fit.plot_triangle() + + assert fig == "figure" + assert captured["labels"][1] == "Distance from fitted b (mirrored)" + assert captured["range"][1][0] == pytest.approx(-0.25) + assert captured["range"][1][1] == pytest.approx(0.25) + assert captured["points"].shape == (10, 3) + expected_b_distance = np.abs(points[:, 1] - fit.sample_parameters["b"]) + np.testing.assert_allclose(captured["points"][:5, 1], expected_b_distance) + np.testing.assert_allclose(captured["points"][5:, 1], -expected_b_distance) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index ae3951d4..246e3d90 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -86,16 +86,26 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: auto_tune_aperture_sigma_grid, check_coordinates, cheap_lightcurve_prescore, + comparison_star_coverage_summary, comparison_star_stability_summary, + detrend_flux_on_out_of_transit_baseline, fit_lightcurve, + fit_final_lightcurve_with_oot_baseline_detrending, + fit_lightcurve_to_every_comparison_candidate, is_adaptive_aperture_mode_enabled, is_comp_star_required, + is_out_of_transit_baseline_detrending_enabled, is_target_driven_comp_selection_enabled, + log_comparison_candidate_fit_summaries, phase_bin_sigma_clip, representative_psf_sigma, resolve_frame_aperture_radii, summarize_adaptive_aperture_usage, should_skip_airmass_fit, + should_fit_lightcurve_to_every_comparison_candidate, + should_use_aperture_photometry, + should_use_psf_photometry, + should_skip_low_comparison_coverage_rejection, should_use_fast_target_centroid, update_coordinates_with_proper_motion, ) @@ -177,6 +187,37 @@ def test_is_target_driven_comp_selection_enabled_parses_values(): assert is_target_driven_comp_selection_enabled("n") is False +def test_should_skip_low_comparison_coverage_rejection_parses_values(): + assert should_skip_low_comparison_coverage_rejection(None) is False + assert should_skip_low_comparison_coverage_rejection("y") is True + assert should_skip_low_comparison_coverage_rejection("n") is False + + +def test_should_fit_lightcurve_to_every_comparison_candidate_parses_values(): + assert should_fit_lightcurve_to_every_comparison_candidate(None) is False + assert should_fit_lightcurve_to_every_comparison_candidate("y") is True + assert should_fit_lightcurve_to_every_comparison_candidate("n") is False + + +def test_is_out_of_transit_baseline_detrending_enabled_parses_values(): + assert is_out_of_transit_baseline_detrending_enabled(None) is True + assert is_out_of_transit_baseline_detrending_enabled("y") is True + assert is_out_of_transit_baseline_detrending_enabled("n") is False + assert is_out_of_transit_baseline_detrending_enabled(True) is True + + +def test_should_use_psf_photometry_parses_values(): + assert should_use_psf_photometry(None) is True + assert should_use_psf_photometry("y") is True + assert should_use_psf_photometry("n") is False + + +def test_should_use_aperture_photometry_parses_values(): + assert should_use_aperture_photometry(None) is True + assert should_use_aperture_photometry("y") is True + assert should_use_aperture_photometry("n") is False + + def test_is_adaptive_aperture_mode_enabled_parses_values(): assert is_adaptive_aperture_mode_enabled(None) is False assert is_adaptive_aperture_mode_enabled("y") is True @@ -268,6 +309,216 @@ def test_auto_tune_aperture_grid_uses_comparison_field_consistency(): assert best_candidate["comp_index"] in (0, 1) +def test_fit_lightcurve_to_every_comparison_candidate_uses_selected_aperture(monkeypatch): + calls = [] + + class DummyFit: + def __init__(self, size): + self.residuals = np.full(size, 0.01) + self.data = np.ones(size) + + def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + calls.append({ + "times": np.asarray(times), + "tflux": np.asarray(tflux), + "cflux": np.asarray(cflux), + "jd_times": np.asarray(jd_times), + "kwargs": dict(kwargs), + }) + return DummyFit(len(times)), np.asarray(tflux), np.asarray(cflux) + + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + times = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) + jd_times = np.array([11.0, 12.0, 13.0, 14.0, 15.0, 16.0]) + airmass = np.array([1.1, 1.2, 1.3, 1.4, 1.5, 1.6]) + aper_data = { + "target": np.array([ + [[1.0], [10.0]], + [[2.0], [11.0]], + [[3.0], [12.0]], + [[4.0], [13.0]], + [[5.0], [14.0]], + [[6.0], [15.0]], + ]), + "comp1": np.array([ + [[4.0], [20.0]], + [[5.0], [np.nan]], + [[6.0], [22.0]], + [[7.0], [23.0]], + [[8.0], [24.0]], + [[9.0], [25.0]], + ]), + "comp2": np.array([ + [[7.0], [30.0]], + [[8.0], [31.0]], + [[9.0], [32.0]], + [[10.0], [33.0]], + [[11.0], [34.0]], + [[12.0], [35.0]], + ]), + } + photometry_info = { + "best_fit_lc": object(), + "comp_star_num": 2, + "min_aperture": 5.0, + "min_annulus": 12.0, + "aperture_index": 1, + "annulus_index": 0, + } + + candidate_fits = fit_lightcurve_to_every_comparison_candidate( + times, + jd_times, + airmass, + ld=np.array([0.1, 0.2, 0.3, 0.4]), + p_dict={"rprs": 0.1}, + comp_stars=[[100, 200], [300, 400]], + psf_data={}, + aper_data=aper_data, + photometry_info=photometry_info, + ) + + assert len(candidate_fits) == 2 + assert candidate_fits[0]["selected"] is False + assert candidate_fits[1]["selected"] is True + assert calls[0]["kwargs"]["final_fit_mode"] == "ns" + assert calls[1]["kwargs"]["final_fit_mode"] == "ns" + np.testing.assert_array_equal(calls[0]["times"], np.array([1.0, 3.0, 4.0, 5.0, 6.0])) + np.testing.assert_array_equal(calls[0]["tflux"], np.array([10.0, 12.0, 13.0, 14.0, 15.0])) + np.testing.assert_array_equal(calls[0]["cflux"], np.array([20.0, 22.0, 23.0, 24.0, 25.0])) + np.testing.assert_array_equal(calls[0]["jd_times"], np.array([11.0, 13.0, 14.0, 15.0, 16.0])) + np.testing.assert_array_equal(calls[1]["times"], np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])) + np.testing.assert_array_equal(calls[1]["tflux"], np.array([10.0, 11.0, 12.0, 13.0, 14.0, 15.0])) + np.testing.assert_array_equal(calls[1]["cflux"], np.array([30.0, 31.0, 32.0, 33.0, 34.0, 35.0])) + + +def test_fit_lightcurve_to_every_comparison_candidate_records_sparse_candidate_failure(monkeypatch): + calls = [] + + class DummyFit: + def __init__(self, size): + self.residuals = np.full(size, 0.01) + self.data = np.ones(size) + + def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + calls.append({ + "times": np.asarray(times), + "tflux": np.asarray(tflux), + "cflux": np.asarray(cflux), + "jd_times": np.asarray(jd_times), + "kwargs": dict(kwargs), + }) + return DummyFit(len(times)), np.asarray(tflux), np.asarray(cflux) + + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + times = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) + jd_times = np.array([11.0, 12.0, 13.0, 14.0, 15.0, 16.0]) + airmass = np.array([1.1, 1.2, 1.3, 1.4, 1.5, 1.6]) + aper_data = { + "target": np.array([ + [[10.0]], + [[11.0]], + [[12.0]], + [[13.0]], + [[14.0]], + [[15.0]], + ]), + "comp1": np.array([ + [[20.0]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + ]), + "comp2": np.array([ + [[30.0]], + [[31.0]], + [[32.0]], + [[33.0]], + [[34.0]], + [[35.0]], + ]), + } + photometry_info = { + "best_fit_lc": object(), + "comp_star_num": 2, + "min_aperture": 5.0, + "min_annulus": 12.0, + "aperture_index": 0, + "annulus_index": 0, + } + + candidate_fits = fit_lightcurve_to_every_comparison_candidate( + times, + jd_times, + airmass, + ld=np.array([0.1, 0.2, 0.3, 0.4]), + p_dict={"rprs": 0.1}, + comp_stars=[[100, 200], [300, 400]], + psf_data={}, + aper_data=aper_data, + photometry_info=photometry_info, + ) + + assert len(calls) == 1 + assert candidate_fits[0]["fit"] is None + assert candidate_fits[0]["coverage_rejected"] is True + assert candidate_fits[0]["fit_diagnostics"]["failed_stage"] == "coverage" + assert "low-coverage clipping" in candidate_fits[0]["failure_reason"] + assert candidate_fits[1]["fit"] is not None + assert candidate_fits[1]["coverage_rejected"] is False + assert candidate_fits[1]["failure_reason"] is None + assert calls[0]["kwargs"]["final_fit_mode"] == "ns" + + +def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): + logged = [] + monkeypatch.setattr("exotic.exotic.log_info", lambda message, warn=False, error=False: logged.append(message)) + + candidate_fit_summaries = [ + { + "label": "Comp 1", + "position": [100, 200], + "selected": False, + "fit": None, + "res_std": np.inf, + "coverage_count": 1, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit_point_count": 0, + "fit_diagnostics": {"usable_point_count": 0}, + "failure_reason": "comparison candidate rejected after iterative low-coverage clipping (1 < 2 valid frame(s); peer median=3.0).", + }, + { + "label": "Comp 2", + "position": [300, 400], + "selected": True, + "fit": object(), + "res_std": 0.01, + "coverage_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit_point_count": 3, + "fit_diagnostics": {"usable_point_count": 3}, + "failure_reason": None, + "parameter_summary": "fit_method=ultranest, Tmid=1.0 +/- 0.1", + }, + ] + + log_comparison_candidate_fit_summaries( + candidate_fit_summaries, + {"selection_basis": "comparison_field", "comp_star_num": 2, "min_std": 0.01}, + ) + + assert any("Selection basis: comparison-field" in message for message in logged) + assert any("Comp 1" in message and "reason=comparison candidate rejected after iterative low-coverage clipping" in message for message in logged) + assert any("Comp 2 [selected]" in message and "comparison-field calibration ranked this star best" in message for message in logged) + assert any("parameters: fit_method=ultranest" in message for message in logged) + + def test_comparison_star_stability_summary_penalizes_variable_candidates(): airmass = np.linspace(1.0, 1.5, 6) summary = comparison_star_stability_summary( @@ -284,6 +535,60 @@ def test_comparison_star_stability_summary_penalizes_variable_candidates(): assert summary["comp_summaries"][2]["aggregate_score"] > summary["comp_summaries"][0]["aggregate_score"] +def test_comparison_star_coverage_summary_rejects_sparse_candidates(): + coverage = comparison_star_coverage_summary( + { + "comp1": np.array([100.0, 101.0, 100.5, 101.5, 100.8, 101.2]), + "comp2": np.array([80.0, 80.8, 80.4, 81.0, 80.6, 80.9]), + "comp3": np.array([60.0, np.nan, np.nan, np.nan, np.nan, 60.3]), + } + ) + + assert not coverage["comp1"]["coverage_rejected"] + assert not coverage["comp2"]["coverage_rejected"] + assert coverage["comp3"]["coverage_rejected"] + assert coverage["comp3"]["coverage_count"] == 2 + + +def test_comparison_star_coverage_summary_iteratively_rejects_low_count_tail(): + coverage = comparison_star_coverage_summary( + { + "comp1": np.array([10.0] * 10), + "comp2": np.array([11.0] * 10), + "comp3": np.array([12.0] * 10), + "comp4": np.array([13.0] * 7 + [np.nan] * 3), + "comp5": np.array([14.0] * 6 + [np.nan] * 4), + "comp6": np.array([15.0] + [np.nan] * 9), + } + ) + + assert not coverage["comp1"]["coverage_rejected"] + assert not coverage["comp2"]["coverage_rejected"] + assert not coverage["comp3"]["coverage_rejected"] + assert coverage["comp4"]["coverage_rejected"] + assert coverage["comp5"]["coverage_rejected"] + assert coverage["comp6"]["coverage_rejected"] + assert coverage["comp1"]["coverage_reference_count"] == pytest.approx(10.0) + assert coverage["comp1"]["coverage_min_required_count"] == 8 + + +def test_comparison_star_stability_summary_rejects_low_coverage_candidates(): + airmass = np.linspace(1.0, 1.5, 6) + summary = comparison_star_stability_summary( + { + "comp1": np.array([100.0, 101.0, 100.5, 101.5, 100.8, 101.2]), + "comp2": np.array([80.0, 80.8, 80.4, 81.0, 80.6, 80.9]), + "comp3": np.array([60.0, np.nan, np.nan, np.nan, np.nan, 60.3]), + }, + airmass, + ) + + assert np.isfinite(summary["field_score"]) + assert summary["best_comp_index"] in (0, 1) + assert summary["comp_summaries"][2]["coverage_rejected"] + assert np.isinf(summary["comp_summaries"][2]["aggregate_score"]) + + def test_cheap_lightcurve_prescore_ignores_relative_flux_above_two(): tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) cflux = np.full(tflux.shape[0], 2.0) @@ -310,6 +615,81 @@ def test_should_skip_airmass_fit_when_airmass_span_is_small(): assert should_skip_airmass_fit(airmass) +def test_detrend_flux_on_out_of_transit_baseline_removes_linear_slope(): + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + baseline = 1.0 + 0.02 * times + transit_profile = np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + flux = baseline * transit_profile + fluxerr = np.full_like(times, 0.01) + fit = types.SimpleNamespace( + transit=transit_profile, + parameters={"tmid": 0.0}, + ) + + result = detrend_flux_on_out_of_transit_baseline(times, flux, fluxerr, fit) + + assert result["applied"] is True + assert np.allclose(result["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert result["flux"][3] == pytest.approx(0.99, abs=1e-8) + assert result["slope"] == pytest.approx(0.02, abs=1e-8) + + +def test_fit_final_lightcurve_with_oot_baseline_detrending_refits_with_flattened_flux(monkeypatch): + import exotic.exotic as exotic_module + + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + flux = (1.0 + 0.02 * times) * np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + fluxerr = np.full_like(times, 0.01) + airmass = np.ones_like(times) + prior = {"rprs": 0.1, "tmid": 0.0, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.2], "tmid": [-0.1, 0.1], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["calls"].append(np.array(call_flux, dtype=float)) + return types.SimpleNamespace( + transit=np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]), + parameters={"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit, refit_flux, refit_unc = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + skip_airmass_fit=False, + disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, + ) + + assert len(captured["calls"]) == 2 + assert np.allclose(captured["calls"][0], flux) + assert np.allclose(captured["calls"][1][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert refit_flux[3] == pytest.approx(0.99, abs=1e-8) + assert np.allclose(refit_unc[[0, 1, 2, 4, 5, 6]], 0.01 / (1.0 + 0.02 * times[[0, 1, 2, 4, 5, 6]])) + assert fit.oot_baseline_detrending_applied is True + assert fit.oot_baseline_pre_points == 3 + assert fit.oot_baseline_post_points == 3 + + def test_phase_bin_sigma_clip_flags_local_phase_outlier(): phase_centers = np.linspace(-0.045, 0.045, 10) phase = np.concatenate([center + np.linspace(-1e-4, 1e-4, 5) for center in phase_centers]) @@ -326,7 +706,17 @@ def test_phase_bin_sigma_clip_flags_local_phase_outlier(): def test_fit_lightcurve_removes_relative_flux_above_two_before_fit(monkeypatch): captured = {} - def fake_lc_fitter(times, fluxes, flux_unc, airmass, prior, bounds, jd_times=None, mode=None): + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): captured["times"] = np.array(times) captured["fluxes"] = np.array(fluxes) captured["flux_unc"] = np.array(flux_unc) @@ -367,10 +757,56 @@ def fake_lc_fitter(times, fluxes, flux_unc, airmass, prior, bounds, jd_times=Non assert np.allclose(fit_cflux, 2.0) +def test_fit_lightcurve_rejects_undersampled_series(monkeypatch): + called = {"count": 0} + + def fake_lc_fitter(*args, **kwargs): + called["count"] += 1 + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + + times = np.linspace(0.0, 0.03, 4) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.3, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, fit_tflux, fit_cflux = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is None + assert fit_tflux is None + assert fit_cflux is None + assert called["count"] == 0 + + def test_fit_lightcurve_refits_after_phase_binned_clip(monkeypatch): captured_calls = [] - def fake_lc_fitter(times, fluxes, flux_unc, airmass, prior, bounds, jd_times=None, mode=None): + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): call_index = len(captured_calls) captured_calls.append({ "times": np.array(times), @@ -426,10 +862,125 @@ def fake_phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max assert len(fit_cflux) == 7 +def test_fit_lightcurve_runs_nested_fit_when_requested(monkeypatch): + captured_modes = [] + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured_modes.append(mode) + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + final_fit_mode="ns", + ) + + assert myfit is not None + assert captured_modes == ["lm", "ns"] + + +def test_fit_lightcurve_can_disable_impact_parameter_parameterization(monkeypatch): + captured = {"flags": []} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["flags"].append(use_impactparameter_rather_than_inclination_to_fit) + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + use_impactparameter_rather_than_inclination_to_fit=False, + ) + + assert captured["flags"] == [False] + + def test_fit_lightcurve_skips_airmass_term_when_airmass_span_is_small(monkeypatch): captured = {} - def fake_lc_fitter(times, fluxes, flux_unc, airmass, prior, bounds, jd_times=None, mode=None): + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): captured["bounds"] = dict(bounds) captured["airmass"] = np.array(airmass) return types.SimpleNamespace() diff --git a/tests/test_inputs.py b/tests/test_inputs.py index df951879..7576fc5b 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -83,6 +83,36 @@ def test_comp_params_defaults_bad_wcs_threshold_percent_to_three(tmp_path): assert inputs.info_dict["bad_wcs_threshold_percent"] == 3.0 +def test_comp_params_defaults_skip_low_comparison_coverage_rejection_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["skip_low_comparison_coverage_rejection"] == "n" + + +def test_comp_params_defaults_fit_lightcurve_to_every_comparison_candidate_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["fit_lightcurve_to_every_comparison_candidate"] == "n" + + def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_path): init_data = { "user_info": {}, @@ -98,6 +128,66 @@ def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_p assert inputs.info_dict["disable_vertical_flux_normalization"] is False +def test_comp_params_defaults_detrend_on_outoftransit_baseline_to_true(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detrend_on_outoftransit_baseline"] is True + + +def test_comp_params_defaults_use_impactparameter_fit_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_impactparameter_rather_than_inclination_to_fit"] == "y" + + +def test_comp_params_defaults_use_psf_photometry_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_psf_photometry"] == "y" + + +def test_comp_params_defaults_use_aperture_photometry_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_aperture_photometry"] == "y" + + def test_comp_params_defaults_use_adaptive_apertures_to_false(tmp_path): init_data = { "user_info": {}, @@ -173,6 +263,36 @@ def test_comp_params_reads_bad_wcs_threshold_percent_from_optional_info(tmp_path assert inputs.info_dict["bad_wcs_threshold_percent"] == 5.5 +def test_comp_params_reads_skip_low_comparison_coverage_rejection_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"skip_low_comparison_coverage_rejection": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["skip_low_comparison_coverage_rejection"] == "y" + + +def test_comp_params_reads_fit_lightcurve_to_every_comparison_candidate_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"fit_lightcurve_to_every_comparison_candidate": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["fit_lightcurve_to_every_comparison_candidate"] == "y" + + def test_comp_params_reads_disable_vertical_flux_normalization_from_optional_info(tmp_path): init_data = { "user_info": {}, @@ -188,6 +308,81 @@ def test_comp_params_reads_disable_vertical_flux_normalization_from_optional_inf assert inputs.info_dict["disable_vertical_flux_normalization"] is True +def test_comp_params_reads_detrend_on_outoftransit_baseline_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"detrend_on_outoftransit_baseline": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detrend_on_outoftransit_baseline"] is True + + +def test_comp_params_reads_detrend_on_outoftransit_baseline_false_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"detrend_on_outoftransit_baseline": False}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detrend_on_outoftransit_baseline"] is False + + +def test_comp_params_reads_use_impactparameter_fit_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_impactparameter_rather_than_inclination_to_fit": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_impactparameter_rather_than_inclination_to_fit"] == "n" + + +def test_comp_params_reads_use_psf_photometry_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_psf_photometry": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_psf_photometry"] == "n" + + +def test_comp_params_reads_use_aperture_photometry_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_aperture_photometry": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_aperture_photometry"] == "n" + + def test_comp_params_reads_use_adaptive_apertures_from_optional_info(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_plots.py b/tests/test_plots.py index 1bd1e5c6..2f3c6c82 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -2,9 +2,15 @@ matplotlib.use("Agg") import numpy as np +import matplotlib.pyplot as plt from matplotlib.axes import Axes -from exotic.plots import plot_adaptive_aperture_diagnostics, plot_obs_stats +from exotic.plots import ( + plot_adaptive_aperture_diagnostics, + plot_comp_star_candidate_lightcurve_fits, + plot_individual_comp_star_calibration_series, + plot_obs_stats, +) class DummyFit: @@ -48,6 +54,42 @@ def spy_plot(self, x, y, *args, **kwargs): assert (tmp_path / "temp" / "Observing_Statistics_target_2026-03-09.png").exists() +def test_plot_obs_stats_uses_supplied_background_series(tmp_path, monkeypatch): + fit = DummyFit() + psf_rows = np.arange(35, dtype=float).reshape(5, 7) + psf = {"target": psf_rows} + si = np.array([2, 0, 4, 1, 3]) + gi = np.array([True, False, True, True, True]) + relative_flux_mask = np.array([True, False, True, True]) + background_series = {"target": np.array([100.0, 200.0, 300.0, 400.0, 500.0])} + captured = [] + + original_plot = Axes.plot + + def spy_plot(self, x, y, *args, **kwargs): + captured.append((np.asarray(x), np.asarray(y))) + return original_plot(self, x, y, *args, **kwargs) + + monkeypatch.setattr(Axes, "plot", spy_plot) + + plot_obs_stats( + fit, + [], + psf, + si, + gi, + "Target", + str(tmp_path), + "2026-03-09", + relative_flux_mask=relative_flux_mask, + background_series=background_series, + ) + + assert len(captured) >= 6 + np.testing.assert_array_equal(captured[5][0], fit.time) + np.testing.assert_array_equal(captured[5][1], np.array([300.0, 200.0, 400.0])) + + def test_plot_adaptive_aperture_diagnostics_writes_outputs(tmp_path): plot_adaptive_aperture_diagnostics( times=np.array([1.0, 2.0, 3.0]), @@ -64,3 +106,59 @@ def test_plot_adaptive_aperture_diagnostics_writes_outputs(tmp_path): assert (tmp_path / "temp" / "AdaptiveApertureDiagnostics_Target_2026-03-09.png").exists() assert (tmp_path / "temp" / "AdaptiveApertureDiagnostics_Target_2026-03-09.pdf").exists() + + +def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): + plot_individual_comp_star_calibration_series( + times=np.array([1.0, 2.0, 3.0]), + comp_summaries=[ + { + "label": "Comp 1", + "selected": True, + "aggregate_score": 0.0012, + "pairwise_ratio_series": {"vs 2": np.array([1.0, 1.01, 0.99])}, + "ensemble_ratio_series": np.array([1.0, 1.005, 0.995]), + }, + { + "label": "Comp 2", + "selected": False, + "aggregate_score": 0.0025, + "pairwise_ratio_series": {"vs 1": np.array([0.99, 1.0, 1.01])}, + "ensemble_ratio_series": np.array([0.995, 1.0, 1.005]), + }, + ], + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + method_label="PSF photometry", + ) + + assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp1_Target_2026-03-09.png").exists() + assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp1_Target_2026-03-09.pdf").exists() + assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.png").exists() + assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.pdf").exists() + + +def test_plot_comp_star_candidate_lightcurve_fits_writes_outputs(tmp_path): + class DummyCandidateFit: + def plot_bestfit(self, phase=False): + fig, axes = plt.subplots(2, 1) + return fig, axes + + plot_comp_star_candidate_lightcurve_fits( + candidate_fit_summaries=[ + {"label": "Comp 1", "selected": True, "fit": DummyCandidateFit(), "res_std": 0.0012}, + {"label": "Comp 2", "selected": False, "fit": DummyCandidateFit(), "res_std": 0.0025}, + {"label": "Comp 3", "selected": False, "fit": None, "res_std": np.inf}, + ], + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + method_label="Aperture photometry (aper=5.00px, annulus=12.00px)", + ) + + assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.png").exists() + assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.pdf").exists() + assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.png").exists() + assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.pdf").exists() + assert not (tmp_path / "temp" / "CompStarLightCurveFit_Comp3_Target_2026-03-09.png").exists() From 626068494807517301b143cb5dbeba3a6d952529 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 11 Apr 2026 17:05:39 +1000 Subject: [PATCH 016/116] Center airmass fits and tighten triangle plot layout --- exotic/api/elca.py | 310 +++++++++++++++++++++++++++++++++--- exotic/exotic.py | 2 +- tests/test_elca_baseline.py | 220 ++++++++++++++++++++++++- 3 files changed, 511 insertions(+), 21 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 48051d94..828a6489 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -147,6 +147,30 @@ def get_flux_baseline(values, fallback=1.0): return fallback +def get_airmass_reference(airmass): + airmass = np.asarray(airmass, dtype=float) + finite_airmass = airmass[np.isfinite(airmass)] + if finite_airmass.size == 0: + return 0.0 + return float(np.nanmean(finite_airmass)) + + +def center_airmass(airmass, reference=None): + airmass = np.asarray(airmass, dtype=float) + if reference is None: + reference = get_airmass_reference(airmass) + return airmass - float(reference) + + +def airmass_trend(a2, airmass, reference=None): + return np.exp(np.asarray(a2, dtype=float) * center_airmass(airmass, reference=reference)) + + +def airmass_trend_grid(a2_values, airmass, reference=None): + centered = center_airmass(airmass, reference=reference) + return np.exp(np.outer(np.asarray(a2_values, dtype=float), centered)) + + def solve_flux_baseline(model, data, dataerr=None): model = np.asarray(model, dtype=float) data = np.asarray(data, dtype=float) @@ -199,7 +223,8 @@ def solve_flux_baseline_uncertainty(model, dataerr): def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000): n = int(n) a2 = np.random.normal(m_a2, sig_a2, n) - model = transit * np.exp(np.outer(a2, airmass)) + reference = get_airmass_reference(airmass) + model = transit * airmass_trend_grid(a2, airmass, reference=reference) weights = np.ones(transit.shape[0], dtype=float) if dataerr is not None: @@ -224,7 +249,7 @@ def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000): valid = np.isfinite(numer) & np.isfinite(denom) & (denom > 0) if not np.any(valid): - best_model = transit * np.exp(m_a2 * airmass) + best_model = transit * airmass_trend(m_a2, airmass, reference=reference) baseline = solve_flux_baseline(best_model, data, dataerr) return baseline, solve_flux_baseline_uncertainty(best_model, dataerr) @@ -233,7 +258,7 @@ def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000): baseline_unc = float(np.nanstd(baselines)) if baseline_unc == 0.0: - best_model = transit * np.exp(m_a2 * airmass) + best_model = transit * airmass_trend(m_a2, airmass, reference=reference) baseline_unc = solve_flux_baseline_uncertainty(best_model, dataerr) return baseline, baseline_unc @@ -310,6 +335,7 @@ def __init__( self.data = data self.dataerr = dataerr self.airmass = airmass + self.airmass_reference = get_airmass_reference(airmass) self.prior = prior self.bounds = bounds self.max_ncalls = 2e5 @@ -343,8 +369,15 @@ def _set_flux_baseline(self, value, error=0.0): self.parameters['a1'] = value self.errors['a1'] = error + def _get_airmass_reference(self): + return getattr(self, 'airmass_reference', get_airmass_reference(self.airmass)) + def _build_systematics_model(self, values): - return get_flux_baseline(values) * np.exp(values.get('a2', 0) * self.airmass) + return get_flux_baseline(values) * airmass_trend( + values.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) def _uses_internal_impact_parameter(self): return ( @@ -459,10 +492,10 @@ def _get_triangle_plot_samples(self): def _get_triangle_plot_display_spec(self, sampled_keys, sample_parameters, sample_errors, sample_points): if 'b' in sampled_keys: key = 'b' - label = 'Distance from fitted b (mirrored)' + label = r'$\Delta b$' elif 'inc' in sampled_keys: key = 'inc' - label = 'Distance from fitted Inc. [deg] (mirrored)' + label = r'$\Delta i$' else: return None @@ -482,18 +515,151 @@ def _get_triangle_plot_display_spec(self, sampled_keys, sample_parameters, sampl 'key': key, 'index': geometry_index, 'label': label, + 'center': center, 'mask_center': 0.0, 'mask_error': error, 'magnitude_samples': magnitude_samples, 'range': [-max_distance, max_distance], } + def _get_triangle_plot_geometry_overlay(self, display_spec, sample_points): + if display_spec is None: + return None + + geometry_index = display_spec['index'] + center = display_spec['center'] + offsets = np.asarray(sample_points[:, geometry_index], dtype=float) - center + left_offsets = offsets[offsets <= 0] + right_offsets = offsets[offsets >= 0] + + def mirrored_offsets(branch_offsets): + if branch_offsets.size == 0: + return np.array([], dtype=float) + return np.concatenate([branch_offsets, -branch_offsets]) + + return { + 'index': geometry_index, + 'left_count': left_offsets.size, + 'right_count': right_offsets.size, + 'left_mirrored': mirrored_offsets(left_offsets), + 'right_mirrored': mirrored_offsets(right_offsets), + } + + def _format_triangle_plot_geometry_value(self, value, error, suffix=''): + if value is None or not np.isfinite(value): + return f"n/a{suffix}" + if error is None or not np.isfinite(error) or error < 0: + return f"{round_to_2(value)}{suffix}" + return f"{round_to_2(value, error)} +- {round_to_2(error)}{suffix}" + + def _get_triangle_plot_geometry_summary(self, sampled_keys, sample_points): + bound_keys = list(self.bounds.keys()) + sample_points = np.asarray(sample_points, dtype=float) + sample_parameters = getattr(self, 'sample_parameters', {}) + sample_errors = getattr(self, 'sample_errors', {}) + + physical_samples = [ + self._physical_values_from_sample_point(point, bound_keys, sampled_keys) + for point in sample_points + ] + inc_samples = np.array([sample['inc'] for sample in physical_samples], dtype=float) + b_samples = np.array([ + sample.get('b', impact_parameter_from_inclination(sample, sample['inc'])) + for sample in physical_samples + ], dtype=float) + + inc_center = float(self.parameters.get('inc', np.nanmedian(inc_samples))) + inc_error = float(self.errors.get('inc', np.nanstd(inc_samples))) + if 'b' in sample_parameters: + b_center = float(sample_parameters['b']) + elif 'b' in self.parameters: + b_center = float(self.parameters['b']) + else: + b_center = float(np.nanmedian(b_samples)) + b_error = float(sample_errors.get('b', self.errors.get('b', np.nanstd(b_samples)))) + + title = ( + f"b={self._format_triangle_plot_geometry_value(b_center, b_error)}\n" + f"i={self._format_triangle_plot_geometry_value(inc_center, inc_error, ' deg')}" + ) + return { + 'b_center': b_center, + 'b_error': b_error, + 'inc_center': inc_center, + 'inc_error': inc_error, + 'title': title, + } + + def _smooth_triangle_plot_counts(self, counts): + counts = np.asarray(counts, dtype=float) + if counts.size <= 1 or not np.any(counts > 0): + return counts + + sigma_bins = max(1.0, counts.size / 18.0) + radius = max(1, int(np.ceil(3 * sigma_bins))) + grid = np.arange(-radius, radius + 1, dtype=float) + kernel = np.exp(-0.5 * (grid / sigma_bins) ** 2) + kernel /= np.sum(kernel) + return np.convolve(counts, kernel, mode='same') + + def _build_triangle_plot_geometry_curves(self, geometry_overlay, hist_range, bins_1d): + hist_range = np.sort(np.asarray(hist_range, dtype=float)) + bins_1d = max(1, int(bins_1d)) + edges = np.linspace(hist_range[0], hist_range[1], bins_1d + 1) + centers = 0.5 * (edges[:-1] + edges[1:]) + half_bin = 0.5 * (edges[1] - edges[0]) if edges.size > 1 else 0.0 + + def branch_curve(samples): + samples = np.asarray(samples, dtype=float) + counts, _ = np.histogram(samples, bins=edges) + support = np.zeros_like(counts, dtype=bool) + if samples.size > 0: + support = np.abs(centers) <= (np.max(np.abs(samples)) + half_bin) + return counts.astype(float), support + + left_curve, left_support = branch_curve(geometry_overlay['left_mirrored']) + right_curve, right_support = branch_curve(geometry_overlay['right_mirrored']) + + left_count = int(geometry_overlay.get('left_count', 0)) + right_count = int(geometry_overlay.get('right_count', 0)) + max_count = max(left_count, right_count) + min_count = min(left_count, right_count) + + if max_count == 0: + main_curve = np.zeros_like(centers, dtype=float) + elif min_count == 0 or (min_count / max_count) < 0.35: + main_curve = left_curve if left_count >= right_count else right_curve + else: + stacked = np.vstack([ + np.where(left_support, left_curve, np.nan), + np.where(right_support, right_curve, np.nan), + ]) + valid_counts = np.sum(np.isfinite(stacked), axis=0) + summed = np.nansum(stacked, axis=0) + main_curve = np.divide( + summed, + valid_counts, + out=np.zeros_like(summed, dtype=float), + where=valid_counts > 0, + ) + + main_curve = self._smooth_triangle_plot_counts(main_curve) + + return { + 'centers': centers, + 'left_curve': left_curve, + 'right_curve': right_curve, + 'main_curve': main_curve, + } + def _get_triangle_plot_payload(self): sampled_keys = getattr(self, 'sampled_keys', list(self.bounds.keys())) sample_parameters = getattr(self, 'sample_parameters', self.parameters) sample_errors = getattr(self, 'sample_errors', self.errors) sample_points, sample_logl = self._get_triangle_plot_samples() display_spec = self._get_triangle_plot_display_spec(sampled_keys, sample_parameters, sample_errors, sample_points) + geometry_overlay = self._get_triangle_plot_geometry_overlay(display_spec, sample_points) + geometry_summary = self._get_triangle_plot_geometry_summary(sampled_keys, sample_points) display_points = np.array(sample_points, copy=True) display_logl = np.array(sample_logl, copy=True) @@ -547,6 +713,7 @@ def _get_triangle_plot_payload(self): if display_spec is not None and key == display_spec['key']: label = display_spec['label'] + title = geometry_summary['title'] plot_range = display_spec['range'] center = display_spec['mask_center'] error = display_spec['mask_error'] @@ -562,6 +729,9 @@ def _get_triangle_plot_payload(self): 'display_points': display_points, 'display_logl': display_logl, 'mask_values': mask_values, + 'display_spec': display_spec, + 'geometry_overlay': geometry_overlay, + 'geometry_summary': geometry_summary, 'labels': labels, 'titles': titles, 'ranges': ranges, @@ -569,6 +739,71 @@ def _get_triangle_plot_payload(self): 'mask_errors': mask_errors, } + def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs=None, label_kwargs=None): + if not hasattr(fig, 'axes'): + return + + display_spec = payload.get('display_spec') + geometry_overlay = payload.get('geometry_overlay') + if display_spec is None or geometry_overlay is None: + return + + sampled_keys = payload['sampled_keys'] + if len(fig.axes) != len(sampled_keys) ** 2: + return + + axes = np.array(fig.axes).reshape((len(sampled_keys), len(sampled_keys))) + geometry_index = geometry_overlay['index'] + ax = axes[geometry_index, geometry_index] + hist_range = np.sort(payload['ranges'][geometry_index]) + bins_1d = int(max(25, np.round(np.sqrt(payload['display_points'].shape[0]) * 6))) + curves = self._build_triangle_plot_geometry_curves(geometry_overlay, hist_range, bins_1d) + + title = payload['titles'][geometry_index] + x_label = payload['labels'][geometry_index] + branch_left_color = '#6f8fcf' + branch_right_color = '#d79b9b' + title_kwargs = {} if title_kwargs is None else dict(title_kwargs) + label_kwargs = {} if label_kwargs is None else dict(label_kwargs) + + ax.cla() + ax.plot(curves['centers'], curves['main_curve'], color='black', linewidth=1.5, zorder=4) + ax.plot(curves['centers'], curves['left_curve'], color=branch_left_color, linestyle='--', + linewidth=0.75, alpha=0.75, zorder=3) + ax.plot(curves['centers'], curves['right_curve'], color=branch_right_color, linestyle='--', + linewidth=0.75, alpha=0.75, zorder=3) + ax.set_title(title, **title_kwargs) + ax.set_xlim(hist_range) + + max_y = max( + np.max(curves['main_curve']) if curves['main_curve'].size > 0 else 0.0, + np.max(curves['left_curve']) if curves['left_curve'].size > 0 else 0.0, + np.max(curves['right_curve']) if curves['right_curve'].size > 0 else 0.0, + ) + ax.set_ylim(0, 1.1 * max(max_y, 1e-6)) + ax.set_yticks([]) + + if geometry_index < len(sampled_keys) - 1: + ax.set_xticklabels([]) + else: + ax.set_xlabel(x_label, **label_kwargs) + + def _adjust_triangle_plot_layout(self, fig): + if not hasattr(fig, 'subplots_adjust'): + return + subplotpars = getattr(fig, 'subplotpars', None) + if subplotpars is None: + return + + fig.subplots_adjust( + left=subplotpars.left, + bottom=max(subplotpars.bottom, 0.10), + right=min(subplotpars.right, 0.95), + top=min(subplotpars.top, 0.955), + wspace=subplotpars.wspace, + hspace=subplotpars.hspace, + ) + def fit_LM(self): freekeys = list(self.bounds.keys()) boundarray = np.array([self.bounds[k] for k in freekeys]) @@ -592,7 +827,11 @@ def lc2min_airmass(pars): for i in range(len(pars)): self.prior[freekeys[i]] = pars[i] model = transit(self.time, self.prior) - model *= np.exp(self.prior.get('a2', 0) * self.airmass) + model *= airmass_trend( + self.prior.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) if self._has_free_flux_baseline(): model *= get_flux_baseline(self.prior) else: @@ -660,7 +899,11 @@ def create_fit_variables(self): self.dataerr, ) else: - systematics = self.transit * np.exp(self.parameters.get('a2', 0) * self.airmass) + systematics = self.transit * airmass_trend( + self.parameters.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) flux_scale = solve_flux_baseline(systematics, self.data, self.dataerr) flux_scale_err = self.errors.get('a0', self.errors.get('a1', solve_flux_baseline_uncertainty(systematics, self.dataerr))) self._set_flux_baseline(flux_scale, flux_scale_err) @@ -729,7 +972,11 @@ def loglike(pars): # chi-squared physical = physical_from_sample_point(pars) model = transit(self.time, physical) - model *= np.exp(physical.get('a2', 0) * self.airmass) + model *= airmass_trend( + physical.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) if self._has_free_flux_baseline(): model *= get_flux_baseline(physical) else: @@ -845,7 +1092,11 @@ def prior_transform(upars): )[0] test_values['a0'] = flux_scale test_values['a1'] = flux_scale - airmass = flux_scale * np.exp(test_values.get('a2', 0) * self.airmass) + airmass = flux_scale * airmass_trend( + test_values.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) residuals = self.data - (lightcurve * airmass) chis.append(np.sum(residuals ** 2)) physical_tests.append(test_values) @@ -1000,6 +1251,14 @@ def plot_triangle(self): if not np.any(mask3): mask3 = np.ones(len(chi2), dtype=bool) + label_kwargs = { + 'labelpad': 10, + } + title_kwargs = { + 'loc': 'left', + 'pad': 4, + } + fig = corner(payload['display_points'], labels=payload['labels'], bins=int(np.sqrt(payload['display_points'].shape[0])), @@ -1015,13 +1274,19 @@ def plot_triangle(self): 'vmax': np.percentile(chi2[mask3], 95), 'cmap': 'viridis' }, - label_kwargs={ - 'labelpad': 15, - }, + label_kwargs=label_kwargs, + title_kwargs=title_kwargs, hist_kwargs={ 'color': 'black', } ) + self._adjust_triangle_plot_layout(fig) + self._overlay_triangle_plot_geometry_histograms( + fig, + payload, + title_kwargs=title_kwargs, + label_kwargs=label_kwargs, + ) return fig # simultaneously fit multiple data sets with global and local parameters @@ -1144,7 +1409,10 @@ def loglike(pars): # compute model model = transit(self.lc_data[i]['time'], self.lc_data[i]['priors']) - model *= np.exp(self.lc_data[i]['priors'].get('a2', 0) * self.lc_data[i]['airmass']) + model *= airmass_trend( + self.lc_data[i]['priors'].get('a2', 0), + self.lc_data[i]['airmass'], + ) if has_explicit_flux_baseline(self.global_bounds) or has_explicit_flux_baseline(self.local_bounds[i]): model *= get_flux_baseline(self.lc_data[i]['priors']) else: @@ -1220,7 +1488,10 @@ def loglike(pars): # solve for the local baseline flux scale model = transit(self.lc_data[n]['time'], self.lc_data[n]['priors']) - airmass = np.exp(self.lc_data[n]['airmass'] * self.lc_data[n]['priors'].get('a2', 0)) + airmass = airmass_trend( + self.lc_data[n]['priors'].get('a2', 0), + self.lc_data[n]['airmass'], + ) if has_explicit_flux_baseline(self.global_bounds) or has_explicit_flux_baseline(self.local_bounds[n]): flux_scale = get_flux_baseline(self.lc_data[n]['priors']) flux_scale_err = self.lc_data[n]['errors'].get('a0', self.lc_data[n]['errors'].get('a1', 0)) @@ -1275,7 +1546,10 @@ def plot_bestfits(self): nmarker = next(markers) model = transit(self.lc_data[i]['time'], self.lc_data[i]['priors']) - airmass = np.exp(self.lc_data[i]['airmass'] * self.lc_data[i]['priors'].get('a2', 0)) + airmass = airmass_trend( + self.lc_data[i]['priors'].get('a2', 0), + self.lc_data[i]['airmass'], + ) detrend = self.lc_data[i]['flux'] / (model * airmass) if ax.ndim == 1: @@ -1553,7 +1827,7 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): 'omega': 120, # Arg of periastron 'tmid': 0.75, # Time of mid transit [day], 'a0': 50, # Baseline flux normalization - 'a2': 0., # trend = a0 * np.exp(a2 * airmass) + 'a2': 0., # trend = a0 * np.exp(a2 * (airmass - mean(airmass))) 'teff': 5000, 'tefferr': 50, @@ -1580,7 +1854,7 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): airmass = np.zeros(time.shape[0]) # GENERATE NOISY DATA - data = transit(time, prior) * prior['a0'] * np.exp(prior['a2'] * airmass) + data = transit(time, prior) * prior['a0'] * airmass_trend(prior['a2'], airmass) data += np.random.normal(0, prior['a0'] * 250e-6, len(time)) dataerr = np.random.normal(300e-6, 50e-6, len(time)) + np.random.normal(300e-6, 50e-6, len(time)) diff --git a/exotic/exotic.py b/exotic/exotic.py index 16873797..20c59fe9 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -3233,7 +3233,7 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs norm_flux_unc = oot_scatter * lc_fit.airmass_model[mask_ref] norm_flux_unc /= np.nanmedian(lc_fit.data[mask_ref]) - model = np.exp(lc_fit.parameters['a2'] * lc_fit.airmass_model[mask_ref]) + model = lc_fit.airmass_model[mask_ref] flux = lc_fit.data[mask_ref] detrended = flux / model diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 0b5fead1..2cda2f8b 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -2,6 +2,7 @@ import sys import types +import matplotlib.pyplot as plt import numpy as np import pytest @@ -89,6 +90,32 @@ def test_lc_fitter_recovers_explicit_a0_baseline(monkeypatch, tmp_path): assert np.median(fit.detrended[oot_mask]) == pytest.approx(1.0, abs=5e-4) +def test_lc_fitter_explicit_a0_tracks_mean_airmass_normalization(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + prior["a2"] = -0.35 + time = np.linspace(-0.03, 0.03, 301) + airmass = np.linspace(1.15, 1.85, len(time)) + dataerr = np.full_like(time, 1e-3) + true_a0 = 0.985 + data = true_a0 * elca.airmass_trend(prior["a2"], airmass) * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + assert fit.airmass_reference == pytest.approx(np.mean(airmass)) + assert fit.parameters["a0"] == pytest.approx(true_a0, abs=1e-4) + assert fit.parameters["a1"] == pytest.approx(true_a0, abs=1e-4) + + def test_lc_fitter_auto_solves_baseline_when_a0_is_not_free(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -114,6 +141,31 @@ def test_lc_fitter_auto_solves_baseline_when_a0_is_not_free(monkeypatch, tmp_pat assert np.median(fit.detrended[oot_mask]) == pytest.approx(1.0, abs=5e-4) +def test_lc_fitter_auto_solves_mean_airmass_normalization(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + prior["a2"] = 0.22 + time = np.linspace(-0.03, 0.03, 301) + airmass = np.linspace(1.05, 1.75, len(time)) + dataerr = np.full_like(time, 1e-3) + true_a0 = 1.018 + data = true_a0 * elca.airmass_trend(prior["a2"], airmass) * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005]}, + mode="lm", + verbose=False, + ) + + assert fit.parameters["a0"] == pytest.approx(true_a0, abs=1e-4) + assert fit.parameters["a1"] == pytest.approx(true_a0, abs=1e-4) + + def test_lc_fitter_rejects_redundant_a0_and_a1_bounds(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -174,6 +226,9 @@ def fake_corner(*args, **kwargs): captured["points"] = args[0] captured["labels"] = kwargs["labels"] captured["range"] = kwargs["range"] + captured["titles"] = kwargs["titles"] + captured["title_kwargs"] = kwargs["title_kwargs"] + captured["label_kwargs"] = kwargs["label_kwargs"] return "figure" monkeypatch.setattr(elca, "corner", fake_corner) @@ -184,6 +239,7 @@ def fake_corner(*args, **kwargs): "inc": [84.0, 90.0], "a0": [0.95, 1.05], } + fit.prior = make_prior() fit.quantiles = {"rprs": [], "inc": [], "a0": []} fit.parameters = {"rprs": 0.10, "inc": 88.42, "a0": 0.94962} fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} @@ -208,7 +264,7 @@ def fake_corner(*args, **kwargs): fig = fit.plot_triangle() assert fig == "figure" - assert captured["labels"][1] == "Distance from fitted Inc. [deg] (mirrored)" + assert captured["labels"][1] == r"$\Delta i$" assert captured["range"][0][0] == pytest.approx(0.05) assert captured["range"][0][1] == pytest.approx(0.125) expected_inc_distance_limit = np.max(np.abs(np.array([84.67, 90.0]) - fit.parameters["inc"])) @@ -220,6 +276,10 @@ def fake_corner(*args, **kwargs): expected_inc_distance = np.abs(points[:, 1] - fit.parameters["inc"]) np.testing.assert_allclose(captured["points"][:5, 1], expected_inc_distance) np.testing.assert_allclose(captured["points"][5:, 1], -expected_inc_distance) + assert captured["titles"][1].startswith("b=") + assert "\ni=" in captured["titles"][1] + assert captured["title_kwargs"]["loc"] == "left" + assert captured["label_kwargs"]["labelpad"] == 10 def test_internal_impact_parameter_transform_round_trips_inclination(monkeypatch, tmp_path): @@ -328,6 +388,8 @@ def fake_corner(*args, **kwargs): captured["points"] = args[0] captured["labels"] = kwargs["labels"] captured["range"] = kwargs["range"] + captured["titles"] = kwargs["titles"] + captured["label_kwargs"] = kwargs["label_kwargs"] return "figure" monkeypatch.setattr(elca, "corner", fake_corner) @@ -338,6 +400,7 @@ def fake_corner(*args, **kwargs): "inc": [84.0, 90.0], "a0": [0.95, 1.05], } + fit.prior = make_prior() fit.sampled_keys = ["rprs", "b", "a0"] fit.sample_bounds = { "rprs": [0.0, 0.125], @@ -369,10 +432,163 @@ def fake_corner(*args, **kwargs): fig = fit.plot_triangle() assert fig == "figure" - assert captured["labels"][1] == "Distance from fitted b (mirrored)" + assert captured["labels"][1] == r"$\Delta b$" assert captured["range"][1][0] == pytest.approx(-0.25) assert captured["range"][1][1] == pytest.approx(0.25) assert captured["points"].shape == (10, 3) expected_b_distance = np.abs(points[:, 1] - fit.sample_parameters["b"]) np.testing.assert_allclose(captured["points"][:5, 1], expected_b_distance) np.testing.assert_allclose(captured["points"][5:, 1], -expected_b_distance) + assert captured["titles"][1].startswith("b=") + assert "\ni=" in captured["titles"][1] + assert captured["label_kwargs"]["labelpad"] == 10 + + +def test_triangle_payload_tracks_left_and_right_geometry_branches_for_inclination(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.125], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.parameters = {"rprs": 0.10, "inc": 88.42, "a0": 0.94962} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} + points = np.array( + [ + [0.099, 88.30, 0.9501], + [0.101, 88.55, 0.9502], + [0.102, 88.10, 0.9515], + [0.098, 88.70, 0.9520], + [0.100, 88.40, 0.9508], + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.array([-5.0, -4.0, -4.5, -5.5, -4.2]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + np.testing.assert_allclose( + payload["geometry_overlay"]["left_mirrored"], + np.array([-0.12, -0.32, -0.02, 0.12, 0.32, 0.02]), + atol=1e-12, + ) + np.testing.assert_allclose( + payload["geometry_overlay"]["right_mirrored"], + np.array([0.13, 0.28, -0.13, -0.28]), + atol=1e-12, + ) + + +def test_triangle_payload_tracks_left_and_right_geometry_branches_for_impact_parameter(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.125], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.sampled_keys = ["rprs", "b", "a0"] + fit.sample_bounds = { + "rprs": [0.0, 0.125], + "b": [0.0, 1.25434156], + "a0": [0.95, 1.05], + } + fit.sample_parameters = {"rprs": 0.10, "b": 0.314, "a0": 0.94962} + fit.sample_errors = {"rprs": 0.01, "b": 0.05, "a0": 0.00394} + fit.parameters = {"rprs": 0.10, "inc": 88.5, "a0": 0.94962} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} + points = np.array( + [ + [0.099, 0.300, 0.9501], + [0.101, 0.330, 0.9502], + [0.102, 0.290, 0.9515], + [0.098, 0.360, 0.9520], + [0.100, 0.314, 0.9508], + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.array([-5.0, -4.0, -4.5, -5.5, -4.2]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + np.testing.assert_allclose( + payload["geometry_overlay"]["left_mirrored"], + np.array([-0.014, -0.024, 0.0, 0.014, 0.024, -0.0]), + atol=1e-12, + ) + np.testing.assert_allclose( + payload["geometry_overlay"]["right_mirrored"], + np.array([0.016, 0.046, 0.0, -0.016, -0.046, -0.0]), + atol=1e-12, + ) + + +def test_triangle_geometry_curves_fall_back_to_surviving_branch(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + curves = fit._build_triangle_plot_geometry_curves( + { + "left_count": 0, + "right_count": 4, + "left_mirrored": np.array([], dtype=float), + "right_mirrored": np.array([0.05, 0.10, 0.15, -0.05, -0.10, -0.15], dtype=float), + }, + [-0.3, 0.3], + 31, + ) + + np.testing.assert_allclose(curves["main_curve"], fit._smooth_triangle_plot_counts(curves["right_curve"])) + assert np.allclose(curves["left_curve"], 0.0) + + +def test_triangle_geometry_overlay_reuses_shared_title_and_label_kwargs(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fig, axes = plt.subplots(2, 2) + payload = { + "sampled_keys": ["rprs", "b"], + "display_points": np.zeros((10, 2)), + "display_spec": {"index": 1}, + "geometry_overlay": { + "index": 1, + "left_count": 2, + "right_count": 2, + "left_mirrored": np.array([-0.1, 0.1]), + "right_mirrored": np.array([-0.2, 0.2]), + }, + "ranges": [[0.0, 0.1], [-0.3, 0.3]], + "titles": ["rprs", "b=0.32 +- 0.18\ni=88.5 +- 1.35 deg"], + "labels": ["rprs", r"$\Delta b$"], + } + + fit._overlay_triangle_plot_geometry_histograms( + fig, + payload, + title_kwargs={"loc": "left", "pad": 4, "fontsize": 12}, + label_kwargs={"labelpad": 10}, + ) + + ax = axes[1, 1] + assert ax.title.get_fontsize() == pytest.approx(12.0) + assert ax.xaxis.label.get_text() == r"$\Delta b$" + assert ax.xaxis.labelpad == pytest.approx(10.0) + plt.close(fig) From fd06ef87f8ff755fab46100808ec924d8ef73db6 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 11 Apr 2026 17:53:01 +1000 Subject: [PATCH 017/116] Remove aperture labels from PSF FOV legend --- exotic/plots.py | 9 ++++++++- tests/test_plots.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/exotic/plots.py b/exotic/plots.py index 1fcc6c4c..cd36fba1 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -96,7 +96,14 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc path_effects=[path_effects.withStroke(linewidth=2, foreground='black')]) handles = [] - label_aper = f"{opt_method} Photometry\n(Min Aper: {abs(min_aper_fov):.2f} px)\n(Min Annulus: {min_annulus_fov:.2f} px)" + if opt_method == "PSF": + label_aper = "PSF Photometry" + else: + label_aper = ( + f"{opt_method} Photometry\n" + f"(Min Aper: {abs(min_aper_fov):.2f} px)\n" + f"(Min Annulus: {min_annulus_fov:.2f} px)" + ) if opt_method == "Aperture": aperture_line = Line2D([], [], color=outer_circle_color, linestyle='-', label=label_aper) diff --git a/tests/test_plots.py b/tests/test_plots.py index 2f3c6c82..a0d9181a 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -6,6 +6,7 @@ from matplotlib.axes import Axes from exotic.plots import ( + plot_fov, plot_adaptive_aperture_diagnostics, plot_comp_star_candidate_lightcurve_fits, plot_individual_comp_star_calibration_series, @@ -108,6 +109,40 @@ def test_plot_adaptive_aperture_diagnostics_writes_outputs(tmp_path): assert (tmp_path / "temp" / "AdaptiveApertureDiagnostics_Target_2026-03-09.pdf").exists() +def test_plot_fov_psf_legend_omits_aperture_annulus_text(tmp_path, monkeypatch): + labels = [] + + original_legend = plt.legend + + def spy_legend(*args, **kwargs): + legend = original_legend(*args, **kwargs) + labels.extend(text.get_text() for text in legend.get_texts()) + return legend + + monkeypatch.setattr(plt, "legend", spy_legend) + + plot_fov( + aper=20.0, + annulus=60.0, + sigma=4.0, + x_targ=50.0, + y_targ=60.0, + x_ref=90.0, + y_ref=100.0, + image=np.ones((200, 200)), + image_scale="Image scale in arcsec/pixel: 0.53", + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + opt_method="PSF", + min_aper_fov=20.44, + min_annulus_fov=61.31, + ) + + assert labels + assert set(labels) == {"PSF Photometry"} + + def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): plot_individual_comp_star_calibration_series( times=np.array([1.0, 2.0, 3.0]), From 10cdd6f2d60a5187935a453dd797fd749e6f7606 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sun, 12 Apr 2026 20:46:33 +1000 Subject: [PATCH 018/116] Fix transit phase plots to use the full dataset span --- exotic/api/elca.py | 124 ++++++++--- exotic/exotic.py | 335 ++++++++++++++++++++++++++++- exotic/exotic_gui.py | 5 + exotic/inputs.py | 5 + exotic/plots.py | 5 +- inits.json | 2 + tests/test_centroid_wcs.py | 55 +++++ tests/test_elca_baseline.py | 85 ++++++++ tests/test_exotic_proper_motion.py | 7 + tests/test_inputs.py | 30 +++ 10 files changed, 617 insertions(+), 36 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 828a6489..50826922 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -131,6 +131,38 @@ def get_phase(times, per, tmid): return (times - tmid + 0.25 * per) / per % 1 - 0.25 +def normalize_time_range(time_range): + if time_range is None: + return None + + values = np.asarray(time_range, dtype=float).reshape(-1) + finite = values[np.isfinite(values)] + if finite.size == 0: + return None + + return float(np.min(finite)), float(np.max(finite)) + + +def get_plot_phase(times, per, tmid, reference_times=None): + times = np.asarray(times, dtype=float) + if not np.isfinite(per) or per == 0: + return times * np.nan + + raw_phase = (times - tmid) / per + + reference_range = normalize_time_range(reference_times) + if reference_range is None: + finite_phase = raw_phase[np.isfinite(raw_phase)] + if finite_phase.size == 0: + return raw_phase + reference_epoch = float(np.rint(0.5 * (np.min(finite_phase) + np.max(finite_phase)))) + else: + ref_phase = (np.asarray(reference_range, dtype=float) - tmid) / per + reference_epoch = float(np.rint(np.mean(ref_phase))) + + return raw_phase - reference_epoch + + def fallback_flux_baseline(): return 1.0 @@ -372,6 +404,29 @@ def _set_flux_baseline(self, value, error=0.0): def _get_airmass_reference(self): return getattr(self, 'airmass_reference', get_airmass_reference(self.airmass)) + def _get_plot_time_range(self): + plot_time_range = normalize_time_range(getattr(self, 'plot_time_range', None)) + if plot_time_range is not None: + return plot_time_range + return normalize_time_range(self.time) + + def _update_plot_geometry(self): + plot_time_range = self._get_plot_time_range() + self.phase = get_plot_phase(self.time, self.parameters['per'], self.parameters['tmid'], plot_time_range) + + if plot_time_range is None: + self.time_upsample = np.linspace(min(self.time), max(self.time), 1000) + else: + self.time_upsample = np.linspace(plot_time_range[0], plot_time_range[1], 1000) + + self.transit_upsample = transit(self.time_upsample, self.parameters) + self.phase_upsample = get_plot_phase( + self.time_upsample, + self.parameters['per'], + self.parameters['tmid'], + plot_time_range, + ) + def _build_systematics_model(self, values): return get_flux_baseline(values) * airmass_trend( values.get('a2', 0), @@ -880,11 +935,8 @@ def lc2min_airmass(pars): self.create_fit_variables() def create_fit_variables(self): - self.phase = get_phase(self.time, self.parameters['per'], self.parameters['tmid']) self.transit = transit(self.time, self.parameters) - self.time_upsample = np.linspace(min(self.time), max(self.time), 1000) - self.transit_upsample = transit(self.time_upsample, self.parameters) - self.phase_upsample = get_phase(self.time_upsample, self.parameters['per'], self.parameters['tmid']) + self._update_plot_geometry() if np.ndim(self.airmass) != 2: if self._has_free_flux_baseline(): flux_scale = get_flux_baseline(self.parameters) @@ -1188,7 +1240,7 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) axs[1].plot(self.phase, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=0.2, label=r'$\sigma$ = {:.2f} %'.format(np.std(self.residuals / np.median(self.data) * 1e2))) axs[1].plot(bt2 / self.parameters['per'], br2, 'bs', alpha=1, zorder=2) - axs[1].set_xlim([min(self.phase), max(self.phase)]) + axs[1].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) axs[1].set_xlabel("Phase", fontsize=14) si = np.argsort(self.phase) @@ -1198,14 +1250,14 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) # axs[0].plot(self.phase[si], self.transit[si], 'r-', zorder=3, label=lclabel) sii = np.argsort(self.phase_upsample) axs[0].plot(self.phase_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel) - axs[0].set_xlim([min(self.phase), max(self.phase)]) + axs[0].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) axs[0].set_xlabel("Phase ", fontsize=14) else: bt, br, _ = time_bin(self.time, self.residuals / np.median(self.data) * 1e2, bin_dt) axs[1].plot(self.time, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=0.2, label=r'$\sigma$ = {:.2f} %'.format(np.std(self.residuals / np.median(self.data) * 1e2))) axs[1].plot(bt, br, 'bs', alpha=1, zorder=2, label=r'$\sigma$ = {:.2f} %'.format(np.std(br))) - axs[1].set_xlim([min(self.time), max(self.time)]) + axs[1].set_xlim([min(self.time_upsample), max(self.time_upsample)]) axs[1].set_xlabel("Time [day]", fontsize=14) bt, bf, bs = time_bin(self.time, self.detrended, bin_dt) @@ -1213,7 +1265,7 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) sii = np.argsort(self.time_upsample) axs[0].errorbar(bt, bf, yerr=bs, alpha=1, zorder=2, color='blue', ls='none', marker='s') axs[0].plot(self.time_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel) - axs[0].set_xlim([min(self.time), max(self.time)]) + axs[0].set_xlim([min(self.time_upsample), max(self.time_upsample)]) axs[0].set_xlabel("Time [day]", fontsize=14) axs[0].get_xaxis().set_visible(False) @@ -1506,9 +1558,23 @@ def loglike(pars): self.lc_data[n]['detrend'] = self.lc_data[n]['flux'] / (airmass * flux_scale) # phase - self.lc_data[n]['phase'] = get_phase(self.lc_data[n]['time'], self.lc_data[n]['priors']['per'], self.lc_data[n]['priors']['tmid']) - self.lc_data[n]['time_upsample'] = np.linspace(min(self.lc_data[n]['time']), max(self.lc_data[n]['time']), 1000) - self.lc_data[n]['phase_upsample'] = get_phase(self.lc_data[n]['time_upsample'], self.lc_data[n]['priors']['per'], self.lc_data[n]['priors']['tmid']) + plot_time_range = normalize_time_range(self.lc_data[n].get('plot_time_range')) + if plot_time_range is None: + plot_time_range = normalize_time_range(self.lc_data[n]['time']) + self.lc_data[n]['plot_time_range'] = plot_time_range + self.lc_data[n]['phase'] = get_plot_phase( + self.lc_data[n]['time'], + self.lc_data[n]['priors']['per'], + self.lc_data[n]['priors']['tmid'], + plot_time_range, + ) + self.lc_data[n]['time_upsample'] = np.linspace(plot_time_range[0], plot_time_range[1], 1000) + self.lc_data[n]['phase_upsample'] = get_plot_phase( + self.lc_data[n]['time_upsample'], + self.lc_data[n]['priors']['per'], + self.lc_data[n]['priors']['tmid'], + plot_time_range, + ) self.lc_data[n]['transit_upsample'] = transit(self.lc_data[n]['time_upsample'], self.lc_data[n]['priors']) # create an average value from all the local fits, used for plotting final best fit @@ -1638,6 +1704,7 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p alldata = { 'time': [], + 'phase': [], 'flux': [], 'detrend': [], 'ferr': [], @@ -1648,12 +1715,13 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p ncolor = next(colors) nmarker = next(markers) alldata['time'].extend(self.lc_data[n]['time'].tolist()) + alldata['phase'].extend(self.lc_data[n]['phase'].tolist()) alldata['detrend'].extend(self.lc_data[n]['detrend'].tolist()) alldata['flux'].extend(self.lc_data[n]['flux'].tolist()) alldata['ferr'].extend(self.lc_data[n]['ferr'].tolist()) alldata['residuals'].extend(self.lc_data[n]['residuals'].tolist()) - phase = get_phase(self.lc_data[n]['time'], self.parameters['per'], self.lc_data[n]['priors']['tmid']) + phase = self.lc_data[n]['phase'] si = np.argsort(phase) #bt2, br2, _ = time_bin(phase[si]*self.parameters['per'], self.lc_data[n]['residuals'][si]/np.median(self.lc_data[n]['flux'])*1e2, bin_dt) @@ -1675,8 +1743,8 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p label=r'{}: {:.2f} %'.format(self.lc_data[n].get('name',''),np.std(self.lc_data[n]['residuals']/np.median(self.lc_data[n]['flux'])*1e2))) # replace min and max for upsampled lc model - minp = min(minp, min(phase)) - maxp = max(maxp, max(phase)) + minp = min(minp, min(self.lc_data[n]['phase_upsample'])) + maxp = max(maxp, max(self.lc_data[n]['phase_upsample'])) min_std = min(min_std, np.std(self.lc_data[n]['residuals']/np.median(self.lc_data[n]['flux']))) # plot individual best fit models @@ -1687,7 +1755,7 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p for k in alldata.keys(): alldata[k] = np.array(alldata[k]) - phase = get_phase(alldata['time'], self.parameters['per'], self.lc_data[n]['priors']['tmid']) + phase = alldata['phase'] si = np.argsort(phase) bt, br, _ = time_bin(phase[si]*self.parameters['per'], alldata['residuals'][si]/np.median(alldata['flux']), 2*bin_dt) bt, bf, bs = time_bin(phase[si]*self.parameters['per'], alldata['detrend'][si], 2*bin_dt) @@ -1700,12 +1768,10 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p axs[1].plot(bt/self.parameters['per'],br*1e2,color='white',ls='none',marker='o',ms=11,markeredgecolor='black') # best fit model - self.time_upsample = np.linspace(minp*self.parameters['per']+self.parameters['tmid'], - maxp*self.parameters['per']+self.parameters['tmid'], 10000) + self.phase_upsample = np.linspace(minp, maxp, 10000) + self.time_upsample = self.parameters['tmid'] + self.phase_upsample * self.parameters['per'] self.transit_upsample = transit(self.time_upsample, self.parameters) - self.phase_upsample = get_phase(self.time_upsample, self.parameters['per'], self.parameters['tmid']) - sii = np.argsort(self.phase_upsample) - axs[0].plot(self.phase_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel, lw=3) + axs[0].plot(self.phase_upsample, self.transit_upsample, 'r-', zorder=3, label=lclabel, lw=3) # set up axes limits axs[0].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) @@ -1718,8 +1784,8 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p # compute average min and max for all the data mins = []; maxs = [] for n in range(len(self.lc_data)): - mins.append(min(self.lc_data[n]['phase'])) - maxs.append(max(self.lc_data[n]['phase'])) + mins.append(min(self.lc_data[n]['phase_upsample'])) + maxs.append(max(self.lc_data[n]['phase_upsample'])) # set up phase limits if isinstance(phase_limits, str): @@ -1782,7 +1848,7 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): ncolor = next(colors) nmarker = next(markers) - phase = get_phase(self.lc_data[n]['time'], self.parameters['per'], self.lc_data[n]['priors']['tmid']) + phase = self.lc_data[n]['phase'] si = np.argsort(phase) bt2, br2, _ = time_bin(phase[si]*self.parameters['per'], self.lc_data[n]['residuals'][si]/np.median(self.lc_data[n]['flux'])*1e2, bin_dt) @@ -1795,17 +1861,15 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): ax.errorbar(bt2/self.lc_data[n]['priors']['per'],bf2,yerr=bs,alpha=1,zorder=2,color=ncolor,ls='none',marker=nmarker) # replace min and max for upsampled lc model - minp = min(minp, min(phase)) - maxp = max(maxp, max(phase)) + minp = min(minp, min(self.lc_data[n]['phase_upsample'])) + maxp = max(maxp, max(self.lc_data[n]['phase_upsample'])) min_std = min(min_std, np.std(self.lc_data[n]['residuals']/np.median(self.lc_data[n]['flux']))) # best fit model - self.time_upsample = np.linspace(minp*self.parameters['per']+self.parameters['tmid'], - maxp*self.parameters['per']+self.parameters['tmid'], 10000) + self.phase_upsample = np.linspace(minp, maxp, 10000) + self.time_upsample = self.parameters['tmid'] + self.phase_upsample * self.parameters['per'] self.transit_upsample = transit(self.time_upsample, self.parameters) - self.phase_upsample = get_phase(self.time_upsample, self.parameters['per'], self.parameters['tmid']) - sii = np.argsort(self.phase_upsample) - ax.plot(self.phase_upsample[sii], self.transit_upsample[sii]-n*dy, ls='-', color=ncolor, zorder=3, label=self.lc_data[n].get('name','')) + ax.plot(self.phase_upsample, self.transit_upsample-n*dy, ls='-', color=ncolor, zorder=3, label=self.lc_data[n].get('name','')) ax.set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) ax.set_xlabel("Phase ", fontsize=14) diff --git a/exotic/exotic.py b/exotic/exotic.py index 20c59fe9..f053140c 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -93,7 +93,7 @@ # scipy imports from scipy.optimize import least_squares from scipy.signal import savgol_filter -from scipy.ndimage import binary_erosion, gaussian_filter +from scipy.ndimage import binary_erosion, gaussian_filter, maximum_filter, median_filter from skimage.registration import phase_cross_correlation from skimage.transform import SimilarityTransform # error handling for scraper @@ -170,6 +170,21 @@ COMPARISON_STAR_COVERAGE_SIGMA = 3.0 COMPARISON_STAR_COVERAGE_MAX_ITERS = 10 OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 +BAD_PIXEL_DETECTION_FRACTION = 0.30 +BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 +BAD_PIXEL_PROGRESS_LOG_INTERVAL = 25 +BAD_PIXEL_OUTLIER_SIGMA = 8.0 +BAD_PIXEL_GLOBAL_SIGMA = 3.0 +BAD_PIXEL_ISOLATION_SIGMA = 5.0 +BAD_PIXEL_ISOLATION_RATIO = 2.0 +BAD_PIXEL_COUNTS_FILENAME = "BadPixelDetectionCounts.fits" +BAD_PIXEL_MASK_FILENAME = "BadPixelMask.fits" +BAD_PIXEL_NEIGHBOR_FOOTPRINT = np.array( + [[1, 1, 1], + [1, 0, 1], + [1, 1, 1]], + dtype=bool, +) def airmass_span(airmass): @@ -397,6 +412,27 @@ def should_use_aperture_photometry(config_value): return True +def should_detect_bad_pixels_before_photometry(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'detect_bad_pixels_before_photometry' value; keeping bad-pixel precheck enabled.", + warn=True, + ) + return True + + def is_adaptive_aperture_mode_enabled(config_value): if config_value is None: return False @@ -678,6 +714,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( disable_vertical_flux_normalization=False, detrend_on_outoftransit_baseline=True, use_impactparameter_rather_than_inclination_to_fit=True, + plot_time_range=None, ): fit = lc_fitter( times, @@ -690,6 +727,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) + fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) if not detrend_on_outoftransit_baseline: @@ -743,6 +781,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) + refit = apply_plot_time_range(refit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, airmass, skip_airmass_fit, note=airmass_skip_note) annotate_out_of_transit_baseline_detrending( refit, @@ -1104,6 +1143,23 @@ def apply_lightcurve_mask(lightcurve, mask, sort_index=None): setattr(lightcurve, attr, array_values[mask]) +def apply_plot_time_range(lightcurve, time_values): + if lightcurve is None: + return lightcurve + + values = np.asarray(time_values, dtype=float).reshape(-1) + finite = values[np.isfinite(values)] + if finite.size == 0: + return lightcurve + + lightcurve.plot_time_range = (float(np.min(finite)), float(np.max(finite))) + updater = getattr(lightcurve, "_update_plot_geometry", None) + if callable(updater): + updater() + + return lightcurve + + def exp_offset(hdr, time_unit, exp): """Returns exposure offset (in days) of more than 0 if headers reveals the time was estimated at the start of the exposure rather than the middle @@ -2542,6 +2598,209 @@ def load_image_data(file_name): return image_data +def persistent_bad_pixel_count_threshold(frame_count, minimum_fraction=BAD_PIXEL_DETECTION_FRACTION): + if frame_count <= 0: + return 1 + return max(1, int(np.floor(float(minimum_fraction) * frame_count)) + 1) + + +def detect_frame_bad_pixels(image_data, + outlier_sigma=BAD_PIXEL_OUTLIER_SIGMA, + isolation_sigma=BAD_PIXEL_ISOLATION_SIGMA, + isolation_ratio=BAD_PIXEL_ISOLATION_RATIO): + values = np.asarray(image_data, dtype=float) + if values.ndim != 2 or values.size == 0: + return np.zeros(values.shape[:2], dtype=bool) + + finite_mask = np.isfinite(values) + if np.count_nonzero(finite_mask) < BAD_PIXEL_NEIGHBOR_FOOTPRINT.sum(): + return np.zeros(values.shape, dtype=bool) + + working = np.array(values, copy=True) + frame_median = bn.nanmedian(working[finite_mask]) + if not np.isfinite(frame_median): + frame_median = 0.0 + working[~finite_mask] = frame_median + + neighbor_median = median_filter(working, footprint=BAD_PIXEL_NEIGHBOR_FOOTPRINT, mode='mirror') + neighbor_max = maximum_filter(working, footprint=BAD_PIXEL_NEIGHBOR_FOOTPRINT, mode='mirror') + residual = working - neighbor_median + + global_scatter = robust_scatter(residual[finite_mask]) + if not np.isfinite(global_scatter) or global_scatter <= 0: + global_scatter = robust_scatter(working[finite_mask]) + if not np.isfinite(global_scatter) or global_scatter <= 0: + return np.zeros(values.shape, dtype=bool) + + local_scatter = 1.4826 * median_filter( + np.abs(residual), + footprint=BAD_PIXEL_NEIGHBOR_FOOTPRINT, + mode='mirror', + ) + diff_threshold = np.maximum(outlier_sigma * local_scatter, BAD_PIXEL_GLOBAL_SIGMA * global_scatter) + isolation_threshold = max(isolation_sigma * global_scatter, 1.0) + neighbor_scale = np.maximum(np.abs(neighbor_max), 1.0) + + with np.errstate(divide='ignore', invalid='ignore'): + isolation_ratio_values = np.divide(np.abs(working), neighbor_scale) + + return ( + finite_mask + & (residual > diff_threshold) + & ((working - neighbor_max) > isolation_threshold) + & (isolation_ratio_values >= isolation_ratio) + ) + + +def build_persistent_bad_pixel_map(inputfiles, frame_loader, save_directory=None, + minimum_fraction=BAD_PIXEL_DETECTION_FRACTION, + minimum_frames=BAD_PIXEL_PRECHECK_MIN_FRAMES): + inputfiles = list(inputfiles) + total_files = len(inputfiles) + if total_files < minimum_frames: + log_info( + f"Bad-pixel precheck skipped: only {total_files} frame(s); need at least {minimum_frames} frames.", + ) + return None + + detection_counts = None + scanned_files = 0 + + for index, file_name in enumerate(inputfiles): + plateStatus.setCurrentFilename(file_name) + try: + frame_data = frame_loader(file_name) + except Exception as exc: + log_info( + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} ({exc}).", + warn=True, + ) + continue + + frame_mask = detect_frame_bad_pixels(frame_data) + if frame_mask.ndim != 2: + log_info( + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} because the frame is not 2-D.", + warn=True, + ) + continue + + if detection_counts is None: + detection_counts = np.zeros(frame_mask.shape, dtype=np.uint32) + elif detection_counts.shape != frame_mask.shape: + log_info( + "Warning: skipping bad-pixel precheck for " + f"{_display_filename(file_name)} because its shape {frame_mask.shape} does not match " + f"the reference frame shape {detection_counts.shape}.", + warn=True, + ) + continue + + detection_counts += frame_mask.astype(np.uint32) + scanned_files += 1 + + completed = index + 1 + if completed == total_files or completed % BAD_PIXEL_PROGRESS_LOG_INTERVAL == 0: + log_info(f"Bad-pixel precheck progress: {completed}/{total_files}") + + if detection_counts is None or scanned_files < minimum_frames: + log_info( + f"Bad-pixel precheck skipped: only {scanned_files} usable frame(s); need at least {minimum_frames}.", + warn=True, + ) + return None + + required_count = persistent_bad_pixel_count_threshold(scanned_files, minimum_fraction) + bad_pixel_mask = detection_counts >= required_count + coord_y, coord_x = np.nonzero(bad_pixel_mask) + + counts_path = None + mask_path = None + if save_directory is not None: + temp_dir = Path(save_directory) / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + counts_path = temp_dir / BAD_PIXEL_COUNTS_FILENAME + mask_path = temp_dir / BAD_PIXEL_MASK_FILENAME + fits.writeto(counts_path, detection_counts.astype(np.int32), overwrite=True) + fits.writeto(mask_path, bad_pixel_mask.astype(np.uint8), overwrite=True) + + threshold_percent = minimum_fraction * 100.0 + summary = ( + f"Bad-pixel precheck: identified {int(np.count_nonzero(bad_pixel_mask))} persistent bad pixel(s) " + f"after scanning {scanned_files}/{total_files} frame(s) with a >{threshold_percent:g}% recurrence threshold " + f"({required_count}+ detections)." + ) + if counts_path is not None and mask_path is not None: + summary += f" Saved {counts_path.name} and {mask_path.name} to temp/." + log_info(summary) + + return { + 'count_image': detection_counts, + 'mask': bad_pixel_mask, + 'coord_y': coord_y.astype(int), + 'coord_x': coord_x.astype(int), + 'required_count': required_count, + 'minimum_fraction': float(minimum_fraction), + 'frame_count': scanned_files, + 'counts_path': counts_path, + 'mask_path': mask_path, + } + + +def repair_bad_pixels_in_frame(image_data, bad_pixel_reference): + if bad_pixel_reference is None: + return image_data + + coord_y = bad_pixel_reference.get('coord_y') + coord_x = bad_pixel_reference.get('coord_x') + if coord_y is None or coord_x is None: + mask = np.asarray(bad_pixel_reference.get('mask'), dtype=bool) + if mask.size == 0: + return image_data + coord_y, coord_x = np.nonzero(mask) + + coord_y = np.asarray(coord_y, dtype=int) + coord_x = np.asarray(coord_x, dtype=int) + if coord_y.size == 0 or coord_x.size == 0: + return image_data + + repaired = np.array(image_data, dtype=float, copy=True) + valid_coords = ( + (coord_y >= 0) & (coord_y < repaired.shape[0]) + & (coord_x >= 0) & (coord_x < repaired.shape[1]) + ) + if not np.any(valid_coords): + return repaired + + coord_y = coord_y[valid_coords] + coord_x = coord_x[valid_coords] + repaired[coord_y, coord_x] = np.nan + + padded = np.pad(repaired, 1, mode='edge') + yp = coord_y + 1 + xp = coord_x + 1 + neighbors = np.stack([ + padded[yp - 1, xp - 1], + padded[yp - 1, xp], + padded[yp - 1, xp + 1], + padded[yp, xp - 1], + padded[yp, xp + 1], + padded[yp + 1, xp - 1], + padded[yp + 1, xp], + padded[yp + 1, xp + 1], + ], axis=0) + + fill_values = np.nanmedian(neighbors, axis=0) + if np.any(~np.isfinite(fill_values)): + frame_median = bn.nanmedian(repaired) + if not np.isfinite(frame_median): + frame_median = 0.0 + fill_values[~np.isfinite(fill_values)] = frame_median + + repaired[coord_y, coord_x] = fill_values + return repaired + + def transformation_task(i, file_name, reference_file): if i == 0: return i, SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) @@ -3296,6 +3555,9 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet timeList, airMassList, exptimes, norm_flux = [], [], [], [] ignore_header_wcs = should_ignore_header_wcs(info_dict.get('ignore_header_wcs')) bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction(info_dict.get('bad_wcs_threshold_percent')) + detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( + info_dict.get('detect_bad_pixels_before_photometry', 'y') + ) plateStatus.initializeFilenames(info_dict['images']) inputfiles = corruption_check(info_dict['images']) @@ -3322,6 +3584,19 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet if dropped_wcs_files: plateStatus.initializeFilenames(list(inputfiles)) + bad_pixel_reference = None + if detect_bad_pixels_before_photometry: + log_info( + "Bad-pixel precheck enabled: scanning frames for persistent isolated high-count outliers before photometry." + ) + bad_pixel_reference = build_persistent_bad_pixel_map( + inputfiles, + load_image_data, + save_directory=info_dict['save'], + ) + else: + log_info("Bad-pixel precheck disabled per optional_info setting.") + use_multiprocess_transform_precompute = should_use_multiprocess_transform_precompute( inputfiles, multiprocess_transformations, ignore_header_wcs=ignore_header_wcs ) @@ -3357,7 +3632,10 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet if tar_radec is not None and comp_radec: target_and_comp_radec = np.array([tar_radec, comp_radec[0]], dtype=float) - targ_sig_xy = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0)[3:5] + centroid_reference_image = load_image_data(inputfiles[0]) + centroid_reference_image = repair_bad_pixels_in_frame(centroid_reference_image, bad_pixel_reference) + targ_sig_xy = fit_centroid(centroid_reference_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0)[3:5] + del centroid_reference_image # aperture and annulus scale factors in PSF sigma units aper_sigma = 3 * max(targ_sig_xy) @@ -3402,6 +3680,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet # IMAGES imageData = hdul[extension].data + imageData = repair_bad_pixels_in_frame(imageData, bad_pixel_reference) if i == 0: firstImage = np.copy(imageData) @@ -3568,6 +3847,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, final_fit_mode='lm', use_impactparameter_rather_than_inclination_to_fit=True): # remove outliers + plot_time_range = np.asarray(times, dtype=float) si = np.argsort(times) times_sorted = times[si] tflux_sorted = tFlux[si] @@ -3692,6 +3972,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, mode='lm', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) + myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) if ( @@ -3723,6 +4004,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, mode='lm', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) + myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) if final_fit_mode == 'ns' and myfit is not None: @@ -3737,6 +4019,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) + myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) return myfit, f1, f2 @@ -4500,7 +4783,8 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta def load_calibrated_reduction_image(file_name, generalDark, generalBias, generalFlat, - demosaic_fmt, demosaic_out, demosaic_mult): + demosaic_fmt, demosaic_out, demosaic_mult, + bad_pixel_reference=None): hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) extension = 0 image_header = hdul[extension].header @@ -4513,6 +4797,7 @@ def load_calibrated_reduction_image(file_name, generalDark, generalBias, general image_data = apply_cals(image_data, generalDark, generalBias, generalFlat, 1) image_data = demosaic_img(image_data, demosaic_fmt, demosaic_out, demosaic_mult, 1) + image_data = repair_bad_pixels_in_frame(image_data, bad_pixel_reference) return image_data @@ -5000,6 +5285,9 @@ def main(): bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction( exotic_infoDict.get('bad_wcs_threshold_percent') ) + detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( + exotic_infoDict.get('detect_bad_pixels_before_photometry', 'y') + ) inputfiles, wcs_keep_mask, dropped_wcs_files = filter_sparse_missing_wcs_frames( inputfiles, ignore_header_wcs=ignore_header_wcs, @@ -5009,7 +5297,29 @@ def main(): times = times[wcs_keep_mask] jd_times = jd_times[wcs_keep_mask] plateStatus.initializeFilenames(list(inputfiles)) - + + bad_pixel_reference = None + if detect_bad_pixels_before_photometry: + log_info( + "Bad-pixel precheck enabled: scanning calibrated frames for persistent isolated " + "high-count outliers before plate-solve checks and photometry." + ) + bad_pixel_reference = build_persistent_bad_pixel_map( + inputfiles, + lambda file_name: load_calibrated_reduction_image( + file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ), + save_directory=exotic_infoDict['save'], + ) + else: + log_info("Bad-pixel precheck disabled per optional_info setting.") + exotic_UIprevTPX = exotic_infoDict['tar_coords'][0] exotic_UIprevTPY = exotic_infoDict['tar_coords'][1] @@ -5017,7 +5327,19 @@ def main(): inc = 0 for ifile in inputfiles: plateStatus.setCurrentFilename(ifile) - first_image = fits.getdata(ifile) + if bad_pixel_reference is not None: + first_image = load_calibrated_reduction_image( + ifile, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + else: + first_image = fits.getdata(ifile) try: initial_centroid = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0) if np.isnan(initial_centroid[0]): @@ -5229,6 +5551,7 @@ def main(): imageData = apply_cals(imageData, generalDark, generalBias, generalFlat, i) # Demosaic, if needed imageData = demosaic_img(imageData, demosaic_fmt, demosaic_out, demosaic_mult, i) + imageData = repair_bad_pixels_in_frame(imageData, bad_pixel_reference) if i == 0: firstImage = np.copy(imageData) @@ -5425,6 +5748,7 @@ def main(): demosaic_fmt, demosaic_out, demosaic_mult, + bad_pixel_reference=bad_pixel_reference, ) loaded_from_disk = True try: @@ -6324,6 +6648,7 @@ def main(): detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=times, ) # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 46608762..d1e63590 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -418,6 +418,7 @@ def save_input(): "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", @@ -441,6 +442,7 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, + "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, @@ -1490,6 +1492,7 @@ def save_input(): "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", @@ -1561,6 +1564,7 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, + "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, @@ -1611,6 +1615,7 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, + "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, diff --git a/exotic/inputs.py b/exotic/inputs.py index 1845a688..15aad82f 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -210,6 +210,7 @@ def __init__(self, init_opt): 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'detrend_on_outoftransit_baseline': True, + 'detect_bad_pixels_before_photometry': 'y', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, @@ -423,6 +424,10 @@ def comp_params(self, init_file, planet_dict): 'disable vertical flux normalization', 'Disable vertical flux normalization', ), + 'detect_bad_pixels_before_photometry': ( + 'detect_bad_pixels_before_photometry', + 'Detect Bad Pixels Before Photometry? (y/n)', + ), 'detrend_on_outoftransit_baseline': ( 'detrend_on_outoftransit_baseline', 'Detrend on Out-of-Transit Baseline', diff --git a/exotic/plots.py b/exotic/plots.py index cd36fba1..13a13f72 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -539,7 +539,10 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date): f, (ax_lc, ax_res) = fit.plot_bestfit() ax_lc.set_title(targ_name) - ax_lc.plot(np.linspace(np.nanmin(fit.phase), np.nanmax(fit.phase), 1000), high_res, 'r', zorder=1000, lw=2) + if hasattr(fit, 'phase_upsample') and hasattr(fit, 'transit_upsample'): + ax_lc.plot(fit.phase_upsample, fit.transit_upsample, 'r', zorder=1000, lw=2) + else: + ax_lc.plot(np.linspace(np.nanmin(fit.phase), np.nanmax(fit.phase), 1000), high_res, 'r', zorder=1000, lw=2) Path(save).mkdir(parents=True, exist_ok=True) try: diff --git a/inits.json b/inits.json index aaf6f40b..adb805c3 100644 --- a/inits.json +++ b/inits.json @@ -26,6 +26,7 @@ "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", @@ -103,6 +104,7 @@ "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": false, + "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": true, "use_impactparameter_rather_than_inclination_to_fit": "y", "use_psf_photometry": "y", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index e5ab9c70..6c515c9c 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -104,6 +104,61 @@ def _write_extension_wcs_fits(tmp_path, shape=(100, 120)): return path +def test_detect_frame_bad_pixels_flags_isolated_hot_pixel_but_not_broad_star_core(): + image = _gaussian_image(shape=(60, 60), center=(30.0, 30.0), amplitude=1200.0, sigma=1.8, background=100.0) + image[10, 15] = 8000.0 + + mask = exotic_module.detect_frame_bad_pixels(image) + + assert mask[10, 15] + assert not mask[30, 30] + + +def test_build_persistent_bad_pixel_map_thresholds_recurrence_and_saves_outputs(tmp_path): + frames = {} + for frame_index in range(10): + frame = np.full((9, 9), 100.0, dtype=float) + if frame_index < 4: + frame[2, 3] = 4000.0 + if frame_index < 3: + frame[6, 5] = 3500.0 + frames[f"frame_{frame_index}.fits"] = frame + + reference = exotic_module.build_persistent_bad_pixel_map( + list(frames.keys()), + lambda file_name: frames[file_name], + save_directory=tmp_path, + ) + + assert reference is not None + assert reference["required_count"] == 4 + assert reference["mask"][2, 3] + assert not reference["mask"][6, 5] + + count_image = fits.getdata(tmp_path / "temp" / "BadPixelDetectionCounts.fits") + mask_image = fits.getdata(tmp_path / "temp" / "BadPixelMask.fits").astype(bool) + + assert count_image[2, 3] == 4 + assert count_image[6, 5] == 3 + assert mask_image[2, 3] + assert not mask_image[6, 5] + + +def test_repair_bad_pixels_in_frame_replaces_known_bad_pixel_with_neighbor_median(): + image = np.arange(25, dtype=float).reshape(5, 5) + image[2, 2] = 9999.0 + reference = { + "mask": np.zeros((5, 5), dtype=bool), + "coord_y": np.array([2]), + "coord_x": np.array([2]), + } + reference["mask"][2, 2] = True + + repaired = exotic_module.repair_bad_pixels_in_frame(image, reference) + + assert repaired[2, 2] == pytest.approx(12.0) + + def test_fit_centroid_uses_moment_fallback_when_psf_fit_fails(monkeypatch): image = _gaussian_image() low_flux_warnings = [] diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 2cda2f8b..381de0ea 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -216,6 +216,91 @@ def test_create_fit_variables_preserves_explicit_baseline_in_nested_mode(monkeyp assert fit.parameters["a1"] == pytest.approx(0.98, abs=1e-9) +def test_create_fit_variables_respects_plot_time_range(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + prior = make_prior() + fit.time = np.array([-0.015, 0.010], dtype=float) + fit.data = 0.99 * elca.transit(fit.time, prior) + fit.dataerr = np.full_like(fit.time, 1e-3) + fit.airmass = np.zeros_like(fit.time) + fit.prior = prior.copy() + fit.bounds = {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]} + fit.mode = "ns" + fit.parameters = prior.copy() + fit.errors = {"rprs": 1e-3, "tmid": 1e-4, "a0": 2e-3} + fit.plot_time_range = (-0.12, 0.18) + + fit.create_fit_variables() + + assert fit.time_upsample[0] == pytest.approx(-0.12, abs=1e-12) + assert fit.time_upsample[-1] == pytest.approx(0.18, abs=1e-12) + assert fit.phase_upsample[0] == pytest.approx(-0.04, abs=1e-12) + assert fit.phase_upsample[-1] == pytest.approx(0.06, abs=1e-12) + + +def test_plot_bestfit_uses_full_plot_time_range_for_phase_xlim(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + fit.plot_time_range = (-0.12, 0.18) + fit._update_plot_geometry() + + fig, axes = fit.plot_bestfit() + + assert axes[0].get_xlim() == pytest.approx((-0.04, 0.06), abs=1e-6) + assert axes[1].get_xlim() == pytest.approx((-0.04, 0.06), abs=1e-6) + plt.close(fig) + + +def test_glc_plot_bestfit_median_limits_use_full_phase_span(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + phase = np.array([0.01, 0.02], dtype=float) + phase_upsample = np.linspace(-0.04, 0.06, 100) + residuals = np.array([1e-4, -1e-4], dtype=float) + times = prior["tmid"] + phase * prior["per"] + + fit = elca.glc_fitter.__new__(elca.glc_fitter) + fit.parameters = prior.copy() + fit.errors = {"rprs": 1e-3, "tmid": 1e-4} + fit.lc_data = [{ + "time": times, + "flux": np.ones_like(times), + "detrend": np.ones_like(times), + "ferr": np.full_like(times, 1e-3), + "residuals": residuals, + "phase": phase, + "phase_upsample": phase_upsample, + "time_upsample": prior["tmid"] + phase_upsample * prior["per"], + "transit_upsample": np.ones_like(phase_upsample), + "priors": prior.copy(), + "errors": {"rprs": 1e-3, "tmid": 1e-4}, + "name": "dataset", + }] + + fig, axes = fit.plot_bestfit(phase_limits="median") + + assert axes[0].get_xlim() == pytest.approx((-0.04, 0.06), abs=1e-6) + assert axes[1].get_xlim() == pytest.approx((-0.04, 0.06), abs=1e-6) + plt.close(fig) + + def test_plot_triangle_clips_ranges_to_parameter_bounds(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 246e3d90..e04eff40 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -103,6 +103,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: summarize_adaptive_aperture_usage, should_skip_airmass_fit, should_fit_lightcurve_to_every_comparison_candidate, + should_detect_bad_pixels_before_photometry, should_use_aperture_photometry, should_use_psf_photometry, should_skip_low_comparison_coverage_rejection, @@ -199,6 +200,12 @@ def test_should_fit_lightcurve_to_every_comparison_candidate_parses_values(): assert should_fit_lightcurve_to_every_comparison_candidate("n") is False +def test_should_detect_bad_pixels_before_photometry_parses_values(): + assert should_detect_bad_pixels_before_photometry(None) is True + assert should_detect_bad_pixels_before_photometry("y") is True + assert should_detect_bad_pixels_before_photometry("n") is False + + def test_is_out_of_transit_baseline_detrending_enabled_parses_values(): assert is_out_of_transit_baseline_detrending_enabled(None) is True assert is_out_of_transit_baseline_detrending_enabled("y") is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 7576fc5b..c459d82c 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -128,6 +128,21 @@ def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_p assert inputs.info_dict["disable_vertical_flux_normalization"] is False +def test_comp_params_defaults_detect_bad_pixels_before_photometry_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detect_bad_pixels_before_photometry"] == "y" + + def test_comp_params_defaults_detrend_on_outoftransit_baseline_to_true(tmp_path): init_data = { "user_info": {}, @@ -308,6 +323,21 @@ def test_comp_params_reads_disable_vertical_flux_normalization_from_optional_inf assert inputs.info_dict["disable_vertical_flux_normalization"] is True +def test_comp_params_reads_detect_bad_pixels_before_photometry_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"detect_bad_pixels_before_photometry": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detect_bad_pixels_before_photometry"] == "n" + + def test_comp_params_reads_detrend_on_outoftransit_baseline_from_optional_info(tmp_path): init_data = { "user_info": {}, From a335391eea68a7a60002a894d946ecb66df11cd5 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sun, 12 Apr 2026 21:29:08 +1000 Subject: [PATCH 019/116] Unify photometry fallback search and log fit failure reasons --- docs/system_prompt.txt | 2 +- exotic/api/colab.py | 2 +- exotic/exotic.py | 550 +++++++++++++++++++---------- exotic/inputs.py | 2 + inits.json | 2 +- tests/test_exotic_proper_motion.py | 119 +++++++ tests/test_inputs.py | 15 + 7 files changed, 497 insertions(+), 195 deletions(-) diff --git a/docs/system_prompt.txt b/docs/system_prompt.txt index 475b3074..7ee1c6c3 100644 --- a/docs/system_prompt.txt +++ b/docs/system_prompt.txt @@ -893,7 +893,7 @@ Example `inits.json` file: "Observing Notes": "Weather, seeing was nice.", "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "y", + "Add Comparison Stars from AAVSO? (y/n)": "n", "Target Star X & Y Pixel": "[424, 286]", "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", diff --git a/exotic/api/colab.py b/exotic/api/colab.py index b745d9c1..db625fb3 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -365,7 +365,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Observing Notes": "%s", "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "y", + "Add Comparison Stars from AAVSO? (y/n)": "n", "Target Star X & Y Pixel": %s, "Comparison Star(s) X & Y Pixel": %s, diff --git a/exotic/exotic.py b/exotic/exotic.py index f053140c..3ee93e68 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -4143,6 +4143,16 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass): return diagnostics +def ensure_lightcurve_fit_failure_reason(diagnostics, fit_result, failed_stage, failure_reason): + diagnostics = {} if diagnostics is None else dict(diagnostics) + if fit_result is None and diagnostics.get('failure_reason') is None: + diagnostics.update({ + 'failed_stage': failed_stage, + 'failure_reason': failure_reason, + }) + return diagnostics + + def cheap_lightcurve_prescore(tFlux, cFlux, airmass): with np.errstate(divide='ignore', invalid='ignore'): flux_ratio = np.divide(tFlux, cFlux) @@ -4201,6 +4211,229 @@ def evaluate_lightcurve_candidate(task): }, tflux_fit, cflux_fit +def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, comp_stars, sigma, + require_comp_star=True, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=True): + candidate_jobs = [] + comp_star_count = len(comp_stars) + + if use_psf_photometry and comp_star_count > 0: + target_flux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + psf_comp_flux_map = { + f"comp{comp_idx + 1}": 2 * np.pi * psf_data[f"comp{comp_idx + 1}"][:, 2] + * psf_data[f"comp{comp_idx + 1}"][:, 3] + * psf_data[f"comp{comp_idx + 1}"][:, 4] + for comp_idx in range(comp_star_count) + } + psf_comp_coverage = comparison_star_coverage_summary( + psf_comp_flux_map, + skip_rejection=skip_low_comparison_coverage_rejection, + ) + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + if psf_comp_coverage[ckey]['coverage_rejected']: + continue + + comp_flux = psf_comp_flux_map[ckey] + candidate_jobs.append({ + 'method': 'psf', + 'a': None, + 'an': None, + 'aper': 0.0, + 'annulus': float(15 * sigma), + 'comp_index': comp_idx, + 'ckey': ckey, + 'mask': np.ones(target_flux.shape[0], dtype=bool), + 'prescore': cheap_lightcurve_prescore(target_flux, comp_flux, airmass), + }) + + if use_aperture_photometry and aper_data is not None and apers is not None and annuli is not None: + for a, aper in enumerate(apers): + for an, annulus in enumerate(annuli): + target_flux = aper_data['target'][:, a, an] + aperture_comp_flux_map = { + f"comp{comp_idx + 1}": aper_data[f"comp{comp_idx + 1}"][:, a, an] + for comp_idx in range(comp_star_count) + } + aperture_comp_coverage = comparison_star_coverage_summary( + aperture_comp_flux_map, + skip_rejection=skip_low_comparison_coverage_rejection, + ) + + if not require_comp_star: + candidate_jobs.append({ + 'method': 'aperture', + 'a': a, + 'an': an, + 'aper': float(aper), + 'annulus': float(annulus), + 'comp_index': None, + 'ckey': None, + 'mask': np.ones(target_flux.shape[0], dtype=bool), + 'prescore': cheap_lightcurve_prescore( + target_flux, + np.ones(target_flux.shape[0]), + airmass, + ), + }) + + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + if aperture_comp_coverage[ckey]['coverage_rejected']: + continue + + comp_series = aperture_comp_flux_map[ckey] + aper_mask = valid_comparison_frame_mask(comp_series) + candidate_jobs.append({ + 'method': 'aperture', + 'a': a, + 'an': an, + 'aper': float(aper), + 'annulus': float(annulus), + 'comp_index': comp_idx, + 'ckey': ckey, + 'mask': aper_mask, + 'prescore': cheap_lightcurve_prescore( + target_flux[aper_mask], + comp_series[aper_mask], + airmass[aper_mask], + ), + }) + + return candidate_jobs + + +def shortlist_target_fit_candidates(candidate_jobs, minimum_count=50, fraction=0.35): + finite_candidates = [candidate for candidate in candidate_jobs if np.isfinite(candidate.get('prescore', np.inf))] + if finite_candidates: + finite_candidates.sort(key=lambda candidate: candidate['prescore']) + shortlist_count = max(int(minimum_count), int(fraction * len(finite_candidates))) + return finite_candidates[:min(len(finite_candidates), shortlist_count)] + return list(candidate_jobs) + + +def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, psf_data, aper_data, + disable_vertical_flux_normalization=False, + use_impactparameter_rather_than_inclination_to_fit=True): + candidate_mask = np.asarray(candidate['mask'], dtype=bool) + + if candidate['method'] == 'psf': + target_flux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + if candidate['ckey'] is None: + comp_flux = np.ones(target_flux.shape[0], dtype=float) + else: + comp_flux = ( + 2 * np.pi * psf_data[candidate['ckey']][:, 2] + * psf_data[candidate['ckey']][:, 3] + * psf_data[candidate['ckey']][:, 4] + ) + else: + target_flux = aper_data['target'][:, candidate['a'], candidate['an']] + if candidate['ckey'] is None: + comp_flux = np.ones(target_flux.shape[0], dtype=float) + else: + comp_flux = aper_data[candidate['ckey']][:, candidate['a'], candidate['an']] + + return ( + times[candidate_mask], + target_flux[candidate_mask], + comp_flux[candidate_mask], + airmass[candidate_mask], + ld, + p_dict, + jd_times[candidate_mask], + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + ) + + +def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, comp_stars, psf_data, aper_data, + apers, annuli, sigma, + require_comp_star=True, + disable_vertical_flux_normalization=False, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=True, + multiprocess_lightcurve_fits=None, + use_impactparameter_rather_than_inclination_to_fit=True): + candidate_jobs = build_target_fit_candidate_jobs( + psf_data, + aper_data, + apers, + annuli, + airmass, + comp_stars, + sigma, + require_comp_star=require_comp_star, + skip_low_comparison_coverage_rejection=skip_low_comparison_coverage_rejection, + use_psf_photometry=use_psf_photometry, + use_aperture_photometry=use_aperture_photometry, + ) + shortlist = shortlist_target_fit_candidates(candidate_jobs) + if not shortlist: + return { + 'candidate_jobs': candidate_jobs, + 'shortlist': shortlist, + 'best_candidate': None, + 'best_fit_lc': None, + 'min_std': np.inf, + 'flux_tar': None, + 'flux_ref': None, + } + + fit_tasks = [ + target_fit_candidate_task( + candidate, + times, + jd_times, + airmass, + ld, + p_dict, + psf_data, + aper_data, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + ) + for candidate in shortlist + ] + + if multiprocess_lightcurve_fits is not None and multiprocess_lightcurve_fits > 0: + log_info(f"Using multiprocessing for candidate lightcurve fits ({multiprocess_lightcurve_fits} processes).") + with ProcessPoolExecutor(max_workers=multiprocess_lightcurve_fits) as executor: + fit_results = list(executor.map(evaluate_lightcurve_candidate, fit_tasks)) + else: + fit_results = [evaluate_lightcurve_candidate(task) for task in fit_tasks] + + best_candidate = None + best_fit_lc = None + best_res_std = np.inf + best_tflux = None + best_cflux = None + for candidate, result in zip(shortlist, fit_results): + fit_meta, tflux_fit, cflux_fit = result + if fit_meta is None: + continue + + if fit_meta['res_std'] < best_res_std: + best_candidate = candidate + best_fit_lc = fit_meta['myfit'] + best_res_std = fit_meta['res_std'] + best_tflux = tflux_fit + best_cflux = cflux_fit + + return { + 'candidate_jobs': candidate_jobs, + 'shortlist': shortlist, + 'best_candidate': best_candidate, + 'best_fit_lc': best_fit_lc, + 'min_std': best_res_std, + 'flux_tar': best_tflux, + 'flux_ref': best_cflux, + } + + def selected_photometry_method_label(photometry_info): min_aperture = photometry_info.get('min_aperture') min_annulus = photometry_info.get('min_annulus') @@ -4454,11 +4687,12 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p final_fit_mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) - if fit_result is None: - fit_diagnostics.update({ - 'failed_stage': 'nested_fit', - 'failure_reason': "the nested lightcurve fitter did not converge to a usable solution.", - }) + fit_diagnostics = ensure_lightcurve_fit_failure_reason( + fit_diagnostics, + fit_result, + failed_stage='nested_fit', + failure_reason="the nested lightcurve fitter did not converge to a usable solution.", + ) res_std = np.inf fit_point_count = 0 if target_fit_flux is None else int(len(target_fit_flux)) @@ -5985,12 +6219,25 @@ def main(): selected_target_flux = aper_data['target'][:, best_a, best_an] selected_comp_flux = aper_data[selected_ckey][:, best_a, best_an] + selected_fit_diagnostics = diagnose_lightcurve_fit_inputs( + times, + selected_target_flux, + selected_comp_flux, + airmass, + ) myfit, tFlux1, cFlux1 = fit_lightcurve( times, selected_target_flux, selected_comp_flux, airmass, ld, pDict, jd_times, disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, ) + selected_fit_diagnostics = ensure_lightcurve_fit_failure_reason( + selected_fit_diagnostics, + myfit, + failed_stage='lightcurve_fit', + failure_reason="the lightcurve fitter did not converge to a usable solution.", + ) + comparison_calibration['selected_fit_diagnostics'] = selected_fit_diagnostics if myfit is not None: res_std = myfit.residuals.std() / np.median(myfit.data) photometry_info.update(best_fit_lc=myfit, @@ -6051,202 +6298,121 @@ def main(): 'pos': exotic_infoDict['comp_stars'][j] } else: - log_info("Warning: Comparison-star calibration selected a photometry setup that failed target fitting." - " Falling back to target-driven photometry selection.", warn=True) - - if photometry_info['best_fit_lc'] is None and use_psf_photometry: - # Legacy fallback when comparison-star-only calibration cannot determine a usable setup. - psf_comp_flux_map = { - f"comp{j + 1}": 2 * np.pi * psf_data[f"comp{j + 1}"][:, 2] - * psf_data[f"comp{j + 1}"][:, 3] - * psf_data[f"comp{j + 1}"][:, 4] - for j in range(len(exotic_infoDict['comp_stars'])) - } - psf_comp_coverage = comparison_star_coverage_summary( - psf_comp_flux_map, - skip_rejection=skip_low_comp_coverage_rejection, - ) - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - if psf_comp_coverage[ckey]['coverage_rejected']: - continue - - cFlux = psf_comp_flux_map[ckey] - myfit, tFlux1, cFlux1 = fit_lightcurve( - times, tFlux, cFlux, airmass, ld, pDict, jd_times, - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, + failure_reason = selected_fit_diagnostics.get( + 'failure_reason', + "the lightcurve fitter did not converge to a usable solution.", + ) + log_info( + "Warning: Comparison-star calibration selected a photometry setup that failed target fitting " + f"(Comp {selected_comp_index + 1}, {comparison_calibration['method_label']}; " + f"reason: {failure_reason}). Falling back to target-driven photometry selection.", + warn=True, ) - res_std = np.inf - - if myfit is not None: - for k in myfit.bounds.keys(): - log.debug(f" {k}: {myfit.parameters[k]:.6f}") - - log.debug("The Residual Standard Deviation is: " - f"{round(100 * myfit.residuals.std() / np.median(myfit.data), 6)}%") - log.debug(f"The Mean Squared Error is: {round(np.sum(myfit.residuals ** 2), 6)}\n") - - res_std = myfit.residuals.std() / np.median(myfit.data) - - if photometry_info['min_std'] > res_std and myfit is not None: - photometry_info.update(best_fit_lc=myfit, - comp_star_num=j + 1, comp_star_coords=exotic_infoDict['comp_stars'][j], - min_std=res_std, min_aperture=0, min_annulus=15 * sigma_display, - aperture_index=None, annulus_index=None, - selection_basis='target_fit') - - flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, - flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) - - centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], - x_ref=psf_data[ckey][:, 0], y_ref=psf_data[ckey][:, 1]) - - if j in vsp_num: - ref_flux[j] = { - 'myfit': myfit, - 'pos': exotic_infoDict['comp_stars'][j] - } - - if photometry_info['best_fit_lc'] is None and use_aperture_photometry: - log_info("\nComputing best comparison star, aperture, and sky annulus from the target lightcurve. Please wait.") - - candidate_jobs = [] - for a, aper in enumerate(apers): - for an, annulus in enumerate(annuli): - target_flux = aper_data['target'][:, a, an] - aperture_comp_flux_map = { - f"comp{j + 1}": aper_data[f"comp{j + 1}"][:, a, an] - for j in range(len(exotic_infoDict['comp_stars'])) - } - aperture_comp_coverage = comparison_star_coverage_summary( - aperture_comp_flux_map, - skip_rejection=skip_low_comp_coverage_rejection, - ) - - if not require_comp_star: - candidate_jobs.append({ - 'a': a, - 'an': an, - 'aper': aper, - 'annulus': annulus, - 'comp_index': None, - 'ckey': None, - 'mask': np.ones(target_flux.shape[0], dtype=bool), - 'prescore': cheap_lightcurve_prescore(target_flux, np.ones(target_flux.shape[0]), airmass), - }) - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - if aperture_comp_coverage[ckey]['coverage_rejected']: - continue - comp_series = aperture_comp_flux_map[ckey] - aper_mask = valid_comparison_frame_mask(comp_series) - comp_flux = comp_series[aper_mask] - candidate_jobs.append({ - 'a': a, - 'an': an, - 'aper': aper, - 'annulus': annulus, - 'comp_index': j, - 'ckey': ckey, - 'mask': aper_mask, - 'prescore': cheap_lightcurve_prescore(target_flux[aper_mask], comp_flux, airmass[aper_mask]), - }) - - finite_candidates = [c for c in candidate_jobs if np.isfinite(c['prescore'])] - if finite_candidates: - finite_candidates.sort(key=lambda candidate: candidate['prescore']) - shortlist_count = max(50, int(0.35 * len(finite_candidates))) - shortlist = finite_candidates[:min(len(finite_candidates), shortlist_count)] + if photometry_info['best_fit_lc'] is None and (use_psf_photometry or use_aperture_photometry): + if use_psf_photometry and use_aperture_photometry: + log_info("\nComputing the best PSF/aperture photometry candidate from the target lightcurve. Please wait.") + elif use_aperture_photometry: + log_info("\nComputing best comparison star, aperture, and sky annulus from the target lightcurve. Please wait.") else: - shortlist = [] + log_info("\nComputing best PSF comparison star from the target lightcurve. Please wait.") - if not shortlist: - shortlist = candidate_jobs + target_driven_search = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + pDict, + exotic_infoDict['comp_stars'], + psf_data, + aper_data, + apers, + annuli, + sigma_display, + require_comp_star=require_comp_star, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=use_psf_photometry, + use_aperture_photometry=use_aperture_photometry, + multiprocess_lightcurve_fits=args.multiprocess_lightcurve_fits, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + ) - fit_tasks = [] - for candidate in shortlist: - candidate_mask = candidate['mask'] - target_flux = aper_data['target'][:, candidate['a'], candidate['an']][candidate_mask] - if candidate['comp_index'] is None: - comp_flux = np.ones(target_flux.shape[0]) - else: - comp_flux = aper_data[candidate['ckey']][:, candidate['a'], candidate['an']][candidate_mask] - - fit_tasks.append(( - times[candidate_mask], - target_flux, - comp_flux, - airmass[candidate_mask], - ld, - pDict, - jd_times[candidate_mask], - disable_vertical_flux_normalization, - use_impactparameter_rather_than_inclination_to_fit, - )) - - fit_results = [] - if args.multiprocess_lightcurve_fits is not None and args.multiprocess_lightcurve_fits > 0: - log_info(f"Using multiprocessing for candidate lightcurve fits ({args.multiprocess_lightcurve_fits} processes).") - with ProcessPoolExecutor(max_workers=args.multiprocess_lightcurve_fits) as executor: - fit_results = list(executor.map(evaluate_lightcurve_candidate, fit_tasks)) - else: - fit_results = [evaluate_lightcurve_candidate(task) for task in fit_tasks] + best_candidate = target_driven_search['best_candidate'] + if best_candidate is not None: + photometry_info.update( + best_fit_lc=target_driven_search['best_fit_lc'], + comp_star_num=(None if best_candidate['comp_index'] is None else best_candidate['comp_index'] + 1), + comp_star_coords=( + None if best_candidate['comp_index'] is None + else exotic_infoDict['comp_stars'][best_candidate['comp_index']] + ), + min_std=target_driven_search['min_std'], + min_aperture=( + 0 if best_candidate['method'] == 'psf' + else (-best_candidate['aper'] if best_candidate['comp_index'] is None else best_candidate['aper']) + ), + min_annulus=best_candidate['annulus'], + aperture_index=best_candidate['a'], + annulus_index=best_candidate['an'], + selection_basis='target_fit', + ) - best_candidate = None - for candidate, result in zip(shortlist, fit_results): - fit_meta, tFlux1, cFlux1 = result - if fit_meta is None: - continue + flux_values.update( + flux_tar=target_driven_search['flux_tar'], + flux_ref=target_driven_search['flux_ref'], + flux_unc_tar=target_driven_search['flux_tar'] ** 0.5, + flux_unc_ref=target_driven_search['flux_ref'] ** 0.5, + ) - myfit = fit_meta['myfit'] - res_std = fit_meta['res_std'] - - if photometry_info['min_std'] > res_std: - best_candidate = candidate - photometry_info.update(best_fit_lc=myfit, - comp_star_num=(None if candidate['comp_index'] is None else candidate['comp_index'] + 1), - comp_star_coords=(None if candidate['comp_index'] is None else exotic_infoDict['comp_stars'][candidate['comp_index']]), - min_std=res_std, - min_aperture=(-candidate['aper'] if candidate['comp_index'] is None else candidate['aper']), - min_annulus=candidate['annulus'], - aperture_index=candidate['a'], - annulus_index=candidate['an'], - selection_basis='target_fit') - - flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, - flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) - - x_ref_data = psf_data['target'][:, 0] - y_ref_data = psf_data['target'][:, 1] - if candidate['ckey'] is not None: - x_ref_data = psf_data[candidate['ckey']][:, 0] - y_ref_data = psf_data[candidate['ckey']][:, 1] - - centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], - x_ref=x_ref_data, y_ref=y_ref_data) + x_ref_data = psf_data['target'][:, 0] + y_ref_data = psf_data['target'][:, 1] + if best_candidate['ckey'] is not None: + x_ref_data = psf_data[best_candidate['ckey']][:, 0] + y_ref_data = psf_data[best_candidate['ckey']][:, 1] + + centroid_positions.update( + x_targ=psf_data["target"][:, 0], + y_targ=psf_data["target"][:, 1], + x_ref=x_ref_data, + y_ref=y_ref_data, + ) if best_candidate is not None and vsp_num: - best_a = best_candidate['a'] - best_an = best_candidate['an'] - best_target_flux = aper_data['target'][:, best_a, best_an] - for j in vsp_num: - ckey = f"comp{j + 1}" - aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) - cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] - vsp_fit, _, _ = fit_lightcurve( - times[aper_mask], best_target_flux[aper_mask], cFlux, - airmass[aper_mask], ld, pDict, jd_times[aper_mask], - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - ) - ref_flux[j] = { - 'myfit': vsp_fit, - 'pos': exotic_infoDict['comp_stars'][j] - } + if best_candidate['method'] == 'psf': + for j in vsp_num: + ckey = f"comp{j + 1}" + cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] + vsp_fit, _, _ = fit_lightcurve( + times, tFlux, cFlux, airmass, ld, pDict, jd_times, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + ) + ref_flux[j] = { + 'myfit': vsp_fit, + 'pos': exotic_infoDict['comp_stars'][j] + } + else: + best_a = best_candidate['a'] + best_an = best_candidate['an'] + best_target_flux = aper_data['target'][:, best_a, best_an] + for j in vsp_num: + ckey = f"comp{j + 1}" + aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) + cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] + vsp_fit, _, _ = fit_lightcurve( + times[aper_mask], best_target_flux[aper_mask], cFlux, + airmass[aper_mask], ld, pDict, jd_times[aper_mask], + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + ) + ref_flux[j] = { + 'myfit': vsp_fit, + 'pos': exotic_infoDict['comp_stars'][j] + } update_photometry_adaptive_summary( photometry_info, diff --git a/exotic/inputs.py b/exotic/inputs.py index 15aad82f..f2f44cf5 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -470,6 +470,8 @@ def comp_params(self, init_file, planet_dict): } self.info_dict = init_params(user_info, self.info_dict, data['user_info']) + if self.info_dict['aavso_comp'] is None: + self.info_dict['aavso_comp'] = 'n' self.info_dict = init_params(opt_info, self.info_dict, data['optional_info']) planet_dict = init_params(planet_params, planet_dict, data['planetary_parameters']) return populate_missing_gaia_astrometry(planet_dict) diff --git a/inits.json b/inits.json index adb805c3..1783e1f0 100644 --- a/inits.json +++ b/inits.json @@ -57,7 +57,7 @@ "Filter Name (aavso.org/filters)": "CV", "Observing Notes": "Weather, seeing was nice.", "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "y", + "Add Comparison Stars from AAVSO? (y/n)": "n", "Target Star X & Y Pixel": "[424, 286]", "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", "Demosaic Format": null, diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index e04eff40..c5f1c04d 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -89,6 +89,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: comparison_star_coverage_summary, comparison_star_stability_summary, detrend_flux_on_out_of_transit_baseline, + ensure_lightcurve_fit_failure_reason, fit_lightcurve, fit_final_lightcurve_with_oot_baseline_detrending, fit_lightcurve_to_every_comparison_candidate, @@ -99,6 +100,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: log_comparison_candidate_fit_summaries, phase_bin_sigma_clip, representative_psf_sigma, + run_target_driven_photometry_search, resolve_frame_aperture_radii, summarize_adaptive_aperture_usage, should_skip_airmass_fit, @@ -922,6 +924,123 @@ def fake_lc_fitter( assert captured_modes == ["lm", "ns"] +def test_run_target_driven_photometry_search_selects_best_method_across_psf_and_aperture(monkeypatch): + evaluated = [] + + class DummyFit: + def __init__(self, residual_level): + self.residuals = np.full(6, residual_level) + self.data = np.ones(6) + + def fake_evaluate(task): + _, tflux, cflux, _, _, _, _, _, _ = task + evaluated.append(np.asarray(cflux)) + cflux = np.asarray(cflux) + tflux = np.asarray(tflux) + if np.allclose(cflux, 20.0): + return {"myfit": DummyFit(0.02), "res_std": 0.02}, tflux, cflux + if np.allclose(cflux, 40.0): + return {"myfit": DummyFit(0.01), "res_std": 0.01}, tflux, cflux + raise AssertionError("Unexpected candidate flux passed to evaluator.") + + monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + psf_target_amp = 20.0 / (2.0 * np.pi) + psf_comp_amp = 20.0 / (2.0 * np.pi) + psf_data = { + "target": np.column_stack([ + np.zeros(6), + np.zeros(6), + np.full(6, psf_target_amp), + np.ones(6), + np.ones(6), + ]), + "comp1": np.column_stack([ + np.ones(6), + np.ones(6), + np.full(6, psf_comp_amp), + np.ones(6), + np.ones(6), + ]), + } + aper_data = { + "target": np.full((6, 1, 1), 40.0), + "comp1": np.full((6, 1, 1), 40.0), + } + + result = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + p_dict, + comp_stars=[[100.0, 200.0]], + psf_data=psf_data, + aper_data=aper_data, + apers=np.array([5.0]), + annuli=np.array([12.0]), + sigma=1.0, + require_comp_star=True, + use_psf_photometry=True, + use_aperture_photometry=True, + ) + + assert len(evaluated) == 2 + assert {tuple(np.unique(values)) for values in evaluated} == {(20.0,), (40.0,)} + assert result["best_candidate"]["method"] == "aperture" + assert result["best_candidate"]["comp_index"] == 0 + assert result["min_std"] == pytest.approx(0.01) + + +def test_ensure_lightcurve_fit_failure_reason_preserves_existing_diagnostic_reason(): + diagnostics = { + "failed_stage": "minimum_points", + "failure_reason": "only 4 usable point(s) remained after filtering; need at least 5 for a lightcurve fit.", + } + + result = ensure_lightcurve_fit_failure_reason( + diagnostics, + fit_result=None, + failed_stage="lightcurve_fit", + failure_reason="the lightcurve fitter did not converge to a usable solution.", + ) + + assert result["failed_stage"] == "minimum_points" + assert result["failure_reason"] == diagnostics["failure_reason"] + + +def test_ensure_lightcurve_fit_failure_reason_adds_generic_reason_when_missing(): + diagnostics = { + "failed_stage": None, + "failure_reason": None, + } + + result = ensure_lightcurve_fit_failure_reason( + diagnostics, + fit_result=None, + failed_stage="lightcurve_fit", + failure_reason="the lightcurve fitter did not converge to a usable solution.", + ) + + assert result["failed_stage"] == "lightcurve_fit" + assert result["failure_reason"] == "the lightcurve fitter did not converge to a usable solution." + + def test_fit_lightcurve_can_disable_impact_parameter_parameterization(monkeypatch): captured = {"flags": []} diff --git a/tests/test_inputs.py b/tests/test_inputs.py index c459d82c..0b7df28a 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -53,6 +53,21 @@ def test_comp_params_defaults_require_comp_star_to_yes(tmp_path): assert inputs.info_dict["require_comp_star"] == "y" +def test_comp_params_defaults_aavso_comp_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["aavso_comp"] == "n" + + def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): init_data = { "user_info": {}, From 974a0a04de9c9f95c45461a7d557cc8ff3828fde Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sat, 18 Apr 2026 08:29:59 +1000 Subject: [PATCH 020/116] filter ID and plate solution reporting. --- exotic/api/elca.py | 17 +- exotic/api/filters.py | 1 + exotic/api/plate_solution.py | 35 +++- exotic/exotic.py | 280 ++++++++++++++++++++++----- tests/test_elca_baseline.py | 46 +++++ tests/test_exotic_proper_motion.py | 298 ++++++++++++++++++++++++++++- tests/test_inputs.py | 25 +++ tests/test_nextastro_astrometry.py | 49 ++++- 8 files changed, 686 insertions(+), 65 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 50826922..01ce9aa4 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -382,10 +382,25 @@ def __init__( self.sample_parameters = {} self.sample_errors = {} self.sample_quantiles = {} + self.nested_fit_fallback = False + self.nested_fit_failure_reason = None if self.mode == "lm": self.fit_LM() elif self.mode == "ns": - self.fit_nested() + try: + self.fit_nested() + except np.linalg.LinAlgError as exc: + self.nested_fit_fallback = True + self.nested_fit_failure_reason = f"{type(exc).__name__}: {exc}" + self.ns_type = 'lm' + self.mode = "lm" + if self.verbose: + print( + "WARNING: Nested light curve fitting failed with a linear algebra error; " + "falling back to least-squares fit." + ) + print(f" Reason: {self.nested_fit_failure_reason}") + self.fit_LM() def _validate_flux_baseline_keys(self): free_flux_keys = [key for key in self.bounds if key in ('a0', 'a1')] diff --git a/exotic/api/filters.py b/exotic/api/filters.py index 49164cae..38c4f560 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -105,6 +105,7 @@ "hb": "Stromgren Hbw", "zs": "PanSTARRS z-short", "clearV": "ClearV", + "C": "MObs CV", "clear": "ClearV", "lum": "ClearV", "w": "ClearV", diff --git a/exotic/api/plate_solution.py b/exotic/api/plate_solution.py index 237403ec..c4336d68 100644 --- a/exotic/api/plate_solution.py +++ b/exotic/api/plate_solution.py @@ -193,7 +193,7 @@ def fail(error_type, service_name='nova.astrometry.net'): class NextAstroPlateSolution: def __init__(self, file=None, directory=None, api_url='https://astrometry.nextastro.org/', ra=None, dec=None, - pixel_scale=None, suppress_fail_warning=False): + pixel_scale=None, suppress_fail_warning=False, message_logger=None): self.api_url = api_url.rstrip('/') self.file = file self.directory = directory @@ -201,6 +201,7 @@ def __init__(self, file=None, directory=None, api_url='https://astrometry.nextas self.dec = dec self.pixel_scale = pixel_scale self.suppress_fail_warning = suppress_fail_warning + self.message_logger = message_logger self.last_error_type = None self.last_http_status = None @@ -227,9 +228,18 @@ def plate_solution(self): return wcs_file def _emit_debug(self, message): - if not self.suppress_fail_warning: + if self.message_logger is not None: + self.message_logger(message) + elif not self.suppress_fail_warning: print(message) + @staticmethod + def _json_message(payload): + try: + return dumps(payload) + except (TypeError, ValueError): + return str(payload) + def _fail(self, error_type): self.last_error_type = error_type if self.suppress_fail_warning: @@ -354,11 +364,11 @@ def _submit_solve_request(self, source_list): if hints is not None: payload["hints"] = hints - self._emit_debug(f"[NextAstro] Solve request payload: {payload}") + self._emit_debug(f"NextAstro astrometry request JSON: {self._json_message(payload)}") response = requests.post(f"{self.api_url}/solve", json=payload, timeout=_RQ_TIMEOUT) response_json = self._decode_response_json(response, 'Solve response') if response_json is not None and response.status_code != 502: - self._emit_debug(f"[NextAstro] Solve response: {response_json}") + self._emit_debug(f"NextAstro astrometry submission response JSON: {self._json_message(response_json)}") if response.status_code >= 400 or response_json is None: return False if response_json.get('status') in {'queued', 'running'}: @@ -388,23 +398,32 @@ def _poll_for_solution(self, request_id): return False if response.status_code >= 400: if response.status_code != 502: - self._emit_debug(f"[NextAstro] Status response (HTTP {response.status_code}): {response_json}") + self._emit_debug( + f"NextAstro astrometry status response JSON (HTTP {response.status_code}): " + f"{self._json_message(response_json)}" + ) return False status = str(response_json.get('status', '')).lower() latest_status = response_json.get('status') if status == 'solved': - self._emit_debug(f"[NextAstro] Status response (solved): {response_json}") + self._emit_debug( + f"NextAstro astrometry status response JSON (solved): {self._json_message(response_json)}" + ) header_dict = response_json.get('solution', {}).get('wcs_header') if isinstance(header_dict, dict): return Header(header_dict) return False if status == 'failed': - self._emit_debug(f"[NextAstro] Status response (failed): {response_json}") + self._emit_debug( + f"NextAstro astrometry status response JSON (failed): {self._json_message(response_json)}" + ) return False if status not in _NEXTASTRO_IN_PROGRESS_STATUSES: - self._emit_debug(f"[NextAstro] Status response (unexpected): {response_json}") + self._emit_debug( + f"NextAstro astrometry status response JSON (unexpected): {self._json_message(response_json)}" + ) return False time.sleep(_NEXTASTRO_STATUS_POLL_SEC) diff --git a/exotic/exotic.py b/exotic/exotic.py index 3ee93e68..78772597 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -565,13 +565,21 @@ def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled finite_flux = np.asarray(flux_values, dtype=float) finite_flux = finite_flux[np.isfinite(finite_flux) & (finite_flux > 0)] baseline_guess = 1.0 if finite_flux.size == 0 else float(np.nanmedian(finite_flux)) - baseline_guess = float(np.clip(baseline_guess, 0.95, 1.05)) + if not np.isfinite(baseline_guess) or baseline_guess <= 0: + baseline_guess = 1.0 prior['a0'] = baseline_guess prior['a1'] = baseline_guess if not disabled: - bounds['a0'] = [0.95, 1.05] + # Raw target/reference flux ratios are often far from unity, so only keep + # the legacy near-unity bound for already-normalized light curves. + if 0.95 <= baseline_guess <= 1.05: + bounds['a0'] = [0.95, 1.05] + else: + lower = max(np.finfo(float).eps, baseline_guess * 0.75) + upper = baseline_guess * 1.25 + bounds['a0'] = [lower, upper] def detrend_flux_on_out_of_transit_baseline( @@ -1810,7 +1818,8 @@ def get_wcs(file, directory="", use_nextastro_astrometry=False, ra=None, dec=Non ra=ra, dec=dec, pixel_scale=pixel_scale, - suppress_fail_warning=True + suppress_fail_warning=True, + message_logger=log_info ) wcs_file = nextastro_solver.plate_solution() if wcs_file: @@ -1848,7 +1857,8 @@ def get_wcs(file, directory="", use_nextastro_astrometry=False, ra=None, dec=Non ra=ra, dec=dec, pixel_scale=pixel_scale, - suppress_fail_warning=True + suppress_fail_warning=True, + message_logger=log_info ) wcs_file = nextastro_solver.plate_solution() animate_toggle() @@ -3845,9 +3855,10 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, allow_mid_transit_range_warning=True, disable_vertical_flux_normalization=False, final_fit_mode='lm', - use_impactparameter_rather_than_inclination_to_fit=True): + use_impactparameter_rather_than_inclination_to_fit=True, + plot_time_range=None): # remove outliers - plot_time_range = np.asarray(times, dtype=float) + plot_time_range = np.asarray(times if plot_time_range is None else plot_time_range, dtype=float) si = np.argsort(times) times_sorted = times[si] tflux_sorted = tFlux[si] @@ -4186,6 +4197,7 @@ def evaluate_lightcurve_candidate(task): ld, p_dict, jd_times, + plot_time_range, disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit, ) = task @@ -4200,6 +4212,7 @@ def evaluate_lightcurve_candidate(task): allow_mid_transit_range_warning=False, disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, ) if myfit is None: return None, tflux_fit, cflux_fit @@ -4315,6 +4328,7 @@ def shortlist_target_fit_candidates(candidate_jobs, minimum_count=50, fraction=0 def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, psf_data, aper_data, + plot_time_range=None, disable_vertical_flux_normalization=False, use_impactparameter_rather_than_inclination_to_fit=True): candidate_mask = np.asarray(candidate['mask'], dtype=bool) @@ -4344,6 +4358,7 @@ def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, p ld, p_dict, jd_times[candidate_mask], + plot_time_range, disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit, ) @@ -4352,6 +4367,7 @@ def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, p def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, comp_stars, psf_data, aper_data, apers, annuli, sigma, require_comp_star=True, + plot_time_range=None, disable_vertical_flux_normalization=False, skip_low_comparison_coverage_rejection=False, use_psf_photometry=True, @@ -4393,6 +4409,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co p_dict, psf_data, aper_data, + plot_time_range=plot_time_range, disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) @@ -4497,12 +4514,24 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): if summary.get('selected'): if selection_basis == 'comparison_field': return "selected: comparison-field calibration ranked this star best for the chosen photometry method" + if selection_basis == 'comparison_field_retry': + return ( + "selected: comparison-field calibration fell back to this star " + "after better-ranked candidates failed target fitting" + ) return "selected: lowest target-fit residual scatter in the chosen search" if selection_basis == 'comparison_field': if selected_comp_num is None: return "not selected: comparison-field calibration chose a different candidate" return f"not selected: comparison-field calibration chose Comp {selected_comp_num}" + if selection_basis == 'comparison_field_retry': + if selected_comp_num is None: + return "not selected: comparison-field fallback chose a different candidate" + return ( + "not selected: comparison-field fallback chose " + f"Comp {selected_comp_num} after better-ranked candidate(s) failed target fitting" + ) if np.isfinite(candidate_res_std) and np.isfinite(selected_res_std): if candidate_res_std > selected_res_std + 1e-12: @@ -4605,6 +4634,7 @@ def log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_i def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p_dict, comp_stars, psf_data, aper_data, photometry_info, + plot_time_range=None, disable_vertical_flux_normalization=False, skip_low_comparison_coverage_rejection=False, use_impactparameter_rather_than_inclination_to_fit=True): @@ -4686,6 +4716,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p disable_vertical_flux_normalization=disable_vertical_flux_normalization, final_fit_mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -5189,6 +5220,117 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, return best_candidate +def ranked_comparison_calibration_summaries(comparison_calibration): + if comparison_calibration is None: + return [] + + ranked_summaries = [] + for summary in comparison_calibration.get('comp_summaries', []): + aggregate_score = summary.get('aggregate_score', np.inf) + if summary.get('coverage_rejected'): + continue + if not np.isfinite(aggregate_score): + continue + ranked_summaries.append(summary) + + ranked_summaries.sort( + key=lambda summary: ( + summary.get('aggregate_score', np.inf), + summary.get('comp_index', np.inf), + ) + ) + return ranked_summaries + + +def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p_dict, comparison_calibration, + psf_data, aper_data, target_psf_flux, + plot_time_range=None, + disable_vertical_flux_normalization=False, + use_impactparameter_rather_than_inclination_to_fit=True): + ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) + if not ranked_summaries: + return { + 'ranked_summaries': [], + 'attempts': [], + 'selected_result': None, + } + + method = comparison_calibration['method'] + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + if method == 'psf': + target_flux = target_psf_flux + else: + target_flux = aper_data['target'][:, aperture_index, annulus_index] + + attempts = [] + selected_result = None + for rank, comp_summary in enumerate(ranked_summaries): + comp_index = comp_summary['comp_index'] + ckey = comp_summary.get('key', f"comp{comp_index + 1}") + if method == 'psf': + comp_flux = ( + 2 * np.pi * psf_data[ckey][:, 2] + * psf_data[ckey][:, 3] + * psf_data[ckey][:, 4] + ) + else: + comp_flux = aper_data[ckey][:, aperture_index, annulus_index] + + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times, + target_flux, + comp_flux, + airmass, + ) + fit_result, tflux_fit, cflux_fit = fit_lightcurve( + times, + target_flux, + comp_flux, + airmass, + ld, + p_dict, + jd_times, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + ) + fit_diagnostics = ensure_lightcurve_fit_failure_reason( + fit_diagnostics, + fit_result, + failed_stage='lightcurve_fit', + failure_reason="the lightcurve fitter did not converge to a usable solution.", + ) + + res_std = np.inf + if fit_result is not None and hasattr(fit_result, 'residuals') and hasattr(fit_result, 'data'): + with np.errstate(divide='ignore', invalid='ignore'): + res_std = float(np.std(fit_result.residuals) / np.median(fit_result.data)) + + attempt = { + 'rank': rank, + 'comp_index': comp_index, + 'ckey': ckey, + 'fit': fit_result, + 'tflux_fit': tflux_fit, + 'cflux_fit': cflux_fit, + 'fit_diagnostics': fit_diagnostics, + 'res_std': res_std, + } + attempts.append(attempt) + + if fit_result is not None: + selected_result = attempt + break + + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': selected_result, + } + + def parse_args(): parser = argparse.ArgumentParser(description="Using a JSON initialization file to bypass user inputs for EXOTIC.") parser.add_argument('-rt', '--realtime', @@ -5515,6 +5657,10 @@ def main(): times = np.array(times)[si] jd_times = np.array(jd_times)[si] inputfiles = np.array(inputfiles)[si] + finite_plot_times = times[np.isfinite(times)] + full_plot_time_range = None + if finite_plot_times.size: + full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction( exotic_infoDict.get('bad_wcs_threshold_percent') @@ -6200,46 +6346,56 @@ def main(): except Exception as e: log_info(f"Warning: Could not save comparison-star calibration outputs ({e}).", warn=True) - selected_comp_index = comparison_calibration['best_comp_index'] - selected_ckey = f"comp{selected_comp_index + 1}" - selected_comp_coords = exotic_infoDict['comp_stars'][selected_comp_index] - selected_min_aperture = 0 if comparison_calibration['method'] == 'psf' else comparison_calibration['aper'] - selected_min_annulus = comparison_calibration['annulus'] - selected_a = None if comparison_calibration['method'] == 'psf' else comparison_calibration['a'] - selected_an = None if comparison_calibration['method'] == 'psf' else comparison_calibration['an'] - - if comparison_calibration['method'] == 'psf': - selected_target_flux = tFlux - selected_comp_flux = ( - 2 * np.pi * psf_data[selected_ckey][:, 2] * psf_data[selected_ckey][:, 3] * psf_data[selected_ckey][:, 4] - ) - else: - best_a = comparison_calibration['a'] - best_an = comparison_calibration['an'] - selected_target_flux = aper_data['target'][:, best_a, best_an] - selected_comp_flux = aper_data[selected_ckey][:, best_a, best_an] - - selected_fit_diagnostics = diagnose_lightcurve_fit_inputs( + comparison_fit_search = fit_ranked_comparison_calibration_candidates( times, - selected_target_flux, - selected_comp_flux, + jd_times, airmass, - ) - myfit, tFlux1, cFlux1 = fit_lightcurve( - times, selected_target_flux, selected_comp_flux, airmass, ld, pDict, jd_times, + ld, + pDict, + comparison_calibration, + psf_data, + aper_data, + tFlux, + plot_time_range=full_plot_time_range, disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, ) - selected_fit_diagnostics = ensure_lightcurve_fit_failure_reason( - selected_fit_diagnostics, - myfit, - failed_stage='lightcurve_fit', - failure_reason="the lightcurve fitter did not converge to a usable solution.", - ) - comparison_calibration['selected_fit_diagnostics'] = selected_fit_diagnostics - if myfit is not None: - res_std = myfit.residuals.std() / np.median(myfit.data) + comparison_calibration['ranked_fit_comp_indices'] = [ + summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] + ] + comparison_calibration['fit_attempt_summaries'] = comparison_fit_search['attempts'] + + selected_attempt = comparison_fit_search['selected_result'] + fit_attempts = comparison_fit_search['attempts'] + if fit_attempts: + comparison_calibration['selected_fit_diagnostics'] = fit_attempts[-1]['fit_diagnostics'] + + if selected_attempt is not None: + selected_comp_index = selected_attempt['comp_index'] + selected_ckey = selected_attempt['ckey'] + selected_comp_coords = exotic_infoDict['comp_stars'][selected_comp_index] + selected_min_aperture = 0 if comparison_calibration['method'] == 'psf' else comparison_calibration['aper'] + selected_min_annulus = comparison_calibration['annulus'] + selected_a = None if comparison_calibration['method'] == 'psf' else comparison_calibration['a'] + selected_an = None if comparison_calibration['method'] == 'psf' else comparison_calibration['an'] + myfit = selected_attempt['fit'] + tFlux1 = selected_attempt['tflux_fit'] + cFlux1 = selected_attempt['cflux_fit'] + res_std = selected_attempt['res_std'] + selection_basis = ( + 'comparison_field' + if selected_comp_index == comparison_calibration['best_comp_index'] + else 'comparison_field_retry' + ) + if selection_basis == 'comparison_field_retry': + retry_count = selected_attempt['rank'] + log_info( + "Comparison-star calibration retry selected " + f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " + f"after {retry_count} better-ranked candidate(s) failed target fitting." + ) + photometry_info.update(best_fit_lc=myfit, comp_star_num=selected_comp_index + 1, comp_star_coords=selected_comp_coords, @@ -6249,7 +6405,7 @@ def main(): aperture_index=selected_a, annulus_index=selected_an, calibration_field_score=comparison_calibration['field_score'], - selection_basis='comparison_field') + selection_basis=selection_basis) flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) @@ -6273,6 +6429,7 @@ def main(): disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -6292,22 +6449,37 @@ def main(): disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, ) ref_flux[j] = { 'myfit': vsp_fit, 'pos': exotic_infoDict['comp_stars'][j] } else: - failure_reason = selected_fit_diagnostics.get( - 'failure_reason', - "the lightcurve fitter did not converge to a usable solution.", - ) - log_info( - "Warning: Comparison-star calibration selected a photometry setup that failed target fitting " - f"(Comp {selected_comp_index + 1}, {comparison_calibration['method_label']}; " - f"reason: {failure_reason}). Falling back to target-driven photometry selection.", - warn=True, - ) + if fit_attempts: + failed_attempt = fit_attempts[-1] + failed_comp_index = failed_attempt['comp_index'] + failure_reason = failed_attempt['fit_diagnostics'].get( + 'failure_reason', + "the lightcurve fitter did not converge to a usable solution.", + ) + attempted_count = len(fit_attempts) + ranked_count = len(comparison_fit_search['ranked_summaries']) + log_info( + "Warning: Comparison-star calibration exhausted " + f"{attempted_count}/{ranked_count} ranked comparison star(s) for " + f"{comparison_calibration['method_label']} without a usable target fit " + f"(last attempt: Comp {failed_comp_index + 1}; reason: {failure_reason}). " + "Falling back to target-driven photometry selection.", + warn=True, + ) + else: + log_info( + "Warning: Comparison-star calibration did not produce any coverage-qualified " + "comparison stars to test against the target fit. Falling back to target-driven " + "photometry selection.", + warn=True, + ) if photometry_info['best_fit_lc'] is None and (use_psf_photometry or use_aperture_photometry): if use_psf_photometry and use_aperture_photometry: @@ -6330,6 +6502,7 @@ def main(): annuli, sigma_display, require_comp_star=require_comp_star, + plot_time_range=full_plot_time_range, disable_vertical_flux_normalization=disable_vertical_flux_normalization, skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, use_psf_photometry=use_psf_photometry, @@ -6389,6 +6562,7 @@ def main(): disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -6408,6 +6582,7 @@ def main(): disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -6480,6 +6655,7 @@ def main(): psf_data, aper_data, photometry_info, + plot_time_range=full_plot_time_range, disable_vertical_flux_normalization=disable_vertical_flux_normalization, skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, use_impactparameter_rather_than_inclination_to_fit= @@ -6814,7 +6990,7 @@ def main(): detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, - plot_time_range=times, + plot_time_range=full_plot_time_range, ) # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 381de0ea..438199d7 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -141,6 +141,52 @@ def test_lc_fitter_auto_solves_baseline_when_a0_is_not_free(monkeypatch, tmp_pat assert np.median(fit.detrended[oot_mask]) == pytest.approx(1.0, abs=5e-4) +def test_lc_fitter_falls_back_to_lm_after_nested_linalg_error(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + calls = [] + + def fake_fit_nested(self): + calls.append("ns") + raise np.linalg.LinAlgError("Singular matrix") + + def fake_fit_LM(self): + calls.append("lm") + self.parameters = self.prior.copy() + self.errors = {} + self.quantiles = {} + self.sampled_keys = [] + self.sample_bounds = {} + self.sample_parameters = {} + self.sample_errors = {} + self.sample_quantiles = {} + + monkeypatch.setattr(elca.lc_fitter, "fit_nested", fake_fit_nested) + monkeypatch.setattr(elca.lc_fitter, "fit_LM", fake_fit_LM) + + prior = make_prior() + time = np.linspace(-0.03, 0.03, 31) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + ) + + assert calls == ["ns", "lm"] + assert fit.mode == "lm" + assert fit.ns_type == "lm" + assert fit.nested_fit_fallback is True + assert fit.nested_fit_failure_reason == "LinAlgError: Singular matrix" + + def test_lc_fitter_auto_solves_mean_airmass_normalization(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index c5f1c04d..fabc0c9f 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -86,6 +86,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: auto_tune_aperture_sigma_grid, check_coordinates, cheap_lightcurve_prescore, + comparison_candidate_fit_selection_reason, comparison_star_coverage_summary, comparison_star_stability_summary, detrend_flux_on_out_of_transit_baseline, @@ -93,6 +94,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: fit_lightcurve, fit_final_lightcurve_with_oot_baseline_detrending, fit_lightcurve_to_every_comparison_candidate, + fit_ranked_comparison_calibration_candidates, is_adaptive_aperture_mode_enabled, is_comp_star_required, is_out_of_transit_baseline_detrending_enabled, @@ -528,6 +530,23 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): assert any("parameters: fit_method=ultranest" in message for message in logged) +def test_comparison_candidate_fit_selection_reason_describes_comparison_field_retry(): + reason = comparison_candidate_fit_selection_reason( + { + "selected": True, + "failure_reason": None, + "res_std": 0.01, + }, + { + "selection_basis": "comparison_field_retry", + "comp_star_num": 2, + "min_std": 0.01, + }, + ) + + assert "fell back to this star" in reason + + def test_comparison_star_stability_summary_penalizes_variable_candidates(): airmass = np.linspace(1.0, 1.5, 6) summary = comparison_star_stability_summary( @@ -712,6 +731,51 @@ def test_phase_bin_sigma_clip_flags_local_phase_outlier(): assert mask[27] +def test_fit_final_lightcurve_preserves_explicit_plot_time_range(monkeypatch): + import exotic.exotic as exotic_module + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + return types.SimpleNamespace( + parameters={"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(0.0, 0.05, 6) + flux = np.ones(6, dtype=float) + fluxerr = np.full(6, 0.01, dtype=float) + airmass = np.linspace(1.0, 1.5, 6) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.05, 0.15], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0]} + plot_time_range = (-0.12, 0.18) + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + plot_time_range=plot_time_range, + ) + + assert fit.plot_time_range == pytest.approx(plot_time_range) + + def test_fit_lightcurve_removes_relative_flux_above_two_before_fit(monkeypatch): captured = {} @@ -766,6 +830,113 @@ def fake_lc_fitter( assert np.allclose(fit_cflux, 2.0) +def test_fit_lightcurve_preserves_explicit_plot_time_range(monkeypatch): + captured = {} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + fit = types.SimpleNamespace() + captured["fit"] = fit + return fit + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + plot_time_range = (-0.12, 0.18) + + myfit, _, _ = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + plot_time_range=plot_time_range, + ) + + assert myfit is captured["fit"] + assert myfit.plot_time_range == pytest.approx(plot_time_range) + + +def test_fit_lightcurve_centers_vertical_flux_bound_on_raw_flux_ratio(monkeypatch): + captured = {} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + fit = types.SimpleNamespace() + captured["fit"] = fit + captured["prior"] = dict(prior) + captured["bounds"] = { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + } + return fit + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 100.0) + cflux = np.full(times.shape[0], 2000.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is captured["fit"] + assert captured["prior"]["a0"] == pytest.approx(0.05) + assert captured["prior"]["a1"] == pytest.approx(0.05) + assert captured["bounds"]["a0"] == pytest.approx([0.0375, 0.0625]) + + def test_fit_lightcurve_rejects_undersampled_series(monkeypatch): called = {"count": 0} @@ -933,7 +1104,7 @@ def __init__(self, residual_level): self.data = np.ones(6) def fake_evaluate(task): - _, tflux, cflux, _, _, _, _, _, _ = task + _, tflux, cflux, _, _, _, _, _, _, _ = task evaluated.append(np.asarray(cflux)) cflux = np.asarray(cflux) tflux = np.asarray(tflux) @@ -1007,6 +1178,131 @@ def fake_evaluate(task): assert result["min_std"] == pytest.approx(0.01) +def test_fit_ranked_comparison_calibration_candidates_retries_next_best_candidate(monkeypatch): + class DummyFit: + def __init__(self): + self.residuals = np.full(6, 0.01) + self.data = np.ones(6) + + def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times, **kwargs): + cflux = np.asarray(cflux, dtype=float) + if np.allclose(cflux, 0.0): + return None, None, None + return DummyFit(), np.asarray(tflux, dtype=float), cflux + + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "best_comp_index": 0, + "comp_summaries": [ + { + "comp_index": 0, + "key": "comp1", + "aggregate_score": 0.01, + "coverage_rejected": False, + }, + { + "comp_index": 1, + "key": "comp2", + "aggregate_score": 0.02, + "coverage_rejected": False, + }, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 10.0), + "comp1": np.zeros((6, 1, 1)), + "comp2": np.full((6, 1, 1), 5.0), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(6), + ) + + assert [attempt["comp_index"] for attempt in result["attempts"]] == [0, 1] + assert result["selected_result"]["comp_index"] == 1 + assert "relative-flux filtering left 0 usable point(s)" in result["attempts"][0]["fit_diagnostics"]["failure_reason"] + assert result["attempts"][1]["fit"] is not None + + +def test_fit_lightcurve_to_every_comparison_candidate_forwards_full_plot_time_range(monkeypatch): + captured_plot_ranges = [] + + class DummyFit: + def __init__(self, plot_time_range): + self.plot_time_range = plot_time_range + self.parameters = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0} + self.errors = {"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01} + self.residuals = np.full(6, 0.01, dtype=float) + self.data = np.ones(6, dtype=float) + + def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + plot_time_range = kwargs.get("plot_time_range") + captured_plot_ranges.append(plot_time_range) + return DummyFit(plot_time_range), np.asarray(tflux, dtype=float), np.asarray(cflux, dtype=float) + + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + monkeypatch.setattr( + "exotic.exotic.diagnose_lightcurve_fit_inputs", + lambda *args, **kwargs: { + "input_point_count": 6, + "has_reference_flux": True, + "relative_flux_point_count": 6, + "sigma_clip_point_count": 6, + "usable_point_count": 6, + "failed_stage": None, + "failure_reason": None, + }, + ) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + psf_series = np.ones((6, 7), dtype=float) + psf_data = { + "target": psf_series.copy(), + "comp1": psf_series.copy(), + } + photometry_info = { + "best_fit_lc": object(), + "comp_star_num": 1, + "min_aperture": 0, + } + plot_time_range = (-0.12, 0.18) + + summaries = fit_lightcurve_to_every_comparison_candidate( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comp_stars=[[100.0, 200.0]], + psf_data=psf_data, + aper_data=None, + photometry_info=photometry_info, + plot_time_range=plot_time_range, + ) + + assert captured_plot_ranges == [plot_time_range] + assert summaries[0]["fit"].plot_time_range == pytest.approx(plot_time_range) + + def test_ensure_lightcurve_fit_failure_reason_preserves_existing_diagnostic_reason(): diagnostics = { "failed_stage": "minimum_points", diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 0b7df28a..2c19724c 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -839,6 +839,14 @@ def test_parse_aavso_prereduced_overrides_uses_known_filter_lookup_when_filter_x assert overrides["wl_max"] == "1000.0" +def test_lookup_aavso_filter_metadata_uses_c_alias_for_cv_filter() -> None: + filter_metadata = inputs_module.lookup_aavso_filter_metadata("C") + + assert filter_metadata["name"] == "CV" + assert filter_metadata["desc"] == "MObs CV" + assert filter_metadata["fwhm"] == ("350.0", "850.0") + + def test_parse_aavso_prereduced_overrides_uses_osc_split_filter_alias_lookup(tmp_path): pre_reduced_file = tmp_path / "aavso_prereduced.txt" pre_reduced_file.write_text( @@ -856,6 +864,23 @@ def test_parse_aavso_prereduced_overrides_uses_osc_split_filter_alias_lookup(tmp assert overrides["wl_max"] == "586.8" +def test_parse_aavso_prereduced_overrides_uses_c_alias_for_cv_filter_lookup(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=C\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "C" + assert overrides["filter_desc"] == "MObs CV" + assert overrides["wl_min"] == "350.0" + assert overrides["wl_max"] == "850.0" + + def test_parse_aavso_prereduced_overrides_marks_airmass_as_already_corrected(tmp_path): pre_reduced_file = tmp_path / "aavso_prereduced.txt" pre_reduced_file.write_text( diff --git a/tests/test_nextastro_astrometry.py b/tests/test_nextastro_astrometry.py index a9308276..7f6303ae 100644 --- a/tests/test_nextastro_astrometry.py +++ b/tests/test_nextastro_astrometry.py @@ -92,6 +92,49 @@ def fake_get(url, timeout): assert header['CTYPE2'] == 'DEC--TAN' +def test_plate_solution_logs_json_via_message_logger_when_fail_warnings_suppressed(tmp_path, monkeypatch): + fits_path = _create_test_fits(tmp_path) + (tmp_path / "temp").mkdir() + logged = [] + + monkeypatch.setattr( + 'exotic.api.plate_solution.requests.post', + lambda url, json, timeout: DummyResponse({'status': 'queued', 'request_id': 'abc123'}) + ) + monkeypatch.setattr( + 'exotic.api.plate_solution.requests.get', + lambda url, timeout: DummyResponse({ + 'status': 'solved', + 'solution': { + 'wcs_header': { + 'SIMPLE': True, + 'BITPIX': -64, + 'NAXIS': 2, + 'NAXIS1': 120, + 'NAXIS2': 100, + 'CTYPE1': 'RA---TAN', + 'CTYPE2': 'DEC--TAN', + } + } + }) + ) + monkeypatch.setattr('exotic.api.plate_solution.time.sleep', lambda _: None) + + solver = NextAstroPlateSolution( + file=fits_path, + directory=tmp_path, + suppress_fail_warning=True, + message_logger=logged.append + ) + + wcs_file = solver.plate_solution() + + assert wcs_file == tmp_path / 'temp' / 'wcs.fits' + assert any('NextAstro astrometry request JSON:' in message for message in logged) + assert any('NextAstro astrometry submission response JSON:' in message for message in logged) + assert any('NextAstro astrometry status response JSON (solved):' in message for message in logged) + + def test_extract_astrometry_hints_with_scale_only(tmp_path): fits_path = _create_test_fits(tmp_path) solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path, pixel_scale=2.0) @@ -143,8 +186,8 @@ def test_poll_for_solution_logs_unexpected_status(tmp_path, monkeypatch, capsys) assert header is False output = capsys.readouterr().out - assert "Status response (unexpected)" in output - assert "'status': 'processing'" in output + assert "NextAstro astrometry status response JSON (unexpected)" in output + assert '"status": "processing"' in output def test_submit_solve_request_handles_non_json_response(tmp_path, monkeypatch, capsys): @@ -169,7 +212,7 @@ def test_submit_solve_request_handles_non_json_response(tmp_path, monkeypatch, c assert request_id is False output = capsys.readouterr().out - assert "Solve request payload" in output + assert "NextAstro astrometry request JSON:" in output assert "Solve response returned non-JSON response" not in output assert "bad gateway" not in output.lower() assert solver.last_http_status == 502 From 4da6f100e55aa2118189e65cb2b7c10a9f6bf939 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sun, 19 Apr 2026 11:10:42 +1000 Subject: [PATCH 021/116] Improve comparison-star rejection diagnostics and final fit retries --- README.md | 1 + exotic/api/elca.py | 184 +++++++ exotic/api/filters.py | 3 + exotic/exotic.py | 746 +++++++++++++++++++++++++++-- exotic/exotic_gui.py | 5 + exotic/inputs.py | 5 + inits.json | 2 + tests/test_elca_baseline.py | 36 ++ tests/test_exotic_proper_motion.py | 300 ++++++++++++ tests/test_inputs.py | 56 +++ 10 files changed, 1302 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 89a75c91..5d97cd0f 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "skip_low_comparison_coverage_rejection": "n", "fit_lightcurve_to_every_comparison_candidate": "n", "detrend_on_outoftransit_baseline": true, + "final_fit_baseline_duration_multiplier": 1.0, "use_impactparameter_rather_than_inclination_to_fit": "y", "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y", diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 01ce9aa4..58a7fe0d 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -559,6 +559,190 @@ def _get_triangle_plot_samples(self): index_samples = np.clip(np.rint(index_samples[:, 0]).astype(int), 0, points.shape[0] - 1) return points[index_samples], np.asarray(self.results.logl, dtype=float)[index_samples] + def get_parameter_posterior_samples(self, key): + try: + sample_points, _ = self._get_triangle_plot_samples() + except Exception: + return np.array([], dtype=float) + + sample_points = np.asarray(sample_points, dtype=float) + if sample_points.ndim != 2 or sample_points.shape[0] == 0: + return np.array([], dtype=float) + + sampled_keys = list(getattr(self, 'sampled_keys', self._get_sampled_keys())) + if key in sampled_keys: + key_index = sampled_keys.index(key) + if key_index < sample_points.shape[1]: + return np.asarray(sample_points[:, key_index], dtype=float) + + bound_keys = list(self.bounds.keys()) + physical_samples = [ + self._physical_values_from_sample_point(point, bound_keys, sampled_keys).get(key, np.nan) + for point in sample_points + ] + return np.asarray(physical_samples, dtype=float) + + def _estimate_histogram_mode(self, samples, bounds=None, bins=None): + samples = np.asarray(samples, dtype=float) + finite_samples = samples[np.isfinite(samples)] + if finite_samples.size == 0: + return np.nan, np.nan + if finite_samples.size == 1: + return float(finite_samples[0]), np.nan + + if bounds is None: + lower = float(np.nanmin(finite_samples)) + upper = float(np.nanmax(finite_samples)) + else: + lower, upper = np.asarray(bounds, dtype=float).reshape(-1)[:2] + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + lower = float(np.nanmin(finite_samples)) + upper = float(np.nanmax(finite_samples)) + + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return float(np.nanmedian(finite_samples)), np.nan + + if bins is None: + bins = int(np.clip(np.sqrt(finite_samples.size), 10, 80)) + bins = max(1, int(bins)) + + counts, edges = np.histogram(finite_samples, bins=bins, range=(lower, upper)) + if counts.size == 0: + return float(np.nanmedian(finite_samples)), np.nan + + mode_index = int(np.argmax(counts)) + mode = float(0.5 * (edges[mode_index] + edges[mode_index + 1])) + bin_width = float(edges[1] - edges[0]) if edges.size > 1 else np.nan + return mode, bin_width + + def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bins=None): + diagnostics = { + 'key': key, + 'clipped': False, + 'edge': None, + 'mode': np.nan, + 'std': np.nan, + 'full_std': np.nan, + 'bounds': None, + 'original_bounds': None, + 'sample_size': 0, + 'reason': None, + } + + bounds = getattr(self, 'sample_bounds', {}).get(key, self.bounds.get(key)) + if bounds is None: + diagnostics['reason'] = "parameter bounds are unavailable." + return diagnostics + + try: + lower_bound, upper_bound = np.asarray(bounds, dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + diagnostics['reason'] = "parameter bounds are malformed." + return diagnostics + + diagnostics['original_bounds'] = [float(lower_bound), float(upper_bound)] + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or lower_bound >= upper_bound: + diagnostics['reason'] = "parameter bounds are not finite." + return diagnostics + + samples = self.get_parameter_posterior_samples(key) + finite_samples = np.asarray(samples, dtype=float) + finite_samples = finite_samples[np.isfinite(finite_samples)] + diagnostics['sample_size'] = int(finite_samples.size) + if finite_samples.size < 8: + diagnostics['reason'] = "too few posterior samples are available." + return diagnostics + + mode, bin_width = self._estimate_histogram_mode(finite_samples, bounds=(lower_bound, upper_bound), bins=bins) + full_std = float(np.nanstd(finite_samples)) + diagnostics['mode'] = mode + diagnostics['full_std'] = full_std + + if not np.isfinite(mode): + diagnostics['reason'] = "posterior mode could not be estimated." + return diagnostics + + q05, q16, q50, q84, q95 = np.nanpercentile(finite_samples, [5, 16, 50, 84, 95]) + width = float(upper_bound - lower_bound) + scale_floor = max( + 2.0 * bin_width if np.isfinite(bin_width) and bin_width > 0 else 0.0, + 0.01 * width, + np.finfo(float).eps, + ) + tail_gap_threshold = max(0.5 * full_std if np.isfinite(full_std) and full_std > 0 else 0.0, scale_floor) + mode_gap_threshold = max( + 1.0 * full_std if np.isfinite(full_std) and full_std > 0 else 0.0, + 3.0 * bin_width if np.isfinite(bin_width) and bin_width > 0 else 0.0, + 0.05 * width, + np.finfo(float).eps, + ) + + upper_gap_q95 = float(upper_bound - q95) + lower_gap_q05 = float(q05 - lower_bound) + upper_gap_mode = float(upper_bound - mode) + lower_gap_mode = float(mode - lower_bound) + + upper_clipped = upper_gap_q95 <= tail_gap_threshold and upper_gap_mode <= mode_gap_threshold + lower_clipped = lower_gap_q05 <= tail_gap_threshold and lower_gap_mode <= mode_gap_threshold + + if upper_clipped and lower_clipped: + clipped_edge = 'upper' if upper_gap_mode <= lower_gap_mode else 'lower' + elif upper_clipped: + clipped_edge = 'upper' + elif lower_clipped: + clipped_edge = 'lower' + else: + diagnostics['bounds'] = [float(lower_bound), float(upper_bound)] + diagnostics['reason'] = "posterior support is comfortably inside the sampled bounds." + return diagnostics + + diagnostics['clipped'] = True + diagnostics['edge'] = clipped_edge + + if clipped_edge == 'upper': + side_distances = mode - finite_samples[finite_samples <= mode] + else: + side_distances = finite_samples[finite_samples >= mode] - mode + + side_distances = np.asarray(side_distances, dtype=float) + side_distances = side_distances[np.isfinite(side_distances)] + side_distances = side_distances[side_distances >= 0] + + if side_distances.size >= 2: + mirrored = np.concatenate([side_distances, -side_distances]) + estimated_std = float(np.nanstd(mirrored)) + else: + estimated_std = full_std + + min_std = max( + bin_width if np.isfinite(bin_width) and bin_width > 0 else 0.0, + width * 1e-3, + np.finfo(float).eps, + ) + if not np.isfinite(estimated_std) or estimated_std <= 0: + estimated_std = full_std + if not np.isfinite(estimated_std) or estimated_std <= 0: + estimated_std = min_std + estimated_std = float(max(estimated_std, min_std)) + diagnostics['std'] = estimated_std + + radius = float(max(sigma_scale * estimated_std, min_std)) + new_lower = float(mode - radius) + new_upper = float(mode + radius) + if lower_bound >= 0: + new_lower = max(float(lower_bound), new_lower) + diagnostics['bounds'] = [new_lower, new_upper] + diagnostics['reason'] = ( + f"posterior peaks against the {clipped_edge} search bound " + f"(mode={mode:.6g}, sigma={estimated_std:.6g})." + ) + diagnostics['q16'] = float(q16) + diagnostics['q50'] = float(q50) + diagnostics['q84'] = float(q84) + diagnostics['q05'] = float(q05) + diagnostics['q95'] = float(q95) + return diagnostics + def _get_triangle_plot_display_spec(self, sampled_keys, sample_parameters, sample_errors, sample_points): if 'b' in sampled_keys: key = 'b' diff --git a/exotic/api/filters.py b/exotic/api/filters.py index 38c4f560..2fac5f55 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -108,6 +108,9 @@ "C": "MObs CV", "clear": "ClearV", "lum": "ClearV", + "Lum": "ClearV", + "Luminosity": "ClearV", + "luminosity": "ClearV", "w": "ClearV", "pl": "ClearV", "exo": "Astrodon ExoPlanet-BB", diff --git a/exotic/exotic.py b/exotic/exotic.py index 78772597..7d2e1be5 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -170,6 +170,8 @@ COMPARISON_STAR_COVERAGE_SIGMA = 3.0 COMPARISON_STAR_COVERAGE_MAX_ITERS = 10 OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 +FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 +FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER = 0.5 BAD_PIXEL_DETECTION_FRACTION = 0.30 BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 BAD_PIXEL_PROGRESS_LOG_INTERVAL = 25 @@ -240,6 +242,187 @@ def annotate_out_of_transit_baseline_detrending( fit.oot_baseline_post_points = int(post_points) if post_points is not None else 0 +def annotate_final_fit_prefit_refinement( + fit, + applied, + note=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + duration=None, + original_point_count=None, + refined_point_count=None, + trimmed_pre_points=0, + trimmed_post_points=0, + original_tmid_bounds=None, + refined_tmid_bounds=None, +): + if fit is None: + return + + fit.prefit_refinement_applied = bool(applied) + fit.prefit_refinement_note = note + fit.prefit_refinement_baseline_duration_multiplier = float(baseline_duration_multiplier) + fit.prefit_refinement_duration = duration + fit.prefit_refinement_original_point_count = ( + int(original_point_count) if original_point_count is not None else None + ) + fit.prefit_refinement_point_count = ( + int(refined_point_count) if refined_point_count is not None else None + ) + fit.prefit_refinement_trimmed_pre_points = int(trimmed_pre_points or 0) + fit.prefit_refinement_trimmed_post_points = int(trimmed_post_points or 0) + fit.prefit_refinement_original_tmid_bounds = original_tmid_bounds + fit.prefit_refinement_tmid_bounds = refined_tmid_bounds + + +def annotate_rprs_posterior_refit(fit, applied, note=None, history=None): + if fit is None: + return + + history = [] if history is None else list(history) + fit.rprs_posterior_refit_applied = bool(applied) + fit.rprs_posterior_refit_note = note + fit.rprs_posterior_refit_count = len(history) + fit.rprs_posterior_refit_history = history + if history: + latest = history[-1] + fit.rprs_posterior_refit_edge = latest.get('edge') + fit.rprs_posterior_refit_mode = latest.get('mode') + fit.rprs_posterior_refit_std = latest.get('std') + fit.rprs_posterior_refit_original_bounds = latest.get('original_bounds') + fit.rprs_posterior_refit_bounds = latest.get('new_bounds') + else: + fit.rprs_posterior_refit_edge = None + fit.rprs_posterior_refit_mode = None + fit.rprs_posterior_refit_std = None + fit.rprs_posterior_refit_original_bounds = None + fit.rprs_posterior_refit_bounds = None + + +def clone_lightcurve_bounds(bounds): + return { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + } + + +def run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + use_impactparameter_rather_than_inclination_to_fit=True, + max_rprs_retries=1, +): + def build_fit(local_prior, local_bounds): + return lc_fitter( + times, + flux_values, + flux_errors, + airmass, + local_prior, + local_bounds, + jd_times=jd_times, + mode='ns', + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + ) + + current_prior = dict(prior) + current_bounds = clone_lightcurve_bounds(bounds) + retry_history = [] + retry_note = None + fit = build_fit(current_prior, current_bounds) + + diagnostics = None + for _ in range(int(max(0, max_rprs_retries))): + diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + if not callable(diagnostics_getter): + diagnostics = None + break + + diagnostics = diagnostics_getter('rprs') + new_bounds = diagnostics.get('bounds') if diagnostics else None + if not diagnostics or not diagnostics.get('clipped') or new_bounds is None: + break + + try: + new_lower, new_upper = [float(value) for value in new_bounds] + except (TypeError, ValueError): + retry_note = "Skipped; the automatic Rp/R* retry proposed malformed bounds." + break + if not np.isfinite(new_lower) or not np.isfinite(new_upper) or new_lower >= new_upper: + retry_note = "Skipped; the automatic Rp/R* retry proposed invalid bounds." + break + + previous_bounds = current_bounds.get('rprs') + if previous_bounds is not None and np.allclose( + np.asarray(previous_bounds, dtype=float), + np.asarray([new_lower, new_upper], dtype=float), + atol=1e-12, + rtol=0.0, + ): + retry_note = "Skipped; the automatic Rp/R* retry did not expand the sampled range." + break + + retry_history.append({ + 'attempt': len(retry_history) + 1, + 'edge': diagnostics.get('edge'), + 'mode': float(diagnostics.get('mode', np.nan)), + 'std': float(diagnostics.get('std', np.nan)), + 'original_bounds': None if previous_bounds is None else [float(previous_bounds[0]), float(previous_bounds[1])], + 'new_bounds': [new_lower, new_upper], + }) + log_info( + "Rp/R* posterior is truncated against the " + f"{diagnostics.get('edge', 'active')} search bound; retrying nested fit " + f"with Rp/R* centered at {diagnostics.get('mode', np.nan):.6f} " + f"and sigma {diagnostics.get('std', np.nan):.6f} " + f"over [{new_lower:.6f}, {new_upper:.6f}]." + ) + + updated_bounds = clone_lightcurve_bounds(current_bounds) + updated_bounds['rprs'] = [new_lower, new_upper] + + updated_prior = dict(current_prior) + fit_parameters = getattr(fit, 'parameters', {}) + if isinstance(fit_parameters, dict): + for key in updated_bounds: + if key in fit_parameters: + updated_prior[key] = fit_parameters[key] + if np.isfinite(diagnostics.get('mode', np.nan)): + updated_prior['rprs'] = float(diagnostics['mode']) + + current_prior = updated_prior + current_bounds = updated_bounds + fit = build_fit(current_prior, current_bounds) + + if retry_history: + final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + final_diagnostics = final_diagnostics_getter('rprs') if callable(final_diagnostics_getter) else None + note = f"Applied {len(retry_history)} automatic Rp/R* posterior range refit(s)." + if final_diagnostics and final_diagnostics.get('clipped'): + note = ( + f"{note} The posterior still hugs the {final_diagnostics.get('edge')} bound after retry." + ) + log_info( + "Warning: Rp/R* posterior still appears truncated after the automatic retry; " + "please inspect the triangle plot carefully.", + warn=True, + ) + elif retry_note is not None: + note = retry_note + elif diagnostics is not None and diagnostics.get('reason'): + note = f"Not needed; {diagnostics['reason']}" + else: + note = "Not evaluated; posterior diagnostics are unavailable for this fit." + + annotate_rprs_posterior_refit(fit, bool(retry_history), note=note, history=retry_history) + return fit + + def log_info(string, warn=False, error=False): if error: print(f"\033[31m {string}\033[0m") @@ -539,6 +722,30 @@ def is_out_of_transit_baseline_detrending_enabled(config_value): return True +def get_final_fit_baseline_duration_multiplier(config_value): + if config_value is None: + return FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT + if isinstance(config_value, (int, float)) and np.isfinite(config_value) and config_value >= 0: + return float(config_value) + if isinstance(config_value, str): + normalized = config_value.strip() + if normalized == "": + return FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT + try: + parsed = float(normalized) + except ValueError: + parsed = np.nan + if np.isfinite(parsed) and parsed >= 0: + return float(parsed) + + log_info( + "Warning: Invalid 'final_fit_baseline_duration_multiplier' value; " + f"using default {FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT:.1f}.", + warn=True, + ) + return FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT + + def should_use_impactparameter_rather_than_inclination_to_fit(config_value): if config_value is None: return True @@ -709,6 +916,188 @@ def detrend_flux_on_out_of_transit_baseline( } +def estimate_transit_duration_from_fit(fit): + if fit is None: + return np.nan + + for attribute_name in ('duration_expected', 'duration_measured'): + duration = getattr(fit, attribute_name, np.nan) + if np.isfinite(duration) and duration > 0: + return float(duration) + + times = np.asarray(getattr(fit, 'time', []), dtype=float) + transit_model = np.asarray(getattr(fit, 'transit', []), dtype=float) + if times.shape != transit_model.shape or times.size == 0: + return np.nan + + in_transit = np.isfinite(times) & np.isfinite(transit_model) & (transit_model < 1) + if not np.any(in_transit): + return np.nan + + transit_times = np.sort(times[in_transit]) + if transit_times.size == 1: + sorted_times = np.sort(times[np.isfinite(times)]) + if sorted_times.size < 2: + return np.nan + cadence = np.nanmedian(np.diff(sorted_times)) + return float(cadence) if np.isfinite(cadence) and cadence > 0 else np.nan + + cadence = np.nanmedian(np.diff(np.sort(times[np.isfinite(times)]))) + if not np.isfinite(cadence) or cadence <= 0: + cadence = 0.0 + duration = (transit_times[-1] - transit_times[0]) + cadence + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + +def build_final_fit_prefit_refinement_plan( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + fit, + jd_times=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + airmass = np.asarray(airmass, dtype=float) + jd_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + + original_tmid_bounds = clone_lightcurve_bounds(bounds).get('tmid') + base_plan = { + 'applied': False, + 'note': None, + 'baseline_duration_multiplier': float(baseline_duration_multiplier), + 'duration': np.nan, + 'trimmed_pre_points': 0, + 'trimmed_post_points': 0, + 'original_point_count': int(times.shape[0]), + 'refined_point_count': int(times.shape[0]), + 'original_tmid_bounds': original_tmid_bounds, + 'refined_tmid_bounds': original_tmid_bounds, + 'times': times, + 'flux': flux_values, + 'unc': flux_errors, + 'airmass': airmass, + 'jd_times': jd_array, + 'prior': dict(prior), + 'bounds': clone_lightcurve_bounds(bounds), + } + + if fit is None: + base_plan['note'] = "Skipped; the initial nested fit did not return a solution." + return base_plan + + duration = estimate_transit_duration_from_fit(fit) + base_plan['duration'] = duration + if not np.isfinite(duration) or duration <= 0: + base_plan['note'] = "Skipped; the initial nested fit did not produce a measurable transit duration." + return base_plan + + fit_parameters = getattr(fit, 'parameters', {}) + tmid = fit_parameters.get('tmid', prior.get('tmid', np.nan)) + if not np.isfinite(tmid): + base_plan['note'] = "Skipped; the initial nested fit did not return a finite Tmid." + return base_plan + + period = fit_parameters.get('per', prior.get('per', np.nan)) + if np.isfinite(period) and duration >= period: + base_plan['note'] = "Skipped; the initial nested fit returned a non-physical transit duration." + return base_plan + + half_duration = FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER * duration + if not np.isfinite(half_duration) or half_duration <= 0: + base_plan['note'] = "Skipped; the initial nested fit returned an invalid transit duration." + return base_plan + + keep_half_width = duration * (0.5 + baseline_duration_multiplier) + lower_window = tmid - keep_half_width + upper_window = tmid + keep_half_width + keep_mask = np.isfinite(times) & (times >= lower_window) & (times <= upper_window) + + trimmed_pre_points = int(np.count_nonzero(np.isfinite(times) & (times < lower_window))) + trimmed_post_points = int(np.count_nonzero(np.isfinite(times) & (times > upper_window))) + kept_points = int(np.count_nonzero(keep_mask)) + min_required_points = max(LIGHTCURVE_MIN_VALID_POINTS, len(bounds) + 1) + if kept_points < min_required_points: + keep_mask = np.ones(times.shape[0], dtype=bool) + kept_points = int(keep_mask.sum()) + trimmed_pre_points = 0 + trimmed_post_points = 0 + trim_note = ( + "kept the full light curve because trimming would leave too few points for a stable refit" + ) + elif trimmed_pre_points or trimmed_post_points: + trim_note = ( + f"trimmed {trimmed_pre_points} pre-ingress and {trimmed_post_points} post-egress point(s)" + ) + else: + trim_note = "kept the full light curve because no extra baseline points fell outside the target window" + + refined_tmid_bounds = [float(tmid - half_duration), float(tmid + half_duration)] + base_plan['refined_tmid_bounds'] = refined_tmid_bounds + + bounds_changed = False + if original_tmid_bounds is None: + bounds_changed = True + else: + bounds_changed = not np.allclose( + np.asarray(original_tmid_bounds, dtype=float), + np.asarray(refined_tmid_bounds, dtype=float), + atol=1e-12, + rtol=0.0, + ) + + if not np.any(keep_mask): + base_plan['note'] = "Skipped; no valid points remained inside the requested prefit window." + return base_plan + + refined_times = times[keep_mask] + refined_flux = flux_values[keep_mask] + refined_unc = flux_errors[keep_mask] + refined_airmass = airmass[keep_mask] + refined_jd_times = None if jd_array is None else jd_array[keep_mask] + + refined_prior = dict(prior) + if isinstance(fit_parameters, dict): + for key in ('rprs', 'tmid', 'inc', 'a2'): + if key in refined_prior and key in fit_parameters: + refined_prior[key] = fit_parameters[key] + + refined_bounds = clone_lightcurve_bounds(bounds) + refined_bounds['tmid'] = refined_tmid_bounds + + base_plan.update({ + 'applied': bool(trimmed_pre_points or trimmed_post_points or bounds_changed), + 'note': ( + "Using an initial nested fit to estimate a transit duration of " + f"{duration:.6f} day(s), {trim_note}, and setting the second-pass " + f"Tmid bounds to [{refined_tmid_bounds[0]:.6f}, {refined_tmid_bounds[1]:.6f}]." + ), + 'trimmed_pre_points': trimmed_pre_points, + 'trimmed_post_points': trimmed_post_points, + 'refined_point_count': kept_points, + 'times': refined_times, + 'flux': refined_flux, + 'unc': refined_unc, + 'airmass': refined_airmass, + 'jd_times': refined_jd_times, + 'prior': refined_prior, + 'bounds': refined_bounds, + }) + + if not base_plan['applied']: + base_plan['note'] = ( + "Not needed; the initial final-fit solution already used the desired baseline window " + "and the Tmid bounds already matched the modeled transit duration." + ) + + return base_plan + + def fit_final_lightcurve_with_oot_baseline_detrending( times, flux_values, @@ -723,8 +1112,9 @@ def fit_final_lightcurve_with_oot_baseline_detrending( detrend_on_outoftransit_baseline=True, use_impactparameter_rather_than_inclination_to_fit=True, plot_time_range=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, ): - fit = lc_fitter( + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( times, flux_values, flux_errors, @@ -732,21 +1122,75 @@ def fit_final_lightcurve_with_oot_baseline_detrending( prior, bounds, jd_times=jd_times, - mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) + prefit_plan = build_final_fit_prefit_refinement_plan( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + fit, + jd_times=jd_times, + baseline_duration_multiplier=baseline_duration_multiplier, + ) + working_times = prefit_plan['times'] + working_flux = prefit_plan['flux'] + working_unc = prefit_plan['unc'] + working_airmass = prefit_plan['airmass'] + working_jd_times = prefit_plan['jd_times'] + working_prior = prefit_plan['prior'] + working_bounds = prefit_plan['bounds'] + + if prefit_plan.get('applied'): + log_info("Applying final-fit prefit refinement before the optional baseline detrending pass.") + log_info(prefit_plan['note']) + apply_vertical_flux_normalization_bound( + working_prior, + working_bounds, + working_flux, + disable_vertical_flux_normalization, + ) + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + working_times, + working_flux, + working_unc, + working_airmass, + working_prior, + working_bounds, + jd_times=working_jd_times, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + ) + fit = apply_plot_time_range(fit, working_times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(fit, working_airmass, skip_airmass_fit, note=airmass_skip_note) + + annotate_final_fit_prefit_refinement( + fit, + prefit_plan.get('applied', False), + note=prefit_plan.get('note'), + baseline_duration_multiplier=baseline_duration_multiplier, + duration=prefit_plan.get('duration'), + original_point_count=prefit_plan.get('original_point_count'), + refined_point_count=prefit_plan.get('refined_point_count'), + trimmed_pre_points=prefit_plan.get('trimmed_pre_points', 0), + trimmed_post_points=prefit_plan.get('trimmed_post_points', 0), + original_tmid_bounds=prefit_plan.get('original_tmid_bounds'), + refined_tmid_bounds=prefit_plan.get('refined_tmid_bounds'), + ) + if not detrend_on_outoftransit_baseline: annotate_out_of_transit_baseline_detrending( fit, False, note="Disabled; using the direct nested-sampling fit.", ) - return fit, flux_values, flux_errors + return fit, working_flux, working_unc - detrend_result = detrend_flux_on_out_of_transit_baseline(times, flux_values, flux_errors, fit) + detrend_result = detrend_flux_on_out_of_transit_baseline(working_times, working_flux, working_unc, fit) if not detrend_result.get('applied'): note = f"Skipped; {detrend_result.get('note', 'unable to fit an out-of-transit baseline.')}" log_info(f"Optional out-of-transit baseline detrending skipped: {detrend_result.get('note', 'unknown reason')}") @@ -757,20 +1201,17 @@ def fit_final_lightcurve_with_oot_baseline_detrending( pre_points=detrend_result.get('pre_points', 0), post_points=detrend_result.get('post_points', 0), ) - return fit, flux_values, flux_errors + return fit, working_flux, working_unc log_info("Applying optional out-of-transit linear baseline detrending and refitting final light curve.") log_info(detrend_result['note']) - refit_prior = dict(prior) + refit_prior = dict(working_prior) for key in ('rprs', 'tmid', 'inc', 'a2'): if key in refit_prior and key in fit.parameters: refit_prior[key] = fit.parameters[key] - refit_bounds = { - key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value - for key, value in bounds.items() - } + refit_bounds = clone_lightcurve_bounds(working_bounds) apply_vertical_flux_normalization_bound( refit_prior, refit_bounds, @@ -778,19 +1219,31 @@ def fit_final_lightcurve_with_oot_baseline_detrending( disable_vertical_flux_normalization, ) - refit = lc_fitter( - times, + refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + working_times, detrend_result['flux'], detrend_result['unc'], - airmass, + working_airmass, refit_prior, refit_bounds, - jd_times=jd_times, - mode='ns', + jd_times=working_jd_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) - refit = apply_plot_time_range(refit, times if plot_time_range is None else plot_time_range) - annotate_airmass_fit(refit, airmass, skip_airmass_fit, note=airmass_skip_note) + refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_final_fit_prefit_refinement( + refit, + prefit_plan.get('applied', False), + note=prefit_plan.get('note'), + baseline_duration_multiplier=baseline_duration_multiplier, + duration=prefit_plan.get('duration'), + original_point_count=prefit_plan.get('original_point_count'), + refined_point_count=prefit_plan.get('refined_point_count'), + trimmed_pre_points=prefit_plan.get('trimmed_pre_points', 0), + trimmed_post_points=prefit_plan.get('trimmed_post_points', 0), + original_tmid_bounds=prefit_plan.get('original_tmid_bounds'), + refined_tmid_bounds=prefit_plan.get('refined_tmid_bounds'), + ) annotate_out_of_transit_baseline_detrending( refit, True, @@ -4019,7 +4472,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) if final_fit_mode == 'ns' and myfit is not None: - myfit = lc_fitter( + myfit = run_nested_lightcurve_fit_with_rprs_posterior_retry( arrayTimes, arrayFinalFlux, arrayNormUnc, @@ -4027,7 +4480,6 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, prior, mybounds, jd_times=arrayJDTimes, - mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) myfit = apply_plot_time_range(myfit, plot_time_range) @@ -4075,6 +4527,18 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass): diagnostics['relative_flux_point_count'] = int(times_sorted.shape[0]) if has_reference_flux: + finite_ratio_mask = np.isfinite(flux_ratio_sorted) + high_ratio_mask = finite_ratio_mask & np.greater(flux_ratio_sorted, RELATIVE_FLUX_MAX) + nonfinite_ratio_count = int(np.count_nonzero(~finite_ratio_mask)) + high_ratio_count = int(np.count_nonzero(high_ratio_mask)) + rejected_ratio_count = nonfinite_ratio_count + high_ratio_count + finite_ratio_values = flux_ratio_sorted[finite_ratio_mask] + ratio_range_text = "finite ratio range=n/a" + if finite_ratio_values.size: + ratio_range_text = ( + f"finite ratio range={np.nanmin(finite_ratio_values):.4f} to " + f"{np.nanmax(finite_ratio_values):.4f}" + ) relative_flux_mask = relative_flux_filter_mask(flux_ratio_sorted) diagnostics['relative_flux_point_count'] = int(np.count_nonzero(relative_flux_mask)) times_sorted = times_sorted[relative_flux_mask] @@ -4087,8 +4551,11 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass): 'failed_stage': 'relative_flux_filter', 'failure_reason': ( "relative-flux filtering left " - f"{diagnostics['relative_flux_point_count']} usable point(s); invalid, non-finite, " - "or >2x target/reference ratios were rejected." + f"{diagnostics['relative_flux_point_count']} usable point(s); " + f"rejected {rejected_ratio_count}/{diagnostics['input_point_count']} frame(s) " + "during target/reference ratio screening " + f"(non-finite={nonfinite_ratio_count}, >{RELATIVE_FLUX_MAX:g}x={high_ratio_count}, " + f"{ratio_range_text})." ), }) return diagnostics @@ -4201,6 +4668,7 @@ def evaluate_lightcurve_candidate(task): disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit, ) = task + fit_diagnostics = diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass) myfit, tflux_fit, cflux_fit = fit_lightcurve( times, tflux, @@ -4214,13 +4682,22 @@ def evaluate_lightcurve_candidate(task): use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, ) - if myfit is None: - return None, tflux_fit, cflux_fit + fit_diagnostics = ensure_lightcurve_fit_failure_reason( + fit_diagnostics, + myfit, + failed_stage='lightcurve_fit', + failure_reason="the lightcurve fitter did not converge to a usable solution.", + ) - res_std = myfit.residuals.std() / np.median(myfit.data) + res_std = np.inf + if myfit is not None: + res_std = myfit.residuals.std() / np.median(myfit.data) return { 'myfit': myfit, 'res_std': res_std, + 'fit_diagnostics': fit_diagnostics, + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'fit_point_count': 0 if tflux_fit is None else int(len(tflux_fit)), }, tflux_fit, cflux_fit @@ -4259,6 +4736,11 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, 'comp_index': comp_idx, 'ckey': ckey, 'mask': np.ones(target_flux.shape[0], dtype=bool), + 'coverage_count': psf_comp_coverage[ckey]['coverage_count'], + 'coverage_total_frame_count': psf_comp_coverage[ckey]['coverage_total_frame_count'], + 'coverage_reference_count': psf_comp_coverage[ckey]['coverage_reference_count'], + 'coverage_min_required_count': psf_comp_coverage[ckey]['coverage_min_required_count'], + 'coverage_rejected': psf_comp_coverage[ckey]['coverage_rejected'], 'prescore': cheap_lightcurve_prescore(target_flux, comp_flux, airmass), }) @@ -4285,6 +4767,11 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, 'comp_index': None, 'ckey': None, 'mask': np.ones(target_flux.shape[0], dtype=bool), + 'coverage_count': int(target_flux.shape[0]), + 'coverage_total_frame_count': int(target_flux.shape[0]), + 'coverage_reference_count': float(target_flux.shape[0]), + 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, + 'coverage_rejected': False, 'prescore': cheap_lightcurve_prescore( target_flux, np.ones(target_flux.shape[0]), @@ -4308,6 +4795,11 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, 'comp_index': comp_idx, 'ckey': ckey, 'mask': aper_mask, + 'coverage_count': aperture_comp_coverage[ckey]['coverage_count'], + 'coverage_total_frame_count': aperture_comp_coverage[ckey]['coverage_total_frame_count'], + 'coverage_reference_count': aperture_comp_coverage[ckey]['coverage_reference_count'], + 'coverage_min_required_count': aperture_comp_coverage[ckey]['coverage_min_required_count'], + 'coverage_rejected': aperture_comp_coverage[ckey]['coverage_rejected'], 'prescore': cheap_lightcurve_prescore( target_flux[aper_mask], comp_series[aper_mask], @@ -4392,6 +4884,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co return { 'candidate_jobs': candidate_jobs, 'shortlist': shortlist, + 'candidate_summaries': [], 'best_candidate': None, 'best_fit_lc': None, 'min_std': np.inf, @@ -4428,9 +4921,12 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co best_res_std = np.inf best_tflux = None best_cflux = None + candidate_summaries = [] for candidate, result in zip(shortlist, fit_results): fit_meta, tflux_fit, cflux_fit = result - if fit_meta is None: + candidate_summaries.append(summarize_target_fit_candidate(candidate, fit_meta, comp_stars)) + + if fit_meta is None or fit_meta.get('myfit') is None: continue if fit_meta['res_std'] < best_res_std: @@ -4440,9 +4936,15 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co best_tflux = tflux_fit best_cflux = cflux_fit + if best_candidate is not None: + best_identity = target_fit_candidate_identity(best_candidate) + for summary in candidate_summaries: + summary['selected'] = target_fit_candidate_identity(summary) == best_identity + return { 'candidate_jobs': candidate_jobs, 'shortlist': shortlist, + 'candidate_summaries': candidate_summaries, 'best_candidate': best_candidate, 'best_fit_lc': best_fit_lc, 'min_std': best_res_std, @@ -4477,6 +4979,127 @@ def format_comp_star_position(position): return f"coords={position}" +def format_comp_star_coverage_text(summary): + coverage_text = ( + f"{summary['coverage_count']} valid frame(s)" + f" out of {summary.get('coverage_total_frame_count', 'n/a')} total" + f"; min_required={summary.get('coverage_min_required_count', 0)}" + ) + coverage_median = summary.get('coverage_reference_count', np.nan) + if np.isfinite(coverage_median): + coverage_text += f"; peer_median={coverage_median:.1f}" + return coverage_text + + +def target_fit_candidate_identity(candidate): + return ( + candidate.get('method'), + candidate.get('a'), + candidate.get('an'), + candidate.get('comp_index'), + ) + + +def summarize_target_fit_candidate(candidate, fit_meta, comp_stars): + comp_index = candidate.get('comp_index') + fit_meta = {} if fit_meta is None else dict(fit_meta) + fit_result = fit_meta.get('myfit') + return { + 'label': "Target-only" if comp_index is None else f"Comp {comp_index + 1}", + 'position': None if comp_index is None else comp_stars[comp_index], + 'selected': False, + 'method_label': comparison_method_label(candidate), + 'prescore': candidate.get('prescore', np.inf), + 'fit': fit_result, + 'res_std': fit_meta.get('res_std', np.inf), + 'coverage_count': candidate.get('coverage_count', 0), + 'coverage_total_frame_count': candidate.get('coverage_total_frame_count', 0), + 'coverage_reference_count': candidate.get('coverage_reference_count', np.nan), + 'coverage_min_required_count': candidate.get('coverage_min_required_count', 0), + 'coverage_rejected': candidate.get('coverage_rejected', False), + 'fit_point_count': fit_meta.get('fit_point_count', 0), + 'fit_diagnostics': fit_meta.get('fit_diagnostics') or {}, + 'failure_reason': fit_meta.get('failure_reason'), + 'parameter_summary': summarize_lightcurve_fit_parameters(fit_result), + 'comp_index': comp_index, + 'a': candidate.get('a'), + 'an': candidate.get('an'), + 'method': candidate.get('method'), + } + + +def log_comparison_calibration_fit_attempt_summaries(attempts, method_label): + if not attempts: + return + + log_info("\nComparison-star calibration target-fit diagnostics:") + log_info(f"Photometry method: {method_label}") + + for attempt in attempts: + selected_label = " [selected]" if attempt.get('selected') else "" + position_text = format_comp_star_position(attempt.get('position')) + diagnostics = attempt.get('fit_diagnostics') or {} + usable_point_count = diagnostics.get('usable_point_count', 0) + coverage_text = format_comp_star_coverage_text(attempt) + suitability_score = attempt.get('aggregate_score', np.inf) + suitability_text = "n/a" if not np.isfinite(suitability_score) else f"{suitability_score * 100.0:.4f}%" + residual_text = "n/a" + if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)): + residual_text = f"{attempt['res_std'] * 100.0:.4f}%" + reason_text = attempt.get( + 'failure_reason', + "selected: first coverage-qualified comparison star with a usable target fit", + ) + log_info( + f" {attempt['label']}{selected_label} ({position_text}): " + f"suitability={suitability_text}, coverage={coverage_text}, " + f"usable_after_filters={usable_point_count}, fit_points={attempt.get('fit_point_count', 0)}, " + f"residual_scatter={residual_text}, reason={reason_text}" + ) + parameter_summary = attempt.get('parameter_summary') + if parameter_summary: + log_info(f" parameters: {parameter_summary}") + + +def log_target_fit_candidate_summaries(candidate_summaries, max_entries=10): + if not candidate_summaries: + return + + log_info("\nTarget-fit candidate diagnostics:") + + displayed_summaries = candidate_summaries[:max_entries] + for summary in displayed_summaries: + selected_label = " [selected]" if summary.get('selected') else "" + position_text = format_comp_star_position(summary.get('position')) + diagnostics = summary.get('fit_diagnostics') or {} + usable_point_count = diagnostics.get('usable_point_count', 0) + coverage_text = format_comp_star_coverage_text(summary) + prescore = summary.get('prescore', np.inf) + prescore_text = "n/a" if not np.isfinite(prescore) else f"{prescore * 100.0:.4f}%" + residual_text = "n/a" + if summary.get('fit') is not None and np.isfinite(summary.get('res_std', np.inf)): + residual_text = f"{summary['res_std'] * 100.0:.4f}%" + reason_text = summary.get( + 'failure_reason', + "selected: lowest target-fit residual scatter in the evaluated shortlist", + ) + log_info( + f" {summary['label']}{selected_label} ({position_text}) with {summary['method_label']}: " + f"prescore={prescore_text}, coverage={coverage_text}, " + f"usable_after_filters={usable_point_count}, fit_points={summary.get('fit_point_count', 0)}, " + f"residual_scatter={residual_text}, reason={reason_text}" + ) + parameter_summary = summary.get('parameter_summary') + if parameter_summary: + log_info(f" parameters: {parameter_summary}") + + if len(candidate_summaries) > len(displayed_summaries): + log_info( + f" ... omitted {len(candidate_summaries) - len(displayed_summaries)} additional " + "target-fit candidate(s); consider increasing the log limit if you need the full list." + ) + + def comparison_calibration_selection_reason(summary, best_comp_score): if summary.get('selected'): return "selected: lowest suitability score among coverage-qualified comparison stars for this method" @@ -4611,12 +5234,7 @@ def log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_i position_text = format_comp_star_position(summary.get('position')) diagnostics = summary.get('fit_diagnostics') or {} usable_point_count = diagnostics.get('usable_point_count', 0) - coverage_median = summary.get('coverage_reference_count', np.nan) - coverage_text = ( - f"{summary['coverage_count']}/{summary.get('coverage_min_required_count', 0)} valid frame(s)" - ) - if np.isfinite(coverage_median): - coverage_text += f", peer_median={coverage_median:.1f}" + coverage_text = format_comp_star_coverage_text(summary) residual_text = "n/a" if summary.get('fit') is not None and np.isfinite(summary.get('res_std', np.inf)): residual_text = f"{summary['res_std'] * 100.0:.4f}%" @@ -4675,6 +5293,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p fit_mask = valid_comparison_frame_mask(comp_flux_series) coverage_count = coverage_summary[ckey]['coverage_count'] + coverage_total_frame_count = coverage_summary[ckey]['coverage_total_frame_count'] coverage_reference_count = coverage_summary[ckey]['coverage_reference_count'] coverage_min_required_count = coverage_summary[ckey]['coverage_min_required_count'] coverage_rejected = coverage_summary[ckey]['coverage_rejected'] @@ -4740,6 +5359,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p 'fit': fit_result, 'res_std': res_std, 'coverage_count': coverage_count, + 'coverage_total_frame_count': coverage_total_frame_count, 'coverage_reference_count': coverage_reference_count, 'coverage_min_required_count': coverage_min_required_count, 'coverage_rejected': coverage_rejected, @@ -4840,6 +5460,7 @@ def comparison_star_coverage_summary(comp_flux_map, for key in comp_keys: coverage_summary[key] = { 'coverage_count': coverage_counts[key], + 'coverage_total_frame_count': total_frame_count, 'coverage_reference_count': coverage_reference_count, 'coverage_median_count': coverage_reference_count, 'coverage_scatter': coverage_scatter, @@ -4924,6 +5545,7 @@ def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_ 'pairwise_ratio_series': pairwise_series, 'ensemble_ratio_series': ensemble_ratio_series, 'coverage_count': coverage_summary[key]['coverage_count'], + 'coverage_total_frame_count': coverage_summary[key]['coverage_total_frame_count'], 'coverage_reference_count': coverage_summary[key]['coverage_reference_count'], 'coverage_min_required_count': coverage_summary[key]['coverage_min_required_count'], 'coverage_rejected': coverage_summary[key]['coverage_rejected'], @@ -5312,11 +5934,23 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'rank': rank, 'comp_index': comp_index, 'ckey': ckey, + 'label': comp_summary.get('label', f"Comp {comp_index + 1}"), + 'position': comp_summary.get('position'), + 'aggregate_score': comp_summary.get('aggregate_score', np.inf), + 'coverage_count': comp_summary.get('coverage_count', 0), + 'coverage_total_frame_count': comp_summary.get('coverage_total_frame_count', 0), + 'coverage_reference_count': comp_summary.get('coverage_reference_count', np.nan), + 'coverage_min_required_count': comp_summary.get('coverage_min_required_count', 0), + 'coverage_rejected': comp_summary.get('coverage_rejected', False), 'fit': fit_result, 'tflux_fit': tflux_fit, 'cflux_fit': cflux_fit, 'fit_diagnostics': fit_diagnostics, 'res_std': res_std, + 'fit_point_count': 0 if tflux_fit is None else int(len(tflux_fit)), + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'parameter_summary': summarize_lightcurve_fit_parameters(fit_result), + 'selected': False, } attempts.append(attempt) @@ -5324,6 +5958,9 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p selected_result = attempt break + if selected_result is not None: + selected_result['selected'] = True + return { 'ranked_summaries': ranked_summaries, 'attempts': attempts, @@ -5527,6 +6164,12 @@ def main(): detrend_on_outoftransit_baseline = is_out_of_transit_baseline_detrending_enabled( exotic_infoDict.get('detrend_on_outoftransit_baseline', True) ) + final_fit_baseline_duration_multiplier = get_final_fit_baseline_duration_multiplier( + exotic_infoDict.get( + 'final_fit_baseline_duration_multiplier', + FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + ) + ) use_impactparameter_rather_than_inclination_to_fit = ( should_use_impactparameter_rather_than_inclination_to_fit( exotic_infoDict.get('use_impactparameter_rather_than_inclination_to_fit', 'y') @@ -6263,6 +6906,7 @@ def main(): } comparison_calibration = None + target_driven_search = None if target_driven_comp_selection: log_info("\nUsing target-driven comparison-star selection per optional_info setting.") else: @@ -6289,10 +6933,7 @@ def main(): pairwise_text = "n/a" if not np.isfinite(summary['pairwise_median_score']) else f"{summary['pairwise_median_score'] * 100.0:.4f}%" selected_label = " [selected]" if summary['selected'] else "" position_text = format_comp_star_position(summary['position']) - coverage_text = ( - f"coverage={summary['coverage_count']}/{summary['coverage_min_required_count']} " - f"(peer_median={summary['coverage_reference_count']:.1f})" - ) + coverage_text = f"coverage={format_comp_star_coverage_text(summary)}" if summary['coverage_rejected']: coverage_text += " [rejected: low coverage]" log_info( @@ -6370,6 +7011,11 @@ def main(): fit_attempts = comparison_fit_search['attempts'] if fit_attempts: comparison_calibration['selected_fit_diagnostics'] = fit_attempts[-1]['fit_diagnostics'] + if selected_attempt is None or len(fit_attempts) > 1: + log_comparison_calibration_fit_attempt_summaries( + fit_attempts, + comparison_calibration['method_label'], + ) if selected_attempt is not None: selected_comp_index = selected_attempt['comp_index'] @@ -6551,6 +7197,22 @@ def main(): x_ref=x_ref_data, y_ref=y_ref_data, ) + else: + candidate_summaries = target_driven_search.get('candidate_summaries', []) + if candidate_summaries: + log_info( + "Warning: Target-driven photometry search evaluated " + f"{len(candidate_summaries)} shortlisted candidate(s) but none yielded " + "a usable lightcurve fit.", + warn=True, + ) + log_target_fit_candidate_summaries(candidate_summaries) + elif require_comp_star: + log_info( + "Warning: Target-driven comparison-star search did not produce any " + "coverage-qualified comparison-star candidates to evaluate.", + warn=True, + ) if best_candidate is not None and vsp_num: if best_candidate['method'] == 'psf': @@ -6599,7 +7261,18 @@ def main(): ) if require_comp_star and photometry_info['comp_star_num'] is None: - log_info("Error: require_comp_star is enabled, but no valid comparison star could be selected.", error=True) + if target_driven_search is not None and target_driven_search.get('candidate_summaries'): + log_info( + "Error: require_comp_star is enabled, but every evaluated comparison-star candidate " + "was rejected. See the target-fit candidate diagnostics above for the per-candidate " + "failure reasons.", + error=True, + ) + else: + log_info( + "Error: require_comp_star is enabled, but no valid comparison star could be selected.", + error=True, + ) return log_info("\n\n*********************************************") @@ -6991,6 +7664,7 @@ def main(): use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, plot_time_range=full_plot_time_range, + baseline_duration_multiplier=final_fit_baseline_duration_multiplier, ) # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index d1e63590..c1bd54ba 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -420,6 +420,7 @@ def save_input(): "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -444,6 +445,7 @@ def save_input(): "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, + "final_fit_baseline_duration_multiplier": 1.0, "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1494,6 +1496,7 @@ def save_input(): "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -1566,6 +1569,7 @@ def save_input(): "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, + "final_fit_baseline_duration_multiplier": 1.0, "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1617,6 +1621,7 @@ def save_input(): "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, + "final_fit_baseline_duration_multiplier": 1.0, "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/exotic/inputs.py b/exotic/inputs.py index f2f44cf5..de805bfe 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -210,6 +210,7 @@ def __init__(self, init_opt): 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'detrend_on_outoftransit_baseline': True, + 'final_fit_baseline_duration_multiplier': 1.0, 'detect_bad_pixels_before_photometry': 'y', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', @@ -433,6 +434,10 @@ def comp_params(self, init_file, planet_dict): 'Detrend on Out-of-Transit Baseline', 'detrend_on_out_of_transit_baseline', ), + 'final_fit_baseline_duration_multiplier': ( + 'final_fit_baseline_duration_multiplier', + 'Final Fit Baseline Duration Multiplier', + ), 'use_impactparameter_rather_than_inclination_to_fit': ( 'use_impactparameter_rather_than_inclination_to_fit', 'Use impact parameter rather than inclination to fit? (y/n)', diff --git a/inits.json b/inits.json index 1783e1f0..aca029bf 100644 --- a/inits.json +++ b/inits.json @@ -28,6 +28,7 @@ "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", @@ -106,6 +107,7 @@ "disable vertical flux normalization": false, "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": true, + "final_fit_baseline_duration_multiplier": 1.0, "use_impactparameter_rather_than_inclination_to_fit": "y", "use_psf_photometry": "y", "use_aperture_photometry": "y", diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 438199d7..908a1916 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -509,6 +509,42 @@ def __init__(self, *args, **kwargs): assert fit.errors["inc"] > 0 +def test_rprs_posterior_recenter_diagnostics_detect_upper_bound_clipping(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + fit.sampled_keys = ["rprs", "tmid"] + fit.sample_bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + + rprs_samples = np.concatenate([ + np.linspace(0.090, 0.120, 12), + np.linspace(0.128, 0.149, 28), + ]) + tmid_samples = np.linspace(-2e-4, 2e-4, rprs_samples.size) + points = np.column_stack([rprs_samples, tmid_samples]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-6.0, -3.0, rprs_samples.size), + }, + "samples": points.copy(), + } + + diagnostics = fit.get_parameter_posterior_recenter_diagnostics("rprs") + + assert diagnostics["clipped"] is True + assert diagnostics["edge"] == "upper" + assert diagnostics["mode"] > 0.13 + assert diagnostics["std"] > 0 + assert diagnostics["bounds"][0] >= 0.0 + assert diagnostics["bounds"][1] > 0.15 + + def test_plot_triangle_uses_mirrored_distance_from_fitted_impact_parameter_axis(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index fabc0c9f..d74de832 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -89,17 +89,21 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: comparison_candidate_fit_selection_reason, comparison_star_coverage_summary, comparison_star_stability_summary, + diagnose_lightcurve_fit_inputs, detrend_flux_on_out_of_transit_baseline, ensure_lightcurve_fit_failure_reason, fit_lightcurve, fit_final_lightcurve_with_oot_baseline_detrending, fit_lightcurve_to_every_comparison_candidate, fit_ranked_comparison_calibration_candidates, + get_final_fit_baseline_duration_multiplier, is_adaptive_aperture_mode_enabled, is_comp_star_required, is_out_of_transit_baseline_detrending_enabled, is_target_driven_comp_selection_enabled, + log_comparison_calibration_fit_attempt_summaries, log_comparison_candidate_fit_summaries, + log_target_fit_candidate_summaries, phase_bin_sigma_clip, representative_psf_sigma, run_target_driven_photometry_search, @@ -217,6 +221,13 @@ def test_is_out_of_transit_baseline_detrending_enabled_parses_values(): assert is_out_of_transit_baseline_detrending_enabled(True) is True +def test_get_final_fit_baseline_duration_multiplier_parses_values(): + assert get_final_fit_baseline_duration_multiplier(None) == pytest.approx(1.0) + assert get_final_fit_baseline_duration_multiplier("2.5") == pytest.approx(2.5) + assert get_final_fit_baseline_duration_multiplier(0) == pytest.approx(0.0) + assert get_final_fit_baseline_duration_multiplier(-1) == pytest.approx(1.0) + + def test_should_use_psf_photometry_parses_values(): assert should_use_psf_photometry(None) is True assert should_use_psf_photometry("y") is True @@ -497,6 +508,7 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): "fit": None, "res_std": np.inf, "coverage_count": 1, + "coverage_total_frame_count": 3, "coverage_reference_count": 3.0, "coverage_min_required_count": 2, "fit_point_count": 0, @@ -510,6 +522,7 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): "fit": object(), "res_std": 0.01, "coverage_count": 3, + "coverage_total_frame_count": 3, "coverage_reference_count": 3.0, "coverage_min_required_count": 2, "fit_point_count": 3, @@ -525,11 +538,95 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): ) assert any("Selection basis: comparison-field" in message for message in logged) + assert any("coverage=1 valid frame(s) out of 3 total; min_required=2; peer_median=3.0" in message for message in logged) assert any("Comp 1" in message and "reason=comparison candidate rejected after iterative low-coverage clipping" in message for message in logged) assert any("Comp 2 [selected]" in message and "comparison-field calibration ranked this star best" in message for message in logged) assert any("parameters: fit_method=ultranest" in message for message in logged) +def test_log_comparison_calibration_fit_attempt_summaries_includes_reasons(monkeypatch): + logged = [] + monkeypatch.setattr("exotic.exotic.log_info", lambda message, warn=False, error=False: logged.append(message)) + + attempts = [ + { + "label": "Comp 1", + "position": [100, 200], + "selected": False, + "aggregate_score": 0.01, + "coverage_count": 3, + "coverage_total_frame_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit": None, + "res_std": np.inf, + "fit_point_count": 0, + "fit_diagnostics": {"usable_point_count": 0}, + "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during target/reference ratio screening (non-finite=0, >2x=3, finite ratio range=3.0000 to 3.0000).", + "parameter_summary": None, + }, + { + "label": "Comp 2", + "position": [300, 400], + "selected": True, + "aggregate_score": 0.02, + "coverage_count": 3, + "coverage_total_frame_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit": object(), + "res_std": 0.01, + "fit_point_count": 3, + "fit_diagnostics": {"usable_point_count": 3}, + "failure_reason": None, + "parameter_summary": "fit_method=ultranest, Tmid=1.0 +/- 0.1", + }, + ] + + log_comparison_calibration_fit_attempt_summaries(attempts, "Aperture photometry (aper=7.05px, annulus=22.73px)") + + assert any("Comparison-star calibration target-fit diagnostics:" in message for message in logged) + assert any("Photometry method: Aperture photometry (aper=7.05px, annulus=22.73px)" in message for message in logged) + assert any("Comp 1" in message and "reason=relative-flux filtering left 0 usable point(s)" in message for message in logged) + assert any("Comp 2 [selected]" in message and "fit_points=3" in message for message in logged) + assert any("parameters: fit_method=ultranest" in message for message in logged) + + +def test_log_target_fit_candidate_summaries_includes_methods_and_reasons(monkeypatch): + logged = [] + monkeypatch.setattr("exotic.exotic.log_info", lambda message, warn=False, error=False: logged.append(message)) + + candidate_summaries = [ + { + "label": "Comp 1", + "position": [100, 200], + "selected": False, + "method_label": "Aperture photometry (aper=7.05px, annulus=22.73px)", + "prescore": 0.005, + "fit": None, + "res_std": np.inf, + "coverage_count": 3, + "coverage_total_frame_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit_point_count": 0, + "fit_diagnostics": {"usable_point_count": 0}, + "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during target/reference ratio screening (non-finite=0, >2x=3, finite ratio range=3.0000 to 3.0000).", + "parameter_summary": None, + }, + ] + + log_target_fit_candidate_summaries(candidate_summaries) + + assert any("Target-fit candidate diagnostics:" in message for message in logged) + assert any( + "Comp 1" in message + and "with Aperture photometry (aper=7.05px, annulus=22.73px)" in message + and "reason=relative-flux filtering left 0 usable point(s)" in message + for message in logged + ) + + def test_comparison_candidate_fit_selection_reason_describes_comparison_field_retry(): reason = comparison_candidate_fit_selection_reason( { @@ -576,6 +673,7 @@ def test_comparison_star_coverage_summary_rejects_sparse_candidates(): assert not coverage["comp2"]["coverage_rejected"] assert coverage["comp3"]["coverage_rejected"] assert coverage["comp3"]["coverage_count"] == 2 + assert coverage["comp3"]["coverage_total_frame_count"] == 6 def test_comparison_star_coverage_summary_iteratively_rejects_low_count_tail(): @@ -596,6 +694,7 @@ def test_comparison_star_coverage_summary_iteratively_rejects_low_count_tail(): assert coverage["comp4"]["coverage_rejected"] assert coverage["comp5"]["coverage_rejected"] assert coverage["comp6"]["coverage_rejected"] + assert coverage["comp1"]["coverage_total_frame_count"] == 10 assert coverage["comp1"]["coverage_reference_count"] == pytest.approx(10.0) assert coverage["comp1"]["coverage_min_required_count"] == 8 @@ -776,6 +875,153 @@ def fake_lc_fitter( assert fit.plot_time_range == pytest.approx(plot_time_range) +def test_fit_final_lightcurve_retries_nested_fit_when_rprs_posterior_is_clipped(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + + def make_fit(call_flux, diagnostics): + fit = types.SimpleNamespace( + parameters={"tmid": 0.0, "rprs": 0.152, "inc": 89.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.002, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: diagnostics if key == "rprs" else None + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": {key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value for key, value in call_bounds.items()}, + }) + if len(captured["calls"]) == 1: + return make_fit( + call_flux, + { + "clipped": True, + "edge": "upper", + "mode": 0.158, + "std": 0.006, + "bounds": [0.128, 0.188], + "reason": "posterior peaks against the upper search bound.", + }, + ) + return make_fit( + call_flux, + { + "clipped": False, + "edge": None, + "mode": 0.159, + "std": 0.005, + "bounds": [0.128, 0.188], + "reason": "posterior support is comfortably inside the sampled bounds.", + }, + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.125], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.125]) + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.158) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.128, 0.188]) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 1 + assert fit.rprs_posterior_refit_edge == "upper" + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.128, 0.188]) + + +def test_fit_final_lightcurve_prefit_refinement_trims_baseline_and_recenters_tmid(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["calls"].append({ + "times": np.array(call_times, dtype=float), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return types.SimpleNamespace( + duration_expected=2.0, + duration_measured=2.0, + parameters={"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.array([-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0], dtype=float) + flux = np.ones(times.shape[0], dtype=float) + fluxerr = np.full(times.shape[0], 0.01, dtype=float) + airmass = np.ones(times.shape[0], dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0} + bounds = {"rprs": [0.0, 0.2], "tmid": [-2.0, 2.0], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit, trimmed_flux, trimmed_unc = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + baseline_duration_multiplier=0.5, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["times"] == pytest.approx(times) + assert captured["calls"][1]["times"] == pytest.approx(np.array([-2.0, -1.0, 0.0, 1.0, 2.0])) + assert captured["calls"][1]["bounds"]["tmid"] == pytest.approx([-1.0, 1.0]) + assert trimmed_flux == pytest.approx(np.ones(5)) + assert trimmed_unc == pytest.approx(np.full(5, 0.01)) + assert fit.prefit_refinement_applied is True + assert fit.prefit_refinement_trimmed_pre_points == 1 + assert fit.prefit_refinement_trimmed_post_points == 1 + assert fit.prefit_refinement_tmid_bounds == pytest.approx([-1.0, 1.0]) + + def test_fit_lightcurve_removes_relative_flux_above_two_before_fit(monkeypatch): captured = {} @@ -1241,6 +1487,60 @@ def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times, **kw assert result["attempts"][1]["fit"] is not None +def test_diagnose_lightcurve_fit_inputs_reports_relative_flux_breakdown(): + diagnostics = diagnose_lightcurve_fit_inputs( + np.linspace(0.0, 0.05, 6), + np.full(6, 30.0), + np.full(6, 10.0), + np.linspace(1.0, 1.5, 6), + ) + + assert diagnostics["failed_stage"] == "relative_flux_filter" + assert "relative-flux filtering left 0 usable point(s)" in diagnostics["failure_reason"] + assert "non-finite=0" in diagnostics["failure_reason"] + assert ">2x=6" in diagnostics["failure_reason"] + assert "finite ratio range=3.0000 to 3.0000" in diagnostics["failure_reason"] + + +def test_run_target_driven_photometry_search_returns_failed_candidate_summaries(monkeypatch): + monkeypatch.setattr("exotic.exotic.fit_lightcurve", lambda *args, **kwargs: (None, None, None)) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + aper_data = { + "target": np.full((6, 1, 1), 30.0), + "comp1": np.full((6, 1, 1), 10.0), + "comp2": np.full((6, 1, 1), 12.0), + } + + result = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comp_stars=[[100.0, 200.0], [300.0, 400.0]], + psf_data={}, + aper_data=aper_data, + apers=np.array([7.05]), + annuli=np.array([22.73]), + sigma=1.0, + require_comp_star=True, + use_psf_photometry=False, + use_aperture_photometry=True, + multiprocess_lightcurve_fits=0, + ) + + assert result["best_candidate"] is None + assert len(result["candidate_summaries"]) == 2 + assert all( + "relative-flux filtering left 0 usable point(s)" in summary["failure_reason"] + for summary in result["candidate_summaries"] + ) + assert result["candidate_summaries"][0]["method_label"] == "Aperture photometry (aper=7.05px, annulus=22.73px)" + + def test_fit_lightcurve_to_every_comparison_candidate_forwards_full_plot_time_range(monkeypatch): captured_plot_ranges = [] diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 2c19724c..b24fd81f 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -173,6 +173,21 @@ def test_comp_params_defaults_detrend_on_outoftransit_baseline_to_true(tmp_path) assert inputs.info_dict["detrend_on_outoftransit_baseline"] is True +def test_comp_params_defaults_final_fit_baseline_duration_multiplier_to_one(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["final_fit_baseline_duration_multiplier"] == pytest.approx(1.0) + + def test_comp_params_defaults_use_impactparameter_fit_to_yes(tmp_path): init_data = { "user_info": {}, @@ -383,6 +398,21 @@ def test_comp_params_reads_detrend_on_outoftransit_baseline_false_from_optional_ assert inputs.info_dict["detrend_on_outoftransit_baseline"] is False +def test_comp_params_reads_final_fit_baseline_duration_multiplier_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"final_fit_baseline_duration_multiplier": 1.75}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["final_fit_baseline_duration_multiplier"] == pytest.approx(1.75) + + def test_comp_params_reads_use_impactparameter_fit_from_optional_info(tmp_path): init_data = { "user_info": {}, @@ -847,6 +877,15 @@ def test_lookup_aavso_filter_metadata_uses_c_alias_for_cv_filter() -> None: assert filter_metadata["fwhm"] == ("350.0", "850.0") +def test_lookup_aavso_filter_metadata_uses_luminosity_aliases_for_clearv_filter() -> None: + for alias in ("lum", "Lum", "Luminosity", "luminosity"): + filter_metadata = inputs_module.lookup_aavso_filter_metadata(alias) + + assert filter_metadata["name"] == "CV" + assert filter_metadata["desc"] == "ClearV" + assert filter_metadata["fwhm"] == ("350.0", "1000.0") + + def test_parse_aavso_prereduced_overrides_uses_osc_split_filter_alias_lookup(tmp_path): pre_reduced_file = tmp_path / "aavso_prereduced.txt" pre_reduced_file.write_text( @@ -881,6 +920,23 @@ def test_parse_aavso_prereduced_overrides_uses_c_alias_for_cv_filter_lookup(tmp_ assert overrides["wl_max"] == "850.0" +def test_parse_aavso_prereduced_overrides_uses_luminosity_alias_for_clearv_lookup(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=Luminosity\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "Luminosity" + assert overrides["filter_desc"] == "ClearV" + assert overrides["wl_min"] == "350.0" + assert overrides["wl_max"] == "1000.0" + + def test_parse_aavso_prereduced_overrides_marks_airmass_as_already_corrected(tmp_path): pre_reduced_file = tmp_path / "aavso_prereduced.txt" pre_reduced_file.write_text( From 3366347511c09d92a9abcbef4afc898af7b1101f Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sun, 19 Apr 2026 19:12:29 +1000 Subject: [PATCH 022/116] Harden NextAstro requests and Rp/R* posterior retries --- exotic/api/elca.py | 50 ++++- exotic/api/filters.py | 1 + exotic/api/http_compression.py | 28 +++ exotic/api/plate_solution.py | 13 +- exotic/exotic.py | 192 ++++++++++++++--- requirements.txt | 1 + tests/test_elca_baseline.py | 73 +++++++ tests/test_exotic_proper_motion.py | 30 ++- tests/test_exotic_rprs_retry.py | 322 ++++++++++++++++++++++++++++ tests/test_inputs.py | 17 ++ tests/test_ld.py | 1 + tests/test_nextastro_astrometry.py | 33 ++- tests/test_nextastro_variability.py | 21 +- 13 files changed, 727 insertions(+), 55 deletions(-) create mode 100644 exotic/api/http_compression.py create mode 100644 tests/test_exotic_rprs_retry.py diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 58a7fe0d..ff0b53ab 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -626,6 +626,11 @@ def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bin 'bounds': None, 'original_bounds': None, 'sample_size': 0, + 'peak_height': np.nan, + 'lower_edge_height': np.nan, + 'upper_edge_height': np.nan, + 'lower_edge_peak_fraction': np.nan, + 'upper_edge_peak_fraction': np.nan, 'reason': None, } @@ -664,6 +669,29 @@ def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bin q05, q16, q50, q84, q95 = np.nanpercentile(finite_samples, [5, 16, 50, 84, 95]) width = float(upper_bound - lower_bound) + histogram_bins = int(np.clip(np.sqrt(finite_samples.size), 10, 80)) if bins is None else max(1, int(bins)) + histogram_counts, _ = np.histogram( + finite_samples, + bins=histogram_bins, + range=(lower_bound, upper_bound), + ) + histogram_counts = np.asarray(histogram_counts, dtype=float) + peak_height = float(np.nanmax(histogram_counts)) if histogram_counts.size else np.nan + lower_edge_height = float(histogram_counts[0]) if histogram_counts.size else np.nan + upper_edge_height = float(histogram_counts[-1]) if histogram_counts.size else np.nan + if np.isfinite(peak_height) and peak_height > 0: + lower_edge_peak_fraction = float(lower_edge_height / peak_height) + upper_edge_peak_fraction = float(upper_edge_height / peak_height) + else: + lower_edge_peak_fraction = np.nan + upper_edge_peak_fraction = np.nan + + diagnostics['peak_height'] = peak_height + diagnostics['lower_edge_height'] = lower_edge_height + diagnostics['upper_edge_height'] = upper_edge_height + diagnostics['lower_edge_peak_fraction'] = lower_edge_peak_fraction + diagnostics['upper_edge_peak_fraction'] = upper_edge_peak_fraction + scale_floor = max( 2.0 * bin_width if np.isfinite(bin_width) and bin_width > 0 else 0.0, 0.01 * width, @@ -684,6 +712,19 @@ def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bin upper_clipped = upper_gap_q95 <= tail_gap_threshold and upper_gap_mode <= mode_gap_threshold lower_clipped = lower_gap_q05 <= tail_gap_threshold and lower_gap_mode <= mode_gap_threshold + edge_peak_fraction_floor = 0.20 + rejected_edges = [] + + if upper_clipped and np.isfinite(upper_edge_peak_fraction) and upper_edge_peak_fraction < edge_peak_fraction_floor: + upper_clipped = False + rejected_edges.append( + f"upper edge histogram height is only {upper_edge_peak_fraction:.3f} of the posterior peak" + ) + if lower_clipped and np.isfinite(lower_edge_peak_fraction) and lower_edge_peak_fraction < edge_peak_fraction_floor: + lower_clipped = False + rejected_edges.append( + f"lower edge histogram height is only {lower_edge_peak_fraction:.3f} of the posterior peak" + ) if upper_clipped and lower_clipped: clipped_edge = 'upper' if upper_gap_mode <= lower_gap_mode else 'lower' @@ -693,7 +734,14 @@ def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bin clipped_edge = 'lower' else: diagnostics['bounds'] = [float(lower_bound), float(upper_bound)] - diagnostics['reason'] = "posterior support is comfortably inside the sampled bounds." + if rejected_edges: + diagnostics['reason'] = ( + "posterior reaches a search bound, but " + + " and ".join(rejected_edges) + + ", so it is not treated as truncated." + ) + else: + diagnostics['reason'] = "posterior support is comfortably inside the sampled bounds." return diagnostics diagnostics['clipped'] = True diff --git a/exotic/api/filters.py b/exotic/api/filters.py index 2fac5f55..fc998414 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -82,6 +82,7 @@ "Clear (unfiltered) reduced to R sequence": "Cousins R", "Clear with blue-blocking": "Astrodon ExoPlanet-BB", + "Astrodon-Exo": "Astrodon ExoPlanet-BB", "Exop": "Astrodon ExoPlanet-BB", # additional short aliases found in FILTER column values diff --git a/exotic/api/http_compression.py b/exotic/api/http_compression.py new file mode 100644 index 00000000..6ae9fd43 --- /dev/null +++ b/exotic/api/http_compression.py @@ -0,0 +1,28 @@ +import gzip +import json + +try: + import zstandard +except ImportError: # pragma: no cover - gzip fallback covers environments without zstandard + zstandard = None + + +_GZIP_LEVEL = 6 +_ZSTD_LEVEL = 3 + + +def build_compressed_json_request(payload): + raw_body = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + if zstandard is not None: + compressed_body = zstandard.ZstdCompressor(level=_ZSTD_LEVEL).compress(raw_body) + encoding = "zstd" + else: + compressed_body = gzip.compress(raw_body, compresslevel=_GZIP_LEVEL) + encoding = "gzip" + + headers = { + "Content-Encoding": encoding, + "Content-Type": "application/json", + } + return compressed_body, headers, encoding, len(raw_body), len(compressed_body) diff --git a/exotic/api/plate_solution.py b/exotic/api/plate_solution.py index c4336d68..4598f1f9 100644 --- a/exotic/api/plate_solution.py +++ b/exotic/api/plate_solution.py @@ -46,6 +46,11 @@ from tenacity import retry, retry_if_exception_type, retry_if_result, \ stop_after_attempt, wait_exponential +try: + from .http_compression import build_compressed_json_request +except ImportError: + from http_compression import build_compressed_json_request + _R_MAX_STOPS_LOW = 7 _R_MAX_STOPS = 10 _R_MAX_SECS = 37 @@ -364,8 +369,14 @@ def _submit_solve_request(self, source_list): if hints is not None: payload["hints"] = hints + request_body, headers, content_encoding, raw_size, compressed_size = build_compressed_json_request(payload) + self._emit_debug(f"NextAstro astrometry request JSON: {self._json_message(payload)}") - response = requests.post(f"{self.api_url}/solve", json=payload, timeout=_RQ_TIMEOUT) + self._emit_debug( + "NextAstro astrometry request compression: " + f"{content_encoding} ({compressed_size} bytes sent; {raw_size} bytes raw)" + ) + response = requests.post(f"{self.api_url}/solve", data=request_body, headers=headers, timeout=_RQ_TIMEOUT) response_json = self._decode_response_json(response, 'Solve response') if response_json is not None and response.status_code != 502: self._emit_debug(f"NextAstro astrometry submission response JSON: {self._json_message(response_json)}") diff --git a/exotic/exotic.py b/exotic/exotic.py index 7d2e1be5..f7a81fc3 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -117,6 +117,10 @@ from .api.plate_solution import NextAstroPlateSolution, PlateSolution except ImportError: # package import from api.plate_solution import NextAstroPlateSolution, PlateSolution +try: + from .api.http_compression import build_compressed_json_request +except ImportError: + from api.http_compression import build_compressed_json_request try: # nea from .api.nea import NASAExoplanetArchive except ImportError: # package import @@ -171,6 +175,9 @@ COMPARISON_STAR_COVERAGE_MAX_ITERS = 10 OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 +RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 +RPRS_SEARCH_BOUND_MIN = 0.0 +RPRS_SEARCH_BOUND_MAX = 0.30 FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER = 0.5 BAD_PIXEL_DETECTION_FRACTION = 0.30 BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 @@ -305,6 +312,50 @@ def clone_lightcurve_bounds(bounds): } +def sanitize_rprs_search_bounds(bounds): + sanitized = clone_lightcurve_bounds(bounds) + if 'rprs' not in sanitized: + return sanitized + + try: + lower_bound, upper_bound = [ + float(value) for value in np.asarray(sanitized['rprs'], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + sanitized['rprs'] = [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + return sanitized + + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound): + sanitized['rprs'] = [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + return sanitized + + lower_bound = float(np.clip(lower_bound, RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX)) + upper_bound = float(np.clip(upper_bound, RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX)) + if lower_bound >= upper_bound: + sanitized['rprs'] = [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + else: + sanitized['rprs'] = [lower_bound, upper_bound] + return sanitized + + +def clamp_rprs_prior_to_bounds(prior, bounds): + clamped = dict(prior) + if 'rprs' not in clamped or 'rprs' not in bounds: + return clamped + + try: + rprs_value = float(clamped['rprs']) + lower_bound, upper_bound = [ + float(value) for value in np.asarray(bounds['rprs'], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return clamped + + if np.isfinite(rprs_value) and np.isfinite(lower_bound) and np.isfinite(upper_bound) and lower_bound < upper_bound: + clamped['rprs'] = float(np.clip(rprs_value, lower_bound, upper_bound)) + return clamped + + def run_nested_lightcurve_fit_with_rprs_posterior_retry( times, flux_values, @@ -314,9 +365,11 @@ def run_nested_lightcurve_fit_with_rprs_posterior_retry( bounds, jd_times=None, use_impactparameter_rather_than_inclination_to_fit=True, - max_rprs_retries=1, + max_rprs_retries=RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, ): def build_fit(local_prior, local_bounds): + local_bounds = sanitize_rprs_search_bounds(local_bounds) + local_prior = clamp_rprs_prior_to_bounds(local_prior, local_bounds) return lc_fitter( times, flux_values, @@ -330,8 +383,8 @@ def build_fit(local_prior, local_bounds): use_impactparameter_rather_than_inclination_to_fit, ) - current_prior = dict(prior) - current_bounds = clone_lightcurve_bounds(bounds) + current_bounds = sanitize_rprs_search_bounds(bounds) + current_prior = clamp_rprs_prior_to_bounds(prior, current_bounds) retry_history = [] retry_note = None fit = build_fit(current_prior, current_bounds) @@ -358,14 +411,34 @@ def build_fit(local_prior, local_bounds): break previous_bounds = current_bounds.get('rprs') - if previous_bounds is not None and np.allclose( - np.asarray(previous_bounds, dtype=float), - np.asarray([new_lower, new_upper], dtype=float), - atol=1e-12, - rtol=0.0, - ): - retry_note = "Skipped; the automatic Rp/R* retry did not expand the sampled range." - break + clamped_bounds = sanitize_rprs_search_bounds({'rprs': [new_lower, new_upper]}).get('rprs', [new_lower, new_upper]) + new_lower, new_upper = [float(value) for value in clamped_bounds] + if previous_bounds is not None: + previous_lower, previous_upper = [float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2]] + clipped_edge = diagnostics.get('edge') + expands_sampled_range = False + if clipped_edge == 'upper': + expands_sampled_range = new_upper > previous_upper + 1e-12 + elif clipped_edge == 'lower': + expands_sampled_range = new_lower < previous_lower - 1e-12 + else: + expands_sampled_range = ( + new_lower < previous_lower - 1e-12 or + new_upper > previous_upper + 1e-12 + ) + + if not expands_sampled_range: + if ( + previous_lower <= RPRS_SEARCH_BOUND_MIN + 1e-12 and + previous_upper >= RPRS_SEARCH_BOUND_MAX - 1e-12 + ): + retry_note = ( + "Skipped; the automatic Rp/R* retry reached the maximum exoplanet " + f"search range [{RPRS_SEARCH_BOUND_MIN:.6f}, {RPRS_SEARCH_BOUND_MAX:.6f}]." + ) + else: + retry_note = "Skipped; the automatic Rp/R* retry did not expand the sampled range." + break retry_history.append({ 'attempt': len(retry_history) + 1, @@ -385,6 +458,7 @@ def build_fit(local_prior, local_bounds): updated_bounds = clone_lightcurve_bounds(current_bounds) updated_bounds['rprs'] = [new_lower, new_upper] + updated_bounds = sanitize_rprs_search_bounds(updated_bounds) updated_prior = dict(current_prior) fit_parameters = getattr(fit, 'parameters', {}) @@ -394,6 +468,7 @@ def build_fit(local_prior, local_bounds): updated_prior[key] = fit_parameters[key] if np.isfinite(diagnostics.get('mode', np.nan)): updated_prior['rprs'] = float(diagnostics['mode']) + updated_prior = clamp_rprs_prior_to_bounds(updated_prior, updated_bounds) current_prior = updated_prior current_bounds = updated_bounds @@ -404,11 +479,13 @@ def build_fit(local_prior, local_bounds): final_diagnostics = final_diagnostics_getter('rprs') if callable(final_diagnostics_getter) else None note = f"Applied {len(retry_history)} automatic Rp/R* posterior range refit(s)." if final_diagnostics and final_diagnostics.get('clipped'): + retry_label = "retry" if len(retry_history) == 1 else "retries" note = ( - f"{note} The posterior still hugs the {final_diagnostics.get('edge')} bound after retry." + f"{note} The posterior still hugs the {final_diagnostics.get('edge')} bound after " + f"{len(retry_history)} {retry_label}." ) log_info( - "Warning: Rp/R* posterior still appears truncated after the automatic retry; " + "Warning: Rp/R* posterior still appears truncated after the automatic retries; " "please inspect the triangle plot carefully.", warn=True, ) @@ -459,7 +536,16 @@ def log_mid_transit_range_warning_once(array_times, tmid_prior): def relative_flux_filter_mask(relative_flux, max_relative_flux=RELATIVE_FLUX_MAX): relative_flux = np.asarray(relative_flux, dtype=float) - return np.isfinite(relative_flux) & np.less_equal(relative_flux, max_relative_flux) + return ( + np.isfinite(relative_flux) + & np.greater(relative_flux, 0) + & np.less_equal(relative_flux, max_relative_flux) + ) + + +def valid_flux_ratio_mask(relative_flux): + relative_flux = np.asarray(relative_flux, dtype=float) + return np.isfinite(relative_flux) & np.greater(relative_flux, 0) def valid_comparison_frame_mask(flux_values): @@ -2596,8 +2682,13 @@ def nextastro_variability_test(comp_ra_dec): api_url = 'https://photometry.nextastro.org/variability_test' payload = [{'ra': float(ra), 'dec': float(dec)} for ra, dec in comp_ra_dec] + request_body, headers, content_encoding, raw_size, compressed_size = build_compressed_json_request(payload) log_info(f"NextAstro variability request JSON: {json.dumps(payload)}") - result = requests.post(api_url, json=payload, timeout=30) + log_info( + "NextAstro variability request compression: " + f"{content_encoding} ({compressed_size} bytes sent; {raw_size} bytes raw)" + ) + result = requests.post(api_url, data=request_body, headers=headers, timeout=30) if result.status_code != 200: raise RuntimeError(f"NextAstro variability server returned HTTP {result.status_code}.") @@ -4321,13 +4412,13 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, has_reference_flux = not np.allclose(cflux_sorted, 1.0) if has_reference_flux: - relative_flux_mask = relative_flux_filter_mask(flux_ratio_sorted) - times_sorted = times_sorted[relative_flux_mask] - tflux_sorted = tflux_sorted[relative_flux_mask] - cflux_sorted = cflux_sorted[relative_flux_mask] - flux_ratio_sorted = flux_ratio_sorted[relative_flux_mask] - jd_times_sorted = jd_times[si][relative_flux_mask] - airmass_sorted = airmass[si][relative_flux_mask] + flux_ratio_mask = valid_flux_ratio_mask(flux_ratio_sorted) + times_sorted = times_sorted[flux_ratio_mask] + tflux_sorted = tflux_sorted[flux_ratio_mask] + cflux_sorted = cflux_sorted[flux_ratio_mask] + flux_ratio_sorted = flux_ratio_sorted[flux_ratio_mask] + jd_times_sorted = jd_times[si][flux_ratio_mask] + airmass_sorted = airmass[si][flux_ratio_mask] else: jd_times_sorted = jd_times[si] airmass_sorted = airmass[si] @@ -4488,7 +4579,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, return myfit, f1, f2 -def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass): +def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass, enforce_relative_flux_max=True): times = np.asarray(times, dtype=float) tflux = np.asarray(tflux, dtype=float) cflux = np.asarray(cflux, dtype=float) @@ -4528,10 +4619,8 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass): if has_reference_flux: finite_ratio_mask = np.isfinite(flux_ratio_sorted) + nonpositive_ratio_mask = finite_ratio_mask & np.less_equal(flux_ratio_sorted, 0) high_ratio_mask = finite_ratio_mask & np.greater(flux_ratio_sorted, RELATIVE_FLUX_MAX) - nonfinite_ratio_count = int(np.count_nonzero(~finite_ratio_mask)) - high_ratio_count = int(np.count_nonzero(high_ratio_mask)) - rejected_ratio_count = nonfinite_ratio_count + high_ratio_count finite_ratio_values = flux_ratio_sorted[finite_ratio_mask] ratio_range_text = "finite ratio range=n/a" if finite_ratio_values.size: @@ -4539,7 +4628,24 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass): f"finite ratio range={np.nanmin(finite_ratio_values):.4f} to " f"{np.nanmax(finite_ratio_values):.4f}" ) - relative_flux_mask = relative_flux_filter_mask(flux_ratio_sorted) + nonfinite_ratio_count = int(np.count_nonzero(~finite_ratio_mask)) + nonpositive_ratio_count = int(np.count_nonzero(nonpositive_ratio_mask)) + high_ratio_count = int(np.count_nonzero(high_ratio_mask)) + rejected_ratio_count = nonfinite_ratio_count + nonpositive_ratio_count + relative_flux_mask = valid_flux_ratio_mask(flux_ratio_sorted) + rejection_detail = ( + f"(non-finite={nonfinite_ratio_count}, non-positive={nonpositive_ratio_count}, " + f"{ratio_range_text})." + ) + rejection_context = "invalid target/reference ratio screening" + if enforce_relative_flux_max: + relative_flux_mask &= ~high_ratio_mask + rejected_ratio_count += high_ratio_count + rejection_detail = ( + f"(non-finite={nonfinite_ratio_count}, non-positive={nonpositive_ratio_count}, " + f">{RELATIVE_FLUX_MAX:g}x={high_ratio_count}, {ratio_range_text})." + ) + rejection_context = "target/reference ratio screening" diagnostics['relative_flux_point_count'] = int(np.count_nonzero(relative_flux_mask)) times_sorted = times_sorted[relative_flux_mask] tflux_sorted = tflux_sorted[relative_flux_mask] @@ -4553,9 +4659,7 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass): "relative-flux filtering left " f"{diagnostics['relative_flux_point_count']} usable point(s); " f"rejected {rejected_ratio_count}/{diagnostics['input_point_count']} frame(s) " - "during target/reference ratio screening " - f"(non-finite={nonfinite_ratio_count}, >{RELATIVE_FLUX_MAX:g}x={high_ratio_count}, " - f"{ratio_range_text})." + f"during {rejection_context} {rejection_detail}" ), }) return diagnostics @@ -4631,13 +4735,16 @@ def ensure_lightcurve_fit_failure_reason(diagnostics, fit_result, failed_stage, return diagnostics -def cheap_lightcurve_prescore(tFlux, cFlux, airmass): +def cheap_lightcurve_prescore(tFlux, cFlux, airmass, enforce_relative_flux_max=True): with np.errstate(divide='ignore', invalid='ignore'): flux_ratio = np.divide(tFlux, cFlux) - finite_mask = np.isfinite(flux_ratio) & np.isfinite(airmass) & (flux_ratio > 0) + finite_mask = valid_flux_ratio_mask(flux_ratio) & np.isfinite(airmass) if not np.allclose(cFlux, 1.0): - finite_mask &= relative_flux_filter_mask(flux_ratio) + if enforce_relative_flux_max: + finite_mask &= relative_flux_filter_mask(flux_ratio) + else: + finite_mask &= valid_flux_ratio_mask(flux_ratio) if np.count_nonzero(finite_mask) < 5: return np.inf @@ -4668,7 +4775,13 @@ def evaluate_lightcurve_candidate(task): disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit, ) = task - fit_diagnostics = diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass) + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times, + tflux, + cflux, + airmass, + enforce_relative_flux_max=False, + ) myfit, tflux_fit, cflux_fit = fit_lightcurve( times, tflux, @@ -4741,7 +4854,12 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, 'coverage_reference_count': psf_comp_coverage[ckey]['coverage_reference_count'], 'coverage_min_required_count': psf_comp_coverage[ckey]['coverage_min_required_count'], 'coverage_rejected': psf_comp_coverage[ckey]['coverage_rejected'], - 'prescore': cheap_lightcurve_prescore(target_flux, comp_flux, airmass), + 'prescore': cheap_lightcurve_prescore( + target_flux, + comp_flux, + airmass, + enforce_relative_flux_max=False, + ), }) if use_aperture_photometry and aper_data is not None and apers is not None and annuli is not None: @@ -4776,6 +4894,7 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, target_flux, np.ones(target_flux.shape[0]), airmass, + enforce_relative_flux_max=False, ), }) @@ -4804,6 +4923,7 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, target_flux[aper_mask], comp_series[aper_mask], airmass[aper_mask], + enforce_relative_flux_max=False, ), }) @@ -5321,6 +5441,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p target_flux[fit_mask], comp_flux_series[fit_mask], airmass[fit_mask], + enforce_relative_flux_max=False, ) if not coverage_rejected and coverage_count > 1 and fit_diagnostics['failure_reason'] is None: fit_result, target_fit_flux, comp_fit_flux = fit_lightcurve( @@ -5904,6 +6025,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p target_flux, comp_flux, airmass, + enforce_relative_flux_max=False, ) fit_result, tflux_fit, cflux_fit = fit_lightcurve( times, diff --git a/requirements.txt b/requirements.txt index fb0af820..ade59384 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ scikit-image~=0.24.0 statsmodels~=0.14.4 tenacity~=9.0 ultranest~=3.6.5;platform_system!='Windows' +zstandard~=0.23.0 diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 908a1916..b33e3709 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -541,10 +541,83 @@ def test_rprs_posterior_recenter_diagnostics_detect_upper_bound_clipping(monkeyp assert diagnostics["edge"] == "upper" assert diagnostics["mode"] > 0.13 assert diagnostics["std"] > 0 + assert diagnostics["upper_edge_peak_fraction"] >= 0.20 assert diagnostics["bounds"][0] >= 0.0 assert diagnostics["bounds"][1] > 0.15 +def test_rprs_posterior_recenter_diagnostics_ignores_upper_edge_below_twenty_percent(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + fit.sampled_keys = ["rprs", "tmid"] + fit.sample_bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + + rprs_samples = np.concatenate([ + np.linspace(0.106, 0.119, 40), + np.linspace(0.120, 0.134, 15), + np.linspace(0.145, 0.149, 5), + ]) + tmid_samples = np.linspace(-2e-4, 2e-4, rprs_samples.size) + points = np.column_stack([rprs_samples, tmid_samples]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-6.0, -3.0, rprs_samples.size), + }, + "samples": points.copy(), + } + + diagnostics = fit.get_parameter_posterior_recenter_diagnostics("rprs") + + assert diagnostics["clipped"] is False + assert diagnostics["edge"] is None + assert diagnostics["upper_edge_peak_fraction"] < 0.20 + assert diagnostics["bounds"] == pytest.approx([0.0, 0.15]) + assert "not treated as truncated" in diagnostics["reason"] + + +def test_rprs_posterior_recenter_diagnostics_ignores_lower_edge_below_twenty_percent(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + fit.sampled_keys = ["rprs", "tmid"] + fit.sample_bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + + rprs_samples = np.concatenate([ + np.linspace(0.001, 0.005, 5), + np.linspace(0.016, 0.029, 15), + np.linspace(0.031, 0.044, 40), + ]) + tmid_samples = np.linspace(-2e-4, 2e-4, rprs_samples.size) + points = np.column_stack([rprs_samples, tmid_samples]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-6.0, -3.0, rprs_samples.size), + }, + "samples": points.copy(), + } + + diagnostics = fit.get_parameter_posterior_recenter_diagnostics("rprs") + + assert diagnostics["clipped"] is False + assert diagnostics["edge"] is None + assert diagnostics["lower_edge_peak_fraction"] < 0.20 + assert diagnostics["bounds"] == pytest.approx([0.0, 0.15]) + assert "not treated as truncated" in diagnostics["reason"] + + def test_plot_triangle_uses_mirrored_distance_from_fitted_impact_parameter_axis(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index d74de832..c1af9160 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -716,16 +716,26 @@ def test_comparison_star_stability_summary_rejects_low_coverage_candidates(): assert np.isinf(summary["comp_summaries"][2]["aggregate_score"]) -def test_cheap_lightcurve_prescore_ignores_relative_flux_above_two(): +def test_cheap_lightcurve_prescore_ignores_large_ratios_when_requested(): tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) cflux = np.full(tflux.shape[0], 2.0) airmass = np.linspace(1.0, 1.5, tflux.shape[0]) - score = cheap_lightcurve_prescore(tflux, cflux, airmass) + score = cheap_lightcurve_prescore(tflux, cflux, airmass, enforce_relative_flux_max=True) assert np.isclose(score, 0.0) +def test_cheap_lightcurve_prescore_allows_large_raw_target_reference_ratios(): + tflux = np.full(6, 30.0) + cflux = np.full(6, 10.0) + airmass = np.linspace(1.0, 1.5, 6) + + score = cheap_lightcurve_prescore(tflux, cflux, airmass, enforce_relative_flux_max=False) + + assert np.isfinite(score) + + def test_cheap_lightcurve_prescore_keeps_target_only_mode_unfiltered(): tflux = np.array([10.0, 11.0, 12.0, 13.0, 14.0, 15.0]) cflux = np.ones(tflux.shape[0]) @@ -1022,7 +1032,7 @@ def fake_lc_fitter( assert fit.prefit_refinement_tmid_bounds == pytest.approx([-1.0, 1.0]) -def test_fit_lightcurve_removes_relative_flux_above_two_before_fit(monkeypatch): +def test_fit_lightcurve_keeps_large_raw_target_reference_ratios(monkeypatch): captured = {} def fake_lc_fitter( @@ -1069,10 +1079,9 @@ def fake_lc_fitter( assert myfit is not None assert captured["mode"] == "lm" - assert len(captured["fluxes"]) == 5 - assert np.all(captured["fluxes"] <= 2.0) - assert np.allclose(captured["fluxes"], 1.0) - assert np.allclose(fit_tflux, 2.0) + assert len(captured["fluxes"]) == 6 + assert np.allclose(captured["fluxes"], np.array([1.0, 1.0, 1.0, 3.0, 1.0, 1.0])) + assert np.allclose(fit_tflux, tflux) assert np.allclose(fit_cflux, 2.0) @@ -1484,6 +1493,7 @@ def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times, **kw assert [attempt["comp_index"] for attempt in result["attempts"]] == [0, 1] assert result["selected_result"]["comp_index"] == 1 assert "relative-flux filtering left 0 usable point(s)" in result["attempts"][0]["fit_diagnostics"]["failure_reason"] + assert "non-finite=6" in result["attempts"][0]["fit_diagnostics"]["failure_reason"] assert result["attempts"][1]["fit"] is not None @@ -1535,7 +1545,11 @@ def test_run_target_driven_photometry_search_returns_failed_candidate_summaries( assert result["best_candidate"] is None assert len(result["candidate_summaries"]) == 2 assert all( - "relative-flux filtering left 0 usable point(s)" in summary["failure_reason"] + summary["failure_reason"] is not None + for summary in result["candidate_summaries"] + ) + assert all( + ">2x=" not in summary["failure_reason"] for summary in result["candidate_summaries"] ) assert result["candidate_summaries"][0]["method_label"] == "Aperture photometry (aper=7.05px, annulus=22.73px)" diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py new file mode 100644 index 00000000..1d4070ea --- /dev/null +++ b/tests/test_exotic_rprs_retry.py @@ -0,0 +1,322 @@ +import importlib.util +import sys +import types + +import numpy as np +import pytest + + +def _module_available(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + +def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: + if not _module_available(name): + sys.modules.setdefault(name, module) + + +fake_barycorrpy = types.ModuleType("barycorrpy") +fake_utc_tdb = types.ModuleType("barycorrpy.utc_tdb") +fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None +fake_astroalign = types.ModuleType("astroalign") +fake_astroalign.PIXEL_TOL = 1 +fake_astroquery = types.ModuleType("astroquery") +fake_astroquery_simbad = types.ModuleType("astroquery.simbad") +fake_astroquery_simbad.Simbad = type("Simbad", (), {}) +fake_astroquery_gaia = types.ModuleType("astroquery.gaia") +fake_astroquery_gaia.Gaia = type("Gaia", (), {}) +fake_imreg_dft = types.ModuleType("imreg_dft") +fake_colour_demosaicing = types.ModuleType("colour_demosaicing") +fake_colour_demosaicing.demosaicing_CFA_Bayer_bilinear = lambda *args, **kwargs: None +fake_photutils = types.ModuleType("photutils") +fake_photutils_aperture = types.ModuleType("photutils.aperture") +fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_detection = types.ModuleType("photutils.detection") +fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) +fake_ldtk = types.ModuleType("ldtk") +fake_ldtk.LDPSet = type("LDPSet", (), {}) +fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) +fake_ldtk_ldmodel = types.ModuleType("ldtk.ldmodel") +fake_ldtk_ldmodel.LinearModel = type("LinearModel", (), {}) +fake_ldtk_ldmodel.QuadraticModel = type("QuadraticModel", (), {}) +fake_ldtk_ldmodel.NonlinearModel = type("NonlinearModel", (), {}) +fake_lmfit = types.ModuleType("lmfit") +fake_pylightcurve = types.ModuleType("pylightcurve") +fake_pylightcurve_models = types.ModuleType("pylightcurve.models") +fake_pylightcurve_exoplanet = types.ModuleType("pylightcurve.models.exoplanet_lc") +fake_pylightcurve_exoplanet.transit = lambda *args, **kwargs: None +fake_pyvo = types.ModuleType("pyvo") +fake_ultranest = types.ModuleType("ultranest") +fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) +fake_elca = types.ModuleType("exotic.api.elca") +fake_elca.lc_fitter = lambda *args, **kwargs: None +fake_elca.binner = lambda *args, **kwargs: None +fake_elca.transit = lambda *args, **kwargs: None +fake_elca.get_phase = lambda *args, **kwargs: None +fake_ld = types.ModuleType("exotic.api.ld") +fake_ld.LimbDarkening = type("LimbDarkening", (), {}) +fake_ld.ld_re_punct_p = lambda *args, **kwargs: None + +_set_stub_if_missing("astroalign", fake_astroalign) +_set_stub_if_missing("astroquery", fake_astroquery) +_set_stub_if_missing("astroquery.simbad", fake_astroquery_simbad) +_set_stub_if_missing("astroquery.gaia", fake_astroquery_gaia) +_set_stub_if_missing("imreg_dft", fake_imreg_dft) +_set_stub_if_missing("colour_demosaicing", fake_colour_demosaicing) +_set_stub_if_missing("photutils", fake_photutils) +_set_stub_if_missing("photutils.aperture", fake_photutils_aperture) +_set_stub_if_missing("photutils.detection", fake_photutils_detection) +_set_stub_if_missing("ldtk", fake_ldtk) +_set_stub_if_missing("ldtk.ldmodel", fake_ldtk_ldmodel) +_set_stub_if_missing("lmfit", fake_lmfit) +_set_stub_if_missing("pylightcurve", fake_pylightcurve) +_set_stub_if_missing("pylightcurve.models", fake_pylightcurve_models) +_set_stub_if_missing("pylightcurve.models.exoplanet_lc", fake_pylightcurve_exoplanet) +_set_stub_if_missing("pyvo", fake_pyvo) +_set_stub_if_missing("ultranest", fake_ultranest) +_set_stub_if_missing("barycorrpy", fake_barycorrpy) +_set_stub_if_missing("barycorrpy.utc_tdb", fake_utc_tdb) +sys.modules.setdefault("exotic.api.elca", fake_elca) +sys.modules.setdefault("exotic.api.ld", fake_ld) + +from exotic.exotic import ( # noqa: E402 + RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, + RPRS_SEARCH_BOUND_MAX, + RPRS_SEARCH_BOUND_MIN, + run_nested_lightcurve_fit_with_rprs_posterior_retry, +) + + +def test_rprs_posterior_retry_walks_bounds_until_retry_cap(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + {"clipped": True, "edge": "upper", "mode": 0.158, "std": 0.006, "bounds": [0.128, 0.188]}, + {"clipped": True, "edge": "upper", "mode": 0.182, "std": 0.005, "bounds": [0.157, 0.207]}, + {"clipped": True, "edge": "upper", "mode": 0.194, "std": 0.004, "bounds": [0.174, 0.214]}, + {"clipped": True, "edge": "upper", "mode": 0.201, "std": 0.003, "bounds": [0.186, 0.216]}, + {"clipped": True, "edge": "upper", "mode": 0.206, "std": 0.003, "bounds": [0.191, 0.221]}, + {"clipped": True, "edge": "upper", "mode": 0.210, "std": 0.003, "bounds": [0.195, 0.225]}, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key == "rprs" + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.125], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert RPRS_POSTERIOR_MAX_RETRIES_DEFAULT == 5 + assert len(captured["calls"]) == 6 + np.testing.assert_allclose( + np.asarray([call["bounds"]["rprs"] for call in captured["calls"]], dtype=float), + np.asarray([ + [0.0, 0.125], + [0.128, 0.188], + [0.157, 0.207], + [0.174, 0.214], + [0.186, 0.216], + [0.191, 0.221], + ], dtype=float), + ) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 5 + assert fit.rprs_posterior_refit_edge == "upper" + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.191, 0.221]) + assert "after 5 retries" in fit.rprs_posterior_refit_note + + +def test_rprs_posterior_retry_caps_retry_bounds_at_exoplanet_limit(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + {"clipped": True, "edge": "upper", "mode": 0.275, "std": 0.020, "bounds": [0.175, 0.375]}, + {"clipped": False, "edge": None, "mode": 0.278, "std": 0.012, "bounds": [0.175, 0.300]}, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key == "rprs" + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.25], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.25]) + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.175, RPRS_SEARCH_BOUND_MAX]) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 1 + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.175, RPRS_SEARCH_BOUND_MAX]) + + +def test_rprs_posterior_retry_stops_at_maximum_exoplanet_range(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics = {"clipped": True, "edge": "upper", "mode": 0.275, "std": 0.020, "bounds": [0.175, 0.375]} + + def make_fit(): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key == "rprs" + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit() + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.35, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [RPRS_SEARCH_BOUND_MIN, 0.35], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert len(captured["calls"]) == 1 + assert captured["calls"][0]["prior"]["rprs"] == pytest.approx(RPRS_SEARCH_BOUND_MAX) + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX]) + assert fit.rprs_posterior_refit_applied is False + assert fit.rprs_posterior_refit_count == 0 + assert "maximum exoplanet search range" in fit.rprs_posterior_refit_note diff --git a/tests/test_inputs.py b/tests/test_inputs.py index b24fd81f..7b08ac07 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -869,6 +869,23 @@ def test_parse_aavso_prereduced_overrides_uses_known_filter_lookup_when_filter_x assert overrides["wl_max"] == "1000.0" +def test_parse_aavso_prereduced_overrides_uses_astrodon_exo_alias_lookup(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=Astrodon-Exo\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "Astrodon-Exo" + assert overrides["filter_desc"] == "Astrodon ExoPlanet-BB" + assert overrides["wl_min"] == "500.0" + assert overrides["wl_max"] == "1000.0" + + def test_lookup_aavso_filter_metadata_uses_c_alias_for_cv_filter() -> None: filter_metadata = inputs_module.lookup_aavso_filter_metadata("C") diff --git a/tests/test_ld.py b/tests/test_ld.py index c0cc8eb0..1e7bec07 100644 --- a/tests/test_ld.py +++ b/tests/test_ld.py @@ -252,6 +252,7 @@ def test_additional_standard_filter_aliases_in_filter_column() -> None: ("w", "MObs CV", "CV", "350.0", "850.0"), ("pl", "MObs CV", "CV", "350.0", "850.0"), ("exo", "Astrodon ExoPlanet-BB", "CBB", "500.0", "1000.0"), + ("Astrodon-Exo", "Astrodon ExoPlanet-BB", "CBB", "500.0", "1000.0"), ] for alias, expected_filter, expected_name, expected_min, expected_max in alias_cases: diff --git a/tests/test_nextastro_astrometry.py b/tests/test_nextastro_astrometry.py index 7f6303ae..29165f60 100644 --- a/tests/test_nextastro_astrometry.py +++ b/tests/test_nextastro_astrometry.py @@ -1,7 +1,10 @@ +import gzip +import json from pathlib import Path import numpy as np from astropy.io.fits import getheader, writeto +import pytest from exotic.api.plate_solution import NextAstroPlateSolution, PlateSolution @@ -19,6 +22,16 @@ def json(self): return self._payload +def _decode_request_body(body, headers): + encoding = headers["Content-Encoding"] + if encoding == "gzip": + return json.loads(gzip.decompress(body).decode("utf-8")) + if encoding == "zstd": + zstandard = pytest.importorskip("zstandard") + return json.loads(zstandard.ZstdDecompressor().decompress(body).decode("utf-8")) + raise AssertionError(f"Unexpected content encoding: {encoding}") + + def _create_test_fits(tmp_path: Path) -> Path: image = np.zeros((100, 120), dtype=float) image[30, 25] = 10000.0 @@ -45,13 +58,16 @@ def test_plate_solution_writes_wcs_file(tmp_path, monkeypatch): fits_path = _create_test_fits(tmp_path) (tmp_path / "temp").mkdir() - def fake_post(url, json, timeout): + def fake_post(url, data, headers, timeout): + payload = _decode_request_body(data, headers) assert url.endswith('/solve') - assert json['image'] == {'width': 120, 'height': 100} - assert json['hints']['ra_deg'] == 210.8023 - assert json['hints']['dec_deg'] == 54.3489 - assert json['hints']['scale_arcsec_per_pix'] == 1.23 - assert json['hints']['scale_tolerance_frac'] == 0.25 + assert headers["Content-Type"] == "application/json" + assert headers["Content-Encoding"] in {"gzip", "zstd"} + assert payload['image'] == {'width': 120, 'height': 100} + assert payload['hints']['ra_deg'] == 210.8023 + assert payload['hints']['dec_deg'] == 54.3489 + assert payload['hints']['scale_arcsec_per_pix'] == 1.23 + assert payload['hints']['scale_tolerance_frac'] == 0.25 return DummyResponse({'status': 'queued', 'request_id': 'abc123'}) def fake_get(url, timeout): @@ -99,7 +115,7 @@ def test_plate_solution_logs_json_via_message_logger_when_fail_warnings_suppress monkeypatch.setattr( 'exotic.api.plate_solution.requests.post', - lambda url, json, timeout: DummyResponse({'status': 'queued', 'request_id': 'abc123'}) + lambda url, data, headers, timeout: DummyResponse({'status': 'queued', 'request_id': 'abc123'}) ) monkeypatch.setattr( 'exotic.api.plate_solution.requests.get', @@ -131,6 +147,7 @@ def test_plate_solution_logs_json_via_message_logger_when_fail_warnings_suppress assert wcs_file == tmp_path / 'temp' / 'wcs.fits' assert any('NextAstro astrometry request JSON:' in message for message in logged) + assert any('NextAstro astrometry request compression:' in message for message in logged) assert any('NextAstro astrometry submission response JSON:' in message for message in logged) assert any('NextAstro astrometry status response JSON (solved):' in message for message in logged) @@ -196,7 +213,7 @@ def test_submit_solve_request_handles_non_json_response(tmp_path, monkeypatch, c monkeypatch.setattr( 'exotic.api.plate_solution.requests.post', - lambda url, json, timeout: DummyResponse( + lambda url, data, headers, timeout: DummyResponse( status_code=502, text='bad gateway', json_error=ValueError('not json') diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index ef504346..5f11db10 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -1,8 +1,11 @@ import importlib.util import sys import types +import gzip +import json import numpy as np +import pytest fake_barycorrpy = types.ModuleType('barycorrpy') fake_utc_tdb = types.ModuleType('barycorrpy.utc_tdb') @@ -99,13 +102,24 @@ def raise_for_status(self): raise RuntimeError(f"HTTP {self.status_code}") +def _decode_request_body(body, headers): + encoding = headers["Content-Encoding"] + if encoding == "gzip": + return json.loads(gzip.decompress(body).decode("utf-8")) + if encoding == "zstd": + zstandard = pytest.importorskip("zstandard") + return json.loads(zstandard.ZstdDecompressor().decompress(body).decode("utf-8")) + raise AssertionError(f"Unexpected content encoding: {encoding}") + + def test_nextastro_variability_logs_json_request_and_response(monkeypatch): captured = {} logged = [] - def fake_post(url, json, timeout): + def fake_post(url, data, headers, timeout): captured['url'] = url - captured['json'] = json + captured['json'] = _decode_request_body(data, headers) + captured['headers'] = headers captured['timeout'] = timeout return DummyResponse([ {'is_in_vsx': 0}, @@ -119,9 +133,12 @@ def fake_post(url, json, timeout): assert captured['url'].endswith('/variability_test') assert captured['timeout'] == 30 + assert captured['headers']['Content-Type'] == 'application/json' + assert captured['headers']['Content-Encoding'] in {'gzip', 'zstd'} assert captured['json'] == [{'ra': 10.1, 'dec': -11.2}, {'ra': 22.3, 'dec': -33.4}] assert variability_flags == [False, True] assert any('NextAstro variability request JSON:' in message for message in logged) + assert any('NextAstro variability request compression:' in message for message in logged) assert any('NextAstro variability response JSON:' in message for message in logged) From 7c5867a1ed7e44c5519772a0d339b1759ebaf46f Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sat, 25 Apr 2026 20:17:27 +1000 Subject: [PATCH 023/116] Improve pointing prechecks and enable EEBLS by default --- .gitignore | 1 + README.md | 1 + exotic/api/colab.py | 3 +- exotic/api/gael_ld.py | 6 +- exotic/api/http_compression.py | 16 +- exotic/api/ld.py | 9 +- exotic/exotic.py | 2476 ++++++++++++++++++++++++--- exotic/exotic_gui.py | 5 + exotic/inputs.py | 12 +- exotic/utils.py | 2 +- inits.json | 4 + tests/test_centroid_wcs.py | 211 ++- tests/test_exotic_proper_motion.py | 925 +++++++++- tests/test_inputs.py | 60 + tests/test_ld.py | 17 + tests/test_nextastro_variability.py | 82 + 16 files changed, 3591 insertions(+), 239 deletions(-) diff --git a/.gitignore b/.gitignore index ab7b73c9..e0757be1 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,4 @@ eaConf.json exotic.log.* pl_names.json /.project +/.deps313 diff --git a/README.md b/README.md index 5d97cd0f..aacd68d4 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "fit_lightcurve_to_every_comparison_candidate": "n", "detrend_on_outoftransit_baseline": true, "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y", diff --git a/exotic/api/colab.py b/exotic/api/colab.py index db625fb3..a08fd9cf 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -190,7 +190,7 @@ def process_lat_long(val, key): else: v = deg + (((60*min) + sec)/3600) return(add_sign(v)) - m = re.search("^\'?([+-]?\d+\.\d+)", val) + m = re.search(r"^'?([+-]?\d+\.\d+)", val) if m: v = float(m.group(1)) return(add_sign(v)) @@ -380,6 +380,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "bad_wcs_threshold_percent": 3.0, "detrend_on_outoftransit_baseline": true, + "use_eebls_to_initialize_tmid_and_bounds": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": false, "require_comp_star": "y" diff --git a/exotic/api/gael_ld.py b/exotic/api/gael_ld.py index 1b42aa10..fd8d396f 100644 --- a/exotic/api/gael_ld.py +++ b/exotic/api/gael_ld.py @@ -129,7 +129,11 @@ def createldgrid(minmu, maxmu, orbp, out['LD'] = allcl.T out['ERR'] = allel.T for i in range(0, len(allcl.T)): - log.warning(f">-- LD{int(i)}: {float(allcl.T[i])} +/- {float(allel.T[i])}") + ld_value = np.ravel(np.asarray(allcl.T[i], dtype=float)) + err_value = np.ravel(np.asarray(allel.T[i], dtype=float)) + if ld_value.size == 0 or err_value.size == 0: + continue + log.warning(f">-- LD{int(i)}: {float(ld_value[0])} +/- {float(err_value[0])}") pass return out diff --git a/exotic/api/http_compression.py b/exotic/api/http_compression.py index 6ae9fd43..3ee06009 100644 --- a/exotic/api/http_compression.py +++ b/exotic/api/http_compression.py @@ -11,10 +11,22 @@ _ZSTD_LEVEL = 3 -def build_compressed_json_request(payload): +def build_compressed_json_request(payload, content_encoding=None): raw_body = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - if zstandard is not None: + normalized_encoding = None if content_encoding is None else str(content_encoding).strip().lower() + if normalized_encoding not in (None, "", "gzip", "zstd"): + raise ValueError(f"Unsupported content encoding: {content_encoding}") + + if normalized_encoding == "zstd": + if zstandard is None: + raise RuntimeError("zstandard compression requested, but the zstandard package is unavailable.") + compressed_body = zstandard.ZstdCompressor(level=_ZSTD_LEVEL).compress(raw_body) + encoding = "zstd" + elif normalized_encoding == "gzip": + compressed_body = gzip.compress(raw_body, compresslevel=_GZIP_LEVEL) + encoding = "gzip" + elif zstandard is not None: compressed_body = zstandard.ZstdCompressor(level=_ZSTD_LEVEL).compress(raw_body) encoding = "zstd" else: diff --git a/exotic/api/ld.py b/exotic/api/ld.py index 9696ed59..ccaa6c39 100644 --- a/exotic/api/ld.py +++ b/exotic/api/ld.py @@ -224,7 +224,14 @@ def check_fwhm(filter_: dict = None) -> bool: return False for k in ('wl_min', 'wl_max'): # clean inputs filter_[k] = filter_.get(k) - filter_[k] = str(filter_[k]).strip().replace(' ', '').rstrip('.') if filter_[k] else filter_[k] + if filter_[k] is None: + continue + filter_[k] = str(filter_[k]).strip().replace(' ', '').rstrip('.') + if not filter_[k]: + filter_[k] = None + if filter_['wl_min'] is None or filter_['wl_max'] is None: + return False + for k in ('wl_min', 'wl_max'): if not 200. <= float(filter_[k]) <= 4000.: # also fails if nan raise ValueError(f"FWHM '{k}' is outside of bounds (200., 4000.). ...") else: # add .0 to end of str to aid literal matching diff --git a/exotic/exotic.py b/exotic/exotic.py index f7a81fc3..95503fa7 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -71,6 +71,7 @@ from astropy.coordinates import SkyCoord, EarthLocation, AltAz from astropy.io import fits from astropy.time import Time +from astropy.timeseries import BoxLeastSquares from astropy.visualization import astropy_mpl_style from astropy.wcs import WCS, FITSFixedWarning # UTC to BJD converter import @@ -97,7 +98,7 @@ from skimage.registration import phase_cross_correlation from skimage.transform import SimilarityTransform # error handling for scraper -from tenacity import retry, stop_after_delay +from tenacity import RetryError, retry, retry_if_exception, stop_after_attempt, stop_after_delay, wait_fixed # color, color_demosaicing from colour_demosaicing import demosaicing_CFA_Bayer_bilinear # ########## EXOTIC imports ########## @@ -178,7 +179,24 @@ RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 RPRS_SEARCH_BOUND_MIN = 0.0 RPRS_SEARCH_BOUND_MAX = 0.30 +RPRS_RETRY_MIN_HALF_WIDTH = 0.05 FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER = 0.5 +EEBLS_DURATION_GRID_SIZE = 15 +EEBLS_DURATION_MIN_FRACTION = 0.5 +EEBLS_DURATION_MAX_FRACTION = 1.75 +EEBLS_TMID_HALF_WIDTH_DURATION_MULTIPLIER = 1.5 +EEBLS_MIN_VALID_POINTS = 10 +EPHEMERIS_BRACKETED_TMID_HALF_WIDTH_DURATION_MULTIPLIER = 2.0 +COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS = 15.0 +ROBUST_FLUX_MIN_FRACTION_OF_MEDIAN = 0.02 +ROBUST_FLUX_MIN_POINTS = 20 +WCS_REFERENCE_GEOMETRY_TOLERANCE_PIXELS = 5.0 +WCS_MIN_GEOMETRY_MATCH_FRACTION = 0.5 +TIME_REJECTION_RANGE_DISPLAY_LIMIT = 6 +TIME_REJECTION_GROUP_GAP_CADENCE_MULTIPLIER = 2.5 +NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS = 5 +NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS = 10 +NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504} BAD_PIXEL_DETECTION_FRACTION = 0.30 BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 BAD_PIXEL_PROGRESS_LOG_INTERVAL = 25 @@ -281,6 +299,118 @@ def annotate_final_fit_prefit_refinement( fit.prefit_refinement_tmid_bounds = refined_tmid_bounds +def annotate_nested_tmid_refinement( + fit, + applied, + note=None, + original_tmid_bounds=None, + refined_tmid_bounds=None, +): + if fit is None: + return + + fit.nested_tmid_refinement_applied = bool(applied) + fit.nested_tmid_refinement_note = note + fit.nested_tmid_refinement_original_tmid_bounds = original_tmid_bounds + fit.nested_tmid_refinement_tmid_bounds = refined_tmid_bounds + + +def annotate_lightcurve_filter_diagnostics(fit, diagnostics): + if fit is None: + return + + fit.frame_filter_diagnostics = [dict(diagnostic) for diagnostic in (diagnostics or [])] + + +def annotate_selected_photometry_debug( + fit, + times, + target_flux, + comp_flux, + raw_ratio, + initial_sigma_keep_mask, + phase_clip_keep_mask_on_sigma_filtered=None, +): + if fit is None: + return + + sigma_keep_mask = np.asarray(initial_sigma_keep_mask, dtype=bool) + sigma_kept_count = int(np.count_nonzero(sigma_keep_mask)) + if phase_clip_keep_mask_on_sigma_filtered is None: + phase_keep_mask = np.ones(sigma_kept_count, dtype=bool) + else: + phase_keep_mask = np.asarray(phase_clip_keep_mask_on_sigma_filtered, dtype=bool) + if phase_keep_mask.shape[0] != sigma_kept_count: + phase_keep_mask = np.ones(sigma_kept_count, dtype=bool) + + fit.selected_photometry_debug = { + 'times': np.asarray(times, dtype=float).copy(), + 'target_flux': np.asarray(target_flux, dtype=float).copy(), + 'comp_flux': np.asarray(comp_flux, dtype=float).copy(), + 'raw_ratio': np.asarray(raw_ratio, dtype=float).copy(), + 'initial_sigma_keep_mask': sigma_keep_mask.copy(), + 'phase_clip_keep_mask_on_sigma_filtered': phase_keep_mask.copy(), + } + + +def save_selected_photometry_debug_series(save_dir, planet_name, observation_date, fit): + if fit is None: + return None + + debug = getattr(fit, 'selected_photometry_debug', None) + if not debug: + return None + + times = np.asarray(debug.get('times'), dtype=float) + target_flux = np.asarray(debug.get('target_flux'), dtype=float) + comp_flux = np.asarray(debug.get('comp_flux'), dtype=float) + raw_ratio = np.asarray(debug.get('raw_ratio'), dtype=float) + initial_sigma_keep_mask = np.asarray(debug.get('initial_sigma_keep_mask'), dtype=bool) + phase_clip_keep_mask = np.asarray( + debug.get('phase_clip_keep_mask_on_sigma_filtered', np.ones(np.count_nonzero(initial_sigma_keep_mask))), + dtype=bool, + ) + + if not ( + times.shape == target_flux.shape == comp_flux.shape == raw_ratio.shape == initial_sigma_keep_mask.shape + ): + return None + + phase_keep_full = np.zeros(times.shape[0], dtype=bool) + sigma_kept_indices = np.flatnonzero(initial_sigma_keep_mask) + if sigma_kept_indices.size: + if phase_clip_keep_mask.shape[0] != sigma_kept_indices.size: + phase_clip_keep_mask = np.ones(sigma_kept_indices.size, dtype=bool) + phase_keep_full[sigma_kept_indices] = phase_clip_keep_mask + + output_dir = Path(save_dir) / "temp" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / f"SelectedPhotometryRawRatio_{planet_name}_{observation_date}.csv" + + output_rows = np.column_stack( + [ + times, + target_flux, + comp_flux, + raw_ratio, + initial_sigma_keep_mask.astype(int), + phase_keep_full.astype(int), + ] + ) + np.savetxt( + output_path, + output_rows, + delimiter=",", + header=( + "BJD_TDB,Target Flux,Comp Flux,Raw Ratio," + "Kept After Initial Sigma Clip,Kept After Phase Residual Clip" + ), + comments="", + fmt=["%.8f", "%.8f", "%.8f", "%.8f", "%d", "%d"], + ) + return output_path + + def annotate_rprs_posterior_refit(fit, applied, note=None, history=None): if fit is None: return @@ -356,6 +486,38 @@ def clamp_rprs_prior_to_bounds(prior, bounds): return clamped +def enforce_minimum_rprs_retry_half_width(mode, bounds, min_half_width=RPRS_RETRY_MIN_HALF_WIDTH): + try: + lower_bound, upper_bound = [ + float(value) for value in np.asarray(bounds, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return bounds + + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or lower_bound >= upper_bound: + return bounds + + center = float(mode) if np.isfinite(mode) else float(0.5 * (lower_bound + upper_bound)) + half_width = max(float(min_half_width), 0.0) + expanded_lower = min(lower_bound, center - half_width) + expanded_upper = max(upper_bound, center + half_width) + + if expanded_lower < RPRS_SEARCH_BOUND_MIN: + expanded_upper = min( + RPRS_SEARCH_BOUND_MAX, + expanded_upper + (RPRS_SEARCH_BOUND_MIN - expanded_lower), + ) + expanded_lower = RPRS_SEARCH_BOUND_MIN + if expanded_upper > RPRS_SEARCH_BOUND_MAX: + expanded_lower = max( + RPRS_SEARCH_BOUND_MIN, + expanded_lower - (expanded_upper - RPRS_SEARCH_BOUND_MAX), + ) + expanded_upper = RPRS_SEARCH_BOUND_MAX + + return [float(expanded_lower), float(expanded_upper)] + + def run_nested_lightcurve_fit_with_rprs_posterior_retry( times, flux_values, @@ -412,6 +574,11 @@ def build_fit(local_prior, local_bounds): previous_bounds = current_bounds.get('rprs') clamped_bounds = sanitize_rprs_search_bounds({'rprs': [new_lower, new_upper]}).get('rprs', [new_lower, new_upper]) + clamped_bounds = enforce_minimum_rprs_retry_half_width( + diagnostics.get('mode', np.nan), + clamped_bounds, + ) + clamped_bounds = sanitize_rprs_search_bounds({'rprs': clamped_bounds}).get('rprs', clamped_bounds) new_lower, new_upper = [float(value) for value in clamped_bounds] if previous_bounds is not None: previous_lower, previous_upper = [float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2]] @@ -553,6 +720,38 @@ def valid_comparison_frame_mask(flux_values): return np.isfinite(flux_values) & (flux_values > 0) +def robust_flux_floor_mask( + flux_values, + min_fraction_of_median=ROBUST_FLUX_MIN_FRACTION_OF_MEDIAN, + min_points=ROBUST_FLUX_MIN_POINTS, +): + flux_values = np.asarray(flux_values, dtype=float) + valid = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(valid) < max(LIGHTCURVE_MIN_VALID_POINTS, int(min_points)): + return valid + + center, _ = sigma_clipped_nanmedian(flux_values[valid], sigma=4.0, max_iters=3) + if not np.isfinite(center) or center <= 0: + center = bn.nanmedian(flux_values[valid]) + if not np.isfinite(center) or center <= 0: + return valid + + floor = float(min_fraction_of_median) * float(center) + if not np.isfinite(floor) or floor <= 0: + return valid + + return valid & np.greater_equal(flux_values, floor) + + +def robust_target_reference_flux_mask(target_flux, reference_flux): + target_mask = robust_flux_floor_mask(target_flux) + if reference_flux is None: + return target_mask + + reference_mask = robust_flux_floor_mask(reference_flux) + return target_mask & reference_mask + + def is_fast_aperture_mask_enabled(config_value): if config_value is None: return True @@ -681,6 +880,28 @@ def should_use_aperture_photometry(config_value): return True +def should_use_eebls_to_initialize_tmid_and_bounds(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_eebls_to_initialize_tmid_and_bounds' value; " + "keeping the EEBLS transit initializer enabled.", + warn=True, + ) + return True + + def should_detect_bad_pixels_before_photometry(config_value): if config_value is None: return True @@ -769,6 +990,47 @@ def get_bad_wcs_threshold_fraction(config_value): return threshold_percent / 100.0 +def get_pointing_rejection_sigma(config_value): + default_sigma = 4.0 + if config_value is None: + return default_sigma + + if isinstance(config_value, str): + normalized = config_value.strip() + if normalized == "": + return default_sigma + else: + normalized = config_value + + try: + sigma = float(normalized) + except (TypeError, ValueError): + log_info( + f"Warning: Invalid 'pointing_rejection_sigma' value; using default {default_sigma:g}.", + warn=True, + ) + return default_sigma + + if not np.isfinite(sigma): + log_info( + f"Warning: Invalid 'pointing_rejection_sigma' value; using default {default_sigma:g}.", + warn=True, + ) + return default_sigma + + if sigma < 0: + log_info( + f"Warning: Invalid 'pointing_rejection_sigma' value; using default {default_sigma:g}.", + warn=True, + ) + return default_sigma + + if sigma == 0: + return None + + return sigma + + def is_vertical_flux_normalization_disabled(config_value): if config_value is None: return False @@ -832,6 +1094,317 @@ def get_final_fit_baseline_duration_multiplier(config_value): return FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT +def estimate_transit_duration_from_prior_geometry(prior): + try: + period = float(prior['per']) + rprs = float(prior['rprs']) + ars = float(prior['ars']) + inc = float(prior['inc']) + except (KeyError, TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(period) or period <= 0 + or not np.isfinite(rprs) or rprs < 0 + or not np.isfinite(ars) or ars <= 0 + or not np.isfinite(inc) + ): + return np.nan + + ecc = prior.get('ecc', 0.0) + omega = np.deg2rad(prior.get('omega', 0.0)) + sin_inc = np.sin(np.deg2rad(inc)) + if not np.isfinite(sin_inc) or sin_inc <= 0: + return np.nan + + impact_scale = ars * (1.0 - ecc ** 2) / max(np.finfo(float).eps, 1.0 + ecc * np.sin(omega)) + impact_parameter = impact_scale * np.cos(np.deg2rad(inc)) + chord_sq = (1.0 + rprs) ** 2 - impact_parameter ** 2 + if not np.isfinite(chord_sq) or chord_sq <= 0 or not np.isfinite(impact_scale) or impact_scale <= 0: + return np.nan + + argument = np.sqrt(chord_sq) / (impact_scale * sin_inc) + argument = float(np.clip(argument, -1.0, 1.0)) + duration = (period / np.pi) * np.arcsin(argument) + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + +def estimate_ephemeris_tmid_and_bounds( + times, + prior_tmid, + period, + midt_unc, + per_unc, + expected_duration=np.nan, + sigma_multiplier=25.0, +): + summary = { + 'method': 'ephemeris', + 'applied': False, + 'tmid': float(prior_tmid) if np.isfinite(prior_tmid) else np.nan, + 'bounds': [np.nan, np.nan], + 'cycle_index': np.nan, + 'propagated_half_width': np.nan, + 'half_width': np.nan, + 'observations_bracket_expected_transit': False, + 'duration_capped': False, + 'observed_window_capped': False, + 'note': 'Using ephemeris-derived Tmid bounds.', + } + + try: + prior_tmid = float(prior_tmid) + period = float(period) + midt_unc = float(midt_unc) + per_unc = float(per_unc) + sigma_multiplier = float(sigma_multiplier) + except (TypeError, ValueError): + summary['note'] = 'Using ephemeris-derived Tmid bounds with invalid prior metadata.' + return summary + + if not np.isfinite(prior_tmid) or not np.isfinite(period) or period <= 0: + summary['note'] = 'Using ephemeris-derived Tmid bounds with invalid Tmid/period metadata.' + return summary + + valid_times = np.asarray(times, dtype=float) + valid_times = valid_times[np.isfinite(valid_times)] + if valid_times.size == 0: + summary['bounds'] = [prior_tmid, prior_tmid] + summary['note'] = 'Using ephemeris-derived Tmid bounds with no finite observation times.' + return summary + + phases = (valid_times - prior_tmid) / period + cycle_index = float(np.floor(phases).max()) + tmid = float(prior_tmid + cycle_index * period) + + propagated_half_width = np.abs(sigma_multiplier * midt_unc + cycle_index * sigma_multiplier * per_unc) + max_half_width = 0.25 * period + if not np.isfinite(propagated_half_width) or propagated_half_width <= 0: + half_width = max_half_width + propagated_half_width = np.nan + else: + half_width = min(float(propagated_half_width), max_half_width) + + cadence = np.nan + if valid_times.size > 1: + cadence = np.nanmedian(np.diff(np.sort(valid_times))) + + lower = float(tmid - half_width) + upper = float(tmid + half_width) + if np.isfinite(expected_duration) and expected_duration > 0: + coverage_margin = 0.5 * float(expected_duration) + if np.isfinite(cadence) and cadence > 0: + coverage_margin = max(coverage_margin, 3.0 * cadence) + + pre_points = int(np.count_nonzero(valid_times < tmid - coverage_margin)) + post_points = int(np.count_nonzero(valid_times > tmid + coverage_margin)) + bracketed = pre_points > 0 and post_points > 0 + summary['observations_bracket_expected_transit'] = bracketed + + cadence_floor = 0.0 + if np.isfinite(cadence) and cadence > 0: + cadence_floor = 5.0 * cadence + duration_cap = max( + EPHEMERIS_BRACKETED_TMID_HALF_WIDTH_DURATION_MULTIPLIER * float(expected_duration), + cadence_floor, + ) + if bracketed and np.isfinite(duration_cap) and duration_cap > 0 and duration_cap < half_width: + half_width = float(duration_cap) + summary['duration_capped'] = True + lower = float(tmid - half_width) + upper = float(tmid + half_width) + + if bracketed: + observed_lower = float(np.nanmin(valid_times) + 0.5 * float(expected_duration)) + observed_upper = float(np.nanmax(valid_times) - 0.5 * float(expected_duration)) + if ( + np.isfinite(observed_lower) + and np.isfinite(observed_upper) + and observed_upper > observed_lower + ): + tightened_lower = max(lower, observed_lower) + tightened_upper = min(upper, observed_upper) + if tightened_upper > tightened_lower and ( + tightened_lower > lower + 1e-12 or tightened_upper < upper - 1e-12 + ): + lower = float(tightened_lower) + upper = float(tightened_upper) + summary['observed_window_capped'] = True + + half_width = max(float(tmid - lower), float(upper - tmid)) + summary.update({ + 'tmid': tmid, + 'bounds': [lower, upper], + 'cycle_index': cycle_index, + 'propagated_half_width': propagated_half_width, + 'half_width': half_width, + 'applied': summary['duration_capped'] or summary['observed_window_capped'], + }) + if summary['observed_window_capped']: + summary['note'] = ( + "Ephemeris-derived Tmid bounds were intersected with the observed time span needed to contain the " + f"full expected transit; using bounds=[{lower:.6f}, {upper:.6f}] instead of the wider propagated " + f"half-width {float(propagated_half_width):.6f} day(s)." + ) + elif summary['duration_capped']: + summary['note'] = ( + "Ephemeris-derived Tmid bounds were narrowed to the expected-transit timescale because the " + f"observations bracket the expected transit; using bounds=[{lower:.6f}, {upper:.6f}] " + f"instead of the wider propagated half-width {float(propagated_half_width):.6f} day(s)." + ) + else: + summary['note'] = f"Using ephemeris-derived Tmid bounds [{lower:.6f}, {upper:.6f}]." + + return summary + + +def estimate_tmid_and_bounds_with_eebls(times, flux_values, flux_errors, prior, fallback_bounds): + summary = { + 'method': 'ephemeris', + 'applied': False, + 'tmid': float(prior.get('tmid', np.nan)), + 'bounds': [float(fallback_bounds[0]), float(fallback_bounds[1])], + 'duration': np.nan, + 'depth': np.nan, + 'depth_snr': np.nan, + 'note': 'EEBLS transit initializer did not run.', + } + + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + period = float(prior.get('per', np.nan)) + if not np.isfinite(period) or period <= 0: + summary['note'] = 'EEBLS transit initializer skipped: invalid orbital period.' + return summary + + valid = np.isfinite(times) & np.isfinite(flux_values) & (flux_values > 0) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) + else: + flux_errors = np.full_like(flux_values, np.nan, dtype=float) + + if np.count_nonzero(valid) < max(LIGHTCURVE_MIN_VALID_POINTS, EEBLS_MIN_VALID_POINTS): + summary['note'] = 'EEBLS transit initializer skipped: not enough valid points.' + return summary + + valid_times = np.asarray(times[valid], dtype=float) + valid_flux = np.asarray(flux_values[valid], dtype=float) + valid_errors = np.asarray(flux_errors[valid], dtype=float) + sort_index = np.argsort(valid_times) + fit_times = valid_times[sort_index] + fit_flux = valid_flux[sort_index] + fit_errors = valid_errors[sort_index] + cadence = np.nanmedian(np.diff(fit_times)) + if not np.isfinite(cadence) or cadence <= 0: + cadence = max(np.finfo(float).eps, 0.005 * period) + + x = fit_times - np.nanmedian(fit_times) + baseline = np.ones_like(fit_flux, dtype=float) + design = np.column_stack((np.ones_like(x), x)) + if np.count_nonzero(np.isfinite(x)) >= 2: + weights = np.ones_like(fit_flux, dtype=float) + finite_error_mask = np.isfinite(fit_errors) & (fit_errors > 0) + if np.any(finite_error_mask): + weights[finite_error_mask] = 1.0 / (fit_errors[finite_error_mask] ** 2) + weights[~finite_error_mask] = 0.0 + if not np.any(weights > 0): + weights = np.ones_like(fit_flux, dtype=float) + sqrt_weights = np.sqrt(weights) + try: + coeffs, _, _, _ = np.linalg.lstsq(design * sqrt_weights[:, None], fit_flux * sqrt_weights, rcond=None) + baseline = coeffs[0] + coeffs[1] * x + if not np.all(np.isfinite(baseline)) or np.any(baseline <= 0): + baseline = np.ones_like(fit_flux, dtype=float) + except np.linalg.LinAlgError: + baseline = np.ones_like(fit_flux, dtype=float) + + detrended_flux = fit_flux / baseline + detrended_flux /= np.nanmedian(detrended_flux) + detrended_errors = fit_errors / baseline + if not np.all(np.isfinite(detrended_errors)) or np.any(detrended_errors <= 0): + detrended_errors = None + + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + if not np.isfinite(expected_duration) or expected_duration <= 0: + expected_duration = 0.05 * period + + min_duration = max(3.0 * cadence, EEBLS_DURATION_MIN_FRACTION * expected_duration) + max_duration = min(0.25 * period, max(min_duration * 1.5, EEBLS_DURATION_MAX_FRACTION * expected_duration)) + if not np.isfinite(min_duration) or not np.isfinite(max_duration) or max_duration <= 0 or min_duration > max_duration: + summary['note'] = 'EEBLS transit initializer skipped: invalid duration search grid.' + return summary + + durations = np.linspace(min_duration, max_duration, EEBLS_DURATION_GRID_SIZE) + durations = np.unique(durations[np.isfinite(durations) & (durations > 0)]) + if durations.size == 0: + summary['note'] = 'EEBLS transit initializer skipped: empty duration search grid.' + return summary + + try: + bls = BoxLeastSquares(fit_times, detrended_flux, dy=detrended_errors) + results = bls.power(period, durations, objective='snr') + except Exception as exc: + summary['note'] = f'EEBLS transit initializer failed: {type(exc).__name__}: {exc}' + return summary + + power = np.asarray(results.power, dtype=float) + if power.size == 0 or not np.any(np.isfinite(power)): + summary['note'] = 'EEBLS transit initializer skipped: no finite search power values were returned.' + return summary + + best_index = int(np.nanargmax(power)) + tmid = float(np.asarray(results.transit_time, dtype=float)[best_index]) + duration = float(np.asarray(results.duration, dtype=float)[best_index]) + depth = float(np.asarray(results.depth, dtype=float)[best_index]) + depth_snr = float(np.asarray(results.depth_snr, dtype=float)[best_index]) + if ( + not np.isfinite(tmid) + or not np.isfinite(duration) or duration <= 0 + or not np.isfinite(depth) or depth <= 0 + or not np.isfinite(depth_snr) or depth_snr <= 0 + ): + summary['note'] = 'EEBLS transit initializer skipped: the best-fitting transit candidate was not physical.' + return summary + + coverage_margin = max(0.5 * duration, 3.0 * cadence) + pre_points = int(np.count_nonzero(np.isfinite(fit_times) & (fit_times < tmid - coverage_margin))) + post_points = int(np.count_nonzero(np.isfinite(fit_times) & (fit_times > tmid + coverage_margin))) + if pre_points == 0 or post_points == 0: + summary['note'] = ( + "EEBLS transit initializer skipped: the strongest box-like signal is not bracketed by data on both sides " + f"({pre_points} pre-point(s), {post_points} post-point(s))." + ) + return summary + + duration_for_bounds = duration + if np.isfinite(expected_duration) and expected_duration > 0: + duration_for_bounds = max(duration_for_bounds, 0.75 * expected_duration) + half_width = min( + 0.25 * period, + max(EEBLS_TMID_HALF_WIDTH_DURATION_MULTIPLIER * duration_for_bounds, 5.0 * cadence), + ) + if not np.isfinite(half_width) or half_width <= 0: + summary['note'] = 'EEBLS transit initializer skipped: invalid Tmid search half-width.' + return summary + + summary.update({ + 'method': 'eebls', + 'applied': True, + 'tmid': tmid, + 'bounds': [float(tmid - half_width), float(tmid + half_width)], + 'duration': duration, + 'depth': depth, + 'depth_snr': depth_snr, + 'note': ( + "EEBLS transit initializer found a box-like transit candidate at " + f"Tmid={tmid:.6f} day(s) with duration={duration:.6f} day(s), depth={depth:.5f}, " + f"depth_snr={depth_snr:.2f}, and bounds=[{tmid - half_width:.6f}, {tmid + half_width:.6f}]." + ), + }) + return summary + + def should_use_impactparameter_rather_than_inclination_to_fit(config_value): if config_value is None: return True @@ -875,21 +1448,23 @@ def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled bounds['a0'] = [lower, upper] -def detrend_flux_on_out_of_transit_baseline( +def summarize_initial_fit_transit_coverage( times, - flux_values, - flux_errors, fit, + flux_values=None, + flux_errors=None, depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, ): times = np.asarray(times, dtype=float) - flux_values = np.asarray(flux_values, dtype=float) - flux_errors = np.asarray(flux_errors, dtype=float) transit_model = np.asarray(getattr(fit, 'transit', []), dtype=float) + if flux_values is None: + flux_values = np.ones_like(times, dtype=float) + else: + flux_values = np.asarray(flux_values, dtype=float) - if transit_model.shape != flux_values.shape: + if transit_model.shape != times.shape or flux_values.shape != times.shape: return { - 'applied': False, + 'valid': False, 'note': 'initial fit did not provide a transit model aligned with the light curve.', } @@ -899,50 +1474,369 @@ def detrend_flux_on_out_of_transit_baseline( & (flux_values > 0) & np.isfinite(transit_model) ) - if flux_errors.shape == flux_values.shape: - valid &= np.isfinite(flux_errors) & (flux_errors > 0) + if flux_errors is not None: + flux_errors = np.asarray(flux_errors, dtype=float) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) + + if np.count_nonzero(valid) < 3: + return { + 'valid': False, + 'note': 'not enough finite flux points remain to isolate the modeled transit window.', + } + + depth = np.clip(1.0 - transit_model, 0.0, None) + max_depth = np.nanmax(depth[valid]) + if not np.isfinite(max_depth) or max_depth <= 0: + return { + 'valid': False, + 'note': 'initial fit did not produce a measurable transit depth for baseline isolation.', + } + + threshold = max(1e-6, depth_fraction * max_depth) + in_transit = valid & (depth > threshold) + if not np.any(in_transit): + return { + 'valid': False, + 'note': 'could not isolate ingress and egress from the initial fit.', + } + + ingress_time = float(np.nanmin(times[in_transit])) + egress_time = float(np.nanmax(times[in_transit])) + oot_mask = valid & ((times < ingress_time) | (times > egress_time)) + + mid_transit = float(getattr(fit, 'parameters', {}).get('tmid', np.nanmedian(times[valid]))) + pre_mask = oot_mask & (times < mid_transit) + post_mask = oot_mask & (times > mid_transit) + pre_points = int(np.count_nonzero(pre_mask)) + post_points = int(np.count_nonzero(post_mask)) + has_two_sided_oot = pre_points > 0 and post_points > 0 + + summary = { + 'valid': True, + 'note': None, + 'mid_transit': mid_transit, + 'ingress_time': ingress_time, + 'egress_time': egress_time, + 'in_transit_mask': in_transit, + 'oot_mask': oot_mask, + 'pre_mask': pre_mask, + 'post_mask': post_mask, + 'pre_points': pre_points, + 'post_points': post_points, + 'has_two_sided_oot': has_two_sided_oot, + } + if not has_two_sided_oot: + summary['note'] = 'need out-of-transit coverage on both sides of transit to fit a linear baseline.' + return summary + + +def summarize_prior_transit_coverage( + times, + prior, + flux_values=None, + flux_errors=None, +): + times = np.asarray(times, dtype=float) + if flux_values is None: + flux_values = np.ones_like(times, dtype=float) else: - flux_errors = np.ones_like(flux_values, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + + if times.shape != flux_values.shape: + return { + 'valid': False, + 'note': 'prior-based transit coverage could not be aligned with the light curve.', + } + + valid = np.isfinite(times) & np.isfinite(flux_values) & (flux_values > 0) + if flux_errors is not None: + flux_errors = np.asarray(flux_errors, dtype=float) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) if np.count_nonzero(valid) < 3: + return { + 'valid': False, + 'note': 'not enough finite flux points remain to evaluate prior-based transit coverage.', + } + + try: + mid_transit = float(prior.get('tmid', np.nan)) + except (AttributeError, TypeError, ValueError): + mid_transit = np.nan + if not np.isfinite(mid_transit): + return { + 'valid': False, + 'note': 'prior-based transit coverage skipped: no finite ephemeris-centered Tmid was available.', + } + + duration = estimate_transit_duration_from_prior_geometry(prior) + if not np.isfinite(duration) or duration <= 0: + return { + 'valid': False, + 'note': 'prior-based transit coverage skipped: could not estimate a physical transit duration from the priors.', + } + + ingress_time = float(mid_transit - 0.5 * duration) + egress_time = float(mid_transit + 0.5 * duration) + in_transit = valid & (times >= ingress_time) & (times <= egress_time) + if not np.any(in_transit): + return { + 'valid': False, + 'note': 'prior-based transit coverage skipped: the ephemeris-centered transit window does not overlap the observations.', + } + + oot_mask = valid & ((times < ingress_time) | (times > egress_time)) + pre_mask = oot_mask & (times < mid_transit) + post_mask = oot_mask & (times > mid_transit) + pre_points = int(np.count_nonzero(pre_mask)) + post_points = int(np.count_nonzero(post_mask)) + has_two_sided_oot = pre_points > 0 and post_points > 0 + + summary = { + 'valid': True, + 'note': None, + 'mid_transit': mid_transit, + 'ingress_time': ingress_time, + 'egress_time': egress_time, + 'in_transit_mask': in_transit, + 'oot_mask': oot_mask, + 'pre_mask': pre_mask, + 'post_mask': post_mask, + 'pre_points': pre_points, + 'post_points': post_points, + 'has_two_sided_oot': has_two_sided_oot, + 'used_prior_ephemeris': True, + 'duration': duration, + } + if not has_two_sided_oot: + summary['note'] = ( + 'need out-of-transit coverage on both sides of the ephemeris-centered transit window ' + 'to fit a linear baseline.' + ) + return summary + + +def extract_baseline_corrected_lightcurve_arrays(fit): + times = np.asarray(getattr(fit, 'time', []), dtype=float) + if times.ndim != 1 or times.size == 0: + return None, None, None + + flux_values = None + flux_source = None + + detrended = np.asarray(getattr(fit, 'detrended', []), dtype=float) + if detrended.shape == times.shape: + valid_detrended = np.isfinite(detrended) & (detrended > 0) + if np.any(valid_detrended): + flux_values = detrended.copy() + flux_source = "provisional detrended light curve" + + if flux_values is None: + data = np.asarray(getattr(fit, 'data', []), dtype=float) + airmass_model = np.asarray(getattr(fit, 'airmass_model', []), dtype=float) + if data.shape == times.shape and airmass_model.shape == times.shape: + with np.errstate(divide='ignore', invalid='ignore'): + flux_values = np.divide(data, airmass_model) + flux_source = "provisional flux ratio divided by the fitted airmass/baseline model" + elif data.shape == times.shape: + flux_values = data.copy() + flux_source = "provisional raw flux ratio" + else: + return None, None, None + + flux_errors = None + detrended_errors = np.asarray(getattr(fit, 'detrendederr', []), dtype=float) + if detrended_errors.shape == times.shape: + valid_detrended_errors = np.isfinite(detrended_errors) & (detrended_errors > 0) + if np.any(valid_detrended_errors): + flux_errors = detrended_errors.copy() + + if flux_errors is None: + data_errors = np.asarray(getattr(fit, 'dataerr', []), dtype=float) + airmass_model = np.asarray(getattr(fit, 'airmass_model', []), dtype=float) + if data_errors.shape == times.shape and airmass_model.shape == times.shape: + with np.errstate(divide='ignore', invalid='ignore'): + flux_errors = np.divide(data_errors, airmass_model) + elif data_errors.shape == times.shape: + flux_errors = data_errors.copy() + else: + flux_errors = np.full(times.shape, np.nan, dtype=float) + + return flux_values, flux_errors, flux_source + + +def prepare_final_fit_lightcurve_series( + fit, + depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, +): + times = np.asarray(getattr(fit, 'time', []), dtype=float) + if times.ndim != 1 or times.size == 0: + return { + 'applied': False, + 'note': 'could not prepare a final-fit light curve because the provisional fit had no time samples.', + } + + flux_values, flux_errors, flux_source = extract_baseline_corrected_lightcurve_arrays(fit) + if flux_values is None: return { 'applied': False, - 'note': 'not enough finite flux points remain to fit an out-of-transit baseline.', + 'note': 'could not derive a baseline-corrected provisional light curve for final-fit preparation.', } - depth = np.clip(1.0 - transit_model, 0.0, None) - max_depth = np.nanmax(depth[valid]) - if not np.isfinite(max_depth) or max_depth <= 0: + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + + valid_flux = np.isfinite(flux_values) & (flux_values > 0) + valid_errors = np.isfinite(flux_errors) & (flux_errors > 0) + + if np.count_nonzero(valid_flux) < LIGHTCURVE_MIN_VALID_POINTS: + return { + 'applied': False, + 'note': 'not enough finite baseline-corrected flux points remained for final-fit preparation.', + } + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors if np.any(valid_errors) else None, + depth_fraction=depth_fraction, + ) + + baseline_mask = valid_flux + used_two_sided_oot = False + pre_points = coverage_summary.get('pre_points', 0) + post_points = coverage_summary.get('post_points', 0) + + if coverage_summary.get('valid') and coverage_summary.get('has_two_sided_oot'): + candidate_baseline_mask = coverage_summary['oot_mask'] & valid_flux + if np.count_nonzero(candidate_baseline_mask) >= LIGHTCURVE_MIN_VALID_POINTS: + baseline_mask = candidate_baseline_mask + used_two_sided_oot = True + + baseline_level, baseline_scatter = sigma_clipped_nanmedian(flux_values[baseline_mask]) + if not np.isfinite(baseline_level) or baseline_level <= 0: + fallback_mask = valid_flux + baseline_level, baseline_scatter = sigma_clipped_nanmedian(flux_values[fallback_mask]) + baseline_mask = fallback_mask + + if not np.isfinite(baseline_level) or baseline_level <= 0: + return { + 'applied': False, + 'note': 'could not determine a positive baseline level for final-fit preparation.', + } + + normalized_flux = flux_values / baseline_level + normalized_unc = flux_errors / baseline_level + + if used_two_sided_oot: + uncertainty_mask = baseline_mask & np.isfinite(normalized_unc) & (normalized_unc > 0) + observed_scatter = np.nanstd(normalized_flux[baseline_mask]) + predicted_unc = np.nanmedian(normalized_unc[uncertainty_mask]) if np.any(uncertainty_mask) else np.nan + if np.isfinite(observed_scatter) and observed_scatter > 0 and np.isfinite(predicted_unc) and predicted_unc > 0: + normalized_unc *= observed_scatter / predicted_unc + + valid_normalized_unc = np.isfinite(normalized_unc) & (normalized_unc > 0) + if not np.any(valid_normalized_unc): + fallback_unc = baseline_scatter / baseline_level + if not np.isfinite(fallback_unc) or fallback_unc <= 0: + fallback_unc = np.nanstd(normalized_flux[baseline_mask]) + if not np.isfinite(fallback_unc) or fallback_unc <= 0: + fallback_unc = np.finfo(float).eps + normalized_unc = np.full(times.shape, fallback_unc, dtype=float) + else: + fallback_unc = np.nanmedian(normalized_unc[valid_normalized_unc]) + if not np.isfinite(fallback_unc) or fallback_unc <= 0: + fallback_unc = np.nanstd(normalized_flux[baseline_mask]) + if not np.isfinite(fallback_unc) or fallback_unc <= 0: + fallback_unc = np.finfo(float).eps + normalized_unc[~valid_normalized_unc] = fallback_unc + + if used_two_sided_oot: + note = ( + f"Prepared the final-fit input light curve from the {flux_source} and normalized it with " + f"{pre_points} pre-ingress and {post_points} post-egress modeled out-of-transit point(s)." + ) + else: + note = ( + f"Prepared the final-fit input light curve from the {flux_source} and normalized it with a " + f"sigma-clipped full-series baseline because the provisional fit only bracketed one side of transit " + f"({pre_points} pre-ingress and {post_points} post-egress modeled out-of-transit point(s))." + ) + + return { + 'applied': True, + 'flux': normalized_flux, + 'unc': normalized_unc, + 'note': note, + 'source': flux_source, + 'coverage_summary': coverage_summary, + 'used_two_sided_oot': used_two_sided_oot, + 'baseline_level': float(baseline_level), + } + + +def detrend_flux_on_out_of_transit_baseline( + times, + flux_values, + flux_errors, + fit, + prior=None, + depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + if flux_errors.shape != flux_values.shape: + flux_errors = np.ones_like(flux_values, dtype=float) + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors, + depth_fraction=depth_fraction, + ) + used_prior_coverage = False + if prior is not None and ( + (not coverage_summary.get('valid')) + or (not coverage_summary.get('has_two_sided_oot')) + ): + prior_coverage_summary = summarize_prior_transit_coverage( + times, + prior, + flux_values=flux_values, + flux_errors=flux_errors, + ) + if prior_coverage_summary.get('valid') and prior_coverage_summary.get('has_two_sided_oot'): + coverage_summary = prior_coverage_summary + used_prior_coverage = True + + if not coverage_summary.get('valid'): return { 'applied': False, - 'note': 'initial fit did not produce a measurable transit depth for baseline isolation.', + 'note': coverage_summary.get('note', 'could not isolate a transit window for baseline fitting.'), } - threshold = max(1e-6, depth_fraction * max_depth) - in_transit = valid & (depth > threshold) - if not np.any(in_transit): + if not coverage_summary.get('has_two_sided_oot'): return { 'applied': False, - 'note': 'could not isolate ingress and egress from the initial fit.', + 'note': coverage_summary.get( + 'note', + 'need out-of-transit coverage on both sides of transit to fit a linear baseline.', + ), + 'pre_points': coverage_summary.get('pre_points', 0), + 'post_points': coverage_summary.get('post_points', 0), } - ingress_time = float(np.nanmin(times[in_transit])) - egress_time = float(np.nanmax(times[in_transit])) - oot_mask = valid & ((times < ingress_time) | (times > egress_time)) - - mid_transit = float(getattr(fit, 'parameters', {}).get('tmid', np.nanmedian(times[valid]))) - pre_mask = oot_mask & (times < mid_transit) - post_mask = oot_mask & (times > mid_transit) - pre_points = int(np.count_nonzero(pre_mask)) - post_points = int(np.count_nonzero(post_mask)) - - if pre_points == 0 or post_points == 0: - return { - 'applied': False, - 'note': 'need out-of-transit coverage on both sides of transit to fit a linear baseline.', - 'pre_points': pre_points, - 'post_points': post_points, - } + oot_mask = coverage_summary['oot_mask'] + mid_transit = coverage_summary['mid_transit'] + ingress_time = coverage_summary['ingress_time'] + egress_time = coverage_summary['egress_time'] + pre_points = coverage_summary['pre_points'] + post_points = coverage_summary['post_points'] x = times[oot_mask] - mid_transit if np.allclose(x, x[0]): @@ -987,8 +1881,14 @@ def detrend_flux_on_out_of_transit_baseline( return { 'applied': True, 'note': ( - f"Applied weighted linear out-of-transit baseline detrending using " - f"{pre_points} pre-ingress and {post_points} post-egress points." + ( + "Applied weighted linear out-of-transit baseline detrending using the " + "ephemeris-centered transit window from the priors because the fitted transit " + "window was one-sided. " + if used_prior_coverage else + "Applied weighted linear out-of-transit baseline detrending using " + ) + + f"{pre_points} pre-ingress and {post_points} post-egress points." ), 'flux': flux_values / baseline, 'unc': flux_errors / baseline, @@ -999,6 +1899,7 @@ def detrend_flux_on_out_of_transit_baseline( 'post_points': post_points, 'ingress_time': ingress_time, 'egress_time': egress_time, + 'used_prior_ephemeris': used_prior_coverage, } @@ -1035,6 +1936,115 @@ def estimate_transit_duration_from_fit(fit): return float(duration) if np.isfinite(duration) and duration > 0 else np.nan +def build_nested_tmid_refinement_from_initial_fit(times, flux_values, flux_errors, prior, bounds, fit): + original_tmid_bounds = clone_lightcurve_bounds(bounds).get('tmid') + base_plan = { + 'applied': False, + 'note': 'Not needed; using the original nested-sampling Tmid bounds.', + 'prior': dict(prior), + 'bounds': clone_lightcurve_bounds(bounds), + 'original_tmid_bounds': original_tmid_bounds, + 'refined_tmid_bounds': original_tmid_bounds, + } + + if fit is None or not hasattr(fit, 'parameters') or not isinstance(fit.parameters, dict): + base_plan['note'] = "Skipped; the initial LM fit did not provide fitted parameters for nested-sampling refinement." + return base_plan + + tmid = fit.parameters.get('tmid', np.nan) + if not np.isfinite(tmid): + base_plan['note'] = "Skipped; the initial LM fit did not return a finite Tmid." + return base_plan + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors, + ) + if not coverage_summary.get('valid'): + base_plan['note'] = ( + "Skipped; the initial LM fit did not provide a usable modeled transit window for nested-sampling refinement." + ) + return base_plan + + if not coverage_summary.get('has_two_sided_oot'): + pre_points = coverage_summary.get('pre_points', 0) + post_points = coverage_summary.get('post_points', 0) + base_plan['note'] = ( + "Skipped; the initial LM fit only captured one side of the modeled transit, " + "so tightening the nested-sampling Tmid bounds would lock onto a partial-transit solution " + f"({pre_points} pre-ingress and {post_points} post-egress out-of-transit point(s))." + ) + return base_plan + + duration = estimate_transit_duration_from_fit(fit) + period = fit.parameters.get('per', prior.get('per', np.nan)) + if not np.isfinite(duration) or duration <= 0: + base_plan['note'] = "Skipped; the initial LM fit did not produce a measurable transit duration." + return base_plan + if np.isfinite(period) and duration >= period: + base_plan['note'] = "Skipped; the initial LM fit returned a non-physical transit duration." + return base_plan + + valid_times = np.asarray(times, dtype=float) + valid_times = valid_times[np.isfinite(valid_times)] + cadence = np.nan + if valid_times.size > 1: + cadence = np.nanmedian(np.diff(np.sort(valid_times))) + + half_width = FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER * duration + if np.isfinite(cadence) and cadence > 0: + half_width = max(half_width, 3.0 * cadence) + if not np.isfinite(half_width) or half_width <= 0: + base_plan['note'] = "Skipped; the nested-sampling Tmid refinement half-width was not physical." + return base_plan + + refined_lower = float(tmid - half_width) + refined_upper = float(tmid + half_width) + if original_tmid_bounds is not None and len(original_tmid_bounds) == 2: + original_lower = float(original_tmid_bounds[0]) + original_upper = float(original_tmid_bounds[1]) + refined_lower = max(original_lower, refined_lower) + refined_upper = min(original_upper, refined_upper) + + if not np.isfinite(refined_lower) or not np.isfinite(refined_upper) or refined_upper <= refined_lower: + base_plan['note'] = "Skipped; the refined nested-sampling Tmid bounds collapsed to an invalid range." + return base_plan + + refined_tmid_bounds = [refined_lower, refined_upper] + base_plan['refined_tmid_bounds'] = refined_tmid_bounds + if original_tmid_bounds is not None and np.allclose( + np.asarray(original_tmid_bounds, dtype=float), + np.asarray(refined_tmid_bounds, dtype=float), + atol=1e-12, + rtol=0.0, + ): + base_plan['note'] = ( + "Not needed; the initial LM fit already sat inside the original nested-sampling Tmid bounds." + ) + return base_plan + + refined_prior = dict(prior) + for key in ('rprs', 'tmid', 'inc', 'a2'): + if key in refined_prior and key in fit.parameters: + refined_prior[key] = fit.parameters[key] + + refined_bounds = clone_lightcurve_bounds(bounds) + refined_bounds['tmid'] = refined_tmid_bounds + + base_plan.update({ + 'applied': True, + 'note': ( + "Using the initial LM fit to recenter nested-sampling Tmid bounds to " + f"[{refined_lower:.6f}, {refined_upper:.6f}] around Tmid={tmid:.6f}." + ), + 'prior': refined_prior, + 'bounds': refined_bounds, + }) + return base_plan + + def build_final_fit_prefit_refinement_plan( times, flux_values, @@ -1089,6 +2099,22 @@ def build_final_fit_prefit_refinement_plan( base_plan['note'] = "Skipped; the initial nested fit did not return a finite Tmid." return base_plan + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors, + ) + if coverage_summary.get('valid') and not coverage_summary.get('has_two_sided_oot'): + pre_points = coverage_summary.get('pre_points', 0) + post_points = coverage_summary.get('post_points', 0) + base_plan['note'] = ( + "Skipped; the initial nested fit only captured one side of the modeled transit, " + "so tightening the second-pass Tmid bounds would lock onto a partial-transit solution " + f"({pre_points} pre-ingress and {post_points} post-egress out-of-transit point(s))." + ) + return base_plan + period = fit_parameters.get('per', prior.get('per', np.nan)) if np.isfinite(period) and duration >= period: base_plan['note'] = "Skipped; the initial nested fit returned a non-physical transit duration." @@ -1123,7 +2149,19 @@ def build_final_fit_prefit_refinement_plan( else: trim_note = "kept the full light curve because no extra baseline points fell outside the target window" - refined_tmid_bounds = [float(tmid - half_duration), float(tmid + half_duration)] + refined_lower = float(tmid - half_duration) + refined_upper = float(tmid + half_duration) + if original_tmid_bounds is not None and len(original_tmid_bounds) == 2: + original_lower = float(original_tmid_bounds[0]) + original_upper = float(original_tmid_bounds[1]) + refined_lower = max(original_lower, refined_lower) + refined_upper = min(original_upper, refined_upper) + + if not np.isfinite(refined_lower) or not np.isfinite(refined_upper) or refined_upper <= refined_lower: + base_plan['note'] = "Skipped; the refined final-fit Tmid bounds collapsed to an invalid range." + return base_plan + + refined_tmid_bounds = [refined_lower, refined_upper] base_plan['refined_tmid_bounds'] = refined_tmid_bounds bounds_changed = False @@ -1276,7 +2314,13 @@ def fit_final_lightcurve_with_oot_baseline_detrending( ) return fit, working_flux, working_unc - detrend_result = detrend_flux_on_out_of_transit_baseline(working_times, working_flux, working_unc, fit) + detrend_result = detrend_flux_on_out_of_transit_baseline( + working_times, + working_flux, + working_unc, + fit, + prior=working_prior, + ) if not detrend_result.get('applied'): note = f"Skipped; {detrend_result.get('note', 'unable to fit an out-of-transit baseline.')}" log_info(f"Optional out-of-transit baseline detrending skipped: {detrend_result.get('note', 'unknown reason')}") @@ -1506,21 +2550,60 @@ def resolve_frame_aperture_radii(apertures, annuli, adaptive_apertures=False, fr # Initialze plate status log plateStatus = PlateStatus(log_info) -def sigma_clip(ogdata, sigma=3, dt=21, po=2): - nanmask = np.isnan(ogdata) +def sigma_clip(ogdata, sigma=3, dt=21, po=2, times=None): + values = np.asarray(ogdata, dtype=float) + nanmask = np.isnan(values) + valid_mask = ~nanmask + valid_indices = np.flatnonzero(valid_mask) + + if not (po < dt <= valid_indices.size): + return nanmask + + segment_ranges = [] + if times is not None: + time_values = np.asarray(times, dtype=float) + if time_values.shape == values.shape: + valid_times = time_values[valid_mask] + finite_valid_times = np.isfinite(valid_times) + if np.all(finite_valid_times): + cadence = np.nanmedian(np.diff(valid_times)) if valid_times.size > 1 else np.nan + if np.isfinite(cadence) and cadence > 0: + gap_threshold = max( + 5.0 * cadence, + 0.25 * int(dt) * cadence, + ) + local_start = 0 + for local_index, gap in enumerate(np.diff(valid_times), start=1): + if gap > gap_threshold: + segment_ranges.append((local_start, local_index)) + local_start = local_index + segment_ranges.append((local_start, valid_times.size)) + + if not segment_ranges: + segment_ranges = [(0, valid_indices.size)] + + clipped_mask = np.zeros(valid_indices.size, dtype=bool) + for start, stop in segment_ranges: + local_values = values[valid_indices[start:stop]] + if not (po < dt <= local_values.size): + continue - if po < dt <= len(ogdata[~nanmask]): - mdata = savgol_filter(ogdata[~nanmask], window_length=dt, polyorder=po) - # mdata = median_filter(ogdata[~nanmask], dt) - res = ogdata[~nanmask] - mdata + mdata = savgol_filter(local_values, window_length=dt, polyorder=po) + # mdata = median_filter(local_values, dt) + res = local_values - mdata + if res.size == 0: + continue # Vectorized bootstrap estimate avoids Python-loop overhead in tight runs. sample_size = min(25, res.size) bootstrap_samples = np.random.choice(res, size=(100, sample_size), replace=True) std = bn.nanmedian(bn.nanstd(bootstrap_samples, axis=1)) # std = np.nanstd(res) # biased from large outliers + if not np.isfinite(std) or std <= 0: + continue sigmask = np.abs(res) > sigma * std - nanmask[~nanmask] = sigmask + clipped_mask[start:stop] = sigmask + nanmask[valid_indices] = clipped_mask return nanmask @@ -1707,6 +2790,123 @@ def apply_plot_time_range(lightcurve, time_values): return lightcurve +def build_time_rejection_diagnostic(stage, times, keep_mask, note=None): + times = np.asarray(times, dtype=float).reshape(-1) + keep_mask = np.asarray(keep_mask, dtype=bool).reshape(-1) + if times.shape[0] != keep_mask.shape[0]: + return None + + finite_mask = np.isfinite(times) + input_point_count = int(np.count_nonzero(finite_mask)) + dropped_times = np.asarray(times[finite_mask & ~keep_mask], dtype=float) + kept_point_count = int(np.count_nonzero(finite_mask & keep_mask)) + dropped_point_count = int(dropped_times.size) + + cadence = np.nan + finite_times = np.sort(times[finite_mask]) + if finite_times.size > 1: + cadence = np.nanmedian(np.diff(finite_times)) + + dropped_ranges = [] + if dropped_times.size: + dropped_times = np.sort(dropped_times) + gap_threshold = np.inf + if np.isfinite(cadence) and cadence > 0: + gap_threshold = max( + TIME_REJECTION_GROUP_GAP_CADENCE_MULTIPLIER * cadence, + np.finfo(float).eps, + ) + + range_start = float(dropped_times[0]) + range_end = float(dropped_times[0]) + range_count = 1 + for current_time in dropped_times[1:]: + current_time = float(current_time) + if np.isfinite(gap_threshold) and (current_time - range_end) <= gap_threshold: + range_end = current_time + range_count += 1 + continue + + dropped_ranges.append({ + 'start': range_start, + 'end': range_end, + 'count': int(range_count), + }) + range_start = current_time + range_end = current_time + range_count = 1 + + dropped_ranges.append({ + 'start': range_start, + 'end': range_end, + 'count': int(range_count), + }) + + return { + 'stage': stage, + 'note': note, + 'input_point_count': input_point_count, + 'kept_point_count': kept_point_count, + 'dropped_point_count': dropped_point_count, + 'cadence': float(cadence) if np.isfinite(cadence) else np.nan, + 'dropped_ranges': dropped_ranges, + 'first_dropped_time': (float(dropped_times[0]) if dropped_times.size else np.nan), + 'last_dropped_time': (float(dropped_times[-1]) if dropped_times.size else np.nan), + } + + +def format_time_rejection_diagnostic(diagnostic, max_ranges=TIME_REJECTION_RANGE_DISPLAY_LIMIT): + if not diagnostic: + return None + + dropped_ranges = diagnostic.get('dropped_ranges') or [] + if not dropped_ranges: + return ( + f"{diagnostic.get('stage', 'frame filter')}: removed 0/" + f"{diagnostic.get('input_point_count', 0)} frame(s)." + ) + + display_ranges = dropped_ranges[:max_ranges] + range_parts = [] + for range_summary in display_ranges: + start = float(range_summary['start']) + end = float(range_summary['end']) + count = int(range_summary['count']) + if count <= 1 or np.isclose(start, end): + range_parts.append(f"{start:.8f} ({count} frame)") + else: + frame_label = "frame" if count == 1 else "frames" + range_parts.append(f"{start:.8f} to {end:.8f} ({count} {frame_label})") + + if len(dropped_ranges) > len(display_ranges): + remaining = len(dropped_ranges) - len(display_ranges) + range_parts.append(f"... {remaining} more range(s)") + + message = ( + f"{diagnostic.get('stage', 'frame filter')}: removed " + f"{diagnostic.get('dropped_point_count', 0)}/{diagnostic.get('input_point_count', 0)} frame(s); " + f"BJD range(s): {'; '.join(range_parts)}." + ) + note = diagnostic.get('note') + if note: + message += f" {note}" + return message + + +def log_lightcurve_filter_diagnostics(diagnostics, header="Lightcurve frame rejection diagnostics", only_removed=True): + normalized = [diagnostic for diagnostic in (diagnostics or []) if diagnostic] + if only_removed: + normalized = [diagnostic for diagnostic in normalized if diagnostic.get('dropped_point_count', 0) > 0] + if not normalized: + return + + log_info(f"\n{header}:") + for diagnostic in normalized: + diagnostic_text = format_time_rejection_diagnostic(diagnostic) + if diagnostic_text: + log_info(f" {diagnostic_text}") + + def exp_offset(hdr, time_unit, exp): """Returns exposure offset (in days) of more than 0 if headers reveals the time was estimated at the start of the exposure rather than the middle @@ -2269,25 +3469,348 @@ def collect_celestial_wcs_coverage(inputfiles): if not file_has_celestial_wcs: missing_wcs_files.append(str(file_name)) - return np.array(has_celestial_wcs, dtype=bool), missing_wcs_files + return np.array(has_celestial_wcs, dtype=bool), missing_wcs_files + + +def evaluate_celestial_wcs_coverage(inputfiles): + has_celestial_wcs, missing_wcs_files = collect_celestial_wcs_coverage(inputfiles) + total_files = len(inputfiles) + all_have_celestial_wcs = total_files > 0 and bool(has_celestial_wcs.all()) + return all_have_celestial_wcs, missing_wcs_files + + +def log_file_preview(file_names, label): + if not file_names: + return + + preview = ", ".join([_display_filename(file_name) for file_name in file_names[:3]]) + remainder = len(file_names) - 3 + if remainder > 0: + preview = f"{preview}, ... (+{remainder} more)" + log.debug(f"{label}: {preview}") + + +def log_missing_celestial_wcs_preview(missing_wcs_files): + log_file_preview(missing_wcs_files, "Files without usable celestial WCS") + + +def format_file_preview_for_user(file_names, limit=6): + if not file_names: + return "" + + display_names = [_display_filename(file_name) for file_name in file_names[:limit]] + remainder = len(file_names) - len(display_names) + if remainder > 0: + display_names.append(f"... (+{remainder} more)") + return ", ".join(display_names) + + +def leading_rejected_reference_prefix(ordered_inputfiles, dropped_files): + if ordered_inputfiles is None: + return [], None + + ordered_inputfiles = [str(file_name) for file_name in ordered_inputfiles] + dropped_lookup = {str(file_name) for file_name in (dropped_files or [])} + leading_rejected = [] + next_candidate = None + + for file_name in ordered_inputfiles: + if file_name in dropped_lookup: + leading_rejected.append(file_name) + continue + next_candidate = file_name + break + + return leading_rejected, next_candidate + + +def abort_if_reference_frame_rejected(reference_file, dropped_files, ordered_inputfiles=None, + rejection_label="Pointing precheck"): + if reference_file is None or not dropped_files: + return False + + reference_file = str(reference_file) + dropped_files = [str(file_name) for file_name in dropped_files] + if reference_file not in dropped_files: + return False + + other_dropped_files = [file_name for file_name in dropped_files if file_name != reference_file] + leading_rejected_files, next_reference_candidate = leading_rejected_reference_prefix( + ordered_inputfiles, + dropped_files, + ) + if not leading_rejected_files: + leading_rejected_files = [reference_file] + + log_info( + f"Error: {rejection_label} rejected the first usable image " + f"({_display_filename(reference_file)}). EXOTIC uses that frame as the reference image for " + "the supplied target and comparison-star pixel coordinates, so it is not safe to continue " + "with a different reference image.", + error=True, + ) + if other_dropped_files: + log_info( + f"{rejection_label} also rejected {len(other_dropped_files)} other frame(s): " + f"{format_file_preview_for_user(other_dropped_files)}", + error=True, + ) + + leading_preview = format_file_preview_for_user(leading_rejected_files) + if len(leading_rejected_files) == 1: + removal_instruction = ( + f"Please remove or move this rejected frame and run again: {leading_preview}." + ) + else: + removal_instruction = ( + f"Please remove or move these leading rejected frames and run again: {leading_preview}." + ) + + if next_reference_candidate is not None: + removal_instruction += ( + f" The next remaining frame would be " + f"{_display_filename(next_reference_candidate)}." + ) + else: + removal_instruction += ( + " No non-rejected frame remains after that prefix, so this dataset still would not " + "have a usable reference image." + ) + + log_info( + removal_instruction, + error=True, + ) + log_info( + "If you need to keep those frames, reorder the dataset so a good reference image comes first, " + "or set optional_info 'pointing_rejection_sigma' to 0 to disable this precheck.", + error=True, + ) + return True + + +def collect_wcs_frame_center_pointings(inputfiles): + positions = np.full((len(inputfiles), 2), np.nan, dtype=float) + usable_mask = np.zeros(len(inputfiles), dtype=bool) + usable_indices = [] + ra_values = [] + dec_values = [] + + for index, file_name in enumerate(inputfiles): + try: + image_header = get_first_image_header(file_name) + wcs = search_wcs_from_header(image_header) + if not wcs.is_celestial: + continue + + width = int(image_header.get("NAXIS1", 0)) + height = int(image_header.get("NAXIS2", 0)) + if width <= 0 or height <= 0: + continue + + center_x = (width - 1) / 2.0 + center_y = (height - 1) / 2.0 + ra_deg, dec_deg = wcs.pixel_to_world_values(center_x, center_y) + if not np.isfinite(ra_deg) or not np.isfinite(dec_deg): + continue + + usable_indices.append(index) + ra_values.append(float(ra_deg)) + dec_values.append(float(dec_deg)) + except Exception: + continue + + if not usable_indices: + return positions, usable_mask + + coords = SkyCoord(ra=np.asarray(ra_values) * u.deg, dec=np.asarray(dec_values) * u.deg, frame='icrs') + reference_coord = coords[0] + delta_lon, delta_lat = reference_coord.spherical_offsets_to(coords) + offsets = np.column_stack((delta_lon.to_value(u.arcsec), delta_lat.to_value(u.arcsec))) + + for index, offset in zip(usable_indices, offsets): + positions[index] = offset + usable_mask[index] = True + + return positions, usable_mask + + +def log_pointing_precheck_alignment_progress(i, total_files, file_name): + log_info( + f"Pointing precheck alignment progress: file {i + 1} of {total_files} : " + f"{_display_filename(file_name)}" + ) + + +def collect_transform_frame_pointings(inputfiles, frame_loader=None): + positions = np.full((len(inputfiles), 2), np.nan, dtype=float) + usable_mask = np.zeros(len(inputfiles), dtype=bool) + + if len(inputfiles) == 0: + return positions, usable_mask + + if frame_loader is None: + frame_loader = load_image_data + + total_files = len(inputfiles) + log_pointing_precheck_alignment_progress(0, total_files, inputfiles[0]) + try: + reference_image = frame_loader(inputfiles[0]) + except Exception as exc: + log_info( + f"Warning: pointing precheck alignment fallback could not load the reference frame " + f"{_display_filename(inputfiles[0])} ({exc}).", + warn=True, + ) + return positions, usable_mask + + if getattr(reference_image, "ndim", 0) != 2: + log_info("Warning: pointing precheck alignment fallback requires 2-D images; skipping.", warn=True) + return positions, usable_mask + + height, width = reference_image.shape + reference_anchor = np.array([[(width - 1) / 2.0, (height - 1) / 2.0]], dtype=float) + positions[0] = reference_anchor[0] + usable_mask[0] = True + + for index, file_name in enumerate(inputfiles[1:], start=1): + log_pointing_precheck_alignment_progress(index, total_files, file_name) + try: + image_data = frame_loader(file_name) + if getattr(image_data, "ndim", 0) != 2: + continue + + tform = transformation( + image_data, + file_name, + report_failure=False, + reference_image=reference_image, + ) + mapped_anchor = np.asarray(tform(reference_anchor), dtype=float).reshape(-1, 2)[0] + if np.all(np.isfinite(mapped_anchor)): + positions[index] = mapped_anchor + usable_mask[index] = True + except Exception: + continue + + return positions, usable_mask + + +def sigma_clip_pointing_positions(positions, sigma=3.0, max_iters=5): + positions = np.asarray(positions, dtype=float) + if positions.ndim != 2 or positions.shape[1] != 2: + raise ValueError("positions must be an Nx2 array") + + finite_mask = np.all(np.isfinite(positions), axis=1) + keep_mask = finite_mask.copy() + if np.count_nonzero(keep_mask) < POINTING_REJECTION_MIN_FRAMES: + return keep_mask + + sigma = float(sigma) + for _ in range(max_iters): + candidate_positions = positions[keep_mask] + if candidate_positions.shape[0] < POINTING_REJECTION_MIN_FRAMES: + break + + center = np.nanmedian(candidate_positions, axis=0) + deltas = candidate_positions - center + radial_offsets = np.hypot(deltas[:, 0], deltas[:, 1]) + + scatter_x = robust_scatter(deltas[:, 0]) + scatter_y = robust_scatter(deltas[:, 1]) + radial_scatter = robust_scatter(radial_offsets) + + if not np.isfinite(scatter_x) or scatter_x <= 0: + scatter_x = radial_scatter + if not np.isfinite(scatter_y) or scatter_y <= 0: + scatter_y = radial_scatter + + if (not np.isfinite(scatter_x) or scatter_x <= 0 + or not np.isfinite(scatter_y) or scatter_y <= 0): + break + + normalized_distance = np.sqrt((deltas[:, 0] / scatter_x) ** 2 + (deltas[:, 1] / scatter_y) ** 2) + current_keep = normalized_distance <= sigma + if np.all(current_keep): + break + + updated_keep = keep_mask.copy() + updated_keep[np.flatnonzero(keep_mask)] = current_keep + if np.array_equal(updated_keep, keep_mask): + break + keep_mask = updated_keep + + return keep_mask + + +def filter_pointing_outlier_frames(inputfiles, pointing_rejection_sigma=None, ignore_header_wcs=False, + frame_loader=None): + inputfiles = np.array(inputfiles) + keep_mask = np.ones(len(inputfiles), dtype=bool) + + if len(inputfiles) == 0 or pointing_rejection_sigma is None: + return inputfiles, keep_mask, [] + + if len(inputfiles) < POINTING_REJECTION_MIN_FRAMES: + log_info( + f"Pointing precheck skipped: only {len(inputfiles)} frame(s); " + f"need at least {POINTING_REJECTION_MIN_FRAMES}.", + ) + return inputfiles, keep_mask, [] + positions = None + usable_mask = None + mode_label = None + + if not ignore_header_wcs: + wcs_positions, wcs_usable_mask = collect_wcs_frame_center_pointings(inputfiles) + usable_wcs_count = int(np.count_nonzero(wcs_usable_mask)) + if usable_wcs_count == len(inputfiles): + positions = wcs_positions + usable_mask = wcs_usable_mask + mode_label = "WCS" + elif usable_wcs_count > 0: + log_info( + f"Pointing precheck: usable WCS-derived pointing centers found for " + f"{usable_wcs_count}/{len(inputfiles)} frame(s); falling back to alignment-derived positions." + ) + else: + log_info("Pointing precheck: no usable WCS-derived pointing centers found; using alignment-derived positions.") -def evaluate_celestial_wcs_coverage(inputfiles): - has_celestial_wcs, missing_wcs_files = collect_celestial_wcs_coverage(inputfiles) - total_files = len(inputfiles) - all_have_celestial_wcs = total_files > 0 and bool(has_celestial_wcs.all()) - return all_have_celestial_wcs, missing_wcs_files + if positions is None: + positions, usable_mask = collect_transform_frame_pointings(inputfiles, frame_loader=frame_loader) + mode_label = "alignment" + usable_count = int(np.count_nonzero(usable_mask)) + if usable_count < POINTING_REJECTION_MIN_FRAMES: + log_info( + f"Pointing precheck skipped: only {usable_count} usable {mode_label}-derived pointing estimate(s); " + f"need at least {POINTING_REJECTION_MIN_FRAMES}.", + ) + return inputfiles, keep_mask, [] -def log_missing_celestial_wcs_preview(missing_wcs_files): - if not missing_wcs_files: - return + keep_mask[np.flatnonzero(usable_mask)] = sigma_clip_pointing_positions( + positions[usable_mask], + sigma=pointing_rejection_sigma, + max_iters=POINTING_REJECTION_MAX_ITERS, + ) - preview = ", ".join(missing_wcs_files[:3]) - remainder = len(missing_wcs_files) - 3 - if remainder > 0: - preview = f"{preview}, ... (+{remainder} more)" - log.debug(f"Files without usable celestial WCS: {preview}") + dropped_files = inputfiles[~keep_mask].tolist() + if not dropped_files: + log_info( + f"Pointing precheck ({mode_label}): no frames exceeded the " + f"{float(pointing_rejection_sigma):g}-sigma pointing threshold." + ) + return inputfiles, keep_mask, [] + + retained_files = inputfiles[keep_mask] + log_info( + f"Pointing precheck ({mode_label}): {len(retained_files)}/{len(inputfiles)} frame(s) remain after " + f"dropping {len(dropped_files)} file(s) beyond {float(pointing_rejection_sigma):g} sigma from the " + "median pointing." + ) + log_file_preview(dropped_files, "Pointing precheck dropped files") + return retained_files, keep_mask, dropped_files def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_missing_fraction=None): @@ -2634,6 +4157,69 @@ def extract_vsx_objects(payload): return [] +def describe_retry_exception(err): + if isinstance(err, RetryError): + last_attempt = getattr(err, 'last_attempt', None) + attempt_number = getattr(last_attempt, 'attempt_number', None) + try: + root_cause = last_attempt.exception() if last_attempt is not None else None + except Exception: + root_cause = None + + if root_cause is not None: + attempt_text = f" after {attempt_number} attempts" if attempt_number else "" + return (f"{err.__class__.__name__}{attempt_text} " + f"({root_cause.__class__.__name__}: {root_cause})") + + return f"{err.__class__.__name__}: {err}" + + +def extract_http_status_code_from_error(err): + if err is None: + return None + + response = getattr(err, 'response', None) + status_code = getattr(response, 'status_code', None) + if status_code is not None: + try: + return int(status_code) + except (TypeError, ValueError): + return None + + status_match = re.search(r"\bHTTP\s+(\d{3})\b", str(err), flags=re.IGNORECASE) + if status_match is None: + return None + + try: + return int(status_match.group(1)) + except (TypeError, ValueError): + return None + + +def should_retry_nextastro_variability_error(err): + if isinstance(err, requests.exceptions.RequestException): + status_code = extract_http_status_code_from_error(err) + return status_code is None or status_code in NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES + + if isinstance(err, RuntimeError): + status_code = extract_http_status_code_from_error(err) + return status_code in NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES + + return False + + +def submit_nextastro_variability_request(api_url, payload, content_encoding=None): + request_body, headers, content_encoding, raw_size, compressed_size = build_compressed_json_request( + payload, + content_encoding=content_encoding, + ) + log_info( + "NextAstro variability request compression: " + f"{content_encoding} ({compressed_size} bytes sent; {raw_size} bytes raw)" + ) + return requests.post(api_url, data=request_body, headers=headers, timeout=30), content_encoding + + @retry(stop=stop_after_delay(30)) def vsx_variable(ra, dec, radius=0.01, maglimit=14): default_vsx_error = None @@ -2677,18 +4263,28 @@ def build_comp_ra_dec(ra_wcs, dec_wcs, comp_stars): return comp_ra_dec -@retry(stop=stop_after_delay(30)) +@retry( + stop=stop_after_attempt(NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS), + wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), + retry=retry_if_exception(should_retry_nextastro_variability_error), +) def nextastro_variability_test(comp_ra_dec): api_url = 'https://photometry.nextastro.org/variability_test' payload = [{'ra': float(ra), 'dec': float(dec)} for ra, dec in comp_ra_dec] - request_body, headers, content_encoding, raw_size, compressed_size = build_compressed_json_request(payload) log_info(f"NextAstro variability request JSON: {json.dumps(payload)}") - log_info( - "NextAstro variability request compression: " - f"{content_encoding} ({compressed_size} bytes sent; {raw_size} bytes raw)" - ) - result = requests.post(api_url, data=request_body, headers=headers, timeout=30) + result, content_encoding = submit_nextastro_variability_request(api_url, payload) + if result.status_code == 415 and content_encoding == 'zstd': + log_info( + "NextAstro variability server rejected zstd-compressed request (HTTP 415); " + "retrying this request once with gzip.", + warn=True, + ) + result, content_encoding = submit_nextastro_variability_request( + api_url, + payload, + content_encoding='gzip', + ) if result.status_code != 200: raise RuntimeError(f"NextAstro variability server returned HTTP {result.status_code}.") @@ -2722,7 +4318,7 @@ def check_for_variable_stars(ra_wcs, dec_wcs, comp_stars, use_nextastro_variabil comp_stars.remove(comp_star) return except Exception as e: - log_info(f"\nWarning: NextAstro variability server check failed ({e}). " + log_info(f"\nWarning: NextAstro variability server check failed ({describe_retry_exception(e)}). " "Falling back to individual VSX variability checks.", warn=True) for i, comp_star in enumerate(comp_stars[:]): @@ -3422,6 +5018,8 @@ def transformation_task_with_cached_reference(i, file_name): MAX_MULTIPROCESS_TRANSFORM_WORKERS = 8 SPARSE_MISSING_WCS_DROP_THRESHOLD = 0.03 +POINTING_REJECTION_MIN_FRAMES = 5 +POINTING_REJECTION_MAX_ITERS = 5 # Automatic aperture-grid tuning constants (in PSF sigma units) APERTURE_SIGMA_MIN = 1.5 @@ -3471,16 +5069,16 @@ def build_multiprocess_transformations(inputfiles, max_processes): return transforms -def log_finding_transformation_progress(i, total_jobs, file_name, use_multiprocess_progress): +def log_alignment_progress(i, total_jobs, file_name, use_multiprocess_progress): if use_multiprocess_progress: completed = i + 1 if completed == total_jobs or completed % 10 == 0: - log_info(f"Multiprocessing finding transformations progress: {completed}/{total_jobs}") + log_info(f"Multiprocessing alignment progress: {completed}/{total_jobs}") return display_file_name = _display_filename(file_name) - sys.stdout.write(f"Finding transformation {i + 1} of {total_jobs} : {display_file_name}\n") - log.debug(f"Finding transformation {i + 1} of {total_jobs} : {display_file_name}\n") + sys.stdout.write(f"Aligning frame {i + 1} of {total_jobs} : {display_file_name}\n") + log.debug(f"Aligning frame {i + 1} of {total_jobs} : {display_file_name}\n") sys.stdout.flush() @@ -3673,18 +5271,118 @@ def fractional_flux_change_within_limit(current_amplitude, previous_amplitude, l return np.abs((current_amplitude - previous_amplitude) / previous_amplitude) <= limit -def centroid_offset_matches_reference(psf_a, psf_b, expected_dx, expected_dy, tolerance=1): - x_values = [psf_a[0], psf_b[0]] - y_values = [psf_a[1], psf_b[1]] - if not np.all(np.isfinite(x_values + y_values)): +def centroid_position_is_finite(psf_row): + try: + coords = np.asarray(psf_row[:2], dtype=float) + except (TypeError, ValueError, IndexError): + return False + + return bool(np.all(np.isfinite(coords))) + + +def centroid_offset_matches_reference(psf_a, psf_b, expected_dx, expected_dy, + tolerance=WCS_REFERENCE_GEOMETRY_TOLERANCE_PIXELS): + if not centroid_position_is_finite(psf_a) or not centroid_position_is_finite(psf_b): return False + if not np.isfinite(expected_dx) or not np.isfinite(expected_dy): + return False + + dx = float(abs(float(psf_a[0]) - float(psf_b[0]))) + dy = float(abs(float(psf_a[1]) - float(psf_b[1]))) + tolerance = float(tolerance) return ( - expected_dx - tolerance <= abs(int(psf_a[0]) - int(psf_b[0])) <= expected_dx + tolerance - and expected_dy - tolerance <= abs(int(psf_a[1]) - int(psf_b[1])) <= expected_dy + tolerance + abs(dx - float(expected_dx)) <= tolerance + and abs(dy - float(expected_dy)) <= tolerance ) +def should_keep_header_wcs_alignment( + projected_off_frame, + frame_index, + target_psf_row, + previous_target_psf_row=None, + comp_psf_rows=None, + previous_comp_psf_rows=None, + expected_offsets=None, + tolerance=WCS_REFERENCE_GEOMETRY_TOLERANCE_PIXELS, + min_geometry_match_fraction=WCS_MIN_GEOMETRY_MATCH_FRACTION, +): + decision = { + 'use_wcs_alignment': False, + 'reason': 'missing_target_centroid', + 'target_flux_change_ok': True, + 'comp_flux_change_ok': True, + 'geometry_match_count': 0, + 'geometry_test_count': 0, + } + + if projected_off_frame: + decision.update(use_wcs_alignment=True, reason='projected_off_frame') + return decision + + if not centroid_position_is_finite(target_psf_row): + return decision + + if frame_index == 0: + decision.update(use_wcs_alignment=True, reason='first_frame') + return decision + + if previous_target_psf_row is not None: + decision['target_flux_change_ok'] = fractional_flux_change_within_limit( + target_psf_row[2], + previous_target_psf_row[2], + ) + + comp_psf_rows = {} if comp_psf_rows is None else dict(comp_psf_rows) + previous_comp_psf_rows = {} if previous_comp_psf_rows is None else dict(previous_comp_psf_rows) + expected_offsets = {} if expected_offsets is None else dict(expected_offsets) + + for key, comp_row in comp_psf_rows.items(): + prev_comp_row = previous_comp_psf_rows.get(key) + if prev_comp_row is not None: + decision['comp_flux_change_ok'] = ( + decision['comp_flux_change_ok'] + and fractional_flux_change_within_limit(comp_row[2], prev_comp_row[2]) + ) + + if not centroid_position_is_finite(comp_row): + continue + + expected_offset = expected_offsets.get(key) + if expected_offset is None: + continue + + try: + expected_dx = float(expected_offset[0]) + expected_dy = float(expected_offset[1]) + except (TypeError, ValueError, IndexError): + continue + + decision['geometry_test_count'] += 1 + if centroid_offset_matches_reference( + comp_row, + target_psf_row, + expected_dx, + expected_dy, + tolerance=tolerance, + ): + decision['geometry_match_count'] += 1 + + geometry_test_count = decision['geometry_test_count'] + if geometry_test_count == 0: + decision.update(use_wcs_alignment=True, reason='finite_target_only') + return decision + + minimum_matches = max(1, int(np.ceil(float(min_geometry_match_fraction) * geometry_test_count))) + if decision['geometry_match_count'] >= minimum_matches: + decision.update(use_wcs_alignment=True, reason='geometry_match') + else: + decision['reason'] = 'geometry_mismatch' + + return decision + + # Method fits a 2D gaussian function that matches the star_psf to the star image and returns its pixel coordinates def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weightedcenter=True, fast_mode=False): stage_start = perf_counter() @@ -4109,6 +5807,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet timeList, airMassList, exptimes, norm_flux = [], [], [], [] ignore_header_wcs = should_ignore_header_wcs(info_dict.get('ignore_header_wcs')) bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction(info_dict.get('bad_wcs_threshold_percent')) + pointing_rejection_sigma = get_pointing_rejection_sigma(info_dict.get('pointing_rejection_sigma')) detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( info_dict.get('detect_bad_pixels_before_photometry', 'y') ) @@ -4137,6 +5836,34 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet ) if dropped_wcs_files: plateStatus.initializeFilenames(list(inputfiles)) + pointing_precheck_inputfiles = np.array(inputfiles, copy=True) + pointing_reference_file = inputfiles[0] if len(inputfiles) else None + inputfiles, _, dropped_pointing_files = filter_pointing_outlier_frames( + inputfiles, + pointing_rejection_sigma=pointing_rejection_sigma, + ignore_header_wcs=ignore_header_wcs, + ) + if dropped_pointing_files: + if abort_if_reference_frame_rejected( + pointing_reference_file, + dropped_pointing_files, + ordered_inputfiles=pointing_precheck_inputfiles, + ): + ax.clear() + ax.set_title(target_name) + ax.set_ylabel('Normalized Flux') + ax.set_xlabel('Time (JD)') + ax.text( + 0.5, + 0.5, + "Reference image rejected by pointing precheck.\nSee log for details.", + transform=ax.transAxes, + ha='center', + va='center', + ) + plt.close(ax.figure) + return + plateStatus.initializeFilenames(list(inputfiles)) bad_pixel_reference = None if detect_bad_pixels_before_photometry: @@ -4239,13 +5966,6 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet if i == 0: firstImage = np.copy(imageData) - log_finding_transformation_progress( - i, - len(inputfiles), - fileName, - use_multiprocess_transform_precompute, - ) - use_wcs_alignment = False if not ignore_header_wcs: try: @@ -4285,35 +6005,29 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet fast_mode=frame_fast_centroid, ) - target_flux_change_ok = True - comp_valid = True - if projected_off_frame: - use_wcs_alignment = True - elif i != 0: - target_flux_change_ok = fractional_flux_change_within_limit( - psf_data['target'][i][2], - psf_data['target'][i - 1][2], - ) - comp_valid = ( - centroid_offset_matches_reference( - psf_data['comp'][i], - psf_data['target'][i], - tar_comp_dist['comp'][0], - tar_comp_dist['comp'][1], - ) - and fractional_flux_change_within_limit( - psf_data['comp'][i][2], - psf_data['comp'][i - 1][2], - ) - ) - use_wcs_alignment = target_flux_change_ok and comp_valid - else: + if i == 0: tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) - use_wcs_alignment = True + wcs_alignment_decision = should_keep_header_wcs_alignment( + projected_off_frame, + i, + psf_data['target'][i], + previous_target_psf_row=None if i == 0 else psf_data['target'][i - 1], + comp_psf_rows={'comp': psf_data['comp'][i]}, + previous_comp_psf_rows={} if i == 0 else {'comp': psf_data['comp'][i - 1]}, + expected_offsets={'comp': tar_comp_dist['comp']}, + ) + use_wcs_alignment = wcs_alignment_decision['use_wcs_alignment'] except Exception: use_wcs_alignment = False + log_alignment_progress( + i, + len(inputfiles), + fileName, + use_multiprocess_transform_precompute, + ) + if not use_wcs_alignment: if i == 0: tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) @@ -4400,7 +6114,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, allow_mid_transit_range_warning=True, disable_vertical_flux_normalization=False, final_fit_mode='lm', use_impactparameter_rather_than_inclination_to_fit=True, - plot_time_range=None): + plot_time_range=None, + use_eebls_to_initialize_tmid_and_bounds=True): # remove outliers plot_time_range = np.asarray(times if plot_time_range is None else plot_time_range, dtype=float) si = np.argsort(times) @@ -4410,9 +6125,16 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, with np.errstate(divide='ignore', invalid='ignore'): flux_ratio_sorted = np.divide(tflux_sorted, cflux_sorted) + filter_diagnostics = [] has_reference_flux = not np.allclose(cflux_sorted, 1.0) if has_reference_flux: flux_ratio_mask = valid_flux_ratio_mask(flux_ratio_sorted) + filter_diagnostics.append(build_time_rejection_diagnostic( + "Target/reference ratio filter", + times_sorted, + flux_ratio_mask, + note="Dropped non-finite or non-positive target/reference ratios before fitting.", + )) times_sorted = times_sorted[flux_ratio_mask] tflux_sorted = tflux_sorted[flux_ratio_mask] cflux_sorted = cflux_sorted[flux_ratio_mask] @@ -4426,12 +6148,24 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, if len(times_sorted) <= 1: return None, None, None + debug_times = np.asarray(times_sorted, dtype=float).copy() + debug_target_flux = np.asarray(tflux_sorted, dtype=float).copy() + debug_comp_flux = np.asarray(cflux_sorted, dtype=float).copy() + debug_raw_ratio = np.asarray(flux_ratio_sorted, dtype=float).copy() + dt = np.mean(np.diff(times_sorted)) ndt = int(25. / 24. / 60. / dt) * 2 + 1 if ndt > len(times_sorted): ndt = int(len(times_sorted)/4) * 2 + 1 - filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt)) + filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt), times=times_sorted) valid_mask = ~filtered_data + debug_initial_sigma_keep_mask = np.asarray(valid_mask, dtype=bool).copy() + filter_diagnostics.append(build_time_rejection_diagnostic( + "Initial sigma clip", + times_sorted, + valid_mask, + note="Dropped 3-sigma target/reference-ratio outliers before the first lightcurve fit.", + )) arrayFinalFlux = flux_ratio_sorted[valid_mask] f1 = tflux_sorted[valid_mask] @@ -4451,6 +6185,12 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, arrayAirmass) | np.less_equal(arrayFinalFlux, 0) | np.less_equal(arrayNormUnc, 0) nanmask = nanmask | np.isinf(arrayFinalFlux) | np.isinf(arrayNormUnc) | np.isinf(arrayTimes) | np.isinf( arrayAirmass) + filter_diagnostics.append(build_time_rejection_diagnostic( + "Finite/positive photometry filter", + arrayTimes, + ~nanmask, + note="Dropped non-finite or non-positive flux, uncertainty, time, or airmass values.", + )) if np.sum(~nanmask) <= 1: return None, None, None @@ -4480,15 +6220,18 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, } arrayPhases = (arrayTimes - pDict['midT']) / prior['per'] - prior['tmid'] = pDict['midT'] + np.floor(arrayPhases).max() * prior['per'] - - upper = prior['tmid'] + np.abs(25 * pDict['midTUnc'] + np.floor(arrayPhases).max() * 25 * pDict['pPerUnc']) - lower = prior['tmid'] - np.abs(25 * pDict['midTUnc'] + np.floor(arrayPhases).max() * 25 * pDict['pPerUnc']) - - if upper > prior['tmid'] + 0.25 * prior['per']: - upper = prior['tmid'] + 0.25 * prior['per'] - if lower < prior['tmid'] - 0.25 * prior['per']: - lower = prior['tmid'] - 0.25 * prior['per'] + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + arrayTimes, + pDict['midT'], + prior['per'], + pDict['midTUnc'], + pDict['pPerUnc'], + expected_duration=expected_duration, + sigma_multiplier=25.0, + ) + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] if ( allow_mid_transit_range_warning @@ -4496,6 +6239,20 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, ): log_mid_transit_range_warning_once(arrayTimes, prior['tmid']) + if tmid_search_summary.get('duration_capped'): + log_info(tmid_search_summary['note']) + if use_eebls_to_initialize_tmid_and_bounds: + tmid_search_summary = estimate_tmid_and_bounds_with_eebls( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + prior, + [lower, upper], + ) + if tmid_search_summary.get('applied'): + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] + mybounds = { 'rprs': [0, prior['rprs'] * 1.25], 'tmid': [lower, upper], @@ -4529,6 +6286,13 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, ) myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) + annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) + if myfit is not None: + myfit.initial_tmid_search_method = tmid_search_summary.get('method') + myfit.initial_tmid_search_applied = bool(tmid_search_summary.get('applied')) + myfit.initial_tmid_search_tmid = tmid_search_summary.get('tmid') + myfit.initial_tmid_search_bounds = tmid_search_summary.get('bounds') + myfit.initial_tmid_search_note = tmid_search_summary.get('note') if ( myfit is not None @@ -4540,6 +6304,12 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, phase_clip_mask = phase_bin_sigma_clip(myfit.residuals, myfit.phase, sigma=3, bins=10) min_required_points = max(len(mybounds) + 1, 5) if np.count_nonzero(~phase_clip_mask) >= min_required_points and np.any(phase_clip_mask): + filter_diagnostics.append(build_time_rejection_diagnostic( + "Phase-binned residual clip", + arrayTimes, + ~phase_clip_mask, + note="Dropped phase-binned residual outliers after the initial LM fit before refitting.", + )) arrayFinalFlux = arrayFinalFlux[~phase_clip_mask] arrayNormUnc = arrayNormUnc[~phase_clip_mask] arrayTimes = arrayTimes[~phase_clip_mask] @@ -4561,20 +6331,63 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, ) myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) - + annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) + if myfit is not None: + myfit.initial_tmid_search_method = tmid_search_summary.get('method') + myfit.initial_tmid_search_applied = bool(tmid_search_summary.get('applied')) + myfit.initial_tmid_search_tmid = tmid_search_summary.get('tmid') + myfit.initial_tmid_search_bounds = tmid_search_summary.get('bounds') + myfit.initial_tmid_search_note = tmid_search_summary.get('note') + + debug_phase_clip_keep_mask = np.ones(np.count_nonzero(debug_initial_sigma_keep_mask), dtype=bool) if final_fit_mode == 'ns' and myfit is not None: - myfit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + nested_refinement = build_nested_tmid_refinement_from_initial_fit( arrayTimes, arrayFinalFlux, arrayNormUnc, - arrayAirmass, prior, mybounds, + myfit, + ) + myfit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + arrayAirmass, + nested_refinement['prior'], + nested_refinement['bounds'], jd_times=arrayJDTimes, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) + annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) + if myfit is not None: + myfit.initial_tmid_search_method = tmid_search_summary.get('method') + myfit.initial_tmid_search_applied = bool(tmid_search_summary.get('applied')) + myfit.initial_tmid_search_tmid = tmid_search_summary.get('tmid') + myfit.initial_tmid_search_bounds = tmid_search_summary.get('bounds') + myfit.initial_tmid_search_note = tmid_search_summary.get('note') + annotate_nested_tmid_refinement( + myfit, + nested_refinement.get('applied', False), + note=nested_refinement.get('note'), + original_tmid_bounds=nested_refinement.get('original_tmid_bounds'), + refined_tmid_bounds=nested_refinement.get('refined_tmid_bounds'), + ) + + if myfit is not None: + if 'phase_clip_mask' in locals(): + debug_phase_clip_keep_mask = np.asarray(~phase_clip_mask, dtype=bool).copy() + annotate_selected_photometry_debug( + myfit, + debug_times, + debug_target_flux, + debug_comp_flux, + debug_raw_ratio, + debug_initial_sigma_keep_mask, + phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, + ) return myfit, f1, f2 @@ -4673,7 +6486,7 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass, enforce_relativ ndt = 5 if ndt > len(times_sorted): ndt = int(len(times_sorted) / 4) * 2 + 1 - filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt)) + filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt), times=times_sorted) valid_mask = ~filtered_data diagnostics['sigma_clip_point_count'] = int(np.count_nonzero(valid_mask)) if diagnostics['sigma_clip_point_count'] <= 1: @@ -4774,6 +6587,7 @@ def evaluate_lightcurve_candidate(task): plot_time_range, disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, ) = task fit_diagnostics = diagnose_lightcurve_fit_inputs( times, @@ -4794,6 +6608,7 @@ def evaluate_lightcurve_candidate(task): disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -4824,6 +6639,7 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, if use_psf_photometry and comp_star_count > 0: target_flux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + target_flux_mask = robust_flux_floor_mask(target_flux) psf_comp_flux_map = { f"comp{comp_idx + 1}": 2 * np.pi * psf_data[f"comp{comp_idx + 1}"][:, 2] * psf_data[f"comp{comp_idx + 1}"][:, 3] @@ -4833,6 +6649,7 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, psf_comp_coverage = comparison_star_coverage_summary( psf_comp_flux_map, skip_rejection=skip_low_comparison_coverage_rejection, + validity_mask_func=robust_flux_floor_mask, ) for comp_idx in range(comp_star_count): ckey = f"comp{comp_idx + 1}" @@ -4840,6 +6657,7 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, continue comp_flux = psf_comp_flux_map[ckey] + psf_mask = target_flux_mask & robust_flux_floor_mask(comp_flux) candidate_jobs.append({ 'method': 'psf', 'a': None, @@ -4848,16 +6666,16 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, 'annulus': float(15 * sigma), 'comp_index': comp_idx, 'ckey': ckey, - 'mask': np.ones(target_flux.shape[0], dtype=bool), + 'mask': psf_mask, 'coverage_count': psf_comp_coverage[ckey]['coverage_count'], 'coverage_total_frame_count': psf_comp_coverage[ckey]['coverage_total_frame_count'], 'coverage_reference_count': psf_comp_coverage[ckey]['coverage_reference_count'], 'coverage_min_required_count': psf_comp_coverage[ckey]['coverage_min_required_count'], 'coverage_rejected': psf_comp_coverage[ckey]['coverage_rejected'], 'prescore': cheap_lightcurve_prescore( - target_flux, - comp_flux, - airmass, + target_flux[psf_mask], + comp_flux[psf_mask], + airmass[psf_mask], enforce_relative_flux_max=False, ), }) @@ -4942,7 +6760,8 @@ def shortlist_target_fit_candidates(candidate_jobs, minimum_count=50, fraction=0 def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, psf_data, aper_data, plot_time_range=None, disable_vertical_flux_normalization=False, - use_impactparameter_rather_than_inclination_to_fit=True): + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True): candidate_mask = np.asarray(candidate['mask'], dtype=bool) if candidate['method'] == 'psf': @@ -4973,6 +6792,7 @@ def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, p plot_time_range, disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, ) @@ -4985,7 +6805,8 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co use_psf_photometry=True, use_aperture_photometry=True, multiprocess_lightcurve_fits=None, - use_impactparameter_rather_than_inclination_to_fit=True): + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True): candidate_jobs = build_target_fit_candidate_jobs( psf_data, aper_data, @@ -5025,6 +6846,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co plot_time_range=plot_time_range, disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, ) for candidate in shortlist ] @@ -5099,6 +6921,47 @@ def format_comp_star_position(position): return f"coords={position}" +def deduplicate_comparison_star_coords(comp_stars, min_separation_pixels=COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS): + if comp_stars is None: + return [], [] + + try: + threshold = float(min_separation_pixels) + except (TypeError, ValueError): + threshold = COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS + if not np.isfinite(threshold) or threshold <= 0: + threshold = COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS + + unique_coords = [] + duplicate_messages = [] + for index, coord in enumerate(comp_stars, start=1): + try: + x_pos, y_pos = float(coord[0]), float(coord[1]) + except (TypeError, ValueError, IndexError): + continue + + duplicate_entry = None + for unique_index, unique_coord in enumerate(unique_coords, start=1): + separation = float(np.hypot(x_pos - unique_coord[0], y_pos - unique_coord[1])) + if separation <= threshold: + duplicate_entry = (unique_index, unique_coord, separation) + break + + if duplicate_entry is not None: + kept_index, kept_coord, separation = duplicate_entry + duplicate_messages.append( + "Merged comparison star " + f"#{index} ({x_pos:.1f}, {y_pos:.1f}) into comparison star " + f"#{kept_index} ({kept_coord[0]:.1f}, {kept_coord[1]:.1f}) " + f"because they were only {separation:.2f} px apart." + ) + continue + + unique_coords.append([x_pos, y_pos]) + + return unique_coords, duplicate_messages + + def format_comp_star_coverage_text(summary): coverage_text = ( f"{summary['coverage_count']} valid frame(s)" @@ -5166,9 +7029,9 @@ def log_comparison_calibration_fit_attempt_summaries(attempts, method_label): residual_text = "n/a" if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)): residual_text = f"{attempt['res_std'] * 100.0:.4f}%" - reason_text = attempt.get( + reason_text = attempt.get('selection_reason') or attempt.get( 'failure_reason', - "selected: first coverage-qualified comparison star with a usable target fit", + "selected: lowest target-fit residual scatter among the evaluated comparison stars", ) log_info( f" {attempt['label']}{selected_label} ({position_text}): " @@ -5375,7 +7238,8 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p plot_time_range=None, disable_vertical_flux_normalization=False, skip_low_comparison_coverage_rejection=False, - use_impactparameter_rather_than_inclination_to_fit=True): + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True): if photometry_info.get('best_fit_lc') is None or not comp_stars: return [] @@ -5404,6 +7268,9 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p coverage_summary = comparison_star_coverage_summary( comp_flux_map, skip_rejection=skip_low_comparison_coverage_rejection, + validity_mask_func=( + robust_flux_floor_mask if use_psf_photometry else valid_comparison_frame_mask + ), ) for comp_index, position in enumerate(comp_stars): @@ -5411,7 +7278,10 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p ckey = f"comp{comp_index + 1}" comp_flux_series = comp_flux_map[ckey] - fit_mask = valid_comparison_frame_mask(comp_flux_series) + if use_psf_photometry: + fit_mask = robust_target_reference_flux_mask(target_flux, comp_flux_series) + else: + fit_mask = valid_comparison_frame_mask(comp_flux_series) coverage_count = coverage_summary[ckey]['coverage_count'] coverage_total_frame_count = coverage_summary[ckey]['coverage_total_frame_count'] coverage_reference_count = coverage_summary[ckey]['coverage_reference_count'] @@ -5457,6 +7327,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p final_fit_mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -5494,10 +7365,10 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p return candidate_fit_summaries -def normalize_flux_series(flux_values): +def normalize_flux_series(flux_values, validity_mask_func=valid_comparison_frame_mask): flux_values = np.asarray(flux_values, dtype=float) normalized = np.full(flux_values.shape, np.nan, dtype=float) - finite_mask = np.isfinite(flux_values) & (flux_values > 0) + finite_mask = validity_mask_func(flux_values) if np.count_nonzero(finite_mask) < 5: return normalized @@ -5538,13 +7409,14 @@ def build_normalized_comp_ensemble(normalized_flux_map, exclude_key): def comparison_star_coverage_summary(comp_flux_map, min_fraction=COMPARISON_STAR_MIN_COVERAGE_FRACTION, min_points=COMPARISON_STAR_MIN_VALID_FRAMES, - skip_rejection=False): + skip_rejection=False, + validity_mask_func=valid_comparison_frame_mask): comp_keys = list(comp_flux_map.keys()) if not comp_keys: return {} coverage_counts = { - key: int(np.count_nonzero(valid_comparison_frame_mask(comp_flux_map[key]))) + key: int(np.count_nonzero(validity_mask_func(comp_flux_map[key]))) for key in comp_keys } total_frame_count = max(np.asarray(comp_flux_map[key]).shape[0] for key in comp_keys) @@ -5592,7 +7464,8 @@ def comparison_star_coverage_summary(comp_flux_map, return coverage_summary -def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_rejection=False): +def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_rejection=False, + validity_mask_func=valid_comparison_frame_mask): if not comp_flux_map: return { 'pairwise_matrix': np.empty((0, 0), dtype=float), @@ -5603,10 +7476,14 @@ def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_ } comp_keys = list(comp_flux_map.keys()) - normalized_flux_map = {key: normalize_flux_series(comp_flux_map[key]) for key in comp_keys} + normalized_flux_map = { + key: normalize_flux_series(comp_flux_map[key], validity_mask_func=validity_mask_func) + for key in comp_keys + } coverage_summary = comparison_star_coverage_summary( comp_flux_map, skip_rejection=skip_low_coverage_rejection, + validity_mask_func=validity_mask_func, ) eligible_keys = { key for key in comp_keys @@ -5905,6 +7782,7 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, psf_flux_map, airmass, skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, + validity_mask_func=robust_flux_floor_mask, ) psf_summary.update({ 'method': 'psf', @@ -5989,7 +7867,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p psf_data, aper_data, target_psf_flux, plot_time_range=None, disable_vertical_flux_normalization=False, - use_impactparameter_rather_than_inclination_to_fit=True): + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True): ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) if not ranked_summaries: return { @@ -6007,7 +7886,6 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p target_flux = aper_data['target'][:, aperture_index, annulus_index] attempts = [] - selected_result = None for rank, comp_summary in enumerate(ranked_summaries): comp_index = comp_summary['comp_index'] ckey = comp_summary.get('key', f"comp{comp_index + 1}") @@ -6020,25 +7898,30 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p else: comp_flux = aper_data[ckey][:, aperture_index, annulus_index] + fit_mask = np.ones(times.shape[0], dtype=bool) + if method == 'psf': + fit_mask = robust_target_reference_flux_mask(target_flux, comp_flux) + fit_diagnostics = diagnose_lightcurve_fit_inputs( - times, - target_flux, - comp_flux, - airmass, + times[fit_mask], + target_flux[fit_mask], + comp_flux[fit_mask], + airmass[fit_mask], enforce_relative_flux_max=False, ) fit_result, tflux_fit, cflux_fit = fit_lightcurve( - times, - target_flux, - comp_flux, - airmass, + times[fit_mask], + target_flux[fit_mask], + comp_flux[fit_mask], + airmass[fit_mask], ld, p_dict, - jd_times, + jd_times[fit_mask], disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -6073,15 +7956,37 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'failure_reason': fit_diagnostics.get('failure_reason'), 'parameter_summary': summarize_lightcurve_fit_parameters(fit_result), 'selected': False, + 'selection_reason': None, } attempts.append(attempt) - if fit_result is not None: - selected_result = attempt - break - - if selected_result is not None: + selected_result = None + successful_attempts = [ + attempt + for attempt in attempts + if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)) + ] + if successful_attempts: + selected_result = min( + successful_attempts, + key=lambda attempt: (attempt.get('res_std', np.inf), attempt.get('rank', np.inf)), + ) selected_result['selected'] = True + selected_residual = selected_result.get('res_std', np.inf) + + for attempt in attempts: + if attempt is selected_result: + attempt['selection_reason'] = ( + "selected: lowest target-fit residual scatter among the evaluated " + "comparison-star calibration candidates" + ) + continue + if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)): + attempt['selection_reason'] = ( + "not selected: target-fit residual scatter " + f"{attempt['res_std'] * 100.0:.4f}% was higher than the selected " + f"{selected_residual * 100.0:.4f}%" + ) return { 'ranked_summaries': ranked_summaries, @@ -6292,6 +8197,9 @@ def main(): FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, ) ) + use_eebls_tmid_initializer = should_use_eebls_to_initialize_tmid_and_bounds( + exotic_infoDict.get('use_eebls_to_initialize_tmid_and_bounds', 'y') + ) use_impactparameter_rather_than_inclination_to_fit = ( should_use_impactparameter_rather_than_inclination_to_fit( exotic_infoDict.get('use_impactparameter_rather_than_inclination_to_fit', 'y') @@ -6430,6 +8338,9 @@ def main(): bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction( exotic_infoDict.get('bad_wcs_threshold_percent') ) + pointing_rejection_sigma = get_pointing_rejection_sigma( + exotic_infoDict.get('pointing_rejection_sigma') + ) detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( exotic_infoDict.get('detect_bad_pixels_before_photometry', 'y') ) @@ -6442,6 +8353,36 @@ def main(): times = times[wcs_keep_mask] jd_times = jd_times[wcs_keep_mask] plateStatus.initializeFilenames(list(inputfiles)) + pointing_precheck_inputfiles = np.array(inputfiles, copy=True) + pointing_reference_file = inputfiles[0] if len(inputfiles) else None + inputfiles, pointing_keep_mask, dropped_pointing_files = filter_pointing_outlier_frames( + inputfiles, + pointing_rejection_sigma=pointing_rejection_sigma, + ignore_header_wcs=ignore_header_wcs, + frame_loader=lambda file_name: load_calibrated_reduction_image( + file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ), + ) + if dropped_pointing_files: + if abort_if_reference_frame_rejected( + pointing_reference_file, + dropped_pointing_files, + ordered_inputfiles=pointing_precheck_inputfiles, + ): + return + times = times[pointing_keep_mask] + jd_times = jd_times[pointing_keep_mask] + finite_plot_times = times[np.isfinite(times)] + full_plot_time_range = None + if finite_plot_times.size: + full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) + plateStatus.initializeFilenames(list(inputfiles)) bad_pixel_reference = None if detect_bad_pixels_before_photometry: @@ -6552,9 +8493,23 @@ def main(): exotic_infoDict['comp_stars'] = comparison_star_coords(exotic_infoDict['comp_stars'], False) check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], use_nextastro_variability_server=args.use_nextastro_variability_server) + + exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( + exotic_infoDict['comp_stars'] + ) + for duplicate_message in duplicate_comp_messages: + log_info(duplicate_message) + # Build RA/Dec for comp after list is finalized (avoid off by one issues, etc ra_dec_wcs = build_comp_ra_dec(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) + else: + exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( + exotic_infoDict['comp_stars'] + ) + for duplicate_message in duplicate_comp_messages: + log_info(duplicate_message) + plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) # alloc psf fitting param psf_data = { @@ -6592,6 +8547,8 @@ def main(): log_info("PSF photometry disabled per optional_info setting.") if not use_aperture_photometry: log_info("Aperture photometry disabled per optional_info setting.") + if not use_eebls_tmid_initializer: + log_info("EEBLS transit initializer disabled per optional_info setting.") for i, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{i + 1}" @@ -6701,13 +8658,6 @@ def main(): if i == 0: firstImage = np.copy(imageData) - log_finding_transformation_progress( - i, - len(inputfiles), - fileName, - use_multiprocess_transform_precompute, - ) - use_wcs_alignment = False if not ignore_header_wcs: try: @@ -6747,14 +8697,8 @@ def main(): # TODO: Add check for flux on target/comp stars relative to others in the field # in case of cloudy data, large changes, etc. - target_flux_change_ok = True - if not projected_off_frame and i != 0: - target_flux_change_ok = fractional_flux_change_within_limit( - psf_data['target'][i][2], - psf_data['target'][i - 1][2], - ) - - comp_valid = True + current_comp_psf_rows = {} + previous_comp_psf_rows = {} for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" @@ -6766,29 +8710,33 @@ def main(): fast_mode=frame_fast_centroid, ) - if projected_off_frame: - continue + current_comp_psf_rows[ckey] = psf_data[ckey][i] if i != 0: - comp_valid = comp_valid and ( - centroid_offset_matches_reference( - psf_data[ckey][i], - psf_data['target'][i], - tar_comp_dist[ckey][0], - tar_comp_dist[ckey][1], - ) - and fractional_flux_change_within_limit( - psf_data[ckey][i][2], - psf_data[ckey][i - 1][2], - ) - ) + previous_comp_psf_rows[ckey] = psf_data[ckey][i - 1] else: tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) - use_wcs_alignment = projected_off_frame or (target_flux_change_ok and comp_valid) + wcs_alignment_decision = should_keep_header_wcs_alignment( + projected_off_frame, + i, + psf_data['target'][i], + previous_target_psf_row=None if i == 0 else psf_data['target'][i - 1], + comp_psf_rows=current_comp_psf_rows, + previous_comp_psf_rows=previous_comp_psf_rows, + expected_offsets=tar_comp_dist, + ) + use_wcs_alignment = wcs_alignment_decision['use_wcs_alignment'] except Exception: use_wcs_alignment = False + log_alignment_progress( + i, + len(inputfiles), + fileName, + use_multiprocess_transform_precompute, + ) + if not use_wcs_alignment: if i == 0: tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) @@ -6954,6 +8902,17 @@ def main(): if aper_data is not None: badmask = badmask | (aper_data["target"][:, 0, 0] == 0) | np.isnan(aper_data["target"][:, 0, 0]) goodmask = ~badmask + global_frame_filter_diagnostic = build_time_rejection_diagnostic( + "Target centroid/aperture validity filter", + times, + goodmask, + note="Dropped frames before photometry selection because the target centroid or target aperture photometry was invalid.", + ) + if global_frame_filter_diagnostic is not None and global_frame_filter_diagnostic['dropped_point_count'] > 0: + log_lightcurve_filter_diagnostics( + [global_frame_filter_diagnostic], + header="Global reduction frame rejections before photometry selection", + ) if np.sum(goodmask) == 0: log_info("No images to fit...check reference image for alignment (first image of sequence)") @@ -7123,6 +9082,7 @@ def main(): disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, ) comparison_calibration['ranked_fit_comp_indices'] = [ summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] @@ -7132,7 +9092,11 @@ def main(): selected_attempt = comparison_fit_search['selected_result'] fit_attempts = comparison_fit_search['attempts'] if fit_attempts: - comparison_calibration['selected_fit_diagnostics'] = fit_attempts[-1]['fit_diagnostics'] + comparison_calibration['selected_fit_diagnostics'] = ( + selected_attempt['fit_diagnostics'] + if selected_attempt is not None + else fit_attempts[-1]['fit_diagnostics'] + ) if selected_attempt is None or len(fit_attempts) > 1: log_comparison_calibration_fit_attempt_summaries( fit_attempts, @@ -7159,9 +9123,10 @@ def main(): if selection_basis == 'comparison_field_retry': retry_count = selected_attempt['rank'] log_info( - "Comparison-star calibration retry selected " + "Comparison-star calibration target-fit selection chose " f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " - f"after {retry_count} better-ranked candidate(s) failed target fitting." + f"after evaluating {retry_count} better-ranked field-stability candidate(s); " + "it delivered the lowest target-fit residual scatter among successful fits." ) photometry_info.update(best_fit_lc=myfit, @@ -7198,6 +9163,7 @@ def main(): use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, plot_time_range=full_plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -7218,6 +9184,7 @@ def main(): use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, plot_time_range=full_plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -7278,6 +9245,7 @@ def main(): multiprocess_lightcurve_fits=args.multiprocess_lightcurve_fits, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, ) best_candidate = target_driven_search['best_candidate'] @@ -7347,6 +9315,7 @@ def main(): use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, plot_time_range=full_plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -7367,6 +9336,7 @@ def main(): use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, plot_time_range=full_plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -7438,6 +9408,27 @@ def main(): best_fit_lc = photometry_info['best_fit_lc'] bestCompStar = photometry_info['comp_star_num'] comp_coords = photometry_info['comp_star_coords'] + log_lightcurve_filter_diagnostics( + getattr(best_fit_lc, 'frame_filter_diagnostics', []), + header="Selected lightcurve frame rejections during target fitting", + ) + try: + selected_photometry_debug_path = save_selected_photometry_debug_series( + exotic_infoDict['save'], + pDict['pName'], + exotic_infoDict['date'], + best_fit_lc, + ) + if selected_photometry_debug_path is not None: + log_info( + f"Saved selected raw target/reference ratio diagnostics to " + f"{selected_photometry_debug_path}." + ) + except Exception as e: + log_info( + f"Warning: Could not save selected raw target/reference ratio diagnostics ({e}).", + warn=True, + ) if fit_every_comparison_candidate and exotic_infoDict['comp_stars']: candidate_fit_summaries = fit_lightcurve_to_every_comparison_candidate( @@ -7455,6 +9446,7 @@ def main(): skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, ) saved_candidate_fit_count = sum(1 for summary in candidate_fit_summaries if summary['fit'] is not None) failed_candidate_fit_count = len(candidate_fit_summaries) - saved_candidate_fit_count @@ -7488,7 +9480,7 @@ def main(): si = np.argsort(best_fit_lc.time) dt = np.mean(np.diff(np.sort(best_fit_lc.time))) ndt = int(30. / 24. / 60. / dt) * 2 + 1 # ~30 minutes - time_clip_mask = sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt) + time_clip_mask = sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt, times=best_fit_lc.time[si]) phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) if hasattr(best_fit_lc, 'residuals') and hasattr(best_fit_lc, 'phase'): phase_clip_mask = phase_bin_sigma_clip(best_fit_lc.residuals[si], best_fit_lc.phase[si], sigma=3, bins=10) @@ -7500,6 +9492,26 @@ def main(): sorted_annuli[~time_clip_mask & ~phase_clip_mask]) adaptive_clip_mask[~time_clip_mask & ~phase_clip_mask] = retained_mask gi = ~(time_clip_mask | phase_clip_mask | adaptive_clip_mask) # good indexs + prefinal_filter_diagnostics = [ + build_time_rejection_diagnostic( + "Final-fit time sigma clip", + np.asarray(best_fit_lc.time, dtype=float)[si], + ~time_clip_mask, + note="Dropped time-series outliers before the final fit.", + ), + build_time_rejection_diagnostic( + "Final-fit phase residual clip", + np.asarray(best_fit_lc.time, dtype=float)[si], + ~phase_clip_mask, + note="Dropped phase-binned residual outliers before the final fit.", + ), + build_time_rejection_diagnostic( + "Final-fit adaptive-aperture clip", + np.asarray(best_fit_lc.time, dtype=float)[si], + ~adaptive_clip_mask, + note="Dropped adaptive-aperture radius outliers before the final fit.", + ), + ] phase_clip_removed = np.count_nonzero(phase_clip_mask & ~time_clip_mask) if phase_clip_removed: log_info(f"Removed {phase_clip_removed} phase-binned residual outlier(s) before final fit.") @@ -7507,20 +9519,6 @@ def main(): if adaptive_clip_removed: log_info(f"Removed {adaptive_clip_removed} adaptive-aperture radius outlier(s) before final fit.") - # Calculate the proper timeseries uncertainties from the residuals of the out-of-transit data - OOT = (best_fit_lc.transit == 1) # find out-of-transit portion of the lightcurve - - if sum(OOT) <= 1: - OOTscatter = np.std(best_fit_lc.residuals) - goodNormUnc = OOTscatter * best_fit_lc.airmass_model - goodNormUnc = goodNormUnc / np.nanmedian(best_fit_lc.data) - goodFluxes = best_fit_lc.data / np.nanmedian(best_fit_lc.data) - else: - OOTscatter = np.std((best_fit_lc.data / best_fit_lc.airmass_model)[OOT]) # calculate the scatter in the data - goodNormUnc = OOTscatter * best_fit_lc.airmass_model # scale this scatter back up by the airmass model and then adopt these as the uncertainties - goodNormUnc = goodNormUnc / np.nanmedian(best_fit_lc.data[OOT]) - goodFluxes = best_fit_lc.data / np.nanmedian(best_fit_lc.data[OOT]) - if np.isnan(best_fit_lc.data).all(): log_info("Error: No valid photometry data found.", error=True) return @@ -7528,10 +9526,22 @@ def main(): apply_lightcurve_mask(best_fit_lc, gi, sort_index=si) goodTimes = best_fit_lc.time - goodFluxes = goodFluxes[si][gi] - goodNormUnc = goodNormUnc[si][gi] goodAirmasses = best_fit_lc.airmass + final_fit_series = prepare_final_fit_lightcurve_series(best_fit_lc) + if not final_fit_series.get('applied'): + log_info( + f"Warning: {final_fit_series.get('note', 'could not prepare the final-fit light curve from the provisional fit.')} " + "Falling back to the provisional detrended light curve arrays.", + warn=True, + ) + goodFluxes = np.asarray(best_fit_lc.detrended, dtype=float) + goodNormUnc = np.asarray(best_fit_lc.detrendederr, dtype=float) + else: + log_info(final_fit_series['note']) + goodFluxes = np.asarray(final_fit_series['flux'], dtype=float) + goodNormUnc = np.asarray(final_fit_series['unc'], dtype=float) + centroid_positions.update(x_targ=centroid_positions['x_targ'][si][gi], y_targ=centroid_positions['y_targ'][si][gi], x_ref=centroid_positions['x_ref'][si][gi], @@ -7543,10 +9553,20 @@ def main(): flux_unc_ref=flux_values['flux_unc_ref'][si][gi]) relative_flux_mask = relative_flux_filter_mask(goodFluxes) + prefinal_filter_diagnostics.append(build_time_rejection_diagnostic( + "Final relative-flux cap", + goodTimes, + relative_flux_mask, + note=f"Dropped points with normalized flux outside (0, {RELATIVE_FLUX_MAX:g}] before the final fit.", + )) if np.count_nonzero(relative_flux_mask) == 0: log_info("Error: No valid photometry data found after removing relative flux values above 2.", error=True) return + log_lightcurve_filter_diagnostics( + prefinal_filter_diagnostics, + header="Selected lightcurve frame rejections before the final fit", + ) apply_lightcurve_mask(best_fit_lc, relative_flux_mask) goodTimes = goodTimes[relative_flux_mask] @@ -7688,6 +9708,10 @@ def main(): goodFluxes = goodFluxes[relative_flux_mask] goodNormUnc = goodNormUnc[relative_flux_mask] goodAirmasses = goodAirmasses[relative_flux_mask] + finite_plot_times = goodTimes[np.isfinite(goodTimes)] + full_plot_time_range = None + if finite_plot_times.size: + full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) # for k in myfit.bounds.keys(): # print(f"{myfit.parameters[k]:.6f} +- {myfit.errors[k]}") @@ -7718,15 +9742,18 @@ def main(): } phase = (goodTimes - prior['tmid']) / prior['per'] - prior['tmid'] = pDict['midT'] + np.floor(phase).max() * prior['per'] - upper = pDict['midT'] + 35 * pDict['midTUnc'] + np.floor(phase).max() * (pDict['pPer'] + 35 * pDict['pPerUnc']) - lower = pDict['midT'] - 35 * pDict['midTUnc'] + np.floor(phase).max() * (pDict['pPer'] - 35 * pDict['pPerUnc']) - - # clip bounds so they're within 1 orbit - if upper > prior['tmid'] + 0.25*prior['per']: - upper = prior['tmid'] + 0.25*prior['per'] - if lower < prior['tmid'] - 0.25*prior['per']: - lower = prior['tmid'] - 0.25*prior['per'] + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + goodTimes, + pDict['midT'], + prior['per'], + pDict['midTUnc'], + pDict['pPerUnc'], + expected_duration=expected_duration, + sigma_multiplier=35.0, + ) + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] if np.floor(phase).max() - np.floor(phase).min() == 0: log_info("Error: Estimated mid-transit not in observation range (check priors or observation time)", error=True) @@ -7734,6 +9761,21 @@ def main(): log_info(f" end:{np.max(goodTimes)}", error=True) log_info(f"prior:{prior['tmid']}", error=True) + if tmid_search_summary.get('duration_capped'): + log_info(tmid_search_summary['note']) + if use_eebls_tmid_initializer: + tmid_search_summary = estimate_tmid_and_bounds_with_eebls( + goodTimes, + goodFluxes, + goodNormUnc, + prior, + [lower, upper], + ) + log_info(tmid_search_summary['note']) + if tmid_search_summary.get('applied'): + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] + final_airmass_span = airmass_span(goodAirmasses) airmass_skip_note = None skip_final_airmass_fit = bool(exotic_infoDict.get('airmass_already_corrected')) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index c1bd54ba..7a558cfd 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -421,6 +421,7 @@ def save_input(): "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", + "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -446,6 +447,7 @@ def save_input(): "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1497,6 +1499,7 @@ def save_input(): "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", + "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -1570,6 +1573,7 @@ def save_input(): "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1622,6 +1626,7 @@ def save_input(): "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/exotic/inputs.py b/exotic/inputs.py index de805bfe..23da40ee 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -27,7 +27,6 @@ log = logging.getLogger(__name__) -logging.basicConfig(filename='exotic.log', level=logging.DEBUG) consoleFormatter = logging.Formatter("%(message)s") consoleHandler = logging.StreamHandler(sys.stdout) consoleHandler.setFormatter(consoleFormatter) @@ -211,10 +210,12 @@ def __init__(self, init_opt): 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'detrend_on_outoftransit_baseline': True, 'final_fit_baseline_duration_multiplier': 1.0, + 'use_eebls_to_initialize_tmid_and_bounds': 'y', 'detect_bad_pixels_before_photometry': 'y', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, + 'pointing_rejection_sigma': 4.0, 'skip_low_comparison_coverage_rejection': 'n', 'fit_lightcurve_to_every_comparison_candidate': 'n', } @@ -438,6 +439,11 @@ def comp_params(self, init_file, planet_dict): 'final_fit_baseline_duration_multiplier', 'Final Fit Baseline Duration Multiplier', ), + 'use_eebls_to_initialize_tmid_and_bounds': ( + 'use_eebls_to_initialize_tmid_and_bounds', + 'Use EEBLS to Initialize Tmid and Bounds? (y/n)', + 'Use EEBLS To Initialize Tmid And Bounds? (y/n)', + ), 'use_impactparameter_rather_than_inclination_to_fit': ( 'use_impactparameter_rather_than_inclination_to_fit', 'Use impact parameter rather than inclination to fit? (y/n)', @@ -468,6 +474,10 @@ def comp_params(self, init_file, planet_dict): 'bad_wcs_threshold_percent', 'Bad WCS Threshold Percent', ), + 'pointing_rejection_sigma': ( + 'pointing_rejection_sigma', + 'Pointing Rejection Sigma', + ), 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (arsec/pixel)'), 'exposure': 'Exposure Time (s)', diff --git a/exotic/utils.py b/exotic/utils.py index cbc7df76..ca266933 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -293,7 +293,7 @@ def process_lat_long(val, key): v = deg + (((60 * min) + sec) / 3600) return add_sign(v) - m = re.search("^\'?([+-]?\d+\.\d+)", val) + m = re.search(r"^'?([+-]?\d+\.\d+)", val) if m: v = float(m.group(1)) diff --git a/inits.json b/inits.json index aca029bf..f346ada2 100644 --- a/inits.json +++ b/inits.json @@ -25,10 +25,12 @@ "Fast Aperture Mask": "Default true/fast mode for quicker aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to false to opt out and use exact masks.", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", + "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Set to 0 to disable. Default 4.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", + "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", @@ -104,10 +106,12 @@ "Fast Aperture Mask (y/n)": true, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, + "pointing_rejection_sigma": 4.0, "disable vertical flux normalization": false, "detect_bad_pixels_before_photometry": "y", "detrend_on_outoftransit_baseline": true, "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_psf_photometry": "y", "use_aperture_photometry": "y", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 6c515c9c..669cf569 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -336,6 +336,22 @@ def test_get_bad_wcs_threshold_fraction_falls_back_for_invalid_values(): assert exotic_module.get_bad_wcs_threshold_fraction(101) == pytest.approx(0.03) +def test_get_pointing_rejection_sigma_defaults_to_four(): + assert exotic_module.get_pointing_rejection_sigma(None) == pytest.approx(4.0) + assert exotic_module.get_pointing_rejection_sigma("") == pytest.approx(4.0) + + +def test_get_pointing_rejection_sigma_reads_positive_numeric_values(): + assert exotic_module.get_pointing_rejection_sigma(3) == pytest.approx(3.0) + assert exotic_module.get_pointing_rejection_sigma("2.75") == pytest.approx(2.75) + + +def test_get_pointing_rejection_sigma_uses_default_for_invalid_text_and_zero_disables(): + assert exotic_module.get_pointing_rejection_sigma("not-a-number") == pytest.approx(4.0) + assert exotic_module.get_pointing_rejection_sigma(-1) == pytest.approx(4.0) + assert exotic_module.get_pointing_rejection_sigma(0) is None + + def test_display_filename_returns_basename_for_unix_and_windows_paths(): assert ( exotic_module._display_filename( @@ -357,22 +373,52 @@ def test_format_plate_solution_reference_uses_basename_only(): ) -def test_log_finding_transformation_progress_prints_basename(monkeypatch): +def test_log_alignment_progress_prints_basename(monkeypatch): stdout = io.StringIO() debug_messages = [] monkeypatch.setattr(exotic_module.sys, "stdout", stdout) monkeypatch.setattr(exotic_module.log, "debug", lambda message: debug_messages.append(message)) - exotic_module.log_finding_transformation_progress( + exotic_module.log_alignment_progress( 144, 220, "/content/drive/MyDrive/0.Exoplanets/2.Transits/run/frame_145.fits.fz", False, ) - assert stdout.getvalue() == "Finding transformation 145 of 220 : frame_145.fits.fz\n" - assert debug_messages == ["Finding transformation 145 of 220 : frame_145.fits.fz\n"] + assert stdout.getvalue() == "Aligning frame 145 of 220 : frame_145.fits.fz\n" + assert debug_messages == ["Aligning frame 145 of 220 : frame_145.fits.fz\n"] + + +def test_collect_transform_frame_pointings_logs_alignment_progress(monkeypatch): + progress_messages = [] + + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, warn=False, error=False: progress_messages.append((message, warn, error)), + ) + monkeypatch.setattr( + exotic_module, + "transformation", + lambda image_data, file_name, report_failure=False, reference_image=None: ( + lambda anchor: np.asarray(anchor, dtype=float) + ), + ) + + frames = ["frame_0001.fits", "frame_0002.fits", "frame_0003.fits"] + frame_loader = lambda file_name: np.ones((8, 8), dtype=float) + + positions, usable_mask = exotic_module.collect_transform_frame_pointings(frames, frame_loader=frame_loader) + + assert positions.shape == (3, 2) + assert usable_mask.tolist() == [True, True, True] + assert [message for message, _, _ in progress_messages] == [ + "Pointing precheck alignment progress: file 1 of 3 : frame_0001.fits", + "Pointing precheck alignment progress: file 2 of 3 : frame_0002.fits", + "Pointing precheck alignment progress: file 3 of 3 : frame_0003.fits", + ] def test_check_wcs_ignores_header_wcs_when_override_enabled(monkeypatch): @@ -484,3 +530,160 @@ def test_filter_sparse_missing_wcs_frames_keeps_files_at_three_percent_or_higher assert filtered.tolist() == frames assert keep_mask.tolist() == [True] * len(frames) assert dropped == [] + + +def test_filter_pointing_outlier_frames_uses_wcs_when_all_frames_have_wcs(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(6)] + wcs_positions = np.array( + [ + [100.0, 100.0], + [101.0, 100.0], + [99.0, 100.0], + [100.0, 101.0], + [100.0, 99.0], + [0.0, 0.0], + ], + dtype=float, + ) + + monkeypatch.setattr( + exotic_module, + "collect_wcs_frame_center_pointings", + lambda inputfiles: (wcs_positions, np.ones(len(inputfiles), dtype=bool)), + ) + monkeypatch.setattr( + exotic_module, + "collect_transform_frame_pointings", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("transform fallback should not be used")), + ) + + filtered, keep_mask, dropped = exotic_module.filter_pointing_outlier_frames( + frames, + pointing_rejection_sigma=3.0, + ) + + assert filtered.tolist() == frames[:-1] + assert keep_mask.tolist() == [True, True, True, True, True, False] + assert dropped == [frames[-1]] + + +def test_filter_pointing_outlier_frames_falls_back_to_transform_when_wcs_is_incomplete(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(6)] + transform_positions = np.array( + [ + [50.0, 50.0], + [50.5, 49.5], + [49.5, 50.5], + [50.0, 51.0], + [50.0, 49.0], + [10.0, 10.0], + ], + dtype=float, + ) + transform_calls = [] + + monkeypatch.setattr( + exotic_module, + "collect_wcs_frame_center_pointings", + lambda inputfiles: ( + np.full((len(inputfiles), 2), np.nan, dtype=float), + np.array([True, True, True, True, False, False], dtype=bool), + ), + ) + + def fake_collect_transform_frame_pointings(inputfiles, frame_loader=None): + transform_calls.append((tuple(inputfiles), frame_loader)) + return transform_positions, np.ones(len(inputfiles), dtype=bool) + + monkeypatch.setattr(exotic_module, "collect_transform_frame_pointings", fake_collect_transform_frame_pointings) + + filtered, keep_mask, dropped = exotic_module.filter_pointing_outlier_frames( + frames, + pointing_rejection_sigma=3.0, + ) + + assert len(transform_calls) == 1 + assert filtered.tolist() == frames[:-1] + assert keep_mask.tolist() == [True, True, True, True, True, False] + assert dropped == [frames[-1]] + + +def test_abort_if_reference_frame_rejected_reports_error_and_removal_recommendation(monkeypatch): + messages = [] + + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, error=False, warn=False: messages.append((message, error, warn)), + ) + + result = exotic_module.abort_if_reference_frame_rejected( + "frame_0001.fits", + ["frame_0001.fits", "frame_0002.fits", "frame_0003.fits"], + ordered_inputfiles=[ + "frame_0001.fits", + "frame_0002.fits", + "frame_0003.fits", + "frame_0004.fits", + ], + ) + + assert result is True + assert any("first usable image" in message and error for message, error, _ in messages) + assert any("frame_0002.fits" in message and error for message, error, _ in messages) + assert any( + "remove or move these leading rejected frames" in message + and "frame_0001.fits, frame_0002.fits, frame_0003.fits" in message + and "frame_0004.fits" in message + and error + for message, error, _ in messages + ) + + +def test_abort_if_reference_frame_rejected_only_recommends_consecutive_leading_rejections(monkeypatch): + messages = [] + + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, error=False, warn=False: messages.append((message, error, warn)), + ) + + result = exotic_module.abort_if_reference_frame_rejected( + "frame_0001.fits", + ["frame_0001.fits", "frame_0003.fits"], + ordered_inputfiles=[ + "frame_0001.fits", + "frame_0002.fits", + "frame_0003.fits", + "frame_0004.fits", + ], + ) + + assert result is True + assert any( + "remove or move this rejected frame" in message + and "frame_0001.fits" in message + and "frame_0002.fits" in message + and "frame_0003.fits" not in message + and error + for message, error, _ in messages + ) + + +def test_abort_if_reference_frame_rejected_ignores_non_reference_rejections(monkeypatch): + messages = [] + + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, error=False, warn=False: messages.append((message, error, warn)), + ) + + result = exotic_module.abort_if_reference_frame_rejected( + "frame_0001.fits", + ["frame_0002.fits", "frame_0003.fits"], + ) + + assert result is False + assert messages == [] diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index c1af9160..0e9310fc 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -84,11 +84,15 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: from exotic.exotic import ( adaptive_aperture_outlier_mask, auto_tune_aperture_sigma_grid, + build_target_fit_candidate_jobs, + build_time_rejection_diagnostic, check_coordinates, cheap_lightcurve_prescore, + centroid_offset_matches_reference, comparison_candidate_fit_selection_reason, comparison_star_coverage_summary, comparison_star_stability_summary, + deduplicate_comparison_star_coords, diagnose_lightcurve_fit_inputs, detrend_flux_on_out_of_transit_baseline, ensure_lightcurve_fit_failure_reason, @@ -97,6 +101,8 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: fit_lightcurve_to_every_comparison_candidate, fit_ranked_comparison_calibration_candidates, get_final_fit_baseline_duration_multiplier, + estimate_ephemeris_tmid_and_bounds, + estimate_tmid_and_bounds_with_eebls, is_adaptive_aperture_mode_enabled, is_comp_star_required, is_out_of_transit_baseline_detrending_enabled, @@ -105,11 +111,19 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: log_comparison_candidate_fit_summaries, log_target_fit_candidate_summaries, phase_bin_sigma_clip, + prepare_final_fit_lightcurve_series, representative_psf_sigma, run_target_driven_photometry_search, resolve_frame_aperture_radii, + robust_flux_floor_mask, + robust_target_reference_flux_mask, + save_selected_photometry_debug_series, + should_keep_header_wcs_alignment, + sigma_clip, summarize_adaptive_aperture_usage, + summarize_prior_transit_coverage, should_skip_airmass_fit, + should_use_eebls_to_initialize_tmid_and_bounds, should_fit_lightcurve_to_every_comparison_candidate, should_detect_bad_pixels_before_photometry, should_use_aperture_photometry, @@ -150,6 +164,277 @@ def test_update_coordinates_accepts_numeric_strings(): assert isinstance(updated_dec, float) +def test_prepare_final_fit_lightcurve_series_uses_two_sided_modeled_oot(): + times = np.linspace(-0.04, 0.04, 9) + detrended = np.array([1.0, 1.0, 1.0, 0.99, 0.98, 0.99, 1.0, 1.0, 1.0], dtype=float) + fit = types.SimpleNamespace( + time=times, + data=detrended.copy(), + dataerr=np.full(times.shape, 0.01, dtype=float), + detrended=detrended.copy(), + detrendederr=np.full(times.shape, 0.01, dtype=float), + airmass_model=np.ones(times.shape, dtype=float), + transit=np.array([1.0, 1.0, 1.0, 0.99, 0.98, 0.99, 1.0, 1.0, 1.0], dtype=float), + parameters={"tmid": 0.0}, + ) + + prepared = prepare_final_fit_lightcurve_series(fit) + + assert prepared["applied"] is True + assert prepared["used_two_sided_oot"] is True + assert "modeled out-of-transit" in prepared["note"] + assert prepared["flux"] == pytest.approx(detrended) + assert np.all(prepared["unc"] > 0) + + +def test_prepare_final_fit_lightcurve_series_avoids_one_sided_oot_raw_flux_bias(): + times = np.linspace(-0.05, 0.05, 11) + detrended = 1.0 - 0.02 * np.exp(-0.5 * (times / 0.012) ** 2) + airmass_model = 1.0 + 2.0 * times + raw_flux = detrended * airmass_model + transit_model = np.where(times < 0.0, 1.0, 0.985) + fit = types.SimpleNamespace( + time=times, + data=raw_flux, + dataerr=np.full(times.shape, 0.01, dtype=float), + detrended=detrended.copy(), + detrendederr=np.full(times.shape, 0.01, dtype=float), + airmass_model=airmass_model, + transit=transit_model, + parameters={"tmid": 0.0}, + ) + + prepared = prepare_final_fit_lightcurve_series(fit) + old_oot_mask = transit_model == 1.0 + legacy_flux = raw_flux / np.nanmedian(raw_flux[old_oot_mask]) + expected_flux = detrended / np.nanmedian(detrended) + + assert prepared["applied"] is True + assert prepared["used_two_sided_oot"] is False + assert "only bracketed one side of transit" in prepared["note"] + assert prepared["flux"] == pytest.approx(expected_flux) + assert prepared["flux"][-1] != pytest.approx(legacy_flux[-1], abs=1e-3) + assert np.all(prepared["unc"] > 0) + + +def test_save_selected_photometry_debug_series_writes_stage_masks(tmp_path): + fit = types.SimpleNamespace( + selected_photometry_debug={ + "times": np.array([1.0, 2.0, 3.0], dtype=float), + "target_flux": np.array([10.0, 11.0, 12.0], dtype=float), + "comp_flux": np.array([5.0, 5.0, 6.0], dtype=float), + "raw_ratio": np.array([2.0, 2.2, 2.0], dtype=float), + "initial_sigma_keep_mask": np.array([True, False, True], dtype=bool), + "phase_clip_keep_mask_on_sigma_filtered": np.array([True, False], dtype=bool), + } + ) + + output_path = save_selected_photometry_debug_series(tmp_path, "Qatar-10 b", "20260420", fit) + + assert output_path is not None + assert output_path.exists() + + rows = np.loadtxt(output_path, delimiter=",", skiprows=1) + assert rows.shape == (3, 6) + assert rows[:, 4].astype(int).tolist() == [1, 0, 1] + assert rows[:, 5].astype(int).tolist() == [1, 0, 0] + + +def test_detrend_flux_on_out_of_transit_baseline_falls_back_to_prior_ephemeris(): + times = np.linspace(-0.08, 0.08, 17) + baseline = 1.0 + 0.25 * times + transit = np.ones_like(times) + transit[(times >= 0.03) & (times <= 0.05)] = 0.99 + flux = baseline.copy() + unc = np.full_like(times, 0.01) + fit = types.SimpleNamespace( + transit=transit, + parameters={"tmid": 0.10}, + ) + prior = { + "tmid": 0.0, + "per": 1.0, + "rprs": 0.1, + "ars": 12.0, + "inc": 88.0, + "ecc": 0.0, + "omega": 0.0, + } + + modeled = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit) + fallback = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit, prior=prior) + prior_coverage = summarize_prior_transit_coverage(times, prior, flux_values=flux, flux_errors=unc) + + assert modeled["applied"] is False + assert prior_coverage["valid"] is True + assert prior_coverage["has_two_sided_oot"] is True + assert fallback["applied"] is True + assert fallback["used_prior_ephemeris"] is True + assert "ephemeris-centered transit window" in fallback["note"] + + +def test_deduplicate_comparison_star_coords_merges_nearby_duplicates(): + unique_coords, duplicate_messages = deduplicate_comparison_star_coords( + [ + [1826.0, 1499.0], + [1827.0, 1511.0], + [1828.0, 1487.0], + [842.0, 1810.0], + ], + min_separation_pixels=15.0, + ) + + assert unique_coords == [[1826.0, 1499.0], [842.0, 1810.0]] + assert len(duplicate_messages) == 2 + assert "Merged comparison star #2" in duplicate_messages[0] + + +def test_robust_flux_floor_mask_rejects_tiny_positive_outliers(): + flux = np.full(30, 100.0) + flux[[5, 17]] = [1.0, 0.5] + + mask = robust_flux_floor_mask(flux) + + assert mask.sum() == 28 + assert not mask[5] + assert not mask[17] + + +def test_robust_target_reference_flux_mask_requires_both_series_to_be_plausible(): + target_flux = np.full(30, 100.0) + reference_flux = np.full(30, 120.0) + target_flux[7] = 1.0 + reference_flux[13] = 1.0 + + mask = robust_target_reference_flux_mask(target_flux, reference_flux) + + assert mask.sum() == 28 + assert not mask[7] + assert not mask[13] + + +def test_build_target_fit_candidate_jobs_masks_psf_target_and_comp_dropouts(): + frame_count = 30 + + def build_psf_rows(amplitudes): + psf_rows = np.zeros((frame_count, 7), dtype=float) + psf_rows[:, 0] = 10.0 + psf_rows[:, 1] = 20.0 + psf_rows[:, 2] = amplitudes + psf_rows[:, 3] = 1.0 + psf_rows[:, 4] = 1.0 + return psf_rows + + target_amplitudes = np.full(frame_count, 100.0) + comp_amplitudes = np.full(frame_count, 120.0) + target_amplitudes[7] = 1.0 + comp_amplitudes[13] = 1.0 + + psf_data = { + "target": build_psf_rows(target_amplitudes), + "comp1": build_psf_rows(comp_amplitudes), + } + + candidate_jobs = build_target_fit_candidate_jobs( + psf_data, + aper_data=None, + apers=None, + annuli=None, + airmass=np.linspace(1.0, 1.3, frame_count), + comp_stars=[[1827.0, 1511.0]], + sigma=3.0, + require_comp_star=True, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=False, + ) + + assert len(candidate_jobs) == 1 + assert candidate_jobs[0]["method"] == "psf" + assert candidate_jobs[0]["mask"].sum() == 28 + assert not candidate_jobs[0]["mask"][7] + assert not candidate_jobs[0]["mask"][13] + assert candidate_jobs[0]["coverage_count"] == 29 + + +def test_centroid_offset_matches_reference_uses_float_geometry_tolerance(): + target = np.array([2383.27, 867.04, 6.3, 9.0, 0.7, 0.0, 223.0]) + comp = np.array([1821.90, 549.21, 131.0, 3.0, 4.9, 0.0, 226.0]) + + assert centroid_offset_matches_reference(comp, target, 562.88, 315.42) + + +def test_should_keep_header_wcs_alignment_prefers_geometry_over_flux_swings(): + decision = should_keep_header_wcs_alignment( + projected_off_frame=False, + frame_index=5, + target_psf_row=np.array([2383.27, 867.04, 6.3, 9.0, 0.7, 0.0, 223.0]), + previous_target_psf_row=np.array([2382.88, 454.14, 154.0, 2.8, 2.7, 0.0, 223.0]), + comp_psf_rows={ + "comp1": np.array([1821.90, 549.21, 131.0, 3.0, 4.9, 0.0, 226.0]), + }, + previous_comp_psf_rows={ + "comp1": np.array([1822.57, 135.19, 3645.8, 2.9, 5.8, 0.0, 226.0]), + }, + expected_offsets={ + "comp1": np.array([562.88, 315.42]), + }, + ) + + assert decision["use_wcs_alignment"] is True + assert decision["reason"] == "geometry_match" + assert not decision["target_flux_change_ok"] + assert not decision["comp_flux_change_ok"] + assert decision["geometry_match_count"] == 1 + + +def test_should_keep_header_wcs_alignment_ignores_one_bad_comp_when_majority_match(): + decision = should_keep_header_wcs_alignment( + projected_off_frame=False, + frame_index=8, + target_psf_row=np.array([2387.50, 868.60, 50.0, 3.0, 3.0, 0.0, 223.0]), + previous_target_psf_row=np.array([2382.88, 454.14, 154.0, 2.8, 2.7, 0.0, 223.0]), + comp_psf_rows={ + "comp1": np.array([1827.0, 558.1, 200.0, 3.0, 5.0, 0.0, 226.0]), + "comp2": np.array([1300.0, 900.0, 300.0, 3.0, 5.0, 0.0, 226.0]), + }, + previous_comp_psf_rows={ + "comp1": np.array([1822.6, 135.2, 3645.8, 2.9, 5.8, 0.0, 226.0]), + "comp2": np.array([1600.0, 700.0, 280.0, 3.0, 5.0, 0.0, 226.0]), + }, + expected_offsets={ + "comp1": np.array([560.7, 310.2]), + "comp2": np.array([1187.5, 31.4]), + }, + ) + + assert decision["use_wcs_alignment"] is True + assert decision["geometry_test_count"] == 2 + assert decision["geometry_match_count"] == 1 + + +def test_should_keep_header_wcs_alignment_rejects_geometry_mismatch(): + decision = should_keep_header_wcs_alignment( + projected_off_frame=False, + frame_index=8, + target_psf_row=np.array([1576.0, 1317.0, 1.1, 20.0, 1.9, 0.0, 223.0]), + previous_target_psf_row=np.array([2382.88, 454.14, 154.0, 2.8, 2.7, 0.0, 223.0]), + comp_psf_rows={ + "comp1": np.array([1826.7, 1510.9, 1.9, 15.4, 1.5, 0.0, 226.0]), + }, + previous_comp_psf_rows={ + "comp1": np.array([1822.57, 135.19, 3645.8, 2.9, 5.8, 0.0, 226.0]), + }, + expected_offsets={ + "comp1": np.array([562.88, 315.42]), + }, + ) + + assert decision["use_wcs_alignment"] is False + assert decision["reason"] == "geometry_mismatch" + assert decision["geometry_match_count"] == 0 + + def test_check_coordinates_non_interactive_prefers_wcs_centroid(): x_pixel, y_pixel = check_coordinates( input_x_pixel=5, @@ -240,6 +525,118 @@ def test_should_use_aperture_photometry_parses_values(): assert should_use_aperture_photometry("n") is False +def test_should_use_eebls_to_initialize_tmid_and_bounds_parses_values(): + assert should_use_eebls_to_initialize_tmid_and_bounds(None) is True + assert should_use_eebls_to_initialize_tmid_and_bounds("y") is True + assert should_use_eebls_to_initialize_tmid_and_bounds("n") is False + assert should_use_eebls_to_initialize_tmid_and_bounds(True) is True + + +def test_build_time_rejection_diagnostic_groups_contiguous_ranges(): + times = np.array([1.0, 1.1, 1.2, 1.5, 1.6, 2.0], dtype=float) + keep_mask = np.array([True, False, False, True, False, True], dtype=bool) + + diagnostic = build_time_rejection_diagnostic("Example stage", times, keep_mask) + + assert diagnostic["stage"] == "Example stage" + assert diagnostic["input_point_count"] == 6 + assert diagnostic["kept_point_count"] == 3 + assert diagnostic["dropped_point_count"] == 3 + assert diagnostic["dropped_ranges"] == [ + {"start": pytest.approx(1.1), "end": pytest.approx(1.2), "count": 2}, + {"start": pytest.approx(1.6), "end": pytest.approx(1.6), "count": 1}, + ] + + +def test_estimate_tmid_and_bounds_with_eebls_identifies_box_like_transit(): + times = np.linspace(0.0, 0.2, 240) + tmid = 0.101 + duration = 0.028 + flux = np.ones(times.shape[0], dtype=float) + in_transit = np.abs(times - tmid) <= duration / 2.0 + flux[in_transit] -= 0.018 + flux += 0.0015 * (times - np.nanmean(times)) + flux_errors = np.full(times.shape[0], 0.002, dtype=float) + prior = { + "tmid": 0.08, + "per": 1.0, + "rprs": np.sqrt(0.018), + "ars": 12.0, + "inc": 88.5, + "ecc": 0.0, + "omega": 0.0, + } + + summary = estimate_tmid_and_bounds_with_eebls( + times, + flux, + flux_errors, + prior, + [0.04, 0.12], + ) + + assert summary["applied"] is True + assert summary["method"] == "eebls" + assert summary["tmid"] == pytest.approx(tmid, abs=0.01) + assert summary["bounds"][0] < summary["tmid"] < summary["bounds"][1] + assert summary["depth"] > 0 + assert summary["depth_snr"] > 0 + + +def test_estimate_ephemeris_tmid_and_bounds_caps_bracketed_runs_to_duration_scale(): + prior = { + "tmid": 2458247.90746, + "per": 1.645321, + "rprs": 0.1265, + "ars": 4.9, + "inc": 85.87, + "ecc": 0.0, + "omega": 0.0, + } + times = np.linspace(2461151.8012, 2461151.9966, 220) + expected_duration = 0.049 + + summary = estimate_ephemeris_tmid_and_bounds( + times, + prior["tmid"], + prior["per"], + midt_unc=0.00036, + per_unc=1.0e-5, + expected_duration=expected_duration, + sigma_multiplier=35.0, + ) + + assert summary["duration_capped"] is True + assert summary["observed_window_capped"] is True + assert summary["observations_bracket_expected_transit"] is True + assert summary["tmid"] == pytest.approx(2461151.899025, abs=1e-6) + assert summary["half_width"] > 0 + assert summary["bounds"][0] == pytest.approx(times.min() + 0.5 * expected_duration) + assert summary["bounds"][1] == pytest.approx(times.max() - 0.5 * expected_duration) + + +def test_estimate_ephemeris_tmid_and_bounds_keeps_wider_bounds_for_one_sided_runs(): + prior = { + "tmid": 2458247.90746, + "per": 1.645321, + } + times = np.linspace(2461151.92, 2461152.02, 120) + + summary = estimate_ephemeris_tmid_and_bounds( + times, + prior["tmid"], + prior["per"], + midt_unc=0.00036, + per_unc=1.0e-5, + expected_duration=0.049, + sigma_multiplier=35.0, + ) + + assert summary["duration_capped"] is False + assert summary["observations_bracket_expected_transit"] is False + assert summary["half_width"] == pytest.approx(0.25 * prior["per"]) + + def test_is_adaptive_aperture_mode_enabled_parses_values(): assert is_adaptive_aperture_mode_enabled(None) is False assert is_adaptive_aperture_mode_enabled("y") is True @@ -840,6 +1237,33 @@ def test_phase_bin_sigma_clip_flags_local_phase_outlier(): assert mask[27] +def test_sigma_clip_respects_large_time_gaps_between_segments(): + times_pre = 2461151.80 + np.arange(50, dtype=float) * 0.00075 + times_post = 2461151.98 + np.arange(8, dtype=float) * 0.00075 + times = np.concatenate([times_pre, times_post]) + + rng = np.random.default_rng(42) + values_pre = ( + 0.0235 + + 0.0006 * np.sin(np.linspace(0, 8 * np.pi, times_pre.size)) + + 0.0004 * np.linspace(0, 1, times_pre.size) + + rng.normal(0, 5e-5, times_pre.size) + ) + values_post = np.full(times_post.size, np.nanmedian(values_pre[-20:]) - 8e-4) + values_post += rng.normal(0, 5e-5, times_post.size) + values = np.concatenate([values_pre, values_post]) + values[54] += 0.8 + + np.random.seed(0) + old_mask = sigma_clip(values, sigma=3, dt=37, times=None) + np.random.seed(0) + gap_aware_mask = sigma_clip(values, sigma=3, dt=37, times=times) + + assert old_mask[54:58].all() + assert not gap_aware_mask[54:58].any() + assert gap_aware_mask.sum() < old_mask.sum() + + def test_fit_final_lightcurve_preserves_explicit_plot_time_range(monkeypatch): import exotic.exotic as exotic_module @@ -961,11 +1385,11 @@ def fake_lc_fitter( assert len(captured["calls"]) == 2 assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.125]) assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.158) - assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.128, 0.188]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.108, 0.208]) assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 1 assert fit.rprs_posterior_refit_edge == "upper" - assert fit.rprs_posterior_refit_bounds == pytest.approx([0.128, 0.188]) + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.108, 0.208]) def test_fit_final_lightcurve_prefit_refinement_trims_baseline_and_recenters_tmid(monkeypatch): @@ -1032,6 +1456,132 @@ def fake_lc_fitter( assert fit.prefit_refinement_tmid_bounds == pytest.approx([-1.0, 1.0]) +def test_fit_final_lightcurve_prefit_refinement_skips_one_sided_transit_solution(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + call_times = np.array(call_times, dtype=float) + captured["calls"].append({ + "times": call_times, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + transit = np.ones_like(call_times, dtype=float) + transit[call_times >= 0.0] = 0.95 + return types.SimpleNamespace( + duration_expected=2.0, + duration_measured=2.0, + parameters={"tmid": 2.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + time=call_times, + transit=transit, + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.array([-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0], dtype=float) + flux = np.ones(times.shape[0], dtype=float) + fluxerr = np.full(times.shape[0], 0.01, dtype=float) + airmass = np.ones(times.shape[0], dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0} + bounds = {"rprs": [0.0, 0.2], "tmid": [-2.0, 2.0], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit, trimmed_flux, trimmed_unc = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + baseline_duration_multiplier=0.5, + ) + + assert len(captured["calls"]) == 1 + assert captured["calls"][0]["times"] == pytest.approx(times) + assert trimmed_flux == pytest.approx(flux) + assert trimmed_unc == pytest.approx(fluxerr) + assert fit.prefit_refinement_applied is False + assert "one side of the modeled transit" in fit.prefit_refinement_note + + +def test_fit_final_lightcurve_prefit_refinement_does_not_expand_tmid_past_original_bounds(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + call_times = np.array(call_times, dtype=float) + captured["calls"].append({ + "times": call_times, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + transit = np.where(np.abs(call_times - 0.4) <= 0.25, 0.98, 1.0) + return types.SimpleNamespace( + duration_expected=0.5, + duration_measured=0.5, + parameters={"tmid": 0.4, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + time=call_times, + transit=transit, + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.array([-0.8, -0.4, 0.0, 0.4, 0.8], dtype=float) + flux = np.ones(times.shape[0], dtype=float) + fluxerr = np.full(times.shape[0], 0.01, dtype=float) + airmass = np.ones(times.shape[0], dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0} + bounds = {"rprs": [0.0, 0.2], "tmid": [-0.5, 0.5], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + baseline_duration_multiplier=0.5, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][1]["bounds"]["tmid"] == pytest.approx([0.15, 0.5]) + assert fit.prefit_refinement_tmid_bounds == pytest.approx([0.15, 0.5]) + + def test_fit_lightcurve_keeps_large_raw_target_reference_ratios(monkeypatch): captured = {} @@ -1055,7 +1605,10 @@ def fake_lc_fitter( return types.SimpleNamespace() monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) - monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) times = np.linspace(0.0, 0.05, 6) tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) @@ -1104,7 +1657,10 @@ def fake_lc_fitter( return fit monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) - monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) times = np.linspace(0.0, 0.05, 6) tflux = np.full(times.shape[0], 2.0) @@ -1164,7 +1720,10 @@ def fake_lc_fitter( return fit monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) - monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) times = np.linspace(0.0, 0.05, 6) tflux = np.full(times.shape[0], 100.0) @@ -1200,7 +1759,10 @@ def fake_lc_fitter(*args, **kwargs): return types.SimpleNamespace() monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) - monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) times = np.linspace(0.0, 0.03, 4) tflux = np.full(times.shape[0], 2.0) @@ -1264,7 +1826,10 @@ def fake_phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max return mask monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) - monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) monkeypatch.setattr("exotic.exotic.phase_bin_sigma_clip", fake_phase_bin_sigma_clip) times = np.linspace(0.0, 0.08, 8) @@ -1315,7 +1880,10 @@ def fake_lc_fitter( return types.SimpleNamespace() monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) - monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) times = np.linspace(0.0, 0.05, 6) tflux = np.full(times.shape[0], 2.0) @@ -1350,6 +1918,335 @@ def fake_lc_fitter( assert captured_modes == ["lm", "ns"] +def test_fit_lightcurve_attaches_frame_filter_diagnostics(monkeypatch): + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + return types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(fluxes, dtype=float), + dataerr=np.asarray(flux_unc, dtype=float), + detrended=np.asarray(fluxes, dtype=float), + detrendederr=np.asarray(flux_unc, dtype=float), + airmass=np.asarray(airmass, dtype=float), + airmass_model=np.ones(len(times), dtype=float), + residuals=np.zeros(len(times), dtype=float), + phase=np.linspace(-0.1, 0.1, len(times)), + transit=np.ones(len(times), dtype=float), + model=np.asarray(fluxes, dtype=float), + wf=np.ones(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a1": 1.0, "a2": 0.0, "per": 1.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a1": 0.01, "a2": 0.01}, + ) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.array([False] * (len(data) - 1) + [True], dtype=bool), + ) + monkeypatch.setattr( + "exotic.exotic.phase_bin_sigma_clip", + lambda values, phase, sigma=3, bins=10, min_points=5, max_iters=3: np.zeros(len(values), dtype=bool), + ) + + times = np.arange(12, dtype=float) + tflux = np.full(times.shape[0], 100.0, dtype=float) + cflux = np.full(times.shape[0], 50.0, dtype=float) + cflux[1] = 0.0 + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.5, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is not None + diagnostics = myfit.frame_filter_diagnostics + assert [diagnostic["stage"] for diagnostic in diagnostics] == [ + "Target/reference ratio filter", + "Initial sigma clip", + "Finite/positive photometry filter", + ] + assert diagnostics[0]["dropped_point_count"] == 1 + assert diagnostics[0]["first_dropped_time"] == pytest.approx(1.0) + assert diagnostics[1]["dropped_point_count"] == 1 + assert diagnostics[1]["first_dropped_time"] == pytest.approx(11.0) + assert diagnostics[2]["dropped_point_count"] == 0 + + +def test_fit_ranked_comparison_calibration_candidates_selects_lowest_residual_success(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + def fake_fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + residual_scale_map = { + 50: 0.05, + 40: 0.02, + 30: 0.03, + } + residual_scale = residual_scale_map[comp_marker] + residuals = residual_scale * np.array([-1.0, 1.0, -1.0, 1.0, -1.0, 1.0], dtype=float) + fit = types.SimpleNamespace( + residuals=residuals, + data=np.ones_like(residuals), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + ) + return fit, np.asarray(tflux, dtype=float), np.asarray(cflux, dtype=float) + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = {"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0} + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + "comp3": np.full((6, 1, 1), 30.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + { + "label": "Comp 1", + "position": (10.0, 10.0), + "aggregate_score": 0.01, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 0, + }, + { + "label": "Comp 2", + "position": (20.0, 20.0), + "aggregate_score": 0.02, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 1, + }, + { + "label": "Comp 3", + "position": (30.0, 30.0), + "aggregate_score": 0.03, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 2, + }, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld, + p_dict, + comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert len(result["attempts"]) == 3 + assert result["selected_result"]["comp_index"] == 1 + assert result["selected_result"]["rank"] == 1 + assert result["selected_result"]["selected"] is True + assert "lowest target-fit residual scatter" in result["selected_result"]["selection_reason"] + assert result["attempts"][0]["selection_reason"].startswith("not selected: target-fit residual scatter") + + +def test_fit_lightcurve_refines_nested_tmid_bounds_from_two_sided_lm_fit(monkeypatch): + captured_calls = [] + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured_calls.append({ + "mode": mode, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + }, + }) + if mode == "lm": + transit = np.ones_like(times, dtype=float) + transit[(times >= 0.018) & (times <= 0.032)] = 0.98 + return types.SimpleNamespace( + transit=transit, + parameters={"tmid": 0.025, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 1.0}, + duration_expected=0.014, + ) + return types.SimpleNamespace(parameters={"tmid": 0.025, "rprs": 0.1, "inc": 89.0, "a2": 0.0}) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 21) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + final_fit_mode="ns", + ) + + assert myfit is not None + assert captured_calls[0]["mode"] == "lm" + assert captured_calls[1]["mode"] == "ns" + assert captured_calls[0]["bounds"]["tmid"] == pytest.approx([0.011347361950458953, 0.03865263804954105]) + assert captured_calls[1]["bounds"]["tmid"] == pytest.approx([0.0175, 0.0325]) + assert myfit.nested_tmid_refinement_applied is True + assert "recenter nested-sampling Tmid bounds" in myfit.nested_tmid_refinement_note + + +def test_fit_lightcurve_skips_nested_tmid_refinement_for_one_sided_lm_fit(monkeypatch): + captured_calls = [] + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured_calls.append({ + "mode": mode, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + }, + }) + if mode == "lm": + transit = np.ones_like(times, dtype=float) + transit[times >= 0.025] = 0.98 + return types.SimpleNamespace( + transit=transit, + parameters={"tmid": 0.025, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 1.0}, + duration_expected=0.014, + ) + return types.SimpleNamespace(parameters={"tmid": 0.025, "rprs": 0.1, "inc": 89.0, "a2": 0.0}) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 21) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + final_fit_mode="ns", + ) + + assert myfit is not None + assert captured_calls[0]["mode"] == "lm" + assert captured_calls[1]["mode"] == "ns" + assert captured_calls[0]["bounds"]["tmid"] == pytest.approx([0.011347361950458953, 0.03865263804954105]) + assert captured_calls[1]["bounds"]["tmid"] == pytest.approx([0.011347361950458953, 0.03865263804954105]) + assert myfit.nested_tmid_refinement_applied is False + assert "one side of the modeled transit" in myfit.nested_tmid_refinement_note + + def test_run_target_driven_photometry_search_selects_best_method_across_psf_and_aperture(monkeypatch): evaluated = [] @@ -1359,7 +2256,7 @@ def __init__(self, residual_level): self.data = np.ones(6) def fake_evaluate(task): - _, tflux, cflux, _, _, _, _, _, _, _ = task + _, tflux, cflux, *_ = task evaluated.append(np.asarray(cflux)) cflux = np.asarray(cflux) tflux = np.asarray(tflux) @@ -1669,7 +2566,10 @@ def fake_lc_fitter( return types.SimpleNamespace() monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) - monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) times = np.linspace(0.0, 0.05, 6) tflux = np.full(times.shape[0], 2.0) @@ -1722,7 +2622,10 @@ def fake_lc_fitter( return types.SimpleNamespace() monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) - monkeypatch.setattr("exotic.exotic.sigma_clip", lambda data, sigma=3, dt=21, po=2: np.zeros(len(data), dtype=bool)) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) times = np.linspace(0.0, 0.05, 6) tflux = np.full(times.shape[0], 2.0) diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 7b08ac07..a89fc904 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -98,6 +98,21 @@ def test_comp_params_defaults_bad_wcs_threshold_percent_to_three(tmp_path): assert inputs.info_dict["bad_wcs_threshold_percent"] == 3.0 +def test_comp_params_defaults_pointing_rejection_sigma_to_four(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["pointing_rejection_sigma"] == pytest.approx(4.0) + + def test_comp_params_defaults_skip_low_comparison_coverage_rejection_to_no(tmp_path): init_data = { "user_info": {}, @@ -188,6 +203,21 @@ def test_comp_params_defaults_final_fit_baseline_duration_multiplier_to_one(tmp_ assert inputs.info_dict["final_fit_baseline_duration_multiplier"] == pytest.approx(1.0) +def test_comp_params_defaults_use_eebls_tmid_initializer_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_eebls_to_initialize_tmid_and_bounds"] == "y" + + def test_comp_params_defaults_use_impactparameter_fit_to_yes(tmp_path): init_data = { "user_info": {}, @@ -308,6 +338,21 @@ def test_comp_params_reads_bad_wcs_threshold_percent_from_optional_info(tmp_path assert inputs.info_dict["bad_wcs_threshold_percent"] == 5.5 +def test_comp_params_reads_pointing_rejection_sigma_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"pointing_rejection_sigma": 3.5}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["pointing_rejection_sigma"] == 3.5 + + def test_comp_params_reads_skip_low_comparison_coverage_rejection_from_optional_info(tmp_path): init_data = { "user_info": {}, @@ -413,6 +458,21 @@ def test_comp_params_reads_final_fit_baseline_duration_multiplier_from_optional_ assert inputs.info_dict["final_fit_baseline_duration_multiplier"] == pytest.approx(1.75) +def test_comp_params_reads_use_eebls_tmid_initializer_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"Use EEBLS to Initialize Tmid and Bounds? (y/n)": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_eebls_to_initialize_tmid_and_bounds"] == "n" + + def test_comp_params_reads_use_impactparameter_fit_from_optional_info(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_ld.py b/tests/test_ld.py index 1e7bec07..5af579bf 100644 --- a/tests/test_ld.py +++ b/tests/test_ld.py @@ -1,3 +1,5 @@ +import logging + from exotic.api.ld import LimbDarkening stellar_params = { @@ -190,6 +192,21 @@ def test_valid_fwhm_range_swapped_min_max() -> None: assert ld_obj.check_fwhm(observed_filter) == True +def test_missing_fwhm_values_do_not_log_errors(caplog) -> None: + observed_filter = { + 'filter': None, + 'name': None, + 'wl_min': None, + 'wl_max': None + } + + ld_obj = LimbDarkening(stellar_params) + + with caplog.at_level(logging.ERROR, logger="exotic.api.ld"): + assert ld_obj.check_fwhm(observed_filter) == False + + assert "FWHM matching failed" not in caplog.text + def test_invalid_fwhm_range_1() -> None: observed_filter = { 'filter': None, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 5f11db10..a42dcc43 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -6,6 +6,7 @@ import numpy as np import pytest +from tenacity import Future, RetryError fake_barycorrpy = types.ModuleType('barycorrpy') fake_utc_tdb = types.ModuleType('barycorrpy.utc_tdb') @@ -142,6 +143,63 @@ def fake_post(url, data, headers, timeout): assert any('NextAstro variability response JSON:' in message for message in logged) +def test_nextastro_variability_retries_zstd_415_once_with_gzip(monkeypatch): + logged = [] + encodings = [] + + def fake_build_compressed_json_request(payload, content_encoding=None): + encoding = content_encoding or 'zstd' + body = json.dumps(payload).encode('utf-8') + headers = { + 'Content-Type': 'application/json', + 'Content-Encoding': encoding, + } + return body, headers, encoding, len(body), len(body) + + def fake_post(url, data, headers, timeout): + encodings.append(headers['Content-Encoding']) + if headers['Content-Encoding'] == 'zstd': + return DummyResponse(None, status_code=415) + return DummyResponse([{'is_in_vsx': 1}]) + + monkeypatch.setattr(exotic_module, 'build_compressed_json_request', fake_build_compressed_json_request) + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + variability_flags = exotic_module.nextastro_variability_test([(10.1, -11.2)]) + + assert variability_flags == [True] + assert encodings == ['zstd', 'gzip'] + assert any('rejected zstd-compressed request (HTTP 415)' in message for message in logged) + + +def test_nextastro_variability_caps_retry_attempts_at_five(monkeypatch): + attempts = [] + + def fake_build_compressed_json_request(payload, content_encoding=None): + encoding = content_encoding or 'gzip' + body = b'{}' + headers = { + 'Content-Type': 'application/json', + 'Content-Encoding': encoding, + } + return body, headers, encoding, len(body), len(body) + + def fake_post(url, data, headers, timeout): + attempts.append(headers['Content-Encoding']) + return DummyResponse(None, status_code=502) + + monkeypatch.setattr(exotic_module, 'build_compressed_json_request', fake_build_compressed_json_request) + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module.nextastro_variability_test.retry, 'sleep', lambda _: None) + + with pytest.raises(RetryError) as excinfo: + exotic_module.nextastro_variability_test([(10.1, -11.2)]) + + assert len(attempts) == 5 + assert excinfo.value.last_attempt.attempt_number == 5 + + def test_check_for_variable_stars_uses_nextastro_flags_to_filter(monkeypatch): logged = [] @@ -161,6 +219,30 @@ def test_check_for_variable_stars_uses_nextastro_flags_to_filter(monkeypatch): assert any('NextAstro flagged variable: True' in message for message in logged) +def test_check_for_variable_stars_logs_underlying_nextastro_retry_error(monkeypatch): + logged = [] + + ra_wcs = np.array([[100.1]]) + dec_wcs = np.array([[-10.1]]) + comp_stars = [[0, 0]] + + last_attempt = Future(5) + last_attempt.set_exception(RuntimeError('HTTP 502')) + + def raise_retry_error(payload): + raise RetryError(last_attempt) + + monkeypatch.setattr(exotic_module, 'nextastro_variability_test', raise_retry_error) + monkeypatch.setattr(exotic_module, 'query_variable_star_apis', lambda ra, dec: False) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + exotic_module.check_for_variable_stars( + ra_wcs, dec_wcs, comp_stars, use_nextastro_variability_server=True + ) + + assert any('RetryError after 5 attempts (RuntimeError: HTTP 502)' in message for message in logged) + + def test_get_wcs_falls_back_to_nextastro_when_nova_fails(monkeypatch): service_calls = [] From d4d8a6c4706aaaf9dde33d3de5d89015715ce561 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Mon, 27 Apr 2026 06:19:56 +1000 Subject: [PATCH 024/116] Improve aperture photometry with FWHM-aware sky annulus --- README.md | 2 +- docs/README.md | 2 +- exotic/exotic.py | 330 ++++++++++++++++++++---- exotic/inputs.py | 2 +- exotic/plots.py | 33 ++- inits.json | 4 +- tests/test_centroid_wcs.py | 78 +++++- tests/test_exotic_proper_motion.py | 10 + tests/test_exotic_rprs_retry.py | 1 + tests/test_inputs.py | 15 ++ tests/test_lazy_pylightcurve_imports.py | 1 + tests/test_nextastro_variability.py | 1 + 12 files changed, 406 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index aacd68d4..3bd0dfe0 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, - "Fast Aperture Mask (y/n)": true, + "Fast Aperture Mask (y/n)": false, "use_psf_photometry": "y", "use_aperture_photometry": "y", "skip_low_comparison_coverage_rejection": "n", diff --git a/docs/README.md b/docs/README.md index a1aaf38a..b1dccc47 100644 --- a/docs/README.md +++ b/docs/README.md @@ -214,7 +214,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Pixel Scale (Ex: 5.21 arcsecs/pixel)": null, "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, - "Fast Aperture Mask (y/n)": true, + "Fast Aperture Mask (y/n)": false, "use_psf_photometry": "y", "use_aperture_photometry": "y", "detrend_on_outoftransit_baseline": true, diff --git a/exotic/exotic.py b/exotic/exotic.py index 95503fa7..d8bd24ea 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -88,7 +88,7 @@ import matplotlib.pyplot as plt import numpy as np # photometry -from photutils.aperture import CircularAperture +from photutils.aperture import CircularAperture, CircularAnnulus import re import requests # scipy imports @@ -754,7 +754,7 @@ def robust_target_reference_flux_mask(target_flux, reference_flux): def is_fast_aperture_mask_enabled(config_value): if config_value is None: - return True + return False if isinstance(config_value, bool): return config_value if isinstance(config_value, (int, float)): @@ -766,8 +766,8 @@ def is_fast_aperture_mask_enabled(config_value): if normalized in ('n', 'no', 'false', '0', 'exact', 'off'): return False - log_info("Warning: Invalid 'Fast Aperture Mask (y/n)' value; using fast mode.", warn=True) - return True + log_info("Warning: Invalid 'Fast Aperture Mask (y/n)' value; using exact mode.", warn=True) + return False def is_comp_star_required(config_value): @@ -2444,7 +2444,26 @@ def summarize_adaptive_aperture_usage(psf_rows, aperture_scale, annulus_scale, f aperture_series = aperture_scale * frame_sigma annulus_series = annulus_scale * frame_sigma - fwhm_series = 2.355 * frame_sigma + fwhm_series = GAUSSIAN_SIGMA_TO_FWHM * frame_sigma + + geometry_rows = [ + resolve_sky_annulus_geometry(aperture_radius, annulus_width, psf_sigma=sigma) + if np.isfinite(aperture_radius) and np.isfinite(annulus_width) and np.isfinite(sigma) + else None + for aperture_radius, annulus_width, sigma in zip(aperture_series, annulus_series, frame_sigma) + ] + sky_inner_series = np.array( + [np.nan if geometry is None else geometry['inner_radius'] for geometry in geometry_rows], + dtype=float, + ) + sky_outer_series = np.array( + [np.nan if geometry is None else geometry['outer_radius'] for geometry in geometry_rows], + dtype=float, + ) + sky_pixel_series = np.array( + [np.nan if geometry is None else geometry['effective_sky_pixels'] for geometry in geometry_rows], + dtype=float, + ) if not np.any(np.isfinite(aperture_series)) or not np.any(np.isfinite(annulus_series)): return None @@ -2456,6 +2475,9 @@ def summarize_adaptive_aperture_usage(psf_rows, aperture_scale, annulus_scale, f 'fwhm_series': fwhm_series, 'aperture_series': aperture_series, 'annulus_series': annulus_series, + 'sky_inner_series': sky_inner_series, + 'sky_outer_series': sky_outer_series, + 'sky_pixel_series': sky_pixel_series, 'aperture_median': float(np.nanmedian(aperture_series)), 'aperture_std': float(np.nanstd(aperture_series)), 'aperture_min': float(np.nanmin(aperture_series)), @@ -5026,6 +5048,12 @@ def transformation_task_with_cached_reference(i, file_name): APERTURE_SIGMA_MAX = 6.0 ANNULUS_SIGMA_MIN = 6.0 ANNULUS_SIGMA_MAX = 15.0 +GAUSSIAN_SIGMA_TO_FWHM = 2.355 +SKY_ANNULUS_MIN_GAP_PIXELS = 2.0 +SKY_ANNULUS_MIN_FWHM_MULTIPLIER = 2.0 +SKY_ANNULUS_MIN_EFFECTIVE_PIXELS = 250.0 +SKY_BACKGROUND_SIGMA_CLIP = 3.0 +SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS = 3 APERTURE_AUTOTUNE_COARSE_APER_POINTS = 5 APERTURE_AUTOTUNE_COARSE_ANNULUS_POINTS = 4 APERTURE_AUTOTUNE_REFINED_APER_POINTS = 6 @@ -5384,7 +5412,7 @@ def should_keep_header_wcs_alignment( # Method fits a 2D gaussian function that matches the star_psf to the star image and returns its pixel coordinates -def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weightedcenter=True, fast_mode=False): +def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weightedcenter=False, fast_mode=False): stage_start = perf_counter() # get sub field in image try: @@ -5450,7 +5478,8 @@ def fcn2min(pars): log.debug(f"Centroid LM fallback failed at {np.round(pos, 2)}: {lm_exc}") return _nan_psf_result() - # override psf fit results with weighted centroid + # Preserve the solved PSF center for subpixel tracking by default. + # The weighted-center override remains available as an explicit legacy option. if weightedcenter: res.x[0] = wx res.x[1] = wy @@ -5487,8 +5516,126 @@ def sigma_clipped_nanmedian(data, sigma=3.0, max_iters=3): return bn.nanmedian(clipped), bn.nanstd(clipped) +def weighted_nanpercentile(values, weights, percentile): + values = np.asarray(values, dtype=float).ravel() + weights = np.asarray(weights, dtype=float).ravel() + valid = np.isfinite(values) & np.isfinite(weights) & (weights > 0) + if not np.any(valid): + return np.nan + + values = values[valid] + weights = weights[valid] + sort_index = np.argsort(values, kind='mergesort') + values = values[sort_index] + weights = weights[sort_index] + + total_weight = float(np.sum(weights)) + if not np.isfinite(total_weight) or total_weight <= 0: + return np.nan + + if values.size == 1: + return float(values[0]) + + cumulative = (np.cumsum(weights) - 0.5 * weights) / total_weight + target = float(np.clip(percentile, 0.0, 100.0)) / 100.0 + return float(np.interp(target, cumulative, values, left=values[0], right=values[-1])) + + +def weighted_nanstd(values, weights): + values = np.asarray(values, dtype=float).ravel() + weights = np.asarray(weights, dtype=float).ravel() + valid = np.isfinite(values) & np.isfinite(weights) & (weights > 0) + if not np.any(valid): + return np.nan + + values = values[valid] + weights = weights[valid] + total_weight = float(np.sum(weights)) + if not np.isfinite(total_weight) or total_weight <= 0: + return np.nan + + mean = float(np.sum(weights * values) / total_weight) + variance = float(np.sum(weights * (values - mean) ** 2) / total_weight) + return float(np.sqrt(max(variance, 0.0))) + + +def sigma_clipped_weighted_median(values, weights, sigma=3.0, max_iters=3, high_only=False): + values = np.asarray(values, dtype=float).ravel() + weights = np.asarray(weights, dtype=float).ravel() + keep = np.isfinite(values) & np.isfinite(weights) & (weights > 0) + if not np.any(keep): + return np.nan, np.nan + + for _ in range(max_iters): + center = weighted_nanpercentile(values[keep], weights[keep], 50.0) + scatter = weighted_nanstd(values[keep], weights[keep]) + if not np.isfinite(center): + return np.nan, np.nan + if not np.isfinite(scatter) or scatter <= 0: + break + + if high_only: + updated_keep = keep & (values <= center + sigma * scatter) + else: + updated_keep = keep & (np.abs(values - center) <= sigma * scatter) + if np.array_equal(updated_keep, keep): + break + keep = updated_keep + + if not np.any(keep): + return np.nan, np.nan + + return weighted_nanpercentile(values[keep], weights[keep], 50.0), weighted_nanstd(values[keep], weights[keep]) + + +def psf_fwhm_from_sigma(sigma): + try: + sigma = float(sigma) + except (TypeError, ValueError): + return np.nan + + if not np.isfinite(sigma) or sigma <= 0: + return np.nan + + return float(GAUSSIAN_SIGMA_TO_FWHM * sigma) + + +def resolve_sky_annulus_geometry( + aperture_radius, + annulus_width, + psf_sigma=np.nan, + minimum_gap_pixels=SKY_ANNULUS_MIN_GAP_PIXELS, + minimum_fwhm_multiplier=SKY_ANNULUS_MIN_FWHM_MULTIPLIER, + minimum_sky_pixels=SKY_ANNULUS_MIN_EFFECTIVE_PIXELS, +): + aperture_radius = abs(float(aperture_radius)) + annulus_width = max(float(annulus_width), 0.0) + + inner_radius = aperture_radius + float(minimum_gap_pixels) + fwhm = psf_fwhm_from_sigma(psf_sigma) + if np.isfinite(fwhm): + inner_radius = max(inner_radius, float(minimum_fwhm_multiplier) * fwhm) + + outer_radius = inner_radius + annulus_width + effective_sky_pixels = np.pi * max(outer_radius ** 2 - inner_radius ** 2, 0.0) + + if minimum_sky_pixels is not None and np.isfinite(minimum_sky_pixels) and minimum_sky_pixels > 0: + minimum_outer_radius = float(np.sqrt(inner_radius ** 2 + float(minimum_sky_pixels) / np.pi)) + if minimum_outer_radius > outer_radius: + outer_radius = minimum_outer_radius + effective_sky_pixels = np.pi * max(outer_radius ** 2 - inner_radius ** 2, 0.0) + + return { + 'inner_radius': float(inner_radius), + 'outer_radius': float(outer_radius), + 'annulus_width': float(max(outer_radius - inner_radius, 0.0)), + 'effective_sky_pixels': float(effective_sky_pixels), + 'fwhm': float(fwhm) if np.isfinite(fwhm) else np.nan, + } + + # Method calculates the flux of the star (uses the skybg_phot method to do background sub) -def aperPhot(data, starIndex, xc, yc, r=5, dr=5, fast_mode=True): +def aperPhot(data, starIndex, xc, yc, r=5, dr=5, fast_mode=False, sigma_hint=np.nan): stage_start = perf_counter() try: # Check for invalid coordinates @@ -5497,7 +5644,16 @@ def aperPhot(data, starIndex, xc, yc, r=5, dr=5, fast_mode=True): # Calculate background if dr > 0 if dr > 0: - bgflux, sigmabg, Nbg = skybg_phot(data, starIndex, xc, yc, r + 2, dr) + sky_geometry = resolve_sky_annulus_geometry(r, dr, psf_sigma=sigma_hint) + bgflux, sigmabg, Nbg = skybg_phot( + data, + starIndex, + xc, + yc, + sky_geometry['inner_radius'], + sky_geometry['annulus_width'], + fast_mode=fast_mode, + ) if not np.isfinite(bgflux): return np.nan, bgflux else: @@ -5521,65 +5677,88 @@ def aperPhot(data, starIndex, xc, yc, r=5, dr=5, fast_mode=True): _record_photometry_stage_timing('aperPhot', perf_counter() - stage_start) -def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False): - # create a crude annulus to mask out bright background pixels - # the box will not extend beyond the borders of the image - image_height, image_width = data.shape - xv, yv = mesh_box([xc, yc], np.round(r + dr), maxx=image_width, maxy=image_height) - if xv.size == 0 or yv.size == 0: - plateStatus.skyBackgroundWarning(starIndex, xc, yc) - log.debug(f"Warning: empty sky background box for {xc:.1f}, {yc:.1f}." - f"\nCheck if star is present or close to border.") - return np.nan, np.nan, 0 +def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False, fast_mode=False): + # The sky annulus uses an inner radius r and an outer radius r + dr. + # Callers are responsible for choosing r and dr from the aperture radius and PSF size. + annulus = CircularAnnulus(positions=[(xc, yc)], r_in=float(r), r_out=float(r + dr)) + mask_method = 'center' if fast_mode else 'exact' + annulus_mask = annulus.to_mask(method=mask_method)[0] + annulus_cutout = annulus_mask.cutout(data, fill_value=np.nan) - r_inner2 = float(r) ** 2 - r_outer2 = float(r + dr) ** 2 - rv2 = (xv - xc) ** 2 + (yv - yc) ** 2 - mask = (rv2 > r_inner2) & (rv2 < r_outer2) - if not np.any(mask): + if annulus_cutout is None: plateStatus.skyBackgroundWarning(starIndex, xc, yc) log.debug(f"Warning: empty sky background annulus for {xc:.1f}, {yc:.1f}." f"\nCheck if star is present or close to border.") return np.nan, np.nan, 0 - annulus_pixels = np.asarray(data[yv, xv][mask], dtype=float) - if annulus_pixels.size == 0: + annulus_cutout = np.asarray(annulus_cutout, dtype=float) + annulus_weights = np.asarray(annulus_mask.data, dtype=float) + valid_mask = np.isfinite(annulus_cutout) & np.isfinite(annulus_weights) & (annulus_weights > 0) + if not np.any(valid_mask): plateStatus.skyBackgroundWarning(starIndex, xc, yc) log.debug(f"Warning: no valid sky background pixels for {xc:.1f}, {yc:.1f}." f"\nCheck if star is present or close to border.") return np.nan, np.nan, 0 + annulus_pixels = annulus_cutout[valid_mask] + annulus_pixel_weights = annulus_weights[valid_mask] + try: - cutoff = np.nanpercentile(annulus_pixels, ptol) + cutoff = weighted_nanpercentile(annulus_pixels, annulus_pixel_weights, ptol) except (IndexError, ValueError): plateStatus.skyBackgroundWarning(starIndex, xc, yc) log.debug(f"Warning: IndexError, problem computing sky bg for {xc:.1f}, {yc:.1f}." f"\nCheck if star is present or close to border.") return np.nan, np.nan, 0 - dat = np.array(data[yv, xv], dtype=float) - dat[dat > cutoff] = np.nan # ignore pixels brighter than percentile + if not np.isfinite(cutoff): + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: invalid cutoff while computing sky bg for {xc:.1f}, {yc:.1f}.") + return np.nan, np.nan, 0 + + clipped_keep = annulus_pixels <= cutoff + clipped_pixels = annulus_pixels[clipped_keep] + clipped_weights = annulus_pixel_weights[clipped_keep] + if clipped_pixels.size == 0: + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: percentile clipping removed all sky background pixels for {xc:.1f}, {yc:.1f}.") + return np.nan, np.nan, 0 + + dat = np.full_like(annulus_cutout, np.nan, dtype=float) + dat[valid_mask] = annulus_cutout[valid_mask] + dat[valid_mask & (annulus_cutout > cutoff)] = np.nan if debug: - minb = data[yv, xv][mask].min() - maxb = data[yv, xv][mask].mean() + 3 * data[yv, xv][mask].std() - nanmask = np.nan * np.zeros(mask.shape) - nanmask[mask] = 1 - bgsky = data[yv, xv] * nanmask - cmed, _ = sigma_clipped_nanmedian(dat.flatten(), sigma=3.0, max_iters=3) - amed, _ = sigma_clipped_nanmedian(bgsky.flatten(), sigma=3.0, max_iters=3) + minb = float(np.nanmin(annulus_pixels)) + maxb = float(np.nanmean(annulus_pixels) + 3 * np.nanstd(annulus_pixels)) + bgsky = np.full_like(annulus_cutout, np.nan, dtype=float) + bgsky[valid_mask] = annulus_cutout[valid_mask] + cmed, _ = sigma_clipped_weighted_median( + clipped_pixels, + clipped_weights, + sigma=SKY_BACKGROUND_SIGMA_CLIP, + max_iters=SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS, + high_only=True, + ) + amed, _ = sigma_clipped_weighted_median( + annulus_pixels, + annulus_pixel_weights, + sigma=SKY_BACKGROUND_SIGMA_CLIP, + max_iters=SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS, + high_only=True, + ) fig, ax = plt.subplots(2, 2, figsize=(9, 9)) - im = ax[0, 0].imshow(data[yv, xv], vmin=minb, vmax=maxb, cmap='inferno') + im = ax[0, 0].imshow(annulus_cutout, vmin=minb, vmax=maxb, cmap='inferno') ax[0, 0].set_title("Original Data") from mpl_toolkits.axes_grid1 import make_axes_locatable divider = make_axes_locatable(ax[0, 0]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') - ax[1, 0].hist(bgsky.flatten(), label=f'Sky Annulus ({np.nanmedian(bgsky):.1f}, {amed:.1f})', + ax[1, 0].hist(annulus_pixels, label=f'Sky Annulus ({np.nanmedian(annulus_pixels):.1f}, {amed:.1f})', alpha=0.5, bins=np.arange(minb, maxb)) - ax[1, 0].hist(dat.flatten(), label=f'Clipped ({np.nanmedian(dat):.1f}, {cmed:.1f})', alpha=0.5, + ax[1, 0].hist(clipped_pixels, label=f'Clipped ({np.nanmedian(clipped_pixels):.1f}, {cmed:.1f})', alpha=0.5, bins=np.arange(minb, maxb)) ax[1, 0].legend(loc='best') ax[1, 0].set_title("Sky Background") @@ -5592,9 +5771,14 @@ def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False): ax[0, 1].set_title("Sky Annulus") plt.tight_layout() plt.show() - dat_flat = dat.ravel() - sky_median, sky_sigma = sigma_clipped_nanmedian(dat_flat, sigma=3.0, max_iters=3) - return sky_median, sky_sigma, np.sum(mask) + sky_median, sky_sigma = sigma_clipped_weighted_median( + clipped_pixels, + clipped_weights, + sigma=SKY_BACKGROUND_SIGMA_CLIP, + max_iters=SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS, + high_only=True, + ) + return sky_median, sky_sigma, float(np.sum(annulus_pixel_weights)) def process_dark_frames(dark_files): """Process dark frames and return the master dark.""" @@ -6087,10 +6271,27 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet aper = float(aper[0]) annulus = float(annulus[0]) - tFlux = aperPhot(imageData, 0, psf_data['target'][i, 0], psf_data['target'][i, 1], aper, annulus, - fast_mode=fast_aperture_mask)[0] - cFlux = aperPhot(imageData, 1, psf_data['comp'][i, 0], psf_data['comp'][i, 1], aper, annulus, - fast_mode=fast_aperture_mask)[0] + comp_frame_sigma = psf_sigma_from_fit(psf_data['comp'][i], fallback_sigma=frame_sigma) + tFlux = aperPhot( + imageData, + 0, + psf_data['target'][i, 0], + psf_data['target'][i, 1], + aper, + annulus, + fast_mode=fast_aperture_mask, + sigma_hint=frame_sigma, + )[0] + cFlux = aperPhot( + imageData, + 1, + psf_data['comp'][i, 0], + psf_data['comp'][i, 1], + aper, + annulus, + fast_mode=fast_aperture_mask, + sigma_hint=comp_frame_sigma, + )[0] norm_flux.append(tFlux / cFlux) # close file + delete from memory @@ -7589,7 +7790,7 @@ def initialize_aperture_data_store(frame_count, aperture_count, annulus_count, c return aper_data -def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=True): +def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=False, sigma_hint=np.nan): flux_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) bg_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) @@ -7613,7 +7814,20 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast stage_start = perf_counter() try: if annulus_width > 0: - bgflux, _, _ = skybg_phot(data, star_index, xc, yc, float(aperture_radius) + 2, float(annulus_width)) + sky_geometry = resolve_sky_annulus_geometry( + aperture_radius=float(aperture_radius), + annulus_width=float(annulus_width), + psf_sigma=sigma_hint, + ) + bgflux, _, _ = skybg_phot( + data, + star_index, + xc, + yc, + sky_geometry['inner_radius'], + sky_geometry['annulus_width'], + fast_mode=fast_mode, + ) else: bgflux = 0 @@ -7648,12 +7862,14 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta frame_apertures, frame_annuli, fast_mode=fast_aperture_mask, + sigma_hint=frame_sigma, ) aper_data['target'][frame_index] = target_flux aper_data['target_bg'][frame_index] = target_bg for comp_idx in range(comp_star_count): ckey = f"comp{comp_idx + 1}" + comp_sigma = psf_sigma_from_fit(psf_data[ckey][frame_index], fallback_sigma=frame_sigma) comp_flux, comp_bg = compute_star_aperture_grid( image_data, comp_idx + 1, @@ -7662,6 +7878,7 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta frame_apertures, frame_annuli, fast_mode=fast_aperture_mask, + sigma_hint=comp_sigma, ) aper_data[ckey][frame_index] = comp_flux aper_data[f"{ckey}_bg"][frame_index] = comp_bg @@ -9605,11 +9822,22 @@ def main(): opt_method = "Aperture" min_aper_fov = float(display_aperture) min_annulus_fov = float(display_annulus) - - plot_fov(display_aperture, display_annulus, sigma_display, + + fov_aperture = min_aper_fov if opt_method == "PSF" else float(display_aperture) + fov_annulus = min_annulus_fov if opt_method == "PSF" else float(display_annulus) + fov_sky_geometry = resolve_sky_annulus_geometry( + fov_aperture, + fov_annulus, + psf_sigma=sigma_display, + ) + + plot_fov(fov_aperture, fov_annulus, sigma_display, centroid_positions['x_targ'][0], centroid_positions['y_targ'][0], centroid_positions['x_ref'][0], centroid_positions['y_ref'][0], - firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov) + firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], + exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov, + sky_inner_radius=fov_sky_geometry['inner_radius'], + sky_outer_radius=fov_sky_geometry['outer_radius']) plot_centroids(centroid_positions['x_targ'], centroid_positions['y_targ'], centroid_positions['x_ref'], centroid_positions['y_ref'], diff --git a/exotic/inputs.py b/exotic/inputs.py index 23da40ee..491e5b4c 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -206,7 +206,7 @@ def __init__(self, init_opt): 'wl_min': None, 'wl_max': None, 'pixel_scale': None, 'exposure': None, 'dist': None, 'pm_ra': None, 'pm_dec': None, 'airmass_already_corrected': False, 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, - 'fast_aperture_mask': True, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', + 'fast_aperture_mask': False, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'detrend_on_outoftransit_baseline': True, 'final_fit_baseline_duration_multiplier': 1.0, diff --git a/exotic/plots.py b/exotic/plots.py index 13a13f72..7d439838 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -49,7 +49,8 @@ def plot_centroids(x_targ, y_targ, x_ref, y_ref, times, target_name, save, date) plt.savefig(Path(save) / "temp" / f"CentroidPositions&Distances_{target_name}_{date}.pdf") plt.close() -def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_scale, targ_name, save, date, opt_method, min_aper_fov, min_annulus_fov): +def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_scale, targ_name, save, date, + opt_method, min_aper_fov, min_annulus_fov, sky_inner_radius=None, sky_outer_radius=None): ref_circle, ref_circle_sky = None, None picframe = 10. * (aper + 15. * sigma) @@ -68,13 +69,27 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc # Create the target circles # We are using abs(aper) to account for a negative aperture in case EXOTIC is not using a comparison star + if sky_inner_radius is None or sky_outer_radius is None: + local_sky_inner_radius = abs(aper) + 2.0 + if np.isfinite(sigma) and sigma > 0: + local_sky_inner_radius = max(local_sky_inner_radius, 2.0 * 2.355 * float(sigma)) + local_sky_outer_radius = max( + local_sky_inner_radius + annulus, + np.sqrt(local_sky_inner_radius ** 2 + 250.0 / np.pi), + ) + else: + local_sky_inner_radius = float(sky_inner_radius) + local_sky_outer_radius = float(sky_outer_radius) + target_circle = plt.Circle((x_targ, y_targ), abs(aper), color=outer_circle_color, fill=False, ls='-') - target_circle_sky = plt.Circle((x_targ, y_targ), abs(aper) + annulus, color=outer_circle_color, fill=False, ls='-') + target_circle_sky_inner = plt.Circle((x_targ, y_targ), local_sky_inner_radius, color=outer_circle_color, fill=False, ls='--') + target_circle_sky_outer = plt.Circle((x_targ, y_targ), local_sky_outer_radius, color=outer_circle_color, fill=False, ls='-') # IF EXOTIC is using a comparison star, create its circles if aper >= 0: ref_circle = plt.Circle((x_ref, y_ref), aper, color=outer_circle_color, fill=False, ls='-') - ref_circle_sky = plt.Circle((x_ref, y_ref), aper + annulus, color=outer_circle_color, fill=False, ls='-') + ref_circle_sky_inner = plt.Circle((x_ref, y_ref), local_sky_inner_radius, color=outer_circle_color, fill=False, ls='--') + ref_circle_sky = plt.Circle((x_ref, y_ref), local_sky_outer_radius, color=outer_circle_color, fill=False, ls='-') interval = ZScaleInterval() vmin, vmax = interval.get_limits(image) @@ -85,14 +100,16 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc fig.colorbar(im) ax.add_artist(target_circle) - ax.add_artist(target_circle_sky) - ax.text(x_targ + abs(aper) + annulus + 5, y_targ, targ_name, color='w', fontsize=10, + ax.add_artist(target_circle_sky_inner) + ax.add_artist(target_circle_sky_outer) + ax.text(x_targ + local_sky_outer_radius + 5, y_targ, targ_name, color='w', fontsize=10, path_effects=[path_effects.withStroke(linewidth=2, foreground='black')]) if aper >= 0: #EXOTIC is using a comparison star ax.add_artist(ref_circle) + ax.add_artist(ref_circle_sky_inner) ax.add_artist(ref_circle_sky) - ax.text(x_ref + aper + annulus + 5, y_ref, 'Comp Star', color='w', fontsize=10, + ax.text(x_ref + local_sky_outer_radius + 5, y_ref, 'Comp Star', color='w', fontsize=10, path_effects=[path_effects.withStroke(linewidth=2, foreground='black')]) handles = [] @@ -387,9 +404,9 @@ def plot_adaptive_aperture_diagnostics(times, aperture_series, annulus_series, f axes[0, 0].grid(alpha=0.25) annulus_mask = valid_time & valid_annulus - axes[0, 1].set_title("Annulus Radius vs Time") + axes[0, 1].set_title("Annulus Width vs Time") axes[0, 1].set_xlabel(f"Time [BJD_TDB-{time_zero:.5f}]") - axes[0, 1].set_ylabel("Annulus Radius [px]") + axes[0, 1].set_ylabel("Annulus Width [px]") if np.any(annulus_mask): axes[0, 1].plot(times[annulus_mask] - time_zero, annulus_series[annulus_mask], color='tab:orange', marker='o', ms=3, lw=1.1) diff --git a/inits.json b/inits.json index f346ada2..31060768 100644 --- a/inits.json +++ b/inits.json @@ -22,7 +22,7 @@ "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", - "Fast Aperture Mask": "Default true/fast mode for quicker aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to false to opt out and use exact masks.", + "Fast Aperture Mask": "Default false/exact mode for fractional-pixel aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to true to opt into center-based masks for speed.", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Set to 0 to disable. Default 4.", @@ -103,7 +103,7 @@ "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, - "Fast Aperture Mask (y/n)": true, + "Fast Aperture Mask (y/n)": false, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "pointing_rejection_sigma": 4.0, diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 669cf569..80716de3 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -51,7 +51,7 @@ def __call__(self, *args, **kwargs): if not _module_available("photutils"): _install_stub_module("photutils") if not _module_available("photutils.aperture"): - _install_stub_module("photutils.aperture", CircularAperture=object) + _install_stub_module("photutils.aperture", CircularAperture=object, CircularAnnulus=object) if not _module_available("photutils.detection"): _install_stub_module("photutils.detection", DAOStarFinder=_DummyDAOStarFinder) if not _module_available("colour_demosaicing"): @@ -193,6 +193,36 @@ def test_fit_centroid_reports_consistent_background_between_fast_and_full_modes( assert full_result[6] == pytest.approx(fast_result[6], abs=1e-8) +def test_fit_centroid_full_mode_preserves_psf_subpixel_solution(): + rng = np.random.default_rng(7) + true_center = (40.3, 35.7) + image = _gaussian_image( + center=true_center, + amplitude=120.0, + sigma=0.8, + background=1000.0, + ) + image += rng.normal(0.0, 20.0, size=image.shape) + + full_result = exotic_module.fit_centroid(image, [40.0, 36.0], 0, fast_mode=False) + psf_result = exotic_module.fit_centroid( + image, + [40.0, 36.0], + 0, + fast_mode=False, + weightedcenter=False, + ) + moment_result = exotic_module.fit_centroid(image, [40.0, 36.0], 0, fast_mode=True) + + assert full_result[0] == pytest.approx(psf_result[0], abs=1e-6) + assert full_result[1] == pytest.approx(psf_result[1], abs=1e-6) + + psf_error = np.hypot(psf_result[0] - true_center[0], psf_result[1] - true_center[1]) + moment_error = np.hypot(moment_result[0] - true_center[0], moment_result[1] - true_center[1]) + + assert psf_error < moment_error + + def test_fit_centroid_or_warn_out_of_frame_skips_centroid_fit(monkeypatch): image = np.zeros((40, 50), dtype=float) out_of_frame_warnings = [] @@ -218,14 +248,20 @@ def test_skybg_phot_returns_nan_when_annulus_box_is_empty(monkeypatch): image = np.zeros((20, 20), dtype=float) sky_warnings = [] - monkeypatch.setattr( - exotic_module, - "mesh_box", - lambda *args, **kwargs: ( - np.empty((0, 0), dtype=int), - np.empty((0, 0), dtype=int), - ), - ) + class _EmptyAnnulusMask: + data = np.empty((0, 0), dtype=float) + + def cutout(self, *args, **kwargs): + return None + + class _EmptyCircularAnnulus: + def __init__(self, *args, **kwargs): + pass + + def to_mask(self, *args, **kwargs): + return [_EmptyAnnulusMask()] + + monkeypatch.setattr(exotic_module, "CircularAnnulus", _EmptyCircularAnnulus) monkeypatch.setattr( exotic_module.plateStatus, "skyBackgroundWarning", @@ -240,6 +276,30 @@ def test_skybg_phot_returns_nan_when_annulus_box_is_empty(monkeypatch): assert sky_warnings == [(0, 30.0, 30.0)] +@pytest.mark.skipif(not _module_available("photutils.aperture"), reason="requires photutils aperture masks") +def test_skybg_phot_exact_annulus_uses_fractional_pixel_area(): + image = np.ones((80, 80), dtype=float) + + bgflux, sigmabg, nbg = exotic_module.skybg_phot(image, 0, 40.3, 35.7, r=3.0, dr=2.0, fast_mode=False) + + assert bgflux == pytest.approx(1.0, abs=1e-8) + assert sigmabg == pytest.approx(0.0, abs=1e-8) + assert nbg == pytest.approx(np.pi * ((3.0 + 2.0) ** 2 - 3.0 ** 2), rel=1e-3) + assert not np.isclose(nbg, round(nbg), atol=1e-6) + + +@pytest.mark.skipif(not _module_available("photutils.aperture"), reason="requires photutils aperture masks") +def test_skybg_phot_high_side_clipping_rejects_hot_pixel(): + image = np.full((80, 80), 100.0, dtype=float) + image[40, 55] = 10000.0 + + bgflux, sigmabg, nbg = exotic_module.skybg_phot(image, 0, 40.0, 40.0, r=10.0, dr=10.0, fast_mode=False) + + assert bgflux == pytest.approx(100.0, abs=1e-8) + assert sigmabg == pytest.approx(0.0, abs=1e-8) + assert nbg > 250.0 + + def test_check_target_pixel_wcs_keeps_input_coords_when_wcs_target_is_off_frame(monkeypatch): image = np.zeros((100, 120), dtype=float) wcs = WCS(naxis=2) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 0e9310fc..da95e3a6 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -33,6 +33,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: fake_photutils = types.ModuleType("photutils") fake_photutils_aperture = types.ModuleType("photutils.aperture") fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_aperture.CircularAnnulus = type("CircularAnnulus", (), {}) fake_photutils_detection = types.ModuleType("photutils.detection") fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) fake_ldtk = types.ModuleType("ldtk") @@ -113,6 +114,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: phase_bin_sigma_clip, prepare_final_fit_lightcurve_series, representative_psf_sigma, + resolve_sky_annulus_geometry, run_target_driven_photometry_search, resolve_frame_aperture_radii, robust_flux_floor_mask, @@ -663,6 +665,14 @@ def test_resolve_frame_aperture_radii_scales_sigma_grid(): assert np.allclose(annuli, np.array([12.0, 15.0])) +def test_resolve_sky_annulus_geometry_enforces_fwhm_floor_and_min_sky_pixels(): + geometry = resolve_sky_annulus_geometry(aperture_radius=1.5, annulus_width=2.0, psf_sigma=1.0) + + assert geometry["inner_radius"] == pytest.approx(2.0 * 2.355) + assert geometry["effective_sky_pixels"] == pytest.approx(250.0, abs=1e-9) + assert geometry["annulus_width"] > 2.0 + + def test_representative_psf_sigma_uses_valid_frames_and_fallback(): psf_rows = np.array([ [0.0, 0.0, 1.0, 2.0, 2.0, 0.0, 0.0], diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 1d4070ea..482ab5b1 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -34,6 +34,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: fake_photutils = types.ModuleType("photutils") fake_photutils_aperture = types.ModuleType("photutils.aperture") fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_aperture.CircularAnnulus = type("CircularAnnulus", (), {}) fake_photutils_detection = types.ModuleType("photutils.detection") fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) fake_ldtk = types.ModuleType("ldtk") diff --git a/tests/test_inputs.py b/tests/test_inputs.py index a89fc904..c2b6ef45 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -263,6 +263,21 @@ def test_comp_params_defaults_use_aperture_photometry_to_yes(tmp_path): assert inputs.info_dict["use_aperture_photometry"] == "y" +def test_comp_params_defaults_fast_aperture_mask_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["fast_aperture_mask"] is False + + def test_comp_params_defaults_use_adaptive_apertures_to_false(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_lazy_pylightcurve_imports.py b/tests/test_lazy_pylightcurve_imports.py index 9602eb2f..72e1e3dc 100644 --- a/tests/test_lazy_pylightcurve_imports.py +++ b/tests/test_lazy_pylightcurve_imports.py @@ -130,6 +130,7 @@ def test_import_exotic_avoids_unused_astroquery_modules(): fake_photutils = types.ModuleType("photutils") fake_photutils_aperture = types.ModuleType("photutils.aperture") fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) + fake_photutils_aperture.CircularAnnulus = type("CircularAnnulus", (), {}) fake_photutils_detection = types.ModuleType("photutils.detection") fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) fake_ldtk = types.ModuleType("ldtk") diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index a42dcc43..4698d641 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -39,6 +39,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: fake_photutils = types.ModuleType("photutils") fake_photutils_aperture = types.ModuleType("photutils.aperture") fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_aperture.CircularAnnulus = type("CircularAnnulus", (), {}) fake_photutils_detection = types.ModuleType("photutils.detection") fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) fake_ldtk = types.ModuleType("ldtk") From 51da382b4f2ec2e7f6b08e52d3344c26dfd2c36e Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Mon, 27 Apr 2026 14:21:14 +1000 Subject: [PATCH 025/116] Iterate comparison-star outlier rejection before target-fit ranking --- README.md | 1 + exotic/api/colab.py | 1 + exotic/exotic.py | 708 +++++++++++++++++++++++------ exotic/exotic_gui.py | 5 + exotic/inputs.py | 6 + exotic/output_files.py | 3 +- inits.json | 2 + tests/test_exotic_proper_motion.py | 330 +++++++++++++- tests/test_output_files.py | 5 + 9 files changed, 924 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index 3bd0dfe0..bab0aee6 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "detrend_on_outoftransit_baseline": true, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y", diff --git a/exotic/api/colab.py b/exotic/api/colab.py index a08fd9cf..13a1662b 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -381,6 +381,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "bad_wcs_threshold_percent": 3.0, "detrend_on_outoftransit_baseline": true, "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": false, "require_comp_star": "y" diff --git a/exotic/exotic.py b/exotic/exotic.py index d8bd24ea..abe633f2 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -174,6 +174,9 @@ COMPARISON_STAR_MIN_VALID_FRAMES = 5 COMPARISON_STAR_COVERAGE_SIGMA = 3.0 COMPARISON_STAR_COVERAGE_MAX_ITERS = 10 +COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA = 4.25 +COMPARISON_STAR_SUITABILITY_MIN_CANDIDATES = 5 +COMPARISON_STAR_SUITABILITY_MAX_ITERS = 10 OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 @@ -844,6 +847,28 @@ def should_fit_lightcurve_to_every_comparison_candidate(config_value): return False +def should_pick_comparison_by_eebls_snr(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'pick_comparison_by_eebls_snr' value; " + "keeping EEBLS SNR comparison selection enabled.", + warn=True, + ) + return True + + def should_use_psf_photometry(config_value): if config_value is None: return True @@ -1405,6 +1430,52 @@ def estimate_tmid_and_bounds_with_eebls(times, flux_values, flux_errors, prior, return summary +def annotate_lightcurve_tmid_search(fit, summary): + if fit is None: + return + + fit.initial_tmid_search_method = summary.get('method') + fit.initial_tmid_search_applied = bool(summary.get('applied')) + fit.initial_tmid_search_tmid = summary.get('tmid') + fit.initial_tmid_search_bounds = summary.get('bounds') + fit.initial_tmid_search_duration = summary.get('duration') + fit.initial_tmid_search_depth = summary.get('depth') + fit.initial_tmid_search_depth_snr = summary.get('depth_snr') + fit.initial_tmid_search_note = summary.get('note') + + +def annotate_lightcurve_eebls_diagnostic(fit, summary): + if fit is None: + return + + summary = {} if summary is None else dict(summary) + fit.eebls_diagnostic_computed = bool(summary) + fit.eebls_diagnostic_method = summary.get('method') + fit.eebls_diagnostic_applied = bool(summary.get('applied')) + fit.eebls_diagnostic_tmid = summary.get('tmid') + fit.eebls_diagnostic_bounds = summary.get('bounds') + fit.eebls_diagnostic_duration = summary.get('duration') + fit.eebls_diagnostic_depth = summary.get('depth') + fit.eebls_diagnostic_depth_snr = summary.get('depth_snr') + fit.eebls_diagnostic_note = summary.get('note') + + +def extract_lightcurve_fit_eebls_snr(fit): + if fit is None: + return np.nan + + for attr_name in ('eebls_diagnostic_depth_snr', 'initial_tmid_search_depth_snr'): + value = getattr(fit, attr_name, np.nan) + try: + numeric_value = float(value) + except (TypeError, ValueError): + continue + if np.isfinite(numeric_value): + return numeric_value + + return np.nan + + def should_use_impactparameter_rather_than_inclination_to_fit(config_value): if config_value is None: return True @@ -5308,6 +5379,21 @@ def centroid_position_is_finite(psf_row): return bool(np.all(np.isfinite(coords))) +def choose_centroid_seed_position(predicted_pos, previous_psf_row=None, max_offset_pixels=5.0): + predicted = np.asarray(predicted_pos, dtype=float).reshape(-1) + if predicted.size < 2 or not np.all(np.isfinite(predicted[:2])): + return np.array([np.nan, np.nan], dtype=float) + + if not centroid_position_is_finite(previous_psf_row): + return np.array(predicted[:2], dtype=float) + + previous = np.asarray(previous_psf_row[:2], dtype=float) + if np.hypot(*(previous - predicted[:2])) > float(max_offset_pixels): + return np.array(predicted[:2], dtype=float) + + return previous.astype(float, copy=True) + + def centroid_offset_matches_reference(psf_a, psf_b, expected_dx, expected_dy, tolerance=WCS_REFERENCE_GEOMETRY_TOLERANCE_PIXELS): if not centroid_position_is_finite(psf_a) or not centroid_position_is_finite(psf_b): @@ -6175,16 +6261,24 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet projected_coords = np.array([[tx, ty], [cx, cy]], dtype=float) projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) + target_seed = choose_centroid_seed_position( + [tx, ty], + None if i == 0 else psf_data['target'][i - 1], + ) + comp_seed = choose_centroid_seed_position( + [cx, cy], + None if i == 0 else psf_data['comp'][i - 1], + ) psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( imageData, - [tx, ty], + target_seed, 0, fast_mode=target_fast_centroid, ) psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( imageData, - [cx, cy], + comp_seed, 1, fast_mode=frame_fast_centroid, ) @@ -6223,17 +6317,25 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet dtype=float, ) tx, ty = transformed_coords[0] + target_seed = choose_centroid_seed_position( + [tx, ty], + None if i == 0 else psf_data['target'][i - 1], + ) psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( imageData, - [tx, ty], + target_seed, 0, fast_mode=target_fast_centroid, ) cx, cy = transformed_coords[1] + comp_seed = choose_centroid_seed_position( + [cx, cy], + None if i == 0 else psf_data['comp'][i - 1], + ) psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( imageData, - [cx, cy], + comp_seed, 1, fast_mode=frame_fast_centroid, ) @@ -6316,7 +6418,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, final_fit_mode='lm', use_impactparameter_rather_than_inclination_to_fit=True, plot_time_range=None, - use_eebls_to_initialize_tmid_and_bounds=True): + use_eebls_to_initialize_tmid_and_bounds=True, + compute_eebls_diagnostics=False): # remove outliers plot_time_range = np.asarray(times if plot_time_range is None else plot_time_range, dtype=float) si = np.argsort(times) @@ -6442,15 +6545,17 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, if tmid_search_summary.get('duration_capped'): log_info(tmid_search_summary['note']) - if use_eebls_to_initialize_tmid_and_bounds: - tmid_search_summary = estimate_tmid_and_bounds_with_eebls( + eebls_search_summary = None + if use_eebls_to_initialize_tmid_and_bounds or compute_eebls_diagnostics: + eebls_search_summary = estimate_tmid_and_bounds_with_eebls( arrayTimes, arrayFinalFlux, arrayNormUnc, prior, [lower, upper], ) - if tmid_search_summary.get('applied'): + if use_eebls_to_initialize_tmid_and_bounds and eebls_search_summary.get('applied'): + tmid_search_summary = eebls_search_summary prior['tmid'] = tmid_search_summary['tmid'] lower, upper = tmid_search_summary['bounds'] @@ -6488,12 +6593,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) - if myfit is not None: - myfit.initial_tmid_search_method = tmid_search_summary.get('method') - myfit.initial_tmid_search_applied = bool(tmid_search_summary.get('applied')) - myfit.initial_tmid_search_tmid = tmid_search_summary.get('tmid') - myfit.initial_tmid_search_bounds = tmid_search_summary.get('bounds') - myfit.initial_tmid_search_note = tmid_search_summary.get('note') + annotate_lightcurve_tmid_search(myfit, tmid_search_summary) + annotate_lightcurve_eebls_diagnostic(myfit, eebls_search_summary) if ( myfit is not None @@ -6533,12 +6634,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) - if myfit is not None: - myfit.initial_tmid_search_method = tmid_search_summary.get('method') - myfit.initial_tmid_search_applied = bool(tmid_search_summary.get('applied')) - myfit.initial_tmid_search_tmid = tmid_search_summary.get('tmid') - myfit.initial_tmid_search_bounds = tmid_search_summary.get('bounds') - myfit.initial_tmid_search_note = tmid_search_summary.get('note') + annotate_lightcurve_tmid_search(myfit, tmid_search_summary) + annotate_lightcurve_eebls_diagnostic(myfit, eebls_search_summary) debug_phase_clip_keep_mask = np.ones(np.count_nonzero(debug_initial_sigma_keep_mask), dtype=bool) if final_fit_mode == 'ns' and myfit is not None: @@ -6564,11 +6661,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) if myfit is not None: - myfit.initial_tmid_search_method = tmid_search_summary.get('method') - myfit.initial_tmid_search_applied = bool(tmid_search_summary.get('applied')) - myfit.initial_tmid_search_tmid = tmid_search_summary.get('tmid') - myfit.initial_tmid_search_bounds = tmid_search_summary.get('bounds') - myfit.initial_tmid_search_note = tmid_search_summary.get('note') + annotate_lightcurve_tmid_search(myfit, tmid_search_summary) + annotate_lightcurve_eebls_diagnostic(myfit, eebls_search_summary) annotate_nested_tmid_refinement( myfit, nested_refinement.get('applied', False), @@ -6789,6 +6883,7 @@ def evaluate_lightcurve_candidate(task): disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, ) = task fit_diagnostics = diagnose_lightcurve_fit_inputs( times, @@ -6810,6 +6905,7 @@ def evaluate_lightcurve_candidate(task): use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics=compute_eebls_diagnostics, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -6824,6 +6920,7 @@ def evaluate_lightcurve_candidate(task): return { 'myfit': myfit, 'res_std': res_std, + 'eebls_snr': extract_lightcurve_fit_eebls_snr(myfit), 'fit_diagnostics': fit_diagnostics, 'failure_reason': fit_diagnostics.get('failure_reason'), 'fit_point_count': 0 if tflux_fit is None else int(len(tflux_fit)), @@ -6962,7 +7059,8 @@ def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, p plot_time_range=None, disable_vertical_flux_normalization=False, use_impactparameter_rather_than_inclination_to_fit=True, - use_eebls_to_initialize_tmid_and_bounds=True): + use_eebls_to_initialize_tmid_and_bounds=True, + compute_eebls_diagnostics=True): candidate_mask = np.asarray(candidate['mask'], dtype=bool) if candidate['method'] == 'psf': @@ -6994,6 +7092,7 @@ def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, p disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, ) @@ -7007,7 +7106,8 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co use_aperture_photometry=True, multiprocess_lightcurve_fits=None, use_impactparameter_rather_than_inclination_to_fit=True, - use_eebls_to_initialize_tmid_and_bounds=True): + use_eebls_to_initialize_tmid_and_bounds=True, + pick_comparison_by_eebls_snr=True): candidate_jobs = build_target_fit_candidate_jobs( psf_data, aper_data, @@ -7030,6 +7130,8 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co 'best_candidate': None, 'best_fit_lc': None, 'min_std': np.inf, + 'selection_metric': 'residual_scatter', + 'selected_eebls_snr': np.nan, 'flux_tar': None, 'flux_ref': None, } @@ -7048,6 +7150,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics=True, ) for candidate in shortlist ] @@ -7059,27 +7162,56 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co else: fit_results = [evaluate_lightcurve_candidate(task) for task in fit_tasks] - best_candidate = None - best_fit_lc = None - best_res_std = np.inf - best_tflux = None - best_cflux = None candidate_summaries = [] + successful_candidates = [] for candidate, result in zip(shortlist, fit_results): fit_meta, tflux_fit, cflux_fit = result - candidate_summaries.append(summarize_target_fit_candidate(candidate, fit_meta, comp_stars)) + summary = summarize_target_fit_candidate(candidate, fit_meta, comp_stars) + candidate_summaries.append(summary) if fit_meta is None or fit_meta.get('myfit') is None: continue + successful_candidates.append((summary, candidate, fit_meta, tflux_fit, cflux_fit)) - if fit_meta['res_std'] < best_res_std: - best_candidate = candidate - best_fit_lc = fit_meta['myfit'] - best_res_std = fit_meta['res_std'] - best_tflux = tflux_fit - best_cflux = cflux_fit + best_candidate = None + best_fit_lc = None + best_res_std = np.inf + best_tflux = None + best_cflux = None + selection_metric = 'residual_scatter' + selected_eebls_snr = np.nan + if successful_candidates: + if pick_comparison_by_eebls_snr and any( + np.isfinite(summary.get('eebls_snr', np.nan)) + for summary, _, _, _, _ in successful_candidates + ): + selection_metric = 'eebls_snr' + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + item[0].get('res_std', np.inf), + item[0].get('prescore', np.inf), + item[0].get('comp_index', np.inf), + ), + ) + else: + selected_entry = min( + successful_candidates, + key=lambda item: ( + item[0].get('res_std', np.inf), + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + item[0].get('prescore', np.inf), + item[0].get('comp_index', np.inf), + ), + ) - if best_candidate is not None: + selected_summary, best_candidate, fit_meta, best_tflux, best_cflux = selected_entry + best_fit_lc = fit_meta['myfit'] + best_res_std = selected_summary.get('res_std', np.inf) + selected_eebls_snr = selected_summary.get('eebls_snr', np.nan) best_identity = target_fit_candidate_identity(best_candidate) for summary in candidate_summaries: summary['selected'] = target_fit_candidate_identity(summary) == best_identity @@ -7091,6 +7223,8 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co 'best_candidate': best_candidate, 'best_fit_lc': best_fit_lc, 'min_std': best_res_std, + 'selection_metric': selection_metric, + 'selected_eebls_snr': selected_eebls_snr, 'flux_tar': best_tflux, 'flux_ref': best_cflux, } @@ -7175,6 +7309,24 @@ def format_comp_star_coverage_text(summary): return coverage_text +def format_eebls_snr(value): + if value is None: + return "n/a" + + try: + numeric_value = float(value) + except (TypeError, ValueError): + return "n/a" + + return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value:.2f}" + + +def comparison_selection_metric_label(selection_metric): + if selection_metric == 'eebls_snr': + return "EEBLS SNR" + return "target-fit residual scatter" + + def target_fit_candidate_identity(candidate): return ( candidate.get('method'), @@ -7202,6 +7354,7 @@ def summarize_target_fit_candidate(candidate, fit_meta, comp_stars): 'coverage_min_required_count': candidate.get('coverage_min_required_count', 0), 'coverage_rejected': candidate.get('coverage_rejected', False), 'fit_point_count': fit_meta.get('fit_point_count', 0), + 'eebls_snr': fit_meta.get('eebls_snr', np.nan), 'fit_diagnostics': fit_meta.get('fit_diagnostics') or {}, 'failure_reason': fit_meta.get('failure_reason'), 'parameter_summary': summarize_lightcurve_fit_parameters(fit_result), @@ -7230,6 +7383,7 @@ def log_comparison_calibration_fit_attempt_summaries(attempts, method_label): residual_text = "n/a" if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)): residual_text = f"{attempt['res_std'] * 100.0:.4f}%" + eebls_text = format_eebls_snr(attempt.get('eebls_snr', np.nan)) reason_text = attempt.get('selection_reason') or attempt.get( 'failure_reason', "selected: lowest target-fit residual scatter among the evaluated comparison stars", @@ -7238,7 +7392,7 @@ def log_comparison_calibration_fit_attempt_summaries(attempts, method_label): f" {attempt['label']}{selected_label} ({position_text}): " f"suitability={suitability_text}, coverage={coverage_text}, " f"usable_after_filters={usable_point_count}, fit_points={attempt.get('fit_point_count', 0)}, " - f"residual_scatter={residual_text}, reason={reason_text}" + f"eebls_snr={eebls_text}, residual_scatter={residual_text}, reason={reason_text}" ) parameter_summary = attempt.get('parameter_summary') if parameter_summary: @@ -7263,6 +7417,7 @@ def log_target_fit_candidate_summaries(candidate_summaries, max_entries=10): residual_text = "n/a" if summary.get('fit') is not None and np.isfinite(summary.get('res_std', np.inf)): residual_text = f"{summary['res_std'] * 100.0:.4f}%" + eebls_text = format_eebls_snr(summary.get('eebls_snr', np.nan)) reason_text = summary.get( 'failure_reason', "selected: lowest target-fit residual scatter in the evaluated shortlist", @@ -7271,7 +7426,7 @@ def log_target_fit_candidate_summaries(candidate_summaries, max_entries=10): f" {summary['label']}{selected_label} ({position_text}) with {summary['method_label']}: " f"prescore={prescore_text}, coverage={coverage_text}, " f"usable_after_filters={usable_point_count}, fit_points={summary.get('fit_point_count', 0)}, " - f"residual_scatter={residual_text}, reason={reason_text}" + f"eebls_snr={eebls_text}, residual_scatter={residual_text}, reason={reason_text}" ) parameter_summary = summary.get('parameter_summary') if parameter_summary: @@ -7286,7 +7441,7 @@ def log_target_fit_candidate_summaries(candidate_summaries, max_entries=10): def comparison_calibration_selection_reason(summary, best_comp_score): if summary.get('selected'): - return "selected: lowest suitability score among coverage-qualified comparison stars for this method" + return "selected: lowest suitability score among coverage-qualified, sigma-clip-qualified comparison stars for this method" if summary.get('coverage_rejected'): return ( @@ -7294,6 +7449,15 @@ def comparison_calibration_selection_reason(summary, best_comp_score): f"({summary['coverage_count']} < {summary['coverage_min_required_count']} valid frames)" ) + if summary.get('suitability_outlier_rejected'): + threshold = summary.get('suitability_high_threshold', np.nan) + if np.isfinite(threshold): + return ( + "not selected: suitability score was rejected by high-side sigma clipping " + f"({summary['aggregate_score'] * 100.0:.4f}% > {threshold * 100.0:.4f}%)" + ) + return "not selected: suitability score was rejected by high-side sigma clipping" + aggregate_score = summary.get('aggregate_score', np.inf) if not np.isfinite(aggregate_score): return "not selected: no usable ensemble or pairwise calibration score" @@ -7314,9 +7478,12 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): return summary['failure_reason'] selection_basis = photometry_info.get('selection_basis', 'target_fit') + selection_metric = photometry_info.get('selection_metric', 'residual_scatter') selected_comp_num = photometry_info.get('comp_star_num') selected_res_std = photometry_info.get('min_std', np.inf) candidate_res_std = summary.get('res_std', np.inf) + selected_eebls_snr = photometry_info.get('comparison_eebls_snr', np.nan) + candidate_eebls_snr = summary.get('eebls_snr', np.nan) if summary.get('selected'): if selection_basis == 'comparison_field': @@ -7326,6 +7493,8 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): "selected: comparison-field calibration fell back to this star " "after better-ranked candidates failed target fitting" ) + if selection_metric == 'eebls_snr' and np.isfinite(candidate_eebls_snr): + return "selected: highest EEBLS SNR in the chosen search" return "selected: lowest target-fit residual scatter in the chosen search" if selection_basis == 'comparison_field': @@ -7340,6 +7509,20 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): f"Comp {selected_comp_num} after better-ranked candidate(s) failed target fitting" ) + if selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + if not np.isfinite(candidate_eebls_snr): + return "not selected: no finite EEBLS SNR was available for this candidate" + if candidate_eebls_snr < selected_eebls_snr - 1e-12: + return ( + "not selected: EEBLS SNR was " + f"{candidate_eebls_snr:.2f} vs {selected_eebls_snr:.2f} for the selected fit" + ) + if candidate_eebls_snr > selected_eebls_snr + 1e-12: + return ( + "not selected: this post-selection diagnostic fit has a stronger " + "EEBLS box signal than the selected fit; the earlier search did not choose it" + ) + if np.isfinite(candidate_res_std) and np.isfinite(selected_res_std): if candidate_res_std > selected_res_std + 1e-12: return ( @@ -7412,6 +7595,10 @@ def log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_i selection_basis = photometry_info.get('selection_basis', 'target_fit').replace('_', '-') log_info("\nComparison-star lightcurve fit diagnostics:") log_info(f"Selection basis: {selection_basis}") + log_info( + "Selection metric: " + f"{comparison_selection_metric_label(photometry_info.get('selection_metric', 'residual_scatter'))}" + ) for summary in candidate_fit_summaries: selected_label = " [selected]" if summary.get('selected') else "" @@ -7422,12 +7609,13 @@ def log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_i residual_text = "n/a" if summary.get('fit') is not None and np.isfinite(summary.get('res_std', np.inf)): residual_text = f"{summary['res_std'] * 100.0:.4f}%" + eebls_text = format_eebls_snr(summary.get('eebls_snr', np.nan)) reason_text = comparison_candidate_fit_selection_reason(summary, photometry_info) log_info( f" {summary['label']}{selected_label} ({position_text}): " f"coverage={coverage_text}, " f"usable_after_filters={usable_point_count}, fit_points={summary['fit_point_count']}, " - f"residual_scatter={residual_text}, reason={reason_text}" + f"eebls_snr={eebls_text}, residual_scatter={residual_text}, reason={reason_text}" ) parameter_summary = summary.get('parameter_summary') if parameter_summary: @@ -7529,6 +7717,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics=True, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -7551,6 +7740,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p 'selected': selected_comp_star_num == comp_index + 1, 'fit': fit_result, 'res_std': res_std, + 'eebls_snr': extract_lightcurve_fit_eebls_snr(fit_result), 'coverage_count': coverage_count, 'coverage_total_frame_count': coverage_total_frame_count, 'coverage_reference_count': coverage_reference_count, @@ -7665,6 +7855,70 @@ def comparison_star_coverage_summary(comp_flux_map, return coverage_summary +def apply_comparison_star_suitability_outlier_rejection( + comp_summaries, + sigma=COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA, + min_candidates=COMPARISON_STAR_SUITABILITY_MIN_CANDIDATES, + eligible_indices=None, +): + if eligible_indices is None: + eligible_indices = [ + index + for index, summary in enumerate(comp_summaries) + if ( + not summary.get('coverage_rejected') + and np.isfinite(summary.get('aggregate_score', np.inf)) + ) + ] + else: + eligible_indices = [ + int(index) + for index in eligible_indices + if ( + 0 <= int(index) < len(comp_summaries) + and not comp_summaries[int(index)].get('coverage_rejected') + and np.isfinite(comp_summaries[int(index)].get('aggregate_score', np.inf)) + ) + ] + clipping_candidate_floor = max(3, int(min_candidates)) + reference_score = np.nan + scatter = np.nan + high_threshold = np.nan + kept_indices = list(eligible_indices) + rejected_index_set = set() + + if len(eligible_indices) >= clipping_candidate_floor: + eligible_scores = np.asarray( + [comp_summaries[index]['aggregate_score'] for index in eligible_indices], + dtype=float, + ) + if eligible_scores.size and np.any(np.isfinite(eligible_scores)): + reference_score = float(np.nanmedian(eligible_scores)) + scatter = robust_scatter(eligible_scores) + if np.isfinite(scatter) and scatter > 0: + high_threshold = reference_score + float(sigma) * scatter + kept_indices = [ + index for index in eligible_indices + if comp_summaries[index]['aggregate_score'] <= high_threshold + ] + rejected_index_set = set(eligible_indices) - set(kept_indices) + + for index, summary in enumerate(comp_summaries): + summary['suitability_outlier_rejected'] = index in rejected_index_set + summary['suitability_reference_score'] = reference_score + summary['suitability_scatter'] = scatter + summary['suitability_high_threshold'] = high_threshold + + return { + 'eligible_indices': eligible_indices, + 'active_indices': kept_indices, + 'rejected_indices': sorted(rejected_index_set), + 'reference_score': reference_score, + 'scatter': scatter, + 'high_threshold': high_threshold, + } + + def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_rejection=False, validity_mask_func=valid_comparison_frame_mask): if not comp_flux_map: @@ -7674,6 +7928,10 @@ def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_ 'field_score': np.inf, 'best_comp_index': None, 'best_comp_score': np.inf, + 'suitability_outlier_rejected_count': 0, + 'suitability_high_threshold': np.nan, + 'suitability_reference_score': np.nan, + 'suitability_scatter': np.nan, } comp_keys = list(comp_flux_map.keys()) @@ -7686,75 +7944,159 @@ def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_ skip_rejection=skip_low_coverage_rejection, validity_mask_func=validity_mask_func, ) - eligible_keys = { + coverage_qualified_keys = [ key for key in comp_keys if not coverage_summary[key]['coverage_rejected'] - } - pairwise_matrix = np.full((len(comp_keys), len(comp_keys)), np.nan, dtype=float) - comp_summaries = [] + ] - for i, key in enumerate(comp_keys): - normalized_flux = normalized_flux_map[key] - self_score = cheap_lightcurve_prescore(normalized_flux, np.ones(normalized_flux.shape[0]), airmass) - pairwise_scores = [] - pairwise_series = {} + def build_stability_iteration(active_keys): + active_key_set = set(active_keys) + active_flux_map = { + eligible_key: normalized_flux_map[eligible_key] + for eligible_key in active_keys + } + pairwise_matrix = np.full((len(comp_keys), len(comp_keys)), np.nan, dtype=float) + comp_summaries = [] - for j, other_key in enumerate(comp_keys): - if i == j or other_key not in eligible_keys: + for i, key in enumerate(comp_keys): + normalized_flux = normalized_flux_map[key] + self_score = cheap_lightcurve_prescore(normalized_flux, np.ones(normalized_flux.shape[0]), airmass) + pairwise_scores = [] + pairwise_series = {} + + for j, other_key in enumerate(comp_keys): + if i == j or other_key not in active_key_set: + continue + other_flux = normalized_flux_map[other_key] + score = cheap_lightcurve_prescore(normalized_flux, other_flux, airmass) + pairwise_matrix[i, j] = score + pairwise_series[f"vs {j + 1}"] = normalized_ratio_series(normalized_flux, other_flux) + if np.isfinite(score): + pairwise_scores.append(float(score)) + + ensemble_flux = build_normalized_comp_ensemble(active_flux_map, key) + ensemble_score = np.inf + ensemble_ratio_series = np.full(normalized_flux.shape, np.nan, dtype=float) + if ensemble_flux is not None: + ensemble_score = cheap_lightcurve_prescore(normalized_flux, ensemble_flux, airmass) + ensemble_ratio_series = normalized_ratio_series(normalized_flux, ensemble_flux) + + if pairwise_scores: + pairwise_median = float(np.nanmedian(pairwise_scores)) + pairwise_max = float(np.nanmax(pairwise_scores)) + pairwise_upper = float(np.nanpercentile(pairwise_scores, 75)) + else: + pairwise_median = np.inf + pairwise_max = np.inf + pairwise_upper = np.inf + + aggregate_inputs = [score for score in (ensemble_score, pairwise_upper) if np.isfinite(score)] + aggregate_score = max(aggregate_inputs) if aggregate_inputs else self_score + if coverage_summary[key]['coverage_rejected']: + aggregate_score = np.inf + + comp_summaries.append({ + 'comp_index': i, + 'key': key, + 'label': f"Comp {i + 1}", + 'pairwise_median_score': pairwise_median, + 'pairwise_max_score': pairwise_max, + 'ensemble_score': float(ensemble_score) if np.isfinite(ensemble_score) else np.inf, + 'self_score': float(self_score) if np.isfinite(self_score) else np.inf, + 'aggregate_score': float(aggregate_score) if np.isfinite(aggregate_score) else np.inf, + 'valid_pair_count': len(pairwise_scores), + 'pairwise_ratio_series': pairwise_series, + 'ensemble_ratio_series': ensemble_ratio_series, + 'coverage_count': coverage_summary[key]['coverage_count'], + 'coverage_total_frame_count': coverage_summary[key]['coverage_total_frame_count'], + 'coverage_reference_count': coverage_summary[key]['coverage_reference_count'], + 'coverage_min_required_count': coverage_summary[key]['coverage_min_required_count'], + 'coverage_rejected': coverage_summary[key]['coverage_rejected'], + 'suitability_outlier_rejected': False, + 'suitability_reference_score': np.nan, + 'suitability_scatter': np.nan, + 'suitability_high_threshold': np.nan, + }) + + return pairwise_matrix, comp_summaries + + active_keys = list(coverage_qualified_keys) + rejected_outlier_keys = set() + rejection_metadata = {} + for _ in range(COMPARISON_STAR_SUITABILITY_MAX_ITERS): + _, iteration_summaries = build_stability_iteration(active_keys) + active_indices = [comp_keys.index(key) for key in active_keys] + outlier_summary = apply_comparison_star_suitability_outlier_rejection( + iteration_summaries, + eligible_indices=active_indices, + ) + newly_rejected_indices = outlier_summary['rejected_indices'] + if not newly_rejected_indices: + break + + newly_rejected_keys = [comp_keys[index] for index in newly_rejected_indices] + if len(active_keys) - len(newly_rejected_keys) < 3: + break + + for index in newly_rejected_indices: + key = comp_keys[index] + if key in rejection_metadata: continue - other_flux = normalized_flux_map[other_key] - score = cheap_lightcurve_prescore(normalized_flux, other_flux, airmass) - pairwise_matrix[i, j] = score - pairwise_series[f"vs {j + 1}"] = normalized_ratio_series(normalized_flux, other_flux) - if np.isfinite(score): - pairwise_scores.append(float(score)) - - eligible_flux_map = {eligible_key: normalized_flux_map[eligible_key] for eligible_key in eligible_keys} - ensemble_flux = build_normalized_comp_ensemble(eligible_flux_map, key) - ensemble_score = np.inf - ensemble_ratio_series = np.full(normalized_flux.shape, np.nan, dtype=float) - if ensemble_flux is not None: - ensemble_score = cheap_lightcurve_prescore(normalized_flux, ensemble_flux, airmass) - ensemble_ratio_series = normalized_ratio_series(normalized_flux, ensemble_flux) - - if pairwise_scores: - pairwise_median = float(np.nanmedian(pairwise_scores)) - pairwise_max = float(np.nanmax(pairwise_scores)) - pairwise_upper = float(np.nanpercentile(pairwise_scores, 75)) - else: - pairwise_median = np.inf - pairwise_max = np.inf - pairwise_upper = np.inf - - aggregate_inputs = [score for score in (ensemble_score, pairwise_upper) if np.isfinite(score)] - aggregate_score = max(aggregate_inputs) if aggregate_inputs else self_score - if coverage_summary[key]['coverage_rejected']: - aggregate_score = np.inf - - comp_summaries.append({ - 'comp_index': i, - 'key': key, - 'label': f"Comp {i + 1}", - 'pairwise_median_score': pairwise_median, - 'pairwise_max_score': pairwise_max, - 'ensemble_score': float(ensemble_score) if np.isfinite(ensemble_score) else np.inf, - 'self_score': float(self_score) if np.isfinite(self_score) else np.inf, - 'aggregate_score': float(aggregate_score) if np.isfinite(aggregate_score) else np.inf, - 'valid_pair_count': len(pairwise_scores), - 'pairwise_ratio_series': pairwise_series, - 'ensemble_ratio_series': ensemble_ratio_series, - 'coverage_count': coverage_summary[key]['coverage_count'], - 'coverage_total_frame_count': coverage_summary[key]['coverage_total_frame_count'], - 'coverage_reference_count': coverage_summary[key]['coverage_reference_count'], - 'coverage_min_required_count': coverage_summary[key]['coverage_min_required_count'], - 'coverage_rejected': coverage_summary[key]['coverage_rejected'], - }) + rejection_metadata[key] = { + 'reference_score': outlier_summary['reference_score'], + 'scatter': outlier_summary['scatter'], + 'high_threshold': outlier_summary['high_threshold'], + } + rejected_outlier_keys.update(newly_rejected_keys) + active_keys = [key for key in active_keys if key not in rejected_outlier_keys] - finite_comp_scores = [summary['aggregate_score'] for summary in comp_summaries if np.isfinite(summary['aggregate_score'])] + pairwise_matrix, comp_summaries = build_stability_iteration(active_keys) + final_active_scores = np.asarray( + [ + summary['aggregate_score'] + for summary in comp_summaries + if summary['key'] in set(active_keys) and np.isfinite(summary['aggregate_score']) + ], + dtype=float, + ) + final_reference_score = np.nan + final_scatter = np.nan + final_high_threshold = np.nan + if final_active_scores.size and np.any(np.isfinite(final_active_scores)): + final_reference_score = float(np.nanmedian(final_active_scores)) + final_scatter = robust_scatter(final_active_scores) + if np.isfinite(final_scatter) and final_scatter > 0: + final_high_threshold = ( + final_reference_score + COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA * final_scatter + ) + + for summary in comp_summaries: + rejection_info = rejection_metadata.get(summary['key']) + if rejection_info is not None: + summary['suitability_outlier_rejected'] = True + summary['suitability_reference_score'] = rejection_info['reference_score'] + summary['suitability_scatter'] = rejection_info['scatter'] + summary['suitability_high_threshold'] = rejection_info['high_threshold'] + else: + summary['suitability_outlier_rejected'] = False + summary['suitability_reference_score'] = final_reference_score + summary['suitability_scatter'] = final_scatter + summary['suitability_high_threshold'] = final_high_threshold + + finite_comp_scores = [ + summary['aggregate_score'] + for summary in comp_summaries + if ( + np.isfinite(summary['aggregate_score']) + and not summary.get('suitability_outlier_rejected') + ) + ] field_score = float(np.nanmedian(finite_comp_scores)) if finite_comp_scores else np.inf best_comp_index = None best_comp_score = np.inf for summary in comp_summaries: + if summary.get('suitability_outlier_rejected'): + continue if summary['aggregate_score'] < best_comp_score: best_comp_score = summary['aggregate_score'] best_comp_index = summary['comp_index'] @@ -7765,6 +8107,10 @@ def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_ 'field_score': field_score, 'best_comp_index': best_comp_index, 'best_comp_score': best_comp_score, + 'suitability_outlier_rejected_count': len(rejected_outlier_keys), + 'suitability_high_threshold': final_high_threshold, + 'suitability_reference_score': final_reference_score, + 'suitability_scatter': final_scatter, } @@ -8067,6 +8413,8 @@ def ranked_comparison_calibration_summaries(comparison_calibration): aggregate_score = summary.get('aggregate_score', np.inf) if summary.get('coverage_rejected'): continue + if summary.get('suitability_outlier_rejected'): + continue if not np.isfinite(aggregate_score): continue ranked_summaries.append(summary) @@ -8085,7 +8433,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p plot_time_range=None, disable_vertical_flux_normalization=False, use_impactparameter_rather_than_inclination_to_fit=True, - use_eebls_to_initialize_tmid_and_bounds=True): + use_eebls_to_initialize_tmid_and_bounds=True, + pick_comparison_by_eebls_snr=True): ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) if not ranked_summaries: return { @@ -8139,6 +8488,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics=True, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -8169,6 +8519,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'cflux_fit': cflux_fit, 'fit_diagnostics': fit_diagnostics, 'res_std': res_std, + 'eebls_snr': extract_lightcurve_fit_eebls_snr(fit_result), 'fit_point_count': 0 if tflux_fit is None else int(len(tflux_fit)), 'failure_reason': fit_diagnostics.get('failure_reason'), 'parameter_summary': summarize_lightcurve_fit_parameters(fit_result), @@ -8183,32 +8534,73 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p for attempt in attempts if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)) ] + selection_metric = 'residual_scatter' if successful_attempts: - selected_result = min( - successful_attempts, - key=lambda attempt: (attempt.get('res_std', np.inf), attempt.get('rank', np.inf)), - ) + if pick_comparison_by_eebls_snr and any( + np.isfinite(attempt.get('eebls_snr', np.nan)) + for attempt in successful_attempts + ): + selection_metric = 'eebls_snr' + selected_result = min( + successful_attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + attempt.get('res_std', np.inf), + attempt.get('rank', np.inf), + ), + ) + else: + selected_result = min( + successful_attempts, + key=lambda attempt: ( + attempt.get('res_std', np.inf), + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) selected_result['selected'] = True selected_residual = selected_result.get('res_std', np.inf) + selected_eebls_snr = selected_result.get('eebls_snr', np.nan) for attempt in attempts: if attempt is selected_result: - attempt['selection_reason'] = ( - "selected: lowest target-fit residual scatter among the evaluated " - "comparison-star calibration candidates" - ) + if selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + attempt['selection_reason'] = ( + "selected: highest EEBLS SNR among the evaluated " + "comparison-star calibration candidates" + ) + else: + attempt['selection_reason'] = ( + "selected: lowest target-fit residual scatter among the evaluated " + "comparison-star calibration candidates" + ) continue if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)): - attempt['selection_reason'] = ( - "not selected: target-fit residual scatter " - f"{attempt['res_std'] * 100.0:.4f}% was higher than the selected " - f"{selected_residual * 100.0:.4f}%" - ) + if selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + if np.isfinite(attempt.get('eebls_snr', np.nan)): + attempt['selection_reason'] = ( + "not selected: EEBLS SNR " + f"{attempt['eebls_snr']:.2f} was lower than the selected " + f"{selected_eebls_snr:.2f}" + ) + else: + attempt['selection_reason'] = ( + "not selected: no finite EEBLS SNR was available for this candidate" + ) + else: + attempt['selection_reason'] = ( + "not selected: target-fit residual scatter " + f"{attempt['res_std'] * 100.0:.4f}% was higher than the selected " + f"{selected_residual * 100.0:.4f}%" + ) return { 'ranked_summaries': ranked_summaries, 'attempts': attempts, 'selected_result': selected_result, + 'selection_metric': selection_metric, } @@ -8417,6 +8809,9 @@ def main(): use_eebls_tmid_initializer = should_use_eebls_to_initialize_tmid_and_bounds( exotic_infoDict.get('use_eebls_to_initialize_tmid_and_bounds', 'y') ) + pick_comparison_by_eebls_snr = should_pick_comparison_by_eebls_snr( + exotic_infoDict.get('pick_comparison_by_eebls_snr', 'y') + ) use_impactparameter_rather_than_inclination_to_fit = ( should_use_impactparameter_rather_than_inclination_to_fit( exotic_infoDict.get('use_impactparameter_rather_than_inclination_to_fit', 'y') @@ -8766,6 +9161,8 @@ def main(): log_info("Aperture photometry disabled per optional_info setting.") if not use_eebls_tmid_initializer: log_info("EEBLS transit initializer disabled per optional_info setting.") + if not pick_comparison_by_eebls_snr: + log_info("Comparison-star selection by EEBLS SNR disabled per optional_info setting.") for i, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{i + 1}" @@ -8841,16 +9238,10 @@ def main(): plateStatus.setCurrentFilename(fileName) hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) - if use_psf_photometry: - # Keep PSF photometry on one consistent measurement path for reduction frames. - frame_fast_centroid = False - target_fast_centroid = False - else: - frame_fast_centroid = should_use_fast_centroid(i) - target_fast_centroid = should_use_fast_target_centroid( - i, - adaptive_apertures=use_adaptive_apertures, - ) + # Final reductions should always use the full centroid fit so the + # centroid series does not inherit the fast moment-estimator cadence. + frame_fast_centroid = False + target_fast_centroid = False extension = 0 image_header = hdul[extension].header @@ -8904,10 +9295,14 @@ def main(): dtype=float, ) projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) + target_seed = choose_centroid_seed_position( + [tx, ty], + None if i == 0 else psf_data['target'][i - 1], + ) psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( imageData, - [tx, ty], + target_seed, 0, fast_mode=target_fast_centroid, ) @@ -8920,9 +9315,13 @@ def main(): ckey = f"comp{j + 1}" cx, cy = pix_x[j + 1], pix_y[j + 1] + comp_seed = choose_centroid_seed_position( + [cx, cy], + None if i == 0 else psf_data[ckey][i - 1], + ) psf_data[ckey][i] = fit_centroid_or_warn_out_of_frame( imageData, - [cx, cy], + comp_seed, j + 1, fast_mode=frame_fast_centroid, ) @@ -8962,9 +9361,13 @@ def main(): transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) tx, ty = transformed_coords[0] + target_seed = choose_centroid_seed_position( + [tx, ty], + None if i == 0 else psf_data['target'][i - 1], + ) psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( imageData, - [tx, ty], + target_seed, 0, fast_mode=target_fast_centroid, ) @@ -8973,9 +9376,13 @@ def main(): ckey = f"comp{j + 1}" cx, cy = transformed_coords[j + 1] + comp_seed = choose_centroid_seed_position( + [cx, cy], + None if i == 0 else psf_data[ckey][i - 1], + ) psf_data[ckey][i] = fit_centroid_or_warn_out_of_frame( imageData, - [cx, cy], + comp_seed, j + 1, fast_mode=frame_fast_centroid, ) @@ -9201,6 +9608,8 @@ def main(): 'adaptive_summary': None, 'calibration_field_score': np.inf, 'selection_basis': 'target_fit', + 'selection_metric': 'residual_scatter', + 'comparison_eebls_snr': np.nan, } comparison_calibration = None @@ -9225,6 +9634,20 @@ def main(): log_info("\nCalibrating comparison stars before target fitting. Please wait.") log_info(f"Comparison-star field method: {comparison_calibration['method_label']}") log_info(f"Comparison-star field score: {comparison_calibration['field_score'] * 100.0:.4f}%") + if comparison_calibration.get('suitability_outlier_rejected_count', 0) > 0: + threshold = comparison_calibration.get('suitability_high_threshold', np.nan) + if np.isfinite(threshold): + log_info( + "Comparison-star field sigma clipping rejected " + f"{comparison_calibration['suitability_outlier_rejected_count']} high-suitability " + f"outlier(s) above {threshold * 100.0:.4f}% before target-fit evaluation." + ) + else: + log_info( + "Comparison-star field sigma clipping rejected " + f"{comparison_calibration['suitability_outlier_rejected_count']} high-suitability " + "outlier(s) before target-fit evaluation." + ) for summary in comparison_calibration['comp_summaries']: aggregate_text = "n/a" if not np.isfinite(summary['aggregate_score']) else f"{summary['aggregate_score'] * 100.0:.4f}%" ensemble_text = "n/a" if not np.isfinite(summary['ensemble_score']) else f"{summary['ensemble_score'] * 100.0:.4f}%" @@ -9234,6 +9657,8 @@ def main(): coverage_text = f"coverage={format_comp_star_coverage_text(summary)}" if summary['coverage_rejected']: coverage_text += " [rejected: low coverage]" + if summary.get('suitability_outlier_rejected'): + coverage_text += " [rejected: high suitability outlier]" log_info( f" {summary['label']}{selected_label} ({position_text}): suitability={aggregate_text}, " f"ensemble={ensemble_text}, pairwise_median={pairwise_text}, " @@ -9300,6 +9725,7 @@ def main(): use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, ) comparison_calibration['ranked_fit_comp_indices'] = [ summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] @@ -9343,7 +9769,8 @@ def main(): "Comparison-star calibration target-fit selection chose " f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " f"after evaluating {retry_count} better-ranked field-stability candidate(s); " - "it delivered the lowest target-fit residual scatter among successful fits." + f"it delivered the best {comparison_selection_metric_label(comparison_fit_search['selection_metric'])} " + "among successful fits." ) photometry_info.update(best_fit_lc=myfit, @@ -9355,7 +9782,9 @@ def main(): aperture_index=selected_a, annulus_index=selected_an, calibration_field_score=comparison_calibration['field_score'], - selection_basis=selection_basis) + selection_basis=selection_basis, + selection_metric=comparison_fit_search.get('selection_metric', 'residual_scatter'), + comparison_eebls_snr=selected_attempt.get('eebls_snr', np.nan)) flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) @@ -9463,6 +9892,7 @@ def main(): use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, ) best_candidate = target_driven_search['best_candidate'] @@ -9483,6 +9913,8 @@ def main(): aperture_index=best_candidate['a'], annulus_index=best_candidate['an'], selection_basis='target_fit', + selection_metric=target_driven_search.get('selection_metric', 'residual_scatter'), + comparison_eebls_snr=target_driven_search.get('selected_eebls_snr', np.nan), ) flux_values.update( @@ -9587,6 +10019,16 @@ def main(): log_info("\n\n*********************************************") if np.isfinite(photometry_info['calibration_field_score']): log_info(f"Comparison-Star Field Score: {round(photometry_info['calibration_field_score'] * 100, 4)}%") + summary_min_aperture = photometry_info.get('min_aperture') + if photometry_info.get('comp_star_num') is not None or ( + summary_min_aperture is not None and summary_min_aperture < 0 + ): + log_info( + "Comparison Selection Metric: " + f"{comparison_selection_metric_label(photometry_info.get('selection_metric', 'residual_scatter'))}" + ) + if np.isfinite(photometry_info.get('comparison_eebls_snr', np.nan)): + log_info(f"Selected Comparison EEBLS SNR: {photometry_info['comparison_eebls_snr']:.2f}") selected_method_label = selected_photometry_method_label(photometry_info) display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) adaptive_summary = photometry_info.get('adaptive_summary') diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 7a558cfd..ec052f6f 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -422,6 +422,7 @@ def save_input(): "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", + "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -448,6 +449,7 @@ def save_input(): "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1500,6 +1502,7 @@ def save_input(): "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", + "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -1574,6 +1577,7 @@ def save_input(): "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1627,6 +1631,7 @@ def save_input(): "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/exotic/inputs.py b/exotic/inputs.py index 491e5b4c..9a06274f 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -211,6 +211,7 @@ def __init__(self, init_opt): 'detrend_on_outoftransit_baseline': True, 'final_fit_baseline_duration_multiplier': 1.0, 'use_eebls_to_initialize_tmid_and_bounds': 'y', + 'pick_comparison_by_eebls_snr': 'y', 'detect_bad_pixels_before_photometry': 'y', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', @@ -444,6 +445,11 @@ def comp_params(self, init_file, planet_dict): 'Use EEBLS to Initialize Tmid and Bounds? (y/n)', 'Use EEBLS To Initialize Tmid And Bounds? (y/n)', ), + 'pick_comparison_by_eebls_snr': ( + 'pick_comparison_by_eebls_snr', + 'Pick Comparison by EEBLS SNR? (y/n)', + 'Pick comparison by EEBLS SNR? (y/n)', + ), 'use_impactparameter_rather_than_inclination_to_fit': ( 'use_impactparameter_rather_than_inclination_to_fit', 'Use impact parameter rather than inclination to fit? (y/n)', diff --git a/exotic/output_files.py b/exotic/output_files.py index 9d7a9274..8cb1282e 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -382,7 +382,7 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label handle.write(f"# Selected comparison star,{'' if best_comp_index is None else best_comp_index + 1}\n") handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,ensemble_score,pairwise_median_score," "pairwise_max_score,self_score,valid_pair_count,coverage_count,coverage_peer_median," - "coverage_min_required,coverage_rejected\n") + "coverage_min_required,coverage_rejected,suitability_outlier_rejected\n") for summary in comp_summaries: position = summary.get('position') or [None, None] @@ -401,6 +401,7 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label summary.get('coverage_reference_count'), summary.get('coverage_min_required_count'), summary.get('coverage_rejected'), + summary.get('suitability_outlier_rejected'), ] handle.write(",".join("" if value is None else str(value) for value in values) + "\n") diff --git a/inits.json b/inits.json index 31060768..38d32220 100644 --- a/inits.json +++ b/inits.json @@ -31,6 +31,7 @@ "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", + "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", @@ -112,6 +113,7 @@ "detrend_on_outoftransit_baseline": true, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_psf_photometry": "y", "use_aperture_photometry": "y", diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index da95e3a6..32e8807d 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -90,6 +90,9 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: check_coordinates, cheap_lightcurve_prescore, centroid_offset_matches_reference, + choose_centroid_seed_position, + apply_comparison_star_suitability_outlier_rejection, + comparison_calibration_selection_reason, comparison_candidate_fit_selection_reason, comparison_star_coverage_summary, comparison_star_stability_summary, @@ -114,6 +117,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: phase_bin_sigma_clip, prepare_final_fit_lightcurve_series, representative_psf_sigma, + ranked_comparison_calibration_summaries, resolve_sky_annulus_geometry, run_target_driven_photometry_search, resolve_frame_aperture_radii, @@ -129,6 +133,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: should_fit_lightcurve_to_every_comparison_candidate, should_detect_bad_pixels_before_photometry, should_use_aperture_photometry, + should_pick_comparison_by_eebls_snr, should_use_psf_photometry, should_skip_low_comparison_coverage_rejection, should_use_fast_target_centroid, @@ -534,6 +539,13 @@ def test_should_use_eebls_to_initialize_tmid_and_bounds_parses_values(): assert should_use_eebls_to_initialize_tmid_and_bounds(True) is True +def test_should_pick_comparison_by_eebls_snr_parses_values(): + assert should_pick_comparison_by_eebls_snr(None) is True + assert should_pick_comparison_by_eebls_snr("y") is True + assert should_pick_comparison_by_eebls_snr("n") is False + assert should_pick_comparison_by_eebls_snr(True) is True + + def test_build_time_rejection_diagnostic_groups_contiguous_ranges(): times = np.array([1.0, 1.1, 1.2, 1.5, 1.6, 2.0], dtype=float) keep_mask = np.array([True, False, False, True, False, True], dtype=bool) @@ -673,6 +685,19 @@ def test_resolve_sky_annulus_geometry_enforces_fwhm_floor_and_min_sky_pixels(): assert geometry["annulus_width"] > 2.0 +def test_choose_centroid_seed_position_prefers_previous_fit_for_small_predicted_jumps(): + seed = choose_centroid_seed_position([100.8, 200.2], previous_psf_row=np.array([100.2, 199.9, 1, 1, 1, 0, 0])) + np.testing.assert_allclose(seed, np.array([100.2, 199.9])) + + +def test_choose_centroid_seed_position_falls_back_to_predicted_for_large_jump_or_invalid_previous(): + seed_far = choose_centroid_seed_position([110.0, 210.0], previous_psf_row=np.array([100.0, 200.0, 1, 1, 1, 0, 0])) + seed_nan = choose_centroid_seed_position([110.0, 210.0], previous_psf_row=np.array([np.nan, 200.0, 1, 1, 1, 0, 0])) + + np.testing.assert_allclose(seed_far, np.array([110.0, 210.0])) + np.testing.assert_allclose(seed_nan, np.array([110.0, 210.0])) + + def test_representative_psf_sigma_uses_valid_frames_and_fallback(): psf_rows = np.array([ [0.0, 0.0, 1.0, 2.0, 2.0, 0.0, 0.0], @@ -914,6 +939,7 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): "selected": False, "fit": None, "res_std": np.inf, + "eebls_snr": np.nan, "coverage_count": 1, "coverage_total_frame_count": 3, "coverage_reference_count": 3.0, @@ -928,6 +954,7 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): "selected": True, "fit": object(), "res_std": 0.01, + "eebls_snr": 6.5, "coverage_count": 3, "coverage_total_frame_count": 3, "coverage_reference_count": 3.0, @@ -941,13 +968,20 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): log_comparison_candidate_fit_summaries( candidate_fit_summaries, - {"selection_basis": "comparison_field", "comp_star_num": 2, "min_std": 0.01}, + { + "selection_basis": "comparison_field", + "selection_metric": "eebls_snr", + "comp_star_num": 2, + "min_std": 0.01, + "comparison_eebls_snr": 6.5, + }, ) assert any("Selection basis: comparison-field" in message for message in logged) + assert any("Selection metric: EEBLS SNR" in message for message in logged) assert any("coverage=1 valid frame(s) out of 3 total; min_required=2; peer_median=3.0" in message for message in logged) assert any("Comp 1" in message and "reason=comparison candidate rejected after iterative low-coverage clipping" in message for message in logged) - assert any("Comp 2 [selected]" in message and "comparison-field calibration ranked this star best" in message for message in logged) + assert any("Comp 2 [selected]" in message and "eebls_snr=6.50" in message and "comparison-field calibration ranked this star best" in message for message in logged) assert any("parameters: fit_method=ultranest" in message for message in logged) @@ -967,6 +1001,7 @@ def test_log_comparison_calibration_fit_attempt_summaries_includes_reasons(monke "coverage_min_required_count": 2, "fit": None, "res_std": np.inf, + "eebls_snr": np.nan, "fit_point_count": 0, "fit_diagnostics": {"usable_point_count": 0}, "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during target/reference ratio screening (non-finite=0, >2x=3, finite ratio range=3.0000 to 3.0000).", @@ -983,6 +1018,7 @@ def test_log_comparison_calibration_fit_attempt_summaries_includes_reasons(monke "coverage_min_required_count": 2, "fit": object(), "res_std": 0.01, + "eebls_snr": 5.2, "fit_point_count": 3, "fit_diagnostics": {"usable_point_count": 3}, "failure_reason": None, @@ -995,7 +1031,7 @@ def test_log_comparison_calibration_fit_attempt_summaries_includes_reasons(monke assert any("Comparison-star calibration target-fit diagnostics:" in message for message in logged) assert any("Photometry method: Aperture photometry (aper=7.05px, annulus=22.73px)" in message for message in logged) assert any("Comp 1" in message and "reason=relative-flux filtering left 0 usable point(s)" in message for message in logged) - assert any("Comp 2 [selected]" in message and "fit_points=3" in message for message in logged) + assert any("Comp 2 [selected]" in message and "eebls_snr=5.20" in message and "fit_points=3" in message for message in logged) assert any("parameters: fit_method=ultranest" in message for message in logged) @@ -1012,6 +1048,7 @@ def test_log_target_fit_candidate_summaries_includes_methods_and_reasons(monkeyp "prescore": 0.005, "fit": None, "res_std": np.inf, + "eebls_snr": np.nan, "coverage_count": 3, "coverage_total_frame_count": 3, "coverage_reference_count": 3.0, @@ -1051,6 +1088,22 @@ def test_comparison_candidate_fit_selection_reason_describes_comparison_field_re assert "fell back to this star" in reason +def test_comparison_calibration_selection_reason_reports_suitability_outlier_rejection(): + reason = comparison_calibration_selection_reason( + { + "selected": False, + "coverage_rejected": False, + "aggregate_score": 0.139668, + "suitability_outlier_rejected": True, + "suitability_high_threshold": 0.0398471675, + }, + best_comp_score=0.021654, + ) + + assert "rejected by high-side sigma clipping" in reason + assert "13.9668%" in reason + + def test_comparison_star_stability_summary_penalizes_variable_candidates(): airmass = np.linspace(1.0, 1.5, 6) summary = comparison_star_stability_summary( @@ -1067,6 +1120,86 @@ def test_comparison_star_stability_summary_penalizes_variable_candidates(): assert summary["comp_summaries"][2]["aggregate_score"] > summary["comp_summaries"][0]["aggregate_score"] +def test_apply_comparison_star_suitability_outlier_rejection_rejects_high_tail(): + comp_summaries = [ + {"label": "Comp 1", "aggregate_score": 0.139668, "coverage_rejected": False}, + {"label": "Comp 2", "aggregate_score": 0.051809, "coverage_rejected": False}, + {"label": "Comp 3", "aggregate_score": 0.024802, "coverage_rejected": False}, + {"label": "Comp 4", "aggregate_score": 0.027680, "coverage_rejected": False}, + {"label": "Comp 5", "aggregate_score": 0.036997, "coverage_rejected": False}, + {"label": "Comp 6", "aggregate_score": 0.037970, "coverage_rejected": False}, + {"label": "Comp 7", "aggregate_score": 0.023458, "coverage_rejected": False}, + {"label": "Comp 8", "aggregate_score": 0.021654, "coverage_rejected": False}, + {"label": "Comp 9", "aggregate_score": 0.025084, "coverage_rejected": False}, + {"label": "Comp 10", "aggregate_score": 0.024918, "coverage_rejected": False}, + ] + + result = apply_comparison_star_suitability_outlier_rejection(comp_summaries) + + assert result["rejected_indices"] == [0, 1] + assert comp_summaries[0]["suitability_outlier_rejected"] is True + assert comp_summaries[1]["suitability_outlier_rejected"] is True + assert comp_summaries[4]["suitability_outlier_rejected"] is False + assert comp_summaries[5]["suitability_outlier_rejected"] is False + assert 0.037970 < result["high_threshold"] < 0.051809 + + +def test_comparison_star_stability_summary_iterates_after_suitability_outlier_rejection(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.normalize_flux_series", + lambda flux_values, validity_mask_func=None: np.asarray(flux_values, dtype=float), + ) + monkeypatch.setattr( + "exotic.exotic.normalized_ratio_series", + lambda flux_a, flux_b: np.array([1.0], dtype=float), + ) + + def fake_build_normalized_comp_ensemble(normalized_flux_map, exclude_key): + comp_index = float(exclude_key.replace("comp", "")) + return np.array([-float(len(normalized_flux_map)), comp_index], dtype=float) + + def fake_prescore(tflux, cflux, airmass, enforce_relative_flux_max=False): + comp_index = int(np.rint(np.asarray(tflux, dtype=float).flat[0])) + reference = np.asarray(cflux, dtype=float).reshape(-1) + if reference.size == 0: + return np.inf + if np.allclose(reference, 1.0): + return 0.0 + if reference[0] < 0: + active_count = int(np.rint(abs(reference[0]))) + ensemble_scores = { + 6: {1: 10.0, 2: 2.5, 3: 1.0, 4: 1.1, 5: 1.2, 6: 1.4}, + 5: {2: 4.0, 3: 1.0, 4: 1.1, 5: 1.2, 6: 1.4}, + 4: {3: 1.0, 4: 1.1, 5: 1.2, 6: 1.4}, + } + return ensemble_scores.get(active_count, {}).get(comp_index, 1.0) + return 0.5 + + monkeypatch.setattr( + "exotic.exotic.build_normalized_comp_ensemble", + fake_build_normalized_comp_ensemble, + ) + monkeypatch.setattr("exotic.exotic.cheap_lightcurve_prescore", fake_prescore) + + summary = comparison_star_stability_summary( + { + "comp1": np.array([1.0], dtype=float), + "comp2": np.array([2.0], dtype=float), + "comp3": np.array([3.0], dtype=float), + "comp4": np.array([4.0], dtype=float), + "comp5": np.array([5.0], dtype=float), + "comp6": np.array([6.0], dtype=float), + }, + np.array([1.0], dtype=float), + ) + + assert summary["suitability_outlier_rejected_count"] == 2 + assert summary["comp_summaries"][0]["suitability_outlier_rejected"] is True + assert summary["comp_summaries"][1]["suitability_outlier_rejected"] is True + assert summary["comp_summaries"][2]["suitability_outlier_rejected"] is False + assert summary["best_comp_index"] == 2 + + def test_comparison_star_coverage_summary_rejects_sparse_candidates(): coverage = comparison_star_coverage_summary( { @@ -2107,6 +2240,116 @@ def fake_fit_lightcurve( assert result["attempts"][0]["selection_reason"].startswith("not selected: target-fit residual scatter") +def test_ranked_comparison_calibration_summaries_skip_suitability_outliers(): + ranked = ranked_comparison_calibration_summaries( + { + "comp_summaries": [ + {"comp_index": 0, "aggregate_score": 0.139668, "coverage_rejected": False, "suitability_outlier_rejected": True}, + {"comp_index": 1, "aggregate_score": 0.051809, "coverage_rejected": False, "suitability_outlier_rejected": True}, + {"comp_index": 2, "aggregate_score": 0.024802, "coverage_rejected": False, "suitability_outlier_rejected": False}, + {"comp_index": 3, "aggregate_score": 0.027680, "coverage_rejected": False, "suitability_outlier_rejected": False}, + ] + } + ) + + assert [summary["comp_index"] for summary in ranked] == [2, 3] + + +def test_fit_ranked_comparison_calibration_candidates_can_prefer_highest_eebls_snr(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + def fake_fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + if comp_marker == 50: + residual_level = 0.01 + eebls_snr = 4.0 + else: + residual_level = 0.02 + eebls_snr = 7.5 + + residuals = residual_level * np.array([-1.0, 1.0, -1.0, 1.0, -1.0, 1.0], dtype=float) + fit = types.SimpleNamespace( + residuals=residuals, + data=np.ones_like(residuals), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + eebls_diagnostic_depth_snr=eebls_snr, + ) + return fit, np.asarray(tflux, dtype=float), np.asarray(cflux, dtype=float) + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = {"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0} + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + { + "label": "Comp 1", + "position": (10.0, 10.0), + "aggregate_score": 0.01, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 0, + }, + { + "label": "Comp 2", + "position": (20.0, 20.0), + "aggregate_score": 0.02, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 1, + }, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld, + p_dict, + comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + pick_comparison_by_eebls_snr=True, + ) + + assert result["selection_metric"] == "eebls_snr" + assert result["selected_result"]["comp_index"] == 1 + assert result["selected_result"]["eebls_snr"] == pytest.approx(7.5) + assert "highest EEBLS SNR" in result["selected_result"]["selection_reason"] + assert result["attempts"][0]["selection_reason"].startswith("not selected: EEBLS SNR") + + def test_fit_lightcurve_refines_nested_tmid_bounds_from_two_sided_lm_fit(monkeypatch): captured_calls = [] @@ -2340,6 +2583,87 @@ def fake_evaluate(task): assert result["min_std"] == pytest.approx(0.01) +def test_run_target_driven_photometry_search_can_prefer_highest_eebls_snr(monkeypatch): + class DummyFit: + def __init__(self, residual_level, eebls_snr): + self.residuals = np.full(6, residual_level) + self.data = np.ones(6) + self.eebls_diagnostic_depth_snr = eebls_snr + + def fake_evaluate(task): + _, tflux, cflux, *_ = task + cflux = np.asarray(cflux, dtype=float) + tflux = np.asarray(tflux, dtype=float) + if np.allclose(cflux, 20.0): + return {"myfit": DummyFit(0.01, 4.0), "res_std": 0.01, "eebls_snr": 4.0}, tflux, cflux + if np.allclose(cflux, 40.0): + return {"myfit": DummyFit(0.02, 9.0), "res_std": 0.02, "eebls_snr": 9.0}, tflux, cflux + raise AssertionError("Unexpected candidate flux passed to evaluator.") + + monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + psf_target_amp = 20.0 / (2.0 * np.pi) + psf_comp_amp = 20.0 / (2.0 * np.pi) + psf_data = { + "target": np.column_stack([ + np.zeros(6), + np.zeros(6), + np.full(6, psf_target_amp), + np.ones(6), + np.ones(6), + ]), + "comp1": np.column_stack([ + np.ones(6), + np.ones(6), + np.full(6, psf_comp_amp), + np.ones(6), + np.ones(6), + ]), + } + aper_data = { + "target": np.full((6, 1, 1), 40.0), + "comp1": np.full((6, 1, 1), 40.0), + } + + result = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + p_dict, + comp_stars=[[100.0, 200.0]], + psf_data=psf_data, + aper_data=aper_data, + apers=np.array([5.0]), + annuli=np.array([12.0]), + sigma=1.0, + require_comp_star=True, + use_psf_photometry=True, + use_aperture_photometry=True, + pick_comparison_by_eebls_snr=True, + ) + + assert result["selection_metric"] == "eebls_snr" + assert result["best_candidate"]["method"] == "aperture" + assert result["selected_eebls_snr"] == pytest.approx(9.0) + assert result["min_std"] == pytest.approx(0.02) + + def test_fit_ranked_comparison_calibration_candidates_retries_next_best_candidate(monkeypatch): class DummyFit: def __init__(self): diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 017d10e5..00232e21 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -158,6 +158,8 @@ def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): "pairwise_max_score": 0.0014, "self_score": 0.0009, "valid_pair_count": 2, + "coverage_rejected": False, + "suitability_outlier_rejected": False, }, { "label": "Comp 2", @@ -169,6 +171,8 @@ def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): "pairwise_max_score": 0.0035, "self_score": 0.0012, "valid_pair_count": 2, + "coverage_rejected": False, + "suitability_outlier_rejected": True, }, ], 0, @@ -176,6 +180,7 @@ def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): text = summary_path.read_text() assert "# Selected comparison star,1" in text + assert "suitability_outlier_rejected" in text assert "Comp 1,101,202,true" in text From cee1633a2b8ec1eb4af5fcff7c637e31018e305a Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 27 Apr 2026 14:39:26 +1000 Subject: [PATCH 026/116] Widen initial Rp/R* bounds for transit fitting --- exotic/exotic.py | 29 +++++++++++++++++++++++++++-- tests/test_exotic_rprs_retry.py | 28 ++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index abe633f2..1670d501 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -183,6 +183,8 @@ RPRS_SEARCH_BOUND_MIN = 0.0 RPRS_SEARCH_BOUND_MAX = 0.30 RPRS_RETRY_MIN_HALF_WIDTH = 0.05 +INITIAL_RPRS_BOUND_LOWER_SCALE = 0.25 +INITIAL_RPRS_BOUND_UPPER_SCALE = 3.0 FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER = 0.5 EEBLS_DURATION_GRID_SIZE = 15 EEBLS_DURATION_MIN_FRACTION = 0.5 @@ -438,6 +440,29 @@ def annotate_rprs_posterior_refit(fit, applied, note=None, history=None): fit.rprs_posterior_refit_bounds = None +def build_initial_rprs_bounds( + rprs, + lower_scale=INITIAL_RPRS_BOUND_LOWER_SCALE, + upper_scale=INITIAL_RPRS_BOUND_UPPER_SCALE, +): + try: + rprs = float(rprs) + lower_scale = float(lower_scale) + upper_scale = float(upper_scale) + except (TypeError, ValueError): + return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + + if not np.isfinite(rprs) or rprs <= 0: + return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + + lower_bound = max(RPRS_SEARCH_BOUND_MIN, lower_scale * rprs) + upper_bound = upper_scale * rprs + if not np.isfinite(upper_bound) or upper_bound <= lower_bound: + upper_bound = max(lower_bound + np.finfo(float).eps, rprs) + + return [float(lower_bound), float(upper_bound)] + + def clone_lightcurve_bounds(bounds): return { key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value @@ -6560,7 +6585,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, lower, upper = tmid_search_summary['bounds'] mybounds = { - 'rprs': [0, prior['rprs'] * 1.25], + 'rprs': build_initial_rprs_bounds(prior['rprs']), 'tmid': [lower, upper], 'inc': [prior['inc'] - 5, min(90, prior['inc'] + 5)], } @@ -10466,7 +10491,7 @@ def main(): ) mybounds = { - 'rprs': [0, prior['rprs'] * 1.25], + 'rprs': build_initial_rprs_bounds(prior['rprs']), 'tmid': [lower, upper], 'inc': [prior['inc'] - 5, min(90, prior['inc'] + 5)], } diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 482ab5b1..61dfc3c1 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -84,13 +84,25 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: sys.modules.setdefault("exotic.api.ld", fake_ld) from exotic.exotic import ( # noqa: E402 + INITIAL_RPRS_BOUND_LOWER_SCALE, + INITIAL_RPRS_BOUND_UPPER_SCALE, RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, RPRS_SEARCH_BOUND_MAX, RPRS_SEARCH_BOUND_MIN, + build_initial_rprs_bounds, run_nested_lightcurve_fit_with_rprs_posterior_retry, ) +def test_build_initial_rprs_bounds_uses_wider_asymmetric_search_box(): + bounds = build_initial_rprs_bounds(0.1) + + assert bounds == pytest.approx([ + INITIAL_RPRS_BOUND_LOWER_SCALE * 0.1, + INITIAL_RPRS_BOUND_UPPER_SCALE * 0.1, + ]) + + def test_rprs_posterior_retry_walks_bounds_until_retry_cap(monkeypatch): import exotic.exotic as exotic_module @@ -166,17 +178,17 @@ def fake_lc_fitter( np.asarray([call["bounds"]["rprs"] for call in captured["calls"]], dtype=float), np.asarray([ [0.0, 0.125], - [0.128, 0.188], - [0.157, 0.207], - [0.174, 0.214], - [0.186, 0.216], - [0.191, 0.221], + [0.108, 0.208], + [0.132, 0.232], + [0.144, 0.244], + [0.151, 0.251], + [0.156, 0.256], ], dtype=float), ) assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 5 assert fit.rprs_posterior_refit_edge == "upper" - assert fit.rprs_posterior_refit_bounds == pytest.approx([0.191, 0.221]) + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.156, 0.256]) assert "after 5 retries" in fit.rprs_posterior_refit_note @@ -248,10 +260,10 @@ def fake_lc_fitter( assert len(captured["calls"]) == 2 assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.25]) assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) - assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.175, RPRS_SEARCH_BOUND_MAX]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.15, RPRS_SEARCH_BOUND_MAX]) assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 1 - assert fit.rprs_posterior_refit_bounds == pytest.approx([0.175, RPRS_SEARCH_BOUND_MAX]) + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.15, RPRS_SEARCH_BOUND_MAX]) def test_rprs_posterior_retry_stops_at_maximum_exoplanet_range(monkeypatch): From 4f8a83406fba29c6dc70c2964b00b91ec9c28568 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 29 Apr 2026 07:04:57 +1000 Subject: [PATCH 027/116] Add a/Rs and impact parameter retries and reporting --- ...ailedFitSummary_HAT-P-32 b_2026-04-28.json | 35 + exotic/api/elca.py | 187 +- exotic/api/joint_fitter.py | 50 +- exotic/exotic.py | 4889 ++++++++++++++--- exotic/inputs.py | 15 + exotic/output_files.py | 154 +- inits.json | 8 +- ...ailedFitSummary_HAT-P-32 b_2026-04-28.json | 35 + setup.cfg | 2 +- tests/test_elca_baseline.py | 243 + tests/test_exotic_proper_motion.py | 1568 +++++- tests/test_exotic_rprs_retry.py | 168 +- tests/test_output_files.py | 97 +- 13 files changed, 6462 insertions(+), 989 deletions(-) create mode 100644 .pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json create mode 100644 manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json diff --git a/.pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json b/.pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json new file mode 100644 index 00000000..fe408498 --- /dev/null +++ b/.pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json @@ -0,0 +1,35 @@ +{ + "planet_name": "HAT-P-32 b", + "observation_date": "2026-04-28", + "comparison_star": 1, + "comparison_label": "Comp 1", + "comparison_position": [ + 100.0, + 200.0 + ], + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "failure_reason": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10).", + "fit_diagnostics": { + "input_point_count": 6, + "has_reference_flux": true, + "relative_flux_point_count": 6, + "sigma_clip_point_count": 4, + "usable_point_count": 4, + "failed_stage": "transit_qc", + "failure_reason": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." + }, + "parameter_summary": null, + "fit_point_count": 6, + "eebls_snr": NaN, + "transit_delta_bic": NaN, + "residual_scatter": 0.0, + "ktmf_metric": NaN, + "ktmf_contributions": [], + "transit_qc": { + "status": "fail", + "summary": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." + }, + "saved_debug_series": null, + "saved_bestfit_plot": null, + "archive_errors": [] +} \ No newline at end of file diff --git a/exotic/api/elca.py b/exotic/api/elca.py index ff0b53ab..d5567f7c 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -43,8 +43,12 @@ import builtins import copy from contextlib import redirect_stderr, redirect_stdout +import faulthandler import io from itertools import cycle +import multiprocessing +import os +import sys import bottleneck as bn import matplotlib.pyplot as plt import numpy as np @@ -69,12 +73,52 @@ except ImportError: from .ultranest_utils import run_reactive_sampler -if not getattr(builtins, "_EXOTIC_IMPORTING_MODULES_PRINTED", False): - print("Importing modules. Please wait.......") +BAD_LOG_LIKELIHOOD = -1.0e100 + +if ( + multiprocessing.current_process().name == "MainProcess" + and not getattr(builtins, "_EXOTIC_IMPORTING_MODULES_PRINTED", False) +): + print("Importing modules. Please wait.......", flush=True) builtins._EXOTIC_IMPORTING_MODULES_PRINTED = True -with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): - from pylightcurve.models.exoplanet_lc import transit as pytransit + +def _pylightcurve_import_watchdog_seconds(): + try: + return float(os.environ.get("EXOTIC_IMPORT_WATCHDOG_SECONDS", "120")) + except (TypeError, ValueError): + return 120.0 + + +def _start_import_watchdog(): + timeout = _pylightcurve_import_watchdog_seconds() + if timeout <= 0: + return False + + try: + if not faulthandler.is_enabled(): + faulthandler.enable(file=sys.__stdout__, all_threads=True) + faulthandler.dump_traceback_later(timeout, repeat=True, file=sys.__stdout__) + return True + except Exception: + return False + + +def _load_pylightcurve_transit(): + watchdog_started = _start_import_watchdog() + try: + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + from pylightcurve.models.exoplanet_lc import transit + return transit + finally: + if watchdog_started: + try: + faulthandler.cancel_dump_traceback_later() + except Exception: + pass + + +pytransit = _load_pylightcurve_transit() def weightedflux(flux, gw, nearest): @@ -127,6 +171,41 @@ def inclination_from_impact_parameter(values, impact_parameter): return np.rad2deg(np.arccos(cosi)) +def transit_duration(values): + try: + period = float(values['per']) + rprs = float(values['rprs']) + ars = float(values['ars']) + inc = float(values['inc']) + except (KeyError, TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(period) or period <= 0 + or not np.isfinite(rprs) or rprs < 0 + or not np.isfinite(ars) or ars <= 0 + or not np.isfinite(inc) + ): + return np.nan + + ecc = values.get('ecc', 0.0) + omega = np.deg2rad(values.get('omega', 0.0)) + sin_inc = np.sin(np.deg2rad(inc)) + if not np.isfinite(sin_inc) or sin_inc <= 0: + return np.nan + + impact_scale = ars * (1.0 - ecc ** 2) / max(np.finfo(float).eps, 1.0 + ecc * np.sin(omega)) + impact_parameter = impact_scale * np.cos(np.deg2rad(inc)) + chord_sq = (1.0 + rprs) ** 2 - impact_parameter ** 2 + if not np.isfinite(chord_sq) or chord_sq <= 0 or not np.isfinite(impact_scale) or impact_scale <= 0: + return np.nan + + argument = np.sqrt(chord_sq) / (impact_scale * sin_inc) + argument = float(np.clip(argument, -1.0, 1.0)) + duration = (period / np.pi) * np.arcsin(argument) + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + def get_phase(times, per, tmid): return (times - tmid + 0.25 * per) / per % 1 - 0.25 @@ -362,6 +441,7 @@ def __init__( jd_times=None, verbose=True, use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, ): self.time = time self.data = data @@ -376,6 +456,7 @@ def __init__( self.mode = mode self.neighbors = neighbors self.use_impactparameter_rather_than_inclination_to_fit = use_impactparameter_rather_than_inclination_to_fit + self.duration_prior = copy.deepcopy(duration_prior) if isinstance(duration_prior, dict) else None self.results = None self.sampled_keys = list(bounds.keys()) self.sample_bounds = copy.deepcopy(bounds) @@ -471,8 +552,26 @@ def _get_sample_bounds(self, bound_keys=None, values=None): for key, sampled_key in zip(bound_keys, sampled_keys): if key == 'inc' and sampled_key == 'b': inc_lower, inc_upper = self.bounds[key] - lower = float(np.min(impact_parameter_from_inclination(values, np.array([inc_lower, inc_upper])))) - upper = float(np.max(impact_parameter_from_inclination(values, np.array([inc_lower, inc_upper])))) + if 'ars' in self.bounds: + ars_lower, ars_upper = self.bounds['ars'] + b_corners = [] + for ars_value in (ars_lower, ars_upper): + corner_values = copy.deepcopy(values) + corner_values['ars'] = float(ars_value) + b_corners.extend( + np.asarray( + impact_parameter_from_inclination( + corner_values, + np.array([inc_lower, inc_upper], dtype=float), + ), + dtype=float, + ).reshape(-1).tolist() + ) + lower = float(np.min(b_corners)) + upper = float(np.max(b_corners)) + else: + lower = float(np.min(impact_parameter_from_inclination(values, np.array([inc_lower, inc_upper])))) + upper = float(np.max(impact_parameter_from_inclination(values, np.array([inc_lower, inc_upper])))) sample_bounds[sampled_key] = [lower, upper] else: sample_bounds[sampled_key] = list(self.bounds[key]) @@ -480,6 +579,13 @@ def _get_sample_bounds(self, bound_keys=None, values=None): def _sample_point_from_unit_cube(self, upars, bound_keys=None): bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + upars_array = np.asarray(upars, dtype=float) + if upars_array.ndim == 2: + return np.asarray([ + self._sample_point_from_unit_cube(row, bound_keys) + for row in upars_array + ], dtype=float) + boundarray = np.array([self.bounds[k] for k in bound_keys], dtype=float) physical = copy.deepcopy(self.prior) sample_point = np.zeros(len(bound_keys), dtype=float) @@ -487,11 +593,11 @@ def _sample_point_from_unit_cube(self, upars, bound_keys=None): for i, key in enumerate(bound_keys): if key == 'inc' and self._uses_internal_impact_parameter(): continue - physical[key] = boundarray[i, 0] + (boundarray[i, 1] - boundarray[i, 0]) * upars[i] + physical[key] = boundarray[i, 0] + (boundarray[i, 1] - boundarray[i, 0]) * upars_array[i] for i, key in enumerate(bound_keys): if key == 'inc' and self._uses_internal_impact_parameter(): - inc = boundarray[i, 0] + (boundarray[i, 1] - boundarray[i, 0]) * upars[i] + inc = boundarray[i, 0] + (boundarray[i, 1] - boundarray[i, 0]) * upars_array[i] sample_point[i] = impact_parameter_from_inclination(physical, inc) else: sample_point[i] = physical[key] @@ -778,7 +884,7 @@ def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bin new_lower = float(mode - radius) new_upper = float(mode + radius) if lower_bound >= 0: - new_lower = max(float(lower_bound), new_lower) + new_lower = max(0.0, new_lower) diagnostics['bounds'] = [new_lower, new_upper] diagnostics['reason'] = ( f"posterior peaks against the {clipped_edge} search bound " @@ -1267,20 +1373,57 @@ def fit_nested(self): def physical_from_sample_point(sample_point): return self._physical_values_from_sample_point(sample_point, bound_keys, sampled_keys) - def loglike(pars): - # chi-squared + def single_loglike(pars): physical = physical_from_sample_point(pars) - model = transit(self.time, physical) - model *= airmass_trend( - physical.get('a2', 0), - self.airmass, - reference=self._get_airmass_reference(), - ) - if self._has_free_flux_baseline(): - model *= get_flux_baseline(physical) - else: - model *= solve_flux_baseline(model, self.data, self.dataerr) - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + duration_prior = self.duration_prior if isinstance(self.duration_prior, dict) else None + duration_loglike = 0.0 + if duration_prior and duration_prior.get('applied'): + expected_duration = duration_prior.get('expected_duration', np.nan) + sigma_log_duration = duration_prior.get('sigma_log_duration', np.nan) + if ( + np.isfinite(expected_duration) + and expected_duration > 0 + and np.isfinite(sigma_log_duration) + and sigma_log_duration > 0 + ): + duration = transit_duration(physical) + if not np.isfinite(duration) or duration <= 0: + return BAD_LOG_LIKELIHOOD + duration_log_residual = np.log(duration / expected_duration) + duration_loglike = -0.5 * (duration_log_residual / sigma_log_duration) ** 2 + try: + model = np.asarray(transit(self.time, physical), dtype=float) + model *= airmass_trend( + physical.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) + if self._has_free_flux_baseline(): + model *= get_flux_baseline(physical) + else: + model *= solve_flux_baseline(model, self.data, self.dataerr) + except Exception: + return BAD_LOG_LIKELIHOOD + + if ( + model.shape != np.asarray(self.data).shape + or not np.all(np.isfinite(model)) + or not np.all(np.isfinite(self.data)) + or not np.all(np.isfinite(self.dataerr)) + or np.any(np.asarray(self.dataerr) <= 0) + ): + return BAD_LOG_LIKELIHOOD + + residuals = (self.data - model) / self.dataerr + chi2 = np.sum(residuals ** 2) + logl = -0.5 * chi2 + duration_loglike + return float(logl) if np.isfinite(logl) else BAD_LOG_LIKELIHOOD + + def loglike(pars): + pars_array = np.asarray(pars, dtype=float) + if pars_array.ndim == 2: + return np.asarray([single_loglike(row) for row in pars_array], dtype=float) + return single_loglike(pars_array) def prior_transform(upars): # transform unit cube to prior volume diff --git a/exotic/api/joint_fitter.py b/exotic/api/joint_fitter.py index 4aadd3e4..f992f40b 100644 --- a/exotic/api/joint_fitter.py +++ b/exotic/api/joint_fitter.py @@ -40,8 +40,12 @@ import builtins from copy import deepcopy from contextlib import redirect_stderr, redirect_stdout +import faulthandler import io from itertools import cycle +import multiprocessing +import os +import sys import matplotlib.pyplot as plt import numpy as np from scipy import stats @@ -63,12 +67,50 @@ except ImportError: from .ultranest_utils import run_reactive_sampler -if not getattr(builtins, "_EXOTIC_IMPORTING_MODULES_PRINTED", False): - print("Importing modules. Please wait.......") +if ( + multiprocessing.current_process().name == "MainProcess" + and not getattr(builtins, "_EXOTIC_IMPORTING_MODULES_PRINTED", False) +): + print("Importing modules. Please wait.......", flush=True) builtins._EXOTIC_IMPORTING_MODULES_PRINTED = True -with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): - from pylightcurve.models.exoplanet_lc import eclipse_mid_time, transit as _pylightcurve_transit + +def _pylightcurve_import_watchdog_seconds(): + try: + return float(os.environ.get("EXOTIC_IMPORT_WATCHDOG_SECONDS", "120")) + except (TypeError, ValueError): + return 120.0 + + +def _start_import_watchdog(): + timeout = _pylightcurve_import_watchdog_seconds() + if timeout <= 0: + return False + + try: + if not faulthandler.is_enabled(): + faulthandler.enable(file=sys.__stdout__, all_threads=True) + faulthandler.dump_traceback_later(timeout, repeat=True, file=sys.__stdout__) + return True + except Exception: + return False + + +def _load_pylightcurve_symbols(): + watchdog_started = _start_import_watchdog() + try: + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + from pylightcurve.models.exoplanet_lc import eclipse_mid_time, transit + return eclipse_mid_time, transit + finally: + if watchdog_started: + try: + faulthandler.cancel_dump_traceback_later() + except Exception: + pass + + +eclipse_mid_time, _pylightcurve_transit = _load_pylightcurve_symbols() AU = const.au.to(u.m).value Mjup = const.M_jup.to(u.kg).value diff --git a/exotic/exotic.py b/exotic/exotic.py index 1670d501..4acab3ca 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -57,9 +57,15 @@ # standard imports import argparse +import copy +import faulthandler +from functools import lru_cache import json import hashlib import os +import sys +import threading +import traceback from concurrent.futures import ProcessPoolExecutor, as_completed from time import sleep, perf_counter # Image alignment import @@ -127,9 +133,21 @@ except ImportError: # package import from api.nea import NASAExoplanetArchive try: # output files - from output_files import OutputFiles, AIDOutputFiles, save_comp_star_calibration_summary + from output_files import ( + OutputFiles, + AIDOutputFiles, + fit_impact_parameter_value_error, + format_parameter_with_error, + save_comp_star_calibration_summary, + ) except ImportError: # package import - from .output_files import OutputFiles, AIDOutputFiles, save_comp_star_calibration_summary + from .output_files import ( + OutputFiles, + AIDOutputFiles, + fit_impact_parameter_value_error, + format_parameter_with_error, + save_comp_star_calibration_summary, + ) try: from plate_status import PlateStatus except ImportError: @@ -166,8 +184,14 @@ # logging -- https://docs.python.org/3/library/logging.html log = logging.getLogger(__name__) +_RUNTIME_LOGGING_CONFIGURED = False +_EXCEPTION_HOOKS_INSTALLED = False +_UNHANDLED_EXCEPTION_LOGGED = False +_BJD_FALLBACK_WARNING_LOGGED = False +_RUNTIME_FILE_HANDLER_NAME = "exotic-runtime-file" +_RUNTIME_CONSOLE_HANDLER_NAME = "exotic-runtime-console" _mid_transit_warning_reported = False -RELATIVE_FLUX_MAX = 2.0 +RELATIVE_FLUX_MAX = 2.0 # Legacy threshold retained for compatibility; no longer used as a hard rejection cap. AIRMASS_FLAT_RANGE_THRESHOLD = 0.05 LIGHTCURVE_MIN_VALID_POINTS = 5 COMPARISON_STAR_MIN_COVERAGE_FRACTION = 0.8 @@ -177,14 +201,28 @@ COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA = 4.25 COMPARISON_STAR_SUITABILITY_MIN_CANDIDATES = 5 COMPARISON_STAR_SUITABILITY_MAX_ITERS = 10 +COMPARISON_IMAGE_OUTLIER_SIGMA = COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA +COMPARISON_IMAGE_OUTLIER_MIN_ACTIVE_STARS = 3 +COMPARISON_IMAGE_OUTLIER_MIN_VALID_PAIRS = 2 +COMPARISON_IMAGE_OUTLIER_MIN_SCATTER = 1e-4 OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 RPRS_SEARCH_BOUND_MIN = 0.0 -RPRS_SEARCH_BOUND_MAX = 0.30 +RPRS_SEARCH_BOUND_MAX = 1.0 RPRS_RETRY_MIN_HALF_WIDTH = 0.05 INITIAL_RPRS_BOUND_LOWER_SCALE = 0.25 INITIAL_RPRS_BOUND_UPPER_SCALE = 3.0 +ARS_SEARCH_BOUND_MIN = 1e-6 +ARS_SEARCH_BOUND_FALLBACK_MAX = 100.0 +ARS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 +ARS_RETRY_MIN_HALF_WIDTH = 0.0 +INITIAL_ARS_BOUND_SIGMA_MULTIPLIER = 5.0 +INITIAL_ARS_BOUND_FALLBACK_RELATIVE_HALF_WIDTH = 0.25 +DURATION_PRIOR_MONTE_CARLO_SAMPLES = 256 +DURATION_PRIOR_MIN_VALID_MONTE_CARLO_SAMPLES = 64 +DURATION_PRIOR_MIN_RELATIVE_SIGMA = 0.05 +DURATION_PRIOR_FALLBACK_RELATIVE_SIGMA = 0.15 FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER = 0.5 EEBLS_DURATION_GRID_SIZE = 15 EEBLS_DURATION_MIN_FRACTION = 0.5 @@ -217,6 +255,24 @@ [1, 1, 1]], dtype=bool, ) +TRANSIT_QC_DELTA_BIC_FAIL_THRESHOLD = 6.0 +TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD = 10.0 +TRANSIT_QC_MIN_RPRS_SIGMA = 3.0 +TRANSIT_QC_MARGINAL_RPRS_SIGMA = 5.0 +TRANSIT_QC_MIN_EEBLS_SNR = 4.0 +TRANSIT_QC_DURATION_RATIO_MIN = 0.5 +TRANSIT_QC_DURATION_RATIO_MAX = 2.0 +TRANSIT_QC_DEFAULT_A2_BOUNDS = (-3.0, 3.0) +TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT = True +TRANSIT_QC_DEVIATION_SIGMA_DEFAULT = 5.0 +TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS = { + 'model_evidence': 0.8, + 'deviation_from_expected_value': 1.5, + 'residual_scatter': 0.7, + 'rprs_significance': 0.5, + 'duration_consistency': 0.75, + 'eebls_depth_snr': 0.75, +} def airmass_span(airmass): @@ -320,6 +376,25 @@ def annotate_nested_tmid_refinement( fit.nested_tmid_refinement_tmid_bounds = refined_tmid_bounds +def annotate_duration_prior(fit, duration_prior): + if fit is None: + return + + summary = duration_prior if isinstance(duration_prior, dict) else {} + fit.duration_prior_applied = bool(summary.get('applied', False)) + fit.duration_prior_note = summary.get('note') + fit.duration_prior_expected_duration = coerce_finite_transit_qc_scalar( + summary.get('expected_duration', np.nan) + ) + fit.duration_prior_sigma_log = coerce_finite_transit_qc_scalar( + summary.get('sigma_log_duration', np.nan) + ) + fit.duration_prior_relative_sigma = coerce_finite_transit_qc_scalar( + summary.get('relative_sigma', np.nan) + ) + fit.duration_prior_source = summary.get('source') + + def annotate_lightcurve_filter_diagnostics(fit, diagnostics): if fit is None: return @@ -327,6 +402,17 @@ def annotate_lightcurve_filter_diagnostics(fit, diagnostics): fit.frame_filter_diagnostics = [dict(diagnostic) for diagnostic in (diagnostics or [])] +def prepend_lightcurve_filter_diagnostic(fit, diagnostic): + if fit is None or not diagnostic: + return + + existing = getattr(fit, 'frame_filter_diagnostics', []) + diagnostics = [dict(diagnostic)] + if isinstance(existing, list): + diagnostics.extend(dict(item) for item in existing if isinstance(item, dict)) + fit.frame_filter_diagnostics = diagnostics + + def annotate_selected_photometry_debug( fit, times, @@ -358,354 +444,2253 @@ def annotate_selected_photometry_debug( } -def save_selected_photometry_debug_series(save_dir, planet_name, observation_date, fit): - if fit is None: - return None +def transit_qc_airmass_reference(airmass): + values = np.asarray(airmass, dtype=float) + finite = values[np.isfinite(values)] + if finite.size == 0: + return 0.0 + return float(np.nanmean(finite)) - debug = getattr(fit, 'selected_photometry_debug', None) - if not debug: - return None - times = np.asarray(debug.get('times'), dtype=float) - target_flux = np.asarray(debug.get('target_flux'), dtype=float) - comp_flux = np.asarray(debug.get('comp_flux'), dtype=float) - raw_ratio = np.asarray(debug.get('raw_ratio'), dtype=float) - initial_sigma_keep_mask = np.asarray(debug.get('initial_sigma_keep_mask'), dtype=bool) - phase_clip_keep_mask = np.asarray( - debug.get('phase_clip_keep_mask_on_sigma_filtered', np.ones(np.count_nonzero(initial_sigma_keep_mask))), - dtype=bool, - ) +def transit_qc_airmass_trend(a2, airmass, reference=None): + values = np.asarray(airmass, dtype=float) + if reference is None: + reference = transit_qc_airmass_reference(values) + return np.exp(float(a2) * (values - float(reference))) - if not ( - times.shape == target_flux.shape == comp_flux.shape == raw_ratio.shape == initial_sigma_keep_mask.shape - ): - return None - phase_keep_full = np.zeros(times.shape[0], dtype=bool) - sigma_kept_indices = np.flatnonzero(initial_sigma_keep_mask) - if sigma_kept_indices.size: - if phase_clip_keep_mask.shape[0] != sigma_kept_indices.size: - phase_clip_keep_mask = np.ones(sigma_kept_indices.size, dtype=bool) - phase_keep_full[sigma_kept_indices] = phase_clip_keep_mask +def solve_transit_qc_flux_baseline(systematics, data, dataerr=None): + systematics = np.asarray(systematics, dtype=float) + data = np.asarray(data, dtype=float) + if systematics.shape != data.shape: + return np.nan - output_dir = Path(save_dir) / "temp" - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / f"SelectedPhotometryRawRatio_{planet_name}_{observation_date}.csv" + weights = np.ones(systematics.shape, dtype=float) + if dataerr is not None: + dataerr = np.asarray(dataerr, dtype=float) + if dataerr.shape != data.shape: + dataerr = None + else: + weights = np.zeros(systematics.shape, dtype=float) + valid_err = np.isfinite(dataerr) & (dataerr > 0) + weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) - output_rows = np.column_stack( - [ - times, - target_flux, - comp_flux, - raw_ratio, - initial_sigma_keep_mask.astype(int), - phase_keep_full.astype(int), - ] - ) - np.savetxt( - output_path, - output_rows, - delimiter=",", - header=( - "BJD_TDB,Target Flux,Comp Flux,Raw Ratio," - "Kept After Initial Sigma Clip,Kept After Phase Residual Clip" - ), - comments="", - fmt=["%.8f", "%.8f", "%.8f", "%.8f", "%d", "%d"], - ) - return output_path + mask = np.isfinite(systematics) & np.isfinite(data) & (systematics != 0) + if dataerr is not None: + mask &= np.isfinite(weights) & (weights > 0) + + if not np.any(mask): + return np.nan + masked_systematics = systematics[mask] + masked_data = data[mask] + masked_weights = weights[mask] + denom = np.sum(masked_weights * masked_systematics ** 2) + if np.isfinite(denom) and denom > 0: + baseline = np.sum(masked_weights * masked_data * masked_systematics) / denom + if np.isfinite(baseline): + return float(baseline) + + ratio = masked_data / masked_systematics + ratio = ratio[np.isfinite(ratio)] + if ratio.size == 0: + return np.nan + return float(np.nanmedian(ratio)) -def annotate_rprs_posterior_refit(fit, applied, note=None, history=None): - if fit is None: - return - history = [] if history is None else list(history) - fit.rprs_posterior_refit_applied = bool(applied) - fit.rprs_posterior_refit_note = note - fit.rprs_posterior_refit_count = len(history) - fit.rprs_posterior_refit_history = history - if history: - latest = history[-1] - fit.rprs_posterior_refit_edge = latest.get('edge') - fit.rprs_posterior_refit_mode = latest.get('mode') - fit.rprs_posterior_refit_std = latest.get('std') - fit.rprs_posterior_refit_original_bounds = latest.get('original_bounds') - fit.rprs_posterior_refit_bounds = latest.get('new_bounds') +def compute_transit_qc_model_chi2(data, model, dataerr=None): + data = np.asarray(data, dtype=float) + model = np.asarray(model, dtype=float) + if data.shape != model.shape: + return np.nan, 0 + + mask = np.isfinite(data) & np.isfinite(model) + if dataerr is not None: + dataerr = np.asarray(dataerr, dtype=float) + if dataerr.shape != data.shape: + dataerr = None + else: + mask &= np.isfinite(dataerr) & (dataerr > 0) + + point_count = int(np.count_nonzero(mask)) + if point_count == 0: + return np.nan, 0 + + residuals = data[mask] - model[mask] + if dataerr is not None: + chi2 = np.sum((residuals / dataerr[mask]) ** 2) else: - fit.rprs_posterior_refit_edge = None - fit.rprs_posterior_refit_mode = None - fit.rprs_posterior_refit_std = None - fit.rprs_posterior_refit_original_bounds = None - fit.rprs_posterior_refit_bounds = None + chi2 = np.sum(residuals ** 2) + return float(chi2), point_count -def build_initial_rprs_bounds( - rprs, - lower_scale=INITIAL_RPRS_BOUND_LOWER_SCALE, - upper_scale=INITIAL_RPRS_BOUND_UPPER_SCALE, -): +def compute_transit_qc_bic(chi2, point_count, parameter_count): try: - rprs = float(rprs) - lower_scale = float(lower_scale) - upper_scale = float(upper_scale) + chi2 = float(chi2) + point_count = int(point_count) + parameter_count = int(parameter_count) except (TypeError, ValueError): - return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + return np.nan - if not np.isfinite(rprs) or rprs <= 0: - return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + if not np.isfinite(chi2) or point_count <= 0 or parameter_count <= 0: + return np.nan + return float(chi2 + parameter_count * np.log(point_count)) - lower_bound = max(RPRS_SEARCH_BOUND_MIN, lower_scale * rprs) - upper_bound = upper_scale * rprs - if not np.isfinite(upper_bound) or upper_bound <= lower_bound: - upper_bound = max(lower_bound + np.finfo(float).eps, rprs) - return [float(lower_bound), float(upper_bound)] +def clip_unit_interval(value): + try: + numeric_value = float(value) + except (TypeError, ValueError): + return np.nan + if not np.isfinite(numeric_value): + return np.nan + return float(np.clip(numeric_value, 0.0, 1.0)) -def clone_lightcurve_bounds(bounds): - return { - key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value - for key, value in bounds.items() - } +def transit_qc_residual_scatter(data, model): + data = np.asarray(data, dtype=float) + model = np.asarray(model, dtype=float) + if data.shape != model.shape or data.size == 0: + return np.nan -def sanitize_rprs_search_bounds(bounds): - sanitized = clone_lightcurve_bounds(bounds) - if 'rprs' not in sanitized: - return sanitized + mask = np.isfinite(data) & np.isfinite(model) + if not np.any(mask): + return np.nan - try: - lower_bound, upper_bound = [ - float(value) for value in np.asarray(sanitized['rprs'], dtype=float).reshape(-1)[:2] - ] - except (TypeError, ValueError, IndexError): - sanitized['rprs'] = [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] - return sanitized + median_flux = np.nanmedian(data[mask]) + if not np.isfinite(median_flux) or median_flux == 0: + return np.nan - if not np.isfinite(lower_bound) or not np.isfinite(upper_bound): - sanitized['rprs'] = [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] - return sanitized + residuals = data[mask] - model[mask] + return float(np.std(residuals) / median_flux) - lower_bound = float(np.clip(lower_bound, RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX)) - upper_bound = float(np.clip(upper_bound, RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX)) - if lower_bound >= upper_bound: - sanitized['rprs'] = [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] - else: - sanitized['rprs'] = [lower_bound, upper_bound] - return sanitized +def transit_qc_deviation_score_from_sigma(sigma_offset, sigma_threshold): + try: + sigma_offset = abs(float(sigma_offset)) + sigma_threshold = float(sigma_threshold) + except (TypeError, ValueError): + return np.nan -def clamp_rprs_prior_to_bounds(prior, bounds): - clamped = dict(prior) - if 'rprs' not in clamped or 'rprs' not in bounds: - return clamped + if not np.isfinite(sigma_offset) or not np.isfinite(sigma_threshold) or sigma_threshold <= 0: + return np.nan + + return float(max(0.0, 1.0 - sigma_offset / sigma_threshold)) + +def transit_qc_duration_score(duration_ratio): try: - rprs_value = float(clamped['rprs']) - lower_bound, upper_bound = [ - float(value) for value in np.asarray(bounds['rprs'], dtype=float).reshape(-1)[:2] - ] - except (TypeError, ValueError, IndexError): - return clamped + duration_ratio = float(duration_ratio) + except (TypeError, ValueError): + return np.nan - if np.isfinite(rprs_value) and np.isfinite(lower_bound) and np.isfinite(upper_bound) and lower_bound < upper_bound: - clamped['rprs'] = float(np.clip(rprs_value, lower_bound, upper_bound)) - return clamped + if not np.isfinite(duration_ratio) or duration_ratio <= 0: + return np.nan + max_log_deviation = np.log(TRANSIT_QC_DURATION_RATIO_MAX) + if not np.isfinite(max_log_deviation) or max_log_deviation <= 0: + return np.nan -def enforce_minimum_rprs_retry_half_width(mode, bounds, min_half_width=RPRS_RETRY_MIN_HALF_WIDTH): - try: - lower_bound, upper_bound = [ - float(value) for value in np.asarray(bounds, dtype=float).reshape(-1)[:2] - ] - except (TypeError, ValueError, IndexError): - return bounds + score = 1.0 - abs(np.log(duration_ratio)) / max_log_deviation + return float(np.clip(score, 0.0, 1.0)) - if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or lower_bound >= upper_bound: - return bounds - center = float(mode) if np.isfinite(mode) else float(0.5 * (lower_bound + upper_bound)) - half_width = max(float(min_half_width), 0.0) - expanded_lower = min(lower_bound, center - half_width) - expanded_upper = max(upper_bound, center + half_width) +def transit_qc_saturating_score(value, scale): + try: + value = float(value) + scale = float(scale) + except (TypeError, ValueError): + return np.nan - if expanded_lower < RPRS_SEARCH_BOUND_MIN: - expanded_upper = min( - RPRS_SEARCH_BOUND_MAX, - expanded_upper + (RPRS_SEARCH_BOUND_MIN - expanded_lower), - ) - expanded_lower = RPRS_SEARCH_BOUND_MIN - if expanded_upper > RPRS_SEARCH_BOUND_MAX: - expanded_lower = max( - RPRS_SEARCH_BOUND_MIN, - expanded_lower - (expanded_upper - RPRS_SEARCH_BOUND_MAX), - ) - expanded_upper = RPRS_SEARCH_BOUND_MAX + if not np.isfinite(value) or not np.isfinite(scale) or scale <= 0: + return np.nan - return [float(expanded_lower), float(expanded_upper)] + return float(np.clip(1.0 - np.exp(-max(value, 0.0) / scale), 0.0, 1.0)) -def run_nested_lightcurve_fit_with_rprs_posterior_retry( - times, - flux_values, - flux_errors, - airmass, - prior, - bounds, - jd_times=None, - use_impactparameter_rather_than_inclination_to_fit=True, - max_rprs_retries=RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, -): - def build_fit(local_prior, local_bounds): - local_bounds = sanitize_rprs_search_bounds(local_bounds) - local_prior = clamp_rprs_prior_to_bounds(local_prior, local_bounds) - return lc_fitter( - times, - flux_values, - flux_errors, - airmass, - local_prior, - local_bounds, - jd_times=jd_times, - mode='ns', - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - ) +def transit_qc_residual_scatter_score(residual_scatter, reference_scatter=0.005): + try: + residual_scatter = float(residual_scatter) + reference_scatter = float(reference_scatter) + except (TypeError, ValueError): + return np.nan - current_bounds = sanitize_rprs_search_bounds(bounds) - current_prior = clamp_rprs_prior_to_bounds(prior, current_bounds) - retry_history = [] - retry_note = None - fit = build_fit(current_prior, current_bounds) + if not np.isfinite(residual_scatter) or residual_scatter < 0 or not np.isfinite(reference_scatter) or reference_scatter <= 0: + return np.nan - diagnostics = None - for _ in range(int(max(0, max_rprs_retries))): - diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) - if not callable(diagnostics_getter): - diagnostics = None - break + return float(np.clip(1.0 / (1.0 + residual_scatter / reference_scatter), 0.0, 1.0)) - diagnostics = diagnostics_getter('rprs') - new_bounds = diagnostics.get('bounds') if diagnostics else None - if not diagnostics or not diagnostics.get('clipped') or new_bounds is None: - break - try: - new_lower, new_upper = [float(value) for value in new_bounds] - except (TypeError, ValueError): - retry_note = "Skipped; the automatic Rp/R* retry proposed malformed bounds." - break - if not np.isfinite(new_lower) or not np.isfinite(new_upper) or new_lower >= new_upper: - retry_note = "Skipped; the automatic Rp/R* retry proposed invalid bounds." - break +def transit_qc_mean_available_score(*scores): + finite_scores = [float(score) for score in scores if np.isfinite(score)] + if not finite_scores: + return np.nan + return float(np.clip(np.mean(finite_scores), 0.0, 1.0)) - previous_bounds = current_bounds.get('rprs') - clamped_bounds = sanitize_rprs_search_bounds({'rprs': [new_lower, new_upper]}).get('rprs', [new_lower, new_upper]) - clamped_bounds = enforce_minimum_rprs_retry_half_width( - diagnostics.get('mode', np.nan), - clamped_bounds, - ) - clamped_bounds = sanitize_rprs_search_bounds({'rprs': clamped_bounds}).get('rprs', clamped_bounds) - new_lower, new_upper = [float(value) for value in clamped_bounds] - if previous_bounds is not None: - previous_lower, previous_upper = [float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2]] - clipped_edge = diagnostics.get('edge') - expands_sampled_range = False - if clipped_edge == 'upper': - expands_sampled_range = new_upper > previous_upper + 1e-12 - elif clipped_edge == 'lower': - expands_sampled_range = new_lower < previous_lower - 1e-12 - else: - expands_sampled_range = ( - new_lower < previous_lower - 1e-12 or - new_upper > previous_upper + 1e-12 - ) - if not expands_sampled_range: - if ( - previous_lower <= RPRS_SEARCH_BOUND_MIN + 1e-12 and - previous_upper >= RPRS_SEARCH_BOUND_MAX - 1e-12 - ): - retry_note = ( - "Skipped; the automatic Rp/R* retry reached the maximum exoplanet " - f"search range [{RPRS_SEARCH_BOUND_MIN:.6f}, {RPRS_SEARCH_BOUND_MAX:.6f}]." - ) - else: - retry_note = "Skipped; the automatic Rp/R* retry did not expand the sampled range." - break +def coerce_finite_transit_qc_scalar(value): + if value is None: + return np.nan - retry_history.append({ - 'attempt': len(retry_history) + 1, - 'edge': diagnostics.get('edge'), - 'mode': float(diagnostics.get('mode', np.nan)), - 'std': float(diagnostics.get('std', np.nan)), - 'original_bounds': None if previous_bounds is None else [float(previous_bounds[0]), float(previous_bounds[1])], - 'new_bounds': [new_lower, new_upper], - }) - log_info( - "Rp/R* posterior is truncated against the " - f"{diagnostics.get('edge', 'active')} search bound; retrying nested fit " - f"with Rp/R* centered at {diagnostics.get('mode', np.nan):.6f} " - f"and sigma {diagnostics.get('std', np.nan):.6f} " - f"over [{new_lower:.6f}, {new_upper:.6f}]." - ) + if isinstance(value, (list, tuple, np.ndarray)): + array_value = np.asarray(value) + if array_value.size != 1: + return np.nan + value = array_value.reshape(-1)[0] - updated_bounds = clone_lightcurve_bounds(current_bounds) - updated_bounds['rprs'] = [new_lower, new_upper] - updated_bounds = sanitize_rprs_search_bounds(updated_bounds) + try: + numeric_value = float(value.strip()) if isinstance(value, str) else float(value) + except (AttributeError, TypeError, ValueError): + return np.nan - updated_prior = dict(current_prior) - fit_parameters = getattr(fit, 'parameters', {}) - if isinstance(fit_parameters, dict): - for key in updated_bounds: - if key in fit_parameters: - updated_prior[key] = fit_parameters[key] - if np.isfinite(diagnostics.get('mode', np.nan)): - updated_prior['rprs'] = float(diagnostics['mode']) - updated_prior = clamp_rprs_prior_to_bounds(updated_prior, updated_bounds) + if not np.isfinite(numeric_value): + return np.nan + return float(numeric_value) - current_prior = updated_prior - current_bounds = updated_bounds - fit = build_fit(current_prior, current_bounds) - if retry_history: - final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) - final_diagnostics = final_diagnostics_getter('rprs') if callable(final_diagnostics_getter) else None - note = f"Applied {len(retry_history)} automatic Rp/R* posterior range refit(s)." - if final_diagnostics and final_diagnostics.get('clipped'): - retry_label = "retry" if len(retry_history) == 1 else "retries" - note = ( - f"{note} The posterior still hugs the {final_diagnostics.get('edge')} bound after " - f"{len(retry_history)} {retry_label}." +def fit_transit_qc_expected_context(fit): + if fit is None: + return {} + + return { + 'expected_tmid': coerce_finite_transit_qc_scalar( + getattr(fit, 'transit_qc_expected_tmid', np.nan) + ), + 'expected_tmid_unc': coerce_finite_transit_qc_scalar( + getattr(fit, 'transit_qc_expected_tmid_unc', np.nan) + ), + 'expected_rprs': coerce_finite_transit_qc_scalar( + getattr(fit, 'transit_qc_expected_rprs', np.nan) + ), + 'expected_rprs_unc': coerce_finite_transit_qc_scalar( + getattr(fit, 'transit_qc_expected_rprs_unc', np.nan) + ), + 'use_deviation_from_expected_transit_in_qc': should_use_deviation_from_expected_transit_in_qc( + getattr( + fit, + 'transit_qc_use_deviation_from_expected_transit_in_qc', + TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT, ) - log_info( - "Warning: Rp/R* posterior still appears truncated after the automatic retries; " - "please inspect the triangle plot carefully.", - warn=True, + ), + 'deviation_sigma_threshold': parse_deviation_from_expected_transit_in_qc_sigma( + getattr( + fit, + 'transit_qc_deviation_sigma_threshold', + TRANSIT_QC_DEVIATION_SIGMA_DEFAULT, + ) + ), + } + + +def annotate_transit_qc_expected_values(fit, planet_dict): + if fit is None or not isinstance(planet_dict, dict): + return + + expected_tmid = coerce_finite_transit_qc_scalar( + getattr(fit, 'initial_tmid_search_tmid', np.nan) + ) + expected_tmid_unc = coerce_finite_transit_qc_scalar( + getattr(fit, 'initial_tmid_search_uncertainty', np.nan) + ) + if not np.isfinite(expected_tmid): + expected_tmid = coerce_finite_transit_qc_scalar(planet_dict.get('midT', np.nan)) + if not np.isfinite(expected_tmid_unc): + expected_tmid_unc = coerce_finite_transit_qc_scalar(planet_dict.get('midTUnc', np.nan)) + + fit.transit_qc_expected_tmid = expected_tmid + fit.transit_qc_expected_tmid_unc = expected_tmid_unc + fit.transit_qc_expected_rprs = coerce_finite_transit_qc_scalar( + planet_dict.get('rprs', np.nan) + ) + fit.transit_qc_expected_rprs_unc = coerce_finite_transit_qc_scalar( + planet_dict.get('rprsUnc', np.nan) + ) + fit.transit_qc_use_deviation_from_expected_transit_in_qc = should_use_deviation_from_expected_transit_in_qc( + planet_dict.get( + 'use_deviation_from_expected_transit_in_qc', + TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT, + ) + ) + fit.transit_qc_deviation_sigma_threshold = parse_deviation_from_expected_transit_in_qc_sigma( + planet_dict.get( + 'deviation_from_expected_transit_in_qc_sigma', + TRANSIT_QC_DEVIATION_SIGMA_DEFAULT, + ) + ) + + +def annotate_transit_qc_fit_context( + fit, + planet_dict=None, + tmid_search_summary=None, + eebls_search_summary=None, +): + if fit is None: + return + + if tmid_search_summary is not None: + annotate_lightcurve_tmid_search(fit, tmid_search_summary) + if eebls_search_summary is not None: + annotate_lightcurve_eebls_diagnostic(fit, eebls_search_summary) + annotate_transit_qc_expected_values(fit, planet_dict) + + +def copy_transit_qc_expected_values(source_fit, target_fit): + if source_fit is None or target_fit is None: + return + + for attr in ( + 'transit_qc_expected_tmid', + 'transit_qc_expected_tmid_unc', + 'transit_qc_expected_rprs', + 'transit_qc_expected_rprs_unc', + 'transit_qc_use_deviation_from_expected_transit_in_qc', + 'transit_qc_deviation_sigma_threshold', + ): + if hasattr(source_fit, attr): + setattr(target_fit, attr, getattr(source_fit, attr)) + + +def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=True): + summary = { + 'enabled': bool(enabled), + 'sigma_threshold': sigma_threshold, + 'tmid_deviation_sigma': np.nan, + 'rprs_deviation_sigma': np.nan, + 'tmid_deviation_score': np.nan, + 'rprs_deviation_score': np.nan, + 'deviation_from_expected_value': np.nan, + 'available': False, + 'failed': False, + 'notes': [], + } + if fit is None or not enabled: + return summary + + parameters = getattr(fit, 'parameters', {}) or {} + expected = fit_transit_qc_expected_context(fit) + sigma_threshold = expected.get('deviation_sigma_threshold', sigma_threshold) + summary['sigma_threshold'] = sigma_threshold + + expected_tmid = expected.get('expected_tmid', np.nan) + expected_tmid_unc = expected.get('expected_tmid_unc', np.nan) + fitted_tmid = parameters.get('tmid', np.nan) + if ( + np.isfinite(expected_tmid) + and np.isfinite(expected_tmid_unc) + and expected_tmid_unc > 0 + and np.isfinite(fitted_tmid) + ): + tmid_sigma = float(abs(fitted_tmid - expected_tmid) / expected_tmid_unc) + summary['tmid_deviation_sigma'] = tmid_sigma + summary['tmid_deviation_score'] = transit_qc_deviation_score_from_sigma(tmid_sigma, sigma_threshold) + + expected_rprs = expected.get('expected_rprs', np.nan) + expected_rprs_unc = expected.get('expected_rprs_unc', np.nan) + fitted_rprs = parameters.get('rprs', np.nan) + if ( + np.isfinite(expected_rprs) + and np.isfinite(expected_rprs_unc) + and expected_rprs_unc > 0 + and np.isfinite(fitted_rprs) + ): + rprs_sigma = float(abs(fitted_rprs - expected_rprs) / expected_rprs_unc) + summary['rprs_deviation_sigma'] = rprs_sigma + summary['rprs_deviation_score'] = transit_qc_deviation_score_from_sigma(rprs_sigma, sigma_threshold) + + component_scores = [ + score + for score in (summary['tmid_deviation_score'], summary['rprs_deviation_score']) + if np.isfinite(score) + ] + summary['available'] = bool(component_scores) + if component_scores: + summary['deviation_from_expected_value'] = float(min(component_scores)) + + if np.isfinite(summary['tmid_deviation_sigma']): + summary['notes'].append( + f"Expected-value Tmid deviation: {summary['tmid_deviation_sigma']:.2f} sigma." + ) + if np.isfinite(summary['rprs_deviation_sigma']): + summary['notes'].append( + f"Expected-value Rp/R* deviation: {summary['rprs_deviation_sigma']:.2f} sigma." + ) + + for label, sigma_value in ( + ('Tmid', summary['tmid_deviation_sigma']), + ('Rp/R*', summary['rprs_deviation_sigma']), + ): + if np.isfinite(sigma_value) and np.isfinite(sigma_threshold) and sigma_threshold > 0 and sigma_value > sigma_threshold: + summary['failed'] = True + summary['notes'].append( + f"{label} differs from the expected value by more than {sigma_threshold:.2f} sigma." + ) + + return summary + + +def compute_transit_qc_ktmf(summary): + if not isinstance(summary, dict): + return np.nan, [] + + delta_bic_score = transit_qc_saturating_score( + summary.get('delta_bic', np.nan), + TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD, + ) + delta_chi2_score = transit_qc_saturating_score(summary.get('delta_chi2', np.nan), 25.0) + model_evidence_score = transit_qc_mean_available_score(delta_bic_score, delta_chi2_score) + model_evidence_detail_parts = [ + f"Delta BIC={format_transit_delta_bic(summary.get('delta_bic', np.nan))}", + f"Delta chi2={summary.get('delta_chi2', np.nan):.2f}" + if np.isfinite(summary.get('delta_chi2', np.nan)) + else "Delta chi2=n/a", + ] + + raw_components = [ + { + 'key': 'model_evidence', + 'label': 'Model Evidence', + 'score': model_evidence_score, + 'detail': ", ".join(model_evidence_detail_parts), + }, + { + 'key': 'deviation_from_expected_value', + 'label': 'Deviation From Expected Value', + 'score': summary.get('deviation_from_expected_value', np.nan), + 'detail': ( + f"score={summary.get('deviation_from_expected_value', np.nan):.2f}, " + f"Tmid sigma={summary.get('tmid_deviation_sigma', np.nan):.2f}, " + f"Rp/R* sigma={summary.get('rprs_deviation_sigma', np.nan):.2f}" + if np.isfinite(summary.get('deviation_from_expected_value', np.nan)) + else "expected-value deviation disabled or unavailable" + ), + }, + { + 'key': 'residual_scatter', + 'label': 'Residual Scatter Around Full Model Fit', + 'score': transit_qc_residual_scatter_score(summary.get('residual_scatter', np.nan)), + 'detail': ( + f"{summary.get('residual_scatter', np.nan) * 100.0:.4f}%" + if np.isfinite(summary.get('residual_scatter', np.nan)) + else "n/a" + ), + }, + { + 'key': 'rprs_significance', + 'label': 'Rp/R* Significance', + 'score': transit_qc_saturating_score(summary.get('rprs_sigma', np.nan), TRANSIT_QC_MIN_RPRS_SIGMA), + 'detail': ( + f"{summary.get('rprs_sigma', np.nan):.2f} sigma" + if np.isfinite(summary.get('rprs_sigma', np.nan)) + else "n/a" + ), + }, + { + 'key': 'duration_consistency', + 'label': 'Duration Consistency', + 'score': transit_qc_duration_score(summary.get('duration_ratio', np.nan)), + 'detail': ( + f"{summary.get('duration_ratio', np.nan):.2f}x expected duration" + if np.isfinite(summary.get('duration_ratio', np.nan)) + else "n/a" + ), + }, + { + 'key': 'eebls_depth_snr', + 'label': 'EEBLS Depth SNR', + 'score': transit_qc_saturating_score(summary.get('eebls_depth_snr', np.nan), TRANSIT_QC_MIN_EEBLS_SNR), + 'detail': ( + f"{summary.get('eebls_depth_snr', np.nan):.2f}" + if np.isfinite(summary.get('eebls_depth_snr', np.nan)) + else "n/a" + ), + }, + ] + + available_components = [ + component + for component in raw_components + if np.isfinite(component.get('score', np.nan)) + and component['key'] in TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS + ] + if not available_components: + return np.nan, [] + + available_max_points = sum(TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS[component['key']] for component in available_components) + if not np.isfinite(available_max_points) or available_max_points <= 0: + return np.nan, [] + + scale_factor = 5.0 / available_max_points + ktmf_contributions = [] + total_points = 0.0 + for component in raw_components: + nominal_max_points = TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS.get(component['key'], 0.0) + score = component.get('score', np.nan) + if np.isfinite(score) and nominal_max_points > 0: + max_points = nominal_max_points * scale_factor + points = float(np.clip(score, 0.0, 1.0) * max_points) + total_points += points + ktmf_contributions.append({ + 'label': component['label'], + 'score': float(np.clip(score, 0.0, 1.0)), + 'max_points': float(max_points), + 'points': points, + 'detail': component.get('detail'), + 'available': True, + }) + else: + ktmf_contributions.append({ + 'label': component['label'], + 'score': np.nan, + 'max_points': 0.0, + 'points': 0.0, + 'detail': component.get('detail'), + 'available': False, + }) + + return float(np.clip(total_points, 0.0, 5.0)), ktmf_contributions + + +def infer_transit_qc_parameter_count(fit, allow_airmass_term): + bounds = getattr(fit, 'bounds', None) + if isinstance(bounds, dict) and bounds: + parameter_count = len(bounds) + if 'a0' not in bounds and 'a1' not in bounds: + parameter_count += 1 + return max(int(parameter_count), 1) + + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + available_keys = set(parameters.keys()) | set(errors.keys()) + + parameter_count = 1 # profiled flux baseline + if 'rprs' in available_keys: + parameter_count += 1 + if 'tmid' in available_keys: + parameter_count += 1 + if 'inc' in available_keys or 'b' in available_keys: + parameter_count += 1 + if allow_airmass_term and 'a2' in available_keys: + parameter_count += 1 + return max(parameter_count, 1) + + +def fit_profiled_flat_null_model(data, dataerr, airmass, initial_a2=0.0, a2_bounds=None, allow_airmass_term=True): + data = np.asarray(data, dtype=float) + dataerr = None if dataerr is None else np.asarray(dataerr, dtype=float) + airmass = None if airmass is None else np.asarray(airmass, dtype=float) + + result = { + 'available': False, + 'used_airmass_term': False, + 'baseline': np.nan, + 'a2': 0.0, + 'model': np.full(data.shape, np.nan, dtype=float), + 'chi2': np.nan, + 'bic': np.nan, + 'point_count': 0, + 'param_count': 1, + 'note': 'Flat/null model was not evaluated.', + } + + if data.ndim != 1 or data.size == 0: + result['note'] = 'Flat/null model comparison unavailable: no 1D lightcurve data were provided.' + return result + + if dataerr is not None and dataerr.shape != data.shape: + dataerr = None + if airmass is not None and airmass.shape != data.shape: + airmass = None + + use_airmass_term = bool( + allow_airmass_term + and airmass is not None + and airmass.ndim == 1 + and airmass.shape == data.shape + and not should_skip_airmass_fit(airmass) + ) + result['used_airmass_term'] = use_airmass_term + result['param_count'] = 1 + int(use_airmass_term) + + if not use_airmass_term: + baseline = solve_transit_qc_flux_baseline(np.ones_like(data, dtype=float), data, dataerr) + if not np.isfinite(baseline): + result['note'] = 'Flat/null model comparison unavailable: could not solve the baseline flux level.' + return result + model = np.full(data.shape, baseline, dtype=float) + chi2, point_count = compute_transit_qc_model_chi2(data, model, dataerr) + result.update({ + 'available': np.isfinite(chi2), + 'baseline': float(baseline), + 'model': model, + 'chi2': chi2, + 'point_count': point_count, + 'bic': compute_transit_qc_bic(chi2, point_count, result['param_count']), + 'note': 'Compared against a flat baseline-only null model.', + }) + return result + + lower, upper = TRANSIT_QC_DEFAULT_A2_BOUNDS + if a2_bounds is not None: + try: + lower, upper = np.asarray(a2_bounds, dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + lower, upper = TRANSIT_QC_DEFAULT_A2_BOUNDS + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + lower, upper = TRANSIT_QC_DEFAULT_A2_BOUNDS + + if not np.isfinite(initial_a2): + initial_a2 = 0.0 + initial_a2 = float(np.clip(initial_a2, lower + np.finfo(float).eps, upper - np.finfo(float).eps)) + reference = transit_qc_airmass_reference(airmass) + + def build_model(a2_value): + systematics = transit_qc_airmass_trend(a2_value, airmass, reference=reference) + baseline = solve_transit_qc_flux_baseline(systematics, data, dataerr) + if not np.isfinite(baseline): + return np.full(data.shape, np.nan, dtype=float), np.nan + return baseline * systematics, baseline + + def residual_vector(params): + model, baseline = build_model(params[0]) + if not np.isfinite(baseline): + return np.full(max(1, data.size), 1e6, dtype=float) + + mask = np.isfinite(data) & np.isfinite(model) + if dataerr is not None: + mask &= np.isfinite(dataerr) & (dataerr > 0) + if not np.any(mask): + return np.full(max(1, data.size), 1e6, dtype=float) + + if dataerr is not None: + return (data[mask] - model[mask]) / dataerr[mask] + return data[mask] - model[mask] + + best_a2 = float(initial_a2) + try: + fit_result = least_squares( + residual_vector, + x0=np.array([initial_a2], dtype=float), + bounds=([lower], [upper]), + ) + if fit_result.x.size: + best_a2 = float(fit_result.x[0]) + except Exception: + pass + + model, baseline = build_model(best_a2) + if not np.isfinite(baseline): + result['note'] = 'Flat/null model comparison unavailable: the null-model fit did not converge.' + return result + + chi2, point_count = compute_transit_qc_model_chi2(data, model, dataerr) + result.update({ + 'available': np.isfinite(chi2), + 'baseline': float(baseline), + 'a2': float(best_a2), + 'model': model, + 'chi2': chi2, + 'point_count': point_count, + 'bic': compute_transit_qc_bic(chi2, point_count, result['param_count']), + 'note': 'Compared against a flat null model with the same profiled baseline and airmass trend.', + }) + return result + + +def evaluate_transit_detection_qc(fit): + expected_context = fit_transit_qc_expected_context(fit) + use_deviation_from_expected_transit_in_qc = expected_context.get( + 'use_deviation_from_expected_transit_in_qc', + TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT, + ) + deviation_sigma_threshold = expected_context.get( + 'deviation_sigma_threshold', + TRANSIT_QC_DEVIATION_SIGMA_DEFAULT, + ) + summary = { + 'computed': False, + 'status': 'unknown', + 'preferred_model': 'unknown', + 'summary': 'Transit QC unavailable: no fit result was provided.', + 'notes': [], + 'transit_chi2': np.nan, + 'flat_chi2': np.nan, + 'delta_chi2': np.nan, + 'transit_bic': np.nan, + 'flat_bic': np.nan, + 'delta_bic': np.nan, + 'transit_parameter_count': 0, + 'flat_parameter_count': 0, + 'flat_baseline': np.nan, + 'flat_a2': np.nan, + 'flat_model_note': None, + 'rprs_sigma': np.nan, + 'duration_ratio': np.nan, + 'eebls_depth_snr': np.nan, + 'residual_scatter': np.nan, + 'use_deviation_from_expected_transit_in_qc': bool(use_deviation_from_expected_transit_in_qc), + 'deviation_sigma_threshold': deviation_sigma_threshold, + 'expected_tmid': expected_context.get('expected_tmid', np.nan), + 'expected_tmid_unc': expected_context.get('expected_tmid_unc', np.nan), + 'expected_rprs': expected_context.get('expected_rprs', np.nan), + 'expected_rprs_unc': expected_context.get('expected_rprs_unc', np.nan), + 'tmid_deviation_sigma': np.nan, + 'rprs_deviation_sigma': np.nan, + 'tmid_deviation_score': np.nan, + 'rprs_deviation_score': np.nan, + 'deviation_from_expected_value': np.nan, + 'ktmf_metric': np.nan, + 'ktmf_contributions': [], + 'point_count': 0, + } + if fit is None: + return summary + + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if data.ndim != 1 or data.size == 0: + summary['summary'] = ( + "Transit QC unavailable: fit results do not expose the 1D lightcurve data needed for " + "a transit-vs-flat comparison." + ) + return summary + + dataerr_obj = getattr(fit, 'dataerr', None) + dataerr = None if dataerr_obj is None else np.asarray(dataerr_obj, dtype=float) + if dataerr is not None and dataerr.shape != data.shape: + dataerr = None + + transit_model_obj = getattr(fit, 'model', None) + if transit_model_obj is None and hasattr(fit, 'residuals'): + residuals = np.asarray(getattr(fit, 'residuals'), dtype=float) + if residuals.shape == data.shape: + transit_model_obj = data - residuals + if transit_model_obj is None: + summary['summary'] = ( + "Transit QC unavailable: fit results do not expose the modeled transit lightcurve needed " + "for a transit-vs-flat comparison." + ) + return summary + + transit_model = np.asarray(transit_model_obj, dtype=float) + if transit_model.shape != data.shape: + summary['summary'] = ( + "Transit QC unavailable: the fitted transit model shape does not match the lightcurve data." + ) + return summary + + airmass_obj = getattr(fit, 'airmass', None) + airmass = None if airmass_obj is None else np.asarray(airmass_obj, dtype=float) + if airmass is not None and airmass.shape != data.shape: + airmass = None + + allow_airmass_term = bool( + airmass is not None + and airmass.ndim == 1 + and not getattr(fit, 'airmass_fit_skipped', False) + ) + bounds = getattr(fit, 'bounds', None) + a2_bounds = bounds.get('a2') if isinstance(bounds, dict) else None + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + initial_a2 = parameters.get('a2', 0.0) + + flat_model = fit_profiled_flat_null_model( + data, + dataerr, + airmass, + initial_a2=initial_a2, + a2_bounds=a2_bounds, + allow_airmass_term=allow_airmass_term, + ) + transit_chi2, point_count = compute_transit_qc_model_chi2(data, transit_model, dataerr) + transit_parameter_count = infer_transit_qc_parameter_count(fit, flat_model.get('used_airmass_term', False)) + transit_bic = compute_transit_qc_bic(transit_chi2, point_count, transit_parameter_count) + flat_bic = flat_model.get('bic', np.nan) + delta_chi2 = flat_model.get('chi2', np.nan) - transit_chi2 + delta_bic = flat_bic - transit_bic + + summary.update({ + 'computed': bool(bool(flat_model.get('available')) and np.isfinite(transit_chi2)), + 'transit_chi2': transit_chi2, + 'flat_chi2': flat_model.get('chi2', np.nan), + 'delta_chi2': delta_chi2, + 'transit_bic': transit_bic, + 'flat_bic': flat_bic, + 'delta_bic': delta_bic, + 'transit_parameter_count': int(transit_parameter_count), + 'flat_parameter_count': int(flat_model.get('param_count', 0)), + 'flat_baseline': flat_model.get('baseline', np.nan), + 'flat_a2': flat_model.get('a2', np.nan), + 'flat_model_note': flat_model.get('note'), + 'residual_scatter': transit_qc_residual_scatter(data, transit_model), + 'point_count': int(point_count), + }) + + if not summary['computed']: + note = flat_model.get('note') or 'flat/null model comparison failed.' + summary['summary'] = f"Transit QC unavailable: {note}" + return summary + + if np.isfinite(delta_chi2): + if delta_chi2 > 1e-12: + summary['preferred_model'] = 'transit' + elif delta_chi2 < -1e-12: + summary['preferred_model'] = 'flat' + else: + summary['preferred_model'] = 'ambiguous' + + rprs = parameters.get('rprs', np.nan) + rprs_err = errors.get('rprs', np.nan) + if np.isfinite(rprs) and np.isfinite(rprs_err) and rprs_err > 0: + summary['rprs_sigma'] = float(abs(rprs) / rprs_err) + + duration_expected = getattr(fit, 'duration_expected', np.nan) + duration_measured = getattr(fit, 'duration_measured', np.nan) + if ( + np.isfinite(duration_expected) + and duration_expected > 0 + and np.isfinite(duration_measured) + and duration_measured >= 0 + ): + summary['duration_ratio'] = float(duration_measured / duration_expected) + + ensure_lightcurve_fit_eebls_diagnostic(fit) + summary['eebls_depth_snr'] = extract_lightcurve_fit_eebls_snr(fit) + deviation_summary = evaluate_transit_qc_expected_value_deviation( + fit, + deviation_sigma_threshold, + enabled=use_deviation_from_expected_transit_in_qc, + ) + summary.update({ + 'tmid_deviation_sigma': deviation_summary.get('tmid_deviation_sigma', np.nan), + 'rprs_deviation_sigma': deviation_summary.get('rprs_deviation_sigma', np.nan), + 'tmid_deviation_score': deviation_summary.get('tmid_deviation_score', np.nan), + 'rprs_deviation_score': deviation_summary.get('rprs_deviation_score', np.nan), + 'deviation_from_expected_value': deviation_summary.get('deviation_from_expected_value', np.nan), + }) + + notes = [] + failure_reasons = [] + status = 'pass' + comparison_text = ( + f"Delta BIC={delta_bic:.2f}, Delta chi2={delta_chi2:.2f}" + if np.isfinite(delta_bic) and np.isfinite(delta_chi2) + else "model comparison unavailable" + ) + + if not np.isfinite(delta_bic) or not np.isfinite(delta_chi2): + status = 'unknown' + notes.append("Transit-vs-flat model comparison was not finite.") + elif delta_chi2 <= 0: + status = 'fail' + notes.append("The flat/null model fits the lightcurve at least as well as the transit model.") + failure_reasons.append("the flat/null model fits the lightcurve at least as well as the transit model") + elif delta_bic < TRANSIT_QC_DELTA_BIC_FAIL_THRESHOLD: + status = 'fail' + notes.append( + "The transit model does not beat the flat/null model strongly enough to claim a detection." + ) + failure_reasons.append( + "the transit model does not beat the flat/null model strongly enough to claim a detection" + ) + elif delta_bic < TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD: + status = 'marginal' + notes.append( + "The transit model is preferred over the flat/null model, but the evidence is only moderate." + ) + else: + notes.append("The transit model is strongly preferred over the flat/null model.") + + if np.isfinite(summary['rprs_sigma']): + if summary['rprs_sigma'] < TRANSIT_QC_MIN_RPRS_SIGMA: + status = 'fail' + notes.append( + f"The fitted transit depth is only {summary['rprs_sigma']:.2f}-sigma." + ) + failure_reasons.append( + f"the fitted transit depth is only {summary['rprs_sigma']:.2f}-sigma" + ) + elif summary['rprs_sigma'] < TRANSIT_QC_MARGINAL_RPRS_SIGMA and status == 'pass': + status = 'marginal' + notes.append( + f"The fitted transit depth is only {summary['rprs_sigma']:.2f}-sigma." + ) + + if np.isfinite(summary['duration_ratio']): + if ( + summary['duration_ratio'] < TRANSIT_QC_DURATION_RATIO_MIN + or summary['duration_ratio'] > TRANSIT_QC_DURATION_RATIO_MAX + ): + if status == 'pass': + status = 'marginal' + notes.append( + f"The measured transit duration is {summary['duration_ratio']:.2f}x the modeled duration." + ) + + if np.isfinite(summary['eebls_depth_snr']) and summary['eebls_depth_snr'] < TRANSIT_QC_MIN_EEBLS_SNR: + if status == 'pass': + status = 'marginal' + notes.append( + f"EEBLS only found a weak box-like event (depth SNR={summary['eebls_depth_snr']:.2f})." + ) + + if use_deviation_from_expected_transit_in_qc: + notes.extend(deviation_summary.get('notes', [])) + if deviation_summary.get('failed'): + status = 'fail' + notes.append( + "The fit deviates too far from the expected published Tmid and/or Rp/R* values." + ) + failure_reasons.append( + "the fit deviates too far from the expected published Tmid and/or Rp/R* values" ) - elif retry_note is not None: - note = retry_note - elif diagnostics is not None and diagnostics.get('reason'): - note = f"Not needed; {diagnostics['reason']}" + + ktmf_metric, ktmf_contributions = compute_transit_qc_ktmf(summary) + summary['ktmf_metric'] = ktmf_metric + summary['ktmf_contributions'] = ktmf_contributions + + if status == 'pass': + summary_text = f"Transit model strongly preferred over flat/null model ({comparison_text})." + elif status == 'marginal': + summary_text = f"Transit model preferred over flat/null model, but the detection is marginal ({comparison_text})." + elif status == 'fail': + if failure_reasons: + flat_model_only_failure = all( + "flat/null model" in reason or "does not beat the flat/null model" in reason + for reason in failure_reasons + ) + if flat_model_only_failure: + summary_text = ( + f"Transit detection not supported strongly enough against a flat/null model ({comparison_text})." + ) + else: + summary_text = ( + "Transit model is preferred over the flat/null model, but QC rejected the fit because " + + "; ".join(failure_reasons) + + f" ({comparison_text})." + ) + else: + summary_text = f"Transit detection QC rejected this fit ({comparison_text})." else: - note = "Not evaluated; posterior diagnostics are unavailable for this fit." + summary_text = f"Transit QC unavailable ({comparison_text})." + + summary.update({ + 'status': status, + 'summary': summary_text, + 'notes': notes, + }) + return summary + + +def annotate_transit_detection_qc(fit, summary=None): + if fit is None: + return + + summary = evaluate_transit_detection_qc(fit) if summary is None else dict(summary) + fit.transit_qc = summary + fit.transit_qc_computed = bool(summary.get('computed')) + fit.transit_qc_status = summary.get('status') + fit.transit_qc_summary = summary.get('summary') + fit.transit_qc_preferred_model = summary.get('preferred_model') + fit.transit_qc_delta_bic = summary.get('delta_bic') + fit.transit_qc_delta_chi2 = summary.get('delta_chi2') + fit.transit_qc_rprs_sigma = summary.get('rprs_sigma') + fit.transit_qc_duration_ratio = summary.get('duration_ratio') + fit.transit_qc_eebls_depth_snr = summary.get('eebls_depth_snr') + fit.transit_qc_residual_scatter = summary.get('residual_scatter') + fit.transit_qc_deviation_from_expected_value = summary.get('deviation_from_expected_value') + fit.transit_qc_deviation_sigma_threshold = summary.get('deviation_sigma_threshold') + fit.transit_qc_tmid_deviation_sigma = summary.get('tmid_deviation_sigma') + fit.transit_qc_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') + fit.transit_qc_expected_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') + fit.transit_qc_ktmf_metric = summary.get('ktmf_metric') + fit.transit_qc_ktmf_contributions = summary.get('ktmf_contributions') + + +def lightcurve_fit_transit_qc_failure_reason(fit): + if fit is None: + return None + + transit_qc = getattr(fit, 'transit_qc', None) + if not isinstance(transit_qc, dict): + return None + + status = str(transit_qc.get('status', '')).strip().lower() + if status != 'fail': + return None + + summary = transit_qc.get('summary') + if isinstance(summary, str) and summary.strip(): + return summary.strip() + + return "Transit detection QC flagged this fit as a poor transit candidate." + + +def make_json_safe(value): + if isinstance(value, dict): + return {str(key): make_json_safe(subvalue) for key, subvalue in value.items()} + if isinstance(value, (list, tuple)): + return [make_json_safe(item) for item in value] + if isinstance(value, np.ndarray): + return [make_json_safe(item) for item in value.tolist()] + if isinstance(value, np.generic): + return value.item() + if isinstance(value, Path): + return str(value) + return value + + +def archive_exception_payload(action, exc): + traceback_text = ''.join( + traceback.format_exception(type(exc), exc, exc.__traceback__) + ).strip() + message = f"{action}: {type(exc).__name__}: {exc}" + try: + log_info(f"Warning: {message}", warn=True) + except Exception: + print(f"Warning: {message}", flush=True) + log.debug("%s\n%s", message, traceback_text) + return { + 'action': action, + 'error_type': type(exc).__name__, + 'message': str(exc), + 'traceback': traceback_text, + } + + +def failed_comparison_archive_dir(save_dir, comp_index): + base_dir = Path(save_dir) + candidate_dir = base_dir / f"comp_{comp_index + 1}_failed" + if not candidate_dir.exists(): + return candidate_dir + + suffix = 2 + while True: + fallback_dir = base_dir / f"comp_{comp_index + 1}_failed_{suffix}" + if not fallback_dir.exists(): + return fallback_dir + suffix += 1 + + +def comparison_candidate_output_dir(save_dir, comp_index): + return Path(save_dir) / f"comp{comp_index + 1}" + + +def estimate_transit_duration_samples_from_fit(fit, sample_count=1000, grid_size=1000): + if fit is None or not hasattr(fit, 'parameters') or not hasattr(fit, 'errors'): + return None, np.array([], dtype=float) + + fit_times = np.asarray(getattr(fit, 'time', []), dtype=float) + fit_times = fit_times[np.isfinite(fit_times)] + if fit_times.size < 2: + return None, np.array([], dtype=float) + + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + transit_times = np.linspace(np.nanmin(fit_times), np.nanmax(fit_times), int(grid_size)) + if transit_times.size < 2: + return None, np.array([], dtype=float) + + baseline_parameters = dict(parameters) + baseline_model = transit(transit_times, baseline_parameters) + dt = float(np.nanmean(np.diff(transit_times))) + if not np.isfinite(dt) or dt <= 0: + return baseline_model, np.array([], dtype=float) + + duration_samples = [] + sample_count = max(1, int(sample_count)) + for _ in range(sample_count): + sampled_parameters = dict(parameters) + for key, error_value in errors.items(): + parameter_value = parameters.get(key) + if parameter_value is None: + continue + try: + numeric_error = float(error_value) + numeric_value = float(parameter_value) + except (TypeError, ValueError): + continue + if not np.isfinite(numeric_error) or numeric_error <= 0 or not np.isfinite(numeric_value): + continue + sampled_parameters[key] = np.random.normal(numeric_value, numeric_error) + + sampled_model = transit(transit_times, sampled_parameters) + in_transit_mask = np.asarray(sampled_model, dtype=float) < 1 + duration_samples.append(float(np.count_nonzero(in_transit_mask)) * dt) + + return baseline_model, np.asarray(duration_samples, dtype=float) + + +def build_comparison_candidate_adaptive_summary(comparison_calibration, psf_data, + use_adaptive_apertures=False, + adaptive_aperture_values=None, + adaptive_annulus_values=None, + fallback_sigma=np.nan): + if ( + not use_adaptive_apertures + or comparison_calibration is None + or comparison_calibration.get('method') == 'psf' + or adaptive_aperture_values is None + or adaptive_annulus_values is None + ): + return None + + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + if aperture_index is None or annulus_index is None: + return None + + aperture_grid = np.asarray(adaptive_aperture_values, dtype=float) + annulus_grid = np.asarray(adaptive_annulus_values, dtype=float) + if aperture_index >= aperture_grid.size or annulus_index >= annulus_grid.size: + return None + + return summarize_adaptive_aperture_usage( + psf_data['target'], + aperture_grid[aperture_index], + annulus_grid[annulus_index], + fallback_sigma=fallback_sigma, + ) + + +def match_time_subset_indices(full_times, subset_times, rtol=1e-10, atol=1e-10): + full_times = np.asarray(full_times, dtype=float).reshape(-1) + subset_times = np.asarray(subset_times, dtype=float).reshape(-1) + if subset_times.size == 0: + return np.array([], dtype=int) + if full_times.size < subset_times.size: + return None + + matched_indices = [] + search_start = 0 + for subset_time in subset_times: + if not np.isfinite(subset_time): + return None + remaining = full_times[search_start:] + matches = np.flatnonzero(np.isclose(remaining, subset_time, rtol=rtol, atol=atol)) + if matches.size == 0: + return None + matched_index = search_start + int(matches[0]) + matched_indices.append(matched_index) + search_start = matched_index + 1 + + return np.asarray(matched_indices, dtype=int) + + +def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, airmass, ld, p_dict, + jd_times=None, + disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True, + plot_time_range=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + adaptive_summary=None): + result = { + 'applied': False, + 'fit': None, + 'good_times': np.array([], dtype=float), + 'good_flux': np.array([], dtype=float), + 'good_unc': np.array([], dtype=float), + 'good_airmass': np.array([], dtype=float), + 'good_jd_times': np.array([], dtype=float), + 'good_target_flux': np.array([], dtype=float), + 'good_comp_flux': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), + 'data_highres': None, + 'duration_samples': np.array([], dtype=float), + 'failure_reason': "full candidate reduction did not run.", + 'filter_diagnostics': [], + 'note': None, + } + prepared = prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + ) + result['filter_diagnostics'] = prepared.get('filter_diagnostics', []) + if not prepared.get('applied'): + result['failure_reason'] = prepared.get( + 'failure_reason', + "the raw comparison-candidate photometry did not yield a usable light curve.", + ) + return result + + good_times = np.asarray(prepared['time'], dtype=float) + good_flux = np.asarray(prepared['flux'], dtype=float) + good_unc = np.asarray(prepared['unc'], dtype=float) + good_airmass = np.asarray(prepared['airmass'], dtype=float) + good_jd_times = np.asarray(prepared['jd_time'], dtype=float) + good_target_flux = np.asarray(prepared['target_flux'], dtype=float) + good_comp_flux = np.asarray(prepared['comp_flux'], dtype=float) + source_indices = np.asarray(prepared['source_indices'], dtype=int) + + adaptive_clip_mask = np.zeros(good_times.shape[0], dtype=bool) + if adaptive_summary is not None: + aperture_series = np.asarray(adaptive_summary.get('aperture_series', []), dtype=float) + annulus_series = np.asarray(adaptive_summary.get('annulus_series', []), dtype=float) + if aperture_series.ndim == 1 and annulus_series.ndim == 1: + try: + selected_apertures = aperture_series[source_indices] + selected_annuli = annulus_series[source_indices] + except IndexError: + selected_apertures = None + selected_annuli = None + if ( + selected_apertures is not None + and selected_apertures.shape == good_times.shape + and selected_annuli.shape == good_times.shape + ): + adaptive_clip_mask = adaptive_aperture_outlier_mask( + selected_apertures, + selected_annuli, + ) + + if np.count_nonzero(~adaptive_clip_mask) < LIGHTCURVE_MIN_VALID_POINTS: + result['failure_reason'] = ( + "adaptive-aperture filtering left too few points for a stable comparison-candidate reduction." + ) + return result + + if np.any(adaptive_clip_mask): + good_times = good_times[~adaptive_clip_mask] + good_flux = good_flux[~adaptive_clip_mask] + good_unc = good_unc[~adaptive_clip_mask] + good_airmass = good_airmass[~adaptive_clip_mask] + good_jd_times = good_jd_times[~adaptive_clip_mask] + good_target_flux = good_target_flux[~adaptive_clip_mask] + good_comp_flux = good_comp_flux[~adaptive_clip_mask] + source_indices = source_indices[~adaptive_clip_mask] + + relative_flux_mask = relative_flux_filter_mask(good_flux) + if np.count_nonzero(relative_flux_mask) < LIGHTCURVE_MIN_VALID_POINTS: + result['failure_reason'] = ( + "the raw comparison-candidate light curve failed the relative-flux filter before full reduction." + ) + return result + + good_times = good_times[relative_flux_mask] + good_flux = good_flux[relative_flux_mask] + good_unc = good_unc[relative_flux_mask] + good_airmass = good_airmass[relative_flux_mask] + good_jd_times = good_jd_times[relative_flux_mask] + good_target_flux = good_target_flux[relative_flux_mask] + good_comp_flux = good_comp_flux[relative_flux_mask] + source_indices = source_indices[relative_flux_mask] + + prior = { + 'rprs': p_dict['rprs'], + 'ars': p_dict['aRs'], + 'per': p_dict['pPer'], + 'inc': p_dict['inc'], + 'u0': ld[0], 'u1': ld[1], 'u2': ld[2], 'u3': ld[3], + 'ecc': p_dict['ecc'], + 'omega': p_dict['omega'], + 'tmid': p_dict['midT'], + 'a2': 0, + } + + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + good_times, + p_dict['midT'], + prior['per'], + p_dict['midTUnc'], + p_dict['pPerUnc'], + expected_duration=expected_duration, + sigma_multiplier=35.0, + ) + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] + + eebls_search_summary = None + if use_eebls_to_initialize_tmid_and_bounds: + eebls_search_summary = estimate_tmid_and_bounds_with_eebls( + good_times, + good_flux, + good_unc, + prior, + [lower, upper], + ) + if eebls_search_summary.get('applied'): + prior['tmid'] = eebls_search_summary['tmid'] + lower, upper = eebls_search_summary['bounds'] + + skip_final_airmass_fit = False + airmass_skip_note = None + final_airmass_span = airmass_span(good_airmass) + if should_skip_airmass_fit(good_airmass): + skip_final_airmass_fit = True + airmass_skip_note = ( + f"Skipped (airmass span {final_airmass_span:.4f} <= {AIRMASS_FLAT_RANGE_THRESHOLD:.2f}); " + "no airmass correction applied." + ) + + bounds = build_initial_transit_bounds( + prior, + [lower, upper], + ars_unc=p_dict.get('aRsUnc'), + ) + apply_vertical_flux_normalization_bound( + prior, + bounds, + good_flux, + disable_vertical_flux_normalization, + ) + if not skip_final_airmass_fit: + bounds['a2'] = [-3, 3] + + debug_phase_clip_keep_mask = None + prefit = lc_fitter( + good_times, + good_flux, + good_unc, + good_airmass, + prior, + bounds, + jd_times=good_jd_times, + mode='lm', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + ) + if ( + prefit is not None + and hasattr(prefit, 'residuals') + and hasattr(prefit, 'phase') + and np.shape(prefit.residuals) == np.shape(good_times) + and np.shape(prefit.phase) == np.shape(good_times) + ): + phase_clip_mask = phase_bin_sigma_clip(prefit.residuals, prefit.phase, sigma=3, bins=10) + min_required_points = max(len(bounds) + 1, LIGHTCURVE_MIN_VALID_POINTS) + if np.any(phase_clip_mask) and np.count_nonzero(~phase_clip_mask) >= min_required_points: + debug_phase_clip_keep_mask = np.asarray(~phase_clip_mask, dtype=bool).copy() + result['filter_diagnostics'].append(build_time_rejection_diagnostic( + "Final-fit phase residual clip", + good_times, + ~phase_clip_mask, + note="Dropped phase-binned residual outliers before the comparison-candidate ultranest fit.", + )) + good_times = good_times[~phase_clip_mask] + good_flux = good_flux[~phase_clip_mask] + good_unc = good_unc[~phase_clip_mask] + good_airmass = good_airmass[~phase_clip_mask] + good_jd_times = good_jd_times[~phase_clip_mask] + good_target_flux = good_target_flux[~phase_clip_mask] + good_comp_flux = good_comp_flux[~phase_clip_mask] + source_indices = source_indices[~phase_clip_mask] + + final_fit, good_flux, good_unc = fit_final_lightcurve_with_oot_baseline_detrending( + good_times, + good_flux, + good_unc, + good_airmass, + prior, + bounds, + jd_times=good_jd_times, + skip_airmass_fit=skip_final_airmass_fit, + airmass_skip_note=airmass_skip_note, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + baseline_duration_multiplier=baseline_duration_multiplier, + expected_planet_dict=p_dict, + expected_tmid_search_summary=tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) + if final_fit is None: + result['failure_reason'] = "the full comparison-candidate reduction did not converge." + return result + + final_fit_times = np.asarray(getattr(final_fit, 'time', good_times), dtype=float) + if ( + final_fit_times.shape != good_times.shape + or not np.allclose(final_fit_times, good_times, rtol=1e-10, atol=1e-10) + ): + final_time_indices = match_time_subset_indices(good_times, final_fit_times) + if final_time_indices is not None: + good_times = good_times[final_time_indices] + good_airmass = good_airmass[final_time_indices] + good_jd_times = good_jd_times[final_time_indices] + good_target_flux = good_target_flux[final_time_indices] + good_comp_flux = good_comp_flux[final_time_indices] + source_indices = source_indices[final_time_indices] + + annotate_lightcurve_filter_diagnostics(final_fit, result['filter_diagnostics']) + annotate_selected_photometry_debug( + final_fit, + prepared['debug_times'], + prepared['debug_target_flux'], + prepared['debug_comp_flux'], + prepared['debug_raw_ratio'], + prepared['initial_sigma_keep_mask'], + phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, + ) + + data_highres, duration_samples = estimate_transit_duration_samples_from_fit(final_fit) + result.update({ + 'applied': True, + 'fit': final_fit, + 'good_times': np.asarray(good_times, dtype=float), + 'good_flux': np.asarray(good_flux, dtype=float), + 'good_unc': np.asarray(good_unc, dtype=float), + 'good_airmass': np.asarray(good_airmass, dtype=float), + 'good_jd_times': np.asarray(good_jd_times, dtype=float), + 'good_target_flux': np.asarray(good_target_flux, dtype=float), + 'good_comp_flux': np.asarray(good_comp_flux, dtype=float), + 'source_indices': np.asarray(source_indices, dtype=int), + 'data_highres': data_highres, + 'duration_samples': duration_samples, + 'failure_reason': None, + 'note': 'completed the full comparison-candidate reduction directly from the raw target/reference light curve.', + }) + return result + + +def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, final_fit, + p_dict, observation_date, comp_index, + comp_coords=None, min_aperture=None, min_annulus=None, + adaptive_summary=None, method_label=None, + selection_summary=None, duration_samples=None, + data_highres=None): + if save_dir is None or final_fit is None or observation_date is None or comp_index is None: + return None + + candidate_dir = comparison_candidate_output_dir(save_dir, comp_index) + temp_dir = candidate_dir / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + + archive_errors = [] + debug_series_path = None + bestfit_plot_path = None + triangle_plot_path = None + + debug_fit = provisional_fit if provisional_fit is not None else final_fit + if debug_fit is not None: + try: + debug_series_path = save_selected_photometry_debug_series( + candidate_dir, + p_dict['pName'], + observation_date, + debug_fit, + ) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the selected raw target/reference ratio diagnostics", + exc, + )) + + plotter = getattr(final_fit, 'plot_bestfit', None) + if callable(plotter): + try: + fig, _ = plotter() + bestfit_plot_path = temp_dir / f"BestFit_{p_dict['pName']}_{observation_date}.png" + fig.savefig(bestfit_plot_path) + plt.close(fig) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the final best-fit plot", + exc, + )) + + triangle_plotter = getattr(final_fit, 'plot_triangle', None) + if callable(triangle_plotter): + try: + fig = triangle_plotter() + triangle_plot_path = temp_dir / f"Triangle_{p_dict['pName']}_{observation_date}.png" + fig.savefig(triangle_plot_path) + plt.close(fig) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the triangle plot", + exc, + )) + + duration_samples = np.asarray([] if duration_samples is None else duration_samples, dtype=float) + if duration_samples.size == 0: + measured_duration = getattr(final_fit, 'duration_measured', np.nan) + if np.isfinite(measured_duration) and measured_duration > 0: + duration_samples = np.asarray([measured_duration], dtype=float) + + candidate_info_dict = { + 'save': str(candidate_dir), + 'date': observation_date, + } + try: + if data_highres is None: + data_highres, _ = estimate_transit_duration_samples_from_fit(final_fit, sample_count=1) + if data_highres is not None: + plot_final_lightcurve(final_fit, data_highres, p_dict['pName'], candidate_info_dict['save'], observation_date) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the final lightcurve plot", + exc, + )) + + output_files = OutputFiles(final_fit, p_dict, candidate_info_dict, duration_samples) + try: + phase = get_phase(final_fit.time, p_dict['pPer'], final_fit.parameters['tmid']) + output_files.final_lightcurve(phase) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save FinalLightCurve CSV", + exc, + )) + + try: + output_files.final_planetary_params( + phot_opt=True, + vsp_params=[], + comp_star=int(comp_index + 1), + comp_coords=comp_coords, + min_aper=0 if min_aperture is None else np.round(min_aperture, 2), + min_annul=(None if min_annulus is None else np.round(min_annulus, 2)), + adaptive_summary=adaptive_summary, + ) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save FinalParams JSON", + exc, + )) + + summary_path = temp_dir / f"ComparisonCandidateSummary_{p_dict['pName']}_{observation_date}.json" + summary_payload = { + 'planet_name': p_dict['pName'], + 'observation_date': observation_date, + 'comparison_star': int(comp_index + 1), + 'comparison_position': comp_coords, + 'method_label': method_label, + 'selection_summary': selection_summary or {}, + 'parameter_summary': summarize_lightcurve_fit_parameters(final_fit), + 'transit_qc': getattr(final_fit, 'transit_qc', None), + 'saved_debug_series': None if debug_series_path is None else str(debug_series_path), + 'saved_bestfit_plot': None if bestfit_plot_path is None else str(bestfit_plot_path), + 'saved_triangle_plot': None if triangle_plot_path is None else str(triangle_plot_path), + 'archive_errors': archive_errors, + } + with summary_path.open('w', encoding='utf-8') as handle: + json.dump(make_json_safe(summary_payload), handle, indent=4) + + return candidate_dir + + +def archive_failed_comparison_fit(save_dir, planet_name, observation_date, attempt, method_label=None): + if save_dir is None or planet_name is None or observation_date is None or not attempt: + return None + + comp_index = attempt.get('comp_index') + if comp_index is None: + return None + + archive_dir = failed_comparison_archive_dir(save_dir, comp_index) + temp_dir = archive_dir / "temp" + temp_dir.mkdir(parents=True, exist_ok=True) + + fit = attempt.get('fit') + archive_errors = [] + debug_series_path = None + bestfit_plot_path = None + + if fit is not None: + try: + debug_series_path = save_selected_photometry_debug_series( + archive_dir, + planet_name, + observation_date, + fit, + ) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the selected raw target/reference ratio diagnostics", + exc, + )) + + plotter = getattr(fit, 'plot_bestfit', None) + if callable(plotter): + try: + fig, _ = plotter() + bestfit_plot_path = temp_dir / f"BestFit_{planet_name}_{observation_date}.png" + fig.savefig(bestfit_plot_path) + plt.close(fig) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the provisional best-fit plot", + exc, + )) + + summary_path = temp_dir / f"FailedFitSummary_{planet_name}_{observation_date}.json" + summary_payload = { + 'planet_name': planet_name, + 'observation_date': observation_date, + 'comparison_star': None if comp_index is None else int(comp_index + 1), + 'comparison_label': attempt.get('label', f"Comp {comp_index + 1}"), + 'comparison_position': attempt.get('position'), + 'method_label': method_label, + 'failure_reason': attempt.get('failure_reason'), + 'fit_diagnostics': attempt.get('fit_diagnostics') or {}, + 'parameter_summary': attempt.get('parameter_summary'), + 'fit_point_count': attempt.get('fit_point_count'), + 'eebls_snr': attempt.get('eebls_snr'), + 'transit_delta_bic': attempt.get('transit_delta_bic'), + 'residual_scatter': attempt.get('residual_scatter'), + 'ktmf_metric': attempt.get('ktmf_metric'), + 'ktmf_contributions': attempt.get('ktmf_contributions') or [], + 'transit_qc': getattr(fit, 'transit_qc', None) if fit is not None else None, + 'saved_debug_series': None if debug_series_path is None else str(debug_series_path), + 'saved_bestfit_plot': None if bestfit_plot_path is None else str(bestfit_plot_path), + 'archive_errors': archive_errors, + } + with summary_path.open('w', encoding='utf-8') as handle: + json.dump(make_json_safe(summary_payload), handle, indent=4) + + return archive_dir + + +def save_selected_photometry_debug_series(save_dir, planet_name, observation_date, fit): + if fit is None: + return None + + debug = getattr(fit, 'selected_photometry_debug', None) + if not debug: + return None + + times = np.asarray(debug.get('times'), dtype=float) + target_flux = np.asarray(debug.get('target_flux'), dtype=float) + comp_flux = np.asarray(debug.get('comp_flux'), dtype=float) + raw_ratio = np.asarray(debug.get('raw_ratio'), dtype=float) + initial_sigma_keep_mask = np.asarray(debug.get('initial_sigma_keep_mask'), dtype=bool) + phase_clip_keep_mask = np.asarray( + debug.get('phase_clip_keep_mask_on_sigma_filtered', np.ones(np.count_nonzero(initial_sigma_keep_mask))), + dtype=bool, + ) + + if not ( + times.shape == target_flux.shape == comp_flux.shape == raw_ratio.shape == initial_sigma_keep_mask.shape + ): + return None + + phase_keep_full = np.zeros(times.shape[0], dtype=bool) + sigma_kept_indices = np.flatnonzero(initial_sigma_keep_mask) + if sigma_kept_indices.size: + if phase_clip_keep_mask.shape[0] != sigma_kept_indices.size: + phase_clip_keep_mask = np.ones(sigma_kept_indices.size, dtype=bool) + phase_keep_full[sigma_kept_indices] = phase_clip_keep_mask + + output_dir = Path(save_dir) / "temp" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / f"SelectedPhotometryRawRatio_{planet_name}_{observation_date}.csv" + + output_rows = np.column_stack( + [ + times, + target_flux, + comp_flux, + raw_ratio, + initial_sigma_keep_mask.astype(int), + phase_keep_full.astype(int), + ] + ) + np.savetxt( + output_path, + output_rows, + delimiter=",", + header=( + "BJD_TDB,Target Flux,Comp Flux,Raw Ratio," + "Kept After Initial Sigma Clip,Kept After Phase Residual Clip" + ), + comments="", + fmt=["%.8f", "%.8f", "%.8f", "%.8f", "%d", "%d"], + ) + return output_path + + +def annotate_rprs_posterior_refit(fit, applied, note=None, history=None): + annotate_parameter_posterior_refit(fit, 'rprs', applied, note=note, history=history) + + +def annotate_ars_posterior_refit(fit, applied, note=None, history=None): + annotate_parameter_posterior_refit(fit, 'ars', applied, note=note, history=history) + + +def annotate_parameter_posterior_refit(fit, parameter_key, applied, note=None, history=None): + if fit is None: + return + + history = [] if history is None else list(history) + attr_prefix = f"{parameter_key}_posterior_refit" + setattr(fit, f"{attr_prefix}_applied", bool(applied)) + setattr(fit, f"{attr_prefix}_note", note) + setattr(fit, f"{attr_prefix}_count", len(history)) + setattr(fit, f"{attr_prefix}_history", history) + if history: + latest = history[-1] + setattr(fit, f"{attr_prefix}_edge", latest.get('edge')) + setattr(fit, f"{attr_prefix}_mode", latest.get('mode')) + setattr(fit, f"{attr_prefix}_std", latest.get('std')) + setattr(fit, f"{attr_prefix}_original_bounds", latest.get('original_bounds')) + setattr(fit, f"{attr_prefix}_bounds", latest.get('new_bounds')) + else: + setattr(fit, f"{attr_prefix}_edge", None) + setattr(fit, f"{attr_prefix}_mode", None) + setattr(fit, f"{attr_prefix}_std", None) + setattr(fit, f"{attr_prefix}_original_bounds", None) + setattr(fit, f"{attr_prefix}_bounds", None) + + +def build_initial_rprs_bounds( + rprs, + lower_scale=INITIAL_RPRS_BOUND_LOWER_SCALE, + upper_scale=INITIAL_RPRS_BOUND_UPPER_SCALE, +): + try: + rprs = float(rprs) + lower_scale = float(lower_scale) + upper_scale = float(upper_scale) + except (TypeError, ValueError): + return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + + if not np.isfinite(rprs) or rprs <= 0: + return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + + lower_bound = max(RPRS_SEARCH_BOUND_MIN, lower_scale * rprs) + upper_bound = upper_scale * rprs + if not np.isfinite(upper_bound) or upper_bound <= lower_bound: + upper_bound = max(lower_bound + np.finfo(float).eps, rprs) + + return [float(lower_bound), float(upper_bound)] + + +def build_initial_ars_bounds( + ars, + ars_unc=None, + sigma_multiplier=INITIAL_ARS_BOUND_SIGMA_MULTIPLIER, + fallback_relative_half_width=INITIAL_ARS_BOUND_FALLBACK_RELATIVE_HALF_WIDTH, +): + try: + ars = float(ars) + except (TypeError, ValueError): + ars = np.nan + + try: + ars_unc = float(ars_unc) + except (TypeError, ValueError): + ars_unc = np.nan + + if not np.isfinite(ars) or ars <= ARS_SEARCH_BOUND_MIN: + return [float(ARS_SEARCH_BOUND_MIN), float(ARS_SEARCH_BOUND_FALLBACK_MAX)] + + if np.isfinite(ars_unc) and ars_unc > 0: + half_width = float(max(ARS_SEARCH_BOUND_MIN, sigma_multiplier * ars_unc)) + else: + half_width = float(max(ARS_SEARCH_BOUND_MIN, fallback_relative_half_width * ars)) + + lower_bound = max(float(ARS_SEARCH_BOUND_MIN), float(ars - half_width)) + upper_bound = float(ars + half_width) + if not np.isfinite(upper_bound) or upper_bound <= lower_bound: + upper_bound = float(lower_bound + max(np.finfo(float).eps, ARS_SEARCH_BOUND_MIN)) + + return [float(lower_bound), float(upper_bound)] + + +def build_initial_transit_bounds(prior, tmid_bounds, ars_unc=None, inclination_half_width=5.0): + lower, upper = [float(value) for value in np.asarray(tmid_bounds, dtype=float).reshape(-1)[:2]] + # Keep ars ahead of inc so the internal impact-parameter parameterization + # uses the sampled ars value when converting inclination to b. + return { + 'rprs': build_initial_rprs_bounds(prior['rprs']), + 'tmid': [lower, upper], + 'ars': build_initial_ars_bounds(prior['ars'], ars_unc=ars_unc), + 'inc': [prior['inc'] - inclination_half_width, min(90, prior['inc'] + inclination_half_width)], + } + + +def clone_lightcurve_bounds(bounds): + return { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + } + + +def sanitize_parameter_search_bounds(bounds, key, minimum_bound, maximum_bound=None, fallback_maximum=None): + sanitized = clone_lightcurve_bounds(bounds) + if key not in sanitized: + return sanitized + + try: + lower_bound, upper_bound = [ + float(value) for value in np.asarray(sanitized[key], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + sanitized[key] = [ + float(minimum_bound), + float(maximum_bound if maximum_bound is not None else fallback_maximum), + ] + return sanitized + + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound): + sanitized[key] = [ + float(minimum_bound), + float(maximum_bound if maximum_bound is not None else fallback_maximum), + ] + return sanitized + + lower_bound = max(float(minimum_bound), float(lower_bound)) + if maximum_bound is not None: + upper_bound = min(float(maximum_bound), float(upper_bound)) + if lower_bound >= upper_bound: + sanitized[key] = [ + float(minimum_bound), + float(maximum_bound if maximum_bound is not None else fallback_maximum), + ] + else: + sanitized[key] = [lower_bound, float(upper_bound)] + return sanitized + + +def sanitize_rprs_search_bounds(bounds): + return sanitize_parameter_search_bounds( + bounds, + 'rprs', + RPRS_SEARCH_BOUND_MIN, + maximum_bound=RPRS_SEARCH_BOUND_MAX, + fallback_maximum=RPRS_SEARCH_BOUND_MAX, + ) + + +def sanitize_ars_search_bounds(bounds): + return sanitize_parameter_search_bounds( + bounds, + 'ars', + ARS_SEARCH_BOUND_MIN, + fallback_maximum=ARS_SEARCH_BOUND_FALLBACK_MAX, + ) + + +def sanitize_retry_search_bounds(bounds): + return sanitize_ars_search_bounds(sanitize_rprs_search_bounds(bounds)) + + +def clamp_parameter_prior_to_bounds(prior, bounds, key): + clamped = dict(prior) + if key not in clamped or key not in bounds: + return clamped + + try: + parameter_value = float(clamped[key]) + lower_bound, upper_bound = [ + float(value) for value in np.asarray(bounds[key], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return clamped + + if ( + np.isfinite(parameter_value) and np.isfinite(lower_bound) and np.isfinite(upper_bound) + and lower_bound < upper_bound + ): + clamped[key] = float(np.clip(parameter_value, lower_bound, upper_bound)) + return clamped + + +def clamp_rprs_prior_to_bounds(prior, bounds): + return clamp_parameter_prior_to_bounds(prior, bounds, 'rprs') + + +def clamp_ars_prior_to_bounds(prior, bounds): + return clamp_parameter_prior_to_bounds(prior, bounds, 'ars') + + +def clamp_retry_priors_to_bounds(prior, bounds): + return clamp_ars_prior_to_bounds(clamp_rprs_prior_to_bounds(prior, bounds), bounds) + + +def enforce_minimum_parameter_retry_half_width( + mode, + bounds, + min_half_width, + minimum_bound, + maximum_bound=None, +): + try: + lower_bound, upper_bound = [ + float(value) for value in np.asarray(bounds, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return bounds + + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or lower_bound >= upper_bound: + return bounds + + center = float(mode) if np.isfinite(mode) else float(0.5 * (lower_bound + upper_bound)) + half_width = max(float(min_half_width), 0.0) + expanded_lower = min(lower_bound, center - half_width) + expanded_upper = max(upper_bound, center + half_width) + + if expanded_lower < minimum_bound: + if maximum_bound is None: + expanded_upper = expanded_upper + (minimum_bound - expanded_lower) + else: + expanded_upper = min( + maximum_bound, + expanded_upper + (minimum_bound - expanded_lower), + ) + expanded_lower = minimum_bound + if maximum_bound is not None and expanded_upper > maximum_bound: + expanded_lower = max( + minimum_bound, + expanded_lower - (expanded_upper - maximum_bound), + ) + expanded_upper = maximum_bound + + return [float(expanded_lower), float(expanded_upper)] + + +def enforce_minimum_rprs_retry_half_width(mode, bounds, min_half_width=RPRS_RETRY_MIN_HALF_WIDTH): + return enforce_minimum_parameter_retry_half_width( + mode, + bounds, + min_half_width, + RPRS_SEARCH_BOUND_MIN, + maximum_bound=RPRS_SEARCH_BOUND_MAX, + ) + + +def enforce_minimum_ars_retry_half_width(mode, bounds, min_half_width=ARS_RETRY_MIN_HALF_WIDTH): + return enforce_minimum_parameter_retry_half_width( + mode, + bounds, + min_half_width, + ARS_SEARCH_BOUND_MIN, + ) + + +def run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + use_impactparameter_rather_than_inclination_to_fit=True, + max_rprs_retries=RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, + duration_prior=None, + max_ars_retries=ARS_POSTERIOR_MAX_RETRIES_DEFAULT, +): + retry_configs = [ + { + 'key': 'rprs', + 'label': 'Rp/R*', + 'sanitize_bounds': sanitize_rprs_search_bounds, + 'enforce_half_width': enforce_minimum_rprs_retry_half_width, + 'max_retries': max_rprs_retries, + 'min_bound': RPRS_SEARCH_BOUND_MIN, + 'max_bound': RPRS_SEARCH_BOUND_MAX, + 'annotate': annotate_rprs_posterior_refit, + }, + { + 'key': 'ars', + 'label': 'a/Rs', + 'sanitize_bounds': sanitize_ars_search_bounds, + 'enforce_half_width': enforce_minimum_ars_retry_half_width, + 'max_retries': max_ars_retries, + 'min_bound': ARS_SEARCH_BOUND_MIN, + 'max_bound': None, + 'annotate': annotate_ars_posterior_refit, + }, + ] + + def build_fit(local_prior, local_bounds): + local_bounds = sanitize_retry_search_bounds(local_bounds) + local_prior = clamp_retry_priors_to_bounds(local_prior, local_bounds) + fit_kwargs = { + 'jd_times': jd_times, + 'mode': 'ns', + 'use_impactparameter_rather_than_inclination_to_fit': + use_impactparameter_rather_than_inclination_to_fit, + } + if isinstance(duration_prior, dict) and duration_prior.get('applied'): + fit_kwargs['duration_prior'] = duration_prior + fit = lc_fitter( + times, + flux_values, + flux_errors, + airmass, + local_prior, + local_bounds, + **fit_kwargs, + ) + annotate_duration_prior(fit, duration_prior) + return fit + + current_bounds = sanitize_retry_search_bounds(bounds) + current_prior = clamp_retry_priors_to_bounds(prior, current_bounds) + retry_histories = {config['key']: [] for config in retry_configs} + retry_notes = {config['key']: None for config in retry_configs} + latest_diagnostics = {config['key']: None for config in retry_configs} + blocked_retry_keys = set() + fit = build_fit(current_prior, current_bounds) + + while True: + diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + if not callable(diagnostics_getter): + latest_diagnostics = {config['key']: None for config in retry_configs} + break + + retry_config = None + diagnostics = None + for config in retry_configs: + key = config['key'] + if key not in current_bounds: + continue + + parameter_diagnostics = diagnostics_getter(key) + latest_diagnostics[key] = parameter_diagnostics + if key in blocked_retry_keys: + continue + + if len(retry_histories[key]) >= int(max(0, config['max_retries'])): + continue + + new_bounds = parameter_diagnostics.get('bounds') if parameter_diagnostics else None + if parameter_diagnostics and parameter_diagnostics.get('clipped') and new_bounds is not None: + retry_config = config + diagnostics = parameter_diagnostics + break + + if retry_config is None: + break + + key = retry_config['key'] + label = retry_config['label'] + new_bounds = diagnostics.get('bounds') + try: + new_lower, new_upper = [float(value) for value in new_bounds] + except (TypeError, ValueError): + retry_notes[key] = f"Skipped; the automatic {label} retry proposed malformed bounds." + blocked_retry_keys.add(key) + continue + if not np.isfinite(new_lower) or not np.isfinite(new_upper) or new_lower >= new_upper: + retry_notes[key] = f"Skipped; the automatic {label} retry proposed invalid bounds." + blocked_retry_keys.add(key) + continue + + previous_bounds = current_bounds.get(key) + clamped_bounds = retry_config['sanitize_bounds']({key: [new_lower, new_upper]}).get( + key, + [new_lower, new_upper], + ) + clamped_bounds = retry_config['enforce_half_width']( + diagnostics.get('mode', np.nan), + clamped_bounds, + ) + clamped_bounds = retry_config['sanitize_bounds']({key: clamped_bounds}).get(key, clamped_bounds) + new_lower, new_upper = [float(value) for value in clamped_bounds] + if previous_bounds is not None: + previous_lower, previous_upper = [float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2]] + clipped_edge = diagnostics.get('edge') + expands_sampled_range = False + if clipped_edge == 'upper': + expands_sampled_range = new_upper > previous_upper + 1e-12 + elif clipped_edge == 'lower': + expands_sampled_range = new_lower < previous_lower - 1e-12 + else: + expands_sampled_range = ( + new_lower < previous_lower - 1e-12 or + new_upper > previous_upper + 1e-12 + ) + + if not expands_sampled_range: + maximum_bound = retry_config['max_bound'] + if ( + maximum_bound is not None and + previous_lower <= retry_config['min_bound'] + 1e-12 and + previous_upper >= maximum_bound - 1e-12 + ): + retry_notes[key] = ( + f"Skipped; the automatic {label} retry reached the maximum exoplanet " + f"search range [{retry_config['min_bound']:.6f}, {maximum_bound:.6f}]." + ) + else: + retry_notes[key] = f"Skipped; the automatic {label} retry did not expand the sampled range." + blocked_retry_keys.add(key) + continue + + retry_histories[key].append({ + 'attempt': len(retry_histories[key]) + 1, + 'edge': diagnostics.get('edge'), + 'mode': float(diagnostics.get('mode', np.nan)), + 'std': float(diagnostics.get('std', np.nan)), + 'original_bounds': None if previous_bounds is None else [float(previous_bounds[0]), float(previous_bounds[1])], + 'new_bounds': [new_lower, new_upper], + }) + log_info( + f"{label} posterior is truncated against the " + f"{diagnostics.get('edge', 'active')} search bound; retrying nested fit " + f"with {label} centered at {diagnostics.get('mode', np.nan):.6f} " + f"and sigma {diagnostics.get('std', np.nan):.6f} " + f"over [{new_lower:.6f}, {new_upper:.6f}]." + ) + + updated_bounds = clone_lightcurve_bounds(current_bounds) + updated_bounds[key] = [new_lower, new_upper] + updated_bounds = sanitize_retry_search_bounds(updated_bounds) + + updated_prior = dict(current_prior) + fit_parameters = getattr(fit, 'parameters', {}) + if isinstance(fit_parameters, dict): + for key in updated_bounds: + if key in fit_parameters: + updated_prior[key] = fit_parameters[key] + if np.isfinite(diagnostics.get('mode', np.nan)): + updated_prior[retry_config['key']] = float(diagnostics['mode']) + updated_prior = clamp_retry_priors_to_bounds(updated_prior, updated_bounds) + + current_prior = updated_prior + current_bounds = updated_bounds + fit = build_fit(current_prior, current_bounds) + + final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + for config in retry_configs: + key = config['key'] + label = config['label'] + history = retry_histories[key] + final_diagnostics = None + if callable(final_diagnostics_getter) and key in current_bounds: + final_diagnostics = final_diagnostics_getter(key) + elif latest_diagnostics.get(key) is not None: + final_diagnostics = latest_diagnostics[key] + + if history: + note = f"Applied {len(history)} automatic {label} posterior range refit(s)." + if final_diagnostics and final_diagnostics.get('clipped'): + retry_label = "retry" if len(history) == 1 else "retries" + note = ( + f"{note} The posterior still hugs the {final_diagnostics.get('edge')} bound after " + f"{len(history)} {retry_label}." + ) + log_info( + f"Warning: {label} posterior still appears truncated after the automatic retries; " + "please inspect the triangle plot carefully.", + warn=True, + ) + elif retry_notes[key] is not None: + note = retry_notes[key] + elif final_diagnostics is not None and final_diagnostics.get('reason'): + note = f"Not needed; {final_diagnostics['reason']}" + else: + note = "Not evaluated; posterior diagnostics are unavailable for this fit." - annotate_rprs_posterior_refit(fit, bool(retry_history), note=note, history=retry_history) + config['annotate'](fit, bool(history), note=note, history=history) return fit def log_info(string, warn=False, error=False): if error: - print(f"\033[31m {string}\033[0m") + print(f"\033[31m {string}\033[0m", flush=True) elif warn: - print(f"\033[34m {string}\033[0m") + print(f"\033[34m {string}\033[0m", flush=True) else: - print(string) + print(string, flush=True) log.debug(string) return True +def _find_runtime_handler(handler_name): + for handler in log.handlers: + if getattr(handler, "_exotic_runtime_handler_name", None) == handler_name: + return handler + return None + + +def configure_runtime_logging(): + global _RUNTIME_LOGGING_CONFIGURED + + if _RUNTIME_LOGGING_CONFIGURED: + return + + logging.root.setLevel(logging.DEBUG) + log.setLevel(logging.DEBUG) + + if _find_runtime_handler(_RUNTIME_FILE_HANDLER_NAME) is None: + try: + file_handler = TimedRotatingFileHandler(filename="exotic.log", when="midnight", backupCount=2) + except Exception as exc: + print(f"Warning: Could not initialize exotic.log ({exc}).") + else: + file_handler._exotic_runtime_handler_name = _RUNTIME_FILE_HANDLER_NAME + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter( + logging.Formatter( + "%(asctime)s.%(msecs)03d [%(threadName)-12.12s] %(levelname)-5.5s " + "%(funcName)s:%(lineno)d - %(message)s", + "%Y-%m-%dT%H:%M:%S", + ) + ) + log.addHandler(file_handler) + + if _find_runtime_handler(_RUNTIME_CONSOLE_HANDLER_NAME) is None: + console_handler = logging.StreamHandler(sys.stdout) + console_handler._exotic_runtime_handler_name = _RUNTIME_CONSOLE_HANDLER_NAME + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(logging.Formatter("%(message)s")) + log.addHandler(console_handler) + + try: + faulthandler.enable(file=sys.stdout, all_threads=True) + except Exception: + pass + + _RUNTIME_LOGGING_CONFIGURED = True + + +def _log_exception_with_fallback(message, exc_type, exc_value, exc_traceback): + try: + log.error(message, exc_info=(exc_type, exc_value, exc_traceback)) + except Exception: + print(message) + traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout) + + +def _handle_unhandled_exception(exc_type, exc_value, exc_traceback): + global _UNHANDLED_EXCEPTION_LOGGED + + if exc_type is not None and issubclass(exc_type, KeyboardInterrupt): + return + + if _UNHANDLED_EXCEPTION_LOGGED: + return + + _UNHANDLED_EXCEPTION_LOGGED = True + _log_exception_with_fallback("Unhandled exception during EXOTIC run", exc_type, exc_value, exc_traceback) + + +def _handle_thread_exception(args): + if args.exc_type is not None and issubclass(args.exc_type, KeyboardInterrupt): + return + + thread_name = args.thread.name if args.thread is not None else "unknown" + _log_exception_with_fallback( + f"Unhandled exception in thread '{thread_name}'", + args.exc_type, + args.exc_value, + args.exc_traceback, + ) + + +def install_exception_hooks(): + global _EXCEPTION_HOOKS_INSTALLED + + if _EXCEPTION_HOOKS_INSTALLED: + return + + sys.excepthook = _handle_unhandled_exception + threading.excepthook = _handle_thread_exception + _EXCEPTION_HOOKS_INSTALLED = True + + def should_log_plate_solution_path(wcs_file): if not wcs_file: return False @@ -734,7 +2719,6 @@ def relative_flux_filter_mask(relative_flux, max_relative_flux=RELATIVE_FLUX_MAX return ( np.isfinite(relative_flux) & np.greater(relative_flux, 0) - & np.less_equal(relative_flux, max_relative_flux) ) @@ -894,6 +2878,72 @@ def should_pick_comparison_by_eebls_snr(config_value): return True +def should_use_deviation_from_expected_transit_in_qc(config_value): + if config_value is None: + return TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_deviation_from_expected_transit_in_qc' value; " + "keeping expected-value deviation QC enabled.", + warn=True, + ) + return TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT + + +def parse_deviation_from_expected_transit_in_qc_sigma(config_value): + if config_value is None: + return TRANSIT_QC_DEVIATION_SIGMA_DEFAULT + + try: + sigma_value = float(config_value) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'deviation_from_expected_transit_in_qc_sigma' value; using the default 5 sigma.", + warn=True, + ) + return TRANSIT_QC_DEVIATION_SIGMA_DEFAULT + + if not np.isfinite(sigma_value) or sigma_value <= 0: + log_info( + "Warning: Non-positive 'deviation_from_expected_transit_in_qc_sigma' value; using the default 5 sigma.", + warn=True, + ) + return TRANSIT_QC_DEVIATION_SIGMA_DEFAULT + return float(sigma_value) + + +def should_assess_all_comparisons_before_selecting_best(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'assess_all_comparisons_before_selecting_best' value; " + "defaulting to assess all comparisons.", + warn=True, + ) + return True + + def should_use_psf_photometry(config_value): if config_value is None: return True @@ -1179,6 +3229,158 @@ def estimate_transit_duration_from_prior_geometry(prior): return float(duration) if np.isfinite(duration) and duration > 0 else np.nan +def _cacheable_duration_prior_scalar(value): + try: + numeric_value = float(value) + except (TypeError, ValueError): + return None + return None if not np.isfinite(numeric_value) else float(numeric_value) + + +def _duration_prior_scalar_from_cache(value): + return np.nan if value is None else float(value) + + +@lru_cache(maxsize=128) +def _cached_single_transit_duration_prior( + period, + rprs, + ars, + inc, + ecc, + omega, + period_unc, + rprs_unc, + ars_unc, + inc_unc, +): + prior = { + 'per': _duration_prior_scalar_from_cache(period), + 'rprs': _duration_prior_scalar_from_cache(rprs), + 'ars': _duration_prior_scalar_from_cache(ars), + 'inc': _duration_prior_scalar_from_cache(inc), + 'ecc': _duration_prior_scalar_from_cache(ecc), + 'omega': _duration_prior_scalar_from_cache(omega), + } + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + if not np.isfinite(expected_duration) or expected_duration <= 0: + return { + 'applied': False, + 'expected_duration': np.nan, + 'sigma_log_duration': np.nan, + 'relative_sigma': np.nan, + 'source': 'unavailable', + 'sample_count': 0, + 'note': ( + "Not applied; could not estimate a physical transit duration from the published single-transit priors." + ), + } + + period_unc = _duration_prior_scalar_from_cache(period_unc) + rprs_unc = _duration_prior_scalar_from_cache(rprs_unc) + ars_unc = _duration_prior_scalar_from_cache(ars_unc) + inc_unc = _duration_prior_scalar_from_cache(inc_unc) + + fallback_sigma_log = float(np.log1p(DURATION_PRIOR_FALLBACK_RELATIVE_SIGMA)) + min_sigma_log = float(np.log1p(DURATION_PRIOR_MIN_RELATIVE_SIGMA)) + sigma_log_duration = fallback_sigma_log + sample_count = 0 + source = "fallback relative width" + + if any( + value is not None and value > 0 + for value in (period_unc, rprs_unc, ars_unc, inc_unc) + ): + rng = np.random.default_rng(0) + sample_draw_count = int(max(DURATION_PRIOR_MONTE_CARLO_SAMPLES, 1)) + period_draws = np.full(sample_draw_count, prior['per'], dtype=float) + rprs_draws = np.full(sample_draw_count, prior['rprs'], dtype=float) + ars_draws = np.full(sample_draw_count, prior['ars'], dtype=float) + inc_draws = np.full(sample_draw_count, prior['inc'], dtype=float) + + if period_unc is not None and period_unc > 0: + period_draws = rng.normal(prior['per'], period_unc, sample_draw_count) + if rprs_unc is not None and rprs_unc > 0: + rprs_draws = rng.normal(prior['rprs'], rprs_unc, sample_draw_count) + if ars_unc is not None and ars_unc > 0: + ars_draws = rng.normal(prior['ars'], ars_unc, sample_draw_count) + if inc_unc is not None and inc_unc > 0: + inc_draws = rng.normal(prior['inc'], inc_unc, sample_draw_count) + + inc_draws = np.clip(inc_draws, 1e-6, 89.999999) + durations = np.full(sample_draw_count, np.nan, dtype=float) + for index in range(sample_draw_count): + durations[index] = estimate_transit_duration_from_prior_geometry({ + 'per': period_draws[index], + 'rprs': rprs_draws[index], + 'ars': ars_draws[index], + 'inc': inc_draws[index], + 'ecc': prior['ecc'], + 'omega': prior['omega'], + }) + + valid_durations = durations[np.isfinite(durations) & (durations > 0)] + sample_count = int(valid_durations.size) + if valid_durations.size >= DURATION_PRIOR_MIN_VALID_MONTE_CARLO_SAMPLES: + log_offsets = np.log(valid_durations / expected_duration) + lower_offset, upper_offset = np.nanpercentile(log_offsets, [16, 84]) + sigma_log_duration = float(max(0.5 * (upper_offset - lower_offset), min_sigma_log)) + source = "published geometry uncertainties" + else: + source = "fallback relative width (insufficient valid uncertainty samples)" + else: + source = "fallback relative width (missing published geometry uncertainties)" + + sigma_log_duration = float(max(sigma_log_duration, min_sigma_log)) + relative_sigma = float(np.expm1(sigma_log_duration)) + note = ( + f"Applied; expected duration={expected_duration:.6f} day(s) with an approximate 1-sigma width of " + f"{relative_sigma * 100.0:.1f}% from {source}" + ) + if sample_count > 0: + note += f" ({sample_count} propagated sample(s))." + else: + note += "." + + return { + 'applied': True, + 'expected_duration': float(expected_duration), + 'sigma_log_duration': float(sigma_log_duration), + 'relative_sigma': relative_sigma, + 'source': source, + 'sample_count': sample_count, + 'note': note, + } + + +def build_single_transit_duration_prior(planet_dict): + if not isinstance(planet_dict, dict): + return { + 'applied': False, + 'expected_duration': np.nan, + 'sigma_log_duration': np.nan, + 'relative_sigma': np.nan, + 'source': 'unavailable', + 'sample_count': 0, + 'note': "Not applied; missing published single-transit planet metadata.", + } + + return dict( + _cached_single_transit_duration_prior( + _cacheable_duration_prior_scalar(planet_dict.get('pPer', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('rprs', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('aRs', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('inc', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('ecc', 0.0)), + _cacheable_duration_prior_scalar(planet_dict.get('omega', 0.0)), + _cacheable_duration_prior_scalar(planet_dict.get('pPerUnc', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('rprsUnc', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('aRsUnc', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('incUnc', np.nan)), + ) + ) + + def estimate_ephemeris_tmid_and_bounds( times, prior_tmid, @@ -1192,6 +3394,7 @@ def estimate_ephemeris_tmid_and_bounds( 'method': 'ephemeris', 'applied': False, 'tmid': float(prior_tmid) if np.isfinite(prior_tmid) else np.nan, + 'uncertainty': np.nan, 'bounds': [np.nan, np.nan], 'cycle_index': np.nan, 'propagated_half_width': np.nan, @@ -1226,6 +3429,7 @@ def estimate_ephemeris_tmid_and_bounds( phases = (valid_times - prior_tmid) / period cycle_index = float(np.floor(phases).max()) tmid = float(prior_tmid + cycle_index * period) + propagated_uncertainty = np.sqrt(midt_unc ** 2 + (cycle_index * per_unc) ** 2) propagated_half_width = np.abs(sigma_multiplier * midt_unc + cycle_index * sigma_multiplier * per_unc) max_half_width = 0.25 * period @@ -1284,6 +3488,7 @@ def estimate_ephemeris_tmid_and_bounds( half_width = max(float(tmid - lower), float(upper - tmid)) summary.update({ 'tmid': tmid, + 'uncertainty': propagated_uncertainty, 'bounds': [lower, upper], 'cycle_index': cycle_index, 'propagated_half_width': propagated_half_width, @@ -1421,9 +3626,18 @@ def estimate_tmid_and_bounds_with_eebls(times, flux_values, flux_errors, prior, pre_points = int(np.count_nonzero(np.isfinite(fit_times) & (fit_times < tmid - coverage_margin))) post_points = int(np.count_nonzero(np.isfinite(fit_times) & (fit_times > tmid + coverage_margin))) if pre_points == 0 or post_points == 0: + summary.update({ + 'method': 'eebls', + 'applied': False, + 'tmid': tmid, + 'duration': duration, + 'depth': depth, + 'depth_snr': depth_snr, + }) summary['note'] = ( - "EEBLS transit initializer skipped: the strongest box-like signal is not bracketed by data on both sides " - f"({pre_points} pre-point(s), {post_points} post-point(s))." + "EEBLS transit initializer found a box-like event, but it is not bracketed by data on both sides " + f"({pre_points} pre-point(s), {post_points} post-point(s)); keeping the EEBLS depth SNR only and " + "falling back to the non-EEBLS Tmid bounds." ) return summary @@ -1462,6 +3676,7 @@ def annotate_lightcurve_tmid_search(fit, summary): fit.initial_tmid_search_method = summary.get('method') fit.initial_tmid_search_applied = bool(summary.get('applied')) fit.initial_tmid_search_tmid = summary.get('tmid') + fit.initial_tmid_search_uncertainty = summary.get('uncertainty') fit.initial_tmid_search_bounds = summary.get('bounds') fit.initial_tmid_search_duration = summary.get('duration') fit.initial_tmid_search_depth = summary.get('depth') @@ -1501,6 +3716,74 @@ def extract_lightcurve_fit_eebls_snr(fit): return np.nan +def ensure_lightcurve_fit_eebls_diagnostic(fit): + if fit is None: + return None + + existing_snr = extract_lightcurve_fit_eebls_snr(fit) + if np.isfinite(existing_snr): + return { + 'applied': True, + 'depth_snr': existing_snr, + 'note': 'Existing EEBLS diagnostic reused.', + } + + times = np.asarray(getattr(fit, 'time', np.array([])), dtype=float) + flux_values = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if times.ndim != 1 or times.size < LIGHTCURVE_MIN_VALID_POINTS or flux_values.shape != times.shape: + return None + + dataerr_obj = getattr(fit, 'dataerr', None) + flux_errors = None if dataerr_obj is None else np.asarray(dataerr_obj, dtype=float) + if flux_errors is None or flux_errors.shape != times.shape: + flux_errors = np.full(times.shape, 1.0, dtype=float) + else: + finite_positive = np.isfinite(flux_errors) & (flux_errors > 0) + if not np.any(finite_positive): + flux_errors = np.full(times.shape, 1.0, dtype=float) + elif not np.all(finite_positive): + replacement = float(np.nanmedian(flux_errors[finite_positive])) + flux_errors = np.where(finite_positive, flux_errors, replacement) + + fit_prior = getattr(fit, 'prior', {}) or {} + parameters = getattr(fit, 'parameters', {}) or {} + period = fit_prior.get('per', parameters.get('per', np.nan)) + tmid = fit_prior.get('tmid', parameters.get('tmid', np.nan)) + try: + period = float(period) + tmid = float(tmid) + except (TypeError, ValueError): + return None + if not np.isfinite(period) or period <= 0 or not np.isfinite(tmid): + return None + + fallback_bounds = getattr(fit, 'initial_tmid_search_bounds', None) + if fallback_bounds is None: + fallback_bounds = getattr(fit, 'eebls_diagnostic_bounds', None) + if fallback_bounds is None: + bounds = getattr(fit, 'bounds', None) + fallback_bounds = bounds.get('tmid') if isinstance(bounds, dict) else None + + try: + lower, upper = [float(value) for value in np.asarray(fallback_bounds, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + lower = float(tmid - 0.25 * period) + upper = float(tmid + 0.25 * period) + if not np.isfinite(lower) or not np.isfinite(upper) or upper <= lower: + lower = float(tmid - 0.25 * period) + upper = float(tmid + 0.25 * period) + + eebls_summary = estimate_tmid_and_bounds_with_eebls( + times, + flux_values, + flux_errors, + {'per': period, 'tmid': tmid}, + [lower, upper], + ) + annotate_lightcurve_eebls_diagnostic(fit, eebls_summary) + return eebls_summary + + def should_use_impactparameter_rather_than_inclination_to_fit(config_value): if config_value is None: return True @@ -1534,8 +3817,9 @@ def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled prior['a1'] = baseline_guess if not disabled: - # Raw target/reference flux ratios are often far from unity, so only keep - # the legacy near-unity bound for already-normalized light curves. + # Some paths deliver an approximately unity-normalized light curve, while + # others still carry an arbitrary positive baseline. Keep the legacy + # near-unity bound only when the working series is already close to 1. if 0.95 <= baseline_guess <= 1.05: bounds['a0'] = [0.95, 1.05] else: @@ -1727,7 +4011,7 @@ def extract_baseline_corrected_lightcurve_arrays(fit): valid_detrended = np.isfinite(detrended) & (detrended > 0) if np.any(valid_detrended): flux_values = detrended.copy() - flux_source = "provisional detrended light curve" + flux_source = "current detrended light curve" if flux_values is None: data = np.asarray(getattr(fit, 'data', []), dtype=float) @@ -1735,10 +4019,10 @@ def extract_baseline_corrected_lightcurve_arrays(fit): if data.shape == times.shape and airmass_model.shape == times.shape: with np.errstate(divide='ignore', invalid='ignore'): flux_values = np.divide(data, airmass_model) - flux_source = "provisional flux ratio divided by the fitted airmass/baseline model" + flux_source = "current flux ratio divided by the fitted airmass/baseline model" elif data.shape == times.shape: flux_values = data.copy() - flux_source = "provisional raw flux ratio" + flux_source = "current raw flux ratio" else: return None, None, None @@ -1771,14 +4055,14 @@ def prepare_final_fit_lightcurve_series( if times.ndim != 1 or times.size == 0: return { 'applied': False, - 'note': 'could not prepare a final-fit light curve because the provisional fit had no time samples.', + 'note': 'could not prepare a final-fit light curve because the current fit had no time samples.', } flux_values, flux_errors, flux_source = extract_baseline_corrected_lightcurve_arrays(fit) if flux_values is None: return { 'applied': False, - 'note': 'could not derive a baseline-corrected provisional light curve for final-fit preparation.', + 'note': 'could not derive a baseline-corrected light curve for final-fit preparation from the current fit.', } flux_values = np.asarray(flux_values, dtype=float) @@ -1858,7 +4142,7 @@ def prepare_final_fit_lightcurve_series( else: note = ( f"Prepared the final-fit input light curve from the {flux_source} and normalized it with a " - f"sigma-clipped full-series baseline because the provisional fit only bracketed one side of transit " + f"sigma-clipped full-series baseline because the current fit only bracketed one side of transit " f"({pre_points} pre-ingress and {post_points} post-egress modeled out-of-transit point(s))." ) @@ -2122,7 +4406,7 @@ def build_nested_tmid_refinement_from_initial_fit(times, flux_values, flux_error return base_plan refined_prior = dict(prior) - for key in ('rprs', 'tmid', 'inc', 'a2'): + for key in ('rprs', 'ars', 'tmid', 'inc', 'a2'): if key in refined_prior and key in fit.parameters: refined_prior[key] = fit.parameters[key] @@ -2283,7 +4567,7 @@ def build_final_fit_prefit_refinement_plan( refined_prior = dict(prior) if isinstance(fit_parameters, dict): - for key in ('rprs', 'tmid', 'inc', 'a2'): + for key in ('rprs', 'ars', 'tmid', 'inc', 'a2'): if key in refined_prior and key in fit_parameters: refined_prior[key] = fit_parameters[key] @@ -2333,7 +4617,14 @@ def fit_final_lightcurve_with_oot_baseline_detrending( use_impactparameter_rather_than_inclination_to_fit=True, plot_time_range=None, baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + expected_planet_dict=None, + expected_tmid_search_summary=None, + eebls_search_summary=None, + duration_prior=None, ): + if duration_prior is None and expected_planet_dict is not None: + duration_prior = build_single_transit_duration_prior(expected_planet_dict) + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( times, flux_values, @@ -2343,9 +4634,16 @@ def fit_final_lightcurve_with_oot_baseline_detrending( bounds, jd_times=jd_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, ) fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_transit_qc_fit_context( + fit, + planet_dict=expected_planet_dict, + tmid_search_summary=expected_tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) prefit_plan = build_final_fit_prefit_refinement_plan( times, @@ -2384,9 +4682,16 @@ def fit_final_lightcurve_with_oot_baseline_detrending( working_bounds, jd_times=working_jd_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, ) fit = apply_plot_time_range(fit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, working_airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_transit_qc_fit_context( + fit, + planet_dict=expected_planet_dict, + tmid_search_summary=expected_tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) annotate_final_fit_prefit_refinement( fit, @@ -2408,6 +4713,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( False, note="Disabled; using the direct nested-sampling fit.", ) + annotate_transit_detection_qc(fit) return fit, working_flux, working_unc detrend_result = detrend_flux_on_out_of_transit_baseline( @@ -2427,13 +4733,14 @@ def fit_final_lightcurve_with_oot_baseline_detrending( pre_points=detrend_result.get('pre_points', 0), post_points=detrend_result.get('post_points', 0), ) + annotate_transit_detection_qc(fit) return fit, working_flux, working_unc log_info("Applying optional out-of-transit linear baseline detrending and refitting final light curve.") log_info(detrend_result['note']) refit_prior = dict(working_prior) - for key in ('rprs', 'tmid', 'inc', 'a2'): + for key in ('rprs', 'ars', 'tmid', 'inc', 'a2'): if key in refit_prior and key in fit.parameters: refit_prior[key] = fit.parameters[key] @@ -2454,9 +4761,16 @@ def fit_final_lightcurve_with_oot_baseline_detrending( refit_bounds, jd_times=working_jd_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_transit_qc_fit_context( + refit, + planet_dict=expected_planet_dict, + tmid_search_summary=expected_tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) annotate_final_fit_prefit_refinement( refit, prefit_plan.get('applied', False), @@ -2479,6 +4793,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( pre_points=detrend_result['pre_points'], post_points=detrend_result['post_points'], ) + annotate_transit_detection_qc(refit) return refit, detrend_result['flux'], detrend_result['unc'] @@ -5627,6 +7942,39 @@ def sigma_clipped_nanmedian(data, sigma=3.0, max_iters=3): return bn.nanmedian(clipped), bn.nanstd(clipped) +def normalize_flux_series_to_approximate_unity( + flux_values, + flux_errors=None, + sigma=3.0, + max_iters=3, + min_points=LIGHTCURVE_MIN_VALID_POINTS, +): + flux_values = np.asarray(flux_values, dtype=float) + normalized_flux = np.array(flux_values, dtype=float, copy=True) + normalized_unc = None if flux_errors is None else np.array(flux_errors, dtype=float, copy=True) + + valid_flux = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(valid_flux) < max(1, int(min_points)): + return normalized_flux, normalized_unc, np.nan + + baseline_level, _ = sigma_clipped_nanmedian( + flux_values[valid_flux], + sigma=sigma, + max_iters=max_iters, + ) + if not np.isfinite(baseline_level) or baseline_level <= 0: + baseline_level = bn.nanmedian(flux_values[valid_flux]) + if not np.isfinite(baseline_level) or baseline_level <= 0: + return normalized_flux, normalized_unc, np.nan + + normalized_flux[valid_flux] = flux_values[valid_flux] / baseline_level + if normalized_unc is not None and normalized_unc.shape == flux_values.shape: + finite_unc = np.isfinite(normalized_unc) + normalized_unc[finite_unc] = normalized_unc[finite_unc] / baseline_level + + return normalized_flux, normalized_unc, float(baseline_level) + + def weighted_nanpercentile(values, weights, percentile): values = np.asarray(values, dtype=float).ravel() weights = np.asarray(weights, dtype=float).ravel() @@ -5942,10 +8290,22 @@ def process_flat_frames(flat_files, master_bias=None): return master_flat / medi def convert_jd_to_bjd(non_bjd, p_dict, info_dict): + global _BJD_FALLBACK_WARNING_LOGGED + try: goodTimes = JDUTC_to_BJDTDB(non_bjd, ra=p_dict['ra'], dec=p_dict['dec'], lat=info_dict['lat'], longi=info_dict['long'], alt=info_dict['elev'])[0] - except: + except Exception as exc: + if not _BJD_FALLBACK_WARNING_LOGGED: + _BJD_FALLBACK_WARNING_LOGGED = True + try: + log.warning( + "barycorrpy JDUTC_to_BJDTDB conversion failed; falling back to astropy light-travel-time " + "conversion for this run.", + exc_info=True, + ) + except Exception: + traceback.print_exception(type(exc), exc, exc.__traceback__, file=sys.stdout) targetloc = SkyCoord(p_dict['ra'], p_dict['dec'], unit=(u.deg, u.deg), frame='icrs') obsloc = EarthLocation(lat=info_dict['lat'], lon=info_dict['long'], height=info_dict['elev']) timesToConvert = Time(non_bjd, format='jd', scale='utc', location=obsloc) @@ -6442,97 +8802,34 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, allow_mid_transit_range_warning=True, disable_vertical_flux_normalization=False, final_fit_mode='lm', use_impactparameter_rather_than_inclination_to_fit=True, - plot_time_range=None, - use_eebls_to_initialize_tmid_and_bounds=True, - compute_eebls_diagnostics=False): - # remove outliers - plot_time_range = np.asarray(times if plot_time_range is None else plot_time_range, dtype=float) - si = np.argsort(times) - times_sorted = times[si] - tflux_sorted = tFlux[si] - cflux_sorted = cFlux[si] - with np.errstate(divide='ignore', invalid='ignore'): - flux_ratio_sorted = np.divide(tflux_sorted, cflux_sorted) - - filter_diagnostics = [] - has_reference_flux = not np.allclose(cflux_sorted, 1.0) - if has_reference_flux: - flux_ratio_mask = valid_flux_ratio_mask(flux_ratio_sorted) - filter_diagnostics.append(build_time_rejection_diagnostic( - "Target/reference ratio filter", - times_sorted, - flux_ratio_mask, - note="Dropped non-finite or non-positive target/reference ratios before fitting.", - )) - times_sorted = times_sorted[flux_ratio_mask] - tflux_sorted = tflux_sorted[flux_ratio_mask] - cflux_sorted = cflux_sorted[flux_ratio_mask] - flux_ratio_sorted = flux_ratio_sorted[flux_ratio_mask] - jd_times_sorted = jd_times[si][flux_ratio_mask] - airmass_sorted = airmass[si][flux_ratio_mask] - else: - jd_times_sorted = jd_times[si] - airmass_sorted = airmass[si] - - if len(times_sorted) <= 1: - return None, None, None - - debug_times = np.asarray(times_sorted, dtype=float).copy() - debug_target_flux = np.asarray(tflux_sorted, dtype=float).copy() - debug_comp_flux = np.asarray(cflux_sorted, dtype=float).copy() - debug_raw_ratio = np.asarray(flux_ratio_sorted, dtype=float).copy() - - dt = np.mean(np.diff(times_sorted)) - ndt = int(25. / 24. / 60. / dt) * 2 + 1 - if ndt > len(times_sorted): - ndt = int(len(times_sorted)/4) * 2 + 1 - filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt), times=times_sorted) - valid_mask = ~filtered_data - debug_initial_sigma_keep_mask = np.asarray(valid_mask, dtype=bool).copy() - filter_diagnostics.append(build_time_rejection_diagnostic( - "Initial sigma clip", - times_sorted, - valid_mask, - note="Dropped 3-sigma target/reference-ratio outliers before the first lightcurve fit.", - )) - - arrayFinalFlux = flux_ratio_sorted[valid_mask] - f1 = tflux_sorted[valid_mask] - sigf1 = f1 ** 0.5 - f2 = cflux_sorted[valid_mask] - sigf2 = f2 ** 0.5 - if np.sum(cFlux) == len(cFlux): - arrayNormUnc = sigf1 - else: - arrayNormUnc = np.sqrt((sigf1 / f2) ** 2 + (sigf2 * f1 / f2 ** 2) ** 2) - arrayTimes = times_sorted[valid_mask] - arrayJDTimes = jd_times_sorted[valid_mask] - arrayAirmass = airmass_sorted[valid_mask] - - # remove nans - nanmask = np.isnan(arrayFinalFlux) | np.isnan(arrayNormUnc) | np.isnan(arrayTimes) | np.isnan( - arrayAirmass) | np.less_equal(arrayFinalFlux, 0) | np.less_equal(arrayNormUnc, 0) - nanmask = nanmask | np.isinf(arrayFinalFlux) | np.isinf(arrayNormUnc) | np.isinf(arrayTimes) | np.isinf( - arrayAirmass) - filter_diagnostics.append(build_time_rejection_diagnostic( - "Finite/positive photometry filter", - arrayTimes, - ~nanmask, - note="Dropped non-finite or non-positive flux, uncertainty, time, or airmass values.", - )) - - if np.sum(~nanmask) <= 1: + plot_time_range=None, + use_eebls_to_initialize_tmid_and_bounds=True, + compute_eebls_diagnostics=False): + plot_time_range = np.asarray(times if plot_time_range is None else plot_time_range, dtype=float) + prepared = prepare_lightcurve_fit_input_series( + times, + tFlux, + cFlux, + airmass, + jd_times=jd_times, + ) + if not prepared.get('applied'): return None, None, None - else: - arrayFinalFlux = arrayFinalFlux[~nanmask] - arrayNormUnc = arrayNormUnc[~nanmask] - arrayTimes = arrayTimes[~nanmask] - arrayJDTimes = arrayJDTimes[~nanmask] - arrayAirmass = arrayAirmass[~nanmask] - f1 = f1[~nanmask] - f2 = f2[~nanmask] - skip_airmass_fit = should_skip_airmass_fit(arrayAirmass) + filter_diagnostics = prepared['filter_diagnostics'] + debug_times = prepared['debug_times'] + debug_target_flux = prepared['debug_target_flux'] + debug_comp_flux = prepared['debug_comp_flux'] + debug_raw_ratio = prepared['debug_raw_ratio'] + debug_initial_sigma_keep_mask = prepared['initial_sigma_keep_mask'] + arrayFinalFlux = prepared['flux'] + f1 = prepared['target_flux'] + f2 = prepared['comp_flux'] + arrayNormUnc = prepared['unc'] + arrayTimes = prepared['time'] + arrayJDTimes = prepared['jd_time'] + arrayAirmass = prepared['airmass'] + skip_airmass_fit = prepared['skip_airmass_fit'] # -----LM LIGHTCURVE FIT-------------------------------------- @@ -6584,11 +8881,11 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, prior['tmid'] = tmid_search_summary['tmid'] lower, upper = tmid_search_summary['bounds'] - mybounds = { - 'rprs': build_initial_rprs_bounds(prior['rprs']), - 'tmid': [lower, upper], - 'inc': [prior['inc'] - 5, min(90, prior['inc'] + 5)], - } + mybounds = build_initial_transit_bounds( + prior, + [lower, upper], + ars_unc=pDict.get('aRsUnc'), + ) apply_vertical_flux_normalization_bound( prior, mybounds, @@ -6664,6 +8961,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, debug_phase_clip_keep_mask = np.ones(np.count_nonzero(debug_initial_sigma_keep_mask), dtype=bool) if final_fit_mode == 'ns' and myfit is not None: + duration_prior = build_single_transit_duration_prior(pDict) nested_refinement = build_nested_tmid_refinement_from_initial_fit( arrayTimes, arrayFinalFlux, @@ -6681,6 +8979,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, nested_refinement['bounds'], jd_times=arrayJDTimes, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, ) myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) @@ -6708,6 +9007,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, debug_initial_sigma_keep_mask, phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, ) + annotate_transit_qc_expected_values(myfit, pDict) + annotate_transit_detection_qc(myfit) return myfit, f1, f2 @@ -6753,7 +9054,6 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass, enforce_relativ if has_reference_flux: finite_ratio_mask = np.isfinite(flux_ratio_sorted) nonpositive_ratio_mask = finite_ratio_mask & np.less_equal(flux_ratio_sorted, 0) - high_ratio_mask = finite_ratio_mask & np.greater(flux_ratio_sorted, RELATIVE_FLUX_MAX) finite_ratio_values = flux_ratio_sorted[finite_ratio_mask] ratio_range_text = "finite ratio range=n/a" if finite_ratio_values.size: @@ -6763,7 +9063,6 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass, enforce_relativ ) nonfinite_ratio_count = int(np.count_nonzero(~finite_ratio_mask)) nonpositive_ratio_count = int(np.count_nonzero(nonpositive_ratio_mask)) - high_ratio_count = int(np.count_nonzero(high_ratio_mask)) rejected_ratio_count = nonfinite_ratio_count + nonpositive_ratio_count relative_flux_mask = valid_flux_ratio_mask(flux_ratio_sorted) rejection_detail = ( @@ -6771,14 +9070,6 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass, enforce_relativ f"{ratio_range_text})." ) rejection_context = "invalid target/reference ratio screening" - if enforce_relative_flux_max: - relative_flux_mask &= ~high_ratio_mask - rejected_ratio_count += high_ratio_count - rejection_detail = ( - f"(non-finite={nonfinite_ratio_count}, non-positive={nonpositive_ratio_count}, " - f">{RELATIVE_FLUX_MAX:g}x={high_ratio_count}, {ratio_range_text})." - ) - rejection_context = "target/reference ratio screening" diagnostics['relative_flux_point_count'] = int(np.count_nonzero(relative_flux_mask)) times_sorted = times_sorted[relative_flux_mask] tflux_sorted = tflux_sorted[relative_flux_mask] @@ -6868,16 +9159,175 @@ def ensure_lightcurve_fit_failure_reason(diagnostics, fit_result, failed_stage, return diagnostics +def prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=None, +): + times = np.asarray(times, dtype=float) + target_flux = np.asarray(target_flux, dtype=float) + comp_flux = np.asarray(comp_flux, dtype=float) + airmass = np.asarray(airmass, dtype=float) + jd_times_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + + prepared = { + 'applied': False, + 'failure_reason': None, + 'filter_diagnostics': [], + 'debug_times': np.array([], dtype=float), + 'debug_target_flux': np.array([], dtype=float), + 'debug_comp_flux': np.array([], dtype=float), + 'debug_raw_ratio': np.array([], dtype=float), + 'initial_sigma_keep_mask': np.array([], dtype=bool), + 'time': np.array([], dtype=float), + 'flux': np.array([], dtype=float), + 'unc': np.array([], dtype=float), + 'jd_time': None, + 'airmass': np.array([], dtype=float), + 'target_flux': np.array([], dtype=float), + 'comp_flux': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), + 'skip_airmass_fit': False, + 'approximate_baseline_level': np.nan, + } + + if not ( + times.ndim == target_flux.ndim == comp_flux.ndim == airmass.ndim == 1 + and times.shape == target_flux.shape == comp_flux.shape == airmass.shape + ): + prepared['failure_reason'] = ( + "lightcurve inputs must be 1D arrays with matching lengths before fitting." + ) + return prepared + + if jd_times_array is not None and jd_times_array.shape != times.shape: + jd_times_array = None + + plot_indices = np.argsort(times) + times_sorted = times[plot_indices] + target_flux_sorted = target_flux[plot_indices] + comp_flux_sorted = comp_flux[plot_indices] + source_indices = np.asarray(plot_indices, dtype=int) + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio_sorted = np.divide(target_flux_sorted, comp_flux_sorted) + + filter_diagnostics = [] + has_reference_flux = not np.allclose(comp_flux_sorted, 1.0) + if has_reference_flux: + flux_ratio_mask = valid_flux_ratio_mask(flux_ratio_sorted) + filter_diagnostics.append(build_time_rejection_diagnostic( + "Target/reference ratio filter", + times_sorted, + flux_ratio_mask, + note="Dropped non-finite or non-positive target/reference ratios before fitting.", + )) + times_sorted = times_sorted[flux_ratio_mask] + target_flux_sorted = target_flux_sorted[flux_ratio_mask] + comp_flux_sorted = comp_flux_sorted[flux_ratio_mask] + flux_ratio_sorted = flux_ratio_sorted[flux_ratio_mask] + source_indices = source_indices[flux_ratio_mask] + if jd_times_array is None: + jd_times_sorted = times_sorted.copy() + else: + jd_times_sorted = jd_times_array[plot_indices][flux_ratio_mask] + airmass_sorted = airmass[plot_indices][flux_ratio_mask] + else: + jd_times_sorted = times_sorted.copy() if jd_times_array is None else jd_times_array[plot_indices] + airmass_sorted = airmass[plot_indices] + + if len(times_sorted) <= 1: + prepared['failure_reason'] = "too few valid points remained after the target/reference ratio filter." + prepared['filter_diagnostics'] = filter_diagnostics + return prepared + + debug_times = np.asarray(times_sorted, dtype=float).copy() + debug_target_flux = np.asarray(target_flux_sorted, dtype=float).copy() + debug_comp_flux = np.asarray(comp_flux_sorted, dtype=float).copy() + debug_raw_ratio = np.asarray(flux_ratio_sorted, dtype=float).copy() + + dt = np.mean(np.diff(times_sorted)) + ndt = int(25. / 24. / 60. / dt) * 2 + 1 + if ndt > len(times_sorted): + ndt = int(len(times_sorted)/4) * 2 + 1 + filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt), times=times_sorted) + valid_mask = ~filtered_data + initial_sigma_keep_mask = np.asarray(valid_mask, dtype=bool).copy() + filter_diagnostics.append(build_time_rejection_diagnostic( + "Initial sigma clip", + times_sorted, + valid_mask, + note="Dropped 3-sigma target/reference-ratio outliers before the first lightcurve fit.", + )) + + flux = flux_ratio_sorted[valid_mask] + filtered_target_flux = target_flux_sorted[valid_mask] + filtered_comp_flux = comp_flux_sorted[valid_mask] + if np.sum(comp_flux) == len(comp_flux): + unc = filtered_target_flux ** 0.5 + else: + sigf1 = filtered_target_flux ** 0.5 + sigf2 = filtered_comp_flux ** 0.5 + unc = np.sqrt((sigf1 / filtered_comp_flux) ** 2 + (sigf2 * filtered_target_flux / filtered_comp_flux ** 2) ** 2) + fit_times = times_sorted[valid_mask] + fit_jd_times = jd_times_sorted[valid_mask] + fit_airmass = airmass_sorted[valid_mask] + source_indices = source_indices[valid_mask] + + nanmask = np.isnan(flux) | np.isnan(unc) | np.isnan(fit_times) | np.isnan(fit_airmass) | np.less_equal(flux, 0) | np.less_equal(unc, 0) + nanmask = nanmask | np.isinf(flux) | np.isinf(unc) | np.isinf(fit_times) | np.isinf(fit_airmass) + filter_diagnostics.append(build_time_rejection_diagnostic( + "Finite/positive photometry filter", + fit_times, + ~nanmask, + note="Dropped non-finite or non-positive flux, uncertainty, time, or airmass values.", + )) + + if np.sum(~nanmask) <= 1: + prepared['failure_reason'] = "too few valid points remained after removing non-finite or non-positive photometry." + prepared.update({ + 'filter_diagnostics': filter_diagnostics, + 'debug_times': debug_times, + 'debug_target_flux': debug_target_flux, + 'debug_comp_flux': debug_comp_flux, + 'debug_raw_ratio': debug_raw_ratio, + 'initial_sigma_keep_mask': initial_sigma_keep_mask, + }) + return prepared + + normalized_flux, normalized_unc, approximate_baseline_level = normalize_flux_series_to_approximate_unity( + flux[~nanmask], + unc[~nanmask], + ) + + prepared.update({ + 'applied': True, + 'filter_diagnostics': filter_diagnostics, + 'debug_times': debug_times, + 'debug_target_flux': debug_target_flux, + 'debug_comp_flux': debug_comp_flux, + 'debug_raw_ratio': debug_raw_ratio, + 'initial_sigma_keep_mask': initial_sigma_keep_mask, + 'time': fit_times[~nanmask], + 'flux': normalized_flux, + 'unc': normalized_unc, + 'jd_time': fit_jd_times[~nanmask], + 'airmass': fit_airmass[~nanmask], + 'target_flux': filtered_target_flux[~nanmask], + 'comp_flux': filtered_comp_flux[~nanmask], + 'source_indices': source_indices[~nanmask], + 'skip_airmass_fit': should_skip_airmass_fit(fit_airmass[~nanmask]), + 'approximate_baseline_level': approximate_baseline_level, + }) + return prepared + + def cheap_lightcurve_prescore(tFlux, cFlux, airmass, enforce_relative_flux_max=True): with np.errstate(divide='ignore', invalid='ignore'): flux_ratio = np.divide(tFlux, cFlux) finite_mask = valid_flux_ratio_mask(flux_ratio) & np.isfinite(airmass) - if not np.allclose(cFlux, 1.0): - if enforce_relative_flux_max: - finite_mask &= relative_flux_filter_mask(flux_ratio) - else: - finite_mask &= valid_flux_ratio_mask(flux_ratio) if np.count_nonzero(finite_mask) < 5: return np.inf @@ -6927,6 +9377,7 @@ def evaluate_lightcurve_candidate(task): jd_times, allow_mid_transit_range_warning=False, disable_vertical_flux_normalization=disable_vertical_flux_normalization, + final_fit_mode='ns', use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, @@ -6938,14 +9389,25 @@ def evaluate_lightcurve_candidate(task): failed_stage='lightcurve_fit', failure_reason="the lightcurve fitter did not converge to a usable solution.", ) + transit_qc_failure_reason = lightcurve_fit_transit_qc_failure_reason(myfit) + if transit_qc_failure_reason is not None: + fit_diagnostics = dict(fit_diagnostics) + fit_diagnostics.update({ + 'failed_stage': 'transit_qc', + 'failure_reason': transit_qc_failure_reason, + }) - res_std = np.inf - if myfit is not None: - res_std = myfit.residuals.std() / np.median(myfit.data) return { 'myfit': myfit, - 'res_std': res_std, + 'accepted': myfit is not None and transit_qc_failure_reason is None, 'eebls_snr': extract_lightcurve_fit_eebls_snr(myfit), + 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(myfit), + 'residual_scatter': extract_lightcurve_fit_residual_scatter(myfit), + 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(myfit), + 'ktmf_contributions': extract_lightcurve_fit_ktmf_contributions(myfit), + 'transit_qc_status': getattr(myfit, 'transit_qc_status', None), + 'transit_qc_summary': getattr(myfit, 'transit_qc_summary', None), + 'rejected_by_transit_qc': transit_qc_failure_reason is not None, 'fit_diagnostics': fit_diagnostics, 'failure_reason': fit_diagnostics.get('failure_reason'), 'fit_point_count': 0 if tflux_fit is None else int(len(tflux_fit)), @@ -7071,15 +9533,6 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, return candidate_jobs -def shortlist_target_fit_candidates(candidate_jobs, minimum_count=50, fraction=0.35): - finite_candidates = [candidate for candidate in candidate_jobs if np.isfinite(candidate.get('prescore', np.inf))] - if finite_candidates: - finite_candidates.sort(key=lambda candidate: candidate['prescore']) - shortlist_count = max(int(minimum_count), int(fraction * len(finite_candidates))) - return finite_candidates[:min(len(finite_candidates), shortlist_count)] - return list(candidate_jobs) - - def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, psf_data, aper_data, plot_time_range=None, disable_vertical_flux_normalization=False, @@ -7146,16 +9599,20 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co use_psf_photometry=use_psf_photometry, use_aperture_photometry=use_aperture_photometry, ) - shortlist = shortlist_target_fit_candidates(candidate_jobs) - if not shortlist: + evaluated_candidates = list(candidate_jobs) + for candidate_order, candidate in enumerate(evaluated_candidates): + candidate.setdefault('candidate_order', candidate_order) + if not evaluated_candidates: return { 'candidate_jobs': candidate_jobs, - 'shortlist': shortlist, + 'evaluated_candidates': evaluated_candidates, + 'shortlist': evaluated_candidates, 'candidate_summaries': [], 'best_candidate': None, 'best_fit_lc': None, - 'min_std': np.inf, - 'selection_metric': 'residual_scatter', + 'selected_ktmf_metric': np.nan, + 'selected_transit_delta_bic': np.nan, + 'selection_metric': 'ktmf', 'selected_eebls_snr': np.nan, 'flux_tar': None, 'flux_ref': None, @@ -7177,7 +9634,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, compute_eebls_diagnostics=True, ) - for candidate in shortlist + for candidate in evaluated_candidates ] if multiprocess_lightcurve_fits is not None and multiprocess_lightcurve_fits > 0: @@ -7189,53 +9646,101 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co candidate_summaries = [] successful_candidates = [] - for candidate, result in zip(shortlist, fit_results): + for candidate, result in zip(evaluated_candidates, fit_results): fit_meta, tflux_fit, cflux_fit = result summary = summarize_target_fit_candidate(candidate, fit_meta, comp_stars) candidate_summaries.append(summary) - if fit_meta is None or fit_meta.get('myfit') is None: + if fit_meta is None or fit_meta.get('myfit') is None or not fit_meta.get('accepted', True): continue successful_candidates.append((summary, candidate, fit_meta, tflux_fit, cflux_fit)) best_candidate = None best_fit_lc = None - best_res_std = np.inf + best_ktmf_metric = np.nan + best_transit_delta_bic = np.nan best_tflux = None best_cflux = None - selection_metric = 'residual_scatter' + selection_metric = 'ktmf' selected_eebls_snr = np.nan if successful_candidates: - if pick_comparison_by_eebls_snr and any( - np.isfinite(summary.get('eebls_snr', np.nan)) - for summary, _, _, _, _ in successful_candidates - ): + has_ktmf = any(np.isfinite(item[0].get('ktmf_metric', np.nan)) for item in successful_candidates) + has_eebls = any(np.isfinite(item[0].get('eebls_snr', np.nan)) for item in successful_candidates) + has_delta_bic = any(np.isfinite(item[0].get('transit_delta_bic', np.nan)) for item in successful_candidates) + + if has_ktmf: + selection_metric = 'ktmf' + if pick_comparison_by_eebls_snr: + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('ktmf_metric', np.nan)) else 1, + -item[0].get('ktmf_metric', np.nan) if np.isfinite(item[0].get('ktmf_metric', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else 1, + -item[0].get('transit_delta_bic', np.nan) if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), + ), + ) + else: + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('ktmf_metric', np.nan)) else 1, + -item[0].get('ktmf_metric', np.nan) if np.isfinite(item[0].get('ktmf_metric', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else 1, + -item[0].get('transit_delta_bic', np.nan) if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), + ), + ) + elif pick_comparison_by_eebls_snr and has_eebls: selection_metric = 'eebls_snr' selected_entry = min( successful_candidates, key=lambda item: ( 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, - item[0].get('res_std', np.inf), - item[0].get('prescore', np.inf), - item[0].get('comp_index', np.inf), + 0 if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else 1, + -item[0].get('transit_delta_bic', np.nan) if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), ), ) - else: + elif has_delta_bic: + selection_metric = 'transit_delta_bic' + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else 1, + -item[0].get('transit_delta_bic', np.nan) if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), + ), + ) + elif has_eebls: + selection_metric = 'eebls_snr' selected_entry = min( successful_candidates, key=lambda item: ( - item[0].get('res_std', np.inf), 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, - item[0].get('prescore', np.inf), - item[0].get('comp_index', np.inf), + item[0].get('candidate_order', np.inf), ), ) + else: + selection_metric = 'comparison_field_rank' + selected_entry = min( + successful_candidates, + key=lambda item: item[0].get('candidate_order', np.inf), + ) selected_summary, best_candidate, fit_meta, best_tflux, best_cflux = selected_entry best_fit_lc = fit_meta['myfit'] - best_res_std = selected_summary.get('res_std', np.inf) + best_ktmf_metric = selected_summary.get('ktmf_metric', np.nan) + best_transit_delta_bic = selected_summary.get('transit_delta_bic', np.nan) selected_eebls_snr = selected_summary.get('eebls_snr', np.nan) best_identity = target_fit_candidate_identity(best_candidate) for summary in candidate_summaries: @@ -7243,11 +9748,13 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co return { 'candidate_jobs': candidate_jobs, - 'shortlist': shortlist, + 'evaluated_candidates': evaluated_candidates, + 'shortlist': evaluated_candidates, 'candidate_summaries': candidate_summaries, 'best_candidate': best_candidate, 'best_fit_lc': best_fit_lc, - 'min_std': best_res_std, + 'selected_ktmf_metric': best_ktmf_metric, + 'selected_transit_delta_bic': best_transit_delta_bic, 'selection_metric': selection_metric, 'selected_eebls_snr': selected_eebls_snr, 'flux_tar': best_tflux, @@ -7346,10 +9853,366 @@ def format_eebls_snr(value): return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value:.2f}" +def extract_lightcurve_fit_transit_delta_bic(fit): + if fit is None: + return np.nan + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + value = transit_qc.get('delta_bic', np.nan) + if np.isfinite(value): + return float(value) + + value = getattr(fit, 'transit_qc_delta_bic', np.nan) + return np.nan if not np.isfinite(value) else float(value) + + +def extract_lightcurve_fit_residual_scatter(fit): + if fit is None: + return np.nan + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + value = transit_qc.get('residual_scatter', np.nan) + if np.isfinite(value): + return float(value) + + value = getattr(fit, 'transit_qc_residual_scatter', np.nan) + if np.isfinite(value): + return float(value) + + value = getattr(fit, 'res_stdev', np.nan) + if np.isfinite(value): + return float(value) + + residuals = np.asarray(getattr(fit, 'residuals', np.array([])), dtype=float) + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if residuals.shape != data.shape or residuals.size == 0: + return np.nan + + median_flux = np.nanmedian(data) + if not np.isfinite(median_flux) or median_flux == 0: + return np.nan + return float(np.std(residuals) / median_flux) + + +def extract_lightcurve_fit_ktmf_metric(fit): + if fit is None: + return np.nan + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + value = transit_qc.get('ktmf_metric', np.nan) + if np.isfinite(value): + return float(value) + + value = getattr(fit, 'transit_qc_ktmf_metric', np.nan) + return np.nan if not np.isfinite(value) else float(value) + + +def extract_lightcurve_fit_ktmf_contributions(fit): + if fit is None: + return [] + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + contributions = transit_qc.get('ktmf_contributions') + if isinstance(contributions, list): + return contributions + + contributions = getattr(fit, 'transit_qc_ktmf_contributions', None) + return contributions if isinstance(contributions, list) else [] + + +def format_transit_delta_bic(value): + if value is None: + return "n/a" + + try: + numeric_value = float(value) + except (TypeError, ValueError): + return "n/a" + + return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value:.2f}" + + +def format_residual_scatter(value): + if value is None: + return "n/a" + + try: + numeric_value = float(value) + except (TypeError, ValueError): + return "n/a" + + return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value * 100.0:.4f}%" + + +def format_ktmf_metric(value): + if value is None: + return "n/a" + + try: + numeric_value = float(value) + except (TypeError, ValueError): + return "n/a" + + return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value:.2f}/5.00" + + +def format_ktmf_contribution(contribution): + if not isinstance(contribution, dict): + return "KTMF contribution: unavailable" + + label = contribution.get('label', 'Unknown component') + detail = contribution.get('detail') or 'n/a' + available = bool(contribution.get('available')) + points = float(contribution.get('points', 0.0) or 0.0) + max_points = float(contribution.get('max_points', 0.0) or 0.0) + score = contribution.get('score', np.nan) + + if available and np.isfinite(score): + return ( + f"KTMF contribution: {label} +{points:.2f}/{max_points:.2f} " + f"(score={score:.2f}; {detail})" + ) + + return f"KTMF contribution: {label} +0.00/0.00 (unavailable; {detail})" + + +def summarize_lightcurve_fit_assessment(fit): + if fit is None: + return None + + fit_method = getattr(fit, 'ns_type', None) or getattr(fit, 'fit_method', None) or 'lm' + try: + rprs_retry_count = int(getattr(fit, 'rprs_posterior_refit_count', 0) or 0) + except (TypeError, ValueError): + rprs_retry_count = 0 + + return { + 'fit_method': fit_method, + 'duration_prior_applied': bool(getattr(fit, 'duration_prior_applied', False)), + 'duration_prior_note': getattr(fit, 'duration_prior_note', None), + 'rprs_posterior_refit_applied': bool(getattr(fit, 'rprs_posterior_refit_applied', False)), + 'rprs_posterior_refit_count': rprs_retry_count, + 'rprs_posterior_refit_note': getattr(fit, 'rprs_posterior_refit_note', None), + 'prefit_refinement_applied': bool(getattr(fit, 'prefit_refinement_applied', False)), + 'prefit_refinement_note': getattr(fit, 'prefit_refinement_note', None), + 'oot_baseline_detrending_applied': bool(getattr(fit, 'oot_baseline_detrending_applied', False)), + 'oot_baseline_detrending_note': getattr(fit, 'oot_baseline_detrending_note', None), + 'airmass_fit_skipped': bool(getattr(fit, 'airmass_fit_skipped', False)), + 'airmass_correction_note': getattr(fit, 'airmass_correction_note', None), + 'nested_tmid_refinement_applied': bool(getattr(fit, 'nested_tmid_refinement_applied', False)), + 'nested_tmid_refinement_note': getattr(fit, 'nested_tmid_refinement_note', None), + } + + +def best_available_attempt_fit(attempt): + if not isinstance(attempt, dict): + return None + return ( + attempt.get('full_reduction_fit') + or attempt.get('fit') + or attempt.get('provisional_fit') + ) + + +def log_lightcurve_fit_assessment_lines(fit, indent=" "): + assessment = summarize_lightcurve_fit_assessment(fit) + if not assessment: + return + + rprs_retry_status = "not applied" + if assessment['rprs_posterior_refit_applied']: + retry_count = assessment['rprs_posterior_refit_count'] + rprs_retry_status = ( + f"applied ({retry_count} refit(s))" + if retry_count > 0 else + "applied" + ) + prefit_status = "applied" if assessment['prefit_refinement_applied'] else "not applied" + oot_status = "applied" if assessment['oot_baseline_detrending_applied'] else "not applied" + duration_prior_status = "applied" if assessment['duration_prior_applied'] else "not applied" + + log_info( + f"{indent}fit assessment: fit_method={assessment['fit_method']}, " + f"duration_prior={duration_prior_status}, " + f"Rp/R* posterior retry={rprs_retry_status}, " + f"prefit_refinement={prefit_status}, " + f"oot_baseline_detrending={oot_status}" + ) + if assessment.get('duration_prior_note'): + log_info(f"{indent}Duration prior note: {assessment['duration_prior_note']}") + if assessment.get('rprs_posterior_refit_note'): + log_info(f"{indent}Rp/R* posterior retry note: {assessment['rprs_posterior_refit_note']}") + if assessment.get('prefit_refinement_note'): + log_info(f"{indent}Prefit refinement note: {assessment['prefit_refinement_note']}") + if assessment.get('oot_baseline_detrending_note'): + log_info(f"{indent}OOT baseline detrending note: {assessment['oot_baseline_detrending_note']}") + if assessment.get('airmass_correction_note'): + log_info(f"{indent}Airmass correction note: {assessment['airmass_correction_note']}") + if assessment.get('nested_tmid_refinement_note'): + log_info(f"{indent}Nested Tmid refinement note: {assessment['nested_tmid_refinement_note']}") + + +def log_comparison_candidate_evaluation_start(comp_summary, rank, ranked_count, method_label, fit_diagnostics): + if comp_summary is None: + return + + label = comp_summary.get('label', f"Comp {comp_summary.get('comp_index', 0) + 1}") + position_text = format_comp_star_position(comp_summary.get('position')) + coverage_text = format_comp_star_coverage_text({ + 'coverage_count': comp_summary.get('coverage_count', 0), + 'coverage_total_frame_count': comp_summary.get('coverage_total_frame_count', 0), + 'coverage_reference_count': comp_summary.get('coverage_reference_count', np.nan), + 'coverage_min_required_count': comp_summary.get('coverage_min_required_count', 0), + }) + suitability_score = comp_summary.get('aggregate_score', np.nan) + suitability_text = "n/a" if not np.isfinite(suitability_score) else f"{suitability_score * 100.0:.4f}%" + usable_point_count = 0 if fit_diagnostics is None else fit_diagnostics.get('usable_point_count', 0) + + log_info( + f"\nStarting comparison-star target-fit evaluation for {label} ({position_text}) " + f"[rank {rank + 1}/{ranked_count}] with {method_label}." + ) + log_info( + f" Candidate inputs: suitability={suitability_text}, coverage={coverage_text}, " + f"usable_after_filters={usable_point_count}." + ) + log_info(" Preparing comparison-candidate light curve for the full reduction.") + + +def log_comparison_candidate_evaluation_result(attempt): + if not attempt: + return + + fit = best_available_attempt_fit(attempt) + fit_method = "n/a" + assessment = summarize_lightcurve_fit_assessment(fit) + if assessment: + fit_method = assessment.get('fit_method', 'n/a') + + qc_status = attempt.get('transit_qc_status') + qc_text = "n/a" if not qc_status else str(qc_status).upper() + if attempt.get('fit') is None: + status_text = "FAILED" + elif attempt.get('rejected_by_transit_qc', False): + status_text = f"REJECTED ({qc_text})" + elif attempt.get('full_reduction_applied', False): + status_text = f"COMPLETE ({qc_text})" + else: + status_text = "PROVISIONAL ONLY" + + reason_text = ( + attempt.get('transit_qc_summary') + or attempt.get('failure_reason') + or attempt.get('selection_reason') + or "completed comparison-star target-fit evaluation." + ) + log_info( + f"Completed comparison-star target-fit evaluation for {attempt.get('label', 'comparison candidate')}: " + f"status={status_text}, fit_method={fit_method}, fit_points={attempt.get('fit_point_count', 0)}, " + f"transit_qc={qc_text}, transit_delta_bic={format_transit_delta_bic(attempt.get('transit_delta_bic', np.nan))}, " + f"residual_scatter={format_residual_scatter(attempt.get('residual_scatter', np.nan))}, " + f"ktmf={format_ktmf_metric(attempt.get('ktmf_metric', np.nan))}, reason={reason_text}" + ) + parameter_summary = attempt.get('parameter_summary') + if parameter_summary: + log_info(f" parameters: {parameter_summary}") + log_lightcurve_fit_assessment_lines(fit, indent=" ") + if attempt.get('final_output_dir'): + log_info(f" outputs: {attempt['final_output_dir']}") + for contribution in attempt.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") + + def comparison_selection_metric_label(selection_metric): + if selection_metric == 'comparison_field_rank': + return "Comparison-Field Rank" + if selection_metric == 'ktmf': + return "KTMF" if selection_metric == 'eebls_snr': return "EEBLS SNR" - return "target-fit residual scatter" + return "transit-vs-flat Delta BIC" + + +def select_preferred_comparison_attempt(attempts, pick_comparison_by_eebls_snr=True): + selected_result = None + selection_metric = 'ktmf' + if not attempts: + return selected_result, selection_metric + + has_ktmf = any(np.isfinite(attempt.get('ktmf_metric', np.nan)) for attempt in attempts) + has_eebls = any(np.isfinite(attempt.get('eebls_snr', np.nan)) for attempt in attempts) + has_delta_bic = any(np.isfinite(attempt.get('transit_delta_bic', np.nan)) for attempt in attempts) + + if has_ktmf: + selection_metric = 'ktmf' + if pick_comparison_by_eebls_snr: + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('ktmf_metric', np.nan)) else 1, + -attempt.get('ktmf_metric', np.nan) if np.isfinite(attempt.get('ktmf_metric', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, + -attempt.get('transit_delta_bic', np.nan) if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + else: + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('ktmf_metric', np.nan)) else 1, + -attempt.get('ktmf_metric', np.nan) if np.isfinite(attempt.get('ktmf_metric', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, + -attempt.get('transit_delta_bic', np.nan) if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + elif pick_comparison_by_eebls_snr and has_eebls: + selection_metric = 'eebls_snr' + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, + -attempt.get('transit_delta_bic', np.nan) if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + elif has_delta_bic: + selection_metric = 'transit_delta_bic' + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, + -attempt.get('transit_delta_bic', np.nan) if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + elif has_eebls: + selection_metric = 'eebls_snr' + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + else: + selected_result = min(attempts, key=lambda attempt: attempt.get('rank', np.inf)) + + return selected_result, selection_metric def target_fit_candidate_identity(candidate): @@ -7372,7 +10235,6 @@ def summarize_target_fit_candidate(candidate, fit_meta, comp_stars): 'method_label': comparison_method_label(candidate), 'prescore': candidate.get('prescore', np.inf), 'fit': fit_result, - 'res_std': fit_meta.get('res_std', np.inf), 'coverage_count': candidate.get('coverage_count', 0), 'coverage_total_frame_count': candidate.get('coverage_total_frame_count', 0), 'coverage_reference_count': candidate.get('coverage_reference_count', np.nan), @@ -7380,6 +10242,10 @@ def summarize_target_fit_candidate(candidate, fit_meta, comp_stars): 'coverage_rejected': candidate.get('coverage_rejected', False), 'fit_point_count': fit_meta.get('fit_point_count', 0), 'eebls_snr': fit_meta.get('eebls_snr', np.nan), + 'transit_delta_bic': fit_meta.get('transit_delta_bic', np.nan), + 'residual_scatter': fit_meta.get('residual_scatter', np.nan), + 'ktmf_metric': fit_meta.get('ktmf_metric', np.nan), + 'ktmf_contributions': fit_meta.get('ktmf_contributions') or [], 'fit_diagnostics': fit_meta.get('fit_diagnostics') or {}, 'failure_reason': fit_meta.get('failure_reason'), 'parameter_summary': summarize_lightcurve_fit_parameters(fit_result), @@ -7387,6 +10253,10 @@ def summarize_target_fit_candidate(candidate, fit_meta, comp_stars): 'a': candidate.get('a'), 'an': candidate.get('an'), 'method': candidate.get('method'), + 'candidate_order': candidate.get('candidate_order'), + 'transit_qc_status': fit_meta.get('transit_qc_status'), + 'transit_qc_summary': fit_meta.get('transit_qc_summary'), + 'rejected_by_transit_qc': fit_meta.get('rejected_by_transit_qc', False), } @@ -7405,23 +10275,29 @@ def log_comparison_calibration_fit_attempt_summaries(attempts, method_label): coverage_text = format_comp_star_coverage_text(attempt) suitability_score = attempt.get('aggregate_score', np.inf) suitability_text = "n/a" if not np.isfinite(suitability_score) else f"{suitability_score * 100.0:.4f}%" - residual_text = "n/a" - if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)): - residual_text = f"{attempt['res_std'] * 100.0:.4f}%" eebls_text = format_eebls_snr(attempt.get('eebls_snr', np.nan)) + transit_delta_bic_text = format_transit_delta_bic(attempt.get('transit_delta_bic', np.nan)) + residual_text = format_residual_scatter(attempt.get('residual_scatter', np.nan)) + ktmf_text = format_ktmf_metric(attempt.get('ktmf_metric', np.nan)) reason_text = attempt.get('selection_reason') or attempt.get( 'failure_reason', - "selected: lowest target-fit residual scatter among the evaluated comparison stars", + "selected: strongest KTMF among the evaluated comparison stars", ) + if attempt.get('failed_run_dir'): + reason_text += f"; archived={attempt['failed_run_dir']}" log_info( f" {attempt['label']}{selected_label} ({position_text}): " f"suitability={suitability_text}, coverage={coverage_text}, " f"usable_after_filters={usable_point_count}, fit_points={attempt.get('fit_point_count', 0)}, " - f"eebls_snr={eebls_text}, residual_scatter={residual_text}, reason={reason_text}" + f"eebls_snr={eebls_text}, transit_delta_bic={transit_delta_bic_text}, " + f"residual_scatter={residual_text}, ktmf={ktmf_text}, reason={reason_text}" ) parameter_summary = attempt.get('parameter_summary') if parameter_summary: log_info(f" parameters: {parameter_summary}") + log_lightcurve_fit_assessment_lines(best_available_attempt_fit(attempt), indent=" ") + for contribution in attempt.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") def log_target_fit_candidate_summaries(candidate_summaries, max_entries=10): @@ -7439,23 +10315,26 @@ def log_target_fit_candidate_summaries(candidate_summaries, max_entries=10): coverage_text = format_comp_star_coverage_text(summary) prescore = summary.get('prescore', np.inf) prescore_text = "n/a" if not np.isfinite(prescore) else f"{prescore * 100.0:.4f}%" - residual_text = "n/a" - if summary.get('fit') is not None and np.isfinite(summary.get('res_std', np.inf)): - residual_text = f"{summary['res_std'] * 100.0:.4f}%" eebls_text = format_eebls_snr(summary.get('eebls_snr', np.nan)) + transit_delta_bic_text = format_transit_delta_bic(summary.get('transit_delta_bic', np.nan)) + residual_text = format_residual_scatter(summary.get('residual_scatter', np.nan)) + ktmf_text = format_ktmf_metric(summary.get('ktmf_metric', np.nan)) reason_text = summary.get( 'failure_reason', - "selected: lowest target-fit residual scatter in the evaluated shortlist", + "selected: strongest KTMF in the evaluated candidate set", ) log_info( f" {summary['label']}{selected_label} ({position_text}) with {summary['method_label']}: " f"prescore={prescore_text}, coverage={coverage_text}, " f"usable_after_filters={usable_point_count}, fit_points={summary.get('fit_point_count', 0)}, " - f"eebls_snr={eebls_text}, residual_scatter={residual_text}, reason={reason_text}" + f"eebls_snr={eebls_text}, transit_delta_bic={transit_delta_bic_text}, " + f"residual_scatter={residual_text}, ktmf={ktmf_text}, reason={reason_text}" ) parameter_summary = summary.get('parameter_summary') if parameter_summary: log_info(f" parameters: {parameter_summary}") + for contribution in summary.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") if len(candidate_summaries) > len(displayed_summaries): log_info( @@ -7503,12 +10382,14 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): return summary['failure_reason'] selection_basis = photometry_info.get('selection_basis', 'target_fit') - selection_metric = photometry_info.get('selection_metric', 'residual_scatter') + selection_metric = photometry_info.get('selection_metric', 'ktmf') selected_comp_num = photometry_info.get('comp_star_num') - selected_res_std = photometry_info.get('min_std', np.inf) - candidate_res_std = summary.get('res_std', np.inf) selected_eebls_snr = photometry_info.get('comparison_eebls_snr', np.nan) + selected_transit_delta_bic = photometry_info.get('comparison_transit_delta_bic', np.nan) + selected_ktmf_metric = photometry_info.get('comparison_ktmf_metric', np.nan) candidate_eebls_snr = summary.get('eebls_snr', np.nan) + candidate_transit_delta_bic = summary.get('transit_delta_bic', np.nan) + candidate_ktmf_metric = summary.get('ktmf_metric', np.nan) if summary.get('selected'): if selection_basis == 'comparison_field': @@ -7518,9 +10399,18 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): "selected: comparison-field calibration fell back to this star " "after better-ranked candidates failed target fitting" ) + if selection_basis == 'comparison_field_qc_fallback': + return ( + "selected: best available comparison-star fit after all completed candidates were rejected " + "by transit QC" + ) + if selection_metric == 'ktmf' and np.isfinite(candidate_ktmf_metric): + return "selected: highest KTMF in the chosen search" if selection_metric == 'eebls_snr' and np.isfinite(candidate_eebls_snr): return "selected: highest EEBLS SNR in the chosen search" - return "selected: lowest target-fit residual scatter in the chosen search" + if np.isfinite(candidate_transit_delta_bic): + return "selected: strongest transit-vs-flat Delta BIC in the chosen search" + return "selected: strongest transit evidence in the chosen search" if selection_basis == 'comparison_field': if selected_comp_num is None: @@ -7533,6 +10423,13 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): "not selected: comparison-field fallback chose " f"Comp {selected_comp_num} after better-ranked candidate(s) failed target fitting" ) + if selection_basis == 'comparison_field_qc_fallback': + if selected_comp_num is None: + return "not selected: comparison-field QC fallback chose a different candidate" + return ( + "not selected: comparison-field QC fallback chose " + f"Comp {selected_comp_num} as the best available rejected fit" + ) if selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): if not np.isfinite(candidate_eebls_snr): @@ -7548,16 +10445,28 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): "EEBLS box signal than the selected fit; the earlier search did not choose it" ) - if np.isfinite(candidate_res_std) and np.isfinite(selected_res_std): - if candidate_res_std > selected_res_std + 1e-12: + if np.isfinite(candidate_ktmf_metric) and np.isfinite(selected_ktmf_metric): + if candidate_ktmf_metric < selected_ktmf_metric - 1e-12: + return ( + "not selected: KTMF was " + f"{candidate_ktmf_metric:.2f} vs {selected_ktmf_metric:.2f} for the selected fit" + ) + if candidate_ktmf_metric > selected_ktmf_metric + 1e-12: + return ( + "not selected: this post-selection diagnostic fit has a higher KTMF than the selected fit; " + "the earlier target-fit search did not choose it" + ) + + if np.isfinite(candidate_transit_delta_bic) and np.isfinite(selected_transit_delta_bic): + if candidate_transit_delta_bic < selected_transit_delta_bic - 1e-12: return ( - "not selected: residual scatter was " - f"{candidate_res_std * 100.0:.4f}% vs {selected_res_std * 100.0:.4f}% for the selected fit" + "not selected: transit-vs-flat Delta BIC was " + f"{candidate_transit_delta_bic:.2f} vs {selected_transit_delta_bic:.2f} for the selected fit" ) - if candidate_res_std < selected_res_std - 1e-12: + if candidate_transit_delta_bic > selected_transit_delta_bic + 1e-12: return ( - "not selected: this post-selection diagnostic fit looks better than the selected fit; " - "the earlier target-fit search did not choose it" + "not selected: this post-selection diagnostic fit has stronger transit evidence " + "than the selected fit; the earlier target-fit search did not choose it" ) if selected_comp_num is None: @@ -7622,7 +10531,7 @@ def log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_i log_info(f"Selection basis: {selection_basis}") log_info( "Selection metric: " - f"{comparison_selection_metric_label(photometry_info.get('selection_metric', 'residual_scatter'))}" + f"{comparison_selection_metric_label(photometry_info.get('selection_metric', 'ktmf'))}" ) for summary in candidate_fit_summaries: @@ -7631,20 +10540,23 @@ def log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_i diagnostics = summary.get('fit_diagnostics') or {} usable_point_count = diagnostics.get('usable_point_count', 0) coverage_text = format_comp_star_coverage_text(summary) - residual_text = "n/a" - if summary.get('fit') is not None and np.isfinite(summary.get('res_std', np.inf)): - residual_text = f"{summary['res_std'] * 100.0:.4f}%" eebls_text = format_eebls_snr(summary.get('eebls_snr', np.nan)) + transit_delta_bic_text = format_transit_delta_bic(summary.get('transit_delta_bic', np.nan)) + residual_text = format_residual_scatter(summary.get('residual_scatter', np.nan)) + ktmf_text = format_ktmf_metric(summary.get('ktmf_metric', np.nan)) reason_text = comparison_candidate_fit_selection_reason(summary, photometry_info) log_info( f" {summary['label']}{selected_label} ({position_text}): " f"coverage={coverage_text}, " f"usable_after_filters={usable_point_count}, fit_points={summary['fit_point_count']}, " - f"eebls_snr={eebls_text}, residual_scatter={residual_text}, reason={reason_text}" + f"eebls_snr={eebls_text}, transit_delta_bic={transit_delta_bic_text}, " + f"residual_scatter={residual_text}, ktmf={ktmf_text}, reason={reason_text}" ) parameter_summary = summary.get('parameter_summary') if parameter_summary: log_info(f" parameters: {parameter_summary}") + for contribution in summary.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p_dict, comp_stars, @@ -7751,11 +10663,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p failure_reason="the nested lightcurve fitter did not converge to a usable solution.", ) - res_std = np.inf fit_point_count = 0 if target_fit_flux is None else int(len(target_fit_flux)) - if fit_result is not None and hasattr(fit_result, 'residuals') and hasattr(fit_result, 'data'): - with np.errstate(divide='ignore', invalid='ignore'): - res_std = float(np.std(fit_result.residuals / np.median(fit_result.data))) parameter_summary = summarize_lightcurve_fit_parameters(fit_result) candidate_fit_summaries.append({ @@ -7764,8 +10672,11 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p 'position': position, 'selected': selected_comp_star_num == comp_index + 1, 'fit': fit_result, - 'res_std': res_std, 'eebls_snr': extract_lightcurve_fit_eebls_snr(fit_result), + 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(fit_result), + 'residual_scatter': extract_lightcurve_fit_residual_scatter(fit_result), + 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(fit_result), + 'ktmf_contributions': extract_lightcurve_fit_ktmf_contributions(fit_result), 'coverage_count': coverage_count, 'coverage_total_frame_count': coverage_total_frame_count, 'coverage_reference_count': coverage_reference_count, @@ -7803,7 +10714,6 @@ def normalized_ratio_series(numerator_flux, denominator_flux): ratio = np.divide(numerator_flux, denominator_flux) ratio[~np.isfinite(ratio)] = np.nan ratio[ratio <= 0] = np.nan - ratio[ratio > RELATIVE_FLUX_MAX] = np.nan return ratio @@ -7944,6 +10854,94 @@ def apply_comparison_star_suitability_outlier_rejection( } +def comparison_star_image_outlier_summary( + normalized_flux_map, + active_keys, + sigma=COMPARISON_IMAGE_OUTLIER_SIGMA, + min_active_stars=COMPARISON_IMAGE_OUTLIER_MIN_ACTIVE_STARS, + min_valid_pairs=COMPARISON_IMAGE_OUTLIER_MIN_VALID_PAIRS, +): + active_keys = [key for key in active_keys if key in normalized_flux_map] + series_length = 0 + for key in active_keys: + flux_values = np.asarray(normalized_flux_map.get(key), dtype=float) + if flux_values.ndim == 1: + series_length = flux_values.shape[0] + break + + keep_mask = np.ones(series_length, dtype=bool) + summary = { + 'frame_keep_mask': keep_mask, + 'rejected_frame_indices': [], + 'rejected_frame_count': 0, + 'valid_pair_counts': np.zeros(series_length, dtype=int), + 'outlier_pair_counts': np.zeros(series_length, dtype=int), + 'available_pair_count': 0, + 'required_valid_pair_count': 0, + 'sigma': float(sigma), + } + + if series_length == 0 or len(active_keys) < max(2, int(min_active_stars)): + return summary + + pairwise_valid_flags = [] + pairwise_outlier_flags = [] + for index, key in enumerate(active_keys): + numerator_flux = np.asarray(normalized_flux_map[key], dtype=float) + if numerator_flux.ndim != 1 or numerator_flux.shape[0] != series_length: + continue + + for other_key in active_keys[index + 1:]: + denominator_flux = np.asarray(normalized_flux_map[other_key], dtype=float) + if denominator_flux.ndim != 1 or denominator_flux.shape[0] != series_length: + continue + + ratio = normalized_ratio_series(numerator_flux, denominator_flux) + valid_mask = np.isfinite(ratio) + if np.count_nonzero(valid_mask) < LIGHTCURVE_MIN_VALID_POINTS: + continue + + center, scatter = sigma_clipped_nanmedian(ratio[valid_mask], sigma=4.0, max_iters=3) + if not np.isfinite(center): + center = float(bn.nanmedian(ratio[valid_mask])) + robust_pair_scatter = robust_scatter(ratio[valid_mask] - center) + if np.isfinite(robust_pair_scatter) and robust_pair_scatter > 0: + scatter = robust_pair_scatter + if not np.isfinite(scatter) or scatter <= 0: + scatter = robust_scatter(ratio[valid_mask] - center) + if not np.isfinite(scatter) or scatter <= 0: + scatter = COMPARISON_IMAGE_OUTLIER_MIN_SCATTER + + outlier_mask = valid_mask & np.greater(np.abs(ratio - center), float(sigma) * scatter) + pairwise_valid_flags.append(valid_mask) + pairwise_outlier_flags.append(outlier_mask) + + available_pair_count = len(pairwise_valid_flags) + required_valid_pair_count = max(int(min_valid_pairs), len(active_keys) - 1) + summary['available_pair_count'] = available_pair_count + summary['required_valid_pair_count'] = required_valid_pair_count + if available_pair_count < required_valid_pair_count: + return summary + + valid_pair_counts = np.sum(np.vstack(pairwise_valid_flags), axis=0).astype(int) + outlier_pair_counts = np.sum(np.vstack(pairwise_outlier_flags), axis=0).astype(int) + rejected_mask = ( + (valid_pair_counts >= required_valid_pair_count) + & (outlier_pair_counts == valid_pair_counts) + & (valid_pair_counts > 0) + ) + keep_mask = ~rejected_mask + + summary.update({ + 'frame_keep_mask': keep_mask, + 'rejected_frame_indices': np.flatnonzero(rejected_mask).astype(int).tolist(), + 'rejected_frame_count': int(np.count_nonzero(rejected_mask)), + 'valid_pair_counts': valid_pair_counts, + 'outlier_pair_counts': outlier_pair_counts, + }) + return summary + + def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_rejection=False, validity_mask_func=valid_comparison_frame_mask): if not comp_flux_map: @@ -7957,6 +10955,13 @@ def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_ 'suitability_high_threshold': np.nan, 'suitability_reference_score': np.nan, 'suitability_scatter': np.nan, + 'field_image_keep_mask': np.ones(0, dtype=bool), + 'image_outlier_rejected_count': 0, + 'image_outlier_sigma': COMPARISON_IMAGE_OUTLIER_SIGMA, + 'image_outlier_required_valid_pairs': 0, + 'image_outlier_available_pairs': 0, + 'image_outlier_valid_pair_counts': np.zeros(0, dtype=int), + 'image_outlier_outlier_pair_counts': np.zeros(0, dtype=int), } comp_keys = list(comp_flux_map.keys()) @@ -7974,17 +10979,32 @@ def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_ if not coverage_summary[key]['coverage_rejected'] ] - def build_stability_iteration(active_keys): + def build_stability_iteration(active_keys, frame_keep_mask=None): active_key_set = set(active_keys) + if comp_keys: + reference_shape = normalized_flux_map[comp_keys[0]].shape + else: + reference_shape = () + if frame_keep_mask is None: + working_flux_map = normalized_flux_map + else: + frame_keep_mask = np.asarray(frame_keep_mask, dtype=bool) + if frame_keep_mask.shape != reference_shape: + working_flux_map = normalized_flux_map + else: + working_flux_map = { + key: np.where(frame_keep_mask, normalized_flux_map[key], np.nan) + for key in comp_keys + } active_flux_map = { - eligible_key: normalized_flux_map[eligible_key] + eligible_key: working_flux_map[eligible_key] for eligible_key in active_keys } pairwise_matrix = np.full((len(comp_keys), len(comp_keys)), np.nan, dtype=float) comp_summaries = [] for i, key in enumerate(comp_keys): - normalized_flux = normalized_flux_map[key] + normalized_flux = working_flux_map[key] self_score = cheap_lightcurve_prescore(normalized_flux, np.ones(normalized_flux.shape[0]), airmass) pairwise_scores = [] pairwise_series = {} @@ -7992,7 +11012,7 @@ def build_stability_iteration(active_keys): for j, other_key in enumerate(comp_keys): if i == j or other_key not in active_key_set: continue - other_flux = normalized_flux_map[other_key] + other_flux = working_flux_map[other_key] score = cheap_lightcurve_prescore(normalized_flux, other_flux, airmass) pairwise_matrix[i, j] = score pairwise_series[f"vs {j + 1}"] = normalized_ratio_series(normalized_flux, other_flux) @@ -8075,7 +11095,15 @@ def build_stability_iteration(active_keys): rejected_outlier_keys.update(newly_rejected_keys) active_keys = [key for key in active_keys if key not in rejected_outlier_keys] - pairwise_matrix, comp_summaries = build_stability_iteration(active_keys) + image_outlier_summary = comparison_star_image_outlier_summary( + normalized_flux_map, + active_keys, + ) + field_image_keep_mask = image_outlier_summary['frame_keep_mask'] + pairwise_matrix, comp_summaries = build_stability_iteration( + active_keys, + frame_keep_mask=field_image_keep_mask, + ) final_active_scores = np.asarray( [ summary['aggregate_score'] @@ -8136,6 +11164,13 @@ def build_stability_iteration(active_keys): 'suitability_high_threshold': final_high_threshold, 'suitability_reference_score': final_reference_score, 'suitability_scatter': final_scatter, + 'field_image_keep_mask': field_image_keep_mask, + 'image_outlier_rejected_count': image_outlier_summary['rejected_frame_count'], + 'image_outlier_sigma': image_outlier_summary['sigma'], + 'image_outlier_required_valid_pairs': image_outlier_summary['required_valid_pair_count'], + 'image_outlier_available_pairs': image_outlier_summary['available_pair_count'], + 'image_outlier_valid_pair_counts': image_outlier_summary['valid_pair_counts'], + 'image_outlier_outlier_pair_counts': image_outlier_summary['outlier_pair_counts'], } @@ -8457,9 +11492,20 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p psf_data, aper_data, target_psf_flux, plot_time_range=None, disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, use_impactparameter_rather_than_inclination_to_fit=True, use_eebls_to_initialize_tmid_and_bounds=True, - pick_comparison_by_eebls_snr=True): + pick_comparison_by_eebls_snr=True, + assess_all_comparisons_before_selecting_best=True, + final_fit_baseline_duration_multiplier= + FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + use_adaptive_apertures=False, + adaptive_aperture_values=None, + adaptive_annulus_values=None, + fallback_sigma=np.nan, + save_dir=None, + planet_name=None, + observation_date=None): ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) if not ranked_summaries: return { @@ -8469,6 +11515,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p } method = comparison_calibration['method'] + method_label = comparison_calibration.get('method_label', method) aperture_index = comparison_calibration.get('a') annulus_index = comparison_calibration.get('an') if method == 'psf': @@ -8476,6 +11523,35 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p else: target_flux = aper_data['target'][:, aperture_index, annulus_index] + adaptive_summary = build_comparison_candidate_adaptive_summary( + comparison_calibration, + psf_data, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=adaptive_aperture_values, + adaptive_annulus_values=adaptive_annulus_values, + fallback_sigma=fallback_sigma, + ) + field_image_keep_mask = np.asarray( + comparison_calibration.get('field_image_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if field_image_keep_mask.shape != times.shape: + field_image_keep_mask = np.ones(times.shape[0], dtype=bool) + field_image_clip_diagnostic = None + if np.any(~field_image_keep_mask): + required_pairs = comparison_calibration.get('image_outlier_required_valid_pairs', 0) + sigma_threshold = comparison_calibration.get('image_outlier_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + field_image_clip_diagnostic = build_time_rejection_diagnostic( + "Comparison-field image clip", + times, + field_image_keep_mask, + note=( + "Dropped frames flagged after comparison-star suitability clipping because every " + f"valid pairwise comparison was more than {sigma_threshold:.2f} sigma from its flat-line median " + f"(min valid pair count={required_pairs})." + ), + ) + attempts = [] for rank, comp_summary in enumerate(ranked_summaries): comp_index = comp_summary['comp_index'] @@ -8489,9 +11565,9 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p else: comp_flux = aper_data[ckey][:, aperture_index, annulus_index] - fit_mask = np.ones(times.shape[0], dtype=bool) + fit_mask = field_image_keep_mask.copy() if method == 'psf': - fit_mask = robust_target_reference_flux_mask(target_flux, comp_flux) + fit_mask &= robust_target_reference_flux_mask(target_flux, comp_flux) fit_diagnostics = diagnose_lightcurve_fit_inputs( times[fit_mask], @@ -8500,32 +11576,66 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p airmass[fit_mask], enforce_relative_flux_max=False, ) - fit_result, tflux_fit, cflux_fit = fit_lightcurve( + log_comparison_candidate_evaluation_start( + comp_summary, + rank, + len(ranked_summaries), + method_label, + fit_diagnostics, + ) + log_info( + " Full reduction starting. Optional out-of-transit baseline detrending is " + f"{'enabled' if detrend_on_outoftransit_baseline else 'disabled'}." + ) + final_reduction = finalize_comparison_candidate_full_reduction( times[fit_mask], target_flux[fit_mask], comp_flux[fit_mask], airmass[fit_mask], ld, p_dict, - jd_times[fit_mask], + jd_times=jd_times[fit_mask], disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, - plot_time_range=plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, - compute_eebls_diagnostics=True, + plot_time_range=plot_time_range, + baseline_duration_multiplier=final_fit_baseline_duration_multiplier, + adaptive_summary=adaptive_summary, ) + fit_result = final_reduction.get('fit') if final_reduction.get('applied') else None + tflux_fit = final_reduction.get('good_target_flux') + cflux_fit = final_reduction.get('good_comp_flux') fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, fit_result, - failed_stage='lightcurve_fit', - failure_reason="the lightcurve fitter did not converge to a usable solution.", + failed_stage='full_candidate_reduction', + failure_reason=final_reduction.get( + 'failure_reason', + "the raw comparison-candidate photometry did not converge to a usable fully reduced solution.", + ), ) - - res_std = np.inf - if fit_result is not None and hasattr(fit_result, 'residuals') and hasattr(fit_result, 'data'): - with np.errstate(divide='ignore', invalid='ignore'): - res_std = float(np.std(fit_result.residuals) / np.median(fit_result.data)) + if fit_result is None: + log_info( + f" {comp_summary.get('label', f'Comp {comp_index + 1}')}: " + "the raw comparison-candidate light curve did not converge to a usable fully reduced fit." + ) + selection_fit = fit_result + transit_qc_failure_reason = lightcurve_fit_transit_qc_failure_reason(selection_fit) + if transit_qc_failure_reason is not None: + fit_diagnostics = dict(fit_diagnostics) + fit_diagnostics.update({ + 'failed_stage': 'transit_qc', + 'failure_reason': transit_qc_failure_reason, + }) + if field_image_clip_diagnostic is not None: + attached_fit_ids = set() + for fit_candidate in (fit_result, final_reduction.get('fit'), selection_fit): + if fit_candidate is None or id(fit_candidate) in attached_fit_ids: + continue + attached_fit_ids.add(id(fit_candidate)) + prepend_lightcurve_filter_diagnostic(fit_candidate, field_image_clip_diagnostic) attempt = { 'rank': rank, @@ -8539,71 +11649,166 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'coverage_reference_count': comp_summary.get('coverage_reference_count', np.nan), 'coverage_min_required_count': comp_summary.get('coverage_min_required_count', 0), 'coverage_rejected': comp_summary.get('coverage_rejected', False), - 'fit': fit_result, + 'fit': selection_fit, + 'provisional_fit': None, + 'full_reduction_fit': final_reduction.get('fit'), + 'good_times': final_reduction.get('good_times'), + 'good_flux': final_reduction.get('good_flux'), + 'good_unc': final_reduction.get('good_unc'), + 'good_airmass': final_reduction.get('good_airmass'), + 'good_jd_times': final_reduction.get('good_jd_times'), 'tflux_fit': tflux_fit, 'cflux_fit': cflux_fit, + 'source_indices': final_reduction.get('source_indices'), + 'duration_samples': final_reduction.get('duration_samples'), + 'data_highres': final_reduction.get('data_highres'), 'fit_diagnostics': fit_diagnostics, - 'res_std': res_std, - 'eebls_snr': extract_lightcurve_fit_eebls_snr(fit_result), - 'fit_point_count': 0 if tflux_fit is None else int(len(tflux_fit)), + 'eebls_snr': extract_lightcurve_fit_eebls_snr(selection_fit), + 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(selection_fit), + 'residual_scatter': extract_lightcurve_fit_residual_scatter(selection_fit), + 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(selection_fit), + 'ktmf_contributions': extract_lightcurve_fit_ktmf_contributions(selection_fit), + 'fit_point_count': 0 if tflux_fit is None else int(len(np.asarray(tflux_fit, dtype=float))), 'failure_reason': fit_diagnostics.get('failure_reason'), - 'parameter_summary': summarize_lightcurve_fit_parameters(fit_result), + 'parameter_summary': summarize_lightcurve_fit_parameters(selection_fit), + 'transit_qc_status': getattr(selection_fit, 'transit_qc_status', None), + 'transit_qc_summary': getattr(selection_fit, 'transit_qc_summary', None), + 'rejected_by_transit_qc': final_reduction.get('applied', False) and transit_qc_failure_reason is not None, 'selected': False, 'selection_reason': None, + 'failed_run_dir': None, + 'final_output_dir': None, + 'full_reduction_applied': final_reduction.get('applied', False), + 'full_reduction_note': final_reduction.get('note'), } + if final_reduction.get('applied') and selection_fit is not None and save_dir is not None: + final_output_dir = save_comparison_candidate_full_reduction_outputs( + save_dir, + None, + selection_fit, + p_dict, + observation_date, + comp_index, + comp_coords=comp_summary.get('position'), + min_aperture=(0 if comparison_calibration['method'] == 'psf' else comparison_calibration.get('aper')), + min_annulus=comparison_calibration.get('annulus'), + adaptive_summary=adaptive_summary, + method_label=comparison_calibration.get('method_label'), + selection_summary={ + 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(selection_fit), + 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(selection_fit), + 'eebls_snr': extract_lightcurve_fit_eebls_snr(selection_fit), + 'transit_qc_status': getattr(selection_fit, 'transit_qc_status', None), + 'transit_qc_summary': getattr(selection_fit, 'transit_qc_summary', None), + }, + duration_samples=final_reduction.get('duration_samples'), + data_highres=final_reduction.get('data_highres'), + ) + if final_output_dir is not None: + attempt['final_output_dir'] = str(final_output_dir) + if transit_qc_failure_reason is not None: + attempt['selection_reason'] = ( + "rejected: transit detection QC flagged this comparison as a poor transit candidate" + ) + archive_dir = archive_failed_comparison_fit( + save_dir, + planet_name, + observation_date, + attempt, + method_label=comparison_calibration.get('method_label'), + ) + if archive_dir is not None: + attempt['failed_run_dir'] = str(archive_dir) + log_comparison_candidate_evaluation_result(attempt) attempts.append(attempt) selected_result = None - successful_attempts = [ + completed_attempts = [ attempt for attempt in attempts - if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)) + if ( + attempt.get('fit') is not None + and attempt.get('full_reduction_applied', False) + ) + ] + successful_attempts = [ + attempt + for attempt in completed_attempts + if not attempt.get('rejected_by_transit_qc', False) ] - selection_metric = 'residual_scatter' + selection_metric = 'ktmf' + fallback_to_qc_rejected = False if successful_attempts: - if pick_comparison_by_eebls_snr and any( - np.isfinite(attempt.get('eebls_snr', np.nan)) - for attempt in successful_attempts - ): - selection_metric = 'eebls_snr' - selected_result = min( - successful_attempts, - key=lambda attempt: ( - 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, - -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, - attempt.get('res_std', np.inf), - attempt.get('rank', np.inf), - ), - ) - else: - selected_result = min( - successful_attempts, - key=lambda attempt: ( - attempt.get('res_std', np.inf), - 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, - -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, - attempt.get('rank', np.inf), - ), - ) + selected_result, selection_metric = select_preferred_comparison_attempt( + successful_attempts, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + ) + else: + qc_rejected_attempts = [ + attempt for attempt in completed_attempts + if attempt.get('rejected_by_transit_qc', False) + ] + selected_result, selection_metric = select_preferred_comparison_attempt( + qc_rejected_attempts, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + ) + fallback_to_qc_rejected = selected_result is not None + + if selected_result is not None: selected_result['selected'] = True - selected_residual = selected_result.get('res_std', np.inf) + selected_result['selected_despite_transit_qc'] = fallback_to_qc_rejected + selected_ktmf_metric = selected_result.get('ktmf_metric', np.nan) + selected_transit_delta_bic = selected_result.get('transit_delta_bic', np.nan) selected_eebls_snr = selected_result.get('eebls_snr', np.nan) for attempt in attempts: if attempt is selected_result: - if selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + if fallback_to_qc_rejected: + attempt['selection_reason'] = ( + "selected as best available fallback: all completed comparison-star " + "target fits were rejected by transit QC" + ) + if selection_metric == 'ktmf' and np.isfinite(selected_ktmf_metric): + attempt['selection_reason'] += ( + f"; this fit had the highest KTMF ({format_ktmf_metric(selected_ktmf_metric)})" + ) + elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + attempt['selection_reason'] += ( + f"; this fit had the highest EEBLS SNR ({selected_eebls_snr:.2f})" + ) + elif np.isfinite(selected_transit_delta_bic): + attempt['selection_reason'] += ( + "; this fit had the strongest transit-vs-flat Delta BIC " + f"({format_transit_delta_bic(selected_transit_delta_bic)})" + ) + elif selection_metric == 'ktmf' and np.isfinite(selected_ktmf_metric): + attempt['selection_reason'] = ( + "selected: highest KTMF among the evaluated " + "comparison-star calibration candidates" + ) + elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): attempt['selection_reason'] = ( "selected: highest EEBLS SNR among the evaluated " "comparison-star calibration candidates" ) else: attempt['selection_reason'] = ( - "selected: lowest target-fit residual scatter among the evaluated " + "selected: strongest transit-vs-flat Delta BIC among the evaluated " "comparison-star calibration candidates" ) continue - if attempt.get('fit') is not None and np.isfinite(attempt.get('res_std', np.inf)): - if selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + if ( + attempt.get('fit') is not None + and attempt.get('full_reduction_applied', False) + and (not attempt.get('rejected_by_transit_qc', False) or fallback_to_qc_rejected) + ): + if selection_metric == 'ktmf' and np.isfinite(selected_ktmf_metric): + attempt['selection_reason'] = ( + "not selected: KTMF " + f"{format_ktmf_metric(attempt.get('ktmf_metric', np.nan))} was lower than the selected " + f"{format_ktmf_metric(selected_ktmf_metric)}" + ) + elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): if np.isfinite(attempt.get('eebls_snr', np.nan)): attempt['selection_reason'] = ( "not selected: EEBLS SNR " @@ -8616,9 +11821,9 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ) else: attempt['selection_reason'] = ( - "not selected: target-fit residual scatter " - f"{attempt['res_std'] * 100.0:.4f}% was higher than the selected " - f"{selected_residual * 100.0:.4f}%" + "not selected: transit-vs-flat Delta BIC " + f"{format_transit_delta_bic(attempt.get('transit_delta_bic', np.nan))} was lower than the selected " + f"{format_transit_delta_bic(selected_transit_delta_bic)}" ) return { @@ -8925,7 +12130,8 @@ def main(): inputfiles = corruption_check(exotic_infoDict['images']) # time sort images times, jd_times = [], [] - for file in inputfiles: + log_info(f"Reading FITS timestamps and converting to BJD_TDB for {len(inputfiles)} frame(s).") + for file_index, file in enumerate(inputfiles): extension = 0 plateStatus.setCurrentFilename(file) header = fits.getheader(filename=file, ext=extension) @@ -8936,6 +12142,9 @@ def main(): times.append(obsTime) plateStatus.setObsTime(obsTime) jd_times.append(img_time_jd(header)) + completed = file_index + 1 + if completed == len(inputfiles) or completed % 25 == 0: + log_info(f"Timestamp conversion progress: {completed}/{len(inputfiles)}") extension = 0 plateStatus.setCurrentFilename(inputfiles[0]) @@ -9168,6 +12377,15 @@ def main(): fit_every_comparison_candidate = should_fit_lightcurve_to_every_comparison_candidate( exotic_infoDict.get('fit_lightcurve_to_every_comparison_candidate', 'n') ) + use_deviation_from_expected_transit_in_qc = should_use_deviation_from_expected_transit_in_qc( + exotic_infoDict.get('use_deviation_from_expected_transit_in_qc', True) + ) + deviation_from_expected_transit_in_qc_sigma = parse_deviation_from_expected_transit_in_qc_sigma( + exotic_infoDict.get('deviation_from_expected_transit_in_qc_sigma', 5.0) + ) + assess_all_comparisons_before_selecting_best = should_assess_all_comparisons_before_selecting_best( + exotic_infoDict.get('assess_all_comparisons_before_selecting_best', 'y') + ) use_psf_photometry = should_use_psf_photometry( exotic_infoDict.get('use_psf_photometry', 'y') ) @@ -9188,6 +12406,23 @@ def main(): log_info("EEBLS transit initializer disabled per optional_info setting.") if not pick_comparison_by_eebls_snr: log_info("Comparison-star selection by EEBLS SNR disabled per optional_info setting.") + if not use_deviation_from_expected_transit_in_qc: + log_info("Expected-value transit QC deviation checks disabled per optional_info setting.") + if target_driven_comp_selection: + log_info( + "Warning: target-driven comparison selection is no longer used; " + "EXOTIC will run comparison-star calibration followed by full candidate reductions.", + warn=True, + ) + if not assess_all_comparisons_before_selecting_best: + log_info( + "Warning: 'assess_all_comparisons_before_selecting_best' is now ignored; " + "all ranked comparison-star candidates will be fully reduced before selection.", + warn=True, + ) + + pDict['use_deviation_from_expected_transit_in_qc'] = use_deviation_from_expected_transit_in_qc + pDict['deviation_from_expected_transit_in_qc_sigma'] = deviation_from_expected_transit_in_qc_sigma for i, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{i + 1}" @@ -9625,7 +12860,6 @@ def main(): 'best_fit_lc': None, 'comp_star_num': None, 'comp_star_coords': None, - 'min_std': 100000, 'min_aperture': None, 'min_annulus': None, 'aperture_index': None, @@ -9633,27 +12867,25 @@ def main(): 'adaptive_summary': None, 'calibration_field_score': np.inf, 'selection_basis': 'target_fit', - 'selection_metric': 'residual_scatter', + 'selection_metric': 'ktmf', + 'comparison_ktmf_metric': np.nan, 'comparison_eebls_snr': np.nan, + 'comparison_transit_delta_bic': np.nan, } comparison_calibration = None - target_driven_search = None - if target_driven_comp_selection: - log_info("\nUsing target-driven comparison-star selection per optional_info setting.") - else: - comparison_calibration = select_comparison_calibrated_photometry( - psf_data, - aper_data, - apers, - annuli, - airmass, - exotic_infoDict['comp_stars'], - sigma_display, - skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, - use_psf_photometry=use_psf_photometry, - use_aperture_photometry=use_aperture_photometry, - ) + comparison_calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + apers, + annuli, + airmass, + exotic_infoDict['comp_stars'], + sigma_display, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=use_psf_photometry, + use_aperture_photometry=use_aperture_photometry, + ) if comparison_calibration is not None: log_info("\nCalibrating comparison stars before target fitting. Please wait.") @@ -9673,6 +12905,15 @@ def main(): f"{comparison_calibration['suitability_outlier_rejected_count']} high-suitability " "outlier(s) before target-fit evaluation." ) + if comparison_calibration.get('image_outlier_rejected_count', 0) > 0: + required_pairs = comparison_calibration.get('image_outlier_required_valid_pairs', 0) + sigma_threshold = comparison_calibration.get('image_outlier_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + log_info( + "Comparison-star field image clipping rejected " + f"{comparison_calibration['image_outlier_rejected_count']} frame(s) after suitability clipping " + f"because every valid pairwise comparison was more than {sigma_threshold:.2f} sigma from " + f"its flat-line median (min valid pair count={required_pairs})." + ) for summary in comparison_calibration['comp_summaries']: aggregate_text = "n/a" if not np.isfinite(summary['aggregate_score']) else f"{summary['aggregate_score'] * 100.0:.4f}%" ensemble_text = "n/a" if not np.isfinite(summary['ensemble_score']) else f"{summary['ensemble_score'] * 100.0:.4f}%" @@ -9747,10 +12988,20 @@ def main(): tFlux, plot_time_range=full_plot_time_range, disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + assess_all_comparisons_before_selecting_best=assess_all_comparisons_before_selecting_best, + final_fit_baseline_duration_multiplier=final_fit_baseline_duration_multiplier, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=aperture_values, + adaptive_annulus_values=annulus_values, + fallback_sigma=sigma_display, + save_dir=exotic_infoDict['save'], + planet_name=pDict['pName'], + observation_date=exotic_infoDict['date'], ) comparison_calibration['ranked_fit_comp_indices'] = [ summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] @@ -9782,13 +13033,39 @@ def main(): myfit = selected_attempt['fit'] tFlux1 = selected_attempt['tflux_fit'] cFlux1 = selected_attempt['cflux_fit'] - res_std = selected_attempt['res_std'] - selection_basis = ( - 'comparison_field' - if selected_comp_index == comparison_calibration['best_comp_index'] - else 'comparison_field_retry' + selected_source_indices = np.asarray( + selected_attempt.get('source_indices', np.arange(len(tFlux1), dtype=int)), + dtype=int, ) - if selection_basis == 'comparison_field_retry': + if selected_attempt.get('selected_despite_transit_qc', False): + selection_basis = 'comparison_field_qc_fallback' + elif selected_comp_index == comparison_calibration['best_comp_index']: + selection_basis = 'comparison_field' + else: + selection_basis = 'comparison_field_retry' + if selection_basis == 'comparison_field_qc_fallback': + fallback_selection_metric = comparison_fit_search.get('selection_metric', 'ktmf') + if fallback_selection_metric == 'ktmf': + fallback_metric_value = format_ktmf_metric( + selected_attempt.get('ktmf_metric', np.nan) + ) + elif fallback_selection_metric == 'eebls_snr': + fallback_metric_value = format_eebls_snr( + selected_attempt.get('eebls_snr', np.nan) + ) + else: + fallback_metric_value = format_transit_delta_bic( + selected_attempt.get('transit_delta_bic', np.nan) + ) + log_info( + "Warning: all completed comparison-star target fits were rejected by transit QC; " + "continuing with the best available fit " + f"(Comp {selected_comp_index + 1}, " + f"{comparison_selection_metric_label(fallback_selection_metric)}=" + f"{fallback_metric_value}) so final outputs are still produced.", + warn=True, + ) + elif selection_basis == 'comparison_field_retry': retry_count = selected_attempt['rank'] log_info( "Comparison-star calibration target-fit selection chose " @@ -9801,21 +13078,35 @@ def main(): photometry_info.update(best_fit_lc=myfit, comp_star_num=selected_comp_index + 1, comp_star_coords=selected_comp_coords, - min_std=res_std, min_aperture=selected_min_aperture, min_annulus=selected_min_annulus, aperture_index=selected_a, annulus_index=selected_an, + reuse_selected_full_reduction_fit=bool( + selected_attempt.get('full_reduction_applied', False) + and selected_attempt.get('fit') is not None + ), + selected_source_indices=selected_source_indices, + selected_fit_good_times=selected_attempt.get('good_times'), + selected_fit_good_flux=selected_attempt.get('good_flux'), + selected_fit_good_unc=selected_attempt.get('good_unc'), + selected_fit_good_airmass=selected_attempt.get('good_airmass'), + selected_fit_duration_samples=selected_attempt.get('duration_samples'), + selected_fit_data_highres=selected_attempt.get('data_highres'), calibration_field_score=comparison_calibration['field_score'], selection_basis=selection_basis, - selection_metric=comparison_fit_search.get('selection_metric', 'residual_scatter'), - comparison_eebls_snr=selected_attempt.get('eebls_snr', np.nan)) + selection_metric=comparison_fit_search.get('selection_metric', 'ktmf'), + comparison_ktmf_metric=selected_attempt.get('ktmf_metric', np.nan), + comparison_eebls_snr=selected_attempt.get('eebls_snr', np.nan), + comparison_transit_delta_bic=selected_attempt.get('transit_delta_bic', np.nan)) flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) - centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], - x_ref=psf_data[selected_ckey][:, 0], y_ref=psf_data[selected_ckey][:, 1]) + centroid_positions.update(x_targ=psf_data["target"][selected_source_indices, 0], + y_targ=psf_data["target"][selected_source_indices, 1], + x_ref=psf_data[selected_ckey][selected_source_indices, 0], + y_ref=psf_data[selected_ckey][selected_source_indices, 1]) if selected_comp_index in vsp_num: ref_flux[selected_comp_index] = { @@ -9867,155 +13158,24 @@ def main(): failed_comp_index = failed_attempt['comp_index'] failure_reason = failed_attempt['fit_diagnostics'].get( 'failure_reason', - "the lightcurve fitter did not converge to a usable solution.", + "the full comparison-star candidate reduction did not converge to a usable solution.", ) attempted_count = len(fit_attempts) ranked_count = len(comparison_fit_search['ranked_summaries']) log_info( - "Warning: Comparison-star calibration exhausted " + "Error: Comparison-star calibration exhausted " f"{attempted_count}/{ranked_count} ranked comparison star(s) for " - f"{comparison_calibration['method_label']} without a usable target fit " - f"(last attempt: Comp {failed_comp_index + 1}; reason: {failure_reason}). " - "Falling back to target-driven photometry selection.", - warn=True, + f"{comparison_calibration['method_label']} without a usable fully reduced target fit " + f"(last attempt: Comp {failed_comp_index + 1}; reason: {failure_reason}).", + error=True, ) else: log_info( - "Warning: Comparison-star calibration did not produce any coverage-qualified " - "comparison stars to test against the target fit. Falling back to target-driven " - "photometry selection.", - warn=True, - ) - - if photometry_info['best_fit_lc'] is None and (use_psf_photometry or use_aperture_photometry): - if use_psf_photometry and use_aperture_photometry: - log_info("\nComputing the best PSF/aperture photometry candidate from the target lightcurve. Please wait.") - elif use_aperture_photometry: - log_info("\nComputing best comparison star, aperture, and sky annulus from the target lightcurve. Please wait.") - else: - log_info("\nComputing best PSF comparison star from the target lightcurve. Please wait.") - - target_driven_search = run_target_driven_photometry_search( - times, - jd_times, - airmass, - ld, - pDict, - exotic_infoDict['comp_stars'], - psf_data, - aper_data, - apers, - annuli, - sigma_display, - require_comp_star=require_comp_star, - plot_time_range=full_plot_time_range, - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, - use_psf_photometry=use_psf_photometry, - use_aperture_photometry=use_aperture_photometry, - multiprocess_lightcurve_fits=args.multiprocess_lightcurve_fits, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, - pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, - ) - - best_candidate = target_driven_search['best_candidate'] - if best_candidate is not None: - photometry_info.update( - best_fit_lc=target_driven_search['best_fit_lc'], - comp_star_num=(None if best_candidate['comp_index'] is None else best_candidate['comp_index'] + 1), - comp_star_coords=( - None if best_candidate['comp_index'] is None - else exotic_infoDict['comp_stars'][best_candidate['comp_index']] - ), - min_std=target_driven_search['min_std'], - min_aperture=( - 0 if best_candidate['method'] == 'psf' - else (-best_candidate['aper'] if best_candidate['comp_index'] is None else best_candidate['aper']) - ), - min_annulus=best_candidate['annulus'], - aperture_index=best_candidate['a'], - annulus_index=best_candidate['an'], - selection_basis='target_fit', - selection_metric=target_driven_search.get('selection_metric', 'residual_scatter'), - comparison_eebls_snr=target_driven_search.get('selected_eebls_snr', np.nan), - ) - - flux_values.update( - flux_tar=target_driven_search['flux_tar'], - flux_ref=target_driven_search['flux_ref'], - flux_unc_tar=target_driven_search['flux_tar'] ** 0.5, - flux_unc_ref=target_driven_search['flux_ref'] ** 0.5, - ) - - x_ref_data = psf_data['target'][:, 0] - y_ref_data = psf_data['target'][:, 1] - if best_candidate['ckey'] is not None: - x_ref_data = psf_data[best_candidate['ckey']][:, 0] - y_ref_data = psf_data[best_candidate['ckey']][:, 1] - - centroid_positions.update( - x_targ=psf_data["target"][:, 0], - y_targ=psf_data["target"][:, 1], - x_ref=x_ref_data, - y_ref=y_ref_data, - ) - else: - candidate_summaries = target_driven_search.get('candidate_summaries', []) - if candidate_summaries: - log_info( - "Warning: Target-driven photometry search evaluated " - f"{len(candidate_summaries)} shortlisted candidate(s) but none yielded " - "a usable lightcurve fit.", - warn=True, - ) - log_target_fit_candidate_summaries(candidate_summaries) - elif require_comp_star: - log_info( - "Warning: Target-driven comparison-star search did not produce any " - "coverage-qualified comparison-star candidates to evaluate.", - warn=True, + "Error: Comparison-star calibration did not produce any coverage-qualified " + "comparison stars to fully reduce against the target fit.", + error=True, ) - - if best_candidate is not None and vsp_num: - if best_candidate['method'] == 'psf': - for j in vsp_num: - ckey = f"comp{j + 1}" - cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] - vsp_fit, _, _ = fit_lightcurve( - times, tFlux, cFlux, airmass, ld, pDict, jd_times, - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - plot_time_range=full_plot_time_range, - use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, - ) - ref_flux[j] = { - 'myfit': vsp_fit, - 'pos': exotic_infoDict['comp_stars'][j] - } - else: - best_a = best_candidate['a'] - best_an = best_candidate['an'] - best_target_flux = aper_data['target'][:, best_a, best_an] - for j in vsp_num: - ckey = f"comp{j + 1}" - aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) - cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] - vsp_fit, _, _ = fit_lightcurve( - times[aper_mask], best_target_flux[aper_mask], cFlux, - airmass[aper_mask], ld, pDict, jd_times[aper_mask], - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - plot_time_range=full_plot_time_range, - use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, - ) - ref_flux[j] = { - 'myfit': vsp_fit, - 'pos': exotic_infoDict['comp_stars'][j] - } + return update_photometry_adaptive_summary( photometry_info, @@ -10027,18 +13187,12 @@ def main(): ) if require_comp_star and photometry_info['comp_star_num'] is None: - if target_driven_search is not None and target_driven_search.get('candidate_summaries'): - log_info( - "Error: require_comp_star is enabled, but every evaluated comparison-star candidate " - "was rejected. See the target-fit candidate diagnostics above for the per-candidate " - "failure reasons.", - error=True, - ) - else: - log_info( - "Error: require_comp_star is enabled, but no valid comparison star could be selected.", - error=True, - ) + log_info( + "Error: require_comp_star is enabled, but every evaluated comparison-star candidate " + "was rejected or failed to complete a usable full reduction. See the comparison-star " + "calibration fit diagnostics above for the per-candidate failure reasons.", + error=True, + ) return log_info("\n\n*********************************************") @@ -10050,20 +13204,25 @@ def main(): ): log_info( "Comparison Selection Metric: " - f"{comparison_selection_metric_label(photometry_info.get('selection_metric', 'residual_scatter'))}" + f"{comparison_selection_metric_label(photometry_info.get('selection_metric', 'ktmf'))}" ) + if np.isfinite(photometry_info.get('comparison_ktmf_metric', np.nan)): + log_info(f"Selected Comparison KTMF: {photometry_info['comparison_ktmf_metric']:.2f} / 5.00") if np.isfinite(photometry_info.get('comparison_eebls_snr', np.nan)): log_info(f"Selected Comparison EEBLS SNR: {photometry_info['comparison_eebls_snr']:.2f}") + if np.isfinite(photometry_info.get('comparison_transit_delta_bic', np.nan)): + log_info( + "Selected Comparison Transit Delta BIC: " + f"{photometry_info['comparison_transit_delta_bic']:.2f}" + ) selected_method_label = selected_photometry_method_label(photometry_info) display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] == 0: # psf log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") - log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") log_info("Optimal Method: PSF photometry") elif photometry_info['min_aperture'] < 0: # no comp star log_info("Best Comparison Star: None") - log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") if adaptive_summary is not None: log_info(f"Optimal Aperture: {abs(display_aperture):.2f} +/- {adaptive_summary['aperture_std']:.2f} px") log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") @@ -10076,7 +13235,6 @@ def main(): log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") else: log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") - log_info(f"Target-Fit Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") if adaptive_summary is not None: log_info(f"Optimal Aperture: {display_aperture:.2f} +/- {adaptive_summary['aperture_std']:.2f} px") log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") @@ -10160,21 +13318,42 @@ def main(): header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f") + reuse_selected_full_reduction_fit = bool( + photometry_info.get('reuse_selected_full_reduction_fit', False) + and best_fit_lc is not None + ) + psf_selection_indices = np.asarray( + photometry_info.get('selected_source_indices', np.arange(len(best_fit_lc.time))), + dtype=int, + ) + if psf_selection_indices.shape[0] != len(best_fit_lc.time): + psf_selection_indices = np.arange(len(best_fit_lc.time), dtype=int) + # sigma clip - si = np.argsort(best_fit_lc.time) - dt = np.mean(np.diff(np.sort(best_fit_lc.time))) - ndt = int(30. / 24. / 60. / dt) * 2 + 1 # ~30 minutes - time_clip_mask = sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt, times=best_fit_lc.time[si]) - phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) - if hasattr(best_fit_lc, 'residuals') and hasattr(best_fit_lc, 'phase'): - phase_clip_mask = phase_bin_sigma_clip(best_fit_lc.residuals[si], best_fit_lc.phase[si], sigma=3, bins=10) - adaptive_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) - if use_adaptive_apertures and adaptive_summary is not None: - sorted_apertures = np.asarray(adaptive_summary['aperture_series'], dtype=float)[si] - sorted_annuli = np.asarray(adaptive_summary['annulus_series'], dtype=float)[si] - retained_mask = adaptive_aperture_outlier_mask(sorted_apertures[~time_clip_mask & ~phase_clip_mask], - sorted_annuli[~time_clip_mask & ~phase_clip_mask]) - adaptive_clip_mask[~time_clip_mask & ~phase_clip_mask] = retained_mask + if reuse_selected_full_reduction_fit: + log_info( + "Reusing the selected comparison-star full-reduction ultranest fit; " + "skipping duplicate selected-only final-fit clipping." + ) + si = np.arange(len(best_fit_lc.time), dtype=int) + time_clip_mask = np.zeros(len(best_fit_lc.time), dtype=bool) + phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + adaptive_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + else: + si = np.argsort(best_fit_lc.time) + dt = np.mean(np.diff(np.sort(best_fit_lc.time))) + ndt = int(30. / 24. / 60. / dt) * 2 + 1 # ~30 minutes + time_clip_mask = sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt, times=best_fit_lc.time[si]) + phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + if hasattr(best_fit_lc, 'residuals') and hasattr(best_fit_lc, 'phase'): + phase_clip_mask = phase_bin_sigma_clip(best_fit_lc.residuals[si], best_fit_lc.phase[si], sigma=3, bins=10) + adaptive_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + if use_adaptive_apertures and adaptive_summary is not None: + sorted_apertures = np.asarray(adaptive_summary['aperture_series'], dtype=float)[si] + sorted_annuli = np.asarray(adaptive_summary['annulus_series'], dtype=float)[si] + retained_mask = adaptive_aperture_outlier_mask(sorted_apertures[~time_clip_mask & ~phase_clip_mask], + sorted_annuli[~time_clip_mask & ~phase_clip_mask]) + adaptive_clip_mask[~time_clip_mask & ~phase_clip_mask] = retained_mask gi = ~(time_clip_mask | phase_clip_mask | adaptive_clip_mask) # good indexs prefinal_filter_diagnostics = [ build_time_rejection_diagnostic( @@ -10212,19 +13391,31 @@ def main(): goodTimes = best_fit_lc.time goodAirmasses = best_fit_lc.airmass - final_fit_series = prepare_final_fit_lightcurve_series(best_fit_lc) - if not final_fit_series.get('applied'): - log_info( - f"Warning: {final_fit_series.get('note', 'could not prepare the final-fit light curve from the provisional fit.')} " - "Falling back to the provisional detrended light curve arrays.", - warn=True, - ) - goodFluxes = np.asarray(best_fit_lc.detrended, dtype=float) - goodNormUnc = np.asarray(best_fit_lc.detrendederr, dtype=float) + if reuse_selected_full_reduction_fit: + selected_good_flux = photometry_info.get('selected_fit_good_flux') + selected_good_unc = photometry_info.get('selected_fit_good_unc') + if selected_good_flux is not None and np.shape(selected_good_flux) == np.shape(goodTimes): + goodFluxes = np.asarray(selected_good_flux, dtype=float) + else: + goodFluxes = np.asarray(best_fit_lc.detrended, dtype=float) + if selected_good_unc is not None and np.shape(selected_good_unc) == np.shape(goodTimes): + goodNormUnc = np.asarray(selected_good_unc, dtype=float) + else: + goodNormUnc = np.asarray(best_fit_lc.detrendederr, dtype=float) else: - log_info(final_fit_series['note']) - goodFluxes = np.asarray(final_fit_series['flux'], dtype=float) - goodNormUnc = np.asarray(final_fit_series['unc'], dtype=float) + final_fit_series = prepare_final_fit_lightcurve_series(best_fit_lc) + if not final_fit_series.get('applied'): + log_info( + f"Warning: {final_fit_series.get('note', 'could not prepare the final-fit light curve from the selected fit.')} " + "Falling back to the current detrended light curve arrays.", + warn=True, + ) + goodFluxes = np.asarray(best_fit_lc.detrended, dtype=float) + goodNormUnc = np.asarray(best_fit_lc.detrendederr, dtype=float) + else: + log_info(final_fit_series['note']) + goodFluxes = np.asarray(final_fit_series['flux'], dtype=float) + goodNormUnc = np.asarray(final_fit_series['unc'], dtype=float) centroid_positions.update(x_targ=centroid_positions['x_targ'][si][gi], y_targ=centroid_positions['y_targ'][si][gi], @@ -10238,13 +13429,16 @@ def main(): relative_flux_mask = relative_flux_filter_mask(goodFluxes) prefinal_filter_diagnostics.append(build_time_rejection_diagnostic( - "Final relative-flux cap", + "Final relative-flux validity filter", goodTimes, relative_flux_mask, - note=f"Dropped points with normalized flux outside (0, {RELATIVE_FLUX_MAX:g}] before the final fit.", + note="Dropped non-finite or non-positive normalized flux values before the final fit.", )) if np.count_nonzero(relative_flux_mask) == 0: - log_info("Error: No valid photometry data found after removing relative flux values above 2.", error=True) + log_info( + "Error: No valid photometry data found after removing non-finite or non-positive relative flux values.", + error=True, + ) return log_lightcurve_filter_diagnostics( @@ -10268,12 +13462,16 @@ def main(): flux_unc_tar=flux_values['flux_unc_tar'][relative_flux_mask], flux_unc_ref=flux_values['flux_unc_ref'][relative_flux_mask]) + psf_selection_indices = psf_selection_indices[si][gi][relative_flux_mask] + obs_stats_sort_index = psf_selection_indices + obs_stats_keep_mask = np.ones(psf_selection_indices.shape[0], dtype=bool) + update_photometry_adaptive_summary( photometry_info, use_adaptive_apertures, aperture_values, annulus_values, - psf_data['target'][si][gi][relative_flux_mask], + psf_data['target'][psf_selection_indices], fallback_sigma=sigma, ) display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) @@ -10396,13 +13594,20 @@ def main(): relative_flux_mask = relative_flux_filter_mask(goodFluxes) if np.count_nonzero(relative_flux_mask) == 0: - log_info("Error: No valid photometry data found after removing relative flux values above 2.", error=True) + log_info( + "Error: No valid photometry data found after removing non-finite or non-positive relative flux values.", + error=True, + ) return goodTimes = goodTimes[relative_flux_mask] goodFluxes = goodFluxes[relative_flux_mask] goodNormUnc = goodNormUnc[relative_flux_mask] goodAirmasses = goodAirmasses[relative_flux_mask] + goodFluxes, goodNormUnc, _ = normalize_flux_series_to_approximate_unity( + goodFluxes, + goodNormUnc, + ) finite_plot_times = goodTimes[np.isfinite(goodTimes)] full_plot_time_range = None if finite_plot_times.size: @@ -10420,6 +13625,12 @@ def main(): log_info("Fitting a Light Curve Model to Your Data") log_info("****************************************\n") + reuse_selected_final_model = bool( + fitsortext == 1 + and photometry_info.get('reuse_selected_full_reduction_fit', False) + and photometry_info.get('best_fit_lc') is not None + ) + ########################## # NESTED SAMPLING FITTING ########################## @@ -10438,7 +13649,7 @@ def main(): phase = (goodTimes - prior['tmid']) / prior['per'] expected_duration = estimate_transit_duration_from_prior_geometry(prior) - tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + ephemeris_tmid_search_summary = estimate_ephemeris_tmid_and_bounds( goodTimes, pDict['midT'], prior['per'], @@ -10447,8 +13658,8 @@ def main(): expected_duration=expected_duration, sigma_multiplier=35.0, ) - prior['tmid'] = tmid_search_summary['tmid'] - lower, upper = tmid_search_summary['bounds'] + prior['tmid'] = ephemeris_tmid_search_summary['tmid'] + lower, upper = ephemeris_tmid_search_summary['bounds'] if np.floor(phase).max() - np.floor(phase).min() == 0: log_info("Error: Estimated mid-transit not in observation range (check priors or observation time)", error=True) @@ -10456,20 +13667,21 @@ def main(): log_info(f" end:{np.max(goodTimes)}", error=True) log_info(f"prior:{prior['tmid']}", error=True) - if tmid_search_summary.get('duration_capped'): - log_info(tmid_search_summary['note']) + if ephemeris_tmid_search_summary.get('duration_capped'): + log_info(ephemeris_tmid_search_summary['note']) + eebls_tmid_search_summary = None if use_eebls_tmid_initializer: - tmid_search_summary = estimate_tmid_and_bounds_with_eebls( + eebls_tmid_search_summary = estimate_tmid_and_bounds_with_eebls( goodTimes, goodFluxes, goodNormUnc, prior, [lower, upper], ) - log_info(tmid_search_summary['note']) - if tmid_search_summary.get('applied'): - prior['tmid'] = tmid_search_summary['tmid'] - lower, upper = tmid_search_summary['bounds'] + log_info(eebls_tmid_search_summary['note']) + if eebls_tmid_search_summary.get('applied'): + prior['tmid'] = eebls_tmid_search_summary['tmid'] + lower, upper = eebls_tmid_search_summary['bounds'] final_airmass_span = airmass_span(goodAirmasses) airmass_skip_note = None @@ -10490,11 +13702,11 @@ def main(): "skipping airmass fitting and applying no airmass correction." ) - mybounds = { - 'rprs': build_initial_rprs_bounds(prior['rprs']), - 'tmid': [lower, upper], - 'inc': [prior['inc'] - 5, min(90, prior['inc'] + 5)], - } + mybounds = build_initial_transit_bounds( + prior, + [lower, upper], + ars_unc=pDict.get('aRsUnc'), + ) apply_vertical_flux_normalization_bound( prior, mybounds, @@ -10508,23 +13720,45 @@ def main(): log_info("Error: No valid photometry data found.", error=True) return - # final light curve fit - myfit, goodFluxes, goodNormUnc = fit_final_lightcurve_with_oot_baseline_detrending( - goodTimes, - goodFluxes, - goodNormUnc, - goodAirmasses, - prior, - mybounds, - skip_airmass_fit=skip_final_airmass_fit, - airmass_skip_note=airmass_skip_note, - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - plot_time_range=full_plot_time_range, - baseline_duration_multiplier=final_fit_baseline_duration_multiplier, - ) + if reuse_selected_final_model: + myfit = photometry_info['best_fit_lc'] + goodTimes = np.asarray(getattr(myfit, 'time', goodTimes), dtype=float) + goodAirmasses = np.asarray(getattr(myfit, 'airmass', goodAirmasses), dtype=float) + reused_flux = photometry_info.get('selected_fit_good_flux') + reused_unc = photometry_info.get('selected_fit_good_unc') + if reused_flux is not None and np.shape(reused_flux) == np.shape(goodTimes): + goodFluxes = np.asarray(reused_flux, dtype=float) + else: + goodFluxes = np.asarray(getattr(myfit, 'detrended', goodFluxes), dtype=float) + if reused_unc is not None and np.shape(reused_unc) == np.shape(goodTimes): + goodNormUnc = np.asarray(reused_unc, dtype=float) + else: + goodNormUnc = np.asarray(getattr(myfit, 'detrendederr', goodNormUnc), dtype=float) + log_info( + "Using the selected comparison-star full-reduction ultranest fit for final outputs; " + "no additional final nested-sampling fit is being run." + ) + else: + # final light curve fit + myfit, goodFluxes, goodNormUnc = fit_final_lightcurve_with_oot_baseline_detrending( + goodTimes, + goodFluxes, + goodNormUnc, + goodAirmasses, + prior, + mybounds, + skip_airmass_fit=skip_final_airmass_fit, + airmass_skip_note=airmass_skip_note, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, + baseline_duration_multiplier=final_fit_baseline_duration_multiplier, + expected_planet_dict=pDict, + expected_tmid_search_summary=ephemeris_tmid_search_summary, + eebls_search_summary=eebls_tmid_search_summary, + ) # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) @@ -10552,9 +13786,10 @@ def main(): photometry_info, len(exotic_infoDict['comp_stars']), ) - plot_obs_stats(myfit, exotic_infoDict['comp_stars'], psf_data, si, gi, pDict['pName'], + plot_obs_stats(myfit, exotic_infoDict['comp_stars'], psf_data, obs_stats_sort_index, + obs_stats_keep_mask, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], - relative_flux_mask=relative_flux_mask, + relative_flux_mask=None, background_series=observing_background_series) ####################################################################### @@ -10567,12 +13802,45 @@ def main(): log_info(f" Radius Ratio (Planet/Star) [Rp/R*]: {round_to_2(myfit.parameters['rprs'], myfit.errors['rprs'])} +/- {round_to_2(myfit.errors['rprs'])}") log_info(f" Transit depth [(Rp/R*)^2]: {round_to_2(100. * (myfit.parameters['rprs'] ** 2.))} +/- {round_to_2(100. * 2. * myfit.parameters['rprs'] * myfit.errors['rprs'])} [%]") log_info(f" Orbital Inclination [inc]: {round_to_2(myfit.parameters['inc'], myfit.errors['inc'])} +/- {round_to_2(myfit.errors['inc'])}") + ars_text = format_parameter_with_error(myfit.parameters.get('ars'), myfit.errors.get('ars')) + if ars_text is not None: + log_info(f" Ratio of Distance to Stellar Radius [a/Rs]: {ars_text}") + impact_parameter, impact_error = fit_impact_parameter_value_error(myfit) + impact_text = format_parameter_with_error(impact_parameter, impact_error) + if impact_text is not None: + log_info(f" Impact Parameter [b]: {impact_text}") if getattr(myfit, 'airmass_fit_skipped', False): log_info(f" Airmass correction: {myfit.airmass_correction_note}") else: log_info(f" Airmass coefficient 1: {round_to_2(myfit.parameters['a1'], myfit.errors['a1'])} +/- {round_to_2(myfit.errors['a1'])}") log_info(f" Airmass coefficient 2: {round_to_2(myfit.parameters['a2'], myfit.errors['a2'])} +/- {round_to_2(myfit.errors['a2'])}") - log_info(f" Residual scatter: {round_to_2(100. * np.std(myfit.residuals / np.median(myfit.data)))} %") + transit_qc = getattr(myfit, 'transit_qc', None) + if transit_qc: + residual_scatter = transit_qc.get('residual_scatter', np.nan) + if np.isfinite(residual_scatter): + log_info(f"Residual scatter around full model fit: {residual_scatter * 100.0:.4f}%") + qc_status = str(transit_qc.get('status', 'unknown')).upper() + qc_summary = transit_qc.get('summary') + if qc_summary: + log_info(f" Transit detection QC: {qc_status} - {qc_summary}") + else: + log_info(f" Transit detection QC: {qc_status}") + if np.isfinite(transit_qc.get('deviation_from_expected_value', np.nan)): + log_info( + f" Deviation From Expected Value: {transit_qc['deviation_from_expected_value']:.2f} / 1.00" + ) + if np.isfinite(transit_qc.get('tmid_deviation_sigma', np.nan)): + log_info( + f" Expected-value Tmid sigma: {transit_qc['tmid_deviation_sigma']:.2f}" + ) + if np.isfinite(transit_qc.get('rprs_deviation_sigma', np.nan)): + log_info( + f" Expected-value Rp/R* sigma: {transit_qc['rprs_deviation_sigma']:.2f}" + ) + if np.isfinite(transit_qc.get('ktmf_metric', np.nan)): + log_info(f" KTMF: {transit_qc['ktmf_metric']:.2f} / 5.00") + for contribution in transit_qc.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") if fitsortext == 1: if np.isfinite(photometry_info.get('calibration_field_score', np.inf)): log_info(f" Comparison-Star Field Score: {round_to_2(100. * photometry_info['calibration_field_score'])} %") @@ -10658,18 +13926,21 @@ def main(): log.debug("Stopped ...") +def cli(): + global _UNHANDLED_EXCEPTION_LOGGED + + _UNHANDLED_EXCEPTION_LOGGED = False + configure_runtime_logging() + install_exception_hooks() + + try: + return main() + except (KeyboardInterrupt, SystemExit): + raise + except Exception as exc: + _handle_unhandled_exception(type(exc), exc, exc.__traceback__) + raise + + if __name__ == "__main__": - # configure logger for standalone execution - logging.root.setLevel(logging.DEBUG) - fileFormatter = logging.Formatter("%(asctime)s.%(msecs)03d [%(threadName)-12.12s] %(levelname)-5.5s " - "%(funcName)s:%(lineno)d - %(message)s", f"%Y-%m-%dT%H:%M:%S") - fileHandler = TimedRotatingFileHandler(filename="exotic.log", when="midnight", backupCount=2) - fileHandler.setLevel(logging.DEBUG) - fileHandler.setFormatter(fileFormatter) - consoleFormatter = logging.Formatter("%(message)s") - consoleHandler = logging.StreamHandler(sys.stdout) - consoleHandler.setFormatter(consoleFormatter) - consoleHandler.setLevel(logging.INFO) - log.addHandler(fileHandler) - log.addHandler(consoleHandler) - main() + raise SystemExit(cli()) diff --git a/exotic/inputs.py b/exotic/inputs.py index 9a06274f..d608f7c8 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -212,6 +212,9 @@ def __init__(self, init_opt): 'final_fit_baseline_duration_multiplier': 1.0, 'use_eebls_to_initialize_tmid_and_bounds': 'y', 'pick_comparison_by_eebls_snr': 'y', + 'use_deviation_from_expected_transit_in_qc': True, + 'deviation_from_expected_transit_in_qc_sigma': 5.0, + 'assess_all_comparisons_before_selecting_best': 'y', 'detect_bad_pixels_before_photometry': 'y', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', @@ -450,6 +453,18 @@ def comp_params(self, init_file, planet_dict): 'Pick Comparison by EEBLS SNR? (y/n)', 'Pick comparison by EEBLS SNR? (y/n)', ), + 'use_deviation_from_expected_transit_in_qc': ( + 'use_deviation_from_expected_transit_in_qc', + 'Use Deviation From Expected Transit In QC? (y/n)', + ), + 'deviation_from_expected_transit_in_qc_sigma': ( + 'deviation_from_expected_transit_in_qc_sigma', + 'Deviation From Expected Transit In QC Sigma', + ), + 'assess_all_comparisons_before_selecting_best': ( + 'assess_all_comparisons_before_selecting_best', + 'Assess All Comparisons Before Selecting Best? (y/n)', + ), 'use_impactparameter_rather_than_inclination_to_fit': ( 'use_impactparameter_rather_than_inclination_to_fit', 'Use impact parameter rather than inclination to fit? (y/n)', diff --git a/exotic/output_files.py b/exotic/output_files.py index 8cb1282e..b396e53a 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1,5 +1,5 @@ from json import dump, dumps -from numpy import mean, median, std +from numpy import mean, std from pathlib import Path import numpy as np @@ -53,6 +53,70 @@ def aavso_detrend_model(fit): return np.asarray(fit.airmass_model, dtype=float) +def finite_float(value, default=np.nan): + try: + value = float(value) + except (TypeError, ValueError): + return default + return value if np.isfinite(value) else default + + +def format_parameter_with_error(value, error): + value = finite_float(value) + error = finite_float(error) + if not np.isfinite(value): + return None + if np.isfinite(error) and error >= 0: + return f"{round_to_2(value, error)} +/- {round_to_2(error)}" + return f"{round_to_2(value)} +/- n/a" + + +def fit_impact_parameter_value_error(fit): + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + sample_parameters = getattr(fit, 'sample_parameters', {}) or {} + sample_errors = getattr(fit, 'sample_errors', {}) or {} + + if 'b' in sample_parameters: + impact_parameter = finite_float(sample_parameters.get('b')) + impact_error = finite_float(sample_errors.get('b')) + if np.isfinite(impact_parameter): + return impact_parameter, impact_error + + if 'b' in parameters: + impact_parameter = finite_float(parameters.get('b')) + impact_error = finite_float(errors.get('b')) + if np.isfinite(impact_parameter): + return impact_parameter, impact_error + + ars = finite_float(parameters.get('ars')) + inc = finite_float(parameters.get('inc')) + if not np.isfinite(ars) or not np.isfinite(inc): + return np.nan, np.nan + + ecc = finite_float(parameters.get('ecc'), 0.0) + omega = np.deg2rad(finite_float(parameters.get('omega'), 0.0)) + denominator = 1.0 + ecc * np.sin(omega) + if not np.isfinite(denominator) or np.isclose(denominator, 0.0): + return np.nan, np.nan + + scale_factor = (1.0 - ecc ** 2) / denominator + inc_rad = np.deg2rad(inc) + impact_parameter = scale_factor * ars * np.cos(inc_rad) + + ars_error = finite_float(errors.get('ars')) + inc_error = finite_float(errors.get('inc')) + if np.isfinite(ars_error) and np.isfinite(inc_error): + impact_error = np.hypot( + scale_factor * np.cos(inc_rad) * ars_error, + scale_factor * ars * np.sin(inc_rad) * np.deg2rad(inc_error), + ) + else: + impact_error = np.nan + + return float(impact_parameter), float(impact_error) if np.isfinite(impact_error) else np.nan + + class OutputFiles: def __init__(self, fit, p_dict, i_dict, durs): self.fit = fit @@ -77,6 +141,23 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor min_annul=None, adaptive_summary=None): params_file = self.dir / "temp" / f"FinalParams_{self.p_dict['pName']}_{self.i_dict['date']}.json" + transit_qc = getattr(self.fit, 'transit_qc', None) + qc_residual_scatter = np.nan + if isinstance(transit_qc, dict): + qc_residual_scatter = transit_qc.get('residual_scatter', np.nan) + if not np.isfinite(qc_residual_scatter): + residuals = np.asarray(getattr(self.fit, 'residuals', np.array([])), dtype=float) + data = np.asarray(getattr(self.fit, 'data', np.array([])), dtype=float) + if residuals.size and data.size: + if residuals.shape == data.shape: + median_flux = np.nanmedian(data) + if np.isfinite(median_flux) and median_flux != 0: + qc_residual_scatter = float(np.std(residuals) / median_flux) + elif residuals.size == 1: + median_flux = np.nanmedian(data) + if np.isfinite(median_flux) and median_flux != 0: + qc_residual_scatter = float(abs(residuals.reshape(-1)[0]) / median_flux) + params_num = { "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB", @@ -86,8 +167,19 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor f"{round_to_2(100. * 2. * self.fit.parameters['rprs'] * self.fit.errors['rprs'])} [%]", "Orbital Inclination (inc)": f"{round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- " f"{round_to_2(self.fit.errors['inc'])} ", - "Scatter in the residuals of the lightcurve fit is": f"{round_to_2(100. * std(self.fit.residuals / median(self.fit.data)))} %", } + ars_text = format_parameter_with_error( + self.fit.parameters.get('ars'), + self.fit.errors.get('ars'), + ) + if ars_text is not None: + params_num["Ratio of Distance to Stellar Radius (a/Rs)"] = ars_text + impact_parameter, impact_error = fit_impact_parameter_value_error(self.fit) + impact_text = format_parameter_with_error(impact_parameter, impact_error) + if impact_text is not None: + params_num["Impact Parameter (b)"] = impact_text + if np.isfinite(qc_residual_scatter): + params_num["Residual scatter around full model fit"] = f"{qc_residual_scatter * 100.0:.4f} %" if getattr(self.fit, 'airmass_fit_skipped', False): params_num["Airmass correction"] = getattr( self.fit, @@ -110,6 +202,64 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor f"{round_to_2(self.fit.errors['a2'])}" ) + if isinstance(transit_qc, dict) and transit_qc: + qc_status = transit_qc.get('status') + qc_summary = transit_qc.get('summary') + qc_notes = transit_qc.get('notes') or [] + qc_delta_bic = transit_qc.get('delta_bic', np.nan) + qc_delta_chi2 = transit_qc.get('delta_chi2', np.nan) + qc_rprs_sigma = transit_qc.get('rprs_sigma', np.nan) + qc_duration_ratio = transit_qc.get('duration_ratio', np.nan) + qc_eebls_depth_snr = transit_qc.get('eebls_depth_snr', np.nan) + qc_deviation_metric = transit_qc.get('deviation_from_expected_value', np.nan) + qc_tmid_deviation_sigma = transit_qc.get('tmid_deviation_sigma', np.nan) + qc_rprs_deviation_sigma = transit_qc.get('rprs_deviation_sigma', np.nan) + qc_sigma_threshold = transit_qc.get('deviation_sigma_threshold', np.nan) + qc_ktmf = transit_qc.get('ktmf_metric', np.nan) + qc_ktmf_contributions = transit_qc.get('ktmf_contributions') or [] + + if qc_status: + params_num["Transit detection QC"] = str(qc_status).upper() + if qc_summary: + params_num["Transit vs flat model"] = qc_summary + if np.isfinite(qc_delta_bic): + params_num["Transit vs flat Delta BIC"] = f"{qc_delta_bic:.2f}" + if np.isfinite(qc_delta_chi2): + params_num["Transit vs flat Delta chi2"] = f"{qc_delta_chi2:.2f}" + if np.isfinite(qc_rprs_sigma): + params_num["Transit depth significance"] = f"{qc_rprs_sigma:.2f} sigma" + if np.isfinite(qc_duration_ratio): + params_num["Transit duration consistency"] = f"{qc_duration_ratio:.2f}x modeled duration" + if np.isfinite(qc_eebls_depth_snr): + params_num["EEBLS depth SNR"] = f"{qc_eebls_depth_snr:.2f}" + if np.isfinite(qc_deviation_metric): + params_num["Deviation From Expected Value"] = f"{qc_deviation_metric:.2f} / 1.00" + if np.isfinite(qc_sigma_threshold): + params_num["Expected-value QC threshold"] = f"{qc_sigma_threshold:.2f} sigma" + if np.isfinite(qc_tmid_deviation_sigma): + params_num["Expected-value Tmid deviation"] = f"{qc_tmid_deviation_sigma:.2f} sigma" + if np.isfinite(qc_rprs_deviation_sigma): + params_num["Expected-value Rp/R* deviation"] = f"{qc_rprs_deviation_sigma:.2f} sigma" + if np.isfinite(qc_ktmf): + params_num["KTMF"] = f"{qc_ktmf:.2f} / 5.00" + for contribution_index, contribution in enumerate(qc_ktmf_contributions, start=1): + label = contribution.get('label', f'Component {contribution_index}') + detail = contribution.get('detail') or 'n/a' + available = bool(contribution.get('available')) + points = float(contribution.get('points', 0.0) or 0.0) + max_points = float(contribution.get('max_points', 0.0) or 0.0) + score = contribution.get('score', np.nan) + if available and np.isfinite(score): + params_num[f"KTMF contribution {contribution_index}"] = ( + f"{label}: +{points:.2f}/{max_points:.2f} (score={score:.2f}; {detail})" + ) + else: + params_num[f"KTMF contribution {contribution_index}"] = ( + f"{label}: +0.00/0.00 (unavailable; {detail})" + ) + if qc_notes: + params_num["Transit QC notes"] = " ".join(str(note) for note in qc_notes) + if vsp_params: params_num["Variable Reference Star"] = f"AAVSO Label: {vsp_params[0]['cname']}, " + \ f"Position: {vsp_params[0]['pos']}" diff --git a/inits.json b/inits.json index 38d32220..ad1f71da 100644 --- a/inits.json +++ b/inits.json @@ -31,7 +31,10 @@ "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", - "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", + "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to use EEBLS depth SNR as an earlier tie-break when KTMF scores do not settle the comparison-star choice. Default y.", + "Expected-Value Transit QC": "Set optional_info 'use_deviation_from_expected_transit_in_qc' to true to reject transit fits whose fitted Tmid or Rp/R* stray too far from the published expected values. Default true.", + "Expected-Value Transit QC Sigma": "Set optional_info 'deviation_from_expected_transit_in_qc_sigma' to the sigma threshold used by the expected-value transit QC rejection. Default 5.", + "Assess All Comparisons Before Selecting Best": "Set optional_info 'assess_all_comparisons_before_selecting_best' to y to fit every comparison-star candidate that survives the earlier screening, report all fits, and select the candidate with the highest KTMF score. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", @@ -114,6 +117,9 @@ "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", "pick_comparison_by_eebls_snr": "y", + "use_deviation_from_expected_transit_in_qc": true, + "deviation_from_expected_transit_in_qc_sigma": 5.0, + "assess_all_comparisons_before_selecting_best": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_psf_photometry": "y", "use_aperture_photometry": "y", diff --git a/manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json b/manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json new file mode 100644 index 00000000..fe408498 --- /dev/null +++ b/manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json @@ -0,0 +1,35 @@ +{ + "planet_name": "HAT-P-32 b", + "observation_date": "2026-04-28", + "comparison_star": 1, + "comparison_label": "Comp 1", + "comparison_position": [ + 100.0, + 200.0 + ], + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "failure_reason": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10).", + "fit_diagnostics": { + "input_point_count": 6, + "has_reference_flux": true, + "relative_flux_point_count": 6, + "sigma_clip_point_count": 4, + "usable_point_count": 4, + "failed_stage": "transit_qc", + "failure_reason": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." + }, + "parameter_summary": null, + "fit_point_count": 6, + "eebls_snr": NaN, + "transit_delta_bic": NaN, + "residual_scatter": 0.0, + "ktmf_metric": NaN, + "ktmf_contributions": [], + "transit_qc": { + "status": "fail", + "summary": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." + }, + "saved_debug_series": null, + "saved_bestfit_plot": null, + "archive_errors": [] +} \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 19c0d3e3..f46e45e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ install_requires = file: requirements.txt [options.entry_points] console_scripts = - exotic = exotic.exotic:main + exotic = exotic.exotic:cli exotic-gui = exotic.exotic_gui:main [options.packages.find] diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index b33e3709..1417bdb3 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -509,6 +509,212 @@ def __init__(self, *args, **kwargs): assert fit.errors["inc"] > 0 +def test_nested_fit_tracks_free_ars_with_internal_impact_parameter(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 101) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + ml_values = prior.copy() + ml_values["ars"] = 12.3 + b_ml = float(elca.impact_parameter_from_inclination(ml_values, 88.8)) + sample_points = np.array( + [ + [0.100, 12.10, b_ml - 0.02, 0.0000], + [0.101, 12.20, b_ml - 0.01, 0.0002], + [0.099, 12.40, b_ml + 0.01, -0.0001], + [0.100, 12.50, b_ml + 0.02, 0.0001], + ] + ) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr( + elca, + "run_reactive_sampler", + lambda *args, **kwargs: { + "maximum_likelihood": {"point": np.array([0.100, 12.30, b_ml, 0.0])}, + "posterior": { + "stdev": np.array([0.005, 0.1, 0.02, 0.0005]), + "errlo": np.array([-0.005, -0.1, -0.02, -0.0005]), + "errup": np.array([0.005, 0.1, 0.02, 0.0005]), + }, + "weighted_samples": { + "points": sample_points, + "logl": np.array([-4.0, -3.0, -3.2, -3.8]), + }, + "samples": sample_points.copy(), + }, + ) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "ars": [11.5, 12.5], "inc": [87.0, 89.5], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + ) + + bounds_values = [] + for ars_value in (11.5, 12.5): + corner_values = prior.copy() + corner_values["ars"] = ars_value + bounds_values.extend( + np.asarray( + elca.impact_parameter_from_inclination(corner_values, np.array([87.0, 89.5])), + dtype=float, + ).reshape(-1).tolist() + ) + + assert fit.sampled_keys == ["rprs", "ars", "b", "tmid"] + assert fit.parameters["ars"] == pytest.approx(12.3, abs=1e-12) + assert fit.parameters["inc"] == pytest.approx(88.8, abs=1e-6) + assert fit.sample_bounds["b"] == pytest.approx([min(bounds_values), max(bounds_values)]) + + +def test_nested_fit_duration_prior_penalizes_wrong_transit_length(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(0.20, 0.30, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = np.ones_like(time) + data[0] += 1e-4 + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + captured = {} + good_point = np.array([prior["rprs"], prior["ars"], prior["inc"], prior["tmid"]], dtype=float) + bad_point = np.array([prior["rprs"], 30.0, prior["inc"], prior["tmid"]], dtype=float) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + + def fake_run_reactive_sampler(sampler, *args, **kwargs): + loglike = sampler.args[1] + captured["good"] = float(loglike(good_point)) + captured["bad"] = float(loglike(bad_point)) + return { + "maximum_likelihood": {"point": good_point.copy()}, + "posterior": { + "stdev": np.array([0.001, 0.1, 0.05, 0.0001]), + "errlo": np.array([-0.001, -0.1, -0.05, -0.0001]), + "errup": np.array([0.001, 0.1, 0.05, 0.0001]), + }, + "weighted_samples": { + "points": np.vstack([good_point, bad_point]), + "logl": np.array([captured["good"], captured["bad"]]), + }, + "samples": np.vstack([good_point, bad_point]), + } + + monkeypatch.setattr(elca, "run_reactive_sampler", fake_run_reactive_sampler) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "ars": [10.0, 35.0], "inc": [88.5, 89.5], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + duration_prior={ + "applied": True, + "expected_duration": elca.transit_duration(prior), + "sigma_log_duration": 0.05, + }, + ) + + expected_penalty = -0.5 * ( + np.log(elca.transit_duration({"per": prior["per"], "rprs": prior["rprs"], "ars": 30.0, "inc": prior["inc"], "ecc": prior["ecc"], "omega": prior["omega"]}) / elca.transit_duration(prior)) + / 0.05 + ) ** 2 + + assert fit.parameters["ars"] == pytest.approx(prior["ars"], abs=1e-12) + assert captured["good"] > captured["bad"] + assert (captured["bad"] - captured["good"]) == pytest.approx(expected_penalty, rel=1e-6, abs=1e-6) + + +def test_nested_fit_duration_prior_returns_finite_floor_for_invalid_geometry(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + captured = {} + good_point = np.array([prior["rprs"], prior["ars"], prior["inc"], prior["tmid"]], dtype=float) + invalid_point = np.array([prior["rprs"], 50.0, 80.0, prior["tmid"]], dtype=float) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + + def fake_run_reactive_sampler(sampler, *args, **kwargs): + loglike = sampler.args[1] + prior_transform = sampler.args[2] + captured["invalid"] = float(loglike(invalid_point)) + captured["vector"] = np.asarray(loglike(np.vstack([good_point, invalid_point])), dtype=float) + captured["transformed"] = prior_transform(np.full((2, 4), 0.5, dtype=float)) + return { + "maximum_likelihood": {"point": good_point.copy()}, + "posterior": { + "stdev": np.array([0.001, 0.1, 0.05, 0.0001]), + "errlo": np.array([-0.001, -0.1, -0.05, -0.0001]), + "errup": np.array([0.001, 0.1, 0.05, 0.0001]), + }, + "weighted_samples": { + "points": np.vstack([good_point, invalid_point]), + "logl": captured["vector"], + }, + "samples": np.vstack([good_point, invalid_point]), + } + + monkeypatch.setattr(elca, "run_reactive_sampler", fake_run_reactive_sampler) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "ars": [10.0, 50.0], "inc": [80.0, 89.5], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + duration_prior={ + "applied": True, + "expected_duration": elca.transit_duration(prior), + "sigma_log_duration": 0.05, + }, + ) + + assert fit.parameters["ars"] == pytest.approx(prior["ars"], abs=1e-12) + assert np.isfinite(captured["invalid"]) + assert captured["invalid"] == pytest.approx(elca.BAD_LOG_LIKELIHOOD) + assert captured["vector"].shape == (2,) + assert np.all(np.isfinite(captured["vector"])) + assert captured["vector"][1] == pytest.approx(elca.BAD_LOG_LIKELIHOOD) + assert np.asarray(captured["transformed"]).shape == (2, 4) + + def test_rprs_posterior_recenter_diagnostics_detect_upper_bound_clipping(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) @@ -546,6 +752,43 @@ def test_rprs_posterior_recenter_diagnostics_detect_upper_bound_clipping(monkeyp assert diagnostics["bounds"][1] > 0.15 +def test_ars_posterior_recenter_diagnostics_detect_lower_bound_clipping(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = {"ars": [10.0, 15.0], "tmid": [-0.005, 0.005]} + fit.sampled_keys = ["ars", "tmid"] + fit.sample_bounds = {"ars": [10.0, 15.0], "tmid": [-0.005, 0.005]} + + ars_samples = np.concatenate([ + np.linspace(10.001, 10.040, 30), + np.linspace(10.060, 10.800, 12), + ]) + tmid_samples = np.linspace(-2e-4, 2e-4, ars_samples.size) + points = np.column_stack([ars_samples, tmid_samples]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-6.0, -3.0, ars_samples.size), + }, + "samples": points.copy(), + } + + diagnostics = fit.get_parameter_posterior_recenter_diagnostics("ars") + + assert diagnostics["clipped"] is True + assert diagnostics["edge"] == "lower" + assert diagnostics["mode"] < 10.5 + assert diagnostics["std"] > 0 + assert diagnostics["lower_edge_peak_fraction"] >= 0.20 + assert diagnostics["bounds"][0] < 10.0 + assert diagnostics["bounds"][0] >= 0.0 + + def test_rprs_posterior_recenter_diagnostics_ignores_upper_edge_below_twenty_percent(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 32e8807d..1cd235d3 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -1,6 +1,8 @@ +import importlib import importlib.util import sys import types +from pathlib import Path import numpy as np import pytest @@ -8,7 +10,7 @@ def _module_available(name: str) -> bool: try: return importlib.util.find_spec(name) is not None - except (ModuleNotFoundError, ValueError): + except Exception: return False @@ -20,6 +22,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: fake_barycorrpy = types.ModuleType("barycorrpy") fake_utc_tdb = types.ModuleType("barycorrpy.utc_tdb") fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None +fake_barycorrpy.utc_tdb = fake_utc_tdb fake_astroalign = types.ModuleType("astroalign") fake_astroalign.PIXEL_TOL = 1 fake_astroquery = types.ModuleType("astroquery") @@ -79,18 +82,27 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: _set_stub_if_missing("ultranest", fake_ultranest) _set_stub_if_missing("barycorrpy", fake_barycorrpy) _set_stub_if_missing("barycorrpy.utc_tdb", fake_utc_tdb) +try: + importlib.import_module("barycorrpy.utc_tdb") +except Exception: + sys.modules["barycorrpy"] = fake_barycorrpy + sys.modules["barycorrpy.utc_tdb"] = fake_utc_tdb sys.modules.setdefault("exotic.api.elca", fake_elca) sys.modules.setdefault("exotic.api.ld", fake_ld) from exotic.exotic import ( adaptive_aperture_outlier_mask, + annotate_transit_qc_expected_values, auto_tune_aperture_sigma_grid, + build_initial_ars_bounds, + build_single_transit_duration_prior, build_target_fit_candidate_jobs, build_time_rejection_diagnostic, check_coordinates, cheap_lightcurve_prescore, centroid_offset_matches_reference, choose_centroid_seed_position, + compute_transit_qc_ktmf, apply_comparison_star_suitability_outlier_rejection, comparison_calibration_selection_reason, comparison_candidate_fit_selection_reason, @@ -100,6 +112,9 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: diagnose_lightcurve_fit_inputs, detrend_flux_on_out_of_transit_baseline, ensure_lightcurve_fit_failure_reason, + evaluate_lightcurve_candidate, + evaluate_transit_detection_qc, + finalize_comparison_candidate_full_reduction, fit_lightcurve, fit_final_lightcurve_with_oot_baseline_detrending, fit_lightcurve_to_every_comparison_candidate, @@ -114,8 +129,11 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: log_comparison_calibration_fit_attempt_summaries, log_comparison_candidate_fit_summaries, log_target_fit_candidate_summaries, + normalize_flux_series_to_approximate_unity, phase_bin_sigma_clip, + parse_deviation_from_expected_transit_in_qc_sigma, prepare_final_fit_lightcurve_series, + prepare_lightcurve_fit_input_series, representative_psf_sigma, ranked_comparison_calibration_summaries, resolve_sky_annulus_geometry, @@ -136,7 +154,9 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: should_pick_comparison_by_eebls_snr, should_use_psf_photometry, should_skip_low_comparison_coverage_rejection, + should_assess_all_comparisons_before_selecting_best, should_use_fast_target_centroid, + should_use_deviation_from_expected_transit_in_qc, update_coordinates_with_proper_motion, ) @@ -247,6 +267,92 @@ def test_save_selected_photometry_debug_series_writes_stage_masks(tmp_path): assert rows[:, 5].astype(int).tolist() == [1, 0, 0] +def test_finalize_comparison_candidate_phase_clips_before_nested_fit(monkeypatch): + captured = {} + + def fake_lc_fitter(times, flux, unc, airmass, prior, bounds, jd_times=None, mode=None, **kwargs): + assert mode == "lm" + return types.SimpleNamespace( + residuals=np.linspace(-0.01, 0.01, len(times)), + phase=np.linspace(-0.5, 0.5, len(times)), + ) + + def fake_phase_clip(residuals, phase, sigma=3, bins=10): + mask = np.zeros(len(residuals), dtype=bool) + mask[3] = True + return mask + + def fake_final_fit( + times, + flux, + unc, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["times"] = np.asarray(times, dtype=float).copy() + captured["jd_times"] = np.asarray(jd_times, dtype=float).copy() + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + airmass=np.asarray(airmass, dtype=float), + data=np.asarray(flux, dtype=float), + dataerr=np.asarray(unc, dtype=float), + detrended=np.asarray(flux, dtype=float), + detrendederr=np.asarray(unc, dtype=float), + airmass_model=np.ones(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + phase=np.linspace(-0.5, 0.5, len(times)), + residuals=np.zeros(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a1": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a1": 0.01, "a2": 0.01}, + ) + return fit, np.asarray(flux, dtype=float), np.asarray(unc, dtype=float) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.phase_bin_sigma_clip", fake_phase_clip) + monkeypatch.setattr("exotic.exotic.fit_final_lightcurve_with_oot_baseline_detrending", fake_final_fit) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.09, 10) + result = finalize_comparison_candidate_full_reduction( + times, + np.full(10, 100.0, dtype=float), + np.full(10, 100.0, dtype=float), + np.linspace(1.0, 1.2, 10), + [0.1, 0.1, 0.1, 0.1], + { + "midT": 0.045, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "aRs": 10.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + jd_times=2460000.0 + times, + ) + + assert result["applied"] is True + assert captured["times"].tolist() == pytest.approx(np.delete(times, 3).tolist()) + assert result["source_indices"].tolist() == [0, 1, 2, 4, 5, 6, 7, 8, 9] + assert any( + diagnostic["stage"] == "Final-fit phase residual clip" + and diagnostic["dropped_point_count"] == 1 + for diagnostic in result["fit"].frame_filter_diagnostics + ) + assert result["fit"].selected_photometry_debug[ + "phase_clip_keep_mask_on_sigma_filtered" + ].tolist() == [True, True, True, False, True, True, True, True, True, True] + + def test_detrend_flux_on_out_of_transit_baseline_falls_back_to_prior_ephemeris(): times = np.linspace(-0.08, 0.08, 17) baseline = 1.0 + 0.25 * times @@ -546,6 +652,27 @@ def test_should_pick_comparison_by_eebls_snr_parses_values(): assert should_pick_comparison_by_eebls_snr(True) is True +def test_should_use_deviation_from_expected_transit_in_qc_parses_values(): + assert should_use_deviation_from_expected_transit_in_qc(None) is True + assert should_use_deviation_from_expected_transit_in_qc("y") is True + assert should_use_deviation_from_expected_transit_in_qc("n") is False + assert should_use_deviation_from_expected_transit_in_qc(True) is True + + +def test_parse_deviation_from_expected_transit_in_qc_sigma_parses_values(): + assert parse_deviation_from_expected_transit_in_qc_sigma(None) == pytest.approx(5.0) + assert parse_deviation_from_expected_transit_in_qc_sigma("7.5") == pytest.approx(7.5) + assert parse_deviation_from_expected_transit_in_qc_sigma(3) == pytest.approx(3.0) + assert parse_deviation_from_expected_transit_in_qc_sigma(-1) == pytest.approx(5.0) + + +def test_should_assess_all_comparisons_before_selecting_best_parses_values(): + assert should_assess_all_comparisons_before_selecting_best(None) is True + assert should_assess_all_comparisons_before_selecting_best("y") is True + assert should_assess_all_comparisons_before_selecting_best("n") is False + assert should_assess_all_comparisons_before_selecting_best(True) is True + + def test_build_time_rejection_diagnostic_groups_contiguous_ranges(): times = np.array([1.0, 1.1, 1.2, 1.5, 1.6, 2.0], dtype=float) keep_mask = np.array([True, False, False, True, False, True], dtype=bool) @@ -597,6 +724,39 @@ def test_estimate_tmid_and_bounds_with_eebls_identifies_box_like_transit(): assert summary["depth_snr"] > 0 +def test_estimate_tmid_and_bounds_with_eebls_keeps_depth_snr_for_one_sided_event(): + times = np.linspace(0.0, 0.11, 160) + tmid = 0.101 + duration = 0.028 + flux = np.ones(times.shape[0], dtype=float) + in_transit = np.abs(times - tmid) <= duration / 2.0 + flux[in_transit] -= 0.018 + flux_errors = np.full(times.shape[0], 0.002, dtype=float) + prior = { + "tmid": 0.08, + "per": 1.0, + "rprs": np.sqrt(0.018), + "ars": 12.0, + "inc": 88.5, + "ecc": 0.0, + "omega": 0.0, + } + + summary = estimate_tmid_and_bounds_with_eebls( + times, + flux, + flux_errors, + prior, + [0.04, 0.12], + ) + + assert summary["method"] == "eebls" + assert summary["applied"] is False + assert summary["depth"] > 0 + assert summary["depth_snr"] > 0 + assert "keeping the EEBLS depth SNR only" in summary["note"] + + def test_estimate_ephemeris_tmid_and_bounds_caps_bracketed_runs_to_duration_scale(): prior = { "tmid": 2458247.90746, @@ -953,8 +1113,20 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): "position": [300, 400], "selected": True, "fit": object(), - "res_std": 0.01, "eebls_snr": 6.5, + "transit_delta_bic": 18.4, + "residual_scatter": 0.0042, + "ktmf_metric": 4.35, + "ktmf_contributions": [ + { + "label": "Delta BIC", + "available": True, + "points": 1.25, + "max_points": 1.40, + "score": 0.89, + "detail": "Delta BIC=18.40", + } + ], "coverage_count": 3, "coverage_total_frame_count": 3, "coverage_reference_count": 3.0, @@ -970,18 +1142,20 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): candidate_fit_summaries, { "selection_basis": "comparison_field", - "selection_metric": "eebls_snr", + "selection_metric": "ktmf", "comp_star_num": 2, - "min_std": 0.01, + "comparison_ktmf_metric": 4.35, "comparison_eebls_snr": 6.5, + "comparison_transit_delta_bic": 18.4, }, ) assert any("Selection basis: comparison-field" in message for message in logged) - assert any("Selection metric: EEBLS SNR" in message for message in logged) + assert any("Selection metric: KTMF" in message for message in logged) assert any("coverage=1 valid frame(s) out of 3 total; min_required=2; peer_median=3.0" in message for message in logged) assert any("Comp 1" in message and "reason=comparison candidate rejected after iterative low-coverage clipping" in message for message in logged) - assert any("Comp 2 [selected]" in message and "eebls_snr=6.50" in message and "comparison-field calibration ranked this star best" in message for message in logged) + assert any("Comp 2 [selected]" in message and "ktmf=4.35/5.00" in message and "comparison-field calibration ranked this star best" in message for message in logged) + assert any("KTMF contribution: Delta BIC +1.25/1.40" in message for message in logged) assert any("parameters: fit_method=ultranest" in message for message in logged) @@ -1004,7 +1178,7 @@ def test_log_comparison_calibration_fit_attempt_summaries_includes_reasons(monke "eebls_snr": np.nan, "fit_point_count": 0, "fit_diagnostics": {"usable_point_count": 0}, - "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during target/reference ratio screening (non-finite=0, >2x=3, finite ratio range=3.0000 to 3.0000).", + "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during invalid target/reference ratio screening (non-finite=0, non-positive=3, finite ratio range=-1.0000 to -1.0000).", "parameter_summary": None, }, { @@ -1017,8 +1191,20 @@ def test_log_comparison_calibration_fit_attempt_summaries_includes_reasons(monke "coverage_reference_count": 3.0, "coverage_min_required_count": 2, "fit": object(), - "res_std": 0.01, "eebls_snr": 5.2, + "transit_delta_bic": 18.4, + "residual_scatter": 0.0035, + "ktmf_metric": 4.60, + "ktmf_contributions": [ + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.63, + "max_points": 0.80, + "score": 0.79, + "detail": "0.3500%", + } + ], "fit_point_count": 3, "fit_diagnostics": {"usable_point_count": 3}, "failure_reason": None, @@ -1031,7 +1217,8 @@ def test_log_comparison_calibration_fit_attempt_summaries_includes_reasons(monke assert any("Comparison-star calibration target-fit diagnostics:" in message for message in logged) assert any("Photometry method: Aperture photometry (aper=7.05px, annulus=22.73px)" in message for message in logged) assert any("Comp 1" in message and "reason=relative-flux filtering left 0 usable point(s)" in message for message in logged) - assert any("Comp 2 [selected]" in message and "eebls_snr=5.20" in message and "fit_points=3" in message for message in logged) + assert any("Comp 2 [selected]" in message and "ktmf=4.60/5.00" in message and "fit_points=3" in message for message in logged) + assert any("KTMF contribution: Residual Scatter Around Full Model Fit +0.63/0.80" in message for message in logged) assert any("parameters: fit_method=ultranest" in message for message in logged) @@ -1047,15 +1234,26 @@ def test_log_target_fit_candidate_summaries_includes_methods_and_reasons(monkeyp "method_label": "Aperture photometry (aper=7.05px, annulus=22.73px)", "prescore": 0.005, "fit": None, - "res_std": np.inf, + "residual_scatter": np.inf, "eebls_snr": np.nan, + "ktmf_metric": 0.0, + "ktmf_contributions": [ + { + "label": "Deviation From Expected Value", + "available": False, + "points": 0.0, + "max_points": 0.0, + "score": np.nan, + "detail": "expected-value deviation disabled or unavailable", + } + ], "coverage_count": 3, "coverage_total_frame_count": 3, "coverage_reference_count": 3.0, "coverage_min_required_count": 2, "fit_point_count": 0, "fit_diagnostics": {"usable_point_count": 0}, - "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during target/reference ratio screening (non-finite=0, >2x=3, finite ratio range=3.0000 to 3.0000).", + "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during invalid target/reference ratio screening (non-finite=0, non-positive=3, finite ratio range=-1.0000 to -1.0000).", "parameter_summary": None, }, ] @@ -1069,6 +1267,44 @@ def test_log_target_fit_candidate_summaries_includes_methods_and_reasons(monkeyp and "reason=relative-flux filtering left 0 usable point(s)" in message for message in logged ) + assert any("KTMF contribution: Deviation From Expected Value +0.00/0.00 (unavailable;" in message for message in logged) + + +def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): + summary = { + "delta_bic": 10.0, + "delta_chi2": 50.0, + "deviation_from_expected_value": 0.6, + "tmid_deviation_sigma": 1.0, + "rprs_deviation_sigma": 2.0, + "residual_scatter": 0.005, + "rprs_sigma": 6.0, + "duration_ratio": 1.0, + "eebls_depth_snr": 8.0, + } + + ktmf_metric, contributions = compute_transit_qc_ktmf(summary) + contributions_by_label = {contribution["label"]: contribution for contribution in contributions} + + assert "Model Evidence" in contributions_by_label + assert "Delta BIC" not in contributions_by_label + assert "Delta chi2" not in contributions_by_label + assert contributions_by_label["Model Evidence"]["max_points"] == pytest.approx(0.8) + assert contributions_by_label["Deviation From Expected Value"]["max_points"] == pytest.approx(1.5) + assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7) + assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75) + assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(0.75) + + model_evidence_score = ((1.0 - np.exp(-1.0)) + (1.0 - np.exp(-2.0))) / 2.0 + expected_ktmf = ( + 0.8 * model_evidence_score + + 1.5 * 0.6 + + 0.7 * 0.5 + + 0.5 * (1.0 - np.exp(-2.0)) + + 0.75 * 1.0 + + 0.75 * (1.0 - np.exp(-2.0)) + ) + assert ktmf_metric == pytest.approx(expected_ktmf) def test_comparison_candidate_fit_selection_reason_describes_comparison_field_retry(): @@ -1076,12 +1312,12 @@ def test_comparison_candidate_fit_selection_reason_describes_comparison_field_re { "selected": True, "failure_reason": None, - "res_std": 0.01, + "transit_delta_bic": 18.4, }, { "selection_basis": "comparison_field_retry", "comp_star_num": 2, - "min_std": 0.01, + "comparison_transit_delta_bic": 18.4, }, ) @@ -1256,14 +1492,49 @@ def test_comparison_star_stability_summary_rejects_low_coverage_candidates(): assert np.isinf(summary["comp_summaries"][2]["aggregate_score"]) -def test_cheap_lightcurve_prescore_ignores_large_ratios_when_requested(): +def test_comparison_star_stability_summary_rejects_shared_bad_frame(): + airmass = np.linspace(1.0, 1.5, 6) + summary = comparison_star_stability_summary( + { + "comp1": np.array([100.0, 100.8, 99.6, 100.4, 100.1, 140.0]), + "comp2": np.array([80.0, 79.5, 80.6, 80.2, 79.8, 40.0]), + "comp3": np.array([120.0, 121.0, 119.2, 120.5, 119.7, 100.0]), + }, + airmass, + ) + + np.testing.assert_array_equal( + summary["field_image_keep_mask"], + np.array([True, True, True, True, True, False], dtype=bool), + ) + assert summary["image_outlier_rejected_count"] == 1 + assert summary["image_outlier_required_valid_pairs"] == 2 + assert summary["image_outlier_available_pairs"] == 3 + assert summary["image_outlier_valid_pair_counts"][-1] == 2 + assert summary["image_outlier_outlier_pair_counts"][-1] == 2 + + +def test_cheap_lightcurve_prescore_treats_large_ratio_flag_as_noop(): tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) cflux = np.full(tflux.shape[0], 2.0) airmass = np.linspace(1.0, 1.5, tflux.shape[0]) - score = cheap_lightcurve_prescore(tflux, cflux, airmass, enforce_relative_flux_max=True) + score_with_flag = cheap_lightcurve_prescore(tflux, cflux, airmass, enforce_relative_flux_max=True) + score_without_flag = cheap_lightcurve_prescore(tflux, cflux, airmass, enforce_relative_flux_max=False) + + assert np.isfinite(score_with_flag) + assert np.isclose(score_with_flag, score_without_flag) + + +def test_normalize_flux_series_to_approximate_unity_scales_by_robust_baseline(): + flux = np.array([3.0, 3.3, 2.7, 3.0, 30.0], dtype=float) + unc = np.full(flux.shape[0], 0.3, dtype=float) + + normalized_flux, normalized_unc, baseline = normalize_flux_series_to_approximate_unity(flux, unc) - assert np.isclose(score, 0.0) + assert baseline == pytest.approx(3.0) + assert np.nanmedian(normalized_flux[:4]) == pytest.approx(1.0) + assert np.nanmedian(normalized_unc[:4]) == pytest.approx(0.1) def test_cheap_lightcurve_prescore_allows_large_raw_target_reference_ratios(): @@ -2007,6 +2278,7 @@ def fake_phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max def test_fit_lightcurve_runs_nested_fit_when_requested(monkeypatch): captured_modes = [] + captured_duration_priors = [] def fake_lc_fitter( times, @@ -2018,8 +2290,10 @@ def fake_lc_fitter( jd_times=None, mode=None, use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, ): captured_modes.append(mode) + captured_duration_priors.append(duration_prior) return types.SimpleNamespace() monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) @@ -2059,6 +2333,11 @@ def fake_lc_fitter( assert myfit is not None assert captured_modes == ["lm", "ns"] + assert captured_duration_priors[0] is None + assert captured_duration_priors[1] is not None + assert captured_duration_priors[1]["applied"] is True + assert captured_duration_priors[1]["expected_duration"] > 0 + assert captured_duration_priors[1]["expected_duration"] > 0 def test_fit_lightcurve_attaches_frame_filter_diagnostics(monkeypatch): @@ -2135,11 +2414,356 @@ def fake_lc_fitter( assert diagnostics[2]["dropped_point_count"] == 0 -def test_fit_ranked_comparison_calibration_candidates_selects_lowest_residual_success(monkeypatch): +def test_evaluate_transit_detection_qc_prefers_transit_model(): + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["preferred_model"] == "transit" + assert summary["status"] == "pass" + assert summary["delta_bic"] > 10.0 + assert summary["delta_chi2"] > 0.0 + + +def test_evaluate_transit_detection_qc_fails_when_flat_model_is_better(): + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = np.ones(21, dtype=float) + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["status"] == "fail" + assert summary["preferred_model"] == "flat" + assert summary["delta_chi2"] < 0.0 + + +def test_evaluate_transit_detection_qc_rejects_large_expected_value_deviation(): + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.18, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.001, + transit_qc_expected_rprs=0.10, + transit_qc_expected_rprs_unc=0.01, + transit_qc_use_deviation_from_expected_transit_in_qc=True, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["status"] == "fail" + assert summary["rprs_deviation_sigma"] == pytest.approx(8.0) + assert summary["deviation_from_expected_value"] == pytest.approx(0.0) + assert summary["ktmf_metric"] <= 5.0 + + +def test_annotate_transit_qc_expected_values_prefers_propagated_epoch_tmid(): + fit = types.SimpleNamespace( + initial_tmid_search_tmid=2460658.8654321, + initial_tmid_search_uncertainty=0.0025, + ) + + annotate_transit_qc_expected_values( + fit, + { + "midT": 2455867.402743, + "midTUnc": 4.9e-05, + "rprs": 0.1488, + "rprsUnc": 0.00055, + }, + ) + + assert fit.transit_qc_expected_tmid == pytest.approx(2460658.8654321) + assert fit.transit_qc_expected_tmid_unc == pytest.approx(0.0025) + assert fit.transit_qc_expected_rprs == pytest.approx(0.1488) + assert fit.transit_qc_expected_rprs_unc == pytest.approx(0.00055) + + +def test_annotate_transit_qc_expected_values_coerces_scalar_like_inputs(): + fit = types.SimpleNamespace( + initial_tmid_search_tmid=np.array(["2460658.8654321"]), + initial_tmid_search_uncertainty="0.0025", + ) + + annotate_transit_qc_expected_values( + fit, + { + "midT": "2455867.402743", + "midTUnc": ["4.9e-05"], + "rprs": "0.1488", + "rprsUnc": np.array(["0.00055"]), + "use_deviation_from_expected_transit_in_qc": "n", + "deviation_from_expected_transit_in_qc_sigma": "7.5", + }, + ) + + assert fit.transit_qc_expected_tmid == pytest.approx(2460658.8654321) + assert fit.transit_qc_expected_tmid_unc == pytest.approx(0.0025) + assert fit.transit_qc_expected_rprs == pytest.approx(0.1488) + assert fit.transit_qc_expected_rprs_unc == pytest.approx(0.00055) + assert fit.transit_qc_use_deviation_from_expected_transit_in_qc is False + assert fit.transit_qc_deviation_sigma_threshold == pytest.approx(7.5) + + +def test_evaluate_transit_detection_qc_failure_summary_reflects_expected_value_rejection(): + times = np.linspace(0.0, 1.0, 21) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[9:12] -= 0.02 + data = transit_model + np.array( + [ + 0.0001, -0.0001, 0.0002, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, 0.0000, -0.0001, 0.0002, -0.0002, + 0.0001, 0.0000, -0.0001, 0.0001, -0.0001, 0.0000, 0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + time=times, + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.18, "tmid": 0.5, "inc": 89.0, "a2": 0.0, "per": 2.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + prior={"per": 2.0, "tmid": 0.5}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.001, + transit_qc_expected_rprs=0.10, + transit_qc_expected_rprs_unc=0.01, + transit_qc_use_deviation_from_expected_transit_in_qc=True, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["status"] == "fail" + assert "QC rejected the fit because" in summary["summary"] + assert "expected published Tmid and/or Rp/R*" in summary["summary"] + assert "not supported strongly enough against a flat/null model" not in summary["summary"] + + +def test_evaluate_transit_detection_qc_computes_missing_eebls_depth_snr(monkeypatch): + def fake_eebls(times, flux_values, flux_errors, prior, fallback_bounds): + return { + "method": "eebls", + "applied": True, + "tmid": 0.5, + "bounds": [0.45, 0.55], + "duration": 0.1, + "depth": 0.01, + "depth_snr": 7.25, + "note": "test eebls diagnostic", + } + + monkeypatch.setattr("exotic.exotic.estimate_tmid_and_bounds_with_eebls", fake_eebls) + + times = np.linspace(0.0, 1.0, 21) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[9:12] -= 0.02 + data = transit_model.copy() + fit = types.SimpleNamespace( + time=times, + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0, "per": 2.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + prior={"per": 2.0, "tmid": 0.5}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.01, + transit_qc_expected_rprs=0.10, + transit_qc_expected_rprs_unc=0.05, + transit_qc_use_deviation_from_expected_transit_in_qc=False, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["eebls_depth_snr"] == pytest.approx(7.25) + assert fit.eebls_diagnostic_depth_snr == pytest.approx(7.25) + + +def test_fit_final_lightcurve_with_oot_baseline_detrending_preserves_expected_tmid_context(monkeypatch): + run_count = {"value": 0, "duration_priors": []} + + def fake_run_nested(times, flux_values, flux_errors, airmass, prior, bounds, **kwargs): + run_count["value"] += 1 + run_count["duration_priors"].append(kwargs.get("duration_prior")) + local_times = np.asarray(times, dtype=float) + model = np.ones(local_times.shape[0], dtype=float) + model[1:-1] -= 0.01 + return types.SimpleNamespace( + time=local_times, + data=model.copy(), + dataerr=np.full(local_times.shape[0], 0.001, dtype=float), + model=model.copy(), + residuals=np.zeros(local_times.shape[0], dtype=float), + airmass=np.asarray(airmass, dtype=float), + prior=dict(prior), + parameters={"rprs": 0.1, "tmid": prior["tmid"], "inc": 89.0, "a2": 0.0, "per": prior["per"]}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds=dict(bounds), + duration_expected=0.1, + duration_measured=0.1, + ) + + monkeypatch.setattr( + "exotic.exotic.run_nested_lightcurve_fit_with_rprs_posterior_retry", + fake_run_nested, + ) + monkeypatch.setattr("exotic.exotic.apply_plot_time_range", lambda fit, plot_time_range: fit) + monkeypatch.setattr("exotic.exotic.apply_vertical_flux_normalization_bound", lambda *args, **kwargs: None) + monkeypatch.setattr( + "exotic.exotic.build_final_fit_prefit_refinement_plan", + lambda times, flux_values, flux_errors, airmass, prior, bounds, fit, **kwargs: { + "applied": True, + "note": "test prefit refinement", + "times": np.asarray(times, dtype=float), + "flux": np.asarray(flux_values, dtype=float), + "unc": np.asarray(flux_errors, dtype=float), + "airmass": np.asarray(airmass, dtype=float), + "jd_times": None, + "prior": dict(prior), + "bounds": dict(bounds), + "duration": 0.1, + "original_point_count": len(times), + "refined_point_count": len(times), + "trimmed_pre_points": 0, + "trimmed_post_points": 0, + "original_tmid_bounds": bounds["tmid"], + "refined_tmid_bounds": bounds["tmid"], + }, + ) + + times = np.linspace(2460000.45, 2460000.55, 8) + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + np.ones(times.shape[0], dtype=float), + np.full(times.shape[0], 0.001, dtype=float), + np.linspace(1.0, 1.1, times.shape[0]), + {"rprs": 0.1, "tmid": 2460000.5, "inc": 89.0, "a2": 0.0, "per": 2.0}, + {"rprs": [0.0, 1.0], "tmid": [2460000.45, 2460000.55], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + detrend_on_outoftransit_baseline=False, + expected_planet_dict={ + "midT": 2455000.0, + "midTUnc": 0.0001, + "pPer": 2.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.01, + "aRs": 15.0, + "aRsUnc": 0.1, + "inc": 89.0, + "incUnc": 0.1, + "ecc": 0.0, + "omega": 0.0, + }, + expected_tmid_search_summary={ + "method": "ephemeris", + "applied": True, + "tmid": 2460000.5, + "uncertainty": 0.002, + "bounds": [2460000.45, 2460000.55], + "duration": 0.1, + "depth": np.nan, + "depth_snr": np.nan, + "note": "test propagated tmid", + }, + eebls_search_summary={ + "method": "eebls", + "applied": True, + "tmid": 2460000.5, + "bounds": [2460000.47, 2460000.53], + "duration": 0.1, + "depth": 0.01, + "depth_snr": 6.5, + "note": "test eebls", + }, + ) + + assert run_count["value"] == 2 + assert all(prior is not None and prior.get("applied") for prior in run_count["duration_priors"]) + assert fit.initial_tmid_search_tmid == pytest.approx(2460000.5) + assert fit.transit_qc_expected_tmid == pytest.approx(2460000.5) + assert fit.transit_qc_expected_tmid_unc == pytest.approx(0.002) + assert fit.eebls_diagnostic_depth_snr == pytest.approx(6.5) + + +def test_fit_ranked_comparison_calibration_candidates_selects_highest_ktmf_success(monkeypatch): def fake_diagnostics(*args, **kwargs): return {"usable_point_count": 6} - def fake_fit_lightcurve( + def fake_finalize( times, tflux, cflux, @@ -2150,11 +2774,9 @@ def fake_fit_lightcurve( **kwargs, ): comp_marker = int(np.nanmedian(cflux)) - residual_scale_map = { - 50: 0.05, - 40: 0.02, - 30: 0.03, - } + residual_scale_map = {50: 0.05, 40: 0.02, 30: 0.03} + delta_bic_map = {50: 8.0, 40: 18.0, 30: 12.0} + ktmf_map = {50: 2.40, 40: 4.70, 30: 3.90} residual_scale = residual_scale_map[comp_marker] residuals = residual_scale * np.array([-1.0, 1.0, -1.0, 1.0, -1.0, 1.0], dtype=float) fit = types.SimpleNamespace( @@ -2162,11 +2784,22 @@ def fake_fit_lightcurve( data=np.ones_like(residuals), parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc_delta_bic=delta_bic_map[comp_marker], + transit_qc_ktmf_metric=ktmf_map[comp_marker], ) - return fit, np.asarray(tflux, dtype=float), np.asarray(cflux, dtype=float) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) - monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) times = np.linspace(0.0, 0.05, 6) jd_times = 2460000.0 + times @@ -2184,39 +2817,9 @@ def fake_fit_lightcurve( "a": 0, "an": 0, "comp_summaries": [ - { - "label": "Comp 1", - "position": (10.0, 10.0), - "aggregate_score": 0.01, - "coverage_count": 6, - "coverage_total_frame_count": 6, - "coverage_reference_count": 6.0, - "coverage_min_required_count": 5, - "coverage_rejected": False, - "comp_index": 0, - }, - { - "label": "Comp 2", - "position": (20.0, 20.0), - "aggregate_score": 0.02, - "coverage_count": 6, - "coverage_total_frame_count": 6, - "coverage_reference_count": 6.0, - "coverage_min_required_count": 5, - "coverage_rejected": False, - "comp_index": 1, - }, - { - "label": "Comp 3", - "position": (30.0, 30.0), - "aggregate_score": 0.03, - "coverage_count": 6, - "coverage_total_frame_count": 6, - "coverage_reference_count": 6.0, - "coverage_min_required_count": 5, - "coverage_rejected": False, - "comp_index": 2, - }, + {"label": "Comp 1", "position": (10.0, 10.0), "aggregate_score": 0.01, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "position": (20.0, 20.0), "aggregate_score": 0.02, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 1}, + {"label": "Comp 3", "position": (30.0, 30.0), "aggregate_score": 0.03, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 2}, ], } @@ -2233,11 +2836,13 @@ def fake_fit_lightcurve( ) assert len(result["attempts"]) == 3 + assert result["selection_metric"] == "ktmf" assert result["selected_result"]["comp_index"] == 1 assert result["selected_result"]["rank"] == 1 assert result["selected_result"]["selected"] is True - assert "lowest target-fit residual scatter" in result["selected_result"]["selection_reason"] - assert result["attempts"][0]["selection_reason"].startswith("not selected: target-fit residual scatter") + assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.70) + assert "highest KTMF" in result["selected_result"]["selection_reason"] + assert result["attempts"][0]["selection_reason"].startswith("not selected: KTMF") def test_ranked_comparison_calibration_summaries_skip_suitability_outliers(): @@ -2255,11 +2860,14 @@ def test_ranked_comparison_calibration_summaries_skip_suitability_outliers(): assert [summary["comp_index"] for summary in ranked] == [2, 3] -def test_fit_ranked_comparison_calibration_candidates_can_prefer_highest_eebls_snr(monkeypatch): - def fake_diagnostics(*args, **kwargs): - return {"usable_point_count": 6} +def test_fit_ranked_comparison_calibration_candidates_applies_field_image_clip(monkeypatch): + observed_lengths = [] - def fake_fit_lightcurve( + def fake_diagnostics(times, *args, **kwargs): + observed_lengths.append(len(times)) + return {"usable_point_count": len(times)} + + def fake_finalize( times, tflux, cflux, @@ -2269,64 +2877,236 @@ def fake_fit_lightcurve( jd_times=None, **kwargs, ): - comp_marker = int(np.nanmedian(cflux)) - if comp_marker == 50: - residual_level = 0.01 - eebls_snr = 4.0 - else: - residual_level = 0.02 - eebls_snr = 7.5 - - residuals = residual_level * np.array([-1.0, 1.0, -1.0, 1.0, -1.0, 1.0], dtype=float) + observed_lengths.append(len(times)) fit = types.SimpleNamespace( - residuals=residuals, - data=np.ones_like(residuals), + residuals=np.full(len(times), 0.01, dtype=float), + data=np.ones(len(times), dtype=float), parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, - eebls_diagnostic_depth_snr=eebls_snr, + transit_qc={"status": "pass", "summary": "ok", "ktmf_metric": 4.2}, + transit_qc_status="pass", + transit_qc_summary="ok", + transit_qc_ktmf_metric=4.2, + transit_qc_delta_bic=16.0, + frame_filter_diagnostics=[{"stage": "Comparison-field image clip", "dropped_point_count": 2}], ) - return fit, np.asarray(tflux, dtype=float), np.asarray(cflux, dtype=float) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) - monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) times = np.linspace(0.0, 0.05, 6) jd_times = 2460000.0 + times airmass = np.linspace(1.0, 1.2, 6) - ld = [0.1, 0.1, 0.1, 0.1] - p_dict = {"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0} - aper_data = { - "target": np.full((6, 1, 1), 100.0, dtype=float), - "comp1": np.full((6, 1, 1), 50.0, dtype=float), - "comp2": np.full((6, 1, 1), 40.0, dtype=float), - } comparison_calibration = { "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", "a": 0, "an": 0, + "aper": 5.0, + "annulus": 12.0, + "field_image_keep_mask": np.array([True, False, True, True, False, True], dtype=bool), + "image_outlier_sigma": 4.25, + "image_outlier_required_valid_pairs": 2, "comp_summaries": [ - { - "label": "Comp 1", - "position": (10.0, 10.0), - "aggregate_score": 0.01, - "coverage_count": 6, - "coverage_total_frame_count": 6, - "coverage_reference_count": 6.0, - "coverage_min_required_count": 5, - "coverage_rejected": False, - "comp_index": 0, - }, - { - "label": "Comp 2", - "position": (20.0, 20.0), - "aggregate_score": 0.02, - "coverage_count": 6, - "coverage_total_frame_count": 6, - "coverage_reference_count": 6.0, - "coverage_min_required_count": 5, - "coverage_rejected": False, - "comp_index": 1, - }, + {"label": "Comp 1", "position": (10.0, 10.0), "aggregate_score": 0.01, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 0}, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert observed_lengths == [4, 4] + diagnostic = result["attempts"][0]["fit"].frame_filter_diagnostics[0] + assert diagnostic["stage"] == "Comparison-field image clip" + assert diagnostic["dropped_point_count"] == 2 + assert result["attempts"][0]["fit_point_count"] == 4 + + +def test_fit_ranked_comparison_calibration_candidates_evaluates_all_candidates_even_when_flag_disabled( + monkeypatch, tmp_path +): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + call_markers = [] + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + call_markers.append(comp_marker) + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc_ktmf_metric={50: 3.2, 40: 4.4}[comp_marker], + transit_qc_delta_bic=12.0 + comp_marker / 100.0, + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + saved_dirs = [] + + def fake_save(save_dir, provisional_fit, final_fit, p_dict, observation_date, comp_index, **kwargs): + candidate_dir = Path(save_dir) / f"comp{comp_index + 1}" + candidate_dir.mkdir(parents=True, exist_ok=True) + saved_dirs.append(candidate_dir) + return candidate_dir + + monkeypatch.setattr( + "exotic.exotic.save_comparison_candidate_full_reduction_outputs", + fake_save, + ) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + assess_all_comparisons_before_selecting_best=False, + save_dir=tmp_path, + planet_name="HAT-P-32 b", + observation_date="2026-04-28", + ) + + assert call_markers == [50, 40] + assert len(result["attempts"]) == 2 + assert result["selection_metric"] == "ktmf" + assert result["selected_result"]["comp_index"] == 1 + assert [attempt["final_output_dir"] for attempt in result["attempts"]] == [ + str(tmp_path / "comp1"), + str(tmp_path / "comp2"), + ] + assert saved_dirs == [tmp_path / "comp1", tmp_path / "comp2"] + + +def test_fit_ranked_comparison_calibration_candidates_can_prefer_highest_eebls_snr(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + if comp_marker == 50: + residual_level = 0.01 + eebls_snr = 4.0 + else: + residual_level = 0.02 + eebls_snr = 7.5 + + residuals = residual_level * np.array([-1.0, 1.0, -1.0, 1.0, -1.0, 1.0], dtype=float) + fit = types.SimpleNamespace( + residuals=residuals, + data=np.ones_like(residuals), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + eebls_diagnostic_depth_snr=eebls_snr, + transit_qc_delta_bic=(10.0 if comp_marker == 50 else 12.0), + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = {"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0} + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "position": (10.0, 10.0), "aggregate_score": 0.01, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "position": (20.0, 20.0), "aggregate_score": 0.02, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 1}, ], } @@ -2350,6 +3130,159 @@ def fake_fit_lightcurve( assert result["attempts"][0]["selection_reason"].startswith("not selected: EEBLS SNR") +def test_fit_ranked_comparison_calibration_candidates_logs_per_comp_run_reporting(monkeypatch): + logged = [] + + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + final_fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + ns_type="ultranest", + transit_qc={ + "status": "pass", + "summary": "Transit model strongly preferred over flat/null model.", + "delta_bic": 18.4, + "residual_scatter": 0.0035, + "ktmf_metric": 4.6, + "ktmf_contributions": [], + }, + transit_qc_status="pass", + transit_qc_summary="Transit model strongly preferred over flat/null model.", + transit_qc_delta_bic=18.4, + transit_qc_residual_scatter=0.0035, + transit_qc_ktmf_metric=4.6, + transit_qc_ktmf_contributions=[], + rprs_posterior_refit_applied=True, + rprs_posterior_refit_count=1, + rprs_posterior_refit_note="Applied 1 automatic Rp/R* posterior range refit(s).", + prefit_refinement_applied=True, + prefit_refinement_note="Applied a focused final-fit prefit refinement window.", + oot_baseline_detrending_applied=False, + oot_baseline_detrending_note="Skipped; need out-of-transit coverage on both sides of transit to fit a linear baseline.", + ) + + def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + return { + "applied": True, + "fit": final_fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "completed the full comparison-candidate reduction.", + } + + monkeypatch.setattr("exotic.exotic.log_info", lambda message, warn=False, error=False: logged.append(message)) + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "comp_summaries": [ + { + "label": "Comp 1", + "position": (10.0, 10.0), + "aggregate_score": 0.01, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 0, + }, + ], + } + + fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert any("Starting comparison-star target-fit evaluation for Comp 1" in message for message in logged) + assert any("Preparing comparison-candidate light curve for the full reduction." in message for message in logged) + assert any("Full reduction starting. Optional out-of-transit baseline detrending is enabled." in message for message in logged) + assert any("Completed comparison-star target-fit evaluation for Comp 1" in message and "transit_qc=PASS" in message for message in logged) + assert any("Rp/R* posterior retry note: Applied 1 automatic Rp/R* posterior range refit(s)." in message for message in logged) + assert any("OOT baseline detrending note: Skipped; need out-of-transit coverage on both sides of transit to fit a linear baseline." in message for message in logged) + + +def test_evaluate_lightcurve_candidate_requests_nested_fit(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + def fake_fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + assert kwargs.get("final_fit_mode") == "ns" + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + ns_type="ultranest", + transit_qc={"status": "pass", "delta_bic": 12.0, "ktmf_metric": 3.8, "ktmf_contributions": []}, + transit_qc_status="pass", + transit_qc_summary="ok", + transit_qc_delta_bic=12.0, + transit_qc_ktmf_metric=3.8, + ) + return fit, np.asarray(tflux, dtype=float), np.asarray(cflux, dtype=float) + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + result, tflux_fit, cflux_fit = evaluate_lightcurve_candidate( + ( + np.linspace(0.0, 0.05, 6), + np.full(6, 20.0), + np.full(6, 10.0), + np.linspace(1.0, 1.2, 6), + [0.1, 0.1, 0.1, 0.1], + {"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + 2460000.0 + np.linspace(0.0, 0.05, 6), + None, + False, + True, + True, + True, + ) + ) + + assert result["accepted"] is True + assert result["ktmf_metric"] == pytest.approx(3.8) + assert tflux_fit.shape == (6,) + assert cflux_fit.shape == (6,) + + def test_fit_lightcurve_refines_nested_tmid_bounds_from_two_sided_lm_fit(monkeypatch): captured_calls = [] @@ -2363,6 +3296,7 @@ def fake_lc_fitter( jd_times=None, mode=None, use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, ): captured_calls.append({ "mode": mode, @@ -2438,6 +3372,7 @@ def fake_lc_fitter( jd_times=None, mode=None, use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, ): captured_calls.append({ "mode": mode, @@ -2504,9 +3439,11 @@ def test_run_target_driven_photometry_search_selects_best_method_across_psf_and_ evaluated = [] class DummyFit: - def __init__(self, residual_level): + def __init__(self, residual_level, delta_bic, ktmf_metric): self.residuals = np.full(6, residual_level) self.data = np.ones(6) + self.transit_qc_delta_bic = delta_bic + self.transit_qc_ktmf_metric = ktmf_metric def fake_evaluate(task): _, tflux, cflux, *_ = task @@ -2514,9 +3451,19 @@ def fake_evaluate(task): cflux = np.asarray(cflux) tflux = np.asarray(tflux) if np.allclose(cflux, 20.0): - return {"myfit": DummyFit(0.02), "res_std": 0.02}, tflux, cflux + return { + "myfit": DummyFit(0.02, 9.0, 2.80), + "res_std": 0.02, + "transit_delta_bic": 9.0, + "ktmf_metric": 2.80, + }, tflux, cflux if np.allclose(cflux, 40.0): - return {"myfit": DummyFit(0.01), "res_std": 0.01}, tflux, cflux + return { + "myfit": DummyFit(0.01, 18.0, 4.85), + "res_std": 0.01, + "transit_delta_bic": 18.0, + "ktmf_metric": 4.85, + }, tflux, cflux raise AssertionError("Unexpected candidate flux passed to evaluator.") monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) @@ -2578,26 +3525,29 @@ def fake_evaluate(task): assert len(evaluated) == 2 assert {tuple(np.unique(values)) for values in evaluated} == {(20.0,), (40.0,)} + assert result["selection_metric"] == "ktmf" assert result["best_candidate"]["method"] == "aperture" assert result["best_candidate"]["comp_index"] == 0 - assert result["min_std"] == pytest.approx(0.01) + assert result["selected_ktmf_metric"] == pytest.approx(4.85) + assert result["selected_transit_delta_bic"] == pytest.approx(18.0) def test_run_target_driven_photometry_search_can_prefer_highest_eebls_snr(monkeypatch): class DummyFit: - def __init__(self, residual_level, eebls_snr): + def __init__(self, residual_level, eebls_snr, delta_bic): self.residuals = np.full(6, residual_level) self.data = np.ones(6) self.eebls_diagnostic_depth_snr = eebls_snr + self.transit_qc_delta_bic = delta_bic def fake_evaluate(task): _, tflux, cflux, *_ = task cflux = np.asarray(cflux, dtype=float) tflux = np.asarray(tflux, dtype=float) if np.allclose(cflux, 20.0): - return {"myfit": DummyFit(0.01, 4.0), "res_std": 0.01, "eebls_snr": 4.0}, tflux, cflux + return {"myfit": DummyFit(0.01, 4.0, 20.0), "res_std": 0.01, "eebls_snr": 4.0, "transit_delta_bic": 20.0}, tflux, cflux if np.allclose(cflux, 40.0): - return {"myfit": DummyFit(0.02, 9.0), "res_std": 0.02, "eebls_snr": 9.0}, tflux, cflux + return {"myfit": DummyFit(0.02, 9.0, 12.0), "res_std": 0.02, "eebls_snr": 9.0, "transit_delta_bic": 12.0}, tflux, cflux raise AssertionError("Unexpected candidate flux passed to evaluator.") monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) @@ -2661,7 +3611,7 @@ def fake_evaluate(task): assert result["selection_metric"] == "eebls_snr" assert result["best_candidate"]["method"] == "aperture" assert result["selected_eebls_snr"] == pytest.approx(9.0) - assert result["min_std"] == pytest.approx(0.02) + assert result["selected_transit_delta_bic"] == pytest.approx(12.0) def test_fit_ranked_comparison_calibration_candidates_retries_next_best_candidate(monkeypatch): @@ -2670,13 +3620,27 @@ def __init__(self): self.residuals = np.full(6, 0.01) self.data = np.ones(6) - def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times, **kwargs): + def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): cflux = np.asarray(cflux, dtype=float) if np.allclose(cflux, 0.0): - return None, None, None - return DummyFit(), np.asarray(tflux, dtype=float), cflux + return { + "applied": False, + "fit": None, + "failure_reason": "the raw comparison-candidate photometry did not yield a usable light curve.", + "note": "test full reduction", + } + return { + "applied": True, + "fit": DummyFit(), + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": cflux, + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } - monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) times = np.linspace(0.0, 0.05, 6) jd_times = 2460000.0 + times @@ -2689,18 +3653,8 @@ def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times, **kw "annulus": 12.0, "best_comp_index": 0, "comp_summaries": [ - { - "comp_index": 0, - "key": "comp1", - "aggregate_score": 0.01, - "coverage_rejected": False, - }, - { - "comp_index": 1, - "key": "comp2", - "aggregate_score": 0.02, - "coverage_rejected": False, - }, + {"comp_index": 0, "key": "comp1", "aggregate_score": 0.01, "coverage_rejected": False}, + {"comp_index": 1, "key": "comp2", "aggregate_score": 0.02, "coverage_rejected": False}, ], } aper_data = { @@ -2723,12 +3677,285 @@ def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times, **kw assert [attempt["comp_index"] for attempt in result["attempts"]] == [0, 1] assert result["selected_result"]["comp_index"] == 1 - assert "relative-flux filtering left 0 usable point(s)" in result["attempts"][0]["fit_diagnostics"]["failure_reason"] - assert "non-finite=6" in result["attempts"][0]["fit_diagnostics"]["failure_reason"] + assert result["attempts"][0]["fit"] is None + assert result["attempts"][0]["fit_diagnostics"]["failure_reason"] is not None assert result["attempts"][1]["fit"] is not None -def test_diagnose_lightcurve_fit_inputs_reports_relative_flux_breakdown(): +def test_fit_ranked_comparison_calibration_candidates_archives_qc_failed_run_and_tries_next(monkeypatch, tmp_path): + class DummyFit: + def __init__(self, qc_status): + self.residuals = np.full(6, 0.01) + self.data = np.ones(6) + self.transit_qc_status = qc_status + self.transit_qc_summary = ( + "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." + if qc_status == "fail" + else "Transit model strongly preferred over flat/null model (Delta BIC=18.40, Delta chi2=27.10)." + ) + self.transit_qc = {"status": qc_status, "summary": self.transit_qc_summary} + + def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + cflux = np.asarray(cflux, dtype=float) + fit = DummyFit("fail" if np.allclose(cflux, 8.0) else "pass") + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": cflux, + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + def fake_save(save_dir, provisional_fit, final_fit, p_dict, observation_date, comp_index, **kwargs): + candidate_dir = Path(save_dir) / f"comp{comp_index + 1}" + candidate_dir.mkdir(parents=True, exist_ok=True) + return candidate_dir + + monkeypatch.setattr( + "exotic.exotic.save_comparison_candidate_full_reduction_outputs", + fake_save, + ) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "best_comp_index": 0, + "comp_summaries": [ + {"comp_index": 0, "key": "comp1", "label": "Comp 1", "position": [100.0, 200.0], "aggregate_score": 0.01, "coverage_rejected": False}, + {"comp_index": 1, "key": "comp2", "label": "Comp 2", "position": [300.0, 400.0], "aggregate_score": 0.02, "coverage_rejected": False}, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 10.0), + "comp1": np.full((6, 1, 1), 8.0), + "comp2": np.full((6, 1, 1), 5.0), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(6), + save_dir=tmp_path, + planet_name="HAT-P-32 b", + observation_date="2026-04-28", + ) + + assert result["selected_result"]["comp_index"] == 1 + assert result["attempts"][0]["rejected_by_transit_qc"] is True + assert result["attempts"][0]["fit_diagnostics"]["failed_stage"] == "transit_qc" + failed_run_dir = result["attempts"][0]["failed_run_dir"] + assert failed_run_dir is not None + assert (tmp_path / "comp_1_failed" / "temp" / "FailedFitSummary_HAT-P-32 b_2026-04-28.json").exists() + assert Path(failed_run_dir).exists() + + +def test_fit_ranked_comparison_calibration_candidates_falls_back_to_best_qc_rejected_fit( + monkeypatch, +): + class DummyFit: + def __init__(self, ktmf, delta_bic): + self.residuals = np.full(6, 0.01) + self.data = np.ones(6) + self.transit_qc_status = "fail" + self.transit_qc_summary = ( + "Transit model is preferred over the flat/null model, but QC rejected the fit because " + "the fit deviates too far from the expected published Tmid and/or Rp/R* values " + f"(Delta BIC={delta_bic:.2f}, Delta chi2=27.10)." + ) + self.transit_qc = { + "status": "fail", + "summary": self.transit_qc_summary, + "ktmf_metric": ktmf, + "delta_bic": delta_bic, + } + self.transit_qc_ktmf_metric = ktmf + self.transit_qc_delta_bic = delta_bic + + def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + cflux = np.asarray(cflux, dtype=float) + comp_marker = int(np.nanmedian(cflux)) + fit = DummyFit( + ktmf={8: 3.10, 5: 4.80}[comp_marker], + delta_bic={8: 18.0, 5: 30.0}[comp_marker], + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": cflux, + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "best_comp_index": 0, + "comp_summaries": [ + {"comp_index": 0, "key": "comp1", "label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False}, + {"comp_index": 1, "key": "comp2", "label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False}, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 10.0), + "comp1": np.full((6, 1, 1), 8.0), + "comp2": np.full((6, 1, 1), 5.0), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(6), + ) + + assert [attempt["rejected_by_transit_qc"] for attempt in result["attempts"]] == [True, True] + assert result["selection_metric"] == "ktmf" + assert result["selected_result"]["comp_index"] == 1 + assert result["selected_result"]["selected_despite_transit_qc"] is True + assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.80) + assert "best available fallback" in result["selected_result"]["selection_reason"] + + +def test_run_target_driven_photometry_search_skips_qc_failed_candidate(monkeypatch): + class DummyFit: + def __init__(self, residual_level, delta_bic=np.nan): + self.residuals = np.full(6, residual_level) + self.data = np.ones(6) + self.transit_qc_delta_bic = delta_bic + + def fake_evaluate(task): + _, tflux, cflux, *_ = task + cflux = np.asarray(cflux, dtype=float) + tflux = np.asarray(tflux, dtype=float) + if np.allclose(cflux, 20.0): + return { + "myfit": DummyFit(0.005, 3.0), + "accepted": False, + "res_std": 0.005, + "eebls_snr": 7.0, + "transit_delta_bic": 3.0, + "transit_qc_status": "fail", + "transit_qc_summary": "Transit detection not supported strongly enough against a flat/null model.", + "rejected_by_transit_qc": True, + "fit_diagnostics": {"failed_stage": "transit_qc", "usable_point_count": 6}, + "failure_reason": "Transit detection not supported strongly enough against a flat/null model.", + "fit_point_count": 6, + }, tflux, cflux + if np.allclose(cflux, 40.0): + return { + "myfit": DummyFit(0.02, 14.0), + "accepted": True, + "res_std": 0.02, + "eebls_snr": 4.0, + "transit_delta_bic": 14.0, + "transit_qc_status": "pass", + "transit_qc_summary": "Transit model strongly preferred over flat/null model.", + "rejected_by_transit_qc": False, + "fit_diagnostics": {"usable_point_count": 6}, + "failure_reason": None, + "fit_point_count": 6, + }, tflux, cflux + raise AssertionError("Unexpected candidate flux passed to evaluator.") + + monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + psf_target_amp = 20.0 / (2.0 * np.pi) + psf_comp_amp = 20.0 / (2.0 * np.pi) + psf_data = { + "target": np.column_stack([ + np.zeros(6), + np.zeros(6), + np.full(6, psf_target_amp), + np.ones(6), + np.ones(6), + ]), + "comp1": np.column_stack([ + np.ones(6), + np.ones(6), + np.full(6, psf_comp_amp), + np.ones(6), + np.ones(6), + ]), + } + aper_data = { + "target": np.full((6, 1, 1), 40.0), + "comp1": np.full((6, 1, 1), 40.0), + } + + result = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + p_dict, + comp_stars=[[100.0, 200.0]], + psf_data=psf_data, + aper_data=aper_data, + apers=np.array([5.0]), + annuli=np.array([12.0]), + sigma=1.0, + require_comp_star=True, + use_psf_photometry=True, + use_aperture_photometry=True, + ) + + assert len(result["candidate_summaries"]) == 2 + assert result["candidate_summaries"][0]["rejected_by_transit_qc"] is True + assert result["best_candidate"]["method"] == "aperture" + assert result["selected_transit_delta_bic"] == pytest.approx(14.0) + + +def test_diagnose_lightcurve_fit_inputs_allows_large_ratios(): diagnostics = diagnose_lightcurve_fit_inputs( np.linspace(0.0, 0.05, 6), np.full(6, 30.0), @@ -2736,11 +3963,24 @@ def test_diagnose_lightcurve_fit_inputs_reports_relative_flux_breakdown(): np.linspace(1.0, 1.5, 6), ) - assert diagnostics["failed_stage"] == "relative_flux_filter" - assert "relative-flux filtering left 0 usable point(s)" in diagnostics["failure_reason"] - assert "non-finite=0" in diagnostics["failure_reason"] - assert ">2x=6" in diagnostics["failure_reason"] - assert "finite ratio range=3.0000 to 3.0000" in diagnostics["failure_reason"] + assert diagnostics["failure_reason"] is None + assert diagnostics["relative_flux_point_count"] == 6 + assert diagnostics["usable_point_count"] >= 5 + + +def test_prepare_lightcurve_fit_input_series_normalizes_ratio_around_unity(): + times = np.linspace(0.0, 0.05, 6) + prepared = prepare_lightcurve_fit_input_series( + times, + np.full(6, 30.0), + np.full(6, 10.0), + np.linspace(1.0, 1.5, 6), + ) + + assert prepared["applied"] is True + assert np.nanmedian(prepared["debug_raw_ratio"]) == pytest.approx(3.0) + assert prepared["approximate_baseline_level"] == pytest.approx(3.0) + assert np.nanmedian(prepared["flux"]) == pytest.approx(1.0) def test_run_target_driven_photometry_search_returns_failed_candidate_summaries(monkeypatch): @@ -2937,6 +4177,33 @@ def fake_lc_fitter( assert captured["flags"] == [False] +def test_build_initial_ars_bounds_prefers_published_uncertainty_when_available(): + assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([14.5, 15.5]) + assert build_initial_ars_bounds(15.0, None) == pytest.approx([11.25, 18.75]) + + +def test_build_single_transit_duration_prior_uses_published_geometry_uncertainties(): + duration_prior = build_single_transit_duration_prior({ + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.01, + "aRs": 15.0, + "aRsUnc": 0.1, + "inc": 89.0, + "incUnc": 0.1, + "ecc": 0.0, + "omega": 0.0, + }) + + assert duration_prior["applied"] is True + assert duration_prior["expected_duration"] > 0 + assert duration_prior["sigma_log_duration"] > 0 + assert duration_prior["relative_sigma"] >= 0.049 + assert duration_prior["source"] == "published geometry uncertainties" + assert "published geometry uncertainties" in duration_prior["note"] + + def test_fit_lightcurve_skips_airmass_term_when_airmass_span_is_small(monkeypatch): captured = {} @@ -2970,6 +4237,7 @@ def fake_lc_fitter( p_dict = { "rprs": 0.1, "aRs": 15.0, + "aRsUnc": 0.1, "pPer": 1.0, "inc": 89.0, "ecc": 0.0, @@ -2982,6 +4250,8 @@ def fake_lc_fitter( myfit, _, _ = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) assert myfit is not None + assert list(captured["bounds"])[:4] == ["rprs", "tmid", "ars", "inc"] + assert captured["bounds"]["ars"] == pytest.approx([14.5, 15.5]) assert "a2" not in captured["bounds"] assert myfit.airmass_fit_skipped is True @@ -3108,3 +4378,23 @@ def test_main_prereduced_respects_disable_vertical_flux_normalization_option(mon disabled = _run_main_until_vertical_flux_bound(monkeypatch, tmp_path, disable_vertical_flux_normalization=True) assert disabled is True + + +def test_cli_logs_unhandled_exception_once(monkeypatch): + import exotic.exotic as exotic_module + + logged = [] + + monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda: None) + monkeypatch.setattr(exotic_module, "install_exception_hooks", lambda: None) + monkeypatch.setattr(exotic_module, "main", lambda: (_ for _ in ()).throw(RuntimeError("boom"))) + + def fake_log_exception(message, exc_type, exc_value, exc_traceback): + logged.append((message, exc_type, str(exc_value), exc_traceback is not None)) + + monkeypatch.setattr(exotic_module, "_log_exception_with_fallback", fake_log_exception) + + with pytest.raises(RuntimeError, match="boom"): + exotic_module.cli() + + assert logged == [("Unhandled exception during EXOTIC run", RuntimeError, "boom", True)] diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 61dfc3c1..b8529a6e 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -89,6 +89,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, RPRS_SEARCH_BOUND_MAX, RPRS_SEARCH_BOUND_MIN, + build_single_transit_duration_prior, build_initial_rprs_bounds, run_nested_lightcurve_fit_with_rprs_posterior_retry, ) @@ -143,10 +144,12 @@ def fake_lc_fitter( jd_times=None, mode=None, use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, ): call_index = len(captured["calls"]) captured["calls"].append({ "prior": dict(call_prior), + "duration_prior": duration_prior, "bounds": { key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value for key, value in call_bounds.items() @@ -192,13 +195,13 @@ def fake_lc_fitter( assert "after 5 retries" in fit.rprs_posterior_refit_note -def test_rprs_posterior_retry_caps_retry_bounds_at_exoplanet_limit(monkeypatch): +def test_rprs_posterior_retry_expands_bounds_without_hitting_the_old_0p3_cap(monkeypatch): import exotic.exotic as exotic_module captured = {"calls": []} diagnostics_sequence = [ {"clipped": True, "edge": "upper", "mode": 0.275, "std": 0.020, "bounds": [0.175, 0.375]}, - {"clipped": False, "edge": None, "mode": 0.278, "std": 0.012, "bounds": [0.175, 0.300]}, + {"clipped": False, "edge": None, "mode": 0.278, "std": 0.012, "bounds": [0.175, RPRS_SEARCH_BOUND_MAX]}, ] def make_fit(diagnostics): @@ -228,10 +231,12 @@ def fake_lc_fitter( jd_times=None, mode=None, use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, ): call_index = len(captured["calls"]) captured["calls"].append({ "prior": dict(call_prior), + "duration_prior": duration_prior, "bounds": { key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value for key, value in call_bounds.items() @@ -260,13 +265,13 @@ def fake_lc_fitter( assert len(captured["calls"]) == 2 assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.25]) assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) - assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.15, RPRS_SEARCH_BOUND_MAX]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.175, 0.375]) assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 1 - assert fit.rprs_posterior_refit_bounds == pytest.approx([0.15, RPRS_SEARCH_BOUND_MAX]) + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.175, 0.375]) -def test_rprs_posterior_retry_stops_at_maximum_exoplanet_range(monkeypatch): +def test_rprs_posterior_retry_can_continue_above_the_old_maximum_exoplanet_range(monkeypatch): import exotic.exotic as exotic_module captured = {"calls": []} @@ -299,9 +304,11 @@ def fake_lc_fitter( jd_times=None, mode=None, use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, ): captured["calls"].append({ "prior": dict(call_prior), + "duration_prior": duration_prior, "bounds": { key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value for key, value in call_bounds.items() @@ -327,9 +334,150 @@ def fake_lc_fitter( bounds, ) - assert len(captured["calls"]) == 1 - assert captured["calls"][0]["prior"]["rprs"] == pytest.approx(RPRS_SEARCH_BOUND_MAX) - assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX]) + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["prior"]["rprs"] == pytest.approx(0.35) + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([RPRS_SEARCH_BOUND_MIN, 0.35]) + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.175, 0.375]) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 1 + + +def test_ars_posterior_retry_expands_bounds_when_upper_edge_is_truncated(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "ars": {"clipped": True, "edge": "upper", "mode": 14.45, "std": 0.37, "bounds": [12.60, 16.30]}, + }, + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "ars": {"clipped": False, "edge": None, "mode": 14.50, "std": 0.20, "bounds": [12.60, 16.30]}, + }, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["rprs"]["mode"], + "ars": diagnostics["ars"]["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + return dict(diagnostics[key]) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 14.0, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.0, 0.25], + "ars": [12.80, 14.80], + "tmid": [-0.01, 0.01], + "inc": [84.0, 90.0], + "a2": [-3.0, 3.0], + }, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["ars"] == pytest.approx([12.80, 14.80]) + assert captured["calls"][1]["prior"]["ars"] == pytest.approx(14.45) + assert captured["calls"][1]["bounds"]["ars"] == pytest.approx([12.60, 16.30]) assert fit.rprs_posterior_refit_applied is False - assert fit.rprs_posterior_refit_count == 0 - assert "maximum exoplanet search range" in fit.rprs_posterior_refit_note + assert fit.ars_posterior_refit_applied is True + assert fit.ars_posterior_refit_count == 1 + assert fit.ars_posterior_refit_edge == "upper" + assert fit.ars_posterior_refit_bounds == pytest.approx([12.60, 16.30]) + + +def test_run_nested_lightcurve_fit_passes_duration_prior_when_available(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + diagnostics = {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured["duration_prior"] = duration_prior + fit = types.SimpleNamespace(parameters={"rprs": 0.1, "ars": 15.0, "tmid": 0.0, "inc": 89.0, "a2": 0.0}) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key in ("rprs", "ars") + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + duration_prior = build_single_transit_duration_prior({ + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.01, + "aRs": 15.0, + "aRsUnc": 0.1, + "inc": 89.0, + "incUnc": 0.1, + "ecc": 0.0, + "omega": 0.0, + }) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 15.0, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "ars": [14.5, 15.5], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + duration_prior=duration_prior, + ) + + assert captured["duration_prior"] == duration_prior + assert fit.duration_prior_applied is True + assert "expected duration=" in fit.duration_prior_note diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 00232e21..32b7a959 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -1,4 +1,9 @@ -from exotic.output_files import OutputFiles, save_comp_star_calibration_summary +import json + +import numpy as np +import pytest + +from exotic.output_files import OutputFiles, fit_impact_parameter_value_error, save_comp_star_calibration_summary class DummyFit: @@ -6,13 +11,17 @@ def __init__(self): self.parameters = { "tmid": 2450000.123456, "rprs": 0.1234, + "ars": 12.0, "inc": 88.5, + "ecc": 0.0, + "omega": 90.0, "a1": 1.0, "a2": 0.0, } self.errors = { "tmid": 0.0001, "rprs": 0.001, + "ars": 0.4, "inc": 0.2, "a1": 0.1, "a2": 0.1, @@ -206,6 +215,33 @@ def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): assert "Airmass coefficient 1 (a1)" not in output_text +def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclination(tmp_path): + fit = DummyFit() + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_data = json.loads(output_file.read_text(encoding="utf-8")) + final_params = output_data["FINAL PLANETARY PARAMETERS"] + keys = list(final_params) + inclination_index = keys.index("Orbital Inclination (inc)") + + assert keys[inclination_index + 1] == "Ratio of Distance to Stellar Radius (a/Rs)" + assert keys[inclination_index + 2] == "Impact Parameter (b)" + assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.0 +/- 0.4" + + expected_b, expected_b_error = fit_impact_parameter_value_error(fit) + assert expected_b == pytest.approx(12.0 * np.cos(np.deg2rad(88.5))) + assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" + + def test_final_planetary_params_reports_adaptive_aperture_summary(tmp_path): fit = DummyFit() (tmp_path / "temp").mkdir() @@ -246,6 +282,65 @@ def test_final_planetary_params_reports_adaptive_aperture_summary(tmp_path): assert "7.12 to 8.76 px" in output_text +def test_final_planetary_params_reports_transit_qc_summary(tmp_path): + fit = DummyFit() + fit.transit_qc = { + "status": "pass", + "summary": "Transit model strongly preferred over flat/null model (Delta BIC=18.40, Delta chi2=27.10).", + "delta_bic": 18.4, + "delta_chi2": 27.1, + "rprs_sigma": 6.2, + "duration_ratio": 1.05, + "eebls_depth_snr": 5.8, + "residual_scatter": 0.0032, + "deviation_from_expected_value": 0.91, + "tmid_deviation_sigma": 1.1, + "rprs_deviation_sigma": 0.8, + "deviation_sigma_threshold": 5.0, + "ktmf_metric": 4.63, + "ktmf_contributions": [ + { + "label": "Delta BIC", + "available": True, + "points": 1.25, + "max_points": 1.40, + "score": 0.89, + "detail": "Delta BIC=18.40", + }, + { + "label": "Deviation From Expected Value", + "available": True, + "points": 0.91, + "max_points": 1.00, + "score": 0.91, + "detail": "score=0.91, Tmid sigma=1.10, Rp/R* sigma=0.80", + }, + ], + "notes": ["The transit model is strongly preferred over the flat/null model."], + } + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_text = output_file.read_text(encoding="utf-8") + + assert "Transit detection QC" in output_text + assert "Transit vs flat model" in output_text + assert "PASS" in output_text + assert "Delta BIC=18.40" in output_text + assert "Residual scatter around full model fit" in output_text + assert "Deviation From Expected Value" in output_text + assert "KTMF" in output_text + assert "KTMF contribution 1" in output_text + + def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_path): fit = DummyFit() fit.airmass_fit_skipped = True From b6046999b64da22d93e346a0720988f5f500ff5c Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 29 Apr 2026 07:27:37 +1000 Subject: [PATCH 028/116] human readable bits in QC --- exotic/exotic.py | 98 ++++++++++++++++++++++++++---- exotic/output_files.py | 9 +++ tests/test_exotic_proper_motion.py | 11 +++- tests/test_output_files.py | 6 ++ 4 files changed, 109 insertions(+), 15 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 4acab3ca..007e4808 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -756,6 +756,13 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T summary = { 'enabled': bool(enabled), 'sigma_threshold': sigma_threshold, + 'expected_tmid': np.nan, + 'expected_tmid_unc': np.nan, + 'expected_tmid_unc_minutes': np.nan, + 'fitted_tmid': np.nan, + 'tmid_deviation_days': np.nan, + 'tmid_deviation_minutes': np.nan, + 'tmid_deviation_threshold_minutes': np.nan, 'tmid_deviation_sigma': np.nan, 'rprs_deviation_sigma': np.nan, 'tmid_deviation_score': np.nan, @@ -764,8 +771,9 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T 'available': False, 'failed': False, 'notes': [], + 'failure_reasons': [], } - if fit is None or not enabled: + if fit is None: return summary parameters = getattr(fit, 'parameters', {}) or {} @@ -776,13 +784,27 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T expected_tmid = expected.get('expected_tmid', np.nan) expected_tmid_unc = expected.get('expected_tmid_unc', np.nan) fitted_tmid = parameters.get('tmid', np.nan) + summary['expected_tmid'] = expected_tmid + summary['expected_tmid_unc'] = expected_tmid_unc + summary['fitted_tmid'] = fitted_tmid + if not enabled: + return summary + if ( np.isfinite(expected_tmid) and np.isfinite(expected_tmid_unc) and expected_tmid_unc > 0 and np.isfinite(fitted_tmid) ): - tmid_sigma = float(abs(fitted_tmid - expected_tmid) / expected_tmid_unc) + tmid_deviation_days = float(abs(fitted_tmid - expected_tmid)) + tmid_sigma = float(tmid_deviation_days / expected_tmid_unc) + summary['tmid_deviation_days'] = tmid_deviation_days + summary['tmid_deviation_minutes'] = tmid_deviation_days * 24.0 * 60.0 + summary['expected_tmid_unc_minutes'] = float(expected_tmid_unc) * 24.0 * 60.0 + if np.isfinite(sigma_threshold) and sigma_threshold > 0: + summary['tmid_deviation_threshold_minutes'] = ( + float(sigma_threshold) * float(expected_tmid_unc) * 24.0 * 60.0 + ) summary['tmid_deviation_sigma'] = tmid_sigma summary['tmid_deviation_score'] = transit_qc_deviation_score_from_sigma(tmid_sigma, sigma_threshold) @@ -810,7 +832,12 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T if np.isfinite(summary['tmid_deviation_sigma']): summary['notes'].append( - f"Expected-value Tmid deviation: {summary['tmid_deviation_sigma']:.2f} sigma." + "Expected-value Tmid deviation: " + f"{summary['tmid_deviation_minutes']:.2f} minutes " + f"({summary['tmid_deviation_sigma']:.2f} sigma; " + f"fit={summary['fitted_tmid']:.6f}, " + f"ephemeris={summary['expected_tmid']:.6f} +/- " + f"{summary['expected_tmid_unc_minutes']:.2f} minutes)." ) if np.isfinite(summary['rprs_deviation_sigma']): summary['notes'].append( @@ -823,9 +850,20 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T ): if np.isfinite(sigma_value) and np.isfinite(sigma_threshold) and sigma_threshold > 0 and sigma_value > sigma_threshold: summary['failed'] = True - summary['notes'].append( - f"{label} differs from the expected value by more than {sigma_threshold:.2f} sigma." - ) + if label == 'Tmid' and np.isfinite(summary['tmid_deviation_minutes']): + threshold_text = "" + if np.isfinite(summary['tmid_deviation_threshold_minutes']): + threshold_text = f" ({summary['tmid_deviation_threshold_minutes']:.2f} minutes)" + reason = ( + "Tmid of the fit is " + f"{summary['tmid_deviation_minutes']:.2f} minutes away from the ephemeris Tmid, " + f"which is {summary['tmid_deviation_sigma']:.2f} sigma from the propagated ephemeris " + f"uncertainty and beyond the {sigma_threshold:.2f}-sigma threshold{threshold_text}" + ) + else: + reason = f"{label} differs from the expected value by more than {sigma_threshold:.2f} sigma" + summary['notes'].append(reason + ".") + summary['failure_reasons'].append(reason) return summary @@ -861,6 +899,7 @@ def compute_transit_qc_ktmf(summary): 'detail': ( f"score={summary.get('deviation_from_expected_value', np.nan):.2f}, " f"Tmid sigma={summary.get('tmid_deviation_sigma', np.nan):.2f}, " + f"Tmid offset={summary.get('tmid_deviation_minutes', np.nan):.2f} min, " f"Rp/R* sigma={summary.get('rprs_deviation_sigma', np.nan):.2f}" if np.isfinite(summary.get('deviation_from_expected_value', np.nan)) else "expected-value deviation disabled or unavailable" @@ -1133,8 +1172,13 @@ def evaluate_transit_detection_qc(fit): 'deviation_sigma_threshold': deviation_sigma_threshold, 'expected_tmid': expected_context.get('expected_tmid', np.nan), 'expected_tmid_unc': expected_context.get('expected_tmid_unc', np.nan), + 'expected_tmid_unc_minutes': np.nan, + 'fitted_tmid': np.nan, 'expected_rprs': expected_context.get('expected_rprs', np.nan), 'expected_rprs_unc': expected_context.get('expected_rprs_unc', np.nan), + 'tmid_deviation_days': np.nan, + 'tmid_deviation_minutes': np.nan, + 'tmid_deviation_threshold_minutes': np.nan, 'tmid_deviation_sigma': np.nan, 'rprs_deviation_sigma': np.nan, 'tmid_deviation_score': np.nan, @@ -1263,6 +1307,13 @@ def evaluate_transit_detection_qc(fit): enabled=use_deviation_from_expected_transit_in_qc, ) summary.update({ + 'expected_tmid': deviation_summary.get('expected_tmid', np.nan), + 'expected_tmid_unc': deviation_summary.get('expected_tmid_unc', np.nan), + 'expected_tmid_unc_minutes': deviation_summary.get('expected_tmid_unc_minutes', np.nan), + 'fitted_tmid': deviation_summary.get('fitted_tmid', np.nan), + 'tmid_deviation_days': deviation_summary.get('tmid_deviation_days', np.nan), + 'tmid_deviation_minutes': deviation_summary.get('tmid_deviation_minutes', np.nan), + 'tmid_deviation_threshold_minutes': deviation_summary.get('tmid_deviation_threshold_minutes', np.nan), 'tmid_deviation_sigma': deviation_summary.get('tmid_deviation_sigma', np.nan), 'rprs_deviation_sigma': deviation_summary.get('rprs_deviation_sigma', np.nan), 'tmid_deviation_score': deviation_summary.get('tmid_deviation_score', np.nan), @@ -1339,12 +1390,19 @@ def evaluate_transit_detection_qc(fit): notes.extend(deviation_summary.get('notes', [])) if deviation_summary.get('failed'): status = 'fail' - notes.append( - "The fit deviates too far from the expected published Tmid and/or Rp/R* values." - ) - failure_reasons.append( - "the fit deviates too far from the expected published Tmid and/or Rp/R* values" - ) + detailed_reasons = [ + reason for reason in deviation_summary.get('failure_reasons', []) + if isinstance(reason, str) and reason.strip() + ] + if detailed_reasons: + failure_reasons.extend(detailed_reasons) + else: + notes.append( + "The fit deviates too far from the expected published Tmid and/or Rp/R* values." + ) + failure_reasons.append( + "the fit deviates too far from the expected published Tmid and/or Rp/R* values" + ) ktmf_metric, ktmf_contributions = compute_transit_qc_ktmf(summary) summary['ktmf_metric'] = ktmf_metric @@ -1401,6 +1459,13 @@ def annotate_transit_detection_qc(fit, summary=None): fit.transit_qc_residual_scatter = summary.get('residual_scatter') fit.transit_qc_deviation_from_expected_value = summary.get('deviation_from_expected_value') fit.transit_qc_deviation_sigma_threshold = summary.get('deviation_sigma_threshold') + fit.transit_qc_expected_tmid_value = summary.get('expected_tmid') + fit.transit_qc_expected_tmid_unc = summary.get('expected_tmid_unc') + fit.transit_qc_expected_tmid_unc_minutes = summary.get('expected_tmid_unc_minutes') + fit.transit_qc_fitted_tmid = summary.get('fitted_tmid') + fit.transit_qc_tmid_deviation_days = summary.get('tmid_deviation_days') + fit.transit_qc_tmid_deviation_minutes = summary.get('tmid_deviation_minutes') + fit.transit_qc_tmid_deviation_threshold_minutes = summary.get('tmid_deviation_threshold_minutes') fit.transit_qc_tmid_deviation_sigma = summary.get('tmid_deviation_sigma') fit.transit_qc_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') fit.transit_qc_expected_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') @@ -13833,6 +13898,15 @@ def main(): log_info( f" Expected-value Tmid sigma: {transit_qc['tmid_deviation_sigma']:.2f}" ) + if np.isfinite(transit_qc.get('tmid_deviation_minutes', np.nan)): + log_info( + f" Expected-value Tmid offset: {transit_qc['tmid_deviation_minutes']:.2f} minutes" + ) + if np.isfinite(transit_qc.get('tmid_deviation_threshold_minutes', np.nan)): + log_info( + " Expected-value Tmid QC window: " + f"{transit_qc['tmid_deviation_threshold_minutes']:.2f} minutes" + ) if np.isfinite(transit_qc.get('rprs_deviation_sigma', np.nan)): log_info( f" Expected-value Rp/R* sigma: {transit_qc['rprs_deviation_sigma']:.2f}" diff --git a/exotic/output_files.py b/exotic/output_files.py index b396e53a..ba96b2ef 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -213,6 +213,9 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor qc_eebls_depth_snr = transit_qc.get('eebls_depth_snr', np.nan) qc_deviation_metric = transit_qc.get('deviation_from_expected_value', np.nan) qc_tmid_deviation_sigma = transit_qc.get('tmid_deviation_sigma', np.nan) + qc_tmid_deviation_minutes = transit_qc.get('tmid_deviation_minutes', np.nan) + qc_tmid_threshold_minutes = transit_qc.get('tmid_deviation_threshold_minutes', np.nan) + qc_expected_tmid_unc_minutes = transit_qc.get('expected_tmid_unc_minutes', np.nan) qc_rprs_deviation_sigma = transit_qc.get('rprs_deviation_sigma', np.nan) qc_sigma_threshold = transit_qc.get('deviation_sigma_threshold', np.nan) qc_ktmf = transit_qc.get('ktmf_metric', np.nan) @@ -238,6 +241,12 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num["Expected-value QC threshold"] = f"{qc_sigma_threshold:.2f} sigma" if np.isfinite(qc_tmid_deviation_sigma): params_num["Expected-value Tmid deviation"] = f"{qc_tmid_deviation_sigma:.2f} sigma" + if np.isfinite(qc_tmid_deviation_minutes): + params_num["Expected-value Tmid offset"] = f"{qc_tmid_deviation_minutes:.2f} minutes" + if np.isfinite(qc_expected_tmid_unc_minutes): + params_num["Expected-value Tmid uncertainty"] = f"{qc_expected_tmid_unc_minutes:.2f} minutes" + if np.isfinite(qc_tmid_threshold_minutes): + params_num["Expected-value Tmid QC window"] = f"{qc_tmid_threshold_minutes:.2f} minutes" if np.isfinite(qc_rprs_deviation_sigma): params_num["Expected-value Rp/R* deviation"] = f"{qc_rprs_deviation_sigma:.2f} sigma" if np.isfinite(qc_ktmf): diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 1cd235d3..807bc8b2 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -2496,7 +2496,7 @@ def test_evaluate_transit_detection_qc_rejects_large_expected_value_deviation(): model=transit_model, airmass=np.ones(data.shape[0], dtype=float), airmass_fit_skipped=True, - parameters={"rprs": 0.18, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + parameters={"rprs": 0.18, "tmid": 0.506, "inc": 89.0, "a2": 0.0}, errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, duration_expected=5.0, @@ -2513,6 +2513,9 @@ def test_evaluate_transit_detection_qc_rejects_large_expected_value_deviation(): assert summary["computed"] is True assert summary["status"] == "fail" + assert summary["tmid_deviation_sigma"] == pytest.approx(6.0) + assert summary["tmid_deviation_minutes"] == pytest.approx(8.64) + assert summary["tmid_deviation_threshold_minutes"] == pytest.approx(7.2) assert summary["rprs_deviation_sigma"] == pytest.approx(8.0) assert summary["deviation_from_expected_value"] == pytest.approx(0.0) assert summary["ktmf_metric"] <= 5.0 @@ -2585,7 +2588,7 @@ def test_evaluate_transit_detection_qc_failure_summary_reflects_expected_value_r model=transit_model, airmass=np.ones(data.shape[0], dtype=float), airmass_fit_skipped=True, - parameters={"rprs": 0.18, "tmid": 0.5, "inc": 89.0, "a2": 0.0, "per": 2.0}, + parameters={"rprs": 0.10, "tmid": 0.528, "inc": 89.0, "a2": 0.0, "per": 2.0}, errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, prior={"per": 2.0, "tmid": 0.5}, @@ -2602,8 +2605,10 @@ def test_evaluate_transit_detection_qc_failure_summary_reflects_expected_value_r summary = evaluate_transit_detection_qc(fit) assert summary["status"] == "fail" + assert summary["tmid_deviation_minutes"] == pytest.approx(40.32) assert "QC rejected the fit because" in summary["summary"] - assert "expected published Tmid and/or Rp/R*" in summary["summary"] + assert "Tmid of the fit is 40.32 minutes away from the ephemeris Tmid" in summary["summary"] + assert "7.20 minutes" in summary["summary"] assert "not supported strongly enough against a flat/null model" not in summary["summary"] diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 32b7a959..720f95dc 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -295,6 +295,9 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): "residual_scatter": 0.0032, "deviation_from_expected_value": 0.91, "tmid_deviation_sigma": 1.1, + "tmid_deviation_minutes": 3.2, + "tmid_deviation_threshold_minutes": 14.4, + "expected_tmid_unc_minutes": 2.88, "rprs_deviation_sigma": 0.8, "deviation_sigma_threshold": 5.0, "ktmf_metric": 4.63, @@ -337,6 +340,9 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): assert "Delta BIC=18.40" in output_text assert "Residual scatter around full model fit" in output_text assert "Deviation From Expected Value" in output_text + assert "Expected-value Tmid offset" in output_text + assert "3.20 minutes" in output_text + assert "Expected-value Tmid QC window" in output_text assert "KTMF" in output_text assert "KTMF contribution 1" in output_text From 6a9554868e072f159fdf275129706bb8560f114b Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 29 Apr 2026 10:17:28 +1000 Subject: [PATCH 029/116] Preserve expanded bounds through baseline refits --- exotic/api/elca.py | 23 ++++- exotic/api/plotting.py | 64 +++++++++---- exotic/exotic.py | 90 +++++++++++++++++- tests/test_elca_baseline.py | 12 +++ tests/test_exotic_proper_motion.py | 145 +++++++++++++++++++++++++++++ 5 files changed, 308 insertions(+), 26 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index d5567f7c..46f3a656 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -1147,6 +1147,26 @@ def _get_triangle_plot_payload(self): 'mask_errors': mask_errors, } + def _triangle_contour_levels(self, chi2, mask1, mask2, mask3): + raw_levels = np.array([ + np.percentile(chi2[mask1], 95), + np.percentile(chi2[mask2], 95), + np.percentile(chi2[mask3], 95), + ], dtype=float) + finite_levels = np.sort(raw_levels[np.isfinite(raw_levels)]) + if finite_levels.size == 0: + return [] + + unique_levels = [] + min_spacing = max( + np.finfo(float).eps, + np.nanmax(np.abs(finite_levels)) * 1e-12, + ) + for level in finite_levels: + if not unique_levels or level > unique_levels[-1] + min_spacing: + unique_levels.append(float(level)) + return unique_levels + def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs=None, label_kwargs=None): if not hasattr(fig, 'axes'): return @@ -1706,8 +1726,7 @@ def plot_triangle(self): bins=int(np.sqrt(payload['display_points'].shape[0])), range=payload['ranges'], plot_contours=True, - levels=[np.percentile(chi2[mask1], 95), np.percentile(chi2[mask2], 95), - np.percentile(chi2[mask3], 95)], + levels=self._triangle_contour_levels(chi2, mask1, mask2, mask3), plot_density=False, titles=payload['titles'], data_kwargs={ diff --git a/exotic/api/plotting.py b/exotic/api/plotting.py index a4525ec9..3c346492 100644 --- a/exotic/api/plotting.py +++ b/exotic/api/plotting.py @@ -494,26 +494,50 @@ def hist2d(x, y, bins=20, range=None, levels=[2], data_kwargs["alpha"] = data_kwargs.get("alpha", 0.2) ax.scatter(x, y, marker="o", zorder=-1, rasterized=True, **data_kwargs) - # Plot the contour edge colors. - if plot_contours: - if contour_kwargs is None: - contour_kwargs = dict() - - # mask data in range + chi2 - maskx = (x > range[0][0]) & (x < range[0][1]) - masky = (y > range[1][0]) & (y < range[1][1]) - mask = maskx & masky & (data_kwargs['c'] < data_kwargs['vmax']*1.2) - - try: # contour - # approx posterior + smooth - xg, yg = np.meshgrid( np.linspace(x[mask].min(),x[mask].max(),256), np.linspace(y[mask].min(),y[mask].max(),256) ) - cg = griddata(np.vstack([x[mask],y[mask]]).T, data_kwargs['c'][mask], (xg,yg), method='nearest', rescale=True) - scg = gaussian_filter(cg,sigma=15) - - ax.contour(xg, yg, scg*np.nanmin(cg)/np.nanmin(scg), np.sort(levels), **contour_kwargs, vmin=data_kwargs['vmin'], vmax=data_kwargs['vmax']) - except Exception as err: - print(err) - print("contour plotting failed") + # Plot the contour edge colors. + if plot_contours: + if contour_kwargs is None: + contour_kwargs = dict() + + contour_levels = np.asarray(levels, dtype=float) + contour_levels = np.unique(contour_levels[np.isfinite(contour_levels)]) + if contour_levels.size == 0: + ax.set_xlim(range[0]) + ax.set_ylim(range[1]) + return + + # mask data in range + chi2 + maskx = (x > range[0][0]) & (x < range[0][1]) + masky = (y > range[1][0]) & (y < range[1][1]) + mask = maskx & masky & (data_kwargs['c'] < data_kwargs['vmax']*1.2) + + try: # contour + if np.count_nonzero(mask) < 3: + raise ValueError("not enough in-range samples for contour plotting") + # approx posterior + smooth + xg, yg = np.meshgrid( np.linspace(x[mask].min(),x[mask].max(),256), np.linspace(y[mask].min(),y[mask].max(),256) ) + cg = griddata(np.vstack([x[mask],y[mask]]).T, data_kwargs['c'][mask], (xg,yg), method='nearest', rescale=True) + scg = gaussian_filter(cg,sigma=15) + cg_min = np.nanmin(cg) + scg_min = np.nanmin(scg) + if not np.isfinite(cg_min) or not np.isfinite(scg_min) or np.isclose(scg_min, 0): + raise ValueError("degenerate contour surface") + + contour_surface = scg * cg_min / scg_min + surface_min = np.nanmin(contour_surface) + surface_max = np.nanmax(contour_surface) + if not np.isfinite(surface_min) or not np.isfinite(surface_max) or np.isclose(surface_min, surface_max): + raise ValueError("degenerate contour surface") + contour_levels = contour_levels[ + (contour_levels > surface_min) & (contour_levels < surface_max) + ] + if contour_levels.size == 0: + raise ValueError("no contour levels fall within the plotted surface") + + ax.contour(xg, yg, contour_surface, contour_levels, **contour_kwargs, vmin=data_kwargs['vmin'], vmax=data_kwargs['vmax']) + except Exception as err: + print(err) + print("contour plotting failed") ax.set_xlim(range[0]) ax.set_ylim(range[1]) diff --git a/exotic/exotic.py b/exotic/exotic.py index 007e4808..c8725839 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -63,6 +63,7 @@ import json import hashlib import os +import shutil import sys import threading import traceback @@ -1542,6 +1543,39 @@ def comparison_candidate_output_dir(save_dir, comp_index): return Path(save_dir) / f"comp{comp_index + 1}" +def triangle_plot_output_path(save_dir, planet_name, observation_date): + return Path(save_dir) / "temp" / f"Triangle_{planet_name}_{observation_date}.png" + + +def save_final_triangle_plot(fit, save_dir, planet_name, observation_date, source_dir=None): + output_path = triangle_plot_output_path(save_dir, planet_name, observation_date) + output_path.parent.mkdir(parents=True, exist_ok=True) + + if source_dir is not None: + source_path = triangle_plot_output_path(source_dir, planet_name, observation_date) + try: + same_path = source_path.resolve() == output_path.resolve() + except OSError: + same_path = False + + if source_path.exists(): + if same_path: + return output_path + try: + shutil.copy2(source_path, output_path) + return output_path + except OSError: + pass + + fig = fit.plot_triangle() + fig.savefig(output_path) + try: + plt.close(fig) + except TypeError: + pass + return output_path + + def estimate_transit_duration_samples_from_fit(fit, sample_count=1000, grid_size=1000): if fit is None or not hasattr(fit, 'parameters') or not hasattr(fit, 'errors'): return None, np.array([], dtype=float) @@ -2279,6 +2313,40 @@ def clone_lightcurve_bounds(bounds): } +def annotate_posterior_refit_final_bounds(fit, bounds): + if fit is None: + return + fit.posterior_refit_final_bounds = clone_lightcurve_bounds(bounds) + + +def get_posterior_refit_final_bounds(fit, fallback_bounds): + effective_bounds = clone_lightcurve_bounds(fallback_bounds) + fit_bounds = getattr(fit, 'posterior_refit_final_bounds', None) + if not isinstance(fit_bounds, dict): + fit_bounds = {} + for key in ('rprs', 'ars'): + refit_bounds = getattr(fit, f'{key}_posterior_refit_bounds', None) + if refit_bounds is not None: + fit_bounds[key] = refit_bounds + + for key, value in fit_bounds.items(): + if not isinstance(value, (list, tuple, np.ndarray)): + continue + try: + lower_bound, upper_bound = [ + float(bound) for bound in np.asarray(value, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + continue + if ( + np.isfinite(lower_bound) + and np.isfinite(upper_bound) + and lower_bound < upper_bound + ): + effective_bounds[key] = [lower_bound, upper_bound] + return effective_bounds + + def sanitize_parameter_search_bounds(bounds, key, minimum_bound, maximum_bound=None, fallback_maximum=None): sanitized = clone_lightcurve_bounds(bounds) if key not in sanitized: @@ -2618,6 +2686,7 @@ def build_fit(local_prior, local_bounds): fit = build_fit(current_prior, current_bounds) final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + annotate_posterior_refit_final_bounds(fit, current_bounds) for config in retry_configs: key = config['key'] label = config['label'] @@ -4710,13 +4779,14 @@ def fit_final_lightcurve_with_oot_baseline_detrending( eebls_search_summary=eebls_search_summary, ) + effective_bounds = get_posterior_refit_final_bounds(fit, bounds) prefit_plan = build_final_fit_prefit_refinement_plan( times, flux_values, flux_errors, airmass, prior, - bounds, + effective_bounds, fit, jd_times=jd_times, baseline_duration_multiplier=baseline_duration_multiplier, @@ -4758,6 +4828,8 @@ def fit_final_lightcurve_with_oot_baseline_detrending( eebls_search_summary=eebls_search_summary, ) + working_bounds = get_posterior_refit_final_bounds(fit, working_bounds) + annotate_final_fit_prefit_refinement( fit, prefit_plan.get('applied', False), @@ -13158,6 +13230,7 @@ def main(): selected_fit_good_airmass=selected_attempt.get('good_airmass'), selected_fit_duration_samples=selected_attempt.get('duration_samples'), selected_fit_data_highres=selected_attempt.get('data_highres'), + selected_fit_final_output_dir=selected_attempt.get('final_output_dir'), calibration_field_score=comparison_calibration['field_score'], selection_basis=selection_basis, selection_metric=comparison_fit_search.get('selection_metric', 'ktmf'), @@ -13944,9 +14017,18 @@ def main(): # SAVE DATA ########## - fig = myfit.plot_triangle() - fig.savefig(Path(exotic_infoDict['save']) / "temp" / - f"Triangle_{pDict['pName']}_{exotic_infoDict['date']}.png") + selected_triangle_source_dir = ( + photometry_info.get('selected_fit_final_output_dir') + if reuse_selected_final_model + else None + ) + save_final_triangle_plot( + myfit, + exotic_infoDict['save'], + pDict['pName'], + exotic_infoDict['date'], + source_dir=selected_triangle_source_dir, + ) if vsp_params: AIDoutput_files = AIDOutputFiles(myfit, pDict, exotic_infoDict, auid, chart_id, vsp_params) diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 1417bdb3..c0ec9cca 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -927,6 +927,18 @@ def fake_corner(*args, **kwargs): assert captured["label_kwargs"]["labelpad"] == 10 +def test_triangle_contour_levels_drop_duplicate_chi2_percentiles(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + chi2 = np.full(12, 42.0) + mask = np.ones(chi2.size, dtype=bool) + + levels = fit._triangle_contour_levels(chi2, mask, mask, mask) + + assert levels == [pytest.approx(42.0)] + + def test_triangle_payload_tracks_left_and_right_geometry_branches_for_inclination(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 807bc8b2..e5ad4631 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -141,6 +141,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: resolve_frame_aperture_radii, robust_flux_floor_mask, robust_target_reference_flux_mask, + save_final_triangle_plot, save_selected_photometry_debug_series, should_keep_header_wcs_alignment, sigma_clip, @@ -161,6 +162,58 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: ) +def test_save_final_triangle_plot_copies_selected_candidate_artifact(tmp_path): + class DummyFit: + def plot_triangle(self): + raise AssertionError("final plot should be copied from the selected candidate") + + planet_name = "TOI-1728 b" + observation_date = "2024-12-14" + source_dir = tmp_path / "comp6" + final_dir = tmp_path / "final" + source_temp = source_dir / "temp" + source_temp.mkdir(parents=True) + source_plot = source_temp / f"Triangle_{planet_name}_{observation_date}.png" + source_plot.write_bytes(b"selected-comp-6") + + output_path = save_final_triangle_plot( + DummyFit(), + final_dir, + planet_name, + observation_date, + source_dir=source_dir, + ) + + assert output_path == final_dir / "temp" / source_plot.name + assert output_path.read_bytes() == b"selected-comp-6" + + +def test_save_final_triangle_plot_regenerates_when_selected_artifact_missing(tmp_path): + class DummyFigure: + def savefig(self, path): + Path(path).write_bytes(b"regenerated") + + class DummyFit: + def __init__(self): + self.called = False + + def plot_triangle(self): + self.called = True + return DummyFigure() + + fit = DummyFit() + output_path = save_final_triangle_plot( + fit, + tmp_path / "final", + "TOI-1728 b", + "2024-12-14", + source_dir=tmp_path / "missing-comp", + ) + + assert fit.called is True + assert output_path.read_bytes() == b"regenerated" + + def test_update_coordinates_handles_non_numeric_proper_motion_values(): info = { "ra": 10.0, @@ -1806,6 +1859,98 @@ def fake_lc_fitter( assert fit.rprs_posterior_refit_bounds == pytest.approx([0.108, 0.208]) +def test_fit_final_lightcurve_carries_retry_bounds_into_oot_baseline_refit(monkeypatch): + import exotic.exotic as exotic_module + + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + flux = (1.0 + 0.02 * times) * np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + fluxerr = np.full_like(times, 0.01) + airmass = np.ones_like(times) + prior = {"rprs": 0.1, "tmid": 0.0, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.125], "tmid": [-0.1, 0.1], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + transit_model = np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + diagnostics = [ + { + "clipped": True, + "edge": "upper", + "mode": 0.158, + "std": 0.006, + "bounds": [0.128, 0.188], + "reason": "posterior peaks against the upper search bound.", + }, + { + "clipped": False, + "edge": None, + "mode": 0.159, + "std": 0.005, + "bounds": [0.108, 0.208], + "reason": "posterior support is comfortably inside the sampled bounds.", + }, + { + "clipped": False, + "edge": None, + "mode": 0.160, + "std": 0.005, + "bounds": [0.108, 0.208], + "reason": "posterior support is comfortably inside the sampled bounds.", + }, + ] + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + call_diagnostics = diagnostics[min(call_index, len(diagnostics) - 1)] + captured["calls"].append({ + "flux": np.array(call_flux, dtype=float), + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + fit = types.SimpleNamespace( + transit=transit_model, + parameters={"tmid": 0.0, "rprs": call_diagnostics["mode"], "inc": 89.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: dict(call_diagnostics) if key == "rprs" else None + ) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=True, + ) + + assert len(captured["calls"]) == 3 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.125]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.108, 0.208]) + assert captured["calls"][2]["bounds"]["rprs"] == pytest.approx([0.108, 0.208]) + assert np.allclose(captured["calls"][2]["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert fit.oot_baseline_detrending_applied is True + + def test_fit_final_lightcurve_prefit_refinement_trims_baseline_and_recenters_tmid(monkeypatch): import exotic.exotic as exotic_module From 65b6e8d9f96070ba65aeed00b622b0b1d075ec8a Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 29 Apr 2026 19:16:18 +1000 Subject: [PATCH 030/116] doing real good. --- exotic/__init__.py | 40 +++- exotic/api/elca.py | 143 +++++++++++- exotic/api/ephemeris.py | 32 ++- exotic/api/nested_linear_fitter.py | 32 ++- exotic/api/ultranest_utils.py | 272 ++++++++++++++++++++--- exotic/exotic.py | 345 ++++++++++++++++++++++++++--- exotic/inputs.py | 16 ++ inits.json | 2 + requirements.txt | 3 +- setup.cfg | 2 +- tests/test_elca_baseline.py | 104 +++++++++ tests/test_exotic_proper_motion.py | 322 ++++++++++++++++++++++++++- tests/test_exotic_rprs_retry.py | 75 ++++++- tests/test_inputs.py | 60 +++++ tests/test_ultranest_utils.py | 153 +++++++++++++ 15 files changed, 1512 insertions(+), 89 deletions(-) diff --git a/exotic/__init__.py b/exotic/__init__.py index 1bf9216e..1861dc23 100644 --- a/exotic/__init__.py +++ b/exotic/__init__.py @@ -44,6 +44,7 @@ import importlib_metadata as metadata # Python <3.8 from pathlib import Path +from importlib import import_module import sys # Extend PYTHONPATH to include current directory and parent directory @@ -71,4 +72,41 @@ __version__ = version_read("exotic.py") except IOError: # Unable to read from exotic script - __version__ = "unknown" \ No newline at end of file + __version__ = "unknown" + + +def _load_runtime_callable(name): + """Load CLI entry points lazily without importing the full runtime at package import.""" + current_module = sys.modules.get(__name__) + module_names = [f"{__name__}.exotic", "exotic.exotic"] + + seen = set() + for module_name in module_names: + if module_name in seen: + continue + seen.add(module_name) + + try: + module = import_module(module_name) + except ModuleNotFoundError as exc: + if exc.name == module_name: + continue + raise + if module is current_module: + continue + runtime_callable = getattr(module, name, None) + if runtime_callable is not None: + return runtime_callable + + raise ImportError(f"cannot import name '{name}' from '{__name__}'") + + +def main(*args, **kwargs): + return _load_runtime_callable("main")(*args, **kwargs) + + +def cli(*args, **kwargs): + return _load_runtime_callable("cli")(*args, **kwargs) + + +__all__ = ["__version__", "main", "cli"] diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 46f3a656..e8a27813 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -629,6 +629,91 @@ def _summarize_derived_parameter(self, samples, point_estimate): upper = float(np.nanpercentile(samples, 84)) return center, std, [lower - center, upper - center] + def _get_ultranest_weighted_sample_arrays(self): + try: + weighted_samples = self.results['weighted_samples'] + points = np.asarray(weighted_samples['points'], dtype=float) + logl = np.asarray(weighted_samples['logl'], dtype=float) + except Exception: + return None, None + + if points.ndim != 2 or points.shape[0] == 0: + return None, None + if logl.shape[0] != points.shape[0]: + return None, None + return points, logl + + def _loglike_neighborhood_uncertainty(self, parameter_index, center, minimum_count=8): + points, logl = self._get_ultranest_weighted_sample_arrays() + if points is None or parameter_index >= points.shape[1]: + return None + + values = np.asarray(points[:, parameter_index], dtype=float) + finite = np.isfinite(values) & np.isfinite(logl) + if np.count_nonzero(finite) < 2: + return None + + finite_values = values[finite] + finite_logl = logl[finite] + max_logl = float(np.nanmax(finite_logl)) + if not np.isfinite(max_logl): + return None + + selected_values = None + selected_delta = np.inf + for delta_chi2 in (1.0, 4.0, 9.0, 16.0, 25.0, np.inf): + if np.isfinite(delta_chi2): + mask = 2.0 * (max_logl - finite_logl) <= delta_chi2 + else: + mask = np.ones(finite_logl.shape, dtype=bool) + if np.count_nonzero(mask) >= minimum_count or delta_chi2 == np.inf: + selected_values = finite_values[mask] + selected_delta = delta_chi2 + break + + if selected_values is None or selected_values.size < 2: + return None + + lower, upper = np.nanpercentile(selected_values, [15.8655, 84.1345]) + std = float(np.nanstd(selected_values)) + half_width = float(0.5 * (upper - lower)) + candidates = [value for value in (std, half_width) if np.isfinite(value) and value > 0] + if not candidates: + return None + + error = float(max(candidates)) + return { + 'error': error, + 'quantiles': [float(lower), float(upper)], + 'sample_count': int(selected_values.size), + 'delta_chi2': float(selected_delta), + } + + def _ultranest_error_needs_sample_fallback(self, parameter_index, center, reported_error): + points, _ = self._get_ultranest_weighted_sample_arrays() + if points is None or parameter_index >= points.shape[1]: + return False + + values = np.asarray(points[:, parameter_index], dtype=float) + finite_values = values[np.isfinite(values)] + if finite_values.size < 2: + return False + + sample_scale = float(np.nanstd(finite_values)) + if not np.isfinite(sample_scale) or sample_scale <= 0: + return False + + try: + reported_error = float(reported_error) + except (TypeError, ValueError): + return True + + if not np.isfinite(reported_error) or reported_error <= 0: + return True + + absolute_floor = max(abs(float(center)) * 1e-12, np.finfo(float).eps) + return reported_error <= absolute_floor or reported_error < sample_scale * 1e-6 + def _get_plot_range(self, key): sample_parameters = getattr(self, 'sample_parameters', {}) sample_errors = getattr(self, 'sample_errors', {}) @@ -653,6 +738,37 @@ def _get_plot_range(self, key): return [lower, upper] + def _expand_plot_range_for_sample_cloud(self, key, plot_range, sample_values, center): + sample_values = np.asarray(sample_values, dtype=float) + finite_values = sample_values[np.isfinite(sample_values)] + if finite_values.size < 2: + return plot_range + + lower, upper = [float(value) for value in plot_range] + in_range = (finite_values >= lower) & (finite_values <= upper) + minimum_in_range = min(finite_values.size, max(8, int(0.05 * finite_values.size))) + if np.count_nonzero(in_range) >= minimum_in_range: + return plot_range + + q_lower, q_upper = np.nanpercentile(finite_values, [0.5, 99.5]) + new_lower = min(float(q_lower), float(center)) + new_upper = max(float(q_upper), float(center)) + padding = 0.05 * (new_upper - new_lower) + if not np.isfinite(padding) or padding <= 0: + padding = max(abs(float(center)) * 1e-6, 1e-6) + new_lower -= padding + new_upper += padding + + sample_bounds = getattr(self, 'sample_bounds', self.bounds) + if key in sample_bounds: + bound_lower, bound_upper = sample_bounds[key] + new_lower = max(new_lower, float(bound_lower)) + new_upper = min(new_upper, float(bound_upper)) + + if not np.isfinite(new_lower) or not np.isfinite(new_upper) or new_lower >= new_upper: + return plot_range + return [float(new_lower), float(new_upper)] + def _get_triangle_plot_samples(self): if self.ns_type == 'ultranest': points = np.asarray(self.results['weighted_samples']['points'], dtype=float) @@ -1112,12 +1228,19 @@ def _get_triangle_plot_payload(self): mask_centers = [] mask_errors = [] - for key in sampled_keys: + for i, key in enumerate(sampled_keys): center = sample_parameters.get(key, self.parameters.get(key, 0.0)) error = sample_errors.get(key, self.errors.get(key, 0.0)) label = flabels.get(key, key) title = f"{center:.5f} +- {error:.5f}" plot_range = self._get_plot_range(key) + if sample_points.ndim == 2 and i < sample_points.shape[1]: + plot_range = self._expand_plot_range_for_sample_cloud( + key, + plot_range, + sample_points[:, i], + center, + ) if display_spec is not None and key == display_spec['key']: label = display_spec['label'] @@ -1451,7 +1574,7 @@ def prior_transform(upars): try: self.ns_type = 'ultranest' - test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform) + test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform, vectorized=True) self.results = run_reactive_sampler( test, @@ -1461,13 +1584,25 @@ def prior_transform(upars): ml_point = self.results['maximum_likelihood']['point'] self.sample_bounds = self._get_sample_bounds(bound_keys, physical_from_sample_point(ml_point)) + self.ultranest_error_fallbacks = {} for i, key in enumerate(sampled_keys): self.sample_parameters[key] = ml_point[i] - self.sample_errors[key] = self.results['posterior']['stdev'][i] - self.sample_quantiles[key] = [ + reported_error = self.results['posterior']['stdev'][i] + reported_quantiles = [ self.results['posterior']['errlo'][i], self.results['posterior']['errup'][i]] + if self._ultranest_error_needs_sample_fallback(i, ml_point[i], reported_error): + fallback = self._loglike_neighborhood_uncertainty(i, ml_point[i]) + else: + fallback = None + if fallback is not None: + self.sample_errors[key] = fallback['error'] + self.sample_quantiles[key] = fallback['quantiles'] + self.ultranest_error_fallbacks[key] = fallback + else: + self.sample_errors[key] = reported_error + self.sample_quantiles[key] = reported_quantiles physical_ml = physical_from_sample_point(ml_point) self.parameters.update(physical_ml) diff --git a/exotic/api/ephemeris.py b/exotic/api/ephemeris.py index 312e2114..523a96bc 100644 --- a/exotic/api/ephemeris.py +++ b/exotic/api/ephemeris.py @@ -109,17 +109,23 @@ def fit_nested(self): def loglike(pars): # chi-squared - model = pars[0] * self.epochs + pars[1] - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + pars_array = np.asarray(pars, dtype=float) + if pars_array.ndim == 2: + model = pars_array[:, 0, None] * self.epochs[None, :] + pars_array[:, 1, None] + return -0.5 * np.sum(((data[None, :] - model) / dataerr[None, :]) ** 2, axis=1) + model = pars_array[0] * self.epochs + pars_array[1] + return -0.5 * np.sum(((data - model) / dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform, vectorized=True) self.results = run_reactive_sampler( sampler, - run_kwargs={"max_ncalls": int(4e5), "min_num_live_points": 420}, + run_kwargs={"max_ncalls": int(4e5)}, verbose=self.verbose, ) # alloc data for best fit + error @@ -698,17 +704,27 @@ def fit_nested(self): def loglike(pars): # chi-squared # tmid = T0 + N*P + 0.5*dPdN*N**2 (eq 3 from paper) - model = pars[0] * self.epochs + pars[1] + 0.5 * pars[2] * self.epochs ** 2 - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + pars_array = np.asarray(pars, dtype=float) + if pars_array.ndim == 2: + model = ( + pars_array[:, 0, None] * self.epochs[None, :] + + pars_array[:, 1, None] + + 0.5 * pars_array[:, 2, None] * self.epochs[None, :] ** 2 + ) + return -0.5 * np.sum(((data[None, :] - model) / dataerr[None, :]) ** 2, axis=1) + model = pars_array[0] * self.epochs + pars_array[1] + 0.5 * pars_array[2] * self.epochs ** 2 + return -0.5 * np.sum(((data - model) / dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform, vectorized=True) self.results = run_reactive_sampler( sampler, - run_kwargs={"max_ncalls": int(4e5), "min_num_live_points": 420}, + run_kwargs={"max_ncalls": int(4e5)}, verbose=self.verbose, ) # alloc data for best fit + error diff --git a/exotic/api/nested_linear_fitter.py b/exotic/api/nested_linear_fitter.py index 2eeb6abb..81baecaf 100644 --- a/exotic/api/nested_linear_fitter.py +++ b/exotic/api/nested_linear_fitter.py @@ -102,17 +102,23 @@ def fit_nested(self): def loglike(pars): # chi-squared - model = pars[0] * self.epochs + pars[1] - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + pars_array = np.asarray(pars, dtype=float) + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + if pars_array.ndim == 2: + model = pars_array[:, 0, None] * self.epochs[None, :] + pars_array[:, 1, None] + return -0.5 * np.sum(((data[None, :] - model) / dataerr[None, :]) ** 2, axis=1) + model = pars_array[0] * self.epochs + pars_array[1] + return -0.5 * np.sum(((data - model) / dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform, vectorized=True) self.results = run_reactive_sampler( sampler, - run_kwargs={"max_ncalls": int(4e5), "min_num_live_points": 420}, + run_kwargs={"max_ncalls": int(4e5)}, verbose=self.verbose, ) # alloc data for best fit + error @@ -670,17 +676,27 @@ def fit_nested(self): def loglike(pars): # chi-squared # tmid = t0 + N*P + 0.5*dPdN*N**2 (eq 3 from paper) - model = pars[0] * self.epochs + pars[1] + 0.5 * pars[2] * self.epochs ** 2 - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + pars_array = np.asarray(pars, dtype=float) + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + if pars_array.ndim == 2: + model = ( + pars_array[:, 0, None] * self.epochs[None, :] + + pars_array[:, 1, None] + + 0.5 * pars_array[:, 2, None] * self.epochs[None, :] ** 2 + ) + return -0.5 * np.sum(((data[None, :] - model) / dataerr[None, :]) ** 2, axis=1) + model = pars_array[0] * self.epochs + pars_array[1] + 0.5 * pars_array[2] * self.epochs ** 2 + return -0.5 * np.sum(((data - model) / dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform, vectorized=True) self.results = run_reactive_sampler( sampler, - run_kwargs={"max_ncalls": int(4e5), "min_num_live_points": 420}, + run_kwargs={"max_ncalls": int(4e5)}, verbose=self.verbose, ) # alloc data for best fit + error diff --git a/exotic/api/ultranest_utils.py b/exotic/api/ultranest_utils.py index 75df5b61..fb91b4af 100644 --- a/exotic/api/ultranest_utils.py +++ b/exotic/api/ultranest_utils.py @@ -1,19 +1,222 @@ import logging import math +import multiprocessing import os import sys import time +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +import numpy as np + _TRUTHY = {"1", "true", "yes", "on"} +_FALSEY = {"0", "false", "no", "off", "n"} DEFAULT_PROGRESS_INTERVAL_SECONDS = 10.0 +DEFAULT_MIN_NUM_LIVE_POINTS = 200 +DEFAULT_RUN_KWARGS = { + "min_num_live_points": DEFAULT_MIN_NUM_LIVE_POINTS, + "min_ess": 200, + "dlogz": 1.0, + "dKL": 1.0, + "frac_remain": 0.05, + "max_num_improvement_loops": 1, +} +MIN_LIVE_POINTS_ENV_KEYS = ( + "EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", + "EXOTIC_ULTRANEST_MIN_LIVE_POINTS", +) +MPI_SIZE_ENV_KEYS = ( + "OMPI_COMM_WORLD_SIZE", + "PMI_SIZE", + "PMIX_SIZE", + "MV2_COMM_WORLD_SIZE", +) +MPI_RANK_ENV_KEYS = ( + "OMPI_COMM_WORLD_RANK", + "PMI_RANK", + "PMIX_RANK", + "MV2_COMM_WORLD_RANK", +) +ULTRANEST_WORKER_ENV_KEYS = ( + "EXOTIC_ULTRANEST_WORKERS", + "NEXTASTRO_EXOTIC_ULTRANEST_WORKERS", +) +ULTRANEST_WORKER_BACKEND_ENV = "EXOTIC_ULTRANEST_WORKER_BACKEND" +_PROCESS_LOGLIKE = None def _is_enabled(value): return str(value).strip().lower() in _TRUTHY +def _is_disabled(value): + return str(value).strip().lower() in _FALSEY + + +def _coerce_positive_int(value, default=None): + try: + parsed = int(float(str(value).strip())) + except (TypeError, ValueError): + return default + + if parsed <= 0: + return default + return parsed + + +def _coerce_int(value, default=None): + try: + return int(float(str(value).strip())) + except (TypeError, ValueError): + return default + + +def _configured_mpi_int(env_keys, default=None): + for env_key in env_keys: + value = _coerce_int(os.environ.get(env_key), default=None) + if value is not None: + return value + return default + + +def get_mpi_status(): + """Return basic MPI status without requiring MPI to be installed.""" + env_size = _configured_mpi_int(MPI_SIZE_ENV_KEYS, default=1) + env_rank = _configured_mpi_int(MPI_RANK_ENV_KEYS, default=0) + try: + from mpi4py import MPI + + comm = MPI.COMM_WORLD + return { + "available": True, + "size": int(comm.Get_size()), + "rank": int(comm.Get_rank()), + "source": "mpi4py", + "error": None, + } + except Exception as exc: + return { + "available": False, + "size": max(int(env_size or 1), 1), + "rank": max(int(env_rank or 0), 0), + "source": "environment", + "error": str(exc), + } + + +def is_mpi_worker_process(): + status = get_mpi_status() + return status["size"] > 1 and status["rank"] > 0 + + +def _is_colab_runtime(): + return bool( + os.environ.get("COLAB_RELEASE_TAG") + or os.environ.get("GOOGLE_COLAB") + or "google.colab" in sys.modules + ) + + +def _configured_ultranest_workers(): + for env_key in ULTRANEST_WORKER_ENV_KEYS: + value = os.environ.get(env_key) + if value in (None, ""): + continue + if _is_disabled(value): + return 1 + parsed = _coerce_positive_int(value, default=None) + if parsed is not None: + return parsed + + return 1 + + +def _configured_ultranest_worker_backend(): + backend = str(os.environ.get(ULTRANEST_WORKER_BACKEND_ENV, "")).strip().lower() + if backend in {"process", "processes", "multiprocessing"}: + return "process" + if backend in {"thread", "threads", "threading"}: + return "thread" + return "none" + + +def _process_loglike_chunk(chunk): + if _PROCESS_LOGLIKE is None: + raise RuntimeError("UltraNest process worker was not initialized.") + return _PROCESS_LOGLIKE(chunk) + + +@contextmanager +def _parallel_vectorized_loglike(sampler, workers=None): + worker_count = max(int(workers or _configured_ultranest_workers()), 1) + backend = _configured_ultranest_worker_backend() + status = get_mpi_status() + if int(status.get("size") or 1) > 1: + worker_count = 1 + + original_loglike = getattr(sampler, "loglike", None) + if worker_count <= 1 or backend == "none" or not callable(original_loglike): + yield 1, "single" + return + + pool = None + executor = None + if backend == "process": + if not sys.platform.startswith("linux") or _is_colab_runtime(): + yield 1, "single" + return + global _PROCESS_LOGLIKE + _PROCESS_LOGLIKE = original_loglike + ctx = multiprocessing.get_context("fork") + pool = ctx.Pool(processes=worker_count) + elif backend == "thread": + executor = ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix="exotic-ultranest") + else: + yield 1, "single" + return + + def parallel_loglike(params): + params_array = np.asarray(params) + if params_array.ndim != 2 or params_array.shape[0] < 2: + return original_loglike(params) + + chunk_count = min(worker_count, params_array.shape[0]) + chunks = [chunk for chunk in np.array_split(params_array, chunk_count) if len(chunk)] + if pool is not None: + results = pool.map(_process_loglike_chunk, chunks) + else: + results = list(executor.map(original_loglike, chunks)) + return np.concatenate([np.atleast_1d(result) for result in results]) + + sampler.loglike = parallel_loglike + try: + yield worker_count, backend + finally: + sampler.loglike = original_loglike + if pool is not None: + pool.close() + pool.join() + _PROCESS_LOGLIKE = None + if executor is not None: + executor.shutdown(wait=True) + + +def _configured_min_num_live_points(default=DEFAULT_MIN_NUM_LIVE_POINTS): + for env_key in MIN_LIVE_POINTS_ENV_KEYS: + value = _coerce_positive_int(os.environ.get(env_key), default=None) + if value is not None: + return value + return default + + +def _apply_default_run_kwargs(run_kwargs): + kwargs = dict(DEFAULT_RUN_KWARGS) + kwargs["min_num_live_points"] = _configured_min_num_live_points() + kwargs.update({} if run_kwargs is None else dict(run_kwargs)) + return kwargs + + def supports_ultranest_live_status(stream=None): """Return True only when rich UltraNest status is explicitly enabled.""" if _is_enabled(os.environ.get("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "")): @@ -184,9 +387,8 @@ def _line(self, done=False): ) def start(self): - self._write( - "[ultranest] Using simple progress updates." - ) + heartbeat = f"{self.interval_seconds:g}s" + self._write(f"[ultranest] Using simple progress updates ({heartbeat} heartbeat).") def update(self, *args, **kwargs): info = _extract_info(args, kwargs) @@ -229,35 +431,45 @@ def run_reactive_sampler( Set EXOTIC_ULTRANEST_RICH_PROGRESS=1 for UltraNest's native status output. Use verbose=False to silence all progress updates. """ - kwargs = {} if run_kwargs is None else dict(run_kwargs) - mode = _progress_mode(verbose=verbose, stream=stream) - - if mode == "silent": - kwargs["show_status"] = False - kwargs["viz_callback"] = False - with _mute_ultranest_logging(sampler): + kwargs = _apply_default_run_kwargs(run_kwargs) + mode = "silent" if is_mpi_worker_process() else _progress_mode(verbose=verbose, stream=stream) + output_stream = stream if stream is not None else sys.stdout + + with _parallel_vectorized_loglike(sampler) as (worker_count, worker_backend): + if worker_count > 1 and mode != "silent": + worker_label = "processes" if worker_backend == "process" else "threads" + print( + f"[ultranest] Using {worker_count} worker {worker_label} for vectorized likelihood batches.", + file=output_stream, + flush=True, + ) + + if mode == "silent": + kwargs["show_status"] = False + kwargs["viz_callback"] = False + with _mute_ultranest_logging(sampler): + return sampler.run(**kwargs) + + if mode == "rich": + kwargs.setdefault("show_status", True) return sampler.run(**kwargs) - if mode == "rich": - kwargs.setdefault("show_status", True) - return sampler.run(**kwargs) - - progress = _UltraNestSimpleProgress(stream=stream, interval_seconds=interval_seconds) - upstream_callback = kwargs.get("viz_callback") + progress = _UltraNestSimpleProgress(stream=stream, interval_seconds=interval_seconds) + upstream_callback = kwargs.get("viz_callback") - def callback(*args, **callback_kwargs): - progress.update(*args, **callback_kwargs) - progress.maybe_emit(force=False) - if callable(upstream_callback): - upstream_callback(*args, **callback_kwargs) + def callback(*args, **callback_kwargs): + progress.update(*args, **callback_kwargs) + progress.maybe_emit(force=False) + if callable(upstream_callback): + upstream_callback(*args, **callback_kwargs) - kwargs["show_status"] = False - kwargs["viz_callback"] = callback + kwargs["show_status"] = False + kwargs["viz_callback"] = callback - progress.start() - try: - with _mute_ultranest_logging(sampler): - result = sampler.run(**kwargs) - finally: - progress.finish() - return result + progress.start() + try: + with _mute_ultranest_logging(sampler): + result = sampler.run(**kwargs) + finally: + progress.finish() + return result diff --git a/exotic/exotic.py b/exotic/exotic.py index c8725839..d212258c 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -125,6 +125,10 @@ from .api.plate_solution import NextAstroPlateSolution, PlateSolution except ImportError: # package import from api.plate_solution import NextAstroPlateSolution, PlateSolution +try: + from .api.ultranest_utils import get_mpi_status +except ImportError: + from api.ultranest_utils import get_mpi_status try: from .api.http_compression import build_compressed_json_request except ImportError: @@ -191,6 +195,9 @@ _BJD_FALLBACK_WARNING_LOGGED = False _RUNTIME_FILE_HANDLER_NAME = "exotic-runtime-file" _RUNTIME_CONSOLE_HANDLER_NAME = "exotic-runtime-console" +_RUNTIME_TRACEBACK_WATCHDOG_SECONDS_ENV = "EXOTIC_RUNTIME_TRACEBACK_WATCHDOG_SECONDS" +_RUNTIME_TRACEBACK_WATCHDOG_DEFAULT_SECONDS = 1800.0 +_RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = False _mid_transit_warning_reported = False RELATIVE_FLUX_MAX = 2.0 # Legacy threshold retained for compatibility; no longer used as a hard rejection cap. AIRMASS_FLAT_RANGE_THRESHOLD = 0.05 @@ -208,11 +215,13 @@ COMPARISON_IMAGE_OUTLIER_MIN_SCATTER = 1e-4 OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 +ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT = 200 +ULTRANEST_MIN_NUM_LIVE_POINTS_ENV = "EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS" RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 RPRS_SEARCH_BOUND_MIN = 0.0 RPRS_SEARCH_BOUND_MAX = 1.0 RPRS_RETRY_MIN_HALF_WIDTH = 0.05 -INITIAL_RPRS_BOUND_LOWER_SCALE = 0.25 +INITIAL_RPRS_BOUND_LOWER_SCALE = 0.0 INITIAL_RPRS_BOUND_UPPER_SCALE = 3.0 ARS_SEARCH_BOUND_MIN = 1e-6 ARS_SEARCH_BOUND_FALLBACK_MAX = 100.0 @@ -1493,6 +1502,17 @@ def lightcurve_fit_transit_qc_failure_reason(fit): return "Transit detection QC flagged this fit as a poor transit candidate." +def lightcurve_fit_transit_qc_passed(fit): + if fit is None: + return False + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + return str(transit_qc.get('status', '')).strip().lower() == 'pass' + + return str(getattr(fit, 'transit_qc_status', '')).strip().lower() == 'pass' + + def make_json_safe(value): if isinstance(value, dict): return {str(key): make_json_safe(subvalue) for key, subvalue in value.items()} @@ -1547,26 +1567,18 @@ def triangle_plot_output_path(save_dir, planet_name, observation_date): return Path(save_dir) / "temp" / f"Triangle_{planet_name}_{observation_date}.png" +def comparison_candidate_triangle_plot_output_path(save_dir, planet_name, observation_date, comp_index): + return ( + Path(save_dir) + / "temp" + / f"Comp{int(comp_index) + 1}_Triangle_{planet_name}_{observation_date}.png" + ) + + def save_final_triangle_plot(fit, save_dir, planet_name, observation_date, source_dir=None): output_path = triangle_plot_output_path(save_dir, planet_name, observation_date) output_path.parent.mkdir(parents=True, exist_ok=True) - if source_dir is not None: - source_path = triangle_plot_output_path(source_dir, planet_name, observation_date) - try: - same_path = source_path.resolve() == output_path.resolve() - except OSError: - same_path = False - - if source_path.exists(): - if same_path: - return output_path - try: - shutil.copy2(source_path, output_path) - return output_path - except OSError: - pass - fig = fit.plot_triangle() fig.savefig(output_path) try: @@ -1999,7 +2011,12 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, if callable(triangle_plotter): try: fig = triangle_plotter() - triangle_plot_path = temp_dir / f"Triangle_{p_dict['pName']}_{observation_date}.png" + triangle_plot_path = comparison_candidate_triangle_plot_output_path( + candidate_dir, + p_dict['pName'], + observation_date, + comp_index, + ) fig.savefig(triangle_plot_path) plt.close(fig) except Exception as exc: @@ -2729,6 +2746,7 @@ def log_info(string, warn=False, error=False): else: print(string, flush=True) log.debug(string) + _reset_runtime_traceback_watchdog() return True @@ -2739,12 +2757,55 @@ def _find_runtime_handler(handler_name): return None -def configure_runtime_logging(): - global _RUNTIME_LOGGING_CONFIGURED +def _runtime_traceback_watchdog_seconds(): + try: + return float(os.environ.get( + _RUNTIME_TRACEBACK_WATCHDOG_SECONDS_ENV, + _RUNTIME_TRACEBACK_WATCHDOG_DEFAULT_SECONDS, + )) + except (TypeError, ValueError): + return _RUNTIME_TRACEBACK_WATCHDOG_DEFAULT_SECONDS + + +def _reset_runtime_traceback_watchdog(): + global _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE + + if not _RUNTIME_LOGGING_CONFIGURED: + return + + timeout = _runtime_traceback_watchdog_seconds() + if timeout <= 0: + cancel_runtime_traceback_watchdog() + return + + try: + faulthandler.cancel_dump_traceback_later() + except Exception: + pass + + try: + faulthandler.dump_traceback_later(timeout, repeat=False, file=sys.stdout) + _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = True + except Exception: + _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = False + - if _RUNTIME_LOGGING_CONFIGURED: +def cancel_runtime_traceback_watchdog(): + global _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE + + if not _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE: return + try: + faulthandler.cancel_dump_traceback_later() + except Exception: + pass + _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = False + + +def configure_runtime_logging(): + global _RUNTIME_LOGGING_CONFIGURED + logging.root.setLevel(logging.DEBUG) log.setLevel(logging.DEBUG) @@ -2765,12 +2826,18 @@ def configure_runtime_logging(): ) log.addHandler(file_handler) - if _find_runtime_handler(_RUNTIME_CONSOLE_HANDLER_NAME) is None: + console_handler = _find_runtime_handler(_RUNTIME_CONSOLE_HANDLER_NAME) + if console_handler is None: console_handler = logging.StreamHandler(sys.stdout) console_handler._exotic_runtime_handler_name = _RUNTIME_CONSOLE_HANDLER_NAME console_handler.setLevel(logging.INFO) console_handler.setFormatter(logging.Formatter("%(message)s")) log.addHandler(console_handler) + else: + try: + console_handler.setStream(sys.stdout) + except Exception: + console_handler.stream = sys.stdout try: faulthandler.enable(file=sys.stdout, all_threads=True) @@ -2778,14 +2845,45 @@ def configure_runtime_logging(): pass _RUNTIME_LOGGING_CONFIGURED = True + _reset_runtime_traceback_watchdog() + + +def _logger_has_current_stdout_handler(logger): + current_stdout = sys.stdout + active_logger = logger + while active_logger: + for handler in active_logger.handlers: + if getattr(handler, "stream", None) is current_stdout: + return True + if not getattr(active_logger, "propagate", False): + break + active_logger = active_logger.parent + return False + + +def _write_exception_traceback_to_stdout(message, exc_type, exc_value, exc_traceback): + traceback_text = ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback)) + try: + print(f"\n{message}", file=sys.stdout, flush=True) + print(traceback_text, file=sys.stdout, end="", flush=True) + except Exception: + try: + print(f"\n{message}", file=sys.__stdout__, flush=True) + print(traceback_text, file=sys.__stdout__, end="", flush=True) + except Exception: + pass def _log_exception_with_fallback(message, exc_type, exc_value, exc_traceback): + wrote_to_logger = False try: log.error(message, exc_info=(exc_type, exc_value, exc_traceback)) + wrote_to_logger = True except Exception: - print(message) - traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout) + pass + + if not wrote_to_logger or not _logger_has_current_stdout_handler(log): + _write_exception_traceback_to_stdout(message, exc_type, exc_value, exc_traceback) def _handle_unhandled_exception(exc_type, exc_value, exc_traceback): @@ -3078,6 +3176,95 @@ def should_assess_all_comparisons_before_selecting_best(config_value): return True +def should_exit_at_first_qc_pass_solution(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'exit_at_first_qc_pass_solution' value; " + "defaulting to exit at the first QC PASS solution.", + warn=True, + ) + return True + + +def parse_ultranest_min_num_live_points(config_value): + if config_value is None: + return ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT + + if isinstance(config_value, str) and config_value.strip() == "": + return ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT + + try: + live_points = int(float(str(config_value).strip())) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'minimum number of live points for ultranest' value; " + f"defaulting to {ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT}.", + warn=True, + ) + return ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT + + if live_points <= 0: + log_info( + "Warning: 'minimum number of live points for ultranest' must be positive; " + f"defaulting to {ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT}.", + warn=True, + ) + return ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT + + return live_points + + +def configure_ultranest_min_num_live_points(config_value): + live_points = parse_ultranest_min_num_live_points(config_value) + os.environ[ULTRANEST_MIN_NUM_LIVE_POINTS_ENV] = str(live_points) + return live_points + + +def log_ultranest_mpi_status(): + status = get_mpi_status() + size = int(status.get("size") or 1) + rank = int(status.get("rank") or 0) + if size <= 1 or rank != 0: + return status + + if status.get("available"): + log_info(f"UltraNest MPI mode detected: {size} process(es).") + else: + log_info( + "Warning: MPI launch detected, but mpi4py is unavailable; " + "UltraNest cannot coordinate MPI workers until mpi4py is installed.", + warn=True, + ) + return status + + +def validate_ultranest_mpi_runtime(): + status = get_mpi_status() + size = int(status.get("size") or 1) + if size <= 1: + return status + + message = ( + "EXOTIC was launched under MPI, which duplicates the full reduction on every rank. " + "Start EXOTIC once and set EXOTIC_ULTRANEST_WORKERS to control UltraNest CPU parallelism." + ) + if int(status.get("rank") or 0) == 0: + log_info(f"Error: {message}", error=True) + raise RuntimeError(message) + + def should_use_psf_photometry(config_value): if config_value is None: return True @@ -10142,6 +10329,7 @@ def summarize_lightcurve_fit_assessment(fit): 'airmass_correction_note': getattr(fit, 'airmass_correction_note', None), 'nested_tmid_refinement_applied': bool(getattr(fit, 'nested_tmid_refinement_applied', False)), 'nested_tmid_refinement_note': getattr(fit, 'nested_tmid_refinement_note', None), + 'ultranest_error_fallbacks': getattr(fit, 'ultranest_error_fallbacks', {}) or {}, } @@ -10191,6 +10379,12 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): log_info(f"{indent}Airmass correction note: {assessment['airmass_correction_note']}") if assessment.get('nested_tmid_refinement_note'): log_info(f"{indent}Nested Tmid refinement note: {assessment['nested_tmid_refinement_note']}") + if assessment.get('ultranest_error_fallbacks'): + fallback_keys = ", ".join(sorted(assessment['ultranest_error_fallbacks'])) + log_info( + f"{indent}UltraNest uncertainty fallback note: replaced degenerate posterior " + f"summary error(s) for {fallback_keys} using the sampled log-likelihood neighborhood." + ) def log_comparison_candidate_evaluation_start(comp_summary, rank, ranked_count, method_label, fit_diagnostics): @@ -10265,6 +10459,8 @@ def log_comparison_candidate_evaluation_result(attempt): def comparison_selection_metric_label(selection_metric): + if selection_metric == 'first_qc_pass': + return "First QC PASS" if selection_metric == 'comparison_field_rank': return "Comparison-Field Rank" if selection_metric == 'ktmf': @@ -10541,6 +10737,8 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): "selected: best available comparison-star fit after all completed candidates were rejected " "by transit QC" ) + if selection_metric == 'first_qc_pass': + return "selected: first completed comparison-star candidate with PASS transit QC" if selection_metric == 'ktmf' and np.isfinite(candidate_ktmf_metric): return "selected: highest KTMF in the chosen search" if selection_metric == 'eebls_snr' and np.isfinite(candidate_eebls_snr): @@ -10567,6 +10765,13 @@ def comparison_candidate_fit_selection_reason(summary, photometry_info): "not selected: comparison-field QC fallback chose " f"Comp {selected_comp_num} as the best available rejected fit" ) + if selection_metric == 'first_qc_pass': + if selected_comp_num is None: + return "not selected: search stopped after another candidate reached PASS transit QC" + return ( + "not selected: search stopped after " + f"Comp {selected_comp_num} reached PASS transit QC" + ) if selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): if not np.isfinite(candidate_eebls_snr): @@ -11634,6 +11839,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p use_eebls_to_initialize_tmid_and_bounds=True, pick_comparison_by_eebls_snr=True, assess_all_comparisons_before_selecting_best=True, + exit_at_first_qc_pass_solution=True, final_fit_baseline_duration_multiplier= FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, use_adaptive_apertures=False, @@ -11690,6 +11896,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ) attempts = [] + stopped_after_first_qc_pass = False for rank, comp_summary in enumerate(ranked_summaries): comp_index = comp_summary['comp_index'] ckey = comp_summary.get('key', f"comp{comp_index + 1}") @@ -11813,6 +12020,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'rejected_by_transit_qc': final_reduction.get('applied', False) and transit_qc_failure_reason is not None, 'selected': False, 'selection_reason': None, + 'search_stopped_after_qc_pass': False, 'failed_run_dir': None, 'final_output_dir': None, 'full_reduction_applied': final_reduction.get('applied', False), @@ -11858,6 +12066,19 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p attempt['failed_run_dir'] = str(archive_dir) log_comparison_candidate_evaluation_result(attempt) attempts.append(attempt) + if ( + exit_at_first_qc_pass_solution + and attempt.get('fit') is not None + and attempt.get('full_reduction_applied', False) + and lightcurve_fit_transit_qc_passed(selection_fit) + ): + attempt['search_stopped_after_qc_pass'] = True + stopped_after_first_qc_pass = True + log_info( + "Stopping comparison-star candidate search after the first transit-QC PASS fit " + f"({attempt['label']})." + ) + break selected_result = None completed_attempts = [ @@ -11873,9 +12094,19 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p for attempt in completed_attempts if not attempt.get('rejected_by_transit_qc', False) ] + first_qc_pass_attempt = next( + ( + attempt for attempt in attempts + if attempt.get('search_stopped_after_qc_pass', False) + ), + None, + ) selection_metric = 'ktmf' fallback_to_qc_rejected = False - if successful_attempts: + if first_qc_pass_attempt is not None: + selected_result = first_qc_pass_attempt + selection_metric = 'first_qc_pass' + elif successful_attempts: selected_result, selection_metric = select_preferred_comparison_attempt( successful_attempts, pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, @@ -11918,6 +12149,10 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p "; this fit had the strongest transit-vs-flat Delta BIC " f"({format_transit_delta_bic(selected_transit_delta_bic)})" ) + elif attempt.get('search_stopped_after_qc_pass', False): + attempt['selection_reason'] = ( + "selected: first completed comparison-star candidate with PASS transit QC" + ) elif selection_metric == 'ktmf' and np.isfinite(selected_ktmf_metric): attempt['selection_reason'] = ( "selected: highest KTMF among the evaluated " @@ -11945,6 +12180,11 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p f"{format_ktmf_metric(attempt.get('ktmf_metric', np.nan))} was lower than the selected " f"{format_ktmf_metric(selected_ktmf_metric)}" ) + elif selection_metric == 'first_qc_pass': + attempt['selection_reason'] = ( + "not selected: search stopped after the first comparison-star candidate " + "with PASS transit QC" + ) elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): if np.isfinite(attempt.get('eebls_snr', np.nan)): attempt['selection_reason'] = ( @@ -11968,6 +12208,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'attempts': attempts, 'selected_result': selected_result, 'selection_metric': selection_metric, + 'stopped_after_first_qc_pass': stopped_after_first_qc_pass, } @@ -12027,13 +12268,14 @@ def parse_args(): return parser.parse_args() -def main(): +def _main_impl(): # command line args args = parse_args() if args.multiprocess_transformations is not None and args.multiprocess_transformations < 1: raise ValueError("--multiprocess-transformations requires an integer greater than 0.") if args.multiprocess_lightcurve_fits is not None and args.multiprocess_lightcurve_fits < 1: raise ValueError("--multiprocess-lightcurve-fits requires an integer greater than 0.") + validate_ultranest_mpi_runtime() log.debug("*************************") log.debug("EXOTIC reduction log file") @@ -12184,6 +12426,14 @@ def main(): exotic_infoDict.get('use_impactparameter_rather_than_inclination_to_fit', 'y') ) ) + ultranest_min_num_live_points = configure_ultranest_min_num_live_points( + exotic_infoDict.get( + 'ultranest_min_num_live_points', + ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT, + ) + ) + log_info(f"UltraNest minimum live points: {ultranest_min_num_live_points}.") + log_ultranest_mpi_status() # Make a temp directory of helpful files Path(Path(exotic_infoDict['save']) / "temp").mkdir(exist_ok=True) @@ -12303,7 +12553,9 @@ def main(): exotic_infoDict['long'] = -110.951376 exotic_infoDict['pixel_bin'] = "2x2" + log_info("Calculating limb-darkening coefficients.") ld, ld0, ld1, ld2, ld3 = get_ld_values(pDict, exotic_infoDict) + log_info("Limb-darkening coefficients ready.") # check for EPW_MD5 checksum if 'EPW_MD5' in header: @@ -12523,6 +12775,9 @@ def main(): assess_all_comparisons_before_selecting_best = should_assess_all_comparisons_before_selecting_best( exotic_infoDict.get('assess_all_comparisons_before_selecting_best', 'y') ) + exit_at_first_qc_pass_solution = should_exit_at_first_qc_pass_solution( + exotic_infoDict.get('exit_at_first_qc_pass_solution', 'y') + ) use_psf_photometry = should_use_psf_photometry( exotic_infoDict.get('use_psf_photometry', 'y') ) @@ -12554,9 +12809,14 @@ def main(): if not assess_all_comparisons_before_selecting_best: log_info( "Warning: 'assess_all_comparisons_before_selecting_best' is now ignored; " - "all ranked comparison-star candidates will be fully reduced before selection.", + "comparison-star target-fit search is controlled by 'exit_at_first_qc_pass_solution'.", warn=True, ) + if not exit_at_first_qc_pass_solution: + log_info( + "Comparison-star candidate search will evaluate all ranked candidates before selection " + "because 'exit_at_first_qc_pass_solution' is disabled." + ) pDict['use_deviation_from_expected_transit_in_qc'] = use_deviation_from_expected_transit_in_qc pDict['deviation_from_expected_transit_in_qc_sigma'] = deviation_from_expected_transit_in_qc_sigma @@ -13131,6 +13391,7 @@ def main(): use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, assess_all_comparisons_before_selecting_best=assess_all_comparisons_before_selecting_best, + exit_at_first_qc_pass_solution=exit_at_first_qc_pass_solution, final_fit_baseline_duration_multiplier=final_fit_baseline_duration_multiplier, use_adaptive_apertures=use_adaptive_apertures, adaptive_aperture_values=aperture_values, @@ -13174,13 +13435,21 @@ def main(): selected_attempt.get('source_indices', np.arange(len(tFlux1), dtype=int)), dtype=int, ) - if selected_attempt.get('selected_despite_transit_qc', False): + if selected_attempt.get('search_stopped_after_qc_pass', False): + selection_basis = 'first_qc_pass' + elif selected_attempt.get('selected_despite_transit_qc', False): selection_basis = 'comparison_field_qc_fallback' elif selected_comp_index == comparison_calibration['best_comp_index']: selection_basis = 'comparison_field' else: selection_basis = 'comparison_field_retry' - if selection_basis == 'comparison_field_qc_fallback': + if selection_basis == 'first_qc_pass': + log_info( + "Comparison-star calibration target-fit selection chose " + f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " + "because it was the first candidate to pass transit QC." + ) + elif selection_basis == 'comparison_field_qc_fallback': fallback_selection_metric = comparison_fit_search.get('selection_metric', 'ktmf') if fallback_selection_metric == 'ktmf': fallback_metric_value = format_ktmf_metric( @@ -14082,6 +14351,24 @@ def main(): log.debug("Stopped ...") +def main(): + global _UNHANDLED_EXCEPTION_LOGGED + + _UNHANDLED_EXCEPTION_LOGGED = False + configure_runtime_logging() + install_exception_hooks() + + try: + return _main_impl() + except (KeyboardInterrupt, SystemExit): + raise + except Exception as exc: + _handle_unhandled_exception(type(exc), exc, exc.__traceback__) + raise + finally: + cancel_runtime_traceback_watchdog() + + def cli(): global _UNHANDLED_EXCEPTION_LOGGED diff --git a/exotic/inputs.py b/exotic/inputs.py index d608f7c8..06935770 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -215,6 +215,7 @@ def __init__(self, init_opt): 'use_deviation_from_expected_transit_in_qc': True, 'deviation_from_expected_transit_in_qc_sigma': 5.0, 'assess_all_comparisons_before_selecting_best': 'y', + 'exit_at_first_qc_pass_solution': 'y', 'detect_bad_pixels_before_photometry': 'y', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', @@ -222,6 +223,7 @@ def __init__(self, init_opt): 'pointing_rejection_sigma': 4.0, 'skip_low_comparison_coverage_rejection': 'n', 'fit_lightcurve_to_every_comparison_candidate': 'n', + 'ultranest_min_num_live_points': 200, } self.params = { 'images': imaging_files, 'save': save_directory, 'aavso_num': obs_code, 'second_obs': second_obs_code, @@ -465,6 +467,13 @@ def comp_params(self, init_file, planet_dict): 'assess_all_comparisons_before_selecting_best', 'Assess All Comparisons Before Selecting Best? (y/n)', ), + 'exit_at_first_qc_pass_solution': ( + 'exit_at_first_qc_pass_solution', + 'exit at first QC PASS solution', + 'Exit at first QC PASS solution', + 'Exit at first QC PASS solution? (y/n)', + 'Exit At First QC PASS Solution? (y/n)', + ), 'use_impactparameter_rather_than_inclination_to_fit': ( 'use_impactparameter_rather_than_inclination_to_fit', 'Use impact parameter rather than inclination to fit? (y/n)', @@ -491,6 +500,13 @@ def comp_params(self, init_file, planet_dict): 'fit_lightcurve_to_every_comparison_candidate', 'Fit Lightcurve to Every Comparison Candidate? (y/n)', ), + 'ultranest_min_num_live_points': ( + 'Minimum Number of Live Points for UltraNest', + 'minimum number of live points for ultranest', + 'ultranest_min_num_live_points', + 'ultranest_min_live_points', + 'min_num_live_points', + ), 'bad_wcs_threshold_percent': ( 'bad_wcs_threshold_percent', 'Bad WCS Threshold Percent', diff --git a/inits.json b/inits.json index ad1f71da..33ce9fea 100644 --- a/inits.json +++ b/inits.json @@ -36,6 +36,7 @@ "Expected-Value Transit QC Sigma": "Set optional_info 'deviation_from_expected_transit_in_qc_sigma' to the sigma threshold used by the expected-value transit QC rejection. Default 5.", "Assess All Comparisons Before Selecting Best": "Set optional_info 'assess_all_comparisons_before_selecting_best' to y to fit every comparison-star candidate that survives the earlier screening, report all fits, and select the candidate with the highest KTMF score. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "UltraNest Live Points": "Set optional_info 'minimum number of live points for ultranest' to a positive integer to control UltraNest's min_num_live_points. Default 200.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", @@ -121,6 +122,7 @@ "deviation_from_expected_transit_in_qc_sigma": 5.0, "assess_all_comparisons_before_selecting_best": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", + "minimum number of live points for ultranest": 200, "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_adaptive_apertures": false, diff --git a/requirements.txt b/requirements.txt index ade59384..7f02c452 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,5 +26,6 @@ scipy~=1.14.1 scikit-image~=0.24.0 statsmodels~=0.14.4 tenacity~=9.0 -ultranest~=3.6.5;platform_system!='Windows' +mpi4py>=4.0;platform_system=='Linux' +ultranest==4.5.0;platform_system!='Windows' zstandard~=0.23.0 diff --git a/setup.cfg b/setup.cfg index f46e45e7..2084197f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ install_requires = file: requirements.txt [options.entry_points] console_scripts = - exotic = exotic.exotic:cli + exotic = exotic:cli exotic-gui = exotic.exotic_gui:main [options.packages.find] diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index c0ec9cca..e720d54c 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -581,6 +581,72 @@ def __init__(self, *args, **kwargs): assert fit.sample_bounds["b"] == pytest.approx([min(bounds_values), max(bounds_values)]) +def test_nested_fit_replaces_degenerate_ultranest_errors_from_loglike_neighborhood(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 101) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + sample_points = np.array( + [ + [0.095, -0.0010], + [0.097, -0.0008], + [0.099, -0.0003], + [0.100, 0.0000], + [0.101, 0.0002], + [0.103, 0.0005], + [0.105, 0.0008], + [0.106, 0.0010], + [0.120, 0.0030], + [0.080, -0.0030], + ], + dtype=float, + ) + logl = np.array([-0.4, -0.3, -0.1, 0.0, -0.1, -0.2, -0.3, -0.4, -2.0, -3.0]) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr( + elca, + "run_reactive_sampler", + lambda *args, **kwargs: { + "maximum_likelihood": {"point": np.array([0.100, 0.0])}, + "posterior": { + "stdev": np.array([1e-15, 1e-15]), + "errlo": np.array([0.100, 0.0]), + "errup": np.array([0.100, 0.0]), + }, + "weighted_samples": { + "points": sample_points, + "logl": logl, + }, + "samples": np.repeat(np.array([[0.100, 0.0]]), 10, axis=0), + }, + ) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.0, 0.2], "tmid": [-0.01, 0.01]}, + mode="ns", + verbose=False, + ) + + assert fit.errors["rprs"] > 1e-3 + assert fit.errors["tmid"] > 1e-4 + assert set(fit.ultranest_error_fallbacks) == {"rprs", "tmid"} + assert fit.ultranest_error_fallbacks["rprs"]["sample_count"] == 8 + + def test_nested_fit_duration_prior_penalizes_wrong_transit_length(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -939,6 +1005,44 @@ def test_triangle_contour_levels_drop_duplicate_chi2_percentiles(monkeypatch, tm assert levels == [pytest.approx(42.0)] +def test_triangle_payload_expands_degenerate_error_ranges_to_sample_cloud(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "tmid": [-0.01, 0.01], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "tmid"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.100, "tmid": 0.0} + fit.errors = {"rprs": 1e-15, "tmid": 1e-15} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + points = np.column_stack( + [ + np.linspace(0.050, 0.150, 50), + np.linspace(-0.004, 0.004, 50), + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": np.repeat(np.array([[0.100, 0.0]]), points.shape[0], axis=0), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["ranges"][0][0] <= 0.052 + assert payload["ranges"][0][1] >= 0.148 + assert payload["ranges"][1][0] <= -0.0038 + assert payload["ranges"][1][1] >= 0.0038 + + def test_triangle_payload_tracks_left_and_right_geometry_branches_for_inclination(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index e5ad4631..d2a2621c 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -105,6 +105,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: compute_transit_qc_ktmf, apply_comparison_star_suitability_outlier_rejection, comparison_calibration_selection_reason, + comparison_candidate_triangle_plot_output_path, comparison_candidate_fit_selection_reason, comparison_star_coverage_summary, comparison_star_stability_summary, @@ -152,6 +153,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: should_fit_lightcurve_to_every_comparison_candidate, should_detect_bad_pixels_before_photometry, should_use_aperture_photometry, + should_exit_at_first_qc_pass_solution, should_pick_comparison_by_eebls_snr, should_use_psf_photometry, should_skip_low_comparison_coverage_rejection, @@ -162,10 +164,18 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: ) -def test_save_final_triangle_plot_copies_selected_candidate_artifact(tmp_path): +def test_save_final_triangle_plot_regenerates_even_when_selected_candidate_artifact_exists(tmp_path): + class DummyFigure: + def savefig(self, path): + Path(path).write_bytes(b"regenerated-final") + class DummyFit: + def __init__(self): + self.called = False + def plot_triangle(self): - raise AssertionError("final plot should be copied from the selected candidate") + self.called = True + return DummyFigure() planet_name = "TOI-1728 b" observation_date = "2024-12-14" @@ -174,10 +184,11 @@ def plot_triangle(self): source_temp = source_dir / "temp" source_temp.mkdir(parents=True) source_plot = source_temp / f"Triangle_{planet_name}_{observation_date}.png" - source_plot.write_bytes(b"selected-comp-6") + source_plot.write_bytes(b"stale-selected-comp-6") + fit = DummyFit() output_path = save_final_triangle_plot( - DummyFit(), + fit, final_dir, planet_name, observation_date, @@ -185,7 +196,20 @@ def plot_triangle(self): ) assert output_path == final_dir / "temp" / source_plot.name - assert output_path.read_bytes() == b"selected-comp-6" + assert output_path.read_bytes() == b"regenerated-final" + assert fit.called is True + + +def test_comparison_candidate_triangle_plot_uses_candidate_specific_name(tmp_path): + output_path = comparison_candidate_triangle_plot_output_path( + tmp_path / "comp7", + "WASP-80 b", + "2025-06-22", + 6, + ) + + assert output_path.name == "Comp7_Triangle_WASP-80 b_2025-06-22.png" + assert output_path.parent == tmp_path / "comp7" / "temp" def test_save_final_triangle_plot_regenerates_when_selected_artifact_missing(tmp_path): @@ -726,6 +750,26 @@ def test_should_assess_all_comparisons_before_selecting_best_parses_values(): assert should_assess_all_comparisons_before_selecting_best(True) is True +def test_should_exit_at_first_qc_pass_solution_parses_values(): + assert should_exit_at_first_qc_pass_solution(None) is True + assert should_exit_at_first_qc_pass_solution("y") is True + assert should_exit_at_first_qc_pass_solution("n") is False + assert should_exit_at_first_qc_pass_solution(True) is True + + +def test_validate_ultranest_mpi_runtime_rejects_whole_program_mpi(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr( + exotic_module, + "get_mpi_status", + lambda: {"available": True, "size": 72, "rank": 0, "source": "mpi4py", "error": None}, + ) + + with pytest.raises(RuntimeError, match="duplicates the full reduction"): + exotic_module.validate_ultranest_mpi_runtime() + + def test_build_time_rejection_diagnostic_groups_contiguous_ranges(): times = np.array([1.0, 1.1, 1.2, 1.5, 1.6, 2.0], dtype=float) keep_mask = np.array([True, False, False, True, False, True], dtype=bool) @@ -2995,6 +3039,188 @@ def fake_finalize( assert result["attempts"][0]["selection_reason"].startswith("not selected: KTMF") +def test_fit_ranked_comparison_calibration_candidates_stops_at_first_qc_pass_by_default(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + call_markers = [] + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + call_markers.append(comp_marker) + status_map = {50: "marginal", 40: "pass", 30: "pass"} + ktmf_map = {50: 4.90, 40: 3.20, 30: 5.00} + status = status_map[comp_marker] + ktmf_metric = ktmf_map[comp_marker] + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={ + "status": status, + "summary": "ok", + "delta_bic": 12.0 + comp_marker / 100.0, + "ktmf_metric": ktmf_metric, + "ktmf_contributions": [], + }, + transit_qc_status=status, + transit_qc_ktmf_metric=ktmf_metric, + transit_qc_delta_bic=12.0 + comp_marker / 100.0, + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + "comp3": np.full((6, 1, 1), 30.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + {"label": "Comp 3", "aggregate_score": 0.03, "coverage_rejected": False, "comp_index": 2}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert call_markers == [50, 40] + assert len(result["attempts"]) == 2 + assert result["stopped_after_first_qc_pass"] is True + assert result["selection_metric"] == "first_qc_pass" + assert result["selected_result"]["comp_index"] == 1 + assert result["selected_result"]["search_stopped_after_qc_pass"] is True + assert "first completed comparison-star candidate" in result["selected_result"]["selection_reason"] + + +def test_fit_ranked_comparison_calibration_candidates_can_evaluate_all_qc_passes_when_exit_disabled(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + call_markers = [] + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + call_markers.append(comp_marker) + ktmf_metric = {50: 3.10, 40: 4.00, 30: 4.80}[comp_marker] + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={ + "status": "pass", + "summary": "ok", + "delta_bic": 12.0 + comp_marker / 100.0, + "ktmf_metric": ktmf_metric, + "ktmf_contributions": [], + }, + transit_qc_status="pass", + transit_qc_ktmf_metric=ktmf_metric, + transit_qc_delta_bic=12.0 + comp_marker / 100.0, + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + "comp3": np.full((6, 1, 1), 30.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + {"label": "Comp 3", "aggregate_score": 0.03, "coverage_rejected": False, "comp_index": 2}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + exit_at_first_qc_pass_solution=False, + ) + + assert call_markers == [50, 40, 30] + assert result["stopped_after_first_qc_pass"] is False + assert result["selection_metric"] == "ktmf" + assert result["selected_result"]["comp_index"] == 2 + assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.80) + + def test_ranked_comparison_calibration_summaries_skip_suitability_outliers(): ranked = ranked_comparison_calibration_summaries( { @@ -4548,3 +4774,89 @@ def fake_log_exception(message, exc_type, exc_value, exc_traceback): exotic_module.cli() assert logged == [("Unhandled exception during EXOTIC run", RuntimeError, "boom", True)] + + +def test_package_init_exports_lazy_main_and_cli(monkeypatch): + import exotic + + monkeypatch.setattr(exotic, "_load_runtime_callable", lambda name: lambda: name) + + assert exotic.main() == "main" + assert exotic.cli() == "cli" + + +def test_package_init_loads_nested_runtime_for_archive_layout(monkeypatch): + import exotic + + def fake_import_module(module_name): + if module_name == "exotic.exotic.exotic": + return types.SimpleNamespace(main=lambda: "nested-main") + raise AssertionError(f"unexpected import: {module_name}") + + monkeypatch.setitem(exotic.__dict__, "__name__", "exotic.exotic") + monkeypatch.setattr(exotic, "import_module", fake_import_module) + + assert exotic._load_runtime_callable("main")() == "nested-main" + + +def test_configure_runtime_logging_rebinds_console_handler_to_current_stdout(monkeypatch): + import io + import exotic.exotic as exotic_module + + original_handlers = list(exotic_module.log.handlers) + original_configured = exotic_module._RUNTIME_LOGGING_CONFIGURED + + try: + exotic_module.log.handlers = [] + exotic_module._RUNTIME_LOGGING_CONFIGURED = False + + first_stdout = io.StringIO() + monkeypatch.setattr(exotic_module.sys, "stdout", first_stdout) + exotic_module.configure_runtime_logging() + handler = exotic_module._find_runtime_handler(exotic_module._RUNTIME_CONSOLE_HANDLER_NAME) + assert handler.stream is first_stdout + + second_stdout = io.StringIO() + monkeypatch.setattr(exotic_module.sys, "stdout", second_stdout) + exotic_module.configure_runtime_logging() + assert handler.stream is second_stdout + finally: + exotic_module.log.handlers = original_handlers + exotic_module._RUNTIME_LOGGING_CONFIGURED = original_configured + + +def test_log_exception_with_fallback_writes_traceback_to_current_stdout(monkeypatch, capsys): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "_logger_has_current_stdout_handler", lambda logger: False) + + try: + raise RuntimeError("boom") + except RuntimeError as exc: + exotic_module._log_exception_with_fallback( + "Unhandled exception during EXOTIC run", + type(exc), + exc, + exc.__traceback__, + ) + + output = capsys.readouterr().out + assert "Unhandled exception during EXOTIC run" in output + assert "Traceback" in output + assert "RuntimeError: boom" in output + + +def test_main_logs_direct_call_exceptions_to_current_stdout(monkeypatch, capsys): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda: None) + monkeypatch.setattr(exotic_module, "install_exception_hooks", lambda: None) + monkeypatch.setattr(exotic_module, "_logger_has_current_stdout_handler", lambda logger: False) + monkeypatch.setattr(exotic_module, "_main_impl", lambda: (_ for _ in ()).throw(RuntimeError("boom"))) + + with pytest.raises(RuntimeError, match="boom"): + exotic_module.main() + + output = capsys.readouterr().out + assert "Unhandled exception during EXOTIC run" in output + assert "RuntimeError: boom" in output diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index b8529a6e..eabacbfe 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -95,13 +95,14 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: ) -def test_build_initial_rprs_bounds_uses_wider_asymmetric_search_box(): +def test_build_initial_rprs_bounds_allows_zero_depth_search_box(): bounds = build_initial_rprs_bounds(0.1) assert bounds == pytest.approx([ - INITIAL_RPRS_BOUND_LOWER_SCALE * 0.1, + RPRS_SEARCH_BOUND_MIN, INITIAL_RPRS_BOUND_UPPER_SCALE * 0.1, ]) + assert INITIAL_RPRS_BOUND_LOWER_SCALE == pytest.approx(0.0) def test_rprs_posterior_retry_walks_bounds_until_retry_cap(monkeypatch): @@ -343,6 +344,76 @@ def fake_lc_fitter( assert fit.rprs_posterior_refit_count == 1 +def test_rprs_posterior_retry_expands_lower_edge_down_to_zero(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + {"clipped": True, "edge": "lower", "mode": 0.030, "std": 0.006, "bounds": [0.000, 0.100]}, + {"clipped": False, "edge": None, "mode": 0.031, "std": 0.005, "bounds": [0.000, 0.120]}, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key == "rprs" + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "duration_prior": duration_prior, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.025, 0.300], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.025, 0.300]) + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.030) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.000, 0.120]) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 1 + assert fit.rprs_posterior_refit_edge == "lower" + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.000, 0.120]) + + def test_ars_posterior_retry_expands_bounds_when_upper_edge_is_truncated(monkeypatch): import exotic.exotic as exotic_module diff --git a/tests/test_inputs.py b/tests/test_inputs.py index c2b6ef45..d2a6503a 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -143,6 +143,36 @@ def test_comp_params_defaults_fit_lightcurve_to_every_comparison_candidate_to_no assert inputs.info_dict["fit_lightcurve_to_every_comparison_candidate"] == "n" +def test_comp_params_defaults_ultranest_live_points_to_200(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["ultranest_min_num_live_points"] == 200 + + +def test_comp_params_defaults_exit_at_first_qc_pass_solution_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["exit_at_first_qc_pass_solution"] == "y" + + def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_path): init_data = { "user_info": {}, @@ -398,6 +428,36 @@ def test_comp_params_reads_fit_lightcurve_to_every_comparison_candidate_from_opt assert inputs.info_dict["fit_lightcurve_to_every_comparison_candidate"] == "y" +def test_comp_params_reads_ultranest_live_points_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"minimum number of live points for ultranest": 275}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["ultranest_min_num_live_points"] == 275 + + +def test_comp_params_reads_exit_at_first_qc_pass_solution_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"exit at first QC PASS solution": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["exit_at_first_qc_pass_solution"] == "n" + + def test_comp_params_reads_disable_vertical_flux_normalization_from_optional_info(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_ultranest_utils.py b/tests/test_ultranest_utils.py index 5d4d11d6..6136ad8e 100644 --- a/tests/test_ultranest_utils.py +++ b/tests/test_ultranest_utils.py @@ -1,10 +1,25 @@ import io import logging +import numpy as np + +import exotic.api.ultranest_utils as ultranest_utils from exotic.api.ultranest_utils import run_reactive_sampler from exotic.api.ultranest_utils import supports_ultranest_live_status +_MPI_ENV_KEYS = ( + "OMPI_COMM_WORLD_SIZE", + "PMI_SIZE", + "PMIX_SIZE", + "MV2_COMM_WORLD_SIZE", + "OMPI_COMM_WORLD_RANK", + "PMI_RANK", + "PMIX_RANK", + "MV2_COMM_WORLD_RANK", +) + + class _FakeStream(io.StringIO): def __init__(self, tty): super().__init__() @@ -17,6 +32,13 @@ def isatty(self): def _reset_ultranest_env(monkeypatch): monkeypatch.delenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", raising=False) monkeypatch.delenv("EXOTIC_ULTRANEST_RICH_PROGRESS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_MIN_LIVE_POINTS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_WORKERS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_WORKER_BACKEND", raising=False) + monkeypatch.delenv("NEXTASTRO_EXOTIC_ULTRANEST_WORKERS", raising=False) + for env_key in _MPI_ENV_KEYS: + monkeypatch.delenv(env_key, raising=False) monkeypatch.delenv("CI", raising=False) @@ -91,6 +113,137 @@ def run(self, **kwargs): assert stream.getvalue() == "" +def test_run_reactive_sampler_applies_fast_defaults(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=False, + ) + + assert sampler.kwargs["min_num_live_points"] == 200 + assert sampler.kwargs["min_ess"] == 200 + assert sampler.kwargs["dlogz"] == 1.0 + assert sampler.kwargs["dKL"] == 1.0 + assert sampler.kwargs["frac_remain"] == 0.05 + assert sampler.kwargs["max_num_improvement_loops"] == 1 + assert sampler.kwargs["max_ncalls"] == 1000 + + +def test_run_reactive_sampler_uses_env_live_point_override(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "320") + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler(sampler, verbose=False) + + assert sampler.kwargs["min_num_live_points"] == 320 + + +def test_run_reactive_sampler_parallelizes_vectorized_loglike_batches(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "3") + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "thread") + + class FakeSampler: + def __init__(self): + self.kwargs = None + self.chunk_sizes = [] + + def loglike(points): + self.chunk_sizes.append(len(points)) + return points[:, 0] + + self.loglike = loglike + + def run(self, **kwargs): + self.kwargs = kwargs + values = self.loglike(np.arange(12, dtype=float).reshape(6, 2)) + return {"values": values} + + sampler = FakeSampler() + result = run_reactive_sampler(sampler, verbose=False) + + assert result["values"].tolist() == [0, 2, 4, 6, 8, 10] + assert sorted(sampler.chunk_sizes) == [2, 2, 2] + + +def test_run_reactive_sampler_preserves_explicit_live_point_override(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "320") + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler( + sampler, + run_kwargs={"min_num_live_points": 450}, + verbose=False, + ) + + assert sampler.kwargs["min_num_live_points"] == 450 + + +def test_run_reactive_sampler_silences_mpi_worker_rank(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setattr( + ultranest_utils, + "get_mpi_status", + lambda: { + "available": True, + "size": 4, + "rank": 2, + "source": "mpi4py", + "error": None, + }, + ) + stream = _FakeStream(tty=True) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + verbose=True, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert sampler.kwargs["viz_callback"] is False + assert stream.getvalue() == "" + + def test_run_reactive_sampler_tty_defaults_to_simple_status(monkeypatch): _reset_ultranest_env(monkeypatch) stream = _FakeStream(tty=True) From 5d4bb6b1b9e8ae5654b779dfe03b65e8ab3da4e7 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 30 Apr 2026 09:21:57 +1000 Subject: [PATCH 031/116] Expand AAVSO exports with richer diagnostics --- exotic/exotic.py | 74 ++++++++- exotic/output_files.py | 324 ++++++++++++++++++++++++++++++++++++- tests/test_output_files.py | 190 ++++++++++++++++++++++ 3 files changed, 586 insertions(+), 2 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index d212258c..bd766953 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -12367,6 +12367,16 @@ def _main_impl(): generalDark, generalBias, generalFlat = np.empty(shape=(0, 0)), np.empty(shape=(0, 0)), np.empty(shape=(0, 0)) demosaic_fmt = None demosaic_out = None + precheck_inputfile_count = None + post_wcs_inputfile_count = None + post_pointing_inputfile_count = None + dropped_wcs_files = [] + dropped_pointing_files = [] + ignore_header_wcs = False + bad_wcs_threshold_fraction = np.nan + pointing_rejection_sigma = np.nan + detect_bad_pixels_before_photometry = None + bad_pixel_reference = None if isinstance(args.reduce, str): fitsortext = 1 @@ -12565,6 +12575,7 @@ def _main_impl(): times = np.array(times)[si] jd_times = np.array(jd_times)[si] inputfiles = np.array(inputfiles)[si] + precheck_inputfile_count = int(len(inputfiles)) finite_plot_times = times[np.isfinite(times)] full_plot_time_range = None if finite_plot_times.size: @@ -12588,6 +12599,7 @@ def _main_impl(): times = times[wcs_keep_mask] jd_times = jd_times[wcs_keep_mask] plateStatus.initializeFilenames(list(inputfiles)) + post_wcs_inputfile_count = int(len(inputfiles)) pointing_precheck_inputfiles = np.array(inputfiles, copy=True) pointing_reference_file = inputfiles[0] if len(inputfiles) else None inputfiles, pointing_keep_mask, dropped_pointing_files = filter_pointing_outlier_frames( @@ -12618,6 +12630,7 @@ def _main_impl(): if finite_plot_times.size: full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) plateStatus.initializeFilenames(list(inputfiles)) + post_pointing_inputfile_count = int(len(inputfiles)) bad_pixel_reference = None if detect_bad_pixels_before_photometry: @@ -14324,7 +14337,66 @@ def _main_impl(): try: if bestCompStar: exotic_infoDict['phot_comp_star'] = save_comp_ra_dec(wcs_file, ra_wcs, dec_wcs, comp_coords) - output_files.aavso(exotic_infoDict['phot_comp_star'], goodAirmasses, ld0, ld1, ld2, ld3, epw_md5) + aavso_photometry_info = photometry_info if fitsortext == 1 else None + aavso_frame_filtering_info = None + aavso_astrometry_info = None + aavso_bad_pixel_info = None + if fitsortext == 1: + aavso_frame_filtering_info = { + 'initial_frame_count': precheck_inputfile_count, + 'after_missing_wcs_filter_frame_count': post_wcs_inputfile_count, + 'final_prephotometry_frame_count': post_pointing_inputfile_count, + 'ignore_header_wcs': ignore_header_wcs, + 'bad_wcs_threshold_percent': ( + 100.0 * bad_wcs_threshold_fraction + if np.isfinite(bad_wcs_threshold_fraction) + else np.nan + ), + 'pointing_rejection_sigma': pointing_rejection_sigma, + 'dropped_missing_wcs_files': dropped_wcs_files, + 'dropped_pointing_files': dropped_pointing_files, + } + aavso_astrometry_info = { + 'wcs_file': str(wcs_file) if wcs_file else None, + 'coordinate_source': 'wcs' if wcs_file else 'input_pixels', + 'ignore_header_wcs': ignore_header_wcs, + 'plate_solution_option': exotic_infoDict.get('plate_opt'), + 'target_input_pixel': exotic_infoDict.get('tar_coords'), + 'comparison_input_pixels': exotic_infoDict.get('comp_stars'), + 'target_ra_dec_deg': ra_dec_tar, + 'comparison_ra_dec_deg': ra_dec_wcs, + 'catalog_ra_dec_deg': [pDict.get('ra'), pDict.get('dec')], + 'gaia_distance_pc': pDict.get('dist'), + 'proper_motion_ra_mas_yr': pDict.get('pm_ra'), + 'proper_motion_dec_mas_yr': pDict.get('pm_dec'), + } + aavso_bad_pixel_info = { + 'enabled': bool(detect_bad_pixels_before_photometry), + 'detected': bad_pixel_reference is not None, + } + if bad_pixel_reference is not None: + bad_pixel_mask = np.asarray(bad_pixel_reference.get('mask'), dtype=bool) + aavso_bad_pixel_info.update({ + 'bad_pixel_count': int(np.count_nonzero(bad_pixel_mask)), + 'frame_count': bad_pixel_reference.get('frame_count'), + 'required_count': bad_pixel_reference.get('required_count'), + 'minimum_fraction': bad_pixel_reference.get('minimum_fraction'), + 'counts_path': bad_pixel_reference.get('counts_path'), + 'mask_path': bad_pixel_reference.get('mask_path'), + }) + output_files.aavso( + exotic_infoDict['phot_comp_star'], + goodAirmasses, + ld0, + ld1, + ld2, + ld3, + epw_md5, + photometry_info=aavso_photometry_info, + astrometry_info=aavso_astrometry_info, + frame_filtering_info=aavso_frame_filtering_info, + bad_pixel_info=aavso_bad_pixel_info, + ) except Exception as e: log_info(f"\nError: Could not create AAVSO.txt. {error_txt}\n\t{e}", error=True) try: diff --git a/exotic/output_files.py b/exotic/output_files.py index ba96b2ef..07790b91 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -61,6 +61,282 @@ def finite_float(value, default=np.nan): return value if np.isfinite(value) else default +def aavso_json_safe(value): + if isinstance(value, dict): + return {str(key): aavso_json_safe(subvalue) for key, subvalue in value.items()} + if isinstance(value, (list, tuple)): + return [aavso_json_safe(item) for item in value] + if isinstance(value, np.ndarray): + if value.ndim == 0: + return aavso_json_safe(value.item()) + return [aavso_json_safe(item) for item in value.tolist()] + if isinstance(value, np.generic): + return aavso_json_safe(value.item()) + if isinstance(value, Path): + return str(value) + if isinstance(value, bool): + return bool(value) + if isinstance(value, float): + return float(value) if np.isfinite(value) else None + if isinstance(value, int): + return int(value) + return value + + +def prune_aavso_metadata(value): + if isinstance(value, dict): + pruned = {} + for key, subvalue in value.items(): + cleaned = prune_aavso_metadata(subvalue) + if cleaned is None or cleaned == "" or cleaned == [] or cleaned == {}: + continue + pruned[key] = cleaned + return pruned + if isinstance(value, np.ndarray): + return prune_aavso_metadata(value.tolist()) + if isinstance(value, (list, tuple)): + return [ + cleaned for cleaned in (prune_aavso_metadata(item) for item in value) + if cleaned is not None and cleaned != "" and cleaned != [] and cleaned != {} + ] + return aavso_json_safe(value) + + +def format_aavso_json_header(name, payload): + payload = prune_aavso_metadata(payload) + if not payload: + return "" + return f"#{name}={dumps(payload, sort_keys=True)}\n" + + +def aavso_result_entry(value, uncertainty=None, units=None): + value = finite_float(value) + uncertainty = finite_float(uncertainty) + if not np.isfinite(value): + return None + + entry = { + 'value': str(round_to_2(value, uncertainty)) if np.isfinite(uncertainty) else str(round_to_2(value)), + } + if np.isfinite(uncertainty): + entry['uncertainty'] = str(round_to_2(uncertainty)) + if units: + entry['units'] = units + return entry + + +def numeric_series_summary(values): + if values is None: + return {} + + try: + series = np.asarray(values, dtype=float).reshape(-1) + except (TypeError, ValueError): + return {} + + finite = series[np.isfinite(series)] + if finite.size == 0: + return {} + + return { + 'count': int(finite.size), + 'median': float(np.nanmedian(finite)), + 'std': float(np.nanstd(finite)), + 'min': float(np.nanmin(finite)), + 'max': float(np.nanmax(finite)), + } + + +def path_name(value): + if value is None: + return None + return Path(str(value)).name + + +def file_list_summary(files, limit=10): + files = list(files or []) + return { + 'count': len(files), + 'files': [path_name(file_name) for file_name in files[:limit]], + 'omitted_file_count': max(0, len(files) - limit), + } + + +def residual_scatter_fraction(fit): + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + qc_residual_scatter = finite_float(transit_qc.get('residual_scatter')) + if np.isfinite(qc_residual_scatter): + return qc_residual_scatter + + residuals = np.asarray(getattr(fit, 'residuals', np.array([])), dtype=float) + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if residuals.size == 0 or data.size == 0: + return np.nan + + median_flux = np.nanmedian(data) + if not np.isfinite(median_flux) or median_flux == 0: + return np.nan + + if residuals.shape == data.shape: + return float(np.nanstd(residuals) / median_flux) + if residuals.size == 1: + return float(abs(residuals.reshape(-1)[0]) / median_flux) + return np.nan + + +def photometry_method_from_info(photometry_info): + if not isinstance(photometry_info, dict): + return None + + min_aperture = photometry_info.get('min_aperture') + min_aperture = finite_float(min_aperture) + if not np.isfinite(min_aperture): + return None + if min_aperture == 0: + return "PSF photometry" + if min_aperture < 0: + return "Aperture photometry without comparison star" + return "Aperture photometry" + + +def build_aavso_qc_metadata(fit): + transit_qc = getattr(fit, 'transit_qc', None) + if not isinstance(transit_qc, dict): + return {} + + fields = ( + 'computed', 'status', 'summary', 'preferred_model', 'point_count', + 'transit_chi2', 'flat_chi2', 'delta_chi2', 'transit_bic', 'flat_bic', + 'delta_bic', 'transit_parameter_count', 'flat_parameter_count', + 'flat_baseline', 'flat_a2', 'flat_model_note', 'residual_scatter', + 'rprs_sigma', 'duration_ratio', 'eebls_depth_snr', + 'use_deviation_from_expected_transit_in_qc', 'deviation_sigma_threshold', + 'expected_tmid', 'expected_tmid_unc', 'expected_tmid_unc_minutes', + 'fitted_tmid', 'expected_rprs', 'expected_rprs_unc', + 'tmid_deviation_days', 'tmid_deviation_minutes', + 'tmid_deviation_threshold_minutes', 'tmid_deviation_sigma', + 'rprs_deviation_sigma', 'tmid_deviation_score', 'rprs_deviation_score', + 'deviation_from_expected_value', 'ktmf_metric', 'ktmf_contributions', + 'notes', + ) + return {field: transit_qc.get(field) for field in fields if field in transit_qc} + + +def build_aavso_photometry_metadata(photometry_info): + if not isinstance(photometry_info, dict): + return {} + + selected_source_indices = photometry_info.get('selected_source_indices') + selected_source_count = None + if selected_source_indices is not None: + try: + selected_source_count = int(np.asarray(selected_source_indices).size) + except (TypeError, ValueError): + selected_source_count = None + + selected_times = numeric_series_summary(photometry_info.get('selected_fit_good_times')) + return { + 'method': photometry_method_from_info(photometry_info), + 'selected_comparison_star': photometry_info.get('comp_star_num'), + 'selected_comparison_coordinates': photometry_info.get('comp_star_coords'), + 'comparison_selection_basis': photometry_info.get('selection_basis'), + 'comparison_selection_metric': photometry_info.get('selection_metric'), + 'comparison_field_score': photometry_info.get('calibration_field_score'), + 'comparison_field_score_percent': ( + 100.0 * finite_float(photometry_info.get('calibration_field_score')) + if np.isfinite(finite_float(photometry_info.get('calibration_field_score'))) + else np.nan + ), + 'selected_comparison_ktmf': photometry_info.get('comparison_ktmf_metric'), + 'selected_comparison_eebls_snr': photometry_info.get('comparison_eebls_snr'), + 'selected_comparison_transit_delta_bic': photometry_info.get('comparison_transit_delta_bic'), + 'reused_selected_full_reduction_fit': photometry_info.get('reuse_selected_full_reduction_fit'), + 'selected_source_point_count': selected_source_count, + 'selected_fit_time_range': selected_times, + } + + +def build_aavso_aperture_metadata(photometry_info): + if not isinstance(photometry_info, dict): + return {} + + adaptive_summary = photometry_info.get('adaptive_summary') + payload = { + 'method': photometry_method_from_info(photometry_info), + 'aperture_index': photometry_info.get('aperture_index'), + 'annulus_index': photometry_info.get('annulus_index'), + 'configured_aperture_px': photometry_info.get('min_aperture'), + 'configured_annulus_px': photometry_info.get('min_annulus'), + 'adaptive': adaptive_summary is not None, + } + if not isinstance(adaptive_summary, dict): + return payload + + payload.update({ + 'aperture_sigma': adaptive_summary.get('aperture_sigma'), + 'annulus_sigma': adaptive_summary.get('annulus_sigma'), + 'aperture_px': { + 'median': adaptive_summary.get('aperture_median'), + 'std': adaptive_summary.get('aperture_std'), + 'min': adaptive_summary.get('aperture_min'), + 'max': adaptive_summary.get('aperture_max'), + }, + 'annulus_px': { + 'median': adaptive_summary.get('annulus_median'), + 'std': adaptive_summary.get('annulus_std'), + 'min': adaptive_summary.get('annulus_min'), + 'max': adaptive_summary.get('annulus_max'), + }, + 'fwhm_px': numeric_series_summary(adaptive_summary.get('fwhm_series')), + 'frame_sigma_px': numeric_series_summary(adaptive_summary.get('frame_sigma')), + 'sky_inner_px': numeric_series_summary(adaptive_summary.get('sky_inner_series')), + 'sky_outer_px': numeric_series_summary(adaptive_summary.get('sky_outer_series')), + 'sky_pixels': numeric_series_summary(adaptive_summary.get('sky_pixel_series')), + }) + return payload + + +def build_aavso_frame_filtering_metadata(fit, frame_filtering_info): + payload = dict(frame_filtering_info or {}) + + for source_key, target_key in ( + ('dropped_missing_wcs_files', 'missing_wcs_rejections'), + ('dropped_pointing_files', 'pointing_rejections'), + ): + if source_key in payload: + payload[target_key] = file_list_summary(payload.pop(source_key)) + + diagnostics = getattr(fit, 'frame_filter_diagnostics', None) + if diagnostics: + payload['lightcurve_filter_diagnostics'] = diagnostics + payload['lightcurve_dropped_point_count'] = sum( + int((diagnostic or {}).get('dropped_point_count', 0)) + for diagnostic in diagnostics + ) + return payload + + +def build_aavso_astrometry_metadata(astrometry_info, comp_star): + payload = dict(astrometry_info or {}) + if payload.get('wcs_file'): + payload['wcs_file'] = path_name(payload['wcs_file']) + if comp_star: + payload['comparison_star_aavso_header'] = comp_star + return payload + + +def build_aavso_bad_pixel_metadata(bad_pixel_info): + if not isinstance(bad_pixel_info, dict): + return {} + + payload = dict(bad_pixel_info) + for key in ('counts_path', 'mask_path'): + if payload.get(key): + payload[key] = path_name(payload[key]) + return payload + + def format_parameter_with_error(value, error): value = finite_float(value) error = finite_float(error) @@ -305,11 +581,19 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor with params_file.open('w') as f: dump(final_params, f, indent=4) - def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): + def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, + photometry_info=None, astrometry_info=None, frame_filtering_info=None, + bad_pixel_info=None): priors_dict, filter_dict, results_dict = aavso_dicts(self.p_dict, self.fit, self.i_dict, self.durs, ld0, ld1, ld2, ld3) aavso_airmass_terms = aavso_airmass_results(self.fit) detrend_model = aavso_detrend_model(self.fit) + qc_metadata = build_aavso_qc_metadata(self.fit) + photometry_metadata = build_aavso_photometry_metadata(photometry_info) + aperture_metadata = build_aavso_aperture_metadata(photometry_info) + frame_filtering_metadata = build_aavso_frame_filtering_metadata(self.fit, frame_filtering_info) + astrometry_metadata = build_aavso_astrometry_metadata(astrometry_info, comp_star) + bad_pixel_metadata = build_aavso_bad_pixel_metadata(bad_pixel_info) obs_name = format_aavso_header_value(self.i_dict.get('obs_name')) obs_name_header = f"#OBSNAME={obs_name}\n" if obs_name else "" gaia_dist = format_aavso_header_value(self.p_dict.get('dist')) @@ -363,6 +647,12 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): f",{aavso_airmass_terms[0][0]}={aavso_airmass_terms[0][1]} +/- {aavso_airmass_terms[0][2]}" f",{aavso_airmass_terms[1][0]}={aavso_airmass_terms[1][1]} +/- {aavso_airmass_terms[1][2]}\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields + f.write(format_aavso_json_header("QC-XC", qc_metadata)) + f.write(format_aavso_json_header("PHOTOMETRY-XC", photometry_metadata)) + f.write(format_aavso_json_header("APERTURE-XC", aperture_metadata)) + f.write(format_aavso_json_header("FRAME_FILTERING-XC", frame_filtering_metadata)) + f.write(format_aavso_json_header("ASTROMETRY-XC", astrometry_metadata)) + f.write(format_aavso_json_header("BAD_PIXEL-XC", bad_pixel_metadata)) if epw_md5: f.write(f"#EPW_MD5-XC={dumps({'epw_checkout_md5': epw_md5})}\n") @@ -512,6 +802,38 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): 'value': aavso_airmass_terms[0][1], 'uncertainty': aavso_airmass_terms[0][2] } + optional_results = { + 'a/R*': aavso_result_entry( + fit.parameters.get('ars'), + fit.errors.get('ars'), + ), + } + impact_parameter, impact_error = fit_impact_parameter_value_error(fit) + optional_results['Impact Parameter (b)'] = aavso_result_entry(impact_parameter, impact_error) + + rprs = finite_float(fit.parameters.get('rprs')) + rprs_error = finite_float(fit.errors.get('rprs')) + if np.isfinite(rprs): + optional_results['Transit depth (Rp/R*)^2'] = aavso_result_entry( + 100.0 * (rprs ** 2.0), + 100.0 * 2.0 * rprs * rprs_error if np.isfinite(rprs_error) else np.nan, + units="percent", + ) + + scatter = residual_scatter_fraction(fit) + optional_results['Residual scatter around full model fit'] = aavso_result_entry( + 100.0 * scatter if np.isfinite(scatter) else np.nan, + units="percent", + ) + if 'a0' in fit.parameters: + optional_results['a0'] = aavso_result_entry(fit.parameters.get('a0'), fit.errors.get('a0')) + elif 'a1' in fit.parameters: + optional_results['a1'] = aavso_result_entry(fit.parameters.get('a1'), fit.errors.get('a1')) + + results.update({ + key: value for key, value in optional_results.items() + if value is not None + }) return priors, filter_type, results diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 720f95dc..0d9662ce 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -33,6 +33,14 @@ def __init__(self): self.airmass_model = [1.0] +def aavso_json_header(output_text, header_name): + prefix = f"#{header_name}=" + for line in output_text.splitlines(): + if line.startswith(prefix): + return json.loads(line[len(prefix):]) + raise AssertionError(f"Missing {header_name} header") + + def test_aavso_output_includes_observatory_location_headers(tmp_path): fit = DummyFit() p_dict = { @@ -403,3 +411,185 @@ def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_p assert "Am1=0 +/- 0" in output_text assert "Am2=0 +/- 0" in output_text assert output_text.strip().endswith("1.0") + + +def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): + fit = DummyFit() + fit.transit_qc = { + "computed": True, + "status": "pass", + "summary": "Transit model strongly preferred over flat/null model.", + "delta_bic": 18.4, + "delta_chi2": 27.1, + "residual_scatter": 0.0032, + "rprs_sigma": 6.2, + "duration_ratio": 1.05, + "eebls_depth_snr": 5.8, + "deviation_from_expected_value": 0.91, + "tmid_deviation_minutes": 3.2, + "tmid_deviation_sigma": 1.1, + "rprs_deviation_sigma": 0.8, + "ktmf_metric": 4.63, + "ktmf_contributions": [ + { + "label": "Model Evidence", + "available": True, + "points": 0.74, + "max_points": 0.80, + "score": 0.93, + "detail": "Delta BIC=18.40", + } + ], + } + fit.frame_filter_diagnostics = [ + { + "stage": "Final-fit phase residual clip", + "input_point_count": 4, + "kept_point_count": 3, + "dropped_point_count": 1, + "dropped_ranges": [{"start": 2450000.2, "end": 2450000.2, "count": 1}], + } + ] + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + "pPer": 2.1500082, + "pPerUnc": 1.3e-07, + "rprs": 0.1488623525, + "rprsUnc": 0.0005539487, + "aRs": 5.344, + "aRsUnc": 0.03949, + "inc": 88.98, + "incUnc": 0.7602, + "ecc": 0.159, + "dist": 245.7, + "pm_ra": 14.25, + "pm_dec": -9.5, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "second_obs": "", + "obs_name": "", + "camera": "CCD", + "pixel_bin": "1x1", + "exposure": 60.0, + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + "notes": "na", + "filter": "CV", + "filter_desc": "Clear with V zero-point", + "wl_min": None, + "wl_max": None, + } + photometry_info = { + "comp_star_num": 2, + "comp_star_coords": [300.5, 400.5], + "min_aperture": 7.5, + "min_annulus": 22.5, + "aperture_index": 1, + "annulus_index": 2, + "calibration_field_score": 0.0042, + "selection_basis": "comparison_field", + "selection_metric": "ktmf", + "comparison_ktmf_metric": 4.6, + "comparison_eebls_snr": 5.2, + "comparison_transit_delta_bic": 18.4, + "reuse_selected_full_reduction_fit": True, + "selected_source_indices": np.array([0, 2, 3]), + "selected_fit_good_times": np.array([2450000.0, 2450000.1, 2450000.2]), + "adaptive_summary": { + "aperture_sigma": 2.62, + "annulus_sigma": 9.00, + "frame_sigma": np.array([2.0, 2.1, 2.2]), + "fwhm_series": np.array([4.7, 4.8, 4.9]), + "sky_inner_series": np.array([12.0, 12.1, 12.2]), + "sky_outer_series": np.array([18.0, 18.1, 18.2]), + "sky_pixel_series": np.array([200.0, 201.0, 202.0]), + "aperture_median": 7.98, + "aperture_std": 0.41, + "aperture_min": 7.12, + "aperture_max": 8.76, + "annulus_median": 27.43, + "annulus_std": 1.39, + "annulus_min": 25.11, + "annulus_max": 30.08, + }, + } + comp_star_header = {"ra": "10.1", "dec": "-20.2", "x": "493", "y": "202"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( + comp_star_header, + [1.0], + (0.1, 0.01), + (0.2, 0.01), + (0.3, 0.01), + (0.4, 0.01), + "abc123", + photometry_info=photometry_info, + frame_filtering_info={ + "initial_frame_count": 5, + "after_missing_wcs_filter_frame_count": 4, + "final_prephotometry_frame_count": 3, + "ignore_header_wcs": False, + "bad_wcs_threshold_percent": 3.0, + "pointing_rejection_sigma": 3.0, + "dropped_missing_wcs_files": [tmp_path / "missing_wcs.fits"], + "dropped_pointing_files": [tmp_path / "bad_pointing.fits"], + }, + astrometry_info={ + "wcs_file": tmp_path / "wcs.fits", + "coordinate_source": "wcs", + "target_ra_dec_deg": [10.0, -20.0], + "comparison_ra_dec_deg": [[10.1, -20.2]], + }, + bad_pixel_info={ + "enabled": True, + "detected": True, + "bad_pixel_count": 3, + "frame_count": 10, + "required_count": 4, + "minimum_fraction": 0.3, + "counts_path": tmp_path / "temp" / "BadPixelDetectionCounts.fits", + "mask_path": tmp_path / "temp" / "BadPixelMask.fits", + }, + ) + + output_text = (tmp_path / "AAVSO_HAT-P-32 b_2020-01-01.txt").read_text(encoding="utf-8") + + results = aavso_json_header(output_text, "RESULTS-XC") + assert "a/R*" in results + assert "Impact Parameter (b)" in results + assert results["Transit depth (Rp/R*)^2"]["units"] == "percent" + assert results["Residual scatter around full model fit"]["value"] == "0.32" + + qc = aavso_json_header(output_text, "QC-XC") + assert qc["status"] == "pass" + assert qc["ktmf_metric"] == pytest.approx(4.63) + assert qc["ktmf_contributions"][0]["label"] == "Model Evidence" + + photometry = aavso_json_header(output_text, "PHOTOMETRY-XC") + assert photometry["selected_comparison_star"] == 2 + assert photometry["comparison_field_score_percent"] == pytest.approx(0.42) + assert photometry["reused_selected_full_reduction_fit"] is True + + aperture = aavso_json_header(output_text, "APERTURE-XC") + assert aperture["adaptive"] is True + assert aperture["aperture_sigma"] == pytest.approx(2.62) + assert aperture["fwhm_px"]["median"] == pytest.approx(4.8) + + frame_filtering = aavso_json_header(output_text, "FRAME_FILTERING-XC") + assert frame_filtering["missing_wcs_rejections"]["files"] == ["missing_wcs.fits"] + assert frame_filtering["pointing_rejections"]["files"] == ["bad_pointing.fits"] + assert frame_filtering["lightcurve_dropped_point_count"] == 1 + + astrometry = aavso_json_header(output_text, "ASTROMETRY-XC") + assert astrometry["wcs_file"] == "wcs.fits" + assert astrometry["comparison_star_aavso_header"] == comp_star_header + + bad_pixel = aavso_json_header(output_text, "BAD_PIXEL-XC") + assert bad_pixel["enabled"] is True + assert bad_pixel["bad_pixel_count"] == 3 + assert bad_pixel["counts_path"] == "BadPixelDetectionCounts.fits" From ee374bf998ef142b04dea9f770c236bbd5732bc6 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 30 Apr 2026 11:28:45 +1000 Subject: [PATCH 032/116] Add KTMF decision metadata to photometry outputs --- exotic/exotic.py | 38 +++- exotic/output_files.py | 414 ++++++++++++++++++++++++++++++++++++- tests/test_output_files.py | 190 ++++++++++++++++- 3 files changed, 638 insertions(+), 4 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index bd766953..2e79df45 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -10248,6 +10248,29 @@ def extract_lightcurve_fit_ktmf_contributions(fit): return contributions if isinstance(contributions, list) else [] +def compact_comparison_attempt_for_output(attempt): + if not isinstance(attempt, dict): + return {} + + return { + 'rank': attempt.get('rank'), + 'comp_index': attempt.get('comp_index'), + 'label': attempt.get('label'), + 'selected': attempt.get('selected'), + 'selection_reason': attempt.get('selection_reason'), + 'ktmf_metric': attempt.get('ktmf_metric'), + 'ktmf_contributions': attempt.get('ktmf_contributions') or [], + 'transit_delta_bic': attempt.get('transit_delta_bic'), + 'eebls_snr': attempt.get('eebls_snr'), + 'residual_scatter': attempt.get('residual_scatter'), + 'fit_point_count': attempt.get('fit_point_count'), + 'transit_qc_status': attempt.get('transit_qc_status'), + 'transit_qc_summary': attempt.get('transit_qc_summary'), + 'rejected_by_transit_qc': attempt.get('rejected_by_transit_qc'), + 'failure_reason': attempt.get('failure_reason'), + } + + def format_transit_delta_bic(value): if value is None: return "n/a" @@ -13516,9 +13539,19 @@ def _main_impl(): calibration_field_score=comparison_calibration['field_score'], selection_basis=selection_basis, selection_metric=comparison_fit_search.get('selection_metric', 'ktmf'), + selected_comparison_selection_reason=selected_attempt.get('selection_reason'), + selected_comparison_attempt=compact_comparison_attempt_for_output(selected_attempt), + comparison_fit_attempt_summaries=[ + compact_comparison_attempt_for_output(attempt) + for attempt in comparison_fit_search.get('attempts', []) + ], comparison_ktmf_metric=selected_attempt.get('ktmf_metric', np.nan), + selected_comparison_ktmf_contributions=selected_attempt.get('ktmf_contributions') or [], comparison_eebls_snr=selected_attempt.get('eebls_snr', np.nan), - comparison_transit_delta_bic=selected_attempt.get('transit_delta_bic', np.nan)) + comparison_transit_delta_bic=selected_attempt.get('transit_delta_bic', np.nan), + selected_comparison_fit_point_count=selected_attempt.get('fit_point_count'), + selected_comparison_transit_qc_status=selected_attempt.get('transit_qc_status'), + selected_comparison_transit_qc_summary=selected_attempt.get('transit_qc_summary')) flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) @@ -14329,7 +14362,8 @@ def _main_impl(): comp_star=bestCompStar, comp_coords=comp_coords, min_aper=np.round(display_aperture, 2), min_annul=np.round(display_annulus, 2), - adaptive_summary=photometry_info.get('adaptive_summary')) + adaptive_summary=photometry_info.get('adaptive_summary'), + photometry_info=photometry_info) else: output_files.final_planetary_params(phot_opt=False, vsp_params=vsp_params) except Exception as e: diff --git a/exotic/output_files.py b/exotic/output_files.py index 07790b91..bc5db7fe 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -185,6 +185,133 @@ def residual_scatter_fraction(fit): return np.nan +def fit_data_model_uncertainty(fit): + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if data.ndim != 1 or data.size == 0: + return None, None, None + + model = getattr(fit, 'model', None) + if model is None: + residuals = np.asarray(getattr(fit, 'residuals', np.array([])), dtype=float) + if residuals.shape == data.shape: + model = data - residuals + if model is None: + transit_model = getattr(fit, 'transit', None) + systematics_model = getattr(fit, 'airmass_model', None) + if transit_model is not None and systematics_model is not None: + model = np.asarray(transit_model, dtype=float) * np.asarray(systematics_model, dtype=float) + if model is None: + return data, None, None + + model = np.asarray(model, dtype=float) + if model.shape != data.shape: + return data, None, None + + uncertainty = getattr(fit, 'dataerr', None) + if uncertainty is not None: + uncertainty = np.asarray(uncertainty, dtype=float) + if uncertainty.shape != data.shape: + uncertainty = None + + return data, model, uncertainty + + +def infer_fit_quality_parameter_count(fit): + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + parameter_count = finite_float(transit_qc.get('transit_parameter_count')) + if np.isfinite(parameter_count) and parameter_count > 0: + return int(parameter_count) + + bounds = getattr(fit, 'bounds', None) + if isinstance(bounds, dict) and bounds: + return len(bounds) + + parameters = getattr(fit, 'parameters', None) + if isinstance(parameters, dict) and parameters: + return len(parameters) + + return 0 + + +def build_fit_quality_metadata(fit): + data, model, uncertainty = fit_data_model_uncertainty(fit) + if data is None or model is None: + return {} + + residuals = data - model + finite_mask = np.isfinite(data) & np.isfinite(model) & np.isfinite(residuals) + if not np.any(finite_mask): + return {} + + finite_residuals = residuals[finite_mask] + median_flux = np.nanmedian(data[finite_mask]) + rms_residual = float(np.sqrt(np.nanmean(finite_residuals ** 2))) + mad_residual = float(np.nanmedian(np.abs(finite_residuals))) + residual_scatter = ( + float(np.nanstd(finite_residuals) / median_flux) + if np.isfinite(median_flux) and median_flux != 0 + else np.nan + ) + point_count = int(np.count_nonzero(finite_mask)) + parameter_count = infer_fit_quality_parameter_count(fit) + degrees_of_freedom = point_count - parameter_count + + payload = { + 'point_count': point_count, + 'parameter_count': parameter_count, + 'degrees_of_freedom': degrees_of_freedom, + 'rms_residual': rms_residual, + 'rms_residual_percent': ( + 100.0 * rms_residual / median_flux + if np.isfinite(median_flux) and median_flux != 0 + else np.nan + ), + 'median_absolute_residual': mad_residual, + 'median_flux': float(median_flux) if np.isfinite(median_flux) else np.nan, + 'residual_scatter': residual_scatter, + 'residual_scatter_percent': 100.0 * residual_scatter if np.isfinite(residual_scatter) else np.nan, + 'uses_uncertainties': False, + } + + if uncertainty is None: + return payload + + uncertainty_mask = finite_mask & np.isfinite(uncertainty) & (uncertainty > 0) + if not np.any(uncertainty_mask): + return payload + + weighted_residuals = residuals[uncertainty_mask] + weighted_uncertainties = uncertainty[uncertainty_mask] + normalized_residuals = weighted_residuals / weighted_uncertainties + chi_square = float(np.sum(normalized_residuals ** 2)) + weighted_point_count = int(np.count_nonzero(uncertainty_mask)) + weighted_degrees_of_freedom = weighted_point_count - parameter_count + median_uncertainty = float(np.nanmedian(weighted_uncertainties)) + + payload.update({ + 'uses_uncertainties': True, + 'weighted_point_count': weighted_point_count, + 'degrees_of_freedom': weighted_degrees_of_freedom, + 'chi_square': chi_square, + 'reduced_chi_square': ( + chi_square / weighted_degrees_of_freedom + if weighted_degrees_of_freedom > 0 + else np.nan + ), + 'rms_normalized_residual': float(np.sqrt(np.nanmean(normalized_residuals ** 2))), + 'median_absolute_normalized_residual': float(np.nanmedian(np.abs(normalized_residuals))), + 'max_absolute_normalized_residual': float(np.nanmax(np.abs(normalized_residuals))), + 'median_uncertainty': median_uncertainty, + 'rms_residual_to_median_uncertainty': ( + rms_residual / median_uncertainty + if np.isfinite(median_uncertainty) and median_uncertainty > 0 + else np.nan + ), + }) + return payload + + def photometry_method_from_info(photometry_info): if not isinstance(photometry_info, dict): return None @@ -223,6 +350,284 @@ def build_aavso_qc_metadata(fit): return {field: transit_qc.get(field) for field in fields if field in transit_qc} +def compact_ktmf_contributions(contributions): + compact = [] + for contribution in contributions or []: + if not isinstance(contribution, dict): + continue + compact.append({ + 'label': contribution.get('label'), + 'available': contribution.get('available'), + 'points': contribution.get('points'), + 'max_points': contribution.get('max_points'), + 'score': contribution.get('score'), + 'detail': contribution.get('detail'), + }) + return compact + + +def compact_comparison_attempt_decision(attempt): + if not isinstance(attempt, dict): + return {} + + comp_index = attempt.get('comp_index') + try: + comp_number = int(comp_index) + 1 + except (TypeError, ValueError): + comp_number = None + + return { + 'rank': attempt.get('rank'), + 'comparison_star': comp_number, + 'label': attempt.get('label'), + 'selected': attempt.get('selected'), + 'selection_reason': attempt.get('selection_reason'), + 'ktmf_metric': attempt.get('ktmf_metric'), + 'ktmf_contributions': compact_ktmf_contributions(attempt.get('ktmf_contributions')), + 'transit_delta_bic': attempt.get('transit_delta_bic'), + 'eebls_snr': attempt.get('eebls_snr'), + 'residual_scatter': attempt.get('residual_scatter'), + 'fit_point_count': attempt.get('fit_point_count'), + 'transit_qc_status': attempt.get('transit_qc_status'), + 'transit_qc_summary': attempt.get('transit_qc_summary'), + 'rejected_by_transit_qc': attempt.get('rejected_by_transit_qc'), + 'failure_reason': attempt.get('failure_reason'), + } + + +def compact_comparison_attempt_decisions(attempts, limit=10): + attempts = list(attempts or []) + return { + 'candidate_count': len(attempts), + 'candidates': [ + compact_comparison_attempt_decision(attempt) + for attempt in attempts[:limit] + ], + 'omitted_candidate_count': max(0, len(attempts) - limit), + } + + +def build_ktmf_decision_metadata(fit, photometry_info=None): + transit_qc = getattr(fit, 'transit_qc', None) + payload = {} + if isinstance(transit_qc, dict): + payload['target_fit'] = { + 'status': transit_qc.get('status'), + 'summary': transit_qc.get('summary'), + 'ktmf_metric': transit_qc.get('ktmf_metric'), + 'ktmf_contributions': compact_ktmf_contributions(transit_qc.get('ktmf_contributions')), + 'delta_bic': transit_qc.get('delta_bic'), + 'delta_chi2': transit_qc.get('delta_chi2'), + 'eebls_depth_snr': transit_qc.get('eebls_depth_snr'), + 'residual_scatter': transit_qc.get('residual_scatter'), + 'deviation_from_expected_value': transit_qc.get('deviation_from_expected_value'), + } + + if isinstance(photometry_info, dict): + selected_attempt = photometry_info.get('selected_comparison_attempt') + selected_payload = compact_comparison_attempt_decision(selected_attempt) + if not selected_payload: + selected_payload = { + 'comparison_star': photometry_info.get('comp_star_num'), + 'selected': photometry_info.get('comp_star_num') is not None, + 'selection_reason': photometry_info.get('selected_comparison_selection_reason'), + 'ktmf_metric': photometry_info.get('comparison_ktmf_metric'), + 'ktmf_contributions': compact_ktmf_contributions( + photometry_info.get('selected_comparison_ktmf_contributions') + ), + 'transit_delta_bic': photometry_info.get('comparison_transit_delta_bic'), + 'eebls_snr': photometry_info.get('comparison_eebls_snr'), + 'fit_point_count': photometry_info.get('selected_comparison_fit_point_count'), + 'transit_qc_status': photometry_info.get('selected_comparison_transit_qc_status'), + 'transit_qc_summary': photometry_info.get('selected_comparison_transit_qc_summary'), + } + + payload['comparison_selection'] = { + 'basis': photometry_info.get('selection_basis'), + 'metric': photometry_info.get('selection_metric'), + 'field_score': photometry_info.get('calibration_field_score'), + 'selected': selected_payload, + } + + attempt_summary = compact_comparison_attempt_decisions( + photometry_info.get('comparison_fit_attempt_summaries') + ) + if attempt_summary['candidate_count']: + payload['comparison_selection'].update(attempt_summary) + + return payload + + +def format_ktmf_metric(value): + value = finite_float(value) + return f"{value:.2f} / 5.00" if np.isfinite(value) else "n/a" + + +def format_optional_metric(label, value, precision=2): + value = finite_float(value) + if not np.isfinite(value): + return None + return f"{label}={value:.{precision}f}" + + +def format_ktmf_candidate_decision(attempt): + attempt = compact_comparison_attempt_decision(attempt) + label = attempt.get('label') or ( + f"Comp {attempt['comparison_star']}" if attempt.get('comparison_star') is not None else "Comparison candidate" + ) + selected_text = " [selected]" if attempt.get('selected') else "" + parts = [ + f"{label}{selected_text}: KTMF={format_ktmf_metric(attempt.get('ktmf_metric'))}", + ] + for metric_text in ( + format_optional_metric("Delta BIC", attempt.get('transit_delta_bic')), + format_optional_metric("EEBLS SNR", attempt.get('eebls_snr')), + ): + if metric_text: + parts.append(metric_text) + qc_status = attempt.get('transit_qc_status') + if qc_status: + parts.append(f"QC={str(qc_status).upper()}") + reason = attempt.get('selection_reason') or attempt.get('failure_reason') + if reason: + parts.append(f"reason={reason}") + return ", ".join(parts) + + +def format_ktmf_decision_final_params(fit, photometry_info=None): + params = {} + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + ktmf_metric = finite_float(transit_qc.get('ktmf_metric')) + if np.isfinite(ktmf_metric): + target_status = str(transit_qc.get('status', 'unknown')).upper() + params["KTMF target-fit decision"] = ( + f"{target_status}: KTMF={format_ktmf_metric(ktmf_metric)}" + ) + for contribution_index, contribution in enumerate( + compact_ktmf_contributions(transit_qc.get('ktmf_contributions')), + start=1, + ): + label = contribution.get('label', f'Component {contribution_index}') + available = bool(contribution.get('available')) + points = finite_float(contribution.get('points'), 0.0) + max_points = finite_float(contribution.get('max_points'), 0.0) + score = finite_float(contribution.get('score')) + detail = contribution.get('detail') or 'n/a' + if available and np.isfinite(score): + params[f"KTMF target contribution {contribution_index}"] = ( + f"{label}: +{points:.2f}/{max_points:.2f} (score={score:.2f}; {detail})" + ) + else: + params[f"KTMF target contribution {contribution_index}"] = ( + f"{label}: +0.00/0.00 (unavailable; {detail})" + ) + + if not isinstance(photometry_info, dict): + return params + + basis = photometry_info.get('selection_basis') + metric = photometry_info.get('selection_metric') + if basis or metric: + params["KTMF comparison selection mode"] = ( + f"basis={basis or 'n/a'}, metric={metric or 'n/a'}" + ) + + selected_attempt = photometry_info.get('selected_comparison_attempt') + if selected_attempt: + params["KTMF selected comparison decision"] = format_ktmf_candidate_decision(selected_attempt) + elif photometry_info.get('comp_star_num') is not None: + selected_payload = { + 'label': f"Comp {photometry_info.get('comp_star_num')}", + 'selected': True, + 'selection_reason': photometry_info.get('selected_comparison_selection_reason'), + 'ktmf_metric': photometry_info.get('comparison_ktmf_metric'), + 'ktmf_contributions': photometry_info.get('selected_comparison_ktmf_contributions'), + 'transit_delta_bic': photometry_info.get('comparison_transit_delta_bic'), + 'eebls_snr': photometry_info.get('comparison_eebls_snr'), + 'transit_qc_status': photometry_info.get('selected_comparison_transit_qc_status'), + } + params["KTMF selected comparison decision"] = format_ktmf_candidate_decision(selected_payload) + + for contribution_index, contribution in enumerate( + compact_ktmf_contributions(photometry_info.get('selected_comparison_ktmf_contributions')), + start=1, + ): + label = contribution.get('label', f'Component {contribution_index}') + available = bool(contribution.get('available')) + points = finite_float(contribution.get('points'), 0.0) + max_points = finite_float(contribution.get('max_points'), 0.0) + score = finite_float(contribution.get('score')) + detail = contribution.get('detail') or 'n/a' + if available and np.isfinite(score): + params[f"KTMF selected comparison contribution {contribution_index}"] = ( + f"{label}: +{points:.2f}/{max_points:.2f} (score={score:.2f}; {detail})" + ) + else: + params[f"KTMF selected comparison contribution {contribution_index}"] = ( + f"{label}: +0.00/0.00 (unavailable; {detail})" + ) + + for attempt_index, attempt in enumerate( + (photometry_info.get('comparison_fit_attempt_summaries') or [])[:10], + start=1, + ): + params[f"KTMF comparison candidate {attempt_index}"] = format_ktmf_candidate_decision(attempt) + + return params + + +def format_fit_quality_final_params(fit_quality): + fit_quality = fit_quality or {} + params = {} + + reduced_chi_square = finite_float(fit_quality.get('reduced_chi_square')) + if np.isfinite(reduced_chi_square): + params["Fit quality reduced chi-square"] = f"{reduced_chi_square:.3f}" + + chi_square = finite_float(fit_quality.get('chi_square')) + if np.isfinite(chi_square): + params["Fit quality chi-square"] = f"{chi_square:.2f}" + + degrees_of_freedom = fit_quality.get('degrees_of_freedom') + try: + degrees_of_freedom = int(degrees_of_freedom) + except (TypeError, ValueError): + degrees_of_freedom = None + if degrees_of_freedom is not None: + params["Fit quality degrees of freedom"] = str(degrees_of_freedom) + + rms_residual_percent = finite_float(fit_quality.get('rms_residual_percent')) + if np.isfinite(rms_residual_percent): + params["Fit quality RMS residual"] = f"{rms_residual_percent:.4f} %" + + median_abs_normalized_residual = finite_float( + fit_quality.get('median_absolute_normalized_residual') + ) + if np.isfinite(median_abs_normalized_residual): + params["Fit quality median absolute normalized residual"] = ( + f"{median_abs_normalized_residual:.2f} sigma" + ) + + rms_uncertainty_ratio = finite_float(fit_quality.get('rms_residual_to_median_uncertainty')) + if np.isfinite(rms_uncertainty_ratio): + params["Fit quality RMS residual / median uncertainty"] = f"{rms_uncertainty_ratio:.2f}" + + point_count = fit_quality.get('weighted_point_count', fit_quality.get('point_count')) + try: + point_count = int(point_count) + except (TypeError, ValueError): + point_count = None + if point_count is not None: + params["Fit quality point count"] = str(point_count) + + if fit_quality and not fit_quality.get('uses_uncertainties'): + params["Fit quality note"] = "Per-point uncertainties unavailable; chi-square metrics not reported." + + return params + + def build_aavso_photometry_metadata(photometry_info): if not isinstance(photometry_info, dict): return {} @@ -414,10 +819,11 @@ def final_lightcurve(self, phase): f.write(f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}\n") def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, - min_annul=None, adaptive_summary=None): + min_annul=None, adaptive_summary=None, photometry_info=None): params_file = self.dir / "temp" / f"FinalParams_{self.p_dict['pName']}_{self.i_dict['date']}.json" transit_qc = getattr(self.fit, 'transit_qc', None) + fit_quality = build_fit_quality_metadata(self.fit) qc_residual_scatter = np.nan if isinstance(transit_qc, dict): qc_residual_scatter = transit_qc.get('residual_scatter', np.nan) @@ -456,6 +862,8 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num["Impact Parameter (b)"] = impact_text if np.isfinite(qc_residual_scatter): params_num["Residual scatter around full model fit"] = f"{qc_residual_scatter * 100.0:.4f} %" + params_num.update(format_fit_quality_final_params(fit_quality)) + params_num.update(format_ktmf_decision_final_params(self.fit, photometry_info)) if getattr(self.fit, 'airmass_fit_skipped', False): params_num["Airmass correction"] = getattr( self.fit, @@ -589,6 +997,8 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, aavso_airmass_terms = aavso_airmass_results(self.fit) detrend_model = aavso_detrend_model(self.fit) qc_metadata = build_aavso_qc_metadata(self.fit) + fit_quality_metadata = build_fit_quality_metadata(self.fit) + ktmf_decision_metadata = build_ktmf_decision_metadata(self.fit, photometry_info) photometry_metadata = build_aavso_photometry_metadata(photometry_info) aperture_metadata = build_aavso_aperture_metadata(photometry_info) frame_filtering_metadata = build_aavso_frame_filtering_metadata(self.fit, frame_filtering_info) @@ -648,6 +1058,8 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, f",{aavso_airmass_terms[1][0]}={aavso_airmass_terms[1][1]} +/- {aavso_airmass_terms[1][2]}\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields f.write(format_aavso_json_header("QC-XC", qc_metadata)) + f.write(format_aavso_json_header("FIT_QUALITY-XC", fit_quality_metadata)) + f.write(format_aavso_json_header("KTMF_DECISION-XC", ktmf_decision_metadata)) f.write(format_aavso_json_header("PHOTOMETRY-XC", photometry_metadata)) f.write(format_aavso_json_header("APERTURE-XC", aperture_metadata)) f.write(format_aavso_json_header("FRAME_FILTERING-XC", frame_filtering_metadata)) diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 0d9662ce..d58afc21 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -355,6 +355,127 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): assert "KTMF contribution 1" in output_text +def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): + fit = DummyFit() + fit.transit_qc = { + "status": "pass", + "summary": "Transit model strongly preferred over flat/null model.", + "ktmf_metric": 4.63, + "delta_bic": 18.4, + "delta_chi2": 27.1, + "ktmf_contributions": [ + { + "label": "Model Evidence", + "available": True, + "points": 0.74, + "max_points": 0.80, + "score": 0.93, + "detail": "Delta BIC=18.40", + } + ], + } + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + photometry_info = { + "selection_basis": "comparison_field_retry", + "selection_metric": "ktmf", + "comp_star_num": 2, + "comparison_ktmf_metric": 4.60, + "comparison_eebls_snr": 5.2, + "comparison_transit_delta_bic": 18.4, + "selected_comparison_selection_reason": "selected: highest KTMF among candidates", + "selected_comparison_ktmf_contributions": [ + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.63, + "max_points": 0.70, + "score": 0.90, + "detail": "0.3500%", + } + ], + "comparison_fit_attempt_summaries": [ + { + "rank": 1, + "comp_index": 0, + "label": "Comp 1", + "selected": False, + "selection_reason": "not selected: KTMF 3.20/5.00 was lower than the selected 4.60/5.00", + "ktmf_metric": 3.2, + "transit_delta_bic": 8.1, + "eebls_snr": 4.2, + "transit_qc_status": "marginal", + }, + { + "rank": 2, + "comp_index": 1, + "label": "Comp 2", + "selected": True, + "selection_reason": "selected: highest KTMF among candidates", + "ktmf_metric": 4.6, + "transit_delta_bic": 18.4, + "eebls_snr": 5.2, + "transit_qc_status": "pass", + }, + ], + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=True, + vsp_params=[], + comp_star=2, + comp_coords=[300.5, 400.5], + min_aper=7.5, + min_annul=22.5, + photometry_info=photometry_info, + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_data = json.loads(output_file.read_text(encoding="utf-8")) + final_params = output_data["FINAL PLANETARY PARAMETERS"] + + assert final_params["KTMF target-fit decision"] == "PASS: KTMF=4.63 / 5.00" + assert final_params["KTMF comparison selection mode"] == "basis=comparison_field_retry, metric=ktmf" + assert "selected: highest KTMF" in final_params["KTMF selected comparison decision"] + assert "Comp 1" in final_params["KTMF comparison candidate 1"] + assert "not selected: KTMF" in final_params["KTMF comparison candidate 1"] + assert "Residual Scatter Around Full Model Fit" in final_params["KTMF selected comparison contribution 1"] + + +def test_final_planetary_params_reports_absolute_fit_quality(tmp_path): + fit = DummyFit() + fit.data = np.array([1.0, 1.02, 0.98, 1.01, 0.99, 1.0]) + fit.model = np.ones(6, dtype=float) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full(6, 0.01, dtype=float) + fit.time = np.arange(6, dtype=float) + fit.airmass_model = np.ones(6, dtype=float) + fit.bounds = {"tmid": [0, 1], "rprs": [0, 1], "a1": [0, 2]} + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_data = json.loads(output_file.read_text(encoding="utf-8")) + final_params = output_data["FINAL PLANETARY PARAMETERS"] + + assert final_params["Fit quality reduced chi-square"] == "3.333" + assert final_params["Fit quality chi-square"] == "10.00" + assert final_params["Fit quality degrees of freedom"] == "3" + assert final_params["Fit quality RMS residual"] == "1.2910 %" + assert final_params["Fit quality median absolute normalized residual"] == "1.00 sigma" + assert final_params["Fit quality RMS residual / median uncertainty"] == "1.29" + assert final_params["Fit quality point count"] == "6" + + def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_path): fit = DummyFit() fit.airmass_fit_skipped = True @@ -415,6 +536,13 @@ def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_p def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): fit = DummyFit() + fit.data = np.array([1.0, 1.02, 0.98, 1.01, 0.99, 1.0]) + fit.model = np.ones(6, dtype=float) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full(6, 0.01, dtype=float) + fit.time = np.arange(6, dtype=float) + 2450000.0 + fit.airmass_model = np.ones(6, dtype=float) + fit.bounds = {"tmid": [0, 1], "rprs": [0, 1], "a1": [0, 2]} fit.transit_qc = { "computed": True, "status": "pass", @@ -497,6 +625,52 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): "comparison_ktmf_metric": 4.6, "comparison_eebls_snr": 5.2, "comparison_transit_delta_bic": 18.4, + "selected_comparison_selection_reason": "selected: highest KTMF among candidates", + "selected_comparison_ktmf_contributions": [ + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.63, + "max_points": 0.70, + "score": 0.90, + "detail": "0.3500%", + } + ], + "comparison_fit_attempt_summaries": [ + { + "rank": 1, + "comp_index": 0, + "label": "Comp 1", + "selected": False, + "selection_reason": "not selected: KTMF 3.20/5.00 was lower than the selected 4.60/5.00", + "ktmf_metric": 3.2, + "transit_delta_bic": 8.1, + "eebls_snr": 4.2, + "transit_qc_status": "marginal", + "ktmf_contributions": [], + }, + { + "rank": 2, + "comp_index": 1, + "label": "Comp 2", + "selected": True, + "selection_reason": "selected: highest KTMF among candidates", + "ktmf_metric": 4.6, + "transit_delta_bic": 18.4, + "eebls_snr": 5.2, + "transit_qc_status": "pass", + "ktmf_contributions": [ + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.63, + "max_points": 0.70, + "score": 0.90, + "detail": "0.3500%", + } + ], + }, + ], "reuse_selected_full_reduction_fit": True, "selected_source_indices": np.array([0, 2, 3]), "selected_fit_good_times": np.array([2450000.0, 2450000.1, 2450000.2]), @@ -522,7 +696,7 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( comp_star_header, - [1.0], + np.ones(6, dtype=float), (0.1, 0.01), (0.2, 0.01), (0.3, 0.01), @@ -570,6 +744,20 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): assert qc["ktmf_metric"] == pytest.approx(4.63) assert qc["ktmf_contributions"][0]["label"] == "Model Evidence" + fit_quality = aavso_json_header(output_text, "FIT_QUALITY-XC") + assert fit_quality["reduced_chi_square"] == pytest.approx(10.0 / 3.0) + assert fit_quality["chi_square"] == pytest.approx(10.0) + assert fit_quality["degrees_of_freedom"] == 3 + assert fit_quality["median_absolute_normalized_residual"] == pytest.approx(1.0) + + ktmf_decision = aavso_json_header(output_text, "KTMF_DECISION-XC") + assert ktmf_decision["target_fit"]["ktmf_metric"] == pytest.approx(4.63) + assert ktmf_decision["comparison_selection"]["basis"] == "comparison_field" + assert ktmf_decision["comparison_selection"]["metric"] == "ktmf" + assert ktmf_decision["comparison_selection"]["selected"]["selection_reason"] == "selected: highest KTMF among candidates" + assert ktmf_decision["comparison_selection"]["candidate_count"] == 2 + assert ktmf_decision["comparison_selection"]["candidates"][0]["selection_reason"].startswith("not selected: KTMF") + photometry = aavso_json_header(output_text, "PHOTOMETRY-XC") assert photometry["selected_comparison_star"] == 2 assert photometry["comparison_field_score_percent"] == pytest.approx(0.42) From 5fff787d747c8a6debbd63ab502610a15a9aa70e Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 30 Apr 2026 14:37:54 +1000 Subject: [PATCH 033/116] Cache and multiprocess pointing precheck alignment --- exotic/api/ultranest_utils.py | 89 ++- exotic/exotic.py | 1036 ++++++++++++++++++++++++++------- tests/test_centroid_wcs.py | 156 ++++- tests/test_ultranest_utils.py | 145 +++++ 4 files changed, 1195 insertions(+), 231 deletions(-) diff --git a/exotic/api/ultranest_utils.py b/exotic/api/ultranest_utils.py index fb91b4af..8b4db06d 100644 --- a/exotic/api/ultranest_utils.py +++ b/exotic/api/ultranest_utils.py @@ -1,3 +1,4 @@ +import gc import logging import math import multiprocessing @@ -12,6 +13,7 @@ _TRUTHY = {"1", "true", "yes", "on"} _FALSEY = {"0", "false", "no", "off", "n"} +_AUTO_WORKER_VALUES = {"auto", "all", "available", "cpu", "cpus", "core", "cores"} DEFAULT_PROGRESS_INTERVAL_SECONDS = 10.0 DEFAULT_MIN_NUM_LIVE_POINTS = 200 DEFAULT_RUN_KWARGS = { @@ -44,6 +46,7 @@ ) ULTRANEST_WORKER_BACKEND_ENV = "EXOTIC_ULTRANEST_WORKER_BACKEND" _PROCESS_LOGLIKE = None +_TK_CLEANUP_CLASSES = ("Image", "Variable") def _is_enabled(value): @@ -54,6 +57,10 @@ def _is_disabled(value): return str(value).strip().lower() in _FALSEY +def _is_auto_worker_count(value): + return str(value).strip().lower() in _AUTO_WORKER_VALUES + + def _coerce_positive_int(value, default=None): try: parsed = int(float(str(value).strip())) @@ -118,6 +125,15 @@ def _is_colab_runtime(): ) +def _available_cpu_count(): + process_cpu_count = getattr(os, "process_cpu_count", None) + if callable(process_cpu_count): + count = process_cpu_count() + else: + count = os.cpu_count() + return max(_coerce_positive_int(count, default=1), 1) + + def _configured_ultranest_workers(): for env_key in ULTRANEST_WORKER_ENV_KEYS: value = os.environ.get(env_key) @@ -125,11 +141,13 @@ def _configured_ultranest_workers(): continue if _is_disabled(value): return 1 + if _is_auto_worker_count(value): + return _available_cpu_count() parsed = _coerce_positive_int(value, default=None) if parsed is not None: return parsed - return 1 + return _available_cpu_count() def _configured_ultranest_worker_backend(): @@ -147,6 +165,53 @@ def _process_loglike_chunk(chunk): return _PROCESS_LOGLIKE(chunk) +def _noop_tk_destructor(_instance): + return None + + +def _suppress_inherited_tk_cleanup(): + """Avoid noisy Tk destructor calls in forked worker processes.""" + tkinter_module = sys.modules.get("tkinter") + if tkinter_module is None: + return False + + patched = False + for class_name in _TK_CLEANUP_CLASSES: + tk_class = getattr(tkinter_module, class_name, None) + if tk_class is None or getattr(tk_class, "_exotic_worker_tk_cleanup_suppressed", False): + continue + + try: + original_del = getattr(tk_class, "__del__", None) + if original_del is None: + continue + setattr(tk_class, "_exotic_worker_original_del", original_del) + setattr(tk_class, "__del__", _noop_tk_destructor) + setattr(tk_class, "_exotic_worker_tk_cleanup_suppressed", True) + patched = True + except Exception: + continue + + return patched + + +def suppress_inherited_tk_cleanup_in_worker(): + return _suppress_inherited_tk_cleanup() + + +@contextmanager +def suppress_tk_cleanup_during_process_pool(): + suppress_inherited_tk_cleanup_in_worker() + restore_gc_after_pool = gc.isenabled() + if restore_gc_after_pool: + gc.disable() + try: + yield + finally: + if restore_gc_after_pool: + gc.enable() + + @contextmanager def _parallel_vectorized_loglike(sampler, workers=None): worker_count = max(int(workers or _configured_ultranest_workers()), 1) @@ -162,14 +227,22 @@ def _parallel_vectorized_loglike(sampler, workers=None): pool = None executor = None + process_pool_guard = None if backend == "process": if not sys.platform.startswith("linux") or _is_colab_runtime(): yield 1, "single" return global _PROCESS_LOGLIKE + process_pool_guard = suppress_tk_cleanup_during_process_pool() + process_pool_guard.__enter__() _PROCESS_LOGLIKE = original_loglike - ctx = multiprocessing.get_context("fork") - pool = ctx.Pool(processes=worker_count) + try: + ctx = multiprocessing.get_context("fork") + pool = ctx.Pool(processes=worker_count, initializer=suppress_inherited_tk_cleanup_in_worker) + except Exception: + _PROCESS_LOGLIKE = None + process_pool_guard.__exit__(*sys.exc_info()) + raise elif backend == "thread": executor = ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix="exotic-ultranest") else: @@ -195,9 +268,13 @@ def parallel_loglike(params): finally: sampler.loglike = original_loglike if pool is not None: - pool.close() - pool.join() - _PROCESS_LOGLIKE = None + try: + pool.close() + pool.join() + finally: + _PROCESS_LOGLIKE = None + if process_pool_guard is not None: + process_pool_guard.__exit__(None, None, None) if executor is not None: executor.shutdown(wait=True) diff --git a/exotic/exotic.py b/exotic/exotic.py index 2e79df45..f9843560 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -126,9 +126,17 @@ except ImportError: # package import from api.plate_solution import NextAstroPlateSolution, PlateSolution try: - from .api.ultranest_utils import get_mpi_status + from .api.ultranest_utils import ( + get_mpi_status, + suppress_inherited_tk_cleanup_in_worker, + suppress_tk_cleanup_during_process_pool, + ) except ImportError: - from api.ultranest_utils import get_mpi_status + from api.ultranest_utils import ( + get_mpi_status, + suppress_inherited_tk_cleanup_in_worker, + suppress_tk_cleanup_during_process_pool, + ) try: from .api.http_compression import build_compressed_json_request except ImportError: @@ -6399,15 +6407,45 @@ def log_pointing_precheck_alignment_progress(i, total_files, file_name): ) -def collect_transform_frame_pointings(inputfiles, frame_loader=None): +def _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms): + if return_transforms: + return positions, usable_mask, alignment_transforms + return positions, usable_mask + + +def _filter_alignment_transform_cache(alignment_transforms, retained_files): + if not alignment_transforms: + return {} + + retained_keys = {str(file_name) for file_name in retained_files} + return { + file_key: tform + for file_key, tform in alignment_transforms.items() + if file_key in retained_keys + } + + +def collect_transform_frame_pointings(inputfiles, frame_loader=None, return_transforms=False, + multiprocess_transformations=None, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None): positions = np.full((len(inputfiles), 2), np.nan, dtype=float) usable_mask = np.zeros(len(inputfiles), dtype=bool) + alignment_transforms = {} if len(inputfiles) == 0: - return positions, usable_mask + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) if frame_loader is None: - frame_loader = load_image_data + frame_loader = lambda file_name: load_calibrated_reduction_image( + file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ) total_files = len(inputfiles) log_pointing_precheck_alignment_progress(0, total_files, inputfiles[0]) @@ -6419,16 +6457,38 @@ def collect_transform_frame_pointings(inputfiles, frame_loader=None): f"{_display_filename(inputfiles[0])} ({exc}).", warn=True, ) - return positions, usable_mask + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) if getattr(reference_image, "ndim", 0) != 2: log_info("Warning: pointing precheck alignment fallback requires 2-D images; skipping.", warn=True) - return positions, usable_mask + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) height, width = reference_image.shape reference_anchor = np.array([[(width - 1) / 2.0, (height - 1) / 2.0]], dtype=float) positions[0] = reference_anchor[0] usable_mask[0] = True + alignment_transforms[str(inputfiles[0])] = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + if multiprocess_transformations is not None and multiprocess_transformations > 0 and len(inputfiles) > 1: + try: + positions, usable_mask, alignment_transforms = build_multiprocess_pointing_precheck_transforms( + inputfiles, + multiprocess_transformations, + reference_anchor, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + ) + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) + except Exception as exc: + log_info( + "Warning: pointing precheck multiprocessing failed; falling back to serial alignment " + f"({exc}).", + warn=True, + ) for index, file_name in enumerate(inputfiles[1:], start=1): log_pointing_precheck_alignment_progress(index, total_files, file_name) @@ -6447,10 +6507,11 @@ def collect_transform_frame_pointings(inputfiles, frame_loader=None): if np.all(np.isfinite(mapped_anchor)): positions[index] = mapped_anchor usable_mask[index] = True + alignment_transforms[str(file_name)] = tform except Exception: continue - return positions, usable_mask + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) def sigma_clip_pointing_positions(positions, sigma=3.0, max_iters=5): @@ -6501,19 +6562,33 @@ def sigma_clip_pointing_positions(positions, sigma=3.0, max_iters=5): def filter_pointing_outlier_frames(inputfiles, pointing_rejection_sigma=None, ignore_header_wcs=False, - frame_loader=None): + frame_loader=None, return_alignment_transforms=False, + multiprocess_transformations=None, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None): inputfiles = np.array(inputfiles) keep_mask = np.ones(len(inputfiles), dtype=bool) + alignment_transforms = {} + + def format_result(result_inputfiles, result_keep_mask, dropped_files): + if return_alignment_transforms: + return ( + result_inputfiles, + result_keep_mask, + dropped_files, + _filter_alignment_transform_cache(alignment_transforms, result_inputfiles), + ) + return result_inputfiles, result_keep_mask, dropped_files if len(inputfiles) == 0 or pointing_rejection_sigma is None: - return inputfiles, keep_mask, [] + return format_result(inputfiles, keep_mask, []) if len(inputfiles) < POINTING_REJECTION_MIN_FRAMES: log_info( f"Pointing precheck skipped: only {len(inputfiles)} frame(s); " f"need at least {POINTING_REJECTION_MIN_FRAMES}.", ) - return inputfiles, keep_mask, [] + return format_result(inputfiles, keep_mask, []) positions = None usable_mask = None @@ -6535,7 +6610,18 @@ def filter_pointing_outlier_frames(inputfiles, pointing_rejection_sigma=None, ig log_info("Pointing precheck: no usable WCS-derived pointing centers found; using alignment-derived positions.") if positions is None: - positions, usable_mask = collect_transform_frame_pointings(inputfiles, frame_loader=frame_loader) + positions, usable_mask, alignment_transforms = collect_transform_frame_pointings( + inputfiles, + frame_loader=frame_loader, + return_transforms=True, + multiprocess_transformations=multiprocess_transformations, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + ) mode_label = "alignment" usable_count = int(np.count_nonzero(usable_mask)) @@ -6544,7 +6630,7 @@ def filter_pointing_outlier_frames(inputfiles, pointing_rejection_sigma=None, ig f"Pointing precheck skipped: only {usable_count} usable {mode_label}-derived pointing estimate(s); " f"need at least {POINTING_REJECTION_MIN_FRAMES}.", ) - return inputfiles, keep_mask, [] + return format_result(inputfiles, keep_mask, []) keep_mask[np.flatnonzero(usable_mask)] = sigma_clip_pointing_positions( positions[usable_mask], @@ -6558,7 +6644,7 @@ def filter_pointing_outlier_frames(inputfiles, pointing_rejection_sigma=None, ig f"Pointing precheck ({mode_label}): no frames exceeded the " f"{float(pointing_rejection_sigma):g}-sigma pointing threshold." ) - return inputfiles, keep_mask, [] + return format_result(inputfiles, keep_mask, []) retained_files = inputfiles[keep_mask] log_info( @@ -6567,7 +6653,7 @@ def filter_pointing_outlier_frames(inputfiles, pointing_rejection_sigma=None, ig "median pointing." ) log_file_preview(dropped_files, "Pointing precheck dropped files") - return retained_files, keep_mask, dropped_files + return format_result(retained_files, keep_mask, dropped_files) def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_missing_fraction=None): @@ -7764,6 +7850,7 @@ def _get_reference_transform_cache(reference_image, roi): def _transformation_pool_initializer(reference_file): global _TRANSFORM_REFERENCE_IMAGE, _TRANSFORM_REFERENCE_CACHE + suppress_inherited_tk_cleanup_in_worker() _TRANSFORM_REFERENCE_IMAGE = load_image_data(reference_file) _TRANSFORM_REFERENCE_CACHE = None @@ -7773,6 +7860,440 @@ def transformation_task_with_cached_reference(i, file_name): return i, transformation(image_data, file_name, report_failure=False, reference_image=_TRANSFORM_REFERENCE_IMAGE) +class _ParallelPlateStatusRecorder: + def __init__(self): + self.warnings = [] + + def setCurrentFilename(self, filename): + return self + + def outOfFrameWarning(self, starIndex): + self.warnings.append(('out_of_frame', int(starIndex), np.nan, np.nan)) + + def lowFluxAmplitudeWarning(self, starIndex, xc, yc): + self.warnings.append(('low_flux', int(starIndex), float(xc), float(yc))) + + def alignmentError(self): + self.warnings.append(('alignment_error', -1, np.nan, np.nan)) + + +_ALIGNMENT_POOL_CONTEXT = {} + + +def _alignment_pool_initializer(reference_file, generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult, bad_pixel_reference): + global _ALIGNMENT_POOL_CONTEXT, _TRANSFORM_REFERENCE_IMAGE, _TRANSFORM_REFERENCE_CACHE + suppress_inherited_tk_cleanup_in_worker() + _ALIGNMENT_POOL_CONTEXT = { + 'generalDark': generalDark, + 'generalBias': generalBias, + 'generalFlat': generalFlat, + 'demosaic_fmt': demosaic_fmt, + 'demosaic_out': demosaic_out, + 'demosaic_mult': demosaic_mult, + 'bad_pixel_reference': bad_pixel_reference, + } + _TRANSFORM_REFERENCE_IMAGE = load_calibrated_reduction_image( + reference_file, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + _TRANSFORM_REFERENCE_CACHE = None + + +def _load_alignment_worker_frame(file_name): + context = _ALIGNMENT_POOL_CONTEXT + hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + image_data = hdul[extension].data + hdul.close() + + image_data = apply_cals( + image_data, + context.get('generalDark'), + context.get('generalBias'), + context.get('generalFlat'), + 1, + ) + image_data = demosaic_img( + image_data, + context.get('demosaic_fmt'), + context.get('demosaic_out'), + context.get('demosaic_mult'), + 1, + ) + image_data = repair_bad_pixels_in_frame(image_data, context.get('bad_pixel_reference')) + return image_header, image_data + + +def _pointing_precheck_alignment_task(task): + i, file_name, reference_anchor = task + try: + _, image_data = _load_alignment_worker_frame(file_name) + if getattr(image_data, "ndim", 0) != 2: + return { + 'index': i, + 'file_name': file_name, + 'usable': False, + 'position': np.array([np.nan, np.nan], dtype=float), + 'transform': None, + } + + tform = transformation( + image_data, + file_name, + report_failure=False, + reference_image=_TRANSFORM_REFERENCE_IMAGE, + ) + mapped_anchor = np.asarray(tform(reference_anchor), dtype=float).reshape(-1, 2)[0] + usable = bool(np.all(np.isfinite(mapped_anchor))) + return { + 'index': i, + 'file_name': file_name, + 'usable': usable, + 'position': mapped_anchor, + 'transform': tform if usable else None, + } + except Exception as exc: + return { + 'index': i, + 'file_name': file_name, + 'usable': False, + 'position': np.array([np.nan, np.nan], dtype=float), + 'transform': None, + 'error': str(exc), + } + + +def build_multiprocess_pointing_precheck_transforms(inputfiles, max_processes, reference_anchor, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None): + total_jobs = len(inputfiles) + positions = np.full((total_jobs, 2), np.nan, dtype=float) + usable_mask = np.zeros(total_jobs, dtype=bool) + alignment_transforms = {} + if total_jobs == 0: + return positions, usable_mask, alignment_transforms + + reference_anchor = np.asarray(reference_anchor, dtype=float).reshape(-1, 2) + positions[0] = reference_anchor[0] + usable_mask[0] = True + alignment_transforms[str(inputfiles[0])] = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + if total_jobs == 1: + return positions, usable_mask, alignment_transforms + + max_workers = min(max_processes, os.cpu_count() or 1, total_jobs - 1, MAX_MULTIPROCESS_TRANSFORM_WORKERS) + log_info( + "Using multiprocessing for pointing precheck alignment " + f"with {max_workers} worker(s) across {total_jobs} image(s)." + ) + + tasks = [ + (i, str(file_name), reference_anchor) + for i, file_name in enumerate(inputfiles) + if i != 0 + ] + + with suppress_tk_cleanup_during_process_pool(): + with ProcessPoolExecutor( + max_workers=max_workers, + initializer=_alignment_pool_initializer, + initargs=( + str(inputfiles[0]), + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + None, + ), + ) as executor: + futures = [executor.submit(_pointing_precheck_alignment_task, task) for task in tasks] + completed = 1 + for future in as_completed(futures): + result = future.result() + index = result['index'] + if result.get('usable'): + positions[index] = result['position'] + usable_mask[index] = True + alignment_transforms[result['file_name']] = result['transform'] + completed += 1 + if completed == total_jobs or completed % 10 == 0: + log_info(f"Pointing precheck alignment progress: {completed}/{total_jobs}") + + return positions, usable_mask, alignment_transforms + + +def _fit_alignment_candidate_psfs(image_data, predicted_coords, target_fast_centroid, frame_fast_centroid): + global plateStatus + predicted_coords = np.asarray(predicted_coords, dtype=float) + original_plate_status = plateStatus + recorder = _ParallelPlateStatusRecorder() + plateStatus = recorder + try: + psf_rows = { + 'target': fit_centroid_or_warn_out_of_frame( + image_data, + choose_centroid_seed_position(predicted_coords[0], None), + 0, + fast_mode=target_fast_centroid, + ) + } + for comp_idx in range(max(0, predicted_coords.shape[0] - 1)): + psf_rows[f"comp{comp_idx + 1}"] = fit_centroid_or_warn_out_of_frame( + image_data, + choose_centroid_seed_position(predicted_coords[comp_idx + 1], None), + comp_idx + 1, + fast_mode=frame_fast_centroid, + ) + return { + 'coords': predicted_coords, + 'psf_rows': psf_rows, + 'warnings': list(recorder.warnings), + } + finally: + plateStatus = original_plate_status + + +def _parallel_alignment_task(task): + ( + i, + file_name, + target_and_comp_pixels, + target_and_comp_radec, + ignore_header_wcs, + target_fast_centroid, + frame_fast_centroid, + compute_fallback_transform, + first_frame_uses_input_comp_pixels, + precomputed_fallback_transform, + ) = task + + target_and_comp_pixels = np.asarray(target_and_comp_pixels, dtype=float) + if target_and_comp_radec is not None: + target_and_comp_radec = np.asarray(target_and_comp_radec, dtype=float) + + image_header, image_data = _load_alignment_worker_frame(file_name) + result = { + 'index': i, + 'file_name': file_name, + 'wcs': None, + 'fallback': None, + } + + if not ignore_header_wcs and target_and_comp_radec is not None: + try: + wcs_hdr = search_wcs_from_header(image_header) + if wcs_hdr.is_celestial: + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + projected_coords = np.column_stack((pix_x, pix_y)) + if i == 0: + projected_coords[0] = target_and_comp_pixels[0] + if first_frame_uses_input_comp_pixels: + projected_coords = np.array(target_and_comp_pixels, dtype=float, copy=True) + + wcs_candidate = _fit_alignment_candidate_psfs( + image_data, + projected_coords, + target_fast_centroid, + frame_fast_centroid, + ) + wcs_candidate['projected_off_frame'] = any_projected_coord_out_of_frame( + projected_coords, + image_data.shape, + ) + result['wcs'] = wcs_candidate + except Exception as exc: + result['wcs_error'] = str(exc) + + if precomputed_fallback_transform is not None or compute_fallback_transform or result['wcs'] is None: + if precomputed_fallback_transform is not None: + tform = precomputed_fallback_transform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + else: + tform = transformation( + image_data, + file_name, + report_failure=False, + reference_image=_TRANSFORM_REFERENCE_IMAGE, + ) + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + result['fallback'] = _fit_alignment_candidate_psfs( + image_data, + transformed_coords, + target_fast_centroid, + frame_fast_centroid, + ) + + return result + + +def _replay_parallel_alignment_warnings(file_name, warnings): + if not warnings: + return + + plateStatus.setCurrentFilename(file_name) + for warning_type, star_index, xc, yc in warnings: + if warning_type == 'out_of_frame': + plateStatus.outOfFrameWarning(star_index) + elif warning_type == 'low_flux': + plateStatus.lowFluxAmplitudeWarning(star_index, xc, yc) + elif warning_type == 'alignment_error': + plateStatus.alignmentError() + + +def _store_alignment_candidate_psfs(candidate, frame_index, psf_data, comp_keys): + psf_data['target'][frame_index] = candidate['psf_rows']['target'] + for comp_idx, comp_key in enumerate(comp_keys): + psf_data[comp_key][frame_index] = candidate['psf_rows'].get( + f"comp{comp_idx + 1}", + _nan_psf_result(), + ) + + +def _update_reference_comp_offsets(psf_data, tar_comp_dist, comp_keys): + target_row = psf_data['target'][0] + if not centroid_position_is_finite(target_row): + return + + for comp_key in comp_keys: + comp_row = psf_data[comp_key][0] + if not centroid_position_is_finite(comp_row): + continue + tar_comp_dist[comp_key][0] = abs(int(comp_row[0]) - int(target_row[0])) + tar_comp_dist[comp_key][1] = abs(int(comp_row[1]) - int(target_row[1])) + + +def apply_parallel_alignment_result(result, frame_index, psf_data, tar_comp_dist, comp_keys): + wcs_candidate = result.get('wcs') + selected_candidate = None + selected_source = 'fallback' + + if wcs_candidate is not None: + comp_psf_rows = { + comp_key: wcs_candidate['psf_rows'].get(f"comp{comp_idx + 1}", _nan_psf_result()) + for comp_idx, comp_key in enumerate(comp_keys) + } + previous_comp_psf_rows = {} + if frame_index != 0: + previous_comp_psf_rows = {comp_key: psf_data[comp_key][frame_index - 1] for comp_key in comp_keys} + + wcs_alignment_decision = should_keep_header_wcs_alignment( + wcs_candidate.get('projected_off_frame', False), + frame_index, + wcs_candidate['psf_rows']['target'], + previous_target_psf_row=None if frame_index == 0 else psf_data['target'][frame_index - 1], + comp_psf_rows=comp_psf_rows, + previous_comp_psf_rows=previous_comp_psf_rows, + expected_offsets=tar_comp_dist, + ) + if wcs_alignment_decision['use_wcs_alignment']: + selected_candidate = wcs_candidate + selected_source = 'wcs' + + if selected_candidate is None: + selected_candidate = result.get('fallback') or wcs_candidate + + if selected_candidate is None: + selected_candidate = { + 'psf_rows': {'target': _nan_psf_result()}, + 'warnings': [('alignment_error', -1, np.nan, np.nan)], + } + + _store_alignment_candidate_psfs(selected_candidate, frame_index, psf_data, comp_keys) + _replay_parallel_alignment_warnings(result.get('file_name'), selected_candidate.get('warnings')) + if frame_index == 0: + _update_reference_comp_offsets(psf_data, tar_comp_dist, comp_keys) + + return selected_source + + +def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_comp_pixels, + target_and_comp_radec=None, ignore_header_wcs=False, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None, + bad_pixel_reference=None, use_fast_centroid_cadence=False, + use_adaptive_apertures=False, compute_fallback_transform=True, + first_frame_uses_input_comp_pixels=False, + precomputed_fallback_transforms=None): + total_jobs = len(inputfiles) + if total_jobs == 0: + return [] + + max_workers = min(max_processes, os.cpu_count() or 1, total_jobs, MAX_MULTIPROCESS_TRANSFORM_WORKERS) + results = [None] * total_jobs + + log_info( + "Using multiprocessing for alignment " + f"with {max_workers} worker(s) across {total_jobs} image(s)." + ) + + tasks = [] + for i, file_name in enumerate(inputfiles): + precomputed_fallback_transform = None + if precomputed_fallback_transforms: + precomputed_fallback_transform = precomputed_fallback_transforms.get(str(file_name)) + frame_fast_centroid = should_use_fast_centroid(i) if use_fast_centroid_cadence else False + target_fast_centroid = ( + should_use_fast_target_centroid(i, adaptive_apertures=use_adaptive_apertures) + if use_fast_centroid_cadence else False + ) + tasks.append(( + i, + str(file_name), + target_and_comp_pixels, + target_and_comp_radec, + ignore_header_wcs, + target_fast_centroid, + frame_fast_centroid, + compute_fallback_transform, + first_frame_uses_input_comp_pixels, + precomputed_fallback_transform, + )) + + with ProcessPoolExecutor( + max_workers=max_workers, + initializer=_alignment_pool_initializer, + initargs=( + str(inputfiles[0]), + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference, + ), + ) as executor: + futures = [executor.submit(_parallel_alignment_task, task) for task in tasks] + completed = 0 + for future in as_completed(futures): + result = future.result() + results[result['index']] = result + completed += 1 + if completed == total_jobs or completed % 10 == 0: + log_info(f"Multiprocessing alignment progress: {completed}/{total_jobs}") + + return results + + MAX_MULTIPROCESS_TRANSFORM_WORKERS = 8 SPARSE_MISSING_WCS_DROP_THRESHOLD = 0.03 POINTING_REJECTION_MIN_FRAMES = 5 @@ -7815,19 +8336,20 @@ def build_multiprocess_transformations(inputfiles, max_processes): transforms[0] = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) - with ProcessPoolExecutor(max_workers=max_workers, initializer=_transformation_pool_initializer, - initargs=(reference_file,)) as executor: - futures = [executor.submit(transformation_task_with_cached_reference, i, str(file_name)) - for i, file_name in enumerate(inputfiles) if i != 0] + with suppress_tk_cleanup_during_process_pool(): + with ProcessPoolExecutor(max_workers=max_workers, initializer=_transformation_pool_initializer, + initargs=(reference_file,)) as executor: + futures = [executor.submit(transformation_task_with_cached_reference, i, str(file_name)) + for i, file_name in enumerate(inputfiles) if i != 0] - completed = 1 - for future in as_completed(futures): - i, tform = future.result() - transforms[i] = tform - completed += 1 + completed = 1 + for future in as_completed(futures): + i, tform = future.result() + transforms[i] = tform + completed += 1 - if completed == total_jobs or completed % 10 == 0: - log_info(f"Multiprocessing transformations progress: {completed}/{total_jobs}") + if completed == total_jobs or completed % 10 == 0: + log_info(f"Multiprocessing transformations progress: {completed}/{total_jobs}") return transforms @@ -8817,10 +9339,12 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet plateStatus.initializeFilenames(list(inputfiles)) pointing_precheck_inputfiles = np.array(inputfiles, copy=True) pointing_reference_file = inputfiles[0] if len(inputfiles) else None - inputfiles, _, dropped_pointing_files = filter_pointing_outlier_frames( + inputfiles, _, dropped_pointing_files, pointing_alignment_transforms = filter_pointing_outlier_frames( inputfiles, pointing_rejection_sigma=pointing_rejection_sigma, ignore_header_wcs=ignore_header_wcs, + return_alignment_transforms=True, + multiprocess_transformations=multiprocess_transformations, ) if dropped_pointing_files: if abort_if_reference_frame_rejected( @@ -8857,13 +9381,6 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet else: log_info("Bad-pixel precheck disabled per optional_info setting.") - use_multiprocess_transform_precompute = should_use_multiprocess_transform_precompute( - inputfiles, multiprocess_transformations, ignore_header_wcs=ignore_header_wcs - ) - fallback_transforms = {} - if use_multiprocess_transform_precompute: - fallback_transforms = build_multiprocess_transformations(inputfiles, multiprocess_transformations) - exotic_UIprevTPX = info_dict['tar_coords'][0] exotic_UIprevTPY = info_dict['tar_coords'][1] @@ -8891,6 +9408,10 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet target_and_comp_radec = None if tar_radec is not None and comp_radec: target_and_comp_radec = np.array([tar_radec, comp_radec[0]], dtype=float) + target_and_comp_pixels = np.array( + [[exotic_UIprevTPX, exotic_UIprevTPY], comp_star], + dtype=float, + ) centroid_reference_image = load_image_data(inputfiles[0]) centroid_reference_image = repair_bad_pixels_in_frame(centroid_reference_image, bad_pixel_reference) @@ -8921,6 +9442,24 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet # open files, calibrate, align, photometry reset_transform_timing_stats() reset_photometry_timing_stats() + multiprocess_alignment_results = None + use_multiprocess_alignment = multiprocess_transformations is not None and multiprocess_transformations > 0 + if use_multiprocess_alignment: + multiprocess_alignment_results = build_multiprocess_alignment_results( + inputfiles, + multiprocess_transformations, + target_and_comp_pixels, + target_and_comp_radec=target_and_comp_radec, + ignore_header_wcs=ignore_header_wcs, + bad_pixel_reference=bad_pixel_reference, + use_fast_centroid_cadence=True, + use_adaptive_apertures=use_adaptive_apertures, + compute_fallback_transform=True, + first_frame_uses_input_comp_pixels=True, + precomputed_fallback_transforms=pointing_alignment_transforms, + ) + use_multiprocess_transform_precompute = False + fallback_transforms = pointing_alignment_transforms for i, fileName in enumerate(inputfiles): plateStatus.setCurrentFilename(fileName) hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, @@ -8945,46 +9484,112 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet if i == 0: firstImage = np.copy(imageData) - use_wcs_alignment = False - if not ignore_header_wcs: - try: - wcs_hdr = search_wcs_from_header(image_header) - use_wcs_alignment = wcs_hdr.is_celestial - except Exception: - use_wcs_alignment = False + if multiprocess_alignment_results is not None: + apply_parallel_alignment_result( + multiprocess_alignment_results[i], + i, + psf_data, + tar_comp_dist, + ['comp'], + ) + else: + use_wcs_alignment = False + if not ignore_header_wcs: + try: + wcs_hdr = search_wcs_from_header(image_header) + use_wcs_alignment = wcs_hdr.is_celestial + except Exception: + use_wcs_alignment = False - if use_wcs_alignment: - try: - if i == 0: - tx, ty = exotic_UIprevTPX, exotic_UIprevTPY - cx, cy = comp_star - else: - pix_x, pix_y = wcs_hdr.world_to_pixel_values( - target_and_comp_radec[:, 0], - target_and_comp_radec[:, 1], + if use_wcs_alignment: + try: + if i == 0: + tx, ty = exotic_UIprevTPX, exotic_UIprevTPY + cx, cy = comp_star + else: + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + tx, ty = pix_x[0], pix_y[0] + cx, cy = pix_x[1], pix_y[1] + + projected_coords = np.array([[tx, ty], [cx, cy]], dtype=float) + projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) + target_seed = choose_centroid_seed_position( + [tx, ty], + None if i == 0 else psf_data['target'][i - 1], + ) + comp_seed = choose_centroid_seed_position( + [cx, cy], + None if i == 0 else psf_data['comp'][i - 1], ) - pix_x = np.asarray(pix_x, dtype=float).reshape(-1) - pix_y = np.asarray(pix_y, dtype=float).reshape(-1) - tx, ty = pix_x[0], pix_y[0] - cx, cy = pix_x[1], pix_y[1] - projected_coords = np.array([[tx, ty], [cx, cy]], dtype=float) - projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) + psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + target_seed, + 0, + fast_mode=target_fast_centroid, + ) + psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + comp_seed, + 1, + fast_mode=frame_fast_centroid, + ) + + if i == 0: + tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) + tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) + wcs_alignment_decision = should_keep_header_wcs_alignment( + projected_off_frame, + i, + psf_data['target'][i], + previous_target_psf_row=None if i == 0 else psf_data['target'][i - 1], + comp_psf_rows={'comp': psf_data['comp'][i]}, + previous_comp_psf_rows={} if i == 0 else {'comp': psf_data['comp'][i - 1]}, + expected_offsets={'comp': tar_comp_dist['comp']}, + ) + use_wcs_alignment = wcs_alignment_decision['use_wcs_alignment'] + except Exception: + use_wcs_alignment = False + + log_alignment_progress( + i, + len(inputfiles), + fileName, + use_multiprocess_transform_precompute, + ) + + if not use_wcs_alignment: + cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None + if cached_tform is not None: + tform = cached_tform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + else: + tform = transformation(imageData, fileName, reference_image=firstImage) + + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + tx, ty = transformed_coords[0] target_seed = choose_centroid_seed_position( [tx, ty], None if i == 0 else psf_data['target'][i - 1], ) - comp_seed = choose_centroid_seed_position( - [cx, cy], - None if i == 0 else psf_data['comp'][i - 1], - ) - psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( imageData, target_seed, 0, fast_mode=target_fast_centroid, ) + + cx, cy = transformed_coords[1] + comp_seed = choose_centroid_seed_position( + [cx, cy], + None if i == 0 else psf_data['comp'][i - 1], + ) psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( imageData, comp_seed, @@ -8995,63 +9600,6 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet if i == 0: tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) - wcs_alignment_decision = should_keep_header_wcs_alignment( - projected_off_frame, - i, - psf_data['target'][i], - previous_target_psf_row=None if i == 0 else psf_data['target'][i - 1], - comp_psf_rows={'comp': psf_data['comp'][i]}, - previous_comp_psf_rows={} if i == 0 else {'comp': psf_data['comp'][i - 1]}, - expected_offsets={'comp': tar_comp_dist['comp']}, - ) - use_wcs_alignment = wcs_alignment_decision['use_wcs_alignment'] - except Exception: - use_wcs_alignment = False - - log_alignment_progress( - i, - len(inputfiles), - fileName, - use_multiprocess_transform_precompute, - ) - - if not use_wcs_alignment: - if i == 0: - tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) - else: - tform = fallback_transforms[i] if i in fallback_transforms else transformation(imageData, fileName, reference_image=firstImage) - - transformed_coords = np.asarray( - tform(np.array([[exotic_UIprevTPX, exotic_UIprevTPY], comp_star], dtype=float)), - dtype=float, - ) - tx, ty = transformed_coords[0] - target_seed = choose_centroid_seed_position( - [tx, ty], - None if i == 0 else psf_data['target'][i - 1], - ) - psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( - imageData, - target_seed, - 0, - fast_mode=target_fast_centroid, - ) - - cx, cy = transformed_coords[1] - comp_seed = choose_centroid_seed_position( - [cx, cy], - None if i == 0 else psf_data['comp'][i - 1], - ) - psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( - imageData, - comp_seed, - 1, - fast_mode=frame_fast_centroid, - ) - - if i == 0: - tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) # aperture photometry frame_sigma = psf_sigma_from_fit(psf_data['target'][i], fallback_sigma=sigma) @@ -9963,8 +10511,12 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co if multiprocess_lightcurve_fits is not None and multiprocess_lightcurve_fits > 0: log_info(f"Using multiprocessing for candidate lightcurve fits ({multiprocess_lightcurve_fits} processes).") - with ProcessPoolExecutor(max_workers=multiprocess_lightcurve_fits) as executor: - fit_results = list(executor.map(evaluate_lightcurve_candidate, fit_tasks)) + with suppress_tk_cleanup_during_process_pool(): + with ProcessPoolExecutor( + max_workers=multiprocess_lightcurve_fits, + initializer=suppress_inherited_tk_cleanup_in_worker, + ) as executor: + fit_results = list(executor.map(evaluate_lightcurve_candidate, fit_tasks)) else: fit_results = [evaluate_lightcurve_candidate(task) for task in fit_tasks] @@ -12281,7 +12833,7 @@ def parse_args(): parser.add_argument('--multiprocess-transformations', type=int, default=None, - help="Use multiprocessing when finding image transformations. " + help="Use multiprocessing for frame alignment and fallback image transformations. " "Provide an integer number of processes to use.") parser.add_argument('--multiprocess-lightcurve-fits', type=int, @@ -12625,7 +13177,7 @@ def _main_impl(): post_wcs_inputfile_count = int(len(inputfiles)) pointing_precheck_inputfiles = np.array(inputfiles, copy=True) pointing_reference_file = inputfiles[0] if len(inputfiles) else None - inputfiles, pointing_keep_mask, dropped_pointing_files = filter_pointing_outlier_frames( + inputfiles, pointing_keep_mask, dropped_pointing_files, pointing_alignment_transforms = filter_pointing_outlier_frames( inputfiles, pointing_rejection_sigma=pointing_rejection_sigma, ignore_header_wcs=ignore_header_wcs, @@ -12638,6 +13190,14 @@ def _main_impl(): demosaic_out, demosaic_mult, ), + return_alignment_transforms=True, + multiprocess_transformations=args.multiprocess_transformations, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, ) if dropped_pointing_files: if abort_if_reference_frame_rejected( @@ -12713,6 +13273,7 @@ def _main_impl(): inputfiles = inputfiles[inc:] times = times[inc:] jd_times = jd_times[inc:] + pointing_alignment_transforms = {} plateStatus.setCurrentFilename(inputfiles[0]) header = get_first_image_header(inputfiles[0]) @@ -12906,13 +13467,6 @@ def _main_impl(): aper_data = None coarse_frame_cache = [None] * coarse_tune_frames if use_aperture_photometry else [] - use_multiprocess_transform_precompute = should_use_multiprocess_transform_precompute( - inputfiles, args.multiprocess_transformations, ignore_header_wcs=ignore_header_wcs - ) - fallback_transforms = {} - if use_multiprocess_transform_precompute: - fallback_transforms = build_multiprocess_transformations(inputfiles, args.multiprocess_transformations) - target_and_comp_radec = None if ra_dec_tar is not None and ra_dec_wcs: target_and_comp_radec = np.array([ra_dec_tar, *ra_dec_wcs], dtype=float) @@ -12927,6 +13481,32 @@ def _main_impl(): # open files, calibrate, align, photometry reset_transform_timing_stats() reset_photometry_timing_stats() + multiprocess_alignment_results = None + use_multiprocess_alignment = ( + args.multiprocess_transformations is not None and args.multiprocess_transformations > 0 + ) + comp_alignment_keys = [f"comp{j + 1}" for j in range(comp_star_count)] + if use_multiprocess_alignment: + multiprocess_alignment_results = build_multiprocess_alignment_results( + inputfiles, + args.multiprocess_transformations, + target_and_comp_pixels, + target_and_comp_radec=target_and_comp_radec, + ignore_header_wcs=ignore_header_wcs, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + use_fast_centroid_cadence=False, + use_adaptive_apertures=use_adaptive_apertures, + compute_fallback_transform=True, + precomputed_fallback_transforms=pointing_alignment_transforms, + ) + use_multiprocess_transform_precompute = False + fallback_transforms = pointing_alignment_transforms for i, fileName in enumerate(inputfiles): plateStatus.setCurrentFilename(fileName) hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, @@ -12959,40 +13539,117 @@ def _main_impl(): if i == 0: firstImage = np.copy(imageData) - use_wcs_alignment = False - if not ignore_header_wcs: - try: - wcs_hdr = search_wcs_from_header(image_header) - use_wcs_alignment = wcs_hdr.is_celestial - except Exception: - use_wcs_alignment = False + if multiprocess_alignment_results is not None: + apply_parallel_alignment_result( + multiprocess_alignment_results[i], + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + else: + use_wcs_alignment = False + if not ignore_header_wcs: + try: + wcs_hdr = search_wcs_from_header(image_header) + use_wcs_alignment = wcs_hdr.is_celestial + except Exception: + use_wcs_alignment = False + + if use_wcs_alignment: + try: + pix_x = pix_y = None + if target_and_comp_radec is not None: + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) - if use_wcs_alignment: - try: - pix_x = pix_y = None - if target_and_comp_radec is not None: - pix_x, pix_y = wcs_hdr.world_to_pixel_values( - target_and_comp_radec[:, 0], - target_and_comp_radec[:, 1], + if i == 0: + tx, ty = exotic_UIprevTPX, exotic_UIprevTPY + else: + tx, ty = pix_x[0], pix_y[0] + + projected_coords = np.array( + [[tx, ty], *np.column_stack((pix_x[1:], pix_y[1:]))] if pix_x is not None else [[tx, ty]], + dtype=float, + ) + projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) + target_seed = choose_centroid_seed_position( + [tx, ty], + None if i == 0 else psf_data['target'][i - 1], + ) + + psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + imageData, + target_seed, + 0, + fast_mode=target_fast_centroid, + ) + + # TODO: Add check for flux on target/comp stars relative to others in the field + # in case of cloudy data, large changes, etc. + current_comp_psf_rows = {} + previous_comp_psf_rows = {} + for j in range(len(exotic_infoDict['comp_stars'])): + ckey = f"comp{j + 1}" + + cx, cy = pix_x[j + 1], pix_y[j + 1] + comp_seed = choose_centroid_seed_position( + [cx, cy], + None if i == 0 else psf_data[ckey][i - 1], + ) + psf_data[ckey][i] = fit_centroid_or_warn_out_of_frame( + imageData, + comp_seed, + j + 1, + fast_mode=frame_fast_centroid, + ) + + current_comp_psf_rows[ckey] = psf_data[ckey][i] + if i != 0: + previous_comp_psf_rows[ckey] = psf_data[ckey][i - 1] + else: + tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) + tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) + + wcs_alignment_decision = should_keep_header_wcs_alignment( + projected_off_frame, + i, + psf_data['target'][i], + previous_target_psf_row=None if i == 0 else psf_data['target'][i - 1], + comp_psf_rows=current_comp_psf_rows, + previous_comp_psf_rows=previous_comp_psf_rows, + expected_offsets=tar_comp_dist, ) - pix_x = np.asarray(pix_x, dtype=float).reshape(-1) - pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + use_wcs_alignment = wcs_alignment_decision['use_wcs_alignment'] + except Exception: + use_wcs_alignment = False - if i == 0: - tx, ty = exotic_UIprevTPX, exotic_UIprevTPY + log_alignment_progress( + i, + len(inputfiles), + fileName, + use_multiprocess_transform_precompute, + ) + + if not use_wcs_alignment: + cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None + if cached_tform is not None: + tform = cached_tform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) else: - tx, ty = pix_x[0], pix_y[0] + tform = transformation(imageData, fileName, reference_image=firstImage) - projected_coords = np.array( - [[tx, ty], *np.column_stack((pix_x[1:], pix_y[1:]))] if pix_x is not None else [[tx, ty]], - dtype=float, - ) - projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + tx, ty = transformed_coords[0] target_seed = choose_centroid_seed_position( [tx, ty], None if i == 0 else psf_data['target'][i - 1], ) - psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( imageData, target_seed, @@ -13000,14 +13657,10 @@ def _main_impl(): fast_mode=target_fast_centroid, ) - # TODO: Add check for flux on target/comp stars relative to others in the field - # in case of cloudy data, large changes, etc. - current_comp_psf_rows = {} - previous_comp_psf_rows = {} - for j in range(len(exotic_infoDict['comp_stars'])): + for j, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{j + 1}" - cx, cy = pix_x[j + 1], pix_y[j + 1] + cx, cy = transformed_coords[j + 1] comp_seed = choose_centroid_seed_position( [cx, cy], None if i == 0 else psf_data[ckey][i - 1], @@ -13019,71 +13672,10 @@ def _main_impl(): fast_mode=frame_fast_centroid, ) - current_comp_psf_rows[ckey] = psf_data[ckey][i] - if i != 0: - previous_comp_psf_rows[ckey] = psf_data[ckey][i - 1] - else: + if i == 0: tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) - wcs_alignment_decision = should_keep_header_wcs_alignment( - projected_off_frame, - i, - psf_data['target'][i], - previous_target_psf_row=None if i == 0 else psf_data['target'][i - 1], - comp_psf_rows=current_comp_psf_rows, - previous_comp_psf_rows=previous_comp_psf_rows, - expected_offsets=tar_comp_dist, - ) - use_wcs_alignment = wcs_alignment_decision['use_wcs_alignment'] - except Exception: - use_wcs_alignment = False - - log_alignment_progress( - i, - len(inputfiles), - fileName, - use_multiprocess_transform_precompute, - ) - - if not use_wcs_alignment: - if i == 0: - tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) - else: - tform = fallback_transforms[i] if i in fallback_transforms else transformation(imageData, fileName, reference_image=firstImage) - - transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) - tx, ty = transformed_coords[0] - target_seed = choose_centroid_seed_position( - [tx, ty], - None if i == 0 else psf_data['target'][i - 1], - ) - psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( - imageData, - target_seed, - 0, - fast_mode=target_fast_centroid, - ) - - for j, coord in enumerate(exotic_infoDict['comp_stars']): - ckey = f"comp{j + 1}" - - cx, cy = transformed_coords[j + 1] - comp_seed = choose_centroid_seed_position( - [cx, cy], - None if i == 0 else psf_data[ckey][i - 1], - ) - psf_data[ckey][i] = fit_centroid_or_warn_out_of_frame( - imageData, - comp_seed, - j + 1, - fast_mode=frame_fast_centroid, - ) - - if i == 0: - tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) - # aperture photometry if use_aperture_photometry and i == 0: sigma = psf_sigma_from_fit(psf_data['target'][0]) diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 80716de3..6afe0dab 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -481,6 +481,35 @@ def test_collect_transform_frame_pointings_logs_alignment_progress(monkeypatch): ] +def test_collect_transform_frame_pointings_can_return_transform_cache(monkeypatch): + monkeypatch.setattr( + exotic_module, + "log_info", + lambda *_args, **_kwargs: None, + ) + + expected_transform = exotic_module.SimilarityTransform(scale=1, rotation=0, translation=[2.0, -1.0]) + monkeypatch.setattr( + exotic_module, + "transformation", + lambda image_data, file_name, report_failure=False, reference_image=None: expected_transform, + ) + + frames = ["frame_0001.fits", "frame_0002.fits"] + frame_loader = lambda file_name: np.ones((8, 8), dtype=float) + + positions, usable_mask, transforms = exotic_module.collect_transform_frame_pointings( + frames, + frame_loader=frame_loader, + return_transforms=True, + ) + + assert usable_mask.tolist() == [True, True] + assert np.allclose(positions[1], [5.5, 2.5]) + assert set(transforms) == set(frames) + assert transforms[frames[1]] is expected_transform + + def test_check_wcs_ignores_header_wcs_when_override_enabled(monkeypatch): monkeypatch.setattr( exotic_module, @@ -556,6 +585,118 @@ def test_should_use_multiprocess_transform_precompute_respects_header_wcs_overri ) is True +def test_transformation_pool_initializer_suppresses_inherited_tk_cleanup(monkeypatch): + calls = [] + reference_image = np.ones((4, 4), dtype=float) + monkeypatch.setattr(exotic_module, "suppress_inherited_tk_cleanup_in_worker", lambda: calls.append(True)) + monkeypatch.setattr(exotic_module, "load_image_data", lambda _file_name: reference_image) + + exotic_module._TRANSFORM_REFERENCE_IMAGE = None + exotic_module._TRANSFORM_REFERENCE_CACHE = {"stale": True} + + exotic_module._transformation_pool_initializer("reference.fits") + + assert calls == [True] + assert exotic_module._TRANSFORM_REFERENCE_IMAGE is reference_image + assert exotic_module._TRANSFORM_REFERENCE_CACHE is None + + +def test_apply_parallel_alignment_result_uses_precomputed_fallback_when_wcs_geometry_fails(monkeypatch): + psf_data = { + "target": np.zeros((2, 7), dtype=float), + "comp1": np.zeros((2, 7), dtype=float), + } + psf_data["target"][0] = np.array([10.0, 10.0, 100.0, 2.0, 2.0, 0.0, 50.0]) + psf_data["comp1"][0] = np.array([20.0, 10.0, 100.0, 2.0, 2.0, 0.0, 50.0]) + tar_comp_dist = {"comp1": np.array([10, 0], dtype=int)} + warnings = [] + + monkeypatch.setattr( + exotic_module.plateStatus, + "lowFluxAmplitudeWarning", + lambda star_index, xc, yc: warnings.append((star_index, xc, yc)), + ) + + result = { + "index": 1, + "file_name": "frame_0002.fits", + "wcs": { + "projected_off_frame": False, + "psf_rows": { + "target": np.array([10.0, 10.0, 100.0, 2.0, 2.0, 0.0, 50.0]), + "comp1": np.array([50.0, 50.0, 100.0, 2.0, 2.0, 0.0, 50.0]), + }, + "warnings": [("low_flux", 1, 50.0, 50.0)], + }, + "fallback": { + "psf_rows": { + "target": np.array([11.0, 10.0, 100.0, 2.0, 2.0, 0.0, 50.0]), + "comp1": np.array([21.0, 10.0, 90.0, 2.0, 2.0, 0.0, 50.0]), + }, + "warnings": [("low_flux", 1, 21.0, 10.0)], + }, + } + + selected = exotic_module.apply_parallel_alignment_result( + result, + 1, + psf_data, + tar_comp_dist, + ["comp1"], + ) + + assert selected == "fallback" + assert psf_data["target"][1, 0] == pytest.approx(11.0) + assert psf_data["comp1"][1, 0] == pytest.approx(21.0) + assert warnings == [(1, 21.0, 10.0)] + + +def test_parallel_alignment_task_uses_precomputed_fallback_transform(monkeypatch): + target_and_comp_pixels = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=float) + precomputed_transform = exotic_module.SimilarityTransform( + scale=1, + rotation=0, + translation=[5.0, -1.0], + ) + + monkeypatch.setattr( + exotic_module, + "_load_alignment_worker_frame", + lambda _file_name: ({}, np.ones((10, 10), dtype=float)), + ) + monkeypatch.setattr( + exotic_module, + "transformation", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("cached transform should be used") + ), + ) + monkeypatch.setattr( + exotic_module, + "_fit_alignment_candidate_psfs", + lambda image_data, predicted_coords, *_args: { + "coords": np.asarray(predicted_coords, dtype=float), + "psf_rows": {"target": np.zeros(7, dtype=float)}, + "warnings": [], + }, + ) + + result = exotic_module._parallel_alignment_task(( + 1, + "frame_0002.fits", + target_and_comp_pixels, + None, + True, + False, + False, + True, + False, + precomputed_transform, + )) + + assert np.allclose(result["fallback"]["coords"], [[6.0, 1.0], [8.0, 3.0]]) + + def test_filter_sparse_missing_wcs_frames_drops_files_below_three_percent(monkeypatch): frames = [f"frame_{i}.fits" for i in range(34)] missing_frame = frames[7] @@ -651,21 +792,30 @@ def test_filter_pointing_outlier_frames_falls_back_to_transform_when_wcs_is_inco ), ) - def fake_collect_transform_frame_pointings(inputfiles, frame_loader=None): - transform_calls.append((tuple(inputfiles), frame_loader)) + def fake_collect_transform_frame_pointings(inputfiles, frame_loader=None, return_transforms=False, **kwargs): + transform_calls.append((tuple(inputfiles), frame_loader, return_transforms, kwargs)) + transforms = { + str(file_name): exotic_module.SimilarityTransform(scale=1, rotation=0, translation=[index, 0]) + for index, file_name in enumerate(inputfiles) + } + if return_transforms: + return transform_positions, np.ones(len(inputfiles), dtype=bool), transforms return transform_positions, np.ones(len(inputfiles), dtype=bool) monkeypatch.setattr(exotic_module, "collect_transform_frame_pointings", fake_collect_transform_frame_pointings) - filtered, keep_mask, dropped = exotic_module.filter_pointing_outlier_frames( + filtered, keep_mask, dropped, cached_transforms = exotic_module.filter_pointing_outlier_frames( frames, pointing_rejection_sigma=3.0, + return_alignment_transforms=True, ) assert len(transform_calls) == 1 + assert transform_calls[0][2] is True assert filtered.tolist() == frames[:-1] assert keep_mask.tolist() == [True, True, True, True, True, False] assert dropped == [frames[-1]] + assert set(cached_transforms) == set(frames[:-1]) def test_abort_if_reference_frame_rejected_reports_error_and_removal_recommendation(monkeypatch): diff --git a/tests/test_ultranest_utils.py b/tests/test_ultranest_utils.py index 6136ad8e..b07a8fa7 100644 --- a/tests/test_ultranest_utils.py +++ b/tests/test_ultranest_utils.py @@ -1,5 +1,7 @@ import io import logging +import sys +import types import numpy as np @@ -158,6 +160,29 @@ def run(self, **kwargs): assert sampler.kwargs["min_num_live_points"] == 320 +def test_configured_ultranest_workers_defaults_to_available_cpu_count(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 12, raising=False) + + assert ultranest_utils._configured_ultranest_workers() == 12 + + +def test_configured_ultranest_workers_accepts_auto_override(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "auto") + monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 10, raising=False) + + assert ultranest_utils._configured_ultranest_workers() == 10 + + +def test_configured_ultranest_workers_preserves_numeric_override(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "3") + monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 12, raising=False) + + assert ultranest_utils._configured_ultranest_workers() == 3 + + def test_run_reactive_sampler_parallelizes_vectorized_loglike_batches(monkeypatch): _reset_ultranest_env(monkeypatch) monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "3") @@ -186,6 +211,126 @@ def run(self, **kwargs): assert sorted(sampler.chunk_sizes) == [2, 2, 2] +def test_run_reactive_sampler_auto_workers_uses_available_cpu_count(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "thread") + monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 4, raising=False) + + class FakeSampler: + def __init__(self): + self.chunk_sizes = [] + + def loglike(points): + self.chunk_sizes.append(len(points)) + return points[:, 0] + + self.loglike = loglike + + def run(self, **kwargs): + return {"values": self.loglike(np.arange(16, dtype=float).reshape(8, 2))} + + sampler = FakeSampler() + result = run_reactive_sampler(sampler, verbose=False) + + assert result["values"].tolist() == [0, 2, 4, 6, 8, 10, 12, 14] + assert sorted(sampler.chunk_sizes) == [2, 2, 2, 2] + + +def test_process_backend_disables_parent_gc_while_pool_is_active(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "2") + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "process") + monkeypatch.setattr(ultranest_utils.sys, "platform", "linux") + monkeypatch.setattr(ultranest_utils, "_is_colab_runtime", lambda: False) + monkeypatch.setattr( + ultranest_utils, + "get_mpi_status", + lambda: {"available": False, "size": 1, "rank": 0, "source": "test", "error": None}, + ) + + cleanup_calls = [] + monkeypatch.setattr( + ultranest_utils, + "suppress_inherited_tk_cleanup_in_worker", + lambda: cleanup_calls.append("suppress"), + ) + + pool_events = [] + + class FakePool: + def __init__(self, processes, initializer): + pool_events.append(("init", processes, initializer)) + + def map(self, func, chunks): + pool_events.append(("map", ultranest_utils.gc.isenabled())) + return [func(chunk) for chunk in chunks] + + def close(self): + pool_events.append(("close", None)) + + def join(self): + pool_events.append(("join", None)) + + class FakeContext: + Pool = FakePool + + monkeypatch.setattr(ultranest_utils.multiprocessing, "get_context", lambda _method: FakeContext()) + + class FakeSampler: + def __init__(self): + self.chunk_sizes = [] + + def loglike(points): + self.chunk_sizes.append(len(points)) + return points[:, 0] + + self.loglike = loglike + + def run(self, **kwargs): + assert ultranest_utils.gc.isenabled() is False + return {"values": self.loglike(np.arange(8, dtype=float).reshape(4, 2))} + + gc_was_enabled = ultranest_utils.gc.isenabled() + ultranest_utils.gc.enable() + try: + sampler = FakeSampler() + result = run_reactive_sampler(sampler, verbose=False) + assert ultranest_utils.gc.isenabled() is True + finally: + if not gc_was_enabled: + ultranest_utils.gc.disable() + + assert result["values"].tolist() == [0, 2, 4, 6] + assert sorted(sampler.chunk_sizes) == [2, 2] + assert cleanup_calls == ["suppress"] + assert pool_events[0] == ("init", 2, ultranest_utils.suppress_inherited_tk_cleanup_in_worker) + assert ("map", False) in pool_events + assert pool_events[-2:] == [("close", None), ("join", None)] + + +def test_process_worker_initializer_suppresses_inherited_tk_destructors(monkeypatch): + class FakeImage: + def __del__(self): + raise RuntimeError("main thread is not in main loop") + + class FakeVariable: + def __del__(self): + raise RuntimeError("main thread is not in main loop") + + original_image_del = FakeImage.__del__ + original_variable_del = FakeVariable.__del__ + fake_tkinter = types.SimpleNamespace(Image=FakeImage, Variable=FakeVariable) + monkeypatch.setitem(sys.modules, "tkinter", fake_tkinter) + + assert ultranest_utils._suppress_inherited_tk_cleanup() is True + + assert FakeImage._exotic_worker_original_del is original_image_del + assert FakeVariable._exotic_worker_original_del is original_variable_del + assert FakeImage().__del__() is None + assert FakeVariable().__del__() is None + assert ultranest_utils._suppress_inherited_tk_cleanup() is False + + def test_run_reactive_sampler_preserves_explicit_live_point_override(monkeypatch): _reset_ultranest_env(monkeypatch) monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "320") From 8a2fdaf61db19b722f087d1dda623094a7fd0ad3 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 7 May 2026 23:55:13 +1000 Subject: [PATCH 034/116] guarding aginst evil colons amongst other things. --- exotic/api/elca.py | 10 --- exotic/api/joint_fitter.py | 10 --- exotic/exotic.py | 90 ++++++++++++---------- exotic/output_files.py | 46 ++++++++++-- exotic/plots.py | 99 ++++++++++++++++++------- exotic/utils.py | 45 +++++++++++ tests/test_exotic_proper_motion.py | 25 +++++-- tests/test_lazy_pylightcurve_imports.py | 2 +- tests/test_utils.py | 17 +++++ 9 files changed, 247 insertions(+), 97 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index e8a27813..9d551aa2 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -40,13 +40,11 @@ # Fit an exoplanet transit model to time series data. # ########################################################################### # from astropy.time import Time -import builtins import copy from contextlib import redirect_stderr, redirect_stdout import faulthandler import io from itertools import cycle -import multiprocessing import os import sys import bottleneck as bn @@ -75,14 +73,6 @@ BAD_LOG_LIKELIHOOD = -1.0e100 -if ( - multiprocessing.current_process().name == "MainProcess" - and not getattr(builtins, "_EXOTIC_IMPORTING_MODULES_PRINTED", False) -): - print("Importing modules. Please wait.......", flush=True) - builtins._EXOTIC_IMPORTING_MODULES_PRINTED = True - - def _pylightcurve_import_watchdog_seconds(): try: return float(os.environ.get("EXOTIC_IMPORT_WATCHDOG_SECONDS", "120")) diff --git a/exotic/api/joint_fitter.py b/exotic/api/joint_fitter.py index f992f40b..8a3e152f 100644 --- a/exotic/api/joint_fitter.py +++ b/exotic/api/joint_fitter.py @@ -37,13 +37,11 @@ # ########################################################################### # from astropy import constants as const from astropy import units as u -import builtins from copy import deepcopy from contextlib import redirect_stderr, redirect_stdout import faulthandler import io from itertools import cycle -import multiprocessing import os import sys import matplotlib.pyplot as plt @@ -67,14 +65,6 @@ except ImportError: from .ultranest_utils import run_reactive_sampler -if ( - multiprocessing.current_process().name == "MainProcess" - and not getattr(builtins, "_EXOTIC_IMPORTING_MODULES_PRINTED", False) -): - print("Importing modules. Please wait.......", flush=True) - builtins._EXOTIC_IMPORTING_MODULES_PRINTED = True - - def _pylightcurve_import_watchdog_seconds(): try: return float(os.environ.get("EXOTIC_IMPORT_WATCHDOG_SECONDS", "120")) diff --git a/exotic/exotic.py b/exotic/exotic.py index f9843560..d74c8cb0 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -178,9 +178,9 @@ plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ plot_adaptive_aperture_diagnostics try: # tools - from utils import round_to_2, user_input + from utils import filename_date_token, round_to_2, safe_output_filename, user_input except ImportError: # package import - from .utils import round_to_2, user_input + from .utils import filename_date_token, round_to_2, safe_output_filename, user_input try: # simple version from .version import __version__ except ImportError: # package import @@ -1572,14 +1572,23 @@ def comparison_candidate_output_dir(save_dir, comp_index): def triangle_plot_output_path(save_dir, planet_name, observation_date): - return Path(save_dir) / "temp" / f"Triangle_{planet_name}_{observation_date}.png" + return ( + Path(save_dir) + / "temp" + / safe_output_filename("Triangle", planet_name, filename_date_token(observation_date), extension="png") + ) def comparison_candidate_triangle_plot_output_path(save_dir, planet_name, observation_date, comp_index): return ( Path(save_dir) / "temp" - / f"Comp{int(comp_index) + 1}_Triangle_{planet_name}_{observation_date}.png" + / safe_output_filename( + f"Comp{int(comp_index) + 1}_Triangle", + planet_name, + filename_date_token(observation_date), + extension="png", + ) ) @@ -2006,7 +2015,12 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, if callable(plotter): try: fig, _ = plotter() - bestfit_plot_path = temp_dir / f"BestFit_{p_dict['pName']}_{observation_date}.png" + bestfit_plot_path = temp_dir / safe_output_filename( + "BestFit", + p_dict['pName'], + filename_date_token(observation_date), + extension="png", + ) fig.savefig(bestfit_plot_path) plt.close(fig) except Exception as exc: @@ -2080,7 +2094,12 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, exc, )) - summary_path = temp_dir / f"ComparisonCandidateSummary_{p_dict['pName']}_{observation_date}.json" + summary_path = temp_dir / safe_output_filename( + "ComparisonCandidateSummary", + p_dict['pName'], + filename_date_token(observation_date), + extension="json", + ) summary_payload = { 'planet_name': p_dict['pName'], 'observation_date': observation_date, @@ -2136,7 +2155,12 @@ def archive_failed_comparison_fit(save_dir, planet_name, observation_date, attem if callable(plotter): try: fig, _ = plotter() - bestfit_plot_path = temp_dir / f"BestFit_{planet_name}_{observation_date}.png" + bestfit_plot_path = temp_dir / safe_output_filename( + "BestFit", + planet_name, + filename_date_token(observation_date), + extension="png", + ) fig.savefig(bestfit_plot_path) plt.close(fig) except Exception as exc: @@ -2145,7 +2169,12 @@ def archive_failed_comparison_fit(save_dir, planet_name, observation_date, attem exc, )) - summary_path = temp_dir / f"FailedFitSummary_{planet_name}_{observation_date}.json" + summary_path = temp_dir / safe_output_filename( + "FailedFitSummary", + planet_name, + filename_date_token(observation_date), + extension="json", + ) summary_payload = { 'planet_name': planet_name, 'observation_date': observation_date, @@ -2205,7 +2234,12 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat output_dir = Path(save_dir) / "temp" output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / f"SelectedPhotometryRawRatio_{planet_name}_{observation_date}.csv" + output_path = output_dir / safe_output_filename( + "SelectedPhotometryRawRatio", + planet_name, + filename_date_token(observation_date), + extension="csv", + ) output_rows = np.column_stack( [ @@ -10665,44 +10699,24 @@ def format_comp_star_position(position): def deduplicate_comparison_star_coords(comp_stars, min_separation_pixels=COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS): + """Normalize comparison-star coordinates without merging nearby stars. + + The function name is historical. User-provided comparison-star selections + are intentional inputs, so nearby stars must remain distinct candidates. + """ if comp_stars is None: return [], [] - try: - threshold = float(min_separation_pixels) - except (TypeError, ValueError): - threshold = COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS - if not np.isfinite(threshold) or threshold <= 0: - threshold = COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS - - unique_coords = [] - duplicate_messages = [] - for index, coord in enumerate(comp_stars, start=1): + normalized_coords = [] + for coord in comp_stars: try: x_pos, y_pos = float(coord[0]), float(coord[1]) except (TypeError, ValueError, IndexError): continue - duplicate_entry = None - for unique_index, unique_coord in enumerate(unique_coords, start=1): - separation = float(np.hypot(x_pos - unique_coord[0], y_pos - unique_coord[1])) - if separation <= threshold: - duplicate_entry = (unique_index, unique_coord, separation) - break - - if duplicate_entry is not None: - kept_index, kept_coord, separation = duplicate_entry - duplicate_messages.append( - "Merged comparison star " - f"#{index} ({x_pos:.1f}, {y_pos:.1f}) into comparison star " - f"#{kept_index} ({kept_coord[0]:.1f}, {kept_coord[1]:.1f}) " - f"because they were only {separation:.2f} px apart." - ) - continue - - unique_coords.append([x_pos, y_pos]) + normalized_coords.append([x_pos, y_pos]) - return unique_coords, duplicate_messages + return normalized_coords, [] def format_comp_star_coverage_text(summary): diff --git a/exotic/output_files.py b/exotic/output_files.py index bc5db7fe..c3a07fa1 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -4,9 +4,9 @@ import numpy as np try: - from utils import round_to_2 + from utils import filename_date_token, round_to_2, safe_output_filename except ImportError: - from .utils import round_to_2 + from .utils import filename_date_token, round_to_2, safe_output_filename try: from version import __version__ except ImportError: @@ -807,7 +807,12 @@ def __init__(self, fit, p_dict, i_dict, durs): self.dir = Path(self.i_dict['save']) def final_lightcurve(self, phase): - params_file = self.dir / "temp" / f"FinalLightCurve_{self.p_dict['pName']}_{self.i_dict['date']}.csv" + params_file = self.dir / "temp" / safe_output_filename( + "FinalLightCurve", + self.p_dict['pName'], + filename_date_token(self.i_dict['date']), + extension="csv", + ) with params_file.open('w') as f: f.write(f"# FINAL TIMESERIES OF {self.p_dict['pName']}\n") @@ -820,7 +825,12 @@ def final_lightcurve(self, phase): def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, min_annul=None, adaptive_summary=None, photometry_info=None): - params_file = self.dir / "temp" / f"FinalParams_{self.p_dict['pName']}_{self.i_dict['date']}.json" + params_file = self.dir / "temp" / safe_output_filename( + "FinalParams", + self.p_dict['pName'], + filename_date_token(self.i_dict['date']), + extension="json", + ) transit_qc = getattr(self.fit, 'transit_qc', None) fit_quality = build_fit_quality_metadata(self.fit) @@ -1013,7 +1023,12 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" - params_file = self.dir / f"AAVSO_{self.p_dict['pName']}_{self.i_dict['date']}.txt" + params_file = self.dir / safe_output_filename( + "AAVSO", + self.p_dict['pName'], + filename_date_token(self.i_dict['date']), + extension="txt", + ) with params_file.open('w', encoding="utf-8") as f: f.write("#TYPE=EXOPLANET\n" # fixed @@ -1086,7 +1101,12 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, f"{round(self.fit.dataerr[aavsoC], 7)},{round(airmasses[aavsoC], 7)}," f"{round(detrend_model[aavsoC], 7)}\n") def plate_status(self, plate_status: PlateStatus): - plate_status_file = self.dir / "temp" / f"PlateStatus_{self.p_dict['pName']}_{self.i_dict['date']}.csv" + plate_status_file = self.dir / "temp" / safe_output_filename( + "PlateStatus", + self.p_dict['pName'], + filename_date_token(self.i_dict['date']), + extension="csv", + ) plate_status.writePlateStatus(plate_status_file) class AIDOutputFiles: @@ -1100,7 +1120,12 @@ def __init__(self, fit, p_dict, i_dict, auid, chart_id, vsp_params): self.vsp_params = vsp_params def aavso(self): - params_file = self.dir / f"AID_AAVSO_{self.p_dict['sName']}_{self.i_dict['date']}.txt" + params_file = self.dir / safe_output_filename( + "AID_AAVSO", + self.p_dict['sName'], + filename_date_token(self.i_dict['date']), + extension="txt", + ) with params_file.open('w', encoding="utf-8") as f: f.write("#TYPE=EXTENDED\n" # fixed f"#OBSCODE={self.i_dict['aavso_num']}\n" # UI @@ -1263,7 +1288,12 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label comp_summaries, best_comp_index): temp_dir = Path(save_dir) / "temp" temp_dir.mkdir(parents=True, exist_ok=True) - summary_file = temp_dir / f"CompStarCalibrationSummary_{target_name}_{date}.csv" + summary_file = temp_dir / safe_output_filename( + "CompStarCalibrationSummary", + target_name, + filename_date_token(date), + extension="csv", + ) with summary_file.open('w') as handle: handle.write(f"# Comparison-star calibration summary for {target_name}\n") diff --git a/exotic/plots.py b/exotic/plots.py index 7d439838..34be75e2 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -6,9 +6,18 @@ import numpy as np from pathlib import Path +try: + from utils import filename_date_token, safe_output_filename +except ImportError: + from .utils import filename_date_token, safe_output_filename + plt.style.use(astropy_mpl_style) +def _dated_plot_filename(prefix, *parts, date, extension): + return safe_output_filename(prefix, *parts, filename_date_token(date), extension=extension) + + # Plots of the centroid positions as a function of time def plot_centroids(x_targ, y_targ, x_ref, y_ref, times, target_name, save, date): fig, axs = plt.subplots(3, 2, figsize=(12, 10)) @@ -46,7 +55,12 @@ def plot_centroids(x_targ, y_targ, x_ref, y_ref, times, target_name, save, date) axs[2, 1].plot(times[e] - np.nanmin(times), abs(y_targ[e] - y_ref[e]), 'k.') plt.tight_layout() - plt.savefig(Path(save) / "temp" / f"CentroidPositions&Distances_{target_name}_{date}.pdf") + plt.savefig(Path(save) / "temp" / _dated_plot_filename( + "CentroidPositions&Distances", + target_name, + date=date, + extension="pdf", + )) plt.close() def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_scale, targ_name, save, date, @@ -146,10 +160,21 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc Path(save).mkdir(parents=True, exist_ok=True) Path(save, "temp").mkdir(parents=True, exist_ok=True) - plt.savefig(Path(save) / "temp" / f"FOV_{targ_name}_{date}_" - f"{str(stretch.__class__).split('.')[-1].split(apos)[0]}.pdf", bbox_inches='tight') - plt.savefig(Path(save) / "temp" / f"FOV_{targ_name}_{date}_" - f"{str(stretch.__class__).split('.')[-1].split(apos)[0]}.png", bbox_inches='tight') + stretch_name = str(stretch.__class__).split('.')[-1].split(apos)[0] + plt.savefig(Path(save) / "temp" / _dated_plot_filename( + "FOV", + targ_name, + stretch_name, + date=date, + extension="pdf", + ), bbox_inches='tight') + plt.savefig(Path(save) / "temp" / _dated_plot_filename( + "FOV", + targ_name, + stretch_name, + date=date, + extension="png", + ), bbox_inches='tight') plt.close() @@ -159,7 +184,7 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Flux [ADU]") plt.errorbar(times, targ, yerr=targ_unc, linestyle='None', fmt='-o') - plt.savefig(Path(save) / "temp" / f"TargetRawFlux_{targ_name}_{date}.pdf") + plt.savefig(Path(save) / "temp" / _dated_plot_filename("TargetRawFlux", targ_name, date=date, extension="pdf")) plt.close() plt.figure() @@ -167,7 +192,7 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Flux [ADU]") plt.errorbar(times, ref, yerr=ref_unc, linestyle='None', fmt='-o') - plt.savefig(Path(save) / "temp" / f"CompRawFlux_{targ_name}_{date}.pdf") + plt.savefig(Path(save) / "temp" / _dated_plot_filename("CompRawFlux", targ_name, date=date, extension="pdf")) plt.close() # Plots final reduced light curve (after the 3 sigma clip) @@ -176,11 +201,11 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Normalized Flux") plt.errorbar(times, norm_flux, yerr=norm_unc, linestyle='None', fmt='-bo') - plt.savefig(Path(save) / "temp" / f"NormalizedFluxTime_{targ_name}_{date}.pdf") + plt.savefig(Path(save) / "temp" / _dated_plot_filename("NormalizedFluxTime", targ_name, date=date, extension="pdf")) plt.close() # Save normalized flux to text file prior to NS - params_file = Path(save) / "temp" / f"NormalizedFlux_{targ_name}_{date}.txt" + params_file = Path(save) / "temp" / _dated_plot_filename("NormalizedFlux", targ_name, date=date, extension="txt") with params_file.open('w') as f: f.write("BJD,Norm Flux,Norm Err,AM\n") @@ -221,8 +246,8 @@ def plot_comp_star_pairwise_matrix(pairwise_matrix, best_comp_index, targ_name, ax.set_xlabel("Reference Comparison Star") ax.set_ylabel("Candidate Comparison Star") fig.tight_layout() - fig.savefig(temp_dir / f"CompStarPairwiseScatter_{targ_name}_{date}.png", bbox_inches="tight") - fig.savefig(temp_dir / f"CompStarPairwiseScatter_{targ_name}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarPairwiseScatter", targ_name, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarPairwiseScatter", targ_name, date=date, extension="pdf"), bbox_inches="tight") plt.close(fig) @@ -246,8 +271,8 @@ def plot_comp_star_calibration_series(times, comp_summaries, targ_name, save, da axes[-1].set_xlabel("Time [BJD_TDB]") fig.suptitle(f"{targ_name} Comparison-Star Calibration Curves\n{method_label}", y=1.01) fig.tight_layout() - fig.savefig(temp_dir / f"CompStarCalibrationCurves_{targ_name}_{date}.png", bbox_inches="tight") - fig.savefig(temp_dir / f"CompStarCalibrationCurves_{targ_name}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarCalibrationCurves", targ_name, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarCalibrationCurves", targ_name, date=date, extension="pdf"), bbox_inches="tight") plt.close(fig) @@ -267,8 +292,20 @@ def plot_individual_comp_star_calibration_series(times, comp_summaries, targ_nam fig.suptitle(f"{targ_name} {summary['label']} Calibration Curves\n{method_label}") fig.tight_layout() label_slug = summary['label'].replace(" ", "") - fig.savefig(temp_dir / f"CompStarCalibrationCurve_{label_slug}_{targ_name}_{date}.png", bbox_inches="tight") - fig.savefig(temp_dir / f"CompStarCalibrationCurve_{label_slug}_{targ_name}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename( + "CompStarCalibrationCurve", + label_slug, + targ_name, + date=date, + extension="png", + ), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename( + "CompStarCalibrationCurve", + label_slug, + targ_name, + date=date, + extension="pdf", + ), bbox_inches="tight") plt.close(fig) @@ -292,8 +329,20 @@ def plot_comp_star_candidate_lightcurve_fits(candidate_fit_summaries, targ_name, ax_res.set_title("") label_slug = summary['label'].replace(" ", "") - fig.savefig(temp_dir / f"CompStarLightCurveFit_{label_slug}_{targ_name}_{date}.png", bbox_inches="tight") - fig.savefig(temp_dir / f"CompStarLightCurveFit_{label_slug}_{targ_name}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename( + "CompStarLightCurveFit", + label_slug, + targ_name, + date=date, + extension="png", + ), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename( + "CompStarLightCurveFit", + label_slug, + targ_name, + date=date, + extension="pdf", + ), bbox_inches="tight") plt.close(fig) @@ -355,8 +404,8 @@ def plot_comp_star_suitability(comp_summaries, targ_name, save, date, method_lab ax.legend() ax.grid(axis='y', alpha=0.25) fig.tight_layout() - fig.savefig(temp_dir / f"CompStarSuitability_{targ_name}_{date}.png", bbox_inches="tight") - fig.savefig(temp_dir / f"CompStarSuitability_{targ_name}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarSuitability", targ_name, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarSuitability", targ_name, date=date, extension="pdf"), bbox_inches="tight") plt.close(fig) @@ -435,8 +484,8 @@ def plot_adaptive_aperture_diagnostics(times, aperture_series, annulus_series, f axes[1, 1].grid(alpha=0.25) fig.tight_layout() - fig.savefig(temp_dir / f"AdaptiveApertureDiagnostics_{targ_name}_{date}.png", bbox_inches="tight") - fig.savefig(temp_dir / f"AdaptiveApertureDiagnostics_{targ_name}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("AdaptiveApertureDiagnostics", targ_name, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("AdaptiveApertureDiagnostics", targ_name, date=date, extension="pdf"), bbox_inches="tight") plt.close(fig) @@ -544,8 +593,8 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relati plt.tight_layout() try: - fig.savefig(temp_dir / f"Observing_Statistics_{key}_{date}.png", bbox_inches="tight") - fig.savefig(temp_dir / f"Observing_Statistics_{key}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("Observing_Statistics", key, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("Observing_Statistics", key, date=date, extension="pdf"), bbox_inches="tight") except Exception: pass plt.close() @@ -563,8 +612,8 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date): Path(save).mkdir(parents=True, exist_ok=True) try: - f.savefig(Path(save) / f"FinalLightCurve_{targ_name}_{date}.png", bbox_inches="tight") - f.savefig(Path(save) / f"FinalLightCurve_{targ_name}_{date}.pdf", bbox_inches="tight") + f.savefig(Path(save) / _dated_plot_filename("FinalLightCurve", targ_name, date=date, extension="png"), bbox_inches="tight") + f.savefig(Path(save) / _dated_plot_filename("FinalLightCurve", targ_name, date=date, extension="pdf"), bbox_inches="tight") except Exception: pass plt.close() diff --git a/exotic/utils.py b/exotic/utils.py index ca266933..63e0900a 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -13,6 +13,51 @@ log = logging.getLogger(__name__) +_WINDOWS_RESERVED_FILENAME_STEMS = { + 'CON', + 'PRN', + 'AUX', + 'NUL', + *(f'COM{i}' for i in range(1, 10)), + *(f'LPT{i}' for i in range(1, 10)), +} +_WINDOWS_ILLEGAL_FILENAME_CHARS_RE = re.compile(r'[<>:"/\\|?*\x00-\x1f\x7f]') + + +def sanitize_filename_component(value, fallback='output'): + """Return one filename component that is safe on Windows, macOS, and Linux.""" + + cleaned = _WINDOWS_ILLEGAL_FILENAME_CHARS_RE.sub('-', str(value or '')) + cleaned = cleaned.rstrip(' .') + if cleaned in {'', '.', '..'}: + cleaned = fallback + device_stem = cleaned.split('.', 1)[0].upper() + if device_stem in _WINDOWS_RESERVED_FILENAME_STEMS: + cleaned = f'_{cleaned}' + return cleaned + + +def filename_date_token(value): + """Return YYYY-MM-DD when a filename date includes a time component.""" + + text = str(value or '').strip() + match = re.match(r'(\d{4})[-/]?(\d{2})[-/]?(\d{2})', text) + if match: + return f'{match.group(1)}-{match.group(2)}-{match.group(3)}' + return text + + +def safe_output_filename(prefix, *parts, extension): + """Build a filename from EXOTIC output labels without illegal path characters.""" + + stem_parts = [str(prefix), *(str(part) for part in parts)] + safe_stem = sanitize_filename_component('_'.join(stem_parts), fallback=str(prefix or 'output')) + ext = str(extension or '') + if ext and not ext.startswith('.'): + ext = f'.{ext}' + return f'{safe_stem}{ext}' + + def user_input(prompt, type_, values=None, max_tries=1000): """ Captures user_input and casts it to the expected type diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index d2a2621c..a83c2ee2 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -212,6 +212,17 @@ def test_comparison_candidate_triangle_plot_uses_candidate_specific_name(tmp_pat assert output_path.parent == tmp_path / "comp7" / "temp" +def test_comparison_candidate_triangle_plot_uses_date_only_from_timestamp(tmp_path): + output_path = comparison_candidate_triangle_plot_output_path( + tmp_path / "comp7", + "XO-1/b", + "2026-05-06T19:51:13.964-0700", + 6, + ) + + assert output_path.name == "Comp7_Triangle_XO-1-b_2026-05-06.png" + + def test_save_final_triangle_plot_regenerates_when_selected_artifact_missing(tmp_path): class DummyFigure: def savefig(self, path): @@ -463,8 +474,8 @@ def test_detrend_flux_on_out_of_transit_baseline_falls_back_to_prior_ephemeris() assert "ephemeris-centered transit window" in fallback["note"] -def test_deduplicate_comparison_star_coords_merges_nearby_duplicates(): - unique_coords, duplicate_messages = deduplicate_comparison_star_coords( +def test_deduplicate_comparison_star_coords_preserves_nearby_user_stars(): + preserved_coords, duplicate_messages = deduplicate_comparison_star_coords( [ [1826.0, 1499.0], [1827.0, 1511.0], @@ -474,9 +485,13 @@ def test_deduplicate_comparison_star_coords_merges_nearby_duplicates(): min_separation_pixels=15.0, ) - assert unique_coords == [[1826.0, 1499.0], [842.0, 1810.0]] - assert len(duplicate_messages) == 2 - assert "Merged comparison star #2" in duplicate_messages[0] + assert preserved_coords == [ + [1826.0, 1499.0], + [1827.0, 1511.0], + [1828.0, 1487.0], + [842.0, 1810.0], + ] + assert duplicate_messages == [] def test_robust_flux_floor_mask_rejects_tiny_positive_outliers(): diff --git a/tests/test_lazy_pylightcurve_imports.py b/tests/test_lazy_pylightcurve_imports.py index 72e1e3dc..46310428 100644 --- a/tests/test_lazy_pylightcurve_imports.py +++ b/tests/test_lazy_pylightcurve_imports.py @@ -85,7 +85,7 @@ def test_imports_eagerly_load_pylightcurve_without_noise(): assert result.returncode == 0, result.stderr or result.stdout assert "imports-ok" in result.stdout - assert result.stdout.count("Importing modules. Please wait.......") == 1 + assert "Importing modules. Please wait......." not in result.stdout assert "LOUD-STDOUT" not in result.stdout assert "LOUD-STDERR" not in result.stderr diff --git a/tests/test_utils.py b/tests/test_utils.py index 9049ecdc..4270a177 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,6 +2,23 @@ from unittest.mock import patch +def test_filename_date_token_uses_date_only_for_iso_timestamp(): + assert filename_date_token("2026-05-06T19:51:13.964-0700") == "2026-05-06" + assert filename_date_token("20260506T195113") == "2026-05-06" + assert filename_date_token("2026/05/06 19:51:13") == "2026-05-06" + + +def test_safe_output_filename_sanitizes_filename_chars(): + filename = safe_output_filename( + "BestFit", + "XO-1/b", + filename_date_token("2026-05-06T19:51:13.964-0700"), + extension="png", + ) + + assert filename == "BestFit_XO-1-b_2026-05-06.png" + + class TestUserInput: """tests the `user_input()` function""" From ce40c40fba69667c25a79efdbabce336cae2ca7e Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 8 May 2026 10:07:16 +1000 Subject: [PATCH 035/116] Update requirements.txt --- requirements.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7f02c452..d8712faf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,8 @@ astropy~=6.1 astroquery~=0.4.7 barycorrpy~=0.4.4 bottleneck~=1.4.2 -colour_demosaicing~=0.2.6 +colour_demosaicing==0.2.6 +colour-science>=0.4.4,<0.4.7 dynesty~=1.2.3;platform_system=='Windows' holoviews~=1.19.1 importlib-metadata>=3.6;python_version<='3.7' @@ -12,9 +13,10 @@ LDTk~=1.8.4 lmfit~=1.3.2 matplotlib~=3.9.2 numba~=0.59.1 -numpy~=1.26.4 -pandas~=2.2.3 +numpy==1.26.4 +pandas>=2.2.2,<2.3 panel~=1.5.2 +bokeh>=3.5,<3.7 photutils~=2.0.0 pylightcurve>=4.0.1,<5 python_dateutil~=2.9 @@ -24,6 +26,7 @@ rebound~=4.4.3 requests~=2.32.3 scipy~=1.14.1 scikit-image~=0.24.0 +tifffile<2026.4.11 statsmodels~=0.14.4 tenacity~=9.0 mpi4py>=4.0;platform_system=='Linux' From 41344bd234fdc13f9d89accac8e06f34ba6af61b Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 8 May 2026 15:56:47 +1000 Subject: [PATCH 036/116] triangle plot bug --- exotic/api/elca.py | 98 ++++++++++++++++--- exotic/api/plotting.py | 53 ++++++++--- tests/test_elca_baseline.py | 163 ++++++++++++++++++++++++++++++++ tests/test_plotting_contours.py | 17 ++++ 4 files changed, 305 insertions(+), 26 deletions(-) create mode 100644 tests/test_plotting_contours.py diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 9d551aa2..e30616b4 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -728,7 +728,14 @@ def _get_plot_range(self, key): return [lower, upper] - def _expand_plot_range_for_sample_cloud(self, key, plot_range, sample_values, center): + def _expand_plot_range_for_sample_cloud( + self, + key, + plot_range, + sample_values, + center, + required_visible_fraction=0.95, + ): sample_values = np.asarray(sample_values, dtype=float) finite_values = sample_values[np.isfinite(sample_values)] if finite_values.size < 2: @@ -736,8 +743,8 @@ def _expand_plot_range_for_sample_cloud(self, key, plot_range, sample_values, ce lower, upper = [float(value) for value in plot_range] in_range = (finite_values >= lower) & (finite_values <= upper) - minimum_in_range = min(finite_values.size, max(8, int(0.05 * finite_values.size))) - if np.count_nonzero(in_range) >= minimum_in_range: + visible_fraction = np.count_nonzero(in_range) / float(finite_values.size) + if visible_fraction >= required_visible_fraction: return plot_range q_lower, q_upper = np.nanpercentile(finite_values, [0.5, 99.5]) @@ -761,19 +768,40 @@ def _expand_plot_range_for_sample_cloud(self, key, plot_range, sample_values, ce def _get_triangle_plot_samples(self): if self.ns_type == 'ultranest': - points = np.asarray(self.results['weighted_samples']['points'], dtype=float) - logl = np.asarray(self.results['weighted_samples']['logl'], dtype=float) - return points, logl + weighted_samples = self.results['weighted_samples'] + points = np.asarray(weighted_samples['points'], dtype=float) + logl = np.asarray(weighted_samples['logl'], dtype=float) + weights = self._get_triangle_plot_sample_weights( + weighted_samples.get('weights'), + points.shape[0], + ) + return points, logl, weights points = np.asarray(self.results.samples, dtype=float) weights = np.exp(self.results.logwt - self.results.logz[-1]) index_samples = resample_equal(np.arange(points.shape[0], dtype=float)[:, None], weights) index_samples = np.clip(np.rint(index_samples[:, 0]).astype(int), 0, points.shape[0] - 1) - return points[index_samples], np.asarray(self.results.logl, dtype=float)[index_samples] + return points[index_samples], np.asarray(self.results.logl, dtype=float)[index_samples], None + + def _get_triangle_plot_sample_weights(self, weights, sample_count): + if weights is None: + return None + + weights = np.asarray(weights, dtype=float) + if weights.ndim != 1 or weights.shape[0] != sample_count: + return None + + finite = np.isfinite(weights) & (weights >= 0) + if not np.all(finite): + weights = np.where(finite, weights, 0.0) + + if not np.isfinite(np.sum(weights)) or np.sum(weights) <= 0: + return None + return weights def get_parameter_posterior_samples(self, key): try: - sample_points, _ = self._get_triangle_plot_samples() + sample_points, _, _ = self._get_triangle_plot_samples() except Exception: return np.array([], dtype=float) @@ -794,9 +822,23 @@ def get_parameter_posterior_samples(self, key): ] return np.asarray(physical_samples, dtype=float) - def _estimate_histogram_mode(self, samples, bounds=None, bins=None): + def _estimate_histogram_mode(self, samples, bounds=None, bins=None, weights=None): samples = np.asarray(samples, dtype=float) - finite_samples = samples[np.isfinite(samples)] + if weights is None: + finite_mask = np.isfinite(samples) + finite_weights = None + else: + weights = np.asarray(weights, dtype=float) + if weights.shape != samples.shape: + finite_mask = np.isfinite(samples) + finite_weights = None + else: + finite_mask = np.isfinite(samples) & np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + if finite_weights.size == 0 or np.sum(finite_weights) <= 0: + finite_weights = None + + finite_samples = samples[finite_mask] if finite_samples.size == 0: return np.nan, np.nan if finite_samples.size == 1: @@ -818,15 +860,23 @@ def _estimate_histogram_mode(self, samples, bounds=None, bins=None): bins = int(np.clip(np.sqrt(finite_samples.size), 10, 80)) bins = max(1, int(bins)) - counts, edges = np.histogram(finite_samples, bins=bins, range=(lower, upper)) + counts, edges = np.histogram(finite_samples, bins=bins, range=(lower, upper), weights=finite_weights) if counts.size == 0: return float(np.nanmedian(finite_samples)), np.nan + if not np.any(counts > 0): + return float(np.nanmedian(finite_samples)), np.nan mode_index = int(np.argmax(counts)) mode = float(0.5 * (edges[mode_index] + edges[mode_index + 1])) bin_width = float(edges[1] - edges[0]) if edges.size > 1 else np.nan return mode, bin_width + def _get_triangle_plot_title_center(self, samples, plot_range, fallback_center, bins, weights=None): + mode, _ = self._estimate_histogram_mode(samples, bounds=plot_range, bins=bins, weights=weights) + if np.isfinite(mode): + return float(mode) + return fallback_center + def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bins=None): diagnostics = { 'key': key, @@ -1020,6 +1070,12 @@ def _get_triangle_plot_display_spec(self, sampled_keys, sample_parameters, sampl if not np.isfinite(error) or error <= 0: error = float(np.nanstd(magnitude_samples)) plot_lower, plot_upper = self._get_plot_range(key) + plot_lower, plot_upper = self._expand_plot_range_for_sample_cloud( + key, + [plot_lower, plot_upper], + sample_points[:, geometry_index], + center, + ) max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) if not np.isfinite(max_distance) or max_distance <= 0: max_distance = float(np.nanmax(magnitude_samples)) @@ -1170,13 +1226,14 @@ def _get_triangle_plot_payload(self): sampled_keys = getattr(self, 'sampled_keys', list(self.bounds.keys())) sample_parameters = getattr(self, 'sample_parameters', self.parameters) sample_errors = getattr(self, 'sample_errors', self.errors) - sample_points, sample_logl = self._get_triangle_plot_samples() + sample_points, sample_logl, sample_weights = self._get_triangle_plot_samples() display_spec = self._get_triangle_plot_display_spec(sampled_keys, sample_parameters, sample_errors, sample_points) geometry_overlay = self._get_triangle_plot_geometry_overlay(display_spec, sample_points) geometry_summary = self._get_triangle_plot_geometry_summary(sampled_keys, sample_points) display_points = np.array(sample_points, copy=True) display_logl = np.array(sample_logl, copy=True) + display_weights = None if sample_weights is None else np.array(sample_weights, copy=True) mask_values = np.array(sample_points, copy=True) if display_spec is not None: @@ -1187,8 +1244,12 @@ def _get_triangle_plot_payload(self): negative_points[:, geometry_index] = -display_spec['magnitude_samples'] display_points = np.vstack([positive_points, negative_points]) display_logl = np.concatenate([sample_logl, sample_logl]) + if sample_weights is not None: + display_weights = np.concatenate([sample_weights, sample_weights]) mask_values = np.array(display_points, copy=True) + plot_bins = int(max(1, np.sqrt(display_points.shape[0]))) + flabels = { 'rprs': r'R$_{p}$/R$_{s}$', 'per': r'Period [day]', @@ -1222,7 +1283,6 @@ def _get_triangle_plot_payload(self): center = sample_parameters.get(key, self.parameters.get(key, 0.0)) error = sample_errors.get(key, self.errors.get(key, 0.0)) label = flabels.get(key, key) - title = f"{center:.5f} +- {error:.5f}" plot_range = self._get_plot_range(key) if sample_points.ndim == 2 and i < sample_points.shape[1]: plot_range = self._expand_plot_range_for_sample_cloud( @@ -1231,6 +1291,16 @@ def _get_triangle_plot_payload(self): sample_points[:, i], center, ) + title_center = center + if display_points.ndim == 2 and i < display_points.shape[1]: + title_center = self._get_triangle_plot_title_center( + display_points[:, i], + plot_range, + center, + plot_bins, + weights=None if display_weights is None else display_weights, + ) + title = f"{title_center:.5f} +- {error:.5f}" if display_spec is not None and key == display_spec['key']: label = display_spec['label'] @@ -1249,6 +1319,7 @@ def _get_triangle_plot_payload(self): 'sampled_keys': sampled_keys, 'display_points': display_points, 'display_logl': display_logl, + 'display_weights': display_weights, 'mask_values': mask_values, 'display_spec': display_spec, 'geometry_overlay': geometry_overlay, @@ -1850,6 +1921,7 @@ def plot_triangle(self): labels=payload['labels'], bins=int(np.sqrt(payload['display_points'].shape[0])), range=payload['ranges'], + weights=payload['display_weights'], plot_contours=True, levels=self._triangle_contour_levels(chi2, mask1, mask2, mask3), plot_density=False, diff --git a/exotic/api/plotting.py b/exotic/api/plotting.py index 3c346492..9bce6220 100644 --- a/exotic/api/plotting.py +++ b/exotic/api/plotting.py @@ -428,7 +428,7 @@ def corner(xs, bins=20, range=None, weights=None, color="k", hist_bin_factor=1, return fig -def quantile(x, q, weights=None): +def quantile(x, q, weights=None): """ Compute sample quantiles with support for weighted samples. @@ -475,15 +475,44 @@ def quantile(x, q, weights=None): raise ValueError("Dimension mismatch: len(weights) != len(x)") idx = np.argsort(x) sw = weights[idx] - cdf = np.cumsum(sw)[:-1] - cdf /= cdf[-1] - cdf = np.append(0, cdf) - return np.interp(q, cdf, x[idx]).tolist() - -def hist2d(x, y, bins=20, range=None, levels=[2], - ax=None, plot_datapoints=True, plot_contours=True, - contour_kwargs=None, contourf_kwargs=None, data_kwargs=None, - **kwargs): + cdf = np.cumsum(sw)[:-1] + cdf /= cdf[-1] + cdf = np.append(0, cdf) + return np.interp(q, cdf, x[idx]).tolist() + +def _contour_levels_within_surface(levels, surface_min, surface_max): + levels = np.asarray(levels, dtype=float) + levels = np.unique(levels[np.isfinite(levels)]) + if levels.size == 0: + return levels + + if ( + not np.isfinite(surface_min) + or not np.isfinite(surface_max) + or surface_min >= surface_max + ): + return np.array([], dtype=float) + + valid_levels = levels[(levels > surface_min) & (levels < surface_max)] + if valid_levels.size > 0: + return valid_levels + + surface_span = float(surface_max - surface_min) + epsilon = max( + surface_span * 1e-9, + np.finfo(float).eps * max(1.0, abs(float(surface_min)), abs(float(surface_max))), + ) + lower = float(surface_min) + epsilon + upper = float(surface_max) - epsilon + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return np.array([0.5 * (float(surface_min) + float(surface_max))], dtype=float) + + return np.unique(np.clip(levels, lower, upper)) + +def hist2d(x, y, bins=20, range=None, levels=[2], + ax=None, plot_datapoints=True, plot_contours=True, + contour_kwargs=None, contourf_kwargs=None, data_kwargs=None, + **kwargs): if ax is None: ax = plt.gca() @@ -528,9 +557,7 @@ def hist2d(x, y, bins=20, range=None, levels=[2], surface_max = np.nanmax(contour_surface) if not np.isfinite(surface_min) or not np.isfinite(surface_max) or np.isclose(surface_min, surface_max): raise ValueError("degenerate contour surface") - contour_levels = contour_levels[ - (contour_levels > surface_min) & (contour_levels < surface_max) - ] + contour_levels = _contour_levels_within_surface(contour_levels, surface_min, surface_max) if contour_levels.size == 0: raise ValueError("no contour levels fall within the plotted surface") diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index e720d54c..0c0d1065 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -1043,6 +1043,169 @@ def test_triangle_payload_expands_degenerate_error_ranges_to_sample_cloud(monkey assert payload["ranges"][1][1] >= 0.0038 +def test_triangle_payload_title_uses_visible_histogram_peak(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.34], + "a0": [0.95, 1.05], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.33796, "a0": 1.0} + fit.errors = {"rprs": 0.09150, "a0": 0.001} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + rprs_samples = np.concatenate([ + np.linspace(0.108, 0.122, 20), + np.linspace(0.318, 0.338, 80), + ]) + weights = np.concatenate([ + np.ones(20, dtype=float), + np.full(80, 0.01, dtype=float), + ]) + points = np.column_stack([rprs_samples, np.linspace(0.998, 1.002, rprs_samples.size)]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + "weights": weights, + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["titles"][0].startswith("0.11900 +-") + assert not payload["titles"][0].startswith("0.33796") + np.testing.assert_allclose(payload["display_weights"], weights) + + +def test_plot_triangle_passes_ultranest_weights_to_visible_histograms(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + captured = {} + + def fake_corner(*args, **kwargs): + captured["weights"] = kwargs["weights"] + return "figure" + + monkeypatch.setattr(elca, "corner", fake_corner) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "a0": [0.95, 1.05], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.1, "a0": 1.0} + fit.errors = {"rprs": 0.01, "a0": 0.001} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + points = np.column_stack([ + np.linspace(0.09, 0.11, 12), + np.linspace(0.998, 1.002, 12), + ]) + weights = np.linspace(1.0, 2.0, points.shape[0]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + "weights": weights, + }, + "samples": points.copy(), + } + + fig = fit.plot_triangle() + + assert fig == "figure" + np.testing.assert_allclose(captured["weights"], weights) + + +def test_triangle_payload_expands_sparse_visible_ranges_to_sample_cloud(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "a0": [0.95, 1.05], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.100, "a0": 1.0} + fit.errors = {"rprs": 0.01, "a0": 1e-4} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + points = np.column_stack( + [ + np.linspace(0.090, 0.110, 100), + np.linspace(0.980, 1.020, 100), + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["ranges"][1][0] <= 0.981 + assert payload["ranges"][1][1] >= 1.019 + + +def test_triangle_payload_expands_mirrored_impact_parameter_range_to_sample_cloud(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.sampled_keys = ["rprs", "b", "a0"] + fit.sample_bounds = { + "rprs": [0.0, 0.2], + "b": [0.0, 1.2], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.sample_parameters = {"rprs": 0.10, "b": 0.30, "a0": 1.0} + fit.sample_errors = {"rprs": 0.01, "b": 0.01, "a0": 0.001} + fit.parameters = {"rprs": 0.10, "inc": 88.6, "a0": 1.0} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.001} + points = np.column_stack( + [ + np.linspace(0.090, 0.110, 100), + np.linspace(0.10, 0.80, 100), + np.linspace(0.998, 1.002, 100), + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["labels"][1] == r"$\Delta b$" + assert payload["ranges"][1][0] <= -0.52 + assert payload["ranges"][1][1] >= 0.52 + + def test_triangle_payload_tracks_left_and_right_geometry_branches_for_inclination(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_plotting_contours.py b/tests/test_plotting_contours.py new file mode 100644 index 00000000..537e630d --- /dev/null +++ b/tests/test_plotting_contours.py @@ -0,0 +1,17 @@ +import numpy as np + +from exotic.api import plotting + + +def test_contour_levels_inside_surface_are_preserved(): + levels = plotting._contour_levels_within_surface([0.2, 0.5, 0.8], 0.0, 1.0) + + np.testing.assert_allclose(levels, np.array([0.2, 0.5, 0.8])) + + +def test_contour_levels_outside_surface_are_clipped_into_drawable_range(): + levels = plotting._contour_levels_within_surface([10.0, 20.0, 30.0], 0.0, 1.0) + + assert levels.size == 1 + assert 0.0 < levels[0] < 1.0 + From a6db247a7d227fac6c601002a4ce056618ccd9a5 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 9 May 2026 12:37:36 +1000 Subject: [PATCH 037/116] better triangle bounds --- exotic/api/elca.py | 230 +++++++++++++++++++++++-- exotic/exotic.py | 296 +++++++++++++++++++++++++++++--- tests/test_elca_baseline.py | 57 +++++- tests/test_exotic_rprs_retry.py | 142 +++++++++++++++ 4 files changed, 675 insertions(+), 50 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index e30616b4..f14a6f56 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -72,6 +72,9 @@ from .ultranest_utils import run_reactive_sampler BAD_LOG_LIKELIHOOD = -1.0e100 +TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX = 0.50 +TRIANGLE_PLOT_EDGE_MIN_SAMPLE_COUNT = 30 +TRIANGLE_PLOT_EDGE_EXPANSION_STEPS = 8 def _pylightcurve_import_watchdog_seconds(): try: @@ -707,26 +710,28 @@ def _ultranest_error_needs_sample_fallback(self, parameter_index, center, report def _get_plot_range(self, key): sample_parameters = getattr(self, 'sample_parameters', {}) sample_errors = getattr(self, 'sample_errors', {}) - sample_bounds = getattr(self, 'sample_bounds', self.bounds) + sample_bounds = getattr(self, 'sample_bounds', getattr(self, 'bounds', {})) center = sample_parameters[key] if key in sample_parameters else self.parameters[key] error = sample_errors[key] if key in sample_errors else self.errors[key] - lower = center - 5 * error - upper = center + 5 * error - - if key in sample_bounds: - bound_lower, bound_upper = sample_bounds[key] - lower = max(lower, bound_lower) - upper = min(upper, bound_upper) - if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: - if key in sample_bounds: - bound_lower, bound_upper = sample_bounds[key] - return [bound_lower, bound_upper] + if isinstance(sample_bounds, dict) and key in sample_bounds: + try: + lower, upper = [ + float(value) for value in np.asarray(sample_bounds[key], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + lower = np.nan + upper = np.nan + if np.isfinite(lower) and np.isfinite(upper) and lower < upper: + return [lower, upper] - pad = error if np.isfinite(error) and error > 0 else max(abs(center) * 1e-6, 1e-6) - return [center - pad, center + pad] + lower = center - 5 * error + upper = center + 5 * error + if np.isfinite(lower) and np.isfinite(upper) and lower < upper: + return [lower, upper] - return [lower, upper] + pad = error if np.isfinite(error) and error > 0 else max(abs(center) * 1e-6, 1e-6) + return [center - pad, center + pad] def _expand_plot_range_for_sample_cloud( self, @@ -766,6 +771,165 @@ def _expand_plot_range_for_sample_cloud( return plot_range return [float(new_lower), float(new_upper)] + def _histogram_edge_peak_fractions(self, sample_values, plot_range, bins, weights=None): + sample_values = np.asarray(sample_values, dtype=float) + finite_mask = np.isfinite(sample_values) + finite_weights = None + + if weights is not None: + weights = np.asarray(weights, dtype=float) + if weights.shape == sample_values.shape: + finite_mask &= np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + finite_weight_sum = np.sum(finite_weights) + if ( + finite_weights.size == 0 + or not np.isfinite(finite_weight_sum) + or finite_weight_sum <= 0 + ): + finite_weights = None + + finite_values = sample_values[finite_mask] + if finite_values.size < 2: + return np.nan, np.nan, np.nan + + try: + lower, upper = [float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + return np.nan, np.nan, np.nan + + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return np.nan, np.nan, np.nan + + bins = max(1, int(bins)) + counts, _ = np.histogram( + finite_values, + bins=bins, + range=(lower, upper), + weights=finite_weights, + ) + counts = np.asarray(counts, dtype=float) + if counts.size == 0: + return np.nan, np.nan, np.nan + + peak = float(np.nanmax(counts)) + if not np.isfinite(peak) or peak <= 0: + return np.nan, np.nan, np.nan + + lower_fraction = float(counts[0] / peak) + upper_fraction = float(counts[-1] / peak) + return lower_fraction, upper_fraction, peak + + def _get_plot_range_expansion_bounds(self, key, sample_values, center): + sample_bounds = getattr(self, 'sample_bounds', getattr(self, 'bounds', {})) + if isinstance(sample_bounds, dict) and key in sample_bounds: + try: + bound_lower, bound_upper = [ + float(value) for value in np.asarray(sample_bounds[key], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + bound_lower = np.nan + bound_upper = np.nan + + if np.isfinite(bound_lower) and np.isfinite(bound_upper) and bound_lower < bound_upper: + return [bound_lower, bound_upper] + + sample_values = np.asarray(sample_values, dtype=float) + finite_values = sample_values[np.isfinite(sample_values)] + try: + center = float(center) + except (TypeError, ValueError): + center = np.nan + if np.isfinite(center): + finite_values = np.concatenate([finite_values, [center]]) + if finite_values.size < 2: + return None + + bound_lower = float(np.nanmin(finite_values)) + bound_upper = float(np.nanmax(finite_values)) + width = bound_upper - bound_lower + if not np.isfinite(width) or width <= 0: + padding = max(abs(float(center)) * 1e-6 if np.isfinite(center) else 0.0, 1e-6) + else: + padding = 0.05 * width + return [bound_lower - padding, bound_upper + padding] + + def _expand_plot_range_for_histogram_edge_dropoff( + self, + key, + plot_range, + sample_values, + center, + bins=None, + weights=None, + max_edge_peak_fraction=TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX, + minimum_count=TRIANGLE_PLOT_EDGE_MIN_SAMPLE_COUNT, + max_steps=TRIANGLE_PLOT_EDGE_EXPANSION_STEPS, + ): + sample_values = np.asarray(sample_values, dtype=float) + finite_values = sample_values[np.isfinite(sample_values)] + if finite_values.size < int(minimum_count): + return plot_range + + try: + lower, upper = [float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + return plot_range + + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return plot_range + + expansion_bounds = self._get_plot_range_expansion_bounds(key, finite_values, center) + if expansion_bounds is None: + return plot_range + + bound_lower, bound_upper = expansion_bounds + if not np.isfinite(bound_lower) or not np.isfinite(bound_upper) or bound_lower >= bound_upper: + return plot_range + + lower = max(lower, bound_lower) + upper = min(upper, bound_upper) + if lower >= upper: + return plot_range + + if bins is None: + bins = int(np.clip(np.sqrt(finite_values.size), 10, 80)) + bins = max(1, int(bins)) + epsilon = max((bound_upper - bound_lower) * 1e-12, np.finfo(float).eps) + + for _ in range(max(0, int(max_steps)) + 1): + lower_fraction, upper_fraction, _ = self._histogram_edge_peak_fractions( + sample_values, + [lower, upper], + bins, + weights=weights, + ) + if not np.isfinite(lower_fraction) or not np.isfinite(upper_fraction): + return [float(lower), float(upper)] + + needs_lower = lower_fraction >= max_edge_peak_fraction + needs_upper = upper_fraction >= max_edge_peak_fraction + if not needs_lower and not needs_upper: + return [float(lower), float(upper)] + + width = upper - lower + if not np.isfinite(width) or width <= 0: + return [float(lower), float(upper)] + + new_lower = lower + new_upper = upper + if needs_lower and lower > bound_lower + epsilon: + new_lower = max(bound_lower, lower - width) + if needs_upper and upper < bound_upper - epsilon: + new_upper = min(bound_upper, upper + width) + + if new_lower == lower and new_upper == upper: + return [float(lower), float(upper)] + + lower, upper = new_lower, new_upper + + return [float(lower), float(upper)] + def _get_triangle_plot_samples(self): if self.ns_type == 'ultranest': weighted_samples = self.results['weighted_samples'] @@ -1053,7 +1217,14 @@ def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bin diagnostics['q95'] = float(q95) return diagnostics - def _get_triangle_plot_display_spec(self, sampled_keys, sample_parameters, sample_errors, sample_points): + def _get_triangle_plot_display_spec( + self, + sampled_keys, + sample_parameters, + sample_errors, + sample_points, + sample_weights=None, + ): if 'b' in sampled_keys: key = 'b' label = r'$\Delta b$' @@ -1076,6 +1247,15 @@ def _get_triangle_plot_display_spec(self, sampled_keys, sample_parameters, sampl sample_points[:, geometry_index], center, ) + plot_bins = int(max(1, np.sqrt(sample_points.shape[0]))) + plot_lower, plot_upper = self._expand_plot_range_for_histogram_edge_dropoff( + key, + [plot_lower, plot_upper], + sample_points[:, geometry_index], + center, + bins=plot_bins, + weights=sample_weights, + ) max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) if not np.isfinite(max_distance) or max_distance <= 0: max_distance = float(np.nanmax(magnitude_samples)) @@ -1227,7 +1407,13 @@ def _get_triangle_plot_payload(self): sample_parameters = getattr(self, 'sample_parameters', self.parameters) sample_errors = getattr(self, 'sample_errors', self.errors) sample_points, sample_logl, sample_weights = self._get_triangle_plot_samples() - display_spec = self._get_triangle_plot_display_spec(sampled_keys, sample_parameters, sample_errors, sample_points) + display_spec = self._get_triangle_plot_display_spec( + sampled_keys, + sample_parameters, + sample_errors, + sample_points, + sample_weights=sample_weights, + ) geometry_overlay = self._get_triangle_plot_geometry_overlay(display_spec, sample_points) geometry_summary = self._get_triangle_plot_geometry_summary(sampled_keys, sample_points) @@ -1291,6 +1477,14 @@ def _get_triangle_plot_payload(self): sample_points[:, i], center, ) + plot_range = self._expand_plot_range_for_histogram_edge_dropoff( + key, + plot_range, + sample_points[:, i], + center, + bins=plot_bins, + weights=sample_weights, + ) title_center = center if display_points.ndim == 2 and i < display_points.shape[1]: title_center = self._get_triangle_plot_title_center( @@ -1385,6 +1579,8 @@ def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs= ax.plot(curves['centers'], curves['right_curve'], color=branch_right_color, linestyle='--', linewidth=0.75, alpha=0.75, zorder=3) ax.set_title(title, **title_kwargs) + if 'fontsize' in title_kwargs: + ax.title.set_fontsize(title_kwargs['fontsize']) ax.set_xlim(hist_range) max_y = max( diff --git a/exotic/exotic.py b/exotic/exotic.py index d74c8cb0..390e6dfc 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -235,6 +235,9 @@ ARS_SEARCH_BOUND_FALLBACK_MAX = 100.0 ARS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 ARS_RETRY_MIN_HALF_WIDTH = 0.0 +IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES_DEFAULT = 5 +INCLINATION_SEARCH_BOUND_MIN = 0.0 +INCLINATION_SEARCH_BOUND_MAX = 90.0 INITIAL_ARS_BOUND_SIGMA_MULTIPLIER = 5.0 INITIAL_ARS_BOUND_FALLBACK_RELATIVE_HALF_WIDTH = 0.25 DURATION_PRIOR_MONTE_CARLO_SAMPLES = 256 @@ -2273,6 +2276,10 @@ def annotate_ars_posterior_refit(fit, applied, note=None, history=None): annotate_parameter_posterior_refit(fit, 'ars', applied, note=note, history=history) +def annotate_impact_parameter_posterior_refit(fit, applied, note=None, history=None): + annotate_parameter_posterior_refit(fit, 'b', applied, note=note, history=history) + + def annotate_parameter_posterior_refit(fit, parameter_key, applied, note=None, history=None): if fit is None: return @@ -2383,7 +2390,7 @@ def get_posterior_refit_final_bounds(fit, fallback_bounds): fit_bounds = getattr(fit, 'posterior_refit_final_bounds', None) if not isinstance(fit_bounds, dict): fit_bounds = {} - for key in ('rprs', 'ars'): + for key in ('rprs', 'ars', 'inc'): refit_bounds = getattr(fit, f'{key}_posterior_refit_bounds', None) if refit_bounds is not None: fit_bounds[key] = refit_bounds @@ -2461,8 +2468,133 @@ def sanitize_ars_search_bounds(bounds): ) +def sanitize_inclination_search_bounds(bounds): + return sanitize_parameter_search_bounds( + bounds, + 'inc', + INCLINATION_SEARCH_BOUND_MIN, + maximum_bound=INCLINATION_SEARCH_BOUND_MAX, + fallback_maximum=INCLINATION_SEARCH_BOUND_MAX, + ) + + def sanitize_retry_search_bounds(bounds): - return sanitize_ars_search_bounds(sanitize_rprs_search_bounds(bounds)) + return sanitize_inclination_search_bounds(sanitize_ars_search_bounds(sanitize_rprs_search_bounds(bounds))) + + +def impact_parameter_scale_for_retry(values): + try: + ars = float(values['ars']) + except (KeyError, TypeError, ValueError): + return np.nan + + try: + ecc = float(values.get('ecc', 0.0)) + except (TypeError, ValueError): + ecc = 0.0 + + try: + omega = np.deg2rad(float(values.get('omega', 0.0))) + except (TypeError, ValueError): + omega = 0.0 + + denom = 1.0 + ecc * np.sin(omega) + if np.isclose(denom, 0.0): + denom = np.finfo(float).eps + scale = ars * (1.0 - ecc ** 2) / denom + return float(scale) if np.isfinite(scale) and scale > 0 else np.nan + + +def impact_parameter_scale_range_for_retry(prior, bounds): + values = dict(prior) + ars_candidates = [] + if 'ars' in bounds: + try: + ars_candidates.extend( + float(value) for value in np.asarray(bounds['ars'], dtype=float).reshape(-1)[:2] + ) + except (TypeError, ValueError, IndexError): + pass + if 'ars' in values: + try: + ars_candidates.append(float(values['ars'])) + except (TypeError, ValueError): + pass + + scales = [] + for ars_value in ars_candidates: + candidate_values = dict(values) + candidate_values['ars'] = ars_value + scale = impact_parameter_scale_for_retry(candidate_values) + if np.isfinite(scale) and scale > 0: + scales.append(scale) + + if not scales: + scale = impact_parameter_scale_for_retry(values) + if np.isfinite(scale) and scale > 0: + scales.append(scale) + + if not scales: + return np.nan, np.nan + return float(np.nanmin(scales)), float(np.nanmax(scales)) + + +def inclination_from_impact_parameter_for_retry(impact_parameter, scale): + if not np.isfinite(scale) or scale <= 0: + return np.nan + try: + impact_parameter = float(impact_parameter) + except (TypeError, ValueError): + return np.nan + cosi = np.clip(impact_parameter / scale, -1.0, 1.0) + return float(np.rad2deg(np.arccos(cosi))) + + +def impact_parameter_retry_proposed_inclination_bounds(diagnostics, current_prior, current_bounds): + if not diagnostics: + return None + if 'inc' not in current_bounds: + return None + + try: + previous_lower, previous_upper = [ + float(value) for value in np.asarray(current_bounds['inc'], dtype=float).reshape(-1)[:2] + ] + proposed_b_lower, proposed_b_upper = [ + float(value) for value in np.asarray(diagnostics.get('bounds'), dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return None + + if ( + not np.isfinite(previous_lower) + or not np.isfinite(previous_upper) + or previous_lower >= previous_upper + or not np.isfinite(proposed_b_lower) + or not np.isfinite(proposed_b_upper) + or proposed_b_lower >= proposed_b_upper + ): + return None + + min_scale, max_scale = impact_parameter_scale_range_for_retry(current_prior, current_bounds) + if not np.isfinite(min_scale) or not np.isfinite(max_scale): + return None + + new_lower = previous_lower + new_upper = previous_upper + clipped_edge = diagnostics.get('edge') + + if clipped_edge in ('upper', None): + inc_for_upper_b = inclination_from_impact_parameter_for_retry(proposed_b_upper, max_scale) + if np.isfinite(inc_for_upper_b): + new_lower = min(new_lower, inc_for_upper_b) + + if clipped_edge in ('lower', None): + inc_for_lower_b = inclination_from_impact_parameter_for_retry(max(0.0, proposed_b_lower), min_scale) + if np.isfinite(inc_for_lower_b): + new_upper = max(new_upper, inc_for_lower_b) + + return [float(new_lower), float(new_upper)] def clamp_parameter_prior_to_bounds(prior, bounds, key): @@ -2494,8 +2626,15 @@ def clamp_ars_prior_to_bounds(prior, bounds): return clamp_parameter_prior_to_bounds(prior, bounds, 'ars') +def clamp_inclination_prior_to_bounds(prior, bounds): + return clamp_parameter_prior_to_bounds(prior, bounds, 'inc') + + def clamp_retry_priors_to_bounds(prior, bounds): - return clamp_ars_prior_to_bounds(clamp_rprs_prior_to_bounds(prior, bounds), bounds) + return clamp_inclination_prior_to_bounds( + clamp_ars_prior_to_bounds(clamp_rprs_prior_to_bounds(prior, bounds), bounds), + bounds, + ) def enforce_minimum_parameter_retry_half_width( @@ -2570,28 +2709,98 @@ def run_nested_lightcurve_fit_with_rprs_posterior_retry( max_rprs_retries=RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, duration_prior=None, max_ars_retries=ARS_POSTERIOR_MAX_RETRIES_DEFAULT, + max_impact_parameter_retries=IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES_DEFAULT, ): + def impact_parameter_retry_available(fit, local_bounds): + if not use_impactparameter_rather_than_inclination_to_fit or 'inc' not in local_bounds: + return False + sampled_keys = getattr(fit, 'sampled_keys', []) or [] + sample_bounds = getattr(fit, 'sample_bounds', {}) + return 'b' in sampled_keys or (isinstance(sample_bounds, dict) and 'b' in sample_bounds) + + def identity_retry_bounds(diagnostics, local_prior, local_bounds, config): + return diagnostics.get('bounds') if diagnostics else None + + def impact_parameter_retry_bounds(diagnostics, local_prior, local_bounds, config): + return impact_parameter_retry_proposed_inclination_bounds( + diagnostics, + local_prior, + local_bounds, + ) + + def normal_retry_expands(previous_bounds, new_bounds, clipped_edge, config): + previous_lower, previous_upper = [ + float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2] + ] + new_lower, new_upper = [ + float(value) for value in np.asarray(new_bounds, dtype=float).reshape(-1)[:2] + ] + if clipped_edge == 'upper': + return new_upper > previous_upper + 1e-12 + if clipped_edge == 'lower': + return new_lower < previous_lower - 1e-12 + return new_lower < previous_lower - 1e-12 or new_upper > previous_upper + 1e-12 + + def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, config): + previous_lower, previous_upper = [ + float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2] + ] + new_lower, new_upper = [ + float(value) for value in np.asarray(new_bounds, dtype=float).reshape(-1)[:2] + ] + if clipped_edge == 'upper': + return new_lower < previous_lower - 1e-12 + if clipped_edge == 'lower': + return new_upper > previous_upper + 1e-12 + return new_lower < previous_lower - 1e-12 or new_upper > previous_upper + 1e-12 + retry_configs = [ { 'key': 'rprs', + 'diagnostic_key': 'rprs', + 'bounds_key': 'rprs', 'label': 'Rp/R*', 'sanitize_bounds': sanitize_rprs_search_bounds, 'enforce_half_width': enforce_minimum_rprs_retry_half_width, + 'propose_bounds': identity_retry_bounds, + 'expands_bounds': normal_retry_expands, 'max_retries': max_rprs_retries, 'min_bound': RPRS_SEARCH_BOUND_MIN, 'max_bound': RPRS_SEARCH_BOUND_MAX, + 'prior_mode_key': 'rprs', 'annotate': annotate_rprs_posterior_refit, }, { 'key': 'ars', + 'diagnostic_key': 'ars', + 'bounds_key': 'ars', 'label': 'a/Rs', 'sanitize_bounds': sanitize_ars_search_bounds, 'enforce_half_width': enforce_minimum_ars_retry_half_width, + 'propose_bounds': identity_retry_bounds, + 'expands_bounds': normal_retry_expands, 'max_retries': max_ars_retries, 'min_bound': ARS_SEARCH_BOUND_MIN, 'max_bound': None, + 'prior_mode_key': 'ars', 'annotate': annotate_ars_posterior_refit, }, + { + 'key': 'b', + 'diagnostic_key': 'b', + 'bounds_key': 'inc', + 'label': 'impact parameter', + 'sanitize_bounds': sanitize_inclination_search_bounds, + 'enforce_half_width': lambda mode, bounds: bounds, + 'propose_bounds': impact_parameter_retry_bounds, + 'expands_bounds': impact_parameter_retry_expands, + 'max_retries': max_impact_parameter_retries, + 'min_bound': INCLINATION_SEARCH_BOUND_MIN, + 'max_bound': INCLINATION_SEARCH_BOUND_MAX, + 'prior_mode_key': None, + 'available': impact_parameter_retry_available, + 'annotate': annotate_impact_parameter_posterior_refit, + }, ] def build_fit(local_prior, local_bounds): @@ -2635,10 +2844,16 @@ def build_fit(local_prior, local_bounds): diagnostics = None for config in retry_configs: key = config['key'] - if key not in current_bounds: + diagnostic_key = config.get('diagnostic_key', key) + bounds_key = config.get('bounds_key', key) + if bounds_key not in current_bounds: + continue + + available = config.get('available') + if callable(available) and not available(fit, current_bounds): continue - parameter_diagnostics = diagnostics_getter(key) + parameter_diagnostics = diagnostics_getter(diagnostic_key) latest_diagnostics[key] = parameter_diagnostics if key in blocked_retry_keys: continue @@ -2656,8 +2871,14 @@ def build_fit(local_prior, local_bounds): break key = retry_config['key'] + bounds_key = retry_config.get('bounds_key', key) label = retry_config['label'] - new_bounds = diagnostics.get('bounds') + new_bounds = retry_config.get('propose_bounds', identity_retry_bounds)( + diagnostics, + current_prior, + current_bounds, + retry_config, + ) try: new_lower, new_upper = [float(value) for value in new_bounds] except (TypeError, ValueError): @@ -2669,30 +2890,26 @@ def build_fit(local_prior, local_bounds): blocked_retry_keys.add(key) continue - previous_bounds = current_bounds.get(key) - clamped_bounds = retry_config['sanitize_bounds']({key: [new_lower, new_upper]}).get( - key, + previous_bounds = current_bounds.get(bounds_key) + clamped_bounds = retry_config['sanitize_bounds']({bounds_key: [new_lower, new_upper]}).get( + bounds_key, [new_lower, new_upper], ) clamped_bounds = retry_config['enforce_half_width']( diagnostics.get('mode', np.nan), clamped_bounds, ) - clamped_bounds = retry_config['sanitize_bounds']({key: clamped_bounds}).get(key, clamped_bounds) + clamped_bounds = retry_config['sanitize_bounds']({bounds_key: clamped_bounds}).get(bounds_key, clamped_bounds) new_lower, new_upper = [float(value) for value in clamped_bounds] if previous_bounds is not None: previous_lower, previous_upper = [float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2]] clipped_edge = diagnostics.get('edge') - expands_sampled_range = False - if clipped_edge == 'upper': - expands_sampled_range = new_upper > previous_upper + 1e-12 - elif clipped_edge == 'lower': - expands_sampled_range = new_lower < previous_lower - 1e-12 - else: - expands_sampled_range = ( - new_lower < previous_lower - 1e-12 or - new_upper > previous_upper + 1e-12 - ) + expands_sampled_range = retry_config.get('expands_bounds', normal_retry_expands)( + previous_bounds, + [new_lower, new_upper], + clipped_edge, + retry_config, + ) if not expands_sampled_range: maximum_bound = retry_config['max_bound'] @@ -2727,17 +2944,18 @@ def build_fit(local_prior, local_bounds): ) updated_bounds = clone_lightcurve_bounds(current_bounds) - updated_bounds[key] = [new_lower, new_upper] + updated_bounds[bounds_key] = [new_lower, new_upper] updated_bounds = sanitize_retry_search_bounds(updated_bounds) updated_prior = dict(current_prior) fit_parameters = getattr(fit, 'parameters', {}) if isinstance(fit_parameters, dict): - for key in updated_bounds: - if key in fit_parameters: - updated_prior[key] = fit_parameters[key] - if np.isfinite(diagnostics.get('mode', np.nan)): - updated_prior[retry_config['key']] = float(diagnostics['mode']) + for bound_key in updated_bounds: + if bound_key in fit_parameters: + updated_prior[bound_key] = fit_parameters[bound_key] + prior_mode_key = retry_config.get('prior_mode_key', key) + if prior_mode_key is not None and np.isfinite(diagnostics.get('mode', np.nan)): + updated_prior[prior_mode_key] = float(diagnostics['mode']) updated_prior = clamp_retry_priors_to_bounds(updated_prior, updated_bounds) current_prior = updated_prior @@ -2748,11 +2966,15 @@ def build_fit(local_prior, local_bounds): annotate_posterior_refit_final_bounds(fit, current_bounds) for config in retry_configs: key = config['key'] + diagnostic_key = config.get('diagnostic_key', key) + bounds_key = config.get('bounds_key', key) label = config['label'] history = retry_histories[key] final_diagnostics = None - if callable(final_diagnostics_getter) and key in current_bounds: - final_diagnostics = final_diagnostics_getter(key) + available = config.get('available') + config_available = not callable(available) or available(fit, current_bounds) + if callable(final_diagnostics_getter) and bounds_key in current_bounds and config_available: + final_diagnostics = final_diagnostics_getter(diagnostic_key) elif latest_diagnostics.get(key) is not None: final_diagnostics = latest_diagnostics[key] @@ -10902,6 +11124,10 @@ def summarize_lightcurve_fit_assessment(fit): rprs_retry_count = int(getattr(fit, 'rprs_posterior_refit_count', 0) or 0) except (TypeError, ValueError): rprs_retry_count = 0 + try: + b_retry_count = int(getattr(fit, 'b_posterior_refit_count', 0) or 0) + except (TypeError, ValueError): + b_retry_count = 0 return { 'fit_method': fit_method, @@ -10910,6 +11136,9 @@ def summarize_lightcurve_fit_assessment(fit): 'rprs_posterior_refit_applied': bool(getattr(fit, 'rprs_posterior_refit_applied', False)), 'rprs_posterior_refit_count': rprs_retry_count, 'rprs_posterior_refit_note': getattr(fit, 'rprs_posterior_refit_note', None), + 'b_posterior_refit_applied': bool(getattr(fit, 'b_posterior_refit_applied', False)), + 'b_posterior_refit_count': b_retry_count, + 'b_posterior_refit_note': getattr(fit, 'b_posterior_refit_note', None), 'prefit_refinement_applied': bool(getattr(fit, 'prefit_refinement_applied', False)), 'prefit_refinement_note': getattr(fit, 'prefit_refinement_note', None), 'oot_baseline_detrending_applied': bool(getattr(fit, 'oot_baseline_detrending_applied', False)), @@ -10945,6 +11174,14 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): if retry_count > 0 else "applied" ) + b_retry_status = "not applied" + if assessment['b_posterior_refit_applied']: + retry_count = assessment['b_posterior_refit_count'] + b_retry_status = ( + f"applied ({retry_count} refit(s))" + if retry_count > 0 else + "applied" + ) prefit_status = "applied" if assessment['prefit_refinement_applied'] else "not applied" oot_status = "applied" if assessment['oot_baseline_detrending_applied'] else "not applied" duration_prior_status = "applied" if assessment['duration_prior_applied'] else "not applied" @@ -10953,6 +11190,7 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): f"{indent}fit assessment: fit_method={assessment['fit_method']}, " f"duration_prior={duration_prior_status}, " f"Rp/R* posterior retry={rprs_retry_status}, " + f"impact parameter posterior retry={b_retry_status}, " f"prefit_refinement={prefit_status}, " f"oot_baseline_detrending={oot_status}" ) @@ -10960,6 +11198,8 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): log_info(f"{indent}Duration prior note: {assessment['duration_prior_note']}") if assessment.get('rprs_posterior_refit_note'): log_info(f"{indent}Rp/R* posterior retry note: {assessment['rprs_posterior_refit_note']}") + if assessment.get('b_posterior_refit_note'): + log_info(f"{indent}Impact parameter posterior retry note: {assessment['b_posterior_refit_note']}") if assessment.get('prefit_refinement_note'): log_info(f"{indent}Prefit refinement note: {assessment['prefit_refinement_note']}") if assessment.get('oot_baseline_detrending_note'): diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 0c0d1065..bc109e40 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -2,6 +2,8 @@ import sys import types +import matplotlib +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import pytest @@ -396,13 +398,13 @@ def fake_corner(*args, **kwargs): assert fig == "figure" assert captured["labels"][1] == r"$\Delta i$" - assert captured["range"][0][0] == pytest.approx(0.05) + assert captured["range"][0][0] == pytest.approx(0.0) assert captured["range"][0][1] == pytest.approx(0.125) - expected_inc_distance_limit = np.max(np.abs(np.array([84.67, 90.0]) - fit.parameters["inc"])) + expected_inc_distance_limit = np.max(np.abs(np.array([84.0, 90.0]) - fit.parameters["inc"])) assert captured["range"][1][0] == pytest.approx(-expected_inc_distance_limit) assert captured["range"][1][1] == pytest.approx(expected_inc_distance_limit) assert captured["range"][2][0] == pytest.approx(0.95) - assert captured["range"][2][1] == pytest.approx(0.96932) + assert captured["range"][2][1] == pytest.approx(1.05) assert captured["points"].shape == (10, 3) expected_inc_distance = np.abs(points[:, 1] - fit.parameters["inc"]) np.testing.assert_allclose(captured["points"][:5, 1], expected_inc_distance) @@ -982,8 +984,9 @@ def fake_corner(*args, **kwargs): assert fig == "figure" assert captured["labels"][1] == r"$\Delta b$" - assert captured["range"][1][0] == pytest.approx(-0.25) - assert captured["range"][1][1] == pytest.approx(0.25) + expected_b_distance_limit = np.max(np.abs(np.array([0.0, 1.25434156]) - fit.sample_parameters["b"])) + assert captured["range"][1][0] == pytest.approx(-expected_b_distance_limit) + assert captured["range"][1][1] == pytest.approx(expected_b_distance_limit) assert captured["points"].shape == (10, 3) expected_b_distance = np.abs(points[:, 1] - fit.sample_parameters["b"]) np.testing.assert_allclose(captured["points"][:5, 1], expected_b_distance) @@ -1163,6 +1166,50 @@ def test_triangle_payload_expands_sparse_visible_ranges_to_sample_cloud(monkeypa assert payload["ranges"][1][1] >= 1.019 +def test_triangle_payload_uses_tested_rprs_range_when_posterior_is_narrow(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "a0": [0.95, 1.05], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.100, "a0": 1.0} + fit.errors = {"rprs": 0.001, "a0": 0.001} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + points = np.column_stack( + [ + np.linspace(0.090, 0.110, 120), + np.linspace(0.998, 1.002, 120), + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + rprs_range = payload["ranges"][0] + plot_bins = int(max(1, np.sqrt(points.shape[0]))) + lower_fraction, upper_fraction, _ = fit._histogram_edge_peak_fractions( + points[:, 0], + rprs_range, + plot_bins, + ) + + assert rprs_range == pytest.approx([0.0, 0.2]) + assert lower_fraction < elca.TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX + assert upper_fraction < elca.TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX + + def test_triangle_payload_expands_mirrored_impact_parameter_range_to_sample_cloud(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index eabacbfe..cf436314 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -496,6 +496,148 @@ def fake_lc_fitter( assert fit.ars_posterior_refit_bounds == pytest.approx([12.60, 16.30]) +def test_impact_parameter_posterior_retry_expands_inclination_bounds(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": True, "edge": "upper", "mode": 1.6, "std": 0.18, "bounds": [0.7, 2.5]}, + }, + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": False, "edge": None, "mode": 1.6, "std": 0.12, "bounds": [0.7, 2.5]}, + }, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + sampled_keys=["rprs", "b", "tmid"], + sample_bounds={"rprs": [0.0, 0.25], "b": [0.0, 2.5], "tmid": [-0.01, 0.01]}, + parameters={ + "rprs": diagnostics["rprs"]["mode"], + "ars": 10.0, + "tmid": 0.0, + "inc": 80.5, + "a2": 0.0, + }, + ) + + def get_parameter_posterior_recenter_diagnostics(key): + return dict(diagnostics[key]) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + "use_impactparameter": use_impactparameter_rather_than_inclination_to_fit, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 85.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "tmid": [-0.01, 0.01], "inc": [80.0, 90.0], "a2": [-3.0, 3.0]}, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["inc"] == pytest.approx([80.0, 90.0]) + assert captured["calls"][1]["bounds"]["inc"][0] == pytest.approx(np.degrees(np.arccos(0.25))) + assert captured["calls"][1]["bounds"]["inc"][1] == pytest.approx(90.0) + assert captured["calls"][1]["prior"]["inc"] == pytest.approx(80.5) + assert fit.b_posterior_refit_applied is True + assert fit.b_posterior_refit_count == 1 + assert fit.b_posterior_refit_edge == "upper" + assert fit.b_posterior_refit_bounds == pytest.approx([np.degrees(np.arccos(0.25)), 90.0]) + + +def test_impact_parameter_posterior_retry_expands_toward_face_on_boundary(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": True, "edge": "lower", "mode": 0.55, "std": 0.10, "bounds": [0.0, 1.0]}, + }, + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": False, "edge": None, "mode": 0.55, "std": 0.08, "bounds": [0.0, 1.0]}, + }, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + sampled_keys=["rprs", "b", "tmid"], + sample_bounds={"rprs": [0.0, 0.25], "b": [0.0, 1.0], "tmid": [-0.01, 0.01]}, + parameters={"rprs": 0.1, "ars": 10.0, "tmid": 0.0, "inc": 86.0, "a2": 0.0}, + ) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: dict(diagnostics[key]) + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 84.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "tmid": [-0.01, 0.01], "inc": [80.0, 87.0], "a2": [-3.0, 3.0]}, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][1]["bounds"]["inc"] == pytest.approx([80.0, 90.0]) + assert fit.b_posterior_refit_applied is True + assert fit.b_posterior_refit_edge == "lower" + + def test_run_nested_lightcurve_fit_passes_duration_prior_when_available(monkeypatch): import exotic.exotic as exotic_module From 308e693a56e817f0816ebdfb7d8efbfa38813c19 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 11 May 2026 09:02:11 +1000 Subject: [PATCH 038/116] better fitting --- exotic/api/elca.py | 776 ++++++++++++++++++++++++----- exotic/exotic.py | 513 ++++++++++++++++++- exotic/exotic_gui.py | 5 + exotic/inputs.py | 7 + exotic/output_files.py | 22 +- exotic/plots.py | 13 +- inits.json | 2 + tests/test_elca_baseline.py | 216 ++++++-- tests/test_exotic_proper_motion.py | 37 +- tests/test_exotic_rprs_retry.py | 153 ++++++ tests/test_inputs.py | 30 ++ tests/test_output_files.py | 21 + tests/test_plots.py | 34 ++ 13 files changed, 1637 insertions(+), 192 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index f14a6f56..0ca8daba 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -44,7 +44,7 @@ from contextlib import redirect_stderr, redirect_stdout import faulthandler import io -from itertools import cycle +from itertools import cycle, product import os import sys import bottleneck as bn @@ -75,6 +75,9 @@ TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX = 0.50 TRIANGLE_PLOT_EDGE_MIN_SAMPLE_COUNT = 30 TRIANGLE_PLOT_EDGE_EXPANSION_STEPS = 8 +TRANSIT_MODEL_UNCERTAINTY_KEYS = ( + 'rprs', 'tmid', 'inc', 'ars', 'per', 'ecc', 'omega', 'u0', 'u1', 'u2', 'u3', +) def _pylightcurve_import_watchdog_seconds(): try: @@ -164,6 +167,17 @@ def inclination_from_impact_parameter(values, impact_parameter): return np.rad2deg(np.arccos(cosi)) +def grazing_impact_parameter(values): + try: + rprs = float(values['rprs']) + except (KeyError, TypeError, ValueError): + return np.nan + + if not np.isfinite(rprs) or rprs < 0: + return np.nan + return 1.0 + rprs + + def transit_duration(values): try: period = float(values['per']) @@ -435,6 +449,7 @@ def __init__( verbose=True, use_impactparameter_rather_than_inclination_to_fit=True, duration_prior=None, + keep_ultranest_sampler=False, ): self.time = time self.data = data @@ -450,9 +465,12 @@ def __init__( self.neighbors = neighbors self.use_impactparameter_rather_than_inclination_to_fit = use_impactparameter_rather_than_inclination_to_fit self.duration_prior = copy.deepcopy(duration_prior) if isinstance(duration_prior, dict) else None + self.keep_ultranest_sampler = bool(keep_ultranest_sampler) + self._ultranest_resume_context = None self.results = None self.sampled_keys = list(bounds.keys()) self.sample_bounds = copy.deepcopy(bounds) + self.impact_parameter_sampled_directly = False self.sample_parameters = {} self.sample_errors = {} self.sample_quantiles = {} @@ -523,6 +541,94 @@ def _build_systematics_model(self, values): reference=self._get_airmass_reference(), ) + def _get_perturbed_transit_parameter_value(self, key, value): + try: + value = float(value) + except (TypeError, ValueError): + return np.nan + + if key in ('rprs', 'ars', 'per'): + return max(value, np.finfo(float).eps) + if key == 'ecc': + return float(np.clip(value, 0.0, 0.999999)) + if key == 'inc': + return float(np.clip(value, 0.0, 180.0)) + return value + + def transit_model_uncertainty(self, times=None, sigma=1.0): + if times is None: + times = getattr(self, 'time_upsample', self.time) + times = np.asarray(times, dtype=float) + if times.size == 0: + return None + + try: + model = transit(times, self.parameters) + except Exception: + return None + + sigma = float(sigma) + variance = np.zeros_like(model, dtype=float) + for key in TRANSIT_MODEL_UNCERTAINTY_KEYS: + if key not in self.parameters: + continue + error = self.errors.get(key) + try: + center = float(self.parameters[key]) + error = float(error) + except (TypeError, ValueError): + continue + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + continue + + lower_value = self._get_perturbed_transit_parameter_value(key, center - error) + upper_value = self._get_perturbed_transit_parameter_value(key, center + error) + if ( + not np.isfinite(lower_value) + or not np.isfinite(upper_value) + or np.isclose(lower_value, upper_value) + ): + continue + + lower_parameters = copy.deepcopy(self.parameters) + upper_parameters = copy.deepcopy(self.parameters) + lower_parameters[key] = lower_value + upper_parameters[key] = upper_value + try: + lower_model = transit(times, lower_parameters) + upper_model = transit(times, upper_parameters) + except Exception: + continue + + derivative = (upper_model - lower_model) / (upper_value - lower_value) + contribution = derivative * error * sigma + finite = np.isfinite(contribution) + variance[finite] += contribution[finite] ** 2 + + model_uncertainty = np.sqrt(variance) + if not np.any(np.isfinite(model_uncertainty) & (model_uncertainty > 0)): + return None + return model - model_uncertainty, model + model_uncertainty + + def _plot_transit_model_uncertainty(self, ax, x_values, times, sort_index, label=None): + envelope = self.transit_model_uncertainty(times) + if envelope is None: + return None + + lower, upper = envelope + x_values = np.asarray(x_values, dtype=float) + sort_index = np.asarray(sort_index, dtype=int) + return ax.fill_between( + x_values[sort_index], + np.asarray(lower, dtype=float)[sort_index], + np.asarray(upper, dtype=float)[sort_index], + color='red', + alpha=0.16, + linewidth=0, + zorder=2.5, + label=label, + ) + def _uses_internal_impact_parameter(self): return ( self.use_impactparameter_rather_than_inclination_to_fit @@ -537,6 +643,59 @@ def _get_sampled_keys(self, bound_keys=None): return bound_keys return ['b' if key == 'inc' else key for key in bound_keys] + def _get_impact_parameter_scale_upper_bound(self, values): + values = copy.deepcopy(values) + scale_keys = ('ars', 'ecc', 'omega') + endpoint_sets = [] + for key in scale_keys: + if key in self.bounds: + endpoints = np.asarray(self.bounds[key], dtype=float).reshape(-1)[:2] + else: + endpoints = np.asarray([values.get(key, 0.0)], dtype=float) + finite_endpoints = [float(value) for value in endpoints if np.isfinite(value)] + if not finite_endpoints: + return np.nan + endpoint_sets.append((key, finite_endpoints)) + + scales = [] + for candidate_values in product(*[endpoints for _, endpoints in endpoint_sets]): + candidate = copy.deepcopy(values) + for key, value in zip([key for key, _ in endpoint_sets], candidate_values): + candidate[key] = value + try: + scale = float(impact_parameter_scale(candidate)) + except (KeyError, TypeError, ValueError): + continue + if np.isfinite(scale) and scale > 0: + scales.append(scale) + + return float(max(scales)) if scales else np.nan + + def _get_impact_parameter_sampling_bounds(self, values=None, use_search_bounds=False): + values = copy.deepcopy(self.prior if values is None else values) + if use_search_bounds and 'rprs' in self.bounds: + rprs_bounds = np.asarray(self.bounds['rprs'], dtype=float).reshape(-1)[:2] + finite_rprs = rprs_bounds[np.isfinite(rprs_bounds) & (rprs_bounds >= 0)] + if finite_rprs.size > 0: + values['rprs'] = float(np.max(finite_rprs)) + + grazing_upper = grazing_impact_parameter(values) + if use_search_bounds: + scale_upper = self._get_impact_parameter_scale_upper_bound(values) + else: + try: + scale_upper = float(impact_parameter_scale(values)) + except (KeyError, TypeError, ValueError): + scale_upper = np.nan + + upper_candidates = [ + float(value) + for value in (grazing_upper, scale_upper) + if np.isfinite(value) and value > 0 + ] + upper = min(upper_candidates) if upper_candidates else 1.0 + return [0.0, float(max(0.0, upper))] + def _get_sample_bounds(self, bound_keys=None, values=None): bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) sampled_keys = self._get_sampled_keys(bound_keys) @@ -544,28 +703,10 @@ def _get_sample_bounds(self, bound_keys=None, values=None): sample_bounds = {} for key, sampled_key in zip(bound_keys, sampled_keys): if key == 'inc' and sampled_key == 'b': - inc_lower, inc_upper = self.bounds[key] - if 'ars' in self.bounds: - ars_lower, ars_upper = self.bounds['ars'] - b_corners = [] - for ars_value in (ars_lower, ars_upper): - corner_values = copy.deepcopy(values) - corner_values['ars'] = float(ars_value) - b_corners.extend( - np.asarray( - impact_parameter_from_inclination( - corner_values, - np.array([inc_lower, inc_upper], dtype=float), - ), - dtype=float, - ).reshape(-1).tolist() - ) - lower = float(np.min(b_corners)) - upper = float(np.max(b_corners)) - else: - lower = float(np.min(impact_parameter_from_inclination(values, np.array([inc_lower, inc_upper])))) - upper = float(np.max(impact_parameter_from_inclination(values, np.array([inc_lower, inc_upper])))) - sample_bounds[sampled_key] = [lower, upper] + sample_bounds[sampled_key] = self._get_impact_parameter_sampling_bounds( + values, + use_search_bounds=True, + ) else: sample_bounds[sampled_key] = list(self.bounds[key]) return sample_bounds @@ -590,8 +731,8 @@ def _sample_point_from_unit_cube(self, upars, bound_keys=None): for i, key in enumerate(bound_keys): if key == 'inc' and self._uses_internal_impact_parameter(): - inc = boundarray[i, 0] + (boundarray[i, 1] - boundarray[i, 0]) * upars_array[i] - sample_point[i] = impact_parameter_from_inclination(physical, inc) + b_lower, b_upper = self._get_impact_parameter_sampling_bounds(physical) + sample_point[i] = b_lower + (b_upper - b_lower) * upars_array[i] else: sample_point[i] = physical[key] @@ -739,7 +880,7 @@ def _expand_plot_range_for_sample_cloud( plot_range, sample_values, center, - required_visible_fraction=0.95, + required_visible_fraction=1.0, ): sample_values = np.asarray(sample_values, dtype=float) finite_values = sample_values[np.isfinite(sample_values)] @@ -752,9 +893,8 @@ def _expand_plot_range_for_sample_cloud( if visible_fraction >= required_visible_fraction: return plot_range - q_lower, q_upper = np.nanpercentile(finite_values, [0.5, 99.5]) - new_lower = min(float(q_lower), float(center)) - new_upper = max(float(q_upper), float(center)) + new_lower = min(float(np.nanmin(finite_values)), float(center)) + new_upper = max(float(np.nanmax(finite_values)), float(center)) padding = 0.05 * (new_upper - new_lower) if not np.isfinite(padding) or padding <= 0: padding = max(abs(float(center)) * 1e-6, 1e-6) @@ -930,6 +1070,81 @@ def _expand_plot_range_for_histogram_edge_dropoff( return [float(lower), float(upper)] + def _get_mirrored_geometry_sample_cloud_range(self, sample_values, center, percentile_padding=0.5): + sample_values = np.asarray(sample_values, dtype=float) + finite_values = sample_values[np.isfinite(sample_values)] + if finite_values.size < 2: + return None + + try: + center = float(center) + except (TypeError, ValueError): + center = np.nan + if not np.isfinite(center): + return None + + percentile_padding = float(percentile_padding) + percentile_padding = min(max(percentile_padding, 0.0), 49.0) + q_lower, q_upper = np.nanpercentile( + finite_values, + [percentile_padding, 100.0 - percentile_padding], + ) + plot_lower = min(float(q_lower), center) + plot_upper = max(float(q_upper), center) + width = plot_upper - plot_lower + if not np.isfinite(width) or width <= 0: + return None + + padding = 0.05 * width + plot_lower -= padding + plot_upper += padding + max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) + if not np.isfinite(max_distance) or max_distance <= 0: + return None + return [-max_distance, max_distance] + + def _get_mirrored_geometry_full_range( + self, + key, + sample_values, + center, + sample_weights=None, + ): + try: + center = float(center) + except (TypeError, ValueError): + center = np.nan + if not np.isfinite(center): + return None + + plot_lower, plot_upper = self._get_plot_range(key) + plot_lower, plot_upper = self._expand_plot_range_for_sample_cloud( + key, + [plot_lower, plot_upper], + sample_values, + center, + required_visible_fraction=1.0, + ) + plot_bins = int(max(1, np.sqrt(np.asarray(sample_values).size))) + plot_lower, plot_upper = self._expand_plot_range_for_histogram_edge_dropoff( + key, + [plot_lower, plot_upper], + sample_values, + center, + bins=plot_bins, + weights=sample_weights, + ) + + max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) + if not np.isfinite(max_distance) or max_distance <= 0: + finite_offsets = np.asarray(sample_values, dtype=float) - center + finite_offsets = finite_offsets[np.isfinite(finite_offsets)] + if finite_offsets.size > 0: + max_distance = float(np.nanmax(np.abs(finite_offsets))) + if not np.isfinite(max_distance) or max_distance <= 0: + max_distance = max(abs(center) * 1e-6, 1e-6) + return [-max_distance, max_distance] + def _get_triangle_plot_samples(self): if self.ns_type == 'ultranest': weighted_samples = self.results['weighted_samples'] @@ -1035,11 +1250,20 @@ def _estimate_histogram_mode(self, samples, bounds=None, bins=None, weights=None bin_width = float(edges[1] - edges[0]) if edges.size > 1 else np.nan return mode, bin_width - def _get_triangle_plot_title_center(self, samples, plot_range, fallback_center, bins, weights=None): - mode, _ = self._estimate_histogram_mode(samples, bounds=plot_range, bins=bins, weights=weights) - if np.isfinite(mode): - return float(mode) - return fallback_center + def _format_triangle_plot_parameter_title(self, value, error): + try: + value = float(value) + except (TypeError, ValueError): + return "n/a" + try: + error = float(error) + except (TypeError, ValueError): + error = np.nan + if not np.isfinite(value): + return "n/a" + if not np.isfinite(error) or error < 0: + return str(round_to_2(value)) + return f"{round_to_2(value, error)} +/- {round_to_2(error)}" def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bins=None): diagnostics = { @@ -1227,54 +1451,216 @@ def _get_triangle_plot_display_spec( ): if 'b' in sampled_keys: key = 'b' - label = r'$\Delta b$' + label = r'Impact parameter $b$' + mirror = False elif 'inc' in sampled_keys: key = 'inc' label = r'$\Delta i$' + mirror = True else: return None geometry_index = sampled_keys.index(key) center = float(sample_parameters.get(key, self.parameters.get(key, 0.0))) - magnitude_samples = np.abs(np.asarray(sample_points[:, geometry_index], dtype=float) - center) + sample_values = np.asarray(sample_points[:, geometry_index], dtype=float) + magnitude_samples = np.abs(sample_values - center) error = float(sample_errors.get(key, np.nanstd(magnitude_samples))) if not np.isfinite(error) or error <= 0: error = float(np.nanstd(magnitude_samples)) - plot_lower, plot_upper = self._get_plot_range(key) - plot_lower, plot_upper = self._expand_plot_range_for_sample_cloud( - key, - [plot_lower, plot_upper], - sample_points[:, geometry_index], - center, - ) - plot_bins = int(max(1, np.sqrt(sample_points.shape[0]))) - plot_lower, plot_upper = self._expand_plot_range_for_histogram_edge_dropoff( - key, - [plot_lower, plot_upper], - sample_points[:, geometry_index], - center, - bins=plot_bins, - weights=sample_weights, - ) - max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) - if not np.isfinite(max_distance) or max_distance <= 0: - max_distance = float(np.nanmax(magnitude_samples)) - if not np.isfinite(max_distance) or max_distance <= 0: - max_distance = max(abs(center) * 1e-6, 1e-6) + + if mirror: + display_range = self._get_mirrored_geometry_full_range( + key, + sample_values, + center, + sample_weights=sample_weights, + ) + else: + display_range = None + + if display_range is None: + plot_lower, plot_upper = self._get_plot_range(key) + plot_lower, plot_upper = self._expand_plot_range_for_sample_cloud( + key, + [plot_lower, plot_upper], + sample_values, + center, + ) + plot_bins = int(max(1, np.sqrt(sample_points.shape[0]))) + plot_lower, plot_upper = self._expand_plot_range_for_histogram_edge_dropoff( + key, + [plot_lower, plot_upper], + sample_values, + center, + bins=plot_bins, + weights=sample_weights, + ) + if mirror: + max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) + if not np.isfinite(max_distance) or max_distance <= 0: + max_distance = float(np.nanmax(magnitude_samples)) + if not np.isfinite(max_distance) or max_distance <= 0: + max_distance = max(abs(center) * 1e-6, 1e-6) + display_range = [-max_distance, max_distance] + else: + display_range = [float(plot_lower), float(plot_upper)] return { 'key': key, 'index': geometry_index, 'label': label, + 'mirror': mirror, 'center': center, - 'mask_center': 0.0, + 'mask_center': 0.0 if mirror else center, 'mask_error': error, 'magnitude_samples': magnitude_samples, - 'range': [-max_distance, max_distance], + 'range': display_range, + 'truth': 0.0 if mirror else center, + 'reference_lines': self._get_triangle_plot_geometry_reference_lines( + key, + center, + sample_parameters, + ), } + def _get_triangle_plot_geometry_reference_lines(self, key, center, sample_parameters): + if key != 'b': + return [] + + try: + center = float(center) + except (TypeError, ValueError): + center = np.nan + if not np.isfinite(center): + return [] + + rprs = sample_parameters.get('rprs') + if rprs is None: + rprs = getattr(self, 'parameters', {}).get( + 'rprs', + getattr(self, 'prior', {}).get('rprs', np.nan), + ) + try: + rprs = float(rprs) + except (TypeError, ValueError): + rprs = np.nan + + reference_lines = [ + { + 'value': 1.0, + 'color': '#707070', + 'linestyle': ':', + 'linewidth': 0.9, + 'alpha': 0.9, + }, + ] + if np.isfinite(rprs) and rprs >= 0: + reference_lines.append( + { + 'value': 1.0 + rprs, + 'color': '#a35d00', + 'linestyle': '-.', + 'linewidth': 0.9, + 'alpha': 0.9, + } + ) + return reference_lines + + def _get_triangle_plot_geometry_reference_offsets(self, display_spec): + reference_lines = display_spec.get('reference_lines', []) if isinstance(display_spec, dict) else [] + if not reference_lines: + return [] + + try: + center = float(display_spec['center']) + except (KeyError, TypeError, ValueError): + return [] + if not np.isfinite(center): + return [] + + offsets = [] + for reference in reference_lines: + try: + value = float(reference['value']) + except (KeyError, TypeError, ValueError): + continue + if not np.isfinite(value): + continue + + distance = abs(value - center) + if not np.isfinite(distance): + continue + reference_offsets = [0.0] if distance <= np.finfo(float).eps else [-distance, distance] + for offset in reference_offsets: + offsets.append({ + 'offset': float(offset), + 'color': reference.get('color', '#707070'), + 'linestyle': reference.get('linestyle', ':'), + 'linewidth': reference.get('linewidth', 0.9), + 'alpha': reference.get('alpha', 0.9), + }) + return offsets + + def _draw_triangle_plot_geometry_reference_lines(self, ax, display_spec, axis='x', limits=None): + if ax is None: + return + + if limits is None: + limits = ax.get_xlim() if axis == 'x' else ax.get_ylim() + lower, upper = np.sort(np.asarray(limits, dtype=float).reshape(-1)[:2]) + if not display_spec.get('mirror', True): + for reference in display_spec.get('reference_lines', []): + try: + value = float(reference['value']) + except (KeyError, TypeError, ValueError): + continue + if not np.isfinite(value) or value < lower or value > upper: + continue + line_kwargs = { + 'color': reference.get('color', '#707070'), + 'linestyle': reference.get('linestyle', ':'), + 'linewidth': reference.get('linewidth', 0.9), + 'alpha': reference.get('alpha', 0.9), + 'zorder': 2, + } + if axis == 'y': + ax.axhline(value, **line_kwargs) + else: + ax.axvline(value, **line_kwargs) + return + + if lower <= 0.0 <= upper: + center_kwargs = { + 'color': '#4682b4', + 'linestyle': '--', + 'linewidth': 0.9, + 'alpha': 0.85, + 'zorder': 2, + } + if axis == 'y': + ax.axhline(0.0, **center_kwargs) + else: + ax.axvline(0.0, **center_kwargs) + for reference in self._get_triangle_plot_geometry_reference_offsets(display_spec): + offset = reference['offset'] + if offset < lower or offset > upper: + continue + line_kwargs = { + 'color': reference['color'], + 'linestyle': reference['linestyle'], + 'linewidth': reference['linewidth'], + 'alpha': reference['alpha'], + 'zorder': 2, + } + if axis == 'y': + ax.axhline(offset, **line_kwargs) + else: + ax.axvline(offset, **line_kwargs) + def _get_triangle_plot_geometry_overlay(self, display_spec, sample_points): if display_spec is None: return None + if not display_spec.get('mirror', True): + return None geometry_index = display_spec['index'] center = display_spec['center'] @@ -1300,7 +1686,7 @@ def _format_triangle_plot_geometry_value(self, value, error, suffix=''): return f"n/a{suffix}" if error is None or not np.isfinite(error) or error < 0: return f"{round_to_2(value)}{suffix}" - return f"{round_to_2(value, error)} +- {round_to_2(error)}{suffix}" + return f"{round_to_2(value, error)} +/- {round_to_2(error)}{suffix}" def _get_triangle_plot_geometry_summary(self, sampled_keys, sample_points): bound_keys = list(self.bounds.keys()) @@ -1422,7 +1808,7 @@ def _get_triangle_plot_payload(self): display_weights = None if sample_weights is None else np.array(sample_weights, copy=True) mask_values = np.array(sample_points, copy=True) - if display_spec is not None: + if display_spec is not None and display_spec.get('mirror', True): geometry_index = display_spec['index'] positive_points = np.array(sample_points, copy=True) negative_points = np.array(sample_points, copy=True) @@ -1464,6 +1850,7 @@ def _get_triangle_plot_payload(self): ranges = [] mask_centers = [] mask_errors = [] + truths = [] for i, key in enumerate(sampled_keys): center = sample_parameters.get(key, self.parameters.get(key, 0.0)) @@ -1485,16 +1872,8 @@ def _get_triangle_plot_payload(self): bins=plot_bins, weights=sample_weights, ) - title_center = center - if display_points.ndim == 2 and i < display_points.shape[1]: - title_center = self._get_triangle_plot_title_center( - display_points[:, i], - plot_range, - center, - plot_bins, - weights=None if display_weights is None else display_weights, - ) - title = f"{title_center:.5f} +- {error:.5f}" + title = self._format_triangle_plot_parameter_title(center, error) + truth = center if display_spec is not None and key == display_spec['key']: label = display_spec['label'] @@ -1502,12 +1881,18 @@ def _get_triangle_plot_payload(self): plot_range = display_spec['range'] center = display_spec['mask_center'] error = display_spec['mask_error'] + truth = display_spec['truth'] labels.append(label) titles.append(title) ranges.append(plot_range) mask_centers.append(center) mask_errors.append(error) + try: + truth = float(truth) + except (TypeError, ValueError): + truth = np.nan + truths.append(truth if np.isfinite(truth) else None) return { 'sampled_keys': sampled_keys, @@ -1523,6 +1908,7 @@ def _get_triangle_plot_payload(self): 'ranges': ranges, 'mask_centers': mask_centers, 'mask_errors': mask_errors, + 'truths': truths, } def _triangle_contour_levels(self, chi2, mask1, mask2, mask3): @@ -1550,8 +1936,7 @@ def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs= return display_spec = payload.get('display_spec') - geometry_overlay = payload.get('geometry_overlay') - if display_spec is None or geometry_overlay is None: + if display_spec is None: return sampled_keys = payload['sampled_keys'] @@ -1559,6 +1944,23 @@ def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs= return axes = np.array(fig.axes).reshape((len(sampled_keys), len(sampled_keys))) + if not display_spec.get('mirror', True): + geometry_index = display_spec['index'] + for row in range(len(sampled_keys)): + for col in range(len(sampled_keys)): + panel = axes[row, col] + if row == geometry_index and col == geometry_index: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='x') + elif col == geometry_index and row > col: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='x') + elif row == geometry_index and col < row: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='y') + return + + geometry_overlay = payload.get('geometry_overlay') + if geometry_overlay is None: + return + geometry_index = geometry_overlay['index'] ax = axes[geometry_index, geometry_index] hist_range = np.sort(payload['ranges'][geometry_index]) @@ -1578,6 +1980,12 @@ def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs= linewidth=0.75, alpha=0.75, zorder=3) ax.plot(curves['centers'], curves['right_curve'], color=branch_right_color, linestyle='--', linewidth=0.75, alpha=0.75, zorder=3) + self._draw_triangle_plot_geometry_reference_lines( + ax, + display_spec, + axis='x', + limits=hist_range, + ) ax.set_title(title, **title_kwargs) if 'fontsize' in title_kwargs: ax.title.set_fontsize(title_kwargs['fontsize']) @@ -1596,6 +2004,16 @@ def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs= else: ax.set_xlabel(x_label, **label_kwargs) + for row in range(len(sampled_keys)): + for col in range(len(sampled_keys)): + if row == geometry_index and col == geometry_index: + continue + panel = axes[row, col] + if col == geometry_index and row > col: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='x') + if row == geometry_index and col < row: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='y') + def _adjust_triangle_plot_layout(self, fig): if not hasattr(fig, 'subplots_adjust'): return @@ -1604,10 +2022,10 @@ def _adjust_triangle_plot_layout(self, fig): return fig.subplots_adjust( - left=subplotpars.left, - bottom=max(subplotpars.bottom, 0.10), - right=min(subplotpars.right, 0.95), - top=min(subplotpars.top, 0.955), + left=max(subplotpars.left, 0.08), + bottom=max(subplotpars.bottom, 0.12), + right=min(subplotpars.right, 0.97), + top=min(subplotpars.top, 0.94), wspace=subplotpars.wspace, hspace=subplotpars.hspace, ) @@ -1752,12 +2170,91 @@ def create_fit_variables(self): self.duration_measured = tdur self.duration_expected = newdur + def _finalize_ultranest_fit_results(self, bound_keys, sampled_keys, physical_from_sample_point): + self.sample_parameters = {} + self.sample_errors = {} + self.sample_quantiles = {} + self.errors = {} + self.quantiles = {} + self.parameters = copy.deepcopy(self.prior) + + ml_point = self.results['maximum_likelihood']['point'] + self.sample_bounds = self._get_sample_bounds(bound_keys, physical_from_sample_point(ml_point)) + self.ultranest_error_fallbacks = {} + + for i, key in enumerate(sampled_keys): + self.sample_parameters[key] = ml_point[i] + reported_error = self.results['posterior']['stdev'][i] + reported_quantiles = [ + self.results['posterior']['errlo'][i], + self.results['posterior']['errup'][i]] + if self._ultranest_error_needs_sample_fallback(i, ml_point[i], reported_error): + fallback = self._loglike_neighborhood_uncertainty(i, ml_point[i]) + else: + fallback = None + if fallback is not None: + self.sample_errors[key] = fallback['error'] + self.sample_quantiles[key] = fallback['quantiles'] + self.ultranest_error_fallbacks[key] = fallback + else: + self.sample_errors[key] = reported_error + self.sample_quantiles[key] = reported_quantiles + + physical_ml = physical_from_sample_point(ml_point) + self.parameters.update(physical_ml) + + for bound_key, sampled_key in zip(bound_keys, sampled_keys): + if bound_key == 'inc' and sampled_key == 'b': + continue + self.errors[bound_key] = self.sample_errors[sampled_key] + self.quantiles[bound_key] = self.sample_quantiles[sampled_key] + + if 'inc' in bound_keys and 'b' in sampled_keys: + inc_samples = np.array([ + physical_from_sample_point(point)['inc'] + for point in self.results['weighted_samples']['points'] + ]) + center, std, quantiles = self._summarize_derived_parameter(inc_samples, physical_ml['inc']) + self.parameters['inc'] = center + self.errors['inc'] = std + self.quantiles['inc'] = quantiles + + def extend_ultranest_fit(self, min_num_live_points=None, max_ncalls=None): + context = getattr(self, '_ultranest_resume_context', None) + if getattr(self, 'ns_type', None) != 'ultranest' or not isinstance(context, dict): + return False + + sampler = context.get('sampler') + if sampler is None: + return False + + run_kwargs = {"max_ncalls": int(max_ncalls if max_ncalls is not None else self.max_ncalls)} + if min_num_live_points is not None: + run_kwargs["min_num_live_points"] = int(min_num_live_points) + + self.results = run_reactive_sampler( + sampler, + run_kwargs=run_kwargs, + verbose=self.verbose, + ) + self._finalize_ultranest_fit_results( + context['bound_keys'], + context['sampled_keys'], + context['physical_from_sample_point'], + ) + self.create_fit_variables() + return True + + def clear_ultranest_resume_state(self): + self._ultranest_resume_context = None + def fit_nested(self): bound_keys = list(self.bounds.keys()) sampled_keys = self._get_sampled_keys(bound_keys) self._validate_flux_baseline_keys() self.sampled_keys = list(sampled_keys) self.sample_bounds = self._get_sample_bounds(bound_keys, self.prior) + self.impact_parameter_sampled_directly = self._uses_internal_impact_parameter() if len(set(self.sampled_keys)) != len(self.sampled_keys): raise ValueError("Free-parameter labels must be unique after internal parameter transforms.") @@ -1839,46 +2336,16 @@ def prior_transform(upars): verbose=self.verbose, ) - ml_point = self.results['maximum_likelihood']['point'] - self.sample_bounds = self._get_sample_bounds(bound_keys, physical_from_sample_point(ml_point)) - self.ultranest_error_fallbacks = {} - - for i, key in enumerate(sampled_keys): - self.sample_parameters[key] = ml_point[i] - reported_error = self.results['posterior']['stdev'][i] - reported_quantiles = [ - self.results['posterior']['errlo'][i], - self.results['posterior']['errup'][i]] - if self._ultranest_error_needs_sample_fallback(i, ml_point[i], reported_error): - fallback = self._loglike_neighborhood_uncertainty(i, ml_point[i]) - else: - fallback = None - if fallback is not None: - self.sample_errors[key] = fallback['error'] - self.sample_quantiles[key] = fallback['quantiles'] - self.ultranest_error_fallbacks[key] = fallback - else: - self.sample_errors[key] = reported_error - self.sample_quantiles[key] = reported_quantiles - - physical_ml = physical_from_sample_point(ml_point) - self.parameters.update(physical_ml) - - for bound_key, sampled_key in zip(bound_keys, sampled_keys): - if bound_key == 'inc' and sampled_key == 'b': - continue - self.errors[bound_key] = self.sample_errors[sampled_key] - self.quantiles[bound_key] = self.sample_quantiles[sampled_key] - - if 'inc' in bound_keys and 'b' in sampled_keys: - inc_samples = np.array([ - physical_from_sample_point(point)['inc'] - for point in self.results['weighted_samples']['points'] - ]) - center, std, quantiles = self._summarize_derived_parameter(inc_samples, physical_ml['inc']) - self.parameters['inc'] = center - self.errors['inc'] = std - self.quantiles['inc'] = quantiles + if self.keep_ultranest_sampler: + self._ultranest_resume_context = { + 'sampler': test, + 'bound_keys': list(bound_keys), + 'sampled_keys': list(sampled_keys), + 'physical_from_sample_point': physical_from_sample_point, + } + else: + self._ultranest_resume_context = None + self._finalize_ultranest_fit_results(bound_keys, sampled_keys, physical_from_sample_point) except NameError: self.ns_type = 'dynesty' dsampler = dynesty.DynamicNestedSampler(loglike, prior_transform, ndim=len(sampled_keys), @@ -1994,7 +2461,15 @@ def prior_transform(upars): # final model self.create_fit_variables() - def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True): + def plot_bestfit( + self, + title="", + bin_dt=30. / (60 * 24), + zoom=False, + phase=True, + show_flux_baseline_label=True, + show_model_uncertainty=False, + ): f = plt.figure(figsize=(9, 6)) f.subplots_adjust(top=0.92, bottom=0.09, left=0.14, right=0.98, hspace=0) ax_lc = plt.subplot2grid((4, 5), (0, 0), colspan=5, rowspan=3) @@ -2018,7 +2493,7 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) ) lclabel = lclabel1 + "\n" + lclabel2 - if 'a0' in self.parameters: + if show_flux_baseline_label and 'a0' in self.parameters: lclabel3 = r"$a_0$ = %s $\pm$ %s" % ( str(round_to_2(self.parameters['a0'], self.errors.get('a0', 0))), str(round_to_2(self.errors.get('a0', 0))) @@ -2051,6 +2526,14 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) marker='s') # axs[0].plot(self.phase[si], self.transit[si], 'r-', zorder=3, label=lclabel) sii = np.argsort(self.phase_upsample) + if show_model_uncertainty: + self._plot_transit_model_uncertainty( + axs[0], + self.phase_upsample, + self.time_upsample, + sii, + label=r'1-$\sigma$ model uncertainty', + ) axs[0].plot(self.phase_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel) axs[0].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) axs[0].set_xlabel("Phase ", fontsize=14) @@ -2066,6 +2549,14 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) si = np.argsort(self.time) sii = np.argsort(self.time_upsample) axs[0].errorbar(bt, bf, yerr=bs, alpha=1, zorder=2, color='blue', ls='none', marker='s') + if show_model_uncertainty: + self._plot_transit_model_uncertainty( + axs[0], + self.time_upsample, + self.time_upsample, + sii, + label=r'1-$\sigma$ model uncertainty', + ) axs[0].plot(self.time_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel) axs[0].set_xlim([min(self.time_upsample), max(self.time_upsample)]) axs[0].set_xlabel("Time [day]", fontsize=14) @@ -2077,9 +2568,11 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) axs[1].grid(True, ls='--', axis='y') return f, axs - def plot_triangle(self): + def plot_triangle(self, plot_title=None): payload = self._get_triangle_plot_payload() chi2 = payload['display_logl'] * -2 + parameter_count = max(1, len(payload['sampled_keys'])) + fig_size = max(9.0, 2.35 * parameter_count) mask1 = np.ones(len(chi2), dtype=bool) mask2 = np.ones(len(chi2), dtype=bool) mask3 = np.ones(len(chi2), dtype=bool) @@ -2107,33 +2600,42 @@ def plot_triangle(self): label_kwargs = { 'labelpad': 10, + 'fontsize': 10, } title_kwargs = { 'loc': 'left', 'pad': 4, + 'fontsize': 11, } fig = corner(payload['display_points'], labels=payload['labels'], bins=int(np.sqrt(payload['display_points'].shape[0])), range=payload['ranges'], - weights=payload['display_weights'], - plot_contours=True, - levels=self._triangle_contour_levels(chi2, mask1, mask2, mask3), - plot_density=False, - titles=payload['titles'], - data_kwargs={ - 'c': chi2, - 'vmin': np.percentile(chi2[mask3], 1), - 'vmax': np.percentile(chi2[mask3], 95), - 'cmap': 'viridis' - }, - label_kwargs=label_kwargs, - title_kwargs=title_kwargs, - hist_kwargs={ - 'color': 'black', - } - ) + weights=payload['display_weights'], + plot_contours=True, + levels=self._triangle_contour_levels(chi2, mask1, mask2, mask3), + plot_density=False, + titles=payload['titles'], + truths=payload['truths'], + data_kwargs={ + 'c': chi2, + 'vmin': np.percentile(chi2[mask3], 1), + 'vmax': np.percentile(chi2[mask3], 95), + 'cmap': 'viridis', + 's': 1.6, + 'alpha': 0.38, + }, + label_kwargs=label_kwargs, + title_kwargs=title_kwargs, + hist_kwargs={ + 'color': 'black', + } + ) + if hasattr(fig, 'set_size_inches'): + fig.set_size_inches(fig_size, fig_size, forward=True) + if plot_title and hasattr(fig, 'suptitle'): + fig.suptitle(plot_title, fontsize=13, y=0.99) self._adjust_triangle_plot_layout(fig) self._overlay_triangle_plot_geometry_histograms( fig, diff --git a/exotic/exotic.py b/exotic/exotic.py index 390e6dfc..bfbca1e0 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -60,6 +60,7 @@ import copy import faulthandler from functools import lru_cache +import inspect import json import hashlib import os @@ -225,6 +226,13 @@ FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT = 200 ULTRANEST_MIN_NUM_LIVE_POINTS_ENV = "EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS" +SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT = True +SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV = "EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY" +SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT = 5 +SPARSE_POSTERIOR_RETRY_PARAMETER_KEYS = ('rprs', 'tmid', 'ars') +SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_FLOOR = 500 +SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_LIVE_POINT = 3.0 +SPARSE_POSTERIOR_MIN_OCCUPIED_BINS = 8 RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 RPRS_SEARCH_BOUND_MIN = 0.0 RPRS_SEARCH_BOUND_MAX = 1.0 @@ -1582,6 +1590,13 @@ def triangle_plot_output_path(save_dir, planet_name, observation_date): ) +def final_triangle_plot_output_path(save_dir, planet_name, observation_date): + return ( + Path(save_dir) + / safe_output_filename("FinalTriangle", planet_name, filename_date_token(observation_date), extension="png") + ) + + def comparison_candidate_triangle_plot_output_path(save_dir, planet_name, observation_date, comp_index): return ( Path(save_dir) @@ -1595,12 +1610,60 @@ def comparison_candidate_triangle_plot_output_path(save_dir, planet_name, observ ) +def comparison_candidate_label_from_output_dir(output_dir): + if output_dir is None: + return None + + for part in reversed(Path(output_dir).parts): + match = re.fullmatch(r"comp(\d+)", str(part), re.IGNORECASE) + if match: + return f"comparison candidate #{int(match.group(1))}" + return None + + +def _plot_triangle_for_output(fit, plot_title=None): + plotter = getattr(fit, 'plot_triangle', None) + if not callable(plotter): + return None + + if plot_title: + try: + signature = inspect.signature(plotter) + accepts_plot_title = ( + 'plot_title' in signature.parameters + or any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + ) + except (TypeError, ValueError): + accepts_plot_title = False + + if accepts_plot_title: + return plotter(plot_title=plot_title) + + return plotter() + + def save_final_triangle_plot(fit, save_dir, planet_name, observation_date, source_dir=None): - output_path = triangle_plot_output_path(save_dir, planet_name, observation_date) + output_path = final_triangle_plot_output_path(save_dir, planet_name, observation_date) + compatibility_path = triangle_plot_output_path(save_dir, planet_name, observation_date) output_path.parent.mkdir(parents=True, exist_ok=True) - - fig = fit.plot_triangle() + compatibility_path.parent.mkdir(parents=True, exist_ok=True) + + source_label = comparison_candidate_label_from_output_dir(source_dir) + plot_title = "Final selected fit" + if source_label: + plot_title = f"{plot_title} ({source_label})" + fig = _plot_triangle_for_output(fit, plot_title=plot_title) + if fig is None: + return None fig.savefig(output_path) + if compatibility_path != output_path: + try: + shutil.copy2(output_path, compatibility_path) + except Exception: + fig.savefig(compatibility_path) try: plt.close(fig) except TypeError: @@ -2032,18 +2095,21 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, exc, )) - triangle_plotter = getattr(final_fit, 'plot_triangle', None) - if callable(triangle_plotter): + if callable(getattr(final_fit, 'plot_triangle', None)): try: - fig = triangle_plotter() + fig = _plot_triangle_for_output( + final_fit, + plot_title=f"Comparison candidate #{int(comp_index) + 1} fit", + ) triangle_plot_path = comparison_candidate_triangle_plot_output_path( candidate_dir, p_dict['pName'], observation_date, comp_index, ) - fig.savefig(triangle_plot_path) - plt.close(fig) + if fig is not None: + fig.savefig(triangle_plot_path) + plt.close(fig) except Exception as exc: archive_errors.append(archive_exception_payload( "Could not save the triangle plot", @@ -2305,6 +2371,340 @@ def annotate_parameter_posterior_refit(fit, parameter_key, applied, note=None, h setattr(fit, f"{attr_prefix}_bounds", None) +def annotate_sparse_posterior_live_point_extension( + fit, + enabled, + applied, + note=None, + diagnostics=None, + post_extension_diagnostics=None, + base_live_points=None, + target_live_points=None, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, +): + if fit is None: + return + + fit.sparse_posterior_live_point_extension_enabled = bool(enabled) + fit.sparse_posterior_live_point_extension_applied = bool(applied) + fit.sparse_posterior_live_point_extension_note = note + fit.sparse_posterior_live_point_extension_diagnostics = diagnostics + fit.sparse_posterior_live_point_extension_post_diagnostics = post_extension_diagnostics + fit.sparse_posterior_live_point_extension_base_live_points = base_live_points + fit.sparse_posterior_live_point_extension_target_live_points = target_live_points + fit.sparse_posterior_live_point_extension_factor = extension_factor + + +def clear_fit_ultranest_resume_state(fit): + clear_resume_state = getattr(fit, 'clear_ultranest_resume_state', None) + if callable(clear_resume_state): + clear_resume_state() + + +def callable_accepts_keyword(callable_obj, keyword): + try: + signature = inspect.signature(callable_obj) + except (TypeError, ValueError): + return False + + if keyword in signature.parameters: + return True + return any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + + +def get_configured_ultranest_min_num_live_points(): + return parse_ultranest_min_num_live_points( + os.environ.get( + ULTRANEST_MIN_NUM_LIVE_POINTS_ENV, + ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT, + ) + ) + + +def _effective_sample_count(weights, fallback_count): + if weights is None: + return float(fallback_count) + + weights = np.asarray(weights, dtype=float) + finite_weights = weights[np.isfinite(weights) & (weights > 0)] + if finite_weights.size == 0: + return float(fallback_count) + + weight_sum = float(np.sum(finite_weights)) + weight_square_sum = float(np.sum(finite_weights ** 2)) + if not np.isfinite(weight_sum) or not np.isfinite(weight_square_sum) or weight_square_sum <= 0: + return float(fallback_count) + return float((weight_sum ** 2) / weight_square_sum) + + +def _fit_posterior_sample_matrix(fit, parameter_keys): + parameter_keys = list(parameter_keys) + if fit is None or not parameter_keys: + return np.empty((0, 0), dtype=float), None + + sample_points = None + sample_weights = None + try: + sample_points, _, sample_weights = fit._get_triangle_plot_samples() + except Exception: + sample_points = None + + if sample_points is not None: + sample_points = np.asarray(sample_points, dtype=float) + if sample_points.ndim == 2 and sample_points.shape[0] > 0: + sampled_keys = list(getattr(fit, 'sampled_keys', [])) + bounds = getattr(fit, 'bounds', {}) + bound_keys = list(bounds.keys()) if isinstance(bounds, dict) else [] + physical_getter = getattr(fit, '_physical_values_from_sample_point', None) + columns = [] + for key in parameter_keys: + if key in sampled_keys: + key_index = sampled_keys.index(key) + if key_index >= sample_points.shape[1]: + return np.empty((0, len(parameter_keys)), dtype=float), None + columns.append(np.asarray(sample_points[:, key_index], dtype=float)) + elif callable(physical_getter) and bound_keys: + columns.append(np.asarray([ + physical_getter(point, bound_keys, sampled_keys).get(key, np.nan) + for point in sample_points + ], dtype=float)) + else: + break + else: + weights = None + if sample_weights is not None: + sample_weights = np.asarray(sample_weights, dtype=float) + if sample_weights.ndim == 1 and sample_weights.shape[0] == sample_points.shape[0]: + weights = sample_weights + return np.column_stack(columns), weights + + sample_getter = getattr(fit, 'get_parameter_posterior_samples', None) + if not callable(sample_getter): + return np.empty((0, len(parameter_keys)), dtype=float), None + + columns = [] + min_size = None + for key in parameter_keys: + values = np.asarray(sample_getter(key), dtype=float).reshape(-1) + columns.append(values) + min_size = values.size if min_size is None else min(min_size, values.size) + + if min_size is None or min_size == 0: + return np.empty((0, len(parameter_keys)), dtype=float), None + + return np.column_stack([values[:min_size] for values in columns]), None + + +def evaluate_sparse_posterior_sample_support( + fit, + parameter_keys=SPARSE_POSTERIOR_RETRY_PARAMETER_KEYS, + base_live_points=None, + minimum_effective_samples=None, + minimum_occupied_bins=SPARSE_POSTERIOR_MIN_OCCUPIED_BINS, +): + if base_live_points is None: + base_live_points = get_configured_ultranest_min_num_live_points() + + if minimum_effective_samples is None: + minimum_effective_samples = max( + SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_FLOOR, + int(np.ceil(SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_LIVE_POINT * float(base_live_points))), + ) + minimum_effective_samples = int(max(1, minimum_effective_samples)) + minimum_occupied_bins = int(max(1, minimum_occupied_bins)) + + sample_matrix, sample_weights = _fit_posterior_sample_matrix(fit, parameter_keys) + diagnostics = { + 'sparse': False, + 'reason': None, + 'parameter_keys': list(parameter_keys), + 'base_live_points': int(base_live_points), + 'minimum_effective_samples': minimum_effective_samples, + 'minimum_occupied_bins': minimum_occupied_bins, + 'parameters': {}, + } + + if sample_matrix.size == 0 or sample_matrix.shape[0] == 0: + diagnostics['sparse'] = True + diagnostics['reason'] = "posterior samples are unavailable for Rp/R*, Tmid, and a/Rs." + return diagnostics + + sparse_reasons = [] + for column_index, key in enumerate(parameter_keys): + if column_index >= sample_matrix.shape[1]: + sample_values = np.array([], dtype=float) + else: + sample_values = np.asarray(sample_matrix[:, column_index], dtype=float) + finite_mask = np.isfinite(sample_values) + finite_values = sample_values[finite_mask] + parameter_weights = sample_weights[finite_mask] if sample_weights is not None else None + sample_count = int(finite_values.size) + effective_count = _effective_sample_count(parameter_weights, sample_count) + + occupied_bins = 0 + central_count = 0 + if sample_count >= 2: + q05, q95 = np.nanpercentile(finite_values, [5, 95]) + central_mask = (finite_values >= q05) & (finite_values <= q95) + central_values = finite_values[central_mask] + central_count = int(central_values.size) + if np.isfinite(q05) and np.isfinite(q95) and q05 < q95 and central_count > 0: + bin_count = int(np.clip(np.sqrt(sample_count), 10, 40)) + hist_counts, _ = np.histogram(central_values, bins=bin_count, range=(q05, q95)) + occupied_bins = int(np.count_nonzero(hist_counts > 0)) + + parameter_diagnostic = { + 'sample_count': sample_count, + 'effective_sample_count': float(effective_count), + 'central_sample_count': central_count, + 'occupied_bins': occupied_bins, + 'sparse': False, + 'reason': None, + } + + if effective_count < minimum_effective_samples: + parameter_diagnostic['sparse'] = True + parameter_diagnostic['reason'] = ( + f"effective samples {effective_count:.0f} < {minimum_effective_samples}" + ) + elif occupied_bins and occupied_bins < minimum_occupied_bins: + parameter_diagnostic['sparse'] = True + parameter_diagnostic['reason'] = ( + f"central posterior occupies {occupied_bins} histogram bins < {minimum_occupied_bins}" + ) + + if parameter_diagnostic['sparse']: + sparse_reasons.append(f"{key}: {parameter_diagnostic['reason']}") + diagnostics['parameters'][key] = parameter_diagnostic + + if sparse_reasons: + diagnostics['sparse'] = True + diagnostics['reason'] = "; ".join(sparse_reasons) + else: + diagnostics['reason'] = "posterior sample support is sufficient for Rp/R*, Tmid, and a/Rs." + + return diagnostics + + +def sparse_posterior_diagnostics_summary(diagnostics): + if not isinstance(diagnostics, dict): + return "posterior sample support diagnostics are unavailable" + reason = diagnostics.get('reason') + if reason: + return str(reason) + return "posterior sample support diagnostics are unavailable" + + +def extend_sparse_posterior_live_points_if_needed( + fit, + enabled=None, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, +): + if enabled is None: + enabled = should_use_sparse_posterior_live_point_retry( + os.environ.get( + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV, + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, + ) + ) + + if not enabled: + annotate_sparse_posterior_live_point_extension(fit, False, False) + clear_fit_ultranest_resume_state(fit) + return fit + + base_live_points = get_configured_ultranest_min_num_live_points() + diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) + if not diagnostics.get('sparse'): + annotate_sparse_posterior_live_point_extension( + fit, + True, + False, + note=f"Not needed; {sparse_posterior_diagnostics_summary(diagnostics)}", + diagnostics=diagnostics, + base_live_points=base_live_points, + extension_factor=extension_factor, + ) + clear_fit_ultranest_resume_state(fit) + return fit + + extender = getattr(fit, 'extend_ultranest_fit', None) + if not callable(extender): + note = ( + "Skipped; sparse posterior support was detected, but the UltraNest sampler state " + "is unavailable for an additive extension." + ) + log_info(f"Warning: {note}", warn=True) + annotate_sparse_posterior_live_point_extension( + fit, + True, + False, + note=note, + diagnostics=diagnostics, + base_live_points=base_live_points, + extension_factor=extension_factor, + ) + clear_fit_ultranest_resume_state(fit) + return fit + + extension_factor = int(max(1, extension_factor)) + target_live_points = int(max( + base_live_points + extension_factor * base_live_points, + base_live_points + 1, + )) + try: + current_max_ncalls = int(float(getattr(fit, 'max_ncalls', 2e5))) + except (TypeError, ValueError): + current_max_ncalls = int(2e5) + target_max_ncalls = int(max(current_max_ncalls, current_max_ncalls * (extension_factor + 1))) + log_info( + "Posterior samples for Rp/R*, Tmid, and a/Rs are sparse " + f"({sparse_posterior_diagnostics_summary(diagnostics)}); continuing UltraNest " + f"from {base_live_points} to {target_live_points} minimum live points." + ) + applied = bool(extender(min_num_live_points=target_live_points, max_ncalls=target_max_ncalls)) + post_diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) + + if applied and post_diagnostics.get('sparse'): + note = ( + f"Applied additive sparse-posterior UltraNest extension " + f"({base_live_points}->{target_live_points} minimum live points), but " + f"{sparse_posterior_diagnostics_summary(post_diagnostics)}" + ) + log_info( + "Warning: sparse posterior support remains after the additive UltraNest extension; " + "please inspect the triangle plot carefully.", + warn=True, + ) + elif applied: + note = ( + f"Applied additive sparse-posterior UltraNest extension " + f"({base_live_points}->{target_live_points} minimum live points)." + ) + else: + note = ( + "Skipped; sparse posterior support was detected, but UltraNest did not continue " + "from the retained sampler state." + ) + + annotate_sparse_posterior_live_point_extension( + fit, + True, + applied, + note=note, + diagnostics=diagnostics, + post_extension_diagnostics=post_diagnostics, + base_live_points=base_live_points, + target_live_points=target_live_points, + extension_factor=extension_factor, + ) + clear_fit_ultranest_resume_state(fit) + return fit + + def build_initial_rprs_bounds( rprs, lower_scale=INITIAL_RPRS_BOUND_LOWER_SCALE, @@ -2710,10 +3110,13 @@ def run_nested_lightcurve_fit_with_rprs_posterior_retry( duration_prior=None, max_ars_retries=ARS_POSTERIOR_MAX_RETRIES_DEFAULT, max_impact_parameter_retries=IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES_DEFAULT, + keep_ultranest_sampler=False, ): def impact_parameter_retry_available(fit, local_bounds): if not use_impactparameter_rather_than_inclination_to_fit or 'inc' not in local_bounds: return False + if getattr(fit, 'impact_parameter_sampled_directly', False): + return False sampled_keys = getattr(fit, 'sampled_keys', []) or [] sample_bounds = getattr(fit, 'sample_bounds', {}) return 'b' in sampled_keys or (isinstance(sample_bounds, dict) and 'b' in sample_bounds) @@ -2814,6 +3217,8 @@ def build_fit(local_prior, local_bounds): } if isinstance(duration_prior, dict) and duration_prior.get('applied'): fit_kwargs['duration_prior'] = duration_prior + if keep_ultranest_sampler and callable_accepts_keyword(lc_fitter, 'keep_ultranest_sampler'): + fit_kwargs['keep_ultranest_sampler'] = True fit = lc_fitter( times, flux_values, @@ -3352,6 +3757,34 @@ def should_fit_lightcurve_to_every_comparison_candidate(config_value): return False +def should_use_sparse_posterior_live_point_retry(config_value): + if config_value is None: + return SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_sparse_posterior_live_point_retry' value; " + "defaulting to enabled.", + warn=True, + ) + return SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT + + +def configure_sparse_posterior_live_point_retry(config_value): + enabled = should_use_sparse_posterior_live_point_retry(config_value) + os.environ[SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV] = "1" if enabled else "0" + return enabled + + def should_pick_comparison_by_eebls_snr(config_value): if config_value is None: return True @@ -5209,6 +5642,12 @@ def fit_final_lightcurve_with_oot_baseline_detrending( ): if duration_prior is None and expected_planet_dict is not None: duration_prior = build_single_transit_duration_prior(expected_planet_dict) + keep_ultranest_for_sparse_extension = should_use_sparse_posterior_live_point_retry( + os.environ.get( + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV, + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, + ) + ) fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( times, @@ -5220,6 +5659,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( jd_times=jd_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, + keep_ultranest_sampler=keep_ultranest_for_sparse_extension, ) fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) @@ -5269,6 +5709,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( jd_times=working_jd_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, + keep_ultranest_sampler=keep_ultranest_for_sparse_extension, ) fit = apply_plot_time_range(fit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -5302,6 +5743,11 @@ def fit_final_lightcurve_with_oot_baseline_detrending( note="Disabled; using the direct nested-sampling fit.", ) annotate_transit_detection_qc(fit) + fit = extend_sparse_posterior_live_points_if_needed( + fit, + enabled=keep_ultranest_for_sparse_extension, + ) + annotate_transit_detection_qc(fit) return fit, working_flux, working_unc detrend_result = detrend_flux_on_out_of_transit_baseline( @@ -5322,6 +5768,11 @@ def fit_final_lightcurve_with_oot_baseline_detrending( post_points=detrend_result.get('post_points', 0), ) annotate_transit_detection_qc(fit) + fit = extend_sparse_posterior_live_points_if_needed( + fit, + enabled=keep_ultranest_for_sparse_extension, + ) + annotate_transit_detection_qc(fit) return fit, working_flux, working_unc log_info("Applying optional out-of-transit linear baseline detrending and refitting final light curve.") @@ -5350,6 +5801,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( jd_times=working_jd_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, + keep_ultranest_sampler=keep_ultranest_for_sparse_extension, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -5382,6 +5834,11 @@ def fit_final_lightcurve_with_oot_baseline_detrending( post_points=detrend_result['post_points'], ) annotate_transit_detection_qc(refit) + refit = extend_sparse_posterior_live_points_if_needed( + refit, + enabled=keep_ultranest_for_sparse_extension, + ) + annotate_transit_detection_qc(refit) return refit, detrend_result['flux'], detrend_result['unc'] @@ -11139,6 +11596,14 @@ def summarize_lightcurve_fit_assessment(fit): 'b_posterior_refit_applied': bool(getattr(fit, 'b_posterior_refit_applied', False)), 'b_posterior_refit_count': b_retry_count, 'b_posterior_refit_note': getattr(fit, 'b_posterior_refit_note', None), + 'sparse_posterior_live_point_extension_applied': bool( + getattr(fit, 'sparse_posterior_live_point_extension_applied', False) + ), + 'sparse_posterior_live_point_extension_note': getattr( + fit, + 'sparse_posterior_live_point_extension_note', + None, + ), 'prefit_refinement_applied': bool(getattr(fit, 'prefit_refinement_applied', False)), 'prefit_refinement_note': getattr(fit, 'prefit_refinement_note', None), 'oot_baseline_detrending_applied': bool(getattr(fit, 'oot_baseline_detrending_applied', False)), @@ -11185,12 +11650,18 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): prefit_status = "applied" if assessment['prefit_refinement_applied'] else "not applied" oot_status = "applied" if assessment['oot_baseline_detrending_applied'] else "not applied" duration_prior_status = "applied" if assessment['duration_prior_applied'] else "not applied" + sparse_extension_status = ( + "applied" + if assessment['sparse_posterior_live_point_extension_applied'] + else "not applied" + ) log_info( f"{indent}fit assessment: fit_method={assessment['fit_method']}, " f"duration_prior={duration_prior_status}, " f"Rp/R* posterior retry={rprs_retry_status}, " f"impact parameter posterior retry={b_retry_status}, " + f"sparse posterior extension={sparse_extension_status}, " f"prefit_refinement={prefit_status}, " f"oot_baseline_detrending={oot_status}" ) @@ -11200,6 +11671,11 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): log_info(f"{indent}Rp/R* posterior retry note: {assessment['rprs_posterior_refit_note']}") if assessment.get('b_posterior_refit_note'): log_info(f"{indent}Impact parameter posterior retry note: {assessment['b_posterior_refit_note']}") + if assessment.get('sparse_posterior_live_point_extension_note'): + log_info( + f"{indent}Sparse posterior live-point extension note: " + f"{assessment['sparse_posterior_live_point_extension_note']}" + ) if assessment.get('prefit_refinement_note'): log_info(f"{indent}Prefit refinement note: {assessment['prefit_refinement_note']}") if assessment.get('oot_baseline_detrending_note'): @@ -13272,6 +13748,18 @@ def _main_impl(): ) ) log_info(f"UltraNest minimum live points: {ultranest_min_num_live_points}.") + use_sparse_posterior_live_point_retry = configure_sparse_posterior_live_point_retry( + exotic_infoDict.get( + 'use_sparse_posterior_live_point_retry', + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, + ) + ) + if use_sparse_posterior_live_point_retry: + log_info( + "Sparse posterior live-point extension enabled: final settled fits can continue " + f"UltraNest with {SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT}x additional " + "minimum live points when Rp/R*, Tmid, or a/Rs are under-sampled." + ) log_ultranest_mpi_status() # Make a temp directory of helpful files @@ -15209,9 +15697,14 @@ def _main_impl(): min_aper=np.round(display_aperture, 2), min_annul=np.round(display_annulus, 2), adaptive_summary=photometry_info.get('adaptive_summary'), - photometry_info=photometry_info) + photometry_info=photometry_info, + publish_to_root=True) else: - output_files.final_planetary_params(phot_opt=False, vsp_params=vsp_params) + output_files.final_planetary_params( + phot_opt=False, + vsp_params=vsp_params, + publish_to_root=True, + ) except Exception as e: log_info(f"\nError: Could not create FinalParams.json. {error_txt}\n\t{e}", error=True) try: diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index ec052f6f..9ce21f6a 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -424,6 +424,7 @@ def save_input(): "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to continue the final settled UltraNest fit with 5x additional minimum live points when Rp/Rs, Tmid, or a/Rs posteriors are too sparse after posterior truncation checks and out-of-transit baseline handling. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", @@ -451,6 +452,7 @@ def save_input(): "use_eebls_to_initialize_tmid_and_bounds": "y", "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", + "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" @@ -1504,6 +1506,7 @@ def save_input(): "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to continue the final settled UltraNest fit with 5x additional minimum live points when Rp/Rs, Tmid, or a/Rs posteriors are too sparse after posterior truncation checks and out-of-transit baseline handling. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", @@ -1579,6 +1582,7 @@ def save_input(): "use_eebls_to_initialize_tmid_and_bounds": "y", "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", + "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" @@ -1633,6 +1637,7 @@ def save_input(): "use_eebls_to_initialize_tmid_and_bounds": "y", "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", + "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" diff --git a/exotic/inputs.py b/exotic/inputs.py index 06935770..982ca9c8 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -224,6 +224,7 @@ def __init__(self, init_opt): 'skip_low_comparison_coverage_rejection': 'n', 'fit_lightcurve_to_every_comparison_candidate': 'n', 'ultranest_min_num_live_points': 200, + 'use_sparse_posterior_live_point_retry': 'y', } self.params = { 'images': imaging_files, 'save': save_directory, 'aavso_num': obs_code, 'second_obs': second_obs_code, @@ -507,6 +508,12 @@ def comp_params(self, init_file, planet_dict): 'ultranest_min_live_points', 'min_num_live_points', ), + 'use_sparse_posterior_live_point_retry': ( + 'use_sparse_posterior_live_point_retry', + 'Use Sparse Posterior Live-Point Retry? (y/n)', + 'Use Sparse Posterior Live Point Retry? (y/n)', + 'Sparse Posterior Live-Point Retry? (y/n)', + ), 'bad_wcs_threshold_percent': ( 'bad_wcs_threshold_percent', 'Bad WCS Threshold Percent', diff --git a/exotic/output_files.py b/exotic/output_files.py index c3a07fa1..9d9e784c 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1,4 +1,5 @@ from json import dump, dumps +import shutil from numpy import mean, std from pathlib import Path import numpy as np @@ -824,7 +825,8 @@ def final_lightcurve(self, phase): f.write(f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}\n") def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, - min_annul=None, adaptive_summary=None, photometry_info=None): + min_annul=None, adaptive_summary=None, photometry_info=None, + publish_to_root=False): params_file = self.dir / "temp" / safe_output_filename( "FinalParams", self.p_dict['pName'], @@ -870,6 +872,19 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor impact_text = format_parameter_with_error(impact_parameter, impact_error) if impact_text is not None: params_num["Impact Parameter (b)"] = impact_text + if getattr(self.fit, 'ns_type', None) is not None: + params_num["Fit parameter point estimate"] = ( + "Best-fit likelihood point; uncertainties are posterior spread." + ) + prefit_refinement_note = getattr(self.fit, 'prefit_refinement_note', None) + if prefit_refinement_note: + params_num["Prefit refinement note"] = str(prefit_refinement_note) + oot_baseline_note = getattr(self.fit, 'oot_baseline_detrending_note', None) + if oot_baseline_note: + params_num["Out-of-transit baseline detrending note"] = str(oot_baseline_note) + sparse_posterior_note = getattr(self.fit, 'sparse_posterior_live_point_extension_note', None) + if sparse_posterior_note: + params_num["Sparse posterior live-point extension note"] = str(sparse_posterior_note) if np.isfinite(qc_residual_scatter): params_num["Residual scatter around full model fit"] = f"{qc_residual_scatter * 100.0:.4f} %" params_num.update(format_fit_quality_final_params(fit_quality)) @@ -998,6 +1013,11 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor with params_file.open('w') as f: dump(final_params, f, indent=4) + if publish_to_root: + root_params_file = self.dir / params_file.name + if root_params_file != params_file: + root_params_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(params_file, root_params_file) def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, photometry_info=None, astrometry_info=None, frame_filtering_info=None, diff --git a/exotic/plots.py b/exotic/plots.py index 34be75e2..101df93d 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -1,5 +1,6 @@ from astropy.visualization import astropy_mpl_style, ZScaleInterval, ImageNormalize from astropy.visualization.stretch import LinearStretch, SquaredStretch, SqrtStretch, LogStretch +import inspect import matplotlib.patheffects as path_effects import matplotlib.pyplot as plt from matplotlib.lines import Line2D @@ -602,7 +603,17 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relati # Plotting Final Lightcurve def plot_final_lightcurve(fit, high_res, targ_name, save, date): - f, (ax_lc, ax_res) = fit.plot_bestfit() + plot_kwargs = {} + try: + plot_parameters = inspect.signature(fit.plot_bestfit).parameters + except (TypeError, ValueError): + plot_parameters = {} + if 'show_flux_baseline_label' in plot_parameters: + plot_kwargs['show_flux_baseline_label'] = False + if 'show_model_uncertainty' in plot_parameters: + plot_kwargs['show_model_uncertainty'] = True + + f, (ax_lc, ax_res) = fit.plot_bestfit(**plot_kwargs) ax_lc.set_title(targ_name) if hasattr(fit, 'phase_upsample') and hasattr(fit, 'transit_upsample'): diff --git a/inits.json b/inits.json index 33ce9fea..f6abf98f 100644 --- a/inits.json +++ b/inits.json @@ -37,6 +37,7 @@ "Assess All Comparisons Before Selecting Best": "Set optional_info 'assess_all_comparisons_before_selecting_best' to y to fit every comparison-star candidate that survives the earlier screening, report all fits, and select the candidate with the highest KTMF score. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "UltraNest Live Points": "Set optional_info 'minimum number of live points for ultranest' to a positive integer to control UltraNest's min_num_live_points. Default 200.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to continue the final settled UltraNest fit with 5x additional minimum live points when Rp/Rs, Tmid, or a/Rs posteriors are too sparse after posterior truncation checks and out-of-transit baseline handling. Set to n to disable. Default y.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", @@ -123,6 +124,7 @@ "assess_all_comparisons_before_selecting_best": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "minimum number of live points for ultranest": 200, + "use_sparse_posterior_live_point_retry": "y", "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_adaptive_apertures": false, diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index bc109e40..296a5885 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -316,6 +316,63 @@ def test_plot_bestfit_uses_full_plot_time_range_for_phase_xlim(monkeypatch, tmp_ plt.close(fig) +def test_plot_bestfit_can_hide_flux_baseline_label(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + fig, axes = fit.plot_bestfit(show_flux_baseline_label=False) + legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) + + assert "$a_0$" not in legend_text + plt.close(fig) + + +def test_plot_bestfit_can_draw_transit_model_uncertainty_band(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + fit.errors["rprs"] = 0.01 + fit.errors["tmid"] = 0.001 + envelope = fit.transit_model_uncertainty(fit.time_upsample) + + fig, axes = fit.plot_bestfit(show_model_uncertainty=True) + labels = [artist.get_label() for artist in axes[0].collections] + + assert envelope is not None + assert np.nanmax(envelope[1] - envelope[0]) > 0 + assert r'1-$\sigma$ model uncertainty' in labels + plt.close(fig) + + def test_glc_plot_bestfit_median_limits_use_full_phase_span(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -429,13 +486,41 @@ def test_internal_impact_parameter_transform_round_trips_inclination(monkeypatch sample_point = fit._sample_point_from_unit_cube(np.array([0.25, 0.4, 0.75])) physical = fit._physical_values_from_sample_point(sample_point) - expected_inc = 87.0 + 0.4 * (90.0 - 87.0) + expected_rprs = 0.08 + 0.25 * (0.12 - 0.08) + expected_b = 0.4 * (1.0 + expected_rprs) + expected_inc = float(elca.inclination_from_impact_parameter( + {**fit.prior, "rprs": expected_rprs}, + expected_b, + )) assert fit._get_sampled_keys() == ["rprs", "b", "tmid"] + assert sample_point[1] == pytest.approx(expected_b) assert physical["inc"] == pytest.approx(expected_inc) assert physical["b"] == pytest.approx(sample_point[1]) +def test_internal_impact_parameter_samples_grazing_range_beyond_one(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "inc": [89.8, 90.0], + "tmid": [-0.005, 0.005], + } + + sample_point = fit._sample_point_from_unit_cube(np.array([1.0, 0.99, 0.5])) + physical = fit._physical_values_from_sample_point(sample_point) + + assert sample_point[0] == pytest.approx(0.12) + assert sample_point[1] == pytest.approx(0.99 * 1.12) + assert sample_point[1] > 1.0 + assert physical["inc"] < 90.0 + assert fit._get_sample_bounds()["b"] == pytest.approx([0.0, 1.12]) + + def test_nested_fit_can_keep_inclination_parameterization_when_requested(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) @@ -566,21 +651,10 @@ def __init__(self, *args, **kwargs): verbose=False, ) - bounds_values = [] - for ars_value in (11.5, 12.5): - corner_values = prior.copy() - corner_values["ars"] = ars_value - bounds_values.extend( - np.asarray( - elca.impact_parameter_from_inclination(corner_values, np.array([87.0, 89.5])), - dtype=float, - ).reshape(-1).tolist() - ) - assert fit.sampled_keys == ["rprs", "ars", "b", "tmid"] assert fit.parameters["ars"] == pytest.approx(12.3, abs=1e-12) assert fit.parameters["inc"] == pytest.approx(88.8, abs=1e-6) - assert fit.sample_bounds["b"] == pytest.approx([min(bounds_values), max(bounds_values)]) + assert fit.sample_bounds["b"] == pytest.approx([0.0, 1.12]) def test_nested_fit_replaces_degenerate_ultranest_errors_from_loglike_neighborhood(monkeypatch, tmp_path): @@ -929,7 +1003,7 @@ def test_rprs_posterior_recenter_diagnostics_ignores_lower_edge_below_twenty_per assert "not treated as truncated" in diagnostics["reason"] -def test_plot_triangle_uses_mirrored_distance_from_fitted_impact_parameter_axis(monkeypatch, tmp_path): +def test_plot_triangle_uses_direct_fitted_impact_parameter_axis(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) @@ -940,6 +1014,7 @@ def fake_corner(*args, **kwargs): captured["labels"] = kwargs["labels"] captured["range"] = kwargs["range"] captured["titles"] = kwargs["titles"] + captured["truths"] = kwargs["truths"] captured["label_kwargs"] = kwargs["label_kwargs"] return "figure" @@ -983,14 +1058,11 @@ def fake_corner(*args, **kwargs): fig = fit.plot_triangle() assert fig == "figure" - assert captured["labels"][1] == r"$\Delta b$" - expected_b_distance_limit = np.max(np.abs(np.array([0.0, 1.25434156]) - fit.sample_parameters["b"])) - assert captured["range"][1][0] == pytest.approx(-expected_b_distance_limit) - assert captured["range"][1][1] == pytest.approx(expected_b_distance_limit) - assert captured["points"].shape == (10, 3) - expected_b_distance = np.abs(points[:, 1] - fit.sample_parameters["b"]) - np.testing.assert_allclose(captured["points"][:5, 1], expected_b_distance) - np.testing.assert_allclose(captured["points"][5:, 1], -expected_b_distance) + assert captured["labels"][1] == r"Impact parameter $b$" + assert captured["range"][1] == pytest.approx([0.0, 1.25434156]) + assert captured["points"].shape == (5, 3) + np.testing.assert_allclose(captured["points"][:, 1], points[:, 1]) + assert captured["truths"][1] == pytest.approx(fit.sample_parameters["b"]) assert captured["titles"][1].startswith("b=") assert "\ni=" in captured["titles"][1] assert captured["label_kwargs"]["labelpad"] == 10 @@ -1046,7 +1118,7 @@ def test_triangle_payload_expands_degenerate_error_ranges_to_sample_cloud(monkey assert payload["ranges"][1][1] >= 0.0038 -def test_triangle_payload_title_uses_visible_histogram_peak(monkeypatch, tmp_path): +def test_triangle_payload_titles_match_reported_parameters(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) @@ -1082,8 +1154,8 @@ def test_triangle_payload_title_uses_visible_histogram_peak(monkeypatch, tmp_pat payload = fit._get_triangle_plot_payload() - assert payload["titles"][0].startswith("0.11900 +-") - assert not payload["titles"][0].startswith("0.33796") + assert payload["titles"][0] == "0.338 +/- 0.091" + assert payload["truths"][0] == pytest.approx(0.33796) np.testing.assert_allclose(payload["display_weights"], weights) @@ -1094,6 +1166,8 @@ def test_plot_triangle_passes_ultranest_weights_to_visible_histograms(monkeypatc def fake_corner(*args, **kwargs): captured["weights"] = kwargs["weights"] + captured["truths"] = kwargs["truths"] + captured["data_kwargs"] = kwargs["data_kwargs"] return "figure" monkeypatch.setattr(elca, "corner", fake_corner) @@ -1128,6 +1202,9 @@ def fake_corner(*args, **kwargs): assert fig == "figure" np.testing.assert_allclose(captured["weights"], weights) + np.testing.assert_allclose(captured["truths"], [0.1, 1.0]) + assert captured["data_kwargs"]["s"] == pytest.approx(1.6) + assert captured["data_kwargs"]["alpha"] == pytest.approx(0.38) def test_triangle_payload_expands_sparse_visible_ranges_to_sample_cloud(monkeypatch, tmp_path): @@ -1210,7 +1287,7 @@ def test_triangle_payload_uses_tested_rprs_range_when_posterior_is_narrow(monkey assert upper_fraction < elca.TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX -def test_triangle_payload_expands_mirrored_impact_parameter_range_to_sample_cloud(monkeypatch, tmp_path): +def test_triangle_payload_keeps_direct_impact_parameter_full_sample_range(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) @@ -1248,9 +1325,57 @@ def test_triangle_payload_expands_mirrored_impact_parameter_range_to_sample_clou payload = fit._get_triangle_plot_payload() - assert payload["labels"][1] == r"$\Delta b$" - assert payload["ranges"][1][0] <= -0.52 - assert payload["ranges"][1][1] >= 0.52 + assert payload["labels"][1] == r"Impact parameter $b$" + assert payload["ranges"][1] == pytest.approx([0.0, 1.2]) + assert payload["display_points"].shape == points.shape + np.testing.assert_allclose(payload["display_points"][:, 1], points[:, 1]) + assert payload["truths"][1] == pytest.approx(fit.sample_parameters["b"]) + assert payload["display_spec"]["mirror"] is False + + +def test_triangle_payload_uses_full_b_range_for_direct_impact_parameter(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.sampled_keys = ["rprs", "b", "a0"] + fit.sample_bounds = { + "rprs": [0.0, 0.2], + "b": [0.0, 1.2], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.sample_parameters = {"rprs": 0.10, "b": 0.856, "a0": 1.0} + fit.sample_errors = {"rprs": 0.01, "b": 0.002, "a0": 0.001} + fit.parameters = {"rprs": 0.10, "inc": 85.0, "a0": 1.0} + fit.errors = {"rprs": 0.01, "inc": 2.7, "a0": 0.001} + b_samples = np.linspace(0.846, 0.866, 100) + points = np.column_stack([ + np.linspace(0.090, 0.110, b_samples.size), + b_samples, + np.linspace(0.998, 1.002, b_samples.size), + ]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + reference_values = [reference["value"] for reference in payload["display_spec"]["reference_lines"]] + + assert payload["labels"][1] == r"Impact parameter $b$" + assert payload["ranges"][1] == pytest.approx([0.0, 1.2]) + assert payload["truths"][1] == pytest.approx(0.856) + assert payload["display_spec"]["mirror"] is False + assert reference_values == pytest.approx([1.0, 1.10]) def test_triangle_payload_tracks_left_and_right_geometry_branches_for_inclination(monkeypatch, tmp_path): @@ -1297,7 +1422,7 @@ def test_triangle_payload_tracks_left_and_right_geometry_branches_for_inclinatio ) -def test_triangle_payload_tracks_left_and_right_geometry_branches_for_impact_parameter(monkeypatch, tmp_path): +def test_triangle_payload_skips_mirrored_overlay_for_direct_impact_parameter(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) @@ -1337,16 +1462,10 @@ def test_triangle_payload_tracks_left_and_right_geometry_branches_for_impact_par payload = fit._get_triangle_plot_payload() - np.testing.assert_allclose( - payload["geometry_overlay"]["left_mirrored"], - np.array([-0.014, -0.024, 0.0, 0.014, 0.024, -0.0]), - atol=1e-12, - ) - np.testing.assert_allclose( - payload["geometry_overlay"]["right_mirrored"], - np.array([0.016, 0.046, 0.0, -0.016, -0.046, -0.0]), - atol=1e-12, - ) + assert payload["geometry_overlay"] is None + assert payload["display_spec"]["mirror"] is False + assert payload["ranges"][1] == pytest.approx([0.0, 1.25434156]) + np.testing.assert_allclose(payload["display_points"][:, 1], points[:, 1]) def test_triangle_geometry_curves_fall_back_to_surviving_branch(monkeypatch, tmp_path): @@ -1376,7 +1495,14 @@ def test_triangle_geometry_overlay_reuses_shared_title_and_label_kwargs(monkeypa payload = { "sampled_keys": ["rprs", "b"], "display_points": np.zeros((10, 2)), - "display_spec": {"index": 1}, + "display_spec": { + "index": 1, + "center": 0.92, + "reference_lines": [ + {"value": 1.0, "linestyle": ":", "color": "#707070"}, + {"value": 1.10, "linestyle": "-.", "color": "#a35d00"}, + ], + }, "geometry_overlay": { "index": 1, "left_count": 2, @@ -1385,7 +1511,7 @@ def test_triangle_geometry_overlay_reuses_shared_title_and_label_kwargs(monkeypa "right_mirrored": np.array([-0.2, 0.2]), }, "ranges": [[0.0, 0.1], [-0.3, 0.3]], - "titles": ["rprs", "b=0.32 +- 0.18\ni=88.5 +- 1.35 deg"], + "titles": ["rprs", "b=0.32 +/- 0.18\ni=88.5 +/- 1.35 deg"], "labels": ["rprs", r"$\Delta b$"], } @@ -1400,4 +1526,10 @@ def test_triangle_geometry_overlay_reuses_shared_title_and_label_kwargs(monkeypa assert ax.title.get_fontsize() == pytest.approx(12.0) assert ax.xaxis.label.get_text() == r"$\Delta b$" assert ax.xaxis.labelpad == pytest.approx(10.0) + reference_offsets = [] + for line in ax.lines: + xdata = np.asarray(line.get_xdata(), dtype=float) + if xdata.size == 2 and np.allclose(xdata, xdata[0]) and line.get_linestyle() in (":", "-."): + reference_offsets.append(float(xdata[0])) + assert sorted(reference_offsets) == pytest.approx([-0.18, -0.08, 0.08, 0.18]) plt.close(fig) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index a83c2ee2..2fa68a5e 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -195,8 +195,9 @@ def plot_triangle(self): source_dir=source_dir, ) - assert output_path == final_dir / "temp" / source_plot.name + assert output_path == final_dir / f"FinalTriangle_{planet_name}_{observation_date}.png" assert output_path.read_bytes() == b"regenerated-final" + assert (final_dir / "temp" / f"Triangle_{planet_name}_{observation_date}.png").read_bytes() == b"regenerated-final" assert fit.called is True @@ -247,6 +248,40 @@ def plot_triangle(self): assert fit.called is True assert output_path.read_bytes() == b"regenerated" + assert output_path.parent == tmp_path / "final" + assert output_path.name == "FinalTriangle_TOI-1728 b_2024-12-14.png" + assert ( + tmp_path + / "final" + / "temp" + / "Triangle_TOI-1728 b_2024-12-14.png" + ).read_bytes() == b"regenerated" + + +def test_save_final_triangle_plot_labels_selected_candidate_when_supported(tmp_path): + class DummyFigure: + def savefig(self, path): + Path(path).write_bytes(b"regenerated") + + class DummyFit: + def __init__(self): + self.plot_title = None + + def plot_triangle(self, plot_title=None): + self.plot_title = plot_title + return DummyFigure() + + fit = DummyFit() + output_path = save_final_triangle_plot( + fit, + tmp_path / "final", + "TOI-1728 b", + "2024-12-14", + source_dir=tmp_path / "comp4", + ) + + assert output_path.read_bytes() == b"regenerated" + assert fit.plot_title == "Final selected fit (comparison candidate #4)" def test_update_coordinates_handles_non_numeric_proper_motion_values(): diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index cf436314..d3dbe8ff 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -89,6 +89,9 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, RPRS_SEARCH_BOUND_MAX, RPRS_SEARCH_BOUND_MIN, + SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + evaluate_sparse_posterior_sample_support, + extend_sparse_posterior_live_points_if_needed, build_single_transit_duration_prior, build_initial_rprs_bounds, run_nested_lightcurve_fit_with_rprs_posterior_retry, @@ -575,6 +578,53 @@ def fake_lc_fitter( assert fit.b_posterior_refit_bounds == pytest.approx([np.degrees(np.arccos(0.25)), 90.0]) +def test_impact_parameter_retry_is_skipped_when_b_is_sampled_directly(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics = { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": True, "edge": "upper", "mode": 1.1, "std": 0.04, "bounds": [0.0, 1.12]}, + } + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured["calls"].append({"prior": dict(call_prior), "bounds": dict(call_bounds)}) + fit = types.SimpleNamespace( + sampled_keys=["rprs", "b", "tmid"], + sample_bounds={"rprs": [0.0, 0.25], "b": [0.0, 1.12], "tmid": [-0.01, 0.01]}, + impact_parameter_sampled_directly=True, + parameters={"rprs": 0.1, "ars": 10.0, "tmid": 0.0, "inc": 83.5, "a2": 0.0}, + ) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: dict(diagnostics[key]) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 85.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "tmid": [-0.01, 0.01], "inc": [80.0, 90.0], "a2": [-3.0, 3.0]}, + ) + + assert len(captured["calls"]) == 1 + assert fit.b_posterior_refit_applied is False + assert fit.b_posterior_refit_count == 0 + + def test_impact_parameter_posterior_retry_expands_toward_face_on_boundary(monkeypatch): import exotic.exotic as exotic_module @@ -694,3 +744,106 @@ def get_parameter_posterior_recenter_diagnostics(key): assert captured["duration_prior"] == duration_prior assert fit.duration_prior_applied is True assert "expected duration=" in fit.duration_prior_note + + +def test_sparse_posterior_metric_flags_under_sampled_key_parameters(): + fit = types.SimpleNamespace() + fit.get_parameter_posterior_samples = lambda key: np.linspace(0.0, 1.0, 100) + + diagnostics = evaluate_sparse_posterior_sample_support( + fit, + base_live_points=200, + ) + + assert diagnostics["sparse"] is True + assert diagnostics["minimum_effective_samples"] == 600 + assert diagnostics["parameters"]["rprs"]["effective_sample_count"] == pytest.approx(100) + assert "rprs" in diagnostics["reason"] + + +def test_sparse_posterior_extension_continues_existing_ultranest_sampler(monkeypatch): + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "200") + + class SparseFit: + def __init__(self): + self.samples = { + "rprs": np.linspace(0.09, 0.11, 100), + "tmid": np.linspace(-0.001, 0.001, 100), + "ars": np.linspace(9.5, 10.5, 100), + } + self.max_ncalls = 1000 + self.extension_calls = [] + self.cleared = False + + def get_parameter_posterior_samples(self, key): + return self.samples[key] + + def extend_ultranest_fit(self, min_num_live_points=None, max_ncalls=None): + self.extension_calls.append({ + "min_num_live_points": min_num_live_points, + "max_ncalls": max_ncalls, + }) + self.samples = { + "rprs": np.linspace(0.09, 0.11, 1000), + "tmid": np.linspace(-0.001, 0.001, 1000), + "ars": np.linspace(9.5, 10.5, 1000), + } + return True + + def clear_ultranest_resume_state(self): + self.cleared = True + + fit = SparseFit() + returned = extend_sparse_posterior_live_points_if_needed( + fit, + enabled=True, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + ) + + assert returned is fit + assert fit.extension_calls == [{ + "min_num_live_points": 1200, + "max_ncalls": 6000, + }] + assert fit.sparse_posterior_live_point_extension_applied is True + assert "200->1200" in fit.sparse_posterior_live_point_extension_note + assert fit.cleared is True + + +def test_run_nested_lightcurve_fit_can_retain_sampler_for_final_extension(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + diagnostics = {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + keep_ultranest_sampler=False, + ): + captured["keep_ultranest_sampler"] = keep_ultranest_sampler + fit = types.SimpleNamespace(parameters={"rprs": 0.1, "ars": 15.0, "tmid": 0.0, "inc": 89.0, "a2": 0.0}) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: dict(diagnostics) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 15.0, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "ars": [14.5, 15.5], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + keep_ultranest_sampler=True, + ) + + assert captured["keep_ultranest_sampler"] is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py index d2a6503a..abcd9647 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -158,6 +158,21 @@ def test_comp_params_defaults_ultranest_live_points_to_200(tmp_path): assert inputs.info_dict["ultranest_min_num_live_points"] == 200 +def test_comp_params_defaults_sparse_posterior_live_point_retry_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_sparse_posterior_live_point_retry"] == "y" + + def test_comp_params_defaults_exit_at_first_qc_pass_solution_to_yes(tmp_path): init_data = { "user_info": {}, @@ -443,6 +458,21 @@ def test_comp_params_reads_ultranest_live_points_from_optional_info(tmp_path): assert inputs.info_dict["ultranest_min_num_live_points"] == 275 +def test_comp_params_reads_sparse_posterior_live_point_retry_off_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_sparse_posterior_live_point_retry": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_sparse_posterior_live_point_retry"] == "n" + + def test_comp_params_reads_exit_at_first_qc_pass_solution_from_optional_info(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_output_files.py b/tests/test_output_files.py index d58afc21..fa57edcc 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -250,6 +250,27 @@ def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclinati assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" +def test_final_planetary_params_can_publish_accepted_copy_to_root(tmp_path): + fit = DummyFit() + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + publish_to_root=True, + ) + + temp_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + root_file = tmp_path / "FinalParams_HAT-P-32 b_2020-01-01.json" + + assert temp_file.exists() + assert root_file.exists() + assert root_file.read_text(encoding="utf-8") == temp_file.read_text(encoding="utf-8") + + def test_final_planetary_params_reports_adaptive_aperture_summary(tmp_path): fit = DummyFit() (tmp_path / "temp").mkdir() diff --git a/tests/test_plots.py b/tests/test_plots.py index a0d9181a..de8a36f4 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -9,6 +9,7 @@ plot_fov, plot_adaptive_aperture_diagnostics, plot_comp_star_candidate_lightcurve_fits, + plot_final_lightcurve, plot_individual_comp_star_calibration_series, plot_obs_stats, ) @@ -197,3 +198,36 @@ def plot_bestfit(self, phase=False): assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.png").exists() assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.pdf").exists() assert not (tmp_path / "temp" / "CompStarLightCurveFit_Comp3_Target_2026-03-09.png").exists() + + +def test_plot_final_lightcurve_requests_uncertainty_band_without_baseline_label(tmp_path): + class DummyFinalFit: + def __init__(self): + self.kwargs = None + self.phase_upsample = np.linspace(-0.05, 0.05, 5) + self.transit_upsample = np.ones(5) + + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False): + self.kwargs = { + "show_flux_baseline_label": show_flux_baseline_label, + "show_model_uncertainty": show_model_uncertainty, + } + fig, axes = plt.subplots(2, 1) + return fig, axes + + fit = DummyFinalFit() + + plot_final_lightcurve( + fit, + high_res=np.ones(5), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + assert fit.kwargs == { + "show_flux_baseline_label": False, + "show_model_uncertainty": True, + } + assert (tmp_path / "FinalLightCurve_Target_2026-03-09.png").exists() + assert (tmp_path / "FinalLightCurve_Target_2026-03-09.pdf").exists() From f26abaac38df8b32758b1605f14e224f342036aa Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 11 May 2026 21:18:11 +1000 Subject: [PATCH 039/116] actual uncertainty in model around fit. --- exotic/api/elca.py | 537 +++++++++++++++++++++++- exotic/api/ultranest_utils.py | 145 ++++++- exotic/exotic.py | 644 ++++++++++++++++++++++++++--- exotic/exotic_gui.py | 4 +- exotic/output_files.py | 3 + exotic/plots.py | 45 +- inits.json | 2 +- tests/test_elca_baseline.py | 219 +++++++++- tests/test_exotic_proper_motion.py | 262 ++++++++++++ tests/test_exotic_rprs_retry.py | 2 +- tests/test_plots.py | 17 +- tests/test_ultranest_utils.py | 63 +++ 12 files changed, 1835 insertions(+), 108 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 0ca8daba..562c0951 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -45,6 +45,7 @@ import faulthandler import io from itertools import cycle, product +import math import os import sys import bottleneck as bn @@ -75,9 +76,14 @@ TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX = 0.50 TRIANGLE_PLOT_EDGE_MIN_SAMPLE_COUNT = 30 TRIANGLE_PLOT_EDGE_EXPANSION_STEPS = 8 +TRIANGLE_PLOT_FALLBACK_EXPANSION_BOUNDS = { + 'rprs': (0.0, 1.0), +} TRANSIT_MODEL_UNCERTAINTY_KEYS = ( 'rprs', 'tmid', 'inc', 'ars', 'per', 'ecc', 'omega', 'u0', 'u1', 'u2', 'u3', ) +BASELINE_MODEL_UNCERTAINTY_KEYS = ('a0', 'a1', 'a2') +MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT = 2000 def _pylightcurve_import_watchdog_seconds(): try: @@ -289,10 +295,22 @@ def airmass_trend_grid(a2_values, airmass, reference=None): return np.exp(np.outer(np.asarray(a2_values, dtype=float), centered)) -def solve_flux_baseline(model, data, dataerr=None): +def normalized_optional_fit_mask(mask, shape): + if mask is None: + return None + fit_mask = np.asarray(mask, dtype=bool) + if fit_mask.shape != tuple(shape): + return None + if not np.any(fit_mask): + return None + return fit_mask + + +def solve_flux_baseline(model, data, dataerr=None, mask=None): model = np.asarray(model, dtype=float) data = np.asarray(data, dtype=float) weights = np.ones(model.shape, dtype=float) + fit_mask = normalized_optional_fit_mask(mask, model.shape) if dataerr is not None: dataerr = np.asarray(dataerr, dtype=float) @@ -301,6 +319,8 @@ def solve_flux_baseline(model, data, dataerr=None): weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) mask = np.isfinite(model) & np.isfinite(data) & (model != 0) + if fit_mask is not None: + mask &= fit_mask if dataerr is not None: mask &= np.isfinite(weights) & (weights > 0) @@ -324,12 +344,15 @@ def solve_flux_baseline(model, data, dataerr=None): return baseline if np.isfinite(baseline) else fallback_flux_baseline() -def solve_flux_baseline_uncertainty(model, dataerr): +def solve_flux_baseline_uncertainty(model, dataerr, mask=None): if dataerr is None: return 0.0 model = np.asarray(model, dtype=float) dataerr = np.asarray(dataerr, dtype=float) + fit_mask = normalized_optional_fit_mask(mask, model.shape) mask = np.isfinite(model) & np.isfinite(dataerr) & (dataerr > 0) + if fit_mask is not None: + mask &= fit_mask if not np.any(mask): return 0.0 denom = np.sum((model[mask] / dataerr[mask]) ** 2) @@ -338,12 +361,13 @@ def solve_flux_baseline_uncertainty(model, dataerr): return (1.0 / denom) ** 0.5 -def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000): +def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000, mask=None): n = int(n) a2 = np.random.normal(m_a2, sig_a2, n) reference = get_airmass_reference(airmass) model = transit * airmass_trend_grid(a2, airmass, reference=reference) weights = np.ones(transit.shape[0], dtype=float) + fit_mask = normalized_optional_fit_mask(mask, transit.shape) if dataerr is not None: dataerr = np.asarray(dataerr, dtype=float) @@ -351,7 +375,9 @@ def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000): valid_err = np.isfinite(dataerr) & (dataerr > 0) weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) - mask = np.isfinite(data) & np.isfinite(transit) + mask = np.isfinite(data) & np.isfinite(transit) & np.isfinite(airmass) + if fit_mask is not None: + mask &= fit_mask if dataerr is not None: mask &= np.isfinite(weights) & (weights > 0) @@ -368,8 +394,8 @@ def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000): if not np.any(valid): best_model = transit * airmass_trend(m_a2, airmass, reference=reference) - baseline = solve_flux_baseline(best_model, data, dataerr) - return baseline, solve_flux_baseline_uncertainty(best_model, dataerr) + baseline = solve_flux_baseline(best_model, data, dataerr, mask=fit_mask) + return baseline, solve_flux_baseline_uncertainty(best_model, dataerr, mask=fit_mask) baselines = numer[valid] / denom[valid] baseline = float(np.nanmean(baselines)) @@ -377,7 +403,7 @@ def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000): if baseline_unc == 0.0: best_model = transit * airmass_trend(m_a2, airmass, reference=reference) - baseline_unc = solve_flux_baseline_uncertainty(best_model, dataerr) + baseline_unc = solve_flux_baseline_uncertainty(best_model, dataerr, mask=fit_mask) return baseline, baseline_unc @@ -450,6 +476,8 @@ def __init__( use_impactparameter_rather_than_inclination_to_fit=True, duration_prior=None, keep_ultranest_sampler=False, + baseline_fit_mask=None, + fixed_parameter_errors=None, ): self.time = time self.data = data @@ -466,6 +494,12 @@ def __init__( self.use_impactparameter_rather_than_inclination_to_fit = use_impactparameter_rather_than_inclination_to_fit self.duration_prior = copy.deepcopy(duration_prior) if isinstance(duration_prior, dict) else None self.keep_ultranest_sampler = bool(keep_ultranest_sampler) + self.baseline_fit_mask = self._coerce_baseline_fit_mask(baseline_fit_mask) + self.fixed_parameter_errors = ( + copy.deepcopy(fixed_parameter_errors) + if isinstance(fixed_parameter_errors, dict) + else {} + ) self._ultranest_resume_context = None self.results = None self.sampled_keys = list(bounds.keys()) @@ -508,6 +542,38 @@ def _set_flux_baseline(self, value, error=0.0): self.parameters['a1'] = value self.errors['a1'] = error + def _coerce_baseline_fit_mask(self, baseline_fit_mask): + fit_mask = normalized_optional_fit_mask(baseline_fit_mask, np.asarray(self.time).shape) + if fit_mask is None: + return None + return fit_mask + + def _get_baseline_fit_mask(self): + return getattr(self, 'baseline_fit_mask', None) + + def _apply_fixed_parameter_errors(self): + fixed_errors = getattr(self, 'fixed_parameter_errors', None) + if not isinstance(fixed_errors, dict): + return + if not hasattr(self, 'parameters') or not isinstance(self.parameters, dict): + return + if not hasattr(self, 'errors') or not isinstance(self.errors, dict): + self.errors = {} + if not hasattr(self, 'quantiles') or not isinstance(self.quantiles, dict): + self.quantiles = {} + + for key, error in fixed_errors.items(): + if key not in self.parameters or key in self.errors: + continue + try: + error = float(error) + except (TypeError, ValueError): + continue + if not np.isfinite(error) or error < 0: + continue + self.errors[key] = error + self.quantiles[key] = [-error, error] + def _get_airmass_reference(self): return getattr(self, 'airmass_reference', get_airmass_reference(self.airmass)) @@ -541,13 +607,46 @@ def _build_systematics_model(self, values): reference=self._get_airmass_reference(), ) + def _build_systematics_model_at(self, values, times=None): + if times is None: + return self._build_systematics_model(values) + + times = np.asarray(times, dtype=float) + if np.ndim(self.airmass) == 2: + return np.full(times.shape, get_flux_baseline(values), dtype=float) + + source_times = np.asarray(self.time, dtype=float) + source_airmass = np.asarray(self.airmass, dtype=float) + finite = np.isfinite(source_times) & np.isfinite(source_airmass) + if times.shape == source_times.shape and np.allclose(times, source_times, rtol=0.0, atol=0.0): + airmass_values = source_airmass + elif np.count_nonzero(finite) >= 2: + order = np.argsort(source_times[finite]) + airmass_values = np.interp( + times, + source_times[finite][order], + source_airmass[finite][order], + left=source_airmass[finite][order][0], + right=source_airmass[finite][order][-1], + ) + elif np.count_nonzero(finite) == 1: + airmass_values = np.full(times.shape, source_airmass[finite][0], dtype=float) + else: + airmass_values = np.zeros(times.shape, dtype=float) + + return get_flux_baseline(values) * airmass_trend( + values.get('a2', 0), + airmass_values, + reference=self._get_airmass_reference(), + ) + def _get_perturbed_transit_parameter_value(self, key, value): try: value = float(value) except (TypeError, ValueError): return np.nan - if key in ('rprs', 'ars', 'per'): + if key in ('rprs', 'ars', 'per', 'a0', 'a1'): return max(value, np.finfo(float).eps) if key == 'ecc': return float(np.clip(value, 0.0, 0.999999)) @@ -555,6 +654,139 @@ def _get_perturbed_transit_parameter_value(self, key, value): return float(np.clip(value, 0.0, 180.0)) return value + def _normalized_model_for_plot_times(self, times, values): + model = np.asarray(transit(times, values), dtype=float) + if np.ndim(self.airmass) == 2: + return model + + try: + sample_systematics = self._build_systematics_model_at(values, times) + best_systematics = self._build_systematics_model_at(self.parameters, times) + except Exception: + return model + + with np.errstate(divide='ignore', invalid='ignore'): + normalized_model = model * sample_systematics / best_systematics + if normalized_model.shape != model.shape or not np.any(np.isfinite(normalized_model)): + return model + return normalized_model + + def _posterior_model_uncertainty(self, times, sigma=1.0): + if getattr(self, 'results', None) is None: + return None + + try: + sample_points, sample_logl, sample_weights = self._get_triangle_plot_samples() + except Exception: + return None + + sample_points = np.asarray(sample_points, dtype=float) + if sample_points.ndim != 2 or sample_points.shape[0] < 2: + return None + + finite_rows = np.all(np.isfinite(sample_points), axis=1) + if sample_logl is not None: + sample_logl = np.asarray(sample_logl, dtype=float) + if sample_logl.shape[0] == sample_points.shape[0]: + finite_rows &= np.isfinite(sample_logl) + + if np.count_nonzero(finite_rows) < 2: + return None + + row_indices = np.flatnonzero(finite_rows) + if row_indices.size > MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT: + if sample_weights is not None: + weights_array = np.asarray(sample_weights, dtype=float) + if weights_array.shape[0] == sample_points.shape[0]: + row_weights = np.where(np.isfinite(weights_array[row_indices]), weights_array[row_indices], 0.0) + order = np.argsort(row_weights)[-MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT:] + row_indices = row_indices[np.sort(order)] + else: + row_indices = row_indices[ + np.linspace(0, row_indices.size - 1, MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT).astype(int) + ] + else: + row_indices = row_indices[ + np.linspace(0, row_indices.size - 1, MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT).astype(int) + ] + + selected_points = sample_points[row_indices] + selected_weights = None + if sample_weights is not None: + weights_array = np.asarray(sample_weights, dtype=float) + if weights_array.shape[0] == sample_points.shape[0]: + selected_weights = weights_array[row_indices] + selected_weights = np.where(np.isfinite(selected_weights) & (selected_weights >= 0), selected_weights, 0.0) + if np.sum(selected_weights) <= 0: + selected_weights = None + + bound_keys = list(self.bounds.keys()) + sampled_keys = getattr(self, 'sampled_keys', None) + if sampled_keys is None: + sampled_keys = self._get_sampled_keys(bound_keys) + models = [] + for point in selected_points: + try: + values = copy.deepcopy(self.parameters) + values.update(self._physical_values_from_sample_point(point, bound_keys, sampled_keys)) + model = self._normalized_model_for_plot_times(times, values) + except Exception: + continue + if model.shape == times.shape and np.all(np.isfinite(model)): + models.append(model) + + if len(models) < 2: + return None + + model_grid = np.asarray(models, dtype=float) + if selected_weights is not None and selected_weights.shape[0] != model_grid.shape[0]: + selected_weights = None + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = 1.0 + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + coverage = math.erf(sigma / np.sqrt(2.0)) + q_lower = 0.5 * (1.0 - coverage) + q_upper = 1.0 - q_lower + + if selected_weights is None: + lower, median, upper = np.nanpercentile( + model_grid, + [100.0 * q_lower, 50.0, 100.0 * q_upper], + axis=0, + ) + else: + lower = np.array([ + self._weighted_quantiles(model_grid[:, i], [q_lower], weights=selected_weights)[0] + for i in range(model_grid.shape[1]) + ]) + median = np.array([ + self._weighted_quantiles(model_grid[:, i], [0.5], weights=selected_weights)[0] + for i in range(model_grid.shape[1]) + ]) + upper = np.array([ + self._weighted_quantiles(model_grid[:, i], [q_upper], weights=selected_weights)[0] + for i in range(model_grid.shape[1]) + ]) + + try: + best_model = self._normalized_model_for_plot_times(times, self.parameters) + except Exception: + best_model = median + + lower_width = median - lower + upper_width = upper - median + lower = best_model - np.maximum(lower_width, 0.0) + upper = best_model + np.maximum(upper_width, 0.0) + + finite = np.isfinite(lower) & np.isfinite(upper) & (lower <= upper) + if not np.any(finite): + return None + return lower, upper + def transit_model_uncertainty(self, times=None, sigma=1.0): if times is None: times = getattr(self, 'time_upsample', self.time) @@ -563,13 +795,23 @@ def transit_model_uncertainty(self, times=None, sigma=1.0): return None try: - model = transit(times, self.parameters) + model = self._normalized_model_for_plot_times(times, self.parameters) except Exception: return None + posterior_envelope = self._posterior_model_uncertainty(times, sigma=sigma) + if posterior_envelope is not None: + return posterior_envelope + sigma = float(sigma) variance = np.zeros_like(model, dtype=float) - for key in TRANSIT_MODEL_UNCERTAINTY_KEYS: + uncertainty_keys = list(TRANSIT_MODEL_UNCERTAINTY_KEYS) + for key in BASELINE_MODEL_UNCERTAINTY_KEYS: + if key == 'a1' and 'a0' in self.parameters: + continue + uncertainty_keys.append(key) + + for key in uncertainty_keys: if key not in self.parameters: continue error = self.errors.get(key) @@ -595,8 +837,8 @@ def transit_model_uncertainty(self, times=None, sigma=1.0): lower_parameters[key] = lower_value upper_parameters[key] = upper_value try: - lower_model = transit(times, lower_parameters) - upper_model = transit(times, upper_parameters) + lower_model = self._normalized_model_for_plot_times(times, lower_parameters) + upper_model = self._normalized_model_for_plot_times(times, upper_parameters) except Exception: continue @@ -618,16 +860,38 @@ def _plot_transit_model_uncertainty(self, ax, x_values, times, sort_index, label lower, upper = envelope x_values = np.asarray(x_values, dtype=float) sort_index = np.asarray(sort_index, dtype=int) - return ax.fill_between( - x_values[sort_index], - np.asarray(lower, dtype=float)[sort_index], - np.asarray(upper, dtype=float)[sort_index], + x_sorted = x_values[sort_index] + lower_sorted = np.asarray(lower, dtype=float)[sort_index] + upper_sorted = np.asarray(upper, dtype=float)[sort_index] + band = ax.fill_between( + x_sorted, + lower_sorted, + upper_sorted, color='red', alpha=0.16, linewidth=0, zorder=2.5, label=label, ) + ax.plot( + x_sorted, + lower_sorted, + color='red', + linestyle='--', + linewidth=0.9, + alpha=0.72, + zorder=3.4, + ) + ax.plot( + x_sorted, + upper_sorted, + color='red', + linestyle='--', + linewidth=0.9, + alpha=0.72, + zorder=3.4, + ) + return band def _uses_internal_impact_parameter(self): return ( @@ -972,8 +1236,26 @@ def _get_plot_range_expansion_bounds(self, key, sample_values, center): bound_upper = np.nan if np.isfinite(bound_lower) and np.isfinite(bound_upper) and bound_lower < bound_upper: + fallback_bounds = TRIANGLE_PLOT_FALLBACK_EXPANSION_BOUNDS.get(key) + if fallback_bounds is not None: + fallback_lower, fallback_upper = fallback_bounds + if ( + np.isfinite(fallback_lower) + and np.isfinite(fallback_upper) + and fallback_lower < fallback_upper + ): + return [ + float(min(bound_lower, fallback_lower)), + float(max(bound_upper, fallback_upper)), + ] return [bound_lower, bound_upper] + fallback_bounds = TRIANGLE_PLOT_FALLBACK_EXPANSION_BOUNDS.get(key) + if fallback_bounds is not None: + fallback_lower, fallback_upper = fallback_bounds + if np.isfinite(fallback_lower) and np.isfinite(fallback_upper) and fallback_lower < fallback_upper: + return [float(fallback_lower), float(fallback_upper)] + sample_values = np.asarray(sample_values, dtype=float) finite_values = sample_values[np.isfinite(sample_values)] try: @@ -1265,6 +1547,128 @@ def _format_triangle_plot_parameter_title(self, value, error): return str(round_to_2(value)) return f"{round_to_2(value, error)} +/- {round_to_2(error)}" + def _weighted_quantiles(self, values, quantiles, weights=None): + values = np.asarray(values, dtype=float) + quantiles = np.asarray(quantiles, dtype=float) + finite_mask = np.isfinite(values) + + finite_weights = None + if weights is not None: + weights = np.asarray(weights, dtype=float) + if weights.shape == values.shape: + finite_mask &= np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + if finite_weights.size == 0 or np.sum(finite_weights) <= 0: + finite_weights = None + + finite_values = values[finite_mask] + if finite_values.size == 0: + return np.full(quantiles.shape, np.nan, dtype=float) + if finite_weights is None: + return np.nanpercentile(finite_values, 100.0 * quantiles) + + order = np.argsort(finite_values) + sorted_values = finite_values[order] + sorted_weights = finite_weights[order] + cumulative = np.cumsum(sorted_weights) + total = cumulative[-1] + if not np.isfinite(total) or total <= 0: + return np.nanpercentile(finite_values, 100.0 * quantiles) + + cumulative = (cumulative - 0.5 * sorted_weights) / total + cumulative = np.clip(cumulative, 0.0, 1.0) + return np.interp(quantiles, cumulative, sorted_values) + + def _triangle_plot_display_estimate( + self, + sample_values, + fallback_center, + fallback_error, + plot_range=None, + weights=None, + min_informative_peak_ratio=1.5, + ): + sample_values = np.asarray(sample_values, dtype=float) + finite_mask = np.isfinite(sample_values) + finite_weights = None + if weights is not None: + weights = np.asarray(weights, dtype=float) + if weights.shape == sample_values.shape: + finite_mask &= np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + if finite_weights.size == 0 or np.sum(finite_weights) <= 0: + finite_weights = None + + finite_values = sample_values[finite_mask] + if finite_values.size < 2: + return fallback_center, fallback_error + + q16, q50, q84 = self._weighted_quantiles( + sample_values, + [0.158655, 0.5, 0.841345], + weights=weights, + ) + estimate = q50 + if plot_range is None: + bounds = [float(np.nanmin(finite_values)), float(np.nanmax(finite_values))] + else: + try: + bounds = [float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + bounds = [float(np.nanmin(finite_values)), float(np.nanmax(finite_values))] + + if np.all(np.isfinite(bounds)) and bounds[0] < bounds[1]: + bins = int(np.clip(np.sqrt(finite_values.size), 10, 80)) + counts, edges = np.histogram( + finite_values, + bins=max(1, bins), + range=bounds, + weights=finite_weights, + ) + positive_counts = counts[counts > 0] + if positive_counts.size > 0: + peak = float(np.nanmax(positive_counts)) + typical = float(np.nanmedian(positive_counts)) + total = float(np.nansum(positive_counts)) + if ( + np.isfinite(peak) + and np.isfinite(typical) + and np.isfinite(total) + and total > 0 + and typical > 0 + and peak >= min_informative_peak_ratio * typical + and peak >= 0.05 * total + ): + mode_index = int(np.argmax(counts)) + estimate = float(0.5 * (edges[mode_index] + edges[mode_index + 1])) + + try: + fallback_center = float(fallback_center) + except (TypeError, ValueError): + fallback_center = np.nan + if not np.isfinite(estimate): + estimate = fallback_center + + spread_candidates = [ + abs(float(q84) - float(estimate)) if np.isfinite(q84) and np.isfinite(estimate) else np.nan, + abs(float(estimate) - float(q16)) if np.isfinite(q16) and np.isfinite(estimate) else np.nan, + 0.5 * abs(float(q84) - float(q16)) if np.isfinite(q16) and np.isfinite(q84) else np.nan, + ] + try: + fallback_error = float(fallback_error) + except (TypeError, ValueError): + fallback_error = np.nan + + finite_spreads = [value for value in spread_candidates if np.isfinite(value) and value >= 0] + if finite_spreads and max(finite_spreads) > 0: + error = float(max(finite_spreads)) + elif np.isfinite(fallback_error) and fallback_error > 0: + error = fallback_error + else: + error = np.nan + + return float(estimate), error + def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bins=None): diagnostics = { 'key': key, @@ -1872,6 +2276,13 @@ def _get_triangle_plot_payload(self): bins=plot_bins, weights=sample_weights, ) + center, error = self._triangle_plot_display_estimate( + sample_points[:, i], + center, + error, + plot_range=plot_range, + weights=sample_weights, + ) title = self._format_triangle_plot_parameter_title(center, error) truth = center @@ -1911,6 +2322,57 @@ def _get_triangle_plot_payload(self): 'truths': truths, } + def _triangle_plot_sigma_window_ranges(self, payload, sigma): + try: + sigma = float(sigma) + except (TypeError, ValueError): + return payload['ranges'] + if not np.isfinite(sigma) or sigma <= 0: + return payload['ranges'] + + zoomed_ranges = [] + display_points = np.asarray(payload.get('display_points', []), dtype=float) + for i, plot_range in enumerate(payload['ranges']): + try: + range_lower, range_upper = [ + float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + zoomed_ranges.append(plot_range) + continue + + if not np.isfinite(range_lower) or not np.isfinite(range_upper) or range_lower >= range_upper: + zoomed_ranges.append(plot_range) + continue + + try: + center = float(payload['mask_centers'][i]) + error = float(payload['mask_errors'][i]) + except (TypeError, ValueError, IndexError): + zoomed_ranges.append(plot_range) + continue + + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + zoomed_ranges.append(plot_range) + continue + + lower = max(range_lower, center - sigma * error) + upper = min(range_upper, center + sigma * error) + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + zoomed_ranges.append(plot_range) + continue + + if display_points.ndim == 2 and i < display_points.shape[1]: + values = display_points[:, i] + finite_values = values[np.isfinite(values)] + if finite_values.size and not np.any((finite_values >= lower) & (finite_values <= upper)): + zoomed_ranges.append(plot_range) + continue + + zoomed_ranges.append([float(lower), float(upper)]) + + return zoomed_ranges + def _triangle_contour_levels(self, chi2, mask1, mask2, mask3): raw_levels = np.array([ np.percentile(chi2[mask1], 95), @@ -2061,7 +2523,12 @@ def lc2min_airmass(pars): if self._has_free_flux_baseline(): model *= get_flux_baseline(self.prior) else: - model *= solve_flux_baseline(model, self.data, self.dataerr) + model *= solve_flux_baseline( + model, + self.data, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ) return ((self.data - model) / self.dataerr) ** 2 try: @@ -2107,6 +2574,7 @@ def lc2min_airmass(pars): def create_fit_variables(self): self.transit = transit(self.time, self.parameters) + self._apply_fixed_parameter_errors() self._update_plot_geometry() if np.ndim(self.airmass) != 2: if self._has_free_flux_baseline(): @@ -2120,6 +2588,7 @@ def create_fit_variables(self): self.airmass, self.data, self.dataerr, + mask=self._get_baseline_fit_mask(), ) else: systematics = self.transit * airmass_trend( @@ -2127,8 +2596,23 @@ def create_fit_variables(self): self.airmass, reference=self._get_airmass_reference(), ) - flux_scale = solve_flux_baseline(systematics, self.data, self.dataerr) - flux_scale_err = self.errors.get('a0', self.errors.get('a1', solve_flux_baseline_uncertainty(systematics, self.dataerr))) + flux_scale = solve_flux_baseline( + systematics, + self.data, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ) + flux_scale_err = self.errors.get( + 'a0', + self.errors.get( + 'a1', + solve_flux_baseline_uncertainty( + systematics, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ), + ), + ) self._set_flux_baseline(flux_scale, flux_scale_err) if np.ndim(self.airmass) == 2: detrended = self.data / self.transit @@ -2218,6 +2702,7 @@ def _finalize_ultranest_fit_results(self, bound_keys, sampled_keys, physical_fro self.parameters['inc'] = center self.errors['inc'] = std self.quantiles['inc'] = quantiles + self._apply_fixed_parameter_errors() def extend_ultranest_fit(self, min_num_live_points=None, max_ncalls=None): context = getattr(self, '_ultranest_resume_context', None) @@ -2298,7 +2783,12 @@ def single_loglike(pars): if self._has_free_flux_baseline(): model *= get_flux_baseline(physical) else: - model *= solve_flux_baseline(model, self.data, self.dataerr) + model *= solve_flux_baseline( + model, + self.data, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ) except Exception: return BAD_LOG_LIKELIHOOD @@ -2410,6 +2900,7 @@ def prior_transform(upars): self.airmass, self.data, self.dataerr, + mask=self._get_baseline_fit_mask(), )[0] test_values['a0'] = flux_scale test_values['a1'] = flux_scale @@ -2568,8 +3059,12 @@ def plot_bestfit( axs[1].grid(True, ls='--', axis='y') return f, axs - def plot_triangle(self, plot_title=None): + def plot_triangle(self, plot_title=None, zoom_sigma=None): payload = self._get_triangle_plot_payload() + if zoom_sigma is not None: + payload = dict(payload) + payload['ranges'] = self._triangle_plot_sigma_window_ranges(payload, zoom_sigma) + chi2 = payload['display_logl'] * -2 parameter_count = max(1, len(payload['sampled_keys'])) fig_size = max(9.0, 2.35 * parameter_count) diff --git a/exotic/api/ultranest_utils.py b/exotic/api/ultranest_utils.py index 8b4db06d..0ad48373 100644 --- a/exotic/api/ultranest_utils.py +++ b/exotic/api/ultranest_utils.py @@ -45,6 +45,14 @@ "NEXTASTRO_EXOTIC_ULTRANEST_WORKERS", ) ULTRANEST_WORKER_BACKEND_ENV = "EXOTIC_ULTRANEST_WORKER_BACKEND" +BYTES_PER_GIB = 1024 ** 3 +MIN_AUTO_POINTS_PER_WORKER = 8 +MEDIUM_AUTO_POINTS_PER_WORKER = 16 +HIGH_AUTO_POINTS_PER_WORKER = 24 +MAX_AUTO_POINTS_PER_WORKER = 32 +AUTO_POINTS_PER_WORKER_MULTIPLIER = 2 +AUTO_DRAW_RAM_FRACTION = 0.005 +MIN_AUTO_DRAW_RAM_BUDGET_BYTES = 64 * 1024 ** 2 _PROCESS_LOGLIKE = None _TK_CLEANUP_CLASSES = ("Image", "Variable") @@ -159,6 +167,120 @@ def _configured_ultranest_worker_backend(): return "none" +def _system_total_memory_bytes(): + if hasattr(os, "sysconf"): + try: + page_size = int(os.sysconf("SC_PAGE_SIZE")) + page_count = int(os.sysconf("SC_PHYS_PAGES")) + if page_size > 0 and page_count > 0: + return page_size * page_count + except (AttributeError, OSError, TypeError, ValueError): + pass + + if sys.platform.startswith("win"): + try: + import ctypes + + class MEMORYSTATUSEX(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("sullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + memory_status = MEMORYSTATUSEX() + memory_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX) + if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory_status)): + return int(memory_status.ullTotalPhys) + except Exception: + pass + + return None + + +def _auto_points_per_worker(cpu_count, total_memory_bytes): + if total_memory_bytes is None: + return MEDIUM_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + + ram_per_cpu_gib = total_memory_bytes / max(int(cpu_count or 1), 1) / BYTES_PER_GIB + if ram_per_cpu_gib >= 2.0: + return MAX_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + if ram_per_cpu_gib >= 1.0: + return HIGH_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + if ram_per_cpu_gib >= 0.5: + return MEDIUM_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + return MIN_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + + +def _estimate_ultranest_draw_point_bytes(sampler): + x_dim = _coerce_positive_int(getattr(sampler, "x_dim", None), default=None) + num_params = _coerce_positive_int(getattr(sampler, "num_params", None), default=None) + coordinate_count = 16 + if x_dim is not None and num_params is not None: + coordinate_count = max(16, 3 + x_dim + num_params) + + return max(1024, coordinate_count * np.dtype(float).itemsize * 16) + + +def _apply_auto_ultranest_draw_sizes(sampler, worker_count): + worker_count = max(int(worker_count or 1), 1) + if worker_count <= 1 or getattr(sampler, "draw_multiple", True) is False: + return None + + current_min = _coerce_positive_int(getattr(sampler, "ndraw_min", None), default=None) + current_max = _coerce_positive_int(getattr(sampler, "ndraw_max", None), default=None) + if current_min is None and current_max is None: + return None + if current_min is None: + current_min = 128 + if current_max is None: + current_max = max(current_min, 65536) + if current_max < current_min: + current_max = current_min + + total_memory_bytes = _system_total_memory_bytes() + cpu_count = _available_cpu_count() + points_per_worker = _auto_points_per_worker(cpu_count, total_memory_bytes) + target_min = max(current_min, worker_count * points_per_worker) + memory_limited_max = current_max + + if total_memory_bytes is not None: + memory_budget = max( + int(total_memory_bytes * AUTO_DRAW_RAM_FRACTION), + MIN_AUTO_DRAW_RAM_BUDGET_BYTES, + ) + point_bytes = _estimate_ultranest_draw_point_bytes(sampler) + memory_limited_max = max(current_min, min(current_max, memory_budget // point_bytes)) + target_min = min(target_min, memory_limited_max) + + target_min = int(max(current_min, min(current_max, target_min))) + target_max = int(max(target_min, min(current_max, memory_limited_max))) + if target_min == current_min and target_max == current_max: + return None + + sampler.ndraw_min = target_min + sampler.ndraw_max = target_max + + ram_gib = None + if total_memory_bytes is not None: + ram_gib = total_memory_bytes / BYTES_PER_GIB + + return { + "ndraw_min": target_min, + "ndraw_max": target_max, + "workers": worker_count, + "cpu_count": cpu_count, + "ram_gib": ram_gib, + "points_per_worker": points_per_worker, + } + + def _process_loglike_chunk(chunk): if _PROCESS_LOGLIKE is None: raise RuntimeError("UltraNest process worker was not initialized.") @@ -222,7 +344,7 @@ def _parallel_vectorized_loglike(sampler, workers=None): original_loglike = getattr(sampler, "loglike", None) if worker_count <= 1 or backend == "none" or not callable(original_loglike): - yield 1, "single" + yield 1, "single", None return pool = None @@ -230,7 +352,7 @@ def _parallel_vectorized_loglike(sampler, workers=None): process_pool_guard = None if backend == "process": if not sys.platform.startswith("linux") or _is_colab_runtime(): - yield 1, "single" + yield 1, "single", None return global _PROCESS_LOGLIKE process_pool_guard = suppress_tk_cleanup_during_process_pool() @@ -246,7 +368,7 @@ def _parallel_vectorized_loglike(sampler, workers=None): elif backend == "thread": executor = ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix="exotic-ultranest") else: - yield 1, "single" + yield 1, "single", None return def parallel_loglike(params): @@ -263,8 +385,9 @@ def parallel_loglike(params): return np.concatenate([np.atleast_1d(result) for result in results]) sampler.loglike = parallel_loglike + draw_sizes = _apply_auto_ultranest_draw_sizes(sampler, worker_count) try: - yield worker_count, backend + yield worker_count, backend, draw_sizes finally: sampler.loglike = original_loglike if pool is not None: @@ -512,7 +635,19 @@ def run_reactive_sampler( mode = "silent" if is_mpi_worker_process() else _progress_mode(verbose=verbose, stream=stream) output_stream = stream if stream is not None else sys.stdout - with _parallel_vectorized_loglike(sampler) as (worker_count, worker_backend): + with _parallel_vectorized_loglike(sampler) as (worker_count, worker_backend, draw_sizes): + if draw_sizes is not None and mode != "silent": + memory_label = "unknown RAM" + if draw_sizes["ram_gib"] is not None: + memory_label = f"{draw_sizes['ram_gib']:.1f} GiB RAM" + print( + "[ultranest] Auto proposal draw sizes: " + f"ndraw_min={draw_sizes['ndraw_min']}, ndraw_max={draw_sizes['ndraw_max']} " + f"({draw_sizes['workers']} workers, {draw_sizes['points_per_worker']} points/worker, " + f"{memory_label}).", + file=output_stream, + flush=True, + ) if worker_count > 1 and mode != "silent": worker_label = "processes" if worker_backend == "process" else "threads" print( diff --git a/exotic/exotic.py b/exotic/exotic.py index bfbca1e0..c23f62b2 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -230,9 +230,11 @@ SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV = "EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY" SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT = 5 SPARSE_POSTERIOR_RETRY_PARAMETER_KEYS = ('rprs', 'tmid', 'ars') -SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_FLOOR = 500 -SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_LIVE_POINT = 3.0 +SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_FLOOR = 1000 +SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_LIVE_POINT = 5.0 SPARSE_POSTERIOR_MIN_OCCUPIED_BINS = 8 +SPARSE_POSTERIOR_MIN_OCCUPIED_BIN_FRACTION = 0.65 +SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_OCCUPIED_BIN = 25.0 RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 RPRS_SEARCH_BOUND_MIN = 0.0 RPRS_SEARCH_BOUND_MAX = 1.0 @@ -357,6 +359,30 @@ def annotate_out_of_transit_baseline_detrending( fit.oot_baseline_post_points = int(post_points) if post_points is not None else 0 +def annotate_out_of_transit_baseline_parameter_fit( + fit, + applied, + note=None, + pre_points=0, + post_points=0, + a0=None, + a0_error=None, + a2=None, + a2_error=None, +): + if fit is None: + return + + fit.oot_baseline_parameter_fit_applied = bool(applied) + fit.oot_baseline_parameter_fit_note = note + fit.oot_baseline_parameter_fit_pre_points = int(pre_points) if pre_points is not None else 0 + fit.oot_baseline_parameter_fit_post_points = int(post_points) if post_points is not None else 0 + fit.oot_baseline_parameter_fit_a0 = a0 + fit.oot_baseline_parameter_fit_a0_error = a0_error + fit.oot_baseline_parameter_fit_a2 = a2 + fit.oot_baseline_parameter_fit_a2_error = a2_error + + def annotate_final_fit_prefit_refinement( fit, applied, @@ -1597,6 +1623,13 @@ def final_triangle_plot_output_path(save_dir, planet_name, observation_date): ) +def zoomed_final_triangle_plot_output_path(save_dir, planet_name, observation_date): + return ( + Path(save_dir) + / safe_output_filename("ZoomedTrianglePlot", planet_name, filename_date_token(observation_date), extension="png") + ) + + def comparison_candidate_triangle_plot_output_path(save_dir, planet_name, observation_date, comp_index): return ( Path(save_dir) @@ -1621,34 +1654,38 @@ def comparison_candidate_label_from_output_dir(output_dir): return None -def _plot_triangle_for_output(fit, plot_title=None): +def _plot_triangle_for_output(fit, plot_title=None, required_keywords=(), **plot_kwargs): plotter = getattr(fit, 'plot_triangle', None) if not callable(plotter): return None - if plot_title: - try: - signature = inspect.signature(plotter) - accepts_plot_title = ( - 'plot_title' in signature.parameters - or any( - parameter.kind == inspect.Parameter.VAR_KEYWORD - for parameter in signature.parameters.values() - ) - ) - except (TypeError, ValueError): - accepts_plot_title = False + supported_kwargs = {} + for keyword in required_keywords: + if not callable_accepts_keyword(plotter, keyword): + return None - if accepts_plot_title: - return plotter(plot_title=plot_title) + if plot_title and callable_accepts_keyword(plotter, 'plot_title'): + supported_kwargs['plot_title'] = plot_title + for keyword, value in plot_kwargs.items(): + if callable_accepts_keyword(plotter, keyword): + supported_kwargs[keyword] = value - return plotter() + return plotter(**supported_kwargs) + + +def _close_plot_figure(fig): + try: + plt.close(fig) + except TypeError: + pass def save_final_triangle_plot(fit, save_dir, planet_name, observation_date, source_dir=None): output_path = final_triangle_plot_output_path(save_dir, planet_name, observation_date) + zoomed_output_path = zoomed_final_triangle_plot_output_path(save_dir, planet_name, observation_date) compatibility_path = triangle_plot_output_path(save_dir, planet_name, observation_date) output_path.parent.mkdir(parents=True, exist_ok=True) + zoomed_output_path.parent.mkdir(parents=True, exist_ok=True) compatibility_path.parent.mkdir(parents=True, exist_ok=True) source_label = comparison_candidate_label_from_output_dir(source_dir) @@ -1664,10 +1701,27 @@ def save_final_triangle_plot(fit, save_dir, planet_name, observation_date, sourc shutil.copy2(output_path, compatibility_path) except Exception: fig.savefig(compatibility_path) + _close_plot_figure(fig) + + zoomed_fig = None try: - plt.close(fig) - except TypeError: - pass + zoomed_title = f"{plot_title} (5-sigma zoom)" + zoomed_fig = _plot_triangle_for_output( + fit, + plot_title=zoomed_title, + required_keywords=('zoom_sigma',), + zoom_sigma=5.0, + ) + if zoomed_fig is not None: + zoomed_fig.savefig(zoomed_output_path) + except Exception as exc: + try: + log_info(f"Warning: Could not save zoomed final triangle plot: {exc}", warn=True) + except Exception: + pass + finally: + if zoomed_fig is not None: + _close_plot_figure(zoomed_fig) return output_path @@ -1994,6 +2048,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, expected_planet_dict=p_dict, expected_tmid_search_summary=tmid_search_summary, eebls_search_summary=eebls_search_summary, + extend_sparse_posterior_live_points=False, + keep_ultranest_sampler_for_deferred_extension=True, ) if final_fit is None: result['failure_reason'] = "the full comparison-candidate reduction did not converge." @@ -2080,7 +2136,10 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, plotter = getattr(final_fit, 'plot_bestfit', None) if callable(plotter): try: - fig, _ = plotter() + plot_kwargs = {} + if callable_accepts_keyword(plotter, 'show_flux_baseline_label'): + plot_kwargs['show_flux_baseline_label'] = False + fig, _ = plotter(**plot_kwargs) bestfit_plot_path = temp_dir / safe_output_filename( "BestFit", p_dict['pName'], @@ -2223,7 +2282,10 @@ def archive_failed_comparison_fit(save_dir, planet_name, observation_date, attem plotter = getattr(fit, 'plot_bestfit', None) if callable(plotter): try: - fig, _ = plotter() + plot_kwargs = {} + if callable_accepts_keyword(plotter, 'show_flux_baseline_label'): + plot_kwargs['show_flux_baseline_label'] = False + fig, _ = plotter(**plot_kwargs) bestfit_plot_path = temp_dir / safe_output_filename( "BestFit", planet_name, @@ -2504,6 +2566,8 @@ def evaluate_sparse_posterior_sample_support( base_live_points=None, minimum_effective_samples=None, minimum_occupied_bins=SPARSE_POSTERIOR_MIN_OCCUPIED_BINS, + minimum_occupied_bin_fraction=SPARSE_POSTERIOR_MIN_OCCUPIED_BIN_FRACTION, + minimum_effective_samples_per_occupied_bin=SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_OCCUPIED_BIN, ): if base_live_points is None: base_live_points = get_configured_ultranest_min_num_live_points() @@ -2515,6 +2579,10 @@ def evaluate_sparse_posterior_sample_support( ) minimum_effective_samples = int(max(1, minimum_effective_samples)) minimum_occupied_bins = int(max(1, minimum_occupied_bins)) + minimum_occupied_bin_fraction = float(np.clip(minimum_occupied_bin_fraction, 0.0, 1.0)) + minimum_effective_samples_per_occupied_bin = float( + max(0.0, minimum_effective_samples_per_occupied_bin) + ) sample_matrix, sample_weights = _fit_posterior_sample_matrix(fit, parameter_keys) diagnostics = { @@ -2524,6 +2592,8 @@ def evaluate_sparse_posterior_sample_support( 'base_live_points': int(base_live_points), 'minimum_effective_samples': minimum_effective_samples, 'minimum_occupied_bins': minimum_occupied_bins, + 'minimum_occupied_bin_fraction': minimum_occupied_bin_fraction, + 'minimum_effective_samples_per_occupied_bin': minimum_effective_samples_per_occupied_bin, 'parameters': {}, } @@ -2546,6 +2616,9 @@ def evaluate_sparse_posterior_sample_support( occupied_bins = 0 central_count = 0 + bin_count = 0 + occupied_bin_fraction = 0.0 + effective_samples_per_occupied_bin = 0.0 if sample_count >= 2: q05, q95 = np.nanpercentile(finite_values, [5, 95]) central_mask = (finite_values >= q05) & (finite_values <= q95) @@ -2555,12 +2628,22 @@ def evaluate_sparse_posterior_sample_support( bin_count = int(np.clip(np.sqrt(sample_count), 10, 40)) hist_counts, _ = np.histogram(central_values, bins=bin_count, range=(q05, q95)) occupied_bins = int(np.count_nonzero(hist_counts > 0)) + occupied_bin_fraction = ( + float(occupied_bins) / float(bin_count) + if bin_count > 0 + else 0.0 + ) + if occupied_bins > 0: + effective_samples_per_occupied_bin = float(effective_count) / float(occupied_bins) parameter_diagnostic = { 'sample_count': sample_count, 'effective_sample_count': float(effective_count), 'central_sample_count': central_count, + 'central_bin_count': bin_count, 'occupied_bins': occupied_bins, + 'occupied_bin_fraction': occupied_bin_fraction, + 'effective_samples_per_occupied_bin': effective_samples_per_occupied_bin, 'sparse': False, 'reason': None, } @@ -2575,6 +2658,22 @@ def evaluate_sparse_posterior_sample_support( parameter_diagnostic['reason'] = ( f"central posterior occupies {occupied_bins} histogram bins < {minimum_occupied_bins}" ) + elif bin_count and occupied_bin_fraction < minimum_occupied_bin_fraction: + parameter_diagnostic['sparse'] = True + parameter_diagnostic['reason'] = ( + f"central posterior occupies {occupied_bin_fraction:.2f} of histogram bins " + f"< {minimum_occupied_bin_fraction:.2f}" + ) + elif ( + occupied_bins + and minimum_effective_samples_per_occupied_bin > 0 + and effective_samples_per_occupied_bin < minimum_effective_samples_per_occupied_bin + ): + parameter_diagnostic['sparse'] = True + parameter_diagnostic['reason'] = ( + f"effective samples per occupied bin {effective_samples_per_occupied_bin:.1f} " + f"< {minimum_effective_samples_per_occupied_bin:.1f}" + ) if parameter_diagnostic['sparse']: sparse_reasons.append(f"{key}: {parameter_diagnostic['reason']}") @@ -2602,6 +2701,8 @@ def extend_sparse_posterior_live_points_if_needed( fit, enabled=None, extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + require_sparse=True, + extension_label="sparse-posterior", ): if enabled is None: enabled = should_use_sparse_posterior_live_point_retry( @@ -2619,23 +2720,24 @@ def extend_sparse_posterior_live_points_if_needed( base_live_points = get_configured_ultranest_min_num_live_points() diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) if not diagnostics.get('sparse'): - annotate_sparse_posterior_live_point_extension( - fit, - True, - False, - note=f"Not needed; {sparse_posterior_diagnostics_summary(diagnostics)}", - diagnostics=diagnostics, - base_live_points=base_live_points, - extension_factor=extension_factor, - ) - clear_fit_ultranest_resume_state(fit) - return fit + if require_sparse: + annotate_sparse_posterior_live_point_extension( + fit, + True, + False, + note=f"Not needed; {sparse_posterior_diagnostics_summary(diagnostics)}", + diagnostics=diagnostics, + base_live_points=base_live_points, + extension_factor=extension_factor, + ) + clear_fit_ultranest_resume_state(fit) + return fit extender = getattr(fit, 'extend_ultranest_fit', None) if not callable(extender): note = ( - "Skipped; sparse posterior support was detected, but the UltraNest sampler state " - "is unavailable for an additive extension." + "Skipped; the retained UltraNest sampler state is unavailable for an additive " + f"{extension_label} live-point extension." ) log_info(f"Warning: {note}", warn=True) annotate_sparse_posterior_live_point_extension( @@ -2660,17 +2762,25 @@ def extend_sparse_posterior_live_points_if_needed( except (TypeError, ValueError): current_max_ncalls = int(2e5) target_max_ncalls = int(max(current_max_ncalls, current_max_ncalls * (extension_factor + 1))) - log_info( - "Posterior samples for Rp/R*, Tmid, and a/Rs are sparse " - f"({sparse_posterior_diagnostics_summary(diagnostics)}); continuing UltraNest " - f"from {base_live_points} to {target_live_points} minimum live points." - ) + if diagnostics.get('sparse'): + log_info( + "Posterior samples for Rp/R*, Tmid, and a/Rs are sparse " + f"({sparse_posterior_diagnostics_summary(diagnostics)}); continuing UltraNest " + f"from {base_live_points} to {target_live_points} minimum live points " + "using the retained final-pass sampler bounds." + ) + else: + log_info( + f"Continuing the {extension_label} UltraNest fit from {base_live_points} " + f"to {target_live_points} minimum live points using the retained final-pass " + f"sampler bounds ({sparse_posterior_diagnostics_summary(diagnostics)})." + ) applied = bool(extender(min_num_live_points=target_live_points, max_ncalls=target_max_ncalls)) post_diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) if applied and post_diagnostics.get('sparse'): note = ( - f"Applied additive sparse-posterior UltraNest extension " + f"Applied additive {extension_label} UltraNest extension " f"({base_live_points}->{target_live_points} minimum live points), but " f"{sparse_posterior_diagnostics_summary(post_diagnostics)}" ) @@ -2681,12 +2791,12 @@ def extend_sparse_posterior_live_points_if_needed( ) elif applied: note = ( - f"Applied additive sparse-posterior UltraNest extension " + f"Applied additive {extension_label} UltraNest extension " f"({base_live_points}->{target_live_points} minimum live points)." ) else: note = ( - "Skipped; sparse posterior support was detected, but UltraNest did not continue " + f"Skipped; UltraNest did not continue the additive {extension_label} extension " "from the retained sampler state." ) @@ -2705,6 +2815,16 @@ def extend_sparse_posterior_live_points_if_needed( return fit +def extend_selected_comparison_live_points_if_needed(fit, enabled=None): + return extend_sparse_posterior_live_points_if_needed( + fit, + enabled=enabled, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + require_sparse=False, + extension_label="selected comparison-star final", + ) + + def build_initial_rprs_bounds( rprs, lower_scale=INITIAL_RPRS_BOUND_LOWER_SCALE, @@ -3111,6 +3231,8 @@ def run_nested_lightcurve_fit_with_rprs_posterior_retry( max_ars_retries=ARS_POSTERIOR_MAX_RETRIES_DEFAULT, max_impact_parameter_retries=IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES_DEFAULT, keep_ultranest_sampler=False, + baseline_fit_mask=None, + fixed_parameter_errors=None, ): def impact_parameter_retry_available(fit, local_bounds): if not use_impactparameter_rather_than_inclination_to_fit or 'inc' not in local_bounds: @@ -3219,6 +3341,10 @@ def build_fit(local_prior, local_bounds): fit_kwargs['duration_prior'] = duration_prior if keep_ultranest_sampler and callable_accepts_keyword(lc_fitter, 'keep_ultranest_sampler'): fit_kwargs['keep_ultranest_sampler'] = True + if baseline_fit_mask is not None and callable_accepts_keyword(lc_fitter, 'baseline_fit_mask'): + fit_kwargs['baseline_fit_mask'] = baseline_fit_mask + if fixed_parameter_errors and callable_accepts_keyword(lc_fitter, 'fixed_parameter_errors'): + fit_kwargs['fixed_parameter_errors'] = fixed_parameter_errors fit = lc_fitter( times, flux_values, @@ -5176,6 +5302,208 @@ def prepare_final_fit_lightcurve_series( } +def fit_airmass_baseline_parameters_on_out_of_transit( + times, + flux_values, + flux_errors, + airmass, + fit, + prior=None, + bounds=None, + depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + airmass = np.asarray(airmass, dtype=float) + prior = {} if prior is None else dict(prior) + bounds = {} if bounds is None else dict(bounds) + + base_result = { + 'applied': False, + 'note': 'out-of-transit baseline parameter fitting did not run.', + 'oot_mask': None, + 'pre_points': 0, + 'post_points': 0, + 'a0': np.nan, + 'a0_error': np.nan, + 'a2': prior.get('a2', 0.0), + 'a2_error': np.nan, + 'used_prior_ephemeris': False, + } + + if not (times.shape == flux_values.shape == flux_errors.shape == airmass.shape): + base_result['note'] = 'light-curve arrays could not be aligned for out-of-transit baseline fitting.' + return base_result + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors, + depth_fraction=depth_fraction, + ) + if not coverage_summary.get('valid') and prior: + prior_coverage = summarize_prior_transit_coverage( + times, + prior, + flux_values=flux_values, + flux_errors=flux_errors, + ) + if prior_coverage.get('valid'): + coverage_summary = prior_coverage + base_result['used_prior_ephemeris'] = True + + if not coverage_summary.get('valid'): + base_result['note'] = coverage_summary.get( + 'note', + 'could not isolate out-of-transit points for baseline parameter fitting.', + ) + return base_result + + oot_mask = np.asarray(coverage_summary.get('oot_mask'), dtype=bool) + finite_mask = ( + oot_mask + & np.isfinite(times) + & np.isfinite(flux_values) + & (flux_values > 0) + & np.isfinite(flux_errors) + & (flux_errors > 0) + & np.isfinite(airmass) + ) + point_count = int(np.count_nonzero(finite_mask)) + fit_a2 = 'a2' in bounds + min_points = 3 if fit_a2 else 2 + base_result['pre_points'] = coverage_summary.get('pre_points', 0) + base_result['post_points'] = coverage_summary.get('post_points', 0) + + if point_count < min_points: + base_result['note'] = ( + f"only {point_count} finite out-of-transit point(s) were available; " + f"need at least {min_points} to fit baseline parameters." + ) + return base_result + + reference_airmass = transit_qc_airmass_reference(airmass) + x = airmass[finite_mask] - reference_airmass + y = flux_values[finite_mask] + yerr = flux_errors[finite_mask] + + a0_bounds = bounds.get('a0') or bounds.get('a1') or [0.5, 1.5] + try: + a0_lower, a0_upper = np.asarray(a0_bounds, dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + a0_lower, a0_upper = 0.5, 1.5 + if not np.isfinite(a0_lower) or a0_lower <= 0: + a0_lower = max(np.nanmedian(y) * 0.5, np.finfo(float).eps) + if not np.isfinite(a0_upper) or a0_upper <= a0_lower: + a0_upper = max(np.nanmedian(y) * 1.5, a0_lower * 1.01) + + if fit_a2: + try: + a2_lower, a2_upper = np.asarray(bounds.get('a2'), dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + a2_lower, a2_upper = -3.0, 3.0 + if not np.isfinite(a2_lower) or not np.isfinite(a2_upper) or a2_lower >= a2_upper: + a2_lower, a2_upper = -3.0, 3.0 + else: + a2_lower = a2_upper = float(prior.get('a2', 0.0) or 0.0) + + initial_a0 = float(np.clip(np.nanmedian(y), a0_lower, a0_upper)) + initial_a2 = float(prior.get('a2', 0.0) or 0.0) + if fit_a2: + initial_a2 = float(np.clip(initial_a2, a2_lower, a2_upper)) + + if fit_a2: + initial = np.array([np.log(initial_a0), initial_a2], dtype=float) + lower_bounds = np.array([np.log(a0_lower), a2_lower], dtype=float) + upper_bounds = np.array([np.log(a0_upper), a2_upper], dtype=float) + else: + initial = np.array([np.log(initial_a0)], dtype=float) + lower_bounds = np.array([np.log(a0_lower)], dtype=float) + upper_bounds = np.array([np.log(a0_upper)], dtype=float) + + def residuals(params): + log_a0 = params[0] + a2_value = params[1] if fit_a2 else initial_a2 + model = np.exp(log_a0) * np.exp(a2_value * x) + return (y - model) / yerr + + try: + result = least_squares( + residuals, + x0=initial, + bounds=(lower_bounds, upper_bounds), + jac='3-point', + loss='linear', + ) + except (ValueError, np.linalg.LinAlgError): + base_result['note'] = 'weighted out-of-transit baseline parameter fit failed.' + return base_result + + if not getattr(result, 'success', False) or not np.all(np.isfinite(result.x)): + base_result['note'] = 'weighted out-of-transit baseline parameter fit did not converge.' + return base_result + + log_a0 = float(result.x[0]) + a0 = float(np.exp(log_a0)) + a2 = float(result.x[1] if fit_a2 else initial_a2) + jacobian = np.asarray(result.jac, dtype=float) + residual_vector = np.asarray(result.fun, dtype=float) + dof = max(1, residual_vector.size - result.x.size) + reduced_chi2 = np.sum(residual_vector ** 2) / dof + covariance = None + if jacobian.ndim == 2 and jacobian.shape[0] >= jacobian.shape[1]: + try: + covariance = np.linalg.pinv(jacobian.T @ jacobian) + covariance *= max(float(reduced_chi2), 1.0) + except np.linalg.LinAlgError: + covariance = None + + if covariance is not None and covariance.shape[0] >= 1: + log_a0_error = float(np.sqrt(max(covariance[0, 0], 0.0))) + a0_error = abs(a0) * log_a0_error + else: + a0_error = np.nan + if covariance is not None and fit_a2 and covariance.shape[0] >= 2: + a2_error = float(np.sqrt(max(covariance[1, 1], 0.0))) + else: + a2_error = 0.0 if not fit_a2 else np.nan + + if not np.isfinite(a0_error) or a0_error <= 0: + a0_error = float(np.nanmedian(yerr)) + if fit_a2 and (not np.isfinite(a2_error) or a2_error <= 0): + airmass_span_value = np.nanmax(x) - np.nanmin(x) + if np.isfinite(airmass_span_value) and airmass_span_value > 0: + a2_error = float(np.nanmedian(yerr / np.maximum(y, np.finfo(float).eps)) / airmass_span_value) + else: + a2_error = 0.0 + + side_note = ( + f"{coverage_summary.get('pre_points', 0)} pre-ingress and " + f"{coverage_summary.get('post_points', 0)} post-egress out-of-transit point(s)" + ) + if base_result['used_prior_ephemeris']: + side_note += " from the ephemeris-centered transit window" + + return { + 'applied': True, + 'note': ( + "Fitted a0" + + (" and a2" if fit_a2 else "") + + f" using only {side_note}; these baseline terms are fixed/profiled in the final transit fit." + ), + 'oot_mask': finite_mask, + 'pre_points': coverage_summary.get('pre_points', 0), + 'post_points': coverage_summary.get('post_points', 0), + 'a0': a0, + 'a0_error': float(a0_error), + 'a2': a2, + 'a2_error': float(a2_error), + 'used_prior_ephemeris': base_result['used_prior_ephemeris'], + } + + def detrend_flux_on_out_of_transit_baseline( times, flux_values, @@ -5639,15 +5967,24 @@ def fit_final_lightcurve_with_oot_baseline_detrending( expected_tmid_search_summary=None, eebls_search_summary=None, duration_prior=None, + extend_sparse_posterior_live_points=True, + keep_ultranest_sampler_for_deferred_extension=False, ): if duration_prior is None and expected_planet_dict is not None: duration_prior = build_single_transit_duration_prior(expected_planet_dict) - keep_ultranest_for_sparse_extension = should_use_sparse_posterior_live_point_retry( + sparse_posterior_live_point_extension_enabled = should_use_sparse_posterior_live_point_retry( os.environ.get( SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV, SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, ) ) + keep_ultranest_for_sparse_extension = ( + sparse_posterior_live_point_extension_enabled + and ( + extend_sparse_posterior_live_points + or keep_ultranest_sampler_for_deferred_extension + ) + ) fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( times, @@ -5736,17 +6073,117 @@ def fit_final_lightcurve_with_oot_baseline_detrending( refined_tmid_bounds=prefit_plan.get('refined_tmid_bounds'), ) + if detrend_on_outoftransit_baseline: + baseline_parameter_result = fit_airmass_baseline_parameters_on_out_of_transit( + working_times, + working_flux, + working_unc, + working_airmass, + fit, + prior=working_prior, + bounds=working_bounds, + ) + else: + baseline_parameter_result = { + 'applied': False, + 'note': 'Disabled with out-of-transit baseline detrending.', + 'pre_points': 0, + 'post_points': 0, + } + baseline_fit_mask = None + baseline_fixed_errors = {} + baseline_constrained_prior = dict(working_prior) + baseline_constrained_bounds = clone_lightcurve_bounds(working_bounds) + if baseline_parameter_result.get('applied'): + log_info("Prepared out-of-transit airmass/baseline parameter constraints for the final transit refit.") + log_info(baseline_parameter_result['note']) + baseline_fit_mask = np.asarray(baseline_parameter_result['oot_mask'], dtype=bool) + baseline_fixed_errors = { + 'a2': baseline_parameter_result.get('a2_error', 0.0), + } + baseline_constrained_prior['a0'] = baseline_parameter_result['a0'] + baseline_constrained_prior['a1'] = baseline_parameter_result['a0'] + baseline_constrained_prior['a2'] = baseline_parameter_result['a2'] + for key in ('rprs', 'ars', 'tmid', 'inc'): + if key in baseline_constrained_prior and key in getattr(fit, 'parameters', {}): + baseline_constrained_prior[key] = fit.parameters[key] + baseline_constrained_bounds.pop('a0', None) + baseline_constrained_bounds.pop('a1', None) + baseline_constrained_bounds.pop('a2', None) + else: + annotate_out_of_transit_baseline_parameter_fit( + fit, + False, + note=baseline_parameter_result.get('note'), + pre_points=baseline_parameter_result.get('pre_points', 0), + post_points=baseline_parameter_result.get('post_points', 0), + ) + + def run_oot_baseline_parameter_refit_if_needed(current_fit): + if not baseline_parameter_result.get('applied'): + return current_fit + + refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + working_times, + working_flux, + working_unc, + working_airmass, + baseline_constrained_prior, + baseline_constrained_bounds, + jd_times=working_jd_times, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, + keep_ultranest_sampler=keep_ultranest_for_sparse_extension, + baseline_fit_mask=baseline_fit_mask, + fixed_parameter_errors=baseline_fixed_errors, + ) + refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_transit_qc_fit_context( + refit, + planet_dict=expected_planet_dict, + tmid_search_summary=expected_tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) + annotate_final_fit_prefit_refinement( + refit, + prefit_plan.get('applied', False), + note=prefit_plan.get('note'), + baseline_duration_multiplier=baseline_duration_multiplier, + duration=prefit_plan.get('duration'), + original_point_count=prefit_plan.get('original_point_count'), + refined_point_count=prefit_plan.get('refined_point_count'), + trimmed_pre_points=prefit_plan.get('trimmed_pre_points', 0), + trimmed_post_points=prefit_plan.get('trimmed_post_points', 0), + original_tmid_bounds=prefit_plan.get('original_tmid_bounds'), + refined_tmid_bounds=prefit_plan.get('refined_tmid_bounds'), + ) + annotate_out_of_transit_baseline_parameter_fit( + refit, + True, + note=baseline_parameter_result.get('note'), + pre_points=baseline_parameter_result.get('pre_points', 0), + post_points=baseline_parameter_result.get('post_points', 0), + a0=baseline_parameter_result.get('a0'), + a0_error=baseline_parameter_result.get('a0_error'), + a2=baseline_parameter_result.get('a2'), + a2_error=baseline_parameter_result.get('a2_error'), + ) + return refit + if not detrend_on_outoftransit_baseline: + fit = run_oot_baseline_parameter_refit_if_needed(fit) annotate_out_of_transit_baseline_detrending( fit, False, note="Disabled; using the direct nested-sampling fit.", ) annotate_transit_detection_qc(fit) - fit = extend_sparse_posterior_live_points_if_needed( - fit, - enabled=keep_ultranest_for_sparse_extension, - ) + if extend_sparse_posterior_live_points: + fit = extend_sparse_posterior_live_points_if_needed( + fit, + enabled=sparse_posterior_live_point_extension_enabled, + ) annotate_transit_detection_qc(fit) return fit, working_flux, working_unc @@ -5760,6 +6197,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( if not detrend_result.get('applied'): note = f"Skipped; {detrend_result.get('note', 'unable to fit an out-of-transit baseline.')}" log_info(f"Optional out-of-transit baseline detrending skipped: {detrend_result.get('note', 'unknown reason')}") + fit = run_oot_baseline_parameter_refit_if_needed(fit) annotate_out_of_transit_baseline_detrending( fit, False, @@ -5768,10 +6206,11 @@ def fit_final_lightcurve_with_oot_baseline_detrending( post_points=detrend_result.get('post_points', 0), ) annotate_transit_detection_qc(fit) - fit = extend_sparse_posterior_live_points_if_needed( - fit, - enabled=keep_ultranest_for_sparse_extension, - ) + if extend_sparse_posterior_live_points: + fit = extend_sparse_posterior_live_points_if_needed( + fit, + enabled=sparse_posterior_live_point_extension_enabled, + ) annotate_transit_detection_qc(fit) return fit, working_flux, working_unc @@ -5790,6 +6229,13 @@ def fit_final_lightcurve_with_oot_baseline_detrending( detrend_result['flux'], disable_vertical_flux_normalization, ) + if baseline_parameter_result.get('applied'): + refit_prior['a0'] = baseline_parameter_result['a0'] + refit_prior['a1'] = baseline_parameter_result['a0'] + refit_prior['a2'] = baseline_parameter_result['a2'] + refit_bounds.pop('a0', None) + refit_bounds.pop('a1', None) + refit_bounds.pop('a2', None) refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( working_times, @@ -5802,6 +6248,8 @@ def fit_final_lightcurve_with_oot_baseline_detrending( use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, + baseline_fit_mask=baseline_fit_mask, + fixed_parameter_errors=baseline_fixed_errors, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -5833,12 +6281,24 @@ def fit_final_lightcurve_with_oot_baseline_detrending( pre_points=detrend_result['pre_points'], post_points=detrend_result['post_points'], ) - annotate_transit_detection_qc(refit) - refit = extend_sparse_posterior_live_points_if_needed( + annotate_out_of_transit_baseline_parameter_fit( refit, - enabled=keep_ultranest_for_sparse_extension, + bool(baseline_parameter_result.get('applied')), + note=baseline_parameter_result.get('note'), + pre_points=baseline_parameter_result.get('pre_points', 0), + post_points=baseline_parameter_result.get('post_points', 0), + a0=baseline_parameter_result.get('a0'), + a0_error=baseline_parameter_result.get('a0_error'), + a2=baseline_parameter_result.get('a2'), + a2_error=baseline_parameter_result.get('a2_error'), ) annotate_transit_detection_qc(refit) + if extend_sparse_posterior_live_points: + refit = extend_sparse_posterior_live_points_if_needed( + refit, + enabled=sparse_posterior_live_point_extension_enabled, + ) + annotate_transit_detection_qc(refit) return refit, detrend_result['flux'], detrend_result['unc'] @@ -11606,6 +12066,10 @@ def summarize_lightcurve_fit_assessment(fit): ), 'prefit_refinement_applied': bool(getattr(fit, 'prefit_refinement_applied', False)), 'prefit_refinement_note': getattr(fit, 'prefit_refinement_note', None), + 'oot_baseline_parameter_fit_applied': bool( + getattr(fit, 'oot_baseline_parameter_fit_applied', False) + ), + 'oot_baseline_parameter_fit_note': getattr(fit, 'oot_baseline_parameter_fit_note', None), 'oot_baseline_detrending_applied': bool(getattr(fit, 'oot_baseline_detrending_applied', False)), 'oot_baseline_detrending_note': getattr(fit, 'oot_baseline_detrending_note', None), 'airmass_fit_skipped': bool(getattr(fit, 'airmass_fit_skipped', False)), @@ -11648,6 +12112,7 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): "applied" ) prefit_status = "applied" if assessment['prefit_refinement_applied'] else "not applied" + oot_parameter_status = "applied" if assessment['oot_baseline_parameter_fit_applied'] else "not applied" oot_status = "applied" if assessment['oot_baseline_detrending_applied'] else "not applied" duration_prior_status = "applied" if assessment['duration_prior_applied'] else "not applied" sparse_extension_status = ( @@ -11663,6 +12128,7 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): f"impact parameter posterior retry={b_retry_status}, " f"sparse posterior extension={sparse_extension_status}, " f"prefit_refinement={prefit_status}, " + f"oot_baseline_parameter_fit={oot_parameter_status}, " f"oot_baseline_detrending={oot_status}" ) if assessment.get('duration_prior_note'): @@ -11678,6 +12144,8 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): ) if assessment.get('prefit_refinement_note'): log_info(f"{indent}Prefit refinement note: {assessment['prefit_refinement_note']}") + if assessment.get('oot_baseline_parameter_fit_note'): + log_info(f"{indent}OOT baseline parameter-fit note: {assessment['oot_baseline_parameter_fit_note']}") if assessment.get('oot_baseline_detrending_note'): log_info(f"{indent}OOT baseline detrending note: {assessment['oot_baseline_detrending_note']}") if assessment.get('airmass_correction_note'): @@ -13508,6 +13976,57 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p f"{format_transit_delta_bic(selected_transit_delta_bic)}" ) + selected_fit = selected_result.get('fit') + if selected_fit is not None: + selected_result['fit'] = extend_selected_comparison_live_points_if_needed(selected_fit) + selected_result['full_reduction_fit'] = selected_result['fit'] + if getattr(selected_result['fit'], 'sparse_posterior_live_point_extension_applied', False): + annotate_transit_detection_qc(selected_result['fit']) + selected_result['eebls_snr'] = extract_lightcurve_fit_eebls_snr(selected_result['fit']) + selected_result['transit_delta_bic'] = extract_lightcurve_fit_transit_delta_bic(selected_result['fit']) + selected_result['residual_scatter'] = extract_lightcurve_fit_residual_scatter(selected_result['fit']) + selected_result['ktmf_metric'] = extract_lightcurve_fit_ktmf_metric(selected_result['fit']) + selected_result['ktmf_contributions'] = extract_lightcurve_fit_ktmf_contributions(selected_result['fit']) + selected_result['parameter_summary'] = summarize_lightcurve_fit_parameters(selected_result['fit']) + selected_result['transit_qc_status'] = getattr(selected_result['fit'], 'transit_qc_status', None) + selected_result['transit_qc_summary'] = getattr(selected_result['fit'], 'transit_qc_summary', None) + data_highres, duration_samples = estimate_transit_duration_samples_from_fit(selected_result['fit']) + selected_result['data_highres'] = data_highres + selected_result['duration_samples'] = duration_samples + if save_dir is not None: + final_output_dir = save_comparison_candidate_full_reduction_outputs( + save_dir, + None, + selected_result['fit'], + p_dict, + observation_date, + selected_result['comp_index'], + comp_coords=selected_result.get('position'), + min_aperture=(0 if comparison_calibration['method'] == 'psf' else comparison_calibration.get('aper')), + min_annulus=comparison_calibration.get('annulus'), + adaptive_summary=adaptive_summary, + method_label=comparison_calibration.get('method_label'), + selection_summary={ + 'ktmf_metric': selected_result.get('ktmf_metric', np.nan), + 'transit_delta_bic': selected_result.get('transit_delta_bic', np.nan), + 'eebls_snr': selected_result.get('eebls_snr', np.nan), + 'transit_qc_status': selected_result.get('transit_qc_status'), + 'transit_qc_summary': selected_result.get('transit_qc_summary'), + }, + duration_samples=selected_result.get('duration_samples'), + data_highres=selected_result.get('data_highres'), + ) + if final_output_dir is not None: + selected_result['final_output_dir'] = str(final_output_dir) + + for attempt in attempts: + if selected_result is not None and attempt is selected_result: + continue + clear_fit_ultranest_resume_state(attempt.get('fit')) + full_reduction_fit = attempt.get('full_reduction_fit') + if full_reduction_fit is not attempt.get('fit'): + clear_fit_ultranest_resume_state(full_reduction_fit) + return { 'ranked_summaries': ranked_summaries, 'attempts': attempts, @@ -13756,9 +14275,10 @@ def _main_impl(): ) if use_sparse_posterior_live_point_retry: log_info( - "Sparse posterior live-point extension enabled: final settled fits can continue " - f"UltraNest with {SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT}x additional " - "minimum live points when Rp/R*, Tmid, or a/Rs are under-sampled." + "Selected comparison-star live-point extension enabled: comparison candidates " + "are ranked at the configured UltraNest live-point count, then the chosen final " + f"comparison fit continues with {SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT}x " + "additional minimum live points using its retained final-pass bounds." ) log_ultranest_mpi_status() @@ -15546,6 +16066,12 @@ def _main_impl(): expected_tmid_search_summary=ephemeris_tmid_search_summary, eebls_search_summary=eebls_tmid_search_summary, ) + if ( + reuse_selected_final_model + and getattr(myfit, 'sparse_posterior_live_point_extension_note', None) is None + ): + myfit = extend_selected_comparison_live_points_if_needed(myfit) + annotate_transit_detection_qc(myfit) # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 9ce21f6a..461d85fb 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -424,7 +424,7 @@ def save_input(): "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", - "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to continue the final settled UltraNest fit with 5x additional minimum live points when Rp/Rs, Tmid, or a/Rs posteriors are too sparse after posterior truncation checks and out-of-transit baseline handling. Set to n to disable. Default y.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", @@ -1506,7 +1506,7 @@ def save_input(): "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", - "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to continue the final settled UltraNest fit with 5x additional minimum live points when Rp/Rs, Tmid, or a/Rs posteriors are too sparse after posterior truncation checks and out-of-transit baseline handling. Set to n to disable. Default y.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", diff --git a/exotic/output_files.py b/exotic/output_files.py index 9d9e784c..40a31aa2 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -879,6 +879,9 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor prefit_refinement_note = getattr(self.fit, 'prefit_refinement_note', None) if prefit_refinement_note: params_num["Prefit refinement note"] = str(prefit_refinement_note) + oot_baseline_parameter_note = getattr(self.fit, 'oot_baseline_parameter_fit_note', None) + if oot_baseline_parameter_note: + params_num["Out-of-transit baseline parameter-fit note"] = str(oot_baseline_parameter_note) oot_baseline_note = getattr(self.fit, 'oot_baseline_detrending_note', None) if oot_baseline_note: params_num["Out-of-transit baseline detrending note"] = str(oot_baseline_note) diff --git a/exotic/plots.py b/exotic/plots.py index 101df93d..9641374e 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -322,7 +322,11 @@ def plot_comp_star_candidate_lightcurve_fits(candidate_fit_summaries, targ_name, if fit is None: continue - fig, (ax_lc, ax_res) = fit.plot_bestfit(phase=False) + fig, (ax_lc, ax_res) = _plot_bestfit_for_lightcurve_png( + fit, + phase=False, + show_flux_baseline_label=False, + ) selected_text = " selected" if summary.get('selected') else "" res_std = summary.get('res_std', np.nan) res_std_text = "n/a" if not np.isfinite(res_std) else f"{res_std * 100.0:.3f}%" @@ -347,6 +351,29 @@ def plot_comp_star_candidate_lightcurve_fits(candidate_fit_summaries, targ_name, plt.close(fig) +def _callable_accepts_keyword(callable_object, keyword): + try: + signature = inspect.signature(callable_object) + except (TypeError, ValueError): + return False + if keyword in signature.parameters: + return True + return any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + + +def _plot_bestfit_for_lightcurve_png(fit, **requested_kwargs): + plotter = fit.plot_bestfit + plot_kwargs = { + key: value + for key, value in requested_kwargs.items() + if _callable_accepts_keyword(plotter, key) + } + return plotter(**plot_kwargs) + + def _draw_comp_star_calibration_axis(axis, times, summary, colors): axis.axhline(1.0, color='lightgray', lw=1.0, zorder=1) pairwise_series = summary.get('pairwise_ratio_series', {}) @@ -603,17 +630,11 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relati # Plotting Final Lightcurve def plot_final_lightcurve(fit, high_res, targ_name, save, date): - plot_kwargs = {} - try: - plot_parameters = inspect.signature(fit.plot_bestfit).parameters - except (TypeError, ValueError): - plot_parameters = {} - if 'show_flux_baseline_label' in plot_parameters: - plot_kwargs['show_flux_baseline_label'] = False - if 'show_model_uncertainty' in plot_parameters: - plot_kwargs['show_model_uncertainty'] = True - - f, (ax_lc, ax_res) = fit.plot_bestfit(**plot_kwargs) + f, (ax_lc, ax_res) = _plot_bestfit_for_lightcurve_png( + fit, + show_flux_baseline_label=False, + show_model_uncertainty=True, + ) ax_lc.set_title(targ_name) if hasattr(fit, 'phase_upsample') and hasattr(fit, 'transit_upsample'): diff --git a/inits.json b/inits.json index f6abf98f..7c5ad3ca 100644 --- a/inits.json +++ b/inits.json @@ -37,7 +37,7 @@ "Assess All Comparisons Before Selecting Best": "Set optional_info 'assess_all_comparisons_before_selecting_best' to y to fit every comparison-star candidate that survives the earlier screening, report all fits, and select the candidate with the highest KTMF score. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "UltraNest Live Points": "Set optional_info 'minimum number of live points for ultranest' to a positive integer to control UltraNest's min_num_live_points. Default 200.", - "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to continue the final settled UltraNest fit with 5x additional minimum live points when Rp/Rs, Tmid, or a/Rs posteriors are too sparse after posterior truncation checks and out-of-transit baseline handling. Set to n to disable. Default y.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 296a5885..546942c2 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -214,6 +214,38 @@ def test_lc_fitter_auto_solves_mean_airmass_normalization(monkeypatch, tmp_path) assert fit.parameters["a1"] == pytest.approx(true_a0, abs=1e-4) +def test_create_fit_variables_solves_baseline_from_out_of_transit_mask(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 301) + dataerr = np.full_like(time, 1e-3) + airmass = np.zeros_like(time) + transit_model = elca.transit(time, prior) + data = 1.02 * transit_model + in_transit = np.abs(time - prior["tmid"]) < 0.012 + data[in_transit] *= 0.90 + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = data + fit.dataerr = dataerr + fit.airmass = airmass + fit.airmass_reference = elca.get_airmass_reference(airmass) + fit.prior = prior.copy() + fit.bounds = {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005]} + fit.mode = "ns" + fit.parameters = prior.copy() + fit.errors = {"rprs": 0.0, "tmid": 0.0, "a2": 0.0} + fit.quantiles = {} + fit.baseline_fit_mask = ~in_transit + fit.fixed_parameter_errors = {} + + fit.create_fit_variables() + + assert fit.parameters["a0"] == pytest.approx(1.02, abs=1e-5) + assert fit.parameters["a1"] == pytest.approx(1.02, abs=1e-5) + + def test_lc_fitter_rejects_redundant_a0_and_a1_bounds(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -366,13 +398,80 @@ def test_plot_bestfit_can_draw_transit_model_uncertainty_band(monkeypatch, tmp_p fig, axes = fit.plot_bestfit(show_model_uncertainty=True) labels = [artist.get_label() for artist in axes[0].collections] + uncertainty_line_count = sum(1 for line in axes[0].lines if line.get_linestyle() == "--") assert envelope is not None assert np.nanmax(envelope[1] - envelope[0]) > 0 assert r'1-$\sigma$ model uncertainty' in labels + assert uncertainty_line_count >= 2 plt.close(fig) +def test_transit_model_uncertainty_includes_baseline_terms(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = elca.transit(time, prior) + fit.dataerr = np.full_like(time, 1e-3) + fit.airmass = np.linspace(1.0, 1.5, time.size) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.prior = prior.copy() + fit.bounds = {} + fit.mode = "ns" + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.1 + fit.errors = {"a0": 0.01, "a1": 0.01, "a2": 0.05} + fit.quantiles = {} + fit.results = None + + envelope = fit.transit_model_uncertainty(time) + + assert envelope is not None + assert np.nanmax(envelope[1] - envelope[0]) > 0 + + +def test_posterior_model_uncertainty_recenters_on_best_fit_model(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = elca.transit(time, prior) + fit.dataerr = np.full_like(time, 1e-3) + fit.airmass = np.zeros_like(time) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.prior = prior.copy() + fit.bounds = {"a0": [0.99, 1.03]} + fit.sampled_keys = ["a0"] + fit.sample_bounds = {"a0": [0.99, 1.03]} + fit.mode = "ns" + fit.ns_type = "ultranest" + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.errors = {"a0": 0.002} + fit.quantiles = {} + fit.results = { + "weighted_samples": { + "points": np.linspace(1.008, 1.012, 41)[:, None], + "logl": np.zeros(41, dtype=float), + "weights": np.ones(41, dtype=float), + } + } + + lower, upper = fit.transit_model_uncertainty(time) + center = 0.5 * (lower + upper) + + np.testing.assert_allclose(center, elca.transit(time, fit.parameters), atol=5e-5) + assert np.nanmedian(center[:3]) < 1.001 + + def test_glc_plot_bestfit_median_limits_use_full_phase_span(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -1080,6 +1179,76 @@ def test_triangle_contour_levels_drop_duplicate_chi2_percentiles(monkeypatch, tm assert levels == [pytest.approx(42.0)] +def test_triangle_plot_sigma_window_ranges_clip_to_solved_point_uncertainties(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + payload = { + "ranges": [[0.0, 1.0], [0.0, 1.2], [0.95, 1.05]], + "mask_centers": [0.20, 0.80, 1.0], + "mask_errors": [0.02, 0.05, 0.001], + "display_points": np.array( + [ + [0.18, 0.75, 0.999], + [0.20, 0.80, 1.000], + [0.22, 0.85, 1.001], + ] + ), + } + + ranges = fit._triangle_plot_sigma_window_ranges(payload, sigma=5.0) + + assert ranges[0] == pytest.approx([0.10, 0.30]) + assert ranges[1] == pytest.approx([0.55, 1.05]) + assert ranges[2] == pytest.approx([0.995, 1.005]) + + +def test_plot_triangle_accepts_zoom_sigma(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + captured = {} + + def fake_corner(*args, **kwargs): + captured["range"] = kwargs["range"] + return "figure" + + monkeypatch.setattr(elca, "corner", fake_corner) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 1.0], + "inc": [84.0, 90.0], + } + fit.sample_bounds = { + "rprs": [0.0, 1.0], + "b": [0.0, 1.2], + } + fit.sampled_keys = ["rprs", "b"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.20, "inc": 86.0} + fit.errors = {"rprs": 0.02, "inc": 0.5} + fit.sample_parameters = {"rprs": 0.20, "b": 0.80} + fit.sample_errors = {"rprs": 0.02, "b": 0.05} + points = np.column_stack([ + np.linspace(0.18, 0.22, 40), + np.linspace(0.75, 0.85, 40), + ]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + fig = fit.plot_triangle(zoom_sigma=5.0) + + assert fig == "figure" + assert captured["range"][0][0] > 0.0 + assert captured["range"][0][1] < 1.0 + assert captured["range"][1][0] > 0.0 + assert captured["range"][1][1] < 1.2 + + def test_triangle_payload_expands_degenerate_error_ranges_to_sample_cloud(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) @@ -1118,7 +1287,7 @@ def test_triangle_payload_expands_degenerate_error_ranges_to_sample_cloud(monkey assert payload["ranges"][1][1] >= 0.0038 -def test_triangle_payload_titles_match_reported_parameters(monkeypatch, tmp_path): +def test_triangle_payload_titles_follow_weighted_posterior_display_estimate(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) @@ -1154,8 +1323,8 @@ def test_triangle_payload_titles_match_reported_parameters(monkeypatch, tmp_path payload = fit._get_triangle_plot_payload() - assert payload["titles"][0] == "0.338 +/- 0.091" - assert payload["truths"][0] == pytest.approx(0.33796) + assert payload["titles"][0] == "0.119 +/- 0.0089" + assert payload["truths"][0] == pytest.approx(0.1186, abs=5e-4) np.testing.assert_allclose(payload["display_weights"], weights) @@ -1202,7 +1371,7 @@ def fake_corner(*args, **kwargs): assert fig == "figure" np.testing.assert_allclose(captured["weights"], weights) - np.testing.assert_allclose(captured["truths"], [0.1, 1.0]) + np.testing.assert_allclose(captured["truths"], [0.1018961, 1.00037922], rtol=1e-6) assert captured["data_kwargs"]["s"] == pytest.approx(1.6) assert captured["data_kwargs"]["alpha"] == pytest.approx(0.38) @@ -1287,6 +1456,48 @@ def test_triangle_payload_uses_tested_rprs_range_when_posterior_is_narrow(monkey assert upper_fraction < elca.TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX +def test_triangle_payload_expands_rprs_lower_edge_past_narrow_recorded_sample_bounds(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.36], + "a0": [0.95, 1.05], + } + fit.sample_bounds = { + "rprs": [0.104, 0.184], + "a0": [0.95, 1.05], + } + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.144, "a0": 1.0} + fit.errors = {"rprs": 0.008, "a0": 0.001} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + rprs_samples = np.concatenate([ + np.linspace(0.104, 0.120, 80), + np.linspace(0.120, 0.180, 20), + ]) + points = np.column_stack([ + rprs_samples, + np.linspace(0.998, 1.002, rprs_samples.size), + ]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["ranges"][0][0] < 0.08 + assert payload["truths"][0] < 0.13 + assert not payload["titles"][0].startswith("0.144") + + def test_triangle_payload_keeps_direct_impact_parameter_full_sample_range(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 2fa68a5e..5f91428c 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -161,6 +161,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: should_use_fast_target_centroid, should_use_deviation_from_expected_transit_in_qc, update_coordinates_with_proper_motion, + zoomed_final_triangle_plot_output_path, ) @@ -224,6 +225,55 @@ def test_comparison_candidate_triangle_plot_uses_date_only_from_timestamp(tmp_pa assert output_path.name == "Comp7_Triangle_XO-1-b_2026-05-06.png" +def test_zoomed_final_triangle_plot_uses_named_artifact(tmp_path): + output_path = zoomed_final_triangle_plot_output_path( + tmp_path / "final", + "XO-1/b", + "2026-05-06T19:51:13.964-0700", + ) + + assert output_path == tmp_path / "final" / "ZoomedTrianglePlot_XO-1-b_2026-05-06.png" + + +def test_save_final_triangle_plot_creates_zoomed_companion_when_supported(tmp_path): + class DummyFigure: + def __init__(self, content): + self.content = content + + def savefig(self, path): + Path(path).write_bytes(self.content) + + class DummyFit: + def __init__(self): + self.calls = [] + + def plot_triangle(self, plot_title=None, zoom_sigma=None): + self.calls.append({"plot_title": plot_title, "zoom_sigma": zoom_sigma}) + content = b"zoomed" if zoom_sigma == 5.0 else b"full" + return DummyFigure(content) + + fit = DummyFit() + output_path = save_final_triangle_plot( + fit, + tmp_path / "final", + "TOI-1728 b", + "2024-12-14", + source_dir=tmp_path / "comp4", + ) + + zoomed_output_path = zoomed_final_triangle_plot_output_path( + tmp_path / "final", + "TOI-1728 b", + "2024-12-14", + ) + assert output_path.read_bytes() == b"full" + assert zoomed_output_path.read_bytes() == b"zoomed" + assert fit.calls == [ + {"plot_title": "Final selected fit (comparison candidate #4)", "zoom_sigma": None}, + {"plot_title": "Final selected fit (comparison candidate #4) (5-sigma zoom)", "zoom_sigma": 5.0}, + ] + + def test_save_final_triangle_plot_regenerates_when_selected_artifact_missing(tmp_path): class DummyFigure: def savefig(self, path): @@ -1785,6 +1835,80 @@ def fake_lc_fitter( assert fit.oot_baseline_post_points == 3 +def test_fit_final_lightcurve_uses_oot_baseline_parameter_refit_when_linear_detrend_skips(monkeypatch): + import exotic.exotic as exotic_module + + times = np.array([-0.03, -0.02, -0.01, 0.00, 0.01, 0.02, 0.03]) + transit_profile = np.array([0.99, 0.99, 0.99, 0.99, 1.0, 1.0, 1.0]) + airmass = np.linspace(1.0, 1.6, times.size) + flux = np.exp(0.2 * (airmass - np.mean(airmass))) * transit_profile + fluxerr = np.full_like(times, 0.01) + prior = {"rprs": 0.1, "tmid": -0.01, "inc": 89.0, "a2": 0.0} + bounds = { + "rprs": [0.0, 0.2], + "tmid": [-0.03, 0.01], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + } + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + baseline_fit_mask=None, + fixed_parameter_errors=None, + ): + captured["calls"].append({ + "bounds": dict(call_bounds), + "baseline_fit_mask": None if baseline_fit_mask is None else np.asarray(baseline_fit_mask, dtype=bool), + "fixed_parameter_errors": dict(fixed_parameter_errors or {}), + "prior": dict(call_prior), + }) + return types.SimpleNamespace( + transit=transit_profile.copy(), + parameters={ + "tmid": -0.01, + "rprs": 0.1, + "inc": 89.0, + "a2": call_prior.get("a2", 0.0), + "a0": call_prior.get("a0", 1.0), + "a1": call_prior.get("a0", 1.0), + }, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01, "a0": 0.001, "a1": 0.001}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=True, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["baseline_fit_mask"] is None + assert captured["calls"][1]["baseline_fit_mask"].tolist() == [False, False, False, False, True, True, True] + assert "a0" not in captured["calls"][1]["bounds"] + assert "a2" not in captured["calls"][1]["bounds"] + assert "a2" in captured["calls"][1]["fixed_parameter_errors"] + assert fit.oot_baseline_parameter_fit_applied is True + assert fit.oot_baseline_detrending_applied is False + + def test_phase_bin_sigma_clip_flags_local_phase_outlier(): phase_centers = np.linspace(-0.045, 0.045, 10) phase = np.concatenate([center + np.linspace(-1e-4, 1e-4, 5) for center in phase_centers]) @@ -3089,6 +3213,144 @@ def fake_finalize( assert result["attempts"][0]["selection_reason"].startswith("not selected: KTMF") +def test_fit_ranked_comparison_calibration_candidates_extends_only_selected_final_fit(monkeypatch): + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "200") + monkeypatch.setenv("EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY", "1") + + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + created_fits = {} + + class RetainedSamplerFit: + def __init__(self, comp_marker, ktmf_metric): + self.comp_marker = comp_marker + self.extension_calls = [] + self.cleared = False + self.max_ncalls = 1000 + self.time = np.linspace(0.0, 0.05, 6) + self.data = np.ones(6, dtype=float) + self.residuals = np.full(6, 0.01, dtype=float) + self.parameters = { + "tmid": 0.5, + "rprs": 0.1, + "inc": 89.0, + "ars": 10.0, + "a0": 1.0, + "a2": 0.0, + } + self.errors = { + "tmid": 0.001, + "rprs": 0.001, + "inc": 0.1, + "ars": 0.1, + "a0": 0.01, + "a2": 0.01, + } + self.bounds = { + "rprs": [0.08, 0.12], + "tmid": [0.49, 0.51], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + "a2": [-3.0, 3.0], + } + self.transit_qc_delta_bic = 12.0 + comp_marker / 100.0 + self.transit_qc_ktmf_metric = ktmf_metric + + def get_parameter_posterior_samples(self, key): + ranges = { + "rprs": (0.09, 0.11), + "tmid": (0.499, 0.501), + "ars": (9.5, 10.5), + } + low, high = ranges[key] + return np.linspace(low, high, 1500) + + def extend_ultranest_fit(self, min_num_live_points=None, max_ncalls=None): + self.extension_calls.append({ + "min_num_live_points": min_num_live_points, + "max_ncalls": max_ncalls, + "bounds": self.bounds.copy(), + }) + return True + + def clear_ultranest_resume_state(self): + self.cleared = True + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + ktmf_map = {50: 2.40, 40: 4.70, 30: 3.90} + fit = RetainedSamplerFit(comp_marker, ktmf_map[comp_marker]) + created_fits[comp_marker] = fit + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + "comp3": np.full((6, 1, 1), 30.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + {"label": "Comp 3", "aggregate_score": 0.03, "coverage_rejected": False, "comp_index": 2}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert result["selected_result"]["comp_index"] == 1 + assert created_fits[40].extension_calls == [{ + "min_num_live_points": 1200, + "max_ncalls": 6000, + "bounds": created_fits[40].bounds, + }] + assert created_fits[50].extension_calls == [] + assert created_fits[30].extension_calls == [] + assert created_fits[40].cleared is True + assert created_fits[50].cleared is True + assert created_fits[30].cleared is True + assert "selected comparison-star final" in created_fits[40].sparse_posterior_live_point_extension_note + + def test_fit_ranked_comparison_calibration_candidates_stops_at_first_qc_pass_by_default(monkeypatch): def fake_diagnostics(*args, **kwargs): return {"usable_point_count": 6} diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index d3dbe8ff..f26daa0c 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -756,7 +756,7 @@ def test_sparse_posterior_metric_flags_under_sampled_key_parameters(): ) assert diagnostics["sparse"] is True - assert diagnostics["minimum_effective_samples"] == 600 + assert diagnostics["minimum_effective_samples"] == 1000 assert diagnostics["parameters"]["rprs"]["effective_sample_count"] == pytest.approx(100) assert "rprs" in diagnostics["reason"] diff --git a/tests/test_plots.py b/tests/test_plots.py index de8a36f4..ab6ef35a 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -177,14 +177,23 @@ def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): def test_plot_comp_star_candidate_lightcurve_fits_writes_outputs(tmp_path): class DummyCandidateFit: - def plot_bestfit(self, phase=False): + def __init__(self): + self.kwargs = None + + def plot_bestfit(self, phase=False, show_flux_baseline_label=True): + self.kwargs = { + "phase": phase, + "show_flux_baseline_label": show_flux_baseline_label, + } fig, axes = plt.subplots(2, 1) return fig, axes + selected_fit = DummyCandidateFit() + other_fit = DummyCandidateFit() plot_comp_star_candidate_lightcurve_fits( candidate_fit_summaries=[ - {"label": "Comp 1", "selected": True, "fit": DummyCandidateFit(), "res_std": 0.0012}, - {"label": "Comp 2", "selected": False, "fit": DummyCandidateFit(), "res_std": 0.0025}, + {"label": "Comp 1", "selected": True, "fit": selected_fit, "res_std": 0.0012}, + {"label": "Comp 2", "selected": False, "fit": other_fit, "res_std": 0.0025}, {"label": "Comp 3", "selected": False, "fit": None, "res_std": np.inf}, ], targ_name="Target", @@ -193,6 +202,8 @@ def plot_bestfit(self, phase=False): method_label="Aperture photometry (aper=5.00px, annulus=12.00px)", ) + assert selected_fit.kwargs == {"phase": False, "show_flux_baseline_label": False} + assert other_fit.kwargs == {"phase": False, "show_flux_baseline_label": False} assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.png").exists() assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.pdf").exists() assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.png").exists() diff --git a/tests/test_ultranest_utils.py b/tests/test_ultranest_utils.py index b07a8fa7..27ac6a0d 100644 --- a/tests/test_ultranest_utils.py +++ b/tests/test_ultranest_utils.py @@ -160,6 +160,69 @@ def run(self, **kwargs): assert sampler.kwargs["min_num_live_points"] == 320 +def test_run_reactive_sampler_auto_scales_draw_size_by_workers_and_ram(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "72") + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "thread") + monkeypatch.setattr(ultranest_utils, "_available_cpu_count", lambda: 72) + monkeypatch.setattr(ultranest_utils, "_system_total_memory_bytes", lambda: 128 * 1024 ** 3) + + class FakeSampler: + def __init__(self): + self.ndraw_min = 128 + self.ndraw_max = 65536 + self.draw_multiple = True + self.x_dim = 6 + self.num_params = 6 + self.loglike = lambda params: np.zeros(np.asarray(params).shape[0]) + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler(sampler, verbose=False) + + assert sampler.ndraw_min == ( + 72 + * ultranest_utils.HIGH_AUTO_POINTS_PER_WORKER + * ultranest_utils.AUTO_POINTS_PER_WORKER_MULTIPLIER + ) + assert sampler.ndraw_max == 65536 + + +def test_run_reactive_sampler_auto_uses_smaller_chunks_when_ram_per_cpu_is_low(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "72") + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "thread") + monkeypatch.setattr(ultranest_utils, "_available_cpu_count", lambda: 72) + monkeypatch.setattr(ultranest_utils, "_system_total_memory_bytes", lambda: 16 * 1024 ** 3) + + class FakeSampler: + def __init__(self): + self.ndraw_min = 128 + self.ndraw_max = 65536 + self.draw_multiple = True + self.x_dim = 6 + self.num_params = 6 + self.loglike = lambda params: np.zeros(np.asarray(params).shape[0]) + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler(sampler, verbose=False) + + assert sampler.ndraw_min == ( + 72 + * ultranest_utils.MIN_AUTO_POINTS_PER_WORKER + * ultranest_utils.AUTO_POINTS_PER_WORKER_MULTIPLIER + ) + + def test_configured_ultranest_workers_defaults_to_available_cpu_count(monkeypatch): _reset_ultranest_env(monkeypatch) monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 12, raising=False) From 2741284b433014108cd8db80775aec886d1064d5 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 12 May 2026 17:47:46 +1000 Subject: [PATCH 040/116] better fits --- exotic/api/elca.py | 25 +- exotic/exotic.py | 591 ++++++++++++++++++++++++++++++-- exotic/inputs.py | 5 + inits.json | 4 +- tests/test_exotic_rprs_retry.py | 223 ++++++++++++ tests/test_inputs.py | 15 + 6 files changed, 834 insertions(+), 29 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 562c0951..881557f4 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -478,6 +478,8 @@ def __init__( keep_ultranest_sampler=False, baseline_fit_mask=None, fixed_parameter_errors=None, + fixed_flux_baseline=False, + ultranest_min_num_live_points=None, ): self.time = time self.data = data @@ -500,6 +502,8 @@ def __init__( if isinstance(fixed_parameter_errors, dict) else {} ) + self.fixed_flux_baseline = bool(fixed_flux_baseline) + self.ultranest_min_num_live_points = ultranest_min_num_live_points self._ultranest_resume_context = None self.results = None self.sampled_keys = list(bounds.keys()) @@ -536,6 +540,9 @@ def _validate_flux_baseline_keys(self): def _has_free_flux_baseline(self): return has_explicit_flux_baseline(self.bounds) + def _uses_fixed_flux_baseline(self): + return bool(getattr(self, 'fixed_flux_baseline', False)) + def _set_flux_baseline(self, value, error=0.0): self.parameters['a0'] = value self.errors['a0'] = error @@ -2522,6 +2529,8 @@ def lc2min_airmass(pars): ) if self._has_free_flux_baseline(): model *= get_flux_baseline(self.prior) + elif self._uses_fixed_flux_baseline(): + model *= get_flux_baseline(self.prior) else: model *= solve_flux_baseline( model, @@ -2580,6 +2589,12 @@ def create_fit_variables(self): if self._has_free_flux_baseline(): flux_scale = get_flux_baseline(self.parameters) flux_scale_err = self.errors.get('a0', self.errors.get('a1', 0.0)) + elif self._uses_fixed_flux_baseline(): + flux_scale = get_flux_baseline(self.parameters) + flux_scale_err = self.errors.get( + 'a0', + self.errors.get('a1', self.fixed_parameter_errors.get('a0', 0.0)), + ) elif self.mode == "ns": flux_scale, flux_scale_err = mc_a1( self.parameters.get('a2', 0), @@ -2782,6 +2797,8 @@ def single_loglike(pars): ) if self._has_free_flux_baseline(): model *= get_flux_baseline(physical) + elif self._uses_fixed_flux_baseline(): + model *= get_flux_baseline(physical) else: model *= solve_flux_baseline( model, @@ -2820,9 +2837,13 @@ def prior_transform(upars): self.ns_type = 'ultranest' test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform, vectorized=True) + run_kwargs = {"max_ncalls": int(self.max_ncalls)} + if self.ultranest_min_num_live_points is not None: + run_kwargs["min_num_live_points"] = int(self.ultranest_min_num_live_points) + self.results = run_reactive_sampler( test, - run_kwargs={"max_ncalls": int(self.max_ncalls)}, + run_kwargs=run_kwargs, verbose=self.verbose, ) @@ -2892,6 +2913,8 @@ def prior_transform(upars): lightcurve = transit(self.time, test_values) if self._has_free_flux_baseline(): flux_scale = get_flux_baseline(test_values) + elif self._uses_fixed_flux_baseline(): + flux_scale = get_flux_baseline(test_values) else: flux_scale = mc_a1( test_values.get('a2', 0), diff --git a/exotic/exotic.py b/exotic/exotic.py index c23f62b2..feddcb8d 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -226,6 +226,9 @@ FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT = 200 ULTRANEST_MIN_NUM_LIVE_POINTS_ENV = "EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS" +FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT = True +FAST_ULTRANEST_MAX_BINNED_POINTS = 20 +FAST_ULTRANEST_MIN_POINTS_TO_BIN = 60 SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT = True SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV = "EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY" SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT = 5 @@ -1834,7 +1837,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, use_eebls_to_initialize_tmid_and_bounds=True, plot_time_range=None, baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, - adaptive_summary=None): + adaptive_summary=None, + run_fast_ultranest_before_final_run=FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT): result = { 'applied': False, 'fit': None, @@ -1990,6 +1994,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, ) if not skip_final_airmass_fit: bounds['a2'] = [-3, 3] + ensure_pre_final_ultranest_baseline_bounds(prior, bounds, good_flux, fit_a2=True) debug_phase_clip_keep_mask = None prefit = lc_fitter( @@ -2029,14 +2034,49 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_comp_flux = good_comp_flux[~phase_clip_mask] source_indices = source_indices[~phase_clip_mask] - final_fit, good_flux, good_unc = fit_final_lightcurve_with_oot_baseline_detrending( - good_times, - good_flux, - good_unc, - good_airmass, - prior, - bounds, - jd_times=good_jd_times, + full_good_times = np.asarray(good_times, dtype=float) + full_good_flux = np.asarray(good_flux, dtype=float) + full_good_unc = np.asarray(good_unc, dtype=float) + full_good_airmass = np.asarray(good_airmass, dtype=float) + full_good_jd_times = np.asarray(good_jd_times, dtype=float) + full_good_target_flux = np.asarray(good_target_flux, dtype=float) + full_good_comp_flux = np.asarray(good_comp_flux, dtype=float) + full_source_indices = np.asarray(source_indices, dtype=int) + + fast_binning = {'applied': False, 'note': None} + fit_times = full_good_times + fit_flux = full_good_flux + fit_unc = full_good_unc + fit_airmass = full_good_airmass + fit_jd_times = full_good_jd_times + if run_fast_ultranest_before_final_run: + fast_binning = build_fast_ultranest_lightcurve_series( + full_good_times, + full_good_flux, + full_good_unc, + full_good_airmass, + jd_times=full_good_jd_times, + ) + if fast_binning.get('applied'): + log_info(fast_binning['note']) + fit_times = fast_binning['time'] + fit_flux = fast_binning['flux'] + fit_unc = fast_binning['unc'] + fit_airmass = fast_binning['airmass'] + fit_jd_times = fast_binning['jd_times'] + + fit_prior = dict(prior) + fit_bounds = clone_lightcurve_bounds(bounds) + ensure_pre_final_ultranest_baseline_bounds(fit_prior, fit_bounds, fit_flux, fit_a2=True) + + final_fit, fitted_flux, fitted_unc = fit_final_lightcurve_with_oot_baseline_detrending( + fit_times, + fit_flux, + fit_unc, + fit_airmass, + fit_prior, + fit_bounds, + jd_times=fit_jd_times, skip_airmass_fit=skip_final_airmass_fit, airmass_skip_note=airmass_skip_note, disable_vertical_flux_normalization=disable_vertical_flux_normalization, @@ -2049,14 +2089,16 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, expected_tmid_search_summary=tmid_search_summary, eebls_search_summary=eebls_search_summary, extend_sparse_posterior_live_points=False, - keep_ultranest_sampler_for_deferred_extension=True, + keep_ultranest_sampler_for_deferred_extension=not bool(fast_binning.get('applied')), + fix_baseline_terms_for_final=not bool(fast_binning.get('applied')), ) + annotate_fast_ultranest_binning(final_fit, fast_binning) if final_fit is None: result['failure_reason'] = "the full comparison-candidate reduction did not converge." return result final_fit_times = np.asarray(getattr(final_fit, 'time', good_times), dtype=float) - if ( + if not fast_binning.get('applied') and ( final_fit_times.shape != good_times.shape or not np.allclose(final_fit_times, good_times, rtol=1e-10, atol=1e-10) ): @@ -2084,14 +2126,24 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, result.update({ 'applied': True, 'fit': final_fit, - 'good_times': np.asarray(good_times, dtype=float), - 'good_flux': np.asarray(good_flux, dtype=float), - 'good_unc': np.asarray(good_unc, dtype=float), - 'good_airmass': np.asarray(good_airmass, dtype=float), - 'good_jd_times': np.asarray(good_jd_times, dtype=float), - 'good_target_flux': np.asarray(good_target_flux, dtype=float), - 'good_comp_flux': np.asarray(good_comp_flux, dtype=float), - 'source_indices': np.asarray(source_indices, dtype=int), + 'good_times': full_good_times if fast_binning.get('applied') else np.asarray(good_times, dtype=float), + 'good_flux': full_good_flux if fast_binning.get('applied') else np.asarray(good_flux, dtype=float), + 'good_unc': full_good_unc if fast_binning.get('applied') else np.asarray(good_unc, dtype=float), + 'good_airmass': full_good_airmass if fast_binning.get('applied') else np.asarray(good_airmass, dtype=float), + 'good_jd_times': full_good_jd_times if fast_binning.get('applied') else np.asarray(good_jd_times, dtype=float), + 'good_target_flux': full_good_target_flux if fast_binning.get('applied') else np.asarray(good_target_flux, dtype=float), + 'good_comp_flux': full_good_comp_flux if fast_binning.get('applied') else np.asarray(good_comp_flux, dtype=float), + 'source_indices': full_source_indices if fast_binning.get('applied') else np.asarray(source_indices, dtype=int), + 'fast_ultranest_binning': fast_binning, + 'fast_fit_good_times': np.asarray(fit_times, dtype=float), + 'fast_fit_good_flux': np.asarray(fitted_flux, dtype=float), + 'fast_fit_good_unc': np.asarray(fitted_unc, dtype=float), + 'fast_fit_good_airmass': np.asarray(fit_airmass, dtype=float), + 'fast_fit_good_jd_times': None if fit_jd_times is None else np.asarray(fit_jd_times, dtype=float), + 'fast_fit_prior': fit_prior, + 'fast_fit_bounds': fit_bounds, + 'skip_airmass_fit': skip_final_airmass_fit, + 'airmass_skip_note': airmass_skip_note, 'data_highres': data_highres, 'duration_samples': duration_samples, 'failure_reason': None, @@ -2100,6 +2152,224 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, return result +def selected_final_live_point_target(enabled=None): + if enabled is None: + enabled = should_use_sparse_posterior_live_point_retry( + os.environ.get( + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV, + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, + ) + ) + base_live_points = get_configured_ultranest_min_num_live_points() + if not enabled: + return base_live_points, None + extension_factor = int(max(1, SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT)) + target_live_points = int(max( + base_live_points + extension_factor * base_live_points, + base_live_points + 1, + )) + return base_live_points, target_live_points + + +def baseline_fixed_errors_from_fit(fit): + errors = getattr(fit, 'errors', {}) if fit is not None else {} + fixed_errors = {} + if isinstance(errors, dict): + for key in ('a0', 'a1', 'a2'): + value = errors.get(key) + try: + value = float(value) + except (TypeError, ValueError): + continue + if np.isfinite(value) and value >= 0: + fixed_errors[key] = value + if 'a0' in fixed_errors and 'a1' not in fixed_errors: + fixed_errors['a1'] = fixed_errors['a0'] + return fixed_errors + + +def build_full_resolution_final_prior_from_previous_fit(previous_fit, p_dict): + previous_parameters = getattr(previous_fit, 'parameters', {}) + if not isinstance(previous_parameters, dict): + previous_parameters = {} + + prior = { + 'rprs': p_dict.get('rprs', previous_parameters.get('rprs')), + 'ars': p_dict.get('aRs', previous_parameters.get('ars')), + 'per': p_dict.get('pPer', previous_parameters.get('per')), + 'inc': p_dict.get('inc', previous_parameters.get('inc')), + 'u0': previous_parameters.get('u0', 0.0), + 'u1': previous_parameters.get('u1', 0.0), + 'u2': previous_parameters.get('u2', 0.0), + 'u3': previous_parameters.get('u3', 0.0), + 'ecc': p_dict.get('ecc', previous_parameters.get('ecc', 0.0)), + 'omega': p_dict.get('omega', previous_parameters.get('omega', 0.0)), + 'tmid': p_dict.get('midT', previous_parameters.get('tmid')), + 'a2': previous_parameters.get('a2', 0.0), + 'a0': previous_parameters.get('a0', previous_parameters.get('a1', 1.0)), + } + prior['a1'] = previous_parameters.get('a1', prior['a0']) + prior.update(previous_parameters) + if 'a0' not in prior and 'a1' in prior: + prior['a0'] = prior['a1'] + if 'a1' not in prior and 'a0' in prior: + prior['a1'] = prior['a0'] + return prior + + +def refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + p_dict, + skip_airmass_fit=False, + airmass_skip_note=None, + detrend_on_outoftransit_baseline=True, + use_impactparameter_rather_than_inclination_to_fit=True, + plot_time_range=None, + duration_prior=None, + sparse_live_point_extension_enabled=None, +): + previous_fit = selected_result.get('fit') if isinstance(selected_result, dict) else None + if previous_fit is None or not getattr(previous_fit, 'fast_ultranest_binning_applied', False): + return None + + times = np.asarray(selected_result.get('good_times'), dtype=float) + flux_values = np.asarray(selected_result.get('good_flux'), dtype=float) + flux_errors = np.asarray(selected_result.get('good_unc'), dtype=float) + airmass = np.asarray(selected_result.get('good_airmass'), dtype=float) + jd_times = selected_result.get('good_jd_times') + jd_times = None if jd_times is None else np.asarray(jd_times, dtype=float) + if not (times.shape == flux_values.shape == flux_errors.shape == airmass.shape): + log_info( + "Warning: Could not run the full-resolution selected comparison-star final fit " + "because the saved fast-fit light-curve arrays were not aligned.", + warn=True, + ) + return None + if jd_times is not None and jd_times.shape != times.shape: + jd_times = None + + prior = build_full_resolution_final_prior_from_previous_fit(previous_fit, p_dict) + fallback_bounds = selected_result.get('fast_fit_bounds') + if not isinstance(fallback_bounds, dict): + fallback_bounds = getattr(previous_fit, 'bounds', {}) + bounds = get_posterior_refit_final_bounds(previous_fit, fallback_bounds) + bounds = clone_lightcurve_bounds(bounds) + for key in ('a0', 'a1', 'a2'): + bounds.pop(key, None) + + for key in ('rprs', 'tmid', 'ars', 'inc'): + if key not in bounds: + if key == 'rprs': + bounds[key] = build_initial_rprs_bounds(prior.get('rprs', p_dict.get('rprs', 0.1))) + elif key == 'tmid': + tmid = prior.get('tmid', p_dict.get('midT', np.nan)) + tmid_unc = p_dict.get('midTUnc', 0.01) + try: + half_width = max(float(tmid_unc) * 3.0, np.finfo(float).eps) + except (TypeError, ValueError): + half_width = 0.01 + bounds[key] = [float(tmid) - half_width, float(tmid) + half_width] + elif key == 'ars': + bounds[key] = build_initial_ars_bounds(prior.get('ars', p_dict.get('aRs')), p_dict.get('aRsUnc')) + elif key == 'inc': + inc = float(prior.get('inc', p_dict.get('inc', 89.0))) + bounds[key] = [inc - 5.0, min(90.0, inc + 5.0)] + + fit_flux = flux_values + fit_unc = flux_errors + detrend_result = {'applied': False, 'note': 'Disabled; using the full-resolution light curve directly.'} + if detrend_on_outoftransit_baseline: + detrend_result = detrend_flux_on_out_of_transit_baseline( + times, + flux_values, + flux_errors, + previous_fit, + prior=prior, + ) + if detrend_result.get('applied'): + fit_flux = np.asarray(detrend_result['flux'], dtype=float) + fit_unc = np.asarray(detrend_result['unc'], dtype=float) + + fixed_errors = baseline_fixed_errors_from_fit(previous_fit) + base_live_points, target_live_points = selected_final_live_point_target( + sparse_live_point_extension_enabled, + ) + min_live_points = target_live_points if target_live_points is not None else base_live_points + + log_info( + "Running the selected comparison-star final UltraNest fit on the full-resolution light curve " + f"with fixed a0/a2 from the previous fast UltraNest fit at {min_live_points} minimum live points." + ) + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + fit_flux, + fit_unc, + airmass, + prior, + bounds, + jd_times=jd_times, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, + keep_ultranest_sampler=False, + fixed_parameter_errors=fixed_errors, + fixed_flux_baseline=True, + ultranest_min_num_live_points=min_live_points, + ) + fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_out_of_transit_baseline_parameter_fit( + fit, + True, + note="Used a0 and a2 from the previous fast UltraNest fit for the full-resolution final run.", + pre_points=0, + post_points=0, + a0=prior.get('a0'), + a0_error=fixed_errors.get('a0'), + a2=prior.get('a2'), + a2_error=fixed_errors.get('a2'), + ) + annotate_out_of_transit_baseline_detrending( + fit, + bool(detrend_result.get('applied')), + note=detrend_result.get('note'), + slope=detrend_result.get('slope'), + intercept=detrend_result.get('intercept'), + pre_points=detrend_result.get('pre_points', 0), + post_points=detrend_result.get('post_points', 0), + ) + annotate_fast_ultranest_binning( + fit, + { + 'applied': False, + 'original_point_count': int(times.shape[0]), + 'binned_point_count': int(times.shape[0]), + 'note': 'Full-resolution selected comparison-star final run; fast binning was not applied.', + }, + ) + if target_live_points is not None: + diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) + annotate_sparse_posterior_live_point_extension( + fit, + True, + True, + note=( + "Applied full-resolution selected comparison-star final UltraNest run " + f"({base_live_points}->{target_live_points} minimum live points) using fixed a0/a2 " + "from the previous fast UltraNest fit." + ), + diagnostics=diagnostics, + post_extension_diagnostics=diagnostics, + base_live_points=base_live_points, + target_live_points=target_live_points, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + ) + else: + annotate_sparse_posterior_live_point_extension(fit, False, False) + annotate_transit_detection_qc(fit) + clear_fit_ultranest_resume_state(fit) + return fit, fit_flux, fit_unc + + def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, final_fit, p_dict, observation_date, comp_index, comp_coords=None, min_aperture=None, min_annulus=None, @@ -3233,6 +3503,8 @@ def run_nested_lightcurve_fit_with_rprs_posterior_retry( keep_ultranest_sampler=False, baseline_fit_mask=None, fixed_parameter_errors=None, + fixed_flux_baseline=False, + ultranest_min_num_live_points=None, ): def impact_parameter_retry_available(fit, local_bounds): if not use_impactparameter_rather_than_inclination_to_fit or 'inc' not in local_bounds: @@ -3345,6 +3617,13 @@ def build_fit(local_prior, local_bounds): fit_kwargs['baseline_fit_mask'] = baseline_fit_mask if fixed_parameter_errors and callable_accepts_keyword(lc_fitter, 'fixed_parameter_errors'): fit_kwargs['fixed_parameter_errors'] = fixed_parameter_errors + if fixed_flux_baseline and callable_accepts_keyword(lc_fitter, 'fixed_flux_baseline'): + fit_kwargs['fixed_flux_baseline'] = True + if ( + ultranest_min_num_live_points is not None + and callable_accepts_keyword(lc_fitter, 'ultranest_min_num_live_points') + ): + fit_kwargs['ultranest_min_num_live_points'] = ultranest_min_num_live_points fit = lc_fitter( times, flux_values, @@ -3905,6 +4184,28 @@ def should_use_sparse_posterior_live_point_retry(config_value): return SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT +def should_run_fast_ultranest_before_final_run(config_value): + if config_value is None: + return FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'run fast ultranest before final run' value; " + "defaulting to enabled.", + warn=True, + ) + return FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT + + def configure_sparse_posterior_live_point_retry(config_value): enabled = should_use_sparse_posterior_live_point_retry(config_value) os.environ[SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV] = "1" if enabled else "0" @@ -4972,6 +5273,181 @@ def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled bounds['a0'] = [lower, upper] +def ensure_pre_final_ultranest_baseline_bounds(prior, bounds, flux_values, fit_a2=True): + finite_flux = np.asarray(flux_values, dtype=float) + finite_flux = finite_flux[np.isfinite(finite_flux) & (finite_flux > 0)] + baseline_guess = prior.get('a0', prior.get('a1', np.nan)) + try: + baseline_guess = float(baseline_guess) + except (TypeError, ValueError): + baseline_guess = np.nan + if not np.isfinite(baseline_guess) or baseline_guess <= 0: + baseline_guess = 1.0 if finite_flux.size == 0 else float(np.nanmedian(finite_flux)) + if not np.isfinite(baseline_guess) or baseline_guess <= 0: + baseline_guess = 1.0 + + prior['a0'] = baseline_guess + prior['a1'] = baseline_guess + if 'a0' not in bounds and 'a1' not in bounds: + if 0.95 <= baseline_guess <= 1.05: + bounds['a0'] = [0.95, 1.05] + else: + lower = max(np.finfo(float).eps, baseline_guess * 0.75) + upper = baseline_guess * 1.25 + bounds['a0'] = [lower, upper] + + if fit_a2: + prior['a2'] = prior.get('a2', 0.0) + if 'a2' not in bounds: + bounds['a2'] = list(TRANSIT_QC_DEFAULT_A2_BOUNDS) + + +def _weighted_mean_with_fallback(values, weights=None): + values = np.asarray(values, dtype=float) + finite = np.isfinite(values) + if not np.any(finite): + return np.nan + + if weights is not None: + weights = np.asarray(weights, dtype=float) + valid_weights = finite & np.isfinite(weights) & (weights > 0) + if np.any(valid_weights): + return float(np.sum(values[valid_weights] * weights[valid_weights]) / np.sum(weights[valid_weights])) + + return float(np.nanmean(values[finite])) + + +def build_fast_ultranest_lightcurve_series( + times, + flux_values, + flux_errors, + airmass, + jd_times=None, + max_points=FAST_ULTRANEST_MAX_BINNED_POINTS, + min_points_to_bin=FAST_ULTRANEST_MIN_POINTS_TO_BIN, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + airmass = np.asarray(airmass, dtype=float) + jd_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + + base_result = { + 'applied': False, + 'note': None, + 'time': times, + 'flux': flux_values, + 'unc': flux_errors, + 'airmass': airmass, + 'jd_times': jd_array, + 'original_point_count': int(times.shape[0]), + 'binned_point_count': int(times.shape[0]), + 'bin_indices': None, + } + + if not (times.shape == flux_values.shape == flux_errors.shape == airmass.shape): + base_result['note'] = 'Skipped; light-curve arrays were not aligned for fast UltraNest binning.' + return base_result + if jd_array is not None and jd_array.shape != times.shape: + base_result['note'] = 'Skipped; JD timestamps were not aligned for fast UltraNest binning.' + return base_result + + point_count = int(times.shape[0]) + if point_count <= int(min_points_to_bin): + base_result['note'] = ( + f"Skipped; {point_count} point(s) did not exceed the fast UltraNest " + f"binning threshold of {int(min_points_to_bin)}." + ) + return base_result + + max_points = int(max(1, max_points)) + target_points = min(max_points, point_count) + valid = ( + np.isfinite(times) + & np.isfinite(flux_values) + & np.isfinite(flux_errors) + & (flux_errors > 0) + & np.isfinite(airmass) + ) + if jd_array is not None: + valid &= np.isfinite(jd_array) + if np.count_nonzero(valid) <= target_points: + base_result['note'] = 'Skipped; too few finite points remained for fast UltraNest binning.' + return base_result + + ordered_indices = np.flatnonzero(valid)[np.argsort(times[valid])] + chunks = [chunk for chunk in np.array_split(ordered_indices, target_points) if chunk.size > 0] + if len(chunks) >= point_count or not chunks: + base_result['note'] = 'Skipped; fast UltraNest binning would not reduce the light curve.' + return base_result + + binned_time = [] + binned_flux = [] + binned_unc = [] + binned_airmass = [] + binned_jd = [] if jd_array is not None else None + for chunk in chunks: + chunk_unc = flux_errors[chunk] + weights = np.zeros(chunk_unc.shape, dtype=float) + valid_unc = np.isfinite(chunk_unc) & (chunk_unc > 0) + weights[valid_unc] = 1.0 / (chunk_unc[valid_unc] ** 2) + binned_time.append(_weighted_mean_with_fallback(times[chunk], weights)) + binned_flux.append(_weighted_mean_with_fallback(flux_values[chunk], weights)) + if np.any(weights > 0): + binned_unc.append(float(np.sqrt(1.0 / np.sum(weights[weights > 0])))) + else: + scatter = float(np.nanstd(flux_values[chunk])) + binned_unc.append(scatter / np.sqrt(max(chunk.size, 1)) if np.isfinite(scatter) else np.nan) + binned_airmass.append(_weighted_mean_with_fallback(airmass[chunk], weights)) + if jd_array is not None: + binned_jd.append(_weighted_mean_with_fallback(jd_array[chunk], weights)) + + binned_time = np.asarray(binned_time, dtype=float) + binned_flux = np.asarray(binned_flux, dtype=float) + binned_unc = np.asarray(binned_unc, dtype=float) + binned_airmass = np.asarray(binned_airmass, dtype=float) + finite_binned = ( + np.isfinite(binned_time) + & np.isfinite(binned_flux) + & np.isfinite(binned_unc) + & (binned_unc > 0) + & np.isfinite(binned_airmass) + ) + if binned_jd is not None: + binned_jd = np.asarray(binned_jd, dtype=float) + finite_binned &= np.isfinite(binned_jd) + + if np.count_nonzero(finite_binned) < LIGHTCURVE_MIN_VALID_POINTS: + base_result['note'] = 'Skipped; fast UltraNest binning produced too few finite bins.' + return base_result + + result = dict(base_result) + result.update({ + 'applied': True, + 'time': binned_time[finite_binned], + 'flux': binned_flux[finite_binned], + 'unc': binned_unc[finite_binned], + 'airmass': binned_airmass[finite_binned], + 'jd_times': None if binned_jd is None else binned_jd[finite_binned], + 'binned_point_count': int(np.count_nonzero(finite_binned)), + 'bin_indices': [chunk.tolist() for i, chunk in enumerate(chunks) if finite_binned[i]], + 'note': ( + f"Using fast UltraNest binning for pre-final runs: " + f"{point_count} point(s) -> {int(np.count_nonzero(finite_binned))} binned point(s)." + ), + }) + return result + + +def annotate_fast_ultranest_binning(fit, binning_result): + if fit is None or not isinstance(binning_result, dict): + return + fit.fast_ultranest_binning_applied = bool(binning_result.get('applied', False)) + fit.fast_ultranest_original_point_count = int(binning_result.get('original_point_count', 0)) + fit.fast_ultranest_binned_point_count = int(binning_result.get('binned_point_count', 0)) + fit.fast_ultranest_binning_note = binning_result.get('note') + + def summarize_initial_fit_transit_coverage( times, fit, @@ -5969,6 +6445,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( duration_prior=None, extend_sparse_posterior_live_points=True, keep_ultranest_sampler_for_deferred_extension=False, + fix_baseline_terms_for_final=True, ): if duration_prior is None and expected_planet_dict is not None: duration_prior = build_single_transit_duration_prior(expected_planet_dict) @@ -6090,6 +6567,13 @@ def fit_final_lightcurve_with_oot_baseline_detrending( 'pre_points': 0, 'post_points': 0, } + if baseline_parameter_result.get('applied') and not fix_baseline_terms_for_final: + baseline_parameter_result = { + 'applied': False, + 'note': 'Deferred; pre-final UltraNest runs keep a0 and a2 as simultaneous fitted parameters.', + 'pre_points': baseline_parameter_result.get('pre_points', 0), + 'post_points': baseline_parameter_result.get('post_points', 0), + } baseline_fit_mask = None baseline_fixed_errors = {} baseline_constrained_prior = dict(working_prior) @@ -6136,6 +6620,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): keep_ultranest_sampler=keep_ultranest_for_sparse_extension, baseline_fit_mask=baseline_fit_mask, fixed_parameter_errors=baseline_fixed_errors, + fixed_flux_baseline=True, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -6250,6 +6735,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): keep_ultranest_sampler=keep_ultranest_for_sparse_extension, baseline_fit_mask=baseline_fit_mask, fixed_parameter_errors=baseline_fixed_errors, + fixed_flux_baseline=bool(baseline_parameter_result.get('applied')), ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -13619,6 +14105,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p adaptive_aperture_values=None, adaptive_annulus_values=None, fallback_sigma=np.nan, + run_fast_ultranest_before_final_run= + FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, save_dir=None, planet_name=None, observation_date=None): @@ -13720,6 +14208,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p plot_time_range=plot_time_range, baseline_duration_multiplier=final_fit_baseline_duration_multiplier, adaptive_summary=adaptive_summary, + run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, ) fit_result = final_reduction.get('fit') if final_reduction.get('applied') else None tflux_fit = final_reduction.get('good_target_flux') @@ -13798,6 +14287,9 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'final_output_dir': None, 'full_reduction_applied': final_reduction.get('applied', False), 'full_reduction_note': final_reduction.get('note'), + 'fast_ultranest_binning': final_reduction.get('fast_ultranest_binning'), + 'skip_airmass_fit': final_reduction.get('skip_airmass_fit', False), + 'airmass_skip_note': final_reduction.get('airmass_skip_note'), } if final_reduction.get('applied') and selection_fit is not None and save_dir is not None: final_output_dir = save_comparison_candidate_full_reduction_outputs( @@ -13978,9 +14470,31 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p selected_fit = selected_result.get('fit') if selected_fit is not None: - selected_result['fit'] = extend_selected_comparison_live_points_if_needed(selected_fit) + full_resolution_refit_applied = False + full_resolution_refit = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + p_dict, + skip_airmass_fit=bool(selected_result.get('skip_airmass_fit', False)), + airmass_skip_note=selected_result.get('airmass_skip_note'), + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + duration_prior=build_single_transit_duration_prior(p_dict), + ) + if full_resolution_refit is not None: + full_resolution_refit_applied = True + selected_result['fit'], selected_result['good_flux'], selected_result['good_unc'] = full_resolution_refit + selected_result['full_reduction_note'] = ( + "selected candidate rerun on the full-resolution light curve after fast UltraNest search." + ) + else: + selected_result['fit'] = extend_selected_comparison_live_points_if_needed(selected_fit) selected_result['full_reduction_fit'] = selected_result['fit'] - if getattr(selected_result['fit'], 'sparse_posterior_live_point_extension_applied', False): + if ( + full_resolution_refit_applied + or getattr(selected_result['fit'], 'sparse_posterior_live_point_extension_applied', False) + ): annotate_transit_detection_qc(selected_result['fit']) selected_result['eebls_snr'] = extract_lightcurve_fit_eebls_snr(selected_result['fit']) selected_result['transit_delta_bic'] = extract_lightcurve_fit_transit_delta_bic(selected_result['fit']) @@ -14260,6 +14774,12 @@ def _main_impl(): exotic_infoDict.get('use_impactparameter_rather_than_inclination_to_fit', 'y') ) ) + run_fast_ultranest_before_final_run = should_run_fast_ultranest_before_final_run( + exotic_infoDict.get( + 'run_fast_ultranest_before_final_run', + FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, + ) + ) ultranest_min_num_live_points = configure_ultranest_min_num_live_points( exotic_infoDict.get( 'ultranest_min_num_live_points', @@ -14267,6 +14787,14 @@ def _main_impl(): ) ) log_info(f"UltraNest minimum live points: {ultranest_min_num_live_points}.") + if run_fast_ultranest_before_final_run: + log_info( + "Fast pre-final UltraNest enabled: comparison-candidate UltraNest search runs " + f"with at most {FAST_ULTRANEST_MAX_BINNED_POINTS} binned light-curve point(s) " + f"when more than {FAST_ULTRANEST_MIN_POINTS_TO_BIN} points are available." + ) + else: + log_info("Fast pre-final UltraNest disabled per optional_info setting.") use_sparse_posterior_live_point_retry = configure_sparse_posterior_live_point_retry( exotic_infoDict.get( 'use_sparse_posterior_live_point_retry', @@ -14274,12 +14802,20 @@ def _main_impl(): ) ) if use_sparse_posterior_live_point_retry: - log_info( - "Selected comparison-star live-point extension enabled: comparison candidates " - "are ranked at the configured UltraNest live-point count, then the chosen final " - f"comparison fit continues with {SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT}x " - "additional minimum live points using its retained final-pass bounds." - ) + if run_fast_ultranest_before_final_run: + log_info( + "Selected comparison-star live-point extension enabled: comparison candidates " + "are ranked with fast pre-final UltraNest fits, then the chosen final comparison " + "fit reruns on the full-resolution light curve with " + f"{SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT}x additional minimum live points." + ) + else: + log_info( + "Selected comparison-star live-point extension enabled: comparison candidates " + "are ranked at the configured UltraNest live-point count, then the chosen final " + f"comparison fit continues with {SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT}x " + "additional minimum live points using its retained final-pass bounds." + ) log_ultranest_mpi_status() # Make a temp directory of helpful files @@ -15287,6 +15823,7 @@ def _main_impl(): adaptive_aperture_values=aperture_values, adaptive_annulus_values=annulus_values, fallback_sigma=sigma_display, + run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, save_dir=exotic_infoDict['save'], planet_name=pDict['pName'], observation_date=exotic_infoDict['date'], diff --git a/exotic/inputs.py b/exotic/inputs.py index 982ca9c8..73b92b4d 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -508,6 +508,11 @@ def comp_params(self, init_file, planet_dict): 'ultranest_min_live_points', 'min_num_live_points', ), + 'run_fast_ultranest_before_final_run': ( + 'run fast ultranest before final run', + 'Run Fast UltraNest Before Final Run? (y/n)', + 'run_fast_ultranest_before_final_run', + ), 'use_sparse_posterior_live_point_retry': ( 'use_sparse_posterior_live_point_retry', 'Use Sparse Posterior Live-Point Retry? (y/n)', diff --git a/inits.json b/inits.json index 7c5ad3ca..60746700 100644 --- a/inits.json +++ b/inits.json @@ -37,7 +37,8 @@ "Assess All Comparisons Before Selecting Best": "Set optional_info 'assess_all_comparisons_before_selecting_best' to y to fit every comparison-star candidate that survives the earlier screening, report all fits, and select the candidate with the highest KTMF score. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "UltraNest Live Points": "Set optional_info 'minimum number of live points for ultranest' to a positive integer to control UltraNest's min_num_live_points. Default 200.", - "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", + "Fast UltraNest Before Final Run": "Set optional_info 'run fast ultranest before final run' to y to run comparison-candidate UltraNest searches on a binned light curve of at most 20 points when more than 60 points are available, then rerun the selected final fit on the full light curve. Default y.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then run or continue the chosen final comparison-star fit with 5x additional minimum live points. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", @@ -124,6 +125,7 @@ "assess_all_comparisons_before_selecting_best": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "minimum number of live points for ultranest": 200, + "run fast ultranest before final run": "y", "use_sparse_posterior_live_point_retry": "y", "use_psf_photometry": "y", "use_aperture_photometry": "y", diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index f26daa0c..6190c233 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -90,8 +90,12 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: RPRS_SEARCH_BOUND_MAX, RPRS_SEARCH_BOUND_MIN, SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + build_fast_ultranest_lightcurve_series, evaluate_sparse_posterior_sample_support, extend_sparse_posterior_live_points_if_needed, + finalize_comparison_candidate_full_reduction, + refit_selected_fast_comparison_on_full_lightcurve, + should_run_fast_ultranest_before_final_run, build_single_transit_duration_prior, build_initial_rprs_bounds, run_nested_lightcurve_fit_with_rprs_posterior_retry, @@ -108,6 +112,225 @@ def test_build_initial_rprs_bounds_allows_zero_depth_search_box(): assert INITIAL_RPRS_BOUND_LOWER_SCALE == pytest.approx(0.0) +def test_fast_ultranest_option_defaults_enabled_and_parses_false_values(): + assert should_run_fast_ultranest_before_final_run(None) is True + assert should_run_fast_ultranest_before_final_run("n") is False + assert should_run_fast_ultranest_before_final_run(False) is False + + +def test_fast_ultranest_binning_reduces_large_light_curve_to_twenty_points(): + times = np.linspace(0.0, 1.0, 80) + flux = 1.0 + 0.01 * np.sin(np.linspace(0.0, 2.0 * np.pi, 80)) + unc = np.full(80, 0.01) + airmass = np.linspace(1.0, 1.5, 80) + + result = build_fast_ultranest_lightcurve_series(times, flux, unc, airmass) + + assert result["applied"] is True + assert result["original_point_count"] == 80 + assert result["binned_point_count"] <= 20 + assert result["time"].shape == result["flux"].shape == result["unc"].shape == result["airmass"].shape + + +def test_fast_ultranest_binning_skips_short_light_curve(): + times = np.linspace(0.0, 1.0, 60) + result = build_fast_ultranest_lightcurve_series( + times, + np.ones(60), + np.full(60, 0.01), + np.linspace(1.0, 1.2, 60), + ) + + assert result["applied"] is False + assert result["binned_point_count"] == 60 + + +def test_finalize_comparison_candidate_runs_pre_final_ultranest_on_binned_series(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + def fake_fit_final( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["point_count"] = len(times) + captured["bounds"] = dict(bounds) + captured["fix_baseline_terms_for_final"] = kwargs.get("fix_baseline_terms_for_final") + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters={ + **dict(prior), + "tmid": 0.5, + "rprs": 0.1, + "ars": 10.0, + "inc": 89.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.02, + }, + errors={"tmid": 0.001, "rprs": 0.001, "ars": 0.1, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit=np.ones(len(times), dtype=float), + residuals=np.zeros(len(times), dtype=float), + duration_measured=0.04, + duration_expected=0.04, + ) + return fit, np.asarray(flux_values, dtype=float), np.asarray(flux_errors, dtype=float) + + monkeypatch.setattr(exotic_module, "fit_final_lightcurve_with_oot_baseline_detrending", fake_fit_final) + + times = np.linspace(0.0, 1.0, 80) + target_flux = 100.0 * (1.0 + 0.002 * np.sin(np.linspace(0.0, 2.0 * np.pi, 80))) + result = finalize_comparison_candidate_full_reduction( + times, + target_flux, + np.full(80, 100.0), + np.linspace(1.0, 1.3, 80), + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={ + "pName": "Test b", + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "aRs": 10.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + jd_times=2460000.0 + times, + run_fast_ultranest_before_final_run=True, + ) + + assert result["applied"] is True + assert captured["point_count"] <= 20 + assert captured["fix_baseline_terms_for_final"] is False + assert "a0" in captured["bounds"] + assert "a2" in captured["bounds"] + assert len(result["good_times"]) == result["fast_ultranest_binning"]["original_point_count"] + assert len(result["good_times"]) > 60 + assert result["fast_ultranest_binning"]["applied"] is True + assert result["fit"].fast_ultranest_binning_applied is True + + +def test_selected_fast_candidate_final_refit_uses_full_series_and_fixed_baseline(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "200") + monkeypatch.setenv("EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY", "1") + captured = {} + + def fake_run_nested( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["point_count"] = len(times) + captured["prior"] = dict(prior) + captured["bounds"] = dict(bounds) + captured["fixed_parameter_errors"] = dict(kwargs.get("fixed_parameter_errors", {})) + captured["fixed_flux_baseline"] = kwargs.get("fixed_flux_baseline") + captured["ultranest_min_num_live_points"] = kwargs.get("ultranest_min_num_live_points") + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters=dict(prior), + errors=dict(kwargs.get("fixed_parameter_errors", {})), + residuals=np.zeros(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + duration_measured=0.04, + duration_expected=0.04, + transit_qc={"status": "pass", "summary": "ok"}, + transit_qc_status="pass", + ) + fit.get_parameter_posterior_samples = lambda key: np.linspace(0.0, 1.0, 1500) + return fit + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + parameters={ + "rprs": 0.1, + "ars": 10.0, + "per": 1.0, + "tmid": 0.5, + "inc": 89.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.03, + "a1": 1.03, + "a2": 0.12, + }, + errors={"a0": 0.02, "a1": 0.02, "a2": 0.03, "rprs": 0.001, "tmid": 0.001, "ars": 0.1}, + bounds={ + "rprs": [0.05, 0.15], + "tmid": [0.49, 0.51], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + }, + ) + times = np.linspace(0.0, 1.0, 80) + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": np.ones(80), + "good_unc": np.full(80, 0.01), + "good_airmass": np.linspace(1.0, 1.3, 80), + "good_jd_times": 2460000.0 + times, + } + + returned = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + { + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "rprs": 0.1, + "aRs": 10.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + detrend_on_outoftransit_baseline=False, + ) + + assert returned is not None + assert captured["point_count"] == 80 + assert captured["fixed_flux_baseline"] is True + assert captured["ultranest_min_num_live_points"] == 1200 + assert captured["prior"]["a0"] == pytest.approx(1.03) + assert captured["prior"]["a2"] == pytest.approx(0.12) + assert captured["fixed_parameter_errors"]["a0"] == pytest.approx(0.02) + assert captured["fixed_parameter_errors"]["a2"] == pytest.approx(0.03) + assert "a0" not in captured["bounds"] + assert "a2" not in captured["bounds"] + + def test_rprs_posterior_retry_walks_bounds_until_retry_cap(monkeypatch): import exotic.exotic as exotic_module diff --git a/tests/test_inputs.py b/tests/test_inputs.py index abcd9647..ea4b09bf 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -458,6 +458,21 @@ def test_comp_params_reads_ultranest_live_points_from_optional_info(tmp_path): assert inputs.info_dict["ultranest_min_num_live_points"] == 275 +def test_comp_params_reads_fast_ultranest_before_final_run_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"run fast ultranest before final run": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["run_fast_ultranest_before_final_run"] == "n" + + def test_comp_params_reads_sparse_posterior_live_point_retry_off_from_optional_info(tmp_path): init_data = { "user_info": {}, From c40de40616eb6f42c9972f4757f0d2d950ae188d Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Tue, 12 May 2026 19:20:32 +1000 Subject: [PATCH 041/116] uncertainty on a0/a2 --- exotic/api/elca.py | 111 ++++++++++++ exotic/exotic.py | 278 ++++++++++++++++++++++++++--- exotic/exotic_gui.py | 5 + exotic/inputs.py | 6 + exotic/plots.py | 1 + inits.json | 2 + tests/test_centroid_wcs.py | 50 ++++++ tests/test_elca_baseline.py | 53 ++++++ tests/test_exotic_proper_motion.py | 10 ++ tests/test_inputs.py | 30 ++++ tests/test_plots.py | 7 +- 11 files changed, 523 insertions(+), 30 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 881557f4..3c2e0f88 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -859,6 +859,78 @@ def transit_model_uncertainty(self, times=None, sigma=1.0): return None return model - model_uncertainty, model + model_uncertainty + def baseline_model_uncertainty(self, times=None, sigma=1.0): + if times is None: + times = getattr(self, 'time_upsample', self.time) + times = np.asarray(times, dtype=float) + if times.size == 0 or np.ndim(getattr(self, 'airmass', np.array([]))) == 2: + return None + + try: + best_systematics = self._build_systematics_model_at(self.parameters, times) + except Exception: + return None + + if ( + np.asarray(best_systematics).shape != times.shape + or not np.any(np.isfinite(best_systematics)) + ): + return None + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = 1.0 + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + + variance = np.zeros_like(times, dtype=float) + for key in ('a0', 'a1', 'a2'): + if key == 'a1' and 'a0' in self.parameters: + continue + if key not in self.parameters: + continue + error = self.errors.get(key) + try: + center = float(self.parameters[key]) + error = float(error) + except (TypeError, ValueError): + continue + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + continue + + lower_value = self._get_perturbed_transit_parameter_value(key, center - error) + upper_value = self._get_perturbed_transit_parameter_value(key, center + error) + if ( + not np.isfinite(lower_value) + or not np.isfinite(upper_value) + or np.isclose(lower_value, upper_value) + ): + continue + + lower_parameters = copy.deepcopy(self.parameters) + upper_parameters = copy.deepcopy(self.parameters) + lower_parameters[key] = lower_value + upper_parameters[key] = upper_value + try: + lower_systematics = self._build_systematics_model_at(lower_parameters, times) + upper_systematics = self._build_systematics_model_at(upper_parameters, times) + except Exception: + continue + + with np.errstate(divide='ignore', invalid='ignore'): + lower_ratio = lower_systematics / best_systematics + upper_ratio = upper_systematics / best_systematics + derivative = (upper_ratio - lower_ratio) / (upper_value - lower_value) + contribution = derivative * error * sigma + finite = np.isfinite(contribution) + variance[finite] += contribution[finite] ** 2 + + baseline_uncertainty = np.sqrt(variance) + if not np.any(np.isfinite(baseline_uncertainty) & (baseline_uncertainty > 0)): + return None + return 1.0 - baseline_uncertainty, 1.0 + baseline_uncertainty + def _plot_transit_model_uncertainty(self, ax, x_values, times, sort_index, label=None): envelope = self.transit_model_uncertainty(times) if envelope is None: @@ -900,6 +972,28 @@ def _plot_transit_model_uncertainty(self, ax, x_values, times, sort_index, label ) return band + def _plot_baseline_model_uncertainty(self, ax, x_values, times, sort_index, label=None): + envelope = self.baseline_model_uncertainty(times) + if envelope is None: + return None + + lower, upper = envelope + x_values = np.asarray(x_values, dtype=float) + sort_index = np.asarray(sort_index, dtype=int) + x_sorted = x_values[sort_index] + lower_sorted = np.asarray(lower, dtype=float)[sort_index] + upper_sorted = np.asarray(upper, dtype=float)[sort_index] + return ax.fill_between( + x_sorted, + lower_sorted, + upper_sorted, + color='gold', + alpha=0.28, + linewidth=0, + zorder=1.7, + label=label, + ) + def _uses_internal_impact_parameter(self): return ( self.use_impactparameter_rather_than_inclination_to_fit @@ -2983,6 +3077,7 @@ def plot_bestfit( phase=True, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False, ): f = plt.figure(figsize=(9, 6)) f.subplots_adjust(top=0.92, bottom=0.09, left=0.14, right=0.98, hspace=0) @@ -3040,6 +3135,14 @@ def plot_bestfit( marker='s') # axs[0].plot(self.phase[si], self.transit[si], 'r-', zorder=3, label=lclabel) sii = np.argsort(self.phase_upsample) + if show_baseline_uncertainty: + self._plot_baseline_model_uncertainty( + axs[0], + self.phase_upsample, + self.time_upsample, + sii, + label=r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty', + ) if show_model_uncertainty: self._plot_transit_model_uncertainty( axs[0], @@ -3063,6 +3166,14 @@ def plot_bestfit( si = np.argsort(self.time) sii = np.argsort(self.time_upsample) axs[0].errorbar(bt, bf, yerr=bs, alpha=1, zorder=2, color='blue', ls='none', marker='s') + if show_baseline_uncertainty: + self._plot_baseline_model_uncertainty( + axs[0], + self.time_upsample, + self.time_upsample, + sii, + label=r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty', + ) if show_model_uncertainty: self._plot_transit_model_uncertainty( axs[0], diff --git a/exotic/exotic.py b/exotic/exotic.py index feddcb8d..2109ffa5 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -281,6 +281,7 @@ BAD_PIXEL_GLOBAL_SIGMA = 3.0 BAD_PIXEL_ISOLATION_SIGMA = 5.0 BAD_PIXEL_ISOLATION_RATIO = 2.0 +MAX_MULTIPROCESS_BAD_PIXEL_WORKERS = 8 BAD_PIXEL_COUNTS_FILENAME = "BadPixelDetectionCounts.fits" BAD_PIXEL_MASK_FILENAME = "BadPixelMask.fits" BAD_PIXEL_NEIGHBOR_FOOTPRINT = np.array( @@ -4468,6 +4469,37 @@ def should_detect_bad_pixels_before_photometry(config_value): return True +def get_multiprocess_bad_pixel_precheck_processes(config_value): + if config_value is None: + return None + if isinstance(config_value, bool): + if not config_value: + return None + return os.cpu_count() or 1 + if isinstance(config_value, (int, float)): + if np.isfinite(config_value) and int(config_value) > 0: + return int(config_value) + return None + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('', 'n', 'no', 'false', '0', 'off'): + return None + if normalized in ('y', 'yes', 'true', '1', 'on'): + return os.cpu_count() or 1 + try: + parsed = float(normalized) + except ValueError: + parsed = np.nan + if np.isfinite(parsed) and int(parsed) > 0: + return int(parsed) + + log_info( + "Warning: Invalid 'multiprocess_bad_pixel_precheck' value; keeping bad-pixel precheck multiprocessing disabled.", + warn=True, + ) + return None + + def is_adaptive_aperture_mode_enabled(config_value): if config_value is None: return False @@ -9304,17 +9336,107 @@ def detect_frame_bad_pixels(image_data, ) -def build_persistent_bad_pixel_map(inputfiles, frame_loader, save_directory=None, - minimum_fraction=BAD_PIXEL_DETECTION_FRACTION, - minimum_frames=BAD_PIXEL_PRECHECK_MIN_FRAMES): - inputfiles = list(inputfiles) - total_files = len(inputfiles) - if total_files < minimum_frames: +_BAD_PIXEL_PRECHECK_POOL_CONTEXT = {} + + +def _bad_pixel_precheck_pool_initializer(generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult): + global _BAD_PIXEL_PRECHECK_POOL_CONTEXT + suppress_inherited_tk_cleanup_in_worker() + _BAD_PIXEL_PRECHECK_POOL_CONTEXT = { + 'generalDark': generalDark, + 'generalBias': generalBias, + 'generalFlat': generalFlat, + 'demosaic_fmt': demosaic_fmt, + 'demosaic_out': demosaic_out, + 'demosaic_mult': demosaic_mult, + } + + +def _load_bad_pixel_precheck_worker_frame(file_name): + context = _BAD_PIXEL_PRECHECK_POOL_CONTEXT + hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + image_data = hdul[extension].data + hdul.close() + + image_data = apply_cals( + image_data, + context.get('generalDark'), + context.get('generalBias'), + context.get('generalFlat'), + 1, + ) + image_data = demosaic_img( + image_data, + context.get('demosaic_fmt'), + context.get('demosaic_out'), + context.get('demosaic_mult'), + 1, + ) + return image_data + + +def _bad_pixel_precheck_task(task): + index, file_name = task + try: + frame_data = _load_bad_pixel_precheck_worker_frame(file_name) + except Exception as exc: + return { + 'index': index, + 'file_name': file_name, + 'usable': False, + 'error': str(exc), + } + + frame_mask = detect_frame_bad_pixels(frame_data) + if frame_mask.ndim != 2: + return { + 'index': index, + 'file_name': file_name, + 'usable': False, + 'not_2d': True, + } + + return { + 'index': index, + 'file_name': file_name, + 'usable': True, + 'mask': frame_mask, + } + + +def _merge_bad_pixel_precheck_mask(detection_counts, frame_mask, file_name): + frame_mask = np.asarray(frame_mask, dtype=bool) + if frame_mask.ndim != 2: log_info( - f"Bad-pixel precheck skipped: only {total_files} frame(s); need at least {minimum_frames} frames.", + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} because the frame is not 2-D.", + warn=True, ) - return None + return detection_counts, False + if detection_counts is None: + detection_counts = np.zeros(frame_mask.shape, dtype=np.uint32) + elif detection_counts.shape != frame_mask.shape: + log_info( + "Warning: skipping bad-pixel precheck for " + f"{_display_filename(file_name)} because its shape {frame_mask.shape} does not match " + f"the reference frame shape {detection_counts.shape}.", + warn=True, + ) + return detection_counts, False + + detection_counts += frame_mask.astype(np.uint32) + return detection_counts, True + + +def _scan_bad_pixel_precheck_frames_serial(inputfiles, frame_loader): + total_files = len(inputfiles) detection_counts = None scanned_files = 0 @@ -9330,31 +9452,117 @@ def build_persistent_bad_pixel_map(inputfiles, frame_loader, save_directory=None continue frame_mask = detect_frame_bad_pixels(frame_data) - if frame_mask.ndim != 2: - log_info( - f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} because the frame is not 2-D.", - warn=True, - ) - continue - - if detection_counts is None: - detection_counts = np.zeros(frame_mask.shape, dtype=np.uint32) - elif detection_counts.shape != frame_mask.shape: - log_info( - "Warning: skipping bad-pixel precheck for " - f"{_display_filename(file_name)} because its shape {frame_mask.shape} does not match " - f"the reference frame shape {detection_counts.shape}.", - warn=True, - ) - continue - - detection_counts += frame_mask.astype(np.uint32) - scanned_files += 1 + detection_counts, usable = _merge_bad_pixel_precheck_mask(detection_counts, frame_mask, file_name) + if usable: + scanned_files += 1 completed = index + 1 if completed == total_files or completed % BAD_PIXEL_PROGRESS_LOG_INTERVAL == 0: log_info(f"Bad-pixel precheck progress: {completed}/{total_files}") + return detection_counts, scanned_files + + +def _scan_bad_pixel_precheck_frames_multiprocess(inputfiles, max_processes, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None): + total_files = len(inputfiles) + max_workers = min(max_processes, os.cpu_count() or 1, total_files, MAX_MULTIPROCESS_BAD_PIXEL_WORKERS) + detection_counts = None + scanned_files = 0 + + log_info( + "Using multiprocessing for bad-pixel precheck " + f"with {max_workers} worker(s) across {total_files} image(s)." + ) + + tasks = [(index, str(file_name)) for index, file_name in enumerate(inputfiles)] + with suppress_tk_cleanup_during_process_pool(): + with ProcessPoolExecutor( + max_workers=max_workers, + initializer=_bad_pixel_precheck_pool_initializer, + initargs=( + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ), + ) as executor: + futures = [executor.submit(_bad_pixel_precheck_task, task) for task in tasks] + completed = 0 + for future in as_completed(futures): + result = future.result() + file_name = result.get('file_name') + if result.get('usable'): + detection_counts, usable = _merge_bad_pixel_precheck_mask( + detection_counts, + result.get('mask'), + file_name, + ) + if usable: + scanned_files += 1 + elif result.get('not_2d'): + log_info( + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} " + "because the frame is not 2-D.", + warn=True, + ) + else: + log_info( + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} " + f"({result.get('error')}).", + warn=True, + ) + + completed += 1 + if completed == total_files or completed % BAD_PIXEL_PROGRESS_LOG_INTERVAL == 0: + log_info(f"Bad-pixel precheck progress: {completed}/{total_files}") + + return detection_counts, scanned_files + + +def build_persistent_bad_pixel_map(inputfiles, frame_loader, save_directory=None, + minimum_fraction=BAD_PIXEL_DETECTION_FRACTION, + minimum_frames=BAD_PIXEL_PRECHECK_MIN_FRAMES, + max_processes=None, generalDark=None, generalBias=None, + generalFlat=None, demosaic_fmt=None, demosaic_out=None, + demosaic_mult=None): + inputfiles = list(inputfiles) + total_files = len(inputfiles) + if total_files < minimum_frames: + log_info( + f"Bad-pixel precheck skipped: only {total_files} frame(s); need at least {minimum_frames} frames.", + ) + return None + + try: + max_processes = int(max_processes) if max_processes is not None else None + except (TypeError, ValueError): + max_processes = None + + if max_processes is not None and max_processes > 1 and total_files > 1: + try: + detection_counts, scanned_files = _scan_bad_pixel_precheck_frames_multiprocess( + inputfiles, + max_processes, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + ) + except Exception as exc: + log_info( + f"Warning: bad-pixel precheck multiprocessing failed ({exc}); falling back to serial scanning.", + warn=True, + ) + detection_counts, scanned_files = _scan_bad_pixel_precheck_frames_serial(inputfiles, frame_loader) + else: + detection_counts, scanned_files = _scan_bad_pixel_precheck_frames_serial(inputfiles, frame_loader) + if detection_counts is None or scanned_files < minimum_frames: log_info( f"Bad-pixel precheck skipped: only {scanned_files} usable frame(s); need at least {minimum_frames}.", @@ -10971,6 +11179,9 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( info_dict.get('detect_bad_pixels_before_photometry', 'y') ) + multiprocess_bad_pixel_precheck = get_multiprocess_bad_pixel_precheck_processes( + info_dict.get('multiprocess_bad_pixel_precheck', 'n') + ) plateStatus.initializeFilenames(info_dict['images']) inputfiles = corruption_check(info_dict['images']) @@ -11036,6 +11247,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet inputfiles, load_image_data, save_directory=info_dict['save'], + max_processes=multiprocess_bad_pixel_precheck, ) else: log_info("Bad-pixel precheck disabled per optional_info setting.") @@ -14963,6 +15175,9 @@ def _main_impl(): detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( exotic_infoDict.get('detect_bad_pixels_before_photometry', 'y') ) + multiprocess_bad_pixel_precheck = get_multiprocess_bad_pixel_precheck_processes( + exotic_infoDict.get('multiprocess_bad_pixel_precheck', 'n') + ) inputfiles, wcs_keep_mask, dropped_wcs_files = filter_sparse_missing_wcs_frames( inputfiles, ignore_header_wcs=ignore_header_wcs, @@ -15031,6 +15246,13 @@ def _main_impl(): demosaic_mult, ), save_directory=exotic_infoDict['save'], + max_processes=multiprocess_bad_pixel_precheck, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, ) else: log_info("Bad-pixel precheck disabled per optional_info setting.") diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 461d85fb..69dbc43c 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -419,6 +419,7 @@ def save_input(): "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", + "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", @@ -447,6 +448,7 @@ def save_input(): "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "y", + "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", @@ -1501,6 +1503,7 @@ def save_input(): "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", + "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", @@ -1577,6 +1580,7 @@ def save_input(): "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "y", + "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", @@ -1632,6 +1636,7 @@ def save_input(): "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "y", + "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", diff --git a/exotic/inputs.py b/exotic/inputs.py index 73b92b4d..dcffde08 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -217,6 +217,7 @@ def __init__(self, init_opt): 'assess_all_comparisons_before_selecting_best': 'y', 'exit_at_first_qc_pass_solution': 'y', 'detect_bad_pixels_before_photometry': 'y', + 'multiprocess_bad_pixel_precheck': 'n', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, @@ -437,6 +438,11 @@ def comp_params(self, init_file, planet_dict): 'detect_bad_pixels_before_photometry', 'Detect Bad Pixels Before Photometry? (y/n)', ), + 'multiprocess_bad_pixel_precheck': ( + 'multiprocess_bad_pixel_precheck', + 'Multiprocess Bad-Pixel Precheck? (y/n or process count)', + 'Multiprocess Bad Pixel Precheck? (y/n or process count)', + ), 'detrend_on_outoftransit_baseline': ( 'detrend_on_outoftransit_baseline', 'Detrend on Out-of-Transit Baseline', diff --git a/exotic/plots.py b/exotic/plots.py index 9641374e..c31a0f10 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -634,6 +634,7 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date): fit, show_flux_baseline_label=False, show_model_uncertainty=True, + show_baseline_uncertainty=True, ) ax_lc.set_title(targ_name) diff --git a/inits.json b/inits.json index 60746700..92371515 100644 --- a/inits.json +++ b/inits.json @@ -28,6 +28,7 @@ "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Set to 0 to disable. Default 4.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", + "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", @@ -116,6 +117,7 @@ "pointing_rejection_sigma": 4.0, "disable vertical flux normalization": false, "detect_bad_pixels_before_photometry": "y", + "multiprocess_bad_pixel_precheck": "y", "detrend_on_outoftransit_baseline": true, "final_fit_baseline_duration_multiplier": 1.0, "use_eebls_to_initialize_tmid_and_bounds": "y", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 6afe0dab..22a7b7d3 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -144,6 +144,56 @@ def test_build_persistent_bad_pixel_map_thresholds_recurrence_and_saves_outputs( assert not mask_image[6, 5] +def test_build_persistent_bad_pixel_map_can_scan_with_multiprocessing(tmp_path, monkeypatch): + paths = [] + for frame_index in range(10): + frame = np.full((9, 9), 100.0, dtype=float) + if frame_index < 4: + frame[2, 3] = 4000.0 + path = tmp_path / f"frame_{frame_index}.fits" + fits.writeto(path, frame, overwrite=True) + paths.append(path) + + captured = {} + + class FakeFuture: + def __init__(self, value): + self._value = value + + def result(self): + return self._value + + class FakeExecutor: + def __init__(self, max_workers, initializer=None, initargs=()): + captured["max_workers"] = max_workers + if initializer is not None: + initializer(*initargs) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def submit(self, fn, task): + return FakeFuture(fn(task)) + + monkeypatch.setattr(exotic_module, "ProcessPoolExecutor", FakeExecutor) + monkeypatch.setattr(exotic_module, "as_completed", lambda futures: futures) + + reference = exotic_module.build_persistent_bad_pixel_map( + paths, + exotic_module.load_image_data, + save_directory=tmp_path, + max_processes=2, + ) + + assert captured["max_workers"] == 2 + assert reference is not None + assert reference["required_count"] == 4 + assert reference["mask"][2, 3] + + def test_repair_bad_pixels_in_frame_replaces_known_bad_pixel_with_neighbor_median(): image = np.arange(25, dtype=float).reshape(5, 5) image[2, 2] = 9999.0 diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 546942c2..56939a3a 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -435,6 +435,59 @@ def test_transit_model_uncertainty_includes_baseline_terms(monkeypatch, tmp_path assert np.nanmax(envelope[1] - envelope[0]) > 0 +def test_baseline_model_uncertainty_is_centered_on_unity_and_includes_a2(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.airmass = np.linspace(1.0, 2.0, time.size) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.1 + fit.errors = {"a0": 0.01, "a1": 0.01, "a2": 0.05} + + lower, upper = fit.baseline_model_uncertainty(time) + width = upper - lower + + np.testing.assert_allclose(0.5 * (lower + upper), np.ones_like(time), atol=1e-12) + assert np.nanmin(lower) < 1.0 + assert np.nanmax(upper) > 1.0 + assert width[0] > width[len(width) // 2] + + +def test_plot_bestfit_can_draw_baseline_uncertainty_band(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + airmass = np.linspace(1.0, 2.0, time.size) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + fit.parameters["a2"] = 0.1 + fit.errors["a0"] = 0.01 + fit.errors["a2"] = 0.05 + + fig, axes = fit.plot_bestfit(show_baseline_uncertainty=True) + labels = [artist.get_label() for artist in axes[0].collections] + + assert r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty' in labels + plt.close(fig) + + def test_posterior_model_uncertainty_recenters_on_best_fit_model(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 5f91428c..57fe2d4a 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -121,6 +121,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: fit_lightcurve_to_every_comparison_candidate, fit_ranked_comparison_calibration_candidates, get_final_fit_baseline_duration_multiplier, + get_multiprocess_bad_pixel_precheck_processes, estimate_ephemeris_tmid_and_bounds, estimate_tmid_and_bounds_with_eebls, is_adaptive_aperture_mode_enabled, @@ -789,6 +790,15 @@ def test_should_detect_bad_pixels_before_photometry_parses_values(): assert should_detect_bad_pixels_before_photometry("n") is False +def test_get_multiprocess_bad_pixel_precheck_processes_parses_values(): + assert get_multiprocess_bad_pixel_precheck_processes(None) is None + assert get_multiprocess_bad_pixel_precheck_processes("n") is None + assert get_multiprocess_bad_pixel_precheck_processes("0") is None + assert get_multiprocess_bad_pixel_precheck_processes("y") >= 1 + assert get_multiprocess_bad_pixel_precheck_processes("3") == 3 + assert get_multiprocess_bad_pixel_precheck_processes(2) == 2 + + def test_is_out_of_transit_baseline_detrending_enabled_parses_values(): assert is_out_of_transit_baseline_detrending_enabled(None) is True assert is_out_of_transit_baseline_detrending_enabled("y") is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py index ea4b09bf..f2f982ba 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -218,6 +218,21 @@ def test_comp_params_defaults_detect_bad_pixels_before_photometry_to_yes(tmp_pat assert inputs.info_dict["detect_bad_pixels_before_photometry"] == "y" +def test_comp_params_defaults_multiprocess_bad_pixel_precheck_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["multiprocess_bad_pixel_precheck"] == "n" + + def test_comp_params_defaults_detrend_on_outoftransit_baseline_to_true(tmp_path): init_data = { "user_info": {}, @@ -533,6 +548,21 @@ def test_comp_params_reads_detect_bad_pixels_before_photometry_from_optional_inf assert inputs.info_dict["detect_bad_pixels_before_photometry"] == "n" +def test_comp_params_reads_multiprocess_bad_pixel_precheck_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"multiprocess_bad_pixel_precheck": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["multiprocess_bad_pixel_precheck"] == "y" + + def test_comp_params_reads_detrend_on_outoftransit_baseline_from_optional_info(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_plots.py b/tests/test_plots.py index ab6ef35a..09f4fd79 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -211,17 +211,19 @@ def plot_bestfit(self, phase=False, show_flux_baseline_label=True): assert not (tmp_path / "temp" / "CompStarLightCurveFit_Comp3_Target_2026-03-09.png").exists() -def test_plot_final_lightcurve_requests_uncertainty_band_without_baseline_label(tmp_path): +def test_plot_final_lightcurve_requests_uncertainty_bands_without_baseline_label(tmp_path): class DummyFinalFit: def __init__(self): self.kwargs = None self.phase_upsample = np.linspace(-0.05, 0.05, 5) self.transit_upsample = np.ones(5) - def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False): + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False): self.kwargs = { "show_flux_baseline_label": show_flux_baseline_label, "show_model_uncertainty": show_model_uncertainty, + "show_baseline_uncertainty": show_baseline_uncertainty, } fig, axes = plt.subplots(2, 1) return fig, axes @@ -239,6 +241,7 @@ def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=Fal assert fit.kwargs == { "show_flux_baseline_label": False, "show_model_uncertainty": True, + "show_baseline_uncertainty": True, } assert (tmp_path / "FinalLightCurve_Target_2026-03-09.png").exists() assert (tmp_path / "FinalLightCurve_Target_2026-03-09.pdf").exists() From 5fe0ab9d3aff0b295fe5f367a2789c1ef0ae8ef4 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 13 May 2026 12:22:51 +1000 Subject: [PATCH 042/116] ultranest in windows working. --- docs/system_prompt.txt | 2 +- exotic/api/elca.py | 158 ++++------------------------- exotic/api/joint_fitter.py | 8 +- exotic/exotic.py | 109 +++++++++++++++----- requirements.txt | 3 +- tests/test_centroid_wcs.py | 40 ++++++++ tests/test_exotic_proper_motion.py | 90 ++++++++++++++++ 7 files changed, 237 insertions(+), 173 deletions(-) diff --git a/docs/system_prompt.txt b/docs/system_prompt.txt index 7ee1c6c3..e79bc8d9 100644 --- a/docs/system_prompt.txt +++ b/docs/system_prompt.txt @@ -724,7 +724,7 @@ where $F_{obs}$ is the flux measured from the detector, $F_{transit}$ is the mod To model the transit lightcurve with {PyLightcurve}, considering the brightness variation from a star's edge (limb) to its center, EXOTIC generates nonlinear four-parameter limb darkening coefficients. The pipeline uses {ldtk} to calculate these coefficients from the star's temperature $T$, metallicity, surface gravity log $g$, and the observational filter based on PHOENIX stellar atmosphere models. ## Nested Sampler -{Ultranest} and {dynesty} are used for Bayesian inference and statistical analysis, each characterized by distinct implementation approaches and features. Nested sampling outperforms Markov Chain Monte Carlo (MCMC) in handling multi-modal and degenerate posteriors by not relying on a thermal transition property and avoiding the burn-in phase, making it a more efficient and robust data analysis method for astrophysical applications. Both nested sampling algorithms employ multiple ellipsoid bounds to outline the parameter space within which the free parameters can traverse. In {dynesty}, we opt to use the {DynamicNestedSampler} method due to its ability to adjust live points to distribute samples more efficiently, thereby speeding up computation. The choice of the nested sampler package depends on the availability of a C compiler and libraries. For users with Mac, Unix, Linux, or Windows systems with a C compiler installed, EXOTIC uses {ultranest}. For Windows systems lacking a C compiler, EXOTIC utilizes {dynesty}. +{Ultranest} is used for Bayesian inference and statistical analysis. Nested sampling outperforms Markov Chain Monte Carlo (MCMC) in handling multi-modal and degenerate posteriors by not relying on a thermal transition property and avoiding the burn-in phase, making it a more efficient and robust data analysis method for astrophysical applications. UltraNest employs multiple ellipsoid bounds to outline the parameter space within which the free parameters can traverse. EXOTIC uses {Ultranest} on Mac, Unix, Linux, and Windows systems. The nested sampling algorithm uses $T_{mid}$, $R_{p}/R_{s}$, and $i$ as free parameters with a uniform distribution in a bounded interval. Meanwhile, the sampler treats the remaining parameters ($a/R_s$, $e$, $w$, along with four-parameter limb darkening coefficients) as fixed. These free parameters are chosen based on their ability to constrain the ephemeris and estimate system parameters. Although we model the lightcurve using a least-squares fit with the Levenberg-Marquardt (LM) algorithm, we derive the final values and uncertainties using the nested sampler. If the Bayesian evidence stabilizes within a threshold (0.05) after each iteration, the algorithm has converged, indicating no further information can be gained from sampling. For well-constrained problems (i.e., each parameter having a single mode), convergence typically occurs within ~10,000-20,000 iterations. This efficiency makes nested sampling faster than Markov Chain Monte Carlo (MCMC), which usually requires around ~100,000 samples and lacks early stopping criteria. The corner plot shown in the triangle plot showcases histograms illustrating the marginalized posterior distributions for each parameter, accompanied by their estimated values and uncertainties. The sampler implements parameter constraints to ensure robust parameter exploration within established physical boundaries (e.g., the orbital inclination can not exceed $90^\circ$ as demonstrated in triangle plot. Scatter plots within the corner plot visually depict correlations among the free parameters, offering insights into their joint posterior distributions. Additionally, the corner plot facilitates the identification of parameter degeneracies, highlighting scenarios where alterations in one parameter impact another (see subplots in triangle plot that demonstrate degeneracies with their elliptical-shaped distributions). diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 3c2e0f88..a1f03ccc 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -54,13 +54,7 @@ from scipy import spatial from scipy.optimize import least_squares from scipy.signal import savgol_filter -try: - from ultranest import ReactiveNestedSampler -except ImportError: - import dynesty - import dynesty.plotting - from dynesty.utils import resample_equal - from scipy.stats import gaussian_kde +from ultranest import ReactiveNestedSampler try: from plotting import corner @@ -1539,11 +1533,7 @@ def _get_triangle_plot_samples(self): ) return points, logl, weights - points = np.asarray(self.results.samples, dtype=float) - weights = np.exp(self.results.logwt - self.results.logz[-1]) - index_samples = resample_equal(np.arange(points.shape[0], dtype=float)[:, None], weights) - index_samples = np.clip(np.rint(index_samples[:, 0]).astype(int), 0, points.shape[0] - 1) - return points[index_samples], np.asarray(self.results.logl, dtype=float)[index_samples], None + raise RuntimeError("Triangle plots require an UltraNest nested-sampling result.") def _get_triangle_plot_sample_weights(self, weights, sample_count): if weights is None: @@ -2927,133 +2917,29 @@ def prior_transform(upars): # transform unit cube to prior volume return self._sample_point_from_unit_cube(upars, bound_keys) - try: - self.ns_type = 'ultranest' - test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform, vectorized=True) - - run_kwargs = {"max_ncalls": int(self.max_ncalls)} - if self.ultranest_min_num_live_points is not None: - run_kwargs["min_num_live_points"] = int(self.ultranest_min_num_live_points) + self.ns_type = 'ultranest' + test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform, vectorized=True) - self.results = run_reactive_sampler( - test, - run_kwargs=run_kwargs, - verbose=self.verbose, - ) - - if self.keep_ultranest_sampler: - self._ultranest_resume_context = { - 'sampler': test, - 'bound_keys': list(bound_keys), - 'sampled_keys': list(sampled_keys), - 'physical_from_sample_point': physical_from_sample_point, - } - else: - self._ultranest_resume_context = None - self._finalize_ultranest_fit_results(bound_keys, sampled_keys, physical_from_sample_point) - except NameError: - self.ns_type = 'dynesty' - dsampler = dynesty.DynamicNestedSampler(loglike, prior_transform, ndim=len(sampled_keys), - bound='multi', sample='unif') - dsampler.run_nested(maxcall=int(1e5), dlogz_init=0.05, - maxbatch=10, nlive_batch=100, print_progress=self.verbose) - self.results = dsampler.results - - tests = [np.zeros(len(sampled_keys), dtype=float) for _ in range(5)] - - # Derive kernel density estimate for best fit - weights = np.exp(self.results.logwt - self.results.logz[-1]) - samples = self.results['samples'] - logvol = self.results['logvol'] - wt_kde = gaussian_kde(resample_equal(-logvol, weights)) # KDE - logvol_grid = np.linspace(logvol[0], logvol[-1], 1000) # resample - wt_grid = wt_kde.pdf(-logvol_grid) # evaluate KDE PDF - self.weights = np.interp(-logvol, -logvol_grid, wt_grid) # interpolate - - # errors + final values - mean, cov = dynesty.utils.mean_and_cov(self.results.samples, weights) - mean2, cov2 = dynesty.utils.mean_and_cov(self.results.samples, self.weights) - for i in range(len(sampled_keys)): - self.sample_errors[sampled_keys[i]] = cov[i, i] ** 0.5 - tests[0][i] = mean[i] - tests[1][i] = mean2[i] - - counts, bins = np.histogram(samples[:, i], bins=100, weights=weights) - mi = np.argmax(counts) - tests[4][i] = bins[mi] + 0.5 * np.mean(np.diff(bins)) - - # finds median and +- 2sigma, will vary from mode if non-gaussian - self.sample_quantiles[sampled_keys[i]] = dynesty.utils.quantile( - self.results.samples[:, i], - [0.025, 0.5, 0.975], - weights=weights, - ) - tests[2][i] = self.sample_quantiles[sampled_keys[i]][1] - - # find minimum near weighted mean - mask = (samples[:, 0] < mean[0] + 2 * self.sample_errors[sampled_keys[0]]) & ( - samples[:, 0] > mean[0] - 2 * self.sample_errors[sampled_keys[0]]) - bi = np.argmin(self.weights[mask]) - - for i in range(len(sampled_keys)): - tests[3][i] = samples[mask][bi, i] - # tests[4][freekeys[i]] = np.average(samples[mask][:, i], weights=self.weights[mask], axis=0) - - # find best fit from chi2 minimization - chis = [] - physical_tests = [] - for i in range(len(tests)): - test_values = physical_from_sample_point(tests[i]) - lightcurve = transit(self.time, test_values) - if self._has_free_flux_baseline(): - flux_scale = get_flux_baseline(test_values) - elif self._uses_fixed_flux_baseline(): - flux_scale = get_flux_baseline(test_values) - else: - flux_scale = mc_a1( - test_values.get('a2', 0), - self.errors.get('a2', 1e-6), - lightcurve, - self.airmass, - self.data, - self.dataerr, - mask=self._get_baseline_fit_mask(), - )[0] - test_values['a0'] = flux_scale - test_values['a1'] = flux_scale - airmass = flux_scale * airmass_trend( - test_values.get('a2', 0), - self.airmass, - reference=self._get_airmass_reference(), - ) - residuals = self.data - (lightcurve * airmass) - chis.append(np.sum(residuals ** 2)) - physical_tests.append(test_values) + run_kwargs = {"max_ncalls": int(self.max_ncalls)} + if self.ultranest_min_num_live_points is not None: + run_kwargs["min_num_live_points"] = int(self.ultranest_min_num_live_points) - mi = np.argmin(chis) - self.parameters = copy.deepcopy(physical_tests[mi]) - self.sample_bounds = self._get_sample_bounds(bound_keys, self.parameters) - self.sample_parameters = {key: tests[mi][i] for i, key in enumerate(sampled_keys)} + self.results = run_reactive_sampler( + test, + run_kwargs=run_kwargs, + verbose=self.verbose, + ) - for bound_key, sampled_key in zip(bound_keys, sampled_keys): - if bound_key == 'inc' and sampled_key == 'b': - continue - self.errors[bound_key] = self.sample_errors[sampled_key] - self.quantiles[bound_key] = self.sample_quantiles[sampled_key] - - if 'inc' in bound_keys and 'b' in sampled_keys: - inc_samples = np.array([ - physical_from_sample_point(point)['inc'] - for point in samples - ]) - center, std, quantiles = self._summarize_derived_parameter(inc_samples, self.parameters['inc']) - self.parameters['inc'] = center - self.errors['inc'] = std - self.quantiles['inc'] = quantiles - else: - for key in sampled_keys: - self.sample_errors.setdefault(key, 0.0) - self.sample_quantiles.setdefault(key, [0, 0, 0]) + if self.keep_ultranest_sampler: + self._ultranest_resume_context = { + 'sampler': test, + 'bound_keys': list(bound_keys), + 'sampled_keys': list(sampled_keys), + 'physical_from_sample_point': physical_from_sample_point, + } + else: + self._ultranest_resume_context = None + self._finalize_ultranest_fit_results(bound_keys, sampled_keys, physical_from_sample_point) if not self.sample_parameters: self.sample_parameters = { diff --git a/exotic/api/joint_fitter.py b/exotic/api/joint_fitter.py index 8a3e152f..8257f009 100644 --- a/exotic/api/joint_fitter.py +++ b/exotic/api/joint_fitter.py @@ -47,13 +47,7 @@ import matplotlib.pyplot as plt import numpy as np from scipy import stats -try: - from ultranest import ReactiveNestedSampler -except ImportError: - import dynesty - import dynesty.plotting - from dynesty.utils import resample_equal - from scipy.stats import gaussian_kde +from ultranest import ReactiveNestedSampler try: from elca import glc_fitter, lc_fitter diff --git a/exotic/exotic.py b/exotic/exotic.py index 2109ffa5..9707dab5 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -63,12 +63,13 @@ import inspect import json import hashlib +import multiprocessing import os import shutil import sys import threading import traceback -from concurrent.futures import ProcessPoolExecutor, as_completed +from concurrent.futures import ProcessPoolExecutor as _ProcessPoolExecutor, ThreadPoolExecutor, as_completed from time import sleep, perf_counter # Image alignment import import astroalign as aa @@ -4390,6 +4391,57 @@ def validate_ultranest_mpi_runtime(): raise RuntimeError(message) +def configure_windows_multiprocessing_main_spec(): + if sys.platform != "win32": + return False + + configured = False + spawn_executable = _windows_python_spawn_executable() + if spawn_executable: + multiprocessing.set_executable(spawn_executable) + if getattr(sys, "frozen", False): + sys.frozen = False + configured = True + + main_module = sys.modules.get("__main__") + if main_module is None: + return configured + + main_file = getattr(main_module, "__file__", None) + if not main_file or os.path.basename(os.fspath(main_file)).lower() not in {"exotic.exe", "exotic-script.py"}: + return configured + + if getattr(main_module, "__spec__", None) is not None: + main_module.__spec__ = None + main_module.__file__ = None + if getattr(main_module, "__package__", None) is not None: + main_module.__package__ = None + + return True + + +def ProcessPoolExecutor(*args, **kwargs): + if sys.platform == "win32": + return ThreadPoolExecutor(*args, **kwargs) + return _ProcessPoolExecutor(*args, **kwargs) + + +def _windows_python_spawn_executable(): + candidates = [ + getattr(sys, "_base_executable", None), + sys.executable, + os.path.join(sys.exec_prefix, "python.exe"), + os.path.join(getattr(sys, "base_exec_prefix", sys.exec_prefix), "python.exe"), + ] + for candidate in candidates: + if not candidate: + continue + executable = os.fspath(candidate) + if os.path.basename(executable).lower() in {"python.exe", "pythonw.exe"}: + return executable + return None + + def should_use_psf_photometry(config_value): if config_value is None: return True @@ -9744,6 +9796,7 @@ def alignmentError(self): self.warnings.append(('alignment_error', -1, np.nan, np.nan)) +_PLATE_STATUS_SWAP_LOCK = threading.RLock() _ALIGNMENT_POOL_CONTEXT = {} @@ -9905,32 +9958,33 @@ def build_multiprocess_pointing_precheck_transforms(inputfiles, max_processes, r def _fit_alignment_candidate_psfs(image_data, predicted_coords, target_fast_centroid, frame_fast_centroid): global plateStatus predicted_coords = np.asarray(predicted_coords, dtype=float) - original_plate_status = plateStatus - recorder = _ParallelPlateStatusRecorder() - plateStatus = recorder - try: - psf_rows = { - 'target': fit_centroid_or_warn_out_of_frame( - image_data, - choose_centroid_seed_position(predicted_coords[0], None), - 0, - fast_mode=target_fast_centroid, - ) - } - for comp_idx in range(max(0, predicted_coords.shape[0] - 1)): - psf_rows[f"comp{comp_idx + 1}"] = fit_centroid_or_warn_out_of_frame( - image_data, - choose_centroid_seed_position(predicted_coords[comp_idx + 1], None), - comp_idx + 1, - fast_mode=frame_fast_centroid, - ) - return { - 'coords': predicted_coords, - 'psf_rows': psf_rows, - 'warnings': list(recorder.warnings), - } - finally: - plateStatus = original_plate_status + with _PLATE_STATUS_SWAP_LOCK: + original_plate_status = plateStatus + recorder = _ParallelPlateStatusRecorder() + plateStatus = recorder + try: + psf_rows = { + 'target': fit_centroid_or_warn_out_of_frame( + image_data, + choose_centroid_seed_position(predicted_coords[0], None), + 0, + fast_mode=target_fast_centroid, + ) + } + for comp_idx in range(max(0, predicted_coords.shape[0] - 1)): + psf_rows[f"comp{comp_idx + 1}"] = fit_centroid_or_warn_out_of_frame( + image_data, + choose_centroid_seed_position(predicted_coords[comp_idx + 1], None), + comp_idx + 1, + fast_mode=frame_fast_centroid, + ) + return { + 'coords': predicted_coords, + 'psf_rows': psf_rows, + 'warnings': list(recorder.warnings), + } + finally: + plateStatus = original_plate_status def _parallel_alignment_task(task): @@ -14825,6 +14879,7 @@ def _main_impl(): raise ValueError("--multiprocess-transformations requires an integer greater than 0.") if args.multiprocess_lightcurve_fits is not None and args.multiprocess_lightcurve_fits < 1: raise ValueError("--multiprocess-lightcurve-fits requires an integer greater than 0.") + configure_windows_multiprocessing_main_spec() validate_ultranest_mpi_runtime() log.debug("*************************") diff --git a/requirements.txt b/requirements.txt index d8712faf..3f4b7730 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ barycorrpy~=0.4.4 bottleneck~=1.4.2 colour_demosaicing==0.2.6 colour-science>=0.4.4,<0.4.7 -dynesty~=1.2.3;platform_system=='Windows' holoviews~=1.19.1 importlib-metadata>=3.6;python_version<='3.7' imreg_dft~=2.0.0 @@ -30,5 +29,5 @@ tifffile<2026.4.11 statsmodels~=0.14.4 tenacity~=9.0 mpi4py>=4.0;platform_system=='Linux' -ultranest==4.5.0;platform_system!='Windows' +ultranest==4.5.0 zstandard~=0.23.0 diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 22a7b7d3..45ba840e 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -2,6 +2,8 @@ import sys import types import importlib.util +import threading +from concurrent.futures import ThreadPoolExecutor import numpy as np import pytest @@ -747,6 +749,44 @@ def test_parallel_alignment_task_uses_precomputed_fallback_transform(monkeypatch assert np.allclose(result["fallback"]["coords"], [[6.0, 1.0], [8.0, 3.0]]) +def test_fit_alignment_candidate_psfs_serializes_plate_status_swap(monkeypatch): + sentinel_status = types.SimpleNamespace(name="original-plate-status") + started = threading.Event() + + def fake_fit_centroid(_data, pos, starIndex, **_kwargs): + started.set() + return np.array([float(starIndex), float(pos[0]), float(pos[1])], dtype=float) + + monkeypatch.setattr(exotic_module, "plateStatus", sentinel_status) + monkeypatch.setattr(exotic_module, "fit_centroid_or_warn_out_of_frame", fake_fit_centroid) + + lock = exotic_module._PLATE_STATUS_SWAP_LOCK + lock.acquire() + executor = ThreadPoolExecutor(max_workers=1) + future = None + try: + future = executor.submit( + exotic_module._fit_alignment_candidate_psfs, + np.ones((5, 5), dtype=float), + np.array([[1.0, 1.0], [2.0, 2.0]], dtype=float), + False, + False, + ) + assert not started.wait(0.2) + finally: + lock.release() + + try: + result = future.result(timeout=2) + finally: + executor.shutdown(wait=True) + + assert started.wait(0.2) + assert result["psf_rows"]["target"].tolist() == [0.0, 1.0, 1.0] + assert result["psf_rows"]["comp1"].tolist() == [1.0, 2.0, 2.0] + assert exotic_module.plateStatus is sentinel_status + + def test_filter_sparse_missing_wcs_frames_drops_files_below_three_percent(monkeypatch): frames = [f"frame_{i}.fits" for i in range(34)] missing_frame = frames[7] diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 57fe2d4a..3361acaf 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -109,6 +109,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: comparison_candidate_fit_selection_reason, comparison_star_coverage_summary, comparison_star_stability_summary, + configure_windows_multiprocessing_main_spec, deduplicate_comparison_star_coords, diagnose_lightcurve_fit_inputs, detrend_flux_on_out_of_transit_baseline, @@ -880,6 +881,95 @@ def test_validate_ultranest_mpi_runtime_rejects_whole_program_mpi(monkeypatch): exotic_module.validate_ultranest_mpi_runtime() +def test_configure_windows_multiprocessing_main_spec_retargets_console_launcher(monkeypatch): + import exotic.exotic as exotic_module + + fake_main = types.SimpleNamespace( + __spec__=types.SimpleNamespace(name="exotic"), + __file__=r"C:\Python312\Scripts\exotic.exe", + __package__="", + ) + spawn_executables = [] + + monkeypatch.setattr(exotic_module.sys, "platform", "win32") + monkeypatch.setattr(exotic_module.sys, "_base_executable", r"C:\Python312\python.exe", raising=False) + monkeypatch.setattr(exotic_module.sys, "executable", r"C:\Python312\Scripts\exotic.exe") + monkeypatch.setattr(exotic_module.sys, "frozen", True, raising=False) + monkeypatch.setattr(exotic_module.multiprocessing, "set_executable", spawn_executables.append) + monkeypatch.setitem(sys.modules, "__main__", fake_main) + + assert configure_windows_multiprocessing_main_spec() is True + assert fake_main.__spec__ is None + assert fake_main.__file__ is None + assert fake_main.__package__ is None + assert spawn_executables == [r"C:\Python312\python.exe"] + assert exotic_module.sys.frozen is False + + +def test_configure_windows_multiprocessing_main_spec_skips_non_windows(monkeypatch): + import exotic.exotic as exotic_module + + fake_main = types.SimpleNamespace(__spec__=types.SimpleNamespace(name="exotic"), __file__="exotic.exe") + + monkeypatch.setattr(exotic_module.sys, "platform", "linux") + monkeypatch.setitem(sys.modules, "__main__", fake_main) + + assert configure_windows_multiprocessing_main_spec() is False + assert fake_main.__spec__.name == "exotic" + + +def test_configure_windows_multiprocessing_main_spec_preserves_regular_script(monkeypatch): + import exotic.exotic as exotic_module + + fake_spec = types.SimpleNamespace(name="run_exotic") + fake_main = types.SimpleNamespace( + __spec__=fake_spec, + __file__=r"C:\work\run_exotic.py", + __package__="", + ) + + monkeypatch.setattr(exotic_module.sys, "platform", "win32") + monkeypatch.setattr(exotic_module.sys, "_base_executable", r"C:\Python312\python.exe", raising=False) + monkeypatch.setattr(exotic_module.sys, "executable", r"C:\Python312\python.exe") + monkeypatch.setattr(exotic_module.multiprocessing, "set_executable", lambda _path: None) + monkeypatch.setitem(sys.modules, "__main__", fake_main) + + assert configure_windows_multiprocessing_main_spec() is True + assert fake_main.__spec__ is fake_spec + assert fake_main.__file__ == r"C:\work\run_exotic.py" + + +def test_windows_python_spawn_executable_falls_back_to_exec_prefix(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module.sys, "_base_executable", r"C:\Python312\Scripts\exotic.exe", raising=False) + monkeypatch.setattr(exotic_module.sys, "executable", r"C:\Python312\Scripts\exotic.exe") + monkeypatch.setattr(exotic_module.sys, "exec_prefix", r"C:\Python312") + monkeypatch.setattr(exotic_module.sys, "base_exec_prefix", r"C:\Python312", raising=False) + + assert exotic_module._windows_python_spawn_executable() == r"C:\Python312\python.exe" + + +def test_process_pool_executor_uses_threads_on_windows(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + class FakeThreadPoolExecutor: + def __init__(self, *args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + + monkeypatch.setattr(exotic_module.sys, "platform", "win32") + monkeypatch.setattr(exotic_module, "ThreadPoolExecutor", FakeThreadPoolExecutor) + + executor = exotic_module.ProcessPoolExecutor(max_workers=3, initializer=lambda: None) + + assert isinstance(executor, FakeThreadPoolExecutor) + assert captured["kwargs"]["max_workers"] == 3 + assert "initializer" in captured["kwargs"] + + def test_build_time_rejection_diagnostic_groups_contiguous_ranges(): times = np.array([1.0, 1.1, 1.2, 1.5, 1.6, 2.0], dtype=float) keep_mask = np.array([True, False, False, True, False, True], dtype=bool) From 5c6751b59db26a1fcbeed3826c040d4d11422b73 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 15 May 2026 08:23:20 +1000 Subject: [PATCH 043/116] uncertaintys and junk --- exotic/api/elca.py | 50 ++- exotic/exotic.py | 537 ++++++++++++++++++++++++++-- exotic/output_files.py | 67 +++- exotic/plots.py | 13 +- tests/test_elca_baseline.py | 89 ++++- tests/test_nextastro_variability.py | 122 +++++++ tests/test_output_files.py | 85 ++++- 7 files changed, 908 insertions(+), 55 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index a1f03ccc..46f5dcf4 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -78,6 +78,8 @@ ) BASELINE_MODEL_UNCERTAINTY_KEYS = ('a0', 'a1', 'a2') MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT = 2000 +ULTRANEST_INFLATED_ERROR_REPLACEMENT_FACTOR = 3.0 +ULTRANEST_LOCAL_UNCERTAINTY_MAX_DELTA_CHI2 = 9.0 def _pylightcurve_import_watchdog_seconds(): try: @@ -1207,6 +1209,29 @@ def _ultranest_error_needs_sample_fallback(self, parameter_index, center, report absolute_floor = max(abs(float(center)) * 1e-12, np.finfo(float).eps) return reported_error <= absolute_floor or reported_error < sample_scale * 1e-6 + def _ultranest_error_is_inflated_relative_to_local_fit(self, reported_error, local_uncertainty): + if not isinstance(local_uncertainty, dict): + return False + + try: + reported_error = float(reported_error) + local_error = float(local_uncertainty.get('error', np.nan)) + delta_chi2 = float(local_uncertainty.get('delta_chi2', np.inf)) + except (TypeError, ValueError): + return False + + if ( + not np.isfinite(reported_error) + or reported_error <= 0 + or not np.isfinite(local_error) + or local_error <= 0 + ): + return False + if not np.isfinite(delta_chi2) or delta_chi2 > ULTRANEST_LOCAL_UNCERTAINTY_MAX_DELTA_CHI2: + return False + + return reported_error > local_error * ULTRANEST_INFLATED_ERROR_REPLACEMENT_FACTOR + def _get_plot_range(self, key): sample_parameters = getattr(self, 'sample_parameters', {}) sample_errors = getattr(self, 'sample_errors', {}) @@ -2773,8 +2798,15 @@ def _finalize_ultranest_fit_results(self, bound_keys, sampled_keys, physical_fro self.results['posterior']['errup'][i]] if self._ultranest_error_needs_sample_fallback(i, ml_point[i], reported_error): fallback = self._loglike_neighborhood_uncertainty(i, ml_point[i]) + if fallback is not None: + fallback['reason'] = 'degenerate_posterior_summary' else: fallback = None + local_uncertainty = self._loglike_neighborhood_uncertainty(i, ml_point[i]) + if self._ultranest_error_is_inflated_relative_to_local_fit(reported_error, local_uncertainty): + fallback = local_uncertainty + fallback['reported_error'] = float(reported_error) + fallback['reason'] = 'posterior_summary_inflated_relative_to_local_fit' if fallback is not None: self.sample_errors[key] = fallback['error'] self.sample_quantiles[key] = fallback['quantiles'] @@ -3000,16 +3032,18 @@ def plot_bestfit( else: if phase: axs[0].errorbar(self.phase, self.detrended, yerr=np.std(self.residuals) / np.median(self.data), - ls='none', marker='.', color='black', zorder=1, alpha=0.2) + ls='none', marker='.', color='black', ecolor='0.72', + elinewidth=1.0, zorder=1, alpha=1.0) else: axs[0].errorbar(self.time, self.detrended, yerr=np.std(self.residuals) / np.median(self.data), - ls='none', marker='.', color='black', zorder=1, alpha=0.2) + ls='none', marker='.', color='black', ecolor='0.72', + elinewidth=1.0, zorder=1, alpha=1.0) if phase: si = np.argsort(self.phase) bt2, br2, _ = time_bin(self.phase[si] * self.parameters['per'], self.residuals[si] / np.median(self.data) * 1e2, bin_dt) - axs[1].plot(self.phase, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=0.2, + axs[1].plot(self.phase, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=1.0, label=r'$\sigma$ = {:.2f} %'.format(np.std(self.residuals / np.median(self.data) * 1e2))) axs[1].plot(bt2 / self.parameters['per'], br2, 'bs', alpha=1, zorder=2) axs[1].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) @@ -3027,7 +3061,7 @@ def plot_bestfit( self.phase_upsample, self.time_upsample, sii, - label=r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty', + label='_nolegend_', ) if show_model_uncertainty: self._plot_transit_model_uncertainty( @@ -3035,14 +3069,14 @@ def plot_bestfit( self.phase_upsample, self.time_upsample, sii, - label=r'1-$\sigma$ model uncertainty', + label='_nolegend_', ) axs[0].plot(self.phase_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel) axs[0].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) axs[0].set_xlabel("Phase ", fontsize=14) else: bt, br, _ = time_bin(self.time, self.residuals / np.median(self.data) * 1e2, bin_dt) - axs[1].plot(self.time, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=0.2, + axs[1].plot(self.time, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=1.0, label=r'$\sigma$ = {:.2f} %'.format(np.std(self.residuals / np.median(self.data) * 1e2))) axs[1].plot(bt, br, 'bs', alpha=1, zorder=2, label=r'$\sigma$ = {:.2f} %'.format(np.std(br))) axs[1].set_xlim([min(self.time_upsample), max(self.time_upsample)]) @@ -3058,7 +3092,7 @@ def plot_bestfit( self.time_upsample, self.time_upsample, sii, - label=r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty', + label='_nolegend_', ) if show_model_uncertainty: self._plot_transit_model_uncertainty( @@ -3066,7 +3100,7 @@ def plot_bestfit( self.time_upsample, self.time_upsample, sii, - label=r'1-$\sigma$ model uncertainty', + label='_nolegend_', ) axs[0].plot(self.time_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel) axs[0].set_xlim([min(self.time_upsample), max(self.time_upsample)]) diff --git a/exotic/exotic.py b/exotic/exotic.py index 9707dab5..f3d98f68 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -275,6 +275,14 @@ NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS = 5 NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS = 10 NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504} +NEXTASTRO_PHOTOMETRY_API_URL = 'https://photometry.nextastro.org' +NEXTASTRO_PHOTOMETRY_COLUMNS = ( + 'id', 'source_id', 'ra', 'dec', + 'Bmag', 'err_Bmag', 'Vmag', 'err_Vmag', + 'umag', 'err_umag', 'g', 'dg', 'r', 'dr', 'i', 'di', 'z', 'dz', +) +NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC = 30.0 +NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 30.0 BAD_PIXEL_DETECTION_FRACTION = 0.30 BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 BAD_PIXEL_PROGRESS_LOG_INTERVAL = 25 @@ -8889,6 +8897,329 @@ def nextastro_variability_test(comp_ra_dec): return variability_flags +def _finite_float(value, default=None): + try: + parsed = float(value) + except (TypeError, ValueError): + return default + return parsed if np.isfinite(parsed) else default + + +def normalize_nextastro_filter_key(obs_filter): + return re.sub(r"[^a-z0-9]", "", str(obs_filter or "").lower()) + + +def nextastro_photometry_band_candidates(obs_filter): + filter_key = normalize_nextastro_filter_key(obs_filter) + direct_map = { + 'u': [('umag', 'err_umag', 'u')], + 'johnsonu': [('umag', 'err_umag', 'u')], + 'su': [('umag', 'err_umag', 'u')], + 'up': [('umag', 'err_umag', 'u')], + 'b': [('Bmag', 'err_Bmag', 'B')], + 'johnsonb': [('Bmag', 'err_Bmag', 'B')], + 'photographicb': [('Bmag', 'err_Bmag', 'B')], + 'bb': [('Bmag', 'err_Bmag', 'B')], + 'pb': [('Bmag', 'err_Bmag', 'B')], + 'v': [('Vmag', 'err_Vmag', 'V')], + 'johnsonv': [('Vmag', 'err_Vmag', 'V')], + 'bv': [('Vmag', 'err_Vmag', 'V')], + 'cv': [('Vmag', 'err_Vmag', 'V')], + 'clearv': [('Vmag', 'err_Vmag', 'V')], + 'clearunfilteredreducedtovsequence': [('Vmag', 'err_Vmag', 'V')], + 'mobscv': [('Vmag', 'err_Vmag', 'V')], + 'c': [('Vmag', 'err_Vmag', 'V')], + 'clear': [('Vmag', 'err_Vmag', 'V')], + 'lum': [('Vmag', 'err_Vmag', 'V')], + 'luminance': [('Vmag', 'err_Vmag', 'V')], + 'sg': [('g', 'dg', 'g')], + 'sloang': [('g', 'dg', 'g')], + 'sdssg': [('g', 'dg', 'g')], + 'photographicg': [('g', 'dg', 'g')], + 'gp': [('g', 'dg', 'g')], + 'g': [('g', 'dg', 'g')], + 'pg': [('g', 'dg', 'g')], + 'tg': [('g', 'dg', 'g')], + 'sr': [('r', 'dr', 'r')], + 'sloanr': [('r', 'dr', 'r')], + 'sdssr': [('r', 'dr', 'r')], + 'johnsonr': [('r', 'dr', 'r')], + 'cousinsr': [('r', 'dr', 'r')], + 'clearunfilteredreducedtorsequence': [('r', 'dr', 'r')], + 'photographicr': [('r', 'dr', 'r')], + 'rp': [('r', 'dr', 'r')], + 'r': [('r', 'dr', 'r')], + 'rc': [('r', 'dr', 'r')], + 'rj': [('r', 'dr', 'r')], + 'pr': [('r', 'dr', 'r')], + 'tr': [('r', 'dr', 'r')], + 'cr': [('r', 'dr', 'r')], + 'si': [('i', 'di', 'i')], + 'sloani': [('i', 'di', 'i')], + 'sdssi': [('i', 'di', 'i')], + 'johnsoni': [('i', 'di', 'i')], + 'cousinsi': [('i', 'di', 'i')], + 'ip': [('i', 'di', 'i')], + 'i': [('i', 'di', 'i')], + 'ic': [('i', 'di', 'i')], + 'ij': [('i', 'di', 'i')], + 'sz': [('z', 'dz', 'z')], + 'sloanz': [('z', 'dz', 'z')], + 'sdssz': [('z', 'dz', 'z')], + 'panstarrszshort': [('z', 'dz', 'z')], + 'zp': [('z', 'dz', 'z')], + 'z': [('z', 'dz', 'z')], + 'zs': [('z', 'dz', 'z')], + } + + fallback = [ + ('Vmag', 'err_Vmag', 'V'), + ('g', 'dg', 'g'), + ('r', 'dr', 'r'), + ('i', 'di', 'i'), + ('Bmag', 'err_Bmag', 'B'), + ('z', 'dz', 'z'), + ('umag', 'err_umag', 'u'), + ] + candidates = list(direct_map.get(filter_key, [])) + candidates.extend(candidate for candidate in fallback if candidate not in candidates) + return candidates + + +def nextastro_catalog_rows(catalog_response): + if not isinstance(catalog_response, dict): + return [] + rows = catalog_response.get('rows', []) + if not isinstance(rows, list): + return [] + columns = catalog_response.get('columns', []) + if catalog_response.get('row_format') == 'arrays': + return [ + {column: row[index] if index < len(row) else None for index, column in enumerate(columns)} + for row in rows + if isinstance(row, list) + ] + return [row for row in rows if isinstance(row, dict)] + + +def row_nextastro_magnitude(row, band_candidates): + for priority, (mag_column, error_column, band_label) in enumerate(band_candidates): + magnitude = _finite_float(row.get(mag_column)) + magnitude_error = _finite_float(row.get(error_column)) + if magnitude is None or magnitude_error is None: + continue + return { + 'priority': priority, + 'mag': magnitude, + 'error': abs(magnitude_error), + 'mag_band': band_label, + 'mag_column': mag_column, + 'mag_error_column': error_column, + } + return None + + +def sky_separation_arcsec(ra_a, dec_a, ra_b, dec_b): + first = SkyCoord(float(ra_a) * u.deg, float(dec_a) * u.deg, frame='fk5') + second = SkyCoord(float(ra_b) * u.deg, float(dec_b) * u.deg, frame='fk5') + return float(first.separation(second).arcsec) + + +def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, + max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + band_candidates = nextastro_photometry_band_candidates(obs_filter) + matches = [] + for row in nextastro_catalog_rows(catalog_response): + row_ra = _finite_float(row.get('ra')) + row_dec = _finite_float(row.get('dec')) + if row_ra is None or row_dec is None: + continue + magnitude = row_nextastro_magnitude(row, band_candidates) + if magnitude is None: + continue + separation = sky_separation_arcsec(ra, dec, row_ra, row_dec) + if separation > max_separation_arcsec: + continue + matches.append({ + **magnitude, + 'catalog_ra': row_ra, + 'catalog_dec': row_dec, + 'source_id': row.get('source_id'), + 'id': row.get('id'), + 'separation_arcsec': separation, + }) + + if not matches: + return None + matches.sort(key=lambda match: (match['priority'], match['separation_arcsec'])) + return matches[0] + + +@retry( + stop=stop_after_attempt(NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS), + wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), + retry=retry_if_exception(should_retry_nextastro_variability_error), +) +def nextastro_photometry_cone_query(ra, dec, radius_arcsec, columns=None): + api_url = f'{NEXTASTRO_PHOTOMETRY_API_URL}/cone_query' + payload = { + 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), + 'ra': float(ra), + 'dec': float(dec), + 'radius_arcsec': float(radius_arcsec), + } + log_info(f"NextAstro photometry catalog request JSON: {json.dumps(payload)}") + result = requests.post(api_url, json=payload, timeout=30) + if result.status_code != 200: + raise RuntimeError(f"NextAstro photometry catalog returned HTTP {result.status_code}.") + + body = result.json() + if not isinstance(body, dict) or not isinstance(body.get('rows'), list): + raise RuntimeError("NextAstro photometry catalog returned an unexpected response format.") + log_info( + "NextAstro photometry catalog response JSON: " + f"{json.dumps({'count': body.get('count'), 'columns': body.get('columns')})}" + ) + return body + + +def nextastro_photometry_catalog_for_wcs(wcs_file, axis, img_scale, obs_filter): + if not wcs_file or img_scale is None: + return None + image_width, image_height = float(axis[0]), float(axis[1]) + if not (np.isfinite(image_width) and np.isfinite(image_height) and np.isfinite(float(img_scale))): + return None + + wcs_hdr = search_wcs(wcs_file) + center_ra, center_dec = wcs_hdr.pixel_to_world_values(image_width / 2.0, image_height / 2.0) + radius_arcsec = 0.5 * float(img_scale) * float(np.hypot(image_width, image_height)) + radius_arcsec += NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC + log_info( + "\nQuerying NextAstro photometry catalog for the full reduced field " + f"(radius={radius_arcsec:.1f} arcsec)." + ) + return nextastro_photometry_cone_query(center_ra, center_dec, radius_arcsec) + + +def nextastro_photometry_for_coordinate(ra, dec, obs_filter, + radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + catalog_response = nextastro_photometry_cone_query(ra, dec, radius_arcsec) + return nextastro_photometry_catalog_match( + catalog_response, + ra, + dec, + obs_filter, + max_separation_arcsec=radius_arcsec, + ) + + +def nextastro_calibration_label(match): + source_id = match.get('source_id') or match.get('id') + if source_id not in (None, ''): + return f"NextAstro-{source_id}" + return f"RA{match['ra']:.6f}_DEC{match['dec']:.6f}" + + +def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, existing_comp_stars=None, + field_catalog=None): + calibration_stars = dict(existing_comp_stars or {}) + existing_positions = { + tuple(value.get('pos', [])) + for value in calibration_stars.values() + if isinstance(value, dict) + } + + added_count = 0 + for index, (comp_pos, comp_radec) in enumerate(zip(comp_stars, comp_ra_dec)): + if tuple(comp_pos) in existing_positions: + continue + comp_ra = _finite_float(comp_radec[0]) + comp_dec = _finite_float(comp_radec[1]) + if comp_ra is None or comp_dec is None: + continue + + match = None + if field_catalog is not None: + match = nextastro_photometry_catalog_match(field_catalog, comp_ra, comp_dec, obs_filter) + if match is None: + try: + match = nextastro_photometry_for_coordinate(comp_ra, comp_dec, obs_filter) + except Exception as exc: + log_info( + f"Warning: NextAstro photometry catalog lookup failed for comparison star #{index + 1} " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + continue + if match is None: + log_info( + f"Warning: NextAstro photometry catalog did not find a usable magnitude for " + f"comparison star #{index + 1}.", + warn=True, + ) + continue + + match.update({ + 'ra': comp_ra, + 'dec': comp_dec, + 'pos': list(comp_pos), + 'catalog_source': 'NextAstro photometry catalog', + 'is_aavso_vsp': False, + }) + label = nextastro_calibration_label(match) + unique_label = label + duplicate_index = 2 + while unique_label in calibration_stars: + unique_label = f"{label}-{duplicate_index}" + duplicate_index += 1 + calibration_stars[unique_label] = match + existing_positions.add(tuple(comp_pos)) + added_count += 1 + log_info( + f"NextAstro photometry calibration for comparison star #{index + 1}: " + f"{match['mag_band']}={match['mag']:.5f} +/- {match['error']:.5f}, " + f"RA={comp_ra:.7f}, Dec={comp_dec:.7f}, " + f"catalog separation={match['separation_arcsec']:.2f} arcsec." + ) + + if added_count: + log_info(f"Added {added_count} NextAstro photometry catalog comparison star calibration(s).") + return calibration_stars + + +def nextastro_prereduced_calibration_star(phot_comp_star, obs_filter): + if not isinstance(phot_comp_star, dict): + return None, None + + comp_ra = _finite_float(phot_comp_star.get('ra')) + comp_dec = _finite_float(phot_comp_star.get('dec')) + if comp_ra is None or comp_dec is None: + return None, None + + match = nextastro_photometry_for_coordinate(comp_ra, comp_dec, obs_filter) + if match is None: + return None, None + + match.update({ + 'ra': comp_ra, + 'dec': comp_dec, + 'pos': [ + phot_comp_star.get('x', ''), + phot_comp_star.get('y', ''), + ], + 'catalog_source': 'NextAstro photometry catalog', + 'is_aavso_vsp': False, + }) + label = nextastro_calibration_label(match) + log_info( + "NextAstro photometry calibration for pre-reduced comparison star: " + f"{match['mag_band']}={match['mag']:.5f} +/- {match['error']:.5f}, " + f"RA={comp_ra:.7f}, Dec={comp_dec:.7f}, " + f"catalog separation={match['separation_arcsec']:.2f} arcsec." + ) + return label, match + + def check_for_variable_stars(ra_wcs, dec_wcs, comp_stars, use_nextastro_variability_server=False): if use_nextastro_variability_server and comp_stars: try: @@ -9013,7 +9344,14 @@ def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=No vsp_comp_stars_info[star['auid']] = { 'pos': vsp_star, 'mag': star_info['mag'], - 'error': star_info['error'] + 'error': star_info['error'], + 'ra': ra_deg, + 'dec': dec_deg, + 'catalog_ra': ra_deg, + 'catalog_dec': dec_deg, + 'mag_band': obs_filter, + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, } if not exist: @@ -11129,11 +11467,109 @@ def choose_comp_star_variability(fit_lc_refs, fit_lc_best, ref_comp, comp_stars, plot_variable_residuals(save) std_devs = {key: np.std(value['res']) for key, value in ref_comp.items() if value} + if not std_devs: + raise RuntimeError("No usable comparison-star residuals were available for stellar variability calibration.") min_std_dev = min(std_devs, key=lambda y: abs(std_devs[y])) return comp_stars[min_std_dev] +def stellar_variability_label(comp_label, comp_star): + if comp_star.get('is_aavso_vsp', True): + return comp_label + comp_ra = _finite_float(comp_star.get('ra')) + comp_dec = _finite_float(comp_star.get('dec')) + if comp_ra is not None and comp_dec is not None: + return f"RA={comp_ra:.7f} Dec={comp_dec:.7f}" + return comp_label + + +def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_label, save, s_name): + comp_mag = _finite_float(comp_star.get('mag')) + comp_mag_error = _finite_float(comp_star.get('error')) + if comp_mag is None or comp_mag_error is None: + raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") + + fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) + fit_airmass_model = np.asarray( + getattr(lc_fit, 'airmass_model', np.ones_like(fit_data)), + dtype=float, + ) + fit_airmass = np.asarray(getattr(lc_fit, 'airmass', np.ones_like(fit_data)), dtype=float) + fit_times = np.asarray(getattr(lc_fit, 'jd_times', getattr(lc_fit, 'time', [])), dtype=float) + transit_model = np.asarray(getattr(lc_fit, 'transit', np.ones_like(fit_data)), dtype=float) + + if not (fit_data.shape == fit_airmass_model.shape == fit_airmass.shape == fit_times.shape): + raise RuntimeError("Lightcurve arrays have inconsistent shapes for stellar variability output.") + + if transit_model.shape == fit_data.shape: + mask_ref = transit_model == 1 + else: + mask_ref = np.ones_like(fit_data, dtype=bool) + + with np.errstate(divide='ignore', invalid='ignore'): + detrended_all = np.divide(fit_data, fit_airmass_model) + if np.count_nonzero(mask_ref) == 0: + mask_ref = np.isfinite(detrended_all) + + detrended = detrended_all[mask_ref] + selected_airmass_model = fit_airmass_model[mask_ref] + selected_data = fit_data[mask_ref] + selected_times = fit_times[mask_ref] + selected_airmass = fit_airmass[mask_ref] + + oot_scatter = np.nanstd(detrended) + median_data = np.nanmedian(selected_data) + with np.errstate(divide='ignore', invalid='ignore'): + norm_flux_unc = oot_scatter * selected_airmass_model / median_data + target_mag = comp_mag - (2.5 * np.log10(detrended)) + target_mag_error = ( + comp_mag_error ** 2 + + (-2.5 * norm_flux_unc / (detrended * np.log(10))) ** 2 + ) ** 0.5 + + valid = ( + np.isfinite(selected_times) + & np.isfinite(selected_airmass) + & np.isfinite(target_mag) + & np.isfinite(target_mag_error) + ) + if np.count_nonzero(valid) == 0: + raise RuntimeError("No finite stellar variability magnitude points were produced.") + + display_label = stellar_variability_label(comp_label, comp_star) + vsp_params = [] + for time_value, airmass_value, mag_value, mag_error_value in zip( + selected_times[valid], + selected_airmass[valid], + target_mag[valid], + target_mag_error[valid], + ): + vsp_params.append({ + 'time': time_value, + 'airmass': airmass_value, + 'mag': mag_value, + 'mag_err': mag_error_value, + 'cname': display_label, + 'cmag': comp_mag, + 'cmag_err': comp_mag_error, + 'pos': comp_pos, + 'comp_ra': comp_star.get('ra'), + 'comp_dec': comp_star.get('dec'), + 'catalog_ra': comp_star.get('catalog_ra'), + 'catalog_dec': comp_star.get('catalog_dec'), + 'catalog_source': comp_star.get('catalog_source', 'AAVSO VSP'), + 'is_aavso_vsp': bool(comp_star.get('is_aavso_vsp', True)), + 'mag_band': comp_star.get('mag_band', 'V'), + 'source_id': comp_star.get('source_id'), + 'catalog_id': comp_star.get('id'), + 'separation_arcsec': comp_star.get('separation_arcsec'), + }) + + plot_stellar_variability(vsp_params, save, s_name, display_label) + return vsp_params + + def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vsp_ind, best_comp, save, s_name): info_comps = {} @@ -11156,22 +11592,15 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs return [] try: - Mc, Mc_err = comp_star['mag'], comp_star['error'] - info_comp = info_comps[comp_stars.index(comp_pos)] - lc_fit = info_comp['fit_lc'] - mask_ref = info_comp['mask_ref'] - - oot_scatter = np.std((lc_fit.data / lc_fit.airmass_model)[mask_ref]) - norm_flux_unc = oot_scatter * lc_fit.airmass_model[mask_ref] - norm_flux_unc /= np.nanmedian(lc_fit.data[mask_ref]) - - model = lc_fit.airmass_model[mask_ref] - flux = lc_fit.data[mask_ref] - detrended = flux / model - - Mt = Mc - (2.5 * np.log10(detrended)) - Mt_err = (Mc_err ** 2 + (-2.5 * norm_flux_unc / (detrended * np.log(10))) ** 2) ** 0.5 + return build_stellar_variability_params_from_fit( + info_comp['fit_lc'], + comp_star, + comp_pos, + vsp_auid_comp, + save, + s_name, + ) except KeyError as e: log_info(f"Key error in processing stellar variability: {e}", warn=True) return [] @@ -11179,24 +11608,6 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs log_info(f"Error in processing stellar variability: {e}", warn=True) return [] - try: - vsp_params = [{ - 'time': lc_fit.jd_times[mask_ref][i], - 'airmass': lc_fit.airmass[mask_ref][i], - 'mag': mt, - 'mag_err': Mt_err[i], - 'cname': vsp_auid_comp, - 'cmag': Mc, - 'pos': comp_pos - } for i, mt in enumerate(Mt)] - - plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp) - except Exception as e: - log_info(f"Error in plotting or finalizing stellar variability data: {e}", warn=True) - return [] - - return vsp_params - # Mid-Transit Time Prior Helper Functions def numberOfTransitsAway(timeData, period, originalT): @@ -12907,8 +13318,8 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): if assessment.get('ultranest_error_fallbacks'): fallback_keys = ", ".join(sorted(assessment['ultranest_error_fallbacks'])) log_info( - f"{indent}UltraNest uncertainty fallback note: replaced degenerate posterior " - f"summary error(s) for {fallback_keys} using the sampled log-likelihood neighborhood." + f"{indent}UltraNest uncertainty fallback note: replaced posterior summary " + f"error(s) for {fallback_keys} using the sampled log-likelihood neighborhood." ) @@ -15364,7 +15775,7 @@ def _main_impl(): img_scale_str, img_scale = get_img_scale(header, wcs_file, exotic_infoDict['pixel_scale']) plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) ra_dec_tar, ra_dec_wcs = None, [] - chart_id, vsp_comp_stars, vsp_list = None, None, [] + chart_id, vsp_comp_stars, vsp_list = None, {}, [] if wcs_file: if should_log_plate_solution_path(wcs_file): @@ -15409,6 +15820,28 @@ def _main_impl(): # Build RA/Dec for comp after list is finalized (avoid off by one issues, etc ra_dec_wcs = build_comp_ra_dec(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) + nextastro_field_catalog = None + try: + nextastro_field_catalog = nextastro_photometry_catalog_for_wcs( + wcs_file, + [header['NAXIS1'], header['NAXIS2']], + img_scale, + exotic_infoDict['filter'], + ) + except Exception as exc: + log_info( + "\nWarning: NextAstro full-field photometry catalog lookup failed " + f"({describe_retry_exception(exc)}). Will try per-comparison catalog lookups.", + warn=True, + ) + vsp_comp_stars = merge_nextastro_calibration_stars( + exotic_infoDict['comp_stars'], + ra_dec_wcs, + exotic_infoDict['filter'], + existing_comp_stars=vsp_comp_stars, + field_catalog=nextastro_field_catalog, + ) + vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) else: exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( @@ -16889,6 +17322,36 @@ def _main_impl(): # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) + if fitsortext != 1 and not vsp_params: + try: + calibration_label, calibration_star = nextastro_prereduced_calibration_star( + exotic_infoDict.get('phot_comp_star'), + exotic_infoDict.get('filter'), + ) + if calibration_star: + vsp_params = build_stellar_variability_params_from_fit( + myfit, + calibration_star, + calibration_star.get('pos'), + calibration_label, + exotic_infoDict['save'], + pDict['sName'], + ) + if not auid: + auid = vsx_auid(pDict['ra'], pDict['dec']) + else: + log_info( + "\nWarning: Could not create pre-reduced stellar variability output because " + "no comparison-star RA/Dec with a usable NextAstro catalog magnitude was available.", + warn=True, + ) + except Exception as exc: + log_info( + f"\nWarning: Could not create pre-reduced stellar variability output " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + # estimate transit duration pars = dict(**myfit.parameters) times = np.linspace(np.min(myfit.time), np.max(myfit.time), 1000) diff --git a/exotic/output_files.py b/exotic/output_files.py index 40a31aa2..3c328512 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -84,6 +84,57 @@ def aavso_json_safe(value): return value +def format_optional_float(value, digits=7): + value = finite_float(value) + if np.isfinite(value): + return f"{value:.{digits}f}" + return "na" + + +def stellar_variability_reference_summary(vsp_param): + if not vsp_param: + return "na" + + cname = vsp_param.get('cname', 'na') + cmag = finite_float(vsp_param.get('cmag')) + cmag_err = finite_float(vsp_param.get('cmag_err')) + band = vsp_param.get('mag_band') or 'V' + if vsp_param.get('is_aavso_vsp', True): + return f"AAVSO Label: {cname}, Position: {vsp_param.get('pos')}" + + source = vsp_param.get('catalog_source') or 'NextAstro photometry catalog' + comp_ra = format_optional_float(vsp_param.get('comp_ra')) + comp_dec = format_optional_float(vsp_param.get('comp_dec')) + if np.isfinite(cmag) and np.isfinite(cmag_err): + mag_text = f"{band}={cmag:.5f} +/- {cmag_err:.5f}" + elif np.isfinite(cmag): + mag_text = f"{band}={cmag:.5f}" + else: + mag_text = f"{band}=na" + return f"{source}: RA={comp_ra}, Dec={comp_dec}, {mag_text}" + + +def aid_comparison_metadata(vsp_param): + if not vsp_param: + return {} + return aavso_json_safe({ + 'source': vsp_param.get('catalog_source', 'AAVSO VSP'), + 'is_aavso_vsp': bool(vsp_param.get('is_aavso_vsp', True)), + 'comparison_name': vsp_param.get('cname'), + 'comparison_position_pixels': vsp_param.get('pos'), + 'comparison_ra_deg': vsp_param.get('comp_ra'), + 'comparison_dec_deg': vsp_param.get('comp_dec'), + 'catalog_ra_deg': vsp_param.get('catalog_ra'), + 'catalog_dec_deg': vsp_param.get('catalog_dec'), + 'catalog_source_id': vsp_param.get('source_id'), + 'catalog_id': vsp_param.get('catalog_id'), + 'catalog_match_separation_arcsec': vsp_param.get('separation_arcsec'), + 'magnitude_band': vsp_param.get('mag_band'), + 'apparent_magnitude': vsp_param.get('cmag'), + 'apparent_magnitude_error': vsp_param.get('cmag_err'), + }) + + def prune_aavso_metadata(value): if isinstance(value, dict): pruned = {} @@ -982,8 +1033,7 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num["Transit QC notes"] = " ".join(str(note) for note in qc_notes) if vsp_params: - params_num["Variable Reference Star"] = f"AAVSO Label: {vsp_params[0]['cname']}, " + \ - f"Position: {vsp_params[0]['pos']}" + params_num["Variable Reference Star"] = stellar_variability_reference_summary(vsp_params[0]) if phot_opt: phot_ext = {"Best Comparison Star": f"#{comp_star} - {comp_coords}" if min_aper >= 0 else str(comp_star)} @@ -1143,6 +1193,10 @@ def __init__(self, fit, p_dict, i_dict, auid, chart_id, vsp_params): self.vsp_params = vsp_params def aavso(self): + first_vsp_param = self.vsp_params[0] if self.vsp_params else {} + comparison_metadata = aid_comparison_metadata(first_vsp_param) + variable_name = self.auid or self.p_dict.get('sName') or self.p_dict.get('pName') + params_file = self.dir / safe_output_filename( "AID_AAVSO", self.p_dict['sName'], @@ -1167,12 +1221,15 @@ def aavso(self): "Space Telescope Science Institute.\n" "# Use of this data is governed by the AAVSO Data Usage Guidelines: " "aavso.org/data-usage-guidelines\n") + if comparison_metadata: + f.write(f"#COMPARISON-CATALOG-XC={dumps(comparison_metadata, sort_keys=True)}\n") f.write("#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,GROUP,CHART,NOTES\n") for vsp_p in self.vsp_params: - f.write(f"{self.auid},{round(vsp_p['time'], 5)},{round(vsp_p['mag'], 5)},{round(vsp_p['mag_err'], 5)}," - f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{round(vsp_p['cmag'], 5)},na,na," - f"{round(vsp_p['airmass'], 7)},na,{self.chart_id},na\n") + chart_id = self.chart_id or vsp_p.get('chart_id') or 'na' + f.write(f"{variable_name},{round(vsp_p['time'], 5)},{round(vsp_p['mag'], 5)},{round(vsp_p['mag_err'], 5)}," + f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{round(vsp_p['cmag'], 5)},na,na," + f"{round(vsp_p['airmass'], 7)},na,{chart_id},na\n") def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): diff --git a/exotic/plots.py b/exotic/plots.py index c31a0f10..51626dcb 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -527,11 +527,20 @@ def plot_variable_residuals(save): def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): + if not vsp_params: + return + for vsp_p in vsp_params: plt.errorbar(vsp_p['time'], vsp_p['mag'], yerr=vsp_p['mag_err'], color="tomato", fmt='.') - plt.title(f"{s_name} (Label: {vsp_auid_comp})") - plt.ylabel("Vmag") + first_param = vsp_params[0] + band = first_param.get('mag_band') or 'V' + if first_param.get('is_aavso_vsp', True): + title = f"{s_name} (Label: {vsp_auid_comp})" + else: + title = f"{s_name} (Calib: {vsp_auid_comp})" + plt.title(title) + plt.ylabel(f"{band}mag") plt.xlabel("Time [JD]") plt.savefig(Path(save) / "temp" / f"Stellar_Variability.png") plt.close() diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 56939a3a..7913d376 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -5,6 +5,7 @@ import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt +from matplotlib.axes import Axes import numpy as np import pytest @@ -398,15 +399,53 @@ def test_plot_bestfit_can_draw_transit_model_uncertainty_band(monkeypatch, tmp_p fig, axes = fit.plot_bestfit(show_model_uncertainty=True) labels = [artist.get_label() for artist in axes[0].collections] + legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) uncertainty_line_count = sum(1 for line in axes[0].lines if line.get_linestyle() == "--") assert envelope is not None assert np.nanmax(envelope[1] - envelope[0]) > 0 - assert r'1-$\sigma$ model uncertainty' in labels + assert "_nolegend_" in labels + assert r'1-$\sigma$ model uncertainty' not in legend_text assert uncertainty_line_count >= 2 plt.close(fig) +def test_plot_bestfit_draws_unbinned_points_black_with_grey_errorbars(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + captured_errorbars = [] + + original_errorbar = Axes.errorbar + + def spy_errorbar(self, *args, **kwargs): + captured_errorbars.append(kwargs.copy()) + return original_errorbar(self, *args, **kwargs) + + monkeypatch.setattr(Axes, "errorbar", spy_errorbar) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + fig, _ = fit.plot_bestfit() + + assert captured_errorbars[0]["color"] == "black" + assert captured_errorbars[0]["ecolor"] == "0.72" + assert captured_errorbars[0]["alpha"] == 1.0 + plt.close(fig) + + def test_transit_model_uncertainty_includes_baseline_terms(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -483,8 +522,10 @@ def test_plot_bestfit_can_draw_baseline_uncertainty_band(monkeypatch, tmp_path): fig, axes = fit.plot_bestfit(show_baseline_uncertainty=True) labels = [artist.get_label() for artist in axes[0].collections] + legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) - assert r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty' in labels + assert "_nolegend_" in labels + assert r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty' not in legend_text plt.close(fig) @@ -875,6 +916,50 @@ def __init__(self, *args, **kwargs): assert fit.ultranest_error_fallbacks["rprs"]["sample_count"] == 8 +def test_nested_fit_replaces_prior_width_like_error_with_local_likelihood_width(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.prior = make_prior() + fit.bounds = {"rprs": [0.0, 0.3]} + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.fixed_parameter_errors = {} + + center = 0.152 + broad_points = np.linspace(0.0, 0.3, 40) + local_points = center + np.linspace(-0.006, 0.006, 17) + points = np.concatenate([broad_points, local_points])[:, None] + broad_logl = np.full(broad_points.shape, -100.0) + local_logl = -0.5 * ((local_points - center) / 0.0038) ** 2 + logl = np.concatenate([broad_logl, local_logl]) + fit.results = { + "maximum_likelihood": {"point": np.array([center])}, + "posterior": { + "stdev": np.array([0.082]), + "errlo": np.array([-0.082]), + "errup": np.array([0.082]), + }, + "weighted_samples": { + "points": points, + "logl": logl, + }, + "samples": points.copy(), + } + + fit._finalize_ultranest_fit_results( + ["rprs"], + ["rprs"], + lambda point: {"rprs": float(point[0])}, + ) + + fallback = fit.ultranest_error_fallbacks["rprs"] + assert fit.errors["rprs"] < 0.01 + assert fit.errors["rprs"] == pytest.approx(fallback["error"]) + assert fallback["reported_error"] == pytest.approx(0.082) + assert fallback["reason"] == "posterior_summary_inflated_relative_to_local_fit" + assert fallback["delta_chi2"] <= 1.0 + + def test_nested_fit_duration_prior_penalizes_wrong_transit_length(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 4698d641..3fe38371 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -201,6 +201,128 @@ def fake_post(url, data, headers, timeout): assert excinfo.value.last_attempt.attempt_number == 5 +def test_nextastro_photometry_catalog_match_prefers_requested_filter(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag', 'g', 'dg'], + 'count': 2, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.0001, + 'dec': 20.0001, + 'Vmag': None, + 'err_Vmag': None, + 'g': 12.1, + 'dg': 0.02, + }, + { + 'id': 2, + 'source_id': 222, + 'ra': 10.0002, + 'dec': 20.0002, + 'Vmag': 12.3, + 'err_Vmag': 0.04, + 'g': 12.0, + 'dg': 0.02, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') + + assert match['source_id'] == 222 + assert match['mag'] == pytest.approx(12.3) + assert match['error'] == pytest.approx(0.04) + assert match['mag_band'] == 'V' + assert match['separation_arcsec'] > 0 + + +def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 9, + 'source_id': 12345, + 'ra': 10.00001, + 'dec': -20.00001, + 'Vmag': 11.2, + 'err_Vmag': 0.03, + } + ], + } + + calibration_stars = exotic_module.merge_nextastro_calibration_stars( + comp_stars=[[100, 200]], + comp_ra_dec=[(10.0, -20.0)], + obs_filter='V', + existing_comp_stars={}, + field_catalog=catalog, + ) + + assert list(calibration_stars) == ['NextAstro-12345'] + calibration = calibration_stars['NextAstro-12345'] + assert calibration['is_aavso_vsp'] is False + assert calibration['catalog_source'] == 'NextAstro photometry catalog' + assert calibration['ra'] == pytest.approx(10.0) + assert calibration['dec'] == pytest.approx(-20.0) + assert calibration['mag'] == pytest.approx(11.2) + assert calibration['error'] == pytest.approx(0.03) + + +def test_build_stellar_variability_params_records_nextastro_reference(monkeypatch, tmp_path): + captured = {} + + class DummyFit: + data = np.array([1.0, 1.02, 0.98], dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.array([1.1, 1.2, 1.3], dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + def fake_plot(params, save, s_name, label): + captured['params'] = params + captured['label'] = label + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', fake_plot) + + calibration_star = { + 'mag': 12.0, + 'error': 0.05, + 'ra': 10.1, + 'dec': -20.2, + 'catalog_ra': 10.10001, + 'catalog_dec': -20.20001, + 'catalog_source': 'NextAstro photometry catalog', + 'is_aavso_vsp': False, + 'mag_band': 'V', + 'source_id': 123, + 'separation_arcsec': 0.2, + } + + params = exotic_module.build_stellar_variability_params_from_fit( + DummyFit(), + calibration_star, + [100, 200], + 'NextAstro-123', + tmp_path, + 'Host Star', + ) + + assert captured['label'] == 'RA=10.1000000 Dec=-20.2000000' + assert len(params) == 3 + assert params[0]['catalog_source'] == 'NextAstro photometry catalog' + assert params[0]['is_aavso_vsp'] is False + assert params[0]['comp_ra'] == pytest.approx(10.1) + assert params[0]['comp_dec'] == pytest.approx(-20.2) + assert params[0]['cmag'] == pytest.approx(12.0) + assert params[0]['cmag_err'] == pytest.approx(0.05) + + def test_check_for_variable_stars_uses_nextastro_flags_to_filter(monkeypatch): logged = [] diff --git a/tests/test_output_files.py b/tests/test_output_files.py index fa57edcc..46c7c9d6 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from exotic.output_files import OutputFiles, fit_impact_parameter_value_error, save_comp_star_calibration_summary +from exotic.output_files import AIDOutputFiles, OutputFiles, fit_impact_parameter_value_error, save_comp_star_calibration_summary class DummyFit: @@ -157,6 +157,56 @@ def test_aavso_output_omits_obsname_header_when_blank(tmp_path): assert "#GAIAPMDEC=" not in output_text +def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + } + vsp_params = [{ + "time": 2450000.12345, + "mag": 12.34, + "mag_err": 0.05, + "airmass": 1.234, + "cname": "RA=10.1000000 Dec=-20.2000000", + "cmag": 12.1, + "cmag_err": 0.03, + "pos": [493, 202], + "comp_ra": 10.1, + "comp_dec": -20.2, + "catalog_ra": 10.10001, + "catalog_dec": -20.20001, + "catalog_source": "NextAstro photometry catalog", + "is_aavso_vsp": False, + "mag_band": "V", + "source_id": 12345, + "separation_arcsec": 0.2, + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = (tmp_path / "AID_AAVSO_HAT-P-32_2020-01-01.txt").read_text(encoding="utf-8") + metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") + + assert metadata["source"] == "NextAstro photometry catalog" + assert metadata["is_aavso_vsp"] is False + assert metadata["comparison_ra_deg"] == pytest.approx(10.1) + assert metadata["comparison_dec_deg"] == pytest.approx(-20.2) + assert metadata["apparent_magnitude"] == pytest.approx(12.1) + assert metadata["apparent_magnitude_error"] == pytest.approx(0.03) + assert "HAT-P-32,2450000.12345" in output_text + + def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): summary_path = save_comp_star_calibration_summary( tmp_path, @@ -223,6 +273,39 @@ def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): assert "Airmass coefficient 1 (a1)" not in output_text +def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path): + fit = DummyFit() + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + vsp_params = [{ + "cname": "RA=10.1000000 Dec=-20.2000000", + "cmag": 12.345, + "cmag_err": 0.067, + "pos": [493, 202], + "comp_ra": 10.1, + "comp_dec": -20.2, + "catalog_source": "NextAstro photometry catalog", + "is_aavso_vsp": False, + "mag_band": "V", + }] + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=vsp_params, + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + reference = final_params["Variable Reference Star"] + assert "NextAstro photometry catalog" in reference + assert "RA=10.1000000" in reference + assert "Dec=-20.2000000" in reference + assert "V=12.34500 +/- 0.06700" in reference + + def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclination(tmp_path): fit = DummyFit() (tmp_path / "temp").mkdir() From e959c0740905f07a94f51eb691cf4cc20c34fc8d Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sat, 16 May 2026 07:01:06 +1000 Subject: [PATCH 044/116] stellar tweaks --- exotic/exotic.py | 28 +++++++++++--- exotic/plots.py | 59 ++++++++++++++++++++++++----- tests/test_nextastro_variability.py | 4 ++ tests/test_plots.py | 46 ++++++++++++++++++++++ 4 files changed, 121 insertions(+), 16 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index f3d98f68..299637c4 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -9165,6 +9165,7 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist 'pos': list(comp_pos), 'catalog_source': 'NextAstro photometry catalog', 'is_aavso_vsp': False, + 'observed_filter': obs_filter, }) label = nextastro_calibration_label(match) unique_label = label @@ -9209,6 +9210,7 @@ def nextastro_prereduced_calibration_star(phot_comp_star, obs_filter): ], 'catalog_source': 'NextAstro photometry catalog', 'is_aavso_vsp': False, + 'observed_filter': obs_filter, }) label = nextastro_calibration_label(match) log_info( @@ -9305,6 +9307,7 @@ def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=No vsp_comp_stars_info = {} vsp_star_count = 0 + observed_filter = obs_filter # Build combined list for comps and target - there are known cases when AAVsO comps have planets (XO-2 N) # Plus, we don't want comp too close to target @@ -9350,6 +9353,7 @@ def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=No 'catalog_ra': ra_deg, 'catalog_dec': dec_deg, 'mag_band': obs_filter, + 'observed_filter': observed_filter, 'catalog_source': 'AAVSO VSP', 'is_aavso_vsp': True, } @@ -11484,11 +11488,13 @@ def stellar_variability_label(comp_label, comp_star): return comp_label -def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_label, save, s_name): +def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_label, save, s_name, + observed_filter=None): comp_mag = _finite_float(comp_star.get('mag')) comp_mag_error = _finite_float(comp_star.get('error')) if comp_mag is None or comp_mag_error is None: raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") + observed_filter = observed_filter or comp_star.get('observed_filter') fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) fit_airmass_model = np.asarray( @@ -11561,6 +11567,7 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ 'catalog_source': comp_star.get('catalog_source', 'AAVSO VSP'), 'is_aavso_vsp': bool(comp_star.get('is_aavso_vsp', True)), 'mag_band': comp_star.get('mag_band', 'V'), + 'observed_filter': observed_filter, 'source_id': comp_star.get('source_id'), 'catalog_id': comp_star.get('id'), 'separation_arcsec': comp_star.get('separation_arcsec'), @@ -11570,7 +11577,8 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ return vsp_params -def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vsp_ind, best_comp, save, s_name): +def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vsp_ind, best_comp, save, s_name, + observed_filter=None): info_comps = {} try: @@ -11600,6 +11608,7 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs vsp_auid_comp, save, s_name, + observed_filter=observed_filter, ) except KeyError as e: log_info(f"Key error in processing stellar variability: {e}", warn=True) @@ -15614,6 +15623,7 @@ def _main_impl(): exotic_infoDict['long'] = -110.951376 exotic_infoDict['pixel_bin'] = "2x2" + exotic_infoDict.setdefault('observed_filter', exotic_infoDict.get('filter')) log_info("Calculating limb-darkening coefficients.") ld, ld0, ld1, ld2, ld3 = get_ld_values(pDict, exotic_infoDict) log_info("Limb-darkening coefficients ready.") @@ -17101,17 +17111,22 @@ def _main_impl(): if vsp_comp_stars: if not bestCompStar: vsp_params = stellar_variability(ref_flux, best_fit_lc, exotic_infoDict['comp_stars'], - vsp_comp_stars, vsp_num, None, exotic_infoDict['save'], - pDict['sName']) + vsp_comp_stars, vsp_num, None, exotic_infoDict['save'], + pDict['sName'], + observed_filter=exotic_infoDict.get('observed_filter', + exotic_infoDict.get('filter'))) else: vsp_params = stellar_variability(ref_flux, best_fit_lc, exotic_infoDict['comp_stars'], - vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], - pDict['sName']) + vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], + pDict['sName'], + observed_filter=exotic_infoDict.get('observed_filter', + exotic_infoDict.get('filter'))) log_info("\n\nOutput File Saved") else: goodTimes, goodFluxes, goodNormUnc, goodAirmasses = [], [], [], [] bestCompStar, comp_coords = None, None + exotic_infoDict.setdefault('observed_filter', exotic_infoDict.get('filter')) ld, ld0, ld1, ld2, ld3 = get_ld_values(pDict, exotic_infoDict) with exotic_infoDict['prered_file'].open('r') as f: @@ -17336,6 +17351,7 @@ def _main_impl(): calibration_label, exotic_infoDict['save'], pDict['sName'], + observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), ) if not auid: auid = vsx_auid(pDict['ra'], pDict['dec']) diff --git a/exotic/plots.py b/exotic/plots.py index 51626dcb..877e8848 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -526,24 +526,63 @@ def plot_variable_residuals(save): plt.close() +def _finite_plot_float(value): + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if np.isfinite(parsed) else None + + +def _stellar_variability_reference_label(vsp_param, comparison_label): + band = vsp_param.get('mag_band') or 'V' + observed_filter = vsp_param.get('observed_filter') + cmag = _finite_plot_float(vsp_param.get('cmag')) + cmag_err = _finite_plot_float(vsp_param.get('cmag_err')) + comp_ra = _finite_plot_float(vsp_param.get('comp_ra')) + comp_dec = _finite_plot_float(vsp_param.get('comp_dec')) + + parts = [] + if vsp_param.get('is_aavso_vsp', True) and comparison_label: + parts.append(f"Label={comparison_label}") + if comp_ra is not None and comp_dec is not None: + parts.append(f"RA={comp_ra:.7f}") + parts.append(f"Dec={comp_dec:.7f}") + elif comparison_label: + parts.append(str(comparison_label)) + + if observed_filter not in (None, ''): + parts.append(f"Observed filter={observed_filter}") + + if cmag is not None and cmag_err is not None: + parts.append(f"{band}={cmag:.5f} +/- {cmag_err:.5f}") + elif cmag is not None: + parts.append(f"{band}={cmag:.5f}") + else: + parts.append(f"{band}=na") + + return ", ".join(parts) + + def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): if not vsp_params: return + fig, ax = plt.subplots(figsize=(8, 5)) for vsp_p in vsp_params: - plt.errorbar(vsp_p['time'], vsp_p['mag'], yerr=vsp_p['mag_err'], color="tomato", fmt='.') + ax.errorbar(vsp_p['time'], vsp_p['mag'], yerr=vsp_p['mag_err'], color="tomato", fmt='.') first_param = vsp_params[0] band = first_param.get('mag_band') or 'V' - if first_param.get('is_aavso_vsp', True): - title = f"{s_name} (Label: {vsp_auid_comp})" - else: - title = f"{s_name} (Calib: {vsp_auid_comp})" - plt.title(title) - plt.ylabel(f"{band}mag") - plt.xlabel("Time [JD]") - plt.savefig(Path(save) / "temp" / f"Stellar_Variability.png") - plt.close() + reference_label = _stellar_variability_reference_label(first_param, vsp_auid_comp) + ax.set_title(f"{s_name}\nComparison: {reference_label}") + ax.set_ylabel(f"Magnitude ({band})") + ax.set_xlabel("Time [JD]") + fig.tight_layout() + output_dir = Path(save) / "temp" + output_dir.mkdir(parents=True, exist_ok=True) + fig.savefig(output_dir / f"Stellar_Variability.png") + plt.close(fig) # Observation statistics series selection diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 3fe38371..b7b432c7 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -272,6 +272,7 @@ def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): assert calibration['dec'] == pytest.approx(-20.0) assert calibration['mag'] == pytest.approx(11.2) assert calibration['error'] == pytest.approx(0.03) + assert calibration['observed_filter'] == 'V' def test_build_stellar_variability_params_records_nextastro_reference(monkeypatch, tmp_path): @@ -300,6 +301,7 @@ def fake_plot(params, save, s_name, label): 'catalog_source': 'NextAstro photometry catalog', 'is_aavso_vsp': False, 'mag_band': 'V', + 'observed_filter': 'V', 'source_id': 123, 'separation_arcsec': 0.2, } @@ -311,6 +313,7 @@ def fake_plot(params, save, s_name, label): 'NextAstro-123', tmp_path, 'Host Star', + observed_filter='CV', ) assert captured['label'] == 'RA=10.1000000 Dec=-20.2000000' @@ -321,6 +324,7 @@ def fake_plot(params, save, s_name, label): assert params[0]['comp_dec'] == pytest.approx(-20.2) assert params[0]['cmag'] == pytest.approx(12.0) assert params[0]['cmag_err'] == pytest.approx(0.05) + assert params[0]['observed_filter'] == 'CV' def test_check_for_variable_stars_uses_nextastro_flags_to_filter(monkeypatch): diff --git a/tests/test_plots.py b/tests/test_plots.py index 09f4fd79..3e58bfa8 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -12,6 +12,7 @@ plot_final_lightcurve, plot_individual_comp_star_calibration_series, plot_obs_stats, + plot_stellar_variability, ) @@ -175,6 +176,51 @@ def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.pdf").exists() +def test_plot_stellar_variability_labels_reference_band_and_coordinates(tmp_path, monkeypatch): + titles = [] + ylabels = [] + original_set_title = Axes.set_title + original_set_ylabel = Axes.set_ylabel + + def spy_set_title(self, label, *args, **kwargs): + titles.append(label) + return original_set_title(self, label, *args, **kwargs) + + def spy_set_ylabel(self, label, *args, **kwargs): + ylabels.append(label) + return original_set_ylabel(self, label, *args, **kwargs) + + monkeypatch.setattr(Axes, "set_title", spy_set_title) + monkeypatch.setattr(Axes, "set_ylabel", spy_set_ylabel) + + plot_stellar_variability( + [ + { + "time": 2450000.1, + "mag": 12.34, + "mag_err": 0.05, + "cmag": 12.345, + "cmag_err": 0.067, + "comp_ra": 10.1, + "comp_dec": -20.2, + "mag_band": "r", + "observed_filter": "CV", + "is_aavso_vsp": False, + } + ], + str(tmp_path), + "Host Star", + "NextAstro-123", + ) + + assert "RA=10.1000000" in titles[-1] + assert "Dec=-20.2000000" in titles[-1] + assert "Observed filter=CV" in titles[-1] + assert "r=12.34500 +/- 0.06700" in titles[-1] + assert ylabels[-1] == "Magnitude (r)" + assert (tmp_path / "temp" / "Stellar_Variability.png").exists() + + def test_plot_comp_star_candidate_lightcurve_fits_writes_outputs(tmp_path): class DummyCandidateFit: def __init__(self): From 99060b0dc1ecc561b60fec1360d1573ce83c8750 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sat, 16 May 2026 07:51:20 +1000 Subject: [PATCH 045/116] plot update --- exotic/plots.py | 27 +++++++++++++++++---------- tests/test_plots.py | 1 + 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/exotic/plots.py b/exotic/plots.py index 877e8848..df3f0de5 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -542,26 +542,33 @@ def _stellar_variability_reference_label(vsp_param, comparison_label): comp_ra = _finite_plot_float(vsp_param.get('comp_ra')) comp_dec = _finite_plot_float(vsp_param.get('comp_dec')) - parts = [] + comparison_parts = [] if vsp_param.get('is_aavso_vsp', True) and comparison_label: - parts.append(f"Label={comparison_label}") + comparison_parts.append(f"Label={comparison_label}") if comp_ra is not None and comp_dec is not None: - parts.append(f"RA={comp_ra:.7f}") - parts.append(f"Dec={comp_dec:.7f}") + comparison_parts.append(f"RA={comp_ra:.7f}") + comparison_parts.append(f"Dec={comp_dec:.7f}") elif comparison_label: - parts.append(str(comparison_label)) + comparison_parts.append(str(comparison_label)) + detail_parts = [] if observed_filter not in (None, ''): - parts.append(f"Observed filter={observed_filter}") + detail_parts.append(f"Observed filter={observed_filter}") if cmag is not None and cmag_err is not None: - parts.append(f"{band}={cmag:.5f} +/- {cmag_err:.5f}") + detail_parts.append(f"{band}={cmag:.5f} +/- {cmag_err:.5f}") elif cmag is not None: - parts.append(f"{band}={cmag:.5f}") + detail_parts.append(f"{band}={cmag:.5f}") else: - parts.append(f"{band}=na") + detail_parts.append(f"{band}=na") - return ", ".join(parts) + label_lines = [] + if comparison_parts: + label_lines.append(", ".join(comparison_parts)) + if detail_parts: + label_lines.append(", ".join(detail_parts)) + + return "\n".join(label_lines) def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): diff --git a/tests/test_plots.py b/tests/test_plots.py index 3e58bfa8..530e11c9 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -217,6 +217,7 @@ def spy_set_ylabel(self, label, *args, **kwargs): assert "Dec=-20.2000000" in titles[-1] assert "Observed filter=CV" in titles[-1] assert "r=12.34500 +/- 0.06700" in titles[-1] + assert "Dec=-20.2000000\nObserved filter=CV" in titles[-1] assert ylabels[-1] == "Magnitude (r)" assert (tmp_path / "temp" / "Stellar_Variability.png").exists() From 628066409fac90c220fe0cb48f8b6606e655fea1 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 19 May 2026 20:04:46 +1000 Subject: [PATCH 046/116] LDtk backup --- exotic/api/gael_ld.py | 112 +++++++++++++++++++++++++++++++ tests/test_ldtk_http_fallback.py | 95 ++++++++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 tests/test_ldtk_http_fallback.py diff --git a/exotic/api/gael_ld.py b/exotic/api/gael_ld.py index fd8d396f..33088114 100644 --- a/exotic/api/gael_ld.py +++ b/exotic/api/gael_ld.py @@ -41,9 +41,19 @@ import logging import matplotlib.pyplot as plt import numpy as np +import os +from pathlib import Path +from urllib.parse import quote + +import requests log = logging.getLogger(__name__) +_LDTK_HTTP_FALLBACK_BASE_URL = "https://downloads.nextastro.org/PHOENIX" +_LDTK_HTTP_FALLBACK_ENV = "EXOTIC_LDTK_FALLBACK_BASE_URL" +_LDTK_DOWNLOAD_TIMEOUT = (10, 120) +_LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES = None + class LDPSet(ldtk.LDPSet): """ @@ -61,6 +71,108 @@ def profile_mu(self): return self._mu setattr(ldtk.ldtk, 'LDPSet', LDPSet) +def _ldtk_http_fallback_base_url(): + return os.environ.get(_LDTK_HTTP_FALLBACK_ENV, _LDTK_HTTP_FALLBACK_BASE_URL).strip().rstrip("/") + + +def _quote_url_path(*parts): + segments = [] + for part in parts: + segments.extend(segment for segment in str(part).strip("/").split("/") if segment) + return "/".join(quote(segment, safe="") for segment in segments) + + +def _ldtk_http_fallback_url(client, ldtk_file): + base_url = _ldtk_http_fallback_base_url() + if not base_url: + return None + path = _quote_url_path(client.edir, ldtk_file._zstr, ldtk_file.name) + return f"{base_url}/{path}" + + +def _download_file(url, local_path): + local_path = Path(local_path) + local_path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = local_path.with_name(f"{local_path.name}.download") + try: + response = requests.get(url, stream=True, timeout=_LDTK_DOWNLOAD_TIMEOUT) + try: + response.raise_for_status() + with open(temporary_path, "wb") as local_file: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if chunk: + local_file.write(chunk) + finally: + response.close() + os.replace(temporary_path, local_path) + except Exception: + if temporary_path.exists(): + temporary_path.unlink() + raise + + +def _download_ldtk_uncached_files_from_http(client, force=False): + files_to_download = [ldtk_file for ldtk_file in client.files if force or not ldtk_file.local_exists] + if not files_to_download: + return False + + base_url = _ldtk_http_fallback_base_url() + if not base_url: + raise RuntimeError( + f"LDTk FTP download failed and {_LDTK_HTTP_FALLBACK_ENV} is empty, " + "so EXOTIC cannot try the HTTP PHOENIX fallback." + ) + + log.warning( + "LDTk FTP download failed; trying PHOENIX HTTP fallback at %s for %d file(s).", + base_url, + len(files_to_download), + ) + + downloaded_paths = [] + for ldtk_file in files_to_download: + url = _ldtk_http_fallback_url(client, ldtk_file) + _download_file(url, ldtk_file.local_path) + downloaded_paths.append(ldtk_file.local_path) + if client.not_cached > 0 and not force: + client.not_cached -= 1 + + return client.check_file_corruption(downloaded_paths) + + +def _install_ldtk_http_fallback(): + global _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES + + try: + from ldtk.client import Client + except Exception: + return + + if getattr(Client.download_uncached_files, "_exotic_http_fallback", False): + return + + _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES = Client.download_uncached_files + + def download_uncached_files_with_http_fallback(self, force=False): + try: + return _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES(self, force=force) + except Exception as ftp_error: + try: + log.warning("LDTk FTP download failed with %s", ftp_error) + return _download_ldtk_uncached_files_from_http(self, force=force) + except Exception as fallback_error: + raise RuntimeError( + "LDTk could not download PHOENIX files from the default FTP server " + "or the EXOTIC HTTP fallback." + ) from fallback_error + + download_uncached_files_with_http_fallback._exotic_http_fallback = True + Client.download_uncached_files = download_uncached_files_with_http_fallback + + +_install_ldtk_http_fallback() + + def createldgrid(minmu, maxmu, orbp, ldmodel='nonlinear', phoenixmin=1e-1, segmentation=int(10), verbose=False): diff --git a/tests/test_ldtk_http_fallback.py b/tests/test_ldtk_http_fallback.py new file mode 100644 index 00000000..291ae1de --- /dev/null +++ b/tests/test_ldtk_http_fallback.py @@ -0,0 +1,95 @@ +from pathlib import Path + +import pytest + +pytest.importorskip("ldtk") + +from exotic.api import gael_ld # noqa: E402 + + +class DummyResponse: + def __init__(self, chunks): + self._chunks = chunks + self.closed = False + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size): + return iter(self._chunks) + + def close(self): + self.closed = True + + +class DummyLDTkFile: + def __init__(self, cache_path): + self.name = "lte02300+0.00+0.5.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits" + self._zstr = "Z+0.5" + self.local_path = str(Path(cache_path) / self._zstr / self.name) + + @property + def local_exists(self): + return Path(self.local_path).exists() + + +class DummyClient: + def __init__(self, cache_path): + self.edir = "SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011" + self.files = [DummyLDTkFile(cache_path)] + self.not_cached = len(self.files) + self.checked_paths = None + + def check_file_corruption(self, paths): + self.checked_paths = paths + return False + + +def test_ldtk_http_fallback_downloads_missing_file_to_ldtk_cache(monkeypatch, tmp_path): + client = DummyClient(tmp_path / "cache_vis-lowres") + captured = {} + + def fake_get(url, stream, timeout): + captured["url"] = url + captured["stream"] = stream + captured["timeout"] = timeout + return DummyResponse([b"phoenix", b"", b"-fits"]) + + monkeypatch.setenv(gael_ld._LDTK_HTTP_FALLBACK_ENV, "https://mirror.example/PHOENIX/") + monkeypatch.setattr(gael_ld.requests, "get", fake_get) + + assert gael_ld._download_ldtk_uncached_files_from_http(client) is False + + assert captured == { + "url": ( + "https://mirror.example/PHOENIX/" + "SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011/" + "Z%2B0.5/lte02300%2B0.00%2B0.5.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits" + ), + "stream": True, + "timeout": gael_ld._LDTK_DOWNLOAD_TIMEOUT, + } + assert Path(client.files[0].local_path).read_bytes() == b"phoenix-fits" + assert client.checked_paths == [client.files[0].local_path] + assert client.not_cached == 0 + + +def test_ldtk_download_wrapper_tries_original_before_http_fallback(monkeypatch, tmp_path): + from ldtk.client import Client + + client = DummyClient(tmp_path / "cache_vis-lowres") + calls = [] + + def fake_original(self, force=False): + calls.append(("ftp", force)) + raise RuntimeError("ftp unavailable") + + def fake_fallback(self, force=False): + calls.append(("http", force)) + return False + + monkeypatch.setattr(gael_ld, "_LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES", fake_original) + monkeypatch.setattr(gael_ld, "_download_ldtk_uncached_files_from_http", fake_fallback) + + assert Client.download_uncached_files(client, force=True) is False + assert calls == [("ftp", True), ("http", True)] From 43910f0db895681a280d98fdea9ef249fd2524dd Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 20 May 2026 15:58:18 +1000 Subject: [PATCH 047/116] rp/rs bounds --- exotic/exotic.py | 487 ++++++++++++++++++++++++++++- exotic/exotic_gui.py | 4 + exotic/inputs.py | 10 + tests/test_exotic_proper_motion.py | 3 +- tests/test_exotic_rprs_retry.py | 182 +++++++++++ tests/test_inputs.py | 30 ++ 6 files changed, 712 insertions(+), 4 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 299637c4..c9de8a1b 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -241,7 +241,9 @@ SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_OCCUPIED_BIN = 25.0 RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 RPRS_SEARCH_BOUND_MIN = 0.0 -RPRS_SEARCH_BOUND_MAX = 1.0 +RPRS_SEARCH_BOUND_MAX_DEFAULT = 0.5 +RPRS_SEARCH_BOUND_ABSOLUTE_MAX = 1.0 +RPRS_SEARCH_BOUND_MAX = RPRS_SEARCH_BOUND_MAX_DEFAULT RPRS_RETRY_MIN_HALF_WIDTH = 0.05 INITIAL_RPRS_BOUND_LOWER_SCALE = 0.0 INITIAL_RPRS_BOUND_UPPER_SCALE = 3.0 @@ -463,6 +465,21 @@ def annotate_duration_prior(fit, duration_prior): fit.duration_prior_source = summary.get('source') +def annotate_pre_ultranest_transit_coverage(fit, assessment): + if fit is None: + return + + assessment = assessment if isinstance(assessment, dict) else {} + fit.pre_ultranest_transit_coverage = dict(assessment) + fit.pre_ultranest_transit_coverage_valid = bool(assessment.get('valid', False)) + fit.pre_ultranest_transit_coverage_status = assessment.get('success_label') + fit.pre_ultranest_transit_coverage_chance = assessment.get('success_chance') + fit.pre_ultranest_transit_coverage_expected_successful = bool( + assessment.get('expected_successful', False) + ) + fit.pre_ultranest_transit_coverage_note = assessment.get('note') + + def annotate_lightcurve_filter_diagnostics(fit, diagnostics): if fit is None: return @@ -2078,6 +2095,14 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, fit_prior = dict(prior) fit_bounds = clone_lightcurve_bounds(bounds) ensure_pre_final_ultranest_baseline_bounds(fit_prior, fit_bounds, fit_flux, fit_a2=True) + pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( + full_good_times, + prior, + flux_values=full_good_flux, + flux_errors=full_good_unc, + tmid_search_summary=tmid_search_summary, + duration_prior=build_single_transit_duration_prior(p_dict), + ) final_fit, fitted_flux, fitted_unc = fit_final_lightcurve_with_oot_baseline_detrending( fit_times, @@ -2101,6 +2126,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, extend_sparse_posterior_live_points=False, keep_ultranest_sampler_for_deferred_extension=not bool(fast_binning.get('applied')), fix_baseline_terms_for_final=not bool(fast_binning.get('applied')), + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, ) annotate_fast_ultranest_binning(final_fit, fast_binning) if final_fit is None: @@ -2305,6 +2331,24 @@ def refit_selected_fast_comparison_on_full_lightcurve( sparse_live_point_extension_enabled, ) min_live_points = target_live_points if target_live_points is not None else base_live_points + coverage_duration_prior = ( + duration_prior if isinstance(duration_prior, dict) else build_single_transit_duration_prior(p_dict) + ) + pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( + times, + prior, + flux_values=fit_flux, + flux_errors=fit_unc, + tmid_search_summary=build_ephemeris_tmid_search_summary_for_coverage( + times, + p_dict, + prior=prior, + duration_prior=coverage_duration_prior, + sigma_multiplier=35.0, + ), + duration_prior=coverage_duration_prior, + ) + log_expected_transit_coverage_assessment(pre_ultranest_coverage_assessment) log_info( "Running the selected comparison-star final UltraNest fit on the full-resolution light curve " @@ -2325,6 +2369,7 @@ def refit_selected_fast_comparison_on_full_lightcurve( fixed_flux_baseline=True, ultranest_min_num_live_points=min_live_points, ) + annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) annotate_out_of_transit_baseline_parameter_fit( @@ -3119,11 +3164,14 @@ def build_initial_rprs_bounds( if not np.isfinite(rprs) or rprs <= 0: return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + if rprs >= RPRS_SEARCH_BOUND_MAX: + return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] lower_bound = max(RPRS_SEARCH_BOUND_MIN, lower_scale * rprs) - upper_bound = upper_scale * rprs + upper_bound = min(RPRS_SEARCH_BOUND_MAX, upper_scale * rprs) if not np.isfinite(upper_bound) or upper_bound <= lower_bound: - upper_bound = max(lower_bound + np.finfo(float).eps, rprs) + lower_bound = RPRS_SEARCH_BOUND_MIN + upper_bound = RPRS_SEARCH_BOUND_MAX return [float(lower_bound), float(upper_bound)] @@ -4366,6 +4414,48 @@ def configure_ultranest_min_num_live_points(config_value): return live_points +def parse_rprs_search_bound_max(config_value): + if config_value is None: + return RPRS_SEARCH_BOUND_MAX_DEFAULT + + if isinstance(config_value, str) and config_value.strip() == "": + return RPRS_SEARCH_BOUND_MAX_DEFAULT + + try: + max_bound = float(str(config_value).strip()) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'rprs_search_bound_max' value; " + f"defaulting to {RPRS_SEARCH_BOUND_MAX_DEFAULT:.3f}.", + warn=True, + ) + return RPRS_SEARCH_BOUND_MAX_DEFAULT + + if not np.isfinite(max_bound) or max_bound <= RPRS_SEARCH_BOUND_MIN: + log_info( + "Warning: 'rprs_search_bound_max' must be finite and positive; " + f"defaulting to {RPRS_SEARCH_BOUND_MAX_DEFAULT:.3f}.", + warn=True, + ) + return RPRS_SEARCH_BOUND_MAX_DEFAULT + + if max_bound > RPRS_SEARCH_BOUND_ABSOLUTE_MAX: + log_info( + "Warning: 'rprs_search_bound_max' exceeds the absolute safety ceiling " + f"of {RPRS_SEARCH_BOUND_ABSOLUTE_MAX:.3f}; clamping to that ceiling.", + warn=True, + ) + return RPRS_SEARCH_BOUND_ABSOLUTE_MAX + + return float(max_bound) + + +def configure_rprs_search_bound_max(config_value): + global RPRS_SEARCH_BOUND_MAX + RPRS_SEARCH_BOUND_MAX = parse_rprs_search_bound_max(config_value) + return RPRS_SEARCH_BOUND_MAX + + def log_ultranest_mpi_status(): status = get_mpi_status() size = int(status.get("size") or 1) @@ -5710,6 +5800,336 @@ def summarize_prior_transit_coverage( return summary +def _coverage_duration_from_context(prior, duration_prior=None): + if isinstance(duration_prior, dict): + duration = coerce_finite_transit_qc_scalar(duration_prior.get('expected_duration', np.nan)) + if np.isfinite(duration) and duration > 0: + return float(duration) + return estimate_transit_duration_from_prior_geometry(prior) + + +def _coverage_tmid_from_context(prior, tmid_search_summary=None): + if isinstance(tmid_search_summary, dict): + tmid = coerce_finite_transit_qc_scalar(tmid_search_summary.get('tmid', np.nan)) + if np.isfinite(tmid): + return float(tmid) + try: + return float(prior.get('tmid', np.nan)) + except (AttributeError, TypeError, ValueError): + return np.nan + + +def build_ephemeris_tmid_search_summary_for_coverage( + times, + planet_dict, + prior=None, + duration_prior=None, + sigma_multiplier=35.0, +): + if not isinstance(planet_dict, dict): + return None + prior = prior if isinstance(prior, dict) else {} + + prior_tmid = coerce_finite_transit_qc_scalar( + planet_dict.get('midT', prior.get('tmid', np.nan)) + ) + period = coerce_finite_transit_qc_scalar( + planet_dict.get('pPer', prior.get('per', np.nan)) + ) + midt_unc = coerce_finite_transit_qc_scalar(planet_dict.get('midTUnc', 0.0)) + per_unc = coerce_finite_transit_qc_scalar(planet_dict.get('pPerUnc', 0.0)) + if not np.isfinite(midt_unc): + midt_unc = 0.0 + if not np.isfinite(per_unc): + per_unc = 0.0 + if not np.isfinite(prior_tmid) or not np.isfinite(period) or period <= 0: + return None + + coverage_prior = dict(prior) + coverage_prior.setdefault('tmid', prior_tmid) + coverage_prior.setdefault('per', period) + coverage_prior.setdefault('rprs', planet_dict.get('rprs', np.nan)) + coverage_prior.setdefault('ars', planet_dict.get('aRs', np.nan)) + coverage_prior.setdefault('inc', planet_dict.get('inc', np.nan)) + coverage_prior.setdefault('ecc', planet_dict.get('ecc', 0.0)) + coverage_prior.setdefault('omega', planet_dict.get('omega', 0.0)) + expected_duration = _coverage_duration_from_context( + coverage_prior, + duration_prior=duration_prior, + ) + + return estimate_ephemeris_tmid_and_bounds( + times, + prior_tmid, + period, + midt_unc, + per_unc, + expected_duration=expected_duration, + sigma_multiplier=sigma_multiplier, + ) + + +def expected_transit_observed_segment( + observed_start, + observed_end, + ingress_time, + mid_transit, + egress_time, +): + if observed_end < ingress_time: + return "pre-transit baseline only" + if observed_start > egress_time: + return "post-transit baseline only" + + pieces = [] + if observed_start < ingress_time: + pieces.append("pre-ingress baseline") + if observed_start <= ingress_time <= observed_end: + pieces.append("ingress") + if observed_start <= mid_transit <= observed_end: + pieces.append("mid-transit") + if observed_start <= egress_time <= observed_end: + pieces.append("egress") + if observed_end > egress_time: + pieces.append("post-egress baseline") + if not pieces: + if observed_end < mid_transit: + return "inside the first half of transit" + if observed_start > mid_transit: + return "inside the second half of transit" + return "inside the expected transit" + return " plus ".join(pieces) + + +def score_expected_transit_model_success( + transit_fraction_observed, + covers_ingress, + covers_mid_transit, + covers_egress, + pre_points, + post_points, +): + has_two_sided_baseline = pre_points > 0 and post_points > 0 + if transit_fraction_observed <= 0: + return "very low", 0.05 + if transit_fraction_observed < 0.25: + return "very low", 0.15 + if not has_two_sided_baseline: + if transit_fraction_observed >= 0.9 and covers_ingress and covers_egress: + return "moderate", 0.50 + if transit_fraction_observed >= 0.5 and covers_mid_transit: + return "low", 0.35 + return "low", 0.25 + if transit_fraction_observed >= 0.9 and covers_ingress and covers_egress: + return "high", 0.85 + if transit_fraction_observed >= 0.65 and covers_mid_transit and (covers_ingress or covers_egress): + return "moderate", 0.65 + if transit_fraction_observed >= 0.4: + return "low", 0.40 + return "low", 0.25 + + +def build_expected_transit_coverage_assessment( + times, + prior, + flux_values=None, + flux_errors=None, + tmid_search_summary=None, + duration_prior=None, +): + times = np.asarray(times, dtype=float) + if flux_values is None: + flux_values = np.ones_like(times, dtype=float) + else: + flux_values = np.asarray(flux_values, dtype=float) + + base = { + 'valid': False, + 'point_count': 0, + 'observed_start': np.nan, + 'observed_end': np.nan, + 'observed_span': np.nan, + 'expected_tmid': np.nan, + 'expected_duration': np.nan, + 'expected_ingress_time': np.nan, + 'expected_egress_time': np.nan, + 'overlap_duration': 0.0, + 'transit_fraction_observed': 0.0, + 'pre_ingress_points': 0, + 'in_transit_points': 0, + 'post_egress_points': 0, + 'covers_ingress': False, + 'covers_mid_transit': False, + 'covers_egress': False, + 'observed_segment': 'unknown', + 'success_label': 'unknown', + 'success_chance': np.nan, + 'expected_successful': False, + 'note': 'Could not evaluate expected transit coverage before UltraNest.', + } + + if times.shape != flux_values.shape: + base['note'] = 'Could not evaluate expected transit coverage because time and flux arrays were misaligned.' + return base + + valid = np.isfinite(times) & np.isfinite(flux_values) & (flux_values > 0) + if flux_errors is not None: + flux_errors = np.asarray(flux_errors, dtype=float) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) + + if np.count_nonzero(valid) < 3: + base['note'] = 'Could not evaluate expected transit coverage because too few finite light-curve points remain.' + return base + + finite_times = np.sort(times[valid]) + observed_start = float(finite_times[0]) + observed_end = float(finite_times[-1]) + observed_span = float(observed_end - observed_start) + mid_transit = _coverage_tmid_from_context(prior, tmid_search_summary=tmid_search_summary) + duration = _coverage_duration_from_context(prior, duration_prior=duration_prior) + base.update({ + 'point_count': int(finite_times.size), + 'observed_start': observed_start, + 'observed_end': observed_end, + 'observed_span': observed_span, + 'expected_tmid': mid_transit, + 'expected_duration': duration, + }) + + if not np.isfinite(mid_transit): + base['note'] = 'Could not evaluate expected transit coverage because no finite ephemeris Tmid was available.' + return base + if not np.isfinite(duration) or duration <= 0: + base['note'] = 'Could not evaluate expected transit coverage because the expected transit duration is unavailable.' + return base + + ingress_time = float(mid_transit - 0.5 * duration) + egress_time = float(mid_transit + 0.5 * duration) + in_transit_mask = valid & (times >= ingress_time) & (times <= egress_time) + pre_mask = valid & (times < ingress_time) + post_mask = valid & (times > egress_time) + overlap_start = max(observed_start, ingress_time) + overlap_end = min(observed_end, egress_time) + overlap_duration = max(0.0, float(overlap_end - overlap_start)) + transit_fraction_observed = float(np.clip(overlap_duration / duration, 0.0, 1.0)) + covers_ingress = observed_start <= ingress_time <= observed_end + covers_mid_transit = observed_start <= mid_transit <= observed_end + covers_egress = observed_start <= egress_time <= observed_end + observed_segment = expected_transit_observed_segment( + observed_start, + observed_end, + ingress_time, + mid_transit, + egress_time, + ) + success_label, success_chance = score_expected_transit_model_success( + transit_fraction_observed, + covers_ingress, + covers_mid_transit, + covers_egress, + int(np.count_nonzero(pre_mask)), + int(np.count_nonzero(post_mask)), + ) + expected_successful = success_chance >= 0.5 + + if expected_successful: + note = ( + "The observed timestamps appear to contain enough of the expected transit window " + "for a constrained nested fit." + ) + elif transit_fraction_observed <= 0: + note = ( + "The observed timestamps do not overlap the expected transit window; " + "UltraNest is unlikely to recover a constrained transit solution." + ) + elif int(np.count_nonzero(pre_mask)) == 0 or int(np.count_nonzero(post_mask)) == 0: + note = ( + "The expected transit is not bracketed by out-of-transit data on both sides; " + "UltraNest may chase partial-transit or baseline-degenerate solutions." + ) + else: + note = ( + "The expected transit is only partially observed; UltraNest may return broad or " + "edge-hugging posteriors." + ) + + base.update({ + 'valid': True, + 'expected_ingress_time': ingress_time, + 'expected_egress_time': egress_time, + 'overlap_duration': overlap_duration, + 'transit_fraction_observed': transit_fraction_observed, + 'pre_ingress_points': int(np.count_nonzero(pre_mask)), + 'in_transit_points': int(np.count_nonzero(in_transit_mask)), + 'post_egress_points': int(np.count_nonzero(post_mask)), + 'covers_ingress': bool(covers_ingress), + 'covers_mid_transit': bool(covers_mid_transit), + 'covers_egress': bool(covers_egress), + 'observed_segment': observed_segment, + 'success_label': success_label, + 'success_chance': float(success_chance), + 'expected_successful': bool(expected_successful), + 'note': note, + }) + return base + + +def _format_minutes_from_days(days): + try: + value = float(days) * 24.0 * 60.0 + except (TypeError, ValueError): + return "n/a" + return "n/a" if not np.isfinite(value) else f"{value:.1f} min" + + +def log_expected_transit_coverage_assessment(assessment, indent=" "): + if not isinstance(assessment, dict): + return + + if not assessment.get('valid'): + log_info( + f"{indent}Warning: pre-UltraNest transit coverage assessment unavailable: " + f"{assessment.get('note', 'unknown reason')}", + warn=True, + ) + return + + success_label = str(assessment.get('success_label', 'unknown')).upper() + success_chance = coerce_finite_transit_qc_scalar(assessment.get('success_chance', np.nan)) + success_text = success_label + if np.isfinite(success_chance): + success_text = f"{success_label} (~{100.0 * float(success_chance):.0f}%)" + + warn = not bool(assessment.get('expected_successful', False)) + log_info(f"{indent}Pre-UltraNest transit coverage assessment:", warn=warn) + log_info( + f"{indent} Data time range: {assessment['observed_start']:.8f} to " + f"{assessment['observed_end']:.8f} BJD_TDB " + f"({_format_minutes_from_days(assessment.get('observed_span'))}, " + f"{assessment.get('point_count', 0)} point(s)).", + warn=warn, + ) + log_info( + f"{indent} Expected transit window: ingress {assessment['expected_ingress_time']:.8f}, " + f"mid {assessment['expected_tmid']:.8f}, egress {assessment['expected_egress_time']:.8f} " + f"BJD_TDB (duration {_format_minutes_from_days(assessment.get('expected_duration'))}).", + warn=warn, + ) + log_info( + f"{indent} Observed coverage: {assessment.get('observed_segment', 'unknown')}; " + f"{100.0 * assessment.get('transit_fraction_observed', 0.0):.1f}% of the expected transit " + f"window with {assessment.get('pre_ingress_points', 0)} pre-ingress, " + f"{assessment.get('in_transit_points', 0)} in-transit, and " + f"{assessment.get('post_egress_points', 0)} post-egress point(s).", + warn=warn, + ) + log_info( + f"{indent} Estimated fit success: {success_text}. {assessment.get('note', '')}", + warn=warn, + ) + + def extract_baseline_corrected_lightcurve_arrays(fit): times = np.asarray(getattr(fit, 'time', []), dtype=float) if times.ndim != 1 or times.size == 0: @@ -6538,9 +6958,20 @@ def fit_final_lightcurve_with_oot_baseline_detrending( extend_sparse_posterior_live_points=True, keep_ultranest_sampler_for_deferred_extension=False, fix_baseline_terms_for_final=True, + pre_ultranest_coverage_assessment=None, ): if duration_prior is None and expected_planet_dict is not None: duration_prior = build_single_transit_duration_prior(expected_planet_dict) + if pre_ultranest_coverage_assessment is None: + pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( + times, + prior, + flux_values=flux_values, + flux_errors=flux_errors, + tmid_search_summary=expected_tmid_search_summary, + duration_prior=duration_prior, + ) + log_expected_transit_coverage_assessment(pre_ultranest_coverage_assessment) sparse_posterior_live_point_extension_enabled = should_use_sparse_posterior_live_point_retry( os.environ.get( SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV, @@ -6575,6 +7006,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( tmid_search_summary=expected_tmid_search_summary, eebls_search_summary=eebls_search_summary, ) + annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) effective_bounds = get_posterior_refit_final_bounds(fit, bounds) prefit_plan = build_final_fit_prefit_refinement_plan( @@ -6625,6 +7057,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( tmid_search_summary=expected_tmid_search_summary, eebls_search_summary=eebls_search_summary, ) + annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) working_bounds = get_posterior_refit_final_bounds(fit, working_bounds) @@ -6722,6 +7155,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): tmid_search_summary=expected_tmid_search_summary, eebls_search_summary=eebls_search_summary, ) + annotate_pre_ultranest_transit_coverage(refit, pre_ultranest_coverage_assessment) annotate_final_fit_prefit_refinement( refit, prefit_plan.get('applied', False), @@ -6837,6 +7271,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): tmid_search_summary=expected_tmid_search_summary, eebls_search_summary=eebls_search_summary, ) + annotate_pre_ultranest_transit_coverage(refit, pre_ultranest_coverage_assessment) annotate_final_fit_prefit_refinement( refit, prefit_plan.get('applied', False), @@ -12075,6 +12510,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, ) prior['tmid'] = tmid_search_summary['tmid'] lower, upper = tmid_search_summary['bounds'] + ephemeris_tmid_search_summary = tmid_search_summary if ( allow_mid_transit_range_warning @@ -12179,6 +12615,15 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, debug_phase_clip_keep_mask = np.ones(np.count_nonzero(debug_initial_sigma_keep_mask), dtype=bool) if final_fit_mode == 'ns' and myfit is not None: duration_prior = build_single_transit_duration_prior(pDict) + pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( + arrayTimes, + prior, + flux_values=arrayFinalFlux, + flux_errors=arrayNormUnc, + tmid_search_summary=ephemeris_tmid_search_summary, + duration_prior=duration_prior, + ) + log_expected_transit_coverage_assessment(pre_ultranest_coverage_assessment) nested_refinement = build_nested_tmid_refinement_from_initial_fit( arrayTimes, arrayFinalFlux, @@ -12198,6 +12643,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, ) + annotate_pre_ultranest_transit_coverage(myfit, pre_ultranest_coverage_assessment) myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) @@ -13222,6 +13668,24 @@ def summarize_lightcurve_fit_assessment(fit): 'fit_method': fit_method, 'duration_prior_applied': bool(getattr(fit, 'duration_prior_applied', False)), 'duration_prior_note': getattr(fit, 'duration_prior_note', None), + 'pre_ultranest_transit_coverage_valid': bool( + getattr(fit, 'pre_ultranest_transit_coverage_valid', False) + ), + 'pre_ultranest_transit_coverage_status': getattr( + fit, + 'pre_ultranest_transit_coverage_status', + None, + ), + 'pre_ultranest_transit_coverage_chance': getattr( + fit, + 'pre_ultranest_transit_coverage_chance', + np.nan, + ), + 'pre_ultranest_transit_coverage_note': getattr( + fit, + 'pre_ultranest_transit_coverage_note', + None, + ), 'rprs_posterior_refit_applied': bool(getattr(fit, 'rprs_posterior_refit_applied', False)), 'rprs_posterior_refit_count': rprs_retry_count, 'rprs_posterior_refit_note': getattr(fit, 'rprs_posterior_refit_note', None), @@ -13305,6 +13769,16 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): ) if assessment.get('duration_prior_note'): log_info(f"{indent}Duration prior note: {assessment['duration_prior_note']}") + if assessment.get('pre_ultranest_transit_coverage_note'): + status = assessment.get('pre_ultranest_transit_coverage_status') or 'unknown' + chance = coerce_finite_transit_qc_scalar( + assessment.get('pre_ultranest_transit_coverage_chance', np.nan) + ) + chance_text = f", chance~{100.0 * float(chance):.0f}%" if np.isfinite(chance) else "" + log_info( + f"{indent}Pre-UltraNest coverage note: status={str(status).upper()}{chance_text}; " + f"{assessment['pre_ultranest_transit_coverage_note']}" + ) if assessment.get('rprs_posterior_refit_note'): log_info(f"{indent}Rp/R* posterior retry note: {assessment['rprs_posterior_refit_note']}") if assessment.get('b_posterior_refit_note'): @@ -15473,7 +15947,14 @@ def _main_impl(): ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT, ) ) + rprs_search_bound_max = configure_rprs_search_bound_max( + exotic_infoDict.get( + 'rprs_search_bound_max', + RPRS_SEARCH_BOUND_MAX_DEFAULT, + ) + ) log_info(f"UltraNest minimum live points: {ultranest_min_num_live_points}.") + log_info(f"Rp/R* maximum search bound: {rprs_search_bound_max:.3f}.") if run_fast_ultranest_before_final_run: log_info( "Fast pre-final UltraNest enabled: comparison-candidate UltraNest search runs " diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 69dbc43c..6703de9f 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -425,6 +425,7 @@ def save_input(): "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "Maximum Rp/Rs Search Bound": "Set optional_info 'rprs_search_bound_max' to cap the nested-fit Rp/Rs search range. Default 0.5.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -454,6 +455,7 @@ def save_input(): "use_eebls_to_initialize_tmid_and_bounds": "y", "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", + "rprs_search_bound_max": 0.5, "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1509,6 +1511,7 @@ def save_input(): "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "Maximum Rp/Rs Search Bound": "Set optional_info 'rprs_search_bound_max' to cap the nested-fit Rp/Rs search range. Default 0.5.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -1586,6 +1589,7 @@ def save_input(): "use_eebls_to_initialize_tmid_and_bounds": "y", "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", + "rprs_search_bound_max": 0.5, "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/exotic/inputs.py b/exotic/inputs.py index dcffde08..064398c2 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -225,6 +225,7 @@ def __init__(self, init_opt): 'skip_low_comparison_coverage_rejection': 'n', 'fit_lightcurve_to_every_comparison_candidate': 'n', 'ultranest_min_num_live_points': 200, + 'rprs_search_bound_max': 0.5, 'use_sparse_posterior_live_point_retry': 'y', } self.params = { @@ -519,6 +520,15 @@ def comp_params(self, init_file, planet_dict): 'Run Fast UltraNest Before Final Run? (y/n)', 'run_fast_ultranest_before_final_run', ), + 'rprs_search_bound_max': ( + 'rprs_search_bound_max', + 'max_rprs_search_bound', + 'maximum rprs search bound', + 'maximum Rp/Rs search bound', + 'maximum Rp/R* search bound', + 'Maximum Rp/Rs Search Bound', + 'Maximum Rp/R* Search Bound', + ), 'use_sparse_posterior_live_point_retry': ( 'use_sparse_posterior_live_point_retry', 'Use Sparse Posterior Live-Point Retry? (y/n)', diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 3361acaf..c41ab3ee 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -2800,7 +2800,8 @@ def fake_lc_fitter( assert captured_duration_priors[1] is not None assert captured_duration_priors[1]["applied"] is True assert captured_duration_priors[1]["expected_duration"] > 0 - assert captured_duration_priors[1]["expected_duration"] > 0 + assert myfit.pre_ultranest_transit_coverage_valid is True + assert myfit.pre_ultranest_transit_coverage_expected_successful is True def test_fit_lightcurve_attaches_frame_filter_diagnostics(monkeypatch): diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 6190c233..5706326b 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -91,10 +91,13 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: RPRS_SEARCH_BOUND_MIN, SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, build_fast_ultranest_lightcurve_series, + build_expected_transit_coverage_assessment, evaluate_sparse_posterior_sample_support, extend_sparse_posterior_live_points_if_needed, + fit_final_lightcurve_with_oot_baseline_detrending, finalize_comparison_candidate_full_reduction, refit_selected_fast_comparison_on_full_lightcurve, + configure_rprs_search_bound_max, should_run_fast_ultranest_before_final_run, build_single_transit_duration_prior, build_initial_rprs_bounds, @@ -112,6 +115,88 @@ def test_build_initial_rprs_bounds_allows_zero_depth_search_box(): assert INITIAL_RPRS_BOUND_LOWER_SCALE == pytest.approx(0.0) +def test_build_initial_rprs_bounds_clamps_to_configured_search_ceiling(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_SEARCH_BOUND_MAX", 0.5) + + assert build_initial_rprs_bounds(0.2) == pytest.approx([RPRS_SEARCH_BOUND_MIN, 0.5]) + assert build_initial_rprs_bounds(0.7) == pytest.approx([RPRS_SEARCH_BOUND_MIN, 0.5]) + + +def test_configure_rprs_search_bound_max_updates_retry_ceiling(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_SEARCH_BOUND_MAX", 0.5) + + assert configure_rprs_search_bound_max("0.4") == pytest.approx(0.4) + assert exotic_module.RPRS_SEARCH_BOUND_MAX == pytest.approx(0.4) + + +def test_rprs_posterior_retry_clamps_to_configured_search_ceiling(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_SEARCH_BOUND_MAX", 0.5) + captured = {"calls": []} + diagnostics_sequence = [ + {"clipped": True, "edge": "upper", "mode": 0.49, "std": 0.10, "bounds": [0.29, 0.89]}, + {"clipped": True, "edge": "upper", "mode": 0.49, "std": 0.08, "bounds": [0.38, 0.78]}, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={"tmid": 0.0, "rprs": diagnostics["mode"], "inc": 89.0, "a2": 0.0} + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: dict(diagnostics) if key == "rprs" else None + ) + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[min(call_index, len(diagnostics_sequence) - 1)]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.4, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.4], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][1]["bounds"]["rprs"][1] == pytest.approx(0.5) + assert fit.rprs_posterior_refit_bounds[1] == pytest.approx(0.5) + + def test_fast_ultranest_option_defaults_enabled_and_parses_false_values(): assert should_run_fast_ultranest_before_final_run(None) is True assert should_run_fast_ultranest_before_final_run("n") is False @@ -145,6 +230,103 @@ def test_fast_ultranest_binning_skips_short_light_curve(): assert result["binned_point_count"] == 60 +def test_expected_transit_coverage_assessment_flags_ingress_only_as_very_low(): + prior = {"tmid": 10.0, "per": 2.0, "rprs": 0.1, "ars": 12.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0} + duration_prior = {"applied": True, "expected_duration": 0.1} + times = np.linspace(9.90, 9.955, 12) + + assessment = build_expected_transit_coverage_assessment( + times, + prior, + flux_values=np.ones(times.shape[0]), + flux_errors=np.full(times.shape[0], 0.001), + duration_prior=duration_prior, + ) + + assert assessment["valid"] is True + assert assessment["observed_segment"] == "pre-ingress baseline plus ingress" + assert assessment["transit_fraction_observed"] == pytest.approx(0.05) + assert assessment["success_label"] == "very low" + assert assessment["expected_successful"] is False + + +def test_final_fit_logs_partial_coverage_before_first_ultranest_call(monkeypatch): + import exotic.exotic as exotic_module + + events = [] + + def fake_log_info(message, warn=False, error=False): + events.append(("log", str(message), warn)) + return True + + def fake_run_nested(times, flux_values, flux_errors, airmass, prior, bounds, **kwargs): + events.append(("run_nested", "", False)) + local_times = np.asarray(times, dtype=float) + model = np.ones(local_times.shape[0], dtype=float) + model[-1:] -= 0.01 + fit = types.SimpleNamespace( + time=local_times, + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + model=model, + residuals=np.zeros(local_times.shape[0], dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters={"rprs": prior["rprs"], "tmid": prior["tmid"], "inc": prior["inc"], "a2": 0.0, "per": prior["per"]}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds=dict(bounds), + duration_expected=0.1, + duration_measured=0.1, + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: {"clipped": False, "edge": None, "mode": fit.parameters.get(key, np.nan), "std": 0.01} + ) + return fit + + monkeypatch.setattr(exotic_module, "log_info", fake_log_info) + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + monkeypatch.setattr(exotic_module, "apply_plot_time_range", lambda fit, plot_time_range: fit) + monkeypatch.setattr( + exotic_module, + "build_final_fit_prefit_refinement_plan", + lambda times, flux_values, flux_errors, airmass, prior, bounds, fit, **kwargs: { + "applied": False, + "note": "not needed", + "times": np.asarray(times, dtype=float), + "flux": np.asarray(flux_values, dtype=float), + "unc": np.asarray(flux_errors, dtype=float), + "airmass": np.asarray(airmass, dtype=float), + "jd_times": None, + "prior": dict(prior), + "bounds": dict(bounds), + "duration": 0.1, + "original_point_count": len(times), + "refined_point_count": len(times), + "trimmed_pre_points": 0, + "trimmed_post_points": 0, + "original_tmid_bounds": bounds["tmid"], + "refined_tmid_bounds": bounds["tmid"], + }, + ) + + times = np.linspace(9.90, 9.955, 12) + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + np.ones(times.shape[0], dtype=float), + np.full(times.shape[0], 0.001, dtype=float), + np.linspace(1.0, 1.1, times.shape[0]), + {"rprs": 0.1, "tmid": 10.0, "inc": 89.0, "a2": 0.0, "per": 2.0, "ars": 12.0, "ecc": 0.0, "omega": 0.0}, + {"rprs": [0.0, 0.5], "tmid": [9.95, 10.05], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + detrend_on_outoftransit_baseline=False, + duration_prior={"applied": True, "expected_duration": 0.1}, + ) + + coverage_index = next(i for i, event in enumerate(events) if "Pre-UltraNest transit coverage assessment" in event[1]) + nested_index = next(i for i, event in enumerate(events) if event[0] == "run_nested") + assert coverage_index < nested_index + assert any("Estimated fit success: VERY LOW" in event[1] and event[2] for event in events) + assert fit.pre_ultranest_transit_coverage_status == "very low" + + def test_finalize_comparison_candidate_runs_pre_final_ultranest_on_binned_series(monkeypatch): import exotic.exotic as exotic_module diff --git a/tests/test_inputs.py b/tests/test_inputs.py index f2f982ba..d9e81aac 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -158,6 +158,21 @@ def test_comp_params_defaults_ultranest_live_points_to_200(tmp_path): assert inputs.info_dict["ultranest_min_num_live_points"] == 200 +def test_comp_params_defaults_rprs_search_bound_max_to_half(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["rprs_search_bound_max"] == 0.5 + + def test_comp_params_defaults_sparse_posterior_live_point_retry_to_yes(tmp_path): init_data = { "user_info": {}, @@ -473,6 +488,21 @@ def test_comp_params_reads_ultranest_live_points_from_optional_info(tmp_path): assert inputs.info_dict["ultranest_min_num_live_points"] == 275 +def test_comp_params_reads_rprs_search_bound_max_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"maximum Rp/Rs search bound": 0.35}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["rprs_search_bound_max"] == 0.35 + + def test_comp_params_reads_fast_ultranest_before_final_run_from_optional_info(tmp_path): init_data = { "user_info": {}, From 6f3f14c27b00582f525b8d32df6471ffb605a7cf Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 21 May 2026 18:21:49 +1000 Subject: [PATCH 048/116] extra LDtk backup server --- exotic/api/gael_ld.py | 64 ++++++++++++++++++++++---------- tests/test_ldtk_http_fallback.py | 20 ++++++++++ 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/exotic/api/gael_ld.py b/exotic/api/gael_ld.py index 33088114..a408c53b 100644 --- a/exotic/api/gael_ld.py +++ b/exotic/api/gael_ld.py @@ -49,7 +49,10 @@ log = logging.getLogger(__name__) -_LDTK_HTTP_FALLBACK_BASE_URL = "https://downloads.nextastro.org/PHOENIX" +_LDTK_HTTP_FALLBACK_BASE_URLS = ( + "https://ftp.gwdg.de/pub/misc/phoenix", + "https://downloads.nextastro.org/PHOENIX", +) _LDTK_HTTP_FALLBACK_ENV = "EXOTIC_LDTK_FALLBACK_BASE_URL" _LDTK_DOWNLOAD_TIMEOUT = (10, 120) _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES = None @@ -71,8 +74,11 @@ def profile_mu(self): return self._mu setattr(ldtk.ldtk, 'LDPSet', LDPSet) -def _ldtk_http_fallback_base_url(): - return os.environ.get(_LDTK_HTTP_FALLBACK_ENV, _LDTK_HTTP_FALLBACK_BASE_URL).strip().rstrip("/") +def _ldtk_http_fallback_base_urls(): + configured_urls = os.environ.get(_LDTK_HTTP_FALLBACK_ENV) + if configured_urls is None: + return [url.rstrip("/") for url in _LDTK_HTTP_FALLBACK_BASE_URLS] + return [url.strip().rstrip("/") for url in configured_urls.split(",") if url.strip()] def _quote_url_path(*parts): @@ -82,10 +88,7 @@ def _quote_url_path(*parts): return "/".join(quote(segment, safe="") for segment in segments) -def _ldtk_http_fallback_url(client, ldtk_file): - base_url = _ldtk_http_fallback_base_url() - if not base_url: - return None +def _ldtk_http_fallback_url(base_url, client, ldtk_file): path = _quote_url_path(client.edir, ldtk_file._zstr, ldtk_file.name) return f"{base_url}/{path}" @@ -111,33 +114,56 @@ def _download_file(url, local_path): raise -def _download_ldtk_uncached_files_from_http(client, force=False): - files_to_download = [ldtk_file for ldtk_file in client.files if force or not ldtk_file.local_exists] +def _ldtk_files_to_download(client, force=False): + return [ldtk_file for ldtk_file in client.files if force or not ldtk_file.local_exists] + + +def _download_ldtk_uncached_files_from_http_mirror(client, base_url, force=False): + files_to_download = _ldtk_files_to_download(client, force=force) if not files_to_download: return False - base_url = _ldtk_http_fallback_base_url() - if not base_url: - raise RuntimeError( - f"LDTk FTP download failed and {_LDTK_HTTP_FALLBACK_ENV} is empty, " - "so EXOTIC cannot try the HTTP PHOENIX fallback." - ) - log.warning( - "LDTk FTP download failed; trying PHOENIX HTTP fallback at %s for %d file(s).", + "Trying PHOENIX HTTP fallback at %s for %d file(s).", base_url, len(files_to_download), ) downloaded_paths = [] for ldtk_file in files_to_download: - url = _ldtk_http_fallback_url(client, ldtk_file) + url = _ldtk_http_fallback_url(base_url, client, ldtk_file) _download_file(url, ldtk_file.local_path) downloaded_paths.append(ldtk_file.local_path) if client.not_cached > 0 and not force: client.not_cached -= 1 - return client.check_file_corruption(downloaded_paths) + if client.check_file_corruption(downloaded_paths): + raise RuntimeError("Downloaded PHOENIX files failed LDTk's FITS corruption check.") + return False + + +def _download_ldtk_uncached_files_from_http(client, force=False): + base_urls = _ldtk_http_fallback_base_urls() + if not base_urls: + raise RuntimeError( + f"LDTk FTP download failed and {_LDTK_HTTP_FALLBACK_ENV} is empty, " + "so EXOTIC cannot try the HTTP PHOENIX fallback." + ) + + log.warning( + "LDTk FTP download failed; trying PHOENIX HTTP fallback mirrors in order: %s", + ", ".join(base_urls), + ) + + last_error = None + for base_url in base_urls: + try: + return _download_ldtk_uncached_files_from_http_mirror(client, base_url, force=force) + except Exception as mirror_error: + last_error = mirror_error + log.warning("PHOENIX HTTP fallback mirror failed at %s: %s", base_url, mirror_error) + + raise RuntimeError("All PHOENIX HTTP fallback mirrors failed.") from last_error def _install_ldtk_http_fallback(): diff --git a/tests/test_ldtk_http_fallback.py b/tests/test_ldtk_http_fallback.py index 291ae1de..57010e7e 100644 --- a/tests/test_ldtk_http_fallback.py +++ b/tests/test_ldtk_http_fallback.py @@ -74,6 +74,26 @@ def fake_get(url, stream, timeout): assert client.not_cached == 0 +def test_ldtk_http_fallback_tries_gwdg_before_nextastro(monkeypatch, tmp_path): + client = DummyClient(tmp_path / "cache_vis-lowres") + requested_urls = [] + + def fake_get(url, stream, timeout): + requested_urls.append(url) + if url.startswith("https://ftp.gwdg.de/"): + raise RuntimeError("gwdg unavailable") + return DummyResponse([b"nextastro"]) + + monkeypatch.delenv(gael_ld._LDTK_HTTP_FALLBACK_ENV, raising=False) + monkeypatch.setattr(gael_ld.requests, "get", fake_get) + + assert gael_ld._download_ldtk_uncached_files_from_http(client) is False + + assert requested_urls[0].startswith("https://ftp.gwdg.de/pub/misc/phoenix/") + assert requested_urls[1].startswith("https://downloads.nextastro.org/PHOENIX/") + assert Path(client.files[0].local_path).read_bytes() == b"nextastro" + + def test_ldtk_download_wrapper_tries_original_before_http_fallback(monkeypatch, tmp_path): from ldtk.client import Client From 52cfd871a4118994ab714bfc67b30d27652da3a0 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 23 May 2026 12:36:22 +1000 Subject: [PATCH 049/116] better handling of partials during ultranest search --- exotic/exotic.py | 676 ++++++++++++++++++++++++----- tests/test_exotic_proper_motion.py | 145 ++++++- tests/test_exotic_rprs_retry.py | 62 +++ 3 files changed, 775 insertions(+), 108 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index c9de8a1b..29d33fb4 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -230,6 +230,12 @@ FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT = True FAST_ULTRANEST_MAX_BINNED_POINTS = 20 FAST_ULTRANEST_MIN_POINTS_TO_BIN = 60 +COMPARISON_PREFLIGHT_FIELD_SCORE_RELATIVE_BAND = 0.25 +COMPARISON_PREFLIGHT_FIELD_SCORE_ABSOLUTE_BAND = 2.5e-4 +PARTIAL_COVERAGE_RPRS_POSTERIOR_MAX_RETRIES = 1 +PARTIAL_COVERAGE_ARS_POSTERIOR_MAX_RETRIES = 0 +PARTIAL_COVERAGE_IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES = 0 +PROMISING_PARTIAL_COMPARISON_KTMF_MIN = 3.0 SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT = True SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV = "EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY" SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT = 5 @@ -906,14 +912,9 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T summary['rprs_deviation_sigma'] = rprs_sigma summary['rprs_deviation_score'] = transit_qc_deviation_score_from_sigma(rprs_sigma, sigma_threshold) - component_scores = [ - score - for score in (summary['tmid_deviation_score'], summary['rprs_deviation_score']) - if np.isfinite(score) - ] - summary['available'] = bool(component_scores) - if component_scores: - summary['deviation_from_expected_value'] = float(min(component_scores)) + if np.isfinite(summary['rprs_deviation_score']): + summary['available'] = True + summary['deviation_from_expected_value'] = float(summary['rprs_deviation_score']) if np.isfinite(summary['tmid_deviation_sigma']): summary['notes'].append( @@ -929,26 +930,12 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T f"Expected-value Rp/R* deviation: {summary['rprs_deviation_sigma']:.2f} sigma." ) - for label, sigma_value in ( - ('Tmid', summary['tmid_deviation_sigma']), - ('Rp/R*', summary['rprs_deviation_sigma']), - ): - if np.isfinite(sigma_value) and np.isfinite(sigma_threshold) and sigma_threshold > 0 and sigma_value > sigma_threshold: - summary['failed'] = True - if label == 'Tmid' and np.isfinite(summary['tmid_deviation_minutes']): - threshold_text = "" - if np.isfinite(summary['tmid_deviation_threshold_minutes']): - threshold_text = f" ({summary['tmid_deviation_threshold_minutes']:.2f} minutes)" - reason = ( - "Tmid of the fit is " - f"{summary['tmid_deviation_minutes']:.2f} minutes away from the ephemeris Tmid, " - f"which is {summary['tmid_deviation_sigma']:.2f} sigma from the propagated ephemeris " - f"uncertainty and beyond the {sigma_threshold:.2f}-sigma threshold{threshold_text}" - ) - else: - reason = f"{label} differs from the expected value by more than {sigma_threshold:.2f} sigma" - summary['notes'].append(reason + ".") - summary['failure_reasons'].append(reason) + rprs_sigma = summary['rprs_deviation_sigma'] + if np.isfinite(rprs_sigma) and np.isfinite(sigma_threshold) and sigma_threshold > 0 and rprs_sigma > sigma_threshold: + summary['failed'] = True + reason = f"Rp/R* differs from the expected value by more than {sigma_threshold:.2f} sigma" + summary['notes'].append(reason + ".") + summary['failure_reasons'].append(reason) return summary @@ -983,8 +970,6 @@ def compute_transit_qc_ktmf(summary): 'score': summary.get('deviation_from_expected_value', np.nan), 'detail': ( f"score={summary.get('deviation_from_expected_value', np.nan):.2f}, " - f"Tmid sigma={summary.get('tmid_deviation_sigma', np.nan):.2f}, " - f"Tmid offset={summary.get('tmid_deviation_minutes', np.nan):.2f} min, " f"Rp/R* sigma={summary.get('rprs_deviation_sigma', np.nan):.2f}" if np.isfinite(summary.get('deviation_from_expected_value', np.nan)) else "expected-value deviation disabled or unavailable" @@ -1483,10 +1468,10 @@ def evaluate_transit_detection_qc(fit): failure_reasons.extend(detailed_reasons) else: notes.append( - "The fit deviates too far from the expected published Tmid and/or Rp/R* values." + "The fit deviates too far from the expected published Rp/R* value." ) failure_reasons.append( - "the fit deviates too far from the expected published Tmid and/or Rp/R* values" + "the fit deviates too far from the expected published Rp/R* value" ) ktmf_metric, ktmf_contributions = compute_transit_qc_ktmf(summary) @@ -1832,57 +1817,41 @@ def build_comparison_candidate_adaptive_summary(comparison_calibration, psf_data ) -def match_time_subset_indices(full_times, subset_times, rtol=1e-10, atol=1e-10): - full_times = np.asarray(full_times, dtype=float).reshape(-1) - subset_times = np.asarray(subset_times, dtype=float).reshape(-1) - if subset_times.size == 0: - return np.array([], dtype=int) - if full_times.size < subset_times.size: - return None - - matched_indices = [] - search_start = 0 - for subset_time in subset_times: - if not np.isfinite(subset_time): - return None - remaining = full_times[search_start:] - matches = np.flatnonzero(np.isclose(remaining, subset_time, rtol=rtol, atol=atol)) - if matches.size == 0: - return None - matched_index = search_start + int(matches[0]) - matched_indices.append(matched_index) - search_start = matched_index + 1 - - return np.asarray(matched_indices, dtype=int) +def build_comparison_candidate_transit_prior(p_dict, ld): + return { + 'rprs': p_dict['rprs'], + 'ars': p_dict['aRs'], + 'per': p_dict['pPer'], + 'inc': p_dict['inc'], + 'u0': ld[0], 'u1': ld[1], 'u2': ld[2], 'u3': ld[3], + 'ecc': p_dict['ecc'], + 'omega': p_dict['omega'], + 'tmid': p_dict['midT'], + 'a2': 0, + } -def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, airmass, ld, p_dict, - jd_times=None, - disable_vertical_flux_normalization=False, - detrend_on_outoftransit_baseline=True, - use_impactparameter_rather_than_inclination_to_fit=True, - use_eebls_to_initialize_tmid_and_bounds=True, - plot_time_range=None, - baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, - adaptive_summary=None, - run_fast_ultranest_before_final_run=FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT): +def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_flux, airmass, + jd_times=None, adaptive_summary=None): result = { 'applied': False, - 'fit': None, - 'good_times': np.array([], dtype=float), - 'good_flux': np.array([], dtype=float), - 'good_unc': np.array([], dtype=float), - 'good_airmass': np.array([], dtype=float), - 'good_jd_times': np.array([], dtype=float), - 'good_target_flux': np.array([], dtype=float), - 'good_comp_flux': np.array([], dtype=float), - 'source_indices': np.array([], dtype=int), - 'data_highres': None, - 'duration_samples': np.array([], dtype=float), - 'failure_reason': "full candidate reduction did not run.", + 'failure_reason': "the raw comparison-candidate photometry did not yield a usable light curve.", 'filter_diagnostics': [], - 'note': None, + 'debug_times': np.array([], dtype=float), + 'debug_target_flux': np.array([], dtype=float), + 'debug_comp_flux': np.array([], dtype=float), + 'debug_raw_ratio': np.array([], dtype=float), + 'initial_sigma_keep_mask': np.array([], dtype=bool), + 'time': np.array([], dtype=float), + 'flux': np.array([], dtype=float), + 'unc': np.array([], dtype=float), + 'airmass': np.array([], dtype=float), + 'jd_time': np.array([], dtype=float), + 'target_flux': np.array([], dtype=float), + 'comp_flux': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), } + prepared = prepare_lightcurve_fit_input_series( times, target_flux, @@ -1891,6 +1860,9 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, jd_times=jd_times, ) result['filter_diagnostics'] = prepared.get('filter_diagnostics', []) + for key in ('debug_times', 'debug_target_flux', 'debug_comp_flux', 'debug_raw_ratio', 'initial_sigma_keep_mask'): + if key in prepared: + result[key] = prepared[key] if not prepared.get('applied'): result['failure_reason'] = prepared.get( 'failure_reason', @@ -1951,26 +1923,372 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, ) return result - good_times = good_times[relative_flux_mask] - good_flux = good_flux[relative_flux_mask] - good_unc = good_unc[relative_flux_mask] - good_airmass = good_airmass[relative_flux_mask] - good_jd_times = good_jd_times[relative_flux_mask] - good_target_flux = good_target_flux[relative_flux_mask] - good_comp_flux = good_comp_flux[relative_flux_mask] - source_indices = source_indices[relative_flux_mask] + result.update({ + 'applied': True, + 'failure_reason': None, + 'time': good_times[relative_flux_mask], + 'flux': good_flux[relative_flux_mask], + 'unc': good_unc[relative_flux_mask], + 'airmass': good_airmass[relative_flux_mask], + 'jd_time': good_jd_times[relative_flux_mask], + 'target_flux': good_target_flux[relative_flux_mask], + 'comp_flux': good_comp_flux[relative_flux_mask], + 'source_indices': source_indices[relative_flux_mask], + }) + return result - prior = { - 'rprs': p_dict['rprs'], - 'ars': p_dict['aRs'], - 'per': p_dict['pPer'], - 'inc': p_dict['inc'], - 'u0': ld[0], 'u1': ld[1], 'u2': ld[2], 'u3': ld[3], - 'ecc': p_dict['ecc'], - 'omega': p_dict['omega'], - 'tmid': p_dict['midT'], - 'a2': 0, + +def comparison_candidate_coverage_priority(assessment): + if not isinstance(assessment, dict) or not assessment.get('valid'): + return 4 + + pre_points = int(assessment.get('pre_ingress_points', 0) or 0) + post_points = int(assessment.get('post_egress_points', 0) or 0) + transit_fraction = coerce_finite_transit_qc_scalar( + assessment.get('transit_fraction_observed', np.nan) + ) + has_two_sided_oot = pre_points > 0 and post_points > 0 + covers_full_window = ( + bool(assessment.get('covers_ingress', False)) + and bool(assessment.get('covers_mid_transit', False)) + and bool(assessment.get('covers_egress', False)) + ) + + if has_two_sided_oot and covers_full_window: + return 0 + if has_two_sided_oot: + return 1 + if np.isfinite(transit_fraction) and transit_fraction >= 0.75 and assessment.get('covers_mid_transit', False): + return 2 + if np.isfinite(transit_fraction) and transit_fraction > 0: + return 3 + return 4 + + +def is_low_one_sided_expected_transit_coverage(assessment): + if not isinstance(assessment, dict) or not assessment.get('valid'): + return False + + pre_points = int(assessment.get('pre_ingress_points', 0) or 0) + post_points = int(assessment.get('post_egress_points', 0) or 0) + success_label = str(assessment.get('success_label', '')).strip().lower() + expected_successful = bool(assessment.get('expected_successful', False)) + return (pre_points == 0 or post_points == 0) and ( + success_label in ('very low', 'low') or not expected_successful + ) + + +def partial_transit_geometry_retry_limits(assessment): + active = is_low_one_sided_expected_transit_coverage(assessment) + note = None + if active: + note = ( + "Skipped; pre-UltraNest coverage is one-sided/LOW, so EXOTIC does not expand this " + "geometry posterior range while the transit shape is baseline-degenerate." + ) + return { + 'active': active, + 'note': note, + 'max_retries': { + 'rprs': PARTIAL_COVERAGE_RPRS_POSTERIOR_MAX_RETRIES, + 'ars': PARTIAL_COVERAGE_ARS_POSTERIOR_MAX_RETRIES, + 'b': PARTIAL_COVERAGE_IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES, + }, + } + + +def score_comparison_candidate_lightcurve_scout(prepared_series, eebls_summary, prior): + result = { + 'score': np.nan, + 'scatter': np.nan, + 'scatter_score': np.nan, + 'depth_score': np.nan, + 'eebls_score': np.nan, + 'eebls_snr': np.nan, + 'eebls_depth': np.nan, + 'expected_depth': np.nan, } + if not isinstance(prepared_series, dict) or not prepared_series.get('applied'): + return result + + flux = np.asarray(prepared_series.get('flux', []), dtype=float) + finite_flux = flux[np.isfinite(flux) & (flux > 0)] + if finite_flux.size < LIGHTCURVE_MIN_VALID_POINTS: + return result + + baseline = bn.nanmedian(finite_flux) + if not np.isfinite(baseline) or baseline <= 0: + return result + + normalized_flux = finite_flux / baseline + scatter = robust_scatter(normalized_flux - bn.nanmedian(normalized_flux)) + rprs = coerce_finite_transit_qc_scalar(prior.get('rprs', np.nan) if isinstance(prior, dict) else np.nan) + expected_depth = rprs ** 2 if np.isfinite(rprs) and rprs >= 0 else np.nan + if np.isfinite(scatter): + result['scatter'] = float(scatter) + if np.isfinite(expected_depth): + result['expected_depth'] = float(expected_depth) + + depth = coerce_finite_transit_qc_scalar((eebls_summary or {}).get('depth', np.nan)) + depth_snr = coerce_finite_transit_qc_scalar((eebls_summary or {}).get('depth_snr', np.nan)) + if np.isfinite(depth): + result['eebls_depth'] = float(depth) + if np.isfinite(depth_snr): + result['eebls_snr'] = float(depth_snr) + + scatter_reference = expected_depth if np.isfinite(expected_depth) and expected_depth > 0 else 0.005 + if np.isfinite(scatter) and scatter >= 0: + result['scatter_score'] = float(np.clip(1.0 / (1.0 + scatter / max(scatter_reference, 1e-6)), 0.0, 1.0)) + + if np.isfinite(depth) and depth > 0 and np.isfinite(expected_depth) and expected_depth > 0: + depth_ratio = depth / expected_depth + if np.isfinite(depth_ratio) and depth_ratio > 0: + result['depth_score'] = float(np.clip(np.exp(-abs(np.log(depth_ratio)) / np.log(2.0)), 0.0, 1.0)) + + if np.isfinite(depth_snr) and depth_snr > 0: + result['eebls_score'] = float(np.clip(depth_snr / 8.0, 0.0, 1.0)) + + components = [ + (0.45, result['scatter_score']), + (0.35, result['depth_score']), + (0.20, result['eebls_score']), + ] + available = [(weight, value) for weight, value in components if np.isfinite(value)] + if available: + weight_sum = sum(weight for weight, _ in available) + result['score'] = float(sum(weight * value for weight, value in available) / weight_sum) + return result + + +def build_comparison_candidate_preflight(times, jd_times, airmass, ld, p_dict, target_flux, comp_flux, + adaptive_summary=None, use_eebls_to_initialize_tmid_and_bounds=True): + prepared = prepare_comparison_candidate_full_reduction_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + adaptive_summary=adaptive_summary, + ) + try: + prior = build_comparison_candidate_transit_prior(p_dict, ld) + except (KeyError, IndexError, TypeError, ValueError): + return { + 'prepared_series': prepared, + 'coverage_assessment': None, + 'coverage_priority': 4, + 'eebls_summary': None, + 'tmid_search_summary': None, + 'duration_prior': None, + 'scout': {'score': np.nan}, + } + + duration_prior = build_single_transit_duration_prior(p_dict) + eebls_summary = None + tmid_search_summary = None + coverage_assessment = None + scout = {'score': np.nan} + + if prepared.get('applied'): + good_times = np.asarray(prepared['time'], dtype=float) + good_flux = np.asarray(prepared['flux'], dtype=float) + good_unc = np.asarray(prepared['unc'], dtype=float) + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + good_times, + p_dict.get('midT', prior.get('tmid', np.nan)), + prior['per'], + p_dict.get('midTUnc', 0.01), + p_dict.get('pPerUnc', 0.0), + expected_duration=expected_duration, + sigma_multiplier=35.0, + ) + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] + if use_eebls_to_initialize_tmid_and_bounds: + eebls_summary = estimate_tmid_and_bounds_with_eebls( + good_times, + good_flux, + good_unc, + prior, + [lower, upper], + ) + else: + eebls_summary = {'applied': False, 'depth': np.nan, 'depth_snr': np.nan} + coverage_assessment = build_expected_transit_coverage_assessment( + good_times, + prior, + flux_values=good_flux, + flux_errors=good_unc, + tmid_search_summary=tmid_search_summary, + duration_prior=duration_prior, + ) + scout = score_comparison_candidate_lightcurve_scout(prepared, eebls_summary, prior) + + return { + 'prepared_series': prepared, + 'coverage_assessment': coverage_assessment, + 'coverage_priority': comparison_candidate_coverage_priority(coverage_assessment), + 'eebls_summary': eebls_summary, + 'tmid_search_summary': tmid_search_summary, + 'duration_prior': duration_prior, + 'scout': scout, + } + + +def comparison_preflight_field_band_limit(plans): + finite_scores = [ + plan['summary'].get('aggregate_score', np.nan) + for plan in plans + if np.isfinite(plan['summary'].get('aggregate_score', np.nan)) + ] + if not finite_scores: + return np.inf + best_score = float(min(finite_scores)) + return best_score + max( + COMPARISON_PREFLIGHT_FIELD_SCORE_ABSOLUTE_BAND, + abs(best_score) * COMPARISON_PREFLIGHT_FIELD_SCORE_RELATIVE_BAND, + ) + + +def rank_comparison_candidate_preflight_plans(plans): + if not plans: + return [] + + field_band_limit = comparison_preflight_field_band_limit(plans) + + def sort_key(plan): + preflight = plan.get('preflight') or {} + scout = preflight.get('scout') or {} + aggregate_score = plan['summary'].get('aggregate_score', np.inf) + finite_aggregate = aggregate_score if np.isfinite(aggregate_score) else np.inf + close_field_band = 0 if finite_aggregate <= field_band_limit else 1 + scout_score = scout.get('score', np.nan) + scout_sort = -float(scout_score) if np.isfinite(scout_score) else np.inf + return ( + int(preflight.get('coverage_priority', 4)), + close_field_band, + scout_sort, + finite_aggregate, + plan.get('field_rank', np.inf), + ) + + return sorted(plans, key=sort_key) + + +def log_comparison_candidate_preflight_order(plans, ranked_plans): + if not plans or not ranked_plans: + return + original_order = [plan['summary'].get('comp_index') for plan in plans] + ranked_order = [plan['summary'].get('comp_index') for plan in ranked_plans] + if original_order == ranked_order: + return + + log_info( + "Comparison-star target-fit order adjusted by pre-UltraNest coverage/scout preflight " + "(Tmid remains free; scout uses coverage, scatter, depth plausibility, and EEBLS SNR)." + ) + for new_rank, plan in enumerate(ranked_plans, start=1): + summary = plan['summary'] + preflight = plan.get('preflight') or {} + coverage = preflight.get('coverage_assessment') or {} + scout = preflight.get('scout') or {} + scout_score = scout.get('score', np.nan) + scout_text = "n/a" if not np.isfinite(scout_score) else f"{scout_score:.3f}" + scatter = scout.get('scatter', np.nan) + scatter_text = "n/a" if not np.isfinite(scatter) else f"{100.0 * scatter:.4f}%" + eebls_snr = scout.get('eebls_snr', np.nan) + eebls_text = "n/a" if not np.isfinite(eebls_snr) else f"{eebls_snr:.2f}" + label = summary.get('label', f"Comp {summary.get('comp_index', 0) + 1}") + log_info( + f" Preflight rank {new_rank}: {label} " + f"(field rank {plan.get('field_rank', 0) + 1}), coverage_priority={preflight.get('coverage_priority', 'n/a')}, " + f"pre/post={coverage.get('pre_ingress_points', 'n/a')}/{coverage.get('post_egress_points', 'n/a')}, " + f"scout={scout_text}, scatter={scatter_text}, eebls_snr={eebls_text}." + ) + + +def match_time_subset_indices(full_times, subset_times, rtol=1e-10, atol=1e-10): + full_times = np.asarray(full_times, dtype=float).reshape(-1) + subset_times = np.asarray(subset_times, dtype=float).reshape(-1) + if subset_times.size == 0: + return np.array([], dtype=int) + if full_times.size < subset_times.size: + return None + + matched_indices = [] + search_start = 0 + for subset_time in subset_times: + if not np.isfinite(subset_time): + return None + remaining = full_times[search_start:] + matches = np.flatnonzero(np.isclose(remaining, subset_time, rtol=rtol, atol=atol)) + if matches.size == 0: + return None + matched_index = search_start + int(matches[0]) + matched_indices.append(matched_index) + search_start = matched_index + 1 + + return np.asarray(matched_indices, dtype=int) + + +def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, airmass, ld, p_dict, + jd_times=None, + disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True, + plot_time_range=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + adaptive_summary=None, + run_fast_ultranest_before_final_run=FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, + precomputed_candidate_series=None): + result = { + 'applied': False, + 'fit': None, + 'good_times': np.array([], dtype=float), + 'good_flux': np.array([], dtype=float), + 'good_unc': np.array([], dtype=float), + 'good_airmass': np.array([], dtype=float), + 'good_jd_times': np.array([], dtype=float), + 'good_target_flux': np.array([], dtype=float), + 'good_comp_flux': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), + 'data_highres': None, + 'duration_samples': np.array([], dtype=float), + 'failure_reason': "full candidate reduction did not run.", + 'filter_diagnostics': [], + 'note': None, + } + if precomputed_candidate_series is None: + prepared = prepare_comparison_candidate_full_reduction_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + adaptive_summary=adaptive_summary, + ) + else: + prepared = precomputed_candidate_series + result['filter_diagnostics'] = prepared.get('filter_diagnostics', []) + if not prepared.get('applied'): + result['failure_reason'] = prepared.get( + 'failure_reason', + "the raw comparison-candidate photometry did not yield a usable light curve.", + ) + return result + + good_times = np.asarray(prepared['time'], dtype=float) + good_flux = np.asarray(prepared['flux'], dtype=float) + good_unc = np.asarray(prepared['unc'], dtype=float) + good_airmass = np.asarray(prepared['airmass'], dtype=float) + good_jd_times = np.asarray(prepared['jd_time'], dtype=float) + good_target_flux = np.asarray(prepared['target_flux'], dtype=float) + good_comp_flux = np.asarray(prepared['comp_flux'], dtype=float) + source_indices = np.asarray(prepared['source_indices'], dtype=int) + + prior = build_comparison_candidate_transit_prior(p_dict, ld) expected_duration = estimate_transit_duration_from_prior_geometry(prior) tmid_search_summary = estimate_ephemeris_tmid_and_bounds( @@ -2368,6 +2686,7 @@ def refit_selected_fast_comparison_on_full_lightcurve( fixed_parameter_errors=fixed_errors, fixed_flux_baseline=True, ultranest_min_num_live_points=min_live_points, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, ) annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) @@ -3563,6 +3882,7 @@ def run_nested_lightcurve_fit_with_rprs_posterior_retry( fixed_parameter_errors=None, fixed_flux_baseline=False, ultranest_min_num_live_points=None, + pre_ultranest_coverage_assessment=None, ): def impact_parameter_retry_available(fit, local_bounds): if not use_impactparameter_rather_than_inclination_to_fit or 'inc' not in local_bounds: @@ -3609,6 +3929,7 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co return new_upper > previous_upper + 1e-12 return new_lower < previous_lower - 1e-12 or new_upper > previous_upper + 1e-12 + partial_retry_limits = partial_transit_geometry_retry_limits(pre_ultranest_coverage_assessment) retry_configs = [ { 'key': 'rprs', @@ -3619,7 +3940,10 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co 'enforce_half_width': enforce_minimum_rprs_retry_half_width, 'propose_bounds': identity_retry_bounds, 'expands_bounds': normal_retry_expands, - 'max_retries': max_rprs_retries, + 'max_retries': min( + max_rprs_retries, + partial_retry_limits['max_retries']['rprs'], + ) if partial_retry_limits['active'] else max_rprs_retries, 'min_bound': RPRS_SEARCH_BOUND_MIN, 'max_bound': RPRS_SEARCH_BOUND_MAX, 'prior_mode_key': 'rprs', @@ -3634,7 +3958,10 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co 'enforce_half_width': enforce_minimum_ars_retry_half_width, 'propose_bounds': identity_retry_bounds, 'expands_bounds': normal_retry_expands, - 'max_retries': max_ars_retries, + 'max_retries': min( + max_ars_retries, + partial_retry_limits['max_retries']['ars'], + ) if partial_retry_limits['active'] else max_ars_retries, 'min_bound': ARS_SEARCH_BOUND_MIN, 'max_bound': None, 'prior_mode_key': 'ars', @@ -3649,7 +3976,10 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co 'enforce_half_width': lambda mode, bounds: bounds, 'propose_bounds': impact_parameter_retry_bounds, 'expands_bounds': impact_parameter_retry_expands, - 'max_retries': max_impact_parameter_retries, + 'max_retries': min( + max_impact_parameter_retries, + partial_retry_limits['max_retries']['b'], + ) if partial_retry_limits['active'] else max_impact_parameter_retries, 'min_bound': INCLINATION_SEARCH_BOUND_MIN, 'max_bound': INCLINATION_SEARCH_BOUND_MAX, 'prior_mode_key': None, @@ -3726,10 +4056,17 @@ def build_fit(local_prior, local_bounds): if key in blocked_retry_keys: continue + new_bounds = parameter_diagnostics.get('bounds') if parameter_diagnostics else None if len(retry_histories[key]) >= int(max(0, config['max_retries'])): + if ( + partial_retry_limits['active'] + and parameter_diagnostics + and parameter_diagnostics.get('clipped') + and retry_notes[key] is None + ): + retry_notes[key] = partial_retry_limits['note'] continue - new_bounds = parameter_diagnostics.get('bounds') if parameter_diagnostics else None if parameter_diagnostics and parameter_diagnostics.get('clipped') and new_bounds is not None: retry_config = config diagnostics = parameter_diagnostics @@ -6997,6 +7334,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, ) fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) @@ -7048,6 +7386,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, ) fit = apply_plot_time_range(fit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -7146,6 +7485,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): baseline_fit_mask=baseline_fit_mask, fixed_parameter_errors=baseline_fixed_errors, fixed_flux_baseline=True, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -7262,6 +7602,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): baseline_fit_mask=baseline_fit_mask, fixed_parameter_errors=baseline_fixed_errors, fixed_flux_baseline=bool(baseline_parameter_result.get('applied')), + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -13659,6 +14000,10 @@ def summarize_lightcurve_fit_assessment(fit): rprs_retry_count = int(getattr(fit, 'rprs_posterior_refit_count', 0) or 0) except (TypeError, ValueError): rprs_retry_count = 0 + try: + ars_retry_count = int(getattr(fit, 'ars_posterior_refit_count', 0) or 0) + except (TypeError, ValueError): + ars_retry_count = 0 try: b_retry_count = int(getattr(fit, 'b_posterior_refit_count', 0) or 0) except (TypeError, ValueError): @@ -13689,6 +14034,9 @@ def summarize_lightcurve_fit_assessment(fit): 'rprs_posterior_refit_applied': bool(getattr(fit, 'rprs_posterior_refit_applied', False)), 'rprs_posterior_refit_count': rprs_retry_count, 'rprs_posterior_refit_note': getattr(fit, 'rprs_posterior_refit_note', None), + 'ars_posterior_refit_applied': bool(getattr(fit, 'ars_posterior_refit_applied', False)), + 'ars_posterior_refit_count': ars_retry_count, + 'ars_posterior_refit_note': getattr(fit, 'ars_posterior_refit_note', None), 'b_posterior_refit_applied': bool(getattr(fit, 'b_posterior_refit_applied', False)), 'b_posterior_refit_count': b_retry_count, 'b_posterior_refit_note': getattr(fit, 'b_posterior_refit_note', None), @@ -13739,6 +14087,14 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): if retry_count > 0 else "applied" ) + ars_retry_status = "not applied" + if assessment['ars_posterior_refit_applied']: + retry_count = assessment['ars_posterior_refit_count'] + ars_retry_status = ( + f"applied ({retry_count} refit(s))" + if retry_count > 0 else + "applied" + ) b_retry_status = "not applied" if assessment['b_posterior_refit_applied']: retry_count = assessment['b_posterior_refit_count'] @@ -13761,6 +14117,7 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): f"{indent}fit assessment: fit_method={assessment['fit_method']}, " f"duration_prior={duration_prior_status}, " f"Rp/R* posterior retry={rprs_retry_status}, " + f"a/Rs posterior retry={ars_retry_status}, " f"impact parameter posterior retry={b_retry_status}, " f"sparse posterior extension={sparse_extension_status}, " f"prefit_refinement={prefit_status}, " @@ -13781,6 +14138,8 @@ def log_lightcurve_fit_assessment_lines(fit, indent=" "): ) if assessment.get('rprs_posterior_refit_note'): log_info(f"{indent}Rp/R* posterior retry note: {assessment['rprs_posterior_refit_note']}") + if assessment.get('ars_posterior_refit_note'): + log_info(f"{indent}a/Rs posterior retry note: {assessment['ars_posterior_refit_note']}") if assessment.get('b_posterior_refit_note'): log_info(f"{indent}Impact parameter posterior retry note: {assessment['b_posterior_refit_note']}") if assessment.get('sparse_posterior_live_point_extension_note'): @@ -13880,6 +14239,8 @@ def log_comparison_candidate_evaluation_result(attempt): def comparison_selection_metric_label(selection_metric): if selection_metric == 'first_qc_pass': return "First QC PASS" + if selection_metric == 'promising_partial': + return "Promising Partial" if selection_metric == 'comparison_field_rank': return "Comparison-Field Rank" if selection_metric == 'ktmf': @@ -13889,6 +14250,35 @@ def comparison_selection_metric_label(selection_metric): return "transit-vs-flat Delta BIC" +def should_stop_after_promising_partial_comparison_attempt(attempt): + if not isinstance(attempt, dict): + return False + if attempt.get('fit') is None or not attempt.get('full_reduction_applied', False): + return False + if attempt.get('rejected_by_transit_qc', False): + return False + + status = str(attempt.get('transit_qc_status') or '').strip().lower() + if status != 'marginal': + return False + + try: + coverage_priority = int(attempt.get('preflight_coverage_priority', 4)) + except (TypeError, ValueError): + coverage_priority = 4 + if coverage_priority > 2: + return False + + ktmf_metric = coerce_finite_transit_qc_scalar(attempt.get('ktmf_metric', np.nan)) + delta_bic = coerce_finite_transit_qc_scalar(attempt.get('transit_delta_bic', np.nan)) + return ( + np.isfinite(ktmf_metric) + and ktmf_metric >= PROMISING_PARTIAL_COMPARISON_KTMF_MIN + and np.isfinite(delta_bic) + and delta_bic >= TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD + ) + + def select_preferred_comparison_attempt(attempts, pick_comparison_by_eebls_snr=True): selected_result = None selection_metric = 'ktmf' @@ -15316,9 +15706,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ), ) - attempts = [] - stopped_after_first_qc_pass = False - for rank, comp_summary in enumerate(ranked_summaries): + preflight_plans = [] + for field_rank, comp_summary in enumerate(ranked_summaries): comp_index = comp_summary['comp_index'] ckey = comp_summary.get('key', f"comp{comp_index + 1}") if method == 'psf': @@ -15341,10 +15730,45 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p airmass[fit_mask], enforce_relative_flux_max=False, ) + preflight = build_comparison_candidate_preflight( + times[fit_mask], + jd_times[fit_mask], + airmass[fit_mask], + ld, + p_dict, + target_flux[fit_mask], + comp_flux[fit_mask], + adaptive_summary=adaptive_summary, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + ) + preflight_plans.append({ + 'field_rank': field_rank, + 'summary': comp_summary, + 'ckey': ckey, + 'comp_flux': comp_flux, + 'fit_mask': fit_mask, + 'fit_diagnostics': fit_diagnostics, + 'preflight': preflight, + }) + + ranked_preflight_plans = rank_comparison_candidate_preflight_plans(preflight_plans) + log_comparison_candidate_preflight_order(preflight_plans, ranked_preflight_plans) + + attempts = [] + stopped_after_first_qc_pass = False + stopped_after_promising_partial = False + for rank, plan in enumerate(ranked_preflight_plans): + comp_summary = plan['summary'] + comp_index = comp_summary['comp_index'] + ckey = plan['ckey'] + comp_flux = plan['comp_flux'] + fit_mask = plan['fit_mask'] + fit_diagnostics = plan['fit_diagnostics'] + preflight = plan.get('preflight') or {} log_comparison_candidate_evaluation_start( comp_summary, rank, - len(ranked_summaries), + len(ranked_preflight_plans), method_label, fit_diagnostics, ) @@ -15369,6 +15793,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p baseline_duration_multiplier=final_fit_baseline_duration_multiplier, adaptive_summary=adaptive_summary, run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, + precomputed_candidate_series=preflight.get('prepared_series'), ) fit_result = final_reduction.get('fit') if final_reduction.get('applied') else None tflux_fit = final_reduction.get('good_target_flux') @@ -15405,6 +15830,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p attempt = { 'rank': rank, + 'field_rank': plan.get('field_rank'), 'comp_index': comp_index, 'ckey': ckey, 'label': comp_summary.get('label', f"Comp {comp_index + 1}"), @@ -15443,6 +15869,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'selected': False, 'selection_reason': None, 'search_stopped_after_qc_pass': False, + 'search_stopped_after_promising_partial': False, 'failed_run_dir': None, 'final_output_dir': None, 'full_reduction_applied': final_reduction.get('applied', False), @@ -15450,6 +15877,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'fast_ultranest_binning': final_reduction.get('fast_ultranest_binning'), 'skip_airmass_fit': final_reduction.get('skip_airmass_fit', False), 'airmass_skip_note': final_reduction.get('airmass_skip_note'), + 'preflight_coverage_priority': preflight.get('coverage_priority'), + 'preflight_scout_score': (preflight.get('scout') or {}).get('score', np.nan), } if final_reduction.get('applied') and selection_fit is not None and save_dir is not None: final_output_dir = save_comparison_candidate_full_reduction_outputs( @@ -15504,6 +15933,17 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p f"({attempt['label']})." ) break + if ( + exit_at_first_qc_pass_solution + and should_stop_after_promising_partial_comparison_attempt(attempt) + ): + attempt['search_stopped_after_promising_partial'] = True + stopped_after_promising_partial = True + log_info( + "Stopping comparison-star candidate search after a promising partial-coverage " + f"MARGINAL fit ({attempt['label']}); proceeding to selected full-resolution confirmation." + ) + break selected_result = None completed_attempts = [ @@ -15526,11 +15966,21 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ), None, ) + first_promising_partial_attempt = next( + ( + attempt for attempt in attempts + if attempt.get('search_stopped_after_promising_partial', False) + ), + None, + ) selection_metric = 'ktmf' fallback_to_qc_rejected = False if first_qc_pass_attempt is not None: selected_result = first_qc_pass_attempt selection_metric = 'first_qc_pass' + elif first_promising_partial_attempt is not None: + selected_result = first_promising_partial_attempt + selection_metric = 'promising_partial' elif successful_attempts: selected_result, selection_metric = select_preferred_comparison_attempt( successful_attempts, @@ -15578,6 +16028,11 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p attempt['selection_reason'] = ( "selected: first completed comparison-star candidate with PASS transit QC" ) + elif attempt.get('search_stopped_after_promising_partial', False): + attempt['selection_reason'] = ( + "selected: first partial-coverage comparison-star candidate with promising " + "MARGINAL transit diagnostics" + ) elif selection_metric == 'ktmf' and np.isfinite(selected_ktmf_metric): attempt['selection_reason'] = ( "selected: highest KTMF among the evaluated " @@ -15610,6 +16065,11 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p "not selected: search stopped after the first comparison-star candidate " "with PASS transit QC" ) + elif selection_metric == 'promising_partial': + attempt['selection_reason'] = ( + "not selected: search stopped after the first partial-coverage comparison-star " + "candidate with promising MARGINAL transit diagnostics" + ) elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): if np.isfinite(attempt.get('eebls_snr', np.nan)): attempt['selection_reason'] = ( @@ -15707,6 +16167,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'selected_result': selected_result, 'selection_metric': selection_metric, 'stopped_after_first_qc_pass': stopped_after_first_qc_pass, + 'stopped_after_promising_partial': stopped_after_promising_partial, } @@ -17065,6 +17526,8 @@ def _main_impl(): ) if selected_attempt.get('search_stopped_after_qc_pass', False): selection_basis = 'first_qc_pass' + elif selected_attempt.get('search_stopped_after_promising_partial', False): + selection_basis = 'promising_partial' elif selected_attempt.get('selected_despite_transit_qc', False): selection_basis = 'comparison_field_qc_fallback' elif selected_comp_index == comparison_calibration['best_comp_index']: @@ -17077,6 +17540,13 @@ def _main_impl(): f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " "because it was the first candidate to pass transit QC." ) + elif selection_basis == 'promising_partial': + log_info( + "Comparison-star calibration target-fit selection chose " + f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " + "because pre-UltraNest preflight and the candidate fit indicated a promising " + "partial-coverage MARGINAL solution." + ) elif selection_basis == 'comparison_field_qc_fallback': fallback_selection_metric = comparison_fit_search.get('selection_metric', 'ktmf') if fallback_selection_metric == 'ktmf': diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index c41ab3ee..2531385c 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -137,6 +137,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: parse_deviation_from_expected_transit_in_qc_sigma, prepare_final_fit_lightcurve_series, prepare_lightcurve_fit_input_series, + rank_comparison_candidate_preflight_plans, representative_psf_sigma, ranked_comparison_calibration_summaries, resolve_sky_annulus_geometry, @@ -157,6 +158,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: should_use_aperture_photometry, should_exit_at_first_qc_pass_solution, should_pick_comparison_by_eebls_snr, + should_stop_after_promising_partial_comparison_attempt, should_use_psf_photometry, should_skip_low_comparison_coverage_rejection, should_assess_all_comparisons_before_selecting_best, @@ -1591,6 +1593,8 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7) assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75) assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(0.75) + assert "Rp/R* sigma=2.00" in contributions_by_label["Deviation From Expected Value"]["detail"] + assert "Tmid" not in contributions_by_label["Deviation From Expected Value"]["detail"] model_evidence_score = ((1.0 - np.exp(-1.0)) + (1.0 - np.exp(-2.0))) / 2.0 expected_ktmf = ( @@ -3033,7 +3037,7 @@ def test_annotate_transit_qc_expected_values_coerces_scalar_like_inputs(): assert fit.transit_qc_deviation_sigma_threshold == pytest.approx(7.5) -def test_evaluate_transit_detection_qc_failure_summary_reflects_expected_value_rejection(): +def test_evaluate_transit_detection_qc_keeps_tmid_deviation_diagnostic_only(): times = np.linspace(0.0, 1.0, 21) transit_model = np.ones(times.shape[0], dtype=float) transit_model[9:12] -= 0.02 @@ -3068,11 +3072,13 @@ def test_evaluate_transit_detection_qc_failure_summary_reflects_expected_value_r summary = evaluate_transit_detection_qc(fit) - assert summary["status"] == "fail" + assert summary["status"] == "pass" assert summary["tmid_deviation_minutes"] == pytest.approx(40.32) - assert "QC rejected the fit because" in summary["summary"] - assert "Tmid of the fit is 40.32 minutes away from the ephemeris Tmid" in summary["summary"] - assert "7.20 minutes" in summary["summary"] + assert summary["rprs_deviation_sigma"] == pytest.approx(0.0) + assert summary["deviation_from_expected_value"] == pytest.approx(1.0) + assert any("Expected-value Tmid deviation: 40.32 minutes" in note for note in summary["notes"]) + assert "QC rejected the fit because" not in summary["summary"] + assert "Tmid of the fit is 40.32 minutes away from the ephemeris Tmid" not in summary["summary"] assert "not supported strongly enough against a flat/null model" not in summary["summary"] @@ -3545,6 +3551,94 @@ def fake_finalize( assert "first completed comparison-star candidate" in result["selected_result"]["selection_reason"] +def test_fit_ranked_comparison_calibration_candidates_stops_at_promising_partial_marginal(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + monkeypatch.setattr( + "exotic.exotic.build_comparison_candidate_preflight", + lambda *args, **kwargs: { + "prepared_series": None, + "coverage_priority": 2, + "scout": {"score": np.nan}, + }, + ) + + call_markers = [] + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + call_markers.append(comp_marker) + status = "marginal" if comp_marker == 50 else "pass" + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={"status": status, "summary": "ok"}, + transit_qc_status=status, + transit_qc_ktmf_metric=3.5, + transit_qc_delta_bic=15.1, + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert call_markers == [50] + assert result["selection_metric"] == "promising_partial" + assert result["selected_result"]["search_stopped_after_promising_partial"] is True + assert result["stopped_after_promising_partial"] is True + + def test_fit_ranked_comparison_calibration_candidates_can_evaluate_all_qc_passes_when_exit_disabled(monkeypatch): def fake_diagnostics(*args, **kwargs): return {"usable_point_count": 6} @@ -3649,6 +3743,47 @@ def test_ranked_comparison_calibration_summaries_skip_suitability_outliers(): assert [summary["comp_index"] for summary in ranked] == [2, 3] +def test_comparison_preflight_ranking_prioritizes_full_coverage_then_scout_score(): + plans = [ + { + "field_rank": 0, + "summary": {"comp_index": 7, "aggregate_score": 0.002175, "label": "Comp 8"}, + "preflight": {"coverage_priority": 2, "scout": {"score": 0.42}}, + }, + { + "field_rank": 1, + "summary": {"comp_index": 0, "aggregate_score": 0.002331, "label": "Comp 1"}, + "preflight": {"coverage_priority": 2, "scout": {"score": 0.91}}, + }, + { + "field_rank": 4, + "summary": {"comp_index": 2, "aggregate_score": 0.002804, "label": "Comp 3"}, + "preflight": {"coverage_priority": 0, "scout": {"score": 0.25}}, + }, + ] + + ranked = rank_comparison_candidate_preflight_plans(plans) + + assert [plan["summary"]["comp_index"] for plan in ranked] == [2, 0, 7] + + +def test_promising_partial_comparison_attempt_can_stop_candidate_search(): + attempt = { + "fit": object(), + "full_reduction_applied": True, + "rejected_by_transit_qc": False, + "transit_qc_status": "marginal", + "preflight_coverage_priority": 2, + "ktmf_metric": 3.50, + "transit_delta_bic": 15.09, + } + + assert should_stop_after_promising_partial_comparison_attempt(attempt) is True + + attempt["preflight_coverage_priority"] = 4 + assert should_stop_after_promising_partial_comparison_attempt(attempt) is False + + def test_fit_ranked_comparison_calibration_candidates_applies_field_image_clip(monkeypatch): observed_lengths = [] diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 5706326b..deb981f6 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -904,6 +904,68 @@ def fake_lc_fitter( assert fit.ars_posterior_refit_bounds == pytest.approx([12.60, 16.30]) +def test_partial_coverage_suppresses_open_geometry_posterior_retries(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics = { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "ars": {"clipped": True, "edge": "lower", "mode": 5.0, "std": 3.0, "bounds": [0.000001, 20.0]}, + "b": {"clipped": True, "edge": "upper", "mode": 1.6, "std": 0.3, "bounds": [0.5, 2.5]}, + } + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured["calls"].append({"prior": dict(call_prior), "bounds": dict(call_bounds)}) + fit = types.SimpleNamespace( + sampled_keys=["rprs", "ars", "b", "tmid"], + sample_bounds={"rprs": [0.0, 0.25], "ars": [0.000001, 20.0], "b": [0.0, 2.5]}, + parameters={"rprs": 0.1, "ars": 5.0, "tmid": 0.0, "inc": 80.0, "a2": 0.0}, + ) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: dict(diagnostics[key]) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.0, 0.25], + "ars": [5.0, 15.0], + "tmid": [-0.01, 0.01], + "inc": [70.0, 90.0], + "a2": [-3.0, 3.0], + }, + pre_ultranest_coverage_assessment={ + "valid": True, + "success_label": "low", + "expected_successful": False, + "pre_ingress_points": 0, + "post_egress_points": 8, + }, + ) + + assert len(captured["calls"]) == 1 + assert fit.ars_posterior_refit_applied is False + assert "one-sided/LOW" in fit.ars_posterior_refit_note + assert fit.b_posterior_refit_applied is False + assert "one-sided/LOW" in fit.b_posterior_refit_note + + def test_impact_parameter_posterior_retry_expands_inclination_bounds(monkeypatch): import exotic.exotic as exotic_module From 840d8834c3a087501a5a8f3daa62416a92255676 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 23 May 2026 17:38:09 +1000 Subject: [PATCH 050/116] uncertainties from data, mate, not priors. --- exotic/exotic.py | 101 ++++++++----------- exotic/output_files.py | 20 +--- inits.json | 2 +- tests/test_exotic_proper_motion.py | 156 +++++++++++++++++++++++++++-- tests/test_output_files.py | 37 ++++++- 5 files changed, 229 insertions(+), 87 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 29d33fb4..12632d57 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -851,10 +851,13 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T 'expected_tmid_unc': np.nan, 'expected_tmid_unc_minutes': np.nan, 'fitted_tmid': np.nan, + 'fitted_rprs': np.nan, + 'fitted_rprs_unc': np.nan, 'tmid_deviation_days': np.nan, 'tmid_deviation_minutes': np.nan, 'tmid_deviation_threshold_minutes': np.nan, 'tmid_deviation_sigma': np.nan, + 'rprs_deviation_fit_unc': np.nan, 'rprs_deviation_sigma': np.nan, 'tmid_deviation_score': np.nan, 'rprs_deviation_score': np.nan, @@ -872,43 +875,26 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T sigma_threshold = expected.get('deviation_sigma_threshold', sigma_threshold) summary['sigma_threshold'] = sigma_threshold - expected_tmid = expected.get('expected_tmid', np.nan) - expected_tmid_unc = expected.get('expected_tmid_unc', np.nan) - fitted_tmid = parameters.get('tmid', np.nan) - summary['expected_tmid'] = expected_tmid - summary['expected_tmid_unc'] = expected_tmid_unc - summary['fitted_tmid'] = fitted_tmid + summary['expected_tmid'] = expected.get('expected_tmid', np.nan) + summary['expected_tmid_unc'] = expected.get('expected_tmid_unc', np.nan) + summary['fitted_tmid'] = parameters.get('tmid', np.nan) if not enabled: return summary - if ( - np.isfinite(expected_tmid) - and np.isfinite(expected_tmid_unc) - and expected_tmid_unc > 0 - and np.isfinite(fitted_tmid) - ): - tmid_deviation_days = float(abs(fitted_tmid - expected_tmid)) - tmid_sigma = float(tmid_deviation_days / expected_tmid_unc) - summary['tmid_deviation_days'] = tmid_deviation_days - summary['tmid_deviation_minutes'] = tmid_deviation_days * 24.0 * 60.0 - summary['expected_tmid_unc_minutes'] = float(expected_tmid_unc) * 24.0 * 60.0 - if np.isfinite(sigma_threshold) and sigma_threshold > 0: - summary['tmid_deviation_threshold_minutes'] = ( - float(sigma_threshold) * float(expected_tmid_unc) * 24.0 * 60.0 - ) - summary['tmid_deviation_sigma'] = tmid_sigma - summary['tmid_deviation_score'] = transit_qc_deviation_score_from_sigma(tmid_sigma, sigma_threshold) - expected_rprs = expected.get('expected_rprs', np.nan) - expected_rprs_unc = expected.get('expected_rprs_unc', np.nan) fitted_rprs = parameters.get('rprs', np.nan) + errors = getattr(fit, 'errors', {}) or {} + fitted_rprs_unc = errors.get('rprs', np.nan) + summary['fitted_rprs'] = fitted_rprs + summary['fitted_rprs_unc'] = fitted_rprs_unc if ( np.isfinite(expected_rprs) - and np.isfinite(expected_rprs_unc) - and expected_rprs_unc > 0 and np.isfinite(fitted_rprs) + and np.isfinite(fitted_rprs_unc) + and fitted_rprs_unc > 0 ): - rprs_sigma = float(abs(fitted_rprs - expected_rprs) / expected_rprs_unc) + rprs_sigma = float(abs(fitted_rprs - expected_rprs) / fitted_rprs_unc) + summary['rprs_deviation_fit_unc'] = fitted_rprs_unc summary['rprs_deviation_sigma'] = rprs_sigma summary['rprs_deviation_score'] = transit_qc_deviation_score_from_sigma(rprs_sigma, sigma_threshold) @@ -916,18 +902,13 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T summary['available'] = True summary['deviation_from_expected_value'] = float(summary['rprs_deviation_score']) - if np.isfinite(summary['tmid_deviation_sigma']): - summary['notes'].append( - "Expected-value Tmid deviation: " - f"{summary['tmid_deviation_minutes']:.2f} minutes " - f"({summary['tmid_deviation_sigma']:.2f} sigma; " - f"fit={summary['fitted_tmid']:.6f}, " - f"ephemeris={summary['expected_tmid']:.6f} +/- " - f"{summary['expected_tmid_unc_minutes']:.2f} minutes)." - ) if np.isfinite(summary['rprs_deviation_sigma']): summary['notes'].append( - f"Expected-value Rp/R* deviation: {summary['rprs_deviation_sigma']:.2f} sigma." + "Expected-value Rp/R* deviation: " + f"{summary['rprs_deviation_sigma']:.2f} sigma " + f"(fit={summary['fitted_rprs']:.6f} +/- {summary['fitted_rprs_unc']:.6f}, " + f"expected={expected_rprs:.6f}; " + f"fit uncertainty={summary['rprs_deviation_fit_unc']:.6f})." ) rprs_sigma = summary['rprs_deviation_sigma'] @@ -956,6 +937,18 @@ def compute_transit_qc_ktmf(summary): if np.isfinite(summary.get('delta_chi2', np.nan)) else "Delta chi2=n/a", ] + deviation_score = summary.get('deviation_from_expected_value', np.nan) + if np.isfinite(deviation_score): + deviation_detail_parts = [ + f"score={deviation_score:.2f}", + f"Rp/R* sigma={summary.get('rprs_deviation_sigma', np.nan):.2f}", + ] + rprs_fit_unc = summary.get('rprs_deviation_fit_unc', np.nan) + if np.isfinite(rprs_fit_unc): + deviation_detail_parts.append(f"fit uncertainty={rprs_fit_unc:.6f}") + deviation_detail = ", ".join(deviation_detail_parts) + else: + deviation_detail = "expected-value deviation disabled or unavailable" raw_components = [ { @@ -967,13 +960,8 @@ def compute_transit_qc_ktmf(summary): { 'key': 'deviation_from_expected_value', 'label': 'Deviation From Expected Value', - 'score': summary.get('deviation_from_expected_value', np.nan), - 'detail': ( - f"score={summary.get('deviation_from_expected_value', np.nan):.2f}, " - f"Rp/R* sigma={summary.get('rprs_deviation_sigma', np.nan):.2f}" - if np.isfinite(summary.get('deviation_from_expected_value', np.nan)) - else "expected-value deviation disabled or unavailable" - ), + 'score': deviation_score, + 'detail': deviation_detail, }, { 'key': 'residual_scatter', @@ -1246,10 +1234,13 @@ def evaluate_transit_detection_qc(fit): 'fitted_tmid': np.nan, 'expected_rprs': expected_context.get('expected_rprs', np.nan), 'expected_rprs_unc': expected_context.get('expected_rprs_unc', np.nan), + 'fitted_rprs': np.nan, + 'fitted_rprs_unc': np.nan, 'tmid_deviation_days': np.nan, 'tmid_deviation_minutes': np.nan, 'tmid_deviation_threshold_minutes': np.nan, 'tmid_deviation_sigma': np.nan, + 'rprs_deviation_fit_unc': np.nan, 'rprs_deviation_sigma': np.nan, 'tmid_deviation_score': np.nan, 'rprs_deviation_score': np.nan, @@ -1381,10 +1372,13 @@ def evaluate_transit_detection_qc(fit): 'expected_tmid_unc': deviation_summary.get('expected_tmid_unc', np.nan), 'expected_tmid_unc_minutes': deviation_summary.get('expected_tmid_unc_minutes', np.nan), 'fitted_tmid': deviation_summary.get('fitted_tmid', np.nan), + 'fitted_rprs': deviation_summary.get('fitted_rprs', np.nan), + 'fitted_rprs_unc': deviation_summary.get('fitted_rprs_unc', np.nan), 'tmid_deviation_days': deviation_summary.get('tmid_deviation_days', np.nan), 'tmid_deviation_minutes': deviation_summary.get('tmid_deviation_minutes', np.nan), 'tmid_deviation_threshold_minutes': deviation_summary.get('tmid_deviation_threshold_minutes', np.nan), 'tmid_deviation_sigma': deviation_summary.get('tmid_deviation_sigma', np.nan), + 'rprs_deviation_fit_unc': deviation_summary.get('rprs_deviation_fit_unc', np.nan), 'rprs_deviation_sigma': deviation_summary.get('rprs_deviation_sigma', np.nan), 'tmid_deviation_score': deviation_summary.get('tmid_deviation_score', np.nan), 'rprs_deviation_score': deviation_summary.get('rprs_deviation_score', np.nan), @@ -1533,10 +1527,13 @@ def annotate_transit_detection_qc(fit, summary=None): fit.transit_qc_expected_tmid_unc = summary.get('expected_tmid_unc') fit.transit_qc_expected_tmid_unc_minutes = summary.get('expected_tmid_unc_minutes') fit.transit_qc_fitted_tmid = summary.get('fitted_tmid') + fit.transit_qc_fitted_rprs = summary.get('fitted_rprs') + fit.transit_qc_fitted_rprs_unc = summary.get('fitted_rprs_unc') fit.transit_qc_tmid_deviation_days = summary.get('tmid_deviation_days') fit.transit_qc_tmid_deviation_minutes = summary.get('tmid_deviation_minutes') fit.transit_qc_tmid_deviation_threshold_minutes = summary.get('tmid_deviation_threshold_minutes') fit.transit_qc_tmid_deviation_sigma = summary.get('tmid_deviation_sigma') + fit.transit_qc_rprs_deviation_fit_unc = summary.get('rprs_deviation_fit_unc') fit.transit_qc_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') fit.transit_qc_expected_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') fit.transit_qc_ktmf_metric = summary.get('ktmf_metric') @@ -2720,6 +2717,7 @@ def refit_selected_fast_comparison_on_full_lightcurve( 'note': 'Full-resolution selected comparison-star final run; fast binning was not applied.', }, ) + annotate_transit_qc_expected_values(fit, p_dict) if target_live_points is not None: diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) annotate_sparse_posterior_live_point_extension( @@ -18386,19 +18384,6 @@ def _main_impl(): log_info( f" Deviation From Expected Value: {transit_qc['deviation_from_expected_value']:.2f} / 1.00" ) - if np.isfinite(transit_qc.get('tmid_deviation_sigma', np.nan)): - log_info( - f" Expected-value Tmid sigma: {transit_qc['tmid_deviation_sigma']:.2f}" - ) - if np.isfinite(transit_qc.get('tmid_deviation_minutes', np.nan)): - log_info( - f" Expected-value Tmid offset: {transit_qc['tmid_deviation_minutes']:.2f} minutes" - ) - if np.isfinite(transit_qc.get('tmid_deviation_threshold_minutes', np.nan)): - log_info( - " Expected-value Tmid QC window: " - f"{transit_qc['tmid_deviation_threshold_minutes']:.2f} minutes" - ) if np.isfinite(transit_qc.get('rprs_deviation_sigma', np.nan)): log_info( f" Expected-value Rp/R* sigma: {transit_qc['rprs_deviation_sigma']:.2f}" diff --git a/exotic/output_files.py b/exotic/output_files.py index 3c328512..0d69594f 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -391,11 +391,9 @@ def build_aavso_qc_metadata(fit): 'flat_baseline', 'flat_a2', 'flat_model_note', 'residual_scatter', 'rprs_sigma', 'duration_ratio', 'eebls_depth_snr', 'use_deviation_from_expected_transit_in_qc', 'deviation_sigma_threshold', - 'expected_tmid', 'expected_tmid_unc', 'expected_tmid_unc_minutes', - 'fitted_tmid', 'expected_rprs', 'expected_rprs_unc', - 'tmid_deviation_days', 'tmid_deviation_minutes', - 'tmid_deviation_threshold_minutes', 'tmid_deviation_sigma', - 'rprs_deviation_sigma', 'tmid_deviation_score', 'rprs_deviation_score', + 'expected_tmid', 'expected_tmid_unc', 'fitted_tmid', + 'expected_rprs', 'expected_rprs_unc', 'fitted_rprs', 'fitted_rprs_unc', + 'rprs_deviation_fit_unc', 'rprs_deviation_sigma', 'rprs_deviation_score', 'deviation_from_expected_value', 'ktmf_metric', 'ktmf_contributions', 'notes', ) @@ -975,10 +973,6 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor qc_duration_ratio = transit_qc.get('duration_ratio', np.nan) qc_eebls_depth_snr = transit_qc.get('eebls_depth_snr', np.nan) qc_deviation_metric = transit_qc.get('deviation_from_expected_value', np.nan) - qc_tmid_deviation_sigma = transit_qc.get('tmid_deviation_sigma', np.nan) - qc_tmid_deviation_minutes = transit_qc.get('tmid_deviation_minutes', np.nan) - qc_tmid_threshold_minutes = transit_qc.get('tmid_deviation_threshold_minutes', np.nan) - qc_expected_tmid_unc_minutes = transit_qc.get('expected_tmid_unc_minutes', np.nan) qc_rprs_deviation_sigma = transit_qc.get('rprs_deviation_sigma', np.nan) qc_sigma_threshold = transit_qc.get('deviation_sigma_threshold', np.nan) qc_ktmf = transit_qc.get('ktmf_metric', np.nan) @@ -1002,14 +996,6 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num["Deviation From Expected Value"] = f"{qc_deviation_metric:.2f} / 1.00" if np.isfinite(qc_sigma_threshold): params_num["Expected-value QC threshold"] = f"{qc_sigma_threshold:.2f} sigma" - if np.isfinite(qc_tmid_deviation_sigma): - params_num["Expected-value Tmid deviation"] = f"{qc_tmid_deviation_sigma:.2f} sigma" - if np.isfinite(qc_tmid_deviation_minutes): - params_num["Expected-value Tmid offset"] = f"{qc_tmid_deviation_minutes:.2f} minutes" - if np.isfinite(qc_expected_tmid_unc_minutes): - params_num["Expected-value Tmid uncertainty"] = f"{qc_expected_tmid_unc_minutes:.2f} minutes" - if np.isfinite(qc_tmid_threshold_minutes): - params_num["Expected-value Tmid QC window"] = f"{qc_tmid_threshold_minutes:.2f} minutes" if np.isfinite(qc_rprs_deviation_sigma): params_num["Expected-value Rp/R* deviation"] = f"{qc_rprs_deviation_sigma:.2f} sigma" if np.isfinite(qc_ktmf): diff --git a/inits.json b/inits.json index 92371515..93b2d956 100644 --- a/inits.json +++ b/inits.json @@ -33,7 +33,7 @@ "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to use EEBLS depth SNR as an earlier tie-break when KTMF scores do not settle the comparison-star choice. Default y.", - "Expected-Value Transit QC": "Set optional_info 'use_deviation_from_expected_transit_in_qc' to true to reject transit fits whose fitted Tmid or Rp/R* stray too far from the published expected values. Default true.", + "Expected-Value Transit QC": "Set optional_info 'use_deviation_from_expected_transit_in_qc' to true to reject transit fits whose fitted Rp/R* strays too far from the published expected value using the fitted Rp/R* uncertainty only. Default true.", "Expected-Value Transit QC Sigma": "Set optional_info 'deviation_from_expected_transit_in_qc_sigma' to the sigma threshold used by the expected-value transit QC rejection. Default 5.", "Assess All Comparisons Before Selecting Best": "Set optional_info 'assess_all_comparisons_before_selecting_best' to y to fit every comparison-star candidate that survives the earlier screening, report all fits, and select the candidate with the highest KTMF score. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 2531385c..6e01d322 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -138,6 +138,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: prepare_final_fit_lightcurve_series, prepare_lightcurve_fit_input_series, rank_comparison_candidate_preflight_plans, + refit_selected_fast_comparison_on_full_lightcurve, representative_psf_sigma, ranked_comparison_calibration_summaries, resolve_sky_annulus_geometry, @@ -2981,12 +2982,12 @@ def test_evaluate_transit_detection_qc_rejects_large_expected_value_deviation(): assert summary["computed"] is True assert summary["status"] == "fail" - assert summary["tmid_deviation_sigma"] == pytest.approx(6.0) - assert summary["tmid_deviation_minutes"] == pytest.approx(8.64) - assert summary["tmid_deviation_threshold_minutes"] == pytest.approx(7.2) assert summary["rprs_deviation_sigma"] == pytest.approx(8.0) assert summary["deviation_from_expected_value"] == pytest.approx(0.0) assert summary["ktmf_metric"] <= 5.0 + assert np.isnan(summary["tmid_deviation_sigma"]) + assert np.isnan(summary["tmid_deviation_minutes"]) + assert summary["rprs_deviation_fit_unc"] == pytest.approx(0.01) def test_annotate_transit_qc_expected_values_prefers_propagated_epoch_tmid(): @@ -3037,7 +3038,7 @@ def test_annotate_transit_qc_expected_values_coerces_scalar_like_inputs(): assert fit.transit_qc_deviation_sigma_threshold == pytest.approx(7.5) -def test_evaluate_transit_detection_qc_keeps_tmid_deviation_diagnostic_only(): +def test_evaluate_transit_detection_qc_does_not_calculate_tmid_expected_value_deviation(): times = np.linspace(0.0, 1.0, 21) transit_model = np.ones(times.shape[0], dtype=float) transit_model[9:12] -= 0.02 @@ -3073,15 +3074,156 @@ def test_evaluate_transit_detection_qc_keeps_tmid_deviation_diagnostic_only(): summary = evaluate_transit_detection_qc(fit) assert summary["status"] == "pass" - assert summary["tmid_deviation_minutes"] == pytest.approx(40.32) + assert np.isnan(summary["tmid_deviation_minutes"]) + assert np.isnan(summary["tmid_deviation_sigma"]) assert summary["rprs_deviation_sigma"] == pytest.approx(0.0) assert summary["deviation_from_expected_value"] == pytest.approx(1.0) - assert any("Expected-value Tmid deviation: 40.32 minutes" in note for note in summary["notes"]) + assert not any("Expected-value Tmid" in note for note in summary["notes"]) assert "QC rejected the fit because" not in summary["summary"] assert "Tmid of the fit is 40.32 minutes away from the ephemeris Tmid" not in summary["summary"] assert "not supported strongly enough against a flat/null model" not in summary["summary"] +def test_expected_value_rprs_deviation_uses_fit_uncertainty_not_prior_uncertainty(): + transit_model = np.ones(21, dtype=float) + transit_model[9:12] -= 0.0287 + data = transit_model.copy() + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.1694, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.0046, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 0.5], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.001, + transit_qc_expected_rprs=0.1589, + transit_qc_expected_rprs_unc=0.0001, + transit_qc_use_deviation_from_expected_transit_in_qc=True, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["rprs_deviation_fit_unc"] == pytest.approx(0.0046) + assert summary["rprs_deviation_sigma"] == pytest.approx(abs(0.1694 - 0.1589) / 0.0046) + assert summary["deviation_from_expected_value"] == pytest.approx( + 1.0 - summary["rprs_deviation_sigma"] / 5.0 + ) + assert summary["deviation_from_expected_value"] > 0.0 + + +def test_selected_full_resolution_refit_keeps_expected_value_context(monkeypatch): + import exotic.exotic as exotic_module + + times = np.linspace(0.0, 1.0, 21) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[9:12] -= 0.0287 + errors = np.full(times.shape[0], 0.0015, dtype=float) + airmass = np.ones(times.shape[0], dtype=float) + + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + parameters={ + "rprs": 0.1694, + "tmid": 0.5, + "ars": 5.0, + "inc": 89.0, + "per": 1.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + }, + errors={"rprs": 0.0046, "tmid": 0.001, "ars": 0.1, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + bounds={"rprs": [0.0, 0.5], "tmid": [0.4, 0.6], "ars": [1.0, 10.0], "inc": [80.0, 90.0]}, + ) + + def fake_run_nested(*args, **kwargs): + return types.SimpleNamespace( + time=times, + data=transit_model.copy(), + dataerr=errors.copy(), + model=transit_model.copy(), + airmass=airmass.copy(), + prior={"per": 1.0, "tmid": 0.5}, + parameters={ + "rprs": 0.1694, + "tmid": 0.5, + "ars": 5.0, + "inc": 89.0, + "per": 1.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + }, + errors={"rprs": 0.0046, "tmid": 0.001, "ars": 0.1, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + bounds={"rprs": [0.0, 0.5], "tmid": [0.4, 0.6], "ars": [1.0, 10.0], "inc": [80.0, 90.0]}, + airmass_fit_skipped=True, + eebls_diagnostic_depth_snr=50.0, + duration_expected=0.1, + duration_measured=0.1, + ) + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + monkeypatch.setattr(exotic_module, "build_expected_transit_coverage_assessment", lambda *args, **kwargs: {}) + monkeypatch.setattr(exotic_module, "log_expected_transit_coverage_assessment", lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, "annotate_pre_ultranest_transit_coverage", lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, "selected_final_live_point_target", lambda *args, **kwargs: (200, None)) + + p_dict = { + "rprs": 0.1589, + "rprsUnc": 0.0001, + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.0, + "aRs": 5.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "use_deviation_from_expected_transit_in_qc": True, + "deviation_from_expected_transit_in_qc_sigma": 5.0, + } + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": transit_model.copy(), + "good_unc": errors.copy(), + "good_airmass": airmass.copy(), + "good_jd_times": times.copy(), + "fast_fit_bounds": previous_fit.bounds, + } + + refit, _, _ = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + p_dict, + detrend_on_outoftransit_baseline=False, + duration_prior={"duration": 0.1}, + ) + + assert refit.transit_qc_rprs_deviation_fit_unc == pytest.approx(0.0046) + assert refit.transit_qc_rprs_deviation_sigma == pytest.approx(abs(0.1694 - 0.1589) / 0.0046) + contribution = next( + item for item in refit.transit_qc_ktmf_contributions + if item["label"] == "Deviation From Expected Value" + ) + assert contribution["score"] > 0.0 + assert contribution["max_points"] > 0.0 + assert "fit uncertainty=0.004600" in contribution["detail"] + assert "Tmid" not in contribution["detail"] + + def test_evaluate_transit_detection_qc_computes_missing_eebls_depth_snr(monkeypatch): def fake_eebls(times, flux_values, flux_errors, prior, fallback_bounds): return { @@ -4701,7 +4843,7 @@ def __init__(self, ktmf, delta_bic): self.transit_qc_status = "fail" self.transit_qc_summary = ( "Transit model is preferred over the flat/null model, but QC rejected the fit because " - "the fit deviates too far from the expected published Tmid and/or Rp/R* values " + "the fit deviates too far from the expected published Rp/R* value " f"(Delta BIC={delta_bic:.2f}, Delta chi2=27.10)." ) self.transit_qc = { diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 46c7c9d6..07d9f734 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -333,6 +333,34 @@ def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclinati assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" +def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertainties(tmp_path): + fit = DummyFit() + (tmp_path / "temp").mkdir() + + p_dict = { + "pName": "HAT-P-32 b", + "midTUnc": 9.9, + "rprsUnc": 8.8, + "aRsUnc": 7.7, + "incUnc": 6.6, + } + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Mid-Transit Time (Tmid)"].endswith("+/- 0.0001 BJD_TDB") + assert final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] == "0.1234 +/- 0.001" + assert final_params["Orbital Inclination (inc)"] == "88.5 +/- 0.2 " + assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.0 +/- 0.4" + assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" + + def test_final_planetary_params_can_publish_accepted_copy_to_root(tmp_path): fit = DummyFit() (tmp_path / "temp").mkdir() @@ -410,6 +438,7 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): "tmid_deviation_minutes": 3.2, "tmid_deviation_threshold_minutes": 14.4, "expected_tmid_unc_minutes": 2.88, + "rprs_deviation_fit_unc": 0.0046, "rprs_deviation_sigma": 0.8, "deviation_sigma_threshold": 5.0, "ktmf_metric": 4.63, @@ -428,7 +457,7 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): "points": 0.91, "max_points": 1.00, "score": 0.91, - "detail": "score=0.91, Tmid sigma=1.10, Rp/R* sigma=0.80", + "detail": "score=0.91, Rp/R* sigma=0.80, fit uncertainty=0.004600", }, ], "notes": ["The transit model is strongly preferred over the flat/null model."], @@ -452,9 +481,9 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): assert "Delta BIC=18.40" in output_text assert "Residual scatter around full model fit" in output_text assert "Deviation From Expected Value" in output_text - assert "Expected-value Tmid offset" in output_text - assert "3.20 minutes" in output_text - assert "Expected-value Tmid QC window" in output_text + assert "3.20 minutes" not in output_text + assert "Expected-value Tmid offset" not in output_text + assert "Expected-value Tmid QC window" not in output_text assert "KTMF" in output_text assert "KTMF contribution 1" in output_text From a51bb8ddbe003176ede394b49d9001795b90c8b1 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 23 May 2026 18:11:40 +1000 Subject: [PATCH 051/116] Bad pixels default and redundant inits removal --- exotic/exotic.py | 64 ++++++++++-------------------- exotic/exotic_gui.py | 10 ++--- exotic/inputs.py | 7 +--- inits.json | 6 +-- tests/test_exotic_proper_motion.py | 13 +----- tests/test_exotic_rprs_retry.py | 6 +++ tests/test_inputs.py | 4 +- 7 files changed, 40 insertions(+), 70 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 12632d57..0b8fc696 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -2678,6 +2678,9 @@ def refit_selected_fast_comparison_on_full_lightcurve( bounds, jd_times=jd_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + max_rprs_retries=0, + max_ars_retries=0, + max_impact_parameter_retries=0, duration_prior=duration_prior, keep_ultranest_sampler=False, fixed_parameter_errors=fixed_errors, @@ -3942,6 +3945,7 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co max_rprs_retries, partial_retry_limits['max_retries']['rprs'], ) if partial_retry_limits['active'] else max_rprs_retries, + 'requested_max_retries': max_rprs_retries, 'min_bound': RPRS_SEARCH_BOUND_MIN, 'max_bound': RPRS_SEARCH_BOUND_MAX, 'prior_mode_key': 'rprs', @@ -3960,6 +3964,7 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co max_ars_retries, partial_retry_limits['max_retries']['ars'], ) if partial_retry_limits['active'] else max_ars_retries, + 'requested_max_retries': max_ars_retries, 'min_bound': ARS_SEARCH_BOUND_MIN, 'max_bound': None, 'prior_mode_key': 'ars', @@ -3978,6 +3983,7 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co max_impact_parameter_retries, partial_retry_limits['max_retries']['b'], ) if partial_retry_limits['active'] else max_impact_parameter_retries, + 'requested_max_retries': max_impact_parameter_retries, 'min_bound': INCLINATION_SEARCH_BOUND_MIN, 'max_bound': INCLINATION_SEARCH_BOUND_MAX, 'prior_mode_key': None, @@ -4055,14 +4061,21 @@ def build_fit(local_prior, local_bounds): continue new_bounds = parameter_diagnostics.get('bounds') if parameter_diagnostics else None - if len(retry_histories[key]) >= int(max(0, config['max_retries'])): + max_retries_allowed = int(max(0, config['max_retries'])) + if len(retry_histories[key]) >= max_retries_allowed: if ( - partial_retry_limits['active'] - and parameter_diagnostics + parameter_diagnostics and parameter_diagnostics.get('clipped') and retry_notes[key] is None ): - retry_notes[key] = partial_retry_limits['note'] + requested_max_retries = int(max(0, config.get('requested_max_retries', config['max_retries']))) + if requested_max_retries <= 0: + retry_notes[key] = ( + f"Skipped; automatic {config['label']} posterior range refits are disabled " + "for this fit." + ) + elif partial_retry_limits['active']: + retry_notes[key] = partial_retry_limits['note'] continue if parameter_diagnostics and parameter_diagnostics.get('clipped') and new_bounds is not None: @@ -4671,28 +4684,6 @@ def parse_deviation_from_expected_transit_in_qc_sigma(config_value): return float(sigma_value) -def should_assess_all_comparisons_before_selecting_best(config_value): - if config_value is None: - return True - if isinstance(config_value, bool): - return config_value - if isinstance(config_value, (int, float)): - return bool(config_value) - if isinstance(config_value, str): - normalized = config_value.strip().lower() - if normalized in ('y', 'yes', 'true', '1', 'on'): - return True - if normalized in ('n', 'no', 'false', '0', 'off', ''): - return False - - log_info( - "Warning: Invalid 'assess_all_comparisons_before_selecting_best' value; " - "defaulting to assess all comparisons.", - warn=True, - ) - return True - - def should_exit_at_first_qc_pass_solution(config_value): if config_value is None: return True @@ -4935,7 +4926,7 @@ def should_use_eebls_to_initialize_tmid_and_bounds(config_value): def should_detect_bad_pixels_before_photometry(config_value): if config_value is None: - return True + return False if isinstance(config_value, bool): return config_value if isinstance(config_value, (int, float)): @@ -4948,10 +4939,10 @@ def should_detect_bad_pixels_before_photometry(config_value): return False log_info( - "Warning: Invalid 'detect_bad_pixels_before_photometry' value; keeping bad-pixel precheck enabled.", + "Warning: Invalid 'detect_bad_pixels_before_photometry' value; keeping bad-pixel precheck disabled.", warn=True, ) - return True + return False def get_multiprocess_bad_pixel_precheck_processes(config_value): @@ -12425,7 +12416,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction(info_dict.get('bad_wcs_threshold_percent')) pointing_rejection_sigma = get_pointing_rejection_sigma(info_dict.get('pointing_rejection_sigma')) detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( - info_dict.get('detect_bad_pixels_before_photometry', 'y') + info_dict.get('detect_bad_pixels_before_photometry', 'n') ) multiprocess_bad_pixel_precheck = get_multiprocess_bad_pixel_precheck_processes( info_dict.get('multiprocess_bad_pixel_precheck', 'n') @@ -15645,7 +15636,6 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p use_impactparameter_rather_than_inclination_to_fit=True, use_eebls_to_initialize_tmid_and_bounds=True, pick_comparison_by_eebls_snr=True, - assess_all_comparisons_before_selecting_best=True, exit_at_first_qc_pass_solution=True, final_fit_baseline_duration_multiplier= FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, @@ -16589,7 +16579,7 @@ def _main_impl(): exotic_infoDict.get('pointing_rejection_sigma') ) detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( - exotic_infoDict.get('detect_bad_pixels_before_photometry', 'y') + exotic_infoDict.get('detect_bad_pixels_before_photometry', 'n') ) multiprocess_bad_pixel_precheck = get_multiprocess_bad_pixel_precheck_processes( exotic_infoDict.get('multiprocess_bad_pixel_precheck', 'n') @@ -16827,9 +16817,6 @@ def _main_impl(): deviation_from_expected_transit_in_qc_sigma = parse_deviation_from_expected_transit_in_qc_sigma( exotic_infoDict.get('deviation_from_expected_transit_in_qc_sigma', 5.0) ) - assess_all_comparisons_before_selecting_best = should_assess_all_comparisons_before_selecting_best( - exotic_infoDict.get('assess_all_comparisons_before_selecting_best', 'y') - ) exit_at_first_qc_pass_solution = should_exit_at_first_qc_pass_solution( exotic_infoDict.get('exit_at_first_qc_pass_solution', 'y') ) @@ -16861,12 +16848,6 @@ def _main_impl(): "EXOTIC will run comparison-star calibration followed by full candidate reductions.", warn=True, ) - if not assess_all_comparisons_before_selecting_best: - log_info( - "Warning: 'assess_all_comparisons_before_selecting_best' is now ignored; " - "comparison-star target-fit search is controlled by 'exit_at_first_qc_pass_solution'.", - warn=True, - ) if not exit_at_first_qc_pass_solution: log_info( "Comparison-star candidate search will evaluate all ranked candidates before selection " @@ -17476,7 +17457,6 @@ def _main_impl(): use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, - assess_all_comparisons_before_selecting_best=assess_all_comparisons_before_selecting_best, exit_at_first_qc_pass_solution=exit_at_first_qc_pass_solution, final_fit_baseline_duration_multiplier=final_fit_baseline_duration_multiplier, use_adaptive_apertures=use_adaptive_apertures, diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 6703de9f..70401985 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -418,7 +418,7 @@ def save_input(): "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", - "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", @@ -448,7 +448,7 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, - "detect_bad_pixels_before_photometry": "y", + "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, @@ -1504,7 +1504,7 @@ def save_input(): "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", - "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", @@ -1582,7 +1582,7 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, - "detect_bad_pixels_before_photometry": "y", + "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, @@ -1639,7 +1639,7 @@ def save_input(): "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "disable vertical flux normalization": False, - "detect_bad_pixels_before_photometry": "y", + "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, "final_fit_baseline_duration_multiplier": 1.0, diff --git a/exotic/inputs.py b/exotic/inputs.py index 064398c2..62b5832e 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -214,9 +214,8 @@ def __init__(self, init_opt): 'pick_comparison_by_eebls_snr': 'y', 'use_deviation_from_expected_transit_in_qc': True, 'deviation_from_expected_transit_in_qc_sigma': 5.0, - 'assess_all_comparisons_before_selecting_best': 'y', 'exit_at_first_qc_pass_solution': 'y', - 'detect_bad_pixels_before_photometry': 'y', + 'detect_bad_pixels_before_photometry': 'n', 'multiprocess_bad_pixel_precheck': 'n', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', @@ -471,10 +470,6 @@ def comp_params(self, init_file, planet_dict): 'deviation_from_expected_transit_in_qc_sigma', 'Deviation From Expected Transit In QC Sigma', ), - 'assess_all_comparisons_before_selecting_best': ( - 'assess_all_comparisons_before_selecting_best', - 'Assess All Comparisons Before Selecting Best? (y/n)', - ), 'exit_at_first_qc_pass_solution': ( 'exit_at_first_qc_pass_solution', 'exit at first QC PASS solution', diff --git a/inits.json b/inits.json index 93b2d956..c97bd758 100644 --- a/inits.json +++ b/inits.json @@ -27,7 +27,7 @@ "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Set to 0 to disable. Default 4.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", - "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default y.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", @@ -35,7 +35,6 @@ "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to use EEBLS depth SNR as an earlier tie-break when KTMF scores do not settle the comparison-star choice. Default y.", "Expected-Value Transit QC": "Set optional_info 'use_deviation_from_expected_transit_in_qc' to true to reject transit fits whose fitted Rp/R* strays too far from the published expected value using the fitted Rp/R* uncertainty only. Default true.", "Expected-Value Transit QC Sigma": "Set optional_info 'deviation_from_expected_transit_in_qc_sigma' to the sigma threshold used by the expected-value transit QC rejection. Default 5.", - "Assess All Comparisons Before Selecting Best": "Set optional_info 'assess_all_comparisons_before_selecting_best' to y to fit every comparison-star candidate that survives the earlier screening, report all fits, and select the candidate with the highest KTMF score. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "UltraNest Live Points": "Set optional_info 'minimum number of live points for ultranest' to a positive integer to control UltraNest's min_num_live_points. Default 200.", "Fast UltraNest Before Final Run": "Set optional_info 'run fast ultranest before final run' to y to run comparison-candidate UltraNest searches on a binned light curve of at most 20 points when more than 60 points are available, then rerun the selected final fit on the full light curve. Default y.", @@ -116,7 +115,7 @@ "bad_wcs_threshold_percent": 3.0, "pointing_rejection_sigma": 4.0, "disable vertical flux normalization": false, - "detect_bad_pixels_before_photometry": "y", + "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "y", "detrend_on_outoftransit_baseline": true, "final_fit_baseline_duration_multiplier": 1.0, @@ -124,7 +123,6 @@ "pick_comparison_by_eebls_snr": "y", "use_deviation_from_expected_transit_in_qc": true, "deviation_from_expected_transit_in_qc_sigma": 5.0, - "assess_all_comparisons_before_selecting_best": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "minimum number of live points for ultranest": 200, "run fast ultranest before final run": "y", diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 6e01d322..87c550b9 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -162,7 +162,6 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: should_stop_after_promising_partial_comparison_attempt, should_use_psf_photometry, should_skip_low_comparison_coverage_rejection, - should_assess_all_comparisons_before_selecting_best, should_use_fast_target_centroid, should_use_deviation_from_expected_transit_in_qc, update_coordinates_with_proper_motion, @@ -789,7 +788,7 @@ def test_should_fit_lightcurve_to_every_comparison_candidate_parses_values(): def test_should_detect_bad_pixels_before_photometry_parses_values(): - assert should_detect_bad_pixels_before_photometry(None) is True + assert should_detect_bad_pixels_before_photometry(None) is False assert should_detect_bad_pixels_before_photometry("y") is True assert should_detect_bad_pixels_before_photometry("n") is False @@ -857,13 +856,6 @@ def test_parse_deviation_from_expected_transit_in_qc_sigma_parses_values(): assert parse_deviation_from_expected_transit_in_qc_sigma(-1) == pytest.approx(5.0) -def test_should_assess_all_comparisons_before_selecting_best_parses_values(): - assert should_assess_all_comparisons_before_selecting_best(None) is True - assert should_assess_all_comparisons_before_selecting_best("y") is True - assert should_assess_all_comparisons_before_selecting_best("n") is False - assert should_assess_all_comparisons_before_selecting_best(True) is True - - def test_should_exit_at_first_qc_pass_solution_parses_values(): assert should_exit_at_first_qc_pass_solution(None) is True assert should_exit_at_first_qc_pass_solution("y") is True @@ -4011,7 +4003,7 @@ def fake_finalize( assert result["attempts"][0]["fit_point_count"] == 4 -def test_fit_ranked_comparison_calibration_candidates_evaluates_all_candidates_even_when_flag_disabled( +def test_fit_ranked_comparison_calibration_candidates_saves_outputs_for_completed_candidates( monkeypatch, tmp_path ): def fake_diagnostics(*args, **kwargs): @@ -4094,7 +4086,6 @@ def fake_save(save_dir, provisional_fit, final_fit, p_dict, observation_date, co psf_data={}, aper_data=aper_data, target_psf_flux=np.full(6, 100.0, dtype=float), - assess_all_comparisons_before_selecting_best=False, save_dir=tmp_path, planet_name="HAT-P-32 b", observation_date="2026-04-28", diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index deb981f6..df147a5e 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -429,6 +429,9 @@ def fake_run_nested( captured["fixed_parameter_errors"] = dict(kwargs.get("fixed_parameter_errors", {})) captured["fixed_flux_baseline"] = kwargs.get("fixed_flux_baseline") captured["ultranest_min_num_live_points"] = kwargs.get("ultranest_min_num_live_points") + captured["max_rprs_retries"] = kwargs.get("max_rprs_retries") + captured["max_ars_retries"] = kwargs.get("max_ars_retries") + captured["max_impact_parameter_retries"] = kwargs.get("max_impact_parameter_retries") fit = types.SimpleNamespace( time=np.asarray(times, dtype=float), data=np.asarray(flux_values, dtype=float), @@ -505,6 +508,9 @@ def fake_run_nested( assert captured["point_count"] == 80 assert captured["fixed_flux_baseline"] is True assert captured["ultranest_min_num_live_points"] == 1200 + assert captured["max_rprs_retries"] == 0 + assert captured["max_ars_retries"] == 0 + assert captured["max_impact_parameter_retries"] == 0 assert captured["prior"]["a0"] == pytest.approx(1.03) assert captured["prior"]["a2"] == pytest.approx(0.12) assert captured["fixed_parameter_errors"]["a0"] == pytest.approx(0.02) diff --git a/tests/test_inputs.py b/tests/test_inputs.py index d9e81aac..57bf5a36 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -218,7 +218,7 @@ def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_p assert inputs.info_dict["disable_vertical_flux_normalization"] is False -def test_comp_params_defaults_detect_bad_pixels_before_photometry_to_yes(tmp_path): +def test_comp_params_defaults_detect_bad_pixels_before_photometry_to_no(tmp_path): init_data = { "user_info": {}, "optional_info": {}, @@ -230,7 +230,7 @@ def test_comp_params_defaults_detect_bad_pixels_before_photometry_to_yes(tmp_pat inputs = Inputs(init_opt="y") inputs.comp_params(init_file, {}) - assert inputs.info_dict["detect_bad_pixels_before_photometry"] == "y" + assert inputs.info_dict["detect_bad_pixels_before_photometry"] == "n" def test_comp_params_defaults_multiprocess_bad_pixel_precheck_to_no(tmp_path): From a89c40043530d3a13608a8c828ea6448e25eff8f Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 25 May 2026 16:33:24 +1000 Subject: [PATCH 052/116] better outlier rejection --- exotic/exotic.py | 475 +++++++++++++++++++++++++++-- exotic/output_files.py | 5 +- exotic/plots.py | 6 + tests/test_exotic_proper_motion.py | 183 ++++++++++- 4 files changed, 641 insertions(+), 28 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 0b8fc696..cd2364cb 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -223,6 +223,7 @@ COMPARISON_IMAGE_OUTLIER_MIN_ACTIVE_STARS = 3 COMPARISON_IMAGE_OUTLIER_MIN_VALID_PAIRS = 2 COMPARISON_IMAGE_OUTLIER_MIN_SCATTER = 1e-4 +COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS = 2 OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT = 200 @@ -511,19 +512,28 @@ def annotate_selected_photometry_debug( comp_flux, raw_ratio, initial_sigma_keep_mask, + prefit_raw_ratio_keep_mask=None, phase_clip_keep_mask_on_sigma_filtered=None, ): if fit is None: return sigma_keep_mask = np.asarray(initial_sigma_keep_mask, dtype=bool) - sigma_kept_count = int(np.count_nonzero(sigma_keep_mask)) + if prefit_raw_ratio_keep_mask is None: + raw_ratio_keep_mask = np.ones(sigma_keep_mask.shape, dtype=bool) + else: + raw_ratio_keep_mask = np.asarray(prefit_raw_ratio_keep_mask, dtype=bool) + if raw_ratio_keep_mask.shape != sigma_keep_mask.shape: + raw_ratio_keep_mask = np.ones(sigma_keep_mask.shape, dtype=bool) + + prefit_keep_mask = sigma_keep_mask & raw_ratio_keep_mask + prefit_kept_count = int(np.count_nonzero(prefit_keep_mask)) if phase_clip_keep_mask_on_sigma_filtered is None: - phase_keep_mask = np.ones(sigma_kept_count, dtype=bool) + phase_keep_mask = np.ones(prefit_kept_count, dtype=bool) else: phase_keep_mask = np.asarray(phase_clip_keep_mask_on_sigma_filtered, dtype=bool) - if phase_keep_mask.shape[0] != sigma_kept_count: - phase_keep_mask = np.ones(sigma_kept_count, dtype=bool) + if phase_keep_mask.shape[0] != prefit_kept_count: + phase_keep_mask = np.ones(prefit_kept_count, dtype=bool) fit.selected_photometry_debug = { 'times': np.asarray(times, dtype=float).copy(), @@ -531,6 +541,7 @@ def annotate_selected_photometry_debug( 'comp_flux': np.asarray(comp_flux, dtype=float).copy(), 'raw_ratio': np.asarray(raw_ratio, dtype=float).copy(), 'initial_sigma_keep_mask': sigma_keep_mask.copy(), + 'prefit_raw_ratio_keep_mask': raw_ratio_keep_mask.copy(), 'phase_clip_keep_mask_on_sigma_filtered': phase_keep_mask.copy(), } @@ -1829,7 +1840,8 @@ def build_comparison_candidate_transit_prior(p_dict, ld): def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_flux, airmass, - jd_times=None, adaptive_summary=None): + jd_times=None, adaptive_summary=None, + expected_transit_depth=None): result = { 'applied': False, 'failure_reason': "the raw comparison-candidate photometry did not yield a usable light curve.", @@ -1839,6 +1851,7 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ 'debug_comp_flux': np.array([], dtype=float), 'debug_raw_ratio': np.array([], dtype=float), 'initial_sigma_keep_mask': np.array([], dtype=bool), + 'prefit_raw_ratio_keep_mask': np.array([], dtype=bool), 'time': np.array([], dtype=float), 'flux': np.array([], dtype=float), 'unc': np.array([], dtype=float), @@ -1855,9 +1868,17 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ comp_flux, airmass, jd_times=jd_times, + expected_transit_depth=expected_transit_depth, ) result['filter_diagnostics'] = prepared.get('filter_diagnostics', []) - for key in ('debug_times', 'debug_target_flux', 'debug_comp_flux', 'debug_raw_ratio', 'initial_sigma_keep_mask'): + for key in ( + 'debug_times', + 'debug_target_flux', + 'debug_comp_flux', + 'debug_raw_ratio', + 'initial_sigma_keep_mask', + 'prefit_raw_ratio_keep_mask', + ): if key in prepared: result[key] = prepared[key] if not prepared.get('applied'): @@ -2066,6 +2087,7 @@ def build_comparison_candidate_preflight(times, jd_times, airmass, ld, p_dict, t airmass, jd_times=jd_times, adaptive_summary=adaptive_summary, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) try: prior = build_comparison_candidate_transit_prior(p_dict, ld) @@ -2265,6 +2287,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, airmass, jd_times=jd_times, adaptive_summary=adaptive_summary, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) else: prepared = precomputed_candidate_series @@ -2470,6 +2493,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, prepared['debug_comp_flux'], prepared['debug_raw_ratio'], prepared['initial_sigma_keep_mask'], + prefit_raw_ratio_keep_mask=prepared.get('prefit_raw_ratio_keep_mask'), phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, ) @@ -2991,8 +3015,15 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat comp_flux = np.asarray(debug.get('comp_flux'), dtype=float) raw_ratio = np.asarray(debug.get('raw_ratio'), dtype=float) initial_sigma_keep_mask = np.asarray(debug.get('initial_sigma_keep_mask'), dtype=bool) + prefit_raw_ratio_keep_mask = np.asarray( + debug.get('prefit_raw_ratio_keep_mask', np.ones(initial_sigma_keep_mask.shape)), + dtype=bool, + ) phase_clip_keep_mask = np.asarray( - debug.get('phase_clip_keep_mask_on_sigma_filtered', np.ones(np.count_nonzero(initial_sigma_keep_mask))), + debug.get( + 'phase_clip_keep_mask_on_sigma_filtered', + np.ones(np.count_nonzero(initial_sigma_keep_mask & prefit_raw_ratio_keep_mask)), + ), dtype=bool, ) @@ -3000,13 +3031,15 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat times.shape == target_flux.shape == comp_flux.shape == raw_ratio.shape == initial_sigma_keep_mask.shape ): return None + if prefit_raw_ratio_keep_mask.shape != initial_sigma_keep_mask.shape: + prefit_raw_ratio_keep_mask = np.ones(initial_sigma_keep_mask.shape, dtype=bool) phase_keep_full = np.zeros(times.shape[0], dtype=bool) - sigma_kept_indices = np.flatnonzero(initial_sigma_keep_mask) - if sigma_kept_indices.size: - if phase_clip_keep_mask.shape[0] != sigma_kept_indices.size: - phase_clip_keep_mask = np.ones(sigma_kept_indices.size, dtype=bool) - phase_keep_full[sigma_kept_indices] = phase_clip_keep_mask + prefit_kept_indices = np.flatnonzero(initial_sigma_keep_mask & prefit_raw_ratio_keep_mask) + if prefit_kept_indices.size: + if phase_clip_keep_mask.shape[0] != prefit_kept_indices.size: + phase_clip_keep_mask = np.ones(prefit_kept_indices.size, dtype=bool) + phase_keep_full[prefit_kept_indices] = phase_clip_keep_mask output_dir = Path(save_dir) / "temp" output_dir.mkdir(parents=True, exist_ok=True) @@ -3024,6 +3057,7 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat comp_flux, raw_ratio, initial_sigma_keep_mask.astype(int), + prefit_raw_ratio_keep_mask.astype(int), phase_keep_full.astype(int), ] ) @@ -3033,10 +3067,11 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat delimiter=",", header=( "BJD_TDB,Target Flux,Comp Flux,Raw Ratio," - "Kept After Initial Sigma Clip,Kept After Phase Residual Clip" + "Kept After Initial Sigma Clip,Kept After Pre-Fit Raw Ratio Clip," + "Kept After Phase Residual Clip" ), comments="", - fmt=["%.8f", "%.8f", "%.8f", "%.8f", "%d", "%d"], + fmt=["%.8f", "%.8f", "%.8f", "%.8f", "%d", "%d", "%d"], ) return output_path @@ -7962,6 +7997,120 @@ def robust_scatter(data): return np.nan +def expected_transit_depth_from_planet_dict(p_dict): + if not isinstance(p_dict, dict): + return np.nan + + try: + rprs = float(p_dict.get('rprs', np.nan)) + except (TypeError, ValueError): + return np.nan + + if not np.isfinite(rprs) or rprs < 0: + return np.nan + return float(rprs ** 2) + + +def prefit_raw_ratio_outlier_mask( + values, + times=None, + sigma=4.0, + window=11, + min_points=5, + max_iters=2, + expected_transit_depth=None, + min_fractional_deviation=0.05, + use_global=True, +): + values = np.asarray(values, dtype=float).reshape(-1) + outlier_mask = ~np.isfinite(values) | (values <= 0) + valid_indices = np.flatnonzero(~outlier_mask) + if valid_indices.size < max(int(min_points), 3): + return outlier_mask + + if times is not None: + times = np.asarray(times, dtype=float).reshape(-1) + if times.shape == values.shape: + order = np.argsort(times[valid_indices]) + valid_indices = valid_indices[order] + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = 4.0 + if not np.isfinite(sigma) or sigma <= 0: + sigma = 4.0 + + try: + expected_depth = float(expected_transit_depth) + except (TypeError, ValueError): + expected_depth = np.nan + if not np.isfinite(expected_depth) or expected_depth < 0: + expected_depth = 0.0 + + fractional_floor = max(float(min_fractional_deviation), 2.0 * expected_depth) + if not np.isfinite(fractional_floor) or fractional_floor <= 0: + fractional_floor = 0.05 + min_log_deviation = np.log1p(fractional_floor) + + window = max(int(window), 2 * int(min_points) + 1) + if window % 2 == 0: + window += 1 + half_window = window // 2 + min_points = max(int(min_points), 3) + max_iters = max(int(max_iters), 1) + + log_values = np.log(values[valid_indices]) + keep = np.ones(valid_indices.size, dtype=bool) + + for _ in range(max_iters): + newly_rejected = np.zeros(valid_indices.size, dtype=bool) + kept_positions = np.flatnonzero(keep) + if kept_positions.size < min_points: + break + + for position in kept_positions: + lower = max(0, int(position) - half_window) + upper = min(valid_indices.size, int(position) + half_window + 1) + local_positions = np.arange(lower, upper) + local_positions = local_positions[(local_positions != position) & keep[local_positions]] + + if local_positions.size < min_points: + local_positions = kept_positions[kept_positions != position] + if local_positions.size < min_points: + continue + + local_values = log_values[local_positions] + center = bn.nanmedian(local_values) + scatter = robust_scatter(local_values - center) + if not np.isfinite(scatter) or scatter <= 0: + continue + + deviation = abs(log_values[position] - center) + if deviation > sigma * scatter and deviation > min_log_deviation: + newly_rejected[position] = True + + if use_global: + kept_values = log_values[kept_positions] + global_center = bn.nanmedian(kept_values) + global_scatter = robust_scatter(kept_values - global_center) + if np.isfinite(global_scatter) and global_scatter > 0: + global_deviation = np.abs(log_values - global_center) + global_outliers = ( + keep + & (global_deviation > sigma * global_scatter) + & (global_deviation > min_log_deviation) + ) + newly_rejected |= global_outliers + + if not np.any(newly_rejected): + break + keep[newly_rejected] = False + + outlier_mask[valid_indices] = ~keep + return outlier_mask + + def phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max_iters=3): values = np.asarray(values, dtype=float) phase = np.asarray(phase, dtype=float) @@ -12794,6 +12943,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, cFlux, airmass, jd_times=jd_times, + expected_transit_depth=expected_transit_depth_from_planet_dict(pDict), ) if not prepared.get('applied'): return None, None, None @@ -12804,6 +12954,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, debug_comp_flux = prepared['debug_comp_flux'] debug_raw_ratio = prepared['debug_raw_ratio'] debug_initial_sigma_keep_mask = prepared['initial_sigma_keep_mask'] + debug_prefit_raw_ratio_keep_mask = prepared['prefit_raw_ratio_keep_mask'] arrayFinalFlux = prepared['flux'] f1 = prepared['target_flux'] f2 = prepared['comp_flux'] @@ -12942,7 +13093,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, annotate_lightcurve_tmid_search(myfit, tmid_search_summary) annotate_lightcurve_eebls_diagnostic(myfit, eebls_search_summary) - debug_phase_clip_keep_mask = np.ones(np.count_nonzero(debug_initial_sigma_keep_mask), dtype=bool) + debug_prefit_keep_mask = debug_initial_sigma_keep_mask & debug_prefit_raw_ratio_keep_mask + debug_phase_clip_keep_mask = np.ones(np.count_nonzero(debug_prefit_keep_mask), dtype=bool) if final_fit_mode == 'ns' and myfit is not None: duration_prior = build_single_transit_duration_prior(pDict) pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( @@ -12998,6 +13150,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, debug_comp_flux, debug_raw_ratio, debug_initial_sigma_keep_mask, + prefit_raw_ratio_keep_mask=debug_prefit_raw_ratio_keep_mask, phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, ) annotate_transit_qc_expected_values(myfit, pDict) @@ -13006,7 +13159,14 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, return myfit, f1, f2 -def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass, enforce_relative_flux_max=True): +def diagnose_lightcurve_fit_inputs( + times, + tflux, + cflux, + airmass, + enforce_relative_flux_max=True, + expected_transit_depth=None, +): times = np.asarray(times, dtype=float) tflux = np.asarray(tflux, dtype=float) cflux = np.asarray(cflux, dtype=float) @@ -13017,6 +13177,7 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass, enforce_relativ 'has_reference_flux': False, 'relative_flux_point_count': 0, 'sigma_clip_point_count': 0, + 'prefit_raw_ratio_clip_point_count': 0, 'usable_point_count': 0, 'failed_stage': None, 'failure_reason': None, @@ -13104,6 +13265,27 @@ def diagnose_lightcurve_fit_inputs(times, tflux, cflux, airmass, enforce_relativ }) return diagnostics + sigma_kept_indices = np.flatnonzero(valid_mask) + if sigma_kept_indices.size: + raw_ratio_outlier_mask = prefit_raw_ratio_outlier_mask( + flux_ratio_sorted[sigma_kept_indices], + times=times_sorted[sigma_kept_indices], + expected_transit_depth=expected_transit_depth, + use_global=has_reference_flux, + ) + valid_mask[sigma_kept_indices] &= ~raw_ratio_outlier_mask + diagnostics['prefit_raw_ratio_clip_point_count'] = int(np.count_nonzero(valid_mask)) + if diagnostics['prefit_raw_ratio_clip_point_count'] <= 1: + diagnostics.update({ + 'failed_stage': 'prefit_raw_ratio_clip', + 'failure_reason': ( + "pre-fit raw target/reference-ratio outlier clipping left " + f"{diagnostics['prefit_raw_ratio_clip_point_count']} usable point(s); not enough data remained " + "for a lightcurve fit." + ), + }) + return diagnostics + arrayFinalFlux = flux_ratio_sorted[valid_mask] f1 = tflux_sorted[valid_mask] sigf1 = f1 ** 0.5 @@ -13158,6 +13340,7 @@ def prepare_lightcurve_fit_input_series( comp_flux, airmass, jd_times=None, + expected_transit_depth=None, ): times = np.asarray(times, dtype=float) target_flux = np.asarray(target_flux, dtype=float) @@ -13174,6 +13357,7 @@ def prepare_lightcurve_fit_input_series( 'debug_comp_flux': np.array([], dtype=float), 'debug_raw_ratio': np.array([], dtype=float), 'initial_sigma_keep_mask': np.array([], dtype=bool), + 'prefit_raw_ratio_keep_mask': np.array([], dtype=bool), 'time': np.array([], dtype=float), 'flux': np.array([], dtype=float), 'unc': np.array([], dtype=float), @@ -13254,6 +13438,39 @@ def prepare_lightcurve_fit_input_series( note="Dropped 3-sigma target/reference-ratio outliers before the first lightcurve fit.", )) + prefit_raw_ratio_keep_mask = np.asarray(initial_sigma_keep_mask, dtype=bool).copy() + sigma_kept_indices = np.flatnonzero(initial_sigma_keep_mask) + if sigma_kept_indices.size: + raw_ratio_outlier_mask = prefit_raw_ratio_outlier_mask( + flux_ratio_sorted[sigma_kept_indices], + times=times_sorted[sigma_kept_indices], + expected_transit_depth=expected_transit_depth, + use_global=has_reference_flux, + ) + raw_ratio_keep_on_sigma_kept = ~np.asarray(raw_ratio_outlier_mask, dtype=bool) + prefit_raw_ratio_keep_mask[sigma_kept_indices] = raw_ratio_keep_on_sigma_kept + filter_diagnostics.append(build_time_rejection_diagnostic( + "Pre-fit raw-ratio outlier clip", + times_sorted[sigma_kept_indices], + raw_ratio_keep_on_sigma_kept, + note=( + "Dropped local log target/reference-ratio outliers before fitting " + "baseline or airmass terms." + ), + )) + else: + filter_diagnostics.append(build_time_rejection_diagnostic( + "Pre-fit raw-ratio outlier clip", + times_sorted, + np.zeros(times_sorted.shape, dtype=bool), + note=( + "Dropped local log target/reference-ratio outliers before fitting " + "baseline or airmass terms." + ), + )) + + valid_mask = prefit_raw_ratio_keep_mask + flux = flux_ratio_sorted[valid_mask] filtered_target_flux = target_flux_sorted[valid_mask] filtered_comp_flux = comp_flux_sorted[valid_mask] @@ -13286,6 +13503,7 @@ def prepare_lightcurve_fit_input_series( 'debug_comp_flux': debug_comp_flux, 'debug_raw_ratio': debug_raw_ratio, 'initial_sigma_keep_mask': initial_sigma_keep_mask, + 'prefit_raw_ratio_keep_mask': prefit_raw_ratio_keep_mask, }) return prepared @@ -13302,6 +13520,7 @@ def prepare_lightcurve_fit_input_series( 'debug_comp_flux': debug_comp_flux, 'debug_raw_ratio': debug_raw_ratio, 'initial_sigma_keep_mask': initial_sigma_keep_mask, + 'prefit_raw_ratio_keep_mask': prefit_raw_ratio_keep_mask, 'time': fit_times[~nanmask], 'flux': normalized_flux, 'unc': normalized_unc, @@ -13359,6 +13578,7 @@ def evaluate_lightcurve_candidate(task): cflux, airmass, enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) myfit, tflux_fit, cflux_fit = fit_lightcurve( times, @@ -14169,6 +14389,7 @@ def log_comparison_candidate_evaluation_start(comp_summary, rank, ranked_count, suitability_score = comp_summary.get('aggregate_score', np.nan) suitability_text = "n/a" if not np.isfinite(suitability_score) else f"{suitability_score * 100.0:.4f}%" usable_point_count = 0 if fit_diagnostics is None else fit_diagnostics.get('usable_point_count', 0) + ensemble_frame_rejected_count = int(comp_summary.get('ensemble_frame_rejected_count', 0) or 0) log_info( f"\nStarting comparison-star target-fit evaluation for {label} ({position_text}) " @@ -14178,6 +14399,11 @@ def log_comparison_candidate_evaluation_start(comp_summary, rank, ranked_count, f" Candidate inputs: suitability={suitability_text}, coverage={coverage_text}, " f"usable_after_filters={usable_point_count}." ) + if ensemble_frame_rejected_count > 0: + log_info( + " Candidate ensemble clipping rejects " + f"{ensemble_frame_rejected_count} comparison-unstable frame(s) before target fitting." + ) log_info(" Preparing comparison-candidate light curve for the full reduction.") @@ -14778,6 +15004,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p comp_flux_series[fit_mask], airmass[fit_mask], enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) if not coverage_rejected and coverage_count > 1 and fit_diagnostics['failure_reason'] is None: fit_result, target_fit_flux, comp_fit_flux = fit_lightcurve( @@ -15082,6 +15309,160 @@ def comparison_star_image_outlier_summary( return summary +def comparison_pairwise_log_ratio_outlier_flags( + ratio, + sigma=COMPARISON_IMAGE_OUTLIER_SIGMA, + min_points=LIGHTCURVE_MIN_VALID_POINTS, + scatter_floor=COMPARISON_IMAGE_OUTLIER_MIN_SCATTER, +): + ratio = np.asarray(ratio, dtype=float).reshape(-1) + valid_mask = np.isfinite(ratio) & (ratio > 0) + outlier_mask = np.zeros(ratio.shape, dtype=bool) + direction = np.zeros(ratio.shape, dtype=int) + + if np.count_nonzero(valid_mask) < max(int(min_points), 3): + return valid_mask, outlier_mask, direction + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = COMPARISON_IMAGE_OUTLIER_SIGMA + if not np.isfinite(sigma) or sigma <= 0: + sigma = COMPARISON_IMAGE_OUTLIER_SIGMA + + valid_indices = np.flatnonzero(valid_mask) + log_ratio = np.log(ratio[valid_indices]) + center, _ = sigma_clipped_nanmedian(log_ratio, sigma=4.0, max_iters=3) + if not np.isfinite(center): + center = bn.nanmedian(log_ratio) + if not np.isfinite(center): + return valid_mask, outlier_mask, direction + + residuals = log_ratio - center + finite_residuals = residuals[np.isfinite(residuals)] + residual_center = bn.nanmedian(finite_residuals) if finite_residuals.size else np.nan + mad = bn.nanmedian(np.abs(finite_residuals - residual_center)) if finite_residuals.size else np.nan + scatter = 1.4826 * mad if np.isfinite(mad) and mad > 0 else np.nan + if np.isfinite(scatter_floor) and scatter_floor > 0: + if not np.isfinite(scatter) or scatter <= 0: + scatter = float(scatter_floor) + else: + scatter = max(float(scatter), float(scatter_floor)) + if not np.isfinite(scatter) or scatter <= 0: + return valid_mask, outlier_mask, direction + + pair_outliers = np.abs(residuals) > sigma * scatter + outlier_indices = valid_indices[pair_outliers] + outlier_mask[outlier_indices] = True + direction[outlier_indices] = np.sign(residuals[pair_outliers]).astype(int) + return valid_mask, outlier_mask, direction + + +def comparison_star_candidate_frame_outlier_summary( + normalized_flux_map, + candidate_key, + active_keys, + field_image_keep_mask=None, + sigma=COMPARISON_IMAGE_OUTLIER_SIGMA, + min_valid_pairs=COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS, +): + active_keys = [key for key in active_keys if key in normalized_flux_map] + series_length = 0 + if candidate_key in normalized_flux_map: + candidate_flux = np.asarray(normalized_flux_map[candidate_key], dtype=float) + if candidate_flux.ndim == 1: + series_length = candidate_flux.shape[0] + else: + candidate_flux = np.asarray([], dtype=float) + else: + candidate_flux = np.asarray([], dtype=float) + + keep_mask = np.ones(series_length, dtype=bool) + summary = { + 'frame_keep_mask': keep_mask, + 'rejected_frame_indices': [], + 'rejected_frame_count': 0, + 'valid_pair_counts': np.zeros(series_length, dtype=int), + 'outlier_pair_counts': np.zeros(series_length, dtype=int), + 'positive_outlier_pair_counts': np.zeros(series_length, dtype=int), + 'negative_outlier_pair_counts': np.zeros(series_length, dtype=int), + 'available_pair_count': 0, + 'required_valid_pair_count': max(int(min_valid_pairs), 1), + 'sigma': float(sigma), + } + + if series_length == 0 or candidate_key not in active_keys: + return summary + + if field_image_keep_mask is None: + field_image_keep_mask = np.ones(series_length, dtype=bool) + else: + field_image_keep_mask = np.asarray(field_image_keep_mask, dtype=bool).reshape(-1) + if field_image_keep_mask.shape[0] != series_length: + field_image_keep_mask = np.ones(series_length, dtype=bool) + + peer_keys = [key for key in active_keys if key != candidate_key] + required_valid_pair_count = max(int(min_valid_pairs), 1) + summary['required_valid_pair_count'] = required_valid_pair_count + if len(peer_keys) < required_valid_pair_count: + return summary + + pairwise_valid_flags = [] + pairwise_outlier_flags = [] + pairwise_positive_flags = [] + pairwise_negative_flags = [] + + for peer_key in peer_keys: + peer_flux = np.asarray(normalized_flux_map.get(peer_key), dtype=float) + if peer_flux.ndim != 1 or peer_flux.shape[0] != series_length: + continue + + ratio = normalized_ratio_series(candidate_flux, peer_flux) + ratio[~field_image_keep_mask] = np.nan + valid_mask, outlier_mask, direction = comparison_pairwise_log_ratio_outlier_flags( + ratio, + sigma=sigma, + ) + valid_mask &= field_image_keep_mask + outlier_mask &= valid_mask + if np.count_nonzero(valid_mask) < LIGHTCURVE_MIN_VALID_POINTS: + continue + + pairwise_valid_flags.append(valid_mask) + pairwise_outlier_flags.append(outlier_mask) + pairwise_positive_flags.append(outlier_mask & (direction > 0)) + pairwise_negative_flags.append(outlier_mask & (direction < 0)) + + available_pair_count = len(pairwise_valid_flags) + summary['available_pair_count'] = available_pair_count + if available_pair_count < required_valid_pair_count: + return summary + + valid_pair_counts = np.sum(np.vstack(pairwise_valid_flags), axis=0).astype(int) + outlier_pair_counts = np.sum(np.vstack(pairwise_outlier_flags), axis=0).astype(int) + positive_outlier_pair_counts = np.sum(np.vstack(pairwise_positive_flags), axis=0).astype(int) + negative_outlier_pair_counts = np.sum(np.vstack(pairwise_negative_flags), axis=0).astype(int) + directional_outlier_counts = np.maximum(positive_outlier_pair_counts, negative_outlier_pair_counts) + rejected_mask = ( + field_image_keep_mask + & (valid_pair_counts >= required_valid_pair_count) + & (directional_outlier_counts >= required_valid_pair_count) + & (directional_outlier_counts > (valid_pair_counts / 2.0)) + ) + keep_mask = ~rejected_mask + + summary.update({ + 'frame_keep_mask': keep_mask, + 'rejected_frame_indices': np.flatnonzero(rejected_mask).astype(int).tolist(), + 'rejected_frame_count': int(np.count_nonzero(rejected_mask)), + 'valid_pair_counts': valid_pair_counts, + 'outlier_pair_counts': outlier_pair_counts, + 'positive_outlier_pair_counts': positive_outlier_pair_counts, + 'negative_outlier_pair_counts': negative_outlier_pair_counts, + }) + return summary + + def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_rejection=False, validity_mask_func=valid_comparison_frame_mask): if not comp_flux_map: @@ -15276,6 +15657,21 @@ def build_stability_iteration(active_keys, frame_keep_mask=None): summary['suitability_scatter'] = final_scatter summary['suitability_high_threshold'] = final_high_threshold + candidate_frame_summary = comparison_star_candidate_frame_outlier_summary( + normalized_flux_map, + summary['key'], + active_keys, + field_image_keep_mask=field_image_keep_mask, + ) + summary['ensemble_frame_keep_mask'] = candidate_frame_summary['frame_keep_mask'] + summary['ensemble_frame_rejected_indices'] = candidate_frame_summary['rejected_frame_indices'] + summary['ensemble_frame_rejected_count'] = candidate_frame_summary['rejected_frame_count'] + summary['ensemble_frame_valid_pair_counts'] = candidate_frame_summary['valid_pair_counts'] + summary['ensemble_frame_outlier_pair_counts'] = candidate_frame_summary['outlier_pair_counts'] + summary['ensemble_frame_required_valid_pairs'] = candidate_frame_summary['required_valid_pair_count'] + summary['ensemble_frame_available_pairs'] = candidate_frame_summary['available_pair_count'] + summary['ensemble_frame_sigma'] = candidate_frame_summary['sigma'] + finite_comp_scores = [ summary['aggregate_score'] for summary in comp_summaries @@ -15707,7 +16103,32 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p else: comp_flux = aper_data[ckey][:, aperture_index, annulus_index] - fit_mask = field_image_keep_mask.copy() + candidate_frame_keep_mask = np.asarray( + comp_summary.get('ensemble_frame_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if candidate_frame_keep_mask.shape != times.shape: + candidate_frame_keep_mask = np.ones(times.shape[0], dtype=bool) + candidate_frame_clip_diagnostic = None + candidate_frame_diagnostic_keep_mask = candidate_frame_keep_mask | ~field_image_keep_mask + if np.any(~candidate_frame_diagnostic_keep_mask): + required_pairs = comp_summary.get( + 'ensemble_frame_required_valid_pairs', + COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS, + ) + sigma_threshold = comp_summary.get('ensemble_frame_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + candidate_frame_clip_diagnostic = build_time_rejection_diagnostic( + "Comparison-candidate ensemble clip", + times, + candidate_frame_diagnostic_keep_mask, + note=( + "Dropped frames where this comparison star disagreed with the comparison-star ensemble " + f"before target fitting; same-direction pairwise majority exceeded {sigma_threshold:.2f} sigma " + f"(min confirming pair count={required_pairs})." + ), + ) + + fit_mask = field_image_keep_mask & candidate_frame_keep_mask if method == 'psf': fit_mask &= robust_target_reference_flux_mask(target_flux, comp_flux) @@ -15717,6 +16138,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p comp_flux[fit_mask], airmass[fit_mask], enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) preflight = build_comparison_candidate_preflight( times[fit_mask], @@ -15735,6 +16157,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'ckey': ckey, 'comp_flux': comp_flux, 'fit_mask': fit_mask, + 'candidate_frame_clip_diagnostic': candidate_frame_clip_diagnostic, 'fit_diagnostics': fit_diagnostics, 'preflight': preflight, }) @@ -15751,6 +16174,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ckey = plan['ckey'] comp_flux = plan['comp_flux'] fit_mask = plan['fit_mask'] + candidate_frame_clip_diagnostic = plan.get('candidate_frame_clip_diagnostic') fit_diagnostics = plan['fit_diagnostics'] preflight = plan.get('preflight') or {} log_comparison_candidate_evaluation_start( @@ -15808,13 +16232,19 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'failed_stage': 'transit_qc', 'failure_reason': transit_qc_failure_reason, }) + external_filter_diagnostics = [] if field_image_clip_diagnostic is not None: + external_filter_diagnostics.append(field_image_clip_diagnostic) + if candidate_frame_clip_diagnostic is not None: + external_filter_diagnostics.append(candidate_frame_clip_diagnostic) + if external_filter_diagnostics: attached_fit_ids = set() for fit_candidate in (fit_result, final_reduction.get('fit'), selection_fit): if fit_candidate is None or id(fit_candidate) in attached_fit_ids: continue attached_fit_ids.add(id(fit_candidate)) - prepend_lightcurve_filter_diagnostic(fit_candidate, field_image_clip_diagnostic) + for diagnostic in reversed(external_filter_diagnostics): + prepend_lightcurve_filter_diagnostic(fit_candidate, diagnostic) attempt = { 'rank': rank, @@ -15829,6 +16259,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'coverage_reference_count': comp_summary.get('coverage_reference_count', np.nan), 'coverage_min_required_count': comp_summary.get('coverage_min_required_count', 0), 'coverage_rejected': comp_summary.get('coverage_rejected', False), + 'ensemble_frame_rejected_count': comp_summary.get('ensemble_frame_rejected_count', 0), + 'ensemble_frame_required_valid_pairs': comp_summary.get('ensemble_frame_required_valid_pairs', 0), 'fit': selection_fit, 'provisional_fit': None, 'full_reduction_fit': final_reduction.get('fit'), @@ -17389,10 +17821,15 @@ def _main_impl(): coverage_text += " [rejected: low coverage]" if summary.get('suitability_outlier_rejected'): coverage_text += " [rejected: high suitability outlier]" + ensemble_frame_text = "" + if summary.get('ensemble_frame_rejected_count', 0) > 0: + ensemble_frame_text = ( + f", ensemble_frame_rejects={summary['ensemble_frame_rejected_count']}" + ) log_info( f" {summary['label']}{selected_label} ({position_text}): suitability={aggregate_text}, " f"ensemble={ensemble_text}, pairwise_median={pairwise_text}, " - f"valid_pairs={summary['valid_pair_count']}, {coverage_text}, " + f"valid_pairs={summary['valid_pair_count']}, {coverage_text}{ensemble_frame_text}, " f"reason={summary['selection_reason']}" ) diff --git a/exotic/output_files.py b/exotic/output_files.py index 0d69594f..419d35bc 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1371,7 +1371,8 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label handle.write(f"# Selected comparison star,{'' if best_comp_index is None else best_comp_index + 1}\n") handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,ensemble_score,pairwise_median_score," "pairwise_max_score,self_score,valid_pair_count,coverage_count,coverage_peer_median," - "coverage_min_required,coverage_rejected,suitability_outlier_rejected\n") + "coverage_min_required,coverage_rejected,suitability_outlier_rejected," + "ensemble_frame_rejected_count,ensemble_frame_required_valid_pairs\n") for summary in comp_summaries: position = summary.get('position') or [None, None] @@ -1391,6 +1392,8 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label summary.get('coverage_min_required_count'), summary.get('coverage_rejected'), summary.get('suitability_outlier_rejected'), + summary.get('ensemble_frame_rejected_count', 0), + summary.get('ensemble_frame_required_valid_pairs', 0), ] handle.write(",".join("" if value is None else str(value) for value in values) + "\n") diff --git a/exotic/plots.py b/exotic/plots.py index df3f0de5..c1c77ccc 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -389,6 +389,12 @@ def _draw_comp_star_calibration_axis(axis, times, summary, colors): if np.any(ensemble_valid): axis.plot(times[ensemble_valid], ensemble_ratio[ensemble_valid], color='black', lw=1.8, label='Ensemble') + ensemble_keep_mask = np.asarray(summary.get('ensemble_frame_keep_mask'), dtype=bool) + if ensemble_keep_mask.shape == times.shape: + rejected = ensemble_valid & ~ensemble_keep_mask + if np.any(rejected): + axis.scatter(times[rejected], ensemble_ratio[rejected], marker='x', s=42, + color='red', linewidths=1.4, label='Ensemble clip') selected_text = " selected" if summary.get('selected') else "" aggregate = summary.get('aggregate_score', np.nan) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 87c550b9..54cd6890 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -429,6 +429,7 @@ def test_save_selected_photometry_debug_series_writes_stage_masks(tmp_path): "comp_flux": np.array([5.0, 5.0, 6.0], dtype=float), "raw_ratio": np.array([2.0, 2.2, 2.0], dtype=float), "initial_sigma_keep_mask": np.array([True, False, True], dtype=bool), + "prefit_raw_ratio_keep_mask": np.array([True, True, True], dtype=bool), "phase_clip_keep_mask_on_sigma_filtered": np.array([True, False], dtype=bool), } ) @@ -439,9 +440,10 @@ def test_save_selected_photometry_debug_series_writes_stage_masks(tmp_path): assert output_path.exists() rows = np.loadtxt(output_path, delimiter=",", skiprows=1) - assert rows.shape == (3, 6) + assert rows.shape == (3, 7) assert rows[:, 4].astype(int).tolist() == [1, 0, 1] - assert rows[:, 5].astype(int).tolist() == [1, 0, 0] + assert rows[:, 5].astype(int).tolist() == [1, 1, 1] + assert rows[:, 6].astype(int).tolist() == [1, 0, 0] def test_finalize_comparison_candidate_phase_clips_before_nested_fit(monkeypatch): @@ -1804,8 +1806,36 @@ def test_comparison_star_stability_summary_rejects_shared_bad_frame(): assert summary["image_outlier_rejected_count"] == 1 assert summary["image_outlier_required_valid_pairs"] == 2 assert summary["image_outlier_available_pairs"] == 3 - assert summary["image_outlier_valid_pair_counts"][-1] == 2 - assert summary["image_outlier_outlier_pair_counts"][-1] == 2 + assert summary["image_outlier_valid_pair_counts"][-1] == 3 + assert summary["image_outlier_outlier_pair_counts"][-1] == 3 + + +def test_comparison_star_stability_summary_flags_candidate_specific_bad_frame(): + airmass = np.linspace(1.0, 1.5, 12) + comp1 = np.full(12, 100.0, dtype=float) + comp2 = np.full(12, 80.0, dtype=float) + comp3 = np.full(12, 120.0, dtype=float) + comp4 = np.full(12, 90.0, dtype=float) + comp1[7] = 60.0 + + summary = comparison_star_stability_summary( + { + "comp1": comp1, + "comp2": comp2, + "comp3": comp3, + "comp4": comp4, + }, + airmass, + ) + + assert summary["field_image_keep_mask"].all() + comp1_summary = summary["comp_summaries"][0] + comp2_summary = summary["comp_summaries"][1] + assert comp1_summary["ensemble_frame_rejected_indices"] == [7] + assert comp1_summary["ensemble_frame_rejected_count"] == 1 + assert comp1_summary["ensemble_frame_valid_pair_counts"][7] == 3 + assert comp1_summary["ensemble_frame_outlier_pair_counts"][7] == 3 + assert comp2_summary["ensemble_frame_rejected_count"] == 0 def test_cheap_lightcurve_prescore_treats_large_ratio_flag_as_noop(): @@ -2570,7 +2600,7 @@ def fake_lc_fitter( assert myfit.plot_time_range == pytest.approx(plot_time_range) -def test_fit_lightcurve_centers_vertical_flux_bound_on_raw_flux_ratio(monkeypatch): +def test_fit_lightcurve_centers_vertical_flux_bound_on_normalized_flux(monkeypatch): captured = {} def fake_lc_fitter( @@ -2620,9 +2650,9 @@ def fake_lc_fitter( myfit, _, _ = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) assert myfit is captured["fit"] - assert captured["prior"]["a0"] == pytest.approx(0.05) - assert captured["prior"]["a1"] == pytest.approx(0.05) - assert captured["bounds"]["a0"] == pytest.approx([0.0375, 0.0625]) + assert captured["prior"]["a0"] == pytest.approx(1.0) + assert captured["prior"]["a1"] == pytest.approx(1.0) + assert captured["bounds"]["a0"] == pytest.approx([0.95, 1.05]) def test_fit_lightcurve_rejects_undersampled_series(monkeypatch): @@ -2866,6 +2896,7 @@ def fake_lc_fitter( assert [diagnostic["stage"] for diagnostic in diagnostics] == [ "Target/reference ratio filter", "Initial sigma clip", + "Pre-fit raw-ratio outlier clip", "Finite/positive photometry filter", ] assert diagnostics[0]["dropped_point_count"] == 1 @@ -2873,6 +2904,7 @@ def fake_lc_fitter( assert diagnostics[1]["dropped_point_count"] == 1 assert diagnostics[1]["first_dropped_time"] == pytest.approx(11.0) assert diagnostics[2]["dropped_point_count"] == 0 + assert diagnostics[3]["dropped_point_count"] == 0 def test_evaluate_transit_detection_qc_prefers_transit_model(): @@ -4003,6 +4035,107 @@ def fake_finalize( assert result["attempts"][0]["fit_point_count"] == 4 +def test_fit_ranked_comparison_calibration_candidates_applies_candidate_ensemble_clip(monkeypatch): + observed_lengths = [] + + def fake_diagnostics(times, *args, **kwargs): + observed_lengths.append(("diagnostics", len(times))) + return {"usable_point_count": len(times), "failure_reason": None} + + def fake_preflight(*args, **kwargs): + return {"coverage_priority": 1, "prepared_series": None} + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + observed_lengths.append(("finalize", len(times))) + fit = types.SimpleNamespace( + residuals=np.full(len(times), 0.01, dtype=float), + data=np.ones(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={"status": "pass", "summary": "ok", "ktmf_metric": 4.2}, + transit_qc_status="pass", + transit_qc_summary="ok", + transit_qc_ktmf_metric=4.2, + transit_qc_delta_bic=16.0, + frame_filter_diagnostics=[], + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.build_comparison_candidate_preflight", fake_preflight) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "field_image_keep_mask": np.ones(6, dtype=bool), + "comp_summaries": [ + { + "label": "Comp 1", + "position": (10.0, 10.0), + "aggregate_score": 0.01, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "suitability_outlier_rejected": False, + "comp_index": 0, + "ensemble_frame_keep_mask": np.array([True, True, False, True, True, True], dtype=bool), + "ensemble_frame_required_valid_pairs": 2, + "ensemble_frame_sigma": 4.25, + }, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert observed_lengths == [("diagnostics", 5), ("finalize", 5)] + diagnostic = result["attempts"][0]["fit"].frame_filter_diagnostics[0] + assert diagnostic["stage"] == "Comparison-candidate ensemble clip" + assert diagnostic["dropped_point_count"] == 1 + assert result["attempts"][0]["fit_point_count"] == 5 + + def test_fit_ranked_comparison_calibration_candidates_saves_outputs_for_completed_candidates( monkeypatch, tmp_path ): @@ -5040,6 +5173,40 @@ def test_prepare_lightcurve_fit_input_series_normalizes_ratio_around_unity(): assert np.nanmedian(prepared["flux"]) == pytest.approx(1.0) +def test_prepare_lightcurve_fit_input_series_clips_prefit_raw_ratio_outliers(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.08, 21) + comp_flux = np.full(times.shape, 1000.0, dtype=float) + raw_ratio = np.ones(times.shape, dtype=float) + raw_ratio[8:13] = 0.98 + raw_ratio[15] = 1.55 + raw_ratio[16] = 0.72 + target_flux = raw_ratio * comp_flux + + prepared = prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + np.linspace(1.0, 1.4, times.shape[0]), + expected_transit_depth=0.02, + ) + + assert prepared["applied"] is True + assert prepared["initial_sigma_keep_mask"].all() + assert prepared["prefit_raw_ratio_keep_mask"].tolist()[15:17] == [False, False] + assert np.any(np.isclose(prepared["time"], times[10])) + assert not np.any(np.isclose(prepared["time"], times[15])) + assert any( + diagnostic["stage"] == "Pre-fit raw-ratio outlier clip" + and diagnostic["dropped_point_count"] == 2 + for diagnostic in prepared["filter_diagnostics"] + ) + + def test_run_target_driven_photometry_search_returns_failed_candidate_summaries(monkeypatch): monkeypatch.setattr("exotic.exotic.fit_lightcurve", lambda *args, **kwargs: (None, None, None)) From 84dc21636d371dd9019a0241ca5c91053e2782ef Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Mon, 25 May 2026 17:35:21 +1000 Subject: [PATCH 053/116] appropriate decimals for magnitudes. also 99.99999 isn't a valid magnitude --- exotic/exotic.py | 44 ++++++++++++++++++---- exotic/output_files.py | 45 +++++++++++++++-------- exotic/plots.py | 40 ++++++++++++++------ exotic/utils.py | 44 ++++++++++++++++++++++ tests/test_nextastro_variability.py | 22 +++++++++++ tests/test_output_files.py | 38 ++++++++++++++++++- tests/test_plots.py | 57 ++++++++++++++++++++++++++++- 7 files changed, 254 insertions(+), 36 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 0b8fc696..81811d49 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -180,9 +180,25 @@ plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ plot_adaptive_aperture_diagnostics try: # tools - from utils import filename_date_token, round_to_2, safe_output_filename, user_input + from utils import ( + MAX_APPARENT_MAGNITUDE, + filename_date_token, + is_usable_apparent_magnitude, + magnitude_text, + round_to_2, + safe_output_filename, + user_input, + ) except ImportError: # package import - from .utils import filename_date_token, round_to_2, safe_output_filename, user_input + from .utils import ( + MAX_APPARENT_MAGNITUDE, + filename_date_token, + is_usable_apparent_magnitude, + magnitude_text, + round_to_2, + safe_output_filename, + user_input, + ) try: # simple version from .version import __version__ except ImportError: # package import @@ -9771,12 +9787,15 @@ def row_nextastro_magnitude(row, band_candidates): for priority, (mag_column, error_column, band_label) in enumerate(band_candidates): magnitude = _finite_float(row.get(mag_column)) magnitude_error = _finite_float(row.get(error_column)) - if magnitude is None or magnitude_error is None: + if not is_usable_apparent_magnitude(magnitude) or magnitude_error is None: + continue + magnitude_error = abs(magnitude_error) + if not is_usable_apparent_magnitude(magnitude_error): continue return { 'priority': priority, 'mag': magnitude, - 'error': abs(magnitude_error), + 'error': magnitude_error, 'mag_band': band_label, 'mag_column': mag_column, 'mag_error_column': error_column, @@ -9941,9 +9960,10 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist calibration_stars[unique_label] = match existing_positions.add(tuple(comp_pos)) added_count += 1 + mag_text = magnitude_text(match['mag_band'], match['mag'], match['error']) log_info( f"NextAstro photometry calibration for comparison star #{index + 1}: " - f"{match['mag_band']}={match['mag']:.5f} +/- {match['error']:.5f}, " + f"{mag_text}, " f"RA={comp_ra:.7f}, Dec={comp_dec:.7f}, " f"catalog separation={match['separation_arcsec']:.2f} arcsec." ) @@ -9978,9 +9998,10 @@ def nextastro_prereduced_calibration_star(phot_comp_star, obs_filter): 'observed_filter': obs_filter, }) label = nextastro_calibration_label(match) + mag_text = magnitude_text(match['mag_band'], match['mag'], match['error']) log_info( "NextAstro photometry calibration for pre-reduced comparison star: " - f"{match['mag_band']}={match['mag']:.5f} +/- {match['error']:.5f}, " + f"{mag_text}, " f"RA={comp_ra:.7f}, Dec={comp_dec:.7f}, " f"catalog separation={match['separation_arcsec']:.2f} arcsec." ) @@ -12257,7 +12278,14 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ observed_filter=None): comp_mag = _finite_float(comp_star.get('mag')) comp_mag_error = _finite_float(comp_star.get('error')) - if comp_mag is None or comp_mag_error is None: + if ( + comp_mag is None + or comp_mag_error is None + or not is_usable_apparent_magnitude(comp_mag) + ): + raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") + comp_mag_error = abs(comp_mag_error) + if not is_usable_apparent_magnitude(comp_mag_error): raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") observed_filter = observed_filter or comp_star.get('observed_filter') @@ -12304,6 +12332,8 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ & np.isfinite(selected_airmass) & np.isfinite(target_mag) & np.isfinite(target_mag_error) + & (target_mag <= MAX_APPARENT_MAGNITUDE) + & (target_mag_error <= MAX_APPARENT_MAGNITUDE) ) if np.count_nonzero(valid) == 0: raise RuntimeError("No finite stellar variability magnitude points were produced.") diff --git a/exotic/output_files.py b/exotic/output_files.py index 0d69594f..f27f260a 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -5,9 +5,23 @@ import numpy as np try: - from utils import filename_date_token, round_to_2, safe_output_filename + from utils import ( + filename_date_token, + format_magnitude, + magnitude_text, + round_to_2, + rounded_magnitude_value, + safe_output_filename, + ) except ImportError: - from .utils import filename_date_token, round_to_2, safe_output_filename + from .utils import ( + filename_date_token, + format_magnitude, + magnitude_text, + round_to_2, + rounded_magnitude_value, + safe_output_filename, + ) try: from version import __version__ except ImportError: @@ -96,8 +110,6 @@ def stellar_variability_reference_summary(vsp_param): return "na" cname = vsp_param.get('cname', 'na') - cmag = finite_float(vsp_param.get('cmag')) - cmag_err = finite_float(vsp_param.get('cmag_err')) band = vsp_param.get('mag_band') or 'V' if vsp_param.get('is_aavso_vsp', True): return f"AAVSO Label: {cname}, Position: {vsp_param.get('pos')}" @@ -105,13 +117,11 @@ def stellar_variability_reference_summary(vsp_param): source = vsp_param.get('catalog_source') or 'NextAstro photometry catalog' comp_ra = format_optional_float(vsp_param.get('comp_ra')) comp_dec = format_optional_float(vsp_param.get('comp_dec')) - if np.isfinite(cmag) and np.isfinite(cmag_err): - mag_text = f"{band}={cmag:.5f} +/- {cmag_err:.5f}" - elif np.isfinite(cmag): - mag_text = f"{band}={cmag:.5f}" - else: - mag_text = f"{band}=na" - return f"{source}: RA={comp_ra}, Dec={comp_dec}, {mag_text}" + details = [f"{source}: RA={comp_ra}", f"Dec={comp_dec}"] + mag_text = magnitude_text(band, vsp_param.get('cmag'), vsp_param.get('cmag_err')) + if mag_text is not None: + details.append(mag_text) + return ", ".join(details) def aid_comparison_metadata(vsp_param): @@ -130,8 +140,8 @@ def aid_comparison_metadata(vsp_param): 'catalog_id': vsp_param.get('catalog_id'), 'catalog_match_separation_arcsec': vsp_param.get('separation_arcsec'), 'magnitude_band': vsp_param.get('mag_band'), - 'apparent_magnitude': vsp_param.get('cmag'), - 'apparent_magnitude_error': vsp_param.get('cmag_err'), + 'apparent_magnitude': rounded_magnitude_value(vsp_param.get('cmag')), + 'apparent_magnitude_error': rounded_magnitude_value(abs(finite_float(vsp_param.get('cmag_err')))), }) @@ -1212,9 +1222,14 @@ def aavso(self): f.write("#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,GROUP,CHART,NOTES\n") for vsp_p in self.vsp_params: + mag = format_magnitude(vsp_p.get('mag'), default=None) + if mag is None: + continue + mag_err = format_magnitude(abs(finite_float(vsp_p.get('mag_err')))) + cmag = format_magnitude(vsp_p.get('cmag')) chart_id = self.chart_id or vsp_p.get('chart_id') or 'na' - f.write(f"{variable_name},{round(vsp_p['time'], 5)},{round(vsp_p['mag'], 5)},{round(vsp_p['mag_err'], 5)}," - f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{round(vsp_p['cmag'], 5)},na,na," + f.write(f"{variable_name},{round(vsp_p['time'], 5)},{mag},{mag_err}," + f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{cmag},na,na," f"{round(vsp_p['airmass'], 7)},na,{chart_id},na\n") diff --git a/exotic/plots.py b/exotic/plots.py index df3f0de5..6a89f1dd 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -8,9 +8,19 @@ from pathlib import Path try: - from utils import filename_date_token, safe_output_filename + from utils import ( + filename_date_token, + is_usable_apparent_magnitude, + magnitude_text, + safe_output_filename, + ) except ImportError: - from .utils import filename_date_token, safe_output_filename + from .utils import ( + filename_date_token, + is_usable_apparent_magnitude, + magnitude_text, + safe_output_filename, + ) plt.style.use(astropy_mpl_style) @@ -537,8 +547,6 @@ def _finite_plot_float(value): def _stellar_variability_reference_label(vsp_param, comparison_label): band = vsp_param.get('mag_band') or 'V' observed_filter = vsp_param.get('observed_filter') - cmag = _finite_plot_float(vsp_param.get('cmag')) - cmag_err = _finite_plot_float(vsp_param.get('cmag_err')) comp_ra = _finite_plot_float(vsp_param.get('comp_ra')) comp_dec = _finite_plot_float(vsp_param.get('comp_dec')) @@ -555,12 +563,9 @@ def _stellar_variability_reference_label(vsp_param, comparison_label): if observed_filter not in (None, ''): detail_parts.append(f"Observed filter={observed_filter}") - if cmag is not None and cmag_err is not None: - detail_parts.append(f"{band}={cmag:.5f} +/- {cmag_err:.5f}") - elif cmag is not None: - detail_parts.append(f"{band}={cmag:.5f}") - else: - detail_parts.append(f"{band}=na") + mag_text = magnitude_text(band, vsp_param.get('cmag'), vsp_param.get('cmag_err')) + if mag_text is not None: + detail_parts.append(mag_text) label_lines = [] if comparison_parts: @@ -576,8 +581,21 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): return fig, ax = plt.subplots(figsize=(8, 5)) + plotted_points = 0 for vsp_p in vsp_params: - ax.errorbar(vsp_p['time'], vsp_p['mag'], yerr=vsp_p['mag_err'], color="tomato", fmt='.') + if not is_usable_apparent_magnitude(vsp_p.get('mag')): + continue + mag_err = _finite_plot_float(vsp_p.get('mag_err')) + if mag_err is not None: + mag_err = abs(mag_err) + if not is_usable_apparent_magnitude(mag_err): + mag_err = None + ax.errorbar(vsp_p['time'], vsp_p['mag'], yerr=mag_err, color="tomato", fmt='.') + plotted_points += 1 + + if plotted_points == 0: + plt.close(fig) + return first_param = vsp_params[0] band = first_param.get('mag_band') or 'V' diff --git a/exotic/utils.py b/exotic/utils.py index 63e0900a..daa1fed4 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -1,4 +1,5 @@ import logging +from math import isfinite import re import requests from numpy import floor, log10 @@ -22,6 +23,8 @@ *(f'LPT{i}' for i in range(1, 10)), } _WINDOWS_ILLEGAL_FILENAME_CHARS_RE = re.compile(r'[<>:"/\\|?*\x00-\x1f\x7f]') +MAX_APPARENT_MAGNITUDE = 30.0 +MAGNITUDE_DECIMAL_PLACES = 3 def sanitize_filename_component(value, fallback='output'): @@ -58,6 +61,47 @@ def safe_output_filename(prefix, *parts, extension): return f'{safe_stem}{ext}' +def parse_finite_float(value, default=None): + try: + parsed = float(value) + except (TypeError, ValueError): + return default + return parsed if isfinite(parsed) else default + + +def is_usable_apparent_magnitude(value, max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = parse_finite_float(value) + return parsed is not None and parsed <= max_magnitude + + +def format_magnitude(value, default="na", digits=MAGNITUDE_DECIMAL_PLACES, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = parse_finite_float(value) + if parsed is None or parsed > max_magnitude: + return default + return f"{parsed:.{digits}f}" + + +def rounded_magnitude_value(value, default=None, digits=MAGNITUDE_DECIMAL_PLACES, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = parse_finite_float(value) + if parsed is None or parsed > max_magnitude: + return default + return round(parsed, digits) + + +def magnitude_text(band, magnitude, magnitude_error=None): + formatted_mag = format_magnitude(magnitude, default=None) + if formatted_mag is None: + return None + + parsed_error = parse_finite_float(magnitude_error) + formatted_error = format_magnitude(abs(parsed_error), default=None) if parsed_error is not None else None + if formatted_error is None: + return f"{band}={formatted_mag}" + return f"{band}={formatted_mag} +/- {formatted_error}" + + def user_input(prompt, type_, values=None, max_tries=1000): """ Captures user_input and casts it to the expected type diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index b7b432c7..b58210af 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -239,6 +239,28 @@ def test_nextastro_photometry_catalog_match_prefers_requested_filter(): assert match['separation_arcsec'] > 0 +def test_nextastro_photometry_catalog_match_ignores_over_30_magnitudes(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.0001, + 'dec': 20.0001, + 'Vmag': 99.99, + 'err_Vmag': 99.99, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') + + assert match is None + + def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 07d9f734..4d6883ed 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -204,7 +204,41 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert metadata["comparison_dec_deg"] == pytest.approx(-20.2) assert metadata["apparent_magnitude"] == pytest.approx(12.1) assert metadata["apparent_magnitude_error"] == pytest.approx(0.03) - assert "HAT-P-32,2450000.12345" in output_text + assert "HAT-P-32,2450000.12345,12.340,0.050,V,NO,STD" in output_text + + +def test_aid_output_skips_over_30_magnitude_rows(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + } + vsp_params = [{ + "time": 2450000.12345, + "mag": 99.99, + "mag_err": 0.05, + "airmass": 1.234, + "cname": "Comp", + "cmag": 12.1, + "cmag_err": 0.03, + "pos": [493, 202], + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = (tmp_path / "AID_AAVSO_HAT-P-32_2020-01-01.txt").read_text(encoding="utf-8") + + assert "HAT-P-32,2450000.12345" not in output_text def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): @@ -303,7 +337,7 @@ def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path assert "NextAstro photometry catalog" in reference assert "RA=10.1000000" in reference assert "Dec=-20.2000000" in reference - assert "V=12.34500 +/- 0.06700" in reference + assert "V=12.345 +/- 0.067" in reference def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclination(tmp_path): diff --git a/tests/test_plots.py b/tests/test_plots.py index 530e11c9..c0e94b39 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -216,12 +216,67 @@ def spy_set_ylabel(self, label, *args, **kwargs): assert "RA=10.1000000" in titles[-1] assert "Dec=-20.2000000" in titles[-1] assert "Observed filter=CV" in titles[-1] - assert "r=12.34500 +/- 0.06700" in titles[-1] + assert "r=12.345 +/- 0.067" in titles[-1] assert "Dec=-20.2000000\nObserved filter=CV" in titles[-1] assert ylabels[-1] == "Magnitude (r)" assert (tmp_path / "temp" / "Stellar_Variability.png").exists() +def test_plot_stellar_variability_omits_invalid_reference_magnitudes(tmp_path, monkeypatch): + titles = [] + original_set_title = Axes.set_title + + def spy_set_title(self, label, *args, **kwargs): + titles.append(label) + return original_set_title(self, label, *args, **kwargs) + + monkeypatch.setattr(Axes, "set_title", spy_set_title) + + plot_stellar_variability( + [ + { + "time": 2450000.1, + "mag": 12.34, + "mag_err": 0.05, + "cmag": 99.99, + "cmag_err": 99.99, + "comp_ra": 10.1, + "comp_dec": -20.2, + "mag_band": "V", + "observed_filter": "MObs CV", + "is_aavso_vsp": False, + } + ], + str(tmp_path), + "Host Star", + "NextAstro-123", + ) + + assert "Observed filter=MObs CV" in titles[-1] + assert "99.99" not in titles[-1] + assert "V=" not in titles[-1] + + +def test_plot_stellar_variability_skips_over_30_measurements(tmp_path): + plot_stellar_variability( + [ + { + "time": 2450000.1, + "mag": 99.99, + "mag_err": 0.05, + "cmag": 12.0, + "cmag_err": 0.05, + "mag_band": "V", + } + ], + str(tmp_path), + "Host Star", + "Comp", + ) + + assert not (tmp_path / "temp" / "Stellar_Variability.png").exists() + + def test_plot_comp_star_candidate_lightcurve_fits_writes_outputs(tmp_path): class DummyCandidateFit: def __init__(self): From 0278c23acac00295a96567dae04969beac9cb1c0 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Tue, 26 May 2026 08:54:36 +1000 Subject: [PATCH 054/116] better matching, aperture corrections --- README.md | 1 + docs/README.md | 1 + exotic/exotic.py | 641 +++++++++++++++++++++++++++- exotic/inputs.py | 6 + inits.json | 2 + tests/test_exotic_proper_motion.py | 156 +++++++ tests/test_inputs.py | 30 ++ tests/test_nextastro_variability.py | 28 ++ 8 files changed, 850 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index bab0aee6..c1bf5b50 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Fast Aperture Mask (y/n)": false, "use_psf_photometry": "y", "use_aperture_photometry": "y", + "use_aperture_corrections_and_full_image_fwhm": false, "skip_low_comparison_coverage_rejection": "n", "fit_lightcurve_to_every_comparison_candidate": "n", "detrend_on_outoftransit_baseline": true, diff --git a/docs/README.md b/docs/README.md index b1dccc47..8d09fae6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -217,6 +217,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Fast Aperture Mask (y/n)": false, "use_psf_photometry": "y", "use_aperture_photometry": "y", + "use_aperture_corrections_and_full_image_fwhm": false, "detrend_on_outoftransit_baseline": true, "use_impactparameter_rather_than_inclination_to_fit": "y", "skip_low_comparison_coverage_rejection": "n", diff --git a/exotic/exotic.py b/exotic/exotic.py index ae2c455c..255f7164 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -307,7 +307,7 @@ 'umag', 'err_umag', 'g', 'dg', 'r', 'dr', 'i', 'di', 'z', 'dz', ) NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC = 30.0 -NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 30.0 +NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 2.0 BAD_PIXEL_DETECTION_FRACTION = 0.30 BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 BAD_PIXEL_PROGRESS_LOG_INTERVAL = 25 @@ -5045,6 +5045,28 @@ def is_adaptive_aperture_mode_enabled(config_value): return False +def should_use_aperture_corrections_and_full_image_fwhm(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_aperture_corrections_and_full_image_fwhm' value; " + "keeping aperture corrections and full-image FWHM estimation disabled.", + warn=True, + ) + return False + + def should_ignore_header_wcs(config_value): if config_value is None: return False @@ -9960,6 +9982,13 @@ def sky_separation_arcsec(ra_a, dec_a, ra_b, dec_b): def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + effective_max_separation_arcsec = _finite_float(max_separation_arcsec) + if effective_max_separation_arcsec is None: + effective_max_separation_arcsec = NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC + effective_max_separation_arcsec = min( + effective_max_separation_arcsec, + NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC, + ) band_candidates = nextastro_photometry_band_candidates(obs_filter) matches = [] for row in nextastro_catalog_rows(catalog_response): @@ -9971,7 +10000,7 @@ def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, if magnitude is None: continue separation = sky_separation_arcsec(ra, dec, row_ra, row_dec) - if separation > max_separation_arcsec: + if separation > effective_max_separation_arcsec: continue matches.append({ **magnitude, @@ -11498,16 +11527,28 @@ def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_c POINTING_REJECTION_MAX_ITERS = 5 # Automatic aperture-grid tuning constants (in PSF sigma units) -APERTURE_SIGMA_MIN = 1.5 -APERTURE_SIGMA_MAX = 6.0 +GAUSSIAN_SIGMA_TO_FWHM = 2.355 +APERTURE_MIN_FWHM_MULTIPLIER = 0.5 +APERTURE_MAX_FWHM_MULTIPLIER = 2.0 +APERTURE_SIGMA_MIN = APERTURE_MIN_FWHM_MULTIPLIER * GAUSSIAN_SIGMA_TO_FWHM +APERTURE_SIGMA_MAX = APERTURE_MAX_FWHM_MULTIPLIER * GAUSSIAN_SIGMA_TO_FWHM ANNULUS_SIGMA_MIN = 6.0 ANNULUS_SIGMA_MAX = 15.0 -GAUSSIAN_SIGMA_TO_FWHM = 2.355 SKY_ANNULUS_MIN_GAP_PIXELS = 2.0 SKY_ANNULUS_MIN_FWHM_MULTIPLIER = 2.0 SKY_ANNULUS_MIN_EFFECTIVE_PIXELS = 250.0 SKY_BACKGROUND_SIGMA_CLIP = 3.0 SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS = 3 +APERTURE_CORRECTION_DETECTION_SIGMA = 5.0 +APERTURE_CORRECTION_MAX_DETECTED_STARS = 500 +APERTURE_CORRECTION_PEAK_TEST_LIMIT = APERTURE_CORRECTION_MAX_DETECTED_STARS * 50 +APERTURE_CORRECTION_PEAK_BLOCK_SIZE = 512 +APERTURE_CORRECTION_PEAK_BLOCK_LIMIT = 128 +APERTURE_CORRECTION_MAX_STARS = 60 +APERTURE_CORRECTION_MIN_STARS = 3 +APERTURE_CORRECTION_MIN_SEPARATION_FWHM = 5.0 +APERTURE_CORRECTION_MIN_BORDER_PIXELS = 20.0 +APERTURE_CORRECTION_MAX_FACTOR = 10.0 APERTURE_AUTOTUNE_COARSE_APER_POINTS = 5 APERTURE_AUTOTUNE_COARSE_ANNULUS_POINTS = 4 APERTURE_AUTOTUNE_REFINED_APER_POINTS = 6 @@ -12137,6 +12178,466 @@ def resolve_sky_annulus_geometry( } +def finite_positive_or_nan(value): + try: + value = float(value) + except (TypeError, ValueError): + return np.nan + + if not np.isfinite(value) or value <= 0: + return np.nan + return value + + +def _aperture_correction_fallback_fwhm(fwhm_hint=np.nan, fallback_sigma=np.nan): + fwhm = finite_positive_or_nan(fwhm_hint) + if np.isfinite(fwhm): + return fwhm + + sigma = finite_positive_or_nan(fallback_sigma) + if np.isfinite(sigma): + return psf_fwhm_from_sigma(sigma) + + return np.nan + + +def _limited_bright_pixel_indices(search_image, bright, max_count): + bright_count = int(np.count_nonzero(bright)) + if bright_count == 0: + return np.empty(0, dtype=np.intp) + if bright_count <= max_count: + return np.flatnonzero(bright.ravel()) + + height, width = bright.shape + selected_blocks = [] + for y0 in range(0, height, APERTURE_CORRECTION_PEAK_BLOCK_SIZE): + y1 = min(y0 + APERTURE_CORRECTION_PEAK_BLOCK_SIZE, height) + for x0 in range(0, width, APERTURE_CORRECTION_PEAK_BLOCK_SIZE): + x1 = min(x0 + APERTURE_CORRECTION_PEAK_BLOCK_SIZE, width) + block_bright = bright[y0:y1, x0:x1] + block_count = int(np.count_nonzero(block_bright)) + if block_count == 0: + continue + + if block_count <= APERTURE_CORRECTION_PEAK_BLOCK_LIMIT: + local_flat = np.flatnonzero(block_bright.ravel()) + else: + block_scores = np.where(block_bright, search_image[y0:y1, x0:x1], -np.inf) + local_flat = np.argpartition( + block_scores.ravel(), + -APERTURE_CORRECTION_PEAK_BLOCK_LIMIT, + )[-APERTURE_CORRECTION_PEAK_BLOCK_LIMIT:] + local_flat = local_flat[np.isfinite(block_scores.ravel()[local_flat])] + + yy, xx = np.divmod(local_flat, x1 - x0) + selected_blocks.append((yy + y0) * width + (xx + x0)) + + if not selected_blocks: + return np.empty(0, dtype=np.intp) + + selected = np.concatenate(selected_blocks).astype(np.intp, copy=False) + if selected.size <= max_count: + return selected + + selected_values = search_image.ravel()[selected] + strongest = np.argpartition(selected_values, -max_count)[-max_count:] + return selected[strongest] + + +def detect_aperture_correction_star_candidates(data, fwhm_hint=np.nan): + image = np.asarray(data, dtype=float) + if image.ndim != 2: + return np.empty((0, 3), dtype=float) + if image.shape[0] < 3 or image.shape[1] < 3: + return np.empty((0, 3), dtype=float) + + finite = np.isfinite(image) + if not np.any(finite): + return np.empty((0, 3), dtype=float) + + background, scatter = sigma_clipped_nanmedian(image[finite], sigma=3.0, max_iters=3) + if not np.isfinite(background): + background = float(np.nanmedian(image[finite])) + if not np.isfinite(scatter) or scatter <= 0: + scatter = float(np.nanstd(image[finite])) + if not np.isfinite(scatter) or scatter <= 0: + return np.empty((0, 3), dtype=float) + + fwhm = finite_positive_or_nan(fwhm_hint) + if not np.isfinite(fwhm): + fwhm = 3.0 + fwhm = float(np.clip(fwhm, 1.0, 20.0)) + + search_image = np.where(finite, image - background, -np.inf) + threshold = APERTURE_CORRECTION_DETECTION_SIGMA * scatter + bright = finite & (search_image > threshold) + bright[0, :] = False + bright[-1, :] = False + bright[:, 0] = False + bright[:, -1] = False + + bright_flat = _limited_bright_pixel_indices( + search_image, + bright, + APERTURE_CORRECTION_PEAK_TEST_LIMIT, + ) + if bright_flat.size == 0: + return np.empty((0, 3), dtype=float) + + y, x = np.divmod(bright_flat, image.shape[1]) + flux = search_image[y, x] + order = np.argsort(flux, kind='mergesort')[::-1] + local_radius = int(np.clip(np.ceil(0.5 * fwhm), 1, 10)) + suppression_radius = max(1.0, 0.75 * fwhm) + suppression_radius_sq = suppression_radius * suppression_radius + + rows = [] + accepted_xy = [] + for idx in order: + xc = int(x[idx]) + yc = int(y[idx]) + center_flux = float(flux[idx]) + if not np.isfinite(center_flux): + continue + + y0 = yc - local_radius + y1 = yc + local_radius + 1 + x0 = xc - local_radius + x1 = xc + local_radius + 1 + if y0 < 0 or x0 < 0 or y1 > image.shape[0] or x1 > image.shape[1]: + continue + if center_flux < float(np.nanmax(search_image[y0:y1, x0:x1])): + continue + + if accepted_xy: + accepted = np.asarray(accepted_xy, dtype=float) + if np.any((accepted[:, 0] - xc) ** 2 + (accepted[:, 1] - yc) ** 2 <= suppression_radius_sq): + continue + + rows.append((float(xc), float(yc), center_flux)) + accepted_xy.append((float(xc), float(yc))) + if len(rows) >= APERTURE_CORRECTION_MAX_DETECTED_STARS: + break + + if not rows: + return np.empty((0, 3), dtype=float) + return np.asarray(rows, dtype=float) + + +def isolated_aperture_correction_candidates(candidates, image_shape, fwhm_hint=np.nan): + candidates = np.asarray(candidates, dtype=float) + if candidates.ndim != 2 or candidates.shape[1] < 2 or candidates.size == 0: + return np.empty((0, 3), dtype=float) + + fwhm = finite_positive_or_nan(fwhm_hint) + if not np.isfinite(fwhm): + fwhm = 3.0 + + height, width = image_shape[:2] + min_separation = APERTURE_CORRECTION_MIN_SEPARATION_FWHM * fwhm + border = max( + APERTURE_CORRECTION_MIN_BORDER_PIXELS, + APERTURE_CORRECTION_MIN_SEPARATION_FWHM * fwhm, + ) + + x = candidates[:, 0] + y = candidates[:, 1] + keep = ( + np.isfinite(x) + & np.isfinite(y) + & (x >= border) + & (x <= (width - 1 - border)) + & (y >= border) + & (y <= (height - 1 - border)) + ) + candidates = candidates[keep] + if candidates.shape[0] <= 1: + return candidates[:APERTURE_CORRECTION_MAX_STARS] + + x = candidates[:, 0] + y = candidates[:, 1] + distances = np.hypot(x[:, None] - x[None, :], y[:, None] - y[None, :]) + np.fill_diagonal(distances, np.inf) + nearest = np.min(distances, axis=1) + candidates = candidates[nearest >= min_separation] + if candidates.size == 0: + return np.empty((0, 3), dtype=float) + + order = np.argsort(candidates[:, 2], kind='mergesort')[::-1] + return candidates[order[:APERTURE_CORRECTION_MAX_STARS]] + + +def estimate_isolated_field_star_psfs(data, fwhm_hint=np.nan): + image = np.asarray(data, dtype=float) + if image.ndim != 2: + return np.empty((0, 7), dtype=float) + + candidates = detect_aperture_correction_star_candidates(image, fwhm_hint=fwhm_hint) + candidates = isolated_aperture_correction_candidates(candidates, image.shape, fwhm_hint=fwhm_hint) + if candidates.size == 0: + return np.empty((0, 7), dtype=float) + + fwhm = finite_positive_or_nan(fwhm_hint) + if not np.isfinite(fwhm): + fwhm = 3.0 + box = int(np.clip(np.ceil(3.0 * fwhm), 6, 30)) + + rows = [] + for x, y, _ in candidates: + try: + xv, yv = mesh_box([x, y], box, maxx=image.shape[1], maxy=image.shape[0]) + subarray = image[yv, xv] + except Exception: + continue + + if subarray.size == 0: + continue + + row = _fit_centroid_moments(subarray, xv, yv, [x, y], box) + if not np.all(np.isfinite(row[:5])): + continue + if not _has_usable_centroid_signal(subarray, row[2], min_snr=5.0): + continue + + solved_fwhm = psf_fwhm_from_sigma(0.5 * (row[3] + row[4])) + if not np.isfinite(solved_fwhm): + continue + if np.hypot(row[0] - x, row[1] - y) > max(2.0, 0.75 * solved_fwhm): + continue + + rows.append(row) + + if not rows: + return np.empty((0, 7), dtype=float) + return np.asarray(rows, dtype=float) + + +def image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=np.nan): + rows = np.asarray(field_star_psfs, dtype=float) + if rows.ndim != 2 or rows.shape[1] < 5 or rows.size == 0: + return finite_positive_or_nan(fallback_fwhm) + + sigmas = 0.5 * (rows[:, 3] + rows[:, 4]) + fwhm_values = GAUSSIAN_SIGMA_TO_FWHM * sigmas + fwhm_values[~np.isfinite(fwhm_values) | (fwhm_values <= 0)] = np.nan + center, _ = sigma_clipped_nanmedian(fwhm_values, sigma=3.0, max_iters=3) + if np.isfinite(center) and center > 0: + return float(center) + + return finite_positive_or_nan(fallback_fwhm) + + +def estimate_image_fwhm_from_isolated_stars(data, fwhm_hint=np.nan, fallback_sigma=np.nan): + fallback_fwhm = _aperture_correction_fallback_fwhm(fwhm_hint, fallback_sigma) + field_star_psfs = estimate_isolated_field_star_psfs(data, fwhm_hint=fallback_fwhm) + return image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=fallback_fwhm) + + +def _aperture_correction_sky_background(data, xc, yc, reference_radius, image_fwhm, fast_mode=False): + sigma_hint = image_fwhm / GAUSSIAN_SIGMA_TO_FWHM if np.isfinite(image_fwhm) and image_fwhm > 0 else np.nan + sky_geometry = resolve_sky_annulus_geometry( + reference_radius, + max(float(image_fwhm), 3.0) if np.isfinite(image_fwhm) else 5.0, + psf_sigma=sigma_hint, + ) + + try: + annulus = CircularAnnulus( + positions=[(xc, yc)], + r_in=sky_geometry['inner_radius'], + r_out=sky_geometry['outer_radius'], + ) + mask_method = 'center' if fast_mode else 'exact' + annulus_mask = annulus.to_mask(method=mask_method)[0] + annulus_cutout = annulus_mask.cutout(data, fill_value=np.nan) + except Exception: + return np.nan + + if annulus_cutout is None: + return np.nan + + annulus_cutout = np.asarray(annulus_cutout, dtype=float) + annulus_weights = np.asarray(annulus_mask.data, dtype=float) + valid_mask = np.isfinite(annulus_cutout) & np.isfinite(annulus_weights) & (annulus_weights > 0) + if not np.any(valid_mask): + return np.nan + + annulus_pixels = annulus_cutout[valid_mask] + annulus_pixel_weights = annulus_weights[valid_mask] + cutoff = weighted_nanpercentile(annulus_pixels, annulus_pixel_weights, 99) + if not np.isfinite(cutoff): + return np.nan + + clipped_keep = annulus_pixels <= cutoff + if not np.any(clipped_keep): + return np.nan + + sky_median, _ = sigma_clipped_weighted_median( + annulus_pixels[clipped_keep], + annulus_pixel_weights[clipped_keep], + sigma=SKY_BACKGROUND_SIGMA_CLIP, + max_iters=SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS, + high_only=True, + ) + return sky_median + + +def _background_subtracted_aperture_sum(data, xc, yc, radius, background, fast_mode=False): + radius = finite_positive_or_nan(radius) + if not np.isfinite(radius) or not np.isfinite(background): + return np.nan + + try: + aperture = CircularAperture(positions=[(xc, yc)], r=radius) + mask_method = 'center' if fast_mode else 'exact' + mask = aperture.to_mask(method=mask_method)[0] + data_cutout = mask.cutout(data) + except Exception: + return np.nan + + if data_cutout is None: + return np.nan + + weights = np.asarray(mask.data, dtype=float) + values = np.asarray(data_cutout, dtype=float) + valid = np.isfinite(weights) & np.isfinite(values) & (weights > 0) + if not np.any(valid): + return np.nan + + return float(np.sum(weights[valid] * (values[valid] - background))) + + +def build_aperture_correction_profile(data, aperture_radii, fwhm_hint=np.nan, fast_mode=False, + field_star_psfs=None): + aperture_radii = np.asarray(aperture_radii, dtype=float).reshape(-1) + correction_factors = np.ones(aperture_radii.shape, dtype=float) + fallback_fwhm = finite_positive_or_nan(fwhm_hint) + + if field_star_psfs is None: + field_star_psfs = estimate_isolated_field_star_psfs(data, fwhm_hint=fallback_fwhm) + else: + field_star_psfs = np.asarray(field_star_psfs, dtype=float) + + image_fwhm = image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=fallback_fwhm) + profile = { + 'applied': False, + 'image_fwhm': image_fwhm, + 'star_count': int(field_star_psfs.shape[0]) if field_star_psfs.ndim == 2 else 0, + 'aperture_radii': aperture_radii, + 'correction_factors': correction_factors, + 'curve_radii': np.array([], dtype=float), + 'enclosed_fraction': np.array([], dtype=float), + 'note': 'Aperture correction skipped; no aperture radii were provided.', + } + + valid_radius_mask = np.isfinite(aperture_radii) & (aperture_radii > 0) + if not np.any(valid_radius_mask): + return profile + + if not np.isfinite(image_fwhm) or image_fwhm <= 0: + profile['note'] = 'Aperture correction skipped; image FWHM could not be estimated.' + return profile + + if profile['star_count'] < APERTURE_CORRECTION_MIN_STARS: + profile['note'] = ( + "Aperture correction skipped; fewer than " + f"{APERTURE_CORRECTION_MIN_STARS} isolated field stars were available." + ) + return profile + + reference_radius = APERTURE_MAX_FWHM_MULTIPLIER * image_fwhm + measurement_radii = np.unique(np.concatenate([aperture_radii[valid_radius_mask], [reference_radius]])) + measurement_radii = measurement_radii[np.isfinite(measurement_radii) & (measurement_radii > 0)] + if measurement_radii.size == 0: + return profile + + fractions_by_radius = {float(radius): [] for radius in measurement_radii} + for row in field_star_psfs: + xc, yc = float(row[0]), float(row[1]) + background = _aperture_correction_sky_background( + data, + xc, + yc, + reference_radius, + image_fwhm, + fast_mode=fast_mode, + ) + reference_flux = _background_subtracted_aperture_sum( + data, + xc, + yc, + reference_radius, + background, + fast_mode=fast_mode, + ) + if not np.isfinite(reference_flux) or reference_flux <= 0: + continue + + for radius in measurement_radii: + flux = _background_subtracted_aperture_sum( + data, + xc, + yc, + radius, + background, + fast_mode=fast_mode, + ) + fraction = flux / reference_flux if np.isfinite(flux) else np.nan + if np.isfinite(fraction) and fraction > 0: + fractions_by_radius[float(radius)].append(float(fraction)) + + curve_radii = [] + enclosed_fraction = [] + for radius in measurement_radii: + fractions = np.asarray(fractions_by_radius[float(radius)], dtype=float) + if fractions.size == 0: + continue + center, _ = sigma_clipped_nanmedian(fractions, sigma=3.0, max_iters=3) + if np.isfinite(center) and center > 0: + curve_radii.append(float(radius)) + enclosed_fraction.append(float(center)) + + if not curve_radii: + profile['note'] = 'Aperture correction skipped; isolated-star curve of growth could not be measured.' + return profile + + curve_radii = np.asarray(curve_radii, dtype=float) + enclosed_fraction = np.asarray(enclosed_fraction, dtype=float) + order = np.argsort(curve_radii, kind='mergesort') + curve_radii = curve_radii[order] + enclosed_fraction = enclosed_fraction[order] + enclosed_fraction = np.clip(enclosed_fraction, 1.0 / APERTURE_CORRECTION_MAX_FACTOR, 1.0) + enclosed_fraction = np.maximum.accumulate(enclosed_fraction) + enclosed_fraction = np.minimum(enclosed_fraction, 1.0) + + interpolated_fraction = np.interp( + aperture_radii[valid_radius_mask], + curve_radii, + enclosed_fraction, + left=enclosed_fraction[0], + right=1.0, + ) + interpolated_fraction = np.clip(interpolated_fraction, 1.0 / APERTURE_CORRECTION_MAX_FACTOR, 1.0) + correction_factors[valid_radius_mask] = np.clip( + 1.0 / interpolated_fraction, + 1.0, + APERTURE_CORRECTION_MAX_FACTOR, + ) + + profile.update({ + 'applied': True, + 'correction_factors': correction_factors, + 'curve_radii': curve_radii, + 'enclosed_fraction': enclosed_fraction, + 'reference_radius': float(reference_radius), + 'note': ( + "Applied aperture correction from " + f"{profile['star_count']} isolated field star(s); image FWHM={image_fwhm:.2f}px." + ), + }) + return profile + + # Method calculates the flux of the star (uses the skybg_phot method to do background sub) def aperPhot(data, starIndex, xc, yc, r=5, dr=5, fast_mode=False, sigma_hint=np.nan): stage_start = perf_counter() @@ -12708,15 +13209,31 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet del centroid_reference_image # aperture and annulus scale factors in PSF sigma units - aper_sigma = 3 * max(targ_sig_xy) + aper_sigma = finite_positive_or_nan(3 * max(targ_sig_xy)) + if not np.isfinite(aper_sigma): + aper_sigma = 3.0 + aper_sigma = float(np.clip(aper_sigma, APERTURE_SIGMA_MIN, APERTURE_SIGMA_MAX)) annulus_sigma = 10 fast_aperture_mask = is_fast_aperture_mask_enabled(info_dict.get('fast_aperture_mask')) use_adaptive_apertures = is_adaptive_aperture_mode_enabled(info_dict.get('use_adaptive_apertures')) + use_aperture_corrections_and_full_image_fwhm = should_use_aperture_corrections_and_full_image_fwhm( + info_dict.get('use_aperture_corrections_and_full_image_fwhm', False) + ) aper = np.nan annulus = np.nan sigma = np.nan if use_adaptive_apertures: - log_info("Adaptive aperture scaling enabled for realtime photometry.") + log_info( + "Adaptive aperture scaling enabled for realtime photometry: " + f"aperture scales are in PSF sigma units (1 image FWHM = {GAUSSIAN_SIGMA_TO_FWHM:.3f} sigma)." + ) + log_info( + "Realtime aperture candidates are limited to " + f"{APERTURE_MIN_FWHM_MULTIPLIER:.1f}-{APERTURE_MAX_FWHM_MULTIPLIER:.1f} image FWHM " + f"({APERTURE_SIGMA_MIN:.2f}-{APERTURE_SIGMA_MAX:.2f} sigma)." + ) + if use_aperture_corrections_and_full_image_fwhm: + log_info("Aperture corrections and full-image FWHM estimation enabled for realtime photometry.") # alloc psf fitting param psf_data = { @@ -12891,7 +13408,18 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) # aperture photometry - frame_sigma = psf_sigma_from_fit(psf_data['target'][i], fallback_sigma=sigma) + target_sigma = psf_sigma_from_fit(psf_data['target'][i], fallback_sigma=sigma) + target_fwhm = psf_fwhm_from_sigma(target_sigma) + field_star_psfs = np.empty((0, 7), dtype=float) + image_fwhm = target_fwhm + if use_aperture_corrections_and_full_image_fwhm: + field_star_psfs = estimate_isolated_field_star_psfs(imageData, fwhm_hint=target_fwhm) + image_fwhm = image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=target_fwhm) + frame_sigma = ( + image_fwhm / GAUSSIAN_SIGMA_TO_FWHM + if np.isfinite(image_fwhm) and image_fwhm > 0 + else target_sigma + ) if i == 0: sigma = frame_sigma if not np.isfinite(sigma) or sigma <= 0: @@ -12919,6 +13447,19 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet aper = float(aper[0]) annulus = float(annulus[0]) + aperture_correction_factor = 1.0 + if use_aperture_corrections_and_full_image_fwhm: + aperture_correction = build_aperture_correction_profile( + imageData, + [aper], + fwhm_hint=image_fwhm, + fast_mode=fast_aperture_mask, + field_star_psfs=field_star_psfs, + ) + correction_factors = np.asarray(aperture_correction.get('correction_factors', [1.0]), dtype=float).reshape(-1) + if correction_factors.size and np.isfinite(correction_factors[0]) and correction_factors[0] > 0: + aperture_correction_factor = float(correction_factors[0]) + comp_frame_sigma = psf_sigma_from_fit(psf_data['comp'][i], fallback_sigma=frame_sigma) tFlux = aperPhot( imageData, @@ -12929,7 +13470,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet annulus, fast_mode=fast_aperture_mask, sigma_hint=frame_sigma, - )[0] + )[0] * aperture_correction_factor cFlux = aperPhot( imageData, 1, @@ -12939,7 +13480,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet annulus, fast_mode=fast_aperture_mask, sigma_hint=comp_frame_sigma, - )[0] + )[0] * aperture_correction_factor norm_flux.append(tFlux / cFlux) # close file + delete from memory @@ -15762,7 +16303,8 @@ def initialize_aperture_data_store(frame_count, aperture_count, annulus_count, c return aper_data -def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=False, sigma_hint=np.nan): +def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=False, sigma_hint=np.nan, + aperture_correction_factors=None): flux_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) bg_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) @@ -15812,12 +16354,31 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast finally: _record_photometry_stage_timing('aperPhot', perf_counter() - stage_start) + if aperture_correction_factors is not None: + factors = np.asarray(aperture_correction_factors, dtype=float).reshape(-1) + if factors.shape[0] == len(apertures): + valid_factors = np.isfinite(factors) & (factors > 0) + if np.any(valid_factors): + flux_grid[valid_factors, :] *= factors[valid_factors, None] + return flux_grid, bg_grid def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_star_count, aper_data, apertures, annuli, - fast_aperture_mask, adaptive_apertures=False, fallback_sigma=np.nan): - frame_sigma = psf_sigma_from_fit(psf_data['target'][frame_index], fallback_sigma=fallback_sigma) + fast_aperture_mask, adaptive_apertures=False, fallback_sigma=np.nan, + use_aperture_corrections_and_full_image_fwhm=False): + target_sigma = psf_sigma_from_fit(psf_data['target'][frame_index], fallback_sigma=fallback_sigma) + target_fwhm = psf_fwhm_from_sigma(target_sigma) + field_star_psfs = np.empty((0, 7), dtype=float) + image_fwhm = target_fwhm + if use_aperture_corrections_and_full_image_fwhm: + field_star_psfs = estimate_isolated_field_star_psfs(image_data, fwhm_hint=target_fwhm) + image_fwhm = image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=target_fwhm) + frame_sigma = ( + image_fwhm / GAUSSIAN_SIGMA_TO_FWHM + if np.isfinite(image_fwhm) and image_fwhm > 0 + else target_sigma + ) frame_apertures, frame_annuli = resolve_frame_aperture_radii( apertures, annuli, @@ -15825,6 +16386,23 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta frame_sigma=frame_sigma, fallback_sigma=fallback_sigma, ) + aperture_correction = { + 'applied': False, + 'image_fwhm': image_fwhm, + 'star_count': 0, + 'correction_factors': np.ones(len(frame_apertures), dtype=float), + 'note': 'Aperture corrections and full-image FWHM estimation disabled.', + } + aperture_correction_factors = None + if use_aperture_corrections_and_full_image_fwhm: + aperture_correction = build_aperture_correction_profile( + image_data, + frame_apertures, + fwhm_hint=image_fwhm, + fast_mode=fast_aperture_mask, + field_star_psfs=field_star_psfs, + ) + aperture_correction_factors = aperture_correction.get('correction_factors') target_flux, target_bg = compute_star_aperture_grid( image_data, @@ -15835,6 +16413,7 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta frame_annuli, fast_mode=fast_aperture_mask, sigma_hint=frame_sigma, + aperture_correction_factors=aperture_correction_factors, ) aper_data['target'][frame_index] = target_flux aper_data['target_bg'][frame_index] = target_bg @@ -15851,10 +16430,13 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta frame_annuli, fast_mode=fast_aperture_mask, sigma_hint=comp_sigma, + aperture_correction_factors=aperture_correction_factors, ) aper_data[ckey][frame_index] = comp_flux aper_data[f"{ckey}_bg"][frame_index] = comp_bg + return aperture_correction + def load_calibrated_reduction_image(file_name, generalDark, generalBias, generalFlat, demosaic_fmt, demosaic_out, demosaic_mult, @@ -17291,6 +17873,9 @@ def _main_impl(): use_adaptive_apertures = is_adaptive_aperture_mode_enabled( exotic_infoDict.get('use_adaptive_apertures', False) ) + use_aperture_corrections_and_full_image_fwhm = should_use_aperture_corrections_and_full_image_fwhm( + exotic_infoDict.get('use_aperture_corrections_and_full_image_fwhm', False) + ) if not use_psf_photometry and not use_aperture_photometry: log_info("Error: both PSF and aperture photometry are disabled in optional_info.", error=True) return @@ -17377,7 +17962,18 @@ def _main_impl(): ) fast_aperture_mask = is_fast_aperture_mask_enabled(exotic_infoDict.get('fast_aperture_mask')) if use_aperture_photometry and use_adaptive_apertures: - log_info("Adaptive aperture scaling enabled: evaluating aperture candidates in PSF sigma units per frame.") + log_info( + "Adaptive aperture scaling enabled: evaluating aperture candidates in PSF sigma units per frame " + f"(1 image FWHM = {GAUSSIAN_SIGMA_TO_FWHM:.3f} sigma)." + ) + if use_aperture_photometry: + log_info( + "Aperture candidates are limited to " + f"{APERTURE_MIN_FWHM_MULTIPLIER:.1f}-{APERTURE_MAX_FWHM_MULTIPLIER:.1f} image FWHM " + f"({APERTURE_SIGMA_MIN:.2f}-{APERTURE_SIGMA_MAX:.2f} sigma)." + ) + if use_aperture_corrections_and_full_image_fwhm: + log_info("Aperture corrections and full-image FWHM estimation enabled per optional_info setting.") # open files, calibrate, align, photometry reset_transform_timing_stats() @@ -17580,6 +18176,14 @@ def _main_impl(): # aperture photometry if use_aperture_photometry and i == 0: sigma = psf_sigma_from_fit(psf_data['target'][0]) + if use_aperture_corrections_and_full_image_fwhm: + image_fwhm = estimate_image_fwhm_from_isolated_stars( + imageData, + fwhm_hint=psf_fwhm_from_sigma(sigma), + fallback_sigma=sigma, + ) + if np.isfinite(image_fwhm) and image_fwhm > 0: + sigma = image_fwhm / GAUSSIAN_SIGMA_TO_FWHM if not np.isfinite(sigma) or sigma <= 0: log_info("Warning: Initial PSF sigma is invalid; using sigma=1.0 for automatic aperture tuning.", warn=True) sigma = 1.0 @@ -17603,6 +18207,7 @@ def _main_impl(): fast_aperture_mask, adaptive_apertures=use_adaptive_apertures, fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=use_aperture_corrections_and_full_image_fwhm, ) if i == coarse_tune_frames - 1: @@ -17631,9 +18236,11 @@ def _main_impl(): if best_coarse_candidate['comp_index'] is not None: best_comp_label = str(best_coarse_candidate['comp_index'] + 1) score_text = "n/a" if not np.isfinite(best_coarse_score) else f"{best_coarse_score:.5f}" + best_aper_sigma = best_coarse_candidate['aper_sigma'] + best_aper_fwhm = best_aper_sigma / GAUSSIAN_SIGMA_TO_FWHM log_info( "Auto-tuned aperture grid: " - f"coarse_best=(aper={best_coarse_candidate['aper_sigma']:.2f} sigma, " + f"coarse_best=(aper={best_aper_sigma:.2f} sigma/{best_aper_fwhm:.2f} FWHM, " f"annulus={best_coarse_candidate['annulus_sigma']:.2f} sigma, comp={best_comp_label}, score={score_text}), " f"refined_grid={len(refined_apertures_sigma)}x{len(refined_annuli_sigma)}." ) @@ -17666,6 +18273,9 @@ def _main_impl(): fast_aperture_mask, adaptive_apertures=use_adaptive_apertures, fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=( + use_aperture_corrections_and_full_image_fwhm + ), ) finally: if loaded_from_disk: @@ -17696,6 +18306,7 @@ def _main_impl(): fast_aperture_mask, adaptive_apertures=use_adaptive_apertures, fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=use_aperture_corrections_and_full_image_fwhm, ) # close file + delete from memory diff --git a/exotic/inputs.py b/exotic/inputs.py index 62b5832e..ff320440 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -220,6 +220,7 @@ def __init__(self, init_opt): 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, + 'use_aperture_corrections_and_full_image_fwhm': False, 'pointing_rejection_sigma': 4.0, 'skip_low_comparison_coverage_rejection': 'n', 'fit_lightcurve_to_every_comparison_candidate': 'n', @@ -495,6 +496,11 @@ def comp_params(self, init_file, planet_dict): 'Use Adaptive Apertures? (y/n)', 'Use Adaptive Apertures (y/n)', ), + 'use_aperture_corrections_and_full_image_fwhm': ( + 'use_aperture_corrections_and_full_image_fwhm', + 'Use Aperture Corrections and Full Image FWHM? (y/n)', + 'Use Aperture Corrections And Full Image FWHM? (y/n)', + ), 'skip_low_comparison_coverage_rejection': ( 'skip_low_comparison_coverage_rejection', 'Skip Low Comparison Coverage Rejection? (y/n)', diff --git a/inits.json b/inits.json index c97bd758..2b4502c9 100644 --- a/inits.json +++ b/inits.json @@ -42,6 +42,7 @@ "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", + "Aperture Corrections and Full Image FWHM": "Set optional_info 'use_aperture_corrections_and_full_image_fwhm' to true to estimate image FWHM from isolated field stars and apply isolated-star aperture corrections. Default false.", "Skip Low Comparison Coverage Rejection": "Set optional_info 'skip_low_comparison_coverage_rejection' to y to disable EXOTIC's default rejection of comparison stars that are valid in far fewer frames than the rest of the comparison-star field. Default n.", "Fit Lightcurve to Every Comparison Candidate": "Set optional_info 'fit_lightcurve_to_every_comparison_candidate' to y to save one target lightcurve fit plot per comparison star into temp/ using the selected photometry setup. Default n.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", @@ -130,6 +131,7 @@ "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_adaptive_apertures": false, + "use_aperture_corrections_and_full_image_fwhm": false, "skip_low_comparison_coverage_rejection": "n", "fit_lightcurve_to_every_comparison_candidate": "n", "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 54cd6890..247c4b0b 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -91,9 +91,15 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: sys.modules.setdefault("exotic.api.ld", fake_ld) from exotic.exotic import ( + APERTURE_MAX_FWHM_MULTIPLIER, + APERTURE_MIN_FWHM_MULTIPLIER, + APERTURE_SIGMA_MAX, + APERTURE_SIGMA_MIN, + GAUSSIAN_SIGMA_TO_FWHM, adaptive_aperture_outlier_mask, annotate_transit_qc_expected_values, auto_tune_aperture_sigma_grid, + build_aperture_correction_profile, build_initial_ars_bounds, build_single_transit_duration_prior, build_target_fit_candidate_jobs, @@ -102,7 +108,9 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: cheap_lightcurve_prescore, centroid_offset_matches_reference, choose_centroid_seed_position, + compute_star_aperture_grid, compute_transit_qc_ktmf, + detect_aperture_correction_star_candidates, apply_comparison_star_suitability_outlier_rejection, comparison_calibration_selection_reason, comparison_candidate_triangle_plot_output_path, @@ -125,6 +133,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: get_multiprocess_bad_pixel_precheck_processes, estimate_ephemeris_tmid_and_bounds, estimate_tmid_and_bounds_with_eebls, + initialize_aperture_data_store, is_adaptive_aperture_mode_enabled, is_comp_star_required, is_out_of_transit_baseline_detrending_enabled, @@ -137,6 +146,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: parse_deviation_from_expected_transit_in_qc_sigma, prepare_final_fit_lightcurve_series, prepare_lightcurve_fit_input_series, + populate_aperture_data_for_frame, rank_comparison_candidate_preflight_plans, refit_selected_fast_comparison_on_full_lightcurve, representative_psf_sigma, @@ -157,6 +167,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: should_fit_lightcurve_to_every_comparison_candidate, should_detect_bad_pixels_before_photometry, should_use_aperture_photometry, + should_use_aperture_corrections_and_full_image_fwhm, should_exit_at_first_qc_pass_solution, should_pick_comparison_by_eebls_snr, should_stop_after_promising_partial_comparison_attempt, @@ -830,6 +841,13 @@ def test_should_use_aperture_photometry_parses_values(): assert should_use_aperture_photometry("n") is False +def test_should_use_aperture_corrections_and_full_image_fwhm_parses_values(): + assert should_use_aperture_corrections_and_full_image_fwhm(None) is False + assert should_use_aperture_corrections_and_full_image_fwhm("y") is True + assert should_use_aperture_corrections_and_full_image_fwhm("n") is False + assert should_use_aperture_corrections_and_full_image_fwhm(True) is True + + def test_should_use_eebls_to_initialize_tmid_and_bounds_parses_values(): assert should_use_eebls_to_initialize_tmid_and_bounds(None) is True assert should_use_eebls_to_initialize_tmid_and_bounds("y") is True @@ -1112,6 +1130,144 @@ def test_is_adaptive_aperture_mode_enabled_parses_values(): assert is_adaptive_aperture_mode_enabled(True) is True +def test_aperture_sigma_bounds_match_physical_fwhm_limits(): + assert APERTURE_SIGMA_MIN == pytest.approx( + APERTURE_MIN_FWHM_MULTIPLIER * GAUSSIAN_SIGMA_TO_FWHM + ) + assert APERTURE_SIGMA_MAX == pytest.approx( + APERTURE_MAX_FWHM_MULTIPLIER * GAUSSIAN_SIGMA_TO_FWHM + ) + + +def test_aperture_correction_profile_recovers_gaussian_curve_of_growth(): + sigma = 2.0 + fwhm = GAUSSIAN_SIGMA_TO_FWHM * sigma + y, x = np.mgrid[0:120, 0:120] + image = np.full((120, 120), 10.0, dtype=float) + positions = np.array([ + [25.0, 25.0], + [25.0, 70.0], + [70.0, 25.0], + [70.0, 70.0], + [95.0, 95.0], + ]) + for xc, yc in positions: + image += 1200.0 * np.exp(-((x - xc) ** 2 + (y - yc) ** 2) / (2.0 * sigma ** 2)) + + field_star_psfs = np.column_stack([ + positions[:, 0], + positions[:, 1], + np.full(positions.shape[0], 1200.0), + np.full(positions.shape[0], sigma), + np.full(positions.shape[0], sigma), + np.zeros(positions.shape[0]), + np.full(positions.shape[0], 10.0), + ]) + radii = np.array([ + APERTURE_MIN_FWHM_MULTIPLIER * fwhm, + fwhm, + APERTURE_MAX_FWHM_MULTIPLIER * fwhm, + ]) + + profile = build_aperture_correction_profile( + image, + radii, + fwhm_hint=fwhm, + field_star_psfs=field_star_psfs, + ) + + assert profile["applied"] is True + assert profile["star_count"] == positions.shape[0] + assert profile["image_fwhm"] == pytest.approx(fwhm) + assert profile["correction_factors"][0] == pytest.approx(2.0, rel=0.15) + assert profile["correction_factors"][1] == pytest.approx(1.066, rel=0.08) + assert profile["correction_factors"][2] == pytest.approx(1.0, abs=0.02) + + +def test_detect_aperture_correction_star_candidates_finds_numpy_local_peaks(): + sigma = 1.8 + fwhm = GAUSSIAN_SIGMA_TO_FWHM * sigma + y, x = np.mgrid[0:140, 0:140] + image = np.full((140, 140), 10.0, dtype=float) + positions = np.array([ + [30.0, 35.0], + [95.0, 42.0], + [58.0, 108.0], + ]) + amplitudes = np.array([1000.0, 850.0, 700.0]) + for (xc, yc), amplitude in zip(positions, amplitudes): + image += amplitude * np.exp(-((x - xc) ** 2 + (y - yc) ** 2) / (2.0 * sigma ** 2)) + + candidates = detect_aperture_correction_star_candidates(image, fwhm_hint=fwhm) + + assert candidates.shape[0] >= positions.shape[0] + for xc, yc in positions: + nearest = np.min(np.hypot(candidates[:, 0] - xc, candidates[:, 1] - yc)) + assert nearest < 1.5 + + +def test_compute_star_aperture_grid_applies_aperture_correction_factors(): + y, x = np.mgrid[0:41, 0:41] + image = 100.0 * np.exp(-((x - 20.0) ** 2 + (y - 20.0) ** 2) / (2.0 * 2.0 ** 2)) + apertures = np.array([2.5, 4.0]) + annuli = np.array([0.0]) + + raw_flux, _ = compute_star_aperture_grid( + image, + 0, + 20.0, + 20.0, + apertures, + annuli, + ) + corrected_flux, _ = compute_star_aperture_grid( + image, + 0, + 20.0, + 20.0, + apertures, + annuli, + aperture_correction_factors=np.array([2.0, 1.25]), + ) + + np.testing.assert_allclose(corrected_flux[:, 0], raw_flux[:, 0] * np.array([2.0, 1.25])) + + +def test_populate_aperture_data_skips_field_star_corrections_when_disabled(monkeypatch): + import exotic.exotic as exotic_module + + def fail_field_star_estimate(*_args, **_kwargs): + raise AssertionError("field-star FWHM estimation should be opt-in") + + monkeypatch.setattr(exotic_module, "estimate_isolated_field_star_psfs", fail_field_star_estimate) + y, x = np.mgrid[0:41, 0:41] + image = 100.0 * np.exp(-((x - 20.0) ** 2 + (y - 20.0) ** 2) / (2.0 * 2.0 ** 2)) + psf_data = { + "target": np.array([[20.0, 20.0, 100.0, 2.0, 2.0, 0.0, 0.0]]), + } + aper_data = initialize_aperture_data_store( + frame_count=1, + aperture_count=1, + annulus_count=1, + comp_star_count=0, + ) + + profile = populate_aperture_data_for_frame( + image, + 0, + psf_data, + 0, + aper_data, + np.array([4.0]), + np.array([0.0]), + fast_aperture_mask=False, + use_aperture_corrections_and_full_image_fwhm=False, + ) + + assert profile["applied"] is False + assert np.isfinite(aper_data["target"][0, 0, 0]) + + def test_should_use_fast_target_centroid_disables_fast_sigma_path_for_adaptive_runs(): assert should_use_fast_target_centroid(1, adaptive_apertures=False) is True assert should_use_fast_target_centroid(6, adaptive_apertures=False) is False diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 57bf5a36..7c75b411 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -368,6 +368,21 @@ def test_comp_params_defaults_use_adaptive_apertures_to_false(tmp_path): assert inputs.info_dict["use_adaptive_apertures"] is False +def test_comp_params_defaults_aperture_corrections_and_full_image_fwhm_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_aperture_corrections_and_full_image_fwhm"] is False + + def test_comp_params_reads_observatory_full_title_from_user_info(tmp_path): init_data = { "user_info": {"Observatory Full Title": "Whipple Observatory"}, @@ -713,6 +728,21 @@ def test_comp_params_reads_use_adaptive_apertures_from_optional_info(tmp_path): assert inputs.info_dict["use_adaptive_apertures"] is True +def test_comp_params_reads_aperture_corrections_and_full_image_fwhm_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_aperture_corrections_and_full_image_fwhm": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_aperture_corrections_and_full_image_fwhm"] is True + + class DummyResponse: def __init__(self, payload): self._payload = payload diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index b58210af..22555704 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -261,6 +261,34 @@ def test_nextastro_photometry_catalog_match_ignores_over_30_magnitudes(): assert match is None +def test_nextastro_photometry_catalog_match_rejects_separations_over_two_arcsec(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.001, + 'dec': 20.0, + 'Vmag': 12.3, + 'err_Vmag': 0.02, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match( + catalog, + 10.0, + 20.0, + 'CV', + max_separation_arcsec=30.0, + ) + + assert match is None + + def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], From 391387ca51741f7c89843fe94bab48c7012d474a Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Wed, 27 May 2026 09:55:21 +1000 Subject: [PATCH 055/116] report 0.001 rather than 0.000 mag error --- README.md | 1 + docs/README.md | 1 + exotic/exotic.py | 65 +++++++++++++++++++++-------- exotic/exotic_gui.py | 5 +++ exotic/inputs.py | 6 +++ exotic/output_files.py | 8 +++- exotic/plots.py | 8 ++-- exotic/utils.py | 43 ++++++++++++++++++- inits.json | 2 + tests/test_exotic_proper_motion.py | 26 ++++++++++++ tests/test_inputs.py | 30 +++++++++++++ tests/test_nextastro_variability.py | 22 ++++++++++ tests/test_output_files.py | 39 +++++++++++++++++ 13 files changed, 230 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index c1bf5b50..1f066ac5 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Maximum Wavelength (nm)": null, "Fast Aperture Mask (y/n)": false, + "prefer_pixel_values_over_wcs_for_target": "n", "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_aperture_corrections_and_full_image_fwhm": false, diff --git a/docs/README.md b/docs/README.md index 8d09fae6..90cb05d8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -215,6 +215,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, "Fast Aperture Mask (y/n)": false, + "prefer_pixel_values_over_wcs_for_target": "n", "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_aperture_corrections_and_full_image_fwhm": false, diff --git a/exotic/exotic.py b/exotic/exotic.py index 255f7164..f04d9e24 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -185,6 +185,7 @@ filename_date_token, is_usable_apparent_magnitude, magnitude_text, + normalized_magnitude_error, round_to_2, safe_output_filename, user_input, @@ -195,6 +196,7 @@ filename_date_token, is_usable_apparent_magnitude, magnitude_text, + normalized_magnitude_error, round_to_2, safe_output_filename, user_input, @@ -5086,6 +5088,25 @@ def should_ignore_header_wcs(config_value): return False +def should_prefer_pixel_values_over_wcs_for_target(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'prefer_pixel_values_over_wcs_for_target' value; " + "using WCS target coordinates.", warn=True) + return False + + def get_bad_wcs_threshold_fraction(config_value): default_fraction = SPARSE_MISSING_WCS_DROP_THRESHOLD default_percent = default_fraction * 100.0 @@ -9546,7 +9567,8 @@ def any_projected_coord_out_of_frame(coords, image_shape): def check_target_pixel_wcs(input_x_pixel, input_y_pixel, info_dict, ra_list, dec_list, image_data, obs_time, - non_interactive_run=False, wcs_header=None): + non_interactive_run=False, wcs_header=None, + prefer_pixel_values_over_wcs_for_target=False): """ Verify the provided pixel coordinates match the target's right ascension and declination. """ @@ -9570,7 +9592,8 @@ def check_target_pixel_wcs(input_x_pixel, input_y_pixel, info_dict, ra_list, dec centroid_x, centroid_y, sigma_x, sigma_y = get_psf_parameters(image_data, calculated_x_pixel, calculated_y_pixel) return check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, - calculated_x_pixel, calculated_y_pixel, non_interactive_run=non_interactive_run) + calculated_x_pixel, calculated_y_pixel, non_interactive_run=non_interactive_run, + prefer_pixel_values_over_wcs_for_target=prefer_pixel_values_over_wcs_for_target) def get_psf_parameters(image_data, x_pixel, y_pixel): @@ -9583,12 +9606,17 @@ def get_psf_parameters(image_data, x_pixel, y_pixel): def check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, - calculated_x_pixel, calculated_y_pixel, non_interactive_run=False): + calculated_x_pixel, calculated_y_pixel, non_interactive_run=False, + prefer_pixel_values_over_wcs_for_target=False): while True: try: validate_pixel_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y) return input_x_pixel, input_y_pixel except ValueError: + if should_prefer_pixel_values_over_wcs_for_target(prefer_pixel_values_over_wcs_for_target): + log_info("Proceeding with provided target pixel coordinates because " + "prefer_pixel_values_over_wcs_for_target is enabled.", warn=True) + return input_x_pixel, input_y_pixel if non_interactive_run: if np.isfinite(centroid_x) and np.isfinite(centroid_y): log_info("Proceeding with WCS-derived centroided target coordinates due to " @@ -9957,12 +9985,9 @@ def nextastro_catalog_rows(catalog_response): def row_nextastro_magnitude(row, band_candidates): for priority, (mag_column, error_column, band_label) in enumerate(band_candidates): magnitude = _finite_float(row.get(mag_column)) - magnitude_error = _finite_float(row.get(error_column)) + magnitude_error = normalized_magnitude_error(row.get(error_column)) if not is_usable_apparent_magnitude(magnitude) or magnitude_error is None: continue - magnitude_error = abs(magnitude_error) - if not is_usable_apparent_magnitude(magnitude_error): - continue return { 'priority': priority, 'mag': magnitude, @@ -12927,16 +12952,13 @@ def stellar_variability_label(comp_label, comp_star): def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_label, save, s_name, observed_filter=None): comp_mag = _finite_float(comp_star.get('mag')) - comp_mag_error = _finite_float(comp_star.get('error')) + comp_mag_error = normalized_magnitude_error(comp_star.get('error')) if ( comp_mag is None or comp_mag_error is None or not is_usable_apparent_magnitude(comp_mag) ): raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") - comp_mag_error = abs(comp_mag_error) - if not is_usable_apparent_magnitude(comp_mag_error): - raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") observed_filter = observed_filter or comp_star.get('observed_filter') fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) @@ -17768,12 +17790,21 @@ def _main_impl(): wcs_header = get_first_image_header(wcs_file) ra_wcs, dec_wcs = get_ra_dec(wcs_header, image_shape=reference_image.shape) - exotic_UIprevTPX, exotic_UIprevTPY = check_target_pixel_wcs(exotic_UIprevTPX, exotic_UIprevTPY, - pDict, ra_wcs, dec_wcs, - reference_image, - jd_times[0], - non_interactive_run=args.non_interactive_run, - wcs_header=wcs_header) + prefer_input_target_pixels = exotic_infoDict.get( + 'prefer_pixel_values_over_wcs_for_target', 'n' + ) + exotic_UIprevTPX, exotic_UIprevTPY = check_target_pixel_wcs( + exotic_UIprevTPX, + exotic_UIprevTPY, + pDict, + ra_wcs, + dec_wcs, + reference_image, + jd_times[0], + non_interactive_run=args.non_interactive_run, + wcs_header=wcs_header, + prefer_pixel_values_over_wcs_for_target=prefer_input_target_pixels, + ) ra_dec_tar = (ra_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], dec_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 70401985..7df0e091 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -417,6 +417,7 @@ def save_input(): "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", + "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", @@ -447,6 +448,7 @@ def save_input(): new_inits['optional_info'] = { "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, + "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", @@ -1503,6 +1505,7 @@ def save_input(): "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", + "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", @@ -1581,6 +1584,7 @@ def save_input(): "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, + "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", @@ -1638,6 +1642,7 @@ def save_input(): "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, + "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", diff --git a/exotic/inputs.py b/exotic/inputs.py index ff320440..46ac1176 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -207,6 +207,7 @@ def __init__(self, init_opt): 'dist': None, 'pm_ra': None, 'pm_dec': None, 'airmass_already_corrected': False, 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, 'fast_aperture_mask': False, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', + 'prefer_pixel_values_over_wcs_for_target': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'detrend_on_outoftransit_baseline': True, 'final_fit_baseline_duration_multiplier': 1.0, @@ -431,6 +432,11 @@ def comp_params(self, init_file, planet_dict): 'Ignore WCS in header and do manual alignment', 'ignore_header_wcs', ), + 'prefer_pixel_values_over_wcs_for_target': ( + 'prefer_pixel_values_over_wcs_for_target', + 'Prefer Pixel Coordinates to WCS Coordinates if there is a conflict', + 'Prefer Pixel Coordinates to WCS Coordinates if there is a conflict? (y/n)', + ), 'disable_vertical_flux_normalization': ( 'disable vertical flux normalization', 'Disable vertical flux normalization', diff --git a/exotic/output_files.py b/exotic/output_files.py index 73992107..0f94d035 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -7,18 +7,22 @@ try: from utils import ( filename_date_token, + format_magnitude_error, format_magnitude, magnitude_text, round_to_2, + rounded_magnitude_error, rounded_magnitude_value, safe_output_filename, ) except ImportError: from .utils import ( filename_date_token, + format_magnitude_error, format_magnitude, magnitude_text, round_to_2, + rounded_magnitude_error, rounded_magnitude_value, safe_output_filename, ) @@ -141,7 +145,7 @@ def aid_comparison_metadata(vsp_param): 'catalog_match_separation_arcsec': vsp_param.get('separation_arcsec'), 'magnitude_band': vsp_param.get('mag_band'), 'apparent_magnitude': rounded_magnitude_value(vsp_param.get('cmag')), - 'apparent_magnitude_error': rounded_magnitude_value(abs(finite_float(vsp_param.get('cmag_err')))), + 'apparent_magnitude_error': rounded_magnitude_error(vsp_param.get('cmag_err')), }) @@ -1225,7 +1229,7 @@ def aavso(self): mag = format_magnitude(vsp_p.get('mag'), default=None) if mag is None: continue - mag_err = format_magnitude(abs(finite_float(vsp_p.get('mag_err')))) + mag_err = format_magnitude_error(vsp_p.get('mag_err')) cmag = format_magnitude(vsp_p.get('cmag')) chart_id = self.chart_id or vsp_p.get('chart_id') or 'na' f.write(f"{variable_name},{round(vsp_p['time'], 5)},{mag},{mag_err}," diff --git a/exotic/plots.py b/exotic/plots.py index 872f557e..4c3755f4 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -12,6 +12,7 @@ filename_date_token, is_usable_apparent_magnitude, magnitude_text, + normalized_magnitude_error, safe_output_filename, ) except ImportError: @@ -19,6 +20,7 @@ filename_date_token, is_usable_apparent_magnitude, magnitude_text, + normalized_magnitude_error, safe_output_filename, ) @@ -591,11 +593,7 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): for vsp_p in vsp_params: if not is_usable_apparent_magnitude(vsp_p.get('mag')): continue - mag_err = _finite_plot_float(vsp_p.get('mag_err')) - if mag_err is not None: - mag_err = abs(mag_err) - if not is_usable_apparent_magnitude(mag_err): - mag_err = None + mag_err = normalized_magnitude_error(vsp_p.get('mag_err')) ax.errorbar(vsp_p['time'], vsp_p['mag'], yerr=mag_err, color="tomato", fmt='.') plotted_points += 1 diff --git a/exotic/utils.py b/exotic/utils.py index daa1fed4..df3e3ac0 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -25,6 +25,7 @@ _WINDOWS_ILLEGAL_FILENAME_CHARS_RE = re.compile(r'[<>:"/\\|?*\x00-\x1f\x7f]') MAX_APPARENT_MAGNITUDE = 30.0 MAGNITUDE_DECIMAL_PLACES = 3 +MINIMUM_MAGNITUDE_ERROR = 0.001 def sanitize_filename_component(value, fallback='output'): @@ -90,13 +91,51 @@ def rounded_magnitude_value(value, default=None, digits=MAGNITUDE_DECIMAL_PLACES return round(parsed, digits) +def normalized_magnitude_error(value, default=None, minimum=MINIMUM_MAGNITUDE_ERROR, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = parse_finite_float(value) + if parsed is None: + return default + parsed = abs(parsed) + if parsed > max_magnitude: + return default + return max(parsed, minimum) + + +def format_magnitude_error(value, default="na", digits=MAGNITUDE_DECIMAL_PLACES, + minimum=MINIMUM_MAGNITUDE_ERROR, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = normalized_magnitude_error( + value, + default=None, + minimum=minimum, + max_magnitude=max_magnitude, + ) + if parsed is None: + return default + return f"{parsed:.{digits}f}" + + +def rounded_magnitude_error(value, default=None, digits=MAGNITUDE_DECIMAL_PLACES, + minimum=MINIMUM_MAGNITUDE_ERROR, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = normalized_magnitude_error( + value, + default=None, + minimum=minimum, + max_magnitude=max_magnitude, + ) + if parsed is None: + return default + return round(parsed, digits) + + def magnitude_text(band, magnitude, magnitude_error=None): formatted_mag = format_magnitude(magnitude, default=None) if formatted_mag is None: return None - parsed_error = parse_finite_float(magnitude_error) - formatted_error = format_magnitude(abs(parsed_error), default=None) if parsed_error is not None else None + formatted_error = format_magnitude_error(magnitude_error, default=None) if formatted_error is None: return f"{band}={formatted_mag}" return f"{band}={formatted_mag} +/- {formatted_error}" diff --git a/inits.json b/inits.json index 2b4502c9..10e42ab3 100644 --- a/inits.json +++ b/inits.json @@ -26,6 +26,7 @@ "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Set to 0 to disable. Default 4.", + "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", @@ -115,6 +116,7 @@ "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, "pointing_rejection_sigma": 4.0, + "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": false, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "y", diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 247c4b0b..4fd5673c 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -159,6 +159,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: save_final_triangle_plot, save_selected_photometry_debug_series, should_keep_header_wcs_alignment, + should_prefer_pixel_values_over_wcs_for_target, sigma_clip, summarize_adaptive_aperture_usage, summarize_prior_transit_coverage, @@ -776,6 +777,31 @@ def test_check_coordinates_non_interactive_uses_wcs_pixel_when_centroid_is_nan() assert y_pixel == 201 +def test_check_coordinates_can_prefer_input_pixels_over_wcs_conflict(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=5, + input_y_pixel=5, + centroid_x=100.25, + centroid_y=200.75, + sigma_x=1.0, + sigma_y=1.0, + calculated_x_pixel=100, + calculated_y_pixel=201, + non_interactive_run=True, + prefer_pixel_values_over_wcs_for_target="y", + ) + + assert x_pixel == 5 + assert y_pixel == 5 + + +def test_should_prefer_pixel_values_over_wcs_for_target_parses_values(): + assert should_prefer_pixel_values_over_wcs_for_target(None) is False + assert should_prefer_pixel_values_over_wcs_for_target("n") is False + assert should_prefer_pixel_values_over_wcs_for_target("y") is True + assert should_prefer_pixel_values_over_wcs_for_target(True) is True + + def test_is_comp_star_required_parses_values(): assert is_comp_star_required(None) is True assert is_comp_star_required("y") is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 7c75b411..55a40c25 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -83,6 +83,21 @@ def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "n" +def test_comp_params_defaults_prefer_pixel_values_over_wcs_for_target_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["prefer_pixel_values_over_wcs_for_target"] == "n" + + def test_comp_params_defaults_bad_wcs_threshold_percent_to_three(tmp_path): init_data = { "user_info": {}, @@ -428,6 +443,21 @@ def test_comp_params_reads_ignore_header_wcs_from_optional_info(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "y" +def test_comp_params_reads_prefer_pixel_values_over_wcs_for_target_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"prefer_pixel_values_over_wcs_for_target": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["prefer_pixel_values_over_wcs_for_target"] == "y" + + def test_comp_params_reads_bad_wcs_threshold_percent_from_optional_info(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 22555704..1d62cbd6 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -239,6 +239,28 @@ def test_nextastro_photometry_catalog_match_prefers_requested_filter(): assert match['separation_arcsec'] > 0 +def test_nextastro_photometry_catalog_match_floors_zero_magnitude_error(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.0001, + 'dec': 20.0001, + 'Vmag': 12.3, + 'err_Vmag': 0.0, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') + + assert match['error'] == pytest.approx(0.001) + + def test_nextastro_photometry_catalog_match_ignores_over_30_magnitudes(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 4d6883ed..9a1765eb 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -207,6 +207,45 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert "HAT-P-32,2450000.12345,12.340,0.050,V,NO,STD" in output_text +def test_aid_output_floors_reported_magnitude_errors(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + } + vsp_params = [{ + "time": 2450000.12345, + "mag": 12.34, + "mag_err": 0.0, + "airmass": 1.234, + "cname": "RA=10.1000000 Dec=-20.2000000", + "cmag": 12.1, + "cmag_err": 0.0, + "pos": [493, 202], + "catalog_source": "NextAstro photometry catalog", + "is_aavso_vsp": False, + "mag_band": "V", + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = (tmp_path / "AID_AAVSO_HAT-P-32_2020-01-01.txt").read_text(encoding="utf-8") + metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") + + assert metadata["apparent_magnitude_error"] == pytest.approx(0.001) + assert "HAT-P-32,2450000.12345,12.340,0.001,V,NO,STD" in output_text + + def test_aid_output_skips_over_30_magnitude_rows(tmp_path): fit = DummyFit() p_dict = { From 28c494b36c046bf0a50676e979f379d4651715db Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 27 May 2026 10:13:22 +1000 Subject: [PATCH 056/116] transit depth =/= rp/rs --- exotic/api/elca.py | 6 +- exotic/api/output_aavso.py | 112 +++++++++++- exotic/exotic.py | 25 ++- exotic/output_files.py | 89 ++++++++- exotic/transit_depth.py | 358 +++++++++++++++++++++++++++++++++++++ tests/test_output_files.py | 70 +++++++- 6 files changed, 636 insertions(+), 24 deletions(-) create mode 100644 exotic/transit_depth.py diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 46f5dcf4..026e3f4a 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -3009,7 +3009,7 @@ def plot_bestfit( rprs2 = self.parameters['rprs'] ** 2 rprs2err = 2 * self.parameters['rprs'] * self.errors['rprs'] - lclabel1 = r"$R^{2}_{p}/R^{2}_{s}$ = %s $\pm$ %s" % ( + lclabel1 = r"Area ratio $(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" % ( str(round_to_2(rprs2, rprs2err)), str(round_to_2(rprs2err)) ) @@ -3536,7 +3536,7 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p rprs2 = self.lc_data[0]['priors']['rprs']**2 rprs2err = 2*self.lc_data[0]['priors']['rprs']*self.lc_data[0]['errors']['rprs'] - lclabel1 = r"$R^{2}_{p}/R^{2}_{s}$ = %s $\pm$ %s" %( + lclabel1 = r"Area ratio $(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( str(round_to_2(rprs2, rprs2err)), str(round_to_2(rprs2err)) ) @@ -3679,7 +3679,7 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): rprs2 = self.parameters['rprs']**2 rprs2err = 2*self.parameters['rprs']*self.errors['rprs'] - lclabel1 = r"$R^{2}_{p}/R^{2}_{s}$ = %s $\pm$ %s" %( + lclabel1 = r"Area ratio $(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( str(round_to_2(rprs2, rprs2err)), str(round_to_2(rprs2err)) ) diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 09afac50..6c70e12b 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -37,6 +37,7 @@ # ########################################################################### # import hashlib from json import dump, dumps +import math from numpy import mean, median, std from pathlib import Path import re @@ -49,6 +50,85 @@ from .version import __version__ except ImportError: from version import __version__ +try: + from ..transit_depth import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + fit_transit_depth_summary, + planet_dict_transit_errors, + planet_dict_transit_parameters, + ) +except ImportError: + from exotic.transit_depth import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + fit_transit_depth_summary, + planet_dict_transit_errors, + planet_dict_transit_parameters, + ) + + +def _format_depth(value, error): + try: + value = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(value): + return None + try: + error = float(error) + except (TypeError, ValueError): + error = math.nan + if math.isfinite(error) and error >= 0: + return f"{round_to_2(value, error)} +/- {round_to_2(error)} [%]" + return f"{round_to_2(value)} +/- n/a [%]" + + +def _depth_final_params(fit, planet_dict=None, limb_darkening=None): + depth_summary = fit_transit_depth_summary( + fit, + prior_parameters=planet_dict_transit_parameters( + planet_dict, + limb_darkening=limb_darkening, + fallback=getattr(fit, 'prior', None), + ), + prior_errors=planet_dict_transit_errors(planet_dict, limb_darkening=limb_darkening), + ) + entries = {} + for label, value_key, error_key in ( + (AREA_DEPTH_LABEL, 'area_depth', 'area_depth_error'), + (OBSERVABLE_DEPTH_LABEL, 'observable_depth', 'observable_depth_error'), + (PRIOR_OBSERVABLE_DEPTH_LABEL, 'prior_observable_depth', 'prior_observable_depth_error'), + (OBSERVABLE_DEPTH_DELTA_LABEL, 'observable_depth_prior_delta', 'observable_depth_prior_delta_error'), + ): + text = _format_depth(depth_summary.get(value_key), depth_summary.get(error_key)) + if text is not None: + entries[label] = text + return entries + + +def _depth_result_entry(value, error): + try: + value = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(value): + return None + try: + error = float(error) + except (TypeError, ValueError): + error = math.nan + entry = { + 'value': str(round_to_2(value, error)) if math.isfinite(error) else str(round_to_2(value)), + 'units': "percent", + } + if math.isfinite(error): + entry['uncertainty'] = str(round_to_2(error)) + return entry class OutputFiles: @@ -81,8 +161,6 @@ def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB", "Ratio of Planet to Stellar Radius (Rp/Rs)": f"{round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- " f"{round_to_2(self.fit.errors['rprs'])}", - "Transit depth (Rp/Rs)^2": f"{round_to_2(100. * (self.fit.parameters['rprs'] ** 2.))} +/- " - f"{round_to_2(100. * 2. * self.fit.parameters['rprs'] * self.fit.errors['rprs'])} [%]", "Semi Major Axis/Star Radius (a/Rs)": f"{round_to_2(self.fit.parameters['ars'], self.fit.errors['ars'])} +/- " f"{round_to_2(self.fit.errors['ars'])} ", "Airmass coefficient 1 (a1)": f"{round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- " @@ -91,6 +169,16 @@ def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min f"{round_to_2(self.fit.errors['a2'])}", "Scatter in the residuals of the lightcurve fit is": f"{round_to_2(100. * std(self.fit.residuals / median(self.fit.data)))} %", } + depth_params = _depth_final_params(self.fit, self.p_dict) + params_num = { + "Mid-Transit Time (Tmid)": params_num["Mid-Transit Time (Tmid)"], + "Ratio of Planet to Stellar Radius (Rp/Rs)": params_num["Ratio of Planet to Stellar Radius (Rp/Rs)"], + **depth_params, + "Semi Major Axis/Star Radius (a/Rs)": params_num["Semi Major Axis/Star Radius (a/Rs)"], + "Airmass coefficient 1 (a1)": params_num["Airmass coefficient 1 (a1)"], + "Airmass coefficient 2 (a2)": params_num["Airmass coefficient 2 (a2)"], + "Scatter in the residuals of the lightcurve fit is": params_num["Scatter in the residuals of the lightcurve fit is"], + } if phot_opt: phot_ext = {"Best Comparison Star": f"#{comp_star} - {comp_coords}" if min_aper >= 0 else str(comp_star)} @@ -338,4 +426,24 @@ def aavso_dicts(planet_dict, fit, i_dict, durs, ld0, ld1, ld2, ld3): 'units': "degrees" } + limb_darkening = (ld0, ld1, ld2, ld3) + depth_summary = fit_transit_depth_summary( + fit, + prior_parameters=planet_dict_transit_parameters( + planet_dict, + limb_darkening=limb_darkening, + fallback=getattr(fit, 'prior', None), + ), + prior_errors=planet_dict_transit_errors(planet_dict, limb_darkening=limb_darkening), + ) + for label, value_key, error_key in ( + (AREA_DEPTH_LABEL, 'area_depth', 'area_depth_error'), + (OBSERVABLE_DEPTH_LABEL, 'observable_depth', 'observable_depth_error'), + (PRIOR_OBSERVABLE_DEPTH_LABEL, 'prior_observable_depth', 'prior_observable_depth_error'), + (OBSERVABLE_DEPTH_DELTA_LABEL, 'observable_depth_prior_delta', 'observable_depth_prior_delta_error'), + ): + entry = _depth_result_entry(depth_summary.get(value_key), depth_summary.get(error_key)) + if entry is not None: + results[label] = entry + return priors, filter_type, results diff --git a/exotic/exotic.py b/exotic/exotic.py index f04d9e24..4cda5c28 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -153,6 +153,7 @@ AIDOutputFiles, fit_impact_parameter_value_error, format_parameter_with_error, + formatted_transit_depth_parameters, save_comp_star_calibration_summary, ) except ImportError: # package import @@ -161,8 +162,13 @@ AIDOutputFiles, fit_impact_parameter_value_error, format_parameter_with_error, + formatted_transit_depth_parameters, save_comp_star_calibration_summary, ) +try: + from transit_depth import fit_transit_depth_summary +except ImportError: + from .transit_depth import fit_transit_depth_summary try: from plate_status import PlateStatus except ImportError: @@ -15458,13 +15464,15 @@ def summarize_lightcurve_fit_parameters(fit): f"Rp/R*={format_fit_parameter_with_uncertainty(parameters.get('rprs'), errors.get('rprs'))}", ] - rprs = parameters.get('rprs') - rprs_err = errors.get('rprs') - depth = None if rprs is None else 100.0 * float(rprs) ** 2 - depth_err = None - if rprs is not None and rprs_err is not None and np.isfinite(rprs) and np.isfinite(rprs_err): - depth_err = 200.0 * float(rprs) * float(rprs_err) - summary_parts.append(f"depth={format_fit_parameter_with_uncertainty(depth, depth_err, suffix='%')}") + depth_summary = fit_transit_depth_summary(fit) + summary_parts.append( + "area_depth=" + f"{format_fit_parameter_with_uncertainty(depth_summary.get('area_depth'), depth_summary.get('area_depth_error'), suffix='%')}" + ) + summary_parts.append( + "observable_depth=" + f"{format_fit_parameter_with_uncertainty(depth_summary.get('observable_depth'), depth_summary.get('observable_depth_error'), suffix='%')}" + ) summary_parts.append(f"inc={format_fit_parameter_with_uncertainty(parameters.get('inc'), errors.get('inc'))}") if getattr(fit, 'airmass_fit_skipped', False): @@ -19444,7 +19452,8 @@ def _main_impl(): log_info("FINAL PLANETARY PARAMETERS\n") log_info(f" Mid-Transit Time [BJD_TDB]: {round_to_2(myfit.parameters['tmid'], myfit.errors['tmid'])} +/- {round_to_2(myfit.errors['tmid'])}") log_info(f" Radius Ratio (Planet/Star) [Rp/R*]: {round_to_2(myfit.parameters['rprs'], myfit.errors['rprs'])} +/- {round_to_2(myfit.errors['rprs'])}") - log_info(f" Transit depth [(Rp/R*)^2]: {round_to_2(100. * (myfit.parameters['rprs'] ** 2.))} +/- {round_to_2(100. * 2. * myfit.parameters['rprs'] * myfit.errors['rprs'])} [%]") + for depth_label, depth_text in formatted_transit_depth_parameters(myfit, pDict).items(): + log_info(f" {depth_label}: {depth_text}") log_info(f" Orbital Inclination [inc]: {round_to_2(myfit.parameters['inc'], myfit.errors['inc'])} +/- {round_to_2(myfit.errors['inc'])}") ars_text = format_parameter_with_error(myfit.parameters.get('ars'), myfit.errors.get('ars')) if ars_text is not None: diff --git a/exotic/output_files.py b/exotic/output_files.py index 0f94d035..57f9f7a1 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -34,6 +34,26 @@ from plate_status import PlateStatus except ImportError: from .plate_status import PlateStatus +try: + from transit_depth import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + fit_transit_depth_summary, + planet_dict_transit_errors, + planet_dict_transit_parameters, + ) +except ImportError: + from .transit_depth import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + fit_transit_depth_summary, + planet_dict_transit_errors, + planet_dict_transit_parameters, + ) def aavso_airmass_results(fit): @@ -816,6 +836,37 @@ def format_parameter_with_error(value, error): return f"{round_to_2(value)} +/- n/a" +def format_percent_parameter_with_error(value, error): + text = format_parameter_with_error(value, error) + return f"{text} [%]" if text is not None else None + + +def formatted_transit_depth_parameters(fit, planet_dict=None, limb_darkening=None): + prior_parameters = planet_dict_transit_parameters( + planet_dict, + limb_darkening=limb_darkening, + fallback=getattr(fit, 'prior', None), + ) + prior_errors = planet_dict_transit_errors(planet_dict, limb_darkening=limb_darkening) + summary = fit_transit_depth_summary( + fit, + prior_parameters=prior_parameters, + prior_errors=prior_errors, + ) + + entries = {} + for label, value_key, error_key in ( + (AREA_DEPTH_LABEL, 'area_depth', 'area_depth_error'), + (OBSERVABLE_DEPTH_LABEL, 'observable_depth', 'observable_depth_error'), + (PRIOR_OBSERVABLE_DEPTH_LABEL, 'prior_observable_depth', 'prior_observable_depth_error'), + (OBSERVABLE_DEPTH_DELTA_LABEL, 'observable_depth_prior_delta', 'observable_depth_prior_delta_error'), + ): + text = format_percent_parameter_with_error(summary.get(value_key), summary.get(error_key)) + if text is not None: + entries[label] = text + return entries + + def fit_impact_parameter_value_error(fit): parameters = getattr(fit, 'parameters', {}) or {} errors = getattr(fit, 'errors', {}) or {} @@ -920,10 +971,15 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB", "Ratio of Planet to Stellar Radius (Rp/R*)": f"{round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- " f"{round_to_2(self.fit.errors['rprs'])}", - "Transit depth (Rp/Rs)^2": f"{round_to_2(100. * (self.fit.parameters['rprs'] ** 2.))} +/- " - f"{round_to_2(100. * 2. * self.fit.parameters['rprs'] * self.fit.errors['rprs'])} [%]", "Orbital Inclination (inc)": f"{round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- " - f"{round_to_2(self.fit.errors['inc'])} ", + f"{round_to_2(self.fit.errors['inc'])} ", + } + depth_params = formatted_transit_depth_parameters(self.fit, self.p_dict) + params_num = { + "Mid-Transit Time (Tmid)": params_num["Mid-Transit Time (Tmid)"], + "Ratio of Planet to Stellar Radius (Rp/R*)": params_num["Ratio of Planet to Stellar Radius (Rp/R*)"], + **depth_params, + "Orbital Inclination (inc)": params_num["Orbital Inclination (inc)"], } ars_text = format_parameter_with_error( self.fit.parameters.get('ars'), @@ -1333,12 +1389,27 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): impact_parameter, impact_error = fit_impact_parameter_value_error(fit) optional_results['Impact Parameter (b)'] = aavso_result_entry(impact_parameter, impact_error) - rprs = finite_float(fit.parameters.get('rprs')) - rprs_error = finite_float(fit.errors.get('rprs')) - if np.isfinite(rprs): - optional_results['Transit depth (Rp/R*)^2'] = aavso_result_entry( - 100.0 * (rprs ** 2.0), - 100.0 * 2.0 * rprs * rprs_error if np.isfinite(rprs_error) else np.nan, + limb_darkening = (ld0, ld1, ld2, ld3) + prior_parameters = planet_dict_transit_parameters( + planet_dict, + limb_darkening=limb_darkening, + fallback=getattr(fit, 'prior', None), + ) + prior_errors = planet_dict_transit_errors(planet_dict, limb_darkening=limb_darkening) + depth_summary = fit_transit_depth_summary( + fit, + prior_parameters=prior_parameters, + prior_errors=prior_errors, + ) + for label, value_key, error_key in ( + (AREA_DEPTH_LABEL, 'area_depth', 'area_depth_error'), + (OBSERVABLE_DEPTH_LABEL, 'observable_depth', 'observable_depth_error'), + (PRIOR_OBSERVABLE_DEPTH_LABEL, 'prior_observable_depth', 'prior_observable_depth_error'), + (OBSERVABLE_DEPTH_DELTA_LABEL, 'observable_depth_prior_delta', 'observable_depth_prior_delta_error'), + ): + optional_results[label] = aavso_result_entry( + depth_summary.get(value_key), + depth_summary.get(error_key), units="percent", ) diff --git a/exotic/transit_depth.py b/exotic/transit_depth.py new file mode 100644 index 00000000..2250fa8d --- /dev/null +++ b/exotic/transit_depth.py @@ -0,0 +1,358 @@ +import math + +import numpy as np + + +AREA_DEPTH_LABEL = "Radius-ratio area depth (Rp/R*)^2" +OBSERVABLE_DEPTH_LABEL = "Observable model transit depth" +PRIOR_OBSERVABLE_DEPTH_LABEL = "Prior observable model transit depth" +OBSERVABLE_DEPTH_DELTA_LABEL = "Observable model depth change from prior" + +_DEPTH_ERROR_KEYS = ("rprs", "ars", "inc", "ecc", "omega", "u0", "u1", "u2", "u3") +_REQUIRED_TRANSIT_KEYS = ("rprs", "per", "ars", "inc", "ecc", "omega", "tmid", "u0", "u1", "u2", "u3") + + +def finite_float(value, default=np.nan): + try: + value = float(value) + except (TypeError, ValueError): + return default + return value if np.isfinite(value) else default + + +def radius_ratio_area_depth_percent(rprs, rprs_error=None): + rprs = finite_float(rprs) + if not np.isfinite(rprs) or rprs < 0: + return np.nan, np.nan + + depth = 100.0 * rprs ** 2 + rprs_error = finite_float(rprs_error) + if np.isfinite(rprs_error) and rprs_error >= 0: + return float(depth), float(200.0 * abs(rprs) * rprs_error) + return float(depth), np.nan + + +def planet_dict_transit_parameters(planet_dict, limb_darkening=None, fallback=None): + values = dict(fallback or {}) + planet_dict = planet_dict or {} + + aliases = { + "rprs": ("rprs", "pl_ratror"), + "per": ("pPer", "pl_orbper", "per", "period"), + "ars": ("aRs", "pl_ratdor", "ars"), + "inc": ("inc", "pl_orbincl"), + "ecc": ("ecc", "pl_orbeccen"), + "omega": ("omega", "pl_orblper"), + "tmid": ("midT", "pl_tranmid", "tmid"), + } + for target, names in aliases.items(): + for name in names: + if name not in planet_dict: + continue + value = finite_float(planet_dict.get(name)) + if np.isfinite(value): + values[target] = value + break + + if limb_darkening is not None: + for index, item in enumerate(limb_darkening): + if index > 3: + break + if isinstance(item, (list, tuple, np.ndarray)): + value = item[0] if len(item) else np.nan + else: + value = item + value = finite_float(value) + if np.isfinite(value): + values[f"u{index}"] = value + + for key in ("u0", "u1", "u2", "u3"): + values.setdefault(key, 0.0) + values.setdefault("ecc", 0.0) + values.setdefault("omega", 0.0) + values.setdefault("tmid", 0.0) + return values + + +def planet_dict_transit_errors(planet_dict, limb_darkening=None, fallback=None): + errors = dict(fallback or {}) + planet_dict = planet_dict or {} + + aliases = { + "rprs": ("rprsUnc", "pl_ratrorerr1"), + "per": ("pPerUnc", "pl_orbpererr1"), + "ars": ("aRsUnc", "pl_ratdorerr1"), + "inc": ("incUnc", "pl_orbinclerr1"), + "tmid": ("midTUnc", "pl_tranmiderr1"), + } + for target, names in aliases.items(): + for name in names: + if name not in planet_dict: + continue + value = abs(finite_float(planet_dict.get(name))) + if np.isfinite(value): + errors[target] = value + break + + if limb_darkening is not None: + for index, item in enumerate(limb_darkening): + if index > 3: + break + if not isinstance(item, (list, tuple, np.ndarray)) or len(item) < 2: + continue + value = abs(finite_float(item[1])) + if np.isfinite(value): + errors[f"u{index}"] = value + return errors + + +def complete_transit_parameters(parameters): + values = planet_dict_transit_parameters(parameters) + if "per" not in values and "period" in values: + values["per"] = values["period"] + return values + + +def transit_duration_days(parameters): + values = complete_transit_parameters(parameters) + period = finite_float(values.get("per")) + rprs = finite_float(values.get("rprs")) + ars = finite_float(values.get("ars")) + inc = finite_float(values.get("inc")) + ecc = finite_float(values.get("ecc"), 0.0) + omega = math.radians(finite_float(values.get("omega"), 0.0)) + + if ( + not np.isfinite(period) or period <= 0 + or not np.isfinite(rprs) or rprs < 0 + or not np.isfinite(ars) or ars <= 0 + or not np.isfinite(inc) + or not np.isfinite(ecc) or ecc < 0 or ecc >= 1 + ): + return np.nan + + sin_inc = math.sin(math.radians(inc)) + if not np.isfinite(sin_inc) or sin_inc <= 0: + return np.nan + + denominator = 1.0 + ecc * math.sin(omega) + if not np.isfinite(denominator) or math.isclose(denominator, 0.0): + return np.nan + + impact_scale = ars * (1.0 - ecc ** 2) / denominator + impact_parameter = impact_scale * math.cos(math.radians(inc)) + chord_sq = (1.0 + rprs) ** 2 - impact_parameter ** 2 + if not np.isfinite(chord_sq) or chord_sq <= 0 or impact_scale <= 0: + return np.nan + + argument = math.sqrt(chord_sq) / (impact_scale * sin_inc) + argument = float(np.clip(argument, -1.0, 1.0)) + duration = (period / math.pi) * math.asin(argument) + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + +def impact_parameter(parameters): + values = complete_transit_parameters(parameters) + ars = finite_float(values.get("ars")) + inc = finite_float(values.get("inc")) + ecc = finite_float(values.get("ecc"), 0.0) + omega = math.radians(finite_float(values.get("omega"), 0.0)) + if not np.isfinite(ars) or not np.isfinite(inc) or not np.isfinite(ecc): + return np.nan + denominator = 1.0 + ecc * math.sin(omega) + if not np.isfinite(denominator) or math.isclose(denominator, 0.0): + return np.nan + return float(ars * (1.0 - ecc ** 2) * math.cos(math.radians(inc)) / denominator) + + +def geometric_observable_depth_fraction(parameters): + values = complete_transit_parameters(parameters) + rprs = finite_float(values.get("rprs")) + b = abs(impact_parameter(values)) + if not np.isfinite(rprs) or rprs < 0 or not np.isfinite(b): + return np.nan + if b >= 1.0 + rprs: + return 0.0 + if b <= abs(1.0 - rprs): + return float(min(rprs ** 2, 1.0)) + if b <= 0: + return float(min(rprs ** 2, 1.0)) + + star_radius = 1.0 + planet_radius = rprs + cos_star = np.clip( + (b ** 2 + star_radius ** 2 - planet_radius ** 2) / (2.0 * b * star_radius), + -1.0, + 1.0, + ) + cos_planet = np.clip( + (b ** 2 + planet_radius ** 2 - star_radius ** 2) / (2.0 * b * planet_radius), + -1.0, + 1.0, + ) + overlap = ( + star_radius ** 2 * math.acos(cos_star) + + planet_radius ** 2 * math.acos(cos_planet) + - 0.5 * math.sqrt( + max( + 0.0, + (-b + star_radius + planet_radius) + * (b + star_radius - planet_radius) + * (b - star_radius + planet_radius) + * (b + star_radius + planet_radius), + ) + ) + ) + return float(np.clip(overlap / math.pi, 0.0, 1.0)) + + +def transit_depth_evaluation_times(parameters, sample_count=2000): + values = complete_transit_parameters(parameters) + tmid = finite_float(values.get("tmid"), 0.0) + period = finite_float(values.get("per")) + duration = transit_duration_days(values) + if np.isfinite(duration) and duration > 0: + half_window = duration + elif np.isfinite(period) and period > 0: + half_window = min(0.2, 0.1 * period) + else: + half_window = 0.2 + half_window = max(float(half_window), 1.0e-4) + return np.linspace(tmid - half_window, tmid + half_window, int(sample_count)) + + +def _load_transit_model(): + try: + from .api.elca import transit + except Exception: + try: + from api.elca import transit + except Exception: + return None + return transit + + +def _depth_fraction_from_model_flux(model_flux): + try: + flux = np.asarray(model_flux, dtype=float).reshape(-1) + except (TypeError, ValueError): + return np.nan + finite = flux[np.isfinite(flux)] + if finite.size == 0: + return np.nan + return float(max(0.0, 1.0 - np.nanmin(finite))) + + +def observable_depth_fraction(parameters, model_flux=None, times=None): + values = complete_transit_parameters(parameters) + if not all(np.isfinite(finite_float(values.get(key))) for key in _REQUIRED_TRANSIT_KEYS): + fallback_depth = _depth_fraction_from_model_flux(model_flux) + return fallback_depth if np.isfinite(fallback_depth) else geometric_observable_depth_fraction(values) + + transit_model = _load_transit_model() + if transit_model is not None: + try: + if times is None: + times = transit_depth_evaluation_times(values) + flux = transit_model(np.asarray(times, dtype=float), values) + depth = _depth_fraction_from_model_flux(flux) + if np.isfinite(depth): + return depth + except Exception: + pass + + fallback_depth = _depth_fraction_from_model_flux(model_flux) + if np.isfinite(fallback_depth): + return fallback_depth + return geometric_observable_depth_fraction(values) + + +def _perturbed_value(key, value): + value = finite_float(value) + if not np.isfinite(value): + return np.nan + if key in ("rprs", "ars", "per"): + return max(value, np.finfo(float).eps) + if key == "ecc": + return float(np.clip(value, 0.0, 0.999999)) + if key == "inc": + return float(np.clip(value, 0.0, 180.0)) + return value + + +def observable_depth_uncertainty_fraction(parameters, errors): + values = complete_transit_parameters(parameters) + errors = errors or {} + contributions = [] + for key in _DEPTH_ERROR_KEYS: + center = finite_float(values.get(key)) + error = abs(finite_float(errors.get(key))) + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + continue + + lower_values = dict(values) + upper_values = dict(values) + lower_values[key] = _perturbed_value(key, center - error) + upper_values[key] = _perturbed_value(key, center + error) + lower_depth = observable_depth_fraction(lower_values) + upper_depth = observable_depth_fraction(upper_values) + if np.isfinite(lower_depth) and np.isfinite(upper_depth): + contributions.append(0.5 * abs(upper_depth - lower_depth)) + + if not contributions: + return np.nan + return float(np.sqrt(np.sum(np.square(contributions)))) + + +def observable_depth_percent(parameters, errors=None, model_flux=None, times=None): + depth_fraction = observable_depth_fraction(parameters, model_flux=model_flux, times=times) + if not np.isfinite(depth_fraction): + return np.nan, np.nan + error_fraction = observable_depth_uncertainty_fraction(parameters, errors or {}) + return ( + float(100.0 * depth_fraction), + float(100.0 * error_fraction) if np.isfinite(error_fraction) else np.nan, + ) + + +def fit_transit_depth_summary(fit, prior_parameters=None, prior_errors=None): + parameters = dict(getattr(fit, "parameters", {}) or {}) + errors = dict(getattr(fit, "errors", {}) or {}) + model_flux = getattr(fit, "transit_upsample", None) + times = getattr(fit, "time_upsample", None) + + area_depth, area_error = radius_ratio_area_depth_percent( + parameters.get("rprs"), + errors.get("rprs"), + ) + observable_depth, observable_error = observable_depth_percent( + parameters, + errors, + model_flux=model_flux, + times=times, + ) + + if prior_parameters is None: + prior_parameters = getattr(fit, "prior", None) + prior_depth = np.nan + prior_error = np.nan + if prior_parameters: + prior_depth, prior_error = observable_depth_percent(prior_parameters, prior_errors or {}) + + delta = np.nan + delta_error = np.nan + if np.isfinite(observable_depth) and np.isfinite(prior_depth): + delta = float(observable_depth - prior_depth) + if np.isfinite(observable_error) and np.isfinite(prior_error): + delta_error = float(np.hypot(observable_error, prior_error)) + + return { + "area_depth": area_depth, + "area_depth_error": area_error, + "observable_depth": observable_depth, + "observable_depth_error": observable_error, + "prior_observable_depth": prior_depth, + "prior_observable_depth_error": prior_error, + "observable_depth_prior_delta": delta, + "observable_depth_prior_delta_error": delta_error, + } diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 9a1765eb..4b369885 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -3,7 +3,17 @@ import numpy as np import pytest -from exotic.output_files import AIDOutputFiles, OutputFiles, fit_impact_parameter_value_error, save_comp_star_calibration_summary +from exotic.output_files import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + AIDOutputFiles, + OutputFiles, + fit_impact_parameter_value_error, + save_comp_star_calibration_summary, +) +from exotic.transit_depth import observable_depth_percent, radius_ratio_area_depth_percent class DummyFit: @@ -12,9 +22,14 @@ def __init__(self): "tmid": 2450000.123456, "rprs": 0.1234, "ars": 12.0, + "per": 2.15, "inc": 88.5, "ecc": 0.0, "omega": 90.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, "a1": 1.0, "a2": 0.0, } @@ -27,10 +42,27 @@ def __init__(self): "a2": 0.1, } self.time = [2450000.123456] + self.time_upsample = np.linspace(2450000.0, 2450000.2, 128) self.data = [1.0] self.dataerr = [0.01] self.residuals = 0.01 self.airmass_model = [1.0] + self.transit = [1.0 - self.parameters["rprs"] ** 2] + self.transit_upsample = np.ones_like(self.time_upsample) + self.transit_upsample[64] = 1.0 - self.parameters["rprs"] ** 2 + self.prior = { + "tmid": 2450000.123456, + "rprs": 0.1, + "ars": 12.0, + "per": 2.15, + "inc": 88.5, + "ecc": 0.0, + "omega": 90.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + } def aavso_json_header(output_text, header_name): @@ -41,6 +73,31 @@ def aavso_json_header(output_text, header_name): raise AssertionError(f"Missing {header_name} header") +def test_observable_depth_is_separate_from_area_depth_for_grazing_geometry(): + parameters = { + "tmid": 0.0, + "rprs": 0.2, + "per": 3.0, + "ars": 10.0, + "inc": np.degrees(np.arccos(1.1 / 10.0)), + "ecc": 0.0, + "omega": 90.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + } + errors = {"rprs": 0.01, "ars": 0.1, "inc": 0.1} + + area_depth, area_error = radius_ratio_area_depth_percent(parameters["rprs"], errors["rprs"]) + observable_depth, observable_error = observable_depth_percent(parameters, errors) + + assert area_depth == pytest.approx(4.0) + assert area_error == pytest.approx(0.4) + assert 0.0 < observable_depth < area_depth + assert observable_error > 0.0 + + def test_aavso_output_includes_observatory_location_headers(tmp_path): fit = DummyFit() p_dict = { @@ -429,6 +486,11 @@ def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertaintie assert final_params["Mid-Transit Time (Tmid)"].endswith("+/- 0.0001 BJD_TDB") assert final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] == "0.1234 +/- 0.001" + assert "Transit depth (Rp/Rs)^2" not in final_params + assert AREA_DEPTH_LABEL in final_params + assert OBSERVABLE_DEPTH_LABEL in final_params + assert PRIOR_OBSERVABLE_DEPTH_LABEL in final_params + assert OBSERVABLE_DEPTH_DELTA_LABEL in final_params assert final_params["Orbital Inclination (inc)"] == "88.5 +/- 0.2 " assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.0 +/- 0.4" assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" @@ -942,7 +1004,11 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): results = aavso_json_header(output_text, "RESULTS-XC") assert "a/R*" in results assert "Impact Parameter (b)" in results - assert results["Transit depth (Rp/R*)^2"]["units"] == "percent" + assert "Transit depth (Rp/R*)^2" not in results + assert results[AREA_DEPTH_LABEL]["units"] == "percent" + assert results[OBSERVABLE_DEPTH_LABEL]["units"] == "percent" + assert results[PRIOR_OBSERVABLE_DEPTH_LABEL]["units"] == "percent" + assert results[OBSERVABLE_DEPTH_DELTA_LABEL]["units"] == "percent" assert results["Residual scatter around full model fit"]["value"] == "0.32" qc = aavso_json_header(output_text, "QC-XC") From cabac52a086b12ea86bb78e6ace2d600ded6e3fc Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 27 May 2026 12:25:50 +1000 Subject: [PATCH 057/116] remove rp/rs fail threshold --- exotic/api/elca.py | 123 +++++++++++++++- exotic/exotic.py | 219 +++++++++++++++++++---------- tests/test_elca_baseline.py | 28 ++++ tests/test_exotic_proper_motion.py | 202 ++++++++++++++++++++++++-- tests/test_exotic_rprs_retry.py | 110 +++++++++++++++ 5 files changed, 592 insertions(+), 90 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 026e3f4a..c60613ff 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -1703,6 +1703,8 @@ def _triangle_plot_display_estimate( plot_range=None, weights=None, min_informative_peak_ratio=1.5, + bins=None, + force_histogram_mode=False, ): sample_values = np.asarray(sample_values, dtype=float) finite_mask = np.isfinite(sample_values) @@ -1734,7 +1736,10 @@ def _triangle_plot_display_estimate( bounds = [float(np.nanmin(finite_values)), float(np.nanmax(finite_values))] if np.all(np.isfinite(bounds)) and bounds[0] < bounds[1]: - bins = int(np.clip(np.sqrt(finite_values.size), 10, 80)) + if bins is None: + bins = int(np.clip(np.sqrt(finite_values.size), 10, 80)) + else: + bins = max(1, int(bins)) counts, edges = np.histogram( finite_values, bins=max(1, bins), @@ -1746,15 +1751,21 @@ def _triangle_plot_display_estimate( peak = float(np.nanmax(positive_counts)) typical = float(np.nanmedian(positive_counts)) total = float(np.nansum(positive_counts)) - if ( + informative_peak = ( np.isfinite(peak) and np.isfinite(typical) and np.isfinite(total) and total > 0 - and typical > 0 - and peak >= min_informative_peak_ratio * typical - and peak >= 0.05 * total - ): + and ( + force_histogram_mode + or ( + typical > 0 + and peak >= min_informative_peak_ratio * typical + and peak >= 0.05 * total + ) + ) + ) + if informative_peak: mode_index = int(np.argmax(counts)) estimate = float(0.5 * (edges[mode_index] + edges[mode_index + 1])) @@ -1785,6 +1796,25 @@ def _triangle_plot_display_estimate( return float(estimate), error + def _visible_triangle_plot_values(self, values, plot_range, weights=None): + values = np.asarray(values, dtype=float) + try: + lower, upper = [float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + finite_mask = np.isfinite(values) + return values[finite_mask], None + + finite_mask = np.isfinite(values) + if np.isfinite(lower) and np.isfinite(upper) and lower < upper: + finite_mask &= (values >= lower) & (values <= upper) + + visible_weights = None + if weights is not None: + weights = np.asarray(weights, dtype=float) + if weights.shape == values.shape: + visible_weights = weights[finite_mask] + return values[finite_mask], visible_weights + def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bins=None): diagnostics = { 'key': key, @@ -2489,6 +2519,86 @@ def _triangle_plot_sigma_window_ranges(self, payload, sigma): return zoomed_ranges + def _recenter_triangle_plot_payload_for_visible_ranges(self, payload): + display_points = np.asarray(payload.get('display_points', []), dtype=float) + if display_points.ndim != 2 or display_points.shape[1] == 0: + return payload + + updated = dict(payload) + titles = list(payload.get('titles', [])) + truths = list(payload.get('truths', [])) + mask_centers = list(payload.get('mask_centers', [])) + mask_errors = list(payload.get('mask_errors', [])) + ranges = list(payload.get('ranges', [])) + sampled_keys = list(payload.get('sampled_keys', [])) + + display_weights = payload.get('display_weights') + if display_weights is not None: + display_weights = np.asarray(display_weights, dtype=float) + if display_weights.ndim != 1 or display_weights.shape[0] != display_points.shape[0]: + display_weights = None + + plot_bins = int(max(1, np.sqrt(display_points.shape[0]))) + display_spec = payload.get('display_spec') + geometry_summary = payload.get('geometry_summary') or {} + + for i, key in enumerate(sampled_keys): + if i >= display_points.shape[1] or i >= len(ranges): + continue + if ( + display_spec is not None + and key == display_spec.get('key') + and display_spec.get('mirror', False) + ): + continue + + visible_values, visible_weights = self._visible_triangle_plot_values( + display_points[:, i], + ranges[i], + weights=display_weights, + ) + if visible_values.size < 2: + continue + + fallback_center = truths[i] if i < len(truths) else np.nan + if fallback_center is None or not np.isfinite(fallback_center): + fallback_center = mask_centers[i] if i < len(mask_centers) else np.nan + fallback_error = mask_errors[i] if i < len(mask_errors) else np.nan + center, error = self._triangle_plot_display_estimate( + visible_values, + fallback_center, + fallback_error, + plot_range=ranges[i], + weights=visible_weights, + bins=plot_bins, + force_histogram_mode=True, + ) + if not np.isfinite(center): + continue + + if i < len(truths): + truths[i] = center + if i < len(mask_centers): + mask_centers[i] = center + if i < len(mask_errors): + mask_errors[i] = error + if i < len(titles): + if display_spec is not None and key == display_spec.get('key'): + inc_center = geometry_summary.get('inc_center') + inc_error = geometry_summary.get('inc_error') + titles[i] = ( + f"b={self._format_triangle_plot_geometry_value(center, error)}\n" + f"i={self._format_triangle_plot_geometry_value(inc_center, inc_error, ' deg')}" + ) + else: + titles[i] = self._format_triangle_plot_parameter_title(center, error) + + updated['titles'] = titles + updated['truths'] = truths + updated['mask_centers'] = mask_centers + updated['mask_errors'] = mask_errors + return updated + def _triangle_contour_levels(self, chi2, mask1, mask2, mask3): raw_levels = np.array([ np.percentile(chi2[mask1], 95), @@ -3118,6 +3228,7 @@ def plot_triangle(self, plot_title=None, zoom_sigma=None): if zoom_sigma is not None: payload = dict(payload) payload['ranges'] = self._triangle_plot_sigma_window_ranges(payload, zoom_sigma) + payload = self._recenter_triangle_plot_payload_for_visible_ranges(payload) chi2 = payload['display_logl'] * -2 parameter_count = max(1, len(payload['sampled_keys'])) diff --git a/exotic/exotic.py b/exotic/exotic.py index 4cda5c28..275228f8 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -238,7 +238,7 @@ LIGHTCURVE_MIN_VALID_POINTS = 5 COMPARISON_STAR_MIN_COVERAGE_FRACTION = 0.8 COMPARISON_STAR_MIN_VALID_FRAMES = 5 -COMPARISON_STAR_COVERAGE_SIGMA = 3.0 +COMPARISON_STAR_COVERAGE_SIGMA = 3.0 # Legacy constant; coverage rejection is fraction-based. COMPARISON_STAR_COVERAGE_MAX_ITERS = 10 COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA = 4.25 COMPARISON_STAR_SUITABILITY_MIN_CANDIDATES = 5 @@ -334,8 +334,8 @@ ) TRANSIT_QC_DELTA_BIC_FAIL_THRESHOLD = 6.0 TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD = 10.0 -TRANSIT_QC_MIN_RPRS_SIGMA = 3.0 -TRANSIT_QC_MARGINAL_RPRS_SIGMA = 5.0 +TRANSIT_QC_KTMF_FAIL_THRESHOLD = 2.5 +TRANSIT_QC_KTMF_PASS_THRESHOLD = 3.5 TRANSIT_QC_MIN_EEBLS_SNR = 4.0 TRANSIT_QC_DURATION_RATIO_MIN = 0.5 TRANSIT_QC_DURATION_RATIO_MAX = 2.0 @@ -346,7 +346,6 @@ 'model_evidence': 0.8, 'deviation_from_expected_value': 1.5, 'residual_scatter': 0.7, - 'rprs_significance': 0.5, 'duration_consistency': 0.75, 'eebls_depth_snr': 0.75, } @@ -1008,16 +1007,6 @@ def compute_transit_qc_ktmf(summary): else "n/a" ), }, - { - 'key': 'rprs_significance', - 'label': 'Rp/R* Significance', - 'score': transit_qc_saturating_score(summary.get('rprs_sigma', np.nan), TRANSIT_QC_MIN_RPRS_SIGMA), - 'detail': ( - f"{summary.get('rprs_sigma', np.nan):.2f} sigma" - if np.isfinite(summary.get('rprs_sigma', np.nan)) - else "n/a" - ), - }, { 'key': 'duration_consistency', 'label': 'Duration Consistency', @@ -1453,34 +1442,21 @@ def evaluate_transit_detection_qc(fit): notes.append("The transit model is strongly preferred over the flat/null model.") if np.isfinite(summary['rprs_sigma']): - if summary['rprs_sigma'] < TRANSIT_QC_MIN_RPRS_SIGMA: - status = 'fail' - notes.append( - f"The fitted transit depth is only {summary['rprs_sigma']:.2f}-sigma." - ) - failure_reasons.append( - f"the fitted transit depth is only {summary['rprs_sigma']:.2f}-sigma" - ) - elif summary['rprs_sigma'] < TRANSIT_QC_MARGINAL_RPRS_SIGMA and status == 'pass': - status = 'marginal' - notes.append( - f"The fitted transit depth is only {summary['rprs_sigma']:.2f}-sigma." - ) + notes.append( + f"Rp/R* fit precision diagnostic: {summary['rprs_sigma']:.2f}-sigma " + "(not used as a transit-detection veto)." + ) if np.isfinite(summary['duration_ratio']): if ( summary['duration_ratio'] < TRANSIT_QC_DURATION_RATIO_MIN or summary['duration_ratio'] > TRANSIT_QC_DURATION_RATIO_MAX ): - if status == 'pass': - status = 'marginal' notes.append( f"The measured transit duration is {summary['duration_ratio']:.2f}x the modeled duration." ) if np.isfinite(summary['eebls_depth_snr']) and summary['eebls_depth_snr'] < TRANSIT_QC_MIN_EEBLS_SNR: - if status == 'pass': - status = 'marginal' notes.append( f"EEBLS only found a weak box-like event (depth SNR={summary['eebls_depth_snr']:.2f})." ) @@ -1488,25 +1464,38 @@ def evaluate_transit_detection_qc(fit): if use_deviation_from_expected_transit_in_qc: notes.extend(deviation_summary.get('notes', [])) if deviation_summary.get('failed'): - status = 'fail' - detailed_reasons = [ - reason for reason in deviation_summary.get('failure_reasons', []) - if isinstance(reason, str) and reason.strip() - ] - if detailed_reasons: - failure_reasons.extend(detailed_reasons) - else: - notes.append( - "The fit deviates too far from the expected published Rp/R* value." - ) - failure_reasons.append( - "the fit deviates too far from the expected published Rp/R* value" - ) + notes.append( + "The fit deviates far from the expected published Rp/R* value; " + "this now contributes through KTMF rather than acting as a hard QC veto." + ) ktmf_metric, ktmf_contributions = compute_transit_qc_ktmf(summary) summary['ktmf_metric'] = ktmf_metric summary['ktmf_contributions'] = ktmf_contributions + if np.isfinite(ktmf_metric): + if status != 'fail' and ktmf_metric < TRANSIT_QC_KTMF_FAIL_THRESHOLD: + status = 'fail' + notes.append( + f"KTMF is {ktmf_metric:.2f}/5.00, below the fail threshold " + f"of {TRANSIT_QC_KTMF_FAIL_THRESHOLD:.2f}." + ) + failure_reasons.append( + f"KTMF is {ktmf_metric:.2f}/5.00, below the fail threshold " + f"of {TRANSIT_QC_KTMF_FAIL_THRESHOLD:.2f}" + ) + elif status == 'pass' and ktmf_metric < TRANSIT_QC_KTMF_PASS_THRESHOLD: + status = 'marginal' + notes.append( + f"KTMF is {ktmf_metric:.2f}/5.00, below the pass threshold " + f"of {TRANSIT_QC_KTMF_PASS_THRESHOLD:.2f}." + ) + elif status == 'marginal' and ktmf_metric < TRANSIT_QC_KTMF_PASS_THRESHOLD: + notes.append( + f"KTMF is {ktmf_metric:.2f}/5.00, below the pass threshold " + f"of {TRANSIT_QC_KTMF_PASS_THRESHOLD:.2f}." + ) + if status == 'pass': summary_text = f"Transit model strongly preferred over flat/null model ({comparison_text})." elif status == 'marginal': @@ -2688,8 +2677,17 @@ def refit_selected_fast_comparison_on_full_lightcurve( if detrend_result.get('applied'): fit_flux = np.asarray(detrend_result['flux'], dtype=float) fit_unc = np.asarray(detrend_result['unc'], dtype=float) + prior['a0'] = 1.0 + prior['a1'] = 1.0 + prior['a2'] = 0.0 fixed_errors = baseline_fixed_errors_from_fit(previous_fit) + if detrend_result.get('applied'): + fixed_errors = { + 'a0': fixed_errors.get('a0', 0.0), + 'a1': fixed_errors.get('a1', fixed_errors.get('a0', 0.0)), + 'a2': fixed_errors.get('a2', 0.0), + } base_live_points, target_live_points = selected_final_live_point_target( sparse_live_point_extension_enabled, ) @@ -2713,9 +2711,14 @@ def refit_selected_fast_comparison_on_full_lightcurve( ) log_expected_transit_coverage_assessment(pre_ultranest_coverage_assessment) + fixed_baseline_source = ( + "with a flat fixed baseline after out-of-transit detrending" + if detrend_result.get('applied') + else "with fixed a0/a2 from the previous fast UltraNest fit" + ) log_info( "Running the selected comparison-star final UltraNest fit on the full-resolution light curve " - f"with fixed a0/a2 from the previous fast UltraNest fit at {min_live_points} minimum live points." + f"{fixed_baseline_source} at {min_live_points} minimum live points." ) fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( times, @@ -2739,10 +2742,16 @@ def refit_selected_fast_comparison_on_full_lightcurve( annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) + baseline_parameter_note = ( + "Full-resolution out-of-transit linear detrending flattened the final-fit light curve; " + "fixed the final baseline to a0=1 and a2=0 instead of reusing the previous fast-fit baseline scale." + if detrend_result.get('applied') + else "Used a0 and a2 from the previous fast UltraNest fit for the full-resolution final run." + ) annotate_out_of_transit_baseline_parameter_fit( fit, - True, - note="Used a0 and a2 from the previous fast UltraNest fit for the full-resolution final run.", + not bool(detrend_result.get('applied')), + note=baseline_parameter_note, pre_points=0, post_points=0, a0=prior.get('a0'), @@ -2777,8 +2786,8 @@ def refit_selected_fast_comparison_on_full_lightcurve( True, note=( "Applied full-resolution selected comparison-star final UltraNest run " - f"({base_live_points}->{target_live_points} minimum live points) using fixed a0/a2 " - "from the previous fast UltraNest fit." + f"({base_live_points}->{target_live_points} minimum live points) " + f"{fixed_baseline_source}." ), diagnostics=diagnostics, post_extension_diagnostics=diagnostics, @@ -7532,7 +7541,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( baseline_constrained_prior = dict(working_prior) baseline_constrained_bounds = clone_lightcurve_bounds(working_bounds) if baseline_parameter_result.get('applied'): - log_info("Prepared out-of-transit airmass/baseline parameter constraints for the final transit refit.") + log_info("Prepared out-of-transit airmass/baseline parameter constraints for a fallback final transit refit.") log_info(baseline_parameter_result['note']) baseline_fit_mask = np.asarray(baseline_parameter_result['oot_mask'], dtype=bool) baseline_fixed_errors = { @@ -7669,13 +7678,15 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): detrend_result['flux'], disable_vertical_flux_normalization, ) + if 'a2' not in refit_bounds: + refit_prior['a2'] = 0.0 + baseline_parameter_fit_note = baseline_parameter_result.get('note') + baseline_parameter_fit_used = False if baseline_parameter_result.get('applied'): - refit_prior['a0'] = baseline_parameter_result['a0'] - refit_prior['a1'] = baseline_parameter_result['a0'] - refit_prior['a2'] = baseline_parameter_result['a2'] - refit_bounds.pop('a0', None) - refit_bounds.pop('a1', None) - refit_bounds.pop('a2', None) + baseline_parameter_fit_note = ( + "Not used in the final refit because the out-of-transit linear detrending " + "already flattened the final-fit flux baseline." + ) refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( working_times, @@ -7690,7 +7701,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): keep_ultranest_sampler=keep_ultranest_for_sparse_extension, baseline_fit_mask=baseline_fit_mask, fixed_parameter_errors=baseline_fixed_errors, - fixed_flux_baseline=bool(baseline_parameter_result.get('applied')), + fixed_flux_baseline=baseline_parameter_fit_used, pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) @@ -7726,14 +7737,14 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): ) annotate_out_of_transit_baseline_parameter_fit( refit, - bool(baseline_parameter_result.get('applied')), - note=baseline_parameter_result.get('note'), + baseline_parameter_fit_used, + note=baseline_parameter_fit_note, pre_points=baseline_parameter_result.get('pre_points', 0), post_points=baseline_parameter_result.get('post_points', 0), - a0=baseline_parameter_result.get('a0'), - a0_error=baseline_parameter_result.get('a0_error'), - a2=baseline_parameter_result.get('a2'), - a2_error=baseline_parameter_result.get('a2_error'), + a0=baseline_parameter_result.get('a0') if baseline_parameter_fit_used else None, + a0_error=baseline_parameter_result.get('a0_error') if baseline_parameter_fit_used else None, + a2=baseline_parameter_result.get('a2') if baseline_parameter_fit_used else None, + a2_error=baseline_parameter_result.get('a2_error') if baseline_parameter_fit_used else None, ) annotate_transit_detection_qc(refit) if extend_sparse_posterior_live_points: @@ -14637,6 +14648,42 @@ def format_comp_star_coverage_text(summary): return coverage_text +def format_comp_star_coverage_rejection_detail(summary): + coverage_count = int(summary.get('coverage_count', 0) or 0) + threshold_values = [] + for threshold in ( + summary.get('coverage_rejection_threshold_count'), + summary.get('coverage_min_required_count'), + ): + try: + numeric_threshold = float(threshold) + except (TypeError, ValueError): + continue + if np.isfinite(numeric_threshold): + threshold_values.append(int(np.ceil(numeric_threshold))) + + threshold_count = max(threshold_values) if threshold_values else None + if threshold_count is None: + detail_parts = [f"{coverage_count} valid frame(s)"] + elif coverage_count < threshold_count: + detail_parts = [f"{coverage_count} < {threshold_count} valid frame(s)"] + else: + detail_parts = [f"{coverage_count} valid frame(s); rejection threshold={threshold_count}"] + + coverage_median = summary.get( + 'coverage_rejection_reference_count', + summary.get('coverage_reference_count', np.nan), + ) + try: + numeric_coverage_median = float(coverage_median) + except (TypeError, ValueError): + numeric_coverage_median = np.nan + if np.isfinite(numeric_coverage_median): + detail_parts.append(f"peer median={numeric_coverage_median:.1f}") + + return "; ".join(detail_parts) + + def format_eebls_snr(value): if value is None: return "n/a" @@ -15306,7 +15353,7 @@ def comparison_calibration_selection_reason(summary, best_comp_score): if summary.get('coverage_rejected'): return ( "not selected: low coverage " - f"({summary['coverage_count']} < {summary['coverage_min_required_count']} valid frames)" + f"({format_comp_star_coverage_rejection_detail(summary)})" ) if summary.get('suitability_outlier_rejected'): @@ -15580,6 +15627,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p coverage_reference_count = coverage_summary[ckey]['coverage_reference_count'] coverage_min_required_count = coverage_summary[ckey]['coverage_min_required_count'] coverage_rejected = coverage_summary[ckey]['coverage_rejected'] + coverage_rejection_detail = format_comp_star_coverage_rejection_detail(coverage_summary[ckey]) fit_result, target_fit_flux, comp_fit_flux = None, None, None fit_diagnostics = { 'input_point_count': int(times.shape[0]), @@ -15595,8 +15643,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p if coverage_rejected: fit_diagnostics['failure_reason'] = ( "comparison candidate rejected after iterative low-coverage clipping " - f"({coverage_count} < {coverage_min_required_count} valid frame(s); " - f"peer median={coverage_reference_count:.1f})." + f"({coverage_rejection_detail})." ) elif coverage_count > 1: fit_diagnostics = diagnose_lightcurve_fit_inputs( @@ -15650,6 +15697,10 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p 'coverage_reference_count': coverage_reference_count, 'coverage_min_required_count': coverage_min_required_count, 'coverage_rejected': coverage_rejected, + 'coverage_rejection_threshold_count': coverage_summary[ckey].get('coverage_rejection_threshold_count'), + 'coverage_rejection_reference_count': coverage_summary[ckey].get('coverage_rejection_reference_count'), + 'coverage_rejection_scatter': coverage_summary[ckey].get('coverage_rejection_scatter'), + 'coverage_rejection_iteration': coverage_summary[ckey].get('coverage_rejection_iteration'), 'fit_point_count': fit_point_count, 'fit_diagnostics': fit_diagnostics, 'failure_reason': fit_diagnostics.get('failure_reason'), @@ -15719,32 +15770,42 @@ def comparison_star_coverage_summary(comp_flux_map, coverage_reference_count = float(np.nanmedian([coverage_counts[key] for key in active_keys])) coverage_min_required_count = max(effective_min_points, 0) coverage_scatter = np.nan + coverage_rejection_info = {} - for _ in range(COMPARISON_STAR_COVERAGE_MAX_ITERS): + for iteration_index in range(COMPARISON_STAR_COVERAGE_MAX_ITERS): active_counts = np.asarray([coverage_counts[key] for key in active_keys], dtype=float) if active_counts.size == 0: break coverage_reference_count = float(np.nanmedian(active_counts)) coverage_scatter = robust_scatter(active_counts) - threshold_candidates = [ + # Coverage is only an availability gate. High-scatter comparison stars + # are handled by the suitability outlier pass after coverage-qualified + # stars have been scored. + coverage_min_required_count = max( effective_min_points, int(np.ceil(float(min_fraction) * coverage_reference_count)), - ] - if np.isfinite(coverage_scatter) and coverage_scatter > 0: - threshold_candidates.append( - int(np.ceil(coverage_reference_count - COMPARISON_STAR_COVERAGE_SIGMA * coverage_scatter)) - ) - coverage_min_required_count = max(threshold_candidates) + ) kept_keys = [key for key in active_keys if coverage_counts[key] >= coverage_min_required_count] if len(kept_keys) == len(active_keys): break + kept_key_set = set(kept_keys) + for key in active_keys: + if key in kept_key_set or key in coverage_rejection_info: + continue + coverage_rejection_info[key] = { + 'coverage_rejection_threshold_count': coverage_min_required_count, + 'coverage_rejection_reference_count': coverage_reference_count, + 'coverage_rejection_scatter': coverage_scatter, + 'coverage_rejection_iteration': iteration_index + 1, + } active_keys = kept_keys coverage_summary = {} active_key_set = set(active_keys) for key in comp_keys: + rejection_info = coverage_rejection_info.get(key, {}) coverage_summary[key] = { 'coverage_count': coverage_counts[key], 'coverage_total_frame_count': total_frame_count, @@ -15753,6 +15814,10 @@ def comparison_star_coverage_summary(comp_flux_map, 'coverage_scatter': coverage_scatter, 'coverage_min_required_count': coverage_min_required_count, 'coverage_rejected': False if skip_rejection else key not in active_key_set, + 'coverage_rejection_threshold_count': rejection_info.get('coverage_rejection_threshold_count'), + 'coverage_rejection_reference_count': rejection_info.get('coverage_rejection_reference_count'), + 'coverage_rejection_scatter': rejection_info.get('coverage_rejection_scatter'), + 'coverage_rejection_iteration': rejection_info.get('coverage_rejection_iteration'), } return coverage_summary @@ -16179,6 +16244,10 @@ def build_stability_iteration(active_keys, frame_keep_mask=None): 'coverage_reference_count': coverage_summary[key]['coverage_reference_count'], 'coverage_min_required_count': coverage_summary[key]['coverage_min_required_count'], 'coverage_rejected': coverage_summary[key]['coverage_rejected'], + 'coverage_rejection_threshold_count': coverage_summary[key].get('coverage_rejection_threshold_count'), + 'coverage_rejection_reference_count': coverage_summary[key].get('coverage_rejection_reference_count'), + 'coverage_rejection_scatter': coverage_summary[key].get('coverage_rejection_scatter'), + 'coverage_rejection_iteration': coverage_summary[key].get('coverage_rejection_iteration'), 'suitability_outlier_rejected': False, 'suitability_reference_score': np.nan, 'suitability_scatter': np.nan, diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 7913d376..89ff38aa 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -1387,6 +1387,34 @@ def fake_corner(*args, **kwargs): assert captured["range"][1][1] < 1.2 +def test_triangle_payload_recenter_uses_visible_zoom_peak(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + ars_values = np.concatenate([ + np.linspace(5.22, 5.30, 60), + np.linspace(6.45, 6.55, 20), + np.linspace(8.0, 9.0, 20), + ]) + payload = { + "sampled_keys": ["ars"], + "display_points": ars_values[:, None], + "display_weights": None, + "ranges": [[5.0, 7.0]], + "titles": ["6.0 +/- 1.0"], + "truths": [6.0], + "mask_centers": [6.0], + "mask_errors": [1.0], + "display_spec": None, + "geometry_summary": {}, + } + + updated = fit._recenter_triangle_plot_payload_for_visible_ranges(payload) + + assert updated["truths"][0] == pytest.approx(5.3) + assert updated["mask_centers"][0] == pytest.approx(5.3) + assert updated["titles"][0].startswith("5.3 +/-") + + def test_triangle_payload_expands_degenerate_error_ranges_to_sample_cloud(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 4fd5673c..4560eada 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -1765,20 +1765,21 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): assert "Model Evidence" in contributions_by_label assert "Delta BIC" not in contributions_by_label assert "Delta chi2" not in contributions_by_label - assert contributions_by_label["Model Evidence"]["max_points"] == pytest.approx(0.8) - assert contributions_by_label["Deviation From Expected Value"]["max_points"] == pytest.approx(1.5) - assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7) - assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75) - assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(0.75) + scale = 5.0 / (0.8 + 1.5 + 0.7 + 0.75 + 0.75) + assert contributions_by_label["Model Evidence"]["max_points"] == pytest.approx(0.8 * scale) + assert contributions_by_label["Deviation From Expected Value"]["max_points"] == pytest.approx(1.5 * scale) + assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) + assert "Rp/R* Significance" not in contributions_by_label + assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75 * scale) + assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(0.75 * scale) assert "Rp/R* sigma=2.00" in contributions_by_label["Deviation From Expected Value"]["detail"] assert "Tmid" not in contributions_by_label["Deviation From Expected Value"]["detail"] model_evidence_score = ((1.0 - np.exp(-1.0)) + (1.0 - np.exp(-2.0))) / 2.0 - expected_ktmf = ( + expected_ktmf = scale * ( 0.8 * model_evidence_score + 1.5 * 0.6 + 0.7 * 0.5 - + 0.5 * (1.0 - np.exp(-2.0)) + 0.75 * 1.0 + 0.75 * (1.0 - np.exp(-2.0)) ) @@ -1953,6 +1954,27 @@ def test_comparison_star_coverage_summary_iteratively_rejects_low_count_tail(): assert coverage["comp1"]["coverage_min_required_count"] == 8 +def test_comparison_star_coverage_summary_keeps_nearly_complete_candidates(): + frame_count = 146 + coverage = comparison_star_coverage_summary( + { + **{ + f"comp{comp_index + 1}": np.ones(frame_count, dtype=float) + for comp_index in range(8) + }, + "comp9": np.concatenate([np.ones(145, dtype=float), [np.nan]]), + "comp10": np.concatenate([np.ones(142, dtype=float), np.full(4, np.nan)]), + } + ) + + assert coverage["comp9"]["coverage_count"] == 145 + assert coverage["comp10"]["coverage_count"] == 142 + assert coverage["comp9"]["coverage_min_required_count"] == 117 + assert coverage["comp10"]["coverage_min_required_count"] == 117 + assert coverage["comp9"]["coverage_rejected"] is False + assert coverage["comp10"]["coverage_rejected"] is False + + def test_comparison_star_stability_summary_rejects_low_coverage_candidates(): airmass = np.linspace(1.0, 1.5, 6) summary = comparison_star_stability_summary( @@ -1970,6 +1992,40 @@ def test_comparison_star_stability_summary_rejects_low_coverage_candidates(): assert np.isinf(summary["comp_summaries"][2]["aggregate_score"]) +def test_comparison_star_stability_summary_rejects_noisy_nearly_complete_candidates_as_outliers(): + frame_count = 146 + airmass = np.linspace(1.0, 1.5, frame_count) + phase = np.linspace(0.0, 4.0 * np.pi, frame_count) + stable_flux = 100.0 * (1.0 + 0.001 * np.sin(phase)) + comp_flux_map = { + f"comp{comp_index + 1}": stable_flux * (1.0 + 0.0001 * comp_index) + for comp_index in range(8) + } + noisy_flux = 100.0 * (1.0 + 0.35 * np.sin(np.linspace(0.0, 14.0 * np.pi, frame_count))) + noisy_flux[-1] = np.nan + choppy_flux = 100.0 * (1.0 + 0.25 * np.sign(np.sin(np.linspace(0.0, 20.0 * np.pi, frame_count)))) + choppy_flux[-4:] = np.nan + comp_flux_map["comp9"] = noisy_flux + comp_flux_map["comp10"] = choppy_flux + + summary = comparison_star_stability_summary(comp_flux_map, airmass) + comp9_summary = summary["comp_summaries"][8] + comp10_summary = summary["comp_summaries"][9] + + assert comp9_summary["coverage_count"] == 145 + assert comp10_summary["coverage_count"] == 142 + assert comp9_summary["coverage_rejected"] is False + assert comp10_summary["coverage_rejected"] is False + assert comp9_summary["suitability_outlier_rejected"] is True + assert comp10_summary["suitability_outlier_rejected"] is True + reason = comparison_calibration_selection_reason( + comp9_summary, + summary["best_comp_score"], + ) + assert "high-side sigma clipping" in reason + assert "low coverage" not in reason + + def test_comparison_star_stability_summary_rejects_shared_bad_frame(): airmass = np.linspace(1.0, 1.5, 6) summary = comparison_star_stability_summary( @@ -2144,6 +2200,100 @@ def fake_lc_fitter( assert fit.oot_baseline_post_points == 3 +def test_fit_final_lightcurve_linear_detrend_does_not_reapply_fixed_airmass_baseline(monkeypatch): + import exotic.exotic as exotic_module + + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + transit_profile = np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + flux = (1.03 + 0.02 * times) * transit_profile + fluxerr = np.full_like(times, 0.01) + airmass = np.linspace(1.0, 1.3, times.size) + prior = {"rprs": 0.1, "tmid": 0.0, "inc": 89.0, "a0": 1.03, "a1": 1.03, "a2": 0.2} + bounds = { + "rprs": [0.0, 0.2], + "tmid": [-0.1, 0.1], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + } + captured = {"calls": []} + + def fake_run_nested( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + **kwargs, + ): + captured["calls"].append({ + "flux": np.asarray(call_flux, dtype=float), + "prior": dict(call_prior), + "bounds": dict(call_bounds), + "fixed_flux_baseline": kwargs.get("fixed_flux_baseline"), + }) + return types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + airmass=np.asarray(call_airmass, dtype=float), + transit=transit_profile.copy(), + parameters={"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a0": call_prior.get("a0", 1.0), "a2": 0.2}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.001, "a2": 0.01}, + residuals=np.zeros_like(call_flux, dtype=float), + duration_expected=0.5, + duration_measured=0.5, + ) + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + monkeypatch.setattr( + exotic_module, + "build_final_fit_prefit_refinement_plan", + lambda call_times, call_flux, call_fluxerr, call_airmass, call_prior, call_bounds, fit, **kwargs: { + "applied": False, + "note": "test no prefit refinement", + "times": np.asarray(call_times, dtype=float), + "flux": np.asarray(call_flux, dtype=float), + "unc": np.asarray(call_fluxerr, dtype=float), + "airmass": np.asarray(call_airmass, dtype=float), + "jd_times": None, + "prior": dict(call_prior), + "bounds": dict(call_bounds), + "duration": 0.5, + "original_point_count": len(call_times), + "refined_point_count": len(call_times), + "trimmed_pre_points": 0, + "trimmed_post_points": 0, + "original_tmid_bounds": call_bounds["tmid"], + "refined_tmid_bounds": call_bounds["tmid"], + }, + ) + monkeypatch.setattr(exotic_module, "annotate_transit_detection_qc", lambda fit: None) + + fit, refit_flux, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=True, + extend_sparse_posterior_live_points=False, + ) + + assert len(captured["calls"]) == 2 + final_call = captured["calls"][1] + assert final_call["fixed_flux_baseline"] is False + assert "a0" in final_call["bounds"] + assert "a2" in final_call["bounds"] + assert np.allclose(final_call["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert final_call["flux"][3] == pytest.approx(0.99, abs=1e-8) + assert np.allclose(refit_flux, final_call["flux"]) + assert fit.oot_baseline_parameter_fit_applied is False + assert "already flattened" in fit.oot_baseline_parameter_fit_note + + def test_fit_final_lightcurve_uses_oot_baseline_parameter_refit_when_linear_detrend_skips(monkeypatch): import exotic.exotic as exotic_module @@ -3122,6 +3272,39 @@ def test_evaluate_transit_detection_qc_prefers_transit_model(): assert summary["delta_chi2"] > 0.0 +def test_evaluate_transit_detection_qc_passes_strong_model_with_low_rprs_precision(): + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.20, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["status"] == "pass" + assert summary["rprs_sigma"] == pytest.approx(0.5) + assert summary["ktmf_metric"] >= 3.5 + assert "not used as a transit-detection veto" in " ".join(summary["notes"]) + + def test_evaluate_transit_detection_qc_fails_when_flat_model_is_better(): transit_model = np.ones(21, dtype=float) transit_model[8:13] = 0.99 @@ -3154,7 +3337,7 @@ def test_evaluate_transit_detection_qc_fails_when_flat_model_is_better(): assert summary["delta_chi2"] < 0.0 -def test_evaluate_transit_detection_qc_rejects_large_expected_value_deviation(): +def test_evaluate_transit_detection_qc_marks_large_expected_value_deviation_marginal_via_ktmf(): transit_model = np.ones(21, dtype=float) transit_model[8:13] = 0.99 data = transit_model + np.array( @@ -3187,10 +3370,11 @@ def test_evaluate_transit_detection_qc_rejects_large_expected_value_deviation(): summary = evaluate_transit_detection_qc(fit) assert summary["computed"] is True - assert summary["status"] == "fail" + assert summary["status"] == "marginal" assert summary["rprs_deviation_sigma"] == pytest.approx(8.0) assert summary["deviation_from_expected_value"] == pytest.approx(0.0) assert summary["ktmf_metric"] <= 5.0 + assert summary["ktmf_metric"] < 3.5 assert np.isnan(summary["tmid_deviation_sigma"]) assert np.isnan(summary["tmid_deviation_minutes"]) assert summary["rprs_deviation_fit_unc"] == pytest.approx(0.01) diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index df147a5e..bd335cd3 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -519,6 +519,116 @@ def fake_run_nested( assert "a2" not in captured["bounds"] +def test_selected_fast_candidate_final_refit_resets_fixed_baseline_after_linear_detrend(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + def fake_run_nested( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["flux"] = np.asarray(flux_values, dtype=float) + captured["prior"] = dict(prior) + captured["bounds"] = dict(bounds) + captured["fixed_parameter_errors"] = dict(kwargs.get("fixed_parameter_errors", {})) + captured["fixed_flux_baseline"] = kwargs.get("fixed_flux_baseline") + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters=dict(prior), + errors=dict(kwargs.get("fixed_parameter_errors", {})), + residuals=np.zeros(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + duration_measured=0.2, + duration_expected=0.2, + transit_qc={"status": "pass", "summary": "ok"}, + transit_qc_status="pass", + ) + return fit + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + monkeypatch.setattr(exotic_module, "selected_final_live_point_target", lambda *args, **kwargs: (200, None)) + + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + transit_profile = np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + baseline = 1.03 + 0.02 * times + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + transit=transit_profile, + parameters={ + "rprs": 0.1, + "ars": 10.0, + "per": 1.0, + "tmid": 0.0, + "inc": 89.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.03, + "a1": 1.03, + "a2": 0.12, + }, + errors={"a0": 0.02, "a1": 0.02, "a2": 0.03, "rprs": 0.001, "tmid": 0.001, "ars": 0.1}, + bounds={ + "rprs": [0.05, 0.15], + "tmid": [-0.1, 0.1], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + }, + ) + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": baseline * transit_profile, + "good_unc": np.full(times.shape, 0.01), + "good_airmass": np.linspace(1.0, 1.3, times.size), + "good_jd_times": 2460000.0 + times, + "fast_fit_bounds": previous_fit.bounds, + } + + returned, fit_flux, _ = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + { + "midT": 0.0, + "midTUnc": 0.001, + "pPer": 1.0, + "rprs": 0.1, + "aRs": 10.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + detrend_on_outoftransit_baseline=True, + ) + + assert returned is not None + assert captured["fixed_flux_baseline"] is True + assert captured["prior"]["a0"] == pytest.approx(1.0) + assert captured["prior"]["a1"] == pytest.approx(1.0) + assert captured["prior"]["a2"] == pytest.approx(0.0) + assert captured["fixed_parameter_errors"]["a0"] == pytest.approx(0.02) + assert captured["fixed_parameter_errors"]["a2"] == pytest.approx(0.03) + assert np.allclose(captured["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert captured["flux"][3] == pytest.approx(0.99, abs=1e-8) + assert np.allclose(fit_flux, captured["flux"]) + assert returned.oot_baseline_parameter_fit_applied is False + assert "instead of reusing" in returned.oot_baseline_parameter_fit_note + + def test_rprs_posterior_retry_walks_bounds_until_retry_cap(monkeypatch): import exotic.exotic as exotic_module From 25414e4c2f86c60c901b92981b826830655db5c9 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 27 May 2026 17:26:44 +1000 Subject: [PATCH 058/116] clearer transit depth (and a bug or two) --- exotic/api/ultranest_utils.py | 39 ++++++++++- exotic/exotic.py | 69 +++++++++++++++++-- exotic/inputs.py | 31 ++++++--- exotic/output_files.py | 4 +- exotic/transit_depth.py | 105 +++++++++++++++++++++++++---- tests/test_exotic_proper_motion.py | 22 ++++-- tests/test_output_files.py | 57 +++++++++++++++- tests/test_ultranest_utils.py | 33 +++++++++ 8 files changed, 324 insertions(+), 36 deletions(-) diff --git a/exotic/api/ultranest_utils.py b/exotic/api/ultranest_utils.py index 0ad48373..c7cb3122 100644 --- a/exotic/api/ultranest_utils.py +++ b/exotic/api/ultranest_utils.py @@ -488,6 +488,39 @@ def _mute_ultranest_logging(sampler): logger.disabled = disabled +def _traceback_mentions_ultranest_mlfriends(exc): + traceback = exc.__traceback__ + while traceback is not None: + filename = str(traceback.tb_frame.f_code.co_filename).replace("\\", "/") + if "ultranest/mlfriends" in filename: + return True + traceback = traceback.tb_next + return False + + +def _is_ultranest_degenerate_region_error(exc): + if not isinstance(exc, ValueError): + return False + + message = str(exc) + if "Buffer has wrong number of dimensions" not in message: + return False + if "expected 2" not in message or "got 0" not in message: + return False + return _traceback_mentions_ultranest_mlfriends(exc) + + +def _run_sampler_with_degenerate_region_guard(sampler, kwargs): + try: + return sampler.run(**kwargs) + except ValueError as exc: + if not _is_ultranest_degenerate_region_error(exc): + raise + raise np.linalg.LinAlgError( + "UltraNest failed while building a degenerate sampling region." + ) from exc + + def _read_float(mapping, *keys): for key in keys: if key not in mapping: @@ -660,11 +693,11 @@ def run_reactive_sampler( kwargs["show_status"] = False kwargs["viz_callback"] = False with _mute_ultranest_logging(sampler): - return sampler.run(**kwargs) + return _run_sampler_with_degenerate_region_guard(sampler, kwargs) if mode == "rich": kwargs.setdefault("show_status", True) - return sampler.run(**kwargs) + return _run_sampler_with_degenerate_region_guard(sampler, kwargs) progress = _UltraNestSimpleProgress(stream=stream, interval_seconds=interval_seconds) upstream_callback = kwargs.get("viz_callback") @@ -681,7 +714,7 @@ def callback(*args, **callback_kwargs): progress.start() try: with _mute_ultranest_logging(sampler): - result = sampler.run(**kwargs) + result = _run_sampler_with_degenerate_region_guard(sampler, kwargs) finally: progress.finish() return result diff --git a/exotic/exotic.py b/exotic/exotic.py index 275228f8..c3451cc7 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -342,6 +342,7 @@ TRANSIT_QC_DEFAULT_A2_BOUNDS = (-3.0, 3.0) TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT = True TRANSIT_QC_DEVIATION_SIGMA_DEFAULT = 5.0 +TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION = 0.05 TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS = { 'model_evidence': 0.8, 'deviation_from_expected_value': 1.5, @@ -704,6 +705,28 @@ def transit_qc_deviation_score_from_sigma(sigma_offset, sigma_threshold): return float(max(0.0, 1.0 - sigma_offset / sigma_threshold)) +def transit_qc_rprs_deviation_uncertainty(fitted_rprs_unc, expected_rprs_unc, expected_rprs): + terms = [] + for value in (fitted_rprs_unc, expected_rprs_unc): + value = coerce_finite_transit_qc_scalar(value) + if np.isfinite(value) and value > 0: + terms.append(float(value)) + + systematic_floor = np.nan + expected_rprs = coerce_finite_transit_qc_scalar(expected_rprs) + if np.isfinite(expected_rprs) and expected_rprs > 0: + systematic_floor = float( + TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION * abs(expected_rprs) + ) + if systematic_floor > 0: + terms.append(systematic_floor) + + if not terms: + return np.nan, systematic_floor + + return float(np.sqrt(np.sum(np.square(terms)))), systematic_floor + + def transit_qc_duration_score(duration_ratio): try: duration_ratio = float(duration_ratio) @@ -892,6 +915,9 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T 'tmid_deviation_threshold_minutes': np.nan, 'tmid_deviation_sigma': np.nan, 'rprs_deviation_fit_unc': np.nan, + 'rprs_deviation_expected_unc': np.nan, + 'rprs_deviation_systematic_floor': np.nan, + 'rprs_deviation_unc': np.nan, 'rprs_deviation_sigma': np.nan, 'tmid_deviation_score': np.nan, 'rprs_deviation_score': np.nan, @@ -916,19 +942,30 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T return summary expected_rprs = expected.get('expected_rprs', np.nan) + expected_rprs_unc = expected.get('expected_rprs_unc', np.nan) fitted_rprs = parameters.get('rprs', np.nan) errors = getattr(fit, 'errors', {}) or {} fitted_rprs_unc = errors.get('rprs', np.nan) + comparison_unc, systematic_floor = transit_qc_rprs_deviation_uncertainty( + fitted_rprs_unc, + expected_rprs_unc, + expected_rprs, + ) + summary['expected_rprs'] = expected_rprs + summary['expected_rprs_unc'] = expected_rprs_unc summary['fitted_rprs'] = fitted_rprs summary['fitted_rprs_unc'] = fitted_rprs_unc + summary['rprs_deviation_fit_unc'] = fitted_rprs_unc + summary['rprs_deviation_expected_unc'] = expected_rprs_unc + summary['rprs_deviation_systematic_floor'] = systematic_floor + summary['rprs_deviation_unc'] = comparison_unc if ( np.isfinite(expected_rprs) and np.isfinite(fitted_rprs) - and np.isfinite(fitted_rprs_unc) - and fitted_rprs_unc > 0 + and np.isfinite(comparison_unc) + and comparison_unc > 0 ): - rprs_sigma = float(abs(fitted_rprs - expected_rprs) / fitted_rprs_unc) - summary['rprs_deviation_fit_unc'] = fitted_rprs_unc + rprs_sigma = float(abs(fitted_rprs - expected_rprs) / comparison_unc) summary['rprs_deviation_sigma'] = rprs_sigma summary['rprs_deviation_score'] = transit_qc_deviation_score_from_sigma(rprs_sigma, sigma_threshold) @@ -941,8 +978,10 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T "Expected-value Rp/R* deviation: " f"{summary['rprs_deviation_sigma']:.2f} sigma " f"(fit={summary['fitted_rprs']:.6f} +/- {summary['fitted_rprs_unc']:.6f}, " - f"expected={expected_rprs:.6f}; " - f"fit uncertainty={summary['rprs_deviation_fit_unc']:.6f})." + f"expected={expected_rprs:.6f} +/- {expected_rprs_unc:.6f}; " + f"comparison uncertainty={summary['rprs_deviation_unc']:.6f}, " + f"including {100.0 * TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION:.1f}% " + f"Rp/R* floor={summary['rprs_deviation_systematic_floor']:.6f})." ) rprs_sigma = summary['rprs_deviation_sigma'] @@ -980,6 +1019,15 @@ def compute_transit_qc_ktmf(summary): rprs_fit_unc = summary.get('rprs_deviation_fit_unc', np.nan) if np.isfinite(rprs_fit_unc): deviation_detail_parts.append(f"fit uncertainty={rprs_fit_unc:.6f}") + expected_unc = summary.get('rprs_deviation_expected_unc', summary.get('expected_rprs_unc', np.nan)) + if np.isfinite(expected_unc): + deviation_detail_parts.append(f"expected uncertainty={expected_unc:.6f}") + comparison_unc = summary.get('rprs_deviation_unc', np.nan) + if np.isfinite(comparison_unc): + deviation_detail_parts.append(f"comparison uncertainty={comparison_unc:.6f}") + systematic_floor = summary.get('rprs_deviation_systematic_floor', np.nan) + if np.isfinite(systematic_floor): + deviation_detail_parts.append(f"systematic floor={systematic_floor:.6f}") deviation_detail = ", ".join(deviation_detail_parts) else: deviation_detail = "expected-value deviation disabled or unavailable" @@ -1265,6 +1313,9 @@ def evaluate_transit_detection_qc(fit): 'tmid_deviation_threshold_minutes': np.nan, 'tmid_deviation_sigma': np.nan, 'rprs_deviation_fit_unc': np.nan, + 'rprs_deviation_expected_unc': np.nan, + 'rprs_deviation_systematic_floor': np.nan, + 'rprs_deviation_unc': np.nan, 'rprs_deviation_sigma': np.nan, 'tmid_deviation_score': np.nan, 'rprs_deviation_score': np.nan, @@ -1403,6 +1454,9 @@ def evaluate_transit_detection_qc(fit): 'tmid_deviation_threshold_minutes': deviation_summary.get('tmid_deviation_threshold_minutes', np.nan), 'tmid_deviation_sigma': deviation_summary.get('tmid_deviation_sigma', np.nan), 'rprs_deviation_fit_unc': deviation_summary.get('rprs_deviation_fit_unc', np.nan), + 'rprs_deviation_expected_unc': deviation_summary.get('rprs_deviation_expected_unc', np.nan), + 'rprs_deviation_systematic_floor': deviation_summary.get('rprs_deviation_systematic_floor', np.nan), + 'rprs_deviation_unc': deviation_summary.get('rprs_deviation_unc', np.nan), 'rprs_deviation_sigma': deviation_summary.get('rprs_deviation_sigma', np.nan), 'tmid_deviation_score': deviation_summary.get('tmid_deviation_score', np.nan), 'rprs_deviation_score': deviation_summary.get('rprs_deviation_score', np.nan), @@ -1558,6 +1612,9 @@ def annotate_transit_detection_qc(fit, summary=None): fit.transit_qc_tmid_deviation_threshold_minutes = summary.get('tmid_deviation_threshold_minutes') fit.transit_qc_tmid_deviation_sigma = summary.get('tmid_deviation_sigma') fit.transit_qc_rprs_deviation_fit_unc = summary.get('rprs_deviation_fit_unc') + fit.transit_qc_rprs_deviation_expected_unc = summary.get('rprs_deviation_expected_unc') + fit.transit_qc_rprs_deviation_systematic_floor = summary.get('rprs_deviation_systematic_floor') + fit.transit_qc_rprs_deviation_unc = summary.get('rprs_deviation_unc') fit.transit_qc_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') fit.transit_qc_expected_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') fit.transit_qc_ktmf_metric = summary.get('ktmf_metric') diff --git a/exotic/inputs.py b/exotic/inputs.py index 46ac1176..c85029e2 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -392,18 +392,33 @@ def comp_params(self, init_file, planet_dict): planet_params = { 'ra': 'Target Star RA', 'dec': 'Target Star Dec', 'pName': "Planet Name", 'sName': "Host Star Name", 'pPer': 'Orbital Period (days)', 'pPerUnc': 'Orbital Period Uncertainty', - 'midT': 'Published Mid-Transit Time (BJD-UTC)', 'midTUnc': 'Mid-Transit Time Uncertainty', - 'rprs': 'Ratio of Planet to Stellar Radius (Rp/Rs)', - 'rprsUnc': 'Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty', - 'aRs': 'Ratio of Distance to Stellar Radius (a/Rs)', - 'aRsUnc': 'Ratio of Distance to Stellar Radius (a/Rs) Uncertainty', + 'midT': ('Published Mid-Transit Time (BJD-UTC)', 'Published Mid-Transit Time'), + 'midTUnc': 'Mid-Transit Time Uncertainty', + 'rprs': ('Ratio of Planet to Stellar Radius (Rp/Rs)', 'Rp/Rs', 'Rp/R*'), + 'rprsUnc': ( + 'Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty', + 'Rp/Rs Uncertainty', + 'Rp/R* Uncertainty', + ), + 'aRs': ('Ratio of Distance to Stellar Radius (a/Rs)', 'a/Rs', 'a/R*'), + 'aRsUnc': ( + 'Ratio of Distance to Stellar Radius (a/Rs) Uncertainty', + 'a/Rs Uncertainty', + 'a/R* Uncertainty', + ), 'inc': 'Orbital Inclination (deg)', - 'incUnc': ('Orbital Inclination (deg) Uncertainty', 'Orbital Inclination (deg) Uncertainity'), - 'ecc': 'Orbital Eccentricity (0 if null)', 'teff': 'Star Effective Temperature (K)', + 'incUnc': ( + 'Orbital Inclination (deg) Uncertainty', + 'Orbital Inclination (deg) Uncertainity', + 'Orbital Inclination Uncertainty', + ), + 'ecc': ('Orbital Eccentricity (0 if null)', 'Orbital Eccentricity'), + 'teff': 'Star Effective Temperature (K)', 'omega': 'Argument of Periastron (deg)', 'teffUncPos': 'Star Effective Temperature (+) Uncertainty', 'teffUncNeg': 'Star Effective Temperature (-) Uncertainty', - 'met': 'Star Metallicity ([FE/H])', 'metUncPos': 'Star Metallicity (+) Uncertainty', + 'met': ('Star Metallicity ([FE/H])', 'Star Metallicity [FE/H]'), + 'metUncPos': 'Star Metallicity (+) Uncertainty', 'metUncNeg': 'Star Metallicity (-) Uncertainty', 'logg': 'Star Surface Gravity (log(g))', 'loggUncPos': 'Star Surface Gravity (+) Uncertainty', 'loggUncNeg': 'Star Surface Gravity (-) Uncertainty', diff --git a/exotic/output_files.py b/exotic/output_files.py index 57f9f7a1..d258a71b 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -427,7 +427,9 @@ def build_aavso_qc_metadata(fit): 'use_deviation_from_expected_transit_in_qc', 'deviation_sigma_threshold', 'expected_tmid', 'expected_tmid_unc', 'fitted_tmid', 'expected_rprs', 'expected_rprs_unc', 'fitted_rprs', 'fitted_rprs_unc', - 'rprs_deviation_fit_unc', 'rprs_deviation_sigma', 'rprs_deviation_score', + 'rprs_deviation_fit_unc', 'rprs_deviation_expected_unc', + 'rprs_deviation_systematic_floor', 'rprs_deviation_unc', + 'rprs_deviation_sigma', 'rprs_deviation_score', 'deviation_from_expected_value', 'ktmf_metric', 'ktmf_contributions', 'notes', ) diff --git a/exotic/transit_depth.py b/exotic/transit_depth.py index 2250fa8d..021e05f3 100644 --- a/exotic/transit_depth.py +++ b/exotic/transit_depth.py @@ -37,13 +37,54 @@ def planet_dict_transit_parameters(planet_dict, limb_darkening=None, fallback=No planet_dict = planet_dict or {} aliases = { - "rprs": ("rprs", "pl_ratror"), - "per": ("pPer", "pl_orbper", "per", "period"), - "ars": ("aRs", "pl_ratdor", "ars"), - "inc": ("inc", "pl_orbincl"), - "ecc": ("ecc", "pl_orbeccen"), - "omega": ("omega", "pl_orblper"), - "tmid": ("midT", "pl_tranmid", "tmid"), + "rprs": ( + "rprs", + "pl_ratror", + "Rp/Rs", + "Rp/R*", + "Ratio of Planet to Stellar Radius (Rp/Rs)", + "Ratio of Planet to Stellar Radius (Rp/R*)", + ), + "per": ( + "pPer", + "pl_orbper", + "per", + "period", + "Orbital Period (days)", + ), + "ars": ( + "aRs", + "pl_ratdor", + "ars", + "a/Rs", + "a/R*", + "Ratio of Distance to Stellar Radius (a/Rs)", + "Ratio of Distance to Stellar Radius (a/R*)", + ), + "inc": ( + "inc", + "pl_orbincl", + "Orbital Inclination (deg)", + ), + "ecc": ( + "ecc", + "pl_orbeccen", + "Orbital Eccentricity", + "Orbital Eccentricity (0 if null)", + ), + "omega": ( + "omega", + "pl_orblper", + "Argument of Periastron (deg)", + ), + "tmid": ( + "midT", + "pl_tranmid", + "tmid", + "Published Mid-Transit Time", + "Published Mid-Transit Time (BJD-UTC)", + "Published Mid-Transit Time (BJD_UTC)", + ), } for target, names in aliases.items(): for name in names: @@ -79,11 +120,38 @@ def planet_dict_transit_errors(planet_dict, limb_darkening=None, fallback=None): planet_dict = planet_dict or {} aliases = { - "rprs": ("rprsUnc", "pl_ratrorerr1"), - "per": ("pPerUnc", "pl_orbpererr1"), - "ars": ("aRsUnc", "pl_ratdorerr1"), - "inc": ("incUnc", "pl_orbinclerr1"), - "tmid": ("midTUnc", "pl_tranmiderr1"), + "rprs": ( + "rprsUnc", + "pl_ratrorerr1", + "Rp/Rs Uncertainty", + "Rp/R* Uncertainty", + "Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty", + "Ratio of Planet to Stellar Radius (Rp/R*) Uncertainty", + ), + "per": ( + "pPerUnc", + "pl_orbpererr1", + "Orbital Period Uncertainty", + ), + "ars": ( + "aRsUnc", + "pl_ratdorerr1", + "a/Rs Uncertainty", + "a/R* Uncertainty", + "Ratio of Distance to Stellar Radius (a/Rs) Uncertainty", + "Ratio of Distance to Stellar Radius (a/R*) Uncertainty", + ), + "inc": ( + "incUnc", + "pl_orbinclerr1", + "Orbital Inclination Uncertainty", + "Orbital Inclination (deg) Uncertainty", + ), + "tmid": ( + "midTUnc", + "pl_tranmiderr1", + "Mid-Transit Time Uncertainty", + ), } for target, names in aliases.items(): for name in names: @@ -113,6 +181,10 @@ def complete_transit_parameters(parameters): return values +def complete_transit_errors(errors): + return planet_dict_transit_errors(errors, fallback=errors) + + def transit_duration_days(parameters): values = complete_transit_parameters(parameters) period = finite_float(values.get("per")) @@ -282,7 +354,7 @@ def _perturbed_value(key, value): def observable_depth_uncertainty_fraction(parameters, errors): values = complete_transit_parameters(parameters) - errors = errors or {} + errors = complete_transit_errors(errors or {}) contributions = [] for key in _DEPTH_ERROR_KEYS: center = finite_float(values.get(key)) @@ -338,6 +410,13 @@ def fit_transit_depth_summary(fit, prior_parameters=None, prior_errors=None): prior_error = np.nan if prior_parameters: prior_depth, prior_error = observable_depth_percent(prior_parameters, prior_errors or {}) + if not np.isfinite(prior_depth): + prior_values = complete_transit_parameters(prior_parameters) + prior_errors = complete_transit_errors(prior_errors or {}) + prior_depth, prior_error = radius_ratio_area_depth_percent( + prior_values.get("rprs"), + prior_errors.get("rprs"), + ) delta = np.nan delta_error = np.nan diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 4560eada..f0204c05 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -3371,7 +3371,10 @@ def test_evaluate_transit_detection_qc_marks_large_expected_value_deviation_marg assert summary["computed"] is True assert summary["status"] == "marginal" - assert summary["rprs_deviation_sigma"] == pytest.approx(8.0) + expected_comparison_unc = np.sqrt(0.01 ** 2 + 0.01 ** 2 + (0.05 * 0.10) ** 2) + assert summary["rprs_deviation_unc"] == pytest.approx(expected_comparison_unc) + assert summary["rprs_deviation_systematic_floor"] == pytest.approx(0.05 * 0.10) + assert summary["rprs_deviation_sigma"] == pytest.approx(abs(0.18 - 0.10) / expected_comparison_unc) assert summary["deviation_from_expected_value"] == pytest.approx(0.0) assert summary["ktmf_metric"] <= 5.0 assert summary["ktmf_metric"] < 3.5 @@ -3474,7 +3477,7 @@ def test_evaluate_transit_detection_qc_does_not_calculate_tmid_expected_value_de assert "not supported strongly enough against a flat/null model" not in summary["summary"] -def test_expected_value_rprs_deviation_uses_fit_uncertainty_not_prior_uncertainty(): +def test_expected_value_rprs_deviation_uses_combined_uncertainty_with_systematic_floor(): transit_model = np.ones(21, dtype=float) transit_model[9:12] -= 0.0287 data = transit_model.copy() @@ -3499,8 +3502,12 @@ def test_expected_value_rprs_deviation_uses_fit_uncertainty_not_prior_uncertaint summary = evaluate_transit_detection_qc(fit) + expected_comparison_unc = np.sqrt(0.0046 ** 2 + 0.0001 ** 2 + (0.05 * 0.1589) ** 2) assert summary["rprs_deviation_fit_unc"] == pytest.approx(0.0046) - assert summary["rprs_deviation_sigma"] == pytest.approx(abs(0.1694 - 0.1589) / 0.0046) + assert summary["rprs_deviation_expected_unc"] == pytest.approx(0.0001) + assert summary["rprs_deviation_systematic_floor"] == pytest.approx(0.05 * 0.1589) + assert summary["rprs_deviation_unc"] == pytest.approx(expected_comparison_unc) + assert summary["rprs_deviation_sigma"] == pytest.approx(abs(0.1694 - 0.1589) / expected_comparison_unc) assert summary["deviation_from_expected_value"] == pytest.approx( 1.0 - summary["rprs_deviation_sigma"] / 5.0 ) @@ -3602,8 +3609,12 @@ def fake_run_nested(*args, **kwargs): duration_prior={"duration": 0.1}, ) + expected_comparison_unc = np.sqrt(0.0046 ** 2 + 0.0001 ** 2 + (0.05 * 0.1589) ** 2) assert refit.transit_qc_rprs_deviation_fit_unc == pytest.approx(0.0046) - assert refit.transit_qc_rprs_deviation_sigma == pytest.approx(abs(0.1694 - 0.1589) / 0.0046) + assert refit.transit_qc_rprs_deviation_expected_unc == pytest.approx(0.0001) + assert refit.transit_qc_rprs_deviation_systematic_floor == pytest.approx(0.05 * 0.1589) + assert refit.transit_qc_rprs_deviation_unc == pytest.approx(expected_comparison_unc) + assert refit.transit_qc_rprs_deviation_sigma == pytest.approx(abs(0.1694 - 0.1589) / expected_comparison_unc) contribution = next( item for item in refit.transit_qc_ktmf_contributions if item["label"] == "Deviation From Expected Value" @@ -3611,6 +3622,9 @@ def fake_run_nested(*args, **kwargs): assert contribution["score"] > 0.0 assert contribution["max_points"] > 0.0 assert "fit uncertainty=0.004600" in contribution["detail"] + assert "expected uncertainty=0.000100" in contribution["detail"] + assert "comparison uncertainty=" in contribution["detail"] + assert "systematic floor=" in contribution["detail"] assert "Tmid" not in contribution["detail"] diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 4b369885..4be3e737 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -13,7 +13,11 @@ fit_impact_parameter_value_error, save_comp_star_calibration_summary, ) -from exotic.transit_depth import observable_depth_percent, radius_ratio_area_depth_percent +from exotic.transit_depth import ( + fit_transit_depth_summary, + observable_depth_percent, + radius_ratio_area_depth_percent, +) class DummyFit: @@ -65,6 +69,57 @@ def __init__(self): } +def test_prior_depth_uses_available_gj436_geometry(): + fit = DummyFit() + prior = { + "Published Mid-Transit Time": 2454510.80162, + "Rp/Rs": 0.0822, + "a/Rs": 13.73, + "Orbital Period (days)": 2.64388312, + "Orbital Inclination (deg)": 86.44, + "Orbital Eccentricity": 0.13827, + "Argument of Periastron (deg)": 351.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + } + + summary = fit_transit_depth_summary( + fit, + prior_parameters=prior, + prior_errors={ + "Rp/Rs Uncertainty": 0.001, + "a/Rs Uncertainty": 0.46, + "Orbital Inclination Uncertainty": 0.17, + }, + ) + + assert summary["prior_observable_depth"] == pytest.approx(0.675684, abs=1.0e-5) + assert summary["prior_observable_depth_error"] == pytest.approx(0.01644, abs=1.0e-6) + + +def test_prior_depth_respects_inclination_for_non_transiting_geometry(): + fit = DummyFit() + prior = { + "tmid": 2454510.80162, + "rprs": 0.0822, + "ars": 13.73, + "per": 2.64388312, + "inc": 0.0, + "ecc": 0.13827, + "omega": 351.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + } + + summary = fit_transit_depth_summary(fit, prior_parameters=prior) + + assert summary["prior_observable_depth"] == pytest.approx(0.0) + + def aavso_json_header(output_text, header_name): prefix = f"#{header_name}=" for line in output_text.splitlines(): diff --git a/tests/test_ultranest_utils.py b/tests/test_ultranest_utils.py index 27ac6a0d..5c213222 100644 --- a/tests/test_ultranest_utils.py +++ b/tests/test_ultranest_utils.py @@ -4,6 +4,7 @@ import types import numpy as np +import pytest import exotic.api.ultranest_utils as ultranest_utils from exotic.api.ultranest_utils import run_reactive_sampler @@ -115,6 +116,38 @@ def run(self, **kwargs): assert stream.getvalue() == "" +def test_run_reactive_sampler_translates_ultranest_degenerate_region_value_error(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def run(self, **kwargs): + exec( + compile( + 'raise ValueError("Buffer has wrong number of dimensions (expected 2, got 0)")', + "ultranest/mlfriends.pyx", + "exec", + ), + {}, + ) + + with pytest.raises(np.linalg.LinAlgError) as excinfo: + run_reactive_sampler(FakeSampler(), verbose=False) + + assert "degenerate sampling region" in str(excinfo.value) + assert isinstance(excinfo.value.__cause__, ValueError) + + +def test_run_reactive_sampler_preserves_unrelated_value_error(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def run(self, **kwargs): + raise ValueError("not an ultranest region error") + + with pytest.raises(ValueError, match="not an ultranest region error"): + run_reactive_sampler(FakeSampler(), verbose=False) + + def test_run_reactive_sampler_applies_fast_defaults(monkeypatch): _reset_ultranest_env(monkeypatch) From 5a4a14a015b7acb047a556c9ecc59cefa962a5b2 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Wed, 27 May 2026 20:05:26 +1000 Subject: [PATCH 059/116] ULTRANEXTMEGASPEEDUPS --- exotic/api/elca.py | 437 ++++++++++++++---- exotic/exotic.py | 694 +++++++++++++++++++++++------ tests/test_centroid_wcs.py | 67 ++- tests/test_elca_baseline.py | 62 +++ tests/test_exotic_proper_motion.py | 16 +- 5 files changed, 1037 insertions(+), 239 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index c60613ff..50b05466 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -361,7 +361,10 @@ def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000, mask=None n = int(n) a2 = np.random.normal(m_a2, sig_a2, n) reference = get_airmass_reference(airmass) - model = transit * airmass_trend_grid(a2, airmass, reference=reference) + transit = np.asarray(transit, dtype=float) + data = np.asarray(data, dtype=float) + airmass = np.asarray(airmass, dtype=float) + centered_airmass = center_airmass(airmass, reference=reference) weights = np.ones(transit.shape[0], dtype=float) fit_mask = normalized_optional_fit_mask(mask, transit.shape) @@ -371,7 +374,7 @@ def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000, mask=None valid_err = np.isfinite(dataerr) & (dataerr > 0) weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) - mask = np.isfinite(data) & np.isfinite(transit) & np.isfinite(airmass) + mask = np.isfinite(data) & np.isfinite(transit) & np.isfinite(centered_airmass) if fit_mask is not None: mask &= fit_mask if dataerr is not None: @@ -380,12 +383,19 @@ def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000, mask=None if not np.any(mask): return fallback_flux_baseline(), 0.0 - masked_model = model[:, mask] - masked_data = np.asarray(data, dtype=float)[mask] + masked_transit = transit[mask] + masked_airmass = centered_airmass[mask] + masked_data = data[mask] masked_weights = weights[mask] - - numer = np.sum(masked_weights * masked_data * masked_model, axis=1) - denom = np.sum(masked_weights * masked_model ** 2, axis=1) + numer = np.empty(n, dtype=float) + denom = np.empty(n, dtype=float) + chunk_size = 1024 + + for start in range(0, n, chunk_size): + stop = min(start + chunk_size, n) + masked_model = masked_transit * np.exp(np.outer(a2[start:stop], masked_airmass)) + numer[start:stop] = np.sum(masked_weights * masked_data * masked_model, axis=1) + denom[start:stop] = np.sum(masked_weights * masked_model ** 2, axis=1) valid = np.isfinite(numer) & np.isfinite(denom) & (denom > 0) if not np.any(valid): @@ -578,7 +588,9 @@ def _apply_fixed_parameter_errors(self): self.quantiles[key] = [-error, error] def _get_airmass_reference(self): - return getattr(self, 'airmass_reference', get_airmass_reference(self.airmass)) + if hasattr(self, 'airmass_reference'): + return self.airmass_reference + return get_airmass_reference(self.airmass) def _get_plot_time_range(self): plot_time_range = normalize_time_range(getattr(self, 'plot_time_range', None)) @@ -1005,7 +1017,7 @@ def _get_sampled_keys(self, bound_keys=None): return ['b' if key == 'inc' else key for key in bound_keys] def _get_impact_parameter_scale_upper_bound(self, values): - values = copy.deepcopy(values) + values = dict(values) scale_keys = ('ars', 'ecc', 'omega') endpoint_sets = [] for key in scale_keys: @@ -1020,7 +1032,7 @@ def _get_impact_parameter_scale_upper_bound(self, values): scales = [] for candidate_values in product(*[endpoints for _, endpoints in endpoint_sets]): - candidate = copy.deepcopy(values) + candidate = dict(values) for key, value in zip([key for key, _ in endpoint_sets], candidate_values): candidate[key] = value try: @@ -1033,8 +1045,9 @@ def _get_impact_parameter_scale_upper_bound(self, values): return float(max(scales)) if scales else np.nan def _get_impact_parameter_sampling_bounds(self, values=None, use_search_bounds=False): - values = copy.deepcopy(self.prior if values is None else values) + values = self.prior if values is None else values if use_search_bounds and 'rprs' in self.bounds: + values = dict(values) rprs_bounds = np.asarray(self.bounds['rprs'], dtype=float).reshape(-1)[:2] finite_rprs = rprs_bounds[np.isfinite(rprs_bounds) & (rprs_bounds >= 0)] if finite_rprs.size > 0: @@ -1057,6 +1070,44 @@ def _get_impact_parameter_sampling_bounds(self, values=None, use_search_bounds=F upper = min(upper_candidates) if upper_candidates else 1.0 return [0.0, float(max(0.0, upper))] + def _get_impact_parameter_upper_bounds_for_sample_points(self, sample_points, bound_keys): + sample_points = np.atleast_2d(np.asarray(sample_points, dtype=float)) + bound_index = {key: index for index, key in enumerate(bound_keys)} + sample_count = sample_points.shape[0] + + def values_for(key, default): + if key in bound_index: + return sample_points[:, bound_index[key]] + value = np.asarray(self.prior.get(key, default), dtype=float) + if value.shape == (): + return np.full(sample_count, float(value), dtype=float) + return np.broadcast_to(value, (sample_count,)).astype(float) + + rprs = values_for('rprs', np.nan) + grazing_upper = np.where(np.isfinite(rprs) & (rprs >= 0), 1.0 + rprs, np.nan) + + ars = values_for('ars', np.nan) + ecc = values_for('ecc', 0.0) + omega = np.deg2rad(values_for('omega', 0.0)) + denom = 1.0 + ecc * np.sin(omega) + denom = np.where(np.isclose(denom, 0.0), np.finfo(float).eps, denom) + scale_upper = ars * (1.0 - ecc ** 2) / denom + + valid_grazing = np.isfinite(grazing_upper) & (grazing_upper > 0) + valid_scale = np.isfinite(scale_upper) & (scale_upper > 0) + upper = np.full(sample_count, 1.0, dtype=float) + + both_valid = valid_grazing & valid_scale + upper[both_valid] = np.minimum(grazing_upper[both_valid], scale_upper[both_valid]) + + grazing_only = valid_grazing & ~valid_scale + upper[grazing_only] = grazing_upper[grazing_only] + + scale_only = valid_scale & ~valid_grazing + upper[scale_only] = scale_upper[scale_only] + + return np.maximum(0.0, upper) + def _get_sample_bounds(self, bound_keys=None, values=None): bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) sampled_keys = self._get_sampled_keys(bound_keys) @@ -1075,44 +1126,69 @@ def _get_sample_bounds(self, bound_keys=None, values=None): def _sample_point_from_unit_cube(self, upars, bound_keys=None): bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) upars_array = np.asarray(upars, dtype=float) - if upars_array.ndim == 2: - return np.asarray([ - self._sample_point_from_unit_cube(row, bound_keys) - for row in upars_array - ], dtype=float) - boundarray = np.array([self.bounds[k] for k in bound_keys], dtype=float) - physical = copy.deepcopy(self.prior) - sample_point = np.zeros(len(bound_keys), dtype=float) + lower_bounds = boundarray[:, 0] + bound_widths = boundarray[:, 1] - lower_bounds + uses_internal_impact_parameter = self._uses_internal_impact_parameter() + inc_indices = [i for i, key in enumerate(bound_keys) if key == 'inc'] + + sample_point = lower_bounds + bound_widths * upars_array + if not uses_internal_impact_parameter or not inc_indices: + return sample_point + + if len(inc_indices) == 1: + inc_index = inc_indices[0] + if upars_array.ndim == 2: + upper_bounds = self._get_impact_parameter_upper_bounds_for_sample_points( + sample_point, + bound_keys, + ) + sample_point[:, inc_index] = upper_bounds * upars_array[:, inc_index] + return sample_point - for i, key in enumerate(bound_keys): - if key == 'inc' and self._uses_internal_impact_parameter(): - continue - physical[key] = boundarray[i, 0] + (boundarray[i, 1] - boundarray[i, 0]) * upars_array[i] + upper_bound = self._get_impact_parameter_upper_bounds_for_sample_points( + sample_point.reshape(1, -1), + bound_keys, + )[0] + sample_point[inc_index] = upper_bound * upars_array[inc_index] + return sample_point + if upars_array.ndim == 2: + for row_index, row_sample_point in enumerate(sample_point): + physical = dict(self.prior) + for i, key in enumerate(bound_keys): + if i not in inc_indices: + physical[key] = row_sample_point[i] + for i in inc_indices: + b_lower, b_upper = self._get_impact_parameter_sampling_bounds(physical) + row_sample_point[i] = b_lower + (b_upper - b_lower) * upars_array[row_index, i] + return sample_point + + physical = dict(self.prior) for i, key in enumerate(bound_keys): - if key == 'inc' and self._uses_internal_impact_parameter(): - b_lower, b_upper = self._get_impact_parameter_sampling_bounds(physical) - sample_point[i] = b_lower + (b_upper - b_lower) * upars_array[i] - else: - sample_point[i] = physical[key] + if i not in inc_indices: + physical[key] = sample_point[i] + for i in inc_indices: + b_lower, b_upper = self._get_impact_parameter_sampling_bounds(physical) + sample_point[i] = b_lower + (b_upper - b_lower) * upars_array[i] return sample_point def _physical_values_from_sample_point(self, sample_point, bound_keys=None, sampled_keys=None): bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) sampled_keys = self._get_sampled_keys(bound_keys) if sampled_keys is None else list(sampled_keys) - physical = copy.deepcopy(self.prior) + physical = dict(self.prior) + impact_parameter = None for value, bound_key, sampled_key in zip(sample_point, bound_keys, sampled_keys): if sampled_key == 'b' and bound_key == 'inc': + impact_parameter = value continue physical[bound_key] = value - for value, bound_key, sampled_key in zip(sample_point, bound_keys, sampled_keys): - if sampled_key == 'b' and bound_key == 'inc': - physical['b'] = value - physical['inc'] = float(inclination_from_impact_parameter(physical, value)) + if impact_parameter is not None: + physical['b'] = impact_parameter + physical['inc'] = float(inclination_from_impact_parameter(physical, impact_parameter)) return physical @@ -1138,8 +1214,9 @@ def _get_ultranest_weighted_sample_arrays(self): return None, None return points, logl - def _loglike_neighborhood_uncertainty(self, parameter_index, center, minimum_count=8): - points, logl = self._get_ultranest_weighted_sample_arrays() + def _loglike_neighborhood_uncertainty(self, parameter_index, center, minimum_count=8, points=None, logl=None): + if points is None or logl is None: + points, logl = self._get_ultranest_weighted_sample_arrays() if points is None or parameter_index >= points.shape[1]: return None @@ -1184,8 +1261,9 @@ def _loglike_neighborhood_uncertainty(self, parameter_index, center, minimum_cou 'delta_chi2': float(selected_delta), } - def _ultranest_error_needs_sample_fallback(self, parameter_index, center, reported_error): - points, _ = self._get_ultranest_weighted_sample_arrays() + def _ultranest_error_needs_sample_fallback(self, parameter_index, center, reported_error, points=None): + if points is None: + points, _ = self._get_ultranest_weighted_sample_arrays() if points is None or parameter_index >= points.shape[1]: return False @@ -2880,10 +2958,12 @@ def create_fit_variables(self): tdur = (self.transit < 1).sum() * np.median(np.diff(np.sort(self.time))) # test for partial transit - newtime = np.linspace(self.parameters['tmid'] - 0.2, self.parameters['tmid'] + 0.2, 10000) - newtran = transit(newtime, self.parameters) - masktran = newtran < 1 - newdur = np.diff(newtime).mean() * masktran.sum() + newdur = transit_duration(self.parameters) + if not np.isfinite(newdur) or newdur <= 0: + newtime = np.linspace(self.parameters['tmid'] - 0.2, self.parameters['tmid'] + 0.2, 10000) + newtran = transit(newtime, self.parameters) + masktran = newtran < 1 + newdur = np.diff(newtime).mean() * masktran.sum() self.duration_measured = tdur self.duration_expected = newdur @@ -2899,6 +2979,7 @@ def _finalize_ultranest_fit_results(self, bound_keys, sampled_keys, physical_fro ml_point = self.results['maximum_likelihood']['point'] self.sample_bounds = self._get_sample_bounds(bound_keys, physical_from_sample_point(ml_point)) self.ultranest_error_fallbacks = {} + weighted_points, weighted_logl = self._get_ultranest_weighted_sample_arrays() for i, key in enumerate(sampled_keys): self.sample_parameters[key] = ml_point[i] @@ -2906,13 +2987,28 @@ def _finalize_ultranest_fit_results(self, bound_keys, sampled_keys, physical_fro reported_quantiles = [ self.results['posterior']['errlo'][i], self.results['posterior']['errup'][i]] - if self._ultranest_error_needs_sample_fallback(i, ml_point[i], reported_error): - fallback = self._loglike_neighborhood_uncertainty(i, ml_point[i]) + if self._ultranest_error_needs_sample_fallback( + i, + ml_point[i], + reported_error, + points=weighted_points, + ): + fallback = self._loglike_neighborhood_uncertainty( + i, + ml_point[i], + points=weighted_points, + logl=weighted_logl, + ) if fallback is not None: fallback['reason'] = 'degenerate_posterior_summary' else: fallback = None - local_uncertainty = self._loglike_neighborhood_uncertainty(i, ml_point[i]) + local_uncertainty = self._loglike_neighborhood_uncertainty( + i, + ml_point[i], + points=weighted_points, + logl=weighted_logl, + ) if self._ultranest_error_is_inflated_relative_to_local_fit(reported_error, local_uncertainty): fallback = local_uncertainty fallback['reported_error'] = float(reported_error) @@ -2934,11 +3030,26 @@ def _finalize_ultranest_fit_results(self, bound_keys, sampled_keys, physical_fro self.errors[bound_key] = self.sample_errors[sampled_key] self.quantiles[bound_key] = self.sample_quantiles[sampled_key] - if 'inc' in bound_keys and 'b' in sampled_keys: - inc_samples = np.array([ - physical_from_sample_point(point)['inc'] - for point in self.results['weighted_samples']['points'] - ]) + if 'inc' in bound_keys and 'b' in sampled_keys and weighted_points is not None: + bound_index = {key: index for index, key in enumerate(bound_keys)} + + def weighted_sample_values(key, default=0.0): + index = bound_index.get(key) + if index is not None and index < weighted_points.shape[1] and sampled_keys[index] != 'b': + return weighted_points[:, index] + value = physical_ml.get(key, self.prior.get(key, default)) + return np.full(weighted_points.shape[0], float(value), dtype=float) + + b_index = sampled_keys.index('b') + scale_values = { + 'ars': weighted_sample_values('ars', np.nan), + 'ecc': weighted_sample_values('ecc', 0.0), + 'omega': weighted_sample_values('omega', 0.0), + } + inc_samples = np.asarray( + inclination_from_impact_parameter(scale_values, weighted_points[:, b_index]), + dtype=float, + ) center, std, quantiles = self._summarize_derived_parameter(inc_samples, physical_ml['inc']) self.parameters['inc'] = center self.errors['inc'] = std @@ -2993,71 +3104,213 @@ def fit_nested(self): self.quantiles = {} self.parameters = copy.deepcopy(self.prior) + base_physical = dict(self.prior) + direct_sample_assignments = [ + (bound_key, index) + for index, (bound_key, sampled_key) in enumerate(zip(bound_keys, sampled_keys)) + if not (sampled_key == 'b' and bound_key == 'inc') + ] + impact_parameter_index = next( + ( + index + for index, (bound_key, sampled_key) in enumerate(zip(bound_keys, sampled_keys)) + if sampled_key == 'b' and bound_key == 'inc' + ), + None, + ) + time = self.time + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + data_shape = data.shape + dataerr_shape_matches = dataerr.shape == data_shape + finite_dataerr = np.isfinite(dataerr) & (dataerr > 0) if dataerr_shape_matches else False + observed_values_valid = ( + dataerr_shape_matches + and np.all(np.isfinite(data)) + and np.all(finite_dataerr) + ) + inverse_dataerr = np.zeros(data_shape, dtype=float) + baseline_weights = np.zeros(data_shape, dtype=float) + baseline_static_mask = np.isfinite(data) + if dataerr_shape_matches: + inverse_dataerr[finite_dataerr] = 1.0 / dataerr[finite_dataerr] + baseline_weights[finite_dataerr] = inverse_dataerr[finite_dataerr] ** 2 + baseline_static_mask &= np.isfinite(baseline_weights) & (baseline_weights > 0) + else: + baseline_static_mask &= False + + baseline_fit_mask = self._get_baseline_fit_mask() + if baseline_fit_mask is not None: + baseline_static_mask &= baseline_fit_mask + + centered_airmass = center_airmass(self.airmass, reference=self._get_airmass_reference()) + has_free_flux_baseline = self._has_free_flux_baseline() + uses_fixed_flux_baseline = self._uses_fixed_flux_baseline() + sampled_key_index = {key: index for index, key in enumerate(sampled_keys)} + sampled_a2_index = sampled_key_index.get('a2') + fixed_airmass_scale = None + if sampled_a2_index is None: + try: + fixed_a2 = float(base_physical.get('a2', 0.0)) + except (TypeError, ValueError): + fixed_a2 = np.nan + if not np.isfinite(fixed_a2): + observed_values_valid = False + elif fixed_a2 != 0.0: + fixed_airmass_scale = np.exp(fixed_a2 * centered_airmass) + + free_flux_baseline_index = next( + (sampled_key_index[key] for key in ('a0', 'a1') if key in sampled_key_index), + None, + ) + fixed_flux_baseline_value = None + if free_flux_baseline_index is None and uses_fixed_flux_baseline: + fixed_flux_baseline_value = get_flux_baseline(base_physical) + + duration_prior = self.duration_prior if isinstance(self.duration_prior, dict) else None + duration_prior_applied = bool(duration_prior and duration_prior.get('applied')) + try: + expected_duration = float(duration_prior.get('expected_duration', np.nan)) if duration_prior else np.nan + sigma_log_duration = float(duration_prior.get('sigma_log_duration', np.nan)) if duration_prior else np.nan + except (TypeError, ValueError): + expected_duration = np.nan + sigma_log_duration = np.nan + duration_prior_valid = ( + duration_prior_applied + and np.isfinite(expected_duration) + and expected_duration > 0 + and np.isfinite(sigma_log_duration) + and sigma_log_duration > 0 + ) + + def solve_flux_baseline_for_model(model): + mask = baseline_static_mask & np.isfinite(model) & (model != 0) + if not np.any(mask): + return fallback_flux_baseline() + + masked_model = model[mask] + masked_data = data[mask] + masked_weights = baseline_weights[mask] + denom = np.sum(masked_weights * masked_model ** 2) + + if not np.isfinite(denom) or denom <= 0: + ratio = masked_data / masked_model + ratio = ratio[np.isfinite(ratio)] + if ratio.size == 0: + return fallback_flux_baseline() + baseline = np.nanmedian(ratio) + return baseline if np.isfinite(baseline) else fallback_flux_baseline() + + baseline = np.sum(masked_weights * masked_data * masked_model) / denom + return baseline if np.isfinite(baseline) else fallback_flux_baseline() + def physical_from_sample_point(sample_point): - return self._physical_values_from_sample_point(sample_point, bound_keys, sampled_keys) + physical = base_physical.copy() + for bound_key, index in direct_sample_assignments: + physical[bound_key] = sample_point[index] + if impact_parameter_index is not None: + impact_parameter = sample_point[impact_parameter_index] + physical['b'] = impact_parameter + physical['inc'] = float(inclination_from_impact_parameter(physical, impact_parameter)) + return physical def single_loglike(pars): + if not observed_values_valid: + return BAD_LOG_LIKELIHOOD + physical = physical_from_sample_point(pars) - duration_prior = self.duration_prior if isinstance(self.duration_prior, dict) else None duration_loglike = 0.0 - if duration_prior and duration_prior.get('applied'): - expected_duration = duration_prior.get('expected_duration', np.nan) - sigma_log_duration = duration_prior.get('sigma_log_duration', np.nan) - if ( - np.isfinite(expected_duration) - and expected_duration > 0 - and np.isfinite(sigma_log_duration) - and sigma_log_duration > 0 - ): - duration = transit_duration(physical) - if not np.isfinite(duration) or duration <= 0: - return BAD_LOG_LIKELIHOOD - duration_log_residual = np.log(duration / expected_duration) - duration_loglike = -0.5 * (duration_log_residual / sigma_log_duration) ** 2 + if duration_prior_valid: + duration = transit_duration(physical) + if not np.isfinite(duration) or duration <= 0: + return BAD_LOG_LIKELIHOOD + duration_log_residual = np.log(duration / expected_duration) + duration_loglike = -0.5 * (duration_log_residual / sigma_log_duration) ** 2 try: - model = np.asarray(transit(self.time, physical), dtype=float) - model *= airmass_trend( - physical.get('a2', 0), - self.airmass, - reference=self._get_airmass_reference(), - ) - if self._has_free_flux_baseline(): - model *= get_flux_baseline(physical) - elif self._uses_fixed_flux_baseline(): + model = np.asarray(transit(time, physical), dtype=float) + if sampled_a2_index is not None: + model *= np.exp(float(pars[sampled_a2_index]) * centered_airmass) + elif fixed_airmass_scale is not None: + model *= fixed_airmass_scale + + if free_flux_baseline_index is not None: + model *= pars[free_flux_baseline_index] + elif fixed_flux_baseline_value is not None: + model *= fixed_flux_baseline_value + elif has_free_flux_baseline: model *= get_flux_baseline(physical) else: - model *= solve_flux_baseline( - model, - self.data, - self.dataerr, - mask=self._get_baseline_fit_mask(), - ) + model *= solve_flux_baseline_for_model(model) except Exception: return BAD_LOG_LIKELIHOOD - if ( - model.shape != np.asarray(self.data).shape - or not np.all(np.isfinite(model)) - or not np.all(np.isfinite(self.data)) - or not np.all(np.isfinite(self.dataerr)) - or np.any(np.asarray(self.dataerr) <= 0) - ): + if model.shape != data_shape or not np.all(np.isfinite(model)): return BAD_LOG_LIKELIHOOD - residuals = (self.data - model) / self.dataerr - chi2 = np.sum(residuals ** 2) + residuals = (data - model) * inverse_dataerr + chi2 = np.sum(residuals * residuals) logl = -0.5 * chi2 + duration_loglike return float(logl) if np.isfinite(logl) else BAD_LOG_LIKELIHOOD def loglike(pars): pars_array = np.asarray(pars, dtype=float) if pars_array.ndim == 2: - return np.asarray([single_loglike(row) for row in pars_array], dtype=float) + return np.fromiter( + (single_loglike(row) for row in pars_array), + dtype=float, + count=pars_array.shape[0], + ) return single_loglike(pars_array) + prior_boundarray = np.array([self.bounds[k] for k in bound_keys], dtype=float) + prior_lower_bounds = prior_boundarray[:, 0] + prior_bound_widths = prior_boundarray[:, 1] - prior_lower_bounds + prior_bound_index = {key: index for index, key in enumerate(bound_keys)} + prior_inc_index = prior_bound_index.get('inc') + + def prior_values_for(sample_points, key, default): + if key in prior_bound_index: + return sample_points[:, prior_bound_index[key]] + value = np.asarray(base_physical.get(key, default), dtype=float) + if value.shape == (): + return np.full(sample_points.shape[0], float(value), dtype=float) + return np.broadcast_to(value, (sample_points.shape[0],)).astype(float) + + def prior_impact_upper_bounds(sample_points): + rprs = prior_values_for(sample_points, 'rprs', np.nan) + grazing_upper = np.where(np.isfinite(rprs) & (rprs >= 0), 1.0 + rprs, np.nan) + + ars = prior_values_for(sample_points, 'ars', np.nan) + ecc = prior_values_for(sample_points, 'ecc', 0.0) + omega = np.deg2rad(prior_values_for(sample_points, 'omega', 0.0)) + denom = 1.0 + ecc * np.sin(omega) + denom = np.where(np.isclose(denom, 0.0), np.finfo(float).eps, denom) + scale_upper = ars * (1.0 - ecc ** 2) / denom + + valid_grazing = np.isfinite(grazing_upper) & (grazing_upper > 0) + valid_scale = np.isfinite(scale_upper) & (scale_upper > 0) + upper = np.full(sample_points.shape[0], 1.0, dtype=float) + + both_valid = valid_grazing & valid_scale + upper[both_valid] = np.minimum(grazing_upper[both_valid], scale_upper[both_valid]) + upper[valid_grazing & ~valid_scale] = grazing_upper[valid_grazing & ~valid_scale] + upper[valid_scale & ~valid_grazing] = scale_upper[valid_scale & ~valid_grazing] + return np.maximum(0.0, upper) + def prior_transform(upars): - # transform unit cube to prior volume - return self._sample_point_from_unit_cube(upars, bound_keys) + upars_array = np.asarray(upars, dtype=float) + sample_points = prior_lower_bounds + prior_bound_widths * upars_array + if not self.impact_parameter_sampled_directly or prior_inc_index is None: + return sample_points + + if upars_array.ndim == 2: + upper_bounds = prior_impact_upper_bounds(sample_points) + sample_points[:, prior_inc_index] = upper_bounds * upars_array[:, prior_inc_index] + return sample_points + + upper_bound = prior_impact_upper_bounds(sample_points.reshape(1, -1))[0] + sample_points[prior_inc_index] = upper_bound * upars_array[prior_inc_index] + return sample_points self.ns_type = 'ultranest' test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform, vectorized=True) diff --git a/exotic/exotic.py b/exotic/exotic.py index c3451cc7..c49ce476 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -316,6 +316,13 @@ ) NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC = 30.0 NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 2.0 +REFERENCE_FALLBACK_COMPARISON_LIMIT = 10 +REFERENCE_FALLBACK_DETECTION_MAX_STARS = 60 +REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS = 12 +REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS = 4 +REFERENCE_FALLBACK_DETECTION_MIN_AREA_PIXELS = 3 +REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS = 10.0 +REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS = 50.0 BAD_PIXEL_DETECTION_FRACTION = 0.30 BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 BAD_PIXEL_PROGRESS_LOG_INTERVAL = 25 @@ -4119,6 +4126,10 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co def build_fit(local_prior, local_bounds): local_bounds = sanitize_retry_search_bounds(local_bounds) + if fixed_flux_baseline: + local_bounds = clone_lightcurve_bounds(local_bounds) + for key in ('a0', 'a1', 'a2'): + local_bounds.pop(key, None) local_prior = clamp_retry_priors_to_bounds(local_prior, local_bounds) fit_kwargs = { 'jd_times': jd_times, @@ -7724,19 +7735,20 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): log_info(detrend_result['note']) refit_prior = dict(working_prior) - for key in ('rprs', 'ars', 'tmid', 'inc', 'a2'): + for key in ('rprs', 'ars', 'tmid', 'inc'): if key in refit_prior and key in fit.parameters: refit_prior[key] = fit.parameters[key] + refit_prior['a0'] = 1.0 + refit_prior['a1'] = 1.0 + refit_prior['a2'] = 0.0 refit_bounds = clone_lightcurve_bounds(working_bounds) - apply_vertical_flux_normalization_bound( - refit_prior, - refit_bounds, - detrend_result['flux'], - disable_vertical_flux_normalization, - ) - if 'a2' not in refit_bounds: - refit_prior['a2'] = 0.0 + for key in ('a0', 'a1', 'a2'): + refit_bounds.pop(key, None) + refit_fixed_parameter_errors = dict(baseline_fixed_errors) + refit_fixed_parameter_errors.setdefault('a0', 0.0) + refit_fixed_parameter_errors.setdefault('a1', refit_fixed_parameter_errors.get('a0', 0.0)) + refit_fixed_parameter_errors['a2'] = 0.0 baseline_parameter_fit_note = baseline_parameter_result.get('note') baseline_parameter_fit_used = False if baseline_parameter_result.get('applied'): @@ -7757,8 +7769,8 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, baseline_fit_mask=baseline_fit_mask, - fixed_parameter_errors=baseline_fixed_errors, - fixed_flux_baseline=baseline_parameter_fit_used, + fixed_parameter_errors=refit_fixed_parameter_errors, + fixed_flux_baseline=True, pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) @@ -9087,15 +9099,15 @@ def leading_rejected_reference_prefix(ordered_inputfiles, dropped_files): return leading_rejected, next_candidate -def abort_if_reference_frame_rejected(reference_file, dropped_files, ordered_inputfiles=None, - rejection_label="Pointing precheck"): +def reference_frame_rejection_fallback_info(reference_file, dropped_files, ordered_inputfiles=None, + rejection_label="Pointing precheck"): if reference_file is None or not dropped_files: - return False + return None reference_file = str(reference_file) dropped_files = [str(file_name) for file_name in dropped_files] if reference_file not in dropped_files: - return False + return None other_dropped_files = [file_name for file_name in dropped_files if file_name != reference_file] leading_rejected_files, next_reference_candidate = leading_rejected_reference_prefix( @@ -9106,50 +9118,61 @@ def abort_if_reference_frame_rejected(reference_file, dropped_files, ordered_inp leading_rejected_files = [reference_file] log_info( - f"Error: {rejection_label} rejected the first usable image " - f"({_display_filename(reference_file)}). EXOTIC uses that frame as the reference image for " - "the supplied target and comparison-star pixel coordinates, so it is not safe to continue " - "with a different reference image.", - error=True, + f"WARNING: {rejection_label} rejected the original reference image " + f"({_display_filename(reference_file)}). EXOTIC is automatically removing the leading rejected " + "frame(s) and continuing with a new reference image.", + warn=True, + ) + log_info( + "IMPORTANT: the supplied target and comparison-star pixel coordinates were tied to the rejected " + "reference image. EXOTIC will estimate the target pixel position from the target RA/Dec on the " + "new reference image and will replace the supplied comparison-star pixels with a new " + "image-detected comparison-star set using the same FITS-image criteria as nextastro_archive.", + warn=True, ) if other_dropped_files: log_info( f"{rejection_label} also rejected {len(other_dropped_files)} other frame(s): " f"{format_file_preview_for_user(other_dropped_files)}", - error=True, + warn=True, ) leading_preview = format_file_preview_for_user(leading_rejected_files) - if len(leading_rejected_files) == 1: - removal_instruction = ( - f"Please remove or move this rejected frame and run again: {leading_preview}." - ) - else: - removal_instruction = ( - f"Please remove or move these leading rejected frames and run again: {leading_preview}." - ) + removal_instruction = ( + f"Automatically removed leading rejected frame(s) from this reduction: {leading_preview}." + ) if next_reference_candidate is not None: removal_instruction += ( - f" The next remaining frame would be " + f" Continuing from new reference image " f"{_display_filename(next_reference_candidate)}." ) else: removal_instruction += ( - " No non-rejected frame remains after that prefix, so this dataset still would not " - "have a usable reference image." + " No non-rejected frame remains after that prefix, so this dataset does not have a usable " + "reference image." ) log_info( removal_instruction, - error=True, - ) - log_info( - "If you need to keep those frames, reorder the dataset so a good reference image comes first, " - "or set optional_info 'pointing_rejection_sigma' to 0 to disable this precheck.", - error=True, + warn=True, ) - return True + return { + 'reference_file': reference_file, + 'leading_rejected_files': leading_rejected_files, + 'next_reference_candidate': next_reference_candidate, + 'other_dropped_files': other_dropped_files, + } + + +def abort_if_reference_frame_rejected(reference_file, dropped_files, ordered_inputfiles=None, + rejection_label="Pointing precheck"): + return reference_frame_rejection_fallback_info( + reference_file, + dropped_files, + ordered_inputfiles=ordered_inputfiles, + rejection_label=rejection_label, + ) is not None def collect_wcs_frame_center_pointings(inputfiles): @@ -10181,6 +10204,317 @@ def nextastro_calibration_label(match): return f"RA{match['ra']:.6f}_DEC{match['dec']:.6f}" +def unique_nextastro_calibration_label(calibration_stars, match): + label = nextastro_calibration_label(match) + unique_label = label + duplicate_index = 2 + while unique_label in calibration_stars: + unique_label = f"{label}-{duplicate_index}" + duplicate_index += 1 + return unique_label + + +def estimate_target_pixel_from_ra_dec(info_dict, wcs_header, image_data, obs_time, + centroid_margin=7.5): + target_ra, target_dec = update_coordinates_with_proper_motion(info_dict, obs_time) + try: + x_pixel, y_pixel = WCS(wcs_header).all_world2pix(target_ra, target_dec, 0) + except Exception as exc: + log_info( + "Warning: Could not project target RA/Dec onto the new reference image " + f"({exc}).", + warn=True, + ) + return None + + x_pixel = float(np.asarray(x_pixel).reshape(-1)[0]) + y_pixel = float(np.asarray(y_pixel).reshape(-1)[0]) + if not pixel_within_image(x_pixel, y_pixel, image_data.shape): + log_info( + "Warning: target RA/Dec projects outside the new reference image; " + "the rejected-reference fallback cannot re-estimate target pixels.", + warn=True, + ) + return None + + centroid_x, centroid_y, sigma_x, sigma_y = np.nan, np.nan, np.nan, np.nan + if pixel_within_image(x_pixel, y_pixel, image_data.shape, margin=centroid_margin): + centroid_x, centroid_y, sigma_x, sigma_y = get_psf_parameters(image_data, x_pixel, y_pixel) + + if np.isfinite(centroid_x) and np.isfinite(centroid_y): + log_info( + "Reference fallback target position: " + f"RA={float(target_ra):.7f}, Dec={float(target_dec):.7f} projected to " + f"[{x_pixel:.2f}, {y_pixel:.2f}] and centroided to " + f"[{centroid_x:.2f}, {centroid_y:.2f}].", + warn=True, + ) + return float(centroid_x), float(centroid_y), float(target_ra), float(target_dec) + + log_info( + "Reference fallback target position: " + f"RA={float(target_ra):.7f}, Dec={float(target_dec):.7f} projected to " + f"[{x_pixel:.2f}, {y_pixel:.2f}]; centroid fit was unavailable, so the WCS-projected " + "pixel position will be used.", + warn=True, + ) + return x_pixel, y_pixel, float(target_ra), float(target_dec) + + +def connected_component_sizes(mask): + mask = np.asarray(mask, dtype=bool) + if mask.ndim != 2 or not np.any(mask): + return None, [] + + labels = np.full(mask.shape, -1, dtype=int) + component_sizes = [] + component_id = 0 + height, width = mask.shape + + for start_y, start_x in np.argwhere(mask): + if labels[start_y, start_x] != -1: + continue + + stack = [(int(start_y), int(start_x))] + labels[start_y, start_x] = component_id + size = 0 + while stack: + y_pos, x_pos = stack.pop() + size += 1 + for neighbor_y in range(max(0, y_pos - 1), min(height, y_pos + 2)): + for neighbor_x in range(max(0, x_pos - 1), min(width, x_pos + 2)): + if not mask[neighbor_y, neighbor_x]: + continue + if labels[neighbor_y, neighbor_x] != -1: + continue + labels[neighbor_y, neighbor_x] = component_id + stack.append((neighbor_y, neighbor_x)) + component_sizes.append(size) + component_id += 1 + + return labels, component_sizes + + +def detect_reference_fallback_bright_stars( + image_data, + max_stars=REFERENCE_FALLBACK_DETECTION_MAX_STARS, + min_sep=REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS, + aperture_radius=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, + min_area=REFERENCE_FALLBACK_DETECTION_MIN_AREA_PIXELS): + if image_data is None: + return [] + + data = np.array(image_data, dtype=np.float64, copy=True) + if data.ndim != 2: + return [] + + finite = np.isfinite(data) + if not finite.any(): + return [] + + median = float(np.nanmedian(data[finite])) + data[~finite] = median + signal = data - median + signal[signal < 0] = 0 + if not np.any(signal > 0): + return [] + + threshold = float(np.percentile(signal, 99.7)) + if threshold <= 0: + threshold = float(np.percentile(signal, 99.0)) + if threshold <= 0: + positive_signal = signal[signal > 0] + if positive_signal.size == 0: + return [] + threshold = float(np.nanmin(positive_signal)) + + height, width = signal.shape + margin = max(int(min_sep), int(aperture_radius) + 2) + flat_order = np.argsort(signal, axis=None)[::-1] + yy, xx = np.indices(signal.shape) + source_labels, source_sizes = connected_component_sizes(signal >= threshold) + stars = [] + + for flat_index in flat_order: + y_pos, x_pos = np.unravel_index(int(flat_index), signal.shape) + peak = float(signal[y_pos, x_pos]) + if peak < threshold: + break + if source_labels is not None: + component_id = int(source_labels[y_pos, x_pos]) + if component_id < 0: + continue + if int(source_sizes[component_id]) < max(int(min_area), 1): + continue + if x_pos < margin or y_pos < margin or x_pos >= (width - margin) or y_pos >= (height - margin): + continue + if any((star['x'] - x_pos) ** 2 + (star['y'] - y_pos) ** 2 < (float(min_sep) ** 2) + for star in stars): + continue + + r2 = (xx - x_pos) ** 2 + (yy - y_pos) ** 2 + flux = float(signal[r2 <= (float(aperture_radius) ** 2)].sum()) + if flux <= 0: + continue + stars.append({'x': float(x_pos), 'y': float(y_pos), 'flux': flux}) + if len(stars) >= max_stars: + break + + stars.sort(key=lambda star: star['flux'], reverse=True) + return stars + + +def dedupe_reference_fallback_stars(stars, dedupe_radius=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + deduped_stars = [] + for star in sorted(stars or [], key=lambda value: float(value.get('flux', 0.0)), reverse=True): + x_pos = _finite_float(star.get('x')) + y_pos = _finite_float(star.get('y')) + if x_pos is None or y_pos is None: + continue + if any( + abs(float(existing.get('x', 0.0)) - x_pos) <= dedupe_radius + and abs(float(existing.get('y', 0.0)) - y_pos) <= dedupe_radius + for existing in deduped_stars + ): + continue + normalized = dict(star) + normalized['x'] = float(x_pos) + normalized['y'] = float(y_pos) + flux = _finite_float(star.get('flux')) + if flux is not None: + normalized['flux'] = float(flux) + deduped_stars.append(normalized) + return deduped_stars + + +def filter_reference_fallback_stars_to_middle_fifty_percent(stars, image_shape): + try: + height, width = image_shape[:2] + width = float(width) + height = float(height) + except Exception: + return list(stars or []) + + if width <= 1.0 or height <= 1.0: + return list(stars or []) + + center_x = (width - 1.0) / 2.0 + center_y = (height - 1.0) / 2.0 + half_width = (width - 1.0) * 0.25 + half_height = (height - 1.0) * 0.25 + + filtered = [] + for star in stars or []: + x_pos = _finite_float(star.get('x')) + y_pos = _finite_float(star.get('y')) + if x_pos is None or y_pos is None: + continue + if abs(x_pos - center_x) > half_width or abs(y_pos - center_y) > half_height: + continue + filtered.append(star) + return filtered + + +def nearest_reference_fallback_star_by_pixels(stars, x_value, y_value, max_sep_pixels=None, used_ids=None): + target_x = _finite_float(x_value) + target_y = _finite_float(y_value) + if target_x is None or target_y is None: + return None + + best_star = None + best_dist2 = None + for star in stars or []: + if used_ids and id(star) in used_ids: + continue + star_x = _finite_float(star.get('x')) + star_y = _finite_float(star.get('y')) + if star_x is None or star_y is None: + continue + dist2 = ((star_x - target_x) ** 2) + ((star_y - target_y) ** 2) + if best_dist2 is None or dist2 < best_dist2: + best_star = star + best_dist2 = dist2 + + if best_star is None: + return None + if max_sep_pixels is not None and best_dist2 is not None and best_dist2 > (float(max_sep_pixels) ** 2): + return None + return best_star + + +def select_reference_fallback_comparison_stars( + image_data, + image_shape, + target_pixel, + comp_count=REFERENCE_FALLBACK_COMPARISON_LIMIT, + min_comp_target_sep=REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS): + max_count = max(0, int(comp_count)) + if max_count == 0 or image_data is None or image_shape is None: + return [], [] + + target_pixel = np.asarray(target_pixel, dtype=float).reshape(-1) + if target_pixel.size < 2 or not np.all(np.isfinite(target_pixel[:2])): + return [], [] + target_x, target_y = float(target_pixel[0]), float(target_pixel[1]) + + stars = detect_reference_fallback_bright_stars(image_data) + comp_pool = filter_reference_fallback_stars_to_middle_fifty_percent( + dedupe_reference_fallback_stars(stars), + image_shape, + ) + detected_target = nearest_reference_fallback_star_by_pixels( + comp_pool, + target_x, + target_y, + max_sep_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + + used_ids = set() + if detected_target is not None: + used_ids.add(id(detected_target)) + + comp_candidates = [] + min_sep2 = max(float(min_comp_target_sep), 0.0) ** 2 + for star in comp_pool: + if id(star) in used_ids: + continue + dx = float(star.get('x', 0.0)) - target_x + dy = float(star.get('y', 0.0)) - target_y + if min_sep2 > 0.0 and ((dx * dx) + (dy * dy)) < min_sep2: + continue + comp_candidates.append(star) + if len(comp_candidates) >= max_count: + break + + comp_stars = [[float(star['x']), float(star['y'])] for star in comp_candidates] + return comp_stars, comp_candidates + + +def log_reference_fallback_comparison_candidates(comp_stars, detected_candidates): + if not comp_stars: + log_info( + "Warning: the nextastro_archive-style bright-star picker did not find any usable replacement " + "comparison stars on the new reference image.", + warn=True, + ) + return + + log_info( + f"Reference fallback replaced supplied comparison-star pixels with {len(comp_stars)} " + "image-detected bright-star candidate(s) selected like nextastro_archive " + "(central 50% of the frame, de-duplicated detections, at least " + f"{REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS:g} px from the target, brightest-first).", + warn=True, + ) + for index, star in enumerate(detected_candidates, start=1): + log_info( + f"Reference fallback comparison candidate #{index}: " + f"pixels=[{float(star['x']):.2f}, {float(star['y']):.2f}], " + f"aperture flux={float(star.get('flux', np.nan)):.3g}.", + warn=True, + ) + + def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, existing_comp_stars=None, field_catalog=None): calibration_stars = dict(existing_comp_stars or {}) @@ -10228,12 +10562,7 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist 'is_aavso_vsp': False, 'observed_filter': obs_filter, }) - label = nextastro_calibration_label(match) - unique_label = label - duplicate_index = 2 - while unique_label in calibration_stars: - unique_label = f"{label}-{duplicate_index}" - duplicate_index += 1 + unique_label = unique_nextastro_calibration_label(calibration_stars, match) calibration_stars[unique_label] = match existing_positions.add(tuple(comp_pos)) added_count += 1 @@ -13232,26 +13561,23 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet multiprocess_transformations=multiprocess_transformations, ) if dropped_pointing_files: - if abort_if_reference_frame_rejected( + reference_fallback = reference_frame_rejection_fallback_info( pointing_reference_file, dropped_pointing_files, ordered_inputfiles=pointing_precheck_inputfiles, - ): - ax.clear() - ax.set_title(target_name) - ax.set_ylabel('Normalized Flux') - ax.set_xlabel('Time (JD)') - ax.text( - 0.5, - 0.5, - "Reference image rejected by pointing precheck.\nSee log for details.", - transform=ax.transAxes, - ha='center', - va='center', - ) - plt.close(ax.figure) - return + ) plateStatus.initializeFilenames(list(inputfiles)) + else: + reference_fallback = None + if reference_fallback is not None and reference_fallback.get('next_reference_candidate') is None: + log_info( + "Error: all leading reference candidates were rejected by the pointing precheck; no usable " + "realtime reference image remains.", + error=True, + ) + return + if reference_fallback is not None: + pointing_alignment_transforms = {} bad_pixel_reference = None if detect_bad_pixels_before_photometry: @@ -13271,6 +13597,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet exotic_UIprevTPY = info_dict['tar_coords'][1] plateStatus.setCurrentFilename(inputfiles[0]) + header = get_first_image_header(inputfiles[0]) wcs_file = check_wcs(inputfiles[0], info_dict['save'], info_dict['plate_opt'], rt=True, use_nextastro_astrometry=use_nextastro_astrometry, ra=p_dict.get('ra'), dec=p_dict.get('dec'), pixel_scale=info_dict.get('pixel_scale'), @@ -13283,13 +13610,59 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet wcs_header = get_first_image_header(wcs_file) ra_file, dec_file = get_ra_dec(wcs_header, image_shape=first_image.shape) - tar_radec = (ra_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], - dec_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) + if reference_fallback is not None: + target_projection = estimate_target_pixel_from_ra_dec( + p_dict, + wcs_header, + first_image, + timeList[0] if timeList else img_time_bjd_tdb(header, p_dict, info_dict), + ) + if target_projection is None: + log_info( + "Error: could not estimate target coordinates from RA/Dec after removing the rejected " + "reference image.", + error=True, + ) + return + exotic_UIprevTPX, exotic_UIprevTPY, target_ra, target_dec = target_projection + info_dict['tar_coords'] = [exotic_UIprevTPX, exotic_UIprevTPY] + tar_radec = (target_ra, target_dec) + fallback_comp_stars, fallback_candidates = select_reference_fallback_comparison_stars( + first_image, + first_image.shape, + [exotic_UIprevTPX, exotic_UIprevTPY], + comp_count=1, + ) + log_reference_fallback_comparison_candidates( + fallback_comp_stars, + fallback_candidates, + ) + if fallback_comp_stars: + comp_star = fallback_comp_stars[0] + info_dict['comp_stars'] = comp_star + else: + log_info( + "Error: no replacement image-detected comparison star was available after removing " + "the rejected realtime reference image.", + error=True, + ) + return + else: + tar_radec = (ra_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], + dec_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) ra = ra_file[int(comp_star[1])][int(comp_star[0])] dec = dec_file[int(comp_star[1])][int(comp_star[0])] comp_radec.append((ra, dec)) + elif reference_fallback is not None: + log_info( + "Error: the original realtime reference image was removed, but the new reference image does not " + "have a usable WCS. EXOTIC cannot estimate target coordinates from RA/Dec or choose a replacement " + "image-detected comparison star without a new reference WCS.", + error=True, + ) + return target_and_comp_radec = None if tar_radec is not None and comp_radec: @@ -17819,12 +18192,11 @@ def _main_impl(): demosaic_mult=demosaic_mult, ) if dropped_pointing_files: - if abort_if_reference_frame_rejected( + reference_fallback = reference_frame_rejection_fallback_info( pointing_reference_file, dropped_pointing_files, ordered_inputfiles=pointing_precheck_inputfiles, - ): - return + ) times = times[pointing_keep_mask] jd_times = jd_times[pointing_keep_mask] finite_plot_times = times[np.isfinite(times)] @@ -17832,6 +18204,17 @@ def _main_impl(): if finite_plot_times.size: full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) plateStatus.initializeFilenames(list(inputfiles)) + else: + reference_fallback = None + if reference_fallback is not None and reference_fallback.get('next_reference_candidate') is None: + log_info( + "Error: all leading reference candidates were rejected by the pointing precheck; no usable " + "reference image remains.", + error=True, + ) + return + if reference_fallback is not None: + pointing_alignment_transforms = {} post_pointing_inputfile_count = int(len(inputfiles)) bad_pixel_reference = None @@ -17868,31 +18251,38 @@ def _main_impl(): # fit target in the first image and use it to determine aperture and annulus range inc = 0 - for ifile in inputfiles: - plateStatus.setCurrentFilename(ifile) - if bad_pixel_reference is not None: - first_image = load_calibrated_reduction_image( - ifile, - generalDark, - generalBias, - generalFlat, - demosaic_fmt, - demosaic_out, - demosaic_mult, - bad_pixel_reference=bad_pixel_reference, - ) - else: - first_image = fits.getdata(ifile) - try: - initial_centroid = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0) - if np.isnan(initial_centroid[0]): - inc += 1 + if reference_fallback is None: + for ifile in inputfiles: + plateStatus.setCurrentFilename(ifile) + if bad_pixel_reference is not None: + first_image = load_calibrated_reduction_image( + ifile, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) else: - break - except Exception: - inc += 1 - finally: - del first_image + first_image = fits.getdata(ifile) + try: + initial_centroid = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0) + if np.isnan(initial_centroid[0]): + inc += 1 + else: + break + except Exception: + inc += 1 + finally: + del first_image + else: + log_info( + "Skipping the old-pixel target precheck because the original reference image was " + "removed; the target will be projected from RA/Dec after the new reference WCS is ready.", + warn=True, + ) if inc > 0: log_info(f"Skipping first {inc} files - Target star not found") @@ -17924,36 +18314,98 @@ def _main_impl(): wcs_header = get_first_image_header(wcs_file) ra_wcs, dec_wcs = get_ra_dec(wcs_header, image_shape=reference_image.shape) - prefer_input_target_pixels = exotic_infoDict.get( - 'prefer_pixel_values_over_wcs_for_target', 'n' - ) - exotic_UIprevTPX, exotic_UIprevTPY = check_target_pixel_wcs( - exotic_UIprevTPX, - exotic_UIprevTPY, - pDict, - ra_wcs, - dec_wcs, - reference_image, - jd_times[0], - non_interactive_run=args.non_interactive_run, - wcs_header=wcs_header, - prefer_pixel_values_over_wcs_for_target=prefer_input_target_pixels, - ) - ra_dec_tar = (ra_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], - dec_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) + if reference_fallback is not None: + target_projection = estimate_target_pixel_from_ra_dec( + pDict, + wcs_header, + reference_image, + jd_times[0], + ) + if target_projection is None: + log_info( + "Error: could not estimate target coordinates from RA/Dec after removing the " + "rejected reference image.", + error=True, + ) + return + exotic_UIprevTPX, exotic_UIprevTPY, target_ra, target_dec = target_projection + exotic_infoDict['tar_coords'] = [exotic_UIprevTPX, exotic_UIprevTPY] + ra_dec_tar = (target_ra, target_dec) + else: + prefer_input_target_pixels = exotic_infoDict.get( + 'prefer_pixel_values_over_wcs_for_target', 'n' + ) + exotic_UIprevTPX, exotic_UIprevTPY = check_target_pixel_wcs( + exotic_UIprevTPX, + exotic_UIprevTPY, + pDict, + ra_wcs, + dec_wcs, + reference_image, + jd_times[0], + non_interactive_run=args.non_interactive_run, + wcs_header=wcs_header, + prefer_pixel_values_over_wcs_for_target=prefer_input_target_pixels, + ) + ra_dec_tar = (ra_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], + dec_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) auid = vsx_auid(ra_dec_tar[0], ra_dec_tar[1]) - check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], - use_nextastro_variability_server=args.use_nextastro_variability_server) + if reference_fallback is not None: + old_comp_count = len(exotic_infoDict['comp_stars']) + exotic_infoDict['comp_stars'] = [] + log_info( + f"Reference fallback discarded {old_comp_count} supplied comparison-star pixel " + "coordinate(s) because they were tied to the rejected reference image.", + warn=True, + ) - if exotic_infoDict['aavso_comp'] == 'y': + if exotic_infoDict['aavso_comp'] == 'y' and reference_fallback is None: vsp_comp_stars, chart_id = vsp_query(wcs_file,[header['NAXIS1'], header['NAXIS2']], exotic_infoDict['filter'], img_scale, user_comp_stars=exotic_infoDict['comp_stars'], user_targ_star = [ exotic_UIprevTPX, exotic_UIprevTPY ]) vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] + nextastro_field_catalog = None + try: + nextastro_field_catalog = nextastro_photometry_catalog_for_wcs( + wcs_file, + [header['NAXIS1'], header['NAXIS2']], + img_scale, + exotic_infoDict['filter'], + ) + except Exception as exc: + log_info( + "\nWarning: NextAstro full-field photometry catalog lookup failed " + f"({describe_retry_exception(exc)}). Will try per-comparison catalog lookups.", + warn=True, + ) + + if reference_fallback is not None: + fallback_comp_stars, fallback_candidates = select_reference_fallback_comparison_stars( + reference_image, + reference_image.shape, + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + ) + log_reference_fallback_comparison_candidates( + fallback_comp_stars, + fallback_candidates, + ) + if fallback_comp_stars: + exotic_infoDict['comp_stars'] = fallback_comp_stars + else: + log_info( + "Error: no replacement image-detected comparison stars were available after " + "removing the rejected reference image.", + error=True, + ) + return + + check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], + use_nextastro_variability_server=args.use_nextastro_variability_server) + while not exotic_infoDict['comp_stars']: log_info("\nThere are no comparison stars left as all of them were indicated as variable stars." "\nPlease reenter new comparison star coordinates.") @@ -17969,20 +18421,6 @@ def _main_impl(): # Build RA/Dec for comp after list is finalized (avoid off by one issues, etc ra_dec_wcs = build_comp_ra_dec(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) - nextastro_field_catalog = None - try: - nextastro_field_catalog = nextastro_photometry_catalog_for_wcs( - wcs_file, - [header['NAXIS1'], header['NAXIS2']], - img_scale, - exotic_infoDict['filter'], - ) - except Exception as exc: - log_info( - "\nWarning: NextAstro full-field photometry catalog lookup failed " - f"({describe_retry_exception(exc)}). Will try per-comparison catalog lookups.", - warn=True, - ) vsp_comp_stars = merge_nextastro_calibration_stars( exotic_infoDict['comp_stars'], ra_dec_wcs, @@ -17993,6 +18431,14 @@ def _main_impl(): vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) else: + if reference_fallback is not None: + log_info( + "Error: the original reference image was removed, but the new reference image does not " + "have a usable WCS. EXOTIC cannot estimate target coordinates from RA/Dec or choose " + "replacement image-detected comparison stars without a new reference WCS.", + error=True, + ) + return exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( exotic_infoDict['comp_stars'] ) diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 45ba840e..ee43825a 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -908,7 +908,7 @@ def fake_collect_transform_frame_pointings(inputfiles, frame_loader=None, return assert set(cached_transforms) == set(frames[:-1]) -def test_abort_if_reference_frame_rejected_reports_error_and_removal_recommendation(monkeypatch): +def test_reference_frame_rejection_fallback_reports_automatic_removal_and_reprojection(monkeypatch): messages = [] monkeypatch.setattr( @@ -917,7 +917,7 @@ def test_abort_if_reference_frame_rejected_reports_error_and_removal_recommendat lambda message, error=False, warn=False: messages.append((message, error, warn)), ) - result = exotic_module.abort_if_reference_frame_rejected( + result = exotic_module.reference_frame_rejection_fallback_info( "frame_0001.fits", ["frame_0001.fits", "frame_0002.fits", "frame_0003.fits"], ordered_inputfiles=[ @@ -928,19 +928,20 @@ def test_abort_if_reference_frame_rejected_reports_error_and_removal_recommendat ], ) - assert result is True - assert any("first usable image" in message and error for message, error, _ in messages) - assert any("frame_0002.fits" in message and error for message, error, _ in messages) + assert result["leading_rejected_files"] == ["frame_0001.fits", "frame_0002.fits", "frame_0003.fits"] + assert result["next_reference_candidate"] == "frame_0004.fits" + assert any("automatically removing" in message and warn for message, _, warn in messages) + assert any("target RA/Dec" in message and "nextastro_archive" in message and warn for message, _, warn in messages) assert any( - "remove or move these leading rejected frames" in message + "Automatically removed leading rejected frame(s)" in message and "frame_0001.fits, frame_0002.fits, frame_0003.fits" in message - and "frame_0004.fits" in message - and error - for message, error, _ in messages + and "Continuing from new reference image frame_0004.fits" in message + and warn + for message, _, warn in messages ) -def test_abort_if_reference_frame_rejected_only_recommends_consecutive_leading_rejections(monkeypatch): +def test_reference_frame_rejection_fallback_only_reports_consecutive_leading_rejections(monkeypatch): messages = [] monkeypatch.setattr( @@ -949,7 +950,7 @@ def test_abort_if_reference_frame_rejected_only_recommends_consecutive_leading_r lambda message, error=False, warn=False: messages.append((message, error, warn)), ) - result = exotic_module.abort_if_reference_frame_rejected( + result = exotic_module.reference_frame_rejection_fallback_info( "frame_0001.fits", ["frame_0001.fits", "frame_0003.fits"], ordered_inputfiles=[ @@ -960,18 +961,19 @@ def test_abort_if_reference_frame_rejected_only_recommends_consecutive_leading_r ], ) - assert result is True + assert result["leading_rejected_files"] == ["frame_0001.fits"] + assert result["next_reference_candidate"] == "frame_0002.fits" assert any( - "remove or move this rejected frame" in message + "Automatically removed leading rejected frame(s)" in message and "frame_0001.fits" in message - and "frame_0002.fits" in message + and "Continuing from new reference image frame_0002.fits" in message and "frame_0003.fits" not in message - and error - for message, error, _ in messages + and warn + for message, _, warn in messages ) -def test_abort_if_reference_frame_rejected_ignores_non_reference_rejections(monkeypatch): +def test_reference_frame_rejection_fallback_ignores_non_reference_rejections(monkeypatch): messages = [] monkeypatch.setattr( @@ -980,10 +982,37 @@ def test_abort_if_reference_frame_rejected_ignores_non_reference_rejections(monk lambda message, error=False, warn=False: messages.append((message, error, warn)), ) - result = exotic_module.abort_if_reference_frame_rejected( + result = exotic_module.reference_frame_rejection_fallback_info( "frame_0001.fits", ["frame_0002.fits", "frame_0003.fits"], ) - assert result is False + assert result is None assert messages == [] + + +def test_reference_fallback_comparison_stars_use_nextastro_archive_image_criteria(): + image = np.zeros((300, 300), dtype=float) + + def add_blob(x_pos, y_pos, value): + image[y_pos - 1:y_pos + 2, x_pos - 1:x_pos + 2] = value * 0.5 + image[y_pos, x_pos] = value + + add_blob(150, 150, 2000.0) # target location, excluded by detected-target match + add_blob(220, 220, 1200.0) + add_blob(80, 80, 900.0) + add_blob(180, 180, 1600.0) # within 50 px of target, excluded + add_blob(25, 25, 5000.0) # outside the central 50% frame, excluded + + comp_stars, candidates = exotic_module.select_reference_fallback_comparison_stars( + image, + image.shape, + target_pixel=[150, 150], + comp_count=2, + ) + + assert comp_stars == [[220.0, 220.0], [80.0, 80.0]] + assert [candidate["flux"] for candidate in candidates] == sorted( + [candidate["flux"] for candidate in candidates], + reverse=True, + ) diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 89ff38aa..864709a6 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -714,6 +714,68 @@ def test_internal_impact_parameter_samples_grazing_range_beyond_one(monkeypatch, assert fit._get_sample_bounds()["b"] == pytest.approx([0.0, 1.12]) +def test_internal_impact_parameter_transform_does_not_deepcopy_prior(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "ars": [10.0, 14.0], + "inc": [87.0, 90.0], + "tmid": [-0.005, 0.005], + } + + def fail_deepcopy(value, memo=None): + raise AssertionError("sampling transforms should not deepcopy parameter dictionaries") + + monkeypatch.setattr(elca.copy, "deepcopy", fail_deepcopy) + + sample_point = fit._sample_point_from_unit_cube(np.array([0.25, 0.5, 0.4, 0.75])) + unit_points = np.array([ + [0.25, 0.5, 0.4, 0.75], + [1.0, 0.25, 0.9, 0.5], + ]) + sample_points = fit._sample_point_from_unit_cube(unit_points) + physical = fit._physical_values_from_sample_point(sample_point) + sample_bounds = fit._get_sample_bounds() + + assert sample_point[0] == pytest.approx(0.09) + np.testing.assert_allclose( + sample_points, + np.vstack([fit._sample_point_from_unit_cube(row) for row in unit_points]), + ) + assert physical["b"] == pytest.approx(sample_point[2]) + assert "b" in sample_bounds + + +def test_unit_cube_transform_vectorizes_simple_bounds(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = False + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "inc": [87.0, 90.0], + "tmid": [-0.005, 0.005], + } + + unit_points = np.array([ + [0.0, 0.5, 1.0], + [1.0, 0.25, 0.0], + ]) + + np.testing.assert_allclose( + fit._sample_point_from_unit_cube(unit_points), + np.array([ + [0.08, 88.5, 0.005], + [0.12, 87.75, -0.005], + ]), + ) + + def test_nested_fit_can_keep_inclination_parameterization_when_requested(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index f0204c05..5a944cdf 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -2239,7 +2239,13 @@ def fake_run_nested( dataerr=np.asarray(call_fluxerr, dtype=float), airmass=np.asarray(call_airmass, dtype=float), transit=transit_profile.copy(), - parameters={"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a0": call_prior.get("a0", 1.0), "a2": 0.2}, + parameters={ + "tmid": 0.0, + "rprs": 0.1, + "inc": 89.0, + "a0": call_prior.get("a0", 1.0), + "a2": call_prior.get("a2", 0.2), + }, errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.001, "a2": 0.01}, residuals=np.zeros_like(call_flux, dtype=float), duration_expected=0.5, @@ -2284,9 +2290,11 @@ def fake_run_nested( assert len(captured["calls"]) == 2 final_call = captured["calls"][1] - assert final_call["fixed_flux_baseline"] is False - assert "a0" in final_call["bounds"] - assert "a2" in final_call["bounds"] + assert final_call["fixed_flux_baseline"] is True + assert final_call["prior"]["a0"] == pytest.approx(1.0) + assert final_call["prior"]["a2"] == pytest.approx(0.0) + assert "a0" not in final_call["bounds"] + assert "a2" not in final_call["bounds"] assert np.allclose(final_call["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) assert final_call["flux"][3] == pytest.approx(0.99, abs=1e-8) assert np.allclose(refit_flux, final_call["flux"]) From e90bf712f60d55ea47f0c73f339c49b8b49aa210 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Thu, 28 May 2026 08:57:41 +1000 Subject: [PATCH 060/116] widen sky annulus --- exotic/exotic.py | 2 +- exotic/plots.py | 2 +- tests/test_exotic_proper_motion.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index c49ce476..bdfb131f 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -11963,7 +11963,7 @@ def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_c ANNULUS_SIGMA_MIN = 6.0 ANNULUS_SIGMA_MAX = 15.0 SKY_ANNULUS_MIN_GAP_PIXELS = 2.0 -SKY_ANNULUS_MIN_FWHM_MULTIPLIER = 2.0 +SKY_ANNULUS_MIN_FWHM_MULTIPLIER = 3.0 SKY_ANNULUS_MIN_EFFECTIVE_PIXELS = 250.0 SKY_BACKGROUND_SIGMA_CLIP = 3.0 SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS = 3 diff --git a/exotic/plots.py b/exotic/plots.py index 4c3755f4..3ea310d1 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -99,7 +99,7 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc if sky_inner_radius is None or sky_outer_radius is None: local_sky_inner_radius = abs(aper) + 2.0 if np.isfinite(sigma) and sigma > 0: - local_sky_inner_radius = max(local_sky_inner_radius, 2.0 * 2.355 * float(sigma)) + local_sky_inner_radius = max(local_sky_inner_radius, 3.0 * 2.355 * float(sigma)) local_sky_outer_radius = max( local_sky_inner_radius + annulus, np.sqrt(local_sky_inner_radius ** 2 + 250.0 / np.pi), diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 5a944cdf..4a4284a5 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -1316,7 +1316,7 @@ def test_resolve_frame_aperture_radii_scales_sigma_grid(): def test_resolve_sky_annulus_geometry_enforces_fwhm_floor_and_min_sky_pixels(): geometry = resolve_sky_annulus_geometry(aperture_radius=1.5, annulus_width=2.0, psf_sigma=1.0) - assert geometry["inner_radius"] == pytest.approx(2.0 * 2.355) + assert geometry["inner_radius"] == pytest.approx(3.0 * 2.355) assert geometry["effective_sky_pixels"] == pytest.approx(250.0, abs=1e-9) assert geometry["annulus_width"] > 2.0 From 346071de3e02837e4a96668c2a266e21227ae4b6 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 28 May 2026 10:29:36 +1000 Subject: [PATCH 061/116] better QC for partial transits. --- exotic/exotic.py | 108 +++++++++++++++++++++++++++-- tests/test_exotic_proper_motion.py | 92 ++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 7 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index bdfb131f..ce21321c 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -751,6 +751,85 @@ def transit_qc_duration_score(duration_ratio): return float(np.clip(score, 0.0, 1.0)) +def estimate_midpoint_anchored_partial_duration(fit, assessment): + times = np.asarray(getattr(fit, 'time', []), dtype=float) + transit_model = np.asarray(getattr(fit, 'transit', []), dtype=float) + if times.shape != transit_model.shape or times.size < 2: + return np.nan + + parameters = getattr(fit, 'parameters', {}) or {} + tmid = coerce_finite_transit_qc_scalar(parameters.get('tmid', assessment.get('expected_tmid', np.nan))) + if not np.isfinite(tmid): + return np.nan + + finite_times = np.sort(times[np.isfinite(times)]) + if finite_times.size < 2: + return np.nan + cadence = float(np.nanmedian(np.diff(finite_times))) + if not np.isfinite(cadence) or cadence <= 0: + return np.nan + + in_transit = np.isfinite(times) & np.isfinite(transit_model) & (transit_model < 1.0) + if not np.any(in_transit): + return np.nan + + covers_ingress = bool(assessment.get('covers_ingress', False)) + covers_egress = bool(assessment.get('covers_egress', False)) + if covers_ingress and not covers_egress: + side_times = times[in_transit & (times <= tmid)] + if side_times.size == 0: + return np.nan + half_duration = tmid - float(np.nanmin(side_times)) + 0.5 * cadence + elif covers_egress and not covers_ingress: + side_times = times[in_transit & (times >= tmid)] + if side_times.size == 0: + return np.nan + half_duration = float(np.nanmax(side_times)) - tmid + 0.5 * cadence + else: + return np.nan + + if not np.isfinite(half_duration) or half_duration <= 0: + return np.nan + return float(2.0 * half_duration) + + +def transit_qc_duration_consistency_measurement(fit): + duration_measured = getattr(fit, 'duration_measured', np.nan) + assessment = getattr(fit, 'pre_ultranest_transit_coverage', None) + if not isinstance(assessment, dict) or not assessment.get('valid', False): + return duration_measured, True, None + + covers_ingress = bool(assessment.get('covers_ingress', False)) + covers_mid_transit = bool(assessment.get('covers_mid_transit', False)) + covers_egress = bool(assessment.get('covers_egress', False)) + if covers_ingress and covers_egress: + return duration_measured, True, None + + observed_segment = assessment.get('observed_segment') or 'partial transit' + if covers_mid_transit and (covers_ingress or covers_egress): + partial_duration = estimate_midpoint_anchored_partial_duration(fit, assessment) + if np.isfinite(partial_duration) and partial_duration > 0: + edge = 'ingress' if covers_ingress else 'egress' + note = ( + "Duration consistency used a midpoint-anchored partial estimate: fitted Tmid to " + f"observed {edge}, doubled to estimate the full duration ({observed_segment})." + ) + return partial_duration, True, note + + note = ( + "Duration consistency was not scored because the expected transit was only partially " + f"observed ({observed_segment}) and the midpoint-to-edge duration could not be measured." + ) + return np.nan, False, note + + note = ( + "Duration consistency was not scored because the expected transit was only partially " + f"observed ({observed_segment}); either both ingress and egress, or mid-transit plus " + "one transit edge, are needed to estimate duration." + ) + return np.nan, False, note + + def transit_qc_saturating_score(value, scale): try: value = float(value) @@ -1039,6 +1118,14 @@ def compute_transit_qc_ktmf(summary): else: deviation_detail = "expected-value deviation disabled or unavailable" + duration_note = summary.get('duration_consistency_note') + if np.isfinite(summary.get('duration_ratio', np.nan)): + duration_detail = f"{summary.get('duration_ratio', np.nan):.2f}x expected duration" + if duration_note: + duration_detail = f"{duration_detail}; {duration_note}" + else: + duration_detail = duration_note or "n/a" + raw_components = [ { 'key': 'model_evidence', @@ -1066,11 +1153,7 @@ def compute_transit_qc_ktmf(summary): 'key': 'duration_consistency', 'label': 'Duration Consistency', 'score': transit_qc_duration_score(summary.get('duration_ratio', np.nan)), - 'detail': ( - f"{summary.get('duration_ratio', np.nan):.2f}x expected duration" - if np.isfinite(summary.get('duration_ratio', np.nan)) - else "n/a" - ), + 'detail': duration_detail, }, { 'key': 'eebls_depth_snr', @@ -1303,6 +1386,9 @@ def evaluate_transit_detection_qc(fit): 'flat_model_note': None, 'rprs_sigma': np.nan, 'duration_ratio': np.nan, + 'duration_measured_for_qc': np.nan, + 'duration_consistency_applicable': True, + 'duration_consistency_note': None, 'eebls_depth_snr': np.nan, 'residual_scatter': np.nan, 'use_deviation_from_expected_transit_in_qc': bool(use_deviation_from_expected_transit_in_qc), @@ -1433,9 +1519,13 @@ def evaluate_transit_detection_qc(fit): summary['rprs_sigma'] = float(abs(rprs) / rprs_err) duration_expected = getattr(fit, 'duration_expected', np.nan) - duration_measured = getattr(fit, 'duration_measured', np.nan) + duration_measured, duration_applicable, duration_note = transit_qc_duration_consistency_measurement(fit) + summary['duration_consistency_applicable'] = bool(duration_applicable) + summary['duration_consistency_note'] = duration_note + summary['duration_measured_for_qc'] = duration_measured if ( - np.isfinite(duration_expected) + duration_applicable + and np.isfinite(duration_expected) and duration_expected > 0 and np.isfinite(duration_measured) and duration_measured >= 0 @@ -1509,6 +1599,8 @@ def evaluate_transit_detection_qc(fit): ) if np.isfinite(summary['duration_ratio']): + if summary.get('duration_consistency_note'): + notes.append(summary['duration_consistency_note']) if ( summary['duration_ratio'] < TRANSIT_QC_DURATION_RATIO_MIN or summary['duration_ratio'] > TRANSIT_QC_DURATION_RATIO_MAX @@ -1516,6 +1608,8 @@ def evaluate_transit_detection_qc(fit): notes.append( f"The measured transit duration is {summary['duration_ratio']:.2f}x the modeled duration." ) + elif summary.get('duration_consistency_note'): + notes.append(summary['duration_consistency_note']) if np.isfinite(summary['eebls_depth_snr']) and summary['eebls_depth_snr'] < TRANSIT_QC_MIN_EEBLS_SNR: notes.append( diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 4a4284a5..bccce867 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -3313,6 +3313,98 @@ def test_evaluate_transit_detection_qc_passes_strong_model_with_low_rprs_precisi assert "not used as a transit-detection veto" in " ".join(summary["notes"]) +def test_evaluate_transit_detection_qc_uses_midpoint_anchored_duration_for_partial(): + times = np.linspace(0.0, 3.0, 13) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[times <= 2.25] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + time=times, + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + transit=transit_model, + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.0, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [-0.1, 0.1], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=2.5, + pre_ultranest_transit_coverage={ + "valid": True, + "covers_ingress": False, + "covers_mid_transit": True, + "covers_egress": True, + "observed_segment": "mid-transit to egress", + "expected_tmid": 0.0, + }, + ) + + summary = evaluate_transit_detection_qc(fit) + contributions_by_label = { + contribution["label"]: contribution + for contribution in summary["ktmf_contributions"] + } + + assert summary["duration_measured_for_qc"] == pytest.approx(4.75) + assert summary["duration_ratio"] == pytest.approx(0.95) + assert contributions_by_label["Duration Consistency"]["available"] is True + assert "midpoint-anchored partial estimate" in contributions_by_label["Duration Consistency"]["detail"] + + +def test_evaluate_transit_detection_qc_skips_duration_for_edge_only_partial(): + times = np.linspace(-3.0, -0.25, 12) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[times >= -2.5] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, + -0.0001, 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + time=times, + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + transit=transit_model, + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.0, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [-0.1, 0.1], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=2.75, + pre_ultranest_transit_coverage={ + "valid": True, + "covers_ingress": True, + "covers_mid_transit": False, + "covers_egress": False, + "observed_segment": "ingress-only partial", + "expected_tmid": 0.0, + }, + ) + + summary = evaluate_transit_detection_qc(fit) + contributions_by_label = { + contribution["label"]: contribution + for contribution in summary["ktmf_contributions"] + } + + assert np.isnan(summary["duration_ratio"]) + assert summary["duration_consistency_applicable"] is False + assert contributions_by_label["Duration Consistency"]["available"] is False + assert "only partially observed" in contributions_by_label["Duration Consistency"]["detail"] + + def test_evaluate_transit_detection_qc_fails_when_flat_model_is_better(): transit_model = np.ones(21, dtype=float) transit_model[8:13] = 0.99 From 4f190373442a77e64a986a36285a80eb0d4a8ec6 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Mon, 1 Jun 2026 06:21:54 +1000 Subject: [PATCH 062/116] fixed missing baseline uncertainty and font adjustments --- exotic/api/elca.py | 25 ++++++++++++++++--- exotic/exotic.py | 39 +++++++++++++++++++++++------- exotic/plots.py | 28 +++------------------ tests/test_elca_baseline.py | 6 +++++ tests/test_exotic_proper_motion.py | 10 ++++++++ tests/test_plots.py | 14 +++++------ 6 files changed, 78 insertions(+), 44 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 50b05466..3e61bbde 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -991,16 +991,35 @@ def _plot_baseline_model_uncertainty(self, ax, x_values, times, sort_index, labe x_sorted = x_values[sort_index] lower_sorted = np.asarray(lower, dtype=float)[sort_index] upper_sorted = np.asarray(upper, dtype=float)[sort_index] - return ax.fill_between( + band = ax.fill_between( x_sorted, lower_sorted, upper_sorted, color='gold', - alpha=0.28, + alpha=0.32, linewidth=0, - zorder=1.7, + zorder=2.1, label=label, ) + ax.plot( + x_sorted, + lower_sorted, + color='gold', + linestyle='--', + linewidth=0.9, + alpha=0.78, + zorder=3.2, + ) + ax.plot( + x_sorted, + upper_sorted, + color='gold', + linestyle='--', + linewidth=0.9, + alpha=0.78, + zorder=3.2, + ) + return band def _uses_internal_impact_parameter(self): return ( diff --git a/exotic/exotic.py b/exotic/exotic.py index ce21321c..04662570 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -2717,23 +2717,46 @@ def selected_final_live_point_target(enabled=None): return base_live_points, target_live_points +def coerce_fixed_baseline_error(value, default=None): + try: + value = float(value) + except (TypeError, ValueError): + return default + if np.isfinite(value) and value >= 0: + return value + return default + + def baseline_fixed_errors_from_fit(fit): errors = getattr(fit, 'errors', {}) if fit is not None else {} fixed_errors = {} if isinstance(errors, dict): for key in ('a0', 'a1', 'a2'): - value = errors.get(key) - try: - value = float(value) - except (TypeError, ValueError): - continue - if np.isfinite(value) and value >= 0: + value = coerce_fixed_baseline_error(errors.get(key)) + if value is not None: fixed_errors[key] = value if 'a0' in fixed_errors and 'a1' not in fixed_errors: fixed_errors['a1'] = fixed_errors['a0'] return fixed_errors +def baseline_fixed_errors_from_oot_parameter_result(result): + fixed_errors = {} + if not isinstance(result, dict): + return fixed_errors + + a0_error = coerce_fixed_baseline_error(result.get('a0_error')) + if a0_error is not None: + fixed_errors['a0'] = a0_error + fixed_errors['a1'] = a0_error + + a2_error = coerce_fixed_baseline_error(result.get('a2_error')) + if a2_error is not None: + fixed_errors['a2'] = a2_error + + return fixed_errors + + def build_full_resolution_final_prior_from_previous_fit(previous_fit, p_dict): previous_parameters = getattr(previous_fit, 'parameters', {}) if not isinstance(previous_parameters, dict): @@ -7706,9 +7729,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( log_info("Prepared out-of-transit airmass/baseline parameter constraints for a fallback final transit refit.") log_info(baseline_parameter_result['note']) baseline_fit_mask = np.asarray(baseline_parameter_result['oot_mask'], dtype=bool) - baseline_fixed_errors = { - 'a2': baseline_parameter_result.get('a2_error', 0.0), - } + baseline_fixed_errors = baseline_fixed_errors_from_oot_parameter_result(baseline_parameter_result) baseline_constrained_prior['a0'] = baseline_parameter_result['a0'] baseline_constrained_prior['a1'] = baseline_parameter_result['a0'] baseline_constrained_prior['a2'] = baseline_parameter_result['a2'] diff --git a/exotic/plots.py b/exotic/plots.py index 3ea310d1..355f50e6 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -553,35 +553,13 @@ def _finite_plot_float(value): def _stellar_variability_reference_label(vsp_param, comparison_label): - band = vsp_param.get('mag_band') or 'V' - observed_filter = vsp_param.get('observed_filter') comp_ra = _finite_plot_float(vsp_param.get('comp_ra')) comp_dec = _finite_plot_float(vsp_param.get('comp_dec')) - comparison_parts = [] - if vsp_param.get('is_aavso_vsp', True) and comparison_label: - comparison_parts.append(f"Label={comparison_label}") if comp_ra is not None and comp_dec is not None: - comparison_parts.append(f"RA={comp_ra:.7f}") - comparison_parts.append(f"Dec={comp_dec:.7f}") - elif comparison_label: - comparison_parts.append(str(comparison_label)) + return f"RA={comp_ra:.6f}, Dec={comp_dec:.6f}" - detail_parts = [] - if observed_filter not in (None, ''): - detail_parts.append(f"Observed filter={observed_filter}") - - mag_text = magnitude_text(band, vsp_param.get('cmag'), vsp_param.get('cmag_err')) - if mag_text is not None: - detail_parts.append(mag_text) - - label_lines = [] - if comparison_parts: - label_lines.append(", ".join(comparison_parts)) - if detail_parts: - label_lines.append(", ".join(detail_parts)) - - return "\n".join(label_lines) + return str(comparison_label) if comparison_label else "" def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): @@ -604,7 +582,7 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): first_param = vsp_params[0] band = first_param.get('mag_band') or 'V' reference_label = _stellar_variability_reference_label(first_param, vsp_auid_comp) - ax.set_title(f"{s_name}\nComparison: {reference_label}") + ax.set_title(f"{s_name}\n{reference_label}" if reference_label else s_name) ax.set_ylabel(f"Magnitude ({band})") ax.set_xlabel("Time [JD]") fig.tight_layout() diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 864709a6..ca6a4aaf 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -523,9 +523,15 @@ def test_plot_bestfit_can_draw_baseline_uncertainty_band(monkeypatch, tmp_path): fig, axes = fit.plot_bestfit(show_baseline_uncertainty=True) labels = [artist.get_label() for artist in axes[0].collections] legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) + baseline_line_count = sum( + 1 + for line in axes[0].lines + if line.get_linestyle() == "--" and line.get_color() == "gold" + ) assert "_nolegend_" in labels assert r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty' not in legend_text + assert baseline_line_count == 2 plt.close(fig) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index bccce867..b6822830 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -2232,6 +2232,7 @@ def fake_run_nested( "prior": dict(call_prior), "bounds": dict(call_bounds), "fixed_flux_baseline": kwargs.get("fixed_flux_baseline"), + "fixed_parameter_errors": dict(kwargs.get("fixed_parameter_errors", {})), }) return types.SimpleNamespace( time=np.asarray(call_times, dtype=float), @@ -2293,6 +2294,11 @@ def fake_run_nested( assert final_call["fixed_flux_baseline"] is True assert final_call["prior"]["a0"] == pytest.approx(1.0) assert final_call["prior"]["a2"] == pytest.approx(0.0) + assert final_call["fixed_parameter_errors"]["a0"] > 0 + assert final_call["fixed_parameter_errors"]["a1"] == pytest.approx( + final_call["fixed_parameter_errors"]["a0"], + ) + assert final_call["fixed_parameter_errors"]["a2"] == pytest.approx(0.0) assert "a0" not in final_call["bounds"] assert "a2" not in final_call["bounds"] assert np.allclose(final_call["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) @@ -2371,6 +2377,10 @@ def fake_lc_fitter( assert captured["calls"][1]["baseline_fit_mask"].tolist() == [False, False, False, False, True, True, True] assert "a0" not in captured["calls"][1]["bounds"] assert "a2" not in captured["calls"][1]["bounds"] + assert captured["calls"][1]["fixed_parameter_errors"]["a0"] > 0 + assert captured["calls"][1]["fixed_parameter_errors"]["a1"] == pytest.approx( + captured["calls"][1]["fixed_parameter_errors"]["a0"], + ) assert "a2" in captured["calls"][1]["fixed_parameter_errors"] assert fit.oot_baseline_parameter_fit_applied is True assert fit.oot_baseline_detrending_applied is False diff --git a/tests/test_plots.py b/tests/test_plots.py index c0e94b39..00499f2f 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -176,7 +176,7 @@ def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.pdf").exists() -def test_plot_stellar_variability_labels_reference_band_and_coordinates(tmp_path, monkeypatch): +def test_plot_stellar_variability_labels_reference_coordinates(tmp_path, monkeypatch): titles = [] ylabels = [] original_set_title = Axes.set_title @@ -213,11 +213,10 @@ def spy_set_ylabel(self, label, *args, **kwargs): "NextAstro-123", ) - assert "RA=10.1000000" in titles[-1] - assert "Dec=-20.2000000" in titles[-1] - assert "Observed filter=CV" in titles[-1] - assert "r=12.345 +/- 0.067" in titles[-1] - assert "Dec=-20.2000000\nObserved filter=CV" in titles[-1] + assert titles[-1] == "Host Star\nRA=10.100000, Dec=-20.200000" + assert "Comparison:" not in titles[-1] + assert "Observed filter" not in titles[-1] + assert "r=12.345 +/- 0.067" not in titles[-1] assert ylabels[-1] == "Magnitude (r)" assert (tmp_path / "temp" / "Stellar_Variability.png").exists() @@ -252,7 +251,8 @@ def spy_set_title(self, label, *args, **kwargs): "NextAstro-123", ) - assert "Observed filter=MObs CV" in titles[-1] + assert titles[-1] == "Host Star\nRA=10.100000, Dec=-20.200000" + assert "Observed filter" not in titles[-1] assert "99.99" not in titles[-1] assert "V=" not in titles[-1] From 8c28d802a7d592eab142f56f8c99a29aa942716d Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 4 Jun 2026 09:36:51 +1000 Subject: [PATCH 063/116] don't produce filenames with spaces --- ...ailedFitSummary_HAT-P-32b_2026-04-28.json} | 0 examples/tess/candidates/toi.py | 28 +++++++++------ examples/tess/candidates/toi_individ_lc.py | 18 ++++++---- examples/tess/tess.py | 35 +++++++++++-------- examples/tess/tess_individ_lc.py | 27 ++++++++------ exotic/api/output_aavso.py | 34 ++++++++++++++---- exotic/utils.py | 16 ++++++--- ...ailedFitSummary_HAT-P-32b_2026-04-28.json} | 0 tests/test_exotic_proper_motion.py | 14 ++++---- tests/test_output_files.py | 28 +++++++-------- tests/test_utils.py | 22 ++++++++++-- 11 files changed, 148 insertions(+), 74 deletions(-) rename .pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/{FailedFitSummary_HAT-P-32 b_2026-04-28.json => FailedFitSummary_HAT-P-32b_2026-04-28.json} (100%) rename manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/{FailedFitSummary_HAT-P-32 b_2026-04-28.json => FailedFitSummary_HAT-P-32b_2026-04-28.json} (100%) diff --git a/.pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json b/.pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json similarity index 100% rename from .pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json rename to .pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json diff --git a/examples/tess/candidates/toi.py b/examples/tess/candidates/toi.py index 09d6b766..58cb7df9 100644 --- a/examples/tess/candidates/toi.py +++ b/examples/tess/candidates/toi.py @@ -27,8 +27,14 @@ from wotan import flatten from exotic.api.elca import transit, lc_fitter from exotic.api.output_aavso import OutputFiles +from exotic.utils import safe_output_filename from transitleastsquares import transitleastsquares + +def output_file_path(output_dir, prefix, *parts, extension): + return os.path.join(output_dir, safe_output_filename(prefix, *parts, extension=extension)) + + def tap_query(base_url, query, dataframe=True): # Table Access Protocol (TAP) query uri_full = base_url @@ -222,7 +228,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # aperture plot tpf.plot(aperture_mask=aper_final) - plt.savefig(os.path.join(planetdir, planetname + f"_sector_{sector}_aperture.png")) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "aperture", extension="png")) plt.close() # remove first ~30 min of data after any big gaps @@ -285,7 +291,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] ax.set_xlabel("Time [TBJD]") ax.set_ylim([np.percentile(flux, 0.1), np.percentile(flux, 99.9)]) plt.tight_layout() - plt.savefig(os.path.join(planetdir, planetname + f"_sector_{sector}_trend.png")) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "trend", extension="png")) plt.close() # combine all sectors @@ -305,7 +311,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create dataframe for entire light curve df = pd.DataFrame({'time': time, 'flux': flux * trend, 'flux_err': flux_err * trend, 'sector': alls}) - df.to_csv(os.path.join(planetdir, planetname + "_lightcurve.csv"), index=False) + df.to_csv(output_file_path(planetdir, planetname, "lightcurve", extension="csv"), index=False) # fit transit for each epoch period = prior['pl_orbper'] @@ -391,11 +397,11 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create plots fig, ax = myfit.plot_bestfit(title=f"{prior['pl_name']} Global Fit") ax[0].set_ylim([np.percentile(flux, 1) * 0.99, np.percentile(flux, 99) * 1.01]) - plt.savefig(os.path.join(planetdir, planetname + "_global_fit.png")) + plt.savefig(output_file_path(planetdir, planetname, "global_fit", extension="png")) plt.close() myfit.plot_triangle() - plt.savefig(os.path.join(planetdir, planetname + "_global_triangle.png")) + plt.savefig(output_file_path(planetdir, planetname, "global_triangle", extension="png")) plt.close() # update priors from best fit @@ -412,7 +418,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] prior['pl_orbinclerr2'] = -myfit.errors['inc'] # save prior to disk - with open(os.path.join(planetdir, planetname + "_prior.json"), 'w', encoding='utf8') as json_file: + with open(output_file_path(planetdir, planetname, "prior", extension="json"), 'w', encoding='utf8') as json_file: json.dump(prior, json_file, indent=4) # save results to state vector @@ -444,7 +450,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] plt.xlabel('Period (days)') plt.plot(results.periods, results.power, color='black', lw=0.5) plt.xlim(0, max(results.periods)) - plt.savefig(os.path.join(planetdir, planetname + "_periodogram.png")) + plt.savefig(output_file_path(planetdir, planetname, "periodogram", extension="png")) plt.close() # save global fit data if above certain SNR @@ -455,7 +461,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] if snr < 1: with open("notes.txt", 'w') as f: f.write(f"Skipping individual light curve fits b.c SNR = {snr:.2f}") - pickle.dump(sv, open(os.path.join(planetdir, planetname + "_data.pkl"), "wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) raise(Exception(f"Skipping individual light curve fits b.c SNR = {snr:.2f}")) period = myfit.parameters['per'] @@ -536,15 +542,15 @@ def check_std(time, flux, dt=0.5): # dt = [hr] tmidstr = str(np.round(myfit.parameters['tmid'], 2)).replace('.', '_') fig, ax = myfit.plot_bestfit(title=f"{prior['pl_name']} - Sector {lcdata['sector']}", bin_dt=0.5 / 24.) - plt.savefig(os.path.join(planetdir, f"{tmidstr}_" + planetname + "_lightcurve.png")) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "lightcurve", extension="png")) plt.close() fig = myfit.plot_triangle() - plt.savefig(os.path.join(planetdir, f"{tmidstr}_" + planetname + "_posterior.png")) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "posterior", extension="png")) plt.close() csv_lk = OutputFiles(myfit, prior, infoDict, planetdir) csv_lk.aavso_csv(airmass, u0, u1, u2, u3, tmidstr) csv_lk.aavso(airmass, u0, u1, u2, u3, tmidstr) - pickle.dump(sv, open(os.path.join(planetdir, planetname + "_data.pkl"), "wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) diff --git a/examples/tess/candidates/toi_individ_lc.py b/examples/tess/candidates/toi_individ_lc.py index 237dfe92..615e7f7f 100644 --- a/examples/tess/candidates/toi_individ_lc.py +++ b/examples/tess/candidates/toi_individ_lc.py @@ -27,8 +27,14 @@ from wotan import flatten from exotic.api.elca import transit, lc_fitter from exotic.api.output_aavso import OutputFiles +from exotic.utils import safe_output_filename from transitleastsquares import transitleastsquares + +def output_file_path(output_dir, prefix, *parts, extension): + return os.path.join(output_dir, safe_output_filename(prefix, *parts, extension=extension)) + + def sigma_clip(ogdata, dt, iterations=1): mask = np.ones(ogdata.shape, dtype=bool) for i in range(iterations): @@ -203,7 +209,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # aperture plot tpf.plot(aperture_mask=aper_final) - plt.savefig(os.path.join(planetdir, planetname + f"_sector_{sector}_aperture.png")) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "aperture", extension="png")) plt.close() # remove first ~30 min of data after any big gaps @@ -267,7 +273,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] ax.set_xlabel("Time [TBJD]") ax.set_ylim([np.percentile(flux, 0.1), np.percentile(flux, 99.9)]) plt.tight_layout() - plt.savefig(os.path.join(planetdir, planetname + f"_sector_{sector}_trend.png")) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "trend", extension="png")) plt.close() # combine all sectors @@ -287,7 +293,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create dataframe for entire light curve df = pd.DataFrame({'time': time, 'flux': flux * trend, 'flux_err': flux_err * trend, 'sector': alls}) - df.to_csv(os.path.join(planetdir, planetname + "_lightcurve.csv"), index=False) + df.to_csv(output_file_path(planetdir, planetname, "lightcurve", extension="csv"), index=False) # fit transit for each epoch period = prior['pl_orbper'] @@ -458,11 +464,11 @@ def check_std(time, flux, dt=0.5): # dt = [hr] tmidstr = str(np.round(myfit.parameters['tmid'], 2)).replace('.', '_') fig, ax = myfit.plot_bestfit(title=f"{prior['pl_name']} - Sector {lcdata['sector']}", bin_dt=0.5 / 24.) - plt.savefig(os.path.join(planetdir, f"{tmidstr}_" + planetname + "_lightcurve.png")) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "lightcurve", extension="png")) plt.close() fig = myfit.plot_triangle() - plt.savefig(os.path.join(planetdir, f"{tmidstr}_" + planetname + "_posterior.png")) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "posterior", extension="png")) plt.close() csv_data = { @@ -475,4 +481,4 @@ def check_std(time, flux, dt=0.5): # dt = [hr] csv_lk.aavso_csv(airmass, u0, u1, u2, u3, tmidstr) csv_lk.aavso(airmass, u0, u1, u2, u3, tmidstr) - pickle.dump(sv, open(os.path.join(planetdir, planetname + "_data.pkl"), "wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) diff --git a/examples/tess/tess.py b/examples/tess/tess.py index db43e8c3..f8017589 100644 --- a/examples/tess/tess.py +++ b/examples/tess/tess.py @@ -33,8 +33,14 @@ from wotan import flatten from exotic.api.elca import transit, lc_fitter from exotic.api.output_aavso import OutputFiles +from exotic.utils import safe_output_filename from transitleastsquares import transitleastsquares + +def output_file_path(output_dir, prefix, *parts, extension): + return os.path.join(output_dir, safe_output_filename(prefix, *parts, extension=extension)) + + def tap_query(base_url, query, dataframe=True): # table access protocol query @@ -166,8 +172,9 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # https://exo.mast.stsci.edu/ # load prior from disk or download - if os.path.exists(os.path.join(planetdir,planetname+"_prior.json")): - prior = json.load(open(os.path.join(planetdir,planetname+"_prior.json"),"r")) + prior_path = output_file_path(planetdir, planetname, "prior", extension="json") + if os.path.exists(prior_path): + prior = json.load(open(prior_path, "r")) else: # download prior from web if "TOI" in args.target: @@ -302,7 +309,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] #os.mkdir(os.path.join(planetdir,"lightcurves")) # save prior to disk - with open(os.path.join(planetdir,planetname+"_prior.json"), 'w', encoding ='utf8') as json_file: + with open(prior_path, 'w', encoding ='utf8') as json_file: json.dump(prior, json_file, indent=4) if len(prior) == 0: @@ -395,7 +402,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # aperture plot tpf.plot(aperture_mask=aper_final) - plt.savefig( os.path.join(planetdir, planetname+f"_sector_{sector}_aperture.png") ) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "aperture", extension="png")) plt.close() # remove first ~30 min of data after any big gaps @@ -467,7 +474,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] plt.tight_layout() #if not os.path.exists(os.path.join(planetdir, "lightcurves")): #os.makedirs(os.path.join(planetdir, "lightcurves")) - plt.savefig( os.path.join(planetdir, planetname+f"_sector_{sector}_trend.png") ) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "trend", extension="png")) plt.close() # combine all sectors @@ -487,7 +494,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create dataframe for entire light curve df = pd.DataFrame({'time':time, 'flux':flux*trend, 'flux_err':flux_err*trend, 'sector':alls}) - df.to_csv( os.path.join(planetdir, planetname+"_lightcurve.csv"), index=False) + df.to_csv(output_file_path(planetdir, planetname, "lightcurve", extension="csv"), index=False) # fit transit for each epoch period = prior['pl_orbper'] @@ -576,11 +583,11 @@ def check_std(time, flux, dt=0.5): # dt = [hr] fig,ax = myfit.plot_bestfit(title=f"{args.target} Global Fit") # set y_limit between 1 and 99 percentile ax[0].set_ylim([np.percentile(flux, 1)*0.99, np.percentile(flux,99)*1.01]) - plt.savefig( os.path.join( planetdir, planetname+"_global_fit.png")) + plt.savefig(output_file_path(planetdir, planetname, "global_fit", extension="png")) plt.close() myfit.plot_triangle() - plt.savefig( os.path.join( planetdir, planetname+"_global_triangle.png")) + plt.savefig(output_file_path(planetdir, planetname, "global_triangle", extension="png")) plt.close() # update priors @@ -606,7 +613,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] prior['pl_orbinclerr2'] = -myfit.errors['inc'] # save prior to disk - with open(os.path.join(planetdir,planetname+"_prior.json"), 'w', encoding ='utf8') as json_file: + with open(prior_path, 'w', encoding ='utf8') as json_file: json.dump(prior, json_file, indent=4) # save results to state vector @@ -639,7 +646,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] plt.xlabel('Period (days)') plt.plot(results.periods, results.power, color='black', lw=0.5) plt.xlim(0, max(results.periods)) - plt.savefig( os.path.join( planetdir, planetname+"_periodogram.png")) + plt.savefig(output_file_path(planetdir, planetname, "periodogram", extension="png")) plt.close() sv['tls'] = { @@ -660,7 +667,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] with open("notes.txt", 'w') as f: f.write(f"Skipping individual light curve fits b.c SNR = {snr:.2f}") # save global fit data - pickle.dump(sv, open(os.path.join(planetdir, planetname+"_data.pkl"),"wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) raise(Exception(f"Skipping individual light curve fits b.c SNR = {snr:.2f}")) # prepare for individual fits @@ -761,12 +768,12 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # save bestfit fig,ax = myfit.plot_bestfit(title=f"{args.target} - Sector {lcdata['sector']}", bin_dt=0.5/24.) - plt.savefig( os.path.join(planetdir, f"{tmidstr}_"+planetname+"_lightcurve.png") ) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "lightcurve", extension="png")) plt.close() # save posterior fig = myfit.plot_triangle() - plt.savefig( os.path.join(planetdir, f"{tmidstr}_"+planetname+"_posterior.png") ) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "posterior", extension="png")) plt.close() csv_data = { @@ -781,4 +788,4 @@ def check_std(time, flux, dt=0.5): # dt = [hr] csv_lk.aavso(airmass,u0,u1,u2,u3, tmidstr) # save sv pickle - pickle.dump(sv, open(os.path.join(planetdir, planetname+"_data.pkl"),"wb")) \ No newline at end of file + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) diff --git a/examples/tess/tess_individ_lc.py b/examples/tess/tess_individ_lc.py index b06e33d2..a96b720d 100644 --- a/examples/tess/tess_individ_lc.py +++ b/examples/tess/tess_individ_lc.py @@ -27,8 +27,14 @@ from wotan import flatten from exotic.api.elca import transit, lc_fitter from exotic.api.output_aavso import OutputFiles +from exotic.utils import safe_output_filename from transitleastsquares import transitleastsquares + +def output_file_path(output_dir, prefix, *parts, extension): + return os.path.join(output_dir, safe_output_filename(prefix, *parts, extension=extension)) + + def tap_query(base_url, query, dataframe=True): # table access protocol query @@ -151,8 +157,9 @@ def check_std(time, flux, dt=0.5): # dt = [hr] raise(Exception(f"no data for: {args.target}")) # https://exo.mast.stsci.edu/ # load prior from disk or download - if os.path.exists(os.path.join(planetdir,planetname+"_prior.json")): - prior = json.load(open(os.path.join(planetdir,planetname+"_prior.json"),"r")) + prior_path = output_file_path(planetdir, planetname, "prior", extension="json") + if os.path.exists(prior_path): + prior = json.load(open(prior_path, "r")) else: # download prior from web if "TOI" in args.target: @@ -275,7 +282,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] #os.mkdir(os.path.join(planetdir,"lightcurves")) # save prior to disk - with open(os.path.join(planetdir,planetname+"_prior.json"), 'w', encoding ='utf8') as json_file: + with open(prior_path, 'w', encoding ='utf8') as json_file: json.dump(prior, json_file, indent=4) if len(prior) == 0: @@ -368,7 +375,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # aperture plot tpf.plot(aperture_mask=aper_final) - plt.savefig( os.path.join(planetdir, planetname+f"_sector_{sector}_aperture.png") ) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "aperture", extension="png")) plt.close() # remove first ~30 min of data after any big gaps @@ -435,7 +442,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] plt.tight_layout() #if not os.path.exists(os.path.join(planetdir, "lightcurves")): #os.makedirs(os.path.join(planetdir, "lightcurves")) - plt.savefig( os.path.join(planetdir, planetname+f"_sector_{sector}_trend.png") ) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "trend", extension="png")) plt.close() # combine all sectors @@ -455,7 +462,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create dataframe for entire light curve df = pd.DataFrame({'time':time, 'flux':flux*trend, 'flux_err':flux_err*trend, 'sector':alls}) - df.to_csv( os.path.join(planetdir, planetname+"_lightcurve.csv"), index=False) + df.to_csv(output_file_path(planetdir, planetname, "lightcurve", extension="csv"), index=False) # fit transit for each epoch period = prior['pl_orbper'] @@ -635,12 +642,12 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # save bestfit fig,ax = myfit.plot_bestfit(title=f"{args.target} - Sector {lcdata['sector']}", bin_dt=0.5/24.) - plt.savefig( os.path.join(planetdir, f"{tmidstr}_"+planetname+"_lightcurve.png") ) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "lightcurve", extension="png")) plt.close() # save posterior fig = myfit.plot_triangle() - plt.savefig( os.path.join(planetdir, f"{tmidstr}_"+planetname+"_posterior.png") ) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "posterior", extension="png")) plt.close() csv_data = { @@ -658,7 +665,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] print(f"Failed to create AAVSO csv for {args.target} - Sector {lcdata['sector']}") # save sv pickle - pickle.dump(sv, open(os.path.join(planetdir, planetname+"_data.pkl"),"wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) # O-C plot tmids = np.array([lc['pars']['tmid'] for lc in sv['lightcurves']]) @@ -674,4 +681,4 @@ def check_std(time, flux, dt=0.5): # dt = [hr] ratios = ratios[dmask] # TODO finish making O-C plot? - # use example from exotic.api.nested_linear_fitter \ No newline at end of file + # use example from exotic.api.nested_linear_fitter diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 6c70e12b..49fbeadc 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -43,9 +43,9 @@ import re try: - from .utils import round_to_2 + from .utils import round_to_2, safe_output_filename except ImportError: - from utils import round_to_2 + from utils import round_to_2, safe_output_filename try: from .version import __version__ except ImportError: @@ -142,7 +142,12 @@ def __init__(self, fit, p_dict, i_dict, planetdir): self.dir = Path(planetdir) def final_lightcurve(self, phase): - params_file = self.dir / f"FinalLightCurve_{self.plname}_TESS.csv" + params_file = self.dir / safe_output_filename( + "FinalLightCurve", + self.plname, + "TESS", + extension="csv", + ) with params_file.open('w') as f: f.write(f"# FINAL TIMESERIES OF {self.p_dict['pl_name']}\n") @@ -154,7 +159,12 @@ def final_lightcurve(self, phase): f.write(f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}\n") def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min_aper=None, min_annul=None): - params_file = self.dir / f"FinalParams_{self.plname}_TESS.json" + params_file = self.dir / safe_output_filename( + "FinalParams", + self.plname, + "TESS", + extension="json", + ) params_num = { "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " @@ -211,7 +221,13 @@ def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): hash_id = hash_object.hexdigest()[:32] #params_file = self.dir / f"TESS_{hash_id}_{self.plname}_{tmidstr}_AAVSO.txt" - params_file = self.dir / f"{tmidstr}_{hash_id}_{self.plname}_AAVSO.txt" + params_file = self.dir / safe_output_filename( + tmidstr, + hash_id, + self.plname, + "AAVSO", + extension="txt", + ) # 2459642_61_5164d266e1755aead98dbec0f26e7b7c_gj436b_AAVSO with params_file.open('w') as f: @@ -278,7 +294,13 @@ def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" - params_file = self.dir / f"TESS_{tmidstr}_{self.p_dict['pl_name']}_lightcurve.csv" + params_file = self.dir / safe_output_filename( + "TESS", + tmidstr, + self.p_dict['pl_name'], + "lightcurve", + extension="csv", + ) with params_file.open('w') as f: f.write("#TYPE=EXOPLANET\n" # fixed diff --git a/exotic/utils.py b/exotic/utils.py index df3e3ac0..2adadd21 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -23,18 +23,26 @@ *(f'LPT{i}' for i in range(1, 10)), } _WINDOWS_ILLEGAL_FILENAME_CHARS_RE = re.compile(r'[<>:"/\\|?*\x00-\x1f\x7f]') +_FILENAME_WHITESPACE_RE = re.compile(r'\s+') MAX_APPARENT_MAGNITUDE = 30.0 MAGNITUDE_DECIMAL_PLACES = 3 MINIMUM_MAGNITUDE_ERROR = 0.001 +def _clean_filename_text(value): + cleaned = _WINDOWS_ILLEGAL_FILENAME_CHARS_RE.sub('-', str(value or '')) + cleaned = _FILENAME_WHITESPACE_RE.sub('', cleaned) + return cleaned.rstrip(' .') + + def sanitize_filename_component(value, fallback='output'): """Return one filename component that is safe on Windows, macOS, and Linux.""" - cleaned = _WINDOWS_ILLEGAL_FILENAME_CHARS_RE.sub('-', str(value or '')) - cleaned = cleaned.rstrip(' .') + cleaned = _clean_filename_text(value) if cleaned in {'', '.', '..'}: - cleaned = fallback + cleaned = _clean_filename_text(fallback) + if cleaned in {'', '.', '..'}: + cleaned = 'output' device_stem = cleaned.split('.', 1)[0].upper() if device_stem in _WINDOWS_RESERVED_FILENAME_STEMS: cleaned = f'_{cleaned}' @@ -56,7 +64,7 @@ def safe_output_filename(prefix, *parts, extension): stem_parts = [str(prefix), *(str(part) for part in parts)] safe_stem = sanitize_filename_component('_'.join(stem_parts), fallback=str(prefix or 'output')) - ext = str(extension or '') + ext = _FILENAME_WHITESPACE_RE.sub('', str(extension or '')) if ext and not ext.startswith('.'): ext = f'.{ext}' return f'{safe_stem}{ext}' diff --git a/manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json b/manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json similarity index 100% rename from manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32 b_2026-04-28.json rename to manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index b6822830..b795273e 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -200,7 +200,7 @@ def plot_triangle(self): final_dir = tmp_path / "final" source_temp = source_dir / "temp" source_temp.mkdir(parents=True) - source_plot = source_temp / f"Triangle_{planet_name}_{observation_date}.png" + source_plot = source_temp / "Triangle_TOI-1728b_2024-12-14.png" source_plot.write_bytes(b"stale-selected-comp-6") fit = DummyFit() @@ -212,9 +212,9 @@ def plot_triangle(self): source_dir=source_dir, ) - assert output_path == final_dir / f"FinalTriangle_{planet_name}_{observation_date}.png" + assert output_path == final_dir / "FinalTriangle_TOI-1728b_2024-12-14.png" assert output_path.read_bytes() == b"regenerated-final" - assert (final_dir / "temp" / f"Triangle_{planet_name}_{observation_date}.png").read_bytes() == b"regenerated-final" + assert (final_dir / "temp" / "Triangle_TOI-1728b_2024-12-14.png").read_bytes() == b"regenerated-final" assert fit.called is True @@ -226,7 +226,7 @@ def test_comparison_candidate_triangle_plot_uses_candidate_specific_name(tmp_pat 6, ) - assert output_path.name == "Comp7_Triangle_WASP-80 b_2025-06-22.png" + assert output_path.name == "Comp7_Triangle_WASP-80b_2025-06-22.png" assert output_path.parent == tmp_path / "comp7" / "temp" @@ -315,12 +315,12 @@ def plot_triangle(self): assert fit.called is True assert output_path.read_bytes() == b"regenerated" assert output_path.parent == tmp_path / "final" - assert output_path.name == "FinalTriangle_TOI-1728 b_2024-12-14.png" + assert output_path.name == "FinalTriangle_TOI-1728b_2024-12-14.png" assert ( tmp_path / "final" / "temp" - / "Triangle_TOI-1728 b_2024-12-14.png" + / "Triangle_TOI-1728b_2024-12-14.png" ).read_bytes() == b"regenerated" @@ -5443,7 +5443,7 @@ def fake_save(save_dir, provisional_fit, final_fit, p_dict, observation_date, co assert result["attempts"][0]["fit_diagnostics"]["failed_stage"] == "transit_qc" failed_run_dir = result["attempts"][0]["failed_run_dir"] assert failed_run_dir is not None - assert (tmp_path / "comp_1_failed" / "temp" / "FailedFitSummary_HAT-P-32 b_2026-04-28.json").exists() + assert (tmp_path / "comp_1_failed" / "temp" / "FailedFitSummary_HAT-P-32b_2026-04-28.json").exists() assert Path(failed_run_dir).exists() diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 4be3e737..5870f457 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -200,7 +200,7 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): None, ) - output_file = tmp_path / "AAVSO_HAT-P-32 b_2020-01-01.txt" + output_file = tmp_path / "AAVSO_HAT-P-32b_2020-01-01.txt" output_text = output_file.read_text(encoding="utf-8") assert "#OBSDATE=2020-01-01" in output_text @@ -260,7 +260,7 @@ def test_aavso_output_omits_obsname_header_when_blank(tmp_path): None, ) - output_file = tmp_path / "AAVSO_HAT-P-32 b_2020-01-01.txt" + output_file = tmp_path / "AAVSO_HAT-P-32b_2020-01-01.txt" output_text = output_file.read_text(encoding="utf-8") assert "#OBSNAME=" not in output_text @@ -450,7 +450,7 @@ def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" output_text = output_file.read_text(encoding="utf-8") assert "Airmass correction" in output_text @@ -481,7 +481,7 @@ def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path vsp_params=vsp_params, ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] reference = final_params["Variable Reference Star"] @@ -503,7 +503,7 @@ def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclinati vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" output_data = json.loads(output_file.read_text(encoding="utf-8")) final_params = output_data["FINAL PLANETARY PARAMETERS"] keys = list(final_params) @@ -536,7 +536,7 @@ def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertaintie vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] assert final_params["Mid-Transit Time (Tmid)"].endswith("+/- 0.0001 BJD_TDB") @@ -564,8 +564,8 @@ def test_final_planetary_params_can_publish_accepted_copy_to_root(tmp_path): publish_to_root=True, ) - temp_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" - root_file = tmp_path / "FinalParams_HAT-P-32 b_2020-01-01.json" + temp_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + root_file = tmp_path / "FinalParams_HAT-P-32b_2020-01-01.json" assert temp_file.exists() assert root_file.exists() @@ -601,7 +601,7 @@ def test_final_planetary_params_reports_adaptive_aperture_summary(tmp_path): adaptive_summary=adaptive_summary, ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" output_text = output_file.read_text(encoding="utf-8") assert "Adaptive Aperture Scale" in output_text @@ -662,7 +662,7 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" output_text = output_file.read_text(encoding="utf-8") assert "Transit detection QC" in output_text @@ -755,7 +755,7 @@ def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): photometry_info=photometry_info, ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" output_data = json.loads(output_file.read_text(encoding="utf-8")) final_params = output_data["FINAL PLANETARY PARAMETERS"] @@ -786,7 +786,7 @@ def test_final_planetary_params_reports_absolute_fit_quality(tmp_path): vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32 b_2020-01-01.json" + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" output_data = json.loads(output_file.read_text(encoding="utf-8")) final_params = output_data["FINAL PLANETARY PARAMETERS"] @@ -849,7 +849,7 @@ def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_p None, ) - output_file = tmp_path / "AAVSO_HAT-P-32 b_2020-01-01.txt" + output_file = tmp_path / "AAVSO_HAT-P-32b_2020-01-01.txt" output_text = output_file.read_text(encoding="utf-8") assert "Am1=0 +/- 0" in output_text @@ -1054,7 +1054,7 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): }, ) - output_text = (tmp_path / "AAVSO_HAT-P-32 b_2020-01-01.txt").read_text(encoding="utf-8") + output_text = (tmp_path / "AAVSO_HAT-P-32b_2020-01-01.txt").read_text(encoding="utf-8") results = aavso_json_header(output_text, "RESULTS-XC") assert "a/R*" in results diff --git a/tests/test_utils.py b/tests/test_utils.py index 4270a177..fea9d693 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,14 +11,32 @@ def test_filename_date_token_uses_date_only_for_iso_timestamp(): def test_safe_output_filename_sanitizes_filename_chars(): filename = safe_output_filename( "BestFit", - "XO-1/b", + "XO-1/b ", filename_date_token("2026-05-06T19:51:13.964-0700"), - extension="png", + extension=" png", ) assert filename == "BestFit_XO-1-b_2026-05-06.png" +def test_safe_output_filename_removes_spaces_from_planet_names(): + filename = safe_output_filename( + "FinalLightCurve", + "Kepler-12 b", + "03-JUN-2026", + extension="png", + ) + + assert filename == "FinalLightCurve_Kepler-12b_03-JUN-2026.png" + assert " " not in filename + + +def test_sanitize_filename_component_cleans_fallback(): + filename = sanitize_filename_component(" ", fallback="bad fallback") + + assert filename == "badfallback" + + class TestUserInput: """tests the `user_input()` function""" From 855794b3e60a3ab9222f5b79da1171dc25540d9c Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 7 Jun 2026 08:40:20 +1000 Subject: [PATCH 064/116] Move QC up higher. Also a protection for RA in hours. --- exotic/exotic.py | 281 ++++++++++++++++++++++++++++++++++--- exotic/output_files.py | 27 +++- tests/test_centroid_wcs.py | 88 ++++++++++++ tests/test_output_files.py | 7 + 4 files changed, 381 insertions(+), 22 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 04662570..42ef7e14 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -9778,6 +9778,174 @@ def any_projected_coord_out_of_frame(coords, image_shape): return False +def project_ra_dec_to_wcs_pixel(ra, dec, wcs_header): + x_pixel, y_pixel = WCS(wcs_header).all_world2pix(ra, dec, 0) + x_pixel = float(np.asarray(x_pixel).reshape(-1)[0]) + y_pixel = float(np.asarray(y_pixel).reshape(-1)[0]) + return x_pixel, y_pixel + + +def _representative_obs_time(obs_times): + if obs_times is None: + return None + + try: + obs_times = np.asarray(obs_times, dtype=float).reshape(-1) + except (TypeError, ValueError): + return None + + finite_obs_times = obs_times[np.isfinite(obs_times)] + if finite_obs_times.size == 0: + return None + return float(np.nanmedian(finite_obs_times)) + + +def target_ra_dec_for_wcs_filter(info_dict, obs_times=None): + obs_time = _representative_obs_time(obs_times) + if obs_time is not None: + target_ra, target_dec = update_coordinates_with_proper_motion(info_dict, obs_time) + return float(target_ra), float(target_dec) + + return float(info_dict['ra']), float(info_dict['dec']) + + +def wcs_target_projection_status(image_header, target_ra, target_dec): + wcs = search_wcs_from_header(image_header) + if not wcs.is_celestial: + return None, None + + x_pixel, y_pixel = project_ra_dec_to_wcs_pixel(target_ra, target_dec, image_header) + width, height = _resolve_wcs_image_dimensions(image_header) + image_shape = (height, width) + return pixel_within_image(x_pixel, y_pixel, image_shape), (x_pixel, y_pixel) + + +def collect_wcs_target_coverage(inputfiles, info_dict, obs_times=None): + keep_mask = np.ones(len(inputfiles), dtype=bool) + dropped_files = [] + + try: + target_ra, target_dec = target_ra_dec_for_wcs_filter(info_dict, obs_times=obs_times) + except Exception as exc: + log_info( + "Warning: target WCS precheck could not determine target RA/Dec " + f"({exc}); skipping target-in-frame filtering.", + warn=True, + ) + return keep_mask, dropped_files + + for index, file_name in enumerate(inputfiles): + try: + image_header = get_first_image_header(file_name) + projection_inside, _ = wcs_target_projection_status(image_header, target_ra, target_dec) + except Exception: + projection_inside = False + + if projection_inside is False: + keep_mask[index] = False + dropped_files.append(str(file_name)) + + return keep_mask, dropped_files + + +def count_wcs_target_projection_hits(inputfiles, target_ra, target_dec): + celestial_count = 0 + inside_count = 0 + + for file_name in inputfiles: + try: + image_header = get_first_image_header(file_name) + projection_inside, _ = wcs_target_projection_status(image_header, target_ra, target_dec) + except Exception: + continue + + if projection_inside is None: + continue + + celestial_count += 1 + if projection_inside: + inside_count += 1 + + return inside_count, celestial_count + + +def maybe_reinterpret_decimal_ra_hours_from_wcs(inputfiles, info_dict, obs_times=None): + if len(inputfiles) == 0: + return False + + try: + original_ra = float(info_dict['ra']) + float(info_dict['dec']) + except (KeyError, TypeError, ValueError): + return False + + if not (0.0 <= original_ra <= 24.0): + return False + + try: + target_ra, target_dec = target_ra_dec_for_wcs_filter(info_dict, obs_times=obs_times) + except Exception: + return False + + primary_inside_count, celestial_count = count_wcs_target_projection_hits( + inputfiles, + target_ra, + target_dec, + ) + if celestial_count == 0 or primary_inside_count > 0: + return False + + alternate_info_dict = dict(info_dict) + alternate_info_dict['ra'] = original_ra * 15.0 + try: + alternate_ra, alternate_dec = target_ra_dec_for_wcs_filter( + alternate_info_dict, + obs_times=obs_times, + ) + except Exception: + return False + + alternate_inside_count, _ = count_wcs_target_projection_hits( + inputfiles, + alternate_ra, + alternate_dec, + ) + if alternate_inside_count <= primary_inside_count: + return False + + info_dict['ra'] = alternate_info_dict['ra'] + log_info( + "Target WCS precheck: interpreted decimal target RA as hours because the supplied RA projected " + f"into 0/{celestial_count} WCS frame(s), while RA*15 projected into " + f"{alternate_inside_count}/{celestial_count} frame(s). Using RA={info_dict['ra']:.7f} deg.", + warn=True, + ) + return True + + +def filter_wcs_target_out_of_frame_frames(inputfiles, info_dict, obs_times=None, ignore_header_wcs=False): + inputfiles = np.array(inputfiles) + keep_mask = np.ones(len(inputfiles), dtype=bool) + if ignore_header_wcs or len(inputfiles) == 0: + return inputfiles, keep_mask, [] + + keep_mask, dropped_files = collect_wcs_target_coverage( + inputfiles, + info_dict, + obs_times=obs_times, + ) + if not dropped_files: + return inputfiles, keep_mask, [] + + retained_files = inputfiles[keep_mask] + log_info( + f"Target WCS precheck: {len(retained_files)}/{len(inputfiles)} frame(s) remain after dropping " + f"{len(dropped_files)} file(s) where the target RA/Dec projects outside the image." + ) + log_file_preview(dropped_files, "Target WCS precheck dropped files") + return retained_files, keep_mask, dropped_files + + def check_target_pixel_wcs(input_x_pixel, input_y_pixel, info_dict, ra_list, dec_list, image_data, obs_time, non_interactive_run=False, wcs_header=None, prefer_pixel_values_over_wcs_for_target=False): @@ -10333,7 +10501,7 @@ def estimate_target_pixel_from_ra_dec(info_dict, wcs_header, image_data, obs_tim centroid_margin=7.5): target_ra, target_dec = update_coordinates_with_proper_motion(info_dict, obs_time) try: - x_pixel, y_pixel = WCS(wcs_header).all_world2pix(target_ra, target_dec, 0) + x_pixel, y_pixel = project_ra_dec_to_wcs_pixel(target_ra, target_dec, wcs_header) except Exception as exc: log_info( "Warning: Could not project target RA/Dec onto the new reference image " @@ -10342,8 +10510,6 @@ def estimate_target_pixel_from_ra_dec(info_dict, wcs_header, image_data, obs_tim ) return None - x_pixel = float(np.asarray(x_pixel).reshape(-1)[0]) - y_pixel = float(np.asarray(y_pixel).reshape(-1)[0]) if not pixel_within_image(x_pixel, y_pixel, image_data.shape): log_info( "Warning: target RA/Dec projects outside the new reference image; " @@ -13644,6 +13810,8 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet plateStatus.initializeFilenames(info_dict['images']) inputfiles = corruption_check(info_dict['images']) + if not ignore_header_wcs: + maybe_reinterpret_decimal_ra_hours_from_wcs(inputfiles, p_dict) # time sort images times = [] for ifile in inputfiles: @@ -13658,14 +13826,42 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet plateStatus.setObsTime(obsTime) si = np.argsort(times) + times = np.array(times)[si] inputfiles = np.array(inputfiles)[si] - inputfiles, _, dropped_wcs_files = filter_sparse_missing_wcs_frames( + inputfiles, wcs_keep_mask, dropped_wcs_files = filter_sparse_missing_wcs_frames( inputfiles, ignore_header_wcs=ignore_header_wcs, max_missing_fraction=bad_wcs_threshold_fraction, ) if dropped_wcs_files: + times = times[wcs_keep_mask] + plateStatus.initializeFilenames(list(inputfiles)) + target_wcs_precheck_inputfiles = np.array(inputfiles, copy=True) + target_wcs_reference_file = inputfiles[0] if len(inputfiles) else None + inputfiles, target_wcs_keep_mask, dropped_target_wcs_files = filter_wcs_target_out_of_frame_frames( + inputfiles, + p_dict, + obs_times=times, + ignore_header_wcs=ignore_header_wcs, + ) + if dropped_target_wcs_files: + target_reference_fallback = reference_frame_rejection_fallback_info( + target_wcs_reference_file, + dropped_target_wcs_files, + ordered_inputfiles=target_wcs_precheck_inputfiles, + rejection_label="Target WCS precheck", + ) + times = times[target_wcs_keep_mask] plateStatus.initializeFilenames(list(inputfiles)) + else: + target_reference_fallback = None + if len(inputfiles) == 0: + log_info( + "Error: target WCS precheck removed every frame because the target RA/Dec projects outside " + "each image.", + error=True, + ) + return pointing_precheck_inputfiles = np.array(inputfiles, copy=True) pointing_reference_file = inputfiles[0] if len(inputfiles) else None inputfiles, _, dropped_pointing_files, pointing_alignment_transforms = filter_pointing_outlier_frames( @@ -13676,14 +13872,15 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet multiprocess_transformations=multiprocess_transformations, ) if dropped_pointing_files: - reference_fallback = reference_frame_rejection_fallback_info( + pointing_reference_fallback = reference_frame_rejection_fallback_info( pointing_reference_file, dropped_pointing_files, ordered_inputfiles=pointing_precheck_inputfiles, ) + reference_fallback = pointing_reference_fallback or target_reference_fallback plateStatus.initializeFilenames(list(inputfiles)) else: - reference_fallback = None + reference_fallback = target_reference_fallback if reference_fallback is not None and reference_fallback.get('next_reference_candidate') is None: log_info( "Error: all leading reference candidates were rejected by the pointing precheck; no usable " @@ -18005,8 +18202,10 @@ def _main_impl(): demosaic_out = None precheck_inputfile_count = None post_wcs_inputfile_count = None + post_target_wcs_inputfile_count = None post_pointing_inputfile_count = None dropped_wcs_files = [] + dropped_target_wcs_files = [] dropped_pointing_files = [] ignore_header_wcs = False bad_wcs_threshold_fraction = np.nan @@ -18203,6 +18402,10 @@ def _main_impl(): plateStatus.initializeFilenames(exotic_infoDict['images']) inputfiles = corruption_check(exotic_infoDict['images']) + early_ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) + if not early_ignore_header_wcs and maybe_reinterpret_decimal_ra_hours_from_wcs(inputfiles, pDict): + userpDict['ra'] = pDict['ra'] + userpDict['dec'] = pDict['dec'] # time sort images times, jd_times = [], [] log_info(f"Reading FITS timestamps and converting to BJD_TDB for {len(inputfiles)} frame(s).") @@ -18282,6 +18485,38 @@ def _main_impl(): jd_times = jd_times[wcs_keep_mask] plateStatus.initializeFilenames(list(inputfiles)) post_wcs_inputfile_count = int(len(inputfiles)) + target_wcs_precheck_inputfiles = np.array(inputfiles, copy=True) + target_wcs_reference_file = inputfiles[0] if len(inputfiles) else None + inputfiles, target_wcs_keep_mask, dropped_target_wcs_files = filter_wcs_target_out_of_frame_frames( + inputfiles, + pDict, + obs_times=jd_times, + ignore_header_wcs=ignore_header_wcs, + ) + if dropped_target_wcs_files: + target_reference_fallback = reference_frame_rejection_fallback_info( + target_wcs_reference_file, + dropped_target_wcs_files, + ordered_inputfiles=target_wcs_precheck_inputfiles, + rejection_label="Target WCS precheck", + ) + times = times[target_wcs_keep_mask] + jd_times = jd_times[target_wcs_keep_mask] + finite_plot_times = times[np.isfinite(times)] + full_plot_time_range = None + if finite_plot_times.size: + full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) + plateStatus.initializeFilenames(list(inputfiles)) + else: + target_reference_fallback = None + if len(inputfiles) == 0: + log_info( + "Error: target WCS precheck removed every frame because the target RA/Dec projects outside " + "each image.", + error=True, + ) + return + post_target_wcs_inputfile_count = int(len(inputfiles)) pointing_precheck_inputfiles = np.array(inputfiles, copy=True) pointing_reference_file = inputfiles[0] if len(inputfiles) else None inputfiles, pointing_keep_mask, dropped_pointing_files, pointing_alignment_transforms = filter_pointing_outlier_frames( @@ -18307,11 +18542,12 @@ def _main_impl(): demosaic_mult=demosaic_mult, ) if dropped_pointing_files: - reference_fallback = reference_frame_rejection_fallback_info( + pointing_reference_fallback = reference_frame_rejection_fallback_info( pointing_reference_file, dropped_pointing_files, ordered_inputfiles=pointing_precheck_inputfiles, ) + reference_fallback = pointing_reference_fallback or target_reference_fallback times = times[pointing_keep_mask] jd_times = jd_times[pointing_keep_mask] finite_plot_times = times[np.isfinite(times)] @@ -18320,7 +18556,7 @@ def _main_impl(): full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) plateStatus.initializeFilenames(list(inputfiles)) else: - reference_fallback = None + reference_fallback = target_reference_fallback if reference_fallback is not None and reference_fallback.get('next_reference_candidate') is None: log_info( "Error: all leading reference candidates were rejected by the pointing precheck; no usable " @@ -20135,8 +20371,24 @@ def _main_impl(): # print final extracted planetary parameters ####################################################################### + transit_qc = getattr(myfit, 'transit_qc', None) + qc_status = None + qc_summary = None + qc_ktmf_metric = np.nan + if isinstance(transit_qc, dict) and transit_qc: + qc_status = str(transit_qc.get('status', 'unknown')).upper() + qc_summary = transit_qc.get('summary') + qc_ktmf_metric = _finite_float(transit_qc.get('ktmf_metric'), default=np.nan) + log_info("\n*********************************************************") log_info("FINAL PLANETARY PARAMETERS\n") + if qc_status: + if qc_summary: + log_info(f" Transit detection QC: {qc_status} - {qc_summary}") + else: + log_info(f" Transit detection QC: {qc_status}") + if np.isfinite(qc_ktmf_metric): + log_info(f" KTMF: {qc_ktmf_metric:.2f} / 5.00") log_info(f" Mid-Transit Time [BJD_TDB]: {round_to_2(myfit.parameters['tmid'], myfit.errors['tmid'])} +/- {round_to_2(myfit.errors['tmid'])}") log_info(f" Radius Ratio (Planet/Star) [Rp/R*]: {round_to_2(myfit.parameters['rprs'], myfit.errors['rprs'])} +/- {round_to_2(myfit.errors['rprs'])}") for depth_label, depth_text in formatted_transit_depth_parameters(myfit, pDict).items(): @@ -20154,17 +20406,10 @@ def _main_impl(): else: log_info(f" Airmass coefficient 1: {round_to_2(myfit.parameters['a1'], myfit.errors['a1'])} +/- {round_to_2(myfit.errors['a1'])}") log_info(f" Airmass coefficient 2: {round_to_2(myfit.parameters['a2'], myfit.errors['a2'])} +/- {round_to_2(myfit.errors['a2'])}") - transit_qc = getattr(myfit, 'transit_qc', None) - if transit_qc: + if isinstance(transit_qc, dict) and transit_qc: residual_scatter = transit_qc.get('residual_scatter', np.nan) if np.isfinite(residual_scatter): log_info(f"Residual scatter around full model fit: {residual_scatter * 100.0:.4f}%") - qc_status = str(transit_qc.get('status', 'unknown')).upper() - qc_summary = transit_qc.get('summary') - if qc_summary: - log_info(f" Transit detection QC: {qc_status} - {qc_summary}") - else: - log_info(f" Transit detection QC: {qc_status}") if np.isfinite(transit_qc.get('deviation_from_expected_value', np.nan)): log_info( f" Deviation From Expected Value: {transit_qc['deviation_from_expected_value']:.2f} / 1.00" @@ -20173,8 +20418,6 @@ def _main_impl(): log_info( f" Expected-value Rp/R* sigma: {transit_qc['rprs_deviation_sigma']:.2f}" ) - if np.isfinite(transit_qc.get('ktmf_metric', np.nan)): - log_info(f" KTMF: {transit_qc['ktmf_metric']:.2f} / 5.00") for contribution in transit_qc.get('ktmf_contributions', []): log_info(f" {format_ktmf_contribution(contribution)}") if fitsortext == 1: @@ -20258,6 +20501,7 @@ def _main_impl(): aavso_frame_filtering_info = { 'initial_frame_count': precheck_inputfile_count, 'after_missing_wcs_filter_frame_count': post_wcs_inputfile_count, + 'after_target_wcs_filter_frame_count': post_target_wcs_inputfile_count, 'final_prephotometry_frame_count': post_pointing_inputfile_count, 'ignore_header_wcs': ignore_header_wcs, 'bad_wcs_threshold_percent': ( @@ -20267,6 +20511,7 @@ def _main_impl(): ), 'pointing_rejection_sigma': pointing_rejection_sigma, 'dropped_missing_wcs_files': dropped_wcs_files, + 'dropped_target_wcs_files': dropped_target_wcs_files, 'dropped_pointing_files': dropped_pointing_files, } aavso_astrometry_info = { diff --git a/exotic/output_files.py b/exotic/output_files.py index d258a71b..a42db10d 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -549,6 +549,22 @@ def format_ktmf_metric(value): return f"{value:.2f} / 5.00" if np.isfinite(value) else "n/a" +def format_transit_qc_headline_final_params(transit_qc): + params = {} + if not isinstance(transit_qc, dict) or not transit_qc: + return params + + qc_status = transit_qc.get('status') + if qc_status: + params["Transit detection QC"] = str(qc_status).upper() + + qc_ktmf = finite_float(transit_qc.get('ktmf_metric')) + if np.isfinite(qc_ktmf): + params["KTMF"] = f"{qc_ktmf:.2f} / 5.00" + + return params + + def format_optional_metric(label, value, precision=2): value = finite_float(value) if not np.isfinite(value): @@ -793,6 +809,7 @@ def build_aavso_frame_filtering_metadata(fit, frame_filtering_info): for source_key, target_key in ( ('dropped_missing_wcs_files', 'missing_wcs_rejections'), + ('dropped_target_wcs_files', 'target_wcs_rejections'), ('dropped_pointing_files', 'pointing_rejections'), ): if source_key in payload: @@ -968,7 +985,8 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor if np.isfinite(median_flux) and median_flux != 0: qc_residual_scatter = float(abs(residuals.reshape(-1)[0]) / median_flux) - params_num = { + headline_params = format_transit_qc_headline_final_params(transit_qc) + core_params = { "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB", "Ratio of Planet to Stellar Radius (Rp/R*)": f"{round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- " @@ -978,10 +996,11 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor } depth_params = formatted_transit_depth_parameters(self.fit, self.p_dict) params_num = { - "Mid-Transit Time (Tmid)": params_num["Mid-Transit Time (Tmid)"], - "Ratio of Planet to Stellar Radius (Rp/R*)": params_num["Ratio of Planet to Stellar Radius (Rp/R*)"], + **headline_params, + "Mid-Transit Time (Tmid)": core_params["Mid-Transit Time (Tmid)"], + "Ratio of Planet to Stellar Radius (Rp/R*)": core_params["Ratio of Planet to Stellar Radius (Rp/R*)"], **depth_params, - "Orbital Inclination (inc)": params_num["Orbital Inclination (inc)"], + "Orbital Inclination (inc)": core_params["Orbital Inclination (inc)"], } ars_text = format_parameter_with_error( self.fit.parameters.get('ars'), diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index ee43825a..62669b69 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -823,6 +823,94 @@ def test_filter_sparse_missing_wcs_frames_keeps_files_at_three_percent_or_higher assert dropped == [] +def test_filter_wcs_target_out_of_frame_frames_drops_only_projected_misses(monkeypatch): + def make_wcs_header(center_ra): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [center_ra, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + header = wcs.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 120 + header["NAXIS2"] = 100 + return header + + no_wcs_header = fits.Header() + no_wcs_header["NAXIS"] = 2 + no_wcs_header["NAXIS1"] = 120 + no_wcs_header["NAXIS2"] = 100 + + headers = { + "target_in_frame.fits": make_wcs_header(210.0), + "target_off_frame.fits": make_wcs_header(212.0), + "no_wcs.fits": no_wcs_header, + } + messages = [] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: headers[file_name]) + monkeypatch.setattr( + exotic_module, + "update_coordinates_with_proper_motion", + lambda info_dict, obs_time: (210.0, 54.0), + ) + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, warn=False, error=False: messages.append((message, warn, error)), + ) + + frames = list(headers) + filtered, keep_mask, dropped = exotic_module.filter_wcs_target_out_of_frame_frames( + frames, + {"ra": 210.0, "dec": 54.0}, + obs_times=[2461196.5, 2461196.6, 2461196.7], + ) + + assert filtered.tolist() == ["target_in_frame.fits", "no_wcs.fits"] + assert keep_mask.tolist() == [True, False, True] + assert dropped == ["target_off_frame.fits"] + assert any("Target WCS precheck" in message for message, _, _ in messages) + + +def test_maybe_reinterpret_decimal_ra_hours_from_wcs_when_only_ra_times_fifteen_matches(monkeypatch): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [16.18494, 74.3313] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + header = wcs.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 120 + header["NAXIS2"] = 100 + messages = [] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda _file_name: header) + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, warn=False, error=False: messages.append((message, warn, error)), + ) + + info = {"ra": 1.078996153, "dec": 74.3313055} + + corrected = exotic_module.maybe_reinterpret_decimal_ra_hours_from_wcs(["frame.fits"], info) + + assert corrected is True + assert info["ra"] == pytest.approx(16.184942295) + assert any("interpreted decimal target RA as hours" in message and warn for message, warn, _ in messages) + + already_degrees = {"ra": 16.184942295, "dec": 74.3313055} + + corrected_again = exotic_module.maybe_reinterpret_decimal_ra_hours_from_wcs( + ["frame.fits"], + already_degrees, + ) + + assert corrected_again is False + assert already_degrees["ra"] == pytest.approx(16.184942295) + + def test_filter_pointing_outlier_frames_uses_wcs_when_all_frames_have_wcs(monkeypatch): frames = [f"frame_{i}.fits" for i in range(6)] wcs_positions = np.array( diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 5870f457..6de57e1d 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -758,7 +758,11 @@ def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" output_data = json.loads(output_file.read_text(encoding="utf-8")) final_params = output_data["FINAL PLANETARY PARAMETERS"] + final_param_keys = list(final_params) + assert final_param_keys[:2] == ["Transit detection QC", "KTMF"] + assert final_params["Transit detection QC"] == "PASS" + assert final_params["KTMF"] == "4.63 / 5.00" assert final_params["KTMF target-fit decision"] == "PASS: KTMF=4.63 / 5.00" assert final_params["KTMF comparison selection mode"] == "basis=comparison_field_retry, metric=ktmf" assert "selected: highest KTMF" in final_params["KTMF selected comparison decision"] @@ -1029,11 +1033,13 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): frame_filtering_info={ "initial_frame_count": 5, "after_missing_wcs_filter_frame_count": 4, + "after_target_wcs_filter_frame_count": 3, "final_prephotometry_frame_count": 3, "ignore_header_wcs": False, "bad_wcs_threshold_percent": 3.0, "pointing_rejection_sigma": 3.0, "dropped_missing_wcs_files": [tmp_path / "missing_wcs.fits"], + "dropped_target_wcs_files": [tmp_path / "target_off_frame.fits"], "dropped_pointing_files": [tmp_path / "bad_pointing.fits"], }, astrometry_info={ @@ -1097,6 +1103,7 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): frame_filtering = aavso_json_header(output_text, "FRAME_FILTERING-XC") assert frame_filtering["missing_wcs_rejections"]["files"] == ["missing_wcs.fits"] + assert frame_filtering["target_wcs_rejections"]["files"] == ["target_off_frame.fits"] assert frame_filtering["pointing_rejections"]["files"] == ["bad_pointing.fits"] assert frame_filtering["lightcurve_dropped_point_count"] == 1 From a072ccf70c5fdfdb897bb0eca18ea00a0da7dd97 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 7 Jun 2026 18:26:38 +1000 Subject: [PATCH 065/116] fix baseline uncertainty and make pointing checks off by default. --- exotic/api/elca.py | 189 +++++++++++++++++++++++++++++++++++- exotic/exotic.py | 19 ++-- exotic/inputs.py | 2 +- inits.json | 4 +- tests/test_centroid_wcs.py | 13 +-- tests/test_elca_baseline.py | 67 +++++++++++++ tests/test_inputs.py | 4 +- 7 files changed, 275 insertions(+), 23 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 3e61bbde..609ee133 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -544,17 +544,52 @@ def _validate_flux_baseline_keys(self): raise ValueError("Use only one of 'a0' or 'a1' as a free baseline parameter.") def _has_free_flux_baseline(self): - return has_explicit_flux_baseline(self.bounds) + return has_explicit_flux_baseline(getattr(self, 'bounds', {})) def _uses_fixed_flux_baseline(self): return bool(getattr(self, 'fixed_flux_baseline', False)) + def _uses_analytic_flux_baseline(self): + return ( + np.ndim(getattr(self, 'airmass', np.array([]))) != 2 + and not self._has_free_flux_baseline() + and not self._uses_fixed_flux_baseline() + and hasattr(self, 'time') + and hasattr(self, 'data') + and hasattr(self, 'dataerr') + ) + def _set_flux_baseline(self, value, error=0.0): self.parameters['a0'] = value self.errors['a0'] = error self.parameters['a1'] = value self.errors['a1'] = error + def _values_with_analytic_flux_baseline(self, values): + values = copy.deepcopy(values) + if not self._uses_analytic_flux_baseline(): + return values + + try: + model = transit(self.time, values) + model = np.asarray(model, dtype=float) * airmass_trend( + values.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) + flux_scale = solve_flux_baseline( + model, + self.data, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ) + except Exception: + return values + + values['a0'] = flux_scale + values['a1'] = flux_scale + return values + def _coerce_baseline_fit_mask(self, baseline_fit_mask): fit_mask = normalized_optional_fit_mask(baseline_fit_mask, np.asarray(self.time).shape) if fit_mask is None: @@ -670,6 +705,7 @@ def _get_perturbed_transit_parameter_value(self, key, value): return value def _normalized_model_for_plot_times(self, times, values): + values = self._values_with_analytic_flux_baseline(values) model = np.asarray(transit(times, values), dtype=float) if np.ndim(self.airmass) == 2: return model @@ -867,6 +903,140 @@ def transit_model_uncertainty(self, times=None, sigma=1.0): return None return model - model_uncertainty, model + model_uncertainty + def _posterior_baseline_model_uncertainty(self, times, sigma=1.0): + if getattr(self, 'results', None) is None or np.ndim(getattr(self, 'airmass', np.array([]))) == 2: + return None + + try: + sample_points, sample_logl, sample_weights = self._get_triangle_plot_samples() + except Exception: + return None + + sample_points = np.asarray(sample_points, dtype=float) + if sample_points.ndim != 2 or sample_points.shape[0] < 2: + return None + + finite_rows = np.all(np.isfinite(sample_points), axis=1) + if sample_logl is not None: + sample_logl = np.asarray(sample_logl, dtype=float) + if sample_logl.shape[0] == sample_points.shape[0]: + finite_rows &= np.isfinite(sample_logl) + + if np.count_nonzero(finite_rows) < 2: + return None + + row_indices = np.flatnonzero(finite_rows) + if row_indices.size > MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT: + if sample_weights is not None: + weights_array = np.asarray(sample_weights, dtype=float) + if weights_array.shape[0] == sample_points.shape[0]: + row_weights = np.where(np.isfinite(weights_array[row_indices]), weights_array[row_indices], 0.0) + order = np.argsort(row_weights)[-MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT:] + row_indices = row_indices[np.sort(order)] + else: + row_indices = row_indices[ + np.linspace(0, row_indices.size - 1, MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT).astype(int) + ] + else: + row_indices = row_indices[ + np.linspace(0, row_indices.size - 1, MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT).astype(int) + ] + + selected_points = sample_points[row_indices] + selected_weights = None + if sample_weights is not None: + weights_array = np.asarray(sample_weights, dtype=float) + if weights_array.shape[0] == sample_points.shape[0]: + selected_weights = weights_array[row_indices] + selected_weights = np.where( + np.isfinite(selected_weights) & (selected_weights >= 0), + selected_weights, + 0.0, + ) + if np.sum(selected_weights) <= 0: + selected_weights = None + + try: + best_parameters = self._values_with_analytic_flux_baseline(self.parameters) + best_systematics = self._build_systematics_model_at(best_parameters, times) + except Exception: + return None + + best_systematics = np.asarray(best_systematics, dtype=float) + if best_systematics.shape != times.shape or not np.all(np.isfinite(best_systematics)): + return None + + bound_keys = list(getattr(self, 'bounds', {}).keys()) + sampled_keys = getattr(self, 'sampled_keys', None) + if sampled_keys is None: + sampled_keys = self._get_sampled_keys(bound_keys) + + ratios = [] + ratio_weights = [] + for index, point in enumerate(selected_points): + try: + values = copy.deepcopy(self.parameters) + values.update(self._physical_values_from_sample_point(point, bound_keys, sampled_keys)) + values = self._values_with_analytic_flux_baseline(values) + sample_systematics = self._build_systematics_model_at(values, times) + with np.errstate(divide='ignore', invalid='ignore'): + ratio = np.asarray(sample_systematics, dtype=float) / best_systematics + except Exception: + continue + if ratio.shape != times.shape or not np.all(np.isfinite(ratio)): + continue + ratios.append(ratio) + if selected_weights is not None: + ratio_weights.append(selected_weights[index]) + + if len(ratios) < 2: + return None + + ratio_grid = np.asarray(ratios, dtype=float) + if selected_weights is not None: + selected_weights = np.asarray(ratio_weights, dtype=float) + if selected_weights.shape[0] != ratio_grid.shape[0] or np.sum(selected_weights) <= 0: + selected_weights = None + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = 1.0 + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + coverage = math.erf(sigma / np.sqrt(2.0)) + q_lower = 0.5 * (1.0 - coverage) + q_upper = 1.0 - q_lower + + if selected_weights is None: + lower, median, upper = np.nanpercentile( + ratio_grid, + [100.0 * q_lower, 50.0, 100.0 * q_upper], + axis=0, + ) + else: + lower = np.array([ + self._weighted_quantiles(ratio_grid[:, i], [q_lower], weights=selected_weights)[0] + for i in range(ratio_grid.shape[1]) + ]) + median = np.array([ + self._weighted_quantiles(ratio_grid[:, i], [0.5], weights=selected_weights)[0] + for i in range(ratio_grid.shape[1]) + ]) + upper = np.array([ + self._weighted_quantiles(ratio_grid[:, i], [q_upper], weights=selected_weights)[0] + for i in range(ratio_grid.shape[1]) + ]) + + lower_width = median - lower + upper_width = upper - median + lower = 1.0 - np.maximum(lower_width, 0.0) + upper = 1.0 + np.maximum(upper_width, 0.0) + finite = np.isfinite(lower) & np.isfinite(upper) & (lower <= upper) + if not np.any(finite): + return None + return lower, upper + def baseline_model_uncertainty(self, times=None, sigma=1.0): if times is None: times = getattr(self, 'time_upsample', self.time) @@ -874,8 +1044,13 @@ def baseline_model_uncertainty(self, times=None, sigma=1.0): if times.size == 0 or np.ndim(getattr(self, 'airmass', np.array([]))) == 2: return None + posterior_envelope = self._posterior_baseline_model_uncertainty(times, sigma=sigma) + if posterior_envelope is not None: + return posterior_envelope + try: - best_systematics = self._build_systematics_model_at(self.parameters, times) + best_parameters = self._values_with_analytic_flux_baseline(self.parameters) + best_systematics = self._build_systematics_model_at(best_parameters, times) except Exception: return None @@ -893,9 +1068,17 @@ def baseline_model_uncertainty(self, times=None, sigma=1.0): sigma = 1.0 variance = np.zeros_like(times, dtype=float) + uses_analytic_flux_baseline = self._uses_analytic_flux_baseline() + a2_error = self.errors.get('a2') + try: + a2_error = float(a2_error) + except (TypeError, ValueError): + a2_error = 0.0 for key in ('a0', 'a1', 'a2'): if key == 'a1' and 'a0' in self.parameters: continue + if key in ('a0', 'a1') and uses_analytic_flux_baseline and a2_error > 0: + continue if key not in self.parameters: continue error = self.errors.get(key) @@ -920,6 +1103,8 @@ def baseline_model_uncertainty(self, times=None, sigma=1.0): upper_parameters = copy.deepcopy(self.parameters) lower_parameters[key] = lower_value upper_parameters[key] = upper_value + lower_parameters = self._values_with_analytic_flux_baseline(lower_parameters) + upper_parameters = self._values_with_analytic_flux_baseline(upper_parameters) try: lower_systematics = self._build_systematics_model_at(lower_parameters, times) upper_systematics = self._build_systematics_model_at(upper_parameters, times) diff --git a/exotic/exotic.py b/exotic/exotic.py index 42ef7e14..bb305cf6 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -5338,14 +5338,13 @@ def get_bad_wcs_threshold_fraction(config_value): def get_pointing_rejection_sigma(config_value): - default_sigma = 4.0 if config_value is None: - return default_sigma + return None if isinstance(config_value, str): normalized = config_value.strip() - if normalized == "": - return default_sigma + if normalized.lower() in ("", "0", "n", "no", "false", "off"): + return None else: normalized = config_value @@ -5353,24 +5352,24 @@ def get_pointing_rejection_sigma(config_value): sigma = float(normalized) except (TypeError, ValueError): log_info( - f"Warning: Invalid 'pointing_rejection_sigma' value; using default {default_sigma:g}.", + "Warning: Invalid 'pointing_rejection_sigma' value; disabling pointing precheck.", warn=True, ) - return default_sigma + return None if not np.isfinite(sigma): log_info( - f"Warning: Invalid 'pointing_rejection_sigma' value; using default {default_sigma:g}.", + "Warning: Invalid 'pointing_rejection_sigma' value; disabling pointing precheck.", warn=True, ) - return default_sigma + return None if sigma < 0: log_info( - f"Warning: Invalid 'pointing_rejection_sigma' value; using default {default_sigma:g}.", + "Warning: Invalid 'pointing_rejection_sigma' value; disabling pointing precheck.", warn=True, ) - return default_sigma + return None if sigma == 0: return None diff --git a/exotic/inputs.py b/exotic/inputs.py index c85029e2..0516522d 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -222,7 +222,7 @@ def __init__(self, init_opt): 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, 'use_aperture_corrections_and_full_image_fwhm': False, - 'pointing_rejection_sigma': 4.0, + 'pointing_rejection_sigma': None, 'skip_low_comparison_coverage_rejection': 'n', 'fit_lightcurve_to_every_comparison_candidate': 'n', 'ultranest_min_num_live_points': 200, diff --git a/inits.json b/inits.json index 10e42ab3..a004a7f9 100644 --- a/inits.json +++ b/inits.json @@ -25,7 +25,7 @@ "Fast Aperture Mask": "Default false/exact mode for fractional-pixel aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to true to opt into center-based masks for speed.", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", - "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Set to 0 to disable. Default 4.", + "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Leave blank/null or set to 0/off to disable. Default disabled.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", @@ -115,7 +115,7 @@ "Fast Aperture Mask (y/n)": false, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", "bad_wcs_threshold_percent": 3.0, - "pointing_rejection_sigma": 4.0, + "pointing_rejection_sigma": null, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": false, "detect_bad_pixels_before_photometry": "n", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 62669b69..60c4b14b 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -448,9 +448,9 @@ def test_get_bad_wcs_threshold_fraction_falls_back_for_invalid_values(): assert exotic_module.get_bad_wcs_threshold_fraction(101) == pytest.approx(0.03) -def test_get_pointing_rejection_sigma_defaults_to_four(): - assert exotic_module.get_pointing_rejection_sigma(None) == pytest.approx(4.0) - assert exotic_module.get_pointing_rejection_sigma("") == pytest.approx(4.0) +def test_get_pointing_rejection_sigma_defaults_to_disabled(): + assert exotic_module.get_pointing_rejection_sigma(None) is None + assert exotic_module.get_pointing_rejection_sigma("") is None def test_get_pointing_rejection_sigma_reads_positive_numeric_values(): @@ -458,10 +458,11 @@ def test_get_pointing_rejection_sigma_reads_positive_numeric_values(): assert exotic_module.get_pointing_rejection_sigma("2.75") == pytest.approx(2.75) -def test_get_pointing_rejection_sigma_uses_default_for_invalid_text_and_zero_disables(): - assert exotic_module.get_pointing_rejection_sigma("not-a-number") == pytest.approx(4.0) - assert exotic_module.get_pointing_rejection_sigma(-1) == pytest.approx(4.0) +def test_get_pointing_rejection_sigma_disables_for_invalid_text_and_zero(): + assert exotic_module.get_pointing_rejection_sigma("not-a-number") is None + assert exotic_module.get_pointing_rejection_sigma(-1) is None assert exotic_module.get_pointing_rejection_sigma(0) is None + assert exotic_module.get_pointing_rejection_sigma("off") is None def test_display_filename_returns_basename_for_unix_and_windows_paths(): diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index ca6a4aaf..8a76ded0 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -498,6 +498,73 @@ def test_baseline_model_uncertainty_is_centered_on_unity_and_includes_a2(monkeyp assert width[0] > width[len(width) // 2] +def test_baseline_model_uncertainty_does_not_double_count_analytic_a0(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + airmass = np.linspace(1.0, 2.0, time.size) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = elca.transit(time, prior) + fit.dataerr = np.full_like(time, 1e-3) + fit.airmass = airmass + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.prior = prior.copy() + fit.bounds = {"a2": [-1.0, 1.0]} + fit.fixed_flux_baseline = False + fit.mode = "ns" + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.0 + fit.errors = {"a0": 0.5, "a1": 0.5, "a2": 0.02} + fit.results = None + + lower, upper = fit.baseline_model_uncertainty(time) + half_width = np.nanmax(np.maximum(1.0 - lower, upper - 1.0)) + + assert half_width < 0.03 + + +def test_baseline_model_uncertainty_prefers_posterior_samples(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + sample_count = 41 + a0_samples = 1.0 + np.linspace(-0.004, 0.004, sample_count) + a2_samples = np.linspace(-0.02, 0.02, sample_count) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = elca.transit(time, prior) + fit.dataerr = np.full_like(time, 1e-3) + fit.airmass = np.linspace(1.0, 2.0, time.size) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.prior = prior.copy() + fit.bounds = {"a0": [0.5, 1.5], "a2": [-1.0, 1.0]} + fit.sampled_keys = ["a0", "a2"] + fit.mode = "ns" + fit.ns_type = "ultranest" + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.0 + fit.errors = {"a0": 0.5, "a1": 0.5, "a2": 0.5} + fit.results = { + "weighted_samples": { + "points": np.column_stack([a0_samples, a2_samples]), + "logl": np.zeros(sample_count, dtype=float), + "weights": np.ones(sample_count, dtype=float), + } + } + + lower, upper = fit.baseline_model_uncertainty(time) + half_width = np.nanmax(np.maximum(1.0 - lower, upper - 1.0)) + + assert half_width < 0.02 + + def test_plot_bestfit_can_draw_baseline_uncertainty_band(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 55a40c25..8d23165d 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -113,7 +113,7 @@ def test_comp_params_defaults_bad_wcs_threshold_percent_to_three(tmp_path): assert inputs.info_dict["bad_wcs_threshold_percent"] == 3.0 -def test_comp_params_defaults_pointing_rejection_sigma_to_four(tmp_path): +def test_comp_params_defaults_pointing_rejection_sigma_to_none(tmp_path): init_data = { "user_info": {}, "optional_info": {}, @@ -125,7 +125,7 @@ def test_comp_params_defaults_pointing_rejection_sigma_to_four(tmp_path): inputs = Inputs(init_opt="y") inputs.comp_params(init_file, {}) - assert inputs.info_dict["pointing_rejection_sigma"] == pytest.approx(4.0) + assert inputs.info_dict["pointing_rejection_sigma"] is None def test_comp_params_defaults_skip_low_comparison_coverage_rejection_to_no(tmp_path): From f6312228c3e50e58e78b736b207164d18dd91ec6 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 8 Jun 2026 15:13:58 +1000 Subject: [PATCH 066/116] Stop AID from picking random unvetted other stars and also uber-inflate uncertainties --- docs/regions/English/example_output.txt | 4 +- docs/regions/German/Beispiel_Output.txt | 4 +- exotic/api/output_aavso.py | 9 +- exotic/exotic.py | 81 +++++++---- exotic/output_files.py | 28 +++- tests/test_nextastro_variability.py | 172 ++++++++++++++++++++++++ tests/test_output_files.py | 80 +++++++++++ 7 files changed, 343 insertions(+), 35 deletions(-) diff --git a/docs/regions/English/example_output.txt b/docs/regions/English/example_output.txt index 7a5d4f62..5154f9c7 100644 --- a/docs/regions/English/example_output.txt +++ b/docs/regions/English/example_output.txt @@ -227,7 +227,7 @@ The Mean Squared Error is: 255638.858211 ********************************************* -Best Comparison Star: #2 +Transit Fit Comparison Star: #2 Minimum Residual Scatter: 0.5414% Optimal Aperture: 4 Optimal Annulus: 5 @@ -261,4 +261,4 @@ Output File Saved ************************ End of Reduction Process -************************ \ No newline at end of file +************************ diff --git a/docs/regions/German/Beispiel_Output.txt b/docs/regions/German/Beispiel_Output.txt index 2e9be3d0..342ce830 100644 --- a/docs/regions/German/Beispiel_Output.txt +++ b/docs/regions/German/Beispiel_Output.txt @@ -227,7 +227,7 @@ The Mean Squared Error is: 255638.858211 ********************************************* -Best Comparison Star: #2 +Transit Fit Comparison Star: #2 Minimum Residual Scatter: 0.5414% Optimal Aperture: 4 Optimal Annulus: 5 @@ -261,4 +261,4 @@ Output File Saved ************************ End of Reduction Process -************************ \ No newline at end of file +************************ diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 49fbeadc..3163fa6b 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -191,7 +191,14 @@ def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min } if phot_opt: - phot_ext = {"Best Comparison Star": f"#{comp_star} - {comp_coords}" if min_aper >= 0 else str(comp_star)} + transit_fit_comp_text = ( + f"#{comp_star} - {comp_coords}" + if comp_star is not None and min_aper >= 0 + else str(comp_star) + ) + phot_ext = { + "Transit Fit Comparison Star": transit_fit_comp_text + } if min_aper == 0: phot_ext["Optimal Method"] = "PSF photometry" else: diff --git a/exotic/exotic.py b/exotic/exotic.py index bb305cf6..c8748c6e 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -316,6 +316,7 @@ ) NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC = 30.0 NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 2.0 +CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX = 0.05 REFERENCE_FALLBACK_COMPARISON_LIMIT = 10 REFERENCE_FALLBACK_DETECTION_MAX_STARS = 60 REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS = 12 @@ -10264,6 +10265,18 @@ def _finite_float(value, default=None): return parsed if np.isfinite(parsed) else default +def usable_catalog_reference_magnitude(magnitude, magnitude_error): + parsed_magnitude = _finite_float(magnitude) + parsed_error = normalized_magnitude_error(magnitude_error) + if ( + not is_usable_apparent_magnitude(parsed_magnitude) + or parsed_error is None + or parsed_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + ): + return None + return parsed_magnitude, parsed_error + + def normalize_nextastro_filter_key(obs_filter): return re.sub(r"[^a-z0-9]", "", str(obs_filter or "").lower()) @@ -10363,10 +10376,10 @@ def nextastro_catalog_rows(catalog_response): def row_nextastro_magnitude(row, band_candidates): for priority, (mag_column, error_column, band_label) in enumerate(band_candidates): - magnitude = _finite_float(row.get(mag_column)) - magnitude_error = normalized_magnitude_error(row.get(error_column)) - if not is_usable_apparent_magnitude(magnitude) or magnitude_error is None: + usable_magnitude = usable_catalog_reference_magnitude(row.get(mag_column), row.get(error_column)) + if usable_magnitude is None: continue + magnitude, magnitude_error = usable_magnitude return { 'priority': priority, 'mag': magnitude, @@ -11015,11 +11028,18 @@ def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=No if obs_filter in [band['band'] for band in star['bands']]: star_info = next(band for band in star['bands'] if band['band'] == obs_filter) + usable_magnitude = usable_catalog_reference_magnitude( + star_info.get('mag'), + star_info.get('error'), + ) + if usable_magnitude is None: + continue + star_mag, star_mag_error = usable_magnitude vsp_comp_stars_info[star['auid']] = { 'pos': vsp_star, - 'mag': star_info['mag'], - 'error': star_info['error'], + 'mag': star_mag, + 'error': star_mag_error, 'ra': ra_deg, 'dec': dec_deg, 'catalog_ra': ra_deg, @@ -13639,6 +13659,7 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ if ( comp_mag is None or comp_mag_error is None + or comp_mag_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX or not is_usable_apparent_magnitude(comp_mag) ): raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") @@ -13667,19 +13688,15 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ mask_ref = np.isfinite(detrended_all) detrended = detrended_all[mask_ref] - selected_airmass_model = fit_airmass_model[mask_ref] - selected_data = fit_data[mask_ref] selected_times = fit_times[mask_ref] selected_airmass = fit_airmass[mask_ref] oot_scatter = np.nanstd(detrended) - median_data = np.nanmedian(selected_data) with np.errstate(divide='ignore', invalid='ignore'): - norm_flux_unc = oot_scatter * selected_airmass_model / median_data target_mag = comp_mag - (2.5 * np.log10(detrended)) target_mag_error = ( comp_mag_error ** 2 - + (-2.5 * norm_flux_unc / (detrended * np.log(10))) ** 2 + + (-2.5 * oot_scatter / (detrended * np.log(10))) ** 2 ) ** 0.5 valid = ( @@ -13732,12 +13749,21 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs info_comps = {} try: - if best_comp is None or (best_comp not in vsp_ind): - comp_pos = choose_comp_star_variability(fit_lc_refs, fit_lc_best, info_comps, comp_stars, vsp_comp_stars, - save) - else: - comp_pos = comp_stars[best_comp] - info_comps[best_comp] = calculate_variablility(fit_lc_refs[best_comp]['myfit'], fit_lc_best) + if best_comp is None: + log_info( + "Skipping AID magnitude output because no transit-fit comparison star was selected.", + warn=True, + ) + return [] + if best_comp not in vsp_ind: + log_info( + "Skipping AID magnitude output because the transit-fit comparison star has no catalog " + f"magnitude with uncertainty <= {CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX:.3f} mag.", + warn=True, + ) + return [] + comp_pos = comp_stars[best_comp] + info_comps[best_comp] = calculate_variablility(fit_lc_refs[best_comp]['myfit'], fit_lc_best) except Exception as e: log_info(f"Error selecting or calculating variability for comparison star: {e}", warn=True) return [] @@ -19751,10 +19777,10 @@ def _main_impl(): display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] == 0: # psf - log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") + log_info(f"Transit Fit Comparison Star: #{photometry_info['comp_star_num']}") log_info("Optimal Method: PSF photometry") elif photometry_info['min_aperture'] < 0: # no comp star - log_info("Best Comparison Star: None") + log_info("Transit Fit Comparison Star: None") if adaptive_summary is not None: log_info(f"Optimal Aperture: {abs(display_aperture):.2f} +/- {adaptive_summary['aperture_std']:.2f} px") log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") @@ -19766,7 +19792,7 @@ def _main_impl(): log_info(f"Optimal Aperture: {abs(np.round(display_aperture, 2))}") log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") else: - log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") + log_info(f"Transit Fit Comparison Star: #{photometry_info['comp_star_num']}") if adaptive_summary is not None: log_info(f"Optimal Aperture: {display_aperture:.2f} +/- {adaptive_summary['aperture_std']:.2f} px") log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") @@ -20077,18 +20103,17 @@ def _main_impl(): # standardDev1 = np.std(goodFluxes) if vsp_comp_stars: - if not bestCompStar: - vsp_params = stellar_variability(ref_flux, best_fit_lc, exotic_infoDict['comp_stars'], - vsp_comp_stars, vsp_num, None, exotic_infoDict['save'], - pDict['sName'], - observed_filter=exotic_infoDict.get('observed_filter', - exotic_infoDict.get('filter'))) - else: + if bestCompStar: vsp_params = stellar_variability(ref_flux, best_fit_lc, exotic_infoDict['comp_stars'], vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], pDict['sName'], observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter'))) + else: + log_info( + "Skipping AID magnitude output because no transit-fit comparison star was selected.", + warn=True, + ) log_info("\n\nOutput File Saved") else: @@ -20425,9 +20450,9 @@ def _main_impl(): display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] >= 0: - log_info(f" Best Comparison Star: #{bestCompStar} - {comp_coords}") + log_info(f" Transit Fit Comparison Star: #{bestCompStar} - {comp_coords}") else: - log_info(" Best Comparison Star: None") + log_info(" Transit Fit Comparison Star: None") if photometry_info['min_aperture'] == 0: log_info(" Optimal Method: PSF photometry") else: diff --git a/exotic/output_files.py b/exotic/output_files.py index a42db10d..b1607bc2 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -148,6 +148,20 @@ def stellar_variability_reference_summary(vsp_param): return ", ".join(details) +def stellar_variability_measurement_summary(vsp_params, transit_fit_comp_star=None): + point_count = len(vsp_params or []) + if point_count == 0: + return None + + if transit_fit_comp_star is None: + return None + + return ( + f"Remeasured {point_count} out-of-transit target/reference point(s) against the transit-fit catalog " + "reference for AID magnitudes; AID rows list the JD timestamps used." + ) + + def aid_comparison_metadata(vsp_param): if not vsp_param: return {} @@ -1109,11 +1123,21 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor if qc_notes: params_num["Transit QC notes"] = " ".join(str(note) for note in qc_notes) - if vsp_params: + if vsp_params and not (phot_opt and comp_star is None): params_num["Variable Reference Star"] = stellar_variability_reference_summary(vsp_params[0]) + measurement_summary = stellar_variability_measurement_summary(vsp_params, comp_star) + if measurement_summary: + params_num["Variable Reference Measurement"] = measurement_summary if phot_opt: - phot_ext = {"Best Comparison Star": f"#{comp_star} - {comp_coords}" if min_aper >= 0 else str(comp_star)} + transit_fit_comp_text = ( + f"#{comp_star} - {comp_coords}" + if comp_star is not None and min_aper >= 0 + else str(comp_star) + ) + phot_ext = { + "Transit Fit Comparison Star": transit_fit_comp_text + } if min_aper == 0: phot_ext["Optimal Method"] = "PSF photometry" else: diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 1d62cbd6..6f1f6ad4 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -261,6 +261,28 @@ def test_nextastro_photometry_catalog_match_floors_zero_magnitude_error(): assert match['error'] == pytest.approx(0.001) +def test_nextastro_photometry_catalog_match_rejects_high_magnitude_error(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.0001, + 'dec': 20.0001, + 'Vmag': 12.3, + 'err_Vmag': 0.051, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') + + assert match is None + + def test_nextastro_photometry_catalog_match_ignores_over_30_magnitudes(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], @@ -347,6 +369,53 @@ def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): assert calibration['observed_filter'] == 'V' +def test_vsp_query_rejects_band_errors_over_limit(monkeypatch): + class DummyWCS: + def pixel_to_world_values(self, x_pixel, y_pixel): + return 10.0, 20.0 + + def world_to_pixel_values(self, ra_deg, dec_deg): + return np.array([40.0]), np.array([50.0]) + + payload = { + 'chartid': 'X123', + 'photometry': [ + { + 'auid': 'HIGH', + 'ra': '00:00:00.0', + 'dec': '+00:00:00.0', + 'bands': [{'band': 'V', 'mag': 12.0, 'error': 0.051}], + }, + { + 'auid': 'LOW', + 'ra': '00:00:00.0', + 'dec': '+00:00:00.0', + 'bands': [{'band': 'V', 'mag': 12.1, 'error': 0.05}], + }, + ], + } + user_comp_stars = [] + + monkeypatch.setattr(exotic_module, 'search_wcs', lambda file: DummyWCS()) + monkeypatch.setattr(exotic_module, 'radec_hours_to_degree', lambda ra, dec: (10.0, 20.0)) + monkeypatch.setattr(exotic_module.requests, 'get', lambda url: DummyResponse(payload)) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + vsp_comp_stars, chart_id = exotic_module.vsp_query( + 'frame.fits', + [100, 100], + 'CV', + 1.0, + user_comp_stars=user_comp_stars, + user_targ_star=[10, 10], + ) + + assert chart_id == 'X123' + assert list(vsp_comp_stars) == ['LOW'] + assert vsp_comp_stars['LOW']['error'] == pytest.approx(0.05) + assert user_comp_stars == [[40, 50]] + + def test_build_stellar_variability_params_records_nextastro_reference(monkeypatch, tmp_path): captured = {} @@ -399,6 +468,109 @@ def fake_plot(params, save, s_name, label): assert params[0]['observed_filter'] == 'CV' +def test_build_stellar_variability_params_keeps_magnitude_errors_in_flux_ratio_units(monkeypatch, tmp_path): + comp_mag = 9.751 + comp_mag_error = 0.018 + target_mag = 13.1 + flux_ratio = 10 ** ((comp_mag - target_mag) / 2.5) + detrended = flux_ratio * np.array([0.94, 1.0, 1.06], dtype=float) + + class DummyFit: + data = detrended + airmass_model = np.ones(3, dtype=float) + airmass = np.array([1.1, 1.2, 1.3], dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + + calibration_star = { + 'mag': comp_mag, + 'error': comp_mag_error, + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + 'mag_band': 'V', + 'observed_filter': 'V', + } + + params = exotic_module.build_stellar_variability_params_from_fit( + DummyFit(), + calibration_star, + [100, 200], + '000-BJX-718', + tmp_path, + 'HAT-P-37', + observed_filter='CV', + ) + + expected_scatter = np.nanstd(detrended) + expected_mag_error = np.hypot( + comp_mag_error, + 2.5 * expected_scatter / (flux_ratio * np.log(10)), + ) + + assert params[1]['mag'] == pytest.approx(target_mag) + assert params[1]['mag_err'] == pytest.approx(expected_mag_error) + assert params[1]['mag_err'] < 0.08 + + +def test_stellar_variability_requires_selected_transit_comparison(monkeypatch, tmp_path): + logged = [] + + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + params = exotic_module.stellar_variability( + {0: {'myfit': DummyFit(), 'pos': [100, 200]}}, + DummyFit(), + [[100, 200]], + {'REF': {'pos': [100, 200], 'mag': 12.0, 'error': 0.02}}, + [0], + None, + tmp_path, + 'Host Star', + ) + + assert params == [] + assert any('no transit-fit comparison star' in message for message in logged) + + +def test_stellar_variability_requires_selected_comparison_catalog_match(monkeypatch, tmp_path): + logged = [] + + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + params = exotic_module.stellar_variability( + { + 0: {'myfit': DummyFit(), 'pos': [100, 200]}, + 1: {'myfit': DummyFit(), 'pos': [300, 400]}, + }, + DummyFit(), + [[100, 200], [300, 400]], + {'REF': {'pos': [300, 400], 'mag': 12.0, 'error': 0.02}}, + [1], + 0, + tmp_path, + 'Host Star', + ) + + assert params == [] + assert any('has no catalog magnitude' in message for message in logged) + + def test_check_for_variable_stars_uses_nextastro_flags_to_filter(monkeypatch): logged = [] diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 6de57e1d..f0888d4b 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -491,6 +491,86 @@ def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path assert "V=12.345 +/- 0.067" in reference +def test_final_planetary_params_reports_transit_comparison_catalog_reference(tmp_path): + fit = DummyFit() + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + vsp_params = [ + { + "cname": "000-BJX-718", + "cmag": 9.751, + "cmag_err": 0.018, + "pos": [616, 113], + "catalog_source": "AAVSO VSP", + "is_aavso_vsp": True, + "mag_band": "V", + }, + { + "cname": "000-BJX-718", + "cmag": 9.751, + "cmag_err": 0.018, + "pos": [616, 113], + "catalog_source": "AAVSO VSP", + "is_aavso_vsp": True, + "mag_band": "V", + }, + ] + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=True, + vsp_params=vsp_params, + comp_star=1, + comp_coords=[616, 113], + min_aper=2.7, + min_annul=10.15, + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Transit Fit Comparison Star"] == "#1 - [616, 113]" + assert "Best Comparison Star" not in final_params + assert final_params["Variable Reference Star"] == "AAVSO Label: 000-BJX-718, Position: [616, 113]" + assert "Remeasured 2 out-of-transit target/reference point(s)" in final_params["Variable Reference Measurement"] + assert "AID rows list the JD timestamps used" in final_params["Variable Reference Measurement"] + assert "transit-fit catalog reference" in final_params["Variable Reference Measurement"] + + +def test_final_planetary_params_suppresses_variable_reference_without_transit_comparison(tmp_path): + fit = DummyFit() + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + vsp_params = [{ + "cname": "000-BJX-718", + "cmag": 9.751, + "cmag_err": 0.018, + "pos": [616, 113], + "catalog_source": "AAVSO VSP", + "is_aavso_vsp": True, + "mag_band": "V", + }] + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=True, + vsp_params=vsp_params, + comp_star=None, + comp_coords=None, + min_aper=-2.7, + min_annul=10.15, + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Transit Fit Comparison Star"] == "None" + assert "Variable Reference Star" not in final_params + assert "Variable Reference Measurement" not in final_params + + def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclination(tmp_path): fit = DummyFit() (tmp_path / "temp").mkdir() From d08b083221274f54534dfdf34d33cb89ac11cc49 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Tue, 9 Jun 2026 07:52:14 +1000 Subject: [PATCH 067/116] Grab pixel binning from header if not in inits.json --- exotic/inputs.py | 35 ++++++++++++++++++++++++++++++-- tests/test_inputs.py | 48 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/exotic/inputs.py b/exotic/inputs.py index 0516522d..dc88682e 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -257,6 +257,8 @@ def complete_red(self, planet): pass elif key in ('lat', 'long'): self.info_dict[key] = self.params[key](self.info_dict[key], hdr) + elif key == 'pixel_bin': + self.info_dict[key] = self.params[key](self.info_dict[key], hdr) else: self.info_dict[key] = self.params[key](self.info_dict[key]) if key == 'save': @@ -850,8 +852,37 @@ def camera(c_type): return "CCD" -def pixel_bin(pix_bin): - if not pix_bin: +def format_fits_binning_axis(value): + if is_blank_value(value): + return None + + try: + binning_value = float(str(value).strip()) + except (TypeError, ValueError): + return None + + if not math.isfinite(binning_value) or binning_value <= 0: + return None + if binning_value.is_integer(): + return str(int(binning_value)) + return str(binning_value) + + +def fits_header_pixel_bin(hdr): + if hdr is None: + return None + + x_binning = format_fits_binning_axis(find(hdr, ['XBINNING'])) + y_binning = format_fits_binning_axis(find(hdr, ['YBINNING'])) + if x_binning is None or y_binning is None: + return None + return f"{x_binning}x{y_binning}" + + +def pixel_bin(pix_bin, hdr=None): + if is_blank_value(pix_bin): + pix_bin = fits_header_pixel_bin(hdr) + if is_blank_value(pix_bin): pix_bin = user_input("Please enter the pixel binning: ", type_=str) return pix_bin diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 8d23165d..bf733aea 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -1,6 +1,8 @@ import json import requests import pytest +import numpy as np +from astropy.io import fits import exotic.inputs as inputs_module from exotic.inputs import Inputs, camera, parse_aavso_prereduced_overrides @@ -68,6 +70,52 @@ def test_comp_params_defaults_aavso_comp_to_no(tmp_path): assert inputs.info_dict["aavso_comp"] == "n" +def test_complete_red_uses_fits_x_y_binning_when_pixel_bin_missing(tmp_path, monkeypatch): + image_dir = tmp_path / "images" + image_dir.mkdir() + save_dir = tmp_path / "results" + save_dir.mkdir() + + header = fits.Header() + header["XBINNING"] = 2 + header["YBINNING"] = 3 + fits.PrimaryHDU(data=np.zeros((2, 2)), header=header).writeto(image_dir / "frame.fits") + + init_data = { + "user_info": { + "Directory with FITS files": str(image_dir), + "Directory to Save Plots": str(save_dir), + "AAVSO Observer Code (blank if none)": "", + "Secondary Observer Codes (blank if none)": "", + "Observation date": "2020-01-01", + "Obs. Latitude": "+32.0", + "Obs. Longitude": "-110.0", + "Obs. Elevation (meters)": 1000, + "Camera Type (CCD or DSLR)": "CCD", + "Observing Notes": "na", + "Plate Solution? (y/n)": "n", + "Add Comparison Stars from AAVSO? (y/n)": "n", + "Target Star X & Y Pixel": [1, 1], + "Comparison Star(s) X & Y Pixel": [[2, 2]], + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + def fail_on_prompt(prompt, type_, values=None, max_tries=1000): + raise AssertionError(f"Unexpected prompt: {prompt}") + + monkeypatch.setattr(inputs_module, "user_input", fail_on_prompt) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + info_dict, _ = inputs.complete_red("HAT-P-32 b") + + assert info_dict["pixel_bin"] == "2x3" + + def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): init_data = { "user_info": {}, From 61c60ff2c825fb9cc7092649da1bd55d6a4e9462 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 16 Jun 2026 09:42:31 +1000 Subject: [PATCH 068/116] Fixing up PSF centroiding for Art I was being clever to reject PSFs quickly at the beginning, but the PSF centroiding wasn't good enough for actual measurements. --- exotic/exotic.py | 1376 ++++++++++++++++++++++------ exotic/output_files.py | 4 +- exotic/plots.py | 27 +- tests/test_centroid_wcs.py | 26 + tests/test_exotic_proper_motion.py | 350 ++++++- tests/test_plots.py | 34 + 6 files changed, 1541 insertions(+), 276 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index c8748c6e..b98080dd 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -243,6 +243,14 @@ COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA = 4.25 COMPARISON_STAR_SUITABILITY_MIN_CANDIDATES = 5 COMPARISON_STAR_SUITABILITY_MAX_ITERS = 10 +COMPARISON_STAR_PRESCORE_SIGMA_CLIP = 3.0 +COMPARISON_STAR_PRESCORE_MAX_CLIP_ITERS = 3 +COMPARISON_STAR_PRESCORE_SCATTER_FLOOR = 1e-4 +PSF_FRAME_QUALITY_SIGMA = 4.0 +PSF_FRAME_QUALITY_MAX_CLIP_ITERS = 3 +PSF_FRAME_QUALITY_SEEING_MIN_FRACTIONAL_DEVIATION = 0.5 +PSF_FRAME_QUALITY_AMPLITUDE_MIN_FRACTIONAL_DEVIATION = 0.25 +PSF_TARGET_QUALITY_MAX_COMP_SIGMA_RATIO = 3.0 COMPARISON_IMAGE_OUTLIER_SIGMA = COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA COMPARISON_IMAGE_OUTLIER_MIN_ACTIVE_STARS = 3 COMPARISON_IMAGE_OUTLIER_MIN_VALID_PAIRS = 2 @@ -303,6 +311,12 @@ ROBUST_FLUX_MIN_POINTS = 20 WCS_REFERENCE_GEOMETRY_TOLERANCE_PIXELS = 5.0 WCS_MIN_GEOMETRY_MATCH_FRACTION = 0.5 +PSF_FIT_MAX_SEED_OFFSET_PIXELS = 6.0 +PSF_FIT_MAX_AXIS_RATIO = 4.0 +PSF_FIT_MAX_SIGMA_PIXELS = 8.0 +PSF_FIT_SELECTION_MARGIN = 0.35 +PSF_ALIGNMENT_TARGET_WIDTH_MAX_COMP_RATIO = 3.0 +PSF_ALIGNMENT_CANDIDATE_SELECTION_MARGIN = 0.20 TIME_REJECTION_RANGE_DISPLAY_LIMIT = 6 TIME_REJECTION_GROUP_GAP_CADENCE_MULTIPLIER = 2.5 NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS = 5 @@ -4730,6 +4744,245 @@ def robust_target_reference_flux_mask(target_flux, reference_flux): return target_mask & reference_mask +def psf_metric_outlier_mask(values, sigma=PSF_FRAME_QUALITY_SIGMA, + max_iters=PSF_FRAME_QUALITY_MAX_CLIP_ITERS, + min_points=LIGHTCURVE_MIN_VALID_POINTS, + high=True, low=True, min_fractional_deviation=0.0): + values = np.asarray(values, dtype=float).reshape(-1) + outlier_mask = ~np.isfinite(values) | (values <= 0) + valid_indices = np.flatnonzero(~outlier_mask) + if valid_indices.size < max(int(min_points), 3): + return outlier_mask + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = PSF_FRAME_QUALITY_SIGMA + if not np.isfinite(sigma) or sigma <= 0: + sigma = PSF_FRAME_QUALITY_SIGMA + + try: + min_fractional_deviation = float(min_fractional_deviation) + except (TypeError, ValueError): + min_fractional_deviation = 0.0 + if not np.isfinite(min_fractional_deviation) or min_fractional_deviation < 0: + min_fractional_deviation = 0.0 + min_log_deviation = np.log1p(min_fractional_deviation) + + log_values = np.log(values[valid_indices]) + keep = np.ones(valid_indices.size, dtype=bool) + max_iters = max(int(max_iters), 1) + min_points = max(int(min_points), 3) + + for _ in range(max_iters): + if np.count_nonzero(keep) < min_points: + break + + kept_values = log_values[keep] + center = bn.nanmedian(kept_values) + if not np.isfinite(center): + break + + scatter = robust_scatter(kept_values - center) + if not np.isfinite(scatter) or scatter <= 0: + break + + deviation = log_values - center + direction_mask = np.zeros(deviation.shape, dtype=bool) + if high: + direction_mask |= deviation > 0 + if low: + direction_mask |= deviation < 0 + + newly_rejected = ( + keep + & direction_mask + & (np.abs(deviation) > sigma * scatter) + & (np.abs(deviation) > min_log_deviation) + ) + if not np.any(newly_rejected): + break + if np.count_nonzero(keep & ~newly_rejected) < min_points: + break + keep[newly_rejected] = False + + outlier_mask[valid_indices] = ~keep + return outlier_mask + + +def psf_frame_quality_components(psf_rows): + psf_rows = np.asarray(psf_rows, dtype=float) + if psf_rows.ndim != 2 or psf_rows.shape[1] < 5: + frame_count = 0 if psf_rows.ndim == 0 else psf_rows.shape[0] + empty = np.zeros(frame_count, dtype=bool) + return { + 'keep_mask': ~empty, + 'invalid_mask': empty, + 'seeing_outlier_mask': empty, + 'amplitude_outlier_mask': empty, + } + + amplitude = psf_rows[:, 2] + sigma_x = psf_rows[:, 3] + sigma_y = psf_rows[:, 4] + seeing = GAUSSIAN_SIGMA_TO_FWHM * 0.5 * (sigma_x + sigma_y) + + invalid_mask = ( + ~np.isfinite(psf_rows[:, 0]) + | ~np.isfinite(psf_rows[:, 1]) + | ~np.isfinite(amplitude) + | ~np.isfinite(sigma_x) + | ~np.isfinite(sigma_y) + | (amplitude <= 0) + | (sigma_x <= 0) + | (sigma_y <= 0) + ) + seeing_outlier_mask = psf_metric_outlier_mask( + seeing, + high=True, + low=False, + min_fractional_deviation=PSF_FRAME_QUALITY_SEEING_MIN_FRACTIONAL_DEVIATION, + ) & ~invalid_mask + amplitude_outlier_mask = psf_metric_outlier_mask( + amplitude, + high=False, + low=True, + min_fractional_deviation=PSF_FRAME_QUALITY_AMPLITUDE_MIN_FRACTIONAL_DEVIATION, + ) & ~invalid_mask + keep_mask = ~(invalid_mask | seeing_outlier_mask | amplitude_outlier_mask) + return { + 'keep_mask': keep_mask, + 'invalid_mask': invalid_mask, + 'seeing_outlier_mask': seeing_outlier_mask, + 'amplitude_outlier_mask': amplitude_outlier_mask, + } + + +def target_psf_shape_quality_components(target_rows, reference_rows=None): + target_rows = np.asarray(target_rows, dtype=float) + if target_rows.ndim != 2 or target_rows.shape[1] < 5: + frame_count = 0 if target_rows.ndim == 0 else target_rows.shape[0] + empty = np.zeros(frame_count, dtype=bool) + return { + 'keep_mask': ~empty, + 'invalid_mask': empty, + 'seeing_outlier_mask': empty, + 'axis_ratio_outlier_mask': empty, + 'reference_width_outlier_mask': empty, + } + + amplitude = target_rows[:, 2] + sigma_x = target_rows[:, 3] + sigma_y = target_rows[:, 4] + seeing = GAUSSIAN_SIGMA_TO_FWHM * 0.5 * (sigma_x + sigma_y) + + invalid_mask = ( + ~np.isfinite(target_rows[:, 0]) + | ~np.isfinite(target_rows[:, 1]) + | ~np.isfinite(amplitude) + | ~np.isfinite(sigma_x) + | ~np.isfinite(sigma_y) + | (amplitude <= 0) + | (sigma_x <= 0) + | (sigma_y <= 0) + ) + seeing_outlier_mask = psf_metric_outlier_mask( + seeing, + high=True, + low=False, + min_fractional_deviation=PSF_FRAME_QUALITY_SEEING_MIN_FRACTIONAL_DEVIATION, + ) & ~invalid_mask + + axis_ratio = np.full(target_rows.shape[0], np.nan, dtype=float) + valid_width = np.isfinite(sigma_x) & np.isfinite(sigma_y) & (sigma_x > 0) & (sigma_y > 0) + axis_ratio[valid_width] = ( + np.maximum(sigma_x[valid_width], sigma_y[valid_width]) + / np.maximum(np.minimum(sigma_x[valid_width], sigma_y[valid_width]), 1e-12) + ) + axis_ratio_outlier_mask = (axis_ratio > PSF_FIT_MAX_AXIS_RATIO) & ~invalid_mask + + reference_width_outlier_mask = np.zeros(target_rows.shape[0], dtype=bool) + if reference_rows is not None: + reference_rows = np.asarray(reference_rows, dtype=float) + if ( + reference_rows.ndim == 2 + and reference_rows.shape[0] == target_rows.shape[0] + and reference_rows.shape[1] >= 5 + ): + reference_sigma_x = reference_rows[:, 3] + reference_sigma_y = reference_rows[:, 4] + reference_sigma = 0.5 * (reference_sigma_x + reference_sigma_y) + target_sigma = 0.5 * (sigma_x + sigma_y) + reference_valid = ( + np.isfinite(reference_sigma) + & np.isfinite(target_sigma) + & (reference_sigma > 0) + & (target_sigma > 0) + ) + reference_width_outlier_mask = ( + reference_valid + & ( + target_sigma + > PSF_TARGET_QUALITY_MAX_COMP_SIGMA_RATIO * reference_sigma + ) + ) + reference_width_outlier_mask &= ~invalid_mask + + keep_mask = ~( + invalid_mask + | seeing_outlier_mask + | axis_ratio_outlier_mask + | reference_width_outlier_mask + ) + return { + 'keep_mask': keep_mask, + 'invalid_mask': invalid_mask, + 'seeing_outlier_mask': seeing_outlier_mask, + 'axis_ratio_outlier_mask': axis_ratio_outlier_mask, + 'reference_width_outlier_mask': reference_width_outlier_mask, + } + + +def target_psf_shape_quality_mask(target_rows, reference_rows=None): + return target_psf_shape_quality_components(target_rows, reference_rows)['keep_mask'] + + +def psf_frame_quality_mask(psf_rows): + return psf_frame_quality_components(psf_rows)['keep_mask'] + + +def psf_quality_mask_for_key(psf_data, key, frame_count): + if not isinstance(psf_data, dict) or key not in psf_data: + return np.ones(int(frame_count), dtype=bool) + + mask = psf_frame_quality_mask(psf_data[key]) + if mask.shape[0] != int(frame_count): + return np.ones(int(frame_count), dtype=bool) + return mask + + +def mask_series_with_quality(values, quality_mask): + masked = np.asarray(values, dtype=float).copy() + quality_mask = np.asarray(quality_mask, dtype=bool) + if masked.shape[0] == quality_mask.shape[0]: + masked[~quality_mask] = np.nan + return masked + + +def psf_flux_series_from_rows(psf_rows, quality_mask=None): + psf_rows = np.asarray(psf_rows, dtype=float) + flux = 2 * np.pi * psf_rows[:, 2] * psf_rows[:, 3] * psf_rows[:, 4] + if quality_mask is not None: + flux = mask_series_with_quality(flux, quality_mask) + return flux + + +def psf_flux_data_source(psf_data, psf_flux_data=None): + if isinstance(psf_flux_data, dict): + return psf_flux_data + return psf_data + + def is_fast_aperture_mask_enabled(config_value): if config_value is None: return False @@ -9969,25 +10222,49 @@ def check_target_pixel_wcs(input_x_pixel, input_y_pixel, info_dict, ra_list, dec "centroid fitting; keeping the input target coordinates.", warn=True) return input_x_pixel, input_y_pixel - centroid_x, centroid_y, sigma_x, sigma_y = get_psf_parameters(image_data, calculated_x_pixel, calculated_y_pixel) + wcs_psf_row = get_psf_fit_row(image_data, calculated_x_pixel, calculated_y_pixel) + centroid_x, centroid_y = wcs_psf_row[0], wcs_psf_row[1] + sigma_x, sigma_y = wcs_psf_row[3], wcs_psf_row[4] + wcs_psf_quality_score = psf_solution_quality_score( + wcs_psf_row, + seed_pos=[calculated_x_pixel, calculated_y_pixel], + ) + + input_psf_quality_score = np.inf + if pixel_within_image(input_x_pixel, input_y_pixel, image_data.shape, margin=centroid_margin): + input_psf_row = get_psf_fit_row(image_data, input_x_pixel, input_y_pixel) + input_psf_quality_score = psf_solution_quality_score( + input_psf_row, + seed_pos=[input_x_pixel, input_y_pixel], + ) return check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, calculated_x_pixel, calculated_y_pixel, non_interactive_run=non_interactive_run, - prefer_pixel_values_over_wcs_for_target=prefer_pixel_values_over_wcs_for_target) + prefer_pixel_values_over_wcs_for_target=prefer_pixel_values_over_wcs_for_target, + wcs_psf_quality_score=wcs_psf_quality_score, + input_psf_quality_score=input_psf_quality_score) -def get_psf_parameters(image_data, x_pixel, y_pixel): +def get_psf_fit_row(image_data, x_pixel, y_pixel): try: - psf_data = fit_centroid(image_data, [x_pixel, y_pixel], 0) + return fit_centroid(image_data, [x_pixel, y_pixel], 0) except Exception as exc: log.debug(f"Centroid fit failed while validating WCS target coordinates: {exc}") + return _nan_psf_result() + + +def get_psf_parameters(image_data, x_pixel, y_pixel): + psf_data = get_psf_fit_row(image_data, x_pixel, y_pixel) + if not np.all(np.isfinite(psf_data[:5])): return np.nan, np.nan, np.nan, np.nan return psf_data[0], psf_data[1], psf_data[3], psf_data[4] def check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, calculated_x_pixel, calculated_y_pixel, non_interactive_run=False, - prefer_pixel_values_over_wcs_for_target=False): + prefer_pixel_values_over_wcs_for_target=False, + wcs_psf_quality_score=None, + input_psf_quality_score=None): while True: try: validate_pixel_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y) @@ -9998,12 +10275,36 @@ def check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigm "prefer_pixel_values_over_wcs_for_target is enabled.", warn=True) return input_x_pixel, input_y_pixel if non_interactive_run: - if np.isfinite(centroid_x) and np.isfinite(centroid_y): + if wcs_psf_quality_score is None: + wcs_psf_quality_score = psf_solution_quality_score( + [centroid_x, centroid_y, 1.0, sigma_x, sigma_y, 0.0, 0.0], + seed_pos=[calculated_x_pixel, calculated_y_pixel], + ) + try: + wcs_score = float(wcs_psf_quality_score) + except (TypeError, ValueError): + wcs_score = np.inf + try: + input_score = float(input_psf_quality_score) + except (TypeError, ValueError): + input_score = np.inf + + if ( + np.isfinite(input_score) + ): + log_info( + "Proceeding with provided target pixel coordinates because they produce a plausible " + "target PSF fit; the WCS-derived target fit points to a different source.", + warn=True, + ) + return input_x_pixel, input_y_pixel + + if np.isfinite(wcs_score) and np.isfinite(centroid_x) and np.isfinite(centroid_y): log_info("Proceeding with WCS-derived centroided target coordinates due to " "--non-interactive-run.", warn=True) return centroid_x, centroid_y log_info("Proceeding with WCS-derived target pixel coordinates due to " - "--non-interactive-run (centroid unavailable).", warn=True) + "--non-interactive-run (centroid unavailable or implausible).", warn=True) return calculated_x_pixel, calculated_y_pixel new_x_pixel, new_y_pixel = prompt_user_for_coordinates(input_x_pixel, input_y_pixel, calculated_x_pixel, calculated_y_pixel) @@ -11989,9 +12290,11 @@ def build_multiprocess_pointing_precheck_transforms(inputfiles, max_processes, r return positions, usable_mask, alignment_transforms -def _fit_alignment_candidate_psfs(image_data, predicted_coords, target_fast_centroid, frame_fast_centroid): +def _fit_alignment_candidate_psfs(image_data, predicted_coords, target_fast_centroid, frame_fast_centroid, + previous_psf_rows=None): global plateStatus predicted_coords = np.asarray(predicted_coords, dtype=float) + previous_psf_rows = {} if previous_psf_rows is None else dict(previous_psf_rows) with _PLATE_STATUS_SWAP_LOCK: original_plate_status = plateStatus recorder = _ParallelPlateStatusRecorder() @@ -12000,15 +12303,16 @@ def _fit_alignment_candidate_psfs(image_data, predicted_coords, target_fast_cent psf_rows = { 'target': fit_centroid_or_warn_out_of_frame( image_data, - choose_centroid_seed_position(predicted_coords[0], None), + choose_centroid_seed_position(predicted_coords[0], previous_psf_rows.get('target')), 0, fast_mode=target_fast_centroid, ) } for comp_idx in range(max(0, predicted_coords.shape[0] - 1)): + comp_key = f"comp{comp_idx + 1}" psf_rows[f"comp{comp_idx + 1}"] = fit_centroid_or_warn_out_of_frame( image_data, - choose_centroid_seed_position(predicted_coords[comp_idx + 1], None), + choose_centroid_seed_position(predicted_coords[comp_idx + 1], previous_psf_rows.get(comp_key)), comp_idx + 1, fast_mode=frame_fast_centroid, ) @@ -12136,35 +12440,181 @@ def _update_reference_comp_offsets(psf_data, tar_comp_dist, comp_keys): tar_comp_dist[comp_key][1] = abs(int(comp_row[1]) - int(target_row[1])) -def apply_parallel_alignment_result(result, frame_index, psf_data, tar_comp_dist, comp_keys): - wcs_candidate = result.get('wcs') +def _candidate_seed_position(candidate, star_index): + if candidate is None: + return None + + try: + coords = np.asarray(candidate.get('coords'), dtype=float) + except (TypeError, ValueError, AttributeError): + return None + + if coords.ndim == 1: + if coords.size % 2 != 0: + return None + coords = coords.reshape(-1, 2) + if coords.ndim != 2 or coords.shape[1] < 2 or star_index >= coords.shape[0]: + return None + + seed = coords[star_index, :2] + if np.all(np.isfinite(seed)): + return seed + return None + + +def alignment_candidate_quality_score(candidate, comp_keys=None, previous_target_psf_row=None, + previous_comp_psf_rows=None, expected_offsets=None, + width_max_comp_ratio=PSF_ALIGNMENT_TARGET_WIDTH_MAX_COMP_RATIO): + if candidate is None: + return np.inf + + psf_rows = candidate.get('psf_rows') if isinstance(candidate, dict) else None + if not isinstance(psf_rows, dict): + return np.inf + + comp_keys = [] if comp_keys is None else list(comp_keys) + previous_comp_psf_rows = {} if previous_comp_psf_rows is None else dict(previous_comp_psf_rows) + expected_offsets = {} if expected_offsets is None else dict(expected_offsets) + + target_row = psf_rows.get('target', _nan_psf_result()) + target_score = psf_solution_quality_score( + target_row, + seed_pos=_candidate_seed_position(candidate, 0), + ) + if not np.isfinite(target_score): + return np.inf + + score = float(target_score) + target_sigma = psf_sigma_from_fit(target_row) + comp_sigmas = [] + geometry_test_count = 0 + geometry_match_count = 0 + + for comp_idx, comp_key in enumerate(comp_keys): + row = psf_rows.get(f"comp{comp_idx + 1}", _nan_psf_result()) + comp_score = psf_solution_quality_score( + row, + seed_pos=_candidate_seed_position(candidate, comp_idx + 1), + ) + if np.isfinite(comp_score): + score += 0.25 * float(comp_score) + comp_sigma = psf_sigma_from_fit(row) + if np.isfinite(comp_sigma) and comp_sigma > 0: + comp_sigmas.append(comp_sigma) + elif comp_keys: + score += 0.75 + + expected_offset = expected_offsets.get(comp_key) + if expected_offset is None: + continue + + try: + expected_dx = float(expected_offset[0]) + expected_dy = float(expected_offset[1]) + except (TypeError, ValueError, IndexError): + continue + if expected_dx == 0.0 and expected_dy == 0.0: + continue + + geometry_test_count += 1 + if centroid_offset_matches_reference(row, target_row, expected_dx, expected_dy): + geometry_match_count += 1 + + previous_comp_row = previous_comp_psf_rows.get(comp_key) + if centroid_position_is_finite(row) and centroid_position_is_finite(previous_comp_row): + comp_jump = float(np.hypot(float(row[0]) - float(previous_comp_row[0]), + float(row[1]) - float(previous_comp_row[1]))) + score += min(comp_jump / 20.0, 1.0) + + if geometry_test_count: + geometry_miss_fraction = (geometry_test_count - geometry_match_count) / geometry_test_count + score += 2.0 * geometry_miss_fraction + + if np.isfinite(target_sigma) and target_sigma > 0 and comp_sigmas: + comp_sigma_center = float(bn.nanmedian(np.asarray(comp_sigmas, dtype=float))) + if np.isfinite(comp_sigma_center) and comp_sigma_center > 0: + width_ratio = target_sigma / comp_sigma_center + if ( + not np.isfinite(width_ratio) + or width_ratio > float(width_max_comp_ratio) + ): + return np.inf + score += 0.5 * abs(np.log(width_ratio)) + + if centroid_position_is_finite(target_row) and centroid_position_is_finite(previous_target_psf_row): + target_jump = float(np.hypot(float(target_row[0]) - float(previous_target_psf_row[0]), + float(target_row[1]) - float(previous_target_psf_row[1]))) + score += min(target_jump / 20.0, 1.0) + + return float(score) + + +def select_alignment_candidate(result, frame_index, psf_data, tar_comp_dist, comp_keys): + wcs_candidate = result.get('wcs') if isinstance(result, dict) else None + fallback_candidate = result.get('fallback') if isinstance(result, dict) else None selected_candidate = None selected_source = 'fallback' + previous_target_psf_row = None if frame_index == 0 else psf_data['target'][frame_index - 1] + previous_comp_psf_rows = {} + if frame_index != 0: + previous_comp_psf_rows = {comp_key: psf_data[comp_key][frame_index - 1] for comp_key in comp_keys} + + wcs_alignment_decision = {'use_wcs_alignment': False, 'reason': 'no_wcs_candidate'} if wcs_candidate is not None: comp_psf_rows = { comp_key: wcs_candidate['psf_rows'].get(f"comp{comp_idx + 1}", _nan_psf_result()) for comp_idx, comp_key in enumerate(comp_keys) } - previous_comp_psf_rows = {} - if frame_index != 0: - previous_comp_psf_rows = {comp_key: psf_data[comp_key][frame_index - 1] for comp_key in comp_keys} wcs_alignment_decision = should_keep_header_wcs_alignment( wcs_candidate.get('projected_off_frame', False), frame_index, wcs_candidate['psf_rows']['target'], - previous_target_psf_row=None if frame_index == 0 else psf_data['target'][frame_index - 1], + previous_target_psf_row=previous_target_psf_row, comp_psf_rows=comp_psf_rows, previous_comp_psf_rows=previous_comp_psf_rows, expected_offsets=tar_comp_dist, ) - if wcs_alignment_decision['use_wcs_alignment']: - selected_candidate = wcs_candidate - selected_source = 'wcs' - if selected_candidate is None: - selected_candidate = result.get('fallback') or wcs_candidate + wcs_score = alignment_candidate_quality_score( + wcs_candidate, + comp_keys=comp_keys, + previous_target_psf_row=previous_target_psf_row, + previous_comp_psf_rows=previous_comp_psf_rows, + expected_offsets=tar_comp_dist, + ) + fallback_score = alignment_candidate_quality_score( + fallback_candidate, + comp_keys=comp_keys, + previous_target_psf_row=previous_target_psf_row, + previous_comp_psf_rows=previous_comp_psf_rows, + expected_offsets=tar_comp_dist, + ) + + if ( + wcs_candidate is not None + and wcs_alignment_decision.get('use_wcs_alignment') + and np.isfinite(wcs_score) + and ( + not np.isfinite(fallback_score) + or wcs_score <= fallback_score + PSF_ALIGNMENT_CANDIDATE_SELECTION_MARGIN + ) + ): + selected_candidate = wcs_candidate + selected_source = 'wcs' + elif fallback_candidate is not None and np.isfinite(fallback_score): + selected_candidate = fallback_candidate + selected_source = 'fallback' + elif wcs_candidate is not None and np.isfinite(wcs_score): + selected_candidate = wcs_candidate + selected_source = 'wcs' + elif fallback_candidate is not None: + selected_candidate = fallback_candidate + selected_source = 'fallback' + elif wcs_candidate is not None: + selected_candidate = wcs_candidate + selected_source = 'wcs' if selected_candidate is None: selected_candidate = { @@ -12172,6 +12622,22 @@ def apply_parallel_alignment_result(result, frame_index, psf_data, tar_comp_dist 'warnings': [('alignment_error', -1, np.nan, np.nan)], } + return selected_source, selected_candidate, { + 'wcs_score': wcs_score, + 'fallback_score': fallback_score, + 'wcs_decision': wcs_alignment_decision, + } + + +def apply_parallel_alignment_result(result, frame_index, psf_data, tar_comp_dist, comp_keys): + selected_source, selected_candidate, _ = select_alignment_candidate( + result, + frame_index, + psf_data, + tar_comp_dist, + comp_keys, + ) + _store_alignment_candidate_psfs(selected_candidate, frame_index, psf_data, comp_keys) _replay_parallel_alignment_warnings(result.get('file_name'), selected_candidate.get('warnings')) if frame_index == 0: @@ -12503,6 +12969,61 @@ def _has_usable_centroid_signal(subarray, amplitude, min_snr=5.0): return amplitude >= (min_snr * scatter) +def _fit_seed_anchored_psf(data, pos, psf_function=gaussian_psf, box=8, bound_radius=4.0): + xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + moment_fit = _fit_centroid_moments(subarray, xv, yv, pos, box) + + try: + init = [np.nanmax(subarray) - np.nanmin(subarray), 1.0, 1.0, 0.0, np.nanmin(subarray)] + except ValueError: + return _nan_psf_result() + + if np.isfinite(moment_fit[0]): + init = [ + moment_fit[2], + min(float(moment_fit[3]), 3.0), + min(float(moment_fit[4]), 3.0), + moment_fit[5], + moment_fit[6], + ] + + bound_radius = float(bound_radius) + sigma_upper = max(float(PSF_FIT_MAX_SIGMA_PIXELS), 0.5) + lo = [ + pos[0] - bound_radius, + pos[1] - bound_radius, + 0, + 0.5, + 0.5, + -np.pi / 4, + np.nanmin(subarray) - 1, + ] + up = [ + pos[0] + bound_radius, + pos[1] + bound_radius, + 1e7, + sigma_upper, + sigma_upper, + np.pi / 4, + np.nanmax(subarray) + 1, + ] + x0 = np.array([pos[0], pos[1], *init], dtype=float) + lo_arr = np.array(lo, dtype=float) + up_arr = np.array(up, dtype=float) + if np.all(np.isfinite(x0)): + x0 = np.clip(x0, lo_arr + 1e-6, up_arr - 1e-6) + + def fcn2min(pars): + model = psf_function(xv, yv, *pars) + return (subarray - model).flatten() + + res = least_squares(fcn2min, x0=x0, bounds=[lo, up], jac='2-point', xtol=None, method='trf') + if np.isfinite(moment_fit[6]): + res.x[6] = moment_fit[6] + return res.x + + def _nan_psf_result(): return np.full(7, np.nan, dtype=float) @@ -12514,6 +13035,60 @@ def fit_centroid_or_warn_out_of_frame(data, pos, starIndex, **kwargs): return fit_centroid(data, pos, starIndex, **kwargs) +def psf_solution_quality_score(psf_row, seed_pos=None, + max_seed_offset_pixels=PSF_FIT_MAX_SEED_OFFSET_PIXELS, + max_axis_ratio=PSF_FIT_MAX_AXIS_RATIO, + max_sigma_pixels=PSF_FIT_MAX_SIGMA_PIXELS): + try: + row = np.asarray(psf_row, dtype=float).reshape(-1) + except (TypeError, ValueError): + return np.inf + if row.size < 5 or not np.all(np.isfinite(row[:5])): + return np.inf + + x_centroid, y_centroid, amplitude, sigma_x, sigma_y = row[:5] + if amplitude <= 0 or sigma_x <= 0 or sigma_y <= 0: + return np.inf + if max(sigma_x, sigma_y) > float(max_sigma_pixels): + return np.inf + + axis_ratio = max(sigma_x, sigma_y) / max(min(sigma_x, sigma_y), 1e-12) + if axis_ratio > float(max_axis_ratio): + return np.inf + + score = 0.25 * np.log(axis_ratio) + if seed_pos is not None: + try: + seed = np.asarray(seed_pos, dtype=float).reshape(-1) + except (TypeError, ValueError): + seed = np.array([], dtype=float) + if seed.size >= 2 and np.all(np.isfinite(seed[:2])): + seed_offset = float(np.hypot(x_centroid - seed[0], y_centroid - seed[1])) + if seed_offset > float(max_seed_offset_pixels): + return np.inf + score += seed_offset / max(float(max_seed_offset_pixels), 1e-12) + + mean_sigma = 0.5 * (sigma_x + sigma_y) + if np.isfinite(mean_sigma) and mean_sigma > 0: + score += 0.05 * abs(np.log(mean_sigma / 1.5)) + + score -= 0.005 * np.log1p(max(float(amplitude), 0.0)) + return float(score) + + +def choose_best_psf_solution(primary_row, fallback_row, seed_pos=None): + primary_score = psf_solution_quality_score(primary_row, seed_pos=seed_pos) + fallback_score = psf_solution_quality_score(fallback_row, seed_pos=seed_pos) + if np.isfinite(primary_score) and ( + not np.isfinite(fallback_score) + or primary_score <= fallback_score + PSF_FIT_SELECTION_MARGIN + ): + return np.asarray(primary_row, dtype=float) + if np.isfinite(fallback_score): + return np.asarray(fallback_row, dtype=float) + return _nan_psf_result() + + def fractional_flux_change_within_limit(current_amplitude, previous_amplitude, limit=0.5): if (not np.isfinite(current_amplitude) or not np.isfinite(previous_amplitude) @@ -12670,7 +13245,10 @@ def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weight wx, wy = moment_fit[0], moment_fit[1] init = [moment_fit[2], moment_fit[3], moment_fit[4], moment_fit[5], moment_fit[6]] if fast_mode: - if _has_usable_centroid_signal(subarray, init[0]): + if ( + _has_usable_centroid_signal(subarray, init[0]) + and centroid_position_is_finite(moment_fit) + ): return moment_fit plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) @@ -12701,7 +13279,10 @@ def fcn2min(pars): try: res = least_squares(fcn2min, x0=x0, bounds=[lo, up], jac='2-point', xtol=None, method='trf') except Exception as exc: - if has_usable_signal and np.isfinite(moment_fit[0]): + if ( + has_usable_signal + and centroid_position_is_finite(moment_fit) + ): log.debug(f"Centroid PSF fit failed at {np.round(pos, 2)}; using moment centroid instead: {exc}") return moment_fit @@ -12725,11 +13306,105 @@ def fcn2min(pars): if np.isfinite(moment_fit[6]): res.x[6] = moment_fit[6] - return res.x + anchored_fit = _nan_psf_result() + if not weightedcenter: + try: + anchored_box = max(4, min(int(box), 8)) + anchored_fit = _fit_seed_anchored_psf( + data, + pos, + psf_function=psf_function, + box=anchored_box, + bound_radius=4.0, + ) + except Exception as exc: + log.debug(f"Seed-anchored PSF fit failed at {np.round(pos, 2)}: {exc}") + + selected_fit = choose_best_psf_solution(res.x, anchored_fit, seed_pos=pos) + selected_fit = choose_best_psf_solution(selected_fit, moment_fit, seed_pos=pos) + if not np.all(np.isfinite(selected_fit[:5])): + log.debug( + f"Centroid PSF fit at {np.round(pos, 2)} rejected as implausible " + "(large seed offset, elongated PSF, or invalid width)." + ) + return selected_fit finally: _record_photometry_stage_timing('fit_centroid', perf_counter() - stage_start) +def fit_psf_photometry_flux_row(data, centroid_row, starIndex, psf_function=gaussian_psf, box=15): + try: + centroid_row = np.asarray(centroid_row, dtype=float).reshape(-1) + except (TypeError, ValueError): + return _nan_psf_result() + if centroid_row.size < 2 or not centroid_position_is_finite(centroid_row): + return _nan_psf_result() + + pos = centroid_row[:2] + try: + xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + init = [ + np.nanmax(subarray) - np.nanmin(subarray), + 1.0, + 1.0, + 0.0, + np.nanmin(subarray), + ] + except Exception: + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + lo = [ + pos[0] - box * 0.5, + pos[1] - box * 0.5, + 0, + 0.5, + 0.5, + -np.pi / 4, + np.nanmin(subarray) - 1, + ] + up = [ + pos[0] + box * 0.5, + pos[1] + box * 0.5, + 1e7, + 20, + 20, + np.pi / 4, + np.nanmax(subarray) + 1, + ] + + def fcn2min(pars): + model = psf_function(xv, yv, *pars) + return (subarray - model).flatten() + + try: + res = least_squares( + fcn2min, + x0=[*pos, *init], + bounds=[lo, up], + jac='3-point', + xtol=None, + method='trf', + ) + flux_row = np.asarray(res.x, dtype=float) + except Exception as exc: + log.debug( + f"Stable PSF photometry flux fit failed at {np.round(pos, 2)} for star {starIndex}; " + f"using centroid fit flux parameters instead: {exc}" + ) + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + if not np.all(np.isfinite(flux_row[:5])): + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + if not np.isfinite(psf_solution_quality_score(flux_row, seed_pos=pos)): + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + # Keep the robust centroid/offset solution for diagnostics and aperture placement, + # but use the legacy-stable Gaussian amplitude/width for PSF flux integration. + flux_row[:2] = centroid_row[:2] + return flux_row + + def sigma_clipped_nanmedian(data, sigma=3.0, max_iters=3): clipped = np.array(data, dtype=float, copy=True) if clipped.size == 0: @@ -14105,19 +14780,31 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet ['comp'], ) else: - use_wcs_alignment = False + alignment_result = { + 'index': i, + 'file_name': fileName, + 'wcs': None, + 'fallback': None, + } + previous_psf_rows = {} + if i != 0: + previous_psf_rows = { + 'target': psf_data['target'][i - 1], + 'comp1': psf_data['comp'][i - 1], + } + + has_wcs_alignment = False if not ignore_header_wcs: try: wcs_hdr = search_wcs_from_header(image_header) - use_wcs_alignment = wcs_hdr.is_celestial + has_wcs_alignment = wcs_hdr.is_celestial except Exception: - use_wcs_alignment = False + has_wcs_alignment = False - if use_wcs_alignment: + if has_wcs_alignment and target_and_comp_radec is not None: try: if i == 0: - tx, ty = exotic_UIprevTPX, exotic_UIprevTPY - cx, cy = comp_star + projected_coords = np.array(target_and_comp_pixels, dtype=float, copy=True) else: pix_x, pix_y = wcs_hdr.world_to_pixel_values( target_and_comp_radec[:, 0], @@ -14125,48 +14812,22 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet ) pix_x = np.asarray(pix_x, dtype=float).reshape(-1) pix_y = np.asarray(pix_y, dtype=float).reshape(-1) - tx, ty = pix_x[0], pix_y[0] - cx, cy = pix_x[1], pix_y[1] - - projected_coords = np.array([[tx, ty], [cx, cy]], dtype=float) - projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) - target_seed = choose_centroid_seed_position( - [tx, ty], - None if i == 0 else psf_data['target'][i - 1], - ) - comp_seed = choose_centroid_seed_position( - [cx, cy], - None if i == 0 else psf_data['comp'][i - 1], - ) + projected_coords = np.column_stack((pix_x, pix_y)) - psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + wcs_candidate = _fit_alignment_candidate_psfs( imageData, - target_seed, - 0, - fast_mode=target_fast_centroid, + projected_coords, + target_fast_centroid, + frame_fast_centroid, + previous_psf_rows=previous_psf_rows, ) - psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( - imageData, - comp_seed, - 1, - fast_mode=frame_fast_centroid, + wcs_candidate['projected_off_frame'] = any_projected_coord_out_of_frame( + projected_coords, + imageData.shape, ) - - if i == 0: - tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) - wcs_alignment_decision = should_keep_header_wcs_alignment( - projected_off_frame, - i, - psf_data['target'][i], - previous_target_psf_row=None if i == 0 else psf_data['target'][i - 1], - comp_psf_rows={'comp': psf_data['comp'][i]}, - previous_comp_psf_rows={} if i == 0 else {'comp': psf_data['comp'][i - 1]}, - expected_offsets={'comp': tar_comp_dist['comp']}, - ) - use_wcs_alignment = wcs_alignment_decision['use_wcs_alignment'] + alignment_result['wcs'] = wcs_candidate except Exception: - use_wcs_alignment = False + alignment_result['wcs'] = None log_alignment_progress( i, @@ -14175,43 +14836,29 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet use_multiprocess_transform_precompute, ) - if not use_wcs_alignment: - cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None - if cached_tform is not None: - tform = cached_tform - elif i == 0: - tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) - else: - tform = transformation(imageData, fileName, reference_image=firstImage) - - transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) - tx, ty = transformed_coords[0] - target_seed = choose_centroid_seed_position( - [tx, ty], - None if i == 0 else psf_data['target'][i - 1], - ) - psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( - imageData, - target_seed, - 0, - fast_mode=target_fast_centroid, - ) - - cx, cy = transformed_coords[1] - comp_seed = choose_centroid_seed_position( - [cx, cy], - None if i == 0 else psf_data['comp'][i - 1], - ) - psf_data['comp'][i] = fit_centroid_or_warn_out_of_frame( - imageData, - comp_seed, - 1, - fast_mode=frame_fast_centroid, - ) + cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None + if cached_tform is not None: + tform = cached_tform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + else: + tform = transformation(imageData, fileName, reference_image=firstImage) - if i == 0: - tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + alignment_result['fallback'] = _fit_alignment_candidate_psfs( + imageData, + transformed_coords, + target_fast_centroid, + frame_fast_centroid, + previous_psf_rows=previous_psf_rows, + ) + apply_parallel_alignment_result( + alignment_result, + i, + psf_data, + tar_comp_dist, + ['comp'], + ) # aperture photometry target_sigma = psf_sigma_from_fit(psf_data['target'][i], fallback_sigma=sigma) @@ -14912,6 +15559,47 @@ def prepare_lightcurve_fit_input_series( return prepared +def comparison_prescore_clip_mask(values, sigma=COMPARISON_STAR_PRESCORE_SIGMA_CLIP, + max_iters=COMPARISON_STAR_PRESCORE_MAX_CLIP_ITERS, + scatter_floor=COMPARISON_STAR_PRESCORE_SCATTER_FLOOR, + min_points=LIGHTCURVE_MIN_VALID_POINTS): + values = np.asarray(values, dtype=float) + finite_mask = np.isfinite(values) + keep_mask = finite_mask.copy() + if np.count_nonzero(keep_mask) < int(min_points): + return keep_mask + + for _ in range(int(max_iters)): + kept_values = values[keep_mask] + center = bn.nanmedian(kept_values) + if not np.isfinite(center): + break + + mad = bn.nanmedian(np.abs(kept_values - center)) + if np.isfinite(mad) and mad > 0: + scatter = 1.4826 * mad + else: + scatter = bn.nanstd(kept_values) + + if np.isfinite(scatter_floor) and scatter_floor > 0: + if not np.isfinite(scatter) or scatter <= 0: + scatter = float(scatter_floor) + else: + scatter = max(float(scatter), float(scatter_floor)) + + if not np.isfinite(scatter) or scatter <= 0: + break + + next_keep_mask = finite_mask & (np.abs(values - center) <= float(sigma) * scatter) + if np.count_nonzero(next_keep_mask) < int(min_points): + break + if np.array_equal(next_keep_mask, keep_mask): + break + keep_mask = next_keep_mask + + return keep_mask + + def cheap_lightcurve_prescore(tFlux, cFlux, airmass, enforce_relative_flux_max=True): with np.errstate(divide='ignore', invalid='ignore'): flux_ratio = np.divide(tFlux, cFlux) @@ -14923,15 +15611,37 @@ def cheap_lightcurve_prescore(tFlux, cFlux, airmass, enforce_relative_flux_max=T x_vals = airmass[finite_mask] y_vals = flux_ratio[finite_mask] - if should_skip_airmass_fit(x_vals): - detrended = y_vals / bn.nanmedian(y_vals) - else: - slope, intercept = np.polyfit(x_vals, y_vals, 1) - trend = slope * x_vals + intercept - with np.errstate(divide='ignore', invalid='ignore'): - detrended = np.divide(y_vals, trend) + score_mask = np.ones(y_vals.shape, dtype=bool) + detrended = np.full(y_vals.shape, np.nan, dtype=float) + for _ in range(COMPARISON_STAR_PRESCORE_MAX_CLIP_ITERS): + if np.count_nonzero(score_mask) < LIGHTCURVE_MIN_VALID_POINTS: + return np.inf + + if should_skip_airmass_fit(x_vals[score_mask]): + baseline = bn.nanmedian(y_vals[score_mask]) + if not np.isfinite(baseline) or baseline == 0: + return np.inf + detrended = y_vals / baseline + else: + slope, intercept = np.polyfit(x_vals[score_mask], y_vals[score_mask], 1) + trend = slope * x_vals + intercept + with np.errstate(divide='ignore', invalid='ignore'): + detrended = np.divide(y_vals, trend) - return bn.nanstd(detrended) + next_score_mask = comparison_prescore_clip_mask(detrended) + if np.array_equal(next_score_mask, score_mask): + break + score_mask = next_score_mask + + finite_detrended = detrended[score_mask & np.isfinite(detrended)] + if finite_detrended.size < LIGHTCURVE_MIN_VALID_POINTS: + return np.inf + + scatter = bn.nanstd(finite_detrended) + if np.isfinite(scatter) and scatter >= 0: + return float(scatter) + + return np.inf def evaluate_lightcurve_candidate(task): @@ -15008,17 +15718,21 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, require_comp_star=True, skip_low_comparison_coverage_rejection=False, use_psf_photometry=True, - use_aperture_photometry=True): + use_aperture_photometry=True, + psf_flux_data=None): candidate_jobs = [] comp_star_count = len(comp_stars) + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) if use_psf_photometry and comp_star_count > 0: - target_flux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + frame_count = psf_data['target'].shape[0] + target_flux = psf_flux_series_from_rows(psf_flux_data['target']) target_flux_mask = robust_flux_floor_mask(target_flux) psf_comp_flux_map = { - f"comp{comp_idx + 1}": 2 * np.pi * psf_data[f"comp{comp_idx + 1}"][:, 2] - * psf_data[f"comp{comp_idx + 1}"][:, 3] - * psf_data[f"comp{comp_idx + 1}"][:, 4] + f"comp{comp_idx + 1}": psf_flux_series_from_rows( + psf_flux_data[f"comp{comp_idx + 1}"], + psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count), + ) for comp_idx in range(comp_star_count) } psf_comp_coverage = comparison_star_coverage_summary( @@ -15032,7 +15746,8 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, continue comp_flux = psf_comp_flux_map[ckey] - psf_mask = target_flux_mask & robust_flux_floor_mask(comp_flux) + target_shape_mask = target_psf_shape_quality_mask(psf_data['target'], psf_data[ckey]) + psf_mask = target_shape_mask & target_flux_mask & robust_flux_floor_mask(comp_flux) candidate_jobs.append({ 'method': 'psf', 'a': None, @@ -15056,11 +15771,15 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, }) if use_aperture_photometry and aper_data is not None and apers is not None and annuli is not None: + frame_count = aper_data['target'].shape[0] for a, aper in enumerate(apers): for an, annulus in enumerate(annuli): - target_flux = aper_data['target'][:, a, an] + target_flux = np.asarray(aper_data['target'][:, a, an], dtype=float) aperture_comp_flux_map = { - f"comp{comp_idx + 1}": aper_data[f"comp{comp_idx + 1}"][:, a, an] + f"comp{comp_idx + 1}": mask_series_with_quality( + aper_data[f"comp{comp_idx + 1}"][:, a, an], + psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count), + ) for comp_idx in range(comp_star_count) } aperture_comp_coverage = comparison_star_coverage_summary( @@ -15097,7 +15816,7 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, continue comp_series = aperture_comp_flux_map[ckey] - aper_mask = valid_comparison_frame_mask(comp_series) + aper_mask = valid_comparison_frame_mask(target_flux) & valid_comparison_frame_mask(comp_series) candidate_jobs.append({ 'method': 'aperture', 'a': a, @@ -15128,18 +15847,24 @@ def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, p disable_vertical_flux_normalization=False, use_impactparameter_rather_than_inclination_to_fit=True, use_eebls_to_initialize_tmid_and_bounds=True, - compute_eebls_diagnostics=True): + compute_eebls_diagnostics=True, + psf_flux_data=None): candidate_mask = np.asarray(candidate['mask'], dtype=bool) if candidate['method'] == 'psf': - target_flux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + target_flux = ( + 2 * np.pi * psf_flux_data['target'][:, 2] + * psf_flux_data['target'][:, 3] + * psf_flux_data['target'][:, 4] + ) if candidate['ckey'] is None: comp_flux = np.ones(target_flux.shape[0], dtype=float) else: comp_flux = ( - 2 * np.pi * psf_data[candidate['ckey']][:, 2] - * psf_data[candidate['ckey']][:, 3] - * psf_data[candidate['ckey']][:, 4] + 2 * np.pi * psf_flux_data[candidate['ckey']][:, 2] + * psf_flux_data[candidate['ckey']][:, 3] + * psf_flux_data[candidate['ckey']][:, 4] ) else: target_flux = aper_data['target'][:, candidate['a'], candidate['an']] @@ -15175,7 +15900,8 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co multiprocess_lightcurve_fits=None, use_impactparameter_rather_than_inclination_to_fit=True, use_eebls_to_initialize_tmid_and_bounds=True, - pick_comparison_by_eebls_snr=True): + pick_comparison_by_eebls_snr=True, + psf_flux_data=None): candidate_jobs = build_target_fit_candidate_jobs( psf_data, aper_data, @@ -15188,6 +15914,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co skip_low_comparison_coverage_rejection=skip_low_comparison_coverage_rejection, use_psf_photometry=use_psf_photometry, use_aperture_photometry=use_aperture_photometry, + psf_flux_data=psf_flux_data, ) evaluated_candidates = list(candidate_jobs) for candidate_order, candidate in enumerate(evaluated_candidates): @@ -15223,6 +15950,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, compute_eebls_diagnostics=True, + psf_flux_data=psf_flux_data, ) for candidate in evaluated_candidates ] @@ -16346,17 +17074,24 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p disable_vertical_flux_normalization=False, skip_low_comparison_coverage_rejection=False, use_impactparameter_rather_than_inclination_to_fit=True, - use_eebls_to_initialize_tmid_and_bounds=True): + use_eebls_to_initialize_tmid_and_bounds=True, + psf_flux_data=None): if photometry_info.get('best_fit_lc') is None or not comp_stars: return [] use_psf_photometry = photometry_info.get('min_aperture') == 0 if use_psf_photometry: - target_flux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + frame_count = psf_data['target'].shape[0] + else: + frame_count = aper_data['target'].shape[0] + if use_psf_photometry: + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + target_flux = psf_flux_series_from_rows(psf_flux_data['target']) comp_flux_map = { - f"comp{comp_index + 1}": 2 * np.pi * psf_data[f"comp{comp_index + 1}"][:, 2] - * psf_data[f"comp{comp_index + 1}"][:, 3] - * psf_data[f"comp{comp_index + 1}"][:, 4] + f"comp{comp_index + 1}": psf_flux_series_from_rows( + psf_flux_data[f"comp{comp_index + 1}"], + psf_quality_mask_for_key(psf_data, f"comp{comp_index + 1}", frame_count), + ) for comp_index in range(len(comp_stars)) } else: @@ -16364,9 +17099,12 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p annulus_index = photometry_info.get('annulus_index') if aperture_index is None or annulus_index is None: return [] - target_flux = aper_data['target'][:, aperture_index, annulus_index] + target_flux = np.asarray(aper_data['target'][:, aperture_index, annulus_index], dtype=float) comp_flux_map = { - f"comp{comp_index + 1}": aper_data[f"comp{comp_index + 1}"][:, aperture_index, annulus_index] + f"comp{comp_index + 1}": mask_series_with_quality( + aper_data[f"comp{comp_index + 1}"][:, aperture_index, annulus_index], + psf_quality_mask_for_key(psf_data, f"comp{comp_index + 1}", frame_count), + ) for comp_index in range(len(comp_stars)) } @@ -16386,9 +17124,14 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p comp_flux_series = comp_flux_map[ckey] if use_psf_photometry: - fit_mask = robust_target_reference_flux_mask(target_flux, comp_flux_series) + target_shape_mask = target_psf_shape_quality_mask(psf_data['target'], psf_data[ckey]) + if target_shape_mask.shape[0] != frame_count: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + fit_mask = target_shape_mask & robust_target_reference_flux_mask(candidate_target_flux, comp_flux_series) else: - fit_mask = valid_comparison_frame_mask(comp_flux_series) + candidate_target_flux = target_flux + fit_mask = valid_comparison_frame_mask(candidate_target_flux) & valid_comparison_frame_mask(comp_flux_series) coverage_count = coverage_summary[ckey]['coverage_count'] coverage_total_frame_count = coverage_summary[ckey]['coverage_total_frame_count'] coverage_reference_count = coverage_summary[ckey]['coverage_reference_count'] @@ -16415,7 +17158,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p elif coverage_count > 1: fit_diagnostics = diagnose_lightcurve_fit_inputs( times[fit_mask], - target_flux[fit_mask], + candidate_target_flux[fit_mask], comp_flux_series[fit_mask], airmass[fit_mask], enforce_relative_flux_max=False, @@ -16424,7 +17167,7 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p if not coverage_rejected and coverage_count > 1 and fit_diagnostics['failure_reason'] is None: fit_result, target_fit_flux, comp_fit_flux = fit_lightcurve( times[fit_mask], - target_flux[fit_mask], + candidate_target_flux[fit_mask], comp_flux_series[fit_mask], airmass[fit_mask], ld, @@ -17333,14 +18076,21 @@ def _refined_sigma_grid(center, lower_bound, upper_bound, half_width, points): def auto_tune_aperture_sigma_grid(coarse_apertures_sigma, coarse_annuli_sigma, coarse_aper_data, comp_star_count, subset_airmass, require_comp_star=True, - skip_low_comparison_coverage_rejection=False): + skip_low_comparison_coverage_rejection=False, + psf_quality_masks=None): best_candidate = None best_score = np.inf for a_idx, aperture_sigma in enumerate(coarse_apertures_sigma): for an_idx, annulus_sigma in enumerate(coarse_annuli_sigma): comp_flux_map = { - f"comp{comp_idx + 1}": coarse_aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx] + f"comp{comp_idx + 1}": mask_series_with_quality( + coarse_aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx], + (psf_quality_masks or {}).get( + f"comp{comp_idx + 1}", + np.ones(coarse_aper_data[f"comp{comp_idx + 1}"].shape[0], dtype=bool), + ), + ) for comp_idx in range(comp_star_count) } field_summary = comparison_star_stability_summary( @@ -17401,18 +18151,27 @@ def comparison_method_label(candidate): def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, airmass, comp_stars, sigma, skip_low_comparison_coverage_rejection=False, use_psf_photometry=True, - use_aperture_photometry=True): + use_aperture_photometry=True, + psf_flux_data=None): candidate_summaries = [] comp_star_count = len(comp_stars) + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) if comp_star_count == 0: return None + frame_count = len(airmass) + psf_quality_masks = { + f"comp{comp_idx + 1}": psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count) + for comp_idx in range(comp_star_count) + } + if use_psf_photometry: psf_flux_map = { - f"comp{comp_idx + 1}": 2 * np.pi * psf_data[f"comp{comp_idx + 1}"][:, 2] - * psf_data[f"comp{comp_idx + 1}"][:, 3] - * psf_data[f"comp{comp_idx + 1}"][:, 4] + f"comp{comp_idx + 1}": psf_flux_series_from_rows( + psf_flux_data[f"comp{comp_idx + 1}"], + psf_quality_masks[f"comp{comp_idx + 1}"], + ) for comp_idx in range(comp_star_count) } psf_summary = comparison_star_stability_summary( @@ -17434,7 +18193,10 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, for a_idx, aperture in enumerate(apers): for an_idx, annulus in enumerate(annuli): comp_flux_map = { - f"comp{comp_idx + 1}": aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx] + f"comp{comp_idx + 1}": mask_series_with_quality( + aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx], + psf_quality_masks[f"comp{comp_idx + 1}"], + ) for comp_idx in range(comp_star_count) } candidate_summary = comparison_star_stability_summary( @@ -17467,6 +18229,12 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, comp_summary = dict(summary) comp_summary['position'] = comp_stars[comp_summary['comp_index']] comp_summary['selected'] = comp_summary['comp_index'] == best_comp_index + quality_mask = psf_quality_masks.get(comp_summary['key']) + if quality_mask is not None: + comp_summary['psf_quality_keep_mask'] = quality_mask + comp_summary['psf_quality_rejected_count'] = int(np.count_nonzero(~quality_mask)) + else: + comp_summary['psf_quality_rejected_count'] = 0 comp_summary['selection_reason'] = comparison_calibration_selection_reason( comp_summary, best_candidate['best_comp_score'], @@ -17504,6 +18272,7 @@ def ranked_comparison_calibration_summaries(comparison_calibration): def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p_dict, comparison_calibration, psf_data, aper_data, target_psf_flux, + psf_flux_data=None, plot_time_range=None, disable_vertical_flux_normalization=False, detrend_on_outoftransit_baseline=True, @@ -17535,9 +18304,14 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p aperture_index = comparison_calibration.get('a') annulus_index = comparison_calibration.get('an') if method == 'psf': - target_flux = target_psf_flux + frame_count = target_psf_flux.shape[0] + else: + frame_count = aper_data['target'].shape[0] + if method == 'psf': + target_flux = np.asarray(target_psf_flux, dtype=float) + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) else: - target_flux = aper_data['target'][:, aperture_index, annulus_index] + target_flux = np.asarray(aper_data['target'][:, aperture_index, annulus_index], dtype=float) adaptive_summary = build_comparison_candidate_adaptive_summary( comparison_calibration, @@ -17572,14 +18346,25 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p for field_rank, comp_summary in enumerate(ranked_summaries): comp_index = comp_summary['comp_index'] ckey = comp_summary.get('key', f"comp{comp_index + 1}") + comp_quality_mask = np.asarray( + comp_summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key(psf_data, ckey, frame_count), + ), + dtype=bool, + ) + if comp_quality_mask.shape[0] != frame_count: + comp_quality_mask = psf_quality_mask_for_key(psf_data, ckey, frame_count) if method == 'psf': - comp_flux = ( - 2 * np.pi * psf_data[ckey][:, 2] - * psf_data[ckey][:, 3] - * psf_data[ckey][:, 4] - ) + target_shape_mask = target_psf_shape_quality_mask(psf_data['target'], psf_data[ckey]) + if target_shape_mask.shape[0] != frame_count: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + comp_flux = psf_flux_series_from_rows(psf_flux_data[ckey], comp_quality_mask) else: - comp_flux = aper_data[ckey][:, aperture_index, annulus_index] + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = target_flux + comp_flux = mask_series_with_quality(aper_data[ckey][:, aperture_index, annulus_index], comp_quality_mask) candidate_frame_keep_mask = np.asarray( comp_summary.get('ensemble_frame_keep_mask', np.ones(times.shape[0], dtype=bool)), @@ -17606,13 +18391,15 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ), ) - fit_mask = field_image_keep_mask & candidate_frame_keep_mask + fit_mask = field_image_keep_mask & candidate_frame_keep_mask & comp_quality_mask & target_shape_mask if method == 'psf': - fit_mask &= robust_target_reference_flux_mask(target_flux, comp_flux) + fit_mask &= robust_target_reference_flux_mask(candidate_target_flux, comp_flux) + else: + fit_mask &= valid_comparison_frame_mask(candidate_target_flux) & valid_comparison_frame_mask(comp_flux) fit_diagnostics = diagnose_lightcurve_fit_inputs( times[fit_mask], - target_flux[fit_mask], + candidate_target_flux[fit_mask], comp_flux[fit_mask], airmass[fit_mask], enforce_relative_flux_max=False, @@ -17624,7 +18411,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p airmass[fit_mask], ld, p_dict, - target_flux[fit_mask], + candidate_target_flux[fit_mask], comp_flux[fit_mask], adaptive_summary=adaptive_summary, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, @@ -17633,6 +18420,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'field_rank': field_rank, 'summary': comp_summary, 'ckey': ckey, + 'target_flux': candidate_target_flux, 'comp_flux': comp_flux, 'fit_mask': fit_mask, 'candidate_frame_clip_diagnostic': candidate_frame_clip_diagnostic, @@ -17650,6 +18438,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p comp_summary = plan['summary'] comp_index = comp_summary['comp_index'] ckey = plan['ckey'] + candidate_target_flux = plan.get('target_flux', target_flux) comp_flux = plan['comp_flux'] fit_mask = plan['fit_mask'] candidate_frame_clip_diagnostic = plan.get('candidate_frame_clip_diagnostic') @@ -17668,7 +18457,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ) final_reduction = finalize_comparison_candidate_full_reduction( times[fit_mask], - target_flux[fit_mask], + candidate_target_flux[fit_mask], comp_flux[fit_mask], airmass[fit_mask], ld, @@ -18827,6 +19616,9 @@ def _main_impl(): # x-cent, y-cent, amplitude, sigma-x, sigma-y, rotation, offset 'target': np.zeros((len(inputfiles), 7)), # PSF fit } + psf_flux_data = { + 'target': np.zeros((len(inputfiles), 7)), + } tar_comp_dist = {} vsp_num = [] comp_star_count = len(exotic_infoDict['comp_stars']) @@ -18896,6 +19688,7 @@ def _main_impl(): if coord in vsp_list: vsp_num.append(i) psf_data[ckey] = np.zeros((len(inputfiles), 7)) + psf_flux_data[ckey] = np.zeros((len(inputfiles), 7)) tar_comp_dist[ckey] = np.zeros(2) coarse_tune_frames = 0 @@ -19032,85 +19825,52 @@ def _main_impl(): comp_alignment_keys, ) else: - use_wcs_alignment = False + alignment_result = { + 'index': i, + 'file_name': fileName, + 'wcs': None, + 'fallback': None, + } + previous_psf_rows = {} + if i != 0: + previous_psf_rows = {'target': psf_data['target'][i - 1]} + for comp_idx, comp_key in enumerate(comp_alignment_keys): + previous_psf_rows[f"comp{comp_idx + 1}"] = psf_data[comp_key][i - 1] + + has_wcs_alignment = False if not ignore_header_wcs: try: wcs_hdr = search_wcs_from_header(image_header) - use_wcs_alignment = wcs_hdr.is_celestial + has_wcs_alignment = wcs_hdr.is_celestial except Exception: - use_wcs_alignment = False + has_wcs_alignment = False - if use_wcs_alignment: + if has_wcs_alignment and target_and_comp_radec is not None: try: - pix_x = pix_y = None - if target_and_comp_radec is not None: - pix_x, pix_y = wcs_hdr.world_to_pixel_values( - target_and_comp_radec[:, 0], - target_and_comp_radec[:, 1], - ) - pix_x = np.asarray(pix_x, dtype=float).reshape(-1) - pix_y = np.asarray(pix_y, dtype=float).reshape(-1) - - if i == 0: - tx, ty = exotic_UIprevTPX, exotic_UIprevTPY - else: - tx, ty = pix_x[0], pix_y[0] - - projected_coords = np.array( - [[tx, ty], *np.column_stack((pix_x[1:], pix_y[1:]))] if pix_x is not None else [[tx, ty]], - dtype=float, - ) - projected_off_frame = any_projected_coord_out_of_frame(projected_coords, imageData.shape) - target_seed = choose_centroid_seed_position( - [tx, ty], - None if i == 0 else psf_data['target'][i - 1], + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + projected_coords = np.column_stack((pix_x, pix_y)) + if i == 0: + projected_coords[0] = target_and_comp_pixels[0] - psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + wcs_candidate = _fit_alignment_candidate_psfs( imageData, - target_seed, - 0, - fast_mode=target_fast_centroid, + projected_coords, + target_fast_centroid, + frame_fast_centroid, + previous_psf_rows=previous_psf_rows, ) - - # TODO: Add check for flux on target/comp stars relative to others in the field - # in case of cloudy data, large changes, etc. - current_comp_psf_rows = {} - previous_comp_psf_rows = {} - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - - cx, cy = pix_x[j + 1], pix_y[j + 1] - comp_seed = choose_centroid_seed_position( - [cx, cy], - None if i == 0 else psf_data[ckey][i - 1], - ) - psf_data[ckey][i] = fit_centroid_or_warn_out_of_frame( - imageData, - comp_seed, - j + 1, - fast_mode=frame_fast_centroid, - ) - - current_comp_psf_rows[ckey] = psf_data[ckey][i] - if i != 0: - previous_comp_psf_rows[ckey] = psf_data[ckey][i - 1] - else: - tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) - - wcs_alignment_decision = should_keep_header_wcs_alignment( - projected_off_frame, - i, - psf_data['target'][i], - previous_target_psf_row=None if i == 0 else psf_data['target'][i - 1], - comp_psf_rows=current_comp_psf_rows, - previous_comp_psf_rows=previous_comp_psf_rows, - expected_offsets=tar_comp_dist, + wcs_candidate['projected_off_frame'] = any_projected_coord_out_of_frame( + projected_coords, + imageData.shape, ) - use_wcs_alignment = wcs_alignment_decision['use_wcs_alignment'] + alignment_result['wcs'] = wcs_candidate except Exception: - use_wcs_alignment = False + alignment_result['wcs'] = None log_alignment_progress( i, @@ -19119,47 +19879,43 @@ def _main_impl(): use_multiprocess_transform_precompute, ) - if not use_wcs_alignment: - cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None - if cached_tform is not None: - tform = cached_tform - elif i == 0: - tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) - else: - tform = transformation(imageData, fileName, reference_image=firstImage) + cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None + if cached_tform is not None: + tform = cached_tform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + else: + tform = transformation(imageData, fileName, reference_image=firstImage) - transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) - tx, ty = transformed_coords[0] - target_seed = choose_centroid_seed_position( - [tx, ty], - None if i == 0 else psf_data['target'][i - 1], - ) - psf_data['target'][i] = fit_centroid_or_warn_out_of_frame( + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + alignment_result['fallback'] = _fit_alignment_candidate_psfs( + imageData, + transformed_coords, + target_fast_centroid, + frame_fast_centroid, + previous_psf_rows=previous_psf_rows, + ) + apply_parallel_alignment_result( + alignment_result, + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + + if use_psf_photometry: + psf_flux_data['target'][i] = fit_psf_photometry_flux_row( + imageData, + psf_data['target'][i], + 0, + ) + for comp_idx, comp_key in enumerate(comp_alignment_keys): + psf_flux_data[comp_key][i] = fit_psf_photometry_flux_row( imageData, - target_seed, - 0, - fast_mode=target_fast_centroid, + psf_data[comp_key][i], + comp_idx + 1, ) - for j, coord in enumerate(exotic_infoDict['comp_stars']): - ckey = f"comp{j + 1}" - - cx, cy = transformed_coords[j + 1] - comp_seed = choose_centroid_seed_position( - [cx, cy], - None if i == 0 else psf_data[ckey][i - 1], - ) - psf_data[ckey][i] = fit_centroid_or_warn_out_of_frame( - imageData, - comp_seed, - j + 1, - fast_mode=frame_fast_centroid, - ) - - if i == 0: - tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) - # aperture photometry if use_aperture_photometry and i == 0: sigma = psf_sigma_from_fit(psf_data['target'][0]) @@ -19199,6 +19955,14 @@ def _main_impl(): if i == coarse_tune_frames - 1: subset_airmass = np.asarray(airMassList[:coarse_tune_frames], dtype=float) + coarse_psf_quality_masks = { + f"comp{comp_idx + 1}": psf_quality_mask_for_key( + {key: value[:coarse_tune_frames] for key, value in psf_data.items()}, + f"comp{comp_idx + 1}", + coarse_tune_frames, + ) + for comp_idx in range(comp_star_count) + } refined_apertures_sigma, refined_annuli_sigma, best_coarse_candidate, best_coarse_score = auto_tune_aperture_sigma_grid( coarse_apertures_sigma, coarse_annuli_sigma, @@ -19207,6 +19971,7 @@ def _main_impl(): subset_airmass, require_comp_star=require_comp_star, skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + psf_quality_masks=coarse_psf_quality_masks, ) if use_adaptive_apertures: aperture_values = refined_apertures_sigma @@ -19329,16 +20094,74 @@ def _main_impl(): jd_times = jd_times[goodmask] airmass = np.array(airMassList)[goodmask] psf_data["target"] = psf_data["target"][goodmask] + psf_flux_data["target"] = psf_flux_data["target"][goodmask] if aper_data is not None: aper_data["target"] = aper_data["target"][goodmask] aper_data["target_bg"] = aper_data["target_bg"][goodmask] for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" psf_data[ckey] = psf_data[ckey][goodmask] + psf_flux_data[ckey] = psf_flux_data[ckey][goodmask] if aper_data is not None: aper_data[ckey] = aper_data[ckey][goodmask] aper_data[f"{ckey}_bg"] = aper_data[f"{ckey}_bg"][goodmask] + psf_quality_diagnostics = [] + target_quality_components = target_psf_shape_quality_components(psf_data['target']) + target_quality_keep_mask = target_quality_components['keep_mask'] + if target_quality_keep_mask.shape == times.shape and np.any(~target_quality_keep_mask): + reason_parts = [] + invalid_count = int(np.count_nonzero(target_quality_components['invalid_mask'])) + seeing_count = int(np.count_nonzero(target_quality_components['seeing_outlier_mask'])) + axis_ratio_count = int(np.count_nonzero(target_quality_components['axis_ratio_outlier_mask'])) + if invalid_count: + reason_parts.append(f"invalid PSF={invalid_count}") + if seeing_count: + reason_parts.append(f"broad PSF outlier={seeing_count}") + if axis_ratio_count: + reason_parts.append(f"elongated PSF={axis_ratio_count}") + reason_text = "; ".join(reason_parts) + psf_quality_diagnostics.append(build_time_rejection_diagnostic( + "Target PSF shape quality filter", + times, + target_quality_keep_mask, + note=( + "Dropped target frame-level PSF photometry before target/comparison fitting " + f"based on robust PSF shape diagnostics ({reason_text})." + ), + )) + for key, label in [ + (f"comp{j + 1}", f"Comp {j + 1}") for j in range(len(exotic_infoDict['comp_stars'])) + ]: + quality_components = psf_frame_quality_components(psf_data[key]) + quality_keep_mask = quality_components['keep_mask'] + if quality_keep_mask.shape == times.shape and np.any(~quality_keep_mask): + reason_parts = [] + invalid_count = int(np.count_nonzero(quality_components['invalid_mask'])) + seeing_count = int(np.count_nonzero(quality_components['seeing_outlier_mask'])) + amplitude_count = int(np.count_nonzero(quality_components['amplitude_outlier_mask'])) + if invalid_count: + reason_parts.append(f"invalid PSF={invalid_count}") + if seeing_count: + reason_parts.append(f"seeing outlier={seeing_count}") + if amplitude_count: + reason_parts.append(f"low amplitude outlier={amplitude_count}") + reason_text = "; ".join(reason_parts) + psf_quality_diagnostics.append(build_time_rejection_diagnostic( + f"{label} PSF seeing/amplitude quality filter", + times, + quality_keep_mask, + note=( + "Dropped this star's frame-level photometry before comparison ensemble scoring " + f"based on robust PSF diagnostics ({reason_text})." + ), + )) + if psf_quality_diagnostics: + log_lightcurve_filter_diagnostics( + psf_quality_diagnostics, + header="PSF frame rejections before comparison-star calibration", + ) + sigma_display = representative_psf_sigma(psf_data['target'], fallback_sigma=sigma) if not np.isfinite(sigma_display) or sigma_display <= 0: sigma_display = 1.0 @@ -19358,9 +20181,25 @@ def _main_impl(): header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f") # x-cent, y-cent, amplitude, sigma-x, sigma-y, rotation, offset + if use_psf_photometry: + np.savetxt( + Path(exotic_infoDict['save']) / "temp" / "psf_flux_data_target.txt", + psf_flux_data["target"], + header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", + fmt="%.6f", + ) + for j in range(len(exotic_infoDict['comp_stars'])): + ckey = f"comp{j + 1}" + np.savetxt( + Path(exotic_infoDict['save']) / "temp" / f"psf_flux_data_{ckey}.txt", + psf_flux_data[ckey], + header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", + fmt="%.6f", + ) # PSF flux - tFlux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + psf_flux_source = psf_flux_data if use_psf_photometry else psf_data + tFlux = psf_flux_series_from_rows(psf_flux_source['target']) ref_flux = {} if vsp_list: @@ -19409,6 +20248,7 @@ def _main_impl(): skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, use_psf_photometry=use_psf_photometry, use_aperture_photometry=use_aperture_photometry, + psf_flux_data=psf_flux_source, ) if comparison_calibration is not None: @@ -19454,10 +20294,16 @@ def _main_impl(): ensemble_frame_text = ( f", ensemble_frame_rejects={summary['ensemble_frame_rejected_count']}" ) + psf_quality_text = "" + if summary.get('psf_quality_rejected_count', 0) > 0: + psf_quality_text = ( + f", psf_quality_rejects={summary['psf_quality_rejected_count']}" + ) log_info( f" {summary['label']}{selected_label} ({position_text}): suitability={aggregate_text}, " f"ensemble={ensemble_text}, pairwise_median={pairwise_text}, " - f"valid_pairs={summary['valid_pair_count']}, {coverage_text}{ensemble_frame_text}, " + f"valid_pairs={summary['valid_pair_count']}, {coverage_text}" + f"{psf_quality_text}{ensemble_frame_text}, " f"reason={summary['selection_reason']}" ) @@ -19515,6 +20361,7 @@ def _main_impl(): psf_data, aper_data, tFlux, + psf_flux_data=psf_flux_source, plot_time_range=full_plot_time_range, disable_vertical_flux_normalization=disable_vertical_flux_normalization, detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, @@ -19676,7 +20523,7 @@ def _main_impl(): if comparison_calibration['method'] == 'psf': for j in vsp_num: ckey = f"comp{j + 1}" - cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] + cFlux = psf_flux_series_from_rows(psf_flux_source[ckey]) vsp_fit, _, _ = fit_lightcurve( times, tFlux, cFlux, airmass, ld, pDict, jd_times, disable_vertical_flux_normalization=disable_vertical_flux_normalization, @@ -19847,6 +20694,7 @@ def _main_impl(): use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + psf_flux_data=psf_flux_source, ) saved_candidate_fit_count = sum(1 for summary in candidate_fit_summaries if summary['fit'] is not None) failed_candidate_fit_count = len(candidate_fit_summaries) - saved_candidate_fit_count diff --git a/exotic/output_files.py b/exotic/output_files.py index b1607bc2..8e5f979b 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1507,7 +1507,8 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,ensemble_score,pairwise_median_score," "pairwise_max_score,self_score,valid_pair_count,coverage_count,coverage_peer_median," "coverage_min_required,coverage_rejected,suitability_outlier_rejected," - "ensemble_frame_rejected_count,ensemble_frame_required_valid_pairs\n") + "psf_quality_rejected_count,ensemble_frame_rejected_count," + "ensemble_frame_required_valid_pairs\n") for summary in comp_summaries: position = summary.get('position') or [None, None] @@ -1527,6 +1528,7 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label summary.get('coverage_min_required_count'), summary.get('coverage_rejected'), summary.get('suitability_outlier_rejected'), + summary.get('psf_quality_rejected_count', 0), summary.get('ensemble_frame_rejected_count', 0), summary.get('ensemble_frame_required_valid_pairs', 0), ] diff --git a/exotic/plots.py b/exotic/plots.py index 355f50e6..e7e99272 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -388,21 +388,32 @@ def _plot_bestfit_for_lightcurve_png(fit, **requested_kwargs): def _draw_comp_star_calibration_axis(axis, times, summary, colors): axis.axhline(1.0, color='lightgray', lw=1.0, zorder=1) + ensemble_keep_mask = np.asarray(summary.get('ensemble_frame_keep_mask'), dtype=bool) + has_ensemble_keep_mask = ensemble_keep_mask.shape == times.shape pairwise_series = summary.get('pairwise_ratio_series', {}) for color_index, (other_label, ratio_series) in enumerate(pairwise_series.items()): ratio_series = np.asarray(ratio_series, dtype=float) - valid = np.isfinite(times) & np.isfinite(ratio_series) - if np.any(valid): - axis.plot(times[valid], ratio_series[valid], color=colors[color_index % len(colors)], + line_ratio = ratio_series.copy() + if has_ensemble_keep_mask and line_ratio.shape == times.shape: + line_ratio[~ensemble_keep_mask] = np.nan + valid_time = np.isfinite(times) + valid_line = valid_time & np.isfinite(line_ratio) + if np.any(valid_line): + axis.plot(times[valid_time], line_ratio[valid_time], color=colors[color_index % len(colors)], alpha=0.55, lw=1.0, label=other_label) ensemble_ratio = np.asarray(summary.get('ensemble_ratio_series'), dtype=float) - ensemble_valid = np.isfinite(times) & np.isfinite(ensemble_ratio) + ensemble_time_valid = np.isfinite(times) + ensemble_valid = ensemble_time_valid & np.isfinite(ensemble_ratio) if np.any(ensemble_valid): - axis.plot(times[ensemble_valid], ensemble_ratio[ensemble_valid], color='black', lw=1.8, - label='Ensemble') - ensemble_keep_mask = np.asarray(summary.get('ensemble_frame_keep_mask'), dtype=bool) - if ensemble_keep_mask.shape == times.shape: + line_ratio = ensemble_ratio.copy() + if has_ensemble_keep_mask and line_ratio.shape == times.shape: + line_ratio[~ensemble_keep_mask] = np.nan + line_valid = ensemble_time_valid & np.isfinite(line_ratio) + if np.any(line_valid): + axis.plot(times[ensemble_time_valid], line_ratio[ensemble_time_valid], color='black', lw=1.8, + label='Ensemble') + if has_ensemble_keep_mask: rejected = ensemble_valid & ~ensemble_keep_mask if np.any(rejected): axis.scatter(times[rejected], ensemble_ratio[rejected], marker='x', s=42, diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 60c4b14b..27bf6c93 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -275,6 +275,32 @@ def test_fit_centroid_full_mode_preserves_psf_subpixel_solution(): assert psf_error < moment_error +def test_fit_psf_photometry_flux_row_preserves_robust_centroid_coordinates(): + image = _gaussian_image(center=(40.3, 35.7), amplitude=180.0, sigma=0.9, background=1000.0) + centroid_row = np.array([40.1, 35.9, 150.0, 0.8, 0.8, 0.0, 1000.0], dtype=float) + + flux_row = exotic_module.fit_psf_photometry_flux_row(image, centroid_row, 0) + + assert flux_row[0] == pytest.approx(centroid_row[0]) + assert flux_row[1] == pytest.approx(centroid_row[1]) + assert flux_row[2] > 0 + assert 0.5 <= flux_row[3] <= 2.0 + assert 0.5 <= flux_row[4] <= 2.0 + + +def test_fit_centroid_prefers_seed_anchored_solution_in_crowded_field(): + yy, xx = np.mgrid[0:80, 0:80] + image = np.full((80, 80), 400.0) + image += 80.0 * np.exp(-((xx - 40.0) ** 2 + (yy - 40.0) ** 2) / (2.0 * 1.0 ** 2)) + image += 220.0 * np.exp(-((xx - 31.5) ** 2 + (yy - 35.0) ** 2) / (2.0 * 1.0 ** 2)) + + result = exotic_module.fit_centroid(image, [40.0, 40.0], 0, fast_mode=False) + + assert np.hypot(result[0] - 40.0, result[1] - 40.0) < 1.5 + assert np.hypot(result[0] - 31.5, result[1] - 35.0) > 5.0 + assert result[2] > 0 + + def test_fit_centroid_or_warn_out_of_frame_skips_centroid_fit(monkeypatch): image = np.zeros((40, 50), dtype=float) out_of_frame_warnings = [] diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index b795273e..86ddc0c4 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -121,6 +121,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: deduplicate_comparison_star_coords, diagnose_lightcurve_fit_inputs, detrend_flux_on_out_of_transit_baseline, + alignment_candidate_quality_score, ensure_lightcurve_fit_failure_reason, evaluate_lightcurve_candidate, evaluate_transit_detection_qc, @@ -146,6 +147,11 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: parse_deviation_from_expected_transit_in_qc_sigma, prepare_final_fit_lightcurve_series, prepare_lightcurve_fit_input_series, + psf_frame_quality_components, + psf_frame_quality_mask, + psf_solution_quality_score, + target_psf_shape_quality_components, + target_psf_shape_quality_mask, populate_aperture_data_for_frame, rank_comparison_candidate_preflight_plans, refit_selected_fast_comparison_on_full_lightcurve, @@ -158,6 +164,8 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: robust_target_reference_flux_mask, save_final_triangle_plot, save_selected_photometry_debug_series, + select_comparison_calibrated_photometry, + select_alignment_candidate, should_keep_header_wcs_alignment, should_prefer_pixel_values_over_wcs_for_target, sigma_clip, @@ -621,7 +629,56 @@ def test_robust_target_reference_flux_mask_requires_both_series_to_be_plausible( assert not mask[13] -def test_build_target_fit_candidate_jobs_masks_psf_target_and_comp_dropouts(): +def test_psf_frame_quality_mask_rejects_high_seeing_and_low_amplitude_outliers(): + frame_count = 30 + phase = np.linspace(0.0, 2.0 * np.pi, frame_count) + psf_rows = np.zeros((frame_count, 7), dtype=float) + psf_rows[:, 0] = 10.0 + psf_rows[:, 1] = 20.0 + psf_rows[:, 2] = 200.0 * (1.0 + 0.02 * np.sin(phase)) + psf_rows[:, 3] = 1.1 * (1.0 + 0.01 * np.cos(phase)) + psf_rows[:, 4] = 1.0 * (1.0 + 0.01 * np.sin(phase)) + psf_rows[5, 2] = 45.0 + psf_rows[12, 3:5] = 6.5 + + components = psf_frame_quality_components(psf_rows) + mask = psf_frame_quality_mask(psf_rows) + + assert mask.sum() == frame_count - 2 + assert not mask[5] + assert not mask[12] + assert components["amplitude_outlier_mask"][5] + assert components["seeing_outlier_mask"][12] + + +def test_target_psf_shape_quality_rejects_broad_target_but_preserves_amplitude_dips(): + frame_count = 30 + phase = np.linspace(0.0, 2.0 * np.pi, frame_count) + target_rows = np.zeros((frame_count, 7), dtype=float) + target_rows[:, 0] = 10.0 + target_rows[:, 1] = 20.0 + target_rows[:, 2] = 200.0 * (1.0 + 0.02 * np.sin(phase)) + target_rows[:, 3] = 1.0 + target_rows[:, 4] = 1.0 + target_rows[5, 2] = 45.0 + target_rows[12, 3:5] = 6.5 + + reference_rows = target_rows.copy() + reference_rows[:, 2] = 250.0 + reference_rows[:, 3:5] = 1.0 + + components = target_psf_shape_quality_components(target_rows, reference_rows) + mask = target_psf_shape_quality_mask(target_rows, reference_rows) + + assert mask.sum() == frame_count - 1 + assert mask[5] + assert not mask[12] + assert not components["invalid_mask"][5] + assert components["seeing_outlier_mask"][12] + assert components["reference_width_outlier_mask"][12] + + +def test_build_target_fit_candidate_jobs_masks_pairwise_psf_failures_but_preserves_target_dips(): frame_count = 30 def build_psf_rows(amplitudes): @@ -635,13 +692,14 @@ def build_psf_rows(amplitudes): target_amplitudes = np.full(frame_count, 100.0) comp_amplitudes = np.full(frame_count, 120.0) - target_amplitudes[7] = 1.0 + target_amplitudes[7] = 80.0 comp_amplitudes[13] = 1.0 psf_data = { "target": build_psf_rows(target_amplitudes), "comp1": build_psf_rows(comp_amplitudes), } + psf_data["target"][19, 3:5] = 6.5 candidate_jobs = build_target_fit_candidate_jobs( psf_data, @@ -660,8 +718,9 @@ def build_psf_rows(amplitudes): assert len(candidate_jobs) == 1 assert candidate_jobs[0]["method"] == "psf" assert candidate_jobs[0]["mask"].sum() == 28 - assert not candidate_jobs[0]["mask"][7] + assert candidate_jobs[0]["mask"][7] assert not candidate_jobs[0]["mask"][13] + assert not candidate_jobs[0]["mask"][19] assert candidate_jobs[0]["coverage_count"] == 29 @@ -760,6 +819,44 @@ def test_check_coordinates_non_interactive_prefers_wcs_centroid(): assert y_pixel == 200.75 +def test_check_coordinates_non_interactive_keeps_input_when_wcs_psf_is_implausible(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=246, + input_y_pixel=271, + centroid_x=238.5, + centroid_y=266.1, + sigma_x=4.6, + sigma_y=0.7, + calculated_x_pixel=245, + calculated_y_pixel=270, + non_interactive_run=True, + wcs_psf_quality_score=np.inf, + input_psf_quality_score=0.1, + ) + + assert x_pixel == 246 + assert y_pixel == 271 + + +def test_check_coordinates_non_interactive_keeps_plausible_input_when_wcs_finds_other_source(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=246, + input_y_pixel=271, + centroid_x=236.6, + centroid_y=265.2, + sigma_x=1.2, + sigma_y=0.8, + calculated_x_pixel=236, + calculated_y_pixel=265, + non_interactive_run=True, + wcs_psf_quality_score=0.05, + input_psf_quality_score=0.25, + ) + + assert x_pixel == 246 + assert y_pixel == 271 + + def test_check_coordinates_non_interactive_uses_wcs_pixel_when_centroid_is_nan(): x_pixel, y_pixel = check_coordinates( input_x_pixel=5, @@ -802,6 +899,67 @@ def test_should_prefer_pixel_values_over_wcs_for_target_parses_values(): assert should_prefer_pixel_values_over_wcs_for_target(True) is True +def test_psf_solution_quality_score_rejects_offset_or_elongated_solutions(): + good = np.array([246.2, 270.8, 140.0, 1.1, 0.9, 0.0, 40.0]) + offset = np.array([238.5, 266.1, 140.0, 1.1, 0.9, 0.0, 40.0]) + elongated = np.array([246.2, 270.8, 140.0, 4.6, 0.7, 0.0, 40.0]) + + assert np.isfinite(psf_solution_quality_score(good, seed_pos=[246.0, 271.0])) + assert not np.isfinite(psf_solution_quality_score(offset, seed_pos=[246.0, 271.0])) + assert not np.isfinite(psf_solution_quality_score(elongated, seed_pos=[246.0, 271.0])) + + +def test_alignment_candidate_selection_rejects_broad_offset_wcs_target_solution(): + psf_data = { + "target": np.zeros((2, 7), dtype=float), + "comp1": np.zeros((2, 7), dtype=float), + } + psf_data["target"][0] = [245.8, 270.7, 110.0, 1.1, 0.8, 0.0, 40.0] + psf_data["comp1"][0] = [360.3, 443.3, 174.0, 1.1, 1.0, 0.0, 40.0] + tar_comp_dist = {"comp1": np.array([115.0, 173.0])} + + wcs_candidate = { + "coords": np.array([[244.0, 268.7], [360.4, 443.2]], dtype=float), + "projected_off_frame": False, + "psf_rows": { + "target": np.array([244.0, 268.7, 210.0, 7.3, 5.6, 0.0, 40.0]), + "comp1": np.array([360.4, 443.2, 174.0, 1.1, 1.0, 0.0, 40.0]), + }, + "warnings": [], + } + fallback_candidate = { + "coords": np.array([[246.0, 270.8], [360.2, 443.3]], dtype=float), + "psf_rows": { + "target": np.array([245.9, 270.8, 111.0, 1.1, 0.8, 0.0, 40.0]), + "comp1": np.array([360.2, 443.3, 173.0, 1.1, 1.0, 0.0, 40.0]), + }, + "warnings": [], + } + + assert not np.isfinite( + alignment_candidate_quality_score( + wcs_candidate, + comp_keys=["comp1"], + previous_target_psf_row=psf_data["target"][0], + previous_comp_psf_rows={"comp1": psf_data["comp1"][0]}, + expected_offsets=tar_comp_dist, + ) + ) + + selected_source, selected_candidate, diagnostics = select_alignment_candidate( + {"wcs": wcs_candidate, "fallback": fallback_candidate, "file_name": "frame.fits"}, + frame_index=1, + psf_data=psf_data, + tar_comp_dist=tar_comp_dist, + comp_keys=["comp1"], + ) + + assert selected_source == "fallback" + assert selected_candidate is fallback_candidate + assert not np.isfinite(diagnostics["wcs_score"]) + assert np.isfinite(diagnostics["fallback_score"]) + + def test_is_comp_star_required_parses_values(): assert is_comp_star_required(None) is True assert is_comp_star_required("y") is True @@ -2076,6 +2234,78 @@ def test_comparison_star_stability_summary_flags_candidate_specific_bad_frame(): assert comp2_summary["ensemble_frame_rejected_count"] == 0 +def test_comparison_star_stability_summary_clips_candidate_psf_spikes_before_suitability_rejection(): + frame_count = 89 + airmass = np.linspace(1.25, 1.06, frame_count) + phase = np.linspace(0.0, 4.0 * np.pi, frame_count) + comp_flux_map = { + f"comp{index + 1}": 100.0 * (1.0 + 0.002 * np.sin(phase + index)) + for index in range(9) + } + spike_indices = np.array([2, 5, 11, 12, 13, 39, 43, 45, 56], dtype=int) + comp_flux_map["comp6"] = comp_flux_map["comp6"].copy() + comp_flux_map["comp6"][spike_indices] *= 0.35 + + summary = comparison_star_stability_summary(comp_flux_map, airmass) + comp6_summary = summary["comp_summaries"][5] + + assert comp6_summary["coverage_count"] == frame_count + assert comp6_summary["suitability_outlier_rejected"] is False + assert comp6_summary["aggregate_score"] < 0.01 + assert comp6_summary["ensemble_frame_rejected_count"] == len(spike_indices) + assert comp6_summary["ensemble_frame_rejected_indices"] == spike_indices.tolist() + + +def test_select_comparison_calibrated_photometry_masks_psf_quality_before_aperture_ensemble(): + frame_count = 30 + airmass = np.linspace(1.2, 1.0, frame_count) + + def build_psf_rows(): + rows = np.zeros((frame_count, 7), dtype=float) + rows[:, 0] = 10.0 + rows[:, 1] = 20.0 + rows[:, 2] = 200.0 + rows[:, 3] = 1.0 + rows[:, 4] = 1.0 + return rows + + psf_data = { + "target": build_psf_rows(), + "comp1": build_psf_rows(), + "comp2": build_psf_rows(), + "comp3": build_psf_rows(), + } + psf_data["comp1"][7, 2] = 40.0 + psf_data["comp1"][7, 3:5] = 6.0 + + aper_data = { + "target": np.full((frame_count, 1, 1), 1000.0), + "target_bg": np.full((frame_count, 1, 1), 10.0), + } + for key in ("comp1", "comp2", "comp3"): + aper_data[key] = np.full((frame_count, 1, 1), 100.0) + aper_data[f"{key}_bg"] = np.full((frame_count, 1, 1), 10.0) + aper_data["comp1"][7, 0, 0] = 1.0 + + calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + apers=np.array([2.5]), + annuli=np.array([10.0]), + airmass=airmass, + comp_stars=[[10.0, 20.0], [30.0, 40.0], [50.0, 60.0]], + sigma=1.0, + use_psf_photometry=False, + use_aperture_photometry=True, + ) + comp1_summary = calibration["comp_summaries"][0] + + assert comp1_summary["psf_quality_rejected_count"] == 1 + assert comp1_summary["coverage_count"] == frame_count - 1 + assert comp1_summary["ensemble_frame_rejected_count"] == 0 + assert np.isnan(comp1_summary["ensemble_ratio_series"][7]) + + def test_cheap_lightcurve_prescore_treats_large_ratio_flag_as_noop(): tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) cflux = np.full(tflux.shape[0], 2.0) @@ -4525,6 +4755,120 @@ def fake_finalize( assert result["attempts"][0]["fit_point_count"] == 4 +def test_fit_ranked_comparison_calibration_candidates_masks_target_psf_shape(monkeypatch): + observed_lengths = [] + + def fake_diagnostics(times, *args, **kwargs): + observed_lengths.append(("diagnostics", len(times))) + return {"usable_point_count": len(times), "failure_reason": None} + + def fake_preflight(*args, **kwargs): + return {"coverage_priority": 1, "prepared_series": None} + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + observed_lengths.append(("finalize", len(times))) + fit = types.SimpleNamespace( + residuals=np.full(len(times), 0.01, dtype=float), + data=np.ones(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={"status": "pass", "summary": "ok", "ktmf_metric": 4.2}, + transit_qc_status="pass", + transit_qc_summary="ok", + transit_qc_ktmf_metric=4.2, + transit_qc_delta_bic=16.0, + frame_filter_diagnostics=[], + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.build_comparison_candidate_preflight", fake_preflight) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + frame_count = 30 + times = np.linspace(0.0, 0.2, frame_count) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, frame_count) + + psf_rows = np.zeros((frame_count, 7), dtype=float) + psf_rows[:, 0] = 10.0 + psf_rows[:, 1] = 20.0 + psf_rows[:, 2] = 100.0 + psf_rows[:, 3] = 1.0 + psf_rows[:, 4] = 1.0 + psf_data = { + "target": psf_rows.copy(), + "comp1": psf_rows.copy(), + } + psf_data["comp1"][:, 2] = 120.0 + psf_data["target"][12, 3:5] = 6.5 + psf_flux_data = { + "target": psf_data["target"].copy(), + "comp1": psf_data["comp1"].copy(), + } + psf_flux_data["comp1"][:, 2] = 240.0 + + target_psf_flux = 2 * np.pi * psf_data["target"][:, 2] * psf_data["target"][:, 3] * psf_data["target"][:, 4] + comparison_calibration = { + "method": "psf", + "method_label": "PSF photometry", + "a": None, + "an": None, + "aper": 0.0, + "annulus": 15.0, + "comp_summaries": [ + { + "label": "Comp 1", + "position": (10.0, 10.0), + "aggregate_score": 0.01, + "coverage_count": frame_count, + "coverage_total_frame_count": frame_count, + "coverage_reference_count": float(frame_count), + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 0, + "key": "comp1", + }, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data=psf_data, + aper_data=None, + target_psf_flux=target_psf_flux, + psf_flux_data=psf_flux_data, + ) + + assert observed_lengths == [("diagnostics", frame_count - 1), ("finalize", frame_count - 1)] + assert result["selected_result"]["fit_point_count"] == frame_count - 1 + assert np.nanmax(result["selected_result"]["tflux_fit"]) < 1000.0 + assert np.nanmedian(result["selected_result"]["cflux_fit"]) == pytest.approx(2.0 * np.pi * 240.0) + + def test_fit_ranked_comparison_calibration_candidates_applies_candidate_ensemble_clip(monkeypatch): observed_lengths = [] diff --git a/tests/test_plots.py b/tests/test_plots.py index 00499f2f..2b2a95a2 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -176,6 +176,40 @@ def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.pdf").exists() +def test_plot_individual_comp_star_calibration_series_masks_rejected_frame_lines(tmp_path, monkeypatch): + captured_lines = {} + original_plot = Axes.plot + + def spy_plot(self, x, y, *args, **kwargs): + label = kwargs.get("label") + if label in {"vs 2", "Ensemble"}: + captured_lines[label] = (np.asarray(x), np.asarray(y)) + return original_plot(self, x, y, *args, **kwargs) + + monkeypatch.setattr(Axes, "plot", spy_plot) + + plot_individual_comp_star_calibration_series( + times=np.array([1.0, 2.0, 3.0, 4.0]), + comp_summaries=[ + { + "label": "Comp 1", + "selected": False, + "aggregate_score": 0.01, + "pairwise_ratio_series": {"vs 2": np.array([1.0, 0.05, 1.01, 0.99])}, + "ensemble_ratio_series": np.array([1.0, 0.02, 1.005, 0.995]), + "ensemble_frame_keep_mask": np.array([True, False, True, True]), + }, + ], + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + method_label="Aperture photometry", + ) + + assert np.isnan(captured_lines["vs 2"][1][1]) + assert np.isnan(captured_lines["Ensemble"][1][1]) + + def test_plot_stellar_variability_labels_reference_coordinates(tmp_path, monkeypatch): titles = [] ylabels = [] From aa50240613aa331f8bc70c66f46350b6aa854faf Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 18 Jun 2026 20:35:37 +1000 Subject: [PATCH 069/116] Major updates to QC, bounds and proper uncertainties. Fixed up some PSF stuff but dealt with a lot of underestimating of uncertaintys and pessimistic QC when bounds are set too tight and the scatter is too high etc. --- exotic/api/elca.py | 87 +- exotic/exotic.py | 2519 +++++++++++++++++++++++++++- exotic/exotic_gui.py | 17 + exotic/inputs.py | 65 + exotic/output_files.py | 669 +++++++- exotic/plots.py | 895 ++++++++++ tests/test_elca_baseline.py | 58 + tests/test_exotic_proper_motion.py | 380 ++++- tests/test_exotic_rprs_retry.py | 469 ++++++ tests/test_inputs.py | 102 ++ tests/test_output_files.py | 166 ++ tests/test_plots.py | 227 +++ 12 files changed, 5523 insertions(+), 131 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 609ee133..41576ba1 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -903,6 +903,63 @@ def transit_model_uncertainty(self, times=None, sigma=1.0): return None return model - model_uncertainty, model + model_uncertainty + def _empirical_uncertainty_value(self, key): + empirical_uncertainty = getattr(self, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + return np.nan + try: + value = float(empirical_uncertainty.get(key)) + except (TypeError, ValueError): + return np.nan + return value if np.isfinite(value) else np.nan + + def _empirical_baseline_uncertainty_fraction(self): + value = self._empirical_uncertainty_value('baseline_red_noise_uncertainty_fraction') + if not np.isfinite(value) or value <= 0: + value = self._empirical_uncertainty_value('depth_uncertainty_fraction') + return value if np.isfinite(value) and value > 0 else np.nan + + def _baseline_envelope_with_empirical_floor(self, envelope): + if envelope is None: + return None + + lower, upper = envelope + lower = np.asarray(lower, dtype=float) + upper = np.asarray(upper, dtype=float) + empirical_baseline_uncertainty = self._empirical_baseline_uncertainty_fraction() + if not np.isfinite(empirical_baseline_uncertainty) or empirical_baseline_uncertainty <= 0: + return lower, upper + + existing_width = np.maximum(1.0 - lower, upper - 1.0) + combined_width = np.sqrt( + np.where(np.isfinite(existing_width), existing_width, 0.0) ** 2 + + empirical_baseline_uncertainty ** 2 + ) + return 1.0 - combined_width, 1.0 + combined_width + + def _combined_rprs_uncertainty_for_reporting(self): + value = self._empirical_uncertainty_value('combined_rprs_uncertainty') + if np.isfinite(value) and value >= 0: + return value + try: + value = float(self.errors.get('rprs')) + except (TypeError, ValueError): + return np.nan + return value if np.isfinite(value) and value >= 0 else np.nan + + def _model_data_uncertainty_for_reporting(self, parameter_name): + try: + value = float(self.errors.get(parameter_name)) + except (TypeError, ValueError): + return np.nan + if not np.isfinite(value) or value < 0: + return np.nan + + beta = self._empirical_uncertainty_value('red_noise_beta_factor') + if not np.isfinite(beta) or beta < 1.0: + beta = 1.0 + return value * beta + def _posterior_baseline_model_uncertainty(self, times, sigma=1.0): if getattr(self, 'results', None) is None or np.ndim(getattr(self, 'airmass', np.array([]))) == 2: return None @@ -1046,7 +1103,7 @@ def baseline_model_uncertainty(self, times=None, sigma=1.0): posterior_envelope = self._posterior_baseline_model_uncertainty(times, sigma=sigma) if posterior_envelope is not None: - return posterior_envelope + return self._baseline_envelope_with_empirical_floor(posterior_envelope) try: best_parameters = self._values_with_analytic_flux_baseline(self.parameters) @@ -1121,8 +1178,12 @@ def baseline_model_uncertainty(self, times=None, sigma=1.0): baseline_uncertainty = np.sqrt(variance) if not np.any(np.isfinite(baseline_uncertainty) & (baseline_uncertainty > 0)): - return None - return 1.0 - baseline_uncertainty, 1.0 + baseline_uncertainty + empirical_baseline_uncertainty = self._empirical_baseline_uncertainty_fraction() + if not np.isfinite(empirical_baseline_uncertainty) or empirical_baseline_uncertainty <= 0: + return None + return self._baseline_envelope_with_empirical_floor( + (1.0 - baseline_uncertainty, 1.0 + baseline_uncertainty) + ) def _plot_transit_model_uncertainty(self, ax, x_values, times, sort_index, label=None): envelope = self.transit_model_uncertainty(times) @@ -3575,15 +3636,21 @@ def plot_bestfit( axs[0].grid(True, ls='--') rprs2 = self.parameters['rprs'] ** 2 - rprs2err = 2 * self.parameters['rprs'] * self.errors['rprs'] - lclabel1 = r"Area ratio $(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" % ( + rprs_error_for_depth = self._combined_rprs_uncertainty_for_reporting() + rprs2err = 2 * self.parameters['rprs'] * rprs_error_for_depth + rprs_prior_marker = " (Prior)" if getattr(self, 'rprs_prior_fallback_applied', False) else "" + lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s%s" % ( str(round_to_2(rprs2, rprs2err)), - str(round_to_2(rprs2err)) + str(round_to_2(rprs2err)), + rprs_prior_marker, ) + tmid_error_for_plot = self._model_data_uncertainty_for_reporting('tmid') + if not np.isfinite(tmid_error_for_plot): + tmid_error_for_plot = self.errors.get('tmid', 0) lclabel2 = r"$T_{mid}$ = %s $\pm$ %s BJD$_{TDB}$" % ( - str(round_to_2(self.parameters['tmid'], self.errors.get('tmid', 0))), - str(round_to_2(self.errors.get('tmid', 0))) + str(round_to_2(self.parameters['tmid'], tmid_error_for_plot)), + str(round_to_2(tmid_error_for_plot)) ) lclabel = lclabel1 + "\n" + lclabel2 @@ -4104,7 +4171,7 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p rprs2 = self.lc_data[0]['priors']['rprs']**2 rprs2err = 2*self.lc_data[0]['priors']['rprs']*self.lc_data[0]['errors']['rprs'] - lclabel1 = r"Area ratio $(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( + lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( str(round_to_2(rprs2, rprs2err)), str(round_to_2(rprs2err)) ) @@ -4247,7 +4314,7 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): rprs2 = self.parameters['rprs']**2 rprs2err = 2*self.parameters['rprs']*self.errors['rprs'] - lclabel1 = r"Area ratio $(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( + lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( str(round_to_2(rprs2, rprs2err)), str(round_to_2(rprs2err)) ) diff --git a/exotic/exotic.py b/exotic/exotic.py index b98080dd..c7104bb2 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -151,7 +151,10 @@ from output_files import ( OutputFiles, AIDOutputFiles, + empirical_red_noise_error_scale, + fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, + fit_parameter_model_data_uncertainty, format_parameter_with_error, formatted_transit_depth_parameters, save_comp_star_calibration_summary, @@ -160,7 +163,10 @@ from .output_files import ( OutputFiles, AIDOutputFiles, + empirical_red_noise_error_scale, + fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, + fit_parameter_model_data_uncertainty, format_parameter_with_error, formatted_transit_depth_parameters, save_comp_star_calibration_summary, @@ -175,12 +181,14 @@ from .plate_status import PlateStatus try: # plots from plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ + plot_prior_posterior_comparison, plot_ktmf_qc_metrics, \ plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ plot_comp_star_calibration_series, plot_individual_comp_star_calibration_series, \ plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ plot_adaptive_aperture_diagnostics except ImportError: # package import from .plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ + plot_prior_posterior_comparison, plot_ktmf_qc_metrics, \ plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ plot_comp_star_calibration_series, plot_individual_comp_star_calibration_series, \ plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ @@ -257,12 +265,19 @@ COMPARISON_IMAGE_OUTLIER_MIN_SCATTER = 1e-4 COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS = 2 OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 +OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT = 12 FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT = 200 ULTRANEST_MIN_NUM_LIVE_POINTS_ENV = "EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS" FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT = True FAST_ULTRANEST_MAX_BINNED_POINTS = 20 FAST_ULTRANEST_MIN_POINTS_TO_BIN = 60 +LEGACY_PSF_FLUX_MODE_DEFAULT = False +FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT = True +FINAL_RESIDUAL_REJECTION_DEFAULT = True +FINAL_RESIDUAL_REJECTION_SIGMA = 3.0 +FINAL_RESIDUAL_REJECTION_MAX_CLIP_ITERS = 10 +FINAL_RESIDUAL_REJECTION_MAX_REFITS = 10 COMPARISON_PREFLIGHT_FIELD_SCORE_RELATIVE_BAND = 0.25 COMPARISON_PREFLIGHT_FIELD_SCORE_ABSOLUTE_BAND = 2.5e-4 PARTIAL_COVERAGE_RPRS_POSTERIOR_MAX_RETRIES = 1 @@ -283,11 +298,22 @@ RPRS_SEARCH_BOUND_MAX_DEFAULT = 0.5 RPRS_SEARCH_BOUND_ABSOLUTE_MAX = 1.0 RPRS_SEARCH_BOUND_MAX = RPRS_SEARCH_BOUND_MAX_DEFAULT +RPRS_RANGE_RESTRICTION_DEFAULT = True +RPRS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT = 10.0 +RPRS_RANGE_RESTRICTION_ENABLED = RPRS_RANGE_RESTRICTION_DEFAULT +RPRS_RANGE_RESTRICTION_PERCENTAGE = RPRS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT +RPRS_DATA_UNCERTAINTY_BOUND_SIGMA = 3.0 +RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR_DEFAULT = True +RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR = RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR_DEFAULT RPRS_RETRY_MIN_HALF_WIDTH = 0.05 INITIAL_RPRS_BOUND_LOWER_SCALE = 0.0 INITIAL_RPRS_BOUND_UPPER_SCALE = 3.0 ARS_SEARCH_BOUND_MIN = 1e-6 ARS_SEARCH_BOUND_FALLBACK_MAX = 100.0 +ARS_RANGE_RESTRICTION_DEFAULT = True +ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT = 10.0 +ARS_RANGE_RESTRICTION_ENABLED = ARS_RANGE_RESTRICTION_DEFAULT +ARS_RANGE_RESTRICTION_PERCENTAGE = ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT ARS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 ARS_RETRY_MIN_HALF_WIDTH = 0.0 IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES_DEFAULT = 5 @@ -356,8 +382,8 @@ ) TRANSIT_QC_DELTA_BIC_FAIL_THRESHOLD = 6.0 TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD = 10.0 -TRANSIT_QC_KTMF_FAIL_THRESHOLD = 2.5 -TRANSIT_QC_KTMF_PASS_THRESHOLD = 3.5 +TRANSIT_QC_KTMF_FAIL_THRESHOLD = 3.0 +TRANSIT_QC_KTMF_PASS_THRESHOLD = 4.0 TRANSIT_QC_MIN_EEBLS_SNR = 4.0 TRANSIT_QC_DURATION_RATIO_MIN = 0.5 TRANSIT_QC_DURATION_RATIO_MAX = 2.0 @@ -366,11 +392,12 @@ TRANSIT_QC_DEVIATION_SIGMA_DEFAULT = 5.0 TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION = 0.05 TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS = { - 'model_evidence': 0.8, + 'model_evidence': 0.3, 'deviation_from_expected_value': 1.5, 'residual_scatter': 0.7, 'duration_consistency': 0.75, - 'eebls_depth_snr': 0.75, + 'eebls_depth_snr': 1.3, + 'sampling': 0.7, } @@ -551,6 +578,334 @@ def prepend_lightcurve_filter_diagnostic(fit, diagnostic): fit.frame_filter_diagnostics = diagnostics +def _fit_residual_percent(fit): + residuals = np.asarray(getattr(fit, 'residuals', np.array([])), dtype=float).reshape(-1) + if residuals.size == 0: + return residuals + + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float).reshape(-1) + median_flux = np.nanmedian(data) if data.size else np.nan + if not np.isfinite(median_flux) or median_flux == 0: + return residuals + return residuals / median_flux * 100.0 + + +def final_residual_rejection_keep_mask( + fit, + sigma=FINAL_RESIDUAL_REJECTION_SIGMA, + min_required_points=LIGHTCURVE_MIN_VALID_POINTS, + max_clip_iters=FINAL_RESIDUAL_REJECTION_MAX_CLIP_ITERS, +): + residual_percent = _fit_residual_percent(fit) + point_count = int(residual_percent.size) + summary = { + 'enabled': True, + 'applied': False, + 'sigma': float(sigma), + 'input_point_count': point_count, + 'kept_point_count': point_count, + 'rejected_point_count': 0, + 'median_residual_percent': np.nan, + 'stdev_residual_percent': np.nan, + 'clip_iteration_count': 0, + 'clip_iterations': [], + 'note': None, + } + if point_count == 0: + summary['note'] = "Skipped; the first final fit did not provide residuals." + return np.ones(0, dtype=bool), summary + + valid = np.isfinite(residual_percent) + valid_count = int(np.count_nonzero(valid)) + if valid_count < max(LIGHTCURVE_MIN_VALID_POINTS, 2): + summary['note'] = "Skipped; too few finite residuals were available." + return np.ones(point_count, dtype=bool), summary + + keep_mask = valid.copy() + max_clip_iters = int(max(1, max_clip_iters or 1)) + stopped_on_iteration_cap = False + last_rejecting_center = np.nan + last_rejecting_scatter = np.nan + for clip_iteration in range(1, max_clip_iters + 1): + active = keep_mask & valid + active_count = int(np.count_nonzero(active)) + if active_count < max(LIGHTCURVE_MIN_VALID_POINTS, 2): + summary['note'] = "Skipped; too few finite residuals remained during iterative clipping." + return valid.copy(), summary + + center = float(np.nanmedian(residual_percent[active])) + scatter = float(np.nanstd(residual_percent[active] - center, ddof=1)) + summary['median_residual_percent'] = center + summary['stdev_residual_percent'] = scatter + if not np.isfinite(scatter) or scatter <= 0: + summary['note'] = "Stopped; the final-fit residual scatter was not finite." + break + + outlier_mask = active & (np.abs(residual_percent - center) > float(sigma) * scatter) + rejected_this_iteration = int(np.count_nonzero(outlier_mask)) + summary['clip_iterations'].append({ + 'iteration': int(clip_iteration), + 'input_point_count': active_count, + 'median_residual_percent': center, + 'stdev_residual_percent': scatter, + 'rejected_point_count': rejected_this_iteration, + }) + summary['clip_iteration_count'] = int(clip_iteration) + if rejected_this_iteration == 0: + break + + last_rejecting_center = center + last_rejecting_scatter = scatter + candidate_keep_mask = keep_mask & ~outlier_mask + if int(np.count_nonzero(candidate_keep_mask)) < int(min_required_points): + summary['note'] = ( + f"Skipped; residual rejection would leave {int(np.count_nonzero(candidate_keep_mask))} point(s), " + f"but at least {int(min_required_points)} are required." + ) + summary['clip_iterations'][-1]['skipped_for_minimum_points'] = True + summary['kept_point_count'] = point_count + summary['rejected_point_count'] = 0 + return np.ones(point_count, dtype=bool), summary + + keep_mask = candidate_keep_mask + else: + stopped_on_iteration_cap = True + + kept_count = int(np.count_nonzero(keep_mask)) + rejected_count = int(point_count - kept_count) + summary['kept_point_count'] = kept_count + summary['rejected_point_count'] = rejected_count + + if rejected_count == 0: + summary['note'] = ( + f"No residual outliers exceeded {float(sigma):.1f} sigma from the iterated median residual " + f"({summary['median_residual_percent']:.4f}%, stdev={summary['stdev_residual_percent']:.4f}%)." + ) + return keep_mask, summary + + summary['applied'] = True + if np.isfinite(last_rejecting_center): + summary['median_residual_percent'] = last_rejecting_center + if np.isfinite(last_rejecting_scatter): + summary['stdev_residual_percent'] = last_rejecting_scatter + cap_text = f" after reaching {max_clip_iters} clip iteration(s)" if stopped_on_iteration_cap else "" + summary['note'] = ( + f"Rejected {rejected_count}/{point_count} final-fit residual outlier(s) with iterative " + f"{float(sigma):.1f}-sigma clipping from the median residual " + f"({summary['median_residual_percent']:.4f}%, stdev={summary['stdev_residual_percent']:.4f}%){cap_text}." + ) + return keep_mask, summary + + +def _fit_array_for_rejection_plot(fit, attr_name, fallback=None): + values = getattr(fit, attr_name, fallback) + if values is None: + values = fallback + if values is None: + return np.array([], dtype=float) + return np.asarray(values, dtype=float).reshape(-1) + + +def build_final_residual_rejection_payload(fit, keep_mask, summary, source_indices=None): + keep_mask = np.asarray(keep_mask, dtype=bool).reshape(-1) + rejected_mask = ~keep_mask + time_values = _fit_array_for_rejection_plot(fit, 'time') + phase_values = _fit_array_for_rejection_plot(fit, 'phase') + data_values = _fit_array_for_rejection_plot(fit, 'data') + flux_values = _fit_array_for_rejection_plot(fit, 'detrended', fallback=data_values) + residual_percent = _fit_residual_percent(fit) + + plot_count = min( + keep_mask.size, + time_values.size, + phase_values.size, + flux_values.size, + residual_percent.size, + ) + if plot_count == 0: + rejected_mask = np.zeros(0, dtype=bool) + else: + rejected_mask = rejected_mask[:plot_count] + time_values = time_values[:plot_count] + phase_values = phase_values[:plot_count] + flux_values = flux_values[:plot_count] + residual_percent = residual_percent[:plot_count] + + payload = dict(summary or {}) + payload['rejected_time'] = time_values[rejected_mask].tolist() + payload['rejected_phase'] = phase_values[rejected_mask].tolist() + payload['rejected_flux'] = flux_values[rejected_mask].tolist() + payload['rejected_residual_percent'] = residual_percent[rejected_mask].tolist() + if source_indices is not None: + index_values = np.asarray(source_indices, dtype=int).reshape(-1) + if index_values.size >= plot_count: + payload['rejected_source_indices'] = index_values[:plot_count][rejected_mask].tolist() + return payload + + +def initialize_final_residual_rejection_payload( + enabled=True, + input_point_count=0, + sigma=FINAL_RESIDUAL_REJECTION_SIGMA, + note=None, +): + point_count = int(input_point_count or 0) + return { + 'enabled': bool(enabled), + 'applied': False, + 'sigma': float(sigma), + 'input_point_count': point_count, + 'kept_point_count': point_count, + 'rejected_point_count': 0, + 'median_residual_percent': np.nan, + 'stdev_residual_percent': np.nan, + 'clip_iteration_count': 0, + 'clip_iterations': [], + 'refit_iteration_count': 0, + 'refit_iterations': [], + 'rejected_time': [], + 'rejected_phase': [], + 'rejected_flux': [], + 'rejected_residual_percent': [], + 'rejected_source_indices': [], + 'note': note, + } + + +def _extend_final_residual_rejection_payload_list(payload, key, values): + existing = payload.get(key) + if not isinstance(existing, list): + existing = [] + if values is None: + values = [] + elif isinstance(values, np.ndarray): + values = values.tolist() + elif not isinstance(values, list): + values = list(values) + payload[key] = existing + values + + +def record_final_residual_rejection_refit_cycle(payload, cycle_payload, refit_iteration): + payload = dict(payload or {}) + cycle_payload = dict(cycle_payload or {}) + rejected_count = int(cycle_payload.get('rejected_point_count', 0) or 0) + payload['enabled'] = True + payload['applied'] = bool(payload.get('applied', False) or rejected_count > 0) + payload['sigma'] = float(cycle_payload.get('sigma', payload.get('sigma', FINAL_RESIDUAL_REJECTION_SIGMA))) + payload['kept_point_count'] = int(cycle_payload.get('kept_point_count', payload.get('kept_point_count', 0)) or 0) + payload['rejected_point_count'] = int(payload.get('rejected_point_count', 0) or 0) + rejected_count + payload['median_residual_percent'] = cycle_payload.get( + 'median_residual_percent', + payload.get('median_residual_percent', np.nan), + ) + payload['stdev_residual_percent'] = cycle_payload.get( + 'stdev_residual_percent', + payload.get('stdev_residual_percent', np.nan), + ) + payload['clip_iteration_count'] = int(payload.get('clip_iteration_count', 0) or 0) + int( + cycle_payload.get('clip_iteration_count', 0) or 0 + ) + _extend_final_residual_rejection_payload_list( + payload, + 'clip_iterations', + cycle_payload.get('clip_iterations', []), + ) + for key in ( + 'rejected_time', + 'rejected_phase', + 'rejected_flux', + 'rejected_residual_percent', + 'rejected_source_indices', + ): + _extend_final_residual_rejection_payload_list(payload, key, cycle_payload.get(key, [])) + + refit_iterations = payload.get('refit_iterations') + if not isinstance(refit_iterations, list): + refit_iterations = [] + refit_iterations.append({ + 'iteration': int(refit_iteration), + 'input_point_count': int(cycle_payload.get('input_point_count', 0) or 0), + 'kept_point_count': int(cycle_payload.get('kept_point_count', 0) or 0), + 'rejected_point_count': rejected_count, + 'median_residual_percent': cycle_payload.get('median_residual_percent', np.nan), + 'stdev_residual_percent': cycle_payload.get('stdev_residual_percent', np.nan), + 'clip_iteration_count': int(cycle_payload.get('clip_iteration_count', 0) or 0), + 'clip_iterations': list(cycle_payload.get('clip_iterations', [])), + 'note': cycle_payload.get('note'), + }) + payload['refit_iterations'] = refit_iterations + payload['refit_iteration_count'] = int(len(refit_iterations)) + return payload + + +def update_final_residual_rejection_final_pass(payload, final_summary): + payload = dict(payload or {}) + final_summary = dict(final_summary or {}) + payload['final_clip_summary'] = final_summary + payload['median_residual_percent'] = final_summary.get( + 'median_residual_percent', + payload.get('median_residual_percent', np.nan), + ) + payload['stdev_residual_percent'] = final_summary.get( + 'stdev_residual_percent', + payload.get('stdev_residual_percent', np.nan), + ) + return payload + + +def finalize_final_residual_rejection_payload( + payload, + current_point_count=None, + stopped_reason=None, +): + payload = dict(payload or {}) + if current_point_count is not None: + payload['kept_point_count'] = int(current_point_count) + + rejected_count = int(payload.get('rejected_point_count', 0) or 0) + input_point_count = int(payload.get('input_point_count', payload.get('kept_point_count', 0)) or 0) + refit_count = int(payload.get('refit_iteration_count', 0) or 0) + payload['applied'] = bool(payload.get('enabled', True) and rejected_count > 0) + if not payload.get('enabled', True): + if not payload.get('note'): + payload['note'] = "Disabled per optional_info setting." + return payload + + if rejected_count <= 0: + if payload.get('note') is None: + payload['note'] = "No final-fit residual outliers were rejected." + return payload + + stop_text = " Final pass found no new residual outliers." + if stopped_reason == 'max_refits': + stop_text = f" Stopped after reaching {FINAL_RESIDUAL_REJECTION_MAX_REFITS} refit cycle(s)." + elif stopped_reason == 'refit_failed': + stop_text = " Stopped because the next residual-rejected UltraNest refit did not converge." + elif stopped_reason == 'shape_mismatch': + stop_text = " Stopped because the next residual array did not align with the light-curve points." + + payload['note'] = ( + f"Rejected {rejected_count}/{input_point_count} final-fit residual outlier(s) over " + f"{refit_count} iterative UltraNest refit cycle(s) using " + f"{float(payload.get('sigma', FINAL_RESIDUAL_REJECTION_SIGMA)):.1f}-sigma median clipping." + f"{stop_text}" + ) + return payload + + +def annotate_final_residual_rejection(fit, payload): + if fit is None: + return + + payload = dict(payload or {}) + fit.final_residual_rejection = payload + fit.final_residual_rejection_applied = bool(payload.get('applied', False)) + fit.final_residual_rejection_sigma = payload.get('sigma') + fit.final_residual_rejection_point_count = int(payload.get('input_point_count', 0) or 0) + fit.final_residual_rejection_rejected_count = int(payload.get('rejected_point_count', 0) or 0) + fit.final_residual_rejection_note = payload.get('note') + + def annotate_selected_photometry_debug( fit, times, @@ -714,6 +1069,24 @@ def transit_qc_residual_scatter(data, model): return float(np.std(residuals) / median_flux) +def transit_qc_model_depth_fraction(model): + model = np.asarray(model, dtype=float) + if model.ndim != 1 or model.size == 0: + return np.nan + + finite = model[np.isfinite(model)] + if finite.size == 0: + return np.nan + + baseline = float(np.nanpercentile(finite, 95.0)) + minimum = float(np.nanmin(finite)) + if not np.isfinite(baseline) or not np.isfinite(minimum) or baseline <= 0: + return np.nan + + depth = (baseline - minimum) / baseline + return float(depth) if np.isfinite(depth) and depth > 0 else np.nan + + def transit_qc_deviation_score_from_sigma(sigma_offset, sigma_threshold): try: sigma_offset = abs(float(sigma_offset)) @@ -766,6 +1139,134 @@ def transit_qc_duration_score(duration_ratio): return float(np.clip(score, 0.0, 1.0)) +def transit_qc_geometry_contact_duration(parameters, contact_radius): + parameters = parameters or {} + try: + period = float(parameters.get('per', np.nan)) + ars = float(parameters.get('ars', np.nan)) + inc = float(parameters.get('inc', np.nan)) + contact_radius = float(contact_radius) + except (TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(period) or period <= 0 + or not np.isfinite(ars) or ars <= 0 + or not np.isfinite(inc) + or not np.isfinite(contact_radius) or contact_radius <= 0 + ): + return np.nan + + ecc = coerce_finite_transit_qc_scalar(parameters.get('ecc', 0.0)) + omega = np.deg2rad(coerce_finite_transit_qc_scalar(parameters.get('omega', 0.0))) + denominator = 1.0 + ecc * np.sin(omega) + if not np.isfinite(denominator) or np.isclose(denominator, 0.0): + return np.nan + + impact_scale = ars * (1.0 - ecc ** 2) / denominator + inc_rad = np.deg2rad(inc) + sin_inc = np.sin(inc_rad) + if not np.isfinite(impact_scale) or impact_scale <= 0 or not np.isfinite(sin_inc) or sin_inc <= 0: + return np.nan + + impact_parameter = impact_scale * np.cos(inc_rad) + chord_sq = contact_radius ** 2 - impact_parameter ** 2 + if not np.isfinite(chord_sq) or chord_sq <= 0: + return np.nan + + argument = np.sqrt(chord_sq) / (impact_scale * sin_inc) + if not np.isfinite(argument): + return np.nan + argument = float(np.clip(argument, -1.0, 1.0)) + duration = (period / np.pi) * np.arcsin(argument) + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + +def transit_qc_sampling_summary(fit): + summary = { + 'available': False, + 'score': np.nan, + 'ingress_count': 0, + 'egress_count': 0, + 'in_transit_count': 0, + 'pre_baseline_count': 0, + 'post_baseline_count': 0, + 'total_duration': np.nan, + 'ingress_duration': np.nan, + 'detail': 'sampling unavailable', + } + if fit is None: + return summary + + times = np.asarray(getattr(fit, 'time', []), dtype=float) + parameters = getattr(fit, 'parameters', {}) or {} + if times.ndim != 1 or times.size == 0: + return summary + + tmid = coerce_finite_transit_qc_scalar(parameters.get('tmid', np.nan)) + rprs = coerce_finite_transit_qc_scalar(parameters.get('rprs', np.nan)) + if not np.isfinite(tmid) or not np.isfinite(rprs) or rprs < 0: + return summary + + total_duration = transit_qc_geometry_contact_duration(parameters, 1.0 + rprs) + full_duration = transit_qc_geometry_contact_duration(parameters, max(1.0 - rprs, 0.0)) + if not np.isfinite(total_duration) or total_duration <= 0: + total_duration = coerce_finite_transit_qc_scalar(getattr(fit, 'duration_expected', np.nan)) + if not np.isfinite(total_duration) or total_duration <= 0: + return summary + + if np.isfinite(full_duration) and full_duration >= 0 and full_duration < total_duration: + ingress_duration = 0.5 * (total_duration - full_duration) + else: + ingress_duration = 0.2 * total_duration + if not np.isfinite(ingress_duration) or ingress_duration <= 0: + return summary + ingress_duration = min(float(ingress_duration), 0.5 * float(total_duration)) + + finite_times = times[np.isfinite(times)] + if finite_times.size == 0: + return summary + + start = tmid - 0.5 * total_duration + end = tmid + 0.5 * total_duration + ingress_end = min(start + ingress_duration, tmid) + egress_start = max(end - ingress_duration, tmid) + + ingress_count = int(np.count_nonzero((finite_times >= start) & (finite_times <= ingress_end))) + egress_count = int(np.count_nonzero((finite_times >= egress_start) & (finite_times <= end))) + in_transit_count = int(np.count_nonzero((finite_times >= start) & (finite_times <= end))) + pre_baseline_count = int(np.count_nonzero(finite_times < start)) + post_baseline_count = int(np.count_nonzero(finite_times > end)) + + ingress_egress_score = min(min(ingress_count, egress_count) / 4.0, 1.0) + in_transit_score = min(in_transit_count / 20.0, 1.0) + baseline_score = min(min(pre_baseline_count, post_baseline_count) / 12.0, 1.0) + score = float(np.clip( + 0.60 * ingress_egress_score + + 0.25 * in_transit_score + + 0.15 * baseline_score, + 0.0, + 1.0, + )) + + summary.update({ + 'available': True, + 'score': score, + 'ingress_count': ingress_count, + 'egress_count': egress_count, + 'in_transit_count': in_transit_count, + 'pre_baseline_count': pre_baseline_count, + 'post_baseline_count': post_baseline_count, + 'total_duration': float(total_duration), + 'ingress_duration': float(ingress_duration), + 'detail': ( + f"ingress={ingress_count}, egress={egress_count}, " + f"in-transit={in_transit_count}, baseline pre/post={pre_baseline_count}/{post_baseline_count}" + ), + }) + return summary + + def estimate_midpoint_anchored_partial_duration(fit, assessment): times = np.asarray(getattr(fit, 'time', []), dtype=float) transit_model = np.asarray(getattr(fit, 'transit', []), dtype=float) @@ -858,17 +1359,50 @@ def transit_qc_saturating_score(value, scale): return float(np.clip(1.0 - np.exp(-max(value, 0.0) / scale), 0.0, 1.0)) -def transit_qc_residual_scatter_score(residual_scatter, reference_scatter=0.005): +def transit_qc_residual_scatter_score( + residual_scatter, + transit_depth=np.nan, + full_credit_ratio=0.5, + zero_credit_ratio=4.0, + decay_rate=3.0, +): try: residual_scatter = float(residual_scatter) - reference_scatter = float(reference_scatter) + transit_depth = float(transit_depth) + full_credit_ratio = float(full_credit_ratio) + zero_credit_ratio = float(zero_credit_ratio) + decay_rate = float(decay_rate) except (TypeError, ValueError): return np.nan - if not np.isfinite(residual_scatter) or residual_scatter < 0 or not np.isfinite(reference_scatter) or reference_scatter <= 0: + if ( + not np.isfinite(residual_scatter) + or residual_scatter < 0 + or not np.isfinite(full_credit_ratio) + or full_credit_ratio < 0 + or not np.isfinite(zero_credit_ratio) + or zero_credit_ratio <= full_credit_ratio + or not np.isfinite(decay_rate) + or decay_rate <= 0 + ): return np.nan - return float(np.clip(1.0 / (1.0 + residual_scatter / reference_scatter), 0.0, 1.0)) + if not np.isfinite(transit_depth) or transit_depth <= 0: + return np.nan + + scatter_ratio = residual_scatter / transit_depth + if scatter_ratio <= full_credit_ratio: + return 1.0 + if scatter_ratio >= zero_credit_ratio: + return 0.0 + + interval_fraction = ( + (scatter_ratio - full_credit_ratio) + / (zero_credit_ratio - full_credit_ratio) + ) + numerator = np.exp(-decay_rate * interval_fraction) - np.exp(-decay_rate) + denominator = 1.0 - np.exp(-decay_rate) + return float(np.clip(numerator / denominator, 0.0, 1.0)) def transit_qc_mean_available_score(*scores): @@ -1016,6 +1550,9 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T 'tmid_deviation_threshold_minutes': np.nan, 'tmid_deviation_sigma': np.nan, 'rprs_deviation_fit_unc': np.nan, + 'rprs_deviation_model_fit_unc': np.nan, + 'rprs_deviation_data_fit_unc': np.nan, + 'rprs_deviation_combined_fit_unc': np.nan, 'rprs_deviation_expected_unc': np.nan, 'rprs_deviation_systematic_floor': np.nan, 'rprs_deviation_unc': np.nan, @@ -1023,6 +1560,8 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T 'tmid_deviation_score': np.nan, 'rprs_deviation_score': np.nan, 'deviation_from_expected_value': np.nan, + 'rprs_prior_assumed': False, + 'rprs_prior_assumed_note': None, 'available': False, 'failed': False, 'notes': [], @@ -1046,7 +1585,32 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T expected_rprs_unc = expected.get('expected_rprs_unc', np.nan) fitted_rprs = parameters.get('rprs', np.nan) errors = getattr(fit, 'errors', {}) or {} - fitted_rprs_unc = errors.get('rprs', np.nan) + fitted_rprs_model_unc = errors.get('rprs', np.nan) + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + try: + fit.empirical_transit_uncertainty = empirical_uncertainty + except Exception: + pass + fitted_rprs_data_unc = np.nan + fitted_rprs_unc = fitted_rprs_model_unc + rprs_prior_assumed = False + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + rprs_prior_assumed = ( + bool(empirical_uncertainty.get('rprs_prior_fallback_applied')) + or empirical_uncertainty.get('rprs_uncertainty_basis') == 'prior_assumed_data_only' + ) + if rprs_prior_assumed: + fitted_rprs_model_unc = np.nan + fitted_rprs_data_unc = empirical_uncertainty.get('data_rprs_uncertainty', np.nan) + combined_uncertainty = _finite_float( + empirical_uncertainty.get('combined_rprs_uncertainty'), + np.nan, + ) + if np.isfinite(combined_uncertainty) and combined_uncertainty >= 0: + fitted_rprs_unc = combined_uncertainty comparison_unc, systematic_floor = transit_qc_rprs_deviation_uncertainty( fitted_rprs_unc, expected_rprs_unc, @@ -1057,9 +1621,18 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T summary['fitted_rprs'] = fitted_rprs summary['fitted_rprs_unc'] = fitted_rprs_unc summary['rprs_deviation_fit_unc'] = fitted_rprs_unc + summary['rprs_deviation_model_fit_unc'] = fitted_rprs_model_unc + summary['rprs_deviation_data_fit_unc'] = fitted_rprs_data_unc + summary['rprs_deviation_combined_fit_unc'] = fitted_rprs_unc summary['rprs_deviation_expected_unc'] = expected_rprs_unc summary['rprs_deviation_systematic_floor'] = systematic_floor summary['rprs_deviation_unc'] = comparison_unc + summary['rprs_prior_assumed'] = bool(rprs_prior_assumed) + if rprs_prior_assumed: + summary['rprs_prior_assumed_note'] = ( + "Rp/R* was fixed to the input prior; the expected-value deviation is circular " + "and is omitted from KTMF scoring." + ) if ( np.isfinite(expected_rprs) and np.isfinite(fitted_rprs) @@ -1075,6 +1648,18 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T summary['deviation_from_expected_value'] = float(summary['rprs_deviation_score']) if np.isfinite(summary['rprs_deviation_sigma']): + uncertainty_parts = [] + model_unc = summary.get('rprs_deviation_model_fit_unc', np.nan) + data_unc = summary.get('rprs_deviation_data_fit_unc', np.nan) + if np.isfinite(model_unc): + uncertainty_parts.append(f"model={model_unc:.6f}") + if np.isfinite(data_unc): + uncertainty_parts.append(f"data/red-noise={data_unc:.6f}") + uncertainty_note = ( + "; fit uncertainty terms: " + ", ".join(uncertainty_parts) + if uncertainty_parts + else "" + ) summary['notes'].append( "Expected-value Rp/R* deviation: " f"{summary['rprs_deviation_sigma']:.2f} sigma " @@ -1082,8 +1667,11 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T f"expected={expected_rprs:.6f} +/- {expected_rprs_unc:.6f}; " f"comparison uncertainty={summary['rprs_deviation_unc']:.6f}, " f"including {100.0 * TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION:.1f}% " - f"Rp/R* floor={summary['rprs_deviation_systematic_floor']:.6f})." + f"Rp/R* floor={summary['rprs_deviation_systematic_floor']:.6f}" + f"{uncertainty_note})." ) + if rprs_prior_assumed: + summary['notes'].append(summary['rprs_prior_assumed_note']) rprs_sigma = summary['rprs_deviation_sigma'] if np.isfinite(rprs_sigma) and np.isfinite(sigma_threshold) and sigma_threshold > 0 and rprs_sigma > sigma_threshold: @@ -1105,6 +1693,12 @@ def compute_transit_qc_ktmf(summary): ) delta_chi2_score = transit_qc_saturating_score(summary.get('delta_chi2', np.nan), 25.0) model_evidence_score = transit_qc_mean_available_score(delta_bic_score, delta_chi2_score) + model_evidence_score_uncertainty = np.nan + model_evidence_scores = [ + float(score) for score in (delta_bic_score, delta_chi2_score) if np.isfinite(score) + ] + if len(model_evidence_scores) > 1: + model_evidence_score_uncertainty = float(np.std(model_evidence_scores)) model_evidence_detail_parts = [ f"Delta BIC={format_transit_delta_bic(summary.get('delta_bic', np.nan))}", f"Delta chi2={summary.get('delta_chi2', np.nan):.2f}" @@ -1112,7 +1706,14 @@ def compute_transit_qc_ktmf(summary): else "Delta chi2=n/a", ] deviation_score = summary.get('deviation_from_expected_value', np.nan) - if np.isfinite(deviation_score): + rprs_prior_assumed = bool(summary.get('rprs_prior_assumed', False)) + if rprs_prior_assumed: + deviation_score = np.nan + deviation_detail = ( + summary.get('rprs_prior_assumed_note') + or "Rp/R* was fixed to the input prior; expected-value deviation is omitted from KTMF scoring." + ) + elif np.isfinite(deviation_score): deviation_detail_parts = [ f"score={deviation_score:.2f}", f"Rp/R* sigma={summary.get('rprs_deviation_sigma', np.nan):.2f}", @@ -1120,6 +1721,12 @@ def compute_transit_qc_ktmf(summary): rprs_fit_unc = summary.get('rprs_deviation_fit_unc', np.nan) if np.isfinite(rprs_fit_unc): deviation_detail_parts.append(f"fit uncertainty={rprs_fit_unc:.6f}") + model_fit_unc = summary.get('rprs_deviation_model_fit_unc', np.nan) + if np.isfinite(model_fit_unc): + deviation_detail_parts.append(f"model uncertainty={model_fit_unc:.6f}") + data_fit_unc = summary.get('rprs_deviation_data_fit_unc', np.nan) + if np.isfinite(data_fit_unc): + deviation_detail_parts.append(f"data/red-noise uncertainty={data_fit_unc:.6f}") expected_unc = summary.get('rprs_deviation_expected_unc', summary.get('expected_rprs_unc', np.nan)) if np.isfinite(expected_unc): deviation_detail_parts.append(f"expected uncertainty={expected_unc:.6f}") @@ -1133,6 +1740,58 @@ def compute_transit_qc_ktmf(summary): else: deviation_detail = "expected-value deviation disabled or unavailable" + residual_scatter = summary.get('residual_scatter', np.nan) + residual_depth = summary.get('transit_depth_for_residual_scatter', np.nan) + residual_scatter_to_depth_ratio = summary.get('residual_scatter_to_depth_ratio', np.nan) + residual_scatter_score = transit_qc_residual_scatter_score( + residual_scatter, + residual_depth, + ) + residual_scatter_score_uncertainty = np.nan + point_count = summary.get('point_count', np.nan) + if ( + np.isfinite(residual_scatter) + and residual_scatter >= 0 + and np.isfinite(residual_depth) + and residual_depth > 0 + and np.isfinite(point_count) + and point_count > 2 + ): + residual_scatter_uncertainty = residual_scatter / np.sqrt(2.0 * (point_count - 1.0)) + residual_ratio_uncertainty = residual_scatter_uncertainty / residual_depth + full_credit_ratio = 0.5 + zero_credit_ratio = 4.0 + decay_rate = 3.0 + if not np.isfinite(residual_scatter_to_depth_ratio): + residual_scatter_to_depth_ratio = residual_scatter / residual_depth + if residual_scatter_to_depth_ratio <= full_credit_ratio or residual_scatter_to_depth_ratio >= zero_credit_ratio: + residual_scatter_score_uncertainty = 0.0 + else: + interval_fraction = ( + (residual_scatter_to_depth_ratio - full_credit_ratio) + / (zero_credit_ratio - full_credit_ratio) + ) + derivative = ( + decay_rate * np.exp(-decay_rate * interval_fraction) + / ((zero_credit_ratio - full_credit_ratio) * (1.0 - np.exp(-decay_rate))) + ) + residual_scatter_score_uncertainty = float(derivative * residual_ratio_uncertainty) + + if ( + np.isfinite(residual_scatter) + and np.isfinite(residual_depth) + and residual_depth > 0 + ): + residual_scatter_detail = ( + f"scatter/depth={residual_scatter / residual_depth:.2f}, " + f"scatter={residual_scatter * 100.0:.4f}%, " + f"depth={residual_depth * 100.0:.3f}%" + ) + elif np.isfinite(residual_scatter): + residual_scatter_detail = f"scatter={residual_scatter * 100.0:.4f}%, depth=n/a" + else: + residual_scatter_detail = "n/a" + duration_note = summary.get('duration_consistency_note') if np.isfinite(summary.get('duration_ratio', np.nan)): duration_detail = f"{summary.get('duration_ratio', np.nan):.2f}x expected duration" @@ -1146,40 +1805,48 @@ def compute_transit_qc_ktmf(summary): 'key': 'model_evidence', 'label': 'Model Evidence', 'score': model_evidence_score, + 'score_uncertainty': model_evidence_score_uncertainty, 'detail': ", ".join(model_evidence_detail_parts), }, { 'key': 'deviation_from_expected_value', 'label': 'Deviation From Expected Value', 'score': deviation_score, + 'score_uncertainty': np.nan, 'detail': deviation_detail, }, { 'key': 'residual_scatter', 'label': 'Residual Scatter Around Full Model Fit', - 'score': transit_qc_residual_scatter_score(summary.get('residual_scatter', np.nan)), - 'detail': ( - f"{summary.get('residual_scatter', np.nan) * 100.0:.4f}%" - if np.isfinite(summary.get('residual_scatter', np.nan)) - else "n/a" - ), + 'score': residual_scatter_score, + 'score_uncertainty': residual_scatter_score_uncertainty, + 'detail': residual_scatter_detail, }, { 'key': 'duration_consistency', 'label': 'Duration Consistency', 'score': transit_qc_duration_score(summary.get('duration_ratio', np.nan)), + 'score_uncertainty': np.nan, 'detail': duration_detail, }, { 'key': 'eebls_depth_snr', 'label': 'EEBLS Depth SNR', 'score': transit_qc_saturating_score(summary.get('eebls_depth_snr', np.nan), TRANSIT_QC_MIN_EEBLS_SNR), + 'score_uncertainty': np.nan, 'detail': ( f"{summary.get('eebls_depth_snr', np.nan):.2f}" if np.isfinite(summary.get('eebls_depth_snr', np.nan)) else "n/a" ), }, + { + 'key': 'sampling', + 'label': 'Sampling / Cadence', + 'score': summary.get('sampling_score', np.nan), + 'score_uncertainty': np.nan, + 'detail': summary.get('sampling_detail') or "n/a", + }, ] available_components = [ @@ -1206,8 +1873,10 @@ def compute_transit_qc_ktmf(summary): points = float(np.clip(score, 0.0, 1.0) * max_points) total_points += points ktmf_contributions.append({ + 'key': component['key'], 'label': component['label'], 'score': float(np.clip(score, 0.0, 1.0)), + 'score_uncertainty': component.get('score_uncertainty', np.nan), 'max_points': float(max_points), 'points': points, 'detail': component.get('detail'), @@ -1215,8 +1884,10 @@ def compute_transit_qc_ktmf(summary): }) else: ktmf_contributions.append({ + 'key': component['key'], 'label': component['label'], 'score': np.nan, + 'score_uncertainty': np.nan, 'max_points': 0.0, 'points': 0.0, 'detail': component.get('detail'), @@ -1406,6 +2077,17 @@ def evaluate_transit_detection_qc(fit): 'duration_consistency_note': None, 'eebls_depth_snr': np.nan, 'residual_scatter': np.nan, + 'transit_depth_for_residual_scatter': np.nan, + 'residual_scatter_to_depth_ratio': np.nan, + 'sampling_score': np.nan, + 'sampling_detail': None, + 'sampling_ingress_count': 0, + 'sampling_egress_count': 0, + 'sampling_in_transit_count': 0, + 'sampling_pre_baseline_count': 0, + 'sampling_post_baseline_count': 0, + 'sampling_total_duration': np.nan, + 'sampling_ingress_duration': np.nan, 'use_deviation_from_expected_transit_in_qc': bool(use_deviation_from_expected_transit_in_qc), 'deviation_sigma_threshold': deviation_sigma_threshold, 'expected_tmid': expected_context.get('expected_tmid', np.nan), @@ -1421,6 +2103,9 @@ def evaluate_transit_detection_qc(fit): 'tmid_deviation_threshold_minutes': np.nan, 'tmid_deviation_sigma': np.nan, 'rprs_deviation_fit_unc': np.nan, + 'rprs_deviation_model_fit_unc': np.nan, + 'rprs_deviation_data_fit_unc': np.nan, + 'rprs_deviation_combined_fit_unc': np.nan, 'rprs_deviation_expected_unc': np.nan, 'rprs_deviation_systematic_floor': np.nan, 'rprs_deviation_unc': np.nan, @@ -1512,8 +2197,17 @@ def evaluate_transit_detection_qc(fit): 'flat_a2': flat_model.get('a2', np.nan), 'flat_model_note': flat_model.get('note'), 'residual_scatter': transit_qc_residual_scatter(data, transit_model), + 'transit_depth_for_residual_scatter': transit_qc_model_depth_fraction(transit_model), 'point_count': int(point_count), }) + if ( + np.isfinite(summary['residual_scatter']) + and np.isfinite(summary['transit_depth_for_residual_scatter']) + and summary['transit_depth_for_residual_scatter'] > 0 + ): + summary['residual_scatter_to_depth_ratio'] = float( + summary['residual_scatter'] / summary['transit_depth_for_residual_scatter'] + ) if not summary['computed']: note = flat_model.get('note') or 'flat/null model comparison failed.' @@ -1549,6 +2243,18 @@ def evaluate_transit_detection_qc(fit): ensure_lightcurve_fit_eebls_diagnostic(fit) summary['eebls_depth_snr'] = extract_lightcurve_fit_eebls_snr(fit) + sampling_summary = transit_qc_sampling_summary(fit) + summary.update({ + 'sampling_score': sampling_summary.get('score', np.nan), + 'sampling_detail': sampling_summary.get('detail'), + 'sampling_ingress_count': sampling_summary.get('ingress_count', 0), + 'sampling_egress_count': sampling_summary.get('egress_count', 0), + 'sampling_in_transit_count': sampling_summary.get('in_transit_count', 0), + 'sampling_pre_baseline_count': sampling_summary.get('pre_baseline_count', 0), + 'sampling_post_baseline_count': sampling_summary.get('post_baseline_count', 0), + 'sampling_total_duration': sampling_summary.get('total_duration', np.nan), + 'sampling_ingress_duration': sampling_summary.get('ingress_duration', np.nan), + }) deviation_summary = evaluate_transit_qc_expected_value_deviation( fit, deviation_sigma_threshold, @@ -1566,6 +2272,9 @@ def evaluate_transit_detection_qc(fit): 'tmid_deviation_threshold_minutes': deviation_summary.get('tmid_deviation_threshold_minutes', np.nan), 'tmid_deviation_sigma': deviation_summary.get('tmid_deviation_sigma', np.nan), 'rprs_deviation_fit_unc': deviation_summary.get('rprs_deviation_fit_unc', np.nan), + 'rprs_deviation_model_fit_unc': deviation_summary.get('rprs_deviation_model_fit_unc', np.nan), + 'rprs_deviation_data_fit_unc': deviation_summary.get('rprs_deviation_data_fit_unc', np.nan), + 'rprs_deviation_combined_fit_unc': deviation_summary.get('rprs_deviation_combined_fit_unc', np.nan), 'rprs_deviation_expected_unc': deviation_summary.get('rprs_deviation_expected_unc', np.nan), 'rprs_deviation_systematic_floor': deviation_summary.get('rprs_deviation_systematic_floor', np.nan), 'rprs_deviation_unc': deviation_summary.get('rprs_deviation_unc', np.nan), @@ -1573,6 +2282,8 @@ def evaluate_transit_detection_qc(fit): 'tmid_deviation_score': deviation_summary.get('tmid_deviation_score', np.nan), 'rprs_deviation_score': deviation_summary.get('rprs_deviation_score', np.nan), 'deviation_from_expected_value': deviation_summary.get('deviation_from_expected_value', np.nan), + 'rprs_prior_assumed': deviation_summary.get('rprs_prior_assumed', False), + 'rprs_prior_assumed_note': deviation_summary.get('rprs_prior_assumed_note'), }) notes = [] @@ -1728,6 +2439,9 @@ def annotate_transit_detection_qc(fit, summary=None): fit.transit_qc_tmid_deviation_threshold_minutes = summary.get('tmid_deviation_threshold_minutes') fit.transit_qc_tmid_deviation_sigma = summary.get('tmid_deviation_sigma') fit.transit_qc_rprs_deviation_fit_unc = summary.get('rprs_deviation_fit_unc') + fit.transit_qc_rprs_deviation_model_fit_unc = summary.get('rprs_deviation_model_fit_unc') + fit.transit_qc_rprs_deviation_data_fit_unc = summary.get('rprs_deviation_data_fit_unc') + fit.transit_qc_rprs_deviation_combined_fit_unc = summary.get('rprs_deviation_combined_fit_unc') fit.transit_qc_rprs_deviation_expected_unc = summary.get('rprs_deviation_expected_unc') fit.transit_qc_rprs_deviation_systematic_floor = summary.get('rprs_deviation_systematic_floor') fit.transit_qc_rprs_deviation_unc = summary.get('rprs_deviation_unc') @@ -2025,6 +2739,234 @@ def build_comparison_candidate_transit_prior(p_dict, ld): } +def build_search_restriction_prior_from_planet_dict(p_dict): + if not isinstance(p_dict, dict): + return {} + return { + 'rprs': p_dict.get('rprs'), + 'ars': p_dict.get('aRs'), + } + + +def estimate_rprs_data_uncertainty_from_lightcurve( + times, + flux_values, + flux_errors, + prior, + transit_depth_threshold_fraction=0.05, +): + payload = { + 'available': False, + 'data_rprs_uncertainty': np.nan, + 'rprs_data_uncertainty': np.nan, + 'note': 'Unavailable; the prior transit model could not be evaluated against the light curve.', + } + if not isinstance(prior, dict): + return payload + + try: + times = np.asarray(times, dtype=float).reshape(-1) + flux_values = np.asarray(flux_values, dtype=float).reshape(-1) + except (TypeError, ValueError): + return payload + if flux_errors is None: + flux_errors = np.full(flux_values.shape, np.nan, dtype=float) + else: + try: + flux_errors = np.asarray(flux_errors, dtype=float).reshape(-1) + except (TypeError, ValueError): + flux_errors = np.full(flux_values.shape, np.nan, dtype=float) + + if not (times.shape == flux_values.shape == flux_errors.shape): + return payload + if times.size < 3: + payload['note'] = 'Unavailable; too few light-curve points for a data-based Rp/R* uncertainty estimate.' + return payload + + try: + prior_transit = np.asarray(transit(times, prior), dtype=float).reshape(-1) + except Exception as exc: + payload['note'] = f"Unavailable; prior transit model evaluation failed ({describe_retry_exception(exc)})." + return payload + if prior_transit.shape != times.shape: + return payload + + finite_model = np.isfinite(prior_transit) + finite_flux = np.isfinite(flux_values) + if not np.any(finite_model & finite_flux): + return payload + + baseline_scale = solve_transit_qc_flux_baseline(prior_transit, flux_values, dataerr=flux_errors) + if not np.isfinite(baseline_scale) or baseline_scale <= 0: + baseline_scale = 1.0 + scaled_transit = prior_transit * baseline_scale + + class PrefitRprsUncertaintyFit: + pass + + dummy_fit = PrefitRprsUncertaintyFit() + dummy_fit.time = times + dummy_fit.data = flux_values + dummy_fit.dataerr = flux_errors + dummy_fit.transit = scaled_transit + dummy_fit.model = scaled_transit + dummy_fit.residuals = flux_values - scaled_transit + dummy_fit.airmass_model = np.ones_like(scaled_transit) + dummy_fit.parameters = dict(prior) + dummy_fit.errors = {'rprs': np.nan} + + empirical = fit_empirical_transit_uncertainty( + dummy_fit, + transit_depth_threshold_fraction=transit_depth_threshold_fraction, + ) + if not isinstance(empirical, dict) or not empirical.get('available'): + payload['note'] = ( + "Unavailable; the prior transit shape did not provide enough in-transit/out-of-transit " + "support for a data-based Rp/R* uncertainty estimate." + ) + return payload + + data_uncertainty = empirical.get('data_rprs_uncertainty') + try: + data_uncertainty = float(data_uncertainty) + except (TypeError, ValueError): + data_uncertainty = np.nan + if not np.isfinite(data_uncertainty) or data_uncertainty < 0: + return payload + + payload.update(empirical) + payload.update({ + 'available': True, + 'data_rprs_uncertainty': data_uncertainty, + 'rprs_data_uncertainty': data_uncertainty, + 'rprs_data_uncertainty_source': 'prefit_flux_residual_red_noise', + 'baseline_scale': float(baseline_scale), + 'note': ( + f"Estimated pre-fit Rp/R* data uncertainty {data_uncertainty:.6f} " + "from residual scatter around the prior transit shape." + ), + }) + return payload + + +def rprs_prior_window_half_widths(prior): + if not isinstance(prior, dict): + return None + try: + rprs = float(prior.get('rprs')) + percentage = float(RPRS_RANGE_RESTRICTION_PERCENTAGE) + except (TypeError, ValueError): + return None + if not np.isfinite(rprs) or rprs <= RPRS_SEARCH_BOUND_MIN: + return None + if not np.isfinite(percentage) or percentage < 0: + return None + + configured_half_width = abs(rprs) * percentage / 100.0 + data_uncertainty = prior.get('rprs_data_uncertainty', prior.get('data_rprs_uncertainty')) + try: + data_uncertainty = float(data_uncertainty) + except (TypeError, ValueError): + data_uncertainty = np.nan + try: + data_sigma = float(prior.get('rprs_data_uncertainty_bound_sigma', RPRS_DATA_UNCERTAINTY_BOUND_SIGMA)) + except (TypeError, ValueError): + data_sigma = RPRS_DATA_UNCERTAINTY_BOUND_SIGMA + if not np.isfinite(data_sigma) or data_sigma <= 0: + data_sigma = RPRS_DATA_UNCERTAINTY_BOUND_SIGMA + + data_half_width = np.nan + use_data_window = False + if np.isfinite(data_uncertainty) and data_uncertainty > configured_half_width: + data_half_width = float(data_sigma * data_uncertainty) + use_data_window = np.isfinite(data_half_width) and data_half_width > configured_half_width + + half_width = data_half_width if use_data_window else configured_half_width + if not np.isfinite(half_width) or half_width <= 0: + return None + return { + 'center': float(rprs), + 'configured_half_width': float(configured_half_width), + 'data_uncertainty': float(data_uncertainty) if np.isfinite(data_uncertainty) else np.nan, + 'data_sigma': float(data_sigma), + 'data_half_width': float(data_half_width) if np.isfinite(data_half_width) else np.nan, + 'half_width': float(half_width), + 'use_data_window': bool(use_data_window), + } + + +def rprs_prior_centered_bounds(prior): + widths = rprs_prior_window_half_widths(prior) + if not widths: + return None + center = widths['center'] + half_width = widths['half_width'] + lower_bound = max(float(RPRS_SEARCH_BOUND_MIN), center - half_width) + upper_bound = min(float(RPRS_SEARCH_BOUND_MAX), center + half_width) + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or upper_bound <= lower_bound: + return None + return [float(lower_bound), float(upper_bound)] + + +def enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_prior, + times, + flux_values, + flux_errors, + transit_prior, + context_label="light curve", +): + enriched = dict(search_prior) if isinstance(search_prior, dict) else {} + if not isinstance(transit_prior, dict): + return enriched + estimate = estimate_rprs_data_uncertainty_from_lightcurve( + times, + flux_values, + flux_errors, + transit_prior, + ) + if not estimate.get('available'): + return enriched + + data_uncertainty = estimate.get('data_rprs_uncertainty') + try: + data_uncertainty = float(data_uncertainty) + except (TypeError, ValueError): + data_uncertainty = np.nan + if not np.isfinite(data_uncertainty) or data_uncertainty < 0: + return enriched + + enriched['rprs_data_uncertainty'] = data_uncertainty + enriched['data_rprs_uncertainty'] = data_uncertainty + enriched['rprs_data_uncertainty_bound_sigma'] = RPRS_DATA_UNCERTAINTY_BOUND_SIGMA + enriched['rprs_data_uncertainty_payload'] = estimate + + widths = rprs_prior_window_half_widths(enriched) + if widths and widths.get('use_data_window'): + log_info( + f"Rp/R* data-derived uncertainty for the {context_label} is " + f"{data_uncertainty:.6f}, larger than the configured " + f"+/-{RPRS_RANGE_RESTRICTION_PERCENTAGE:.1f}% prior window " + f"({widths['configured_half_width']:.6f}); widening the prior-centered " + f"Rp/R* search half-width to {widths['data_sigma']:.1f} sigma " + f"({widths['half_width']:.6f})." + ) + return enriched + + +def widen_rprs_bounds_to_data_uncertainty_window(bounds, prior): + widened = clone_lightcurve_bounds(bounds) + if 'rprs' not in widened or not RPRS_RANGE_RESTRICTION_ENABLED: + return widened + widths = rprs_prior_window_half_widths(prior) + if not widths or not widths.get('use_data_window'): + return widened + window = rprs_prior_centered_bounds(prior) + if window is not None: + widened['rprs'] = window + return widened + + def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_flux, airmass, jd_times=None, adaptive_summary=None, expected_transit_depth=None): @@ -2447,6 +3389,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, adaptive_summary=None, run_fast_ultranest_before_final_run=FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, + run_final_fit_phase_residual_clip=FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT, + run_final_residual_rejection=FINAL_RESIDUAL_REJECTION_DEFAULT, precomputed_candidate_series=None): result = { 'applied': False, @@ -2560,7 +3504,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, ) if ( - prefit is not None + run_final_fit_phase_residual_clip + and prefit is not None and hasattr(prefit, 'residuals') and hasattr(prefit, 'phase') and np.shape(prefit.residuals) == np.shape(good_times) @@ -2651,6 +3596,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, keep_ultranest_sampler_for_deferred_extension=not bool(fast_binning.get('applied')), fix_baseline_terms_for_final=not bool(fast_binning.get('applied')), pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=build_search_restriction_prior_from_planet_dict(p_dict), ) annotate_fast_ultranest_binning(final_fit, fast_binning) if final_fit is None: @@ -2671,6 +3617,176 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_comp_flux = good_comp_flux[final_time_indices] source_indices = source_indices[final_time_indices] + if run_final_residual_rejection and not fast_binning.get('applied'): + residual_payload = initialize_final_residual_rejection_payload( + enabled=True, + input_point_count=int(good_times.shape[0]), + ) + residual_stop_reason = None + for residual_refit_iteration in range(1, FINAL_RESIDUAL_REJECTION_MAX_REFITS + 1): + min_required_points = max(len(fit_bounds) + 1, LIGHTCURVE_MIN_VALID_POINTS) + residual_keep_mask, residual_summary = final_residual_rejection_keep_mask( + final_fit, + min_required_points=min_required_points, + ) + cycle_payload = build_final_residual_rejection_payload( + final_fit, + residual_keep_mask, + residual_summary, + source_indices=source_indices, + ) + if residual_keep_mask.shape != good_times.shape: + residual_stop_reason = 'shape_mismatch' + if int(residual_payload.get('rejected_point_count', 0) or 0) == 0: + residual_payload['note'] = ( + "Skipped; the final-fit residual array did not align with the retained light-curve points." + ) + else: + log_info( + "Warning: final residual rejection stopped because the residual array no longer " + "aligned with the retained light-curve points.", + warn=True, + ) + break + + if not residual_summary.get('applied'): + if int(residual_payload.get('rejected_point_count', 0) or 0) == 0: + residual_payload = dict(cycle_payload) + residual_payload.setdefault('refit_iteration_count', 0) + residual_payload.setdefault('refit_iterations', []) + else: + residual_payload = update_final_residual_rejection_final_pass( + residual_payload, + residual_summary, + ) + break + + clipped_times = good_times[residual_keep_mask] + clipped_flux = good_flux[residual_keep_mask] + clipped_unc = good_unc[residual_keep_mask] + clipped_airmass = good_airmass[residual_keep_mask] + clipped_jd_times = good_jd_times[residual_keep_mask] + clipped_target_flux = good_target_flux[residual_keep_mask] + clipped_comp_flux = good_comp_flux[residual_keep_mask] + clipped_source_indices = source_indices[residual_keep_mask] + + residual_refit_prior = dict(fit_prior) + final_parameters = getattr(final_fit, 'parameters', {}) + if isinstance(final_parameters, dict): + for key in ('rprs', 'ars', 'tmid', 'inc', 'a0', 'a1', 'a2'): + if key in residual_refit_prior and key in final_parameters: + residual_refit_prior[key] = final_parameters[key] + residual_refit_bounds = get_posterior_refit_final_bounds(final_fit, fit_bounds) + residual_coverage_assessment = build_expected_transit_coverage_assessment( + clipped_times, + residual_refit_prior, + flux_values=clipped_flux, + flux_errors=clipped_unc, + tmid_search_summary=build_ephemeris_tmid_search_summary_for_coverage( + clipped_times, + p_dict, + prior=residual_refit_prior, + duration_prior=build_single_transit_duration_prior(p_dict), + sigma_multiplier=35.0, + ), + duration_prior=build_single_transit_duration_prior(p_dict), + ) + refit, refit_flux, refit_unc = fit_final_lightcurve_with_oot_baseline_detrending( + clipped_times, + clipped_flux, + clipped_unc, + clipped_airmass, + residual_refit_prior, + residual_refit_bounds, + jd_times=clipped_jd_times, + skip_airmass_fit=skip_final_airmass_fit, + airmass_skip_note=airmass_skip_note, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + baseline_duration_multiplier=baseline_duration_multiplier, + expected_planet_dict=p_dict, + expected_tmid_search_summary=tmid_search_summary, + eebls_search_summary=eebls_search_summary, + extend_sparse_posterior_live_points=False, + keep_ultranest_sampler_for_deferred_extension=True, + fix_baseline_terms_for_final=True, + pre_ultranest_coverage_assessment=residual_coverage_assessment, + search_restriction_prior=build_search_restriction_prior_from_planet_dict(p_dict), + ) + if refit is None: + residual_stop_reason = 'refit_failed' + if int(residual_payload.get('rejected_point_count', 0) or 0) == 0: + residual_payload['note'] = ( + "Skipped; the residual-rejected final UltraNest refit did not converge, " + "so the unrejected final fit was retained." + ) + log_info( + "Warning: the residual-rejected final UltraNest refit did not converge; " + "retaining the last successful final fit.", + warn=True, + ) + break + + cycle_note = ( + f"Iteration {residual_refit_iteration}: {residual_summary.get('note')} " + f"Reran UltraNest on {int(clipped_times.shape[0])} point(s)." + ) + diagnostic = build_time_rejection_diagnostic( + f"Final residual rejection refit {residual_refit_iteration}", + good_times, + residual_keep_mask, + note=cycle_note, + ) + if diagnostic is not None: + result['filter_diagnostics'].append(diagnostic) + log_info(cycle_note) + residual_payload = record_final_residual_rejection_refit_cycle( + residual_payload, + cycle_payload, + residual_refit_iteration, + ) + annotate_fast_ultranest_binning(refit, fast_binning) + final_fit = refit + fitted_flux = np.asarray(refit_flux, dtype=float) + fitted_unc = np.asarray(refit_unc, dtype=float) + good_times = clipped_times + good_flux = clipped_flux + good_unc = clipped_unc + good_airmass = clipped_airmass + good_jd_times = clipped_jd_times + good_target_flux = clipped_target_flux + good_comp_flux = clipped_comp_flux + source_indices = clipped_source_indices + else: + residual_stop_reason = 'max_refits' + + residual_payload = finalize_final_residual_rejection_payload( + residual_payload, + current_point_count=int(good_times.shape[0]), + stopped_reason=residual_stop_reason, + ) + annotate_final_residual_rejection(final_fit, residual_payload) + else: + annotate_final_residual_rejection( + final_fit, + { + 'enabled': bool(run_final_residual_rejection), + 'applied': False, + 'note': ( + "Deferred to the selected full-resolution final refit." + if fast_binning.get('applied') + else "Disabled per optional_info setting." + ), + 'input_point_count': int(final_fit_times.size), + 'kept_point_count': int(final_fit_times.size), + 'rejected_point_count': 0, + 'sigma': FINAL_RESIDUAL_REJECTION_SIGMA, + }, + ) + annotate_lightcurve_filter_diagnostics(final_fit, result['filter_diagnostics']) annotate_selected_photometry_debug( final_fit, @@ -2807,10 +3923,12 @@ def refit_selected_fast_comparison_on_full_lightcurve( skip_airmass_fit=False, airmass_skip_note=None, detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT, use_impactparameter_rather_than_inclination_to_fit=True, plot_time_range=None, duration_prior=None, sparse_live_point_extension_enabled=None, + run_final_residual_rejection=FINAL_RESIDUAL_REJECTION_DEFAULT, ): previous_fit = selected_result.get('fit') if isinstance(selected_result, dict) else None if previous_fit is None or not getattr(previous_fit, 'fast_ultranest_binning_applied', False): @@ -2832,7 +3950,28 @@ def refit_selected_fast_comparison_on_full_lightcurve( if jd_times is not None and jd_times.shape != times.shape: jd_times = None + original_times = times.copy() + base_filter_diagnostics = [ + dict(diagnostic) + for diagnostic in getattr(previous_fit, 'frame_filter_diagnostics', []) + if isinstance(diagnostic, dict) + ] + residual_rejection_diagnostics = [] + residual_rejection_payload = None + + def aligned_selected_array(key, dtype=float): + values = selected_result.get(key) + if values is None: + return None + array = np.asarray(values, dtype=dtype).reshape(-1) + return array if array.shape == times.shape else None + + target_flux_values = aligned_selected_array('good_target_flux') + comp_flux_values = aligned_selected_array('good_comp_flux') + source_indices = aligned_selected_array('source_indices', dtype=int) + prior = build_full_resolution_final_prior_from_previous_fit(previous_fit, p_dict) + search_restriction_prior = build_search_restriction_prior_from_planet_dict(p_dict) fallback_bounds = selected_result.get('fast_fit_bounds') if not isinstance(fallback_bounds, dict): fallback_bounds = getattr(previous_fit, 'bounds', {}) @@ -2869,6 +4008,7 @@ def refit_selected_fast_comparison_on_full_lightcurve( flux_errors, previous_fit, prior=prior, + min_side_points=oot_baseline_min_points_per_side, ) if detrend_result.get('applied'): fit_flux = np.asarray(detrend_result['flux'], dtype=float) @@ -2876,6 +4016,25 @@ def refit_selected_fast_comparison_on_full_lightcurve( prior['a0'] = 1.0 prior['a1'] = 1.0 prior['a2'] = 0.0 + log_info( + "Applying selected full-resolution out-of-transit linear baseline detrending: " + f"{detrend_result.get('note', 'baseline fit details unavailable')}" + ) + else: + log_info( + "Selected full-resolution out-of-transit baseline detrending skipped: " + f"{detrend_result.get('note', 'baseline fit details unavailable')}" + ) + + search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_restriction_prior, + times, + fit_flux, + fit_unc, + prior, + context_label="selected full-resolution final light curve", + ) + bounds = widen_rprs_bounds_to_data_uncertainty_window(bounds, search_restriction_prior) fixed_errors = baseline_fixed_errors_from_fit(previous_fit) if detrend_result.get('applied'): @@ -2907,36 +4066,215 @@ def refit_selected_fast_comparison_on_full_lightcurve( ) log_expected_transit_coverage_assessment(pre_ultranest_coverage_assessment) - fixed_baseline_source = ( - "with a flat fixed baseline after out-of-transit detrending" - if detrend_result.get('applied') - else "with fixed a0/a2 from the previous fast UltraNest fit" - ) - log_info( - "Running the selected comparison-star final UltraNest fit on the full-resolution light curve " - f"{fixed_baseline_source} at {min_live_points} minimum live points." - ) - fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( - times, - fit_flux, - fit_unc, - airmass, - prior, - bounds, - jd_times=jd_times, - use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, - max_rprs_retries=0, - max_ars_retries=0, - max_impact_parameter_retries=0, - duration_prior=duration_prior, - keep_ultranest_sampler=False, - fixed_parameter_errors=fixed_errors, - fixed_flux_baseline=True, - ultranest_min_num_live_points=min_live_points, - pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, - ) + fixed_baseline_source = ( + "with a flat fixed baseline after out-of-transit detrending" + if detrend_result.get('applied') + else "with fixed a0/a2 from the previous fast UltraNest fit" + ) + log_info( + "Running the selected comparison-star final UltraNest fit on the full-resolution light curve " + f"{fixed_baseline_source} at {min_live_points} minimum live points." + ) + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + fit_flux, + fit_unc, + airmass, + prior, + bounds, + jd_times=jd_times, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + max_rprs_retries=0, + max_ars_retries=0, + max_impact_parameter_retries=0, + duration_prior=duration_prior, + keep_ultranest_sampler=False, + fixed_parameter_errors=fixed_errors, + fixed_flux_baseline=True, + ultranest_min_num_live_points=min_live_points, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, + ) + if fit is None: + return None + + if run_final_residual_rejection: + residual_rejection_payload = initialize_final_residual_rejection_payload( + enabled=True, + input_point_count=int(times.shape[0]), + ) + residual_stop_reason = None + for residual_refit_iteration in range(1, FINAL_RESIDUAL_REJECTION_MAX_REFITS + 1): + min_required_points = max(len(bounds) + 1, LIGHTCURVE_MIN_VALID_POINTS) + residual_keep_mask, residual_summary = final_residual_rejection_keep_mask( + fit, + min_required_points=min_required_points, + ) + cycle_payload = build_final_residual_rejection_payload( + fit, + residual_keep_mask, + residual_summary, + source_indices=source_indices, + ) + if residual_keep_mask.shape != times.shape: + residual_stop_reason = 'shape_mismatch' + if int(residual_rejection_payload.get('rejected_point_count', 0) or 0) == 0: + residual_rejection_payload['note'] = ( + "Skipped; the final-fit residual array did not align with the retained light-curve points." + ) + else: + log_info( + "Warning: final residual rejection stopped because the residual array no longer " + "aligned with the retained light-curve points.", + warn=True, + ) + break + + if not residual_summary.get('applied'): + if int(residual_rejection_payload.get('rejected_point_count', 0) or 0) == 0: + residual_rejection_payload = dict(cycle_payload) + residual_rejection_payload.setdefault('refit_iteration_count', 0) + residual_rejection_payload.setdefault('refit_iterations', []) + else: + residual_rejection_payload = update_final_residual_rejection_final_pass( + residual_rejection_payload, + residual_summary, + ) + break + + retained_times = times[residual_keep_mask] + retained_flux = fit_flux[residual_keep_mask] + retained_unc = fit_unc[residual_keep_mask] + retained_airmass = airmass[residual_keep_mask] + retained_jd_times = None if jd_times is None else jd_times[residual_keep_mask] + retained_target_flux_values = ( + None if target_flux_values is None else target_flux_values[residual_keep_mask] + ) + retained_comp_flux_values = ( + None if comp_flux_values is None else comp_flux_values[residual_keep_mask] + ) + retained_source_indices = ( + None if source_indices is None else source_indices[residual_keep_mask] + ) + + residual_refit_prior = dict(prior) + final_parameters = getattr(fit, 'parameters', {}) + if isinstance(final_parameters, dict): + for key in ('rprs', 'ars', 'tmid', 'inc'): + if key in residual_refit_prior and key in final_parameters: + residual_refit_prior[key] = final_parameters[key] + residual_refit_bounds = get_posterior_refit_final_bounds(fit, bounds) + residual_search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_restriction_prior, + retained_times, + retained_flux, + retained_unc, + residual_refit_prior, + context_label="residual-rejected selected final light curve", + ) + residual_refit_bounds = widen_rprs_bounds_to_data_uncertainty_window( + residual_refit_bounds, + residual_search_restriction_prior, + ) + residual_coverage_assessment = build_expected_transit_coverage_assessment( + retained_times, + residual_refit_prior, + flux_values=retained_flux, + flux_errors=retained_unc, + tmid_search_summary=build_ephemeris_tmid_search_summary_for_coverage( + retained_times, + p_dict, + prior=residual_refit_prior, + duration_prior=coverage_duration_prior, + sigma_multiplier=35.0, + ), + duration_prior=coverage_duration_prior, + ) + log_expected_transit_coverage_assessment(residual_coverage_assessment) + refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + retained_times, + retained_flux, + retained_unc, + retained_airmass, + residual_refit_prior, + residual_refit_bounds, + jd_times=retained_jd_times, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + max_rprs_retries=0, + max_ars_retries=0, + max_impact_parameter_retries=0, + duration_prior=duration_prior, + keep_ultranest_sampler=False, + fixed_parameter_errors=fixed_errors, + fixed_flux_baseline=True, + ultranest_min_num_live_points=min_live_points, + pre_ultranest_coverage_assessment=residual_coverage_assessment, + search_restriction_prior=residual_search_restriction_prior, + ) + if refit is None: + residual_stop_reason = 'refit_failed' + if int(residual_rejection_payload.get('rejected_point_count', 0) or 0) == 0: + residual_rejection_payload['note'] = ( + "Skipped; the residual-rejected selected full-resolution UltraNest refit " + "did not converge, so the unrejected final fit was retained." + ) + log_info( + "Warning: the residual-rejected selected full-resolution UltraNest refit did not " + "converge; retaining the last successful final fit.", + warn=True, + ) + break + + cycle_note = ( + f"Iteration {residual_refit_iteration}: {residual_summary.get('note')} " + f"Reran UltraNest on {int(retained_times.shape[0])} point(s)." + ) + residual_rejection_diagnostic = build_time_rejection_diagnostic( + f"Final residual rejection refit {residual_refit_iteration}", + times, + residual_keep_mask, + note=cycle_note, + ) + if residual_rejection_diagnostic is not None: + residual_rejection_diagnostics.append(residual_rejection_diagnostic) + log_info(cycle_note) + residual_rejection_payload = record_final_residual_rejection_refit_cycle( + residual_rejection_payload, + cycle_payload, + residual_refit_iteration, + ) + fit = refit + times = retained_times + fit_flux = retained_flux + fit_unc = retained_unc + airmass = retained_airmass + jd_times = retained_jd_times + target_flux_values = retained_target_flux_values + comp_flux_values = retained_comp_flux_values + source_indices = retained_source_indices + pre_ultranest_coverage_assessment = residual_coverage_assessment + else: + residual_stop_reason = 'max_refits' + + residual_rejection_payload = finalize_final_residual_rejection_payload( + residual_rejection_payload, + current_point_count=int(times.shape[0]), + stopped_reason=residual_stop_reason, + ) + else: + residual_rejection_payload = { + 'enabled': False, + 'applied': False, + 'note': "Disabled per optional_info setting.", + 'input_point_count': int(times.shape[0]), + 'kept_point_count': int(times.shape[0]), + 'rejected_point_count': 0, + 'sigma': FINAL_RESIDUAL_REJECTION_SIGMA, + } + annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) - fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) + fit = apply_plot_time_range(fit, original_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) baseline_parameter_note = ( "Full-resolution out-of-transit linear detrending flattened the final-fit light curve; " @@ -2968,7 +4306,7 @@ def refit_selected_fast_comparison_on_full_lightcurve( fit, { 'applied': False, - 'original_point_count': int(times.shape[0]), + 'original_point_count': int(original_times.shape[0]), 'binned_point_count': int(times.shape[0]), 'note': 'Full-resolution selected comparison-star final run; fast binning was not applied.', }, @@ -2993,6 +4331,26 @@ def refit_selected_fast_comparison_on_full_lightcurve( ) else: annotate_sparse_posterior_live_point_extension(fit, False, False) + if residual_rejection_diagnostics: + base_filter_diagnostics.extend(residual_rejection_diagnostics) + annotate_lightcurve_filter_diagnostics(fit, base_filter_diagnostics) + annotate_final_residual_rejection(fit, residual_rejection_payload) + selected_debug = getattr(previous_fit, 'selected_photometry_debug', None) + if selected_debug is not None: + fit.selected_photometry_debug = copy.deepcopy(selected_debug) + selected_result['good_times'] = np.asarray(times, dtype=float) + selected_result['good_flux'] = np.asarray(fit_flux, dtype=float) + selected_result['good_unc'] = np.asarray(fit_unc, dtype=float) + selected_result['good_airmass'] = np.asarray(airmass, dtype=float) + selected_result['good_jd_times'] = None if jd_times is None else np.asarray(jd_times, dtype=float) + if target_flux_values is not None: + selected_result['good_target_flux'] = np.asarray(target_flux_values, dtype=float) + selected_result['tflux_fit'] = np.asarray(target_flux_values, dtype=float) + if comp_flux_values is not None: + selected_result['good_comp_flux'] = np.asarray(comp_flux_values, dtype=float) + selected_result['cflux_fit'] = np.asarray(comp_flux_values, dtype=float) + if source_indices is not None: + selected_result['source_indices'] = np.asarray(source_indices, dtype=int) annotate_transit_detection_qc(fit) clear_fit_ultranest_resume_state(fit) return fit, fit_flux, fit_unc @@ -3088,9 +4446,11 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, data_highres, _ = estimate_transit_duration_samples_from_fit(final_fit, sample_count=1) if data_highres is not None: plot_final_lightcurve(final_fit, data_highres, p_dict['pName'], candidate_info_dict['save'], observation_date) + plot_prior_posterior_comparison(final_fit, p_dict, p_dict['pName'], candidate_info_dict['save'], observation_date) + plot_ktmf_qc_metrics(final_fit, p_dict['pName'], candidate_info_dict['save'], observation_date) except Exception as exc: archive_errors.append(archive_exception_payload( - "Could not save the final lightcurve plot", + "Could not save the final lightcurve, prior/posterior comparison, or KTMF QC plot", exc, )) @@ -3734,10 +5094,99 @@ def extend_selected_comparison_live_points_if_needed(fit, enabled=None): ) +def prior_centered_parameter_bounds( + prior_value, + percentage, + minimum_bound, + maximum_bound=None, +): + try: + center = float(prior_value) + percentage = float(percentage) + except (TypeError, ValueError): + return None + + if not np.isfinite(center) or center <= minimum_bound: + return None + if not np.isfinite(percentage) or percentage < 0: + return None + + fraction = percentage / 100.0 + lower_bound = max(float(minimum_bound), center * (1.0 - fraction)) + upper_bound = center * (1.0 + fraction) + if maximum_bound is not None: + upper_bound = min(float(maximum_bound), upper_bound) + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or upper_bound <= lower_bound: + return None + return [float(lower_bound), float(upper_bound)] + + +def configured_prior_centered_bounds_for_key(key, prior): + if not isinstance(prior, dict): + return None + if key == 'rprs': + if not RPRS_RANGE_RESTRICTION_ENABLED: + return None + return rprs_prior_centered_bounds(prior) + if key == 'ars': + if not ARS_RANGE_RESTRICTION_ENABLED: + return None + return prior_centered_parameter_bounds( + prior.get('ars'), + ARS_RANGE_RESTRICTION_PERCENTAGE, + ARS_SEARCH_BOUND_MIN, + ) + return None + + +def intersect_parameter_bounds(bounds, restriction): + if restriction is None: + return None + try: + lower_bound, upper_bound = [ + float(value) for value in np.asarray(bounds, dtype=float).reshape(-1)[:2] + ] + restrict_lower, restrict_upper = [ + float(value) for value in np.asarray(restriction, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return None + + lower_bound = max(lower_bound, restrict_lower) + upper_bound = min(upper_bound, restrict_upper) + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or upper_bound <= lower_bound: + return None + return [float(lower_bound), float(upper_bound)] + + +def apply_configured_prior_search_restrictions(bounds, prior): + restricted = widen_rprs_bounds_to_data_uncertainty_window(bounds, prior) + for key in ('rprs', 'ars'): + if key not in restricted: + continue + restriction = configured_prior_centered_bounds_for_key(key, prior) + intersection = intersect_parameter_bounds(restricted[key], restriction) + if intersection is not None: + restricted[key] = intersection + return restricted + + +def bounds_are_close(bounds_a, bounds_b, atol=1e-12): + if bounds_a is None or bounds_b is None: + return False + try: + array_a = np.asarray(bounds_a, dtype=float).reshape(-1)[:2] + array_b = np.asarray(bounds_b, dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + return False + return array_a.shape == array_b.shape and bool(np.allclose(array_a, array_b, rtol=0.0, atol=atol)) + + def build_initial_rprs_bounds( rprs, lower_scale=INITIAL_RPRS_BOUND_LOWER_SCALE, upper_scale=INITIAL_RPRS_BOUND_UPPER_SCALE, + rprs_data_uncertainty=None, ): try: rprs = float(rprs) @@ -3757,7 +5206,15 @@ def build_initial_rprs_bounds( lower_bound = RPRS_SEARCH_BOUND_MIN upper_bound = RPRS_SEARCH_BOUND_MAX - return [float(lower_bound), float(upper_bound)] + bounds = [float(lower_bound), float(upper_bound)] + restriction_prior = {'rprs': rprs} + if rprs_data_uncertainty is not None: + restriction_prior['rprs_data_uncertainty'] = rprs_data_uncertainty + restricted_bounds = intersect_parameter_bounds( + bounds, + configured_prior_centered_bounds_for_key('rprs', restriction_prior), + ) + return restricted_bounds if restricted_bounds is not None else bounds def build_initial_ars_bounds( @@ -3789,15 +5246,29 @@ def build_initial_ars_bounds( if not np.isfinite(upper_bound) or upper_bound <= lower_bound: upper_bound = float(lower_bound + max(np.finfo(float).eps, ARS_SEARCH_BOUND_MIN)) - return [float(lower_bound), float(upper_bound)] + bounds = [float(lower_bound), float(upper_bound)] + restricted_bounds = intersect_parameter_bounds( + bounds, + configured_prior_centered_bounds_for_key('ars', {'ars': ars}), + ) + return restricted_bounds if restricted_bounds is not None else bounds -def build_initial_transit_bounds(prior, tmid_bounds, ars_unc=None, inclination_half_width=5.0): +def build_initial_transit_bounds( + prior, + tmid_bounds, + ars_unc=None, + inclination_half_width=5.0, + rprs_data_uncertainty=None, +): lower, upper = [float(value) for value in np.asarray(tmid_bounds, dtype=float).reshape(-1)[:2]] # Keep ars ahead of inc so the internal impact-parameter parameterization # uses the sampled ars value when converting inclination to b. return { - 'rprs': build_initial_rprs_bounds(prior['rprs']), + 'rprs': build_initial_rprs_bounds( + prior['rprs'], + rprs_data_uncertainty=rprs_data_uncertainty, + ), 'tmid': [lower, upper], 'ars': build_initial_ars_bounds(prior['ars'], ars_unc=ars_unc), 'inc': [prior['inc'] - inclination_half_width, min(90, prior['inc'] + inclination_half_width)], @@ -4148,7 +5619,19 @@ def run_nested_lightcurve_fit_with_rprs_posterior_retry( fixed_flux_baseline=False, ultranest_min_num_live_points=None, pre_ultranest_coverage_assessment=None, + search_restriction_prior=None, + use_prior_rprs_when_posterior_pinned=None, ): + if use_prior_rprs_when_posterior_pinned is None: + use_prior_rprs_when_posterior_pinned = RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR + else: + use_prior_rprs_when_posterior_pinned = bool(use_prior_rprs_when_posterior_pinned) + restriction_reference_prior = ( + dict(search_restriction_prior) + if isinstance(search_restriction_prior, dict) + else dict(prior) if isinstance(prior, dict) else {} + ) + def impact_parameter_retry_available(fit, local_bounds): if not use_impactparameter_rather_than_inclination_to_fit or 'inc' not in local_bounds: return False @@ -4256,8 +5739,10 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co }, ] - def build_fit(local_prior, local_bounds): - local_bounds = sanitize_retry_search_bounds(local_bounds) + def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): + local_bounds = sanitize_retry_search_bounds( + apply_configured_prior_search_restrictions(local_bounds, restriction_reference_prior) + ) if fixed_flux_baseline: local_bounds = clone_lightcurve_bounds(local_bounds) for key in ('a0', 'a1', 'a2'): @@ -4275,8 +5760,15 @@ def build_fit(local_prior, local_bounds): fit_kwargs['keep_ultranest_sampler'] = True if baseline_fit_mask is not None and callable_accepts_keyword(lc_fitter, 'baseline_fit_mask'): fit_kwargs['baseline_fit_mask'] = baseline_fit_mask - if fixed_parameter_errors and callable_accepts_keyword(lc_fitter, 'fixed_parameter_errors'): - fit_kwargs['fixed_parameter_errors'] = fixed_parameter_errors + effective_fixed_parameter_errors = ( + dict(fixed_parameter_errors) + if isinstance(fixed_parameter_errors, dict) + else {} + ) + if isinstance(fixed_parameter_errors_override, dict): + effective_fixed_parameter_errors.update(fixed_parameter_errors_override) + if effective_fixed_parameter_errors and callable_accepts_keyword(lc_fitter, 'fixed_parameter_errors'): + fit_kwargs['fixed_parameter_errors'] = effective_fixed_parameter_errors if fixed_flux_baseline and callable_accepts_keyword(lc_fitter, 'fixed_flux_baseline'): fit_kwargs['fixed_flux_baseline'] = True if ( @@ -4296,7 +5788,9 @@ def build_fit(local_prior, local_bounds): annotate_duration_prior(fit, duration_prior) return fit - current_bounds = sanitize_retry_search_bounds(bounds) + current_bounds = sanitize_retry_search_bounds( + apply_configured_prior_search_restrictions(bounds, restriction_reference_prior) + ) current_prior = clamp_retry_priors_to_bounds(prior, current_bounds) retry_histories = {config['key']: [] for config in retry_configs} retry_notes = {config['key']: None for config in retry_configs} @@ -4384,6 +5878,11 @@ def build_fit(local_prior, local_bounds): clamped_bounds, ) clamped_bounds = retry_config['sanitize_bounds']({bounds_key: clamped_bounds}).get(bounds_key, clamped_bounds) + clamped_bounds = apply_configured_prior_search_restrictions( + {bounds_key: clamped_bounds}, + restriction_reference_prior, + ).get(bounds_key, clamped_bounds) + clamped_bounds = retry_config['sanitize_bounds']({bounds_key: clamped_bounds}).get(bounds_key, clamped_bounds) new_lower, new_upper = [float(value) for value in clamped_bounds] if previous_bounds is not None: previous_lower, previous_upper = [float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2]] @@ -4397,7 +5896,20 @@ def build_fit(local_prior, local_bounds): if not expands_sampled_range: maximum_bound = retry_config['max_bound'] + restricted_bounds = configured_prior_centered_bounds_for_key( + bounds_key, + restriction_reference_prior, + ) if ( + restricted_bounds is not None + and bounds_are_close(previous_bounds, restricted_bounds) + ): + retry_notes[key] = ( + f"Skipped; the automatic {label} retry reached the configured " + f"prior-centered search range [{restricted_bounds[0]:.6f}, " + f"{restricted_bounds[1]:.6f}]." + ) + elif ( maximum_bound is not None and previous_lower <= retry_config['min_bound'] + 1e-12 and previous_upper >= maximum_bound - 1e-12 @@ -4429,7 +5941,9 @@ def build_fit(local_prior, local_bounds): updated_bounds = clone_lightcurve_bounds(current_bounds) updated_bounds[bounds_key] = [new_lower, new_upper] - updated_bounds = sanitize_retry_search_bounds(updated_bounds) + updated_bounds = sanitize_retry_search_bounds( + apply_configured_prior_search_restrictions(updated_bounds, restriction_reference_prior) + ) updated_prior = dict(current_prior) fit_parameters = getattr(fit, 'parameters', {}) @@ -4447,6 +5961,117 @@ def build_fit(local_prior, local_bounds): fit = build_fit(current_prior, current_bounds) final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + rprs_final_diagnostics = None + if callable(final_diagnostics_getter) and 'rprs' in current_bounds: + rprs_final_diagnostics = final_diagnostics_getter('rprs') + latest_diagnostics['rprs'] = rprs_final_diagnostics + elif latest_diagnostics.get('rprs') is not None: + rprs_final_diagnostics = latest_diagnostics['rprs'] + + if ( + use_prior_rprs_when_posterior_pinned + and int(max(0, max_rprs_retries)) <= 0 + and 'rprs' in current_bounds + and isinstance(rprs_final_diagnostics, dict) + and rprs_final_diagnostics.get('clipped') + ): + prior_rprs = restriction_reference_prior.get('rprs', prior.get('rprs') if isinstance(prior, dict) else np.nan) + try: + prior_rprs = float(prior_rprs) + except (TypeError, ValueError): + prior_rprs = np.nan + if np.isfinite(prior_rprs) and prior_rprs > 0: + original_fit = fit + original_bounds = clone_lightcurve_bounds(current_bounds) + original_rprs_value = (getattr(original_fit, 'parameters', {}) or {}).get('rprs', np.nan) + fixed_prior = dict(current_prior) + fixed_prior['rprs'] = prior_rprs + fixed_bounds = clone_lightcurve_bounds(current_bounds) + fixed_bounds.pop('rprs', None) + + prefit_uncertainty = estimate_rprs_data_uncertainty_from_lightcurve( + times, + flux_values, + flux_errors, + fixed_prior, + ) + data_rprs_uncertainty = prefit_uncertainty.get( + 'data_rprs_uncertainty', + restriction_reference_prior.get('rprs_data_uncertainty', np.nan), + ) + try: + data_rprs_uncertainty = float(data_rprs_uncertainty) + except (TypeError, ValueError): + data_rprs_uncertainty = np.nan + + fixed_error_override = {} + if np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: + fixed_error_override['rprs'] = data_rprs_uncertainty + + log_info( + "Rp/R* posterior is pinned against the " + f"{rprs_final_diagnostics.get('edge', 'active')} bound while Rp/R* " + "posterior expansion is disabled; rerunning UltraNest with Rp/R* fixed " + f"to the input prior ({prior_rprs:.6f}) and using a data-only Rp/R* uncertainty." + ) + fallback_fit = build_fit( + fixed_prior, + fixed_bounds, + fixed_parameter_errors_override=fixed_error_override, + ) + fallback_parameters = getattr(fallback_fit, 'parameters', None) + if isinstance(fallback_parameters, dict): + fallback_parameters['rprs'] = prior_rprs + fallback_errors = getattr(fallback_fit, 'errors', None) + if not isinstance(fallback_errors, dict): + fallback_fit.errors = {} + fallback_errors = fallback_fit.errors + + fallback_fit.rprs_prior_fallback_applied = True + fallback_fit.rprs_prior_fallback_prior_value = prior_rprs + fallback_fit.rprs_prior_fallback_original_fit_value = original_rprs_value + fallback_fit.rprs_prior_fallback_original_bounds = original_bounds.get('rprs') + fallback_fit.rprs_prior_fallback_edge = rprs_final_diagnostics.get('edge') + fallback_fit.rprs_prior_fallback_original_diagnostics = dict(rprs_final_diagnostics) + + empirical_uncertainty = fit_empirical_transit_uncertainty(fallback_fit) + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + fallback_fit.empirical_transit_uncertainty = empirical_uncertainty + empirical_data_uncertainty = empirical_uncertainty.get('data_rprs_uncertainty') + try: + empirical_data_uncertainty = float(empirical_data_uncertainty) + except (TypeError, ValueError): + empirical_data_uncertainty = np.nan + if np.isfinite(empirical_data_uncertainty) and empirical_data_uncertainty >= 0: + data_rprs_uncertainty = empirical_data_uncertainty + + if np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: + fallback_errors['rprs'] = data_rprs_uncertainty + fixed_parameter_errors_payload = getattr(fallback_fit, 'fixed_parameter_errors', None) + if not isinstance(fixed_parameter_errors_payload, dict): + fallback_fit.fixed_parameter_errors = {} + fixed_parameter_errors_payload = fallback_fit.fixed_parameter_errors + fixed_parameter_errors_payload['rprs'] = data_rprs_uncertainty + + fallback_note = ( + "Applied Rp/R* prior fallback; the sampled Rp/R* posterior hugged the " + f"{rprs_final_diagnostics.get('edge', 'active')} search bound while automatic " + "Rp/R* posterior expansion was disabled. EXOTIC reran UltraNest with Rp/R* fixed " + f"to the input prior ({prior_rprs:.6f}) and treats Tmid, a/Rs, and " + "impact parameter/inclination as the fitted transit-shape parameters. " + "The quoted Rp/R* uncertainty is a data-only red-noise estimate rather than a " + "model posterior uncertainty." + ) + if np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: + fallback_note += f" Data-only Rp/R* uncertainty: {data_rprs_uncertainty:.6f}." + fallback_fit.rprs_prior_fallback_data_uncertainty = data_rprs_uncertainty + fallback_fit.rprs_prior_fallback_note = fallback_note + retry_notes['rprs'] = fallback_note + current_prior = fixed_prior + current_bounds = fixed_bounds + fit = fallback_fit + final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + annotate_posterior_refit_final_bounds(fit, current_bounds) for config in retry_configs: key = config['key'] @@ -4951,16 +6576,29 @@ def psf_frame_quality_mask(psf_rows): return psf_frame_quality_components(psf_rows)['keep_mask'] -def psf_quality_mask_for_key(psf_data, key, frame_count): - if not isinstance(psf_data, dict) or key not in psf_data: +def psf_quality_rows_for_key(psf_data, key, psf_flux_data=None): + if isinstance(psf_flux_data, dict) and key in psf_flux_data: + return psf_flux_data[key] + if isinstance(psf_data, dict) and key in psf_data: + return psf_data[key] + return None + + +def psf_quality_mask_for_key(psf_data, key, frame_count, psf_flux_data=None): + rows = psf_quality_rows_for_key(psf_data, key, psf_flux_data=psf_flux_data) + if rows is None: return np.ones(int(frame_count), dtype=bool) - mask = psf_frame_quality_mask(psf_data[key]) + mask = psf_frame_quality_mask(rows) if mask.shape[0] != int(frame_count): return np.ones(int(frame_count), dtype=bool) return mask +def target_psf_quality_rows(psf_data, psf_flux_data=None): + return psf_quality_rows_for_key(psf_data, 'target', psf_flux_data=psf_flux_data) + + def mask_series_with_quality(values, quality_mask): masked = np.asarray(values, dtype=float).copy() quality_mask = np.asarray(quality_mask, dtype=bool) @@ -4978,7 +6616,7 @@ def psf_flux_series_from_rows(psf_rows, quality_mask=None): def psf_flux_data_source(psf_data, psf_flux_data=None): - if isinstance(psf_flux_data, dict): + if isinstance(psf_flux_data, dict) and 'target' in psf_flux_data: return psf_flux_data return psf_data @@ -5119,6 +6757,87 @@ def should_run_fast_ultranest_before_final_run(config_value): return FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT +def should_run_final_residual_rejection(config_value): + if config_value is None: + return FINAL_RESIDUAL_REJECTION_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'run final residual rejection and extra ultranest run' value; " + "defaulting to enabled.", + warn=True, + ) + return FINAL_RESIDUAL_REJECTION_DEFAULT + + +def should_use_legacy_psf_flux_mode(config_value): + if config_value is None: + return LEGACY_PSF_FLUX_MODE_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on', 'legacy'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_legacy_psf_flux' value; " + "defaulting to modern PSF flux mode.", + warn=True, + ) + return LEGACY_PSF_FLUX_MODE_DEFAULT + + +def should_run_final_fit_phase_residual_clip(config_value): + if config_value is None: + return FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'run_final_fit_phase_residual_clip' value; " + "defaulting to enabled.", + warn=True, + ) + return FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT + + +def psf_seed_track_directory_from_config(config_value): + if config_value is None: + return None + if isinstance(config_value, bool): + return None + if isinstance(config_value, (int, float)): + return None + if isinstance(config_value, str): + value = config_value.strip() + if value.lower() in ('', 'n', 'no', 'false', '0', 'off', 'none', 'null'): + return None + return value + return str(config_value) + + def configure_sparse_posterior_live_point_retry(config_value): enabled = should_use_sparse_posterior_live_point_retry(config_value) os.environ[SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV] = "1" if enabled else "0" @@ -5280,13 +6999,112 @@ def parse_rprs_search_bound_max(config_value): ) return RPRS_SEARCH_BOUND_ABSOLUTE_MAX - return float(max_bound) + return float(max_bound) + + +def configure_rprs_search_bound_max(config_value): + global RPRS_SEARCH_BOUND_MAX + RPRS_SEARCH_BOUND_MAX = parse_rprs_search_bound_max(config_value) + return RPRS_SEARCH_BOUND_MAX + + +def parse_bool_config_value(config_value, default, option_name): + if config_value is None: + return default + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + default_text = "enabled" if default else "disabled" + log_info( + f"Warning: Invalid '{option_name}' value; defaulting to {default_text}.", + warn=True, + ) + return default + + +def parse_range_restriction_percentage(config_value, default, option_name): + if config_value is None: + return default + if isinstance(config_value, str) and config_value.strip() == "": + return default + + try: + percentage = float(str(config_value).strip().rstrip('%')) + except (TypeError, ValueError): + log_info( + f"Warning: Invalid '{option_name}' value; defaulting to {default:.1f}%.", + warn=True, + ) + return default + + if not np.isfinite(percentage) or percentage <= 0: + log_info( + f"Warning: '{option_name}' must be finite and positive; defaulting to {default:.1f}%.", + warn=True, + ) + return default + + return float(percentage) + + +def should_restrict_rprs_range(config_value): + return parse_bool_config_value( + config_value, + RPRS_RANGE_RESTRICTION_DEFAULT, + 'restrict_Rp/Rs_range', + ) + + +def should_use_prior_rprs_when_posterior_pinned(config_value): + return parse_bool_config_value( + config_value, + RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR_DEFAULT, + 'use_prior_Rp/Rs_when_posterior_pinned', + ) + + +def should_restrict_ars_range(config_value): + return parse_bool_config_value( + config_value, + ARS_RANGE_RESTRICTION_DEFAULT, + 'restrict_a/Rs_range', + ) + + +def configure_rprs_range_restriction(enabled_value, percentage_value): + global RPRS_RANGE_RESTRICTION_ENABLED, RPRS_RANGE_RESTRICTION_PERCENTAGE + RPRS_RANGE_RESTRICTION_ENABLED = should_restrict_rprs_range(enabled_value) + RPRS_RANGE_RESTRICTION_PERCENTAGE = parse_range_restriction_percentage( + percentage_value, + RPRS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + 'restrict_Rp/Rs_range_percentage', + ) + return RPRS_RANGE_RESTRICTION_ENABLED, RPRS_RANGE_RESTRICTION_PERCENTAGE + +def configure_prior_rprs_fallback_on_pinned_posterior(config_value): + global RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR + RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR = should_use_prior_rprs_when_posterior_pinned(config_value) + return RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR -def configure_rprs_search_bound_max(config_value): - global RPRS_SEARCH_BOUND_MAX - RPRS_SEARCH_BOUND_MAX = parse_rprs_search_bound_max(config_value) - return RPRS_SEARCH_BOUND_MAX + +def configure_ars_range_restriction(enabled_value, percentage_value): + global ARS_RANGE_RESTRICTION_ENABLED, ARS_RANGE_RESTRICTION_PERCENTAGE + ARS_RANGE_RESTRICTION_ENABLED = should_restrict_ars_range(enabled_value) + ARS_RANGE_RESTRICTION_PERCENTAGE = parse_range_restriction_percentage( + percentage_value, + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + 'restrict_a/Rs_range_percentage', + ) + return ARS_RANGE_RESTRICTION_ENABLED, ARS_RANGE_RESTRICTION_PERCENTAGE def log_ultranest_mpi_status(): @@ -7163,6 +8981,15 @@ def prepare_final_fit_lightcurve_series( } +def normalize_out_of_transit_baseline_min_side_points(value): + if value is None: + return 0 + try: + return max(0, int(value)) + except (TypeError, ValueError): + return OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT + + def fit_airmass_baseline_parameters_on_out_of_transit( times, flux_values, @@ -7172,6 +8999,7 @@ def fit_airmass_baseline_parameters_on_out_of_transit( prior=None, bounds=None, depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, + min_side_points=OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT, ): times = np.asarray(times, dtype=float) flux_values = np.asarray(flux_values, dtype=float) @@ -7237,6 +9065,20 @@ def fit_airmass_baseline_parameters_on_out_of_transit( min_points = 3 if fit_a2 else 2 base_result['pre_points'] = coverage_summary.get('pre_points', 0) base_result['post_points'] = coverage_summary.get('post_points', 0) + min_side_points = normalize_out_of_transit_baseline_min_side_points(min_side_points) + if ( + min_side_points > 0 + and ( + base_result['pre_points'] <= min_side_points + or base_result['post_points'] <= min_side_points + ) + ): + base_result['note'] = ( + f"only {base_result['pre_points']} pre-ingress and " + f"{base_result['post_points']} post-egress out-of-transit point(s) were available; " + f"need more than {min_side_points} on each side to fit baseline parameters." + ) + return base_result if point_count < min_points: base_result['note'] = ( @@ -7372,6 +9214,7 @@ def detrend_flux_on_out_of_transit_baseline( fit, prior=None, depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, + min_side_points=OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT, ): times = np.asarray(times, dtype=float) flux_values = np.asarray(flux_values, dtype=float) @@ -7424,6 +9267,17 @@ def detrend_flux_on_out_of_transit_baseline( egress_time = coverage_summary['egress_time'] pre_points = coverage_summary['pre_points'] post_points = coverage_summary['post_points'] + min_side_points = normalize_out_of_transit_baseline_min_side_points(min_side_points) + if min_side_points > 0 and (pre_points <= min_side_points or post_points <= min_side_points): + return { + 'applied': False, + 'note': ( + f"only {pre_points} pre-ingress and {post_points} post-egress out-of-transit point(s) " + f"were available; need more than {min_side_points} on each side to fit a linear baseline." + ), + 'pre_points': pre_points, + 'post_points': post_points, + } x = times[oot_mask] - mid_transit if np.allclose(x, x[0]): @@ -7821,6 +9675,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( airmass_skip_note=None, disable_vertical_flux_normalization=False, detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT, use_impactparameter_rather_than_inclination_to_fit=True, plot_time_range=None, baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, @@ -7832,7 +9687,22 @@ def fit_final_lightcurve_with_oot_baseline_detrending( keep_ultranest_sampler_for_deferred_extension=False, fix_baseline_terms_for_final=True, pre_ultranest_coverage_assessment=None, + search_restriction_prior=None, ): + search_restriction_prior = ( + dict(search_restriction_prior) + if isinstance(search_restriction_prior, dict) + else dict(prior) if isinstance(prior, dict) else {} + ) + search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_restriction_prior, + times, + flux_values, + flux_errors, + prior, + context_label="final light curve", + ) + bounds = widen_rprs_bounds_to_data_uncertainty_window(bounds, search_restriction_prior) if duration_prior is None and expected_planet_dict is not None: duration_prior = build_single_transit_duration_prior(expected_planet_dict) if pre_ultranest_coverage_assessment is None: @@ -7871,6 +9741,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, ) fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) @@ -7923,6 +9794,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, ) fit = apply_plot_time_range(fit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(fit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -7959,6 +9831,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( fit, prior=working_prior, bounds=working_bounds, + min_side_points=oot_baseline_min_points_per_side, ) else: baseline_parameter_result = { @@ -8020,6 +9893,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): fixed_parameter_errors=baseline_fixed_errors, fixed_flux_baseline=True, pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -8078,6 +9952,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): working_unc, fit, prior=working_prior, + min_side_points=oot_baseline_min_points_per_side, ) if not detrend_result.get('applied'): note = f"Skipped; {detrend_result.get('note', 'unable to fit an out-of-transit baseline.')}" @@ -8125,6 +10000,18 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): "already flattened the final-fit flux baseline." ) + refit_search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_restriction_prior, + working_times, + detrend_result['flux'], + detrend_result['unc'], + refit_prior, + context_label="out-of-transit detrended final light curve", + ) + refit_bounds = widen_rprs_bounds_to_data_uncertainty_window( + refit_bounds, + refit_search_restriction_prior, + ) refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( working_times, detrend_result['flux'], @@ -8140,6 +10027,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): fixed_parameter_errors=refit_fixed_parameter_errors, fixed_flux_baseline=True, pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=refit_search_restriction_prior, ) refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) @@ -13405,6 +15293,196 @@ def fcn2min(pars): return flux_row +def fit_legacy_psf_photometry_flux_row(data, centroid_row, starIndex, psf_function=gaussian_psf, box=15): + try: + centroid_row = np.asarray(centroid_row, dtype=float).reshape(-1) + except (TypeError, ValueError): + return _nan_psf_result() + if centroid_row.size < 2 or not centroid_position_is_finite(centroid_row): + return _nan_psf_result() + + pos = centroid_row[:2] + try: + xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + init = [ + np.nanmax(subarray) - np.nanmin(subarray), + 1, + 1, + 0, + np.nanmin(subarray), + ] + except ValueError: + plateStatus.outOfFrameWarning(starIndex) + log.debug(f"Warning: empty subfield for legacy PSF flux fit at {np.round(pos, 2)}") + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + except Exception as exc: + log.debug(f"Legacy PSF flux setup failed at {np.round(pos, 2)} for star {starIndex}: {exc}") + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + try: + wx = np.sum(xv[0] * subarray.sum(0)) / subarray.sum(0).sum() + wy = np.sum(yv[:, 0] * subarray.sum(1)) / subarray.sum(1).sum() + except Exception: + wx, wy = pos[0], pos[1] + + lo = [ + pos[0] - box * 0.5, + pos[1] - box * 0.5, + 0, + 0.5, + 0.5, + -np.pi / 4, + np.nanmin(subarray) - 1, + ] + up = [ + pos[0] + box * 0.5, + pos[1] + box * 0.5, + 1e7, + 20, + 20, + np.pi / 4, + np.nanmax(subarray) + 1, + ] + + def fcn2min(pars): + model = psf_function(xv, yv, *pars) + return (subarray - model).flatten() + + try: + res = least_squares( + fcn2min, + x0=[*pos, *init], + bounds=[lo, up], + jac='3-point', + xtol=None, + method='trf', + ) + except Exception as exc: + plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) + log.debug( + f"Legacy PSF flux fit failed at {np.round(pos, 2)} for star {starIndex}; " + f"trying unbounded LM fallback: {exc}" + ) + try: + res = least_squares( + fcn2min, + x0=[*pos, *init], + jac='3-point', + xtol=None, + method='lm', + ) + except Exception as lm_exc: + log.debug( + f"Legacy PSF flux LM fallback failed at {np.round(pos, 2)} for star {starIndex}; " + f"using centroid fit flux parameters instead: {lm_exc}" + ) + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + flux_row = np.asarray(res.x, dtype=float) + if flux_row.size >= 2: + flux_row[0] = wx + flux_row[1] = wy + return flux_row + + +def _psf_seed_track_candidate_paths(seed_track_directory, key): + root = Path(seed_track_directory).expanduser() + search_dirs = [root] + if root.name.lower() != 'temp': + search_dirs.append(root / 'temp') + + if key == 'target': + names = ( + 'psf_data_target.txt', + 'psf_flux_data_target.txt', + ) + else: + comp_index = key[4:] if key.startswith('comp') else '' + names = ( + f'psf_data_{key}.txt', + f'psf_flux_data_{key}.txt', + f'psf_data_comp{comp_index}.txt' if comp_index else '', + f'psf_flux_data_comp{comp_index}.txt' if comp_index else '', + 'psf_data_comp.txt' if key == 'comp1' else '', + 'psf_flux_data_comp.txt' if key == 'comp1' else '', + ) + + for directory in search_dirs: + for name in names: + if not name: + continue + path = directory / name + if path.exists(): + return path + return None + + +def _load_psf_seed_track_file(path, expected_frame_count, key): + try: + rows = np.loadtxt(path, comments='#') + except Exception as exc: + log_info( + f"Warning: Could not load PSF seed track for {key} from {path}: {exc}", + warn=True, + ) + return None + + rows = np.asarray(rows, dtype=float) + if rows.ndim == 1: + rows = rows.reshape(1, -1) + + if rows.ndim != 2 or rows.shape[1] < 7: + log_info( + f"Warning: Ignoring PSF seed track for {key}; expected at least 7 columns in {path}.", + warn=True, + ) + return None + + if rows.shape[0] != int(expected_frame_count): + log_info( + f"Warning: Ignoring PSF seed track for {key}; {path} has {rows.shape[0]} row(s), " + f"but the reduction has {int(expected_frame_count)} frame(s).", + warn=True, + ) + return None + + return np.array(rows[:, :7], dtype=float, copy=True) + + +def load_psf_flux_seed_tracks(seed_track_directory, expected_frame_count, comp_alignment_keys): + seed_track_directory = psf_seed_track_directory_from_config(seed_track_directory) + if seed_track_directory is None: + return {} + + seed_tracks = {} + for key in ('target', *tuple(comp_alignment_keys or ())): + path = _psf_seed_track_candidate_paths(seed_track_directory, key) + if path is None: + log_info( + f"Warning: PSF seed track directory {seed_track_directory} did not contain a seed file for {key}.", + warn=True, + ) + continue + rows = _load_psf_seed_track_file(path, expected_frame_count, key) + if rows is not None: + seed_tracks[key] = rows + + if seed_tracks: + loaded_keys = ', '.join(seed_tracks) + log_info( + f"Loaded PSF flux seed track row(s) from {seed_track_directory} for: {loaded_keys}. " + "These rows will seed PSF flux fits only; alignment centroids remain unchanged." + ) + else: + log_info( + f"Warning: No usable PSF flux seed track rows were loaded from {seed_track_directory}.", + warn=True, + ) + + return seed_tracks + + def sigma_clipped_nanmedian(data, sigma=3.0, max_iters=3): clipped = np.array(data, dtype=float, copy=True) if clipped.size == 0: @@ -15039,10 +17117,19 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, prior['tmid'] = tmid_search_summary['tmid'] lower, upper = tmid_search_summary['bounds'] + search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + build_search_restriction_prior_from_planet_dict(pDict), + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + prior, + context_label="initial light curve", + ) mybounds = build_initial_transit_bounds( prior, [lower, upper], ars_unc=pDict.get('aRsUnc'), + rprs_data_uncertainty=search_restriction_prior.get('rprs_data_uncertainty'), ) apply_vertical_flux_normalization_bound( prior, @@ -15148,6 +17235,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, jd_times=arrayJDTimes, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, + search_restriction_prior=search_restriction_prior, ) annotate_pre_ultranest_transit_coverage(myfit, pre_ultranest_coverage_assessment) myfit = apply_plot_time_range(myfit, plot_time_range) @@ -15731,7 +17819,12 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, psf_comp_flux_map = { f"comp{comp_idx + 1}": psf_flux_series_from_rows( psf_flux_data[f"comp{comp_idx + 1}"], - psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count), + psf_quality_mask_for_key( + psf_data, + f"comp{comp_idx + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ), ) for comp_idx in range(comp_star_count) } @@ -15746,7 +17839,10 @@ def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, continue comp_flux = psf_comp_flux_map[ckey] - target_shape_mask = target_psf_shape_quality_mask(psf_data['target'], psf_data[ckey]) + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), + ) psf_mask = target_shape_mask & target_flux_mask & robust_flux_floor_mask(comp_flux) candidate_jobs.append({ 'method': 'psf', @@ -17090,7 +19186,12 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p comp_flux_map = { f"comp{comp_index + 1}": psf_flux_series_from_rows( psf_flux_data[f"comp{comp_index + 1}"], - psf_quality_mask_for_key(psf_data, f"comp{comp_index + 1}", frame_count), + psf_quality_mask_for_key( + psf_data, + f"comp{comp_index + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ), ) for comp_index in range(len(comp_stars)) } @@ -17124,7 +19225,10 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p comp_flux_series = comp_flux_map[ckey] if use_psf_photometry: - target_shape_mask = target_psf_shape_quality_mask(psf_data['target'], psf_data[ckey]) + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), + ) if target_shape_mask.shape[0] != frame_count: target_shape_mask = np.ones(frame_count, dtype=bool) candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) @@ -18161,10 +20265,19 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, return None frame_count = len(airmass) - psf_quality_masks = { + centroid_psf_quality_masks = { f"comp{comp_idx + 1}": psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count) for comp_idx in range(comp_star_count) } + psf_quality_masks = { + f"comp{comp_idx + 1}": psf_quality_mask_for_key( + psf_data, + f"comp{comp_idx + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ) + for comp_idx in range(comp_star_count) + } if use_psf_photometry: psf_flux_map = { @@ -18195,7 +20308,7 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, comp_flux_map = { f"comp{comp_idx + 1}": mask_series_with_quality( aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx], - psf_quality_masks[f"comp{comp_idx + 1}"], + centroid_psf_quality_masks[f"comp{comp_idx + 1}"], ) for comp_idx in range(comp_star_count) } @@ -18225,11 +20338,12 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, best_comp_index = best_candidate['best_comp_index'] method_label = comparison_method_label(best_candidate) comp_summaries = [] + best_quality_masks = psf_quality_masks if best_candidate.get('method') == 'psf' else centroid_psf_quality_masks for summary in best_candidate['comp_summaries']: comp_summary = dict(summary) comp_summary['position'] = comp_stars[comp_summary['comp_index']] comp_summary['selected'] = comp_summary['comp_index'] == best_comp_index - quality_mask = psf_quality_masks.get(comp_summary['key']) + quality_mask = best_quality_masks.get(comp_summary['key']) if quality_mask is not None: comp_summary['psf_quality_keep_mask'] = quality_mask comp_summary['psf_quality_rejected_count'] = int(np.count_nonzero(~quality_mask)) @@ -18288,6 +20402,9 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p fallback_sigma=np.nan, run_fast_ultranest_before_final_run= FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, + run_final_fit_phase_residual_clip= + FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT, + run_final_residual_rejection=FINAL_RESIDUAL_REJECTION_DEFAULT, save_dir=None, planet_name=None, observation_date=None): @@ -18349,14 +20466,27 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p comp_quality_mask = np.asarray( comp_summary.get( 'psf_quality_keep_mask', - psf_quality_mask_for_key(psf_data, ckey, frame_count), + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ), ), dtype=bool, ) if comp_quality_mask.shape[0] != frame_count: - comp_quality_mask = psf_quality_mask_for_key(psf_data, ckey, frame_count) + comp_quality_mask = psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ) if method == 'psf': - target_shape_mask = target_psf_shape_quality_mask(psf_data['target'], psf_data[ckey]) + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), + ) if target_shape_mask.shape[0] != frame_count: target_shape_mask = np.ones(frame_count, dtype=bool) candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) @@ -18472,6 +20602,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p baseline_duration_multiplier=final_fit_baseline_duration_multiplier, adaptive_summary=adaptive_summary, run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, + run_final_fit_phase_residual_clip=run_final_fit_phase_residual_clip, + run_final_residual_rejection=run_final_residual_rejection, precomputed_candidate_series=preflight.get('prepared_series'), ) fit_result = final_reduction.get('fit') if final_reduction.get('applied') else None @@ -18788,6 +20920,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p use_impactparameter_rather_than_inclination_to_fit, plot_time_range=plot_time_range, duration_prior=build_single_transit_duration_prior(p_dict), + run_final_residual_rejection=run_final_residual_rejection, ) if full_resolution_refit is not None: full_resolution_refit_applied = True @@ -19091,6 +21224,42 @@ def _main_impl(): FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, ) ) + run_final_residual_rejection = should_run_final_residual_rejection( + exotic_infoDict.get( + 'run_final_residual_rejection', + exotic_infoDict.get( + 'Run Final Residual Rejection and Extra UltraNest Run', + FINAL_RESIDUAL_REJECTION_DEFAULT, + ), + ) + ) + run_final_fit_phase_residual_clip = should_run_final_fit_phase_residual_clip( + exotic_infoDict.get( + 'run_final_fit_phase_residual_clip', + exotic_infoDict.get( + 'Run Final-Fit Phase Residual Clip? (y/n)', + FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT, + ), + ) + ) + use_legacy_psf_flux_mode = should_use_legacy_psf_flux_mode( + exotic_infoDict.get( + 'use_legacy_psf_flux', + exotic_infoDict.get( + 'legacy_psf_flux_mode', + LEGACY_PSF_FLUX_MODE_DEFAULT, + ), + ) + ) + psf_seed_track_directory = psf_seed_track_directory_from_config( + exotic_infoDict.get( + 'psf_seed_track_directory', + exotic_infoDict.get( + 'legacy_psf_seed_track_directory', + None, + ), + ) + ) ultranest_min_num_live_points = configure_ultranest_min_num_live_points( exotic_infoDict.get( 'ultranest_min_num_live_points', @@ -19103,8 +21272,59 @@ def _main_impl(): RPRS_SEARCH_BOUND_MAX_DEFAULT, ) ) + restrict_rprs_range, restrict_rprs_percentage = configure_rprs_range_restriction( + exotic_infoDict.get( + 'restrict_rprs_range', + RPRS_RANGE_RESTRICTION_DEFAULT, + ), + exotic_infoDict.get( + 'restrict_rprs_range_percentage', + RPRS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ), + ) + use_prior_rprs_fallback_on_pinned_posterior = configure_prior_rprs_fallback_on_pinned_posterior( + exotic_infoDict.get( + 'use_prior_rprs_when_posterior_pinned', + exotic_infoDict.get( + 'use_prior_Rp/Rs_when_posterior_pinned', + RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR_DEFAULT, + ), + ) + ) + restrict_ars_range, restrict_ars_percentage = configure_ars_range_restriction( + exotic_infoDict.get( + 'restrict_ars_range', + ARS_RANGE_RESTRICTION_DEFAULT, + ), + exotic_infoDict.get( + 'restrict_ars_range_percentage', + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ), + ) log_info(f"UltraNest minimum live points: {ultranest_min_num_live_points}.") log_info(f"Rp/R* maximum search bound: {rprs_search_bound_max:.3f}.") + if restrict_rprs_range: + log_info( + "Rp/R* prior-centered search restriction enabled: " + f"+/- {restrict_rprs_percentage:.1f}% around the input prior." + ) + else: + log_info("Rp/R* prior-centered search restriction disabled.") + if use_prior_rprs_fallback_on_pinned_posterior: + log_info( + "Rp/R* pinned-posterior prior fallback enabled: when Rp/R* expansion is disabled " + "and the posterior is edge-pinned, EXOTIC reruns with Rp/R* fixed to the input " + "prior and quotes a data-only Rp/R* uncertainty." + ) + else: + log_info("Rp/R* pinned-posterior prior fallback disabled per optional_info setting.") + if restrict_ars_range: + log_info( + "a/Rs prior-centered search restriction enabled: " + f"+/- {restrict_ars_percentage:.1f}% around the input prior." + ) + else: + log_info("a/Rs prior-centered search restriction disabled.") if run_fast_ultranest_before_final_run: log_info( "Fast pre-final UltraNest enabled: comparison-candidate UltraNest search runs " @@ -19113,6 +21333,28 @@ def _main_impl(): ) else: log_info("Fast pre-final UltraNest disabled per optional_info setting.") + if run_final_residual_rejection: + log_info( + "Final residual rejection enabled: the selected final light-curve fit will reject " + f"residual outliers beyond {FINAL_RESIDUAL_REJECTION_SIGMA:.1f} sigma and rerun UltraNest." + ) + else: + log_info("Final residual rejection disabled per optional_info setting.") + if run_final_fit_phase_residual_clip: + log_info("Final-fit phase residual clipping enabled.") + else: + log_info("Final-fit phase residual clipping disabled per optional_info setting.") + if use_legacy_psf_flux_mode: + log_info( + "Legacy PSF flux mode enabled: PSF photometry flux rows will use the 4.3.1-style " + "Gaussian fit with weighted-center override." + ) + else: + log_info("Modern PSF flux mode enabled.") + if psf_seed_track_directory is not None: + log_info( + f"PSF flux seed-track directory requested: {psf_seed_track_directory}" + ) use_sparse_posterior_live_point_retry = configure_sparse_posterior_live_point_retry( exotic_infoDict.get( 'use_sparse_posterior_live_point_retry', @@ -19763,6 +22005,11 @@ def _main_impl(): args.multiprocess_transformations is not None and args.multiprocess_transformations > 0 ) comp_alignment_keys = [f"comp{j + 1}" for j in range(comp_star_count)] + psf_flux_seed_tracks = load_psf_flux_seed_tracks( + psf_seed_track_directory, + len(inputfiles), + comp_alignment_keys, + ) if use_multiprocess_alignment: multiprocess_alignment_results = build_multiprocess_alignment_results( inputfiles, @@ -19904,15 +22151,26 @@ def _main_impl(): ) if use_psf_photometry: - psf_flux_data['target'][i] = fit_psf_photometry_flux_row( + psf_flux_row_fitter = ( + fit_legacy_psf_photometry_flux_row + if use_legacy_psf_flux_mode + else fit_psf_photometry_flux_row + ) + target_psf_flux_seed_row = psf_data['target'][i] + if 'target' in psf_flux_seed_tracks: + target_psf_flux_seed_row = psf_flux_seed_tracks['target'][i] + psf_flux_data['target'][i] = psf_flux_row_fitter( imageData, - psf_data['target'][i], + target_psf_flux_seed_row, 0, ) for comp_idx, comp_key in enumerate(comp_alignment_keys): - psf_flux_data[comp_key][i] = fit_psf_photometry_flux_row( + comp_psf_flux_seed_row = psf_data[comp_key][i] + if comp_key in psf_flux_seed_tracks: + comp_psf_flux_seed_row = psf_flux_seed_tracks[comp_key][i] + psf_flux_data[comp_key][i] = psf_flux_row_fitter( imageData, - psf_data[comp_key][i], + comp_psf_flux_seed_row, comp_idx + 1, ) @@ -20107,7 +22365,10 @@ def _main_impl(): aper_data[f"{ckey}_bg"] = aper_data[f"{ckey}_bg"][goodmask] psf_quality_diagnostics = [] - target_quality_components = target_psf_shape_quality_components(psf_data['target']) + if use_psf_photometry: + target_quality_components = target_psf_shape_quality_components(psf_flux_data['target']) + else: + target_quality_components = target_psf_shape_quality_components(psf_data['target']) target_quality_keep_mask = target_quality_components['keep_mask'] if target_quality_keep_mask.shape == times.shape and np.any(~target_quality_keep_mask): reason_parts = [] @@ -20133,7 +22394,12 @@ def _main_impl(): for key, label in [ (f"comp{j + 1}", f"Comp {j + 1}") for j in range(len(exotic_infoDict['comp_stars'])) ]: - quality_components = psf_frame_quality_components(psf_data[key]) + quality_rows = psf_quality_rows_for_key( + psf_data, + key, + psf_flux_data=psf_flux_data if use_psf_photometry else None, + ) + quality_components = psf_frame_quality_components(quality_rows) quality_keep_mask = quality_components['keep_mask'] if quality_keep_mask.shape == times.shape and np.any(~quality_keep_mask): reason_parts = [] @@ -20376,6 +22642,8 @@ def _main_impl(): adaptive_annulus_values=annulus_values, fallback_sigma=sigma_display, run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, + run_final_fit_phase_residual_clip=run_final_fit_phase_residual_clip, + run_final_residual_rejection=run_final_residual_rejection, save_dir=exotic_infoDict['save'], planet_name=pDict['pName'], observation_date=exotic_infoDict['date'], @@ -20751,7 +23019,11 @@ def _main_impl(): ndt = int(30. / 24. / 60. / dt) * 2 + 1 # ~30 minutes time_clip_mask = sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt, times=best_fit_lc.time[si]) phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) - if hasattr(best_fit_lc, 'residuals') and hasattr(best_fit_lc, 'phase'): + if ( + run_final_fit_phase_residual_clip + and hasattr(best_fit_lc, 'residuals') + and hasattr(best_fit_lc, 'phase') + ): phase_clip_mask = phase_bin_sigma_clip(best_fit_lc.residuals[si], best_fit_lc.phase[si], sigma=3, bins=10) adaptive_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) if use_adaptive_apertures and adaptive_summary is not None: @@ -21112,10 +23384,19 @@ def _main_impl(): "skipping airmass fitting and applying no airmass correction." ) + search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + build_search_restriction_prior_from_planet_dict(pDict), + goodTimes, + goodFluxes, + goodNormUnc, + prior, + context_label="final light curve", + ) mybounds = build_initial_transit_bounds( prior, [lower, upper], ars_unc=pDict.get('aRsUnc'), + rprs_data_uncertainty=search_restriction_prior.get('rprs_data_uncertainty'), ) apply_vertical_flux_normalization_bound( prior, @@ -21168,6 +23449,7 @@ def _main_impl(): expected_planet_dict=pDict, expected_tmid_search_summary=ephemeris_tmid_search_summary, eebls_search_summary=eebls_tmid_search_summary, + search_restriction_prior=search_restriction_prior, ) if ( reuse_selected_final_model @@ -21225,6 +23507,8 @@ def _main_impl(): durs.append(tmask.sum() * dt) plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + plot_prior_posterior_comparison(myfit, pDict, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + plot_ktmf_qc_metrics(myfit, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) if fitsortext == 1: observing_background_series = build_observing_background_series( @@ -21261,15 +23545,68 @@ def _main_impl(): log_info(f" Transit detection QC: {qc_status}") if np.isfinite(qc_ktmf_metric): log_info(f" KTMF: {qc_ktmf_metric:.2f} / 5.00") - log_info(f" Mid-Transit Time [BJD_TDB]: {round_to_2(myfit.parameters['tmid'], myfit.errors['tmid'])} +/- {round_to_2(myfit.errors['tmid'])}") - log_info(f" Radius Ratio (Planet/Star) [Rp/R*]: {round_to_2(myfit.parameters['rprs'], myfit.errors['rprs'])} +/- {round_to_2(myfit.errors['rprs'])}") - for depth_label, depth_text in formatted_transit_depth_parameters(myfit, pDict).items(): + empirical_uncertainty = getattr(myfit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(myfit) + rprs_report_error = _finite_float( + (empirical_uncertainty or {}).get('combined_rprs_uncertainty'), + default=myfit.errors['rprs'], + ) + if not np.isfinite(rprs_report_error) or rprs_report_error < 0: + rprs_report_error = myfit.errors['rprs'] + tmid_report_error = fit_parameter_model_data_uncertainty( + myfit, + 'tmid', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(tmid_report_error) or tmid_report_error < 0: + tmid_report_error = myfit.errors['tmid'] + inc_report_error = fit_parameter_model_data_uncertainty( + myfit, + 'inc', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(inc_report_error) or inc_report_error < 0: + inc_report_error = myfit.errors['inc'] + ars_report_error = fit_parameter_model_data_uncertainty( + myfit, + 'ars', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(ars_report_error) or ars_report_error < 0: + ars_report_error = myfit.errors.get('ars', np.nan) + log_info(f" Mid-Transit Time [BJD_TDB]: {round_to_2(myfit.parameters['tmid'], tmid_report_error)} +/- {round_to_2(tmid_report_error)}") + log_info(f" Radius Ratio (Planet/Star) [Rp/R*]: {round_to_2(myfit.parameters['rprs'], rprs_report_error)} +/- {round_to_2(rprs_report_error)}") + rprs_prior_fallback_note = getattr(myfit, 'rprs_prior_fallback_note', None) + if rprs_prior_fallback_note: + log_info(f" Rp/R* fallback note: {rprs_prior_fallback_note}") + for depth_label, depth_text in formatted_transit_depth_parameters( + myfit, + pDict, + empirical_uncertainty=empirical_uncertainty, + ).items(): log_info(f" {depth_label}: {depth_text}") - log_info(f" Orbital Inclination [inc]: {round_to_2(myfit.parameters['inc'], myfit.errors['inc'])} +/- {round_to_2(myfit.errors['inc'])}") - ars_text = format_parameter_with_error(myfit.parameters.get('ars'), myfit.errors.get('ars')) + log_info(f" Orbital Inclination [inc]: {round_to_2(myfit.parameters['inc'], inc_report_error)} +/- {round_to_2(inc_report_error)}") + ars_text = format_parameter_with_error(myfit.parameters.get('ars'), ars_report_error) if ars_text is not None: log_info(f" Ratio of Distance to Stellar Radius [a/Rs]: {ars_text}") - impact_parameter, impact_error = fit_impact_parameter_value_error(myfit) + impact_error_overrides = { + 'ars': ars_report_error, + 'inc': inc_report_error, + } + model_impact_parameter, model_impact_error = fit_impact_parameter_value_error(myfit) + if ( + np.isfinite(model_impact_error) + and ('b' in (getattr(myfit, 'parameters', {}) or {}) + or 'b' in (getattr(myfit, 'sample_parameters', {}) or {})) + ): + impact_error_overrides['b'] = model_impact_error * empirical_red_noise_error_scale( + empirical_uncertainty + ) + impact_parameter, impact_error = fit_impact_parameter_value_error( + myfit, + errors_override=impact_error_overrides, + ) impact_text = format_parameter_with_error(impact_parameter, impact_error) if impact_text is not None: log_info(f" Impact Parameter [b]: {impact_text}") diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 7df0e091..de4f1393 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -427,6 +427,9 @@ def save_input(): "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Maximum Rp/Rs Search Bound": "Set optional_info 'rprs_search_bound_max' to cap the nested-fit Rp/Rs search range. Default 0.5.", + "Restrict Rp/Rs Search Range": "Set optional_info 'restrict_Rp/Rs_range' to y to restrict Rp/Rs to a prior-centered percentage window. Set 'restrict_Rp/Rs_range_percentage' to control the half-width. Defaults y and 10.", + "Prior Rp/Rs Fallback For Pinned Posterior": "Set optional_info 'use_prior_Rp/Rs_when_posterior_pinned' to y to rerun a fit with Rp/Rs fixed to the input prior and quote a data-only Rp/Rs uncertainty when the Rp/Rs posterior is edge-pinned and automatic Rp/Rs expansion is disabled. Default y.", + "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -458,6 +461,11 @@ def save_input(): "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "rprs_search_bound_max": 0.5, + "restrict_Rp/Rs_range": "y", + "restrict_Rp/Rs_range_percentage": 10.0, + "use_prior_Rp/Rs_when_posterior_pinned": "y", + "restrict_a/Rs_range": "y", + "restrict_a/Rs_range_percentage": 10.0, "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1515,6 +1523,9 @@ def save_input(): "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Maximum Rp/Rs Search Bound": "Set optional_info 'rprs_search_bound_max' to cap the nested-fit Rp/Rs search range. Default 0.5.", + "Restrict Rp/Rs Search Range": "Set optional_info 'restrict_Rp/Rs_range' to y to restrict Rp/Rs to a prior-centered percentage window. Set 'restrict_Rp/Rs_range_percentage' to control the half-width. Defaults y and 10.", + "Prior Rp/Rs Fallback For Pinned Posterior": "Set optional_info 'use_prior_Rp/Rs_when_posterior_pinned' to y to rerun a fit with Rp/Rs fixed to the input prior and quote a data-only Rp/Rs uncertainty when the Rp/Rs posterior is edge-pinned and automatic Rp/Rs expansion is disabled. Default y.", + "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", @@ -1594,6 +1605,11 @@ def save_input(): "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "rprs_search_bound_max": 0.5, + "restrict_Rp/Rs_range": "y", + "restrict_Rp/Rs_range_percentage": 10.0, + "use_prior_Rp/Rs_when_posterior_pinned": "y", + "restrict_a/Rs_range": "y", + "restrict_a/Rs_range_percentage": 10.0, "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", @@ -1651,6 +1667,7 @@ def save_input(): "use_eebls_to_initialize_tmid_and_bounds": "y", "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", + "use_prior_Rp/Rs_when_posterior_pinned": "y", "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/exotic/inputs.py b/exotic/inputs.py index dc88682e..e3efb2a3 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -215,11 +215,14 @@ def __init__(self, init_opt): 'pick_comparison_by_eebls_snr': 'y', 'use_deviation_from_expected_transit_in_qc': True, 'deviation_from_expected_transit_in_qc_sigma': 5.0, + 'run_final_fit_phase_residual_clip': 'y', 'exit_at_first_qc_pass_solution': 'y', 'detect_bad_pixels_before_photometry': 'n', 'multiprocess_bad_pixel_precheck': 'n', 'use_impactparameter_rather_than_inclination_to_fit': 'y', 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', + 'use_legacy_psf_flux': 'n', + 'psf_seed_track_directory': None, 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, 'use_aperture_corrections_and_full_image_fwhm': False, 'pointing_rejection_sigma': None, @@ -227,6 +230,11 @@ def __init__(self, init_opt): 'fit_lightcurve_to_every_comparison_candidate': 'n', 'ultranest_min_num_live_points': 200, 'rprs_search_bound_max': 0.5, + 'restrict_rprs_range': 'y', + 'restrict_rprs_range_percentage': 10.0, + 'use_prior_rprs_when_posterior_pinned': 'y', + 'restrict_ars_range': 'y', + 'restrict_ars_range_percentage': 10.0, 'use_sparse_posterior_live_point_retry': 'y', } self.params = { @@ -494,6 +502,11 @@ def comp_params(self, init_file, planet_dict): 'deviation_from_expected_transit_in_qc_sigma', 'Deviation From Expected Transit In QC Sigma', ), + 'run_final_fit_phase_residual_clip': ( + 'run_final_fit_phase_residual_clip', + 'Run Final-Fit Phase Residual Clip? (y/n)', + 'Run Final Fit Phase Residual Clip? (y/n)', + ), 'exit_at_first_qc_pass_solution': ( 'exit_at_first_qc_pass_solution', 'exit at first QC PASS solution', @@ -510,6 +523,18 @@ def comp_params(self, init_file, planet_dict): 'use_psf_photometry', 'Use PSF Photometry? (y/n)', ), + 'use_legacy_psf_flux': ( + 'use_legacy_psf_flux', + 'legacy_psf_flux_mode', + 'Use Legacy PSF Flux? (y/n)', + 'Use Legacy PSF Flux Mode? (y/n)', + ), + 'psf_seed_track_directory': ( + 'psf_seed_track_directory', + 'legacy_psf_seed_track_directory', + 'PSF Seed Track Directory', + 'Legacy PSF Seed Track Directory', + ), 'use_aperture_photometry': ( 'use_aperture_photometry', 'Use Aperture Photometry? (y/n)', @@ -553,6 +578,46 @@ def comp_params(self, init_file, planet_dict): 'Maximum Rp/Rs Search Bound', 'Maximum Rp/R* Search Bound', ), + 'restrict_rprs_range': ( + 'restrict_Rp/Rs_range', + 'restrict_Rp/R*_range', + 'restrict_rprs_range', + 'restrict_RpRs_range', + 'Restrict Rp/Rs Range? (y/n)', + 'Restrict Rp/R* Range? (y/n)', + ), + 'restrict_rprs_range_percentage': ( + 'restrict_Rp/Rs_range_percentage', + 'restrict_Rp/R*_range_percentage', + 'restrict_rprs_range_percentage', + 'restrict_RpRs_range_percentage', + 'Restrict Rp/Rs Range Percentage', + 'Restrict Rp/R* Range Percentage', + ), + 'use_prior_rprs_when_posterior_pinned': ( + 'use_prior_Rp/Rs_when_posterior_pinned', + 'use_prior_Rp/R*_when_posterior_pinned', + 'use_prior_rprs_when_posterior_pinned', + 'use_prior_RpRs_when_posterior_pinned', + 'Use Prior Rp/Rs When Posterior Pinned? (y/n)', + 'Use Prior Rp/R* When Posterior Pinned? (y/n)', + ), + 'restrict_ars_range': ( + 'restrict_a/Rs_range', + 'restrict_a/R*_range', + 'restrict_ars_range', + 'restrict_aRs_range', + 'Restrict a/Rs Range? (y/n)', + 'Restrict a/R* Range? (y/n)', + ), + 'restrict_ars_range_percentage': ( + 'restrict_a/Rs_range_percentage', + 'restrict_a/R*_range_percentage', + 'restrict_ars_range_percentage', + 'restrict_aRs_range_percentage', + 'Restrict a/Rs Range Percentage', + 'Restrict a/R* Range Percentage', + ), 'use_sparse_posterior_live_point_retry': ( 'use_sparse_posterior_live_point_retry', 'Use Sparse Posterior Live-Point Retry? (y/n)', diff --git a/exotic/output_files.py b/exotic/output_files.py index 8e5f979b..a010879b 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -412,6 +412,258 @@ def build_fit_quality_metadata(fit): return payload +def red_noise_beta_factor(residual_fraction, coordinates=None, min_bin_size=2, max_bin_size=None): + residual_fraction = np.asarray(residual_fraction, dtype=float) + finite_mask = np.isfinite(residual_fraction) + + coordinates_array = None + if coordinates is not None: + coordinates_array = np.asarray(coordinates, dtype=float) + if coordinates_array.shape == residual_fraction.shape: + finite_mask &= np.isfinite(coordinates_array) + else: + coordinates_array = None + + residual_fraction = residual_fraction[finite_mask] + if coordinates_array is not None: + coordinates_array = coordinates_array[finite_mask] + sort_index = np.argsort(coordinates_array) + residual_fraction = residual_fraction[sort_index] + + point_count = int(residual_fraction.size) + payload = { + 'factor': 1.0, + 'point_count': point_count, + 'bin_sizes': [], + 'beta_by_bin': {}, + 'max_bin_size': np.nan, + } + min_bin_size = int(max(2, finite_float(min_bin_size, 2))) + if point_count < min_bin_size * 2: + return payload + + residual_fraction = residual_fraction - np.nanmedian(residual_fraction) + unbinned_rms = finite_float(np.nanstd(residual_fraction, ddof=1)) + if not np.isfinite(unbinned_rms) or unbinned_rms <= 0: + return payload + + if max_bin_size is None: + max_bin_size = min(10, max(min_bin_size, point_count // 4)) + max_bin_size = int(max(min_bin_size, finite_float(max_bin_size, min_bin_size))) + max_bin_size = min(max_bin_size, point_count // 2) + if max_bin_size < min_bin_size: + return payload + + beta_values = [] + for bin_size in range(min_bin_size, max_bin_size + 1): + bin_count = point_count // bin_size + if bin_count < 2: + continue + trimmed = residual_fraction[:bin_count * bin_size] + binned_means = np.nanmean(trimmed.reshape(bin_count, bin_size), axis=1) + binned_rms = finite_float(np.nanstd(binned_means, ddof=1)) + expected_rms = ( + unbinned_rms + / np.sqrt(bin_size) + * np.sqrt(bin_count / (bin_count - 1.0)) + ) + if not np.isfinite(binned_rms) or not np.isfinite(expected_rms) or expected_rms <= 0: + continue + beta = max(1.0, float(binned_rms / expected_rms)) + payload['bin_sizes'].append(bin_size) + payload['beta_by_bin'][str(bin_size)] = beta + beta_values.append(beta) + + if beta_values: + payload['factor'] = float(max(beta_values)) + payload['max_bin_size'] = max(payload['bin_sizes']) + return payload + + +def _fit_uncertainty_time_coordinates(fit, expected_shape): + for name in ('time', 'phase'): + values = getattr(fit, name, None) + if values is None: + continue + try: + values = np.asarray(values, dtype=float) + except (TypeError, ValueError): + continue + if values.shape == expected_shape: + return values + return None + + +def fit_empirical_transit_uncertainty(fit, fit_quality=None, transit_depth_threshold_fraction=0.05): + data, model, _ = fit_data_model_uncertainty(fit) + if data is None or model is None: + return {} + + data = np.asarray(data, dtype=float) + model = np.asarray(model, dtype=float) + if data.shape != model.shape or data.ndim != 1: + return {} + + residuals = data - model + finite_mask = np.isfinite(data) & np.isfinite(model) & np.isfinite(residuals) + if not np.any(finite_mask): + return {} + + fit_quality = fit_quality or {} + median_flux = np.nanmedian(data[finite_mask]) + residual_scatter = finite_float(fit_quality.get('residual_scatter')) + if not np.isfinite(residual_scatter): + if np.isfinite(median_flux) and median_flux != 0: + residual_scatter = float(np.nanstd(residuals[finite_mask]) / median_flux) + if not np.isfinite(residual_scatter) or residual_scatter < 0: + return {} + + transit = np.asarray(getattr(fit, 'transit', np.array([])), dtype=float) + if transit.shape != data.shape: + return {} + + transit_mask = finite_mask & np.isfinite(transit) + if np.count_nonzero(transit_mask) < 2: + return {} + + transit_values = transit[transit_mask] + baseline = finite_float(np.nanpercentile(transit_values, 95)) + if not np.isfinite(baseline): + return {} + + transit_depth_profile = baseline - transit + max_depth = finite_float(np.nanmax(transit_depth_profile[transit_mask])) + if not np.isfinite(max_depth) or max_depth <= 0: + return {} + + threshold_fraction = finite_float(transit_depth_threshold_fraction, 0.05) + if not np.isfinite(threshold_fraction) or threshold_fraction <= 0: + threshold_fraction = 0.05 + depth_threshold = max_depth * threshold_fraction + in_transit_mask = transit_mask & (transit_depth_profile >= depth_threshold) + out_of_transit_mask = transit_mask & (transit_depth_profile < depth_threshold) + + in_transit_count = int(np.count_nonzero(in_transit_mask)) + out_of_transit_count = int(np.count_nonzero(out_of_transit_mask)) + if in_transit_count <= 0: + return {} + + sample_term = 1.0 / in_transit_count + if out_of_transit_count > 0: + sample_term += 1.0 / out_of_transit_count + + depth_standard_error_fraction = float(residual_scatter * np.sqrt(sample_term)) + if not np.isfinite(depth_standard_error_fraction) or depth_standard_error_fraction < 0: + return {} + if out_of_transit_count > 0: + baseline_standard_error_fraction = float(residual_scatter / np.sqrt(out_of_transit_count)) + else: + baseline_standard_error_fraction = float(depth_standard_error_fraction) + + residual_fraction = residuals + if np.isfinite(median_flux) and median_flux != 0: + residual_fraction = residuals / median_flux + coordinates = _fit_uncertainty_time_coordinates(fit, data.shape) + max_beta_bin_size = min(10, max(2, in_transit_count // 4)) + beta_payload = red_noise_beta_factor( + residual_fraction[finite_mask], + coordinates=coordinates[finite_mask] if coordinates is not None else None, + min_bin_size=2, + max_bin_size=max_beta_bin_size, + ) + red_noise_beta = finite_float(beta_payload.get('factor'), 1.0) + if not np.isfinite(red_noise_beta) or red_noise_beta < 1.0: + red_noise_beta = 1.0 + depth_uncertainty_fraction = float(depth_standard_error_fraction * red_noise_beta) + baseline_uncertainty_fraction = float(baseline_standard_error_fraction * red_noise_beta) + depth_flux_scatter_fraction = float(residual_scatter) + + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + rprs = finite_float(parameters.get('rprs')) + model_rprs_uncertainty = finite_float(errors.get('rprs')) + rprs_prior_fallback = bool(getattr(fit, 'rprs_prior_fallback_applied', False)) + if rprs_prior_fallback: + model_rprs_uncertainty = np.nan + data_rprs_uncertainty = np.nan + data_rprs_standard_error = np.nan + data_rprs_flux_scatter_uncertainty = np.nan + combined_rprs_uncertainty = np.nan + combined_rprs_standard_error = np.nan + conservative_rprs_uncertainty = np.nan + if np.isfinite(rprs) and rprs > 0: + data_rprs_uncertainty = float(depth_uncertainty_fraction / (2.0 * rprs)) + data_rprs_standard_error = float(depth_standard_error_fraction / (2.0 * rprs)) + data_rprs_flux_scatter_uncertainty = float(depth_flux_scatter_fraction / (2.0 * rprs)) + if rprs_prior_fallback: + combined_rprs_uncertainty = data_rprs_uncertainty + combined_rprs_standard_error = data_rprs_standard_error + conservative_rprs_uncertainty = data_rprs_uncertainty + elif np.isfinite(model_rprs_uncertainty) and model_rprs_uncertainty >= 0: + combined_rprs_uncertainty = float( + np.sqrt(model_rprs_uncertainty ** 2 + data_rprs_uncertainty ** 2) + ) + combined_rprs_standard_error = float( + np.sqrt(model_rprs_uncertainty ** 2 + data_rprs_standard_error ** 2) + ) + conservative_rprs_uncertainty = float( + max(model_rprs_uncertainty, data_rprs_uncertainty) + ) + + return { + 'available': True, + 'residual_scatter': residual_scatter, + 'residual_scatter_percent': residual_scatter * 100.0, + 'in_transit_point_count': in_transit_count, + 'out_of_transit_point_count': out_of_transit_count, + 'transit_depth_threshold_fraction': threshold_fraction, + 'model_depth_fraction': max_depth, + 'model_depth_percent': max_depth * 100.0, + 'depth_uncertainty_fraction': depth_uncertainty_fraction, + 'depth_uncertainty_percent': depth_uncertainty_fraction * 100.0, + 'depth_red_noise_uncertainty_fraction': depth_uncertainty_fraction, + 'depth_red_noise_uncertainty_percent': depth_uncertainty_fraction * 100.0, + 'baseline_standard_error_fraction': baseline_standard_error_fraction, + 'baseline_standard_error_percent': baseline_standard_error_fraction * 100.0, + 'baseline_red_noise_uncertainty_fraction': baseline_uncertainty_fraction, + 'baseline_red_noise_uncertainty_percent': baseline_uncertainty_fraction * 100.0, + 'depth_flux_scatter_fraction': depth_flux_scatter_fraction, + 'depth_flux_scatter_percent': depth_flux_scatter_fraction * 100.0, + 'depth_standard_error_fraction': depth_standard_error_fraction, + 'depth_standard_error_percent': depth_standard_error_fraction * 100.0, + 'red_noise_beta_factor': red_noise_beta, + 'red_noise_beta_bin_sizes': beta_payload.get('bin_sizes', []), + 'red_noise_beta_by_bin': beta_payload.get('beta_by_bin', {}), + 'red_noise_beta_max_bin_size': beta_payload.get('max_bin_size', np.nan), + 'rprs': rprs, + 'model_rprs_uncertainty': model_rprs_uncertainty, + 'data_rprs_uncertainty': data_rprs_uncertainty, + 'data_rprs_red_noise_uncertainty': data_rprs_uncertainty, + 'data_rprs_standard_error': data_rprs_standard_error, + 'data_rprs_flux_scatter_uncertainty': data_rprs_flux_scatter_uncertainty, + 'combined_rprs_uncertainty': combined_rprs_uncertainty, + 'combined_rprs_red_noise_uncertainty': combined_rprs_uncertainty, + 'combined_rprs_standard_error': combined_rprs_standard_error, + 'conservative_rprs_uncertainty': conservative_rprs_uncertainty, + 'rprs_uncertainty_basis': ( + 'prior_assumed_data_only' + if rprs_prior_fallback + else 'model_plus_red_noise' + ), + 'rprs_prior_fallback_applied': rprs_prior_fallback, + 'rprs_prior_fallback_prior_value': finite_float( + getattr(fit, 'rprs_prior_fallback_prior_value', np.nan) + ), + 'rprs_prior_fallback_original_fit_value': finite_float( + getattr(fit, 'rprs_prior_fallback_original_fit_value', np.nan) + ), + 'rprs_prior_fallback_data_uncertainty': finite_float( + getattr(fit, 'rprs_prior_fallback_data_uncertainty', np.nan) + ), + 'rprs_prior_fallback_note': getattr(fit, 'rprs_prior_fallback_note', None), + } + + def photometry_method_from_info(photometry_info): if not isinstance(photometry_info, dict): return None @@ -437,13 +689,21 @@ def build_aavso_qc_metadata(fit): 'transit_chi2', 'flat_chi2', 'delta_chi2', 'transit_bic', 'flat_bic', 'delta_bic', 'transit_parameter_count', 'flat_parameter_count', 'flat_baseline', 'flat_a2', 'flat_model_note', 'residual_scatter', + 'transit_depth_for_residual_scatter', 'residual_scatter_to_depth_ratio', 'rprs_sigma', 'duration_ratio', 'eebls_depth_snr', + 'sampling_score', 'sampling_detail', 'sampling_ingress_count', + 'sampling_egress_count', 'sampling_in_transit_count', + 'sampling_pre_baseline_count', 'sampling_post_baseline_count', + 'sampling_total_duration', 'sampling_ingress_duration', 'use_deviation_from_expected_transit_in_qc', 'deviation_sigma_threshold', 'expected_tmid', 'expected_tmid_unc', 'fitted_tmid', 'expected_rprs', 'expected_rprs_unc', 'fitted_rprs', 'fitted_rprs_unc', - 'rprs_deviation_fit_unc', 'rprs_deviation_expected_unc', + 'rprs_deviation_fit_unc', 'rprs_deviation_model_fit_unc', + 'rprs_deviation_data_fit_unc', 'rprs_deviation_combined_fit_unc', + 'rprs_deviation_expected_unc', 'rprs_deviation_systematic_floor', 'rprs_deviation_unc', 'rprs_deviation_sigma', 'rprs_deviation_score', + 'rprs_prior_assumed', 'rprs_prior_assumed_note', 'deviation_from_expected_value', 'ktmf_metric', 'ktmf_contributions', 'notes', ) @@ -461,6 +721,7 @@ def compact_ktmf_contributions(contributions): 'points': contribution.get('points'), 'max_points': contribution.get('max_points'), 'score': contribution.get('score'), + 'score_uncertainty': contribution.get('score_uncertainty'), 'detail': contribution.get('detail'), }) return compact @@ -520,7 +781,21 @@ def build_ktmf_decision_metadata(fit, photometry_info=None): 'delta_chi2': transit_qc.get('delta_chi2'), 'eebls_depth_snr': transit_qc.get('eebls_depth_snr'), 'residual_scatter': transit_qc.get('residual_scatter'), + 'transit_depth_for_residual_scatter': transit_qc.get('transit_depth_for_residual_scatter'), + 'residual_scatter_to_depth_ratio': transit_qc.get('residual_scatter_to_depth_ratio'), + 'sampling_score': transit_qc.get('sampling_score'), + 'sampling_detail': transit_qc.get('sampling_detail'), + 'sampling_ingress_count': transit_qc.get('sampling_ingress_count'), + 'sampling_egress_count': transit_qc.get('sampling_egress_count'), + 'sampling_in_transit_count': transit_qc.get('sampling_in_transit_count'), + 'sampling_pre_baseline_count': transit_qc.get('sampling_pre_baseline_count'), + 'sampling_post_baseline_count': transit_qc.get('sampling_post_baseline_count'), 'deviation_from_expected_value': transit_qc.get('deviation_from_expected_value'), + 'rprs_deviation_fit_unc': transit_qc.get('rprs_deviation_fit_unc'), + 'rprs_deviation_model_fit_unc': transit_qc.get('rprs_deviation_model_fit_unc'), + 'rprs_deviation_data_fit_unc': transit_qc.get('rprs_deviation_data_fit_unc'), + 'rprs_deviation_expected_unc': transit_qc.get('rprs_deviation_expected_unc'), + 'rprs_deviation_unc': transit_qc.get('rprs_deviation_unc'), } if isinstance(photometry_info, dict): @@ -563,6 +838,17 @@ def format_ktmf_metric(value): return f"{value:.2f} / 5.00" if np.isfinite(value) else "n/a" +def format_ktmf_status(value): + value = finite_float(value) + if not np.isfinite(value): + return "UNKNOWN" + if value >= 4.0: + return "PASS" + if value >= 3.0: + return "MARGINAL" + return "FAIL" + + def format_transit_qc_headline_final_params(transit_qc): params = {} if not isinstance(transit_qc, dict) or not transit_qc: @@ -617,7 +903,7 @@ def format_ktmf_decision_final_params(fit, photometry_info=None): if isinstance(transit_qc, dict): ktmf_metric = finite_float(transit_qc.get('ktmf_metric')) if np.isfinite(ktmf_metric): - target_status = str(transit_qc.get('status', 'unknown')).upper() + target_status = format_ktmf_status(ktmf_metric) params["KTMF target-fit decision"] = ( f"{target_status}: KTMF={format_ktmf_metric(ktmf_metric)}" ) @@ -744,6 +1030,177 @@ def format_fit_quality_final_params(fit_quality): return params +def format_empirical_transit_uncertainty_final_params(empirical_uncertainty): + empirical_uncertainty = empirical_uncertainty or {} + if not empirical_uncertainty.get('available'): + return {} + + params = {} + rprs = finite_float(empirical_uncertainty.get('rprs')) + model_rprs_uncertainty = finite_float(empirical_uncertainty.get('model_rprs_uncertainty')) + data_rprs_uncertainty = finite_float(empirical_uncertainty.get('data_rprs_uncertainty')) + combined_rprs_uncertainty = finite_float(empirical_uncertainty.get('combined_rprs_uncertainty')) + conservative_rprs_uncertainty = finite_float( + empirical_uncertainty.get('conservative_rprs_uncertainty') + ) + data_rprs_standard_error = finite_float(empirical_uncertainty.get('data_rprs_standard_error')) + data_rprs_flux_scatter_uncertainty = finite_float( + empirical_uncertainty.get('data_rprs_flux_scatter_uncertainty') + ) + combined_rprs_standard_error = finite_float( + empirical_uncertainty.get('combined_rprs_standard_error') + ) + depth_uncertainty_percent = finite_float(empirical_uncertainty.get('depth_uncertainty_percent')) + depth_flux_scatter_percent = finite_float( + empirical_uncertainty.get('depth_flux_scatter_percent') + ) + depth_standard_error_percent = finite_float( + empirical_uncertainty.get('depth_standard_error_percent') + ) + baseline_red_noise_percent = finite_float( + empirical_uncertainty.get('baseline_red_noise_uncertainty_percent') + ) + baseline_standard_error_percent = finite_float( + empirical_uncertainty.get('baseline_standard_error_percent') + ) + residual_scatter_percent = finite_float(empirical_uncertainty.get('residual_scatter_percent')) + red_noise_beta = finite_float(empirical_uncertainty.get('red_noise_beta_factor')) + rprs_prior_fallback = bool(empirical_uncertainty.get('rprs_prior_fallback_applied')) + rprs_uncertainty_basis = empirical_uncertainty.get('rprs_uncertainty_basis') + + if ( + not rprs_prior_fallback + and np.isfinite(rprs) + and np.isfinite(model_rprs_uncertainty) + and model_rprs_uncertainty >= 0 + ): + params["Ratio of Planet to Stellar Radius (Rp/R*) model-fit uncertainty"] = ( + f"{round_to_2(rprs, model_rprs_uncertainty)} +/- {round_to_2(model_rprs_uncertainty)}" + ) + if np.isfinite(rprs) and np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: + if rprs_prior_fallback: + params["Ratio of Planet to Stellar Radius (Rp/R*) prior-assumed data-only uncertainty"] = ( + f"{round_to_2(rprs, data_rprs_uncertainty)} +/- {round_to_2(data_rprs_uncertainty)}" + ) + else: + params["Ratio of Planet to Stellar Radius (Rp/R*) data-fit red-noise uncertainty"] = ( + f"{round_to_2(rprs, data_rprs_uncertainty)} +/- {round_to_2(data_rprs_uncertainty)}" + ) + if np.isfinite(rprs) and np.isfinite(combined_rprs_uncertainty) and combined_rprs_uncertainty >= 0: + if rprs_prior_fallback: + params["Ratio of Planet to Stellar Radius (Rp/R*) data-only uncertainty used for primary value"] = ( + f"{round_to_2(rprs, combined_rprs_uncertainty)} +/- " + f"{round_to_2(combined_rprs_uncertainty)}" + ) + else: + params["Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty"] = ( + f"{round_to_2(rprs, combined_rprs_uncertainty)} +/- " + f"{round_to_2(combined_rprs_uncertainty)}" + ) + if np.isfinite(conservative_rprs_uncertainty): + params["Conservative Rp/R* uncertainty to quote"] = ( + f"+/- {round_to_2(conservative_rprs_uncertainty)}" + ) + if np.isfinite(rprs) and np.isfinite(data_rprs_standard_error) and data_rprs_standard_error >= 0: + params["Ratio of Planet to Stellar Radius (Rp/R*) data-fit standard-error estimate"] = ( + f"{round_to_2(rprs, data_rprs_standard_error)} +/- {round_to_2(data_rprs_standard_error)}" + ) + if ( + not rprs_prior_fallback + and np.isfinite(rprs) + and np.isfinite(combined_rprs_standard_error) + and combined_rprs_standard_error >= 0 + ): + params["Ratio of Planet to Stellar Radius (Rp/R*) model+standard-error estimate"] = ( + f"{round_to_2(rprs, combined_rprs_standard_error)} +/- " + f"{round_to_2(combined_rprs_standard_error)}" + ) + if ( + np.isfinite(rprs) + and np.isfinite(data_rprs_flux_scatter_uncertainty) + and data_rprs_flux_scatter_uncertainty >= 0 + ): + params["Ratio of Planet to Stellar Radius (Rp/R*) flux-scatter equivalent"] = ( + f"{round_to_2(rprs, data_rprs_flux_scatter_uncertainty)} +/- " + f"{round_to_2(data_rprs_flux_scatter_uncertainty)}" + ) + if np.isfinite(depth_uncertainty_percent): + params["Transit depth red-noise uncertainty"] = ( + f"+/- {depth_uncertainty_percent:.4f} %" + ) + if np.isfinite(depth_flux_scatter_percent): + params["Transit depth flux-scatter equivalent"] = ( + f"+/- {depth_flux_scatter_percent:.4f} %" + ) + if np.isfinite(depth_standard_error_percent): + params["Transit depth data-fit standard-error estimate"] = ( + f"+/- {depth_standard_error_percent:.4f} %" + ) + if np.isfinite(baseline_red_noise_percent): + params["Flux baseline red-noise uncertainty"] = ( + f"+/- {baseline_red_noise_percent:.4f} %" + ) + if np.isfinite(baseline_standard_error_percent): + params["Flux baseline standard-error estimate"] = ( + f"+/- {baseline_standard_error_percent:.4f} %" + ) + if np.isfinite(red_noise_beta): + params["Red-noise beta factor"] = f"{red_noise_beta:.3f}" + if rprs_uncertainty_basis: + params["Rp/R* uncertainty basis"] = str(rprs_uncertainty_basis) + fallback_note = empirical_uncertainty.get('rprs_prior_fallback_note') + if fallback_note: + params["Rp/R* prior fallback note"] = str(fallback_note) + + beta_bins = empirical_uncertainty.get('red_noise_beta_bin_sizes') + if beta_bins: + params["Red-noise beta bin sizes"] = ", ".join(str(int(item)) for item in beta_bins) + + in_count = empirical_uncertainty.get('in_transit_point_count') + out_count = empirical_uncertainty.get('out_of_transit_point_count') + try: + in_count = int(in_count) + out_count = int(out_count) + except (TypeError, ValueError): + in_count = None + out_count = None + if in_count is not None and out_count is not None: + params["Data-fit uncertainty point counts"] = ( + f"{in_count} in transit, {out_count} out of transit" + ) + + if np.isfinite(residual_scatter_percent): + params["Flux residual scatter around model"] = ( + f"{residual_scatter_percent:.4f} %" + ) + if rprs_prior_fallback: + params["Uncertainty interpretation note"] = ( + "The primary Rp/R* and radius-ratio area-depth uncertainties use the input " + "prior Rp/R* value with a data-only red-noise uncertainty because the sampled " + "Rp/R* posterior was pinned against a search bound and automatic Rp/R* " + "posterior expansion was disabled. Tmid, a/Rs, inclination, and impact " + "parameter remain fitted parameters; their primary uncertainties apply the " + "same residual time-binning beta factor to the posterior uncertainties. " + "The flux baseline red-noise uncertainty is the out-of-transit baseline " + "component used for the final-plot baseline band." + ) + else: + params["Uncertainty interpretation note"] = ( + "The primary Rp/R* and radius-ratio area-depth uncertainties use model+red-noise " + "when available. The red-noise uncertainty inflates the data-fit standard error " + "by a residual time-binning beta factor before combining it with the model " + "posterior. The flux baseline red-noise uncertainty is the out-of-transit " + "baseline component used for the final-plot baseline band; the transit-depth " + "red-noise uncertainty already includes both the in-transit and baseline terms. " + "The same residual time-binning beta factor is applied to the posterior " + "uncertainties for Tmid, a/Rs, inclination, and impact parameter when reporting " + "their primary model+red-noise uncertainties. " + "The flux-scatter equivalent is also shown as a diagnostic of the full residual " + "scatter around the model." + ) + return params + + def build_aavso_photometry_metadata(photometry_info): if not isinstance(photometry_info, dict): return {} @@ -836,6 +1293,9 @@ def build_aavso_frame_filtering_metadata(fit, frame_filtering_info): int((diagnostic or {}).get('dropped_point_count', 0)) for diagnostic in diagnostics ) + final_residual_rejection = getattr(fit, 'final_residual_rejection', None) + if isinstance(final_residual_rejection, dict) and final_residual_rejection: + payload['final_residual_rejection'] = aavso_json_safe(final_residual_rejection) return payload @@ -874,7 +1334,7 @@ def format_percent_parameter_with_error(value, error): return f"{text} [%]" if text is not None else None -def formatted_transit_depth_parameters(fit, planet_dict=None, limb_darkening=None): +def formatted_transit_depth_parameters(fit, planet_dict=None, limb_darkening=None, empirical_uncertainty=None): prior_parameters = planet_dict_transit_parameters( planet_dict, limb_darkening=limb_darkening, @@ -897,24 +1357,93 @@ def formatted_transit_depth_parameters(fit, planet_dict=None, limb_darkening=Non text = format_percent_parameter_with_error(summary.get(value_key), summary.get(error_key)) if text is not None: entries[label] = text + + empirical_uncertainty = empirical_uncertainty or {} + if empirical_uncertainty.get('available'): + area_depth = finite_float(summary.get('area_depth')) + rprs = finite_float(empirical_uncertainty.get('rprs')) + rprs_prior_fallback = bool(empirical_uncertainty.get('rprs_prior_fallback_applied')) + + def area_error_percent_from_rprs_error(error): + error = finite_float(error) + if np.isfinite(area_depth) and np.isfinite(rprs) and np.isfinite(error) and error >= 0: + return float(200.0 * abs(rprs) * error) + return np.nan + + model_area_error = area_error_percent_from_rprs_error( + empirical_uncertainty.get('model_rprs_uncertainty') + ) + data_area_error = area_error_percent_from_rprs_error( + empirical_uncertainty.get('data_rprs_uncertainty') + ) + combined_area_error = area_error_percent_from_rprs_error( + empirical_uncertainty.get('combined_rprs_uncertainty') + ) + flux_scatter_area_error = area_error_percent_from_rprs_error( + empirical_uncertainty.get('data_rprs_flux_scatter_uncertainty') + ) + if np.isfinite(area_depth) and np.isfinite(combined_area_error): + combined_text = format_percent_parameter_with_error(area_depth, combined_area_error) + if combined_text is not None: + entries[AREA_DEPTH_LABEL] = combined_text + if rprs_prior_fallback: + entries[f"{AREA_DEPTH_LABEL} prior-assumed data-only uncertainty"] = combined_text + else: + entries[f"{AREA_DEPTH_LABEL} model+red-noise uncertainty"] = combined_text + if not rprs_prior_fallback and np.isfinite(area_depth) and np.isfinite(model_area_error): + text = format_percent_parameter_with_error(area_depth, model_area_error) + if text is not None: + entries[f"{AREA_DEPTH_LABEL} model-fit uncertainty"] = text + if np.isfinite(area_depth) and np.isfinite(data_area_error): + text = format_percent_parameter_with_error(area_depth, data_area_error) + if text is not None: + if rprs_prior_fallback: + entries[f"{AREA_DEPTH_LABEL} prior-assumed data-fit uncertainty"] = text + else: + entries[f"{AREA_DEPTH_LABEL} data-fit red-noise uncertainty"] = text + if np.isfinite(area_depth) and np.isfinite(flux_scatter_area_error): + text = format_percent_parameter_with_error(area_depth, flux_scatter_area_error) + if text is not None: + entries[f"{AREA_DEPTH_LABEL} flux-scatter equivalent"] = text return entries -def fit_impact_parameter_value_error(fit): +def empirical_red_noise_error_scale(empirical_uncertainty): + empirical_uncertainty = empirical_uncertainty or {} + if not empirical_uncertainty.get('available'): + return 1.0 + + beta = finite_float(empirical_uncertainty.get('red_noise_beta_factor')) + if np.isfinite(beta) and beta > 1.0: + return float(beta) + return 1.0 + + +def fit_parameter_model_data_uncertainty(fit, parameter_name, empirical_uncertainty=None): + errors = getattr(fit, 'errors', {}) or {} + model_error = finite_float(errors.get(parameter_name)) + if not np.isfinite(model_error) or model_error < 0: + return np.nan + + return float(model_error * empirical_red_noise_error_scale(empirical_uncertainty)) + + +def fit_impact_parameter_value_error(fit, errors_override=None): parameters = getattr(fit, 'parameters', {}) or {} errors = getattr(fit, 'errors', {}) or {} + errors_override = errors_override or {} sample_parameters = getattr(fit, 'sample_parameters', {}) or {} sample_errors = getattr(fit, 'sample_errors', {}) or {} if 'b' in sample_parameters: impact_parameter = finite_float(sample_parameters.get('b')) - impact_error = finite_float(sample_errors.get('b')) + impact_error = finite_float(errors_override.get('b'), finite_float(sample_errors.get('b'))) if np.isfinite(impact_parameter): return impact_parameter, impact_error if 'b' in parameters: impact_parameter = finite_float(parameters.get('b')) - impact_error = finite_float(errors.get('b')) + impact_error = finite_float(errors_override.get('b'), finite_float(errors.get('b'))) if np.isfinite(impact_parameter): return impact_parameter, impact_error @@ -933,8 +1462,8 @@ def fit_impact_parameter_value_error(fit): inc_rad = np.deg2rad(inc) impact_parameter = scale_factor * ars * np.cos(inc_rad) - ars_error = finite_float(errors.get('ars')) - inc_error = finite_float(errors.get('inc')) + ars_error = finite_float(errors_override.get('ars'), finite_float(errors.get('ars'))) + inc_error = finite_float(errors_override.get('inc'), finite_float(errors.get('inc'))) if np.isfinite(ars_error) and np.isfinite(inc_error): impact_error = np.hypot( scale_factor * np.cos(inc_rad) * ars_error, @@ -983,6 +1512,7 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor transit_qc = getattr(self.fit, 'transit_qc', None) fit_quality = build_fit_quality_metadata(self.fit) + empirical_uncertainty = fit_empirical_transit_uncertainty(self.fit, fit_quality=fit_quality) qc_residual_scatter = np.nan if isinstance(transit_qc, dict): qc_residual_scatter = transit_qc.get('residual_scatter', np.nan) @@ -1000,15 +1530,43 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor qc_residual_scatter = float(abs(residuals.reshape(-1)[0]) / median_flux) headline_params = format_transit_qc_headline_final_params(transit_qc) + rprs_report_error = finite_float(empirical_uncertainty.get('combined_rprs_uncertainty')) + if not np.isfinite(rprs_report_error) or rprs_report_error < 0: + rprs_report_error = self.fit.errors['rprs'] + tmid_report_error = fit_parameter_model_data_uncertainty( + self.fit, + 'tmid', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(tmid_report_error) or tmid_report_error < 0: + tmid_report_error = self.fit.errors['tmid'] + inc_report_error = fit_parameter_model_data_uncertainty( + self.fit, + 'inc', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(inc_report_error) or inc_report_error < 0: + inc_report_error = self.fit.errors['inc'] + ars_report_error = fit_parameter_model_data_uncertainty( + self.fit, + 'ars', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(ars_report_error) or ars_report_error < 0: + ars_report_error = self.fit.errors.get('ars', np.nan) core_params = { - "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " - f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB", - "Ratio of Planet to Stellar Radius (Rp/R*)": f"{round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- " - f"{round_to_2(self.fit.errors['rprs'])}", - "Orbital Inclination (inc)": f"{round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- " - f"{round_to_2(self.fit.errors['inc'])} ", + "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], tmid_report_error)} +/- " + f"{round_to_2(tmid_report_error)} BJD_TDB", + "Ratio of Planet to Stellar Radius (Rp/R*)": f"{round_to_2(self.fit.parameters['rprs'], rprs_report_error)} +/- " + f"{round_to_2(rprs_report_error)}", + "Orbital Inclination (inc)": f"{round_to_2(self.fit.parameters['inc'], inc_report_error)} +/- " + f"{round_to_2(inc_report_error)} ", } - depth_params = formatted_transit_depth_parameters(self.fit, self.p_dict) + depth_params = formatted_transit_depth_parameters( + self.fit, + self.p_dict, + empirical_uncertainty=empirical_uncertainty, + ) params_num = { **headline_params, "Mid-Transit Time (Tmid)": core_params["Mid-Transit Time (Tmid)"], @@ -1018,14 +1576,62 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor } ars_text = format_parameter_with_error( self.fit.parameters.get('ars'), - self.fit.errors.get('ars'), + ars_report_error, ) if ars_text is not None: params_num["Ratio of Distance to Stellar Radius (a/Rs)"] = ars_text - impact_parameter, impact_error = fit_impact_parameter_value_error(self.fit) + model_impact_parameter, model_impact_error = fit_impact_parameter_value_error(self.fit) + impact_error_overrides = { + 'ars': ars_report_error, + 'inc': inc_report_error, + } + if ( + np.isfinite(model_impact_error) + and ('b' in (getattr(self.fit, 'parameters', {}) or {}) + or 'b' in (getattr(self.fit, 'sample_parameters', {}) or {})) + ): + impact_error_overrides['b'] = model_impact_error * empirical_red_noise_error_scale( + empirical_uncertainty + ) + impact_parameter, impact_error = fit_impact_parameter_value_error( + self.fit, + errors_override=impact_error_overrides, + ) impact_text = format_parameter_with_error(impact_parameter, impact_error) if impact_text is not None: params_num["Impact Parameter (b)"] = impact_text + if empirical_uncertainty.get('available'): + tmid_model_text = ( + f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " + f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB" + ) + tmid_combined_text = core_params["Mid-Transit Time (Tmid)"] + params_num["Mid-Transit Time (Tmid) model-fit uncertainty"] = tmid_model_text + params_num["Mid-Transit Time (Tmid) model+red-noise uncertainty"] = tmid_combined_text + + inc_model_text = ( + f"{round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- " + f"{round_to_2(self.fit.errors['inc'])} " + ) + params_num["Orbital Inclination (inc) model-fit uncertainty"] = inc_model_text + params_num["Orbital Inclination (inc) model+red-noise uncertainty"] = core_params[ + "Orbital Inclination (inc)" + ] + + ars_model_text = format_parameter_with_error( + self.fit.parameters.get('ars'), + self.fit.errors.get('ars'), + ) + if ars_model_text is not None: + params_num["Ratio of Distance to Stellar Radius (a/Rs) model-fit uncertainty"] = ars_model_text + if ars_text is not None: + params_num["Ratio of Distance to Stellar Radius (a/Rs) model+red-noise uncertainty"] = ars_text + + impact_model_text = format_parameter_with_error(model_impact_parameter, model_impact_error) + if impact_model_text is not None: + params_num["Impact Parameter (b) model-fit uncertainty"] = impact_model_text + if impact_text is not None: + params_num["Impact Parameter (b) model+red-noise uncertainty"] = impact_text if getattr(self.fit, 'ns_type', None) is not None: params_num["Fit parameter point estimate"] = ( "Best-fit likelihood point; uncertainties are posterior spread." @@ -1042,9 +1648,13 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor sparse_posterior_note = getattr(self.fit, 'sparse_posterior_live_point_extension_note', None) if sparse_posterior_note: params_num["Sparse posterior live-point extension note"] = str(sparse_posterior_note) + final_residual_note = getattr(self.fit, 'final_residual_rejection_note', None) + if final_residual_note: + params_num["Final residual rejection note"] = str(final_residual_note) if np.isfinite(qc_residual_scatter): params_num["Residual scatter around full model fit"] = f"{qc_residual_scatter * 100.0:.4f} %" params_num.update(format_fit_quality_final_params(fit_quality)) + params_num.update(format_empirical_transit_uncertainty_final_params(empirical_uncertainty)) params_num.update(format_ktmf_decision_final_params(self.fit, photometry_info)) if getattr(self.fit, 'airmass_fit_skipped', False): params_num["Airmass correction"] = getattr( @@ -1079,6 +1689,11 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor qc_eebls_depth_snr = transit_qc.get('eebls_depth_snr', np.nan) qc_deviation_metric = transit_qc.get('deviation_from_expected_value', np.nan) qc_rprs_deviation_sigma = transit_qc.get('rprs_deviation_sigma', np.nan) + qc_rprs_deviation_fit_unc = transit_qc.get('rprs_deviation_fit_unc', np.nan) + qc_rprs_deviation_model_unc = transit_qc.get('rprs_deviation_model_fit_unc', np.nan) + qc_rprs_deviation_data_unc = transit_qc.get('rprs_deviation_data_fit_unc', np.nan) + qc_rprs_deviation_expected_unc = transit_qc.get('rprs_deviation_expected_unc', np.nan) + qc_rprs_deviation_comparison_unc = transit_qc.get('rprs_deviation_unc', np.nan) qc_sigma_threshold = transit_qc.get('deviation_sigma_threshold', np.nan) qc_ktmf = transit_qc.get('ktmf_metric', np.nan) qc_ktmf_contributions = transit_qc.get('ktmf_contributions') or [] @@ -1103,6 +1718,26 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num["Expected-value QC threshold"] = f"{qc_sigma_threshold:.2f} sigma" if np.isfinite(qc_rprs_deviation_sigma): params_num["Expected-value Rp/R* deviation"] = f"{qc_rprs_deviation_sigma:.2f} sigma" + if np.isfinite(qc_rprs_deviation_fit_unc): + params_num["Expected-value Rp/R* fit uncertainty used"] = ( + f"+/- {round_to_2(qc_rprs_deviation_fit_unc)}" + ) + if np.isfinite(qc_rprs_deviation_model_unc): + params_num["Expected-value Rp/R* model-fit uncertainty"] = ( + f"+/- {round_to_2(qc_rprs_deviation_model_unc)}" + ) + if np.isfinite(qc_rprs_deviation_data_unc): + params_num["Expected-value Rp/R* data-fit red-noise uncertainty"] = ( + f"+/- {round_to_2(qc_rprs_deviation_data_unc)}" + ) + if np.isfinite(qc_rprs_deviation_expected_unc): + params_num["Expected-value Rp/R* prior uncertainty"] = ( + f"+/- {round_to_2(qc_rprs_deviation_expected_unc)}" + ) + if np.isfinite(qc_rprs_deviation_comparison_unc): + params_num["Expected-value Rp/R* total comparison uncertainty"] = ( + f"+/- {round_to_2(qc_rprs_deviation_comparison_unc)}" + ) if np.isfinite(qc_ktmf): params_num["KTMF"] = f"{qc_ktmf:.2f} / 5.00" for contribution_index, contribution in enumerate(qc_ktmf_contributions, start=1): diff --git a/exotic/plots.py b/exotic/plots.py index e7e99272..9fe4ee1a 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -6,6 +6,7 @@ from matplotlib.lines import Line2D import numpy as np from pathlib import Path +import textwrap try: from utils import ( @@ -24,6 +25,21 @@ safe_output_filename, ) +try: + from output_files import ( + empirical_red_noise_error_scale, + fit_empirical_transit_uncertainty, + fit_impact_parameter_value_error, + fit_parameter_model_data_uncertainty, + ) +except ImportError: + from .output_files import ( + empirical_red_noise_error_scale, + fit_empirical_transit_uncertainty, + fit_impact_parameter_value_error, + fit_parameter_model_data_uncertainty, + ) + plt.style.use(astropy_mpl_style) @@ -695,7 +711,196 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relati # Plotting Final Lightcurve +def _final_lightcurve_model_grid(fit, high_res): + if hasattr(fit, 'phase_upsample') and hasattr(fit, 'transit_upsample'): + x_values = np.asarray(fit.phase_upsample, dtype=float) + model = np.asarray(fit.transit_upsample, dtype=float) + times = getattr(fit, 'time_upsample', None) + if times is not None: + times = np.asarray(times, dtype=float) + if times.shape != model.shape: + times = None + return x_values, model, times + + phase = np.asarray(getattr(fit, 'phase', np.array([])), dtype=float) + model = np.asarray(high_res, dtype=float) + if phase.size == 0 or model.size == 0: + return None, None, None + x_values = np.linspace(np.nanmin(phase), np.nanmax(phase), model.size) + return x_values, model, None + + +def _transit_model_uncertainty_envelope_for_grid(fit, times, model_shape): + if times is None or times.shape != model_shape: + return None + + uncertainty_func = getattr(fit, 'transit_model_uncertainty', None) + if not callable(uncertainty_func): + return None + + try: + envelope = uncertainty_func(times) + except Exception: + return None + if envelope is None or len(envelope) != 2: + return None + + lower = np.asarray(envelope[0], dtype=float) + upper = np.asarray(envelope[1], dtype=float) + if lower.shape != model_shape or upper.shape != model_shape: + return None + return lower, upper + + +def _plot_final_data_scatter_uncertainty_band(ax_lc, fit, high_res): + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + return False + + depth_uncertainty = empirical_uncertainty.get('depth_uncertainty_fraction') + try: + depth_uncertainty = float(depth_uncertainty) + except (TypeError, ValueError): + return False + if not np.isfinite(depth_uncertainty) or depth_uncertainty <= 0: + return False + + x_values, model, times = _final_lightcurve_model_grid(fit, high_res) + if x_values is None or model is None or x_values.shape != model.shape: + return False + + finite = np.isfinite(x_values) & np.isfinite(model) + if not np.any(finite): + return False + + empirical_lower = model - depth_uncertainty + empirical_upper = model + depth_uncertainty + sort_index = np.argsort(x_values) + x_sorted = x_values[sort_index] + finite_sorted = finite[sort_index] + empirical_lower_sorted = empirical_lower[sort_index] + empirical_upper_sorted = empirical_upper[sort_index] + + model_envelope = _transit_model_uncertainty_envelope_for_grid(fit, times, model.shape) + drew_band = False + def next_label(): + nonlocal drew_band + drew_band = True + return '_nolegend_' + + if model_envelope is not None: + model_lower, model_upper = model_envelope + model_lower_sorted = np.asarray(model_lower, dtype=float)[sort_index] + model_upper_sorted = np.asarray(model_upper, dtype=float)[sort_index] + + upper_region = ( + finite_sorted + & np.isfinite(empirical_upper_sorted) + & np.isfinite(model_upper_sorted) + & (empirical_upper_sorted > model_upper_sorted) + ) + lower_region = ( + finite_sorted + & np.isfinite(empirical_lower_sorted) + & np.isfinite(model_lower_sorted) + & (empirical_lower_sorted < model_lower_sorted) + ) + if np.any(upper_region): + ax_lc.fill_between( + x_sorted, + model_upper_sorted, + empirical_upper_sorted, + where=upper_region, + interpolate=True, + color='#6a1b9a', + alpha=0.16, + linewidth=0, + zorder=2.35, + label=next_label(), + ) + if np.any(lower_region): + ax_lc.fill_between( + x_sorted, + empirical_lower_sorted, + model_lower_sorted, + where=lower_region, + interpolate=True, + color='#6a1b9a', + alpha=0.16, + linewidth=0, + zorder=2.35, + label=next_label(), + ) + return drew_band + + ax_lc.fill_between( + x_sorted, + empirical_lower_sorted, + empirical_upper_sorted, + where=finite_sorted, + interpolate=True, + color='#6a1b9a', + alpha=0.14, + linewidth=0, + zorder=2.0, + label=next_label(), + ) + return drew_band + + +def _plot_final_residual_rejected_points(ax_lc, ax_res, fit): + rejection = getattr(fit, 'final_residual_rejection', None) + if not isinstance(rejection, dict) or not rejection.get('applied'): + return + + phase = np.asarray(rejection.get('rejected_phase', []), dtype=float) + flux = np.asarray(rejection.get('rejected_flux', []), dtype=float) + residual_percent = np.asarray(rejection.get('rejected_residual_percent', []), dtype=float) + plot_count = min(phase.size, flux.size, residual_percent.size) + if plot_count == 0: + return + + phase = phase[:plot_count] + flux = flux[:plot_count] + residual_percent = residual_percent[:plot_count] + finite = np.isfinite(phase) & np.isfinite(flux) & np.isfinite(residual_percent) + if not np.any(finite): + return + + ax_lc.scatter( + phase[finite], + flux[finite], + marker='x', + s=58, + linewidths=1.6, + color='red', + zorder=1200, + label='_nolegend_', + ) + ax_res.scatter( + phase[finite], + residual_percent[finite], + marker='x', + s=58, + linewidths=1.6, + color='red', + zorder=1200, + label='_nolegend_', + ) + + def plot_final_lightcurve(fit, high_res, targ_name, save, date): + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + try: + fit.empirical_transit_uncertainty = empirical_uncertainty + except Exception: + pass + f, (ax_lc, ax_res) = _plot_bestfit_for_lightcurve_png( fit, show_flux_baseline_label=False, @@ -704,10 +909,14 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date): ) ax_lc.set_title(targ_name) + drew_data_scatter_band = _plot_final_data_scatter_uncertainty_band(ax_lc, fit, high_res) if hasattr(fit, 'phase_upsample') and hasattr(fit, 'transit_upsample'): ax_lc.plot(fit.phase_upsample, fit.transit_upsample, 'r', zorder=1000, lw=2) else: ax_lc.plot(np.linspace(np.nanmin(fit.phase), np.nanmax(fit.phase), 1000), high_res, 'r', zorder=1000, lw=2) + _plot_final_residual_rejected_points(ax_lc, ax_res, fit) + if drew_data_scatter_band: + ax_lc.legend(loc='best') Path(save).mkdir(parents=True, exist_ok=True) try: @@ -716,3 +925,689 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date): except Exception: pass plt.close() + + +def _plot_scalar(value, default=np.nan): + try: + result = np.asarray(value, dtype=float).reshape(-1) + except (TypeError, ValueError): + return default + if result.size == 0: + return default + result = float(result[0]) + return result if np.isfinite(result) else default + + +def _plot_positive_error(value): + value = _plot_scalar(value) + if not np.isfinite(value) or value < 0: + return np.nan + return value + + +def _decimal_places_for_two_sigfig_error(error): + error = _plot_positive_error(error) + if not np.isfinite(error) or error == 0: + return None + + exponent = int(np.floor(np.log10(abs(error)))) + return max(0, 1 - exponent) + + +def _format_parameter_value(value, error=None, unit="", split_error=False): + value = _plot_scalar(value) + if not np.isfinite(value): + return "n/a" + + suffix = f" {unit}" if unit else "" + if error is None: + return f"{value:.6f}".rstrip('0').rstrip('.') + suffix + + error = _plot_positive_error(error) + if np.isfinite(error): + decimal_places = _decimal_places_for_two_sigfig_error(error) + if decimal_places is None: + decimal_places = 0 + value_text = f"{value:.{decimal_places}f}" + error_text = f"{error:.{decimal_places}f}" + if split_error: + return f"{value_text}\n+/- {error_text}{suffix}" + return f"{value_text} +/- {error_text}{suffix}" + return f"{value:.6f}".rstrip('0').rstrip('.') + suffix + + +def _prior_impact_parameter_value_error(planet_dict): + ars = _plot_scalar(planet_dict.get('aRs')) + inc = _plot_scalar(planet_dict.get('inc')) + if not np.isfinite(ars) or not np.isfinite(inc): + return np.nan, np.nan + + ecc = _plot_scalar(planet_dict.get('ecc'), 0.0) + omega = np.deg2rad(_plot_scalar(planet_dict.get('omega'), 0.0)) + denominator = 1.0 + ecc * np.sin(omega) + if not np.isfinite(denominator) or np.isclose(denominator, 0.0): + return np.nan, np.nan + + scale_factor = (1.0 - ecc ** 2) / denominator + inc_rad = np.deg2rad(inc) + impact_parameter = scale_factor * ars * np.cos(inc_rad) + + ars_error = _plot_positive_error(planet_dict.get('aRsUnc')) + inc_error = _plot_positive_error(planet_dict.get('incUnc')) + if np.isfinite(ars_error) and np.isfinite(inc_error): + impact_error = np.hypot( + scale_factor * np.cos(inc_rad) * ars_error, + scale_factor * ars * np.sin(inc_rad) * np.deg2rad(inc_error), + ) + else: + impact_error = np.nan + + return float(impact_parameter), float(impact_error) if np.isfinite(impact_error) else np.nan + + +def _ephemeris_prior_at_posterior_epoch(planet_dict, posterior_tmid): + mid_t = _plot_scalar(planet_dict.get('midT')) + period = _plot_scalar(planet_dict.get('pPer')) + posterior_tmid = _plot_scalar(posterior_tmid) + if not np.isfinite(mid_t) or not np.isfinite(period) or period <= 0 or not np.isfinite(posterior_tmid): + return mid_t, _plot_positive_error(planet_dict.get('midTUnc')), None + + epoch = int(np.round((posterior_tmid - mid_t) / period)) + expected_tmid = mid_t + epoch * period + + error_terms = [] + mid_t_error = _plot_positive_error(planet_dict.get('midTUnc')) + period_error = _plot_positive_error(planet_dict.get('pPerUnc')) + if np.isfinite(mid_t_error): + error_terms.append(mid_t_error) + if np.isfinite(period_error): + error_terms.append(abs(epoch) * period_error) + + if error_terms: + expected_error = float(np.sqrt(np.sum(np.square(error_terms)))) + else: + expected_error = np.nan + return float(expected_tmid), expected_error, epoch + + +def _posterior_parameter_value_error(fit, parameter_key, empirical_uncertainty): + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + + if parameter_key == 'b': + errors_override = {} + errors = getattr(fit, 'errors', {}) or {} + sample_errors = getattr(fit, 'sample_errors', {}) or {} + b_error = _plot_positive_error(errors.get('b')) + if not np.isfinite(b_error): + b_error = _plot_positive_error(sample_errors.get('b')) + if np.isfinite(b_error): + errors_override['b'] = float(b_error * empirical_red_noise_error_scale(empirical_uncertainty)) + ars_error = fit_parameter_model_data_uncertainty( + fit, + 'ars', + empirical_uncertainty=empirical_uncertainty, + ) + inc_error = fit_parameter_model_data_uncertainty( + fit, + 'inc', + empirical_uncertainty=empirical_uncertainty, + ) + if np.isfinite(ars_error): + errors_override['ars'] = ars_error + if np.isfinite(inc_error): + errors_override['inc'] = inc_error + return fit_impact_parameter_value_error(fit, errors_override=errors_override) + + value = _plot_scalar(parameters.get(parameter_key)) + if parameter_key == 'rprs': + error = _plot_positive_error( + (empirical_uncertainty or {}).get('combined_rprs_uncertainty') + ) + if not np.isfinite(error): + error = _plot_positive_error(errors.get(parameter_key)) + return value, error + + error = fit_parameter_model_data_uncertainty( + fit, + parameter_key, + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(error): + error = _plot_positive_error(errors.get(parameter_key)) + return value, error + + +def _prior_posterior_comparison_rows(fit, planet_dict): + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + try: + fit.empirical_transit_uncertainty = empirical_uncertainty + except Exception: + pass + + definitions = [ + ("Tmid", "tmid", "midT", "midTUnc", "", True), + ("Rp/R*", "rprs", "rprs", "rprsUnc", "", False), + ("a/Rs", "ars", "aRs", "aRsUnc", "", False), + ("Inc.", "inc", "inc", "incUnc", "deg", False), + ("b", "b", None, None, "", False), + ] + + rows = [] + rprs_prior_fallback = bool( + getattr(fit, 'rprs_prior_fallback_applied', False) + or (isinstance(empirical_uncertainty, dict) + and empirical_uncertainty.get('rprs_prior_fallback_applied')) + or (isinstance(empirical_uncertainty, dict) + and empirical_uncertainty.get('rprs_uncertainty_basis') == 'prior_assumed_data_only') + ) + omitted_notes = [] + for label, parameter_key, prior_key, prior_error_key, unit, split_error in definitions: + if parameter_key == 'rprs' and rprs_prior_fallback: + prior_value = _plot_scalar(planet_dict.get(prior_key)) + prior_error = _plot_positive_error(planet_dict.get(prior_error_key)) + posterior_value, posterior_error = _posterior_parameter_value_error( + fit, + parameter_key, + empirical_uncertainty, + ) + omitted_notes.append( + "Rp/R* omitted: prior value assumed, not measured " + f"({_format_parameter_value(prior_value, prior_error)}; " + f"data-only uncertainty {_format_parameter_value(posterior_value, posterior_error)})." + ) + continue + + posterior_value, posterior_error = _posterior_parameter_value_error( + fit, + parameter_key, + empirical_uncertainty, + ) + + if parameter_key == 'b': + prior_value, prior_error = _prior_impact_parameter_value_error(planet_dict) + prior_label = "Prior" + elif parameter_key == 'tmid': + prior_value, prior_error, _ = _ephemeris_prior_at_posterior_epoch( + planet_dict, + posterior_value, + ) + prior_label = "Prior" + else: + prior_value = _plot_scalar(planet_dict.get(prior_key)) + prior_error = _plot_positive_error(planet_dict.get(prior_error_key)) + prior_label = "Prior" + + if not np.isfinite(prior_value) or not np.isfinite(posterior_value): + continue + + error_terms = [ + term for term in (prior_error, posterior_error) + if np.isfinite(term) and term > 0 + ] + if error_terms: + combined_sigma = float(np.sqrt(np.sum(np.square(error_terms)))) + else: + separation = abs(posterior_value - prior_value) + combined_sigma = float(separation) if separation > 0 else np.nan + if not np.isfinite(combined_sigma) or combined_sigma <= 0: + continue + + posterior_offset = (posterior_value - prior_value) / combined_sigma + prior_error_sigma = prior_error / combined_sigma if np.isfinite(prior_error) else 0.0 + posterior_error_sigma = ( + posterior_error / combined_sigma if np.isfinite(posterior_error) else 0.0 + ) + prior_assumed = parameter_key == 'rprs' and rprs_prior_fallback + + rows.append({ + "label": label, + "parameter_key": parameter_key, + "posterior_offset": float(posterior_offset), + "prior_error_sigma": float(prior_error_sigma), + "posterior_error_sigma": float(posterior_error_sigma), + "prior_text": _format_parameter_value( + prior_value, + prior_error, + unit=unit, + split_error=split_error, + ), + "posterior_text": _format_parameter_value( + posterior_value, + posterior_error, + unit=unit, + split_error=split_error, + ), + "prior_label": prior_label, + "prior_assumed": prior_assumed, + }) + + return rows, omitted_notes + + +def plot_prior_posterior_comparison(fit, planet_dict, targ_name, save, date): + rows, omitted_notes = _prior_posterior_comparison_rows(fit, planet_dict) + if not rows: + return None + + note_height = 0.34 * len(omitted_notes) + row_spacing = 1.35 + fig_height = max(5.0, 1.02 * len(rows) + 2.0 + note_height) + fig, ax = plt.subplots(figsize=(11.8, fig_height)) + + y_positions = np.arange(len(rows), dtype=float) * row_spacing + posterior_offsets = np.array([row["posterior_offset"] for row in rows], dtype=float) + prior_errors = np.array([row["prior_error_sigma"] for row in rows], dtype=float) + posterior_errors = np.array([row["posterior_error_sigma"] for row in rows], dtype=float) + + xmin = min(-3.5, np.nanmin(np.r_[posterior_offsets - posterior_errors, -prior_errors]) - 0.45) + xmax = max(3.5, np.nanmax(np.r_[posterior_offsets + posterior_errors, prior_errors]) + 0.45) + + ax.axvspan(-1.0, 1.0, color='#2e7d32', alpha=0.08, linewidth=0) + ax.axvspan(-3.0, 3.0, color='#f9a825', alpha=0.06, linewidth=0) + ax.axvline(0.0, color='0.25', lw=1.2, ls='--', zorder=1) + + ax.errorbar( + np.zeros_like(y_positions), + y_positions + 0.13, + xerr=prior_errors, + fmt='o', + ms=6, + color='#1565c0', + ecolor='#1565c0', + elinewidth=1.4, + capsize=3, + label='Prior', + zorder=5, + ) + ax.errorbar( + posterior_offsets, + y_positions - 0.13, + xerr=posterior_errors, + fmt='s', + ms=6, + color='#c62828', + ecolor='#c62828', + elinewidth=1.4, + capsize=3, + label='Posterior', + zorder=6, + ) + + for y_position, row in zip(y_positions, rows): + annotation = ( + f"{row['prior_label']}\n" + f"{row['prior_text']}\n" + "Posterior\n" + f"{row['posterior_text']}" + ) + ax.text( + 1.015, + y_position, + annotation, + transform=ax.get_yaxis_transform(), + ha='left', + va='center', + fontsize=8.5, + linespacing=1.12, + color='0.18', + ) + + ax.set_yticks(y_positions) + ax.set_yticklabels([row["label"] for row in rows]) + ax.invert_yaxis() + ax.set_xlim(xmin, xmax) + ax.set_xlabel("Posterior offset from prior [combined sigma]") + ax.set_title(f"{targ_name} Prior vs Posterior Transit Parameters") + ax.grid(axis='x', alpha=0.28) + if omitted_notes: + ax.text( + 0.0, + -0.16, + "\n".join(omitted_notes), + transform=ax.transAxes, + ha='left', + va='top', + fontsize=9, + color='0.22', + ) + ax.legend( + handles=[ + Line2D([0], [0], marker='o', color='none', markerfacecolor='#1565c0', + markeredgecolor='#1565c0', markersize=7, label='Prior'), + Line2D([0], [0], marker='s', color='none', markerfacecolor='#c62828', + markeredgecolor='#c62828', markersize=7, label='Posterior'), + ], + loc='lower right', + ) + fig.subplots_adjust(right=0.64) + + Path(save).mkdir(parents=True, exist_ok=True) + png_path = Path(save) / _dated_plot_filename( + "PriorPosteriorComparison", + targ_name, + date=date, + extension="png", + ) + pdf_path = Path(save) / _dated_plot_filename( + "PriorPosteriorComparison", + targ_name, + date=date, + extension="pdf", + ) + try: + fig.savefig(png_path, bbox_inches="tight") + fig.savefig(pdf_path, bbox_inches="tight") + except Exception: + png_path = None + plt.close(fig) + return png_path + + +def _fit_ktmf_metric_contributions_status(fit): + transit_qc = getattr(fit, 'transit_qc', None) + if not isinstance(transit_qc, dict): + transit_qc = {} + + metric = _plot_scalar( + getattr(fit, 'transit_qc_ktmf_metric', transit_qc.get('ktmf_metric', np.nan)) + ) + contributions = getattr(fit, 'transit_qc_ktmf_contributions', None) + if not contributions: + contributions = transit_qc.get('ktmf_contributions', []) + if not isinstance(contributions, (list, tuple)): + contributions = [] + + status = _ktmf_status_from_metric(metric) + if not status: + status = getattr(fit, 'transit_qc_status', transit_qc.get('status', None)) + return metric, list(contributions), status + + +def _ktmf_status_from_metric(metric): + metric = _plot_scalar(metric) + if not np.isfinite(metric): + return None + if metric >= 4.0: + return "pass" + if metric >= 3.0: + return "marginal" + return "fail" + + +def _short_ktmf_label(label): + replacements = { + "Deviation From Expected Value": "Expected Rp/R*", + "Residual Scatter Around Full Model Fit": "Residual scatter", + "Duration Consistency": "Duration", + "EEBLS Depth SNR": "EEBLS SNR", + "Sampling / Cadence": "Sampling", + } + return replacements.get(str(label), str(label)) + + +def _ktmf_marker_color(score): + score = _plot_scalar(score) + if not np.isfinite(score): + return '0.45' + if score >= 0.8: + return '#2e7d32' + if score >= 0.6: + return '#f9a825' + return '#c62828' + + +def _format_ktmf_metric(value, maximum=5.0): + value = _plot_scalar(value) + maximum = _plot_scalar(maximum) + if not np.isfinite(value): + return "n/a" + if np.isfinite(maximum) and maximum > 0: + return f"{value:.2f} / {maximum:.2f}" + return f"{value:.2f}" + + +def _format_ktmf_component_annotation(row): + if row.get('kind') == 'total': + status = row.get('status') + status_text = f"\n{status.upper()}" if status else "" + uncertainty = _plot_positive_error(row.get('score_uncertainty')) + uncertainty_text = f"\nscore spread +/- {uncertainty:.2f}" if np.isfinite(uncertainty) else "" + return f"KTMF\n{_format_ktmf_metric(row.get('points'), row.get('max_points'))}{status_text}{uncertainty_text}" + + if not row.get('available', True): + detail = row.get('detail') or "unavailable" + return f"Not scored\n{textwrap.fill(str(detail), width=44)}" + + score_uncertainty = _plot_positive_error(row.get('score_uncertainty')) + if np.isfinite(score_uncertainty): + score_text = _format_parameter_value(row.get('score'), score_uncertainty) + else: + score_text = _format_parameter_value(row.get('score')) + detail = _compact_ktmf_detail(row) + detail_text = f"\n{textwrap.fill(str(detail), width=44)}" if detail else "" + return ( + f"Score\n{score_text}\n" + f"Points\n{_format_ktmf_metric(row.get('points'), row.get('max_points'))}" + f"{detail_text}" + ) + + +def _compact_ktmf_detail(row): + detail = row.get('detail') + if not detail: + return None + detail = str(detail) + if row.get('label') == "Expected Rp/R*": + if "fixed to the input prior" in detail or "prior" in detail.lower(): + return "Rp/R* prior assumed; not scored." + keep = [] + for part in detail.split(','): + part = part.strip() + if part.startswith("Rp/R* sigma="): + keep.append(part) + return ", ".join(keep) if keep else detail + return detail + + +def _ktmf_plot_rows(fit): + metric, contributions, status = _fit_ktmf_metric_contributions_status(fit) + component_rows = [] + for contribution in contributions: + if not isinstance(contribution, dict): + continue + available = bool(contribution.get('available', True)) + score = _plot_scalar(contribution.get('score')) + if not available or not np.isfinite(score): + score = np.nan + component_rows.append({ + "kind": "component", + "label": _short_ktmf_label(contribution.get('label', 'KTMF component')), + "score": float(np.clip(score, 0.0, 1.0)) if np.isfinite(score) else np.nan, + "score_uncertainty": _plot_positive_error(contribution.get('score_uncertainty')), + "points": _plot_scalar(contribution.get('points'), 0.0), + "max_points": _plot_scalar(contribution.get('max_points'), 0.0), + "available": available and np.isfinite(score), + "detail": contribution.get('detail'), + }) + + rows = [] + if np.isfinite(metric): + total_score = float(np.clip(metric / 5.0, 0.0, 1.0)) + available_component_rows = [ + row for row in component_rows + if row.get('available') + and np.isfinite(row.get('score', np.nan)) + and np.isfinite(row.get('max_points', np.nan)) + and row.get('max_points', 0.0) > 0 + ] + score_uncertainty = np.nan + if len(available_component_rows) > 1: + scores = np.asarray([row['score'] for row in available_component_rows], dtype=float) + weights = np.asarray([row['max_points'] for row in available_component_rows], dtype=float) + if np.isfinite(weights).all() and np.sum(weights) > 0: + score_uncertainty = float( + np.sqrt(np.average((scores - total_score) ** 2, weights=weights)) + ) + rows.append({ + "kind": "total", + "label": "KTMF total", + "score": total_score, + "score_uncertainty": score_uncertainty, + "points": float(metric), + "max_points": 5.0, + "available": True, + "status": status, + }) + rows.extend(component_rows) + return rows + + +def _score_errorbar_limits(score, uncertainty): + score = _plot_scalar(score) + uncertainty = _plot_positive_error(uncertainty) + if not np.isfinite(score) or not np.isfinite(uncertainty) or uncertainty <= 0: + return None + lower = min(uncertainty, max(score, 0.0)) + upper = min(uncertainty, max(1.0 - score, 0.0)) + if lower <= 0 and upper <= 0: + return None + return np.asarray([[lower], [upper]], dtype=float) + + +def _draw_ktmf_score_background(ax): + ax.axvspan(0.0, 0.6, color='#c62828', alpha=0.055, linewidth=0) + ax.axvspan(0.6, 0.8, color='#f9a825', alpha=0.09, linewidth=0) + ax.axvspan(0.8, 1.0, color='#2e7d32', alpha=0.08, linewidth=0) + ax.axvline(0.6, color='0.55', lw=1.0, ls=':', zorder=1) + ax.axvline(0.8, color='0.45', lw=1.1, ls='--', zorder=1) + ax.grid(axis='x', alpha=0.28) + + +def _plot_ktmf_score_marker(ax, row, y_position): + score = _plot_scalar(row.get('score')) + color = _ktmf_marker_color(score) + marker = 'D' if row.get('kind') == 'total' else 's' + marker_size = 62 if row.get('kind') == 'total' else 48 + marker_scale = 3.0 + if np.isfinite(score): + ax.errorbar( + [score], + [y_position], + xerr=_score_errorbar_limits(score, row.get('score_uncertainty')), + fmt=marker, + ms=np.sqrt(marker_size) * marker_scale, + color=color, + ecolor=color, + elinewidth=1.8, + capsize=4, + markeredgecolor='white', + markeredgewidth=1.2, + zorder=5, + ) + else: + ax.scatter( + [0.0], + [y_position], + marker='x', + s=52 * marker_scale ** 2, + color='0.45', + linewidths=2.0, + zorder=5, + ) + + +def plot_ktmf_qc_metrics(fit, targ_name, save, date): + rows = _ktmf_plot_rows(fit) + if not rows: + return None + + total_rows = [row for row in rows if row.get('kind') == 'total'] + component_rows = [row for row in rows if row.get('kind') != 'total'] + row_spacing = 1.35 + component_height = max(3.6, 0.98 * max(len(component_rows), 1) + 1.3) + fig_height = component_height + (1.55 if total_rows else 0.0) + if total_rows: + fig, (ax_total, ax_components) = plt.subplots( + 2, + 1, + figsize=(11.8, fig_height), + sharex=True, + gridspec_kw={'height_ratios': [1.0, component_height]}, + ) + axes = [ax_total, ax_components] + else: + fig, ax_components = plt.subplots(figsize=(11.8, fig_height)) + ax_total = None + axes = [ax_components] + + for axis in axes: + _draw_ktmf_score_background(axis) + axis.set_xlim(-0.05, 1.05) + + if total_rows: + total_row = total_rows[0] + _plot_ktmf_score_marker(ax_total, total_row, 0.0) + ax_total.text( + 1.025, + 0.0, + _format_ktmf_component_annotation(total_row), + transform=ax_total.get_yaxis_transform(), + ha='left', + va='center', + fontsize=8.5, + linespacing=1.12, + color='0.18', + ) + ax_total.set_yticks([0.0]) + ax_total.set_yticklabels([total_row["label"]]) + ax_total.set_ylim(0.65, -0.65) + ax_total.tick_params(axis='x', labelbottom=False) + ax_total.set_title(f"{targ_name} KTMF QC Metrics") + + component_positions = np.arange(len(component_rows), dtype=float) * row_spacing + for y_position, row in zip(component_positions, component_rows): + _plot_ktmf_score_marker(ax_components, row, y_position) + ax_components.text( + 1.025, + y_position, + _format_ktmf_component_annotation(row), + transform=ax_components.get_yaxis_transform(), + ha='left', + va='center', + fontsize=8.5, + linespacing=1.12, + color='0.18', + ) + + ax_components.set_yticks(component_positions) + ax_components.set_yticklabels([row["label"] for row in component_rows]) + ax_components.invert_yaxis() + ax_components.set_xlabel("KTMF component score fraction") + if not total_rows: + ax_components.set_title(f"{targ_name} KTMF QC Metrics") + fig.subplots_adjust(right=0.62, hspace=0.12) + + Path(save).mkdir(parents=True, exist_ok=True) + png_path = Path(save) / _dated_plot_filename( + "KTMF_QC", + targ_name, + date=date, + extension="png", + ) + pdf_path = Path(save) / _dated_plot_filename( + "KTMF_QC", + targ_name, + date=date, + extension="pdf", + ) + try: + fig.savefig(png_path, bbox_inches="tight") + fig.savefig(pdf_path, bbox_inches="tight") + except Exception: + png_path = None + plt.close(fig) + return png_path diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 8a76ded0..9ee0cf65 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -375,6 +375,38 @@ def test_plot_bestfit_can_hide_flux_baseline_label(monkeypatch, tmp_path): plt.close(fig) +def test_plot_bestfit_marks_prior_rprs_fallback_uncertainty(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + fixed_parameter_errors={"rprs": 0.02}, + ) + fit.rprs_prior_fallback_applied = True + fit.empirical_transit_uncertainty = { + "available": True, + "combined_rprs_uncertainty": 0.02, + } + + fig, axes = fit.plot_bestfit(show_flux_baseline_label=False) + legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) + + assert "(Prior)" in legend_text + plt.close(fig) + + def test_plot_bestfit_can_draw_transit_model_uncertainty_band(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -527,6 +559,32 @@ def test_baseline_model_uncertainty_does_not_double_count_analytic_a0(monkeypatc assert half_width < 0.03 +def test_baseline_model_uncertainty_includes_empirical_flux_floor(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.airmass = np.linspace(1.0, 2.0, time.size) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.0 + fit.errors = {"a0": 0.001, "a1": 0.001, "a2": 0.001} + fit.results = None + fit.empirical_transit_uncertainty = { + "available": True, + "baseline_red_noise_uncertainty_fraction": 0.02, + } + + lower, upper = fit.baseline_model_uncertainty(time) + half_width = np.nanmax(np.maximum(1.0 - lower, upper - 1.0)) + + assert half_width >= 0.02 + + def test_baseline_model_uncertainty_prefers_posterior_samples(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 86ddc0c4..3dc3fd0e 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -111,6 +111,8 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: compute_star_aperture_grid, compute_transit_qc_ktmf, detect_aperture_correction_star_candidates, + transit_qc_residual_scatter_score, + transit_qc_sampling_summary, apply_comparison_star_suitability_outlier_rejection, comparison_calibration_selection_reason, comparison_candidate_triangle_plot_output_path, @@ -552,6 +554,84 @@ def fake_final_fit( ].tolist() == [True, True, True, False, True, True, True, True, True, True] +def test_finalize_comparison_candidate_can_disable_phase_residual_clip(monkeypatch): + captured = {"phase_clip_called": False} + + def fake_lc_fitter(times, flux, unc, airmass, prior, bounds, jd_times=None, mode=None, **kwargs): + return types.SimpleNamespace( + residuals=np.linspace(-0.01, 0.01, len(times)), + phase=np.linspace(-0.5, 0.5, len(times)), + ) + + def fake_phase_clip(residuals, phase, sigma=3, bins=10): + captured["phase_clip_called"] = True + mask = np.zeros(len(residuals), dtype=bool) + mask[3] = True + return mask + + def fake_final_fit(times, flux, unc, airmass, prior, bounds, jd_times=None, **kwargs): + captured["times"] = np.asarray(times, dtype=float).copy() + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + airmass=np.asarray(airmass, dtype=float), + data=np.asarray(flux, dtype=float), + dataerr=np.asarray(unc, dtype=float), + detrended=np.asarray(flux, dtype=float), + detrendederr=np.asarray(unc, dtype=float), + airmass_model=np.ones(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + phase=np.linspace(-0.5, 0.5, len(times)), + residuals=np.zeros(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a1": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a1": 0.01, "a2": 0.01}, + ) + return fit, np.asarray(flux, dtype=float), np.asarray(unc, dtype=float) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.phase_bin_sigma_clip", fake_phase_clip) + monkeypatch.setattr("exotic.exotic.fit_final_lightcurve_with_oot_baseline_detrending", fake_final_fit) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.09, 10) + result = finalize_comparison_candidate_full_reduction( + times, + np.full(10, 100.0, dtype=float), + np.full(10, 100.0, dtype=float), + np.linspace(1.0, 1.2, 10), + [0.1, 0.1, 0.1, 0.1], + { + "midT": 0.045, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "aRs": 10.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + jd_times=2460000.0 + times, + run_final_fit_phase_residual_clip=False, + ) + + assert result["applied"] is True + assert captured["phase_clip_called"] is False + assert captured["times"].tolist() == pytest.approx(times.tolist()) + assert result["source_indices"].tolist() == list(range(10)) + assert not any( + diagnostic["stage"] == "Final-fit phase residual clip" + and diagnostic["dropped_point_count"] > 0 + for diagnostic in result["fit"].frame_filter_diagnostics + ) + assert result["fit"].selected_photometry_debug[ + "phase_clip_keep_mask_on_sigma_filtered" + ].tolist() == [True] * 10 + + def test_detrend_flux_on_out_of_transit_baseline_falls_back_to_prior_ephemeris(): times = np.linspace(-0.08, 0.08, 17) baseline = 1.0 + 0.25 * times @@ -573,8 +653,8 @@ def test_detrend_flux_on_out_of_transit_baseline_falls_back_to_prior_ephemeris() "omega": 0.0, } - modeled = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit) - fallback = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit, prior=prior) + modeled = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit, min_side_points=2) + fallback = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit, prior=prior, min_side_points=2) prior_coverage = summarize_prior_transit_coverage(times, prior, flux_values=flux, flux_errors=unc) assert modeled["applied"] is False @@ -585,6 +665,26 @@ def test_detrend_flux_on_out_of_transit_baseline_falls_back_to_prior_ephemeris() assert "ephemeris-centered transit window" in fallback["note"] +def test_detrend_flux_on_out_of_transit_baseline_requires_default_side_coverage(): + times = np.linspace(-0.08, 0.08, 17) + baseline = 1.0 + 0.25 * times + transit_profile = np.ones_like(times) + transit_profile[(times >= -0.01) & (times <= 0.01)] = 0.99 + flux = baseline * transit_profile + unc = np.full_like(times, 0.01) + fit = types.SimpleNamespace( + transit=transit_profile, + parameters={"tmid": 0.0}, + ) + + result = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit) + + assert result["applied"] is False + assert result["pre_points"] <= 12 + assert result["post_points"] <= 12 + assert "need more than 12 on each side" in result["note"] + + def test_deduplicate_comparison_star_coords_preserves_nearby_user_stars(): preserved_coords, duplicate_messages = deduplicate_comparison_star_coords( [ @@ -724,6 +824,109 @@ def build_psf_rows(amplitudes): assert candidate_jobs[0]["coverage_count"] == 29 +def test_build_target_fit_candidate_jobs_uses_psf_flux_rows_for_psf_quality(): + frame_count = 30 + + def build_psf_rows(amplitudes): + psf_rows = np.zeros((frame_count, 7), dtype=float) + psf_rows[:, 0] = 10.0 + psf_rows[:, 1] = 20.0 + psf_rows[:, 2] = amplitudes + psf_rows[:, 3] = 1.0 + psf_rows[:, 4] = 1.0 + return psf_rows + + psf_data = { + "target": build_psf_rows(np.full(frame_count, 100.0)), + "comp1": build_psf_rows(np.full(frame_count, 120.0)), + } + psf_data["target"][19, 3:5] = 6.5 + + psf_flux_data = { + "target": build_psf_rows(np.full(frame_count, 100.0)), + "comp1": build_psf_rows(np.full(frame_count, 120.0)), + } + psf_flux_data["target"][7, 2] = 80.0 + psf_flux_data["target"][21, 3:5] = 6.5 + psf_flux_data["comp1"][13, 2] = 1.0 + + candidate_jobs = build_target_fit_candidate_jobs( + psf_data, + aper_data=None, + apers=None, + annuli=None, + airmass=np.linspace(1.0, 1.3, frame_count), + comp_stars=[[1827.0, 1511.0]], + sigma=3.0, + require_comp_star=True, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=False, + psf_flux_data=psf_flux_data, + ) + + assert len(candidate_jobs) == 1 + assert candidate_jobs[0]["method"] == "psf" + assert candidate_jobs[0]["mask"].sum() == 28 + assert candidate_jobs[0]["mask"][7] + assert candidate_jobs[0]["mask"][19] + assert not candidate_jobs[0]["mask"][13] + assert not candidate_jobs[0]["mask"][21] + assert candidate_jobs[0]["coverage_count"] == 29 + + +@pytest.mark.filterwarnings("ignore::RuntimeWarning") +def test_legacy_psf_photometry_flux_row_uses_weighted_centroid_override(): + import exotic.exotic as exotic_module + + y_grid, x_grid = np.mgrid[0:31, 0:31] + data = exotic_module.gaussian_psf( + x_grid, + y_grid, + 15.25, + 14.65, + 200.0, + 1.8, + 2.2, + 0.05, + 30.0, + ) + seed_row = np.array([15.0, 15.0, 100.0, 1.0, 1.0, 0.0, 30.0], dtype=float) + + row = exotic_module.fit_legacy_psf_photometry_flux_row(data, seed_row, 0, box=8) + + xv, yv = exotic_module.mesh_box(seed_row[:2], 8, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + expected_wx = np.sum(xv[0] * subarray.sum(0)) / subarray.sum(0).sum() + expected_wy = np.sum(yv[:, 0] * subarray.sum(1)) / subarray.sum(1).sum() + + assert row[0] == pytest.approx(expected_wx) + assert row[1] == pytest.approx(expected_wy) + assert row[2] > 0 + assert row[3] > 0 + assert row[4] > 0 + + +def test_load_psf_flux_seed_tracks_accepts_legacy_selected_comp_file(tmp_path): + import exotic.exotic as exotic_module + + run_dir = tmp_path / "old_run" + temp_dir = run_dir / "temp" + temp_dir.mkdir(parents=True) + target_rows = np.tile(np.array([[10.0, 20.0, 100.0, 1.0, 1.1, 0.0, 30.0]]), (3, 1)) + comp_rows = np.tile(np.array([[30.0, 40.0, 150.0, 1.2, 1.3, 0.0, 31.0]]), (3, 1)) + np.savetxt(temp_dir / "psf_data_target.txt", target_rows) + np.savetxt(temp_dir / "psf_data_comp.txt", comp_rows) + + seed_tracks = exotic_module.load_psf_flux_seed_tracks(str(run_dir), 3, ["comp1"]) + + assert set(seed_tracks) == {"target", "comp1"} + assert seed_tracks["target"].shape == (3, 7) + assert seed_tracks["comp1"].shape == (3, 7) + assert seed_tracks["target"][0, 0] == pytest.approx(10.0) + assert seed_tracks["comp1"][0, 0] == pytest.approx(30.0) + + def test_centroid_offset_matches_reference_uses_float_geometry_tolerance(): target = np.array([2383.27, 867.04, 6.3, 9.0, 0.7, 0.0, 223.0]) comp = np.array([1821.90, 549.21, 131.0, 3.0, 4.9, 0.0, 226.0]) @@ -1912,6 +2115,8 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): "tmid_deviation_sigma": 1.0, "rprs_deviation_sigma": 2.0, "residual_scatter": 0.005, + "transit_depth_for_residual_scatter": 0.01, + "residual_scatter_to_depth_ratio": 0.5, "rprs_sigma": 6.0, "duration_ratio": 1.0, "eebls_depth_snr": 8.0, @@ -1923,23 +2128,102 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): assert "Model Evidence" in contributions_by_label assert "Delta BIC" not in contributions_by_label assert "Delta chi2" not in contributions_by_label - scale = 5.0 / (0.8 + 1.5 + 0.7 + 0.75 + 0.75) - assert contributions_by_label["Model Evidence"]["max_points"] == pytest.approx(0.8 * scale) + scale = 5.0 / (0.3 + 1.5 + 0.7 + 0.75 + 1.3) + assert contributions_by_label["Model Evidence"]["max_points"] == pytest.approx(0.3 * scale) assert contributions_by_label["Deviation From Expected Value"]["max_points"] == pytest.approx(1.5 * scale) assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) assert "Rp/R* Significance" not in contributions_by_label assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75 * scale) - assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(0.75 * scale) + assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) assert "Rp/R* sigma=2.00" in contributions_by_label["Deviation From Expected Value"]["detail"] assert "Tmid" not in contributions_by_label["Deviation From Expected Value"]["detail"] model_evidence_score = ((1.0 - np.exp(-1.0)) + (1.0 - np.exp(-2.0))) / 2.0 expected_ktmf = scale * ( - 0.8 * model_evidence_score + 0.3 * model_evidence_score + 1.5 * 0.6 - + 0.7 * 0.5 + + 0.7 * 1.0 + 0.75 * 1.0 - + 0.75 * (1.0 - np.exp(-2.0)) + + 1.3 * (1.0 - np.exp(-2.0)) + ) + assert ktmf_metric == pytest.approx(expected_ktmf) + + +def test_transit_qc_residual_scatter_score_full_credit_floor_and_zero_ceiling(): + transit_depth = 0.02 + assert transit_qc_residual_scatter_score(0.0, transit_depth) == pytest.approx(1.0) + assert transit_qc_residual_scatter_score(0.01, transit_depth) == pytest.approx(1.0) + assert transit_qc_residual_scatter_score(0.08, transit_depth) == pytest.approx(0.0) + assert transit_qc_residual_scatter_score(0.09, transit_depth) == pytest.approx(0.0) + assert not np.isfinite(transit_qc_residual_scatter_score(0.005)) + + mid_score = transit_qc_residual_scatter_score(0.03, transit_depth) + assert 0.0 < mid_score < 1.0 + assert mid_score < transit_qc_residual_scatter_score(0.02, transit_depth) + + +def test_transit_qc_sampling_summary_scores_ingress_egress_and_baseline_counts(): + fit = types.SimpleNamespace( + time=np.array([ + -0.090, -0.075, -0.060, + -0.050, -0.045, -0.040, -0.035, -0.030, + -0.020, -0.010, 0.000, 0.010, 0.020, + 0.030, 0.035, 0.040, 0.045, 0.050, + 0.060, 0.075, 0.090, + ]), + parameters={"tmid": 0.0, "rprs": 0.1}, + duration_expected=0.1, + ) + + summary = transit_qc_sampling_summary(fit) + + assert summary["available"] is True + assert summary["ingress_count"] == 5 + assert summary["egress_count"] == 5 + assert summary["in_transit_count"] == 15 + assert summary["pre_baseline_count"] == 3 + assert summary["post_baseline_count"] == 3 + assert 0.0 < summary["score"] < 1.0 + assert "ingress=5, egress=5" in summary["detail"] + + +def test_compute_transit_qc_ktmf_omits_prior_assumed_rprs_component(): + summary = { + "delta_bic": 10.0, + "delta_chi2": 50.0, + "deviation_from_expected_value": 1.0, + "rprs_deviation_sigma": 0.0, + "rprs_prior_assumed": True, + "rprs_prior_assumed_note": "Rp/R* was fixed to the input prior.", + "residual_scatter": 0.005, + "transit_depth_for_residual_scatter": 0.01, + "residual_scatter_to_depth_ratio": 0.5, + "point_count": 86, + "duration_ratio": 1.0, + "eebls_depth_snr": 8.0, + } + + ktmf_metric, contributions = compute_transit_qc_ktmf(summary) + contributions_by_label = {contribution["label"]: contribution for contribution in contributions} + + omitted = contributions_by_label["Deviation From Expected Value"] + assert omitted["available"] is False + assert omitted["points"] == pytest.approx(0.0) + assert omitted["max_points"] == pytest.approx(0.0) + assert "fixed to the input prior" in omitted["detail"] + + scale = 5.0 / (0.3 + 0.7 + 0.75 + 1.3) + assert contributions_by_label["Model Evidence"]["max_points"] == pytest.approx(0.3 * scale) + assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) + assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75 * scale) + assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) + + model_evidence_score = ((1.0 - np.exp(-1.0)) + (1.0 - np.exp(-2.0))) / 2.0 + expected_ktmf = scale * ( + 0.3 * model_evidence_score + + 0.7 * 1.0 + + 0.75 * 1.0 + + 1.3 * (1.0 - np.exp(-2.0)) ) assert ktmf_metric == pytest.approx(expected_ktmf) @@ -2366,7 +2650,7 @@ def test_detrend_flux_on_out_of_transit_baseline_removes_linear_slope(): parameters={"tmid": 0.0}, ) - result = detrend_flux_on_out_of_transit_baseline(times, flux, fluxerr, fit) + result = detrend_flux_on_out_of_transit_baseline(times, flux, fluxerr, fit, min_side_points=2) assert result["applied"] is True assert np.allclose(result["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) @@ -2418,6 +2702,7 @@ def fake_lc_fitter( skip_airmass_fit=False, disable_vertical_flux_normalization=False, detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=2, ) assert len(captured["calls"]) == 2 @@ -2516,6 +2801,7 @@ def fake_run_nested( prior, bounds, detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=2, extend_sparse_posterior_live_points=False, ) @@ -2600,6 +2886,7 @@ def fake_lc_fitter( prior, bounds, detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=0, ) assert len(captured["calls"]) == 2 @@ -2704,6 +2991,7 @@ def fake_lc_fitter( def test_fit_final_lightcurve_retries_nested_fit_when_rprs_posterior_is_clipped(monkeypatch): import exotic.exotic as exotic_module + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", False) captured = {"calls": []} def make_fit(call_flux, diagnostics): @@ -2787,6 +3075,7 @@ def fake_lc_fitter( def test_fit_final_lightcurve_carries_retry_bounds_into_oot_baseline_refit(monkeypatch): import exotic.exotic as exotic_module + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", False) times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) flux = (1.0 + 0.02 * times) * np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) fluxerr = np.full_like(times, 0.01) @@ -2866,6 +3155,7 @@ def fake_lc_fitter( prior, bounds, detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=2, ) assert len(captured["calls"]) == 3 @@ -3677,7 +3967,7 @@ def test_evaluate_transit_detection_qc_fails_when_flat_model_is_better(): assert summary["delta_chi2"] < 0.0 -def test_evaluate_transit_detection_qc_marks_large_expected_value_deviation_marginal_via_ktmf(): +def test_evaluate_transit_detection_qc_marks_large_expected_value_deviation_fail_via_ktmf(): transit_model = np.ones(21, dtype=float) transit_model[8:13] = 0.99 data = transit_model + np.array( @@ -3710,14 +4000,14 @@ def test_evaluate_transit_detection_qc_marks_large_expected_value_deviation_marg summary = evaluate_transit_detection_qc(fit) assert summary["computed"] is True - assert summary["status"] == "marginal" + assert summary["status"] == "fail" expected_comparison_unc = np.sqrt(0.01 ** 2 + 0.01 ** 2 + (0.05 * 0.10) ** 2) assert summary["rprs_deviation_unc"] == pytest.approx(expected_comparison_unc) assert summary["rprs_deviation_systematic_floor"] == pytest.approx(0.05 * 0.10) assert summary["rprs_deviation_sigma"] == pytest.approx(abs(0.18 - 0.10) / expected_comparison_unc) assert summary["deviation_from_expected_value"] == pytest.approx(0.0) assert summary["ktmf_metric"] <= 5.0 - assert summary["ktmf_metric"] < 3.5 + assert summary["ktmf_metric"] < 3.0 assert np.isnan(summary["tmid_deviation_sigma"]) assert np.isnan(summary["tmid_deviation_minutes"]) assert summary["rprs_deviation_fit_unc"] == pytest.approx(0.01) @@ -3854,6 +4144,62 @@ def test_expected_value_rprs_deviation_uses_combined_uncertainty_with_systematic assert summary["deviation_from_expected_value"] > 0.0 +def test_expected_value_rprs_deviation_uses_model_data_fit_uncertainty(): + transit_model = np.ones(31, dtype=float) + transit_model[12:19] -= 0.0287 + residual_pattern = np.array( + [ + 0.0, 0.006, -0.005, 0.004, -0.006, 0.005, -0.004, 0.006, + -0.005, 0.004, -0.006, 0.005, -0.004, 0.006, -0.005, 0.004, + -0.006, 0.005, -0.004, 0.006, -0.005, 0.004, -0.006, 0.005, + -0.004, 0.006, -0.005, 0.004, -0.006, 0.005, 0.0, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + time=np.linspace(0.0, 1.0, transit_model.size), + data=transit_model + residual_pattern, + dataerr=np.full(transit_model.size, 0.003, dtype=float), + model=transit_model, + transit=transit_model, + airmass=np.ones(transit_model.size, dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.1694, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.0046, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 0.5], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.001, + transit_qc_expected_rprs=0.1589, + transit_qc_expected_rprs_unc=0.0001, + transit_qc_use_deviation_from_expected_transit_in_qc=True, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + expected_fit_unc = np.sqrt( + summary["rprs_deviation_model_fit_unc"] ** 2 + + summary["rprs_deviation_data_fit_unc"] ** 2 + ) + expected_comparison_unc = np.sqrt( + expected_fit_unc ** 2 + + summary["rprs_deviation_expected_unc"] ** 2 + + summary["rprs_deviation_systematic_floor"] ** 2 + ) + assert summary["rprs_deviation_model_fit_unc"] == pytest.approx(0.0046) + assert summary["rprs_deviation_data_fit_unc"] > 0.0 + assert summary["rprs_deviation_fit_unc"] == pytest.approx(expected_fit_unc) + assert summary["rprs_deviation_unc"] == pytest.approx(expected_comparison_unc) + contribution = next( + item for item in summary["ktmf_contributions"] + if item["label"] == "Deviation From Expected Value" + ) + assert "model uncertainty=0.004600" in contribution["detail"] + assert "data/red-noise uncertainty=" in contribution["detail"] + + def test_selected_full_resolution_refit_keeps_expected_value_context(monkeypatch): import exotic.exotic as exotic_module @@ -6235,8 +6581,16 @@ def fake_lc_fitter( assert captured["flags"] == [False] -def test_build_initial_ars_bounds_prefers_published_uncertainty_when_available(): +def test_build_initial_ars_bounds_prefers_published_uncertainty_when_available(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([14.5, 15.5]) + assert build_initial_ars_bounds(15.0, None) == pytest.approx([13.5, 16.5]) + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", False) assert build_initial_ars_bounds(15.0, None) == pytest.approx([11.25, 18.75]) diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index bd335cd3..169c8c29 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -84,27 +84,50 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: sys.modules.setdefault("exotic.api.ld", fake_ld) from exotic.exotic import ( # noqa: E402 + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, INITIAL_RPRS_BOUND_LOWER_SCALE, INITIAL_RPRS_BOUND_UPPER_SCALE, RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, RPRS_SEARCH_BOUND_MAX, RPRS_SEARCH_BOUND_MIN, SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + build_initial_ars_bounds, build_fast_ultranest_lightcurve_series, build_expected_transit_coverage_assessment, evaluate_sparse_posterior_sample_support, extend_sparse_posterior_live_points_if_needed, + final_residual_rejection_keep_mask, fit_final_lightcurve_with_oot_baseline_detrending, finalize_comparison_candidate_full_reduction, refit_selected_fast_comparison_on_full_lightcurve, + configure_ars_range_restriction, + configure_prior_rprs_fallback_on_pinned_posterior, configure_rprs_search_bound_max, + configure_rprs_range_restriction, + should_use_legacy_psf_flux_mode, + should_run_final_fit_phase_residual_clip, + should_run_final_residual_rejection, should_run_fast_ultranest_before_final_run, + should_restrict_ars_range, + should_restrict_rprs_range, + should_use_prior_rprs_when_posterior_pinned, build_single_transit_duration_prior, build_initial_rprs_bounds, run_nested_lightcurve_fit_with_rprs_posterior_retry, ) +@pytest.fixture(autouse=True) +def _disable_prior_centered_range_restrictions(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", False) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + monkeypatch.setattr(exotic_module, "RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR", True) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", False) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + def test_build_initial_rprs_bounds_allows_zero_depth_search_box(): bounds = build_initial_rprs_bounds(0.1) @@ -115,6 +138,35 @@ def test_build_initial_rprs_bounds_allows_zero_depth_search_box(): assert INITIAL_RPRS_BOUND_LOWER_SCALE == pytest.approx(0.0) +def test_build_initial_rprs_bounds_restricts_to_prior_centered_window(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + assert build_initial_rprs_bounds(0.1) == pytest.approx([0.09, 0.11]) + + +def test_build_initial_rprs_bounds_widens_prior_window_for_data_uncertainty(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + assert build_initial_rprs_bounds(0.1, rprs_data_uncertainty=0.006) == pytest.approx([0.09, 0.11]) + assert build_initial_rprs_bounds(0.1, rprs_data_uncertainty=0.02) == pytest.approx([0.04, 0.16]) + + +def test_build_initial_ars_bounds_restricts_fallback_window_to_prior_centered_range(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + assert build_initial_ars_bounds(15.0, None) == pytest.approx([13.5, 16.5]) + assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([14.5, 15.5]) + + def test_build_initial_rprs_bounds_clamps_to_configured_search_ceiling(monkeypatch): import exotic.exotic as exotic_module @@ -133,6 +185,31 @@ def test_configure_rprs_search_bound_max_updates_retry_ceiling(monkeypatch): assert exotic_module.RPRS_SEARCH_BOUND_MAX == pytest.approx(0.4) +def test_configure_prior_centered_range_restrictions_parse_values(): + import exotic.exotic as exotic_module + + assert should_restrict_rprs_range(None) is True + assert should_restrict_ars_range(None) is True + assert should_use_prior_rprs_when_posterior_pinned(None) is True + assert should_restrict_rprs_range("n") is False + assert should_restrict_ars_range(False) is False + assert should_use_prior_rprs_when_posterior_pinned("n") is False + + enabled, percentage = configure_rprs_range_restriction("y", "12.5%") + assert enabled is True + assert percentage == pytest.approx(12.5) + assert exotic_module.RPRS_RANGE_RESTRICTION_ENABLED is True + assert exotic_module.RPRS_RANGE_RESTRICTION_PERCENTAGE == pytest.approx(12.5) + + enabled, percentage = configure_ars_range_restriction("n", None) + assert enabled is False + assert percentage == pytest.approx(ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT) + assert exotic_module.ARS_RANGE_RESTRICTION_ENABLED is False + + assert configure_prior_rprs_fallback_on_pinned_posterior("n") is False + assert exotic_module.RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR is False + + def test_rprs_posterior_retry_clamps_to_configured_search_ceiling(monkeypatch): import exotic.exotic as exotic_module @@ -197,12 +274,282 @@ def fake_lc_fitter( assert fit.rprs_posterior_refit_bounds[1] == pytest.approx(0.5) +def test_rprs_posterior_retry_does_not_escape_configured_prior_range(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + monkeypatch.setattr(exotic_module, "RPRS_SEARCH_BOUND_MAX", 0.5) + captured = {"calls": []} + diagnostics = {"clipped": True, "edge": "upper", "mode": 0.109, "std": 0.04, "bounds": [0.09, 0.25]} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + **kwargs, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + fit = types.SimpleNamespace( + parameters={"tmid": 0.0, "rprs": diagnostics["mode"], "inc": 89.0, "a2": 0.0} + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: dict(diagnostics) if key == "rprs" else None + ) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.3], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert len(captured["calls"]) == 1 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.09, 0.11]) + assert fit.rprs_posterior_refit_applied is False + assert "configured prior-centered search range" in fit.rprs_posterior_refit_note + + +def test_rprs_restriction_uses_explicit_search_prior_instead_of_refined_prior(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + captured = {"bounds": None} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + **kwargs, + ): + captured["bounds"] = { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + } + fit = types.SimpleNamespace(parameters=dict(call_prior)) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: { + "clipped": False, + "edge": None, + "mode": call_prior.get(key, np.nan), + "std": 0.01, + "bounds": captured["bounds"].get(key), + "reason": "posterior support is comfortably inside the sampled bounds.", + } if key == "rprs" else None + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + + run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + {"tmid": 0.0, "rprs": 0.145, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.0, 0.5], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + search_restriction_prior={"rprs": 0.1}, + ) + + assert captured["bounds"]["rprs"] == pytest.approx([0.09, 0.11]) + + +def test_pinned_rprs_without_expansion_reruns_with_prior_value_and_data_error(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR", True) + + def fake_transit(call_times, call_prior): + call_times = np.asarray(call_times, dtype=float) + depth = float(call_prior["rprs"]) ** 2 + return 1.0 - depth * (np.abs(call_times) <= 0.01) + + monkeypatch.setattr(exotic_module, "transit", fake_transit) + + captured = {"calls": []} + pinned_diagnostics = { + "clipped": True, + "edge": "upper", + "mode": 0.11, + "std": 0.006, + "bounds": [0.09, 0.11], + } + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + fixed_parameter_errors=None, + **kwargs, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + "fixed_parameter_errors": dict(fixed_parameter_errors or {}), + }) + model = fake_transit(call_times, call_prior) + fit = types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + transit=np.asarray(model, dtype=float), + model=np.asarray(model, dtype=float), + residuals=np.asarray(call_flux, dtype=float) - np.asarray(model, dtype=float), + airmass_model=np.ones_like(model), + parameters=dict(call_prior), + errors=dict(fixed_parameter_errors or {}), + fixed_parameter_errors=dict(fixed_parameter_errors or {}), + ) + fit.errors.setdefault("tmid", 0.001) + fit.errors.setdefault("ars", 0.1) + fit.errors.setdefault("inc", 0.1) + + def diagnostics(key): + if key == "rprs" and "rprs" in call_bounds: + return dict(pinned_diagnostics) + return { + "clipped": False, + "edge": None, + "mode": call_prior.get(key, np.nan), + "std": 0.001, + "bounds": call_bounds.get(key), + "reason": "posterior support is comfortably inside the sampled bounds.", + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 9) + prior = {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0} + flux = fake_transit(times, prior) + np.array([0.0, 0.004, -0.003, 0.002, -0.004, 0.003, -0.002, 0.004, 0.0]) + fluxerr = np.full(times.shape, 0.003) + airmass = np.ones(times.shape) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + {"tmid": 0.0, "rprs": 0.11, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.09, 0.11], "tmid": [-0.01, 0.01], "ars": [8.0, 12.0], "inc": [84.0, 90.0]}, + max_rprs_retries=0, + max_ars_retries=0, + max_impact_parameter_retries=0, + search_restriction_prior={"rprs": 0.1, "ars": 10.0}, + ) + + assert len(captured["calls"]) == 2 + assert "rprs" in captured["calls"][0]["bounds"] + assert "rprs" not in captured["calls"][1]["bounds"] + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.1) + assert captured["calls"][1]["fixed_parameter_errors"]["rprs"] > 0 + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.1) + assert fit.errors["rprs"] == pytest.approx(fit.empirical_transit_uncertainty["data_rprs_uncertainty"]) + assert fit.empirical_transit_uncertainty["combined_rprs_uncertainty"] == pytest.approx( + fit.empirical_transit_uncertainty["data_rprs_uncertainty"] + ) + assert "prior fallback" in fit.rprs_prior_fallback_note + + def test_fast_ultranest_option_defaults_enabled_and_parses_false_values(): assert should_run_fast_ultranest_before_final_run(None) is True assert should_run_fast_ultranest_before_final_run("n") is False assert should_run_fast_ultranest_before_final_run(False) is False +def test_final_residual_rejection_option_defaults_enabled_and_parses_false_values(): + assert should_run_final_residual_rejection(None) is True + assert should_run_final_residual_rejection("n") is False + assert should_run_final_residual_rejection(False) is False + + +def test_final_fit_phase_residual_clip_option_defaults_enabled_and_parses_false_values(): + assert should_run_final_fit_phase_residual_clip(None) is True + assert should_run_final_fit_phase_residual_clip("n") is False + assert should_run_final_fit_phase_residual_clip(False) is False + + +def test_legacy_psf_flux_mode_option_defaults_modern_and_parses_true_values(): + assert should_use_legacy_psf_flux_mode(None) is False + assert should_use_legacy_psf_flux_mode("legacy") is True + assert should_use_legacy_psf_flux_mode("y") is True + assert should_use_legacy_psf_flux_mode(False) is False + + +def test_final_residual_rejection_keep_mask_flags_large_residual_outlier(): + fit = types.SimpleNamespace( + data=np.ones(8, dtype=float), + residuals=np.array([0.0, 0.001, -0.001, 0.0, 0.001, -0.001, 0.0, 0.20], dtype=float), + ) + + keep_mask, summary = final_residual_rejection_keep_mask(fit, sigma=2.0, min_required_points=5) + + assert keep_mask.tolist() == [True, True, True, True, True, True, True, False] + assert summary["applied"] is True + assert summary["rejected_point_count"] == 1 + assert summary["kept_point_count"] == 7 + + +def test_final_residual_rejection_keep_mask_iterates_until_clean(): + fit = types.SimpleNamespace( + data=np.ones(10, dtype=float), + residuals=np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05, 0.20], dtype=float), + ) + + keep_mask, summary = final_residual_rejection_keep_mask(fit, sigma=2.0, min_required_points=5) + + assert keep_mask.tolist() == [True, True, True, True, True, True, True, True, False, False] + assert summary["applied"] is True + assert summary["rejected_point_count"] == 2 + assert summary["clip_iteration_count"] == 2 + + def test_fast_ultranest_binning_reduces_large_light_curve_to_twenty_points(): times = np.linspace(0.0, 1.0, 80) flux = 1.0 + 0.01 * np.sin(np.linspace(0.0, 2.0 * np.pi, 80)) @@ -519,6 +866,127 @@ def fake_run_nested( assert "a2" not in captured["bounds"] +def test_selected_fast_candidate_final_refit_reruns_after_residual_rejection(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "selected_final_live_point_target", lambda *args, **kwargs: (200, None)) + calls = [] + + def fake_run_nested( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + call_index = len(calls) + calls.append({ + "times": np.asarray(times, dtype=float), + "flux": np.asarray(flux_values, dtype=float), + "source_count": len(times), + }) + residuals = np.zeros(len(times), dtype=float) + if call_index == 0: + residuals[-1] = 1.0 + elif call_index == 1: + residuals[0] = 1.0 + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters={**dict(prior), "rprs": 0.1, "tmid": 0.5, "ars": 10.0, "inc": 89.0}, + errors={"rprs": 0.001, "tmid": 0.001, "ars": 0.1, "inc": 0.1}, + residuals=residuals, + phase=np.linspace(-0.05, 0.05, len(times)), + detrended=np.asarray(flux_values, dtype=float), + transit=np.ones(len(times), dtype=float), + duration_measured=0.04, + duration_expected=0.04, + transit_qc={"status": "pass", "summary": "ok"}, + transit_qc_status="pass", + ) + return fit + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + frame_filter_diagnostics=[], + parameters={ + "rprs": 0.1, + "ars": 10.0, + "per": 1.0, + "tmid": 0.5, + "inc": 89.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + }, + errors={"a0": 0.0, "a1": 0.0, "a2": 0.0, "rprs": 0.001, "tmid": 0.001, "ars": 0.1}, + bounds={ + "rprs": [0.05, 0.15], + "tmid": [0.49, 0.51], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + }, + ) + times = np.linspace(0.0, 1.0, 80) + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": np.ones(80), + "good_unc": np.full(80, 0.01), + "good_airmass": np.linspace(1.0, 1.3, 80), + "good_jd_times": 2460000.0 + times, + "good_target_flux": np.linspace(1000.0, 1080.0, 80), + "good_comp_flux": np.linspace(500.0, 540.0, 80), + "source_indices": np.arange(80), + } + + returned, fit_flux, fit_unc = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + { + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "rprs": 0.1, + "aRs": 10.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + detrend_on_outoftransit_baseline=False, + ) + + assert returned is not None + assert [call["source_count"] for call in calls] == [80, 79, 78] + assert len(fit_flux) == 78 + assert len(fit_unc) == 78 + assert selected_result["good_times"].shape == (78,) + assert selected_result["tflux_fit"].shape == (78,) + assert selected_result["cflux_fit"].shape == (78,) + assert selected_result["source_indices"][0] == 1 + assert selected_result["source_indices"][-1] == 78 + assert returned.final_residual_rejection_applied is True + assert returned.final_residual_rejection_rejected_count == 2 + assert returned.final_residual_rejection["refit_iteration_count"] == 2 + assert returned.final_residual_rejection["rejected_source_indices"] == [79, 0] + assert [item["stage"] for item in returned.frame_filter_diagnostics[-2:]] == [ + "Final residual rejection refit 1", + "Final residual rejection refit 2", + ] + + def test_selected_fast_candidate_final_refit_resets_fixed_baseline_after_linear_detrend(monkeypatch): import exotic.exotic as exotic_module @@ -613,6 +1081,7 @@ def fake_run_nested( "omega": 0.0, }, detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=2, ) assert returned is not None diff --git a/tests/test_inputs.py b/tests/test_inputs.py index bf733aea..68b8c939 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -236,6 +236,24 @@ def test_comp_params_defaults_rprs_search_bound_max_to_half(tmp_path): assert inputs.info_dict["rprs_search_bound_max"] == 0.5 +def test_comp_params_defaults_prior_centered_search_restrictions_to_on(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["restrict_rprs_range"] == "y" + assert inputs.info_dict["restrict_rprs_range_percentage"] == 10.0 + assert inputs.info_dict["restrict_ars_range"] == "y" + assert inputs.info_dict["restrict_ars_range_percentage"] == 10.0 + + def test_comp_params_defaults_sparse_posterior_live_point_retry_to_yes(tmp_path): init_data = { "user_info": {}, @@ -596,6 +614,44 @@ def test_comp_params_reads_rprs_search_bound_max_from_optional_info(tmp_path): assert inputs.info_dict["rprs_search_bound_max"] == 0.35 +def test_comp_params_reads_prior_centered_search_restrictions_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "restrict_Rp/Rs_range": "n", + "restrict_Rp/Rs_range_percentage": 15, + "restrict_a/Rs_range": "y", + "restrict_a/Rs_range_percentage": "12.5%", + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["restrict_rprs_range"] == "n" + assert inputs.info_dict["restrict_rprs_range_percentage"] == 15 + assert inputs.info_dict["restrict_ars_range"] == "y" + assert inputs.info_dict["restrict_ars_range_percentage"] == "12.5%" + + +def test_comp_params_reads_prior_rprs_fallback_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"Use Prior Rp/Rs When Posterior Pinned? (y/n)": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_prior_rprs_when_posterior_pinned"] == "n" + + def test_comp_params_reads_fast_ultranest_before_final_run_from_optional_info(tmp_path): init_data = { "user_info": {}, @@ -776,6 +832,52 @@ def test_comp_params_reads_use_psf_photometry_from_optional_info(tmp_path): assert inputs.info_dict["use_psf_photometry"] == "n" +def test_comp_params_reads_use_legacy_psf_flux_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_legacy_psf_flux": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_legacy_psf_flux"] == "y" + + +def test_comp_params_reads_psf_seed_track_directory_from_optional_info(tmp_path): + seed_dir = tmp_path / "old_run" + init_data = { + "user_info": {}, + "optional_info": {"psf_seed_track_directory": str(seed_dir)}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["psf_seed_track_directory"] == str(seed_dir) + + +def test_comp_params_reads_final_fit_phase_residual_clip_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"run_final_fit_phase_residual_clip": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["run_final_fit_phase_residual_clip"] == "n" + + def test_comp_params_reads_use_aperture_photometry_from_optional_info(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_output_files.py b/tests/test_output_files.py index f0888d4b..fc01d308 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -10,6 +10,7 @@ PRIOR_OBSERVABLE_DEPTH_LABEL, AIDOutputFiles, OutputFiles, + fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, save_comp_star_calibration_summary, ) @@ -631,6 +632,171 @@ def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertaintie assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" +def test_fit_empirical_transit_uncertainty_uses_residual_scatter_and_point_counts(): + fit = DummyFit() + fit.parameters["rprs"] = 0.1 + fit.errors["rprs"] = 0.002 + fit.transit = np.array([1.0, 1.0, 0.99, 0.99, 1.0, 1.0]) + fit.model = np.array(fit.transit) + fit.data = fit.model + np.array([0.0, 0.01, -0.01, 0.01, -0.01, 0.0]) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full_like(fit.model, 0.01) + fit.airmass_model = np.ones_like(fit.model) + + empirical = fit_empirical_transit_uncertainty(fit) + + assert empirical["available"] is True + assert empirical["in_transit_point_count"] == 2 + assert empirical["out_of_transit_point_count"] == 4 + assert empirical["data_rprs_uncertainty"] == pytest.approx( + empirical["depth_uncertainty_fraction"] / 0.2 + ) + assert empirical["depth_flux_scatter_fraction"] == pytest.approx( + empirical["residual_scatter"] + ) + assert empirical["data_rprs_standard_error"] == pytest.approx( + empirical["depth_standard_error_fraction"] / 0.2 + ) + assert empirical["data_rprs_flux_scatter_uncertainty"] == pytest.approx( + empirical["depth_flux_scatter_fraction"] / 0.2 + ) + assert empirical["red_noise_beta_factor"] >= 1.0 + assert empirical["data_rprs_uncertainty"] >= empirical["data_rprs_standard_error"] + assert empirical["data_rprs_flux_scatter_uncertainty"] > empirical["data_rprs_standard_error"] + assert empirical["combined_rprs_uncertainty"] > empirical["model_rprs_uncertainty"] + assert empirical["baseline_red_noise_uncertainty_fraction"] >= ( + empirical["baseline_standard_error_fraction"] + ) + assert empirical["depth_uncertainty_fraction"] >= empirical["baseline_red_noise_uncertainty_fraction"] + + +def test_fit_empirical_transit_uncertainty_uses_data_only_for_prior_fallback(): + fit = DummyFit() + fit.parameters["rprs"] = 0.1 + fit.errors["rprs"] = 0.5 + fit.rprs_prior_fallback_applied = True + fit.rprs_prior_fallback_note = "Applied Rp/R* prior fallback." + fit.transit = np.array([1.0, 1.0, 0.99, 0.99, 1.0, 1.0]) + fit.model = np.array(fit.transit) + fit.data = fit.model + np.array([0.0, 0.01, -0.01, 0.01, -0.01, 0.0]) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full_like(fit.model, 0.01) + fit.airmass_model = np.ones_like(fit.model) + + empirical = fit_empirical_transit_uncertainty(fit) + + assert empirical["rprs_uncertainty_basis"] == "prior_assumed_data_only" + assert np.isnan(empirical["model_rprs_uncertainty"]) + assert empirical["combined_rprs_uncertainty"] == pytest.approx( + empirical["data_rprs_uncertainty"] + ) + assert empirical["conservative_rprs_uncertainty"] == pytest.approx( + empirical["data_rprs_uncertainty"] + ) + + +def test_final_planetary_params_reports_model_and_red_noise_uncertainties(tmp_path): + fit = DummyFit() + fit.parameters["rprs"] = 0.1 + fit.errors["rprs"] = 0.002 + fit.transit = np.array([1.0, 1.0, 0.99, 0.99, 1.0, 1.0]) + fit.model = np.array(fit.transit) + fit.data = fit.model + np.array([0.0, 0.01, -0.01, 0.01, -0.01, 0.0]) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full_like(fit.model, 0.01) + fit.airmass_model = np.ones_like(fit.model) + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] == ( + final_params["Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty"] + ) + assert final_params["Ratio of Planet to Stellar Radius (Rp/R*) model-fit uncertainty"] == ( + "0.1 +/- 0.002" + ) + assert "Ratio of Planet to Stellar Radius (Rp/R*) data-fit red-noise uncertainty" in final_params + assert "Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty" in final_params + assert "Ratio of Planet to Stellar Radius (Rp/R*) data-fit standard-error estimate" in final_params + assert "Ratio of Planet to Stellar Radius (Rp/R*) flux-scatter equivalent" in final_params + assert "Transit depth red-noise uncertainty" in final_params + assert "Transit depth data-fit standard-error estimate" in final_params + assert "Transit depth flux-scatter equivalent" in final_params + assert final_params[AREA_DEPTH_LABEL] == ( + final_params[f"{AREA_DEPTH_LABEL} model+red-noise uncertainty"] + ) + assert final_params["Mid-Transit Time (Tmid)"] == ( + final_params["Mid-Transit Time (Tmid) model+red-noise uncertainty"] + ) + assert "Mid-Transit Time (Tmid) model-fit uncertainty" in final_params + assert final_params["Orbital Inclination (inc)"] == ( + final_params["Orbital Inclination (inc) model+red-noise uncertainty"] + ) + assert "Orbital Inclination (inc) model-fit uncertainty" in final_params + assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == ( + final_params["Ratio of Distance to Stellar Radius (a/Rs) model+red-noise uncertainty"] + ) + assert "Ratio of Distance to Stellar Radius (a/Rs) model-fit uncertainty" in final_params + assert final_params["Impact Parameter (b)"] == ( + final_params["Impact Parameter (b) model+red-noise uncertainty"] + ) + assert "Impact Parameter (b) model-fit uncertainty" in final_params + assert f"{AREA_DEPTH_LABEL} model-fit uncertainty" in final_params + assert f"{AREA_DEPTH_LABEL} data-fit red-noise uncertainty" in final_params + assert "Flux baseline red-noise uncertainty" in final_params + assert "Flux baseline standard-error estimate" in final_params + assert "Red-noise beta factor" in final_params + assert final_params["Data-fit uncertainty point counts"] == "2 in transit, 4 out of transit" + assert "primary Rp/R*" in final_params["Uncertainty interpretation note"] + assert "baseline component" in final_params["Uncertainty interpretation note"] + assert "time-binning" in final_params["Uncertainty interpretation note"] + + +def test_final_planetary_params_reports_prior_fallback_data_only_uncertainty(tmp_path): + fit = DummyFit() + fit.parameters["rprs"] = 0.1 + fit.errors["rprs"] = 0.5 + fit.rprs_prior_fallback_applied = True + fit.rprs_prior_fallback_prior_value = 0.1 + fit.rprs_prior_fallback_original_fit_value = 0.11 + fit.rprs_prior_fallback_data_uncertainty = 0.02 + fit.rprs_prior_fallback_note = "Applied Rp/R* prior fallback." + fit.transit = np.array([1.0, 1.0, 0.99, 0.99, 1.0, 1.0]) + fit.model = np.array(fit.transit) + fit.data = fit.model + np.array([0.0, 0.01, -0.01, 0.01, -0.01, 0.0]) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full_like(fit.model, 0.01) + fit.airmass_model = np.ones_like(fit.model) + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Rp/R* uncertainty basis"] == "prior_assumed_data_only" + assert not any("Rp/R*) model-fit uncertainty" in key for key in final_params) + assert not any("Rp/R*) model+standard-error" in key for key in final_params) + assert "input prior Rp/R* value with a data-only" in final_params["Uncertainty interpretation note"] + assert "Rp/R* prior fallback note" in final_params + assert any("prior-assumed data-only uncertainty" in key for key in final_params) + + def test_final_planetary_params_can_publish_accepted_copy_to_root(tmp_path): fit = DummyFit() (tmp_path / "temp").mkdir() diff --git a/tests/test_plots.py b/tests/test_plots.py index 2b2a95a2..e8ef80bb 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -6,12 +6,15 @@ from matplotlib.axes import Axes from exotic.plots import ( + _format_parameter_value, plot_fov, plot_adaptive_aperture_diagnostics, plot_comp_star_candidate_lightcurve_fits, plot_final_lightcurve, plot_individual_comp_star_calibration_series, + plot_ktmf_qc_metrics, plot_obs_stats, + plot_prior_posterior_comparison, plot_stellar_variability, ) @@ -22,6 +25,14 @@ def __init__(self): self.airmass = np.array([1.1, 1.2, 1.3]) +def test_format_parameter_value_uses_uncertainty_precision_without_scientific_notation(): + assert ( + _format_parameter_value(2461197.8645824, 0.0005037355680314821, split_error=True) + == "2461197.86458\n+/- 0.00050" + ) + assert _format_parameter_value(89.3511, 2.16, unit="deg") == "89.4 +/- 2.2 deg" + + def test_plot_obs_stats_applies_relative_flux_mask(tmp_path, monkeypatch): fit = DummyFit() psf_rows = np.arange(35, dtype=float).reshape(5, 7) @@ -381,3 +392,219 @@ def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=Fal } assert (tmp_path / "FinalLightCurve_Target_2026-03-09.png").exists() assert (tmp_path / "FinalLightCurve_Target_2026-03-09.pdf").exists() + + +def test_plot_final_lightcurve_draws_data_scatter_uncertainty_band(tmp_path, monkeypatch): + captured = [] + original_fill_between = Axes.fill_between + + def spy_fill_between(self, x, y1, y2=0, *args, **kwargs): + captured.append({ + "x": np.asarray(x, dtype=float), + "y1": np.asarray(y1, dtype=float), + "y2": np.asarray(y2, dtype=float), + "color": kwargs.get("color"), + "alpha": kwargs.get("alpha"), + "label": kwargs.get("label"), + }) + return original_fill_between(self, x, y1, y2, *args, **kwargs) + + monkeypatch.setattr(Axes, "fill_between", spy_fill_between) + + class DummyFinalFit: + def __init__(self): + self.phase_upsample = np.linspace(-0.05, 0.05, 41) + depth_shape = np.exp(-0.5 * (self.phase_upsample / 0.015) ** 2) + self.transit_upsample = 1.0 - 0.01 * depth_shape + self.time_upsample = self.phase_upsample.copy() + self.phase = self.phase_upsample.copy() + self.transit = self.transit_upsample.copy() + self.model = self.transit.copy() + residual_pattern = 0.02 * np.sin(np.linspace(0, 6 * np.pi, self.model.size)) + self.data = self.model + residual_pattern + self.residuals = self.data - self.model + self.dataerr = np.full_like(self.model, 0.02) + self.parameters = {"rprs": 0.1} + self.errors = {"rprs": 0.001} + + def transit_model_uncertainty(self, times): + return self.transit_upsample - 0.001, self.transit_upsample + 0.001 + + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False): + fig, axes = plt.subplots(2, 1) + axes[0].plot(self.phase_upsample, self.transit_upsample, 'r-', label='model') + axes[0].legend(loc='best') + return fig, axes + + plot_final_lightcurve( + DummyFinalFit(), + high_res=np.ones(41), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + purple_bands = [item for item in captured if item["color"] == "#6a1b9a"] + assert purple_bands + assert all(item["label"] == "_nolegend_" for item in purple_bands) + assert all(item["alpha"] <= 0.16 for item in purple_bands) + assert any(np.nanmax(np.abs(item["y2"] - item["y1"])) > 0.001 for item in purple_bands) + + +def test_plot_final_lightcurve_marks_final_residual_rejections(tmp_path, monkeypatch): + captured = [] + original_scatter = Axes.scatter + + def spy_scatter(self, x, y, *args, **kwargs): + captured.append({ + "x": np.asarray(x, dtype=float), + "y": np.asarray(y, dtype=float), + "label": kwargs.get("label"), + "color": kwargs.get("color"), + }) + return original_scatter(self, x, y, *args, **kwargs) + + monkeypatch.setattr(Axes, "scatter", spy_scatter) + + class DummyFinalFit: + def __init__(self): + self.phase_upsample = np.linspace(-0.05, 0.05, 5) + self.transit_upsample = np.ones(5) + self.final_residual_rejection = { + "applied": True, + "rejected_phase": [0.01], + "rejected_flux": [0.92], + "rejected_residual_percent": [-8.0], + } + + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False): + fig, axes = plt.subplots(2, 1) + return fig, axes + + plot_final_lightcurve( + DummyFinalFit(), + high_res=np.ones(5), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + residual_rejection_points = [ + item for item in captured + if item["label"] == "_nolegend_" and item["color"] == "red" + ] + assert len(residual_rejection_points) == 2 + np.testing.assert_allclose(residual_rejection_points[0]["x"], [0.01]) + np.testing.assert_allclose(residual_rejection_points[1]["y"], [-8.0]) + + +def test_plot_prior_posterior_comparison_omits_prior_fallback_rprs(tmp_path, monkeypatch): + captured_text = [] + original_text = Axes.text + + def spy_text(self, x, y, s, *args, **kwargs): + captured_text.append(str(s)) + return original_text(self, x, y, s, *args, **kwargs) + + monkeypatch.setattr(Axes, "text", spy_text) + + class DummyFit: + def __init__(self): + self.parameters = { + "tmid": 1.012, + "rprs": 0.1, + "ars": 10.6, + "inc": 88.2, + } + self.errors = { + "tmid": 0.002, + "rprs": 0.005, + "ars": 0.4, + "inc": 0.3, + } + self.rprs_prior_fallback_applied = True + self.empirical_transit_uncertainty = { + "available": True, + "combined_rprs_uncertainty": 0.02, + "rprs_uncertainty_basis": "prior_assumed_data_only", + } + + planet_dict = { + "midT": 1.0, + "midTUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.003, + "aRs": 10.0, + "aRsUnc": 0.2, + "inc": 89.0, + "incUnc": 0.4, + } + + output = plot_prior_posterior_comparison( + DummyFit(), + planet_dict, + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + assert output == tmp_path / "PriorPosteriorComparison_Target_2026-03-09.png" + assert output.exists() + assert (tmp_path / "PriorPosteriorComparison_Target_2026-03-09.pdf").exists() + assert any("Rp/R* omitted: prior value assumed, not measured" in text for text in captured_text) + assert any("Prior\n" in text and "Posterior\n" in text for text in captured_text) + assert not any("BJD_TDB" in text for text in captured_text) + assert not any("Prior epoch" in text for text in captured_text) + assert not any("Posterior (Prior)" in text for text in captured_text) + + +def test_plot_ktmf_qc_metrics_writes_outputs_and_annotations(tmp_path, monkeypatch): + captured_text = [] + original_text = Axes.text + + def spy_text(self, x, y, s, *args, **kwargs): + captured_text.append(str(s)) + return original_text(self, x, y, s, *args, **kwargs) + + monkeypatch.setattr(Axes, "text", spy_text) + + class DummyKTMFFit: + def __init__(self): + self.transit_qc = { + "status": "fail", + "ktmf_metric": 3.07, + "ktmf_contributions": [ + { + "label": "Model Evidence", + "available": True, + "points": 0.11, + "max_points": 0.89, + "score": 0.12, + "detail": "Delta BIC=-2.00, Delta chi2=6.91", + }, + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.14, + "max_points": 0.78, + "score": 0.18, + "detail": "2.3437%", + }, + ], + } + + output = plot_ktmf_qc_metrics( + DummyKTMFFit(), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + assert output == tmp_path / "KTMF_QC_Target_2026-03-09.png" + assert output.exists() + assert (tmp_path / "KTMF_QC_Target_2026-03-09.pdf").exists() + assert any("KTMF\n3.07 / 5.00\nMARGINAL" in text for text in captured_text) + assert any("0.11 / 0.89" in text for text in captured_text) + assert any("Delta BIC=-2.00" in text for text in captured_text) From 7bf391ef337767ff161139d535e0535e7fdf20a0 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 20 Jun 2026 08:56:56 +1000 Subject: [PATCH 070/116] decimal places --- exotic/api/elca.py | 84 ++++++++++++++++++++++++++++++------- tests/test_elca_baseline.py | 10 +++++ 2 files changed, 80 insertions(+), 14 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 41576ba1..32206491 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -430,6 +430,43 @@ def round_to_2(*args): return round(x, roundval) +def _decimal_places_for_two_sigfig_error(error): + try: + error = float(error) + except (TypeError, ValueError): + return None + + if not np.isfinite(error): + return None + if error == 0: + return 2 + + exponent = int(np.floor(np.log10(abs(error)))) + return max(0, 1 - exponent) + + +def format_value_error_for_plot(value, error): + """Format value/error text with a two-significant-figure uncertainty.""" + try: + value = float(value) + except (TypeError, ValueError): + value = np.nan + + try: + error = float(error) + except (TypeError, ValueError): + error = np.nan + + decimal_places = _decimal_places_for_two_sigfig_error(error) + if decimal_places is None: + value_text = f"{value:.6f}".rstrip('0').rstrip('.') if np.isfinite(value) else "n/a" + return value_text, "n/a" + + value_text = f"{value:.{decimal_places}f}" if np.isfinite(value) else "n/a" + error_text = f"{error:.{decimal_places}f}" if np.isfinite(error) else "n/a" + return value_text, error_text + + # average data into bins of dt from start to finish def time_bin(time, flux, dt=1. / (60 * 24)): bins = int(np.floor((max(time) - min(time)) / dt)) @@ -3639,25 +3676,34 @@ def plot_bestfit( rprs_error_for_depth = self._combined_rprs_uncertainty_for_reporting() rprs2err = 2 * self.parameters['rprs'] * rprs_error_for_depth rprs_prior_marker = " (Prior)" if getattr(self, 'rprs_prior_fallback_applied', False) else "" + rprs2_text, rprs2err_text = format_value_error_for_plot(rprs2, rprs2err) lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s%s" % ( - str(round_to_2(rprs2, rprs2err)), - str(round_to_2(rprs2err)), + rprs2_text, + rprs2err_text, rprs_prior_marker, ) tmid_error_for_plot = self._model_data_uncertainty_for_reporting('tmid') if not np.isfinite(tmid_error_for_plot): tmid_error_for_plot = self.errors.get('tmid', 0) + tmid_text, tmid_error_text = format_value_error_for_plot( + self.parameters['tmid'], + tmid_error_for_plot, + ) lclabel2 = r"$T_{mid}$ = %s $\pm$ %s BJD$_{TDB}$" % ( - str(round_to_2(self.parameters['tmid'], tmid_error_for_plot)), - str(round_to_2(tmid_error_for_plot)) + tmid_text, + tmid_error_text, ) lclabel = lclabel1 + "\n" + lclabel2 if show_flux_baseline_label and 'a0' in self.parameters: + a0_text, a0_error_text = format_value_error_for_plot( + self.parameters['a0'], + self.errors.get('a0', 0), + ) lclabel3 = r"$a_0$ = %s $\pm$ %s" % ( - str(round_to_2(self.parameters['a0'], self.errors.get('a0', 0))), - str(round_to_2(self.errors.get('a0', 0))) + a0_text, + a0_error_text, ) lclabel += "\n" + lclabel3 @@ -4171,14 +4217,19 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p rprs2 = self.lc_data[0]['priors']['rprs']**2 rprs2err = 2*self.lc_data[0]['priors']['rprs']*self.lc_data[0]['errors']['rprs'] + rprs2_text, rprs2err_text = format_value_error_for_plot(rprs2, rprs2err) lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( - str(round_to_2(rprs2, rprs2err)), - str(round_to_2(rprs2err)) + rprs2_text, + rprs2err_text, ) + tmid_text, tmid_error_text = format_value_error_for_plot( + self.parameters['tmid'], + self.errors.get('tmid',0), + ) lclabel2 = r"$T_{mid}$ = %s $\pm$ %s BJD$_{TDB}$" %( - str(round_to_2(self.parameters['tmid'], self.errors.get('tmid',0))), - str(round_to_2(self.errors.get('tmid',0))) + tmid_text, + tmid_error_text, ) lclabel = lclabel1 + "\n" + lclabel2 @@ -4314,14 +4365,19 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): rprs2 = self.parameters['rprs']**2 rprs2err = 2*self.parameters['rprs']*self.errors['rprs'] + rprs2_text, rprs2err_text = format_value_error_for_plot(rprs2, rprs2err) lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( - str(round_to_2(rprs2, rprs2err)), - str(round_to_2(rprs2err)) + rprs2_text, + rprs2err_text, ) + tmid_text, tmid_error_text = format_value_error_for_plot( + self.parameters['tmid'], + self.errors.get('tmid',0), + ) lclabel2 = r"$T_{mid}$ = %s $\pm$ %s BJD$_{TDB}$" %( - str(round_to_2(self.parameters['tmid'], self.errors.get('tmid',0))), - str(round_to_2(self.errors.get('tmid',0))) + tmid_text, + tmid_error_text, ) lclabel = lclabel1 + "\n" + lclabel2 diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 9ee0cf65..5f74ccc1 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -375,6 +375,14 @@ def test_plot_bestfit_can_hide_flux_baseline_label(monkeypatch, tmp_path): plt.close(fig) +def test_format_value_error_for_plot_preserves_two_sigfig_uncertainty_places(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + + assert elca.format_value_error_for_plot(0.027, 0.05) == ("0.027", "0.050") + assert elca.format_value_error_for_plot(2461209.81, 0.087) == ("2461209.810", "0.087") + assert elca.format_value_error_for_plot(89.3511, 2.16) == ("89.4", "2.2") + + def test_plot_bestfit_marks_prior_rprs_fallback_uncertainty(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -404,6 +412,8 @@ def test_plot_bestfit_marks_prior_rprs_fallback_uncertainty(monkeypatch, tmp_pat legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) assert "(Prior)" in legend_text + assert "0.0100" in legend_text + assert "0.0040" in legend_text plt.close(fig) From 67f98ae9c7fa3bcd9f7c4d7527b5e6359dd4856d Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 22 Jun 2026 00:35:50 +1000 Subject: [PATCH 071/116] A lot of AAVSO AID related adjustements but also some partial tranit stuff Added partial-transit geometry prior handling: fixes geometry to priors for one-sided/no-out-of-transit partial light curves, annotates final params/QC, and adjusts KTMF scoring accordingly. Reworked transit QC/KTMF logic: removed model-evidence contribution, updated pass/marginal wording and threshold behavior, and expanded diagnostic reporting. Added automatic comparison/calibration star selection from field detections/catalog color matching, with new optional input settings. Added ensemble comparison photometry support, including output/log handling and plot skips where a single reference centroid is unavailable. Improved comparison-candidate ranking with scatter-gated KTMF/projected-scatter scoring and richer selection-pass/final-refit diagnostics. Enhanced stellar variability/AID reference handling: derives selected comparison catalog magnitudes from anchors/direct field catalog data and records derived-reference metadata. Updated plots/output files: single-parameter triangle Gaussian overlay, AAVSO assumption labels, prior-geometry notes, ensemble labels, and expanded candidate decision fields. Added/updated tests across QC, partial coverage, comparison selection, ensemble flux, inputs, AID/variability, plots, and output files. --- ...FailedFitSummary_HAT-P-32b_2026-04-28.json | 35 - exotic/api/elca.py | 54 + exotic/exotic.py | 2222 +++++++++++++++-- exotic/inputs.py | 20 +- exotic/output_files.py | 158 +- exotic/plots.py | 29 +- ...FailedFitSummary_HAT-P-32b_2026-04-28.json | 35 - tests/test_centroid_wcs.py | 66 + tests/test_exotic_proper_motion.py | 257 +- tests/test_exotic_rprs_retry.py | 176 ++ tests/test_inputs.py | 6 + tests/test_nextastro_variability.py | 249 +- tests/test_output_files.py | 81 +- tests/test_plots.py | 40 +- 14 files changed, 3157 insertions(+), 271 deletions(-) delete mode 100644 .pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json delete mode 100644 manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json diff --git a/.pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json b/.pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json deleted file mode 100644 index fe408498..00000000 --- a/.pytest_tmp_codex/test_fit_ranked_comparison_cal0/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "planet_name": "HAT-P-32 b", - "observation_date": "2026-04-28", - "comparison_star": 1, - "comparison_label": "Comp 1", - "comparison_position": [ - 100.0, - 200.0 - ], - "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", - "failure_reason": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10).", - "fit_diagnostics": { - "input_point_count": 6, - "has_reference_flux": true, - "relative_flux_point_count": 6, - "sigma_clip_point_count": 4, - "usable_point_count": 4, - "failed_stage": "transit_qc", - "failure_reason": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." - }, - "parameter_summary": null, - "fit_point_count": 6, - "eebls_snr": NaN, - "transit_delta_bic": NaN, - "residual_scatter": 0.0, - "ktmf_metric": NaN, - "ktmf_contributions": [], - "transit_qc": { - "status": "fail", - "summary": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." - }, - "saved_debug_series": null, - "saved_bestfit_plot": null, - "archive_errors": [] -} \ No newline at end of file diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 32206491..12a59929 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -3082,6 +3082,59 @@ def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs= if row == geometry_index and col < row: self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='y') + def _overlay_single_parameter_triangle_gaussian(self, fig, payload): + if not hasattr(fig, 'axes') or len(fig.axes) != 1: + return + sampled_keys = list(payload.get('sampled_keys', [])) + if len(sampled_keys) != 1: + return + + ax = fig.axes[0] + display_points = np.asarray(payload.get('display_points', []), dtype=float) + if display_points.ndim != 2 or display_points.shape[1] != 1: + return + + values = display_points[:, 0] + values = values[np.isfinite(values)] + if values.size < 2: + return + + try: + center = float(payload.get('mask_centers', [np.nan])[0]) + sigma = float(payload.get('mask_errors', [np.nan])[0]) + lower, upper = [ + float(value) + for value in np.asarray(payload.get('ranges', [[np.nan, np.nan]])[0], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return + + if not np.isfinite(center): + center = float(np.nanmedian(values)) + if not np.isfinite(sigma) or sigma <= 0: + sigma = float(np.nanstd(values)) + if not np.isfinite(sigma) or sigma <= 0: + return + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + lower, upper = float(np.nanmin(values)), float(np.nanmax(values)) + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return + + x_values = np.linspace(lower, upper, 300) + y_values = np.exp(-0.5 * ((x_values - center) / sigma) ** 2) + y_max = y_values.max() if y_values.size else np.nan + if not np.isfinite(y_max) or y_max <= 0: + return + axis_top = ax.get_ylim()[1] + if not np.isfinite(axis_top) or axis_top <= 0: + axis_top = 1.0 + y_values = y_values / y_max * axis_top * 0.90 + + ax.plot(x_values, y_values, color='#c2410c', linewidth=1.5, label='Gaussian') + ax.axvline(center, color='#c2410c', linestyle='--', linewidth=1.0, label='Peak fit') + ax.set_ylim(0, max(axis_top, float(np.nanmax(y_values)) * 1.05)) + ax.legend(loc='best', fontsize=8, frameon=False) + def _adjust_triangle_plot_layout(self, fig): if not hasattr(fig, 'subplots_adjust'): return @@ -3866,6 +3919,7 @@ def plot_triangle(self, plot_title=None, zoom_sigma=None): fig.set_size_inches(fig_size, fig_size, forward=True) if plot_title and hasattr(fig, 'suptitle'): fig.suptitle(plot_title, fontsize=13, y=0.99) + self._overlay_single_parameter_triangle_gaussian(fig, payload) self._adjust_triangle_plot_layout(fig) self._overlay_triangle_plot_geometry_histograms( fig, diff --git a/exotic/exotic.py b/exotic/exotic.py index c7104bb2..e9e7d597 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -284,6 +284,7 @@ PARTIAL_COVERAGE_ARS_POSTERIOR_MAX_RETRIES = 0 PARTIAL_COVERAGE_IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES = 0 PROMISING_PARTIAL_COMPARISON_KTMF_MIN = 3.0 +COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER = 1.5 SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT = True SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV = "EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY" SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT = 5 @@ -364,6 +365,12 @@ REFERENCE_FALLBACK_DETECTION_MIN_AREA_PIXELS = 3 REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS = 10.0 REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS = 50.0 +AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT = 10 +AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MIN_RATIO = 0.5 +AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MAX_RATIO = 2.0 +AUTOMATIC_CALIBRATION_SELECTOR_MAX_DETECTIONS = 1000 +AUTOMATIC_CALIBRATION_SELECTOR_DETECTION_PERCENTILE = 98.0 +AUTOMATIC_CALIBRATION_SELECTOR_COLOR_MATCH_RADIUS_ARCSEC = 20.0 BAD_PIXEL_DETECTION_FRACTION = 0.30 BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 BAD_PIXEL_PROGRESS_LOG_INTERVAL = 25 @@ -392,9 +399,9 @@ TRANSIT_QC_DEVIATION_SIGMA_DEFAULT = 5.0 TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION = 0.05 TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS = { - 'model_evidence': 0.3, - 'deviation_from_expected_value': 1.5, + 'deviation_from_expected_value': 2.0, 'residual_scatter': 0.7, + 'residual_flatness': 1.0, 'duration_consistency': 0.75, 'eebls_depth_snr': 1.3, 'sampling': 0.7, @@ -478,6 +485,22 @@ def annotate_out_of_transit_baseline_parameter_fit( fit.oot_baseline_parameter_fit_a2_error = a2_error +def annotate_partial_transit_geometry_prior_assumption(fit, payload): + if fit is None: + return + + payload = payload if isinstance(payload, dict) else {} + fit.partial_transit_geometry_prior_assumption_applied = bool(payload.get('applied', False)) + fit.partial_transit_geometry_prior_assumption_mode = payload.get('mode') + fit.partial_transit_geometry_prior_assumption_note = payload.get('note') + fit.partial_transit_geometry_prior_assumption_fixed_parameters = list( + payload.get('fixed_parameters') or [] + ) + fit.partial_transit_geometry_prior_assumption_sampled_parameters = list( + payload.get('sampled_parameters') or [] + ) + + def annotate_final_fit_prefit_refinement( fit, applied, @@ -1405,6 +1428,247 @@ def transit_qc_residual_scatter_score( return float(np.clip(numerator / denominator, 0.0, 1.0)) +def transit_qc_flatness_declining_score(value, full_credit, zero_credit, decay_rate=3.0): + try: + value = float(value) + full_credit = float(full_credit) + zero_credit = float(zero_credit) + decay_rate = float(decay_rate) + except (TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(value) + or value < 0 + or not np.isfinite(full_credit) + or full_credit < 0 + or not np.isfinite(zero_credit) + or zero_credit <= full_credit + or not np.isfinite(decay_rate) + or decay_rate <= 0 + ): + return np.nan + + if value <= full_credit: + return 1.0 + if value >= zero_credit: + return 0.0 + + interval_fraction = (value - full_credit) / (zero_credit - full_credit) + numerator = np.exp(-decay_rate * interval_fraction) - np.exp(-decay_rate) + denominator = 1.0 - np.exp(-decay_rate) + return float(np.clip(numerator / denominator, 0.0, 1.0)) + + +def robust_sigma(values): + values = np.asarray(values, dtype=float) + finite = values[np.isfinite(values)] + if finite.size == 0: + return np.nan + + center = float(np.nanmedian(finite)) + mad = float(np.nanmedian(np.abs(finite - center))) + if np.isfinite(mad) and mad > 0: + return float(1.4826 * mad) + + scatter = float(np.nanstd(finite, ddof=1)) if finite.size > 1 else 0.0 + return scatter if np.isfinite(scatter) else np.nan + + +def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points=12): + summary = { + 'available': False, + 'score': np.nan, + 'point_count': 0, + 'scatter': np.nan, + 'trend_strength': np.nan, + 'curve_strength': np.nan, + 'scatter_ratio': np.nan, + 'trend_score': np.nan, + 'curve_score': np.nan, + 'scatter_stability_score': np.nan, + 'dominant': None, + 'detail': 'residual flatness unavailable', + } + + residuals = np.asarray(residuals, dtype=float).reshape(-1) + if residuals.size == 0: + return summary + + if coordinates is None: + coordinates = np.arange(residuals.size, dtype=float) + else: + coordinates = np.asarray(coordinates, dtype=float).reshape(-1) + if coordinates.shape != residuals.shape: + coordinates = np.arange(residuals.size, dtype=float) + + finite = np.isfinite(residuals) & np.isfinite(coordinates) + point_count = int(np.count_nonzero(finite)) + summary['point_count'] = point_count + if point_count < int(min_points): + summary['detail'] = f"not enough residual points ({point_count} < {int(min_points)})" + return summary + + residuals = residuals[finite] + coordinates = coordinates[finite] + order = np.argsort(coordinates) + residuals = residuals[order] + coordinates = coordinates[order] + + centered = residuals - float(np.nanmedian(residuals)) + scatter = robust_sigma(centered) + if not np.isfinite(scatter): + summary['detail'] = "residual scatter unavailable" + return summary + if scatter <= np.finfo(float).eps: + summary.update({ + 'available': True, + 'score': 1.0, + 'scatter': float(scatter), + 'trend_strength': 0.0, + 'curve_strength': 0.0, + 'scatter_ratio': 1.0, + 'trend_score': 1.0, + 'curve_score': 1.0, + 'scatter_stability_score': 1.0, + 'dominant': 'flat', + 'detail': f"flat residuals; n={point_count}", + }) + return summary + + normalized = centered / scatter + coordinate_min = float(np.nanmin(coordinates)) + coordinate_max = float(np.nanmax(coordinates)) + if not np.isfinite(coordinate_min) or not np.isfinite(coordinate_max) or coordinate_max <= coordinate_min: + x01 = np.linspace(0.0, 1.0, point_count) + else: + x01 = (coordinates - coordinate_min) / (coordinate_max - coordinate_min) + x = 2.0 * x01 - 1.0 + + trend_strength = np.nan + try: + trend_design = np.column_stack([np.ones(point_count, dtype=float), x]) + trend_coeff, *_ = np.linalg.lstsq(trend_design, normalized, rcond=None) + trend_model = trend_design @ trend_coeff + trend_strength = float(np.nanstd(trend_model - np.nanmean(trend_model))) + except Exception: + trend_strength = np.nan + + curve_strength = np.nan + try: + centered_quadratic = x ** 2 - float(np.nanmean(x ** 2)) + structure_design = np.column_stack([ + np.ones(point_count, dtype=float), + x, + centered_quadratic, + np.sin(2.0 * np.pi * x01), + np.cos(2.0 * np.pi * x01), + np.sin(4.0 * np.pi * x01), + np.cos(4.0 * np.pi * x01), + ]) + structure_coeff, *_ = np.linalg.lstsq(structure_design, normalized, rcond=None) + structure_model = structure_design @ structure_coeff + raw_structure_strength = float(np.nanstd(structure_model - np.nanmean(structure_model))) + expected_noise_projection = float(np.sqrt((structure_design.shape[1] - 1) / max(point_count, 1))) + curve_strength = max(0.0, raw_structure_strength - expected_noise_projection) + except Exception: + curve_strength = np.nan + + binned_curve_strength = np.nan + scatter_ratio = np.nan + bin_count = int(np.clip(point_count // 8, 4, 8)) + bins = [ + chunk for chunk in np.array_split(np.arange(point_count), bin_count) + if chunk.size >= 3 + ] + if len(bins) >= 3: + bin_medians = np.asarray([ + float(np.nanmedian(normalized[chunk])) + for chunk in bins + ], dtype=float) + median_bin_size = float(np.nanmedian([chunk.size for chunk in bins])) + expected_binned_median_noise = 1.253 / np.sqrt(max(median_bin_size, 1.0)) + binned_curve_strength = max( + 0.0, + float(np.nanstd(bin_medians - np.nanmedian(bin_medians))) + - expected_binned_median_noise, + ) + + bin_sigmas = np.asarray([ + robust_sigma(normalized[chunk] - float(np.nanmedian(normalized[chunk]))) + for chunk in bins + ], dtype=float) + finite_sigmas = bin_sigmas[np.isfinite(bin_sigmas) & (bin_sigmas > np.finfo(float).eps)] + if finite_sigmas.size >= 2: + median_sigma = float(np.nanmedian(finite_sigmas)) + if np.isfinite(median_sigma) and median_sigma > np.finfo(float).eps: + finite_sizes = np.asarray([ + chunk.size + for chunk, sigma in zip(bins, bin_sigmas) + if np.isfinite(sigma) and sigma > np.finfo(float).eps + ], dtype=float) + log_scatter_offsets = np.abs(np.log(finite_sigmas / median_sigma)) + expected_log_scatter_noise = 1.0 / np.sqrt(2.0 * np.maximum(finite_sizes - 1.0, 1.0)) + excess_log_scatter_offsets = np.maximum( + 0.0, + log_scatter_offsets - expected_log_scatter_noise, + ) + if excess_log_scatter_offsets.size: + scatter_ratio = float(np.exp(np.nanpercentile(excess_log_scatter_offsets, 80.0))) + + if np.isfinite(binned_curve_strength): + curve_strength = ( + max(curve_strength, binned_curve_strength) + if np.isfinite(curve_strength) + else binned_curve_strength + ) + + trend_score = transit_qc_flatness_declining_score(trend_strength, 0.20, 0.85) + curve_score = transit_qc_flatness_declining_score(curve_strength, 0.25, 1.00) + scatter_stability_score = ( + transit_qc_flatness_declining_score(np.log(scatter_ratio), np.log(1.5), np.log(3.0)) + if np.isfinite(scatter_ratio) and scatter_ratio > 0 + else np.nan + ) + component_scores = { + 'trend': trend_score, + 'curvature/sinusoid': curve_score, + 'scatter stability': scatter_stability_score, + } + finite_component_scores = { + key: float(value) + for key, value in component_scores.items() + if np.isfinite(value) + } + if not finite_component_scores: + summary['detail'] = "residual flatness components unavailable" + return summary + + dominant = min(finite_component_scores, key=finite_component_scores.get) + score = finite_component_scores[dominant] + detail_parts = [ + f"{dominant} limited", + f"trend={trend_strength:.2f}" if np.isfinite(trend_strength) else "trend=n/a", + f"curve={curve_strength:.2f}" if np.isfinite(curve_strength) else "curve=n/a", + f"scatter ratio={scatter_ratio:.2f}" if np.isfinite(scatter_ratio) else "scatter ratio=n/a", + f"n={point_count}", + ] + summary.update({ + 'available': True, + 'score': float(np.clip(score, 0.0, 1.0)), + 'scatter': float(scatter), + 'trend_strength': trend_strength, + 'curve_strength': curve_strength, + 'scatter_ratio': scatter_ratio, + 'trend_score': trend_score, + 'curve_score': curve_score, + 'scatter_stability_score': scatter_stability_score, + 'dominant': dominant, + 'detail': ", ".join(detail_parts), + }) + return summary + + def transit_qc_mean_available_score(*scores): finite_scores = [float(score) for score in scores if np.isfinite(score)] if not finite_scores: @@ -1687,26 +1951,9 @@ def compute_transit_qc_ktmf(summary): if not isinstance(summary, dict): return np.nan, [] - delta_bic_score = transit_qc_saturating_score( - summary.get('delta_bic', np.nan), - TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD, - ) - delta_chi2_score = transit_qc_saturating_score(summary.get('delta_chi2', np.nan), 25.0) - model_evidence_score = transit_qc_mean_available_score(delta_bic_score, delta_chi2_score) - model_evidence_score_uncertainty = np.nan - model_evidence_scores = [ - float(score) for score in (delta_bic_score, delta_chi2_score) if np.isfinite(score) - ] - if len(model_evidence_scores) > 1: - model_evidence_score_uncertainty = float(np.std(model_evidence_scores)) - model_evidence_detail_parts = [ - f"Delta BIC={format_transit_delta_bic(summary.get('delta_bic', np.nan))}", - f"Delta chi2={summary.get('delta_chi2', np.nan):.2f}" - if np.isfinite(summary.get('delta_chi2', np.nan)) - else "Delta chi2=n/a", - ] deviation_score = summary.get('deviation_from_expected_value', np.nan) rprs_prior_assumed = bool(summary.get('rprs_prior_assumed', False)) + geometry_prior_assumed = bool(summary.get('geometry_prior_assumed', False)) if rprs_prior_assumed: deviation_score = np.nan deviation_detail = ( @@ -1740,6 +1987,16 @@ def compute_transit_qc_ktmf(summary): else: deviation_detail = "expected-value deviation disabled or unavailable" + if geometry_prior_assumed: + geometry_note = ( + summary.get('geometry_prior_assumed_note') + or "Transit geometry was fixed to input priors for partial-coverage fitting." + ) + deviation_score = np.nan + deviation_detail = ( + f"{geometry_note} Expected-value deviation is omitted from KTMF scoring." + ) + residual_scatter = summary.get('residual_scatter', np.nan) residual_depth = summary.get('transit_depth_for_residual_scatter', np.nan) residual_scatter_to_depth_ratio = summary.get('residual_scatter_to_depth_ratio', np.nan) @@ -1747,6 +2004,8 @@ def compute_transit_qc_ktmf(summary): residual_scatter, residual_depth, ) + residual_flatness_score = summary.get('residual_flatness_score', np.nan) + residual_flatness_detail = summary.get('residual_flatness_detail') or "n/a" residual_scatter_score_uncertainty = np.nan point_count = summary.get('point_count', np.nan) if ( @@ -1800,14 +2059,24 @@ def compute_transit_qc_ktmf(summary): else: duration_detail = duration_note or "n/a" + duration_score = transit_qc_duration_score(summary.get('duration_ratio', np.nan)) + sampling_score = summary.get('sampling_score', np.nan) + sampling_detail = summary.get('sampling_detail') or "n/a" + if geometry_prior_assumed: + geometry_note = ( + summary.get('geometry_prior_assumed_note') + or "Transit geometry was fixed to input priors for partial-coverage fitting." + ) + duration_score = np.nan + duration_detail = ( + f"{geometry_note} Duration consistency is omitted from KTMF scoring." + ) + sampling_score = np.nan + sampling_detail = ( + f"{geometry_note} Sampling/cadence is omitted from KTMF scoring." + ) + raw_components = [ - { - 'key': 'model_evidence', - 'label': 'Model Evidence', - 'score': model_evidence_score, - 'score_uncertainty': model_evidence_score_uncertainty, - 'detail': ", ".join(model_evidence_detail_parts), - }, { 'key': 'deviation_from_expected_value', 'label': 'Deviation From Expected Value', @@ -1822,10 +2091,17 @@ def compute_transit_qc_ktmf(summary): 'score_uncertainty': residual_scatter_score_uncertainty, 'detail': residual_scatter_detail, }, + { + 'key': 'residual_flatness', + 'label': 'Residual Flatness', + 'score': residual_flatness_score, + 'score_uncertainty': np.nan, + 'detail': residual_flatness_detail, + }, { 'key': 'duration_consistency', 'label': 'Duration Consistency', - 'score': transit_qc_duration_score(summary.get('duration_ratio', np.nan)), + 'score': duration_score, 'score_uncertainty': np.nan, 'detail': duration_detail, }, @@ -1843,9 +2119,9 @@ def compute_transit_qc_ktmf(summary): { 'key': 'sampling', 'label': 'Sampling / Cadence', - 'score': summary.get('sampling_score', np.nan), + 'score': sampling_score, 'score_uncertainty': np.nan, - 'detail': summary.get('sampling_detail') or "n/a", + 'detail': sampling_detail, }, ] @@ -2079,6 +2355,15 @@ def evaluate_transit_detection_qc(fit): 'residual_scatter': np.nan, 'transit_depth_for_residual_scatter': np.nan, 'residual_scatter_to_depth_ratio': np.nan, + 'residual_flatness_score': np.nan, + 'residual_flatness_trend_strength': np.nan, + 'residual_flatness_curve_strength': np.nan, + 'residual_flatness_scatter_ratio': np.nan, + 'residual_flatness_trend_score': np.nan, + 'residual_flatness_curve_score': np.nan, + 'residual_flatness_scatter_stability_score': np.nan, + 'residual_flatness_dominant_metric': None, + 'residual_flatness_detail': None, 'sampling_score': np.nan, 'sampling_detail': None, 'sampling_ingress_count': 0, @@ -2116,6 +2401,8 @@ def evaluate_transit_detection_qc(fit): 'ktmf_metric': np.nan, 'ktmf_contributions': [], 'point_count': 0, + 'geometry_prior_assumed': False, + 'geometry_prior_assumed_note': None, } if fit is None: return summary @@ -2166,6 +2453,16 @@ def evaluate_transit_detection_qc(fit): a2_bounds = bounds.get('a2') if isinstance(bounds, dict) else None parameters = getattr(fit, 'parameters', {}) or {} errors = getattr(fit, 'errors', {}) or {} + geometry_prior_assumed = bool( + getattr(fit, 'partial_transit_geometry_prior_assumption_applied', False) + ) + geometry_prior_assumed_note = getattr( + fit, + 'partial_transit_geometry_prior_assumption_note', + None, + ) + summary['geometry_prior_assumed'] = geometry_prior_assumed + summary['geometry_prior_assumed_note'] = geometry_prior_assumed_note initial_a2 = parameters.get('a2', 0.0) flat_model = fit_profiled_flat_null_model( @@ -2200,6 +2497,24 @@ def evaluate_transit_detection_qc(fit): 'transit_depth_for_residual_scatter': transit_qc_model_depth_fraction(transit_model), 'point_count': int(point_count), }) + residual_coordinates = getattr(fit, 'phase', None) + if residual_coordinates is None or np.asarray(residual_coordinates).shape != data.shape: + residual_coordinates = getattr(fit, 'time', None) + residual_flatness = transit_qc_residual_flatness_summary( + data - transit_model, + coordinates=residual_coordinates, + ) + summary.update({ + 'residual_flatness_score': residual_flatness.get('score', np.nan), + 'residual_flatness_trend_strength': residual_flatness.get('trend_strength', np.nan), + 'residual_flatness_curve_strength': residual_flatness.get('curve_strength', np.nan), + 'residual_flatness_scatter_ratio': residual_flatness.get('scatter_ratio', np.nan), + 'residual_flatness_trend_score': residual_flatness.get('trend_score', np.nan), + 'residual_flatness_curve_score': residual_flatness.get('curve_score', np.nan), + 'residual_flatness_scatter_stability_score': residual_flatness.get('scatter_stability_score', np.nan), + 'residual_flatness_dominant_metric': residual_flatness.get('dominant'), + 'residual_flatness_detail': residual_flatness.get('detail'), + }) if ( np.isfinite(summary['residual_scatter']) and np.isfinite(summary['transit_depth_for_residual_scatter']) @@ -2282,8 +2597,15 @@ def evaluate_transit_detection_qc(fit): 'tmid_deviation_score': deviation_summary.get('tmid_deviation_score', np.nan), 'rprs_deviation_score': deviation_summary.get('rprs_deviation_score', np.nan), 'deviation_from_expected_value': deviation_summary.get('deviation_from_expected_value', np.nan), - 'rprs_prior_assumed': deviation_summary.get('rprs_prior_assumed', False), - 'rprs_prior_assumed_note': deviation_summary.get('rprs_prior_assumed_note'), + 'rprs_prior_assumed': ( + deviation_summary.get('rprs_prior_assumed', False) + or geometry_prior_assumed + ), + 'rprs_prior_assumed_note': ( + geometry_prior_assumed_note + if geometry_prior_assumed + else deviation_summary.get('rprs_prior_assumed_note') + ), }) notes = [] @@ -2294,6 +2616,8 @@ def evaluate_transit_detection_qc(fit): if np.isfinite(delta_bic) and np.isfinite(delta_chi2) else "model comparison unavailable" ) + if geometry_prior_assumed_note: + notes.append(str(geometry_prior_assumed_note)) if not np.isfinite(delta_bic) or not np.isfinite(delta_chi2): status = 'unknown' @@ -2355,7 +2679,7 @@ def evaluate_transit_detection_qc(fit): summary['ktmf_contributions'] = ktmf_contributions if np.isfinite(ktmf_metric): - if status != 'fail' and ktmf_metric < TRANSIT_QC_KTMF_FAIL_THRESHOLD: + if ktmf_metric < TRANSIT_QC_KTMF_FAIL_THRESHOLD: status = 'fail' notes.append( f"KTMF is {ktmf_metric:.2f}/5.00, below the fail threshold " @@ -2365,22 +2689,23 @@ def evaluate_transit_detection_qc(fit): f"KTMF is {ktmf_metric:.2f}/5.00, below the fail threshold " f"of {TRANSIT_QC_KTMF_FAIL_THRESHOLD:.2f}" ) - elif status == 'pass' and ktmf_metric < TRANSIT_QC_KTMF_PASS_THRESHOLD: + elif ktmf_metric < TRANSIT_QC_KTMF_PASS_THRESHOLD: status = 'marginal' notes.append( - f"KTMF is {ktmf_metric:.2f}/5.00, below the pass threshold " - f"of {TRANSIT_QC_KTMF_PASS_THRESHOLD:.2f}." + f"KTMF is {ktmf_metric:.2f}/5.00, in the marginal range " + f"[{TRANSIT_QC_KTMF_FAIL_THRESHOLD:.2f}, {TRANSIT_QC_KTMF_PASS_THRESHOLD:.2f})." ) - elif status == 'marginal' and ktmf_metric < TRANSIT_QC_KTMF_PASS_THRESHOLD: + else: + status = 'pass' notes.append( - f"KTMF is {ktmf_metric:.2f}/5.00, below the pass threshold " + f"KTMF is {ktmf_metric:.2f}/5.00, meeting the pass threshold " f"of {TRANSIT_QC_KTMF_PASS_THRESHOLD:.2f}." ) if status == 'pass': - summary_text = f"Transit model strongly preferred over flat/null model ({comparison_text})." + summary_text = f"KTMF supports a pass-quality transit fit ({comparison_text})." elif status == 'marginal': - summary_text = f"Transit model preferred over flat/null model, but the detection is marginal ({comparison_text})." + summary_text = f"KTMF indicates a marginal transit fit ({comparison_text})." elif status == 'fail': if failure_reasons: flat_model_only_failure = all( @@ -2744,7 +3069,11 @@ def build_search_restriction_prior_from_planet_dict(p_dict): return {} return { 'rprs': p_dict.get('rprs'), + 'rprs_unc': p_dict.get('rprsUnc'), 'ars': p_dict.get('aRs'), + 'ars_unc': p_dict.get('aRsUnc'), + 'inc': p_dict.get('inc'), + 'inc_unc': p_dict.get('incUnc'), } @@ -3124,6 +3453,144 @@ def is_low_one_sided_expected_transit_coverage(assessment): ) +def partial_transit_geometry_prior_assumption_mode(assessment): + if not isinstance(assessment, dict) or not assessment.get('valid'): + return None + + transit_fraction = coerce_finite_transit_qc_scalar( + assessment.get('transit_fraction_observed', np.nan) + ) + in_transit_points = int(assessment.get('in_transit_points', 0) or 0) + if ( + (not np.isfinite(transit_fraction) or transit_fraction <= 0) + and in_transit_points <= 0 + ): + return None + + pre_points = int(assessment.get('pre_ingress_points', 0) or 0) + post_points = int(assessment.get('post_egress_points', 0) or 0) + if pre_points == 0 and post_points == 0: + return 'tmid_baseline_airmass' + if pre_points == 0 or post_points == 0: + return 'tmid_only' + return None + + +def partial_transit_geometry_prior_assumption_fixed_error(key, prior, search_restriction_prior): + aliases = { + 'rprs': ('rprs_unc', 'rprsUnc', 'rprs_error', 'rprsErr', 'rprs_data_uncertainty'), + 'ars': ('ars_unc', 'aRsUnc', 'ars_error', 'aRsErr'), + 'inc': ('inc_unc', 'incUnc', 'inc_error', 'incErr'), + 'b': ('b_unc', 'impact_parameter_unc', 'impactParameterUnc'), + } + for source in (search_restriction_prior, prior): + if not isinstance(source, dict): + continue + for alias in aliases.get(key, ()): + value = coerce_finite_transit_qc_scalar(source.get(alias, np.nan)) + if np.isfinite(value) and value >= 0: + return float(value) + return 0.0 + + +def partial_transit_geometry_prior_assumption_note(mode, assessment, sampled_parameters): + observed_segment = assessment.get('observed_segment') or 'partial transit' + pre_points = int(assessment.get('pre_ingress_points', 0) or 0) + post_points = int(assessment.get('post_egress_points', 0) or 0) + sampled_text = ", ".join(sampled_parameters) if sampled_parameters else "no free parameters" + if mode == 'tmid_baseline_airmass': + return ( + "Applied prior-assumed transit geometry for a no-out-of-transit partial light curve; " + "Rp/R*, a/Rs, and inclination/impact parameter were fixed to the input priors because " + f"the observation contains {pre_points} pre-ingress and {post_points} post-egress " + f"out-of-transit point(s) ({observed_segment}). The nested fit keeps baseline/airmass " + f"terms simultaneous with Tmid; sampled parameter(s): {sampled_text}." + ) + return ( + "Applied prior-assumed transit geometry for a one-sided partial light curve; Rp/R*, a/Rs, " + "and inclination/impact parameter were fixed to the input priors because the transit shape " + f"is baseline-degenerate with {pre_points} pre-ingress and {post_points} post-egress " + f"out-of-transit point(s) ({observed_segment}). Sampled parameter(s): {sampled_text}." + ) + + +def ensure_simultaneous_baseline_airmass_bounds_for_no_oot(prior, bounds, flux_values, airmass): + fit_a2 = not should_skip_airmass_fit(airmass) + ensure_pre_final_ultranest_baseline_bounds( + prior, + bounds, + flux_values, + fit_a2=fit_a2, + ) + + +def apply_partial_transit_geometry_prior_assumption( + prior, + bounds, + assessment, + flux_values=None, + airmass=None, + fixed_parameter_errors=None, + search_restriction_prior=None, +): + mode = partial_transit_geometry_prior_assumption_mode(assessment) + local_prior = dict(prior) if isinstance(prior, dict) else {} + local_bounds = clone_lightcurve_bounds(bounds) + local_fixed_errors = ( + dict(fixed_parameter_errors) + if isinstance(fixed_parameter_errors, dict) + else {} + ) + payload = { + 'applied': False, + 'mode': None, + 'note': None, + 'fixed_parameters': [], + 'sampled_parameters': list(local_bounds.keys()), + } + if mode is None: + return local_prior, local_bounds, local_fixed_errors, payload + + fixed_parameters = [] + for key in ('rprs', 'ars', 'inc', 'b'): + if key in local_bounds: + local_bounds.pop(key, None) + fixed_parameters.append(key) + elif key in local_prior: + fixed_parameters.append(key) + if key in local_prior and key not in local_fixed_errors: + local_fixed_errors[key] = partial_transit_geometry_prior_assumption_fixed_error( + key, + local_prior, + search_restriction_prior, + ) + + if mode == 'tmid_baseline_airmass': + ensure_simultaneous_baseline_airmass_bounds_for_no_oot( + local_prior, + local_bounds, + flux_values, + airmass, + ) + else: + for key in ('a0', 'a1', 'a2'): + local_bounds.pop(key, None) + + sampled_parameters = list(local_bounds.keys()) + payload.update({ + 'applied': True, + 'mode': mode, + 'fixed_parameters': sorted(set(fixed_parameters)), + 'sampled_parameters': sampled_parameters, + }) + payload['note'] = partial_transit_geometry_prior_assumption_note( + mode, + assessment, + sampled_parameters, + ) + return local_prior, local_bounds, local_fixed_errors, payload + + def partial_transit_geometry_retry_limits(assessment): active = is_low_one_sided_expected_transit_coverage(assessment) note = None @@ -5738,12 +6205,39 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co 'annotate': annotate_impact_parameter_posterior_refit, }, ] + base_fixed_parameter_errors = ( + dict(fixed_parameter_errors) + if isinstance(fixed_parameter_errors, dict) + else {} + ) def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): local_bounds = sanitize_retry_search_bounds( apply_configured_prior_search_restrictions(local_bounds, restriction_reference_prior) ) - if fixed_flux_baseline: + effective_fixed_parameter_errors = ( + dict(base_fixed_parameter_errors) + if isinstance(base_fixed_parameter_errors, dict) + else {} + ) + if isinstance(fixed_parameter_errors_override, dict): + effective_fixed_parameter_errors.update(fixed_parameter_errors_override) + local_prior, local_bounds, effective_fixed_parameter_errors, prior_assumption = ( + apply_partial_transit_geometry_prior_assumption( + local_prior, + local_bounds, + pre_ultranest_coverage_assessment, + flux_values=flux_values, + airmass=airmass, + fixed_parameter_errors=effective_fixed_parameter_errors, + search_restriction_prior=restriction_reference_prior, + ) + ) + effective_fixed_flux_baseline = ( + fixed_flux_baseline + and prior_assumption.get('mode') != 'tmid_baseline_airmass' + ) + if effective_fixed_flux_baseline: local_bounds = clone_lightcurve_bounds(local_bounds) for key in ('a0', 'a1', 'a2'): local_bounds.pop(key, None) @@ -5760,16 +6254,9 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): fit_kwargs['keep_ultranest_sampler'] = True if baseline_fit_mask is not None and callable_accepts_keyword(lc_fitter, 'baseline_fit_mask'): fit_kwargs['baseline_fit_mask'] = baseline_fit_mask - effective_fixed_parameter_errors = ( - dict(fixed_parameter_errors) - if isinstance(fixed_parameter_errors, dict) - else {} - ) - if isinstance(fixed_parameter_errors_override, dict): - effective_fixed_parameter_errors.update(fixed_parameter_errors_override) if effective_fixed_parameter_errors and callable_accepts_keyword(lc_fitter, 'fixed_parameter_errors'): fit_kwargs['fixed_parameter_errors'] = effective_fixed_parameter_errors - if fixed_flux_baseline and callable_accepts_keyword(lc_fitter, 'fixed_flux_baseline'): + if effective_fixed_flux_baseline and callable_accepts_keyword(lc_fitter, 'fixed_flux_baseline'): fit_kwargs['fixed_flux_baseline'] = True if ( ultranest_min_num_live_points is not None @@ -5786,12 +6273,24 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): **fit_kwargs, ) annotate_duration_prior(fit, duration_prior) + annotate_partial_transit_geometry_prior_assumption(fit, prior_assumption) return fit current_bounds = sanitize_retry_search_bounds( apply_configured_prior_search_restrictions(bounds, restriction_reference_prior) ) - current_prior = clamp_retry_priors_to_bounds(prior, current_bounds) + current_prior, current_bounds, base_fixed_parameter_errors, initial_prior_assumption = ( + apply_partial_transit_geometry_prior_assumption( + prior, + current_bounds, + pre_ultranest_coverage_assessment, + flux_values=flux_values, + airmass=airmass, + fixed_parameter_errors=base_fixed_parameter_errors, + search_restriction_prior=restriction_reference_prior, + ) + ) + current_prior = clamp_retry_priors_to_bounds(current_prior, current_bounds) retry_histories = {config['key']: [] for config in retry_configs} retry_notes = {config['key']: None for config in retry_configs} latest_diagnostics = {config['key']: None for config in retry_configs} @@ -6713,6 +7212,44 @@ def should_fit_lightcurve_to_every_comparison_candidate(config_value): return False +def should_use_automatic_optimal_calibration_selector(config_value): + return parse_bool_config_value( + config_value, + False, + 'automatic_optimal_calibration_selector', + ) + + +def should_use_ensemble_photometry_rather_than_single_comp(config_value): + return parse_bool_config_value( + config_value, + False, + 'use_ensemble_photometry_rather_than_single_comp', + ) + + +def parse_automatic_calibration_selector_count(config_value): + if config_value is None or config_value == '': + return AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT + try: + count = int(float(config_value)) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'automatic_optimal_calibration_selector_count' value; " + f"defaulting to {AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT}.", + warn=True, + ) + return AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT + if count < 1: + log_info( + "Warning: 'automatic_optimal_calibration_selector_count' must be at least 1; " + f"defaulting to {AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT}.", + warn=True, + ) + return AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT + return count + + def should_use_sparse_posterior_live_point_retry(config_value): if config_value is None: return SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT @@ -12454,13 +12991,17 @@ def _finite_float(value, default=None): return parsed if np.isfinite(parsed) else default -def usable_catalog_reference_magnitude(magnitude, magnitude_error): +def usable_catalog_reference_magnitude(magnitude, magnitude_error, + max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX): parsed_magnitude = _finite_float(magnitude) parsed_error = normalized_magnitude_error(magnitude_error) + error_limit = _finite_float(max_error) + if error_limit is None: + error_limit = CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX if ( not is_usable_apparent_magnitude(parsed_magnitude) or parsed_error is None - or parsed_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + or parsed_error > error_limit ): return None return parsed_magnitude, parsed_error @@ -12563,9 +13104,13 @@ def nextastro_catalog_rows(catalog_response): return [row for row in rows if isinstance(row, dict)] -def row_nextastro_magnitude(row, band_candidates): +def row_nextastro_magnitude(row, band_candidates, max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX): for priority, (mag_column, error_column, band_label) in enumerate(band_candidates): - usable_magnitude = usable_catalog_reference_magnitude(row.get(mag_column), row.get(error_column)) + usable_magnitude = usable_catalog_reference_magnitude( + row.get(mag_column), + row.get(error_column), + max_error=max_error, + ) if usable_magnitude is None: continue magnitude, magnitude_error = usable_magnitude @@ -12587,7 +13132,8 @@ def sky_separation_arcsec(ra_a, dec_a, ra_b, dec_b): def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, - max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC, + max_magnitude_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX): effective_max_separation_arcsec = _finite_float(max_separation_arcsec) if effective_max_separation_arcsec is None: effective_max_separation_arcsec = NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC @@ -12602,7 +13148,7 @@ def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, row_dec = _finite_float(row.get('dec')) if row_ra is None or row_dec is None: continue - magnitude = row_nextastro_magnitude(row, band_candidates) + magnitude = row_nextastro_magnitude(row, band_candidates, max_error=max_magnitude_error) if magnitude is None: continue separation = sky_separation_arcsec(ra, dec, row_ra, row_dec) @@ -12615,6 +13161,7 @@ def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, 'source_id': row.get('source_id'), 'id': row.get('id'), 'separation_arcsec': separation, + 'catalog_row': row, }) if not matches: @@ -12782,7 +13329,8 @@ def detect_reference_fallback_bright_stars( max_stars=REFERENCE_FALLBACK_DETECTION_MAX_STARS, min_sep=REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS, aperture_radius=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, - min_area=REFERENCE_FALLBACK_DETECTION_MIN_AREA_PIXELS): + min_area=REFERENCE_FALLBACK_DETECTION_MIN_AREA_PIXELS, + threshold_percentile=99.7): if image_data is None: return [] @@ -12801,7 +13349,12 @@ def detect_reference_fallback_bright_stars( if not np.any(signal > 0): return [] - threshold = float(np.percentile(signal, 99.7)) + try: + threshold_percentile = float(threshold_percentile) + except (TypeError, ValueError): + threshold_percentile = 99.7 + threshold_percentile = min(max(threshold_percentile, 0.0), 100.0) + threshold = float(np.percentile(signal, threshold_percentile)) if threshold <= 0: threshold = float(np.percentile(signal, 99.0)) if threshold <= 0: @@ -12997,29 +13550,291 @@ def log_reference_fallback_comparison_candidates(comp_stars, detected_candidates ) -def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, existing_comp_stars=None, - field_catalog=None): - calibration_stars = dict(existing_comp_stars or {}) - existing_positions = { - tuple(value.get('pos', [])) - for value in calibration_stars.values() - if isinstance(value, dict) - } +def image_aperture_signal_flux(image_data, x_pos, y_pos, + aperture_radius=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS): + if image_data is None: + return np.nan + data = np.asarray(image_data, dtype=float) + if data.ndim != 2: + return np.nan + x_pos = _finite_float(x_pos) + y_pos = _finite_float(y_pos) + if x_pos is None or y_pos is None: + return np.nan - added_count = 0 - for index, (comp_pos, comp_radec) in enumerate(zip(comp_stars, comp_ra_dec)): - if tuple(comp_pos) in existing_positions: - continue - comp_ra = _finite_float(comp_radec[0]) - comp_dec = _finite_float(comp_radec[1]) - if comp_ra is None or comp_dec is None: + height, width = data.shape + radius = float(aperture_radius) + x_min = max(0, int(np.floor(x_pos - radius))) + x_max = min(width, int(np.ceil(x_pos + radius)) + 1) + y_min = max(0, int(np.floor(y_pos - radius))) + y_max = min(height, int(np.ceil(y_pos + radius)) + 1) + if x_min >= x_max or y_min >= y_max: + return np.nan + + finite = np.isfinite(data) + if not np.any(finite): + return np.nan + background = float(np.nanmedian(data[finite])) + yy, xx = np.indices(data.shape) + mask = (xx - x_pos) ** 2 + (yy - y_pos) ** 2 <= radius ** 2 + aperture_values = data[mask] + aperture_values = aperture_values[np.isfinite(aperture_values)] + if aperture_values.size == 0: + return np.nan + flux = float(np.sum(aperture_values - background)) + return flux if np.isfinite(flux) and flux > 0 else np.nan + + +def nextastro_color_candidate_pairs(obs_filter): + filter_key = normalize_nextastro_filter_key(obs_filter) + if filter_key in ('u', 'johnsonu', 'su', 'up'): + preferred = [('umag', 'g', 'u-g')] + elif filter_key in ('b', 'johnsonb', 'photographicb', 'bb', 'pb'): + preferred = [('Bmag', 'Vmag', 'B-V')] + elif filter_key in ('v', 'johnsonv', 'bv', 'cv', 'clearv', 'c', 'clear', 'lum', 'luminance'): + preferred = [('Bmag', 'Vmag', 'B-V')] + elif filter_key in ('sg', 'sloang', 'sdssg', 'photographicg', 'gp', 'g', 'pg', 'tg'): + preferred = [('g', 'r', 'g-r')] + elif filter_key in ('sr', 'sloanr', 'sdssr', 'johnsonr', 'cousinsr', 'rp', 'r', 'rc', 'rj', 'pr', 'tr', 'cr'): + preferred = [('r', 'i', 'r-i')] + elif filter_key in ('si', 'sloani', 'sdssi', 'johnsoni', 'cousinsi', 'ip', 'i', 'ic', 'ij'): + preferred = [('r', 'i', 'r-i')] + elif filter_key in ('sz', 'sloanz', 'sdssz', 'zp', 'z', 'zs'): + preferred = [('i', 'z', 'i-z')] + else: + preferred = [] + + fallback = [ + ('Bmag', 'Vmag', 'B-V'), + ('g', 'r', 'g-r'), + ('r', 'i', 'r-i'), + ('i', 'z', 'i-z'), + ('umag', 'g', 'u-g'), + ] + pairs = list(preferred) + pairs.extend(pair for pair in fallback if pair not in pairs) + return pairs + + +def nextastro_catalog_color(row, obs_filter): + if not isinstance(row, dict): + return None + for first_column, second_column, label in nextastro_color_candidate_pairs(obs_filter): + first = _finite_float(row.get(first_column)) + second = _finite_float(row.get(second_column)) + if first is None or second is None: continue + return { + 'color': float(first - second), + 'label': label, + 'first_column': first_column, + 'second_column': second_column, + } + return None - match = None - if field_catalog is not None: - match = nextastro_photometry_catalog_match(field_catalog, comp_ra, comp_dec, obs_filter) - if match is None: - try: + +def nextastro_catalog_nearest_color_row(catalog_response, ra, dec, obs_filter, + max_separation_arcsec= + AUTOMATIC_CALIBRATION_SELECTOR_COLOR_MATCH_RADIUS_ARCSEC): + best_match = None + best_separation = None + for row in nextastro_catalog_rows(catalog_response): + row_ra = _finite_float(row.get('ra')) + row_dec = _finite_float(row.get('dec')) + if row_ra is None or row_dec is None: + continue + color = nextastro_catalog_color(row, obs_filter) + if color is None: + continue + separation = sky_separation_arcsec(ra, dec, row_ra, row_dec) + if separation > float(max_separation_arcsec): + continue + if best_separation is None or separation < best_separation: + best_match = { + 'catalog_row': row, + 'catalog_ra': row_ra, + 'catalog_dec': row_dec, + 'source_id': row.get('source_id'), + 'id': row.get('id'), + 'separation_arcsec': separation, + 'color': color, + } + best_separation = separation + return best_match + + +def select_automatic_optimal_calibration_stars( + image_data, + image_shape, + target_pixel, + ra_wcs, + dec_wcs, + obs_filter, + field_catalog, + count=AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT, + min_comp_target_sep=REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS): + max_count = parse_automatic_calibration_selector_count(count) + if image_data is None or field_catalog is None: + return [], [] + + target_pixel = np.asarray(target_pixel, dtype=float).reshape(-1) + if target_pixel.size < 2 or not np.all(np.isfinite(target_pixel[:2])): + return [], [] + target_x, target_y = float(target_pixel[0]), float(target_pixel[1]) + + target_flux = image_aperture_signal_flux(image_data, target_x, target_y) + if not np.isfinite(target_flux) or target_flux <= 0: + log_info( + "Warning: automatic calibration selector could not measure a positive target flux " + "on the reference image.", + warn=True, + ) + return [], [] + + height, width = image_shape[:2] + target_xi = int(np.clip(round(target_x), 0, width - 1)) + target_yi = int(np.clip(round(target_y), 0, height - 1)) + target_match = nextastro_catalog_nearest_color_row( + field_catalog, + ra_wcs[target_yi][target_xi], + dec_wcs[target_yi][target_xi], + obs_filter, + ) + target_color = nextastro_catalog_color((target_match or {}).get('catalog_row'), obs_filter) + if target_color is None: + log_info( + "Warning: automatic calibration selector could not derive a target color from the " + "NextAstro photometry catalog.", + warn=True, + ) + return [], [] + + detected_stars = detect_reference_fallback_bright_stars( + image_data, + max_stars=max( + AUTOMATIC_CALIBRATION_SELECTOR_MAX_DETECTIONS, + max_count * 8, + ), + threshold_percentile=AUTOMATIC_CALIBRATION_SELECTOR_DETECTION_PERCENTILE, + ) + comp_pool = filter_reference_fallback_stars_to_middle_fifty_percent( + dedupe_reference_fallback_stars(detected_stars), + image_shape, + ) + target_detection = nearest_reference_fallback_star_by_pixels( + comp_pool, + target_x, + target_y, + max_sep_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + used_ids = {id(target_detection)} if target_detection is not None else set() + + min_sep2 = max(float(min_comp_target_sep), 0.0) ** 2 + candidates = [] + for star in comp_pool: + if id(star) in used_ids: + continue + x_pos = _finite_float(star.get('x')) + y_pos = _finite_float(star.get('y')) + star_flux = _finite_float(star.get('flux')) + if x_pos is None or y_pos is None or star_flux is None: + continue + if not pixel_within_image(x_pos, y_pos, image_shape): + continue + if (x_pos - target_x) ** 2 + (y_pos - target_y) ** 2 < min_sep2: + continue + brightness_ratio = float(star_flux / target_flux) + if not ( + AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MIN_RATIO + <= brightness_ratio + <= AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MAX_RATIO + ): + continue + xi = int(np.clip(round(x_pos), 0, width - 1)) + yi = int(np.clip(round(y_pos), 0, height - 1)) + comp_ra = _finite_float(ra_wcs[yi][xi]) + comp_dec = _finite_float(dec_wcs[yi][xi]) + if comp_ra is None or comp_dec is None: + continue + match = nextastro_catalog_nearest_color_row(field_catalog, comp_ra, comp_dec, obs_filter) + color = nextastro_catalog_color((match or {}).get('catalog_row'), obs_filter) + if match is None or color is None: + continue + color_delta = abs(color['color'] - target_color['color']) + candidates.append({ + 'x': float(x_pos), + 'y': float(y_pos), + 'flux': float(star_flux), + 'brightness_ratio': brightness_ratio, + 'ra': comp_ra, + 'dec': comp_dec, + 'catalog_match': match, + 'color': color['color'], + 'color_label': color['label'], + 'target_color': target_color['color'], + 'color_delta': float(color_delta), + 'target_flux': float(target_flux), + }) + + candidates.sort( + key=lambda candidate: ( + candidate['color_delta'], + abs(np.log(candidate['brightness_ratio'])), + -candidate['flux'], + ) + ) + selected_candidates = candidates[:max_count] + comp_stars = [[candidate['x'], candidate['y']] for candidate in selected_candidates] + return comp_stars, selected_candidates + + +def log_automatic_optimal_calibration_selection(comp_stars, candidates, requested_count): + if not comp_stars: + log_info( + "Warning: automatic optimal calibration selector did not find any usable comparison stars; " + "the existing comparison-star list will be kept.", + warn=True, + ) + return + log_info( + "Automatic optimal calibration selector chose " + f"{len(comp_stars)} comparison star(s) out of the requested {requested_count}. " + "Candidates were image-detected, flux-matched to 0.5-2.0x the target, NextAstro matched, " + "and ranked by catalog color similarity to the target." + ) + for index, candidate in enumerate(candidates, start=1): + log_info( + f" Auto comp #{index}: pixels=[{candidate['x']:.2f}, {candidate['y']:.2f}], " + f"flux_ratio={candidate['brightness_ratio']:.3f}, " + f"{candidate['color_label']}={candidate['color']:.3f}, " + f"target_{candidate['color_label']}={candidate['target_color']:.3f}, " + f"delta={candidate['color_delta']:.3f}." + ) + + +def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, existing_comp_stars=None, + field_catalog=None): + calibration_stars = dict(existing_comp_stars or {}) + existing_positions = { + tuple(value.get('pos', [])) + for value in calibration_stars.values() + if isinstance(value, dict) + } + + added_count = 0 + for index, (comp_pos, comp_radec) in enumerate(zip(comp_stars, comp_ra_dec)): + if tuple(comp_pos) in existing_positions: + continue + comp_ra = _finite_float(comp_radec[0]) + comp_dec = _finite_float(comp_radec[1]) + if comp_ra is None or comp_dec is None: + continue + + match = None + if field_catalog is not None: + match = nextastro_photometry_catalog_match(field_catalog, comp_ra, comp_dec, obs_filter) + if match is None: + try: match = nextastro_photometry_for_coordinate(comp_ra, comp_dec, obs_filter) except Exception as exc: log_info( @@ -16409,10 +17224,16 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ observed_filter=None): comp_mag = _finite_float(comp_star.get('mag')) comp_mag_error = normalized_magnitude_error(comp_star.get('error')) + derived_catalog_reference = bool(comp_star.get('derived_catalog_reference', False)) + allow_high_error_catalog_reference = bool(comp_star.get('allow_high_error_catalog_reference', False)) if ( comp_mag is None or comp_mag_error is None - or comp_mag_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + or ( + comp_mag_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + and not derived_catalog_reference + and not allow_high_error_catalog_reference + ) or not is_usable_apparent_magnitude(comp_mag) ): raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") @@ -16491,14 +17312,395 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ 'source_id': comp_star.get('source_id'), 'catalog_id': comp_star.get('id'), 'separation_arcsec': comp_star.get('separation_arcsec'), + 'derived_catalog_reference': derived_catalog_reference, + 'derived_reference_anchor_count': comp_star.get('derived_reference_anchor_count'), + 'derived_reference_anchor_labels': comp_star.get('derived_reference_anchor_labels'), + 'allow_high_error_catalog_reference': allow_high_error_catalog_reference, }) plot_stellar_variability(vsp_params, save, s_name, display_label) return vsp_params +def stellar_variability_reference_series(lc_fit): + fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) + fit_airmass_model = np.asarray( + getattr(lc_fit, 'airmass_model', np.ones_like(fit_data)), + dtype=float, + ) + fit_times = np.asarray(getattr(lc_fit, 'jd_times', getattr(lc_fit, 'time', [])), dtype=float) + transit_model = np.asarray(getattr(lc_fit, 'transit', np.ones_like(fit_data)), dtype=float) + + if not (fit_data.shape == fit_airmass_model.shape == fit_times.shape): + return np.array([], dtype=float), np.array([], dtype=float) + + with np.errstate(divide='ignore', invalid='ignore'): + reference_curve = np.divide(fit_data, fit_airmass_model) + + valid = ( + np.isfinite(fit_times) + & np.isfinite(reference_curve) + & (reference_curve > 0) + ) + if transit_model.shape == fit_data.shape: + valid &= transit_model == 1 + + return fit_times[valid], reference_curve[valid] + + +def aligned_reference_curve_ratio(selected_fit, anchor_fit): + selected_times, selected_curve = stellar_variability_reference_series(selected_fit) + anchor_times, anchor_curve = stellar_variability_reference_series(anchor_fit) + if selected_times.size == 0 or anchor_times.size == 0: + return np.array([], dtype=float) + + selected_keys = np.round(selected_times.astype(float), 8) + anchor_keys = np.round(anchor_times.astype(float), 8) + _, selected_idx, anchor_idx = np.intersect1d( + selected_keys, + anchor_keys, + return_indices=True, + ) + if selected_idx.size == 0: + return np.array([], dtype=float) + + with np.errstate(divide='ignore', invalid='ignore'): + selected_to_anchor_flux_ratio = np.divide(anchor_curve[anchor_idx], selected_curve[selected_idx]) + + return selected_to_anchor_flux_ratio[ + np.isfinite(selected_to_anchor_flux_ratio) + & (selected_to_anchor_flux_ratio > 0) + ] + + +def build_direct_selected_catalog_candidate(comp_stars, comp_ra_dec, field_catalog, best_comp, + observed_filter=None): + if best_comp is None or best_comp < 0 or best_comp >= len(comp_stars): + return None + if field_catalog is None or not comp_ra_dec or best_comp >= len(comp_ra_dec): + return None + + comp_ra = _finite_float(comp_ra_dec[best_comp][0]) + comp_dec = _finite_float(comp_ra_dec[best_comp][1]) + if comp_ra is None or comp_dec is None: + return None + + match = nextastro_photometry_catalog_match( + field_catalog, + comp_ra, + comp_dec, + observed_filter, + max_magnitude_error=MAX_APPARENT_MAGNITUDE, + ) + if match is None: + return None + if catalog_band_priority(match.get('mag_band'), observed_filter) != 0: + return None + + match.update({ + 'ra': comp_ra, + 'dec': comp_dec, + 'pos': list(comp_stars[best_comp]), + 'catalog_source': 'NextAstro photometry catalog', + 'is_aavso_vsp': False, + 'observed_filter': observed_filter, + 'derived_catalog_reference': False, + 'allow_high_error_catalog_reference': True, + }) + label = unique_nextastro_calibration_label({}, match) + return { + 'label': label, + 'star': match, + 'source': 'direct_catalog', + 'error': match.get('error'), + } + + +def combine_catalog_reference_estimates(derived_estimates, selected_pos, observed_filter, source_label, + mag_band='V', label_prefix='Derived Comp', + selected_ra=None, selected_dec=None): + if not derived_estimates: + return None, None + + mags = np.asarray([estimate['mag'] for estimate in derived_estimates], dtype=float) + errors = np.asarray([estimate['error'] for estimate in derived_estimates], dtype=float) + weights = np.divide( + 1.0, + errors ** 2, + out=np.zeros_like(errors, dtype=float), + where=np.isfinite(errors) & (errors > 0), + ) + if not np.any(weights > 0): + return None, None + + combined_mag = float(np.average(mags, weights=weights)) + formal_error = float((1.0 / np.sum(weights)) ** 0.5) + if len(derived_estimates) > 1: + anchor_scatter = float(np.sqrt(np.average((mags - combined_mag) ** 2, weights=weights))) + combined_error = float(np.hypot(formal_error, anchor_scatter / len(derived_estimates) ** 0.5)) + else: + combined_error = formal_error + + anchor_labels = [ + estimate['anchor_label'] for estimate in derived_estimates + if estimate.get('anchor_label') is not None + ] + derived_star = { + 'pos': selected_pos, + 'ra': selected_ra, + 'dec': selected_dec, + 'mag': combined_mag, + 'error': combined_error, + 'catalog_source': source_label, + 'is_aavso_vsp': False, + 'mag_band': mag_band or 'V', + 'observed_filter': observed_filter, + 'derived_catalog_reference': True, + 'derived_reference_anchor_count': len(derived_estimates), + 'derived_reference_anchor_labels': anchor_labels, + } + return f"{label_prefix}", derived_star + + +def preferred_catalog_magnitude_band_for_filter(observed_filter): + candidates = nextastro_photometry_band_candidates(observed_filter) + if not candidates: + return None + return candidates[0][2] + + +def catalog_band_priority(mag_band, observed_filter): + preferred_band = preferred_catalog_magnitude_band_for_filter(observed_filter) + if preferred_band is None: + return 0 + return 0 if str(mag_band or '').strip().lower() == str(preferred_band).strip().lower() else 1 + + +def derived_catalog_reference_for_selected_comp(fit_lc_refs, comp_stars, vsp_comp_stars, vsp_ind, + best_comp, observed_filter=None, comp_ra_dec=None): + if best_comp is None or best_comp not in fit_lc_refs: + return None, None + + selected_fit = fit_lc_refs[best_comp].get('myfit') + selected_pos = comp_stars[best_comp] + selected_ra, selected_dec = None, None + if comp_ra_dec is not None and best_comp < len(comp_ra_dec): + selected_ra = _finite_float(comp_ra_dec[best_comp][0]) + selected_dec = _finite_float(comp_ra_dec[best_comp][1]) + derived_estimates = [] + + for anchor_index in vsp_ind: + if anchor_index == best_comp or anchor_index not in fit_lc_refs: + continue + if anchor_index < 0 or anchor_index >= len(comp_stars): + continue + + anchor_pos = comp_stars[anchor_index] + anchor_label = None + anchor_star = None + for label, star in vsp_comp_stars.items(): + if star.get('pos') == anchor_pos: + anchor_label = label + anchor_star = star + break + if anchor_star is None: + continue + + anchor_mag = _finite_float(anchor_star.get('mag')) + anchor_mag_error = normalized_magnitude_error(anchor_star.get('error')) + if catalog_band_priority(anchor_star.get('mag_band'), observed_filter) != 0: + continue + if ( + anchor_mag is None + or anchor_mag_error is None + or anchor_mag_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + or not is_usable_apparent_magnitude(anchor_mag) + ): + continue + + ratio = aligned_reference_curve_ratio(selected_fit, fit_lc_refs[anchor_index].get('myfit')) + if ratio.size == 0: + continue + + with np.errstate(divide='ignore', invalid='ignore'): + selected_mag_points = anchor_mag - (2.5 * np.log10(ratio)) + selected_mag_points = selected_mag_points[ + np.isfinite(selected_mag_points) + & (selected_mag_points <= MAX_APPARENT_MAGNITUDE) + ] + if selected_mag_points.size == 0: + continue + + median_mag = float(np.nanmedian(selected_mag_points)) + mad = float(np.nanmedian(np.abs(selected_mag_points - median_mag))) + robust_scatter = 1.4826 * mad if np.isfinite(mad) else np.nan + if not np.isfinite(robust_scatter): + robust_scatter = float(np.nanstd(selected_mag_points)) + scatter_error = robust_scatter / max(selected_mag_points.size, 1) ** 0.5 + total_error = float(np.hypot(anchor_mag_error, scatter_error)) + if not np.isfinite(total_error) or total_error <= 0: + total_error = anchor_mag_error + + derived_estimates.append({ + 'mag': median_mag, + 'error': total_error, + 'mag_band': anchor_star.get('mag_band'), + 'anchor_label': anchor_label, + 'anchor_index': anchor_index, + 'points': int(selected_mag_points.size), + }) + + return combine_catalog_reference_estimates( + derived_estimates, + selected_pos, + observed_filter, + 'Derived from catalog-calibrated comparison stars', + mag_band=( + next((estimate.get('mag_band') for estimate in derived_estimates if estimate.get('mag_band')), None) + or 'V' + ), + label_prefix=f"Derived Comp {best_comp + 1}", + selected_ra=selected_ra, + selected_dec=selected_dec, + ) + + +def derive_selected_comp_catalog_reference_from_field(reference_image, wcs_file, field_catalog, comp_stars, + best_comp, observed_filter=None, + aperture_radius=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, + min_separation_pixels=REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS): + if ( + reference_image is None + or not wcs_file + or field_catalog is None + or best_comp is None + or best_comp < 0 + or best_comp >= len(comp_stars) + ): + return None, None + + selected_pos = comp_stars[best_comp] + selected_x = _finite_float(selected_pos[0]) + selected_y = _finite_float(selected_pos[1]) + if selected_x is None or selected_y is None: + return None, None + + selected_flux = image_aperture_signal_flux( + reference_image, + selected_x, + selected_y, + aperture_radius=aperture_radius, + ) + if not np.isfinite(selected_flux) or selected_flux <= 0: + return None, None + + try: + wcs_hdr = search_wcs(wcs_file) + except Exception: + return None, None + try: + selected_ra, selected_dec = wcs_hdr.pixel_to_world_values(selected_x, selected_y) + selected_ra = float(np.asarray(selected_ra).reshape(-1)[0]) + selected_dec = float(np.asarray(selected_dec).reshape(-1)[0]) + except Exception: + selected_ra, selected_dec = None, None + + image_shape = np.asarray(reference_image).shape + band_candidates = nextastro_photometry_band_candidates(observed_filter) + derived_estimates = [] + + for row in nextastro_catalog_rows(field_catalog): + row_ra = _finite_float(row.get('ra')) + row_dec = _finite_float(row.get('dec')) + if row_ra is None or row_dec is None: + continue + magnitude = row_nextastro_magnitude(row, band_candidates, max_error=MAX_APPARENT_MAGNITUDE) + if magnitude is None: + continue + if catalog_band_priority(magnitude.get('mag_band'), observed_filter) != 0: + continue + try: + anchor_x, anchor_y = wcs_hdr.world_to_pixel_values(row_ra, row_dec) + anchor_x = float(np.asarray(anchor_x).reshape(-1)[0]) + anchor_y = float(np.asarray(anchor_y).reshape(-1)[0]) + except Exception: + continue + if not pixel_within_image(anchor_x, anchor_y, image_shape, margin=float(aperture_radius) + 2.0): + continue + if (anchor_x - selected_x) ** 2 + (anchor_y - selected_y) ** 2 < float(min_separation_pixels) ** 2: + continue + + anchor_flux = image_aperture_signal_flux( + reference_image, + anchor_x, + anchor_y, + aperture_radius=aperture_radius, + ) + if not np.isfinite(anchor_flux) or anchor_flux <= 0: + continue + + with np.errstate(divide='ignore', invalid='ignore'): + selected_mag = magnitude['mag'] - (2.5 * np.log10(selected_flux / anchor_flux)) + if not is_usable_apparent_magnitude(selected_mag): + continue + + derived_estimates.append({ + 'mag': float(selected_mag), + 'error': float(magnitude['error']), + 'mag_band': magnitude.get('mag_band'), + 'anchor_label': ( + f"NextAstro-{row.get('source_id') or row.get('id')}" + if (row.get('source_id') or row.get('id')) not in (None, '') + else f"RA={row_ra:.6f} Dec={row_dec:.6f}" + ), + 'anchor_x': anchor_x, + 'anchor_y': anchor_y, + 'anchor_flux': float(anchor_flux), + }) + + label, star = combine_catalog_reference_estimates( + derived_estimates, + selected_pos, + observed_filter, + 'Derived from full-field catalog-calibrated stars', + mag_band=( + next((estimate.get('mag_band') for estimate in derived_estimates if estimate.get('mag_band')), None) + or 'V' + ), + label_prefix=f"Derived Field Comp {best_comp + 1}", + selected_ra=selected_ra, + selected_dec=selected_dec, + ) + return label, star + + +def catalog_reference_candidate_error(candidate): + if not isinstance(candidate, dict): + return np.inf + star = candidate.get('star') + if not isinstance(star, dict): + return np.inf + error = normalized_magnitude_error(star.get('error')) + return float(error) if error is not None and np.isfinite(error) else np.inf + + +def choose_selected_comp_catalog_reference_candidate(candidates, observed_filter=None): + usable = [ + candidate for candidate in candidates + if np.isfinite(catalog_reference_candidate_error(candidate)) + ] + if not usable: + return None + usable.sort(key=lambda candidate: ( + catalog_band_priority((candidate.get('star') or {}).get('mag_band'), observed_filter), + catalog_reference_candidate_error(candidate), + )) + return usable[0] + + def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vsp_ind, best_comp, save, s_name, - observed_filter=None): + observed_filter=None, comp_ra_dec=None, field_catalog=None, reference_image=None, + wcs_file=None): info_comps = {} try: @@ -16508,26 +17710,104 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs warn=True, ) return [] - if best_comp not in vsp_ind: - log_info( - "Skipping AID magnitude output because the transit-fit comparison star has no catalog " - f"magnitude with uncertainty <= {CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX:.3f} mag.", - warn=True, - ) - return [] comp_pos = comp_stars[best_comp] info_comps[best_comp] = calculate_variablility(fit_lc_refs[best_comp]['myfit'], fit_lc_best) except Exception as e: log_info(f"Error selecting or calculating variability for comparison star: {e}", warn=True) return [] - try: - comp_star = next(vsp_comp_stars[ckey] for ckey in vsp_comp_stars.keys() if comp_pos == vsp_comp_stars[ckey]['pos']) - vsp_auid_comp = next(key for key, value in vsp_comp_stars.items() if value['pos'] == comp_pos) - except StopIteration: - log_info("Comparison star or VSP AUID not found.", warn=True) + candidates = [] + for key, value in vsp_comp_stars.items(): + if value.get('pos') == comp_pos: + candidates.append({ + 'label': key, + 'star': value, + 'source': 'direct_catalog', + 'error': value.get('error'), + }) + break + + direct_relaxed = build_direct_selected_catalog_candidate( + comp_stars, + comp_ra_dec, + field_catalog, + best_comp, + observed_filter=observed_filter, + ) + if direct_relaxed is not None: + candidates.append(direct_relaxed) + + derived_label, derived_star = derived_catalog_reference_for_selected_comp( + fit_lc_refs, + comp_stars, + vsp_comp_stars, + vsp_ind, + best_comp, + observed_filter=observed_filter, + comp_ra_dec=comp_ra_dec, + ) + if derived_star is not None: + candidates.append({ + 'label': derived_label, + 'star': derived_star, + 'source': 'provided_comp_derived', + 'error': derived_star.get('error'), + }) + + has_preferred_band_candidate = any( + catalog_band_priority((candidate.get('star') or {}).get('mag_band'), observed_filter) == 0 + for candidate in candidates + ) + if not has_preferred_band_candidate: + field_label, field_star = derive_selected_comp_catalog_reference_from_field( + reference_image, + wcs_file, + field_catalog, + comp_stars, + best_comp, + observed_filter=observed_filter, + ) + if field_star is not None: + candidates.append({ + 'label': field_label, + 'star': field_star, + 'source': 'field_derived', + 'error': field_star.get('error'), + }) + + selected_candidate = choose_selected_comp_catalog_reference_candidate( + candidates, + observed_filter=observed_filter, + ) + if selected_candidate is None: + log_info( + "Skipping AID magnitude output because the transit-fit comparison star has no catalog " + f"magnitude and no derived magnitude could be inferred from calibrated comparison stars " + "or full-field catalog stars.", + warn=True, + ) return [] + comp_star = selected_candidate['star'] + vsp_auid_comp = selected_candidate['label'] + direct_error = catalog_reference_candidate_error( + next((candidate for candidate in candidates if candidate.get('source') == 'direct_catalog'), None) + ) + chosen_error = catalog_reference_candidate_error(selected_candidate) + if comp_star.get('derived_catalog_reference'): + log_info( + "AID magnitude output will use a derived catalog magnitude for the transit-fit comparison star " + f"from {comp_star.get('derived_reference_anchor_count', 0)} calibrated star(s): " + f"{comp_star.get('mag', np.nan):.3f} +/- {comp_star.get('error', np.nan):.3f} mag." + ) + elif np.isfinite(direct_error) and direct_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX: + log_info( + "AID magnitude output will use the direct selected-comparison catalog magnitude even though " + f"its uncertainty exceeds {CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX:.3f} mag because it is no worse " + f"than the derived alternatives: {comp_star.get('mag', np.nan):.3f} +/- " + f"{chosen_error:.3f} mag." + ) + try: info_comp = info_comps[comp_stars.index(comp_pos)] return build_stellar_variability_params_from_fit( @@ -17732,6 +19012,103 @@ def cheap_lightcurve_prescore(tFlux, cFlux, airmass, enforce_relative_flux_max=T return np.inf +def target_comp_flux_scatter(tFlux, cFlux, min_points=LIGHTCURVE_MIN_VALID_POINTS): + if tFlux is None or cFlux is None: + return np.nan + + tFlux = np.asarray(tFlux, dtype=float) + cFlux = np.asarray(cFlux, dtype=float) + if tFlux.shape != cFlux.shape: + return np.nan + + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio = np.divide(tFlux, cFlux) + + ratio_mask = valid_flux_ratio_mask(flux_ratio) + if np.count_nonzero(ratio_mask) < int(min_points): + return np.nan + + ratio_values = np.asarray(flux_ratio[ratio_mask], dtype=float) + baseline = bn.nanmedian(ratio_values) + if not np.isfinite(baseline) or baseline <= 0: + return np.nan + + normalized_ratio = ratio_values / baseline + keep_mask = comparison_prescore_clip_mask( + normalized_ratio, + min_points=min_points, + ) + kept_ratio = normalized_ratio[keep_mask & np.isfinite(normalized_ratio)] + if kept_ratio.size < int(min_points): + return np.nan + + scatter = robust_scatter(kept_ratio - bn.nanmedian(kept_ratio)) + return float(scatter) if np.isfinite(scatter) and scatter >= 0 else np.nan + + +def fitted_lightcurve_model_at(fit, times, airmass): + if fit is None: + return np.array([], dtype=float) + + parameters = getattr(fit, 'parameters', None) + if not isinstance(parameters, dict): + return np.array([], dtype=float) + + times = np.asarray(times, dtype=float) + airmass = np.asarray(airmass, dtype=float) + if times.ndim != 1 or airmass.shape != times.shape: + return np.array([], dtype=float) + + try: + transit_model = np.asarray(transit(times, parameters), dtype=float) + except Exception: + return np.array([], dtype=float) + if transit_model.shape != times.shape: + return np.array([], dtype=float) + + baseline = parameters.get('a0', parameters.get('a1', 1.0)) + try: + baseline = float(baseline) + except (TypeError, ValueError): + baseline = 1.0 + if not np.isfinite(baseline): + baseline = 1.0 + + try: + a2 = float(parameters.get('a2', 0.0)) + except (TypeError, ValueError): + a2 = 0.0 + if not np.isfinite(a2): + a2 = 0.0 + + reference = getattr(fit, 'airmass_reference', None) + try: + reference = float(reference) + except (TypeError, ValueError): + reference = transit_qc_airmass_reference(getattr(fit, 'airmass', airmass)) + if not np.isfinite(reference): + reference = transit_qc_airmass_reference(airmass) + + systematics = baseline * transit_qc_airmass_trend(a2, airmass, reference=reference) + model = transit_model * systematics + return model if model.shape == times.shape else np.array([], dtype=float) + + +def fitted_lightcurve_scatter_on_dataset(fit, times, flux_values, airmass): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + airmass = np.asarray(airmass, dtype=float) + if not (times.shape == flux_values.shape == airmass.shape): + return np.nan + + model = fitted_lightcurve_model_at(fit, times, airmass) + if model.shape != flux_values.shape: + return np.nan + + scatter = transit_qc_residual_scatter(flux_values, model) + return float(scatter) if np.isfinite(scatter) and scatter >= 0 else np.nan + + def evaluate_lightcurve_candidate(task): ( times, @@ -18368,6 +19745,27 @@ def compact_comparison_attempt_for_output(attempt): 'label': attempt.get('label'), 'selected': attempt.get('selected'), 'selection_reason': attempt.get('selection_reason'), + 'selection_pass_ktmf_metric': attempt.get('selection_pass_ktmf_metric'), + 'selection_pass_transit_delta_bic': attempt.get('selection_pass_transit_delta_bic'), + 'selection_pass_eebls_snr': attempt.get('selection_pass_eebls_snr'), + 'selection_pass_residual_scatter': attempt.get('selection_pass_residual_scatter'), + 'target_model_scatter_basis': attempt.get('target_model_scatter_basis'), + 'projected_full_residual_scatter': attempt.get('projected_full_residual_scatter'), + 'selection_scatter': attempt.get('selection_scatter'), + 'selection_scatter_basis': attempt.get('selection_scatter_basis'), + 'target_comp_scatter': attempt.get('target_comp_scatter'), + 'selection_pass_target_comp_scatter': attempt.get('selection_pass_target_comp_scatter'), + 'selection_pass_transit_qc_status': attempt.get('selection_pass_transit_qc_status'), + 'selection_pass_transit_qc_summary': attempt.get('selection_pass_transit_qc_summary'), + 'scatter_gate_passed': attempt.get('scatter_gate_passed'), + 'scatter_gate_lowest_residual_scatter': attempt.get('scatter_gate_lowest_residual_scatter'), + 'scatter_gate_threshold': attempt.get('scatter_gate_threshold'), + 'scatter_adjusted_ktmf_metric': attempt.get('scatter_adjusted_ktmf_metric'), + 'combined_quality_ktmf_metric': attempt.get('combined_quality_ktmf_metric'), + 'combined_quality_best_residual_scatter': attempt.get('combined_quality_best_residual_scatter'), + 'combined_quality_best_target_comp_scatter': attempt.get('combined_quality_best_target_comp_scatter'), + 'combined_quality_best_comp_stability': attempt.get('combined_quality_best_comp_stability'), + 'final_refit_metric_note': attempt.get('final_refit_metric_note'), 'ktmf_metric': attempt.get('ktmf_metric'), 'ktmf_contributions': attempt.get('ktmf_contributions') or [], 'transit_delta_bic': attempt.get('transit_delta_bic'), @@ -18381,6 +19779,82 @@ def compact_comparison_attempt_for_output(attempt): } +def record_comparison_attempt_selection_pass_metrics(attempt): + if not isinstance(attempt, dict): + return + + attempt['selection_pass_ktmf_metric'] = attempt.get('ktmf_metric', np.nan) + attempt['selection_pass_transit_delta_bic'] = attempt.get('transit_delta_bic', np.nan) + attempt['selection_pass_eebls_snr'] = attempt.get('eebls_snr', np.nan) + attempt['selection_pass_residual_scatter'] = attempt.get('residual_scatter', np.nan) + attempt['selection_pass_selection_scatter'] = attempt.get('selection_scatter', np.nan) + attempt['selection_pass_target_comp_scatter'] = attempt.get('target_comp_scatter', np.nan) + attempt['selection_pass_fit_point_count'] = attempt.get('fit_point_count') + attempt['selection_pass_transit_qc_status'] = attempt.get('transit_qc_status') + attempt['selection_pass_transit_qc_summary'] = attempt.get('transit_qc_summary') + + +def comparison_selection_metric_changed(before, after, tolerance=5.0e-3): + try: + before = float(before) + after = float(after) + except (TypeError, ValueError): + return False + return np.isfinite(before) and np.isfinite(after) and abs(before - after) > tolerance + + +def update_selected_comparison_final_refit_note(selected_result): + if not isinstance(selected_result, dict): + return + + notes = [] + if comparison_selection_metric_changed( + selected_result.get('selection_pass_ktmf_metric'), + selected_result.get('ktmf_metric'), + ): + notes.append( + "KTMF " + f"{format_ktmf_metric(selected_result.get('selection_pass_ktmf_metric'))} -> " + f"{format_ktmf_metric(selected_result.get('ktmf_metric'))}" + ) + if comparison_selection_metric_changed( + selected_result.get('selection_pass_eebls_snr'), + selected_result.get('eebls_snr'), + tolerance=1.0e-2, + ): + selection_pass_eebls_snr = float(selected_result.get('selection_pass_eebls_snr')) + final_eebls_snr = float(selected_result.get('eebls_snr')) + notes.append( + "EEBLS SNR " + f"{selection_pass_eebls_snr:.2f} -> " + f"{final_eebls_snr:.2f}" + ) + if comparison_selection_metric_changed( + selected_result.get('selection_pass_transit_delta_bic'), + selected_result.get('transit_delta_bic'), + tolerance=1.0e-2, + ): + notes.append( + "Delta BIC " + f"{format_transit_delta_bic(selected_result.get('selection_pass_transit_delta_bic'))} -> " + f"{format_transit_delta_bic(selected_result.get('transit_delta_bic'))}" + ) + + if not notes: + return + + selected_result['final_refit_metric_note'] = ( + "final full-resolution refit updated selected-candidate metrics: " + + "; ".join(notes) + ) + reason = selected_result.get('selection_reason') or '' + if selected_result['final_refit_metric_note'] not in reason: + selected_result['selection_reason'] = ( + f"{reason}; {selected_result['final_refit_metric_note']}" + if reason else selected_result['final_refit_metric_note'] + ) + + def format_transit_delta_bic(value): if value is None: return "n/a" @@ -18615,7 +20089,10 @@ def log_comparison_candidate_evaluation_start(comp_summary, rank, ranked_count, if comp_summary is None: return - label = comp_summary.get('label', f"Comp {comp_summary.get('comp_index', 0) + 1}") + label = comp_summary.get('label') + if label is None: + comp_index = comp_summary.get('comp_index') + label = "comparison candidate" if comp_index is None else f"Comp {comp_index + 1}" position_text = format_comp_star_position(comp_summary.get('position')) coverage_text = format_comp_star_coverage_text({ 'coverage_count': comp_summary.get('coverage_count', 0), @@ -18697,6 +20174,10 @@ def comparison_selection_metric_label(selection_metric): return "Comparison-Field Rank" if selection_metric == 'ktmf': return "KTMF" + if selection_metric == 'ktmf_scatter': + return "KTMF / scatter" + if selection_metric == 'ktmf_combined_quality': + return "KTMF / projected scatter" if selection_metric == 'eebls_snr': return "EEBLS SNR" return "transit-vs-flat Delta BIC" @@ -18731,22 +20212,121 @@ def should_stop_after_promising_partial_comparison_attempt(attempt): ) +def scatter_gate_comparison_attempts( + attempts, + max_scatter_multiplier=COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER): + attempts = list(attempts or []) + finite_scatters = [ + float(attempt.get('selection_scatter', attempt.get('residual_scatter', np.nan))) + for attempt in attempts + if np.isfinite(attempt.get('selection_scatter', attempt.get('residual_scatter', np.nan))) + ] + if not finite_scatters: + for attempt in attempts: + attempt['scatter_gate_passed'] = True + attempt['scatter_gate_lowest_residual_scatter'] = np.nan + attempt['scatter_gate_threshold'] = np.nan + return attempts, np.nan, np.nan + + lowest_scatter = min(finite_scatters) + scatter_threshold = lowest_scatter * float(max_scatter_multiplier) + eligible_attempts = [] + for attempt in attempts: + residual_scatter = attempt.get('selection_scatter', attempt.get('residual_scatter', np.nan)) + scatter_passed = ( + np.isfinite(residual_scatter) + and residual_scatter <= scatter_threshold + ) + attempt['scatter_gate_passed'] = bool(scatter_passed) + attempt['scatter_gate_lowest_residual_scatter'] = lowest_scatter + attempt['scatter_gate_threshold'] = scatter_threshold + if scatter_passed: + eligible_attempts.append(attempt) + + return eligible_attempts or attempts, lowest_scatter, scatter_threshold + + +def finite_positive_attempt_values(attempts, key): + values = [] + for attempt in attempts: + value = attempt.get(key, np.nan) + if np.isfinite(value) and value > 0: + values.append(float(value)) + return values + + +def comparison_attempt_combined_quality_ktmf(attempt): + ktmf_metric = attempt.get('ktmf_metric', np.nan) + if not np.isfinite(ktmf_metric): + return np.nan + + selection_scatter = attempt.get('selection_scatter', np.nan) + if ( + np.isfinite(selection_scatter) + and selection_scatter > 0 + ): + return float(ktmf_metric) / float(selection_scatter * 100.0) + return float(ktmf_metric) + + +def annotate_comparison_attempt_combined_quality_scores(attempts): + attempts = list(attempts or []) + for attempt in attempts: + selection_scatter = attempt.get('selection_scatter', np.nan) + if not np.isfinite(selection_scatter): + residual_scatter = attempt.get('residual_scatter', np.nan) + if np.isfinite(residual_scatter): + attempt['selection_scatter'] = residual_scatter + attempt.setdefault( + 'selection_scatter_basis', + "candidate UltraNest model residual scatter", + ) + + best_residual_scatter = min(finite_positive_attempt_values(attempts, 'selection_scatter'), default=np.nan) + for attempt in attempts: + attempt['combined_quality_best_residual_scatter'] = best_residual_scatter + attempt['combined_quality_best_target_comp_scatter'] = np.nan + attempt['combined_quality_best_comp_stability'] = np.nan + attempt['combined_quality_ktmf_metric'] = comparison_attempt_combined_quality_ktmf(attempt) + attempt['scatter_adjusted_ktmf_metric'] = attempt['combined_quality_ktmf_metric'] + + return attempts + + +def comparison_attempt_ranking_score(attempt): + value = attempt.get('combined_quality_ktmf_metric', np.nan) + if np.isfinite(value): + return value + value = attempt.get('scatter_adjusted_ktmf_metric', np.nan) + if np.isfinite(value): + return value + value = attempt.get('ktmf_metric', np.nan) + if np.isfinite(value): + return value + return np.nan + + def select_preferred_comparison_attempt(attempts, pick_comparison_by_eebls_snr=True): selected_result = None selection_metric = 'ktmf' if not attempts: return selected_result, selection_metric + attempts, _, _ = scatter_gate_comparison_attempts(attempts) + annotate_comparison_attempt_combined_quality_scores(attempts) has_ktmf = any(np.isfinite(attempt.get('ktmf_metric', np.nan)) for attempt in attempts) has_eebls = any(np.isfinite(attempt.get('eebls_snr', np.nan)) for attempt in attempts) has_delta_bic = any(np.isfinite(attempt.get('transit_delta_bic', np.nan)) for attempt in attempts) if has_ktmf: - selection_metric = 'ktmf' + selection_metric = 'ktmf_combined_quality' if pick_comparison_by_eebls_snr: selected_result = min( attempts, key=lambda attempt: ( + 0 if np.isfinite(comparison_attempt_ranking_score(attempt)) else 1, + -comparison_attempt_ranking_score(attempt) + if np.isfinite(comparison_attempt_ranking_score(attempt)) else np.inf, 0 if np.isfinite(attempt.get('ktmf_metric', np.nan)) else 1, -attempt.get('ktmf_metric', np.nan) if np.isfinite(attempt.get('ktmf_metric', np.nan)) else np.inf, 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, @@ -18760,6 +20340,9 @@ def select_preferred_comparison_attempt(attempts, pick_comparison_by_eebls_snr=T selected_result = min( attempts, key=lambda attempt: ( + 0 if np.isfinite(comparison_attempt_ranking_score(attempt)) else 1, + -comparison_attempt_ranking_score(attempt) + if np.isfinite(comparison_attempt_ranking_score(attempt)) else np.inf, 0 if np.isfinite(attempt.get('ktmf_metric', np.nan)) else 1, -attempt.get('ktmf_metric', np.nan) if np.isfinite(attempt.get('ktmf_metric', np.nan)) else np.inf, 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, @@ -19365,6 +20948,40 @@ def build_normalized_comp_ensemble(normalized_flux_map, exclude_key): return ensemble +def build_absolute_comp_ensemble_flux(comp_flux_map, active_keys, + validity_mask_func=valid_comparison_frame_mask): + normalized_members = [] + member_medians = [] + member_keys = [] + for key in active_keys: + if key not in comp_flux_map: + continue + flux_values = np.asarray(comp_flux_map[key], dtype=float) + valid_mask = validity_mask_func(flux_values) + if np.count_nonzero(valid_mask) < 5: + continue + member_median = float(bn.nanmedian(flux_values[valid_mask])) + if not np.isfinite(member_median) or member_median <= 0: + continue + normalized_flux = np.full(flux_values.shape, np.nan, dtype=float) + normalized_flux[valid_mask] = flux_values[valid_mask] / member_median + normalized_members.append(normalized_flux) + member_medians.append(member_median) + member_keys.append(key) + + if not normalized_members: + return None, [] + + ensemble_stack = np.vstack(normalized_members) + valid_mask = np.any(np.isfinite(ensemble_stack), axis=0) + ensemble = np.full(ensemble_stack.shape[1], np.nan, dtype=float) + ensemble[valid_mask] = np.nanmedian(ensemble_stack[:, valid_mask], axis=0) + scale = float(np.nanmedian(member_medians)) + if not np.isfinite(scale) or scale <= 0: + scale = 1.0 + return ensemble * scale, member_keys + + def comparison_star_coverage_summary(comp_flux_map, min_fraction=COMPARISON_STAR_MIN_COVERAGE_FRACTION, min_points=COMPARISON_STAR_MIN_VALID_FRAMES, @@ -20407,7 +22024,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p run_final_residual_rejection=FINAL_RESIDUAL_REJECTION_DEFAULT, save_dir=None, planet_name=None, - observation_date=None): + observation_date=None, + use_ensemble_photometry_rather_than_single_comp=False): ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) if not ranked_summaries: return { @@ -20460,7 +22078,142 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ) preflight_plans = [] - for field_rank, comp_summary in enumerate(ranked_summaries): + if use_ensemble_photometry_rather_than_single_comp: + active_keys = [summary.get('key') for summary in ranked_summaries if summary.get('key')] + if method == 'psf': + comp_flux_map = { + summary['key']: psf_flux_series_from_rows( + psf_flux_data[summary['key']], + np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + summary['key'], + frame_count, + psf_flux_data=psf_flux_data, + ), + ), + dtype=bool, + ), + ) + for summary in ranked_summaries + if summary.get('key') in psf_flux_data + } + ensemble_flux, member_keys = build_absolute_comp_ensemble_flux( + comp_flux_map, + active_keys, + validity_mask_func=robust_flux_floor_mask, + ) + target_shape_mask = target_psf_shape_quality_mask(target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data)) + if target_shape_mask.shape[0] != frame_count: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + else: + comp_flux_map = { + summary['key']: mask_series_with_quality( + aper_data[summary['key']][:, aperture_index, annulus_index], + np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key(psf_data, summary['key'], frame_count), + ), + dtype=bool, + ), + ) + for summary in ranked_summaries + if summary.get('key') in aper_data + } + ensemble_flux, member_keys = build_absolute_comp_ensemble_flux( + comp_flux_map, + active_keys, + validity_mask_func=valid_comparison_frame_mask, + ) + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = target_flux + + if ensemble_flux is not None and member_keys: + candidate_frame_keep_mask = np.ones(times.shape[0], dtype=bool) + for summary in ranked_summaries: + if summary.get('key') not in set(member_keys): + continue + member_keep_mask = np.asarray( + summary.get('ensemble_frame_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if member_keep_mask.shape == times.shape: + candidate_frame_keep_mask &= member_keep_mask + fit_mask = field_image_keep_mask & candidate_frame_keep_mask & target_shape_mask + if method == 'psf': + fit_mask &= robust_target_reference_flux_mask(candidate_target_flux, ensemble_flux) + else: + fit_mask &= ( + valid_comparison_frame_mask(candidate_target_flux) + & valid_comparison_frame_mask(ensemble_flux) + ) + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + candidate_target_flux[fit_mask], + ensemble_flux[fit_mask], + airmass[fit_mask], + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + preflight = build_comparison_candidate_preflight( + times[fit_mask], + jd_times[fit_mask], + airmass[fit_mask], + ld, + p_dict, + candidate_target_flux[fit_mask], + ensemble_flux[fit_mask], + adaptive_summary=adaptive_summary, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + ) + ensemble_summary = { + 'comp_index': None, + 'key': 'ensemble', + 'label': f"Comparison ensemble ({len(member_keys)} comps)", + 'position': None, + 'aggregate_score': comparison_calibration.get('field_score', np.inf), + 'coverage_count': int(np.count_nonzero(valid_comparison_frame_mask(ensemble_flux))), + 'coverage_total_frame_count': int(frame_count), + 'coverage_reference_count': np.nan, + 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, + 'coverage_rejected': False, + 'ensemble_frame_rejected_count': int(np.count_nonzero(~candidate_frame_keep_mask)), + 'ensemble_frame_required_valid_pairs': len(member_keys), + 'ensemble_member_keys': member_keys, + } + preflight_plans.append({ + 'field_rank': 0, + 'summary': ensemble_summary, + 'ckey': None, + 'target_flux': candidate_target_flux, + 'comp_flux': ensemble_flux, + 'fit_mask': fit_mask, + 'candidate_frame_clip_diagnostic': None, + 'fit_diagnostics': fit_diagnostics, + 'preflight': preflight, + }) + log_info( + "Ensemble comparison photometry enabled: target fit will use " + f"{len(member_keys)} non-rejected comparison star(s) as a median normalized ensemble " + "rather than fitting each comparison star independently." + ) + else: + log_info( + "Warning: ensemble comparison photometry was enabled, but no usable non-rejected " + "comparison-star ensemble could be built; falling back to ranked single-comp fits.", + warn=True, + ) + + if not preflight_plans: + ranked_summaries_to_fit = ranked_summaries + else: + ranked_summaries_to_fit = [] + + for field_rank, comp_summary in enumerate(ranked_summaries_to_fit): comp_index = comp_summary['comp_index'] ckey = comp_summary.get('key', f"comp{comp_index + 1}") comp_quality_mask = np.asarray( @@ -20620,10 +22373,37 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ) if fit_result is None: log_info( - f" {comp_summary.get('label', f'Comp {comp_index + 1}')}: " + f" {comp_summary.get('label', 'comparison candidate')}: " "the raw comparison-candidate light curve did not converge to a usable fully reduced fit." ) selection_fit = fit_result + fast_binning = final_reduction.get('fast_ultranest_binning') or {} + fast_binned_ultranest = bool(fast_binning.get('applied')) + residual_scatter = extract_lightcurve_fit_residual_scatter(selection_fit) + target_comp_scatter_value = target_comp_flux_scatter(tflux_fit, cflux_fit) + projected_full_scatter = fitted_lightcurve_scatter_on_dataset( + selection_fit, + final_reduction.get('good_times'), + final_reduction.get('good_flux'), + final_reduction.get('good_airmass'), + ) + if fast_binned_ultranest and np.isfinite(projected_full_scatter): + selection_scatter = projected_full_scatter + selection_scatter_basis = ( + "fast-binned UltraNest model residual scatter evaluated on full unbinned light curve" + ) + else: + selection_scatter = residual_scatter + selection_scatter_basis = ( + "full-resolution UltraNest model residual scatter" + if not fast_binned_ultranest + else "fast-binned UltraNest model residual scatter" + ) + target_model_scatter_basis = ( + "fast-binned UltraNest model residual scatter" + if fast_binned_ultranest + else "full-resolution UltraNest model residual scatter" + ) transit_qc_failure_reason = lightcurve_fit_transit_qc_failure_reason(selection_fit) if transit_qc_failure_reason is not None: fit_diagnostics = dict(fit_diagnostics) @@ -20650,7 +22430,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'field_rank': plan.get('field_rank'), 'comp_index': comp_index, 'ckey': ckey, - 'label': comp_summary.get('label', f"Comp {comp_index + 1}"), + 'label': comp_summary.get('label', 'comparison candidate'), 'position': comp_summary.get('position'), 'aggregate_score': comp_summary.get('aggregate_score', np.inf), 'coverage_count': comp_summary.get('coverage_count', 0), @@ -20676,7 +22456,12 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'fit_diagnostics': fit_diagnostics, 'eebls_snr': extract_lightcurve_fit_eebls_snr(selection_fit), 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(selection_fit), - 'residual_scatter': extract_lightcurve_fit_residual_scatter(selection_fit), + 'residual_scatter': residual_scatter, + 'target_model_scatter_basis': target_model_scatter_basis, + 'projected_full_residual_scatter': projected_full_scatter, + 'selection_scatter': selection_scatter, + 'selection_scatter_basis': selection_scatter_basis, + 'target_comp_scatter': target_comp_scatter_value, 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(selection_fit), 'ktmf_contributions': extract_lightcurve_fit_ktmf_contributions(selection_fit), 'fit_point_count': 0 if tflux_fit is None else int(len(np.asarray(tflux_fit, dtype=float))), @@ -20693,7 +22478,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'final_output_dir': None, 'full_reduction_applied': final_reduction.get('applied', False), 'full_reduction_note': final_reduction.get('note'), - 'fast_ultranest_binning': final_reduction.get('fast_ultranest_binning'), + 'fast_ultranest_binning': fast_binning, 'skip_airmass_fit': final_reduction.get('skip_airmass_fit', False), 'airmass_skip_note': final_reduction.get('airmass_skip_note'), 'preflight_coverage_priority': preflight.get('coverage_priority'), @@ -20819,9 +22604,13 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p if selected_result is not None: selected_result['selected'] = True selected_result['selected_despite_transit_qc'] = fallback_to_qc_rejected - selected_ktmf_metric = selected_result.get('ktmf_metric', np.nan) - selected_transit_delta_bic = selected_result.get('transit_delta_bic', np.nan) - selected_eebls_snr = selected_result.get('eebls_snr', np.nan) + for attempt in attempts: + record_comparison_attempt_selection_pass_metrics(attempt) + selected_ktmf_metric = selected_result.get('selection_pass_ktmf_metric', np.nan) + selected_transit_delta_bic = selected_result.get('selection_pass_transit_delta_bic', np.nan) + selected_eebls_snr = selected_result.get('selection_pass_eebls_snr', np.nan) + selected_scatter_adjusted_ktmf = selected_result.get('scatter_adjusted_ktmf_metric', np.nan) + selected_combined_quality_ktmf = selected_result.get('combined_quality_ktmf_metric', np.nan) for attempt in attempts: if attempt is selected_result: @@ -20830,17 +22619,28 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p "selected as best available fallback: all completed comparison-star " "target fits were rejected by transit QC" ) - if selection_metric == 'ktmf' and np.isfinite(selected_ktmf_metric): + if ( + selection_metric in ('ktmf', 'ktmf_scatter', 'ktmf_combined_quality') + and np.isfinite(selected_ktmf_metric) + ): attempt['selection_reason'] += ( - f"; this fit had the highest KTMF ({format_ktmf_metric(selected_ktmf_metric)})" + f"; this candidate had the highest KTMF/projected-scatter score " + f"among candidates with projected scatter " + f"<= {COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER:.1f}x the lowest scatter" ) + if np.isfinite(selected_combined_quality_ktmf): + attempt['selection_reason'] += ( + f"; KTMF/projected-scatter score={selected_combined_quality_ktmf:.2f}; " + f"raw selection-pass KTMF={format_ktmf_metric(selected_ktmf_metric)}" + ) elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): attempt['selection_reason'] += ( - f"; this fit had the highest EEBLS SNR ({selected_eebls_snr:.2f})" + f"; this candidate had the highest selection-pass EEBLS SNR " + f"({selected_eebls_snr:.2f})" ) elif np.isfinite(selected_transit_delta_bic): attempt['selection_reason'] += ( - "; this fit had the strongest transit-vs-flat Delta BIC " + "; this candidate had the strongest selection-pass transit-vs-flat Delta BIC " f"({format_transit_delta_bic(selected_transit_delta_bic)})" ) elif attempt.get('search_stopped_after_qc_pass', False): @@ -20852,20 +22652,29 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p "selected: first partial-coverage comparison-star candidate with promising " "MARGINAL transit diagnostics" ) - elif selection_metric == 'ktmf' and np.isfinite(selected_ktmf_metric): + elif ( + selection_metric in ('ktmf', 'ktmf_scatter', 'ktmf_combined_quality') + and np.isfinite(selected_ktmf_metric) + ): attempt['selection_reason'] = ( - "selected: highest KTMF among the evaluated " - "comparison-star calibration candidates" + "selected: highest KTMF/projected-scatter score among comparison-star calibration " + f"candidates with projected scatter <= " + f"{COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER:.1f}x the lowest scatter" ) + if np.isfinite(selected_combined_quality_ktmf): + attempt['selection_reason'] += ( + f"; KTMF/projected-scatter score={selected_combined_quality_ktmf:.2f}; " + f"raw selection-pass KTMF={format_ktmf_metric(selected_ktmf_metric)}" + ) elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): attempt['selection_reason'] = ( - "selected: highest EEBLS SNR among the evaluated " + "selected: highest selection-pass EEBLS SNR among the evaluated " "comparison-star calibration candidates" ) else: attempt['selection_reason'] = ( - "selected: strongest transit-vs-flat Delta BIC among the evaluated " - "comparison-star calibration candidates" + "selected: strongest selection-pass transit-vs-flat Delta BIC among " + "the evaluated comparison-star calibration candidates" ) continue if ( @@ -20873,11 +22682,31 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p and attempt.get('full_reduction_applied', False) and (not attempt.get('rejected_by_transit_qc', False) or fallback_to_qc_rejected) ): - if selection_metric == 'ktmf' and np.isfinite(selected_ktmf_metric): + if attempt.get('scatter_gate_passed') is False: + selection_scatter_basis = attempt.get('selection_scatter_basis') or "selection scatter" attempt['selection_reason'] = ( - "not selected: KTMF " - f"{format_ktmf_metric(attempt.get('ktmf_metric', np.nan))} was lower than the selected " - f"{format_ktmf_metric(selected_ktmf_metric)}" + f"not selected: {selection_scatter_basis} " + f"{format_residual_scatter(attempt.get('selection_scatter', np.nan))} " + f"exceeded {COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER:.1f}x the lowest candidate scatter " + f"({format_residual_scatter(attempt.get('scatter_gate_lowest_residual_scatter', np.nan))}); " + "excluded before KTMF ranking" + ) + elif ( + selection_metric in ('ktmf', 'ktmf_scatter', 'ktmf_combined_quality') + and np.isfinite(selected_ktmf_metric) + ): + attempt_score = attempt.get('combined_quality_ktmf_metric', np.nan) + selected_score = selected_combined_quality_ktmf + score_text = ( + f"KTMF/projected-scatter score {attempt_score:.2f} was lower than the selected " + f"{selected_score:.2f}" + if np.isfinite(attempt_score) and np.isfinite(selected_score) + else "selection-pass KTMF was lower than the selected candidate" + ) + attempt['selection_reason'] = ( + "not selected: " + f"{score_text}; raw selection-pass KTMF=" + f"{format_ktmf_metric(attempt.get('selection_pass_ktmf_metric', np.nan))}" ) elif selection_metric == 'first_qc_pass': attempt['selection_reason'] = ( @@ -20890,10 +22719,10 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p "candidate with promising MARGINAL transit diagnostics" ) elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): - if np.isfinite(attempt.get('eebls_snr', np.nan)): + if np.isfinite(attempt.get('selection_pass_eebls_snr', np.nan)): attempt['selection_reason'] = ( - "not selected: EEBLS SNR " - f"{attempt['eebls_snr']:.2f} was lower than the selected " + "not selected: selection-pass EEBLS SNR " + f"{attempt['selection_pass_eebls_snr']:.2f} was lower than the selected " f"{selected_eebls_snr:.2f}" ) else: @@ -20902,8 +22731,9 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ) else: attempt['selection_reason'] = ( - "not selected: transit-vs-flat Delta BIC " - f"{format_transit_delta_bic(attempt.get('transit_delta_bic', np.nan))} was lower than the selected " + "not selected: selection-pass transit-vs-flat Delta BIC " + f"{format_transit_delta_bic(attempt.get('selection_pass_transit_delta_bic', np.nan))} " + "was lower than the selected " f"{format_transit_delta_bic(selected_transit_delta_bic)}" ) @@ -20939,11 +22769,18 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p selected_result['eebls_snr'] = extract_lightcurve_fit_eebls_snr(selected_result['fit']) selected_result['transit_delta_bic'] = extract_lightcurve_fit_transit_delta_bic(selected_result['fit']) selected_result['residual_scatter'] = extract_lightcurve_fit_residual_scatter(selected_result['fit']) + final_target_comp_scatter = target_comp_flux_scatter( + selected_result.get('good_target_flux'), + selected_result.get('good_comp_flux'), + ) + if np.isfinite(final_target_comp_scatter): + selected_result['target_comp_scatter'] = final_target_comp_scatter selected_result['ktmf_metric'] = extract_lightcurve_fit_ktmf_metric(selected_result['fit']) selected_result['ktmf_contributions'] = extract_lightcurve_fit_ktmf_contributions(selected_result['fit']) selected_result['parameter_summary'] = summarize_lightcurve_fit_parameters(selected_result['fit']) selected_result['transit_qc_status'] = getattr(selected_result['fit'], 'transit_qc_status', None) selected_result['transit_qc_summary'] = getattr(selected_result['fit'], 'transit_qc_summary', None) + update_selected_comparison_final_refit_note(selected_result) data_highres, duration_samples = estimate_transit_duration_samples_from_fit(selected_result['fit']) selected_result['data_highres'] = data_highres selected_result['duration_samples'] = duration_samples @@ -21810,6 +23647,31 @@ def _main_impl(): ) return + if should_use_automatic_optimal_calibration_selector( + exotic_infoDict.get('automatic_optimal_calibration_selector', 'n') + ): + automatic_comp_count = parse_automatic_calibration_selector_count( + exotic_infoDict.get('automatic_optimal_calibration_selector_count') + ) + automatic_comp_stars, automatic_candidates = select_automatic_optimal_calibration_stars( + reference_image, + reference_image.shape, + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter=exotic_infoDict['filter'], + field_catalog=nextastro_field_catalog, + count=automatic_comp_count, + ) + log_automatic_optimal_calibration_selection( + automatic_comp_stars, + automatic_candidates, + automatic_comp_count, + ) + if automatic_comp_stars: + exotic_infoDict['comp_stars'] = automatic_comp_stars + vsp_comp_stars = {} + check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], use_nextastro_variability_server=args.use_nextastro_variability_server) @@ -21876,6 +23738,9 @@ def _main_impl(): fit_every_comparison_candidate = should_fit_lightcurve_to_every_comparison_candidate( exotic_infoDict.get('fit_lightcurve_to_every_comparison_candidate', 'n') ) + use_ensemble_photometry_rather_than_single_comp = should_use_ensemble_photometry_rather_than_single_comp( + exotic_infoDict.get('use_ensemble_photometry_rather_than_single_comp', 'n') + ) use_deviation_from_expected_transit_in_qc = should_use_deviation_from_expected_transit_in_qc( exotic_infoDict.get('use_deviation_from_expected_transit_in_qc', True) ) @@ -21908,6 +23773,11 @@ def _main_impl(): log_info("EEBLS transit initializer disabled per optional_info setting.") if not pick_comparison_by_eebls_snr: log_info("Comparison-star selection by EEBLS SNR disabled per optional_info setting.") + if use_ensemble_photometry_rather_than_single_comp: + log_info( + "Ensemble comparison photometry enabled per optional_info setting; the final target " + "light curve will use non-rejected comparison stars as a combined reference." + ) if not use_deviation_from_expected_transit_in_qc: log_info("Expected-value transit QC deviation checks disabled per optional_info setting.") if target_driven_comp_selection: @@ -22647,6 +24517,8 @@ def _main_impl(): save_dir=exotic_infoDict['save'], planet_name=pDict['pName'], observation_date=exotic_infoDict['date'], + use_ensemble_photometry_rather_than_single_comp= + use_ensemble_photometry_rather_than_single_comp, ) comparison_calibration['ranked_fit_comp_indices'] = [ summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] @@ -22670,7 +24542,12 @@ def _main_impl(): if selected_attempt is not None: selected_comp_index = selected_attempt['comp_index'] selected_ckey = selected_attempt['ckey'] - selected_comp_coords = exotic_infoDict['comp_stars'][selected_comp_index] + selected_is_ensemble = selected_comp_index is None + selected_comp_coords = ( + None + if selected_is_ensemble + else exotic_infoDict['comp_stars'][selected_comp_index] + ) selected_min_aperture = 0 if comparison_calibration['method'] == 'psf' else comparison_calibration['aper'] selected_min_annulus = comparison_calibration['annulus'] selected_a = None if comparison_calibration['method'] == 'psf' else comparison_calibration['a'] @@ -22682,12 +24559,15 @@ def _main_impl(): selected_attempt.get('source_indices', np.arange(len(tFlux1), dtype=int)), dtype=int, ) + selected_attempt_label = selected_attempt.get('label', 'comparison candidate') if selected_attempt.get('search_stopped_after_qc_pass', False): selection_basis = 'first_qc_pass' elif selected_attempt.get('search_stopped_after_promising_partial', False): selection_basis = 'promising_partial' elif selected_attempt.get('selected_despite_transit_qc', False): selection_basis = 'comparison_field_qc_fallback' + elif selected_is_ensemble: + selection_basis = 'comparison_ensemble' elif selected_comp_index == comparison_calibration['best_comp_index']: selection_basis = 'comparison_field' else: @@ -22695,13 +24575,13 @@ def _main_impl(): if selection_basis == 'first_qc_pass': log_info( "Comparison-star calibration target-fit selection chose " - f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " + f"{selected_attempt_label} with {comparison_calibration['method_label']} " "because it was the first candidate to pass transit QC." ) elif selection_basis == 'promising_partial': log_info( "Comparison-star calibration target-fit selection chose " - f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " + f"{selected_attempt_label} with {comparison_calibration['method_label']} " "because pre-UltraNest preflight and the candidate fit indicated a promising " "partial-coverage MARGINAL solution." ) @@ -22711,6 +24591,16 @@ def _main_impl(): fallback_metric_value = format_ktmf_metric( selected_attempt.get('ktmf_metric', np.nan) ) + elif fallback_selection_metric in ('ktmf_scatter', 'ktmf_combined_quality'): + score_key = ( + 'combined_quality_ktmf_metric' + if fallback_selection_metric == 'ktmf_combined_quality' + else 'scatter_adjusted_ktmf_metric' + ) + score = selected_attempt.get(score_key, np.nan) + fallback_metric_value = ( + f"{score:.2f}" if np.isfinite(score) else "n/a" + ) elif fallback_selection_metric == 'eebls_snr': fallback_metric_value = format_eebls_snr( selected_attempt.get('eebls_snr', np.nan) @@ -22722,11 +24612,17 @@ def _main_impl(): log_info( "Warning: all completed comparison-star target fits were rejected by transit QC; " "continuing with the best available fit " - f"(Comp {selected_comp_index + 1}, " + f"({selected_attempt_label}, " f"{comparison_selection_metric_label(fallback_selection_metric)}=" f"{fallback_metric_value}) so final outputs are still produced.", warn=True, ) + elif selection_basis == 'comparison_ensemble': + log_info( + "Comparison-star calibration target-fit selection chose the comparison-star ensemble " + f"with {comparison_calibration['method_label']} because " + "'use_ensemble_photometry_rather_than_single_comp' is enabled." + ) elif selection_basis == 'comparison_field_retry': retry_count = selected_attempt['rank'] log_info( @@ -22738,7 +24634,9 @@ def _main_impl(): ) photometry_info.update(best_fit_lc=myfit, - comp_star_num=selected_comp_index + 1, + comp_star_num=( + 'ensemble' if selected_is_ensemble else selected_comp_index + 1 + ), comp_star_coords=selected_comp_coords, min_aperture=selected_min_aperture, min_annulus=selected_min_annulus, @@ -22776,12 +24674,17 @@ def _main_impl(): flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) + ref_centroid_x = np.full(selected_source_indices.shape, np.nan, dtype=float) + ref_centroid_y = np.full(selected_source_indices.shape, np.nan, dtype=float) + if selected_ckey in psf_data: + ref_centroid_x = psf_data[selected_ckey][selected_source_indices, 0] + ref_centroid_y = psf_data[selected_ckey][selected_source_indices, 1] centroid_positions.update(x_targ=psf_data["target"][selected_source_indices, 0], y_targ=psf_data["target"][selected_source_indices, 1], - x_ref=psf_data[selected_ckey][selected_source_indices, 0], - y_ref=psf_data[selected_ckey][selected_source_indices, 1]) + x_ref=ref_centroid_x, + y_ref=ref_centroid_y) - if selected_comp_index in vsp_num: + if selected_comp_index is not None: ref_flux[selected_comp_index] = { 'myfit': myfit, 'pos': exotic_infoDict['comp_stars'][selected_comp_index] @@ -22892,7 +24795,10 @@ def _main_impl(): display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] == 0: # psf - log_info(f"Transit Fit Comparison Star: #{photometry_info['comp_star_num']}") + if photometry_info.get('comp_star_num') == 'ensemble': + log_info("Transit Fit Comparison Star: ensemble") + else: + log_info(f"Transit Fit Comparison Star: #{photometry_info['comp_star_num']}") log_info("Optimal Method: PSF photometry") elif photometry_info['min_aperture'] < 0: # no comp star log_info("Transit Fit Comparison Star: None") @@ -22907,7 +24813,10 @@ def _main_impl(): log_info(f"Optimal Aperture: {abs(np.round(display_aperture, 2))}") log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") else: - log_info(f"Transit Fit Comparison Star: #{photometry_info['comp_star_num']}") + if photometry_info.get('comp_star_num') == 'ensemble': + log_info("Transit Fit Comparison Star: ensemble") + else: + log_info(f"Transit Fit Comparison Star: #{photometry_info['comp_star_num']}") if adaptive_summary is not None: log_info(f"Optimal Aperture: {display_aperture:.2f} +/- {adaptive_summary['aperture_std']:.2f} px") log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") @@ -22987,7 +24896,7 @@ def _main_impl(): log_info(f"Warning: Could not save comparison-candidate lightcurve plots ({e}).", warn=True) # save psf_data to disk for best comparison star - if bestCompStar: + if isinstance(bestCompStar, int): np.savetxt(Path(exotic_infoDict['save']) / "temp" / "psf_data_comp.txt", psf_data[f"comp{bestCompStar}"], header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f") @@ -23174,17 +25083,27 @@ def _main_impl(): psf_sigma=sigma_display, ) - plot_fov(fov_aperture, fov_annulus, sigma_display, - centroid_positions['x_targ'][0], centroid_positions['y_targ'][0], - centroid_positions['x_ref'][0], centroid_positions['y_ref'][0], - firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], - exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov, - sky_inner_radius=fov_sky_geometry['inner_radius'], - sky_outer_radius=fov_sky_geometry['outer_radius']) - - plot_centroids(centroid_positions['x_targ'], centroid_positions['y_targ'], - centroid_positions['x_ref'], centroid_positions['y_ref'], - goodTimes, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + reference_centroid_available = ( + np.isfinite(centroid_positions['x_ref'][0]) + and np.isfinite(centroid_positions['y_ref'][0]) + ) + if reference_centroid_available: + plot_fov(fov_aperture, fov_annulus, sigma_display, + centroid_positions['x_targ'][0], centroid_positions['y_targ'][0], + centroid_positions['x_ref'][0], centroid_positions['y_ref'][0], + firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], + exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov, + sky_inner_radius=fov_sky_geometry['inner_radius'], + sky_outer_radius=fov_sky_geometry['outer_radius']) + + plot_centroids(centroid_positions['x_targ'], centroid_positions['y_targ'], + centroid_positions['x_ref'], centroid_positions['y_ref'], + goodTimes, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + else: + log_info( + "Skipping reference-star FOV and centroid plots because the selected reference is " + "a comparison-star ensemble rather than a single star." + ) plot_flux(goodTimes, flux_values['flux_tar'], flux_values['flux_unc_tar'], flux_values['flux_ref'], flux_values['flux_unc_ref'], @@ -23223,12 +25142,16 @@ def _main_impl(): # standardDev1 = np.std(goodFluxes) if vsp_comp_stars: - if bestCompStar: + if isinstance(bestCompStar, int): vsp_params = stellar_variability(ref_flux, best_fit_lc, exotic_infoDict['comp_stars'], vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], pDict['sName'], observed_filter=exotic_infoDict.get('observed_filter', - exotic_infoDict.get('filter'))) + exotic_infoDict.get('filter')), + comp_ra_dec=ra_dec_wcs, + field_catalog=nextastro_field_catalog, + reference_image=reference_image, + wcs_file=wcs_file) else: log_info( "Skipping AID magnitude output because no transit-fit comparison star was selected.", @@ -23635,7 +25558,10 @@ def _main_impl(): display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] >= 0: - log_info(f" Transit Fit Comparison Star: #{bestCompStar} - {comp_coords}") + if bestCompStar == 'ensemble': + log_info(" Transit Fit Comparison Star: ensemble") + else: + log_info(f" Transit Fit Comparison Star: #{bestCompStar} - {comp_coords}") else: log_info(" Transit Fit Comparison Star: None") if photometry_info['min_aperture'] == 0: @@ -23700,7 +25626,7 @@ def _main_impl(): except Exception as e: log_info(f"\nError: Could not create FinalParams.json. {error_txt}\n\t{e}", error=True) try: - if bestCompStar: + if isinstance(bestCompStar, int): exotic_infoDict['phot_comp_star'] = save_comp_ra_dec(wcs_file, ra_wcs, dec_wcs, comp_coords) aavso_photometry_info = photometry_info if fitsortext == 1 else None aavso_frame_filtering_info = None diff --git a/exotic/inputs.py b/exotic/inputs.py index e3efb2a3..15e8add6 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -557,6 +557,20 @@ def comp_params(self, init_file, planet_dict): 'fit_lightcurve_to_every_comparison_candidate', 'Fit Lightcurve to Every Comparison Candidate? (y/n)', ), + 'automatic_optimal_calibration_selector': ( + 'automatic_optimal_calibration_selector', + 'Automatic Optimal Calibration Selector? (y/n)', + ), + 'automatic_optimal_calibration_selector_count': ( + 'automatic_optimal_calibration_selector_count', + 'Automatic Optimal Calibration Selector Count', + 'automatic_optimal_calibration_selector_max_stars', + 'Automatic Optimal Calibration Selector Max Stars', + ), + 'use_ensemble_photometry_rather_than_single_comp': ( + 'use_ensemble_photometry_rather_than_single_comp', + 'Use Ensemble Photometry Rather Than Single Comp? (y/n)', + ), 'ultranest_min_num_live_points': ( 'Minimum Number of Live Points for UltraNest', 'minimum number of live points for ultranest', @@ -995,7 +1009,7 @@ def target_star_coords(coords, planet): def comparison_star_coords(comp_stars, rt_bool): - if isinstance(comp_stars, list) and 1 <= len(comp_stars) <= 10 and \ + if isinstance(comp_stars, list) and len(comp_stars) >= 1 and \ all(isinstance(star, list) for star in comp_stars): comp_stars = [star for star in comp_stars if star != []] elif isinstance(comp_stars, str) and any(str.isdigit(x) for x in comp_stars): @@ -1008,8 +1022,8 @@ def comparison_star_coords(comp_stars, rt_bool): if not comp_stars: while True: if not rt_bool: - num_comp_stars = user_input("\nHow many Comparison Stars would you like to use? (1-10): ", type_=int) - if 1 <= num_comp_stars <= 10: + num_comp_stars = user_input("\nHow many Comparison Stars would you like to use? (1 or more): ", type_=int) + if num_comp_stars >= 1: break log_info("\nError: The number of Comparison Stars entered is incorrect.", error=True) else: diff --git a/exotic/output_files.py b/exotic/output_files.py index a010879b..78b23579 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -157,15 +157,25 @@ def stellar_variability_measurement_summary(vsp_params, transit_fit_comp_star=No return None return ( - f"Remeasured {point_count} out-of-transit target/reference point(s) against the transit-fit catalog " - "reference for AID magnitudes; AID rows list the JD timestamps used." + f"Remeasured {point_count} out-of-transit target/reference point(s) against the transit-fit " + f"{'derived ' if vsp_params[0].get('derived_catalog_reference') else ''}catalog reference " + "for AID magnitudes; AID rows list the BJD_TDB timestamps used." ) def aid_comparison_metadata(vsp_param): if not vsp_param: return {} - return aavso_json_safe({ + anchor_labels = vsp_param.get('derived_reference_anchor_labels') + anchor_label_sample = None + if isinstance(anchor_labels, np.ndarray): + anchor_labels = anchor_labels.tolist() + if isinstance(anchor_labels, (list, tuple)): + anchor_labels = list(anchor_labels) + if len(anchor_labels) > 10: + anchor_label_sample = anchor_labels[:10] + anchor_labels = None + metadata = { 'source': vsp_param.get('catalog_source', 'AAVSO VSP'), 'is_aavso_vsp': bool(vsp_param.get('is_aavso_vsp', True)), 'comparison_name': vsp_param.get('cname'), @@ -177,10 +187,17 @@ def aid_comparison_metadata(vsp_param): 'catalog_source_id': vsp_param.get('source_id'), 'catalog_id': vsp_param.get('catalog_id'), 'catalog_match_separation_arcsec': vsp_param.get('separation_arcsec'), + 'derived_catalog_reference': bool(vsp_param.get('derived_catalog_reference', False)), + 'derived_reference_anchor_count': vsp_param.get('derived_reference_anchor_count'), 'magnitude_band': vsp_param.get('mag_band'), 'apparent_magnitude': rounded_magnitude_value(vsp_param.get('cmag')), 'apparent_magnitude_error': rounded_magnitude_error(vsp_param.get('cmag_err')), - }) + } + if anchor_labels is not None: + metadata['derived_reference_anchor_labels'] = anchor_labels + if anchor_label_sample is not None: + metadata['derived_reference_anchor_label_sample'] = anchor_label_sample + return aavso_json_safe(metadata) def prune_aavso_metadata(value): @@ -690,6 +707,11 @@ def build_aavso_qc_metadata(fit): 'delta_bic', 'transit_parameter_count', 'flat_parameter_count', 'flat_baseline', 'flat_a2', 'flat_model_note', 'residual_scatter', 'transit_depth_for_residual_scatter', 'residual_scatter_to_depth_ratio', + 'residual_flatness_score', 'residual_flatness_trend_strength', + 'residual_flatness_curve_strength', 'residual_flatness_scatter_ratio', + 'residual_flatness_trend_score', 'residual_flatness_curve_score', + 'residual_flatness_scatter_stability_score', + 'residual_flatness_dominant_metric', 'residual_flatness_detail', 'rprs_sigma', 'duration_ratio', 'eebls_depth_snr', 'sampling_score', 'sampling_detail', 'sampling_ingress_count', 'sampling_egress_count', 'sampling_in_transit_count', @@ -743,6 +765,27 @@ def compact_comparison_attempt_decision(attempt): 'label': attempt.get('label'), 'selected': attempt.get('selected'), 'selection_reason': attempt.get('selection_reason'), + 'selection_pass_ktmf_metric': attempt.get('selection_pass_ktmf_metric'), + 'selection_pass_transit_delta_bic': attempt.get('selection_pass_transit_delta_bic'), + 'selection_pass_eebls_snr': attempt.get('selection_pass_eebls_snr'), + 'selection_pass_residual_scatter': attempt.get('selection_pass_residual_scatter'), + 'target_model_scatter_basis': attempt.get('target_model_scatter_basis'), + 'projected_full_residual_scatter': attempt.get('projected_full_residual_scatter'), + 'selection_scatter': attempt.get('selection_scatter'), + 'selection_scatter_basis': attempt.get('selection_scatter_basis'), + 'target_comp_scatter': attempt.get('target_comp_scatter'), + 'selection_pass_target_comp_scatter': attempt.get('selection_pass_target_comp_scatter'), + 'selection_pass_transit_qc_status': attempt.get('selection_pass_transit_qc_status'), + 'selection_pass_transit_qc_summary': attempt.get('selection_pass_transit_qc_summary'), + 'scatter_gate_passed': attempt.get('scatter_gate_passed'), + 'scatter_gate_lowest_residual_scatter': attempt.get('scatter_gate_lowest_residual_scatter'), + 'scatter_gate_threshold': attempt.get('scatter_gate_threshold'), + 'scatter_adjusted_ktmf_metric': attempt.get('scatter_adjusted_ktmf_metric'), + 'combined_quality_ktmf_metric': attempt.get('combined_quality_ktmf_metric'), + 'combined_quality_best_residual_scatter': attempt.get('combined_quality_best_residual_scatter'), + 'combined_quality_best_target_comp_scatter': attempt.get('combined_quality_best_target_comp_scatter'), + 'combined_quality_best_comp_stability': attempt.get('combined_quality_best_comp_stability'), + 'final_refit_metric_note': attempt.get('final_refit_metric_note'), 'ktmf_metric': attempt.get('ktmf_metric'), 'ktmf_contributions': compact_ktmf_contributions(attempt.get('ktmf_contributions')), 'transit_delta_bic': attempt.get('transit_delta_bic'), @@ -783,6 +826,11 @@ def build_ktmf_decision_metadata(fit, photometry_info=None): 'residual_scatter': transit_qc.get('residual_scatter'), 'transit_depth_for_residual_scatter': transit_qc.get('transit_depth_for_residual_scatter'), 'residual_scatter_to_depth_ratio': transit_qc.get('residual_scatter_to_depth_ratio'), + 'residual_flatness_score': transit_qc.get('residual_flatness_score'), + 'residual_flatness_detail': transit_qc.get('residual_flatness_detail'), + 'residual_flatness_trend_strength': transit_qc.get('residual_flatness_trend_strength'), + 'residual_flatness_curve_strength': transit_qc.get('residual_flatness_curve_strength'), + 'residual_flatness_scatter_ratio': transit_qc.get('residual_flatness_scatter_ratio'), 'sampling_score': transit_qc.get('sampling_score'), 'sampling_detail': transit_qc.get('sampling_detail'), 'sampling_ingress_count': transit_qc.get('sampling_ingress_count'), @@ -872,21 +920,99 @@ def format_optional_metric(label, value, precision=2): return f"{label}={value:.{precision}f}" +def format_transit_delta_bic(value): + value = finite_float(value) + return f"{value:.2f}" if np.isfinite(value) else "n/a" + + +def format_percent_metric(label, value, precision=4): + value = finite_float(value) + if not np.isfinite(value): + return None + return f"{label}={value * 100.0:.{precision}f}%" + + +def metric_values_differ(first_value, second_value, tolerance=5.0e-3): + first_value = finite_float(first_value) + second_value = finite_float(second_value) + return ( + np.isfinite(first_value) + and np.isfinite(second_value) + and abs(first_value - second_value) > tolerance + ) + + def format_ktmf_candidate_decision(attempt): attempt = compact_comparison_attempt_decision(attempt) label = attempt.get('label') or ( f"Comp {attempt['comparison_star']}" if attempt.get('comparison_star') is not None else "Comparison candidate" ) selected_text = " [selected]" if attempt.get('selected') else "" - parts = [ - f"{label}{selected_text}: KTMF={format_ktmf_metric(attempt.get('ktmf_metric'))}", - ] + ktmf_text = f"KTMF={format_ktmf_metric(attempt.get('ktmf_metric'))}" + if metric_values_differ( + attempt.get('selection_pass_ktmf_metric'), + attempt.get('ktmf_metric'), + ): + ktmf_text += ( + f" (selection-pass {format_ktmf_metric(attempt.get('selection_pass_ktmf_metric'))})" + ) + parts = [f"{label}{selected_text}: {ktmf_text}"] for metric_text in ( format_optional_metric("Delta BIC", attempt.get('transit_delta_bic')), format_optional_metric("EEBLS SNR", attempt.get('eebls_snr')), + format_percent_metric("Target/comp scatter", attempt.get('target_comp_scatter')), + format_percent_metric("Target model scatter", attempt.get('residual_scatter')), + format_percent_metric("Selection scatter", attempt.get('selection_scatter')), + format_optional_metric("KTMF/projected-scatter score", attempt.get('combined_quality_ktmf_metric')), ): if metric_text: parts.append(metric_text) + target_model_basis = attempt.get('target_model_scatter_basis') + selection_scatter_basis = attempt.get('selection_scatter_basis') + if target_model_basis: + parts.append(f"target model scatter basis={target_model_basis}") + if selection_scatter_basis: + parts.append(f"selection scatter basis={selection_scatter_basis}") + selection_pass_target_comp_scatter = finite_float(attempt.get('selection_pass_target_comp_scatter')) + target_comp_scatter = finite_float(attempt.get('target_comp_scatter')) + if ( + np.isfinite(selection_pass_target_comp_scatter) + and ( + not np.isfinite(target_comp_scatter) + or abs(selection_pass_target_comp_scatter - target_comp_scatter) > 1.0e-5 + ) + ): + parts.append( + "selection-pass target/comp scatter=" + f"{selection_pass_target_comp_scatter * 100.0:.4f}%" + ) + if metric_values_differ( + attempt.get('selection_pass_residual_scatter'), + attempt.get('residual_scatter'), + tolerance=1.0e-5, + ): + parts.append( + "selection-pass target model scatter=" + f"{finite_float(attempt.get('selection_pass_residual_scatter')) * 100.0:.4f}%" + ) + if metric_values_differ( + attempt.get('selection_pass_transit_delta_bic'), + attempt.get('transit_delta_bic'), + tolerance=1.0e-2, + ): + parts.append( + "selection-pass Delta BIC=" + f"{format_transit_delta_bic(attempt.get('selection_pass_transit_delta_bic'))}" + ) + if metric_values_differ( + attempt.get('selection_pass_eebls_snr'), + attempt.get('eebls_snr'), + tolerance=1.0e-2, + ): + parts.append( + "selection-pass EEBLS SNR=" + f"{finite_float(attempt.get('selection_pass_eebls_snr')):.2f}" + ) qc_status = attempt.get('transit_qc_status') if qc_status: parts.append(f"QC={str(qc_status).upper()}") @@ -1639,6 +1765,9 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor prefit_refinement_note = getattr(self.fit, 'prefit_refinement_note', None) if prefit_refinement_note: params_num["Prefit refinement note"] = str(prefit_refinement_note) + geometry_prior_note = getattr(self.fit, 'partial_transit_geometry_prior_assumption_note', None) + if geometry_prior_note: + params_num["Prior-assumed partial-transit geometry note"] = str(geometry_prior_note) oot_baseline_parameter_note = getattr(self.fit, 'oot_baseline_parameter_fit_note', None) if oot_baseline_parameter_note: params_num["Out-of-transit baseline parameter-fit note"] = str(oot_baseline_parameter_note) @@ -1765,11 +1894,14 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num["Variable Reference Measurement"] = measurement_summary if phot_opt: - transit_fit_comp_text = ( - f"#{comp_star} - {comp_coords}" - if comp_star is not None and min_aper >= 0 - else str(comp_star) - ) + if comp_star == 'ensemble': + transit_fit_comp_text = "ensemble" + else: + transit_fit_comp_text = ( + f"#{comp_star} - {comp_coords}" + if comp_star is not None and min_aper >= 0 + else str(comp_star) + ) phot_ext = { "Transit Fit Comparison Star": transit_fit_comp_text } @@ -1944,7 +2076,7 @@ def aavso(self): f"#OBSCODE={self.i_dict['aavso_num']}\n" # UI f"#SOFTWARE=EXOTIC v{__version__}\n" # fixed "#DELIM=,\n" # fixed - "#DATE=JD\n" # fixed + "#DATE=BJD_TDB\n" # fixed f"#OBSDATE={format_aavso_header_value(self.i_dict.get('date'))}\n" f"#OBSTYPE={self.i_dict['camera']}\n" f"#OBSLAT={format_aavso_header_value(self.i_dict.get('lat'))}\n" diff --git a/exotic/plots.py b/exotic/plots.py index 9fe4ee1a..1aacab07 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -589,6 +589,26 @@ def _stellar_variability_reference_label(vsp_param, comparison_label): return str(comparison_label) if comparison_label else "" +def _stellar_variability_aavso_assumption_label(vsp_param): + if not vsp_param.get('is_aavso_vsp', True): + return "" + + details = [] + observed_filter = vsp_param.get('observed_filter') + if observed_filter: + details.append(f"Observed filter: {observed_filter}") + + comparison_mag = magnitude_text( + vsp_param.get('mag_band') or 'V', + vsp_param.get('cmag'), + vsp_param.get('cmag_err'), + ) + if comparison_mag is not None: + details.append(f"Assumed comparison: {comparison_mag}") + + return " | ".join(details) + + def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): if not vsp_params: return @@ -609,7 +629,13 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): first_param = vsp_params[0] band = first_param.get('mag_band') or 'V' reference_label = _stellar_variability_reference_label(first_param, vsp_auid_comp) - ax.set_title(f"{s_name}\n{reference_label}" if reference_label else s_name) + title_lines = [s_name] + if reference_label: + title_lines.append(reference_label) + assumption_label = _stellar_variability_aavso_assumption_label(first_param) + if assumption_label: + title_lines.append(assumption_label) + ax.set_title("\n".join(title_lines)) ax.set_ylabel(f"Magnitude ({band})") ax.set_xlabel("Time [JD]") fig.tight_layout() @@ -1342,6 +1368,7 @@ def _short_ktmf_label(label): replacements = { "Deviation From Expected Value": "Expected Rp/R*", "Residual Scatter Around Full Model Fit": "Residual scatter", + "Residual Flatness": "Residual flatness", "Duration Consistency": "Duration", "EEBLS Depth SNR": "EEBLS SNR", "Sampling / Cadence": "Sampling", diff --git a/manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json b/manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json deleted file mode 100644 index fe408498..00000000 --- a/manual_comp_refactor_tmp/archives_qc_failed/comp_1_failed/temp/FailedFitSummary_HAT-P-32b_2026-04-28.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "planet_name": "HAT-P-32 b", - "observation_date": "2026-04-28", - "comparison_star": 1, - "comparison_label": "Comp 1", - "comparison_position": [ - 100.0, - 200.0 - ], - "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", - "failure_reason": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10).", - "fit_diagnostics": { - "input_point_count": 6, - "has_reference_flux": true, - "relative_flux_point_count": 6, - "sigma_clip_point_count": 4, - "usable_point_count": 4, - "failed_stage": "transit_qc", - "failure_reason": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." - }, - "parameter_summary": null, - "fit_point_count": 6, - "eebls_snr": NaN, - "transit_delta_bic": NaN, - "residual_scatter": 0.0, - "ktmf_metric": NaN, - "ktmf_contributions": [], - "transit_qc": { - "status": "fail", - "summary": "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." - }, - "saved_debug_series": null, - "saved_bestfit_plot": null, - "archive_errors": [] -} \ No newline at end of file diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 27bf6c93..5ecdce3d 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -1131,3 +1131,69 @@ def add_blob(x_pos, y_pos, value): [candidate["flux"] for candidate in candidates], reverse=True, ) + + +def test_automatic_optimal_calibration_selector_filters_flux_and_ranks_color(monkeypatch): + image = np.zeros((300, 300), dtype=float) + + def add_blob(x_pos, y_pos, value): + image[y_pos - 1:y_pos + 2, x_pos - 1:x_pos + 2] = value * 0.5 + image[y_pos, x_pos] = value + + add_blob(150, 150, 2000.0) + add_blob(220, 220, 1800.0) + add_blob(80, 80, 1700.0) + add_blob(230, 80, 6000.0) + + ra_wcs = np.tile(np.arange(300, dtype=float), (300, 1)) + dec_wcs = np.tile(np.arange(300, dtype=float)[:, None], (1, 300)) + catalog = {"rows": []} + + def fake_color_match(_catalog, ra, dec, obs_filter, max_separation_arcsec=5.0): + colors = { + (150, 150): (12.0, 11.4), + (220, 220): (13.0, 12.41), + (80, 80): (13.0, 12.0), + (230, 80): (10.0, 9.4), + } + key = (int(round(float(ra))), int(round(float(dec)))) + if key not in colors: + return None + b_mag, v_mag = colors[key] + return {"catalog_row": {"Bmag": b_mag, "Vmag": v_mag}} + + monkeypatch.setattr(exotic_module, "nextastro_catalog_nearest_color_row", fake_color_match) + + comp_stars, candidates = exotic_module.select_automatic_optimal_calibration_stars( + image, + image.shape, + target_pixel=[150, 150], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter="V", + field_catalog=catalog, + count=2, + ) + + assert comp_stars[0] == [220.0, 220.0] + assert [candidate["color_delta"] for candidate in candidates] == sorted( + candidate["color_delta"] for candidate in candidates + ) + assert all(0.5 <= candidate["brightness_ratio"] <= 2.0 for candidate in candidates) + + +def test_build_absolute_comp_ensemble_flux_uses_median_normalized_members(): + comp_flux_map = { + "comp1": np.array([100.0, 102.0, 98.0, 100.0, 101.0, 99.0]), + "comp2": np.array([200.0, 204.0, 196.0, 200.0, 202.0, 198.0]), + "comp3": np.array([np.nan, np.nan, np.nan, np.nan, np.nan, np.nan]), + } + + ensemble_flux, member_keys = exotic_module.build_absolute_comp_ensemble_flux( + comp_flux_map, + ["comp1", "comp2", "comp3"], + ) + + assert member_keys == ["comp1", "comp2"] + assert np.nanmedian(ensemble_flux) == pytest.approx(150.0) + assert ensemble_flux[1] / np.nanmedian(ensemble_flux) == pytest.approx(1.02) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 3dc3fd0e..9abab86f 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -112,6 +112,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: compute_transit_qc_ktmf, detect_aperture_correction_star_candidates, transit_qc_residual_scatter_score, + transit_qc_residual_flatness_summary, transit_qc_sampling_summary, apply_comparison_star_suitability_outlier_rejection, comparison_calibration_selection_reason, @@ -154,6 +155,8 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: psf_solution_quality_score, target_psf_shape_quality_components, target_psf_shape_quality_mask, + target_comp_flux_scatter, + fitted_lightcurve_scatter_on_dataset, populate_aperture_data_for_frame, rank_comparison_candidate_preflight_plans, refit_selected_fast_comparison_on_full_lightcurve, @@ -168,6 +171,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: save_selected_photometry_debug_series, select_comparison_calibrated_photometry, select_alignment_candidate, + select_preferred_comparison_attempt, should_keep_header_wcs_alignment, should_prefer_pixel_values_over_wcs_for_target, sigma_clip, @@ -1956,12 +1960,12 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): "ktmf_metric": 4.35, "ktmf_contributions": [ { - "label": "Delta BIC", + "label": "EEBLS Depth SNR", "available": True, "points": 1.25, "max_points": 1.40, "score": 0.89, - "detail": "Delta BIC=18.40", + "detail": "6.50", } ], "coverage_count": 3, @@ -1992,7 +1996,7 @@ def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): assert any("coverage=1 valid frame(s) out of 3 total; min_required=2; peer_median=3.0" in message for message in logged) assert any("Comp 1" in message and "reason=comparison candidate rejected after iterative low-coverage clipping" in message for message in logged) assert any("Comp 2 [selected]" in message and "ktmf=4.35/5.00" in message and "comparison-field calibration ranked this star best" in message for message in logged) - assert any("KTMF contribution: Delta BIC +1.25/1.40" in message for message in logged) + assert any("KTMF contribution: EEBLS Depth SNR +1.25/1.40" in message for message in logged) assert any("parameters: fit_method=ultranest" in message for message in logged) @@ -2117,6 +2121,8 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): "residual_scatter": 0.005, "transit_depth_for_residual_scatter": 0.01, "residual_scatter_to_depth_ratio": 0.5, + "residual_flatness_score": 0.5, + "residual_flatness_detail": "curve=0.50", "rprs_sigma": 6.0, "duration_ratio": 1.0, "eebls_depth_snr": 8.0, @@ -2125,24 +2131,24 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): ktmf_metric, contributions = compute_transit_qc_ktmf(summary) contributions_by_label = {contribution["label"]: contribution for contribution in contributions} - assert "Model Evidence" in contributions_by_label + assert "Model Evidence" not in contributions_by_label assert "Delta BIC" not in contributions_by_label assert "Delta chi2" not in contributions_by_label - scale = 5.0 / (0.3 + 1.5 + 0.7 + 0.75 + 1.3) - assert contributions_by_label["Model Evidence"]["max_points"] == pytest.approx(0.3 * scale) - assert contributions_by_label["Deviation From Expected Value"]["max_points"] == pytest.approx(1.5 * scale) + scale = 5.0 / (2.0 + 0.7 + 1.0 + 0.75 + 1.3) + assert contributions_by_label["Deviation From Expected Value"]["max_points"] == pytest.approx(2.0 * scale) assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) + assert contributions_by_label["Residual Flatness"]["max_points"] == pytest.approx(1.0 * scale) assert "Rp/R* Significance" not in contributions_by_label assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75 * scale) assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) assert "Rp/R* sigma=2.00" in contributions_by_label["Deviation From Expected Value"]["detail"] assert "Tmid" not in contributions_by_label["Deviation From Expected Value"]["detail"] + assert contributions_by_label["Residual Flatness"]["score"] == pytest.approx(0.5) - model_evidence_score = ((1.0 - np.exp(-1.0)) + (1.0 - np.exp(-2.0))) / 2.0 expected_ktmf = scale * ( - 0.3 * model_evidence_score - + 1.5 * 0.6 + 2.0 * 0.6 + 0.7 * 1.0 + + 1.0 * 0.5 + 0.75 * 1.0 + 1.3 * (1.0 - np.exp(-2.0)) ) @@ -2162,6 +2168,46 @@ def test_transit_qc_residual_scatter_score_full_credit_floor_and_zero_ceiling(): assert mid_score < transit_qc_residual_scatter_score(0.02, transit_depth) +def test_transit_qc_residual_flatness_summary_penalizes_residual_structure(): + phase = np.linspace(-0.05, 0.05, 80) + alternating_noise = 0.001 * np.where(np.arange(phase.size) % 2 == 0, -1.0, 1.0) + + flat_summary = transit_qc_residual_flatness_summary(alternating_noise, phase) + trend_summary = transit_qc_residual_flatness_summary( + alternating_noise + 0.004 * np.linspace(-1.0, 1.0, phase.size), + phase, + ) + curve_summary = transit_qc_residual_flatness_summary( + alternating_noise + 0.004 * np.sin(2.0 * np.pi * np.linspace(0.0, 1.0, phase.size)), + phase, + ) + heteroscedastic_summary = transit_qc_residual_flatness_summary( + alternating_noise * np.r_[np.ones(40), np.full(40, 4.0)], + phase, + ) + + assert flat_summary["available"] is True + assert flat_summary["score"] > 0.9 + assert trend_summary["score"] < flat_summary["score"] + assert trend_summary["score"] < 0.5 + assert curve_summary["score"] < flat_summary["score"] + assert curve_summary["score"] < 0.6 + assert heteroscedastic_summary["score"] < flat_summary["score"] + assert heteroscedastic_summary["score"] < 0.8 + + +def test_transit_qc_residual_flatness_summary_tolerates_one_quiet_patch(): + phase = np.linspace(-0.05, 0.05, 84) + residuals = 0.001 * np.sin(np.arange(phase.size) * 2.3999632) + residuals[-10:] *= 0.12 + + summary = transit_qc_residual_flatness_summary(residuals, phase) + + assert summary["available"] is True + assert summary["score"] > 0.5 + assert summary["scatter_ratio"] < 3.0 + + def test_transit_qc_sampling_summary_scores_ingress_egress_and_baseline_counts(): fit = types.SimpleNamespace( time=np.array([ @@ -2212,22 +2258,54 @@ def test_compute_transit_qc_ktmf_omits_prior_assumed_rprs_component(): assert omitted["max_points"] == pytest.approx(0.0) assert "fixed to the input prior" in omitted["detail"] - scale = 5.0 / (0.3 + 0.7 + 0.75 + 1.3) - assert contributions_by_label["Model Evidence"]["max_points"] == pytest.approx(0.3 * scale) + assert "Model Evidence" not in contributions_by_label + scale = 5.0 / (0.7 + 0.75 + 1.3) assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75 * scale) assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) - model_evidence_score = ((1.0 - np.exp(-1.0)) + (1.0 - np.exp(-2.0))) / 2.0 expected_ktmf = scale * ( - 0.3 * model_evidence_score - + 0.7 * 1.0 + 0.7 * 1.0 + 0.75 * 1.0 + 1.3 * (1.0 - np.exp(-2.0)) ) assert ktmf_metric == pytest.approx(expected_ktmf) +def test_compute_transit_qc_ktmf_uses_only_residual_and_eebls_for_prior_assumed_geometry(): + summary = { + "geometry_prior_assumed": True, + "geometry_prior_assumed_note": "Transit geometry was fixed to priors.", + "deviation_from_expected_value": 1.0, + "residual_scatter": 0.005, + "transit_depth_for_residual_scatter": 0.01, + "residual_scatter_to_depth_ratio": 0.5, + "point_count": 86, + "duration_ratio": 1.0, + "sampling_score": 1.0, + "sampling_detail": "ingress=4, egress=4", + "eebls_depth_snr": 8.0, + } + + ktmf_metric, contributions = compute_transit_qc_ktmf(summary) + contributions_by_label = {contribution["label"]: contribution for contribution in contributions} + + assert contributions_by_label["Deviation From Expected Value"]["available"] is False + assert contributions_by_label["Duration Consistency"]["available"] is False + assert contributions_by_label["Sampling / Cadence"]["available"] is False + assert "fixed to priors" in contributions_by_label["Duration Consistency"]["detail"] + + scale = 5.0 / (0.7 + 1.3) + assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) + assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) + + expected_ktmf = scale * ( + 0.7 * 1.0 + + 1.3 * (1.0 - np.exp(-2.0)) + ) + assert ktmf_metric == pytest.approx(expected_ktmf) + + def test_comparison_candidate_fit_selection_reason_describes_comparison_field_retry(): reason = comparison_candidate_fit_selection_reason( { @@ -3843,6 +3921,36 @@ def test_evaluate_transit_detection_qc_passes_strong_model_with_low_rprs_precisi assert "not used as a transit-detection veto" in " ".join(summary["notes"]) +def test_evaluate_transit_detection_qc_uses_ktmf_marginal_band_despite_weak_bic(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.compute_transit_qc_ktmf", + lambda summary: (3.34, []), + ) + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = np.ones(21, dtype=float) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.02, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.02, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["delta_bic"] < 6.0 + assert summary["ktmf_metric"] == pytest.approx(3.34) + assert summary["status"] == "marginal" + assert "KTMF indicates a marginal transit fit" in summary["summary"] + + def test_evaluate_transit_detection_qc_uses_midpoint_anchored_duration_for_partial(): times = np.linspace(0.0, 3.0, 13) transit_model = np.ones(times.shape[0], dtype=float) @@ -4543,13 +4651,126 @@ def fake_finalize( ) assert len(result["attempts"]) == 3 - assert result["selection_metric"] == "ktmf" + assert result["selection_metric"] == "ktmf_combined_quality" assert result["selected_result"]["comp_index"] == 1 assert result["selected_result"]["rank"] == 1 assert result["selected_result"]["selected"] is True assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.70) - assert "highest KTMF" in result["selected_result"]["selection_reason"] - assert result["attempts"][0]["selection_reason"].startswith("not selected: KTMF") + assert "highest KTMF/projected-scatter" in result["selected_result"]["selection_reason"] + assert result["attempts"][0]["selection_reason"].startswith( + "not selected: full-resolution UltraNest model residual scatter" + ) + + +def test_select_preferred_comparison_attempt_rejects_noisy_high_ktmf_before_ranking(): + attempts = [ + { + "label": "Comp 1", + "rank": 0, + "ktmf_metric": 4.8, + "residual_scatter": 0.040, + "eebls_snr": 3.0, + "transit_delta_bic": 10.0, + }, + { + "label": "Comp 2", + "rank": 1, + "ktmf_metric": 3.6, + "residual_scatter": 0.010, + "eebls_snr": 2.8, + "transit_delta_bic": 8.0, + }, + { + "label": "Comp 3", + "rank": 2, + "ktmf_metric": 3.8, + "residual_scatter": 0.014, + "eebls_snr": 2.6, + "transit_delta_bic": 7.0, + }, + ] + + selected, metric = select_preferred_comparison_attempt(attempts) + + assert metric == "ktmf_combined_quality" + assert selected["label"] == "Comp 2" + assert attempts[0]["scatter_gate_passed"] is False + assert attempts[0]["scatter_gate_threshold"] == pytest.approx(0.015) + assert selected["scatter_adjusted_ktmf_metric"] == pytest.approx(3.6) + assert attempts[2]["scatter_adjusted_ktmf_metric"] == pytest.approx(3.8 * 0.010 / 0.014) + + +def test_select_preferred_comparison_attempt_uses_ktmf_and_projected_selection_scatter_only(): + attempts = [ + { + "label": "Comp 1", + "rank": 0, + "ktmf_metric": 2.59, + "selection_scatter": 0.022659, + "target_comp_scatter": 0.005693, + "aggregate_score": 0.010, + "eebls_snr": 3.27, + "transit_delta_bic": 2.71, + }, + { + "label": "Comp 2", + "rank": 1, + "ktmf_metric": 3.23, + "selection_scatter": 0.027437, + "target_comp_scatter": 0.003000, + "aggregate_score": 0.001, + "eebls_snr": 3.53, + "transit_delta_bic": 2.28, + }, + { + "label": "Comp 9", + "rank": 2, + "ktmf_metric": 3.25, + "selection_scatter": 0.023904, + "target_comp_scatter": 0.030258, + "aggregate_score": 0.100, + "eebls_snr": 1.61, + "transit_delta_bic": -8.36, + }, + ] + + selected, metric = select_preferred_comparison_attempt(attempts) + + assert metric == "ktmf_combined_quality" + assert selected["label"] == "Comp 9" + assert attempts[0]["combined_quality_ktmf_metric"] == pytest.approx(2.59 / 2.2659) + assert attempts[1]["combined_quality_ktmf_metric"] == pytest.approx(3.23 / 2.7437) + assert attempts[2]["combined_quality_ktmf_metric"] == pytest.approx(3.25 / 2.3904) + assert selected["combined_quality_ktmf_metric"] > attempts[1]["combined_quality_ktmf_metric"] + assert selected["combined_quality_ktmf_metric"] > attempts[0]["combined_quality_ktmf_metric"] + + +def test_target_comp_flux_scatter_measures_normalized_target_reference_ratio(): + comp_flux = np.full(8, 100.0, dtype=float) + ratio = np.array([1.00, 1.01, 0.99, 1.02, 0.98, 1.00, 1.01, 0.99], dtype=float) + target_flux = comp_flux * ratio + + scatter = target_comp_flux_scatter(target_flux, comp_flux, min_points=5) + + assert scatter == pytest.approx(0.014826, rel=1.0e-3) + + +def test_fitted_lightcurve_scatter_on_dataset_projects_fit_to_full_flux(monkeypatch): + def fake_transit(times, parameters): + return np.ones_like(np.asarray(times, dtype=float)) + + monkeypatch.setattr("exotic.exotic.transit", fake_transit) + fit = types.SimpleNamespace( + parameters={"a0": 1.0, "a2": 0.0}, + airmass_reference=1.0, + ) + times = np.arange(8, dtype=float) + flux_values = np.array([1.0, 1.01, 0.99, 1.02, 0.98, 1.0, 1.01, 0.99], dtype=float) + airmass = np.ones_like(times) + + scatter = fitted_lightcurve_scatter_on_dataset(fit, times, flux_values, airmass) + + assert scatter == pytest.approx(np.std(flux_values - 1.0) / np.median(flux_values)) def test_fit_ranked_comparison_calibration_candidates_extends_only_selected_final_fit(monkeypatch): diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 169c8c29..78f9c156 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -1551,6 +1551,182 @@ def fake_lc_fitter( assert "one-sided/LOW" in fit.b_posterior_refit_note +def test_one_sided_partial_coverage_fixes_geometry_and_samples_tmid_only(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + fixed_parameter_errors=None, + fixed_flux_baseline=False, + **kwargs, + ): + captured["prior"] = dict(call_prior) + captured["bounds"] = dict(call_bounds) + captured["fixed_parameter_errors"] = dict(fixed_parameter_errors or {}) + captured["fixed_flux_baseline"] = bool(fixed_flux_baseline) + fit = types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + airmass=np.asarray(call_airmass, dtype=float), + sampled_keys=list(call_bounds.keys()), + sample_bounds=dict(call_bounds), + parameters=dict(call_prior), + errors=dict(fixed_parameter_errors or {}), + transit=np.ones(len(call_times), dtype=float), + residuals=np.zeros(len(call_times), dtype=float), + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: {"clipped": False, "reason": "parameter was fixed to the prior"} + ) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.06, 0.02, 20), + np.ones(20, dtype=float), + np.full(20, 0.01, dtype=float), + np.linspace(1.0, 1.4, 20), + { + "tmid": 0.0, + "rprs": 0.1, + "ars": 12.0, + "inc": 89.0, + "per": 1.0, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.0, + "a2": 0.1, + }, + { + "rprs": [0.0, 0.2], + "tmid": [-0.05, 0.05], + "ars": [10.0, 14.0], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + }, + pre_ultranest_coverage_assessment={ + "valid": True, + "transit_fraction_observed": 0.55, + "in_transit_points": 12, + "pre_ingress_points": 5, + "post_egress_points": 0, + "observed_segment": "pre-ingress baseline plus ingress plus mid-transit", + }, + search_restriction_prior={"rprs_unc": 0.002, "ars_unc": 0.3, "inc_unc": 0.4}, + ) + + assert list(captured["bounds"]) == ["tmid"] + assert captured["fixed_flux_baseline"] is False + assert captured["fixed_parameter_errors"]["rprs"] == pytest.approx(0.002) + assert captured["fixed_parameter_errors"]["ars"] == pytest.approx(0.3) + assert captured["fixed_parameter_errors"]["inc"] == pytest.approx(0.4) + assert fit.partial_transit_geometry_prior_assumption_applied is True + assert fit.partial_transit_geometry_prior_assumption_mode == "tmid_only" + assert fit.partial_transit_geometry_prior_assumption_sampled_parameters == ["tmid"] + + +def test_no_oot_partial_coverage_keeps_baseline_airmass_in_ultranest(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + fixed_parameter_errors=None, + fixed_flux_baseline=False, + **kwargs, + ): + captured["prior"] = dict(call_prior) + captured["bounds"] = dict(call_bounds) + captured["fixed_parameter_errors"] = dict(fixed_parameter_errors or {}) + captured["fixed_flux_baseline"] = bool(fixed_flux_baseline) + fit = types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + airmass=np.asarray(call_airmass, dtype=float), + sampled_keys=list(call_bounds.keys()), + sample_bounds=dict(call_bounds), + parameters=dict(call_prior), + errors=dict(fixed_parameter_errors or {}), + transit=np.ones(len(call_times), dtype=float), + residuals=np.zeros(len(call_times), dtype=float), + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: {"clipped": False, "reason": "parameter was fixed to the prior"} + ) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 20), + np.full(20, 1.02, dtype=float), + np.full(20, 0.01, dtype=float), + np.linspace(1.0, 1.5, 20), + { + "tmid": 0.0, + "rprs": 0.1, + "ars": 12.0, + "inc": 89.0, + "per": 1.0, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.02, + "a2": 0.1, + }, + { + "rprs": [0.0, 0.2], + "tmid": [-0.05, 0.05], + "ars": [10.0, 14.0], + "inc": [84.0, 90.0], + }, + fixed_flux_baseline=True, + pre_ultranest_coverage_assessment={ + "valid": True, + "transit_fraction_observed": 0.90, + "in_transit_points": 18, + "pre_ingress_points": 0, + "post_egress_points": 0, + "observed_segment": "inside the expected transit", + }, + search_restriction_prior={"rprs_unc": 0.002, "ars_unc": 0.3, "inc_unc": 0.4}, + ) + + assert "rprs" not in captured["bounds"] + assert "ars" not in captured["bounds"] + assert "inc" not in captured["bounds"] + assert set(captured["bounds"]) == {"tmid", "a0", "a2"} + assert captured["fixed_flux_baseline"] is False + assert captured["fixed_parameter_errors"]["rprs"] == pytest.approx(0.002) + assert captured["fixed_parameter_errors"]["ars"] == pytest.approx(0.3) + assert captured["fixed_parameter_errors"]["inc"] == pytest.approx(0.4) + assert fit.partial_transit_geometry_prior_assumption_applied is True + assert fit.partial_transit_geometry_prior_assumption_mode == "tmid_baseline_airmass" + assert set(fit.partial_transit_geometry_prior_assumption_sampled_parameters) == {"tmid", "a0", "a2"} + + def test_impact_parameter_posterior_retry_expands_inclination_bounds(monkeypatch): import exotic.exotic as exotic_module diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 68b8c939..22e869c3 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -23,6 +23,12 @@ def test_camera_keeps_dslr_as_dslr(): assert camera("canon dslr") == "DSLR" +def test_comparison_star_coords_accepts_more_than_ten_manual_comps(): + comp_stars = [[float(index), float(index + 1)] for index in range(12)] + + assert inputs_module.comparison_star_coords(comp_stars, rt_bool=False) == comp_stars + + def test_comp_params_accepts_verbose_camera_key(tmp_path): init_data = { "user_info": { diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 6f1f6ad4..392140ce 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -541,9 +541,108 @@ class DummyFit: assert any('no transit-fit comparison star' in message for message in logged) -def test_stellar_variability_requires_selected_comparison_catalog_match(monkeypatch, tmp_path): +def test_stellar_variability_derives_selected_comparison_catalog_magnitude(monkeypatch, tmp_path): logged = [] + captured = {} + + class DummyFit: + def __init__(self, data): + self.data = np.array(data, dtype=float) + self.airmass_model = np.ones(3, dtype=float) + self.airmass = np.ones(3, dtype=float) + self.jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + self.transit = np.ones(3, dtype=float) + + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + monkeypatch.setattr( + exotic_module, + 'plot_stellar_variability', + lambda params, save, s_name, label: captured.update(params=params, label=label), + ) + + selected_mag = 11.0 + anchor_mag = 12.0 + selected_to_anchor_flux_ratio = 10 ** ((anchor_mag - selected_mag) / 2.5) + selected_fit = DummyFit([1.0, 1.01, 0.99]) + anchor_fit = DummyFit(selected_to_anchor_flux_ratio * np.array([1.0, 1.01, 0.99])) + + params = exotic_module.stellar_variability( + { + 0: {'myfit': selected_fit, 'pos': [100, 200]}, + 1: {'myfit': anchor_fit, 'pos': [300, 400]}, + }, + DummyFit([1.0, 1.01, 0.99]), + [[100, 200], [300, 400]], + {'REF': {'pos': [300, 400], 'mag': anchor_mag, 'error': 0.02}}, + [1], + 0, + tmp_path, + 'Host Star', + comp_ra_dec=[(10.0, -20.0), (11.0, -21.0)], + ) + + assert len(params) == 3 + assert params[0]['cmag'] == pytest.approx(selected_mag) + assert params[0]['cmag_err'] == pytest.approx(0.02) + assert params[0]['comp_ra'] == pytest.approx(10.0) + assert params[0]['comp_dec'] == pytest.approx(-20.0) + assert params[0]['derived_catalog_reference'] is True + assert params[0]['derived_reference_anchor_count'] == 1 + assert params[0]['derived_reference_anchor_labels'] == ['REF'] + assert captured['label'] == 'RA=10.0000000 Dec=-20.0000000' + assert any('derived catalog magnitude' in message for message in logged) + + +def test_stellar_variability_uses_direct_catalog_when_derived_error_is_worse(monkeypatch, tmp_path): + logged = [] + + class DummyFit: + def __init__(self, data): + self.data = np.array(data, dtype=float) + self.airmass_model = np.ones(4, dtype=float) + self.airmass = np.ones(4, dtype=float) + self.jd_times = np.array([2450000.1, 2450000.2, 2450000.3, 2450000.4], dtype=float) + self.transit = np.ones(4, dtype=float) + + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + + selected_fit = DummyFit([1.0, 1.0, 1.0, 1.0]) + noisy_anchor_fit = DummyFit([2.0, 0.8, 2.2, 0.7]) + params = exotic_module.stellar_variability( + { + 0: {'myfit': selected_fit, 'pos': [100, 200]}, + 1: {'myfit': noisy_anchor_fit, 'pos': [300, 400]}, + }, + DummyFit([1.0, 1.0, 1.0, 1.0]), + [[100, 200], [300, 400]], + {'ANCHOR': {'pos': [300, 400], 'mag': 12.0, 'error': 0.02, 'mag_band': 'g'}}, + [1], + 0, + tmp_path, + 'Host Star', + observed_filter='g', + comp_ra_dec=[(10.0, -20.0), (11.0, -21.0)], + field_catalog={ + 'rows': [{ + 'ra': 10.0, + 'dec': -20.0, + 'g': 11.5, + 'dg': 0.08, + 'source_id': 12345, + }] + }, + ) + assert len(params) == 4 + assert params[0]['cmag'] == pytest.approx(11.5) + assert params[0]['cmag_err'] == pytest.approx(0.08) + assert params[0]['derived_catalog_reference'] is False + assert params[0]['allow_high_error_catalog_reference'] is True + assert any('direct selected-comparison catalog magnitude' in message for message in logged) + + +def test_stellar_variability_derives_catalog_magnitude_from_full_field(monkeypatch, tmp_path): class DummyFit: data = np.array([1.0, 1.01, 0.99], dtype=float) airmass_model = np.ones(3, dtype=float) @@ -551,24 +650,156 @@ class DummyFit: jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) transit = np.ones(3, dtype=float) - monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + class DummyWcs: + def world_to_pixel_values(self, ra, dec): + return float(ra), float(dec) + + def pixel_to_world_values(self, x, y): + return 123.4, -45.6 + + image = np.full((60, 60), 10.0, dtype=float) + image[20, 20] = 50.0 + image[40, 40] = 110.0 + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'search_wcs', lambda _path: DummyWcs()) params = exotic_module.stellar_variability( - { - 0: {'myfit': DummyFit(), 'pos': [100, 200]}, - 1: {'myfit': DummyFit(), 'pos': [300, 400]}, + {0: {'myfit': DummyFit(), 'pos': [20, 20]}}, + DummyFit(), + [[20, 20]], + {}, + [], + 0, + tmp_path, + 'Host Star', + observed_filter='g', + field_catalog={ + 'rows': [{ + 'ra': 40.0, + 'dec': 40.0, + 'g': 12.0, + 'dg': 0.03, + 'source_id': 67890, + }] }, + reference_image=image, + wcs_file='dummy.wcs', + ) + + expected_mag = 12.0 - 2.5 * np.log10(40.0 / 100.0) + assert len(params) == 3 + assert params[0]['cmag'] == pytest.approx(expected_mag) + assert params[0]['cmag_err'] == pytest.approx(0.03) + assert params[0]['comp_ra'] == pytest.approx(123.4) + assert params[0]['comp_dec'] == pytest.approx(-45.6) + assert params[0]['derived_catalog_reference'] is True + assert params[0]['derived_reference_anchor_count'] == 1 + assert params[0]['derived_reference_anchor_labels'] == ['NextAstro-67890'] + + +def test_stellar_variability_rejects_g_catalog_anchor_for_clearv(monkeypatch, tmp_path): + logged = [] + + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + class DummyWcs: + def world_to_pixel_values(self, ra, dec): + return float(ra), float(dec) + + def pixel_to_world_values(self, x, y): + return 123.4, -45.6 + + image = np.full((60, 60), 10.0, dtype=float) + image[20, 20] = 50.0 + image[40, 40] = 110.0 + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'search_wcs', lambda _path: DummyWcs()) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + params = exotic_module.stellar_variability( + {0: {'myfit': DummyFit(), 'pos': [20, 20]}}, DummyFit(), - [[100, 200], [300, 400]], - {'REF': {'pos': [300, 400], 'mag': 12.0, 'error': 0.02}}, - [1], + [[20, 20]], + {}, + [], 0, tmp_path, 'Host Star', + observed_filter='CV', + field_catalog={ + 'rows': [{ + 'ra': 40.0, + 'dec': 40.0, + 'g': 12.0, + 'dg': 0.03, + 'source_id': 67890, + }] + }, + reference_image=image, + wcs_file='dummy.wcs', ) assert params == [] - assert any('has no catalog magnitude' in message for message in logged) + assert any('no derived magnitude could be inferred' in message for message in logged) + + +def test_stellar_variability_uses_v_catalog_anchor_for_clearv(monkeypatch, tmp_path): + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + class DummyWcs: + def world_to_pixel_values(self, ra, dec): + return float(ra), float(dec) + + def pixel_to_world_values(self, x, y): + return 123.4, -45.6 + + image = np.full((60, 60), 10.0, dtype=float) + image[20, 20] = 50.0 + image[40, 40] = 110.0 + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'search_wcs', lambda _path: DummyWcs()) + + params = exotic_module.stellar_variability( + {0: {'myfit': DummyFit(), 'pos': [20, 20]}}, + DummyFit(), + [[20, 20]], + {}, + [], + 0, + tmp_path, + 'Host Star', + observed_filter='CV', + field_catalog={ + 'rows': [{ + 'ra': 40.0, + 'dec': 40.0, + 'Vmag': 12.0, + 'err_Vmag': 0.03, + 'g': 11.7, + 'dg': 0.01, + 'source_id': 67890, + }] + }, + reference_image=image, + wcs_file='dummy.wcs', + ) + + assert len(params) == 3 + assert params[0]['mag_band'] == 'V' + assert params[0]['cmag_err'] == pytest.approx(0.03) def test_check_for_variable_stars_uses_nextastro_flags_to_filter(monkeypatch): diff --git a/tests/test_output_files.py b/tests/test_output_files.py index fc01d308..05d7c781 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -317,9 +317,54 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert metadata["comparison_dec_deg"] == pytest.approx(-20.2) assert metadata["apparent_magnitude"] == pytest.approx(12.1) assert metadata["apparent_magnitude_error"] == pytest.approx(0.03) + assert "#DATE=BJD_TDB" in output_text assert "HAT-P-32,2450000.12345,12.340,0.050,V,NO,STD" in output_text +def test_aid_output_samples_large_derived_anchor_label_lists(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + } + anchor_labels = [f"NextAstro-{index}" for index in range(20)] + vsp_params = [{ + "time": 2450000.12345, + "mag": 12.34, + "mag_err": 0.05, + "airmass": 1.234, + "cname": "RA=10.1000000 Dec=-20.2000000", + "cmag": 12.1, + "cmag_err": 0.03, + "pos": [493, 202], + "catalog_source": "Derived from full-field catalog-calibrated stars", + "is_aavso_vsp": False, + "derived_catalog_reference": True, + "derived_reference_anchor_count": len(anchor_labels), + "derived_reference_anchor_labels": anchor_labels, + "mag_band": "V", + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = (tmp_path / "AID_AAVSO_HAT-P-32_2020-01-01.txt").read_text(encoding="utf-8") + metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") + + assert metadata["derived_reference_anchor_count"] == 20 + assert "derived_reference_anchor_labels" not in metadata + assert metadata["derived_reference_anchor_label_sample"] == anchor_labels[:10] + + def test_aid_output_floors_reported_magnitude_errors(tmp_path): fit = DummyFit() p_dict = { @@ -535,7 +580,7 @@ def test_final_planetary_params_reports_transit_comparison_catalog_reference(tmp assert "Best Comparison Star" not in final_params assert final_params["Variable Reference Star"] == "AAVSO Label: 000-BJX-718, Position: [616, 113]" assert "Remeasured 2 out-of-transit target/reference point(s)" in final_params["Variable Reference Measurement"] - assert "AID rows list the JD timestamps used" in final_params["Variable Reference Measurement"] + assert "AID rows list the BJD_TDB timestamps used" in final_params["Variable Reference Measurement"] assert "transit-fit catalog reference" in final_params["Variable Reference Measurement"] @@ -934,12 +979,12 @@ def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): "delta_chi2": 27.1, "ktmf_contributions": [ { - "label": "Model Evidence", + "label": "EEBLS Depth SNR", "available": True, "points": 0.74, "max_points": 0.80, "score": 0.93, - "detail": "Delta BIC=18.40", + "detail": "5.80", } ], } @@ -1049,6 +1094,30 @@ def test_final_planetary_params_reports_absolute_fit_quality(tmp_path): assert final_params["Fit quality point count"] == "6" +def test_final_planetary_params_reports_prior_assumed_geometry_note(tmp_path): + fit = DummyFit() + fit.partial_transit_geometry_prior_assumption_note = ( + "Applied prior-assumed transit geometry for a one-sided partial light curve." + ) + (tmp_path / "temp").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert ( + final_params["Prior-assumed partial-transit geometry note"] + == "Applied prior-assumed transit geometry for a one-sided partial light curve." + ) + + def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_path): fit = DummyFit() fit.airmass_fit_skipped = True @@ -1133,12 +1202,12 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): "ktmf_metric": 4.63, "ktmf_contributions": [ { - "label": "Model Evidence", + "label": "EEBLS Depth SNR", "available": True, "points": 0.74, "max_points": 0.80, "score": 0.93, - "detail": "Delta BIC=18.40", + "detail": "5.80", } ], } @@ -1321,7 +1390,7 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): qc = aavso_json_header(output_text, "QC-XC") assert qc["status"] == "pass" assert qc["ktmf_metric"] == pytest.approx(4.63) - assert qc["ktmf_contributions"][0]["label"] == "Model Evidence" + assert qc["ktmf_contributions"][0]["label"] == "EEBLS Depth SNR" fit_quality = aavso_json_header(output_text, "FIT_QUALITY-XC") assert fit_quality["reduced_chi_square"] == pytest.approx(10.0 / 3.0) diff --git a/tests/test_plots.py b/tests/test_plots.py index e8ef80bb..9a0a9b21 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -266,6 +266,40 @@ def spy_set_ylabel(self, label, *args, **kwargs): assert (tmp_path / "temp" / "Stellar_Variability.png").exists() +def test_plot_stellar_variability_labels_aavso_filter_and_assumed_comparison(tmp_path, monkeypatch): + titles = [] + original_set_title = Axes.set_title + + def spy_set_title(self, label, *args, **kwargs): + titles.append(label) + return original_set_title(self, label, *args, **kwargs) + + monkeypatch.setattr(Axes, "set_title", spy_set_title) + + plot_stellar_variability( + [ + { + "time": 2450000.1, + "mag": 12.34, + "mag_err": 0.05, + "cmag": 12.345, + "cmag_err": 0.067, + "comp_ra": 10.1, + "comp_dec": -20.2, + "mag_band": "V", + "observed_filter": "CV", + "is_aavso_vsp": True, + } + ], + str(tmp_path), + "Host Star", + "000-BJX-718", + ) + + assert "Observed filter: CV" in titles[-1] + assert "Assumed comparison: V=12.345 +/- 0.067" in titles[-1] + + def test_plot_stellar_variability_omits_invalid_reference_magnitudes(tmp_path, monkeypatch): titles = [] original_set_title = Axes.set_title @@ -577,12 +611,12 @@ def __init__(self): "ktmf_metric": 3.07, "ktmf_contributions": [ { - "label": "Model Evidence", + "label": "EEBLS Depth SNR", "available": True, "points": 0.11, "max_points": 0.89, "score": 0.12, - "detail": "Delta BIC=-2.00, Delta chi2=6.91", + "detail": "2.00", }, { "label": "Residual Scatter Around Full Model Fit", @@ -607,4 +641,4 @@ def __init__(self): assert (tmp_path / "KTMF_QC_Target_2026-03-09.pdf").exists() assert any("KTMF\n3.07 / 5.00\nMARGINAL" in text for text in captured_text) assert any("0.11 / 0.89" in text for text in captured_text) - assert any("Delta BIC=-2.00" in text for text in captured_text) + assert any("2.00" in text for text in captured_text) From 272dc2025962643f8bb4c4a0b71a0232edcb796e Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Thu, 25 Jun 2026 08:54:46 +1000 Subject: [PATCH 072/116] Glitch finding in residual QC items --- exotic/exotic.py | 101 ++++++++++++++++----- exotic/exotic_gui.py | 4 +- exotic/output_files.py | 5 ++ exotic/plots.py | 7 +- tests/test_exotic_proper_motion.py | 51 +++++++++++ tests/test_exotic_rprs_retry.py | 138 +++++++++++++++++++++++++++-- tests/test_plots.py | 3 +- 7 files changed, 269 insertions(+), 40 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index e9e7d597..efb27dfc 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -1484,6 +1484,9 @@ def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points 'trend_strength': np.nan, 'curve_strength': np.nan, 'scatter_ratio': np.nan, + 'zero_offset_strength': np.nan, + 'sign_imbalance': np.nan, + 'zero_bias_score': np.nan, 'trend_score': np.nan, 'curve_score': np.nan, 'scatter_stability_score': np.nan, @@ -1515,27 +1518,44 @@ def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points residuals = residuals[order] coordinates = coordinates[order] - centered = residuals - float(np.nanmedian(residuals)) + median_residual = float(np.nanmedian(residuals)) + centered = residuals - median_residual scatter = robust_sigma(centered) if not np.isfinite(scatter): summary['detail'] = "residual scatter unavailable" return summary if scatter <= np.finfo(float).eps: + zero_bias_score = 1.0 if abs(median_residual) <= np.finfo(float).eps else 0.0 summary.update({ 'available': True, - 'score': 1.0, + 'score': zero_bias_score, 'scatter': float(scatter), 'trend_strength': 0.0, 'curve_strength': 0.0, 'scatter_ratio': 1.0, + 'zero_offset_strength': 0.0 if zero_bias_score == 1.0 else np.inf, + 'sign_imbalance': 0.0 if zero_bias_score == 1.0 else 1.0, + 'zero_bias_score': zero_bias_score, 'trend_score': 1.0, 'curve_score': 1.0, 'scatter_stability_score': 1.0, - 'dominant': 'flat', - 'detail': f"flat residuals; n={point_count}", + 'dominant': 'flat' if zero_bias_score == 1.0 else 'zero bias', + 'detail': ( + f"flat residuals; n={point_count}" + if zero_bias_score == 1.0 + else f"zero bias limited, median offset=inf, sign imbalance=1.00, n={point_count}" + ), }) return summary + zero_offset_strength = abs(median_residual) / scatter + sign_tolerance = 0.05 * scatter + signed = residuals[np.abs(residuals) > sign_tolerance] + sign_imbalance = np.nan + if signed.size >= max(6, point_count // 3): + positive_fraction = float(np.count_nonzero(signed > 0.0) / signed.size) + sign_imbalance = float(abs(2.0 * positive_fraction - 1.0)) + normalized = centered / scatter coordinate_min = float(np.nanmin(coordinates)) coordinate_max = float(np.nanmax(coordinates)) @@ -1568,9 +1588,7 @@ def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points ]) structure_coeff, *_ = np.linalg.lstsq(structure_design, normalized, rcond=None) structure_model = structure_design @ structure_coeff - raw_structure_strength = float(np.nanstd(structure_model - np.nanmean(structure_model))) - expected_noise_projection = float(np.sqrt((structure_design.shape[1] - 1) / max(point_count, 1))) - curve_strength = max(0.0, raw_structure_strength - expected_noise_projection) + curve_strength = float(np.nanstd(structure_model - np.nanmean(structure_model))) except Exception: curve_strength = np.nan @@ -1586,13 +1604,7 @@ def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points float(np.nanmedian(normalized[chunk])) for chunk in bins ], dtype=float) - median_bin_size = float(np.nanmedian([chunk.size for chunk in bins])) - expected_binned_median_noise = 1.253 / np.sqrt(max(median_bin_size, 1.0)) - binned_curve_strength = max( - 0.0, - float(np.nanstd(bin_medians - np.nanmedian(bin_medians))) - - expected_binned_median_noise, - ) + binned_curve_strength = float(np.nanstd(bin_medians - np.nanmedian(bin_medians))) bin_sigmas = np.asarray([ robust_sigma(normalized[chunk] - float(np.nanmedian(normalized[chunk]))) @@ -1630,7 +1642,18 @@ def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points if np.isfinite(scatter_ratio) and scatter_ratio > 0 else np.nan ) + zero_offset_score = transit_qc_flatness_declining_score(zero_offset_strength, 0.25, 1.25) + sign_balance_score = ( + transit_qc_flatness_declining_score(sign_imbalance, 0.35, 0.85) + if np.isfinite(sign_imbalance) + else np.nan + ) + zero_bias_score = np.nanmin([ + value for value in (zero_offset_score, sign_balance_score) + if np.isfinite(value) + ]) if np.isfinite(zero_offset_score) or np.isfinite(sign_balance_score) else np.nan component_scores = { + 'zero bias': zero_bias_score, 'trend': trend_score, 'curvature/sinusoid': curve_score, 'scatter stability': scatter_stability_score, @@ -1648,6 +1671,8 @@ def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points score = finite_component_scores[dominant] detail_parts = [ f"{dominant} limited", + f"median offset={zero_offset_strength:.2f}" if np.isfinite(zero_offset_strength) else "median offset=n/a", + f"sign imbalance={sign_imbalance:.2f}" if np.isfinite(sign_imbalance) else "sign imbalance=n/a", f"trend={trend_strength:.2f}" if np.isfinite(trend_strength) else "trend=n/a", f"curve={curve_strength:.2f}" if np.isfinite(curve_strength) else "curve=n/a", f"scatter ratio={scatter_ratio:.2f}" if np.isfinite(scatter_ratio) else "scatter ratio=n/a", @@ -1660,6 +1685,9 @@ def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points 'trend_strength': trend_strength, 'curve_strength': curve_strength, 'scatter_ratio': scatter_ratio, + 'zero_offset_strength': zero_offset_strength, + 'sign_imbalance': sign_imbalance, + 'zero_bias_score': zero_bias_score, 'trend_score': trend_score, 'curve_score': curve_score, 'scatter_stability_score': scatter_stability_score, @@ -2359,6 +2387,9 @@ def evaluate_transit_detection_qc(fit): 'residual_flatness_trend_strength': np.nan, 'residual_flatness_curve_strength': np.nan, 'residual_flatness_scatter_ratio': np.nan, + 'residual_flatness_zero_offset_strength': np.nan, + 'residual_flatness_sign_imbalance': np.nan, + 'residual_flatness_zero_bias_score': np.nan, 'residual_flatness_trend_score': np.nan, 'residual_flatness_curve_score': np.nan, 'residual_flatness_scatter_stability_score': np.nan, @@ -2464,6 +2495,14 @@ def evaluate_transit_detection_qc(fit): summary['geometry_prior_assumed'] = geometry_prior_assumed summary['geometry_prior_assumed_note'] = geometry_prior_assumed_note initial_a2 = parameters.get('a2', 0.0) + transit_depth_model_obj = getattr(fit, 'transit', None) + transit_depth_model = None + if transit_depth_model_obj is not None: + transit_depth_model = np.asarray(transit_depth_model_obj, dtype=float) + if transit_depth_model.shape != data.shape: + transit_depth_model = None + if transit_depth_model is None: + transit_depth_model = transit_model flat_model = fit_profiled_flat_null_model( data, @@ -2494,7 +2533,7 @@ def evaluate_transit_detection_qc(fit): 'flat_a2': flat_model.get('a2', np.nan), 'flat_model_note': flat_model.get('note'), 'residual_scatter': transit_qc_residual_scatter(data, transit_model), - 'transit_depth_for_residual_scatter': transit_qc_model_depth_fraction(transit_model), + 'transit_depth_for_residual_scatter': transit_qc_model_depth_fraction(transit_depth_model), 'point_count': int(point_count), }) residual_coordinates = getattr(fit, 'phase', None) @@ -2509,6 +2548,9 @@ def evaluate_transit_detection_qc(fit): 'residual_flatness_trend_strength': residual_flatness.get('trend_strength', np.nan), 'residual_flatness_curve_strength': residual_flatness.get('curve_strength', np.nan), 'residual_flatness_scatter_ratio': residual_flatness.get('scatter_ratio', np.nan), + 'residual_flatness_zero_offset_strength': residual_flatness.get('zero_offset_strength', np.nan), + 'residual_flatness_sign_imbalance': residual_flatness.get('sign_imbalance', np.nan), + 'residual_flatness_zero_bias_score': residual_flatness.get('zero_bias_score', np.nan), 'residual_flatness_trend_score': residual_flatness.get('trend_score', np.nan), 'residual_flatness_curve_score': residual_flatness.get('curve_score', np.nan), 'residual_flatness_scatter_stability_score': residual_flatness.get('scatter_stability_score', np.nan), @@ -6469,7 +6511,6 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): if ( use_prior_rprs_when_posterior_pinned - and int(max(0, max_rprs_retries)) <= 0 and 'rprs' in current_bounds and isinstance(rprs_final_diagnostics, dict) and rprs_final_diagnostics.get('clipped') @@ -6510,7 +6551,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): log_info( "Rp/R* posterior is pinned against the " f"{rprs_final_diagnostics.get('edge', 'active')} bound while Rp/R* " - "posterior expansion is disabled; rerunning UltraNest with Rp/R* fixed " + "posterior expansion is disabled, exhausted, or blocked; rerunning UltraNest with Rp/R* fixed " f"to the input prior ({prior_rprs:.6f}) and using a data-only Rp/R* uncertainty." ) fallback_fit = build_fit( @@ -6555,7 +6596,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): fallback_note = ( "Applied Rp/R* prior fallback; the sampled Rp/R* posterior hugged the " f"{rprs_final_diagnostics.get('edge', 'active')} search bound while automatic " - "Rp/R* posterior expansion was disabled. EXOTIC reran UltraNest with Rp/R* fixed " + "Rp/R* posterior expansion was disabled, exhausted, or blocked. EXOTIC reran UltraNest with Rp/R* fixed " f"to the input prior ({prior_rprs:.6f}) and treats Tmid, a/Rs, and " "impact parameter/inclination as the fitted transit-shape parameters. " "The quoted Rp/R* uncertainty is a data-only red-noise estimate rather than a " @@ -6581,12 +6622,26 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): final_diagnostics = None available = config.get('available') config_available = not callable(available) or available(fit, current_bounds) - if callable(final_diagnostics_getter) and bounds_key in current_bounds and config_available: + prior_fallback_applied = ( + key == 'rprs' + and bool(getattr(fit, 'rprs_prior_fallback_applied', False)) + ) + if prior_fallback_applied: + final_diagnostics = None + elif callable(final_diagnostics_getter) and bounds_key in current_bounds and config_available: final_diagnostics = final_diagnostics_getter(diagnostic_key) elif latest_diagnostics.get(key) is not None: final_diagnostics = latest_diagnostics[key] - if history: + if prior_fallback_applied: + fallback_note = getattr(fit, 'rprs_prior_fallback_note', None) or retry_notes.get(key) + note = fallback_note + if history: + note = ( + f"Applied {len(history)} automatic {label} posterior range refit(s), " + "then applied the Rp/R* prior fallback." + ) + elif history: note = f"Applied {len(history)} automatic {label} posterior range refit(s)." if final_diagnostics and final_diagnostics.get('clipped'): retry_label = "retry" if len(history) == 1 else "retries" @@ -23149,9 +23204,9 @@ def _main_impl(): log_info("Rp/R* prior-centered search restriction disabled.") if use_prior_rprs_fallback_on_pinned_posterior: log_info( - "Rp/R* pinned-posterior prior fallback enabled: when Rp/R* expansion is disabled " - "and the posterior is edge-pinned, EXOTIC reruns with Rp/R* fixed to the input " - "prior and quotes a data-only Rp/R* uncertainty." + "Rp/R* pinned-posterior prior fallback enabled: when the posterior remains " + "edge-pinned after Rp/R* retry handling, EXOTIC reruns with Rp/R* fixed to " + "the input prior and quotes a data-only Rp/R* uncertainty." ) else: log_info("Rp/R* pinned-posterior prior fallback disabled per optional_info setting.") diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index de4f1393..fdaed4bd 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -428,7 +428,7 @@ def save_input(): "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Maximum Rp/Rs Search Bound": "Set optional_info 'rprs_search_bound_max' to cap the nested-fit Rp/Rs search range. Default 0.5.", "Restrict Rp/Rs Search Range": "Set optional_info 'restrict_Rp/Rs_range' to y to restrict Rp/Rs to a prior-centered percentage window. Set 'restrict_Rp/Rs_range_percentage' to control the half-width. Defaults y and 10.", - "Prior Rp/Rs Fallback For Pinned Posterior": "Set optional_info 'use_prior_Rp/Rs_when_posterior_pinned' to y to rerun a fit with Rp/Rs fixed to the input prior and quote a data-only Rp/Rs uncertainty when the Rp/Rs posterior is edge-pinned and automatic Rp/Rs expansion is disabled. Default y.", + "Prior Rp/Rs Fallback For Pinned Posterior": "Set optional_info 'use_prior_Rp/Rs_when_posterior_pinned' to y to rerun a fit with Rp/Rs fixed to the input prior and quote a data-only Rp/Rs uncertainty when the Rp/Rs posterior remains edge-pinned after retry handling. Default y.", "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", @@ -1524,7 +1524,7 @@ def save_input(): "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", "Maximum Rp/Rs Search Bound": "Set optional_info 'rprs_search_bound_max' to cap the nested-fit Rp/Rs search range. Default 0.5.", "Restrict Rp/Rs Search Range": "Set optional_info 'restrict_Rp/Rs_range' to y to restrict Rp/Rs to a prior-centered percentage window. Set 'restrict_Rp/Rs_range_percentage' to control the half-width. Defaults y and 10.", - "Prior Rp/Rs Fallback For Pinned Posterior": "Set optional_info 'use_prior_Rp/Rs_when_posterior_pinned' to y to rerun a fit with Rp/Rs fixed to the input prior and quote a data-only Rp/Rs uncertainty when the Rp/Rs posterior is edge-pinned and automatic Rp/Rs expansion is disabled. Default y.", + "Prior Rp/Rs Fallback For Pinned Posterior": "Set optional_info 'use_prior_Rp/Rs_when_posterior_pinned' to y to rerun a fit with Rp/Rs fixed to the input prior and quote a data-only Rp/Rs uncertainty when the Rp/Rs posterior remains edge-pinned after retry handling. Default y.", "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", diff --git a/exotic/output_files.py b/exotic/output_files.py index 78b23579..ceb66ec7 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -709,6 +709,8 @@ def build_aavso_qc_metadata(fit): 'transit_depth_for_residual_scatter', 'residual_scatter_to_depth_ratio', 'residual_flatness_score', 'residual_flatness_trend_strength', 'residual_flatness_curve_strength', 'residual_flatness_scatter_ratio', + 'residual_flatness_zero_offset_strength', + 'residual_flatness_sign_imbalance', 'residual_flatness_zero_bias_score', 'residual_flatness_trend_score', 'residual_flatness_curve_score', 'residual_flatness_scatter_stability_score', 'residual_flatness_dominant_metric', 'residual_flatness_detail', @@ -831,6 +833,9 @@ def build_ktmf_decision_metadata(fit, photometry_info=None): 'residual_flatness_trend_strength': transit_qc.get('residual_flatness_trend_strength'), 'residual_flatness_curve_strength': transit_qc.get('residual_flatness_curve_strength'), 'residual_flatness_scatter_ratio': transit_qc.get('residual_flatness_scatter_ratio'), + 'residual_flatness_zero_offset_strength': transit_qc.get('residual_flatness_zero_offset_strength'), + 'residual_flatness_sign_imbalance': transit_qc.get('residual_flatness_sign_imbalance'), + 'residual_flatness_zero_bias_score': transit_qc.get('residual_flatness_zero_bias_score'), 'sampling_score': transit_qc.get('sampling_score'), 'sampling_detail': transit_qc.get('sampling_detail'), 'sampling_ingress_count': transit_qc.get('sampling_ingress_count'), diff --git a/exotic/plots.py b/exotic/plots.py index 1aacab07..ec2d62bc 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -6,7 +6,6 @@ from matplotlib.lines import Line2D import numpy as np from pathlib import Path -import textwrap try: from utils import ( @@ -1406,20 +1405,16 @@ def _format_ktmf_component_annotation(row): return f"KTMF\n{_format_ktmf_metric(row.get('points'), row.get('max_points'))}{status_text}{uncertainty_text}" if not row.get('available', True): - detail = row.get('detail') or "unavailable" - return f"Not scored\n{textwrap.fill(str(detail), width=44)}" + return "Not scored" score_uncertainty = _plot_positive_error(row.get('score_uncertainty')) if np.isfinite(score_uncertainty): score_text = _format_parameter_value(row.get('score'), score_uncertainty) else: score_text = _format_parameter_value(row.get('score')) - detail = _compact_ktmf_detail(row) - detail_text = f"\n{textwrap.fill(str(detail), width=44)}" if detail else "" return ( f"Score\n{score_text}\n" f"Points\n{_format_ktmf_metric(row.get('points'), row.get('max_points'))}" - f"{detail_text}" ) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 9abab86f..a17d655e 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -2168,6 +2168,42 @@ def test_transit_qc_residual_scatter_score_full_credit_floor_and_zero_ceiling(): assert mid_score < transit_qc_residual_scatter_score(0.02, transit_depth) +def test_evaluate_transit_detection_qc_uses_transit_component_depth_for_residual_scatter(): + transit_component = np.ones(21, dtype=float) + transit_component[8:13] = 0.984 + baseline_trend = np.linspace(0.0, 0.04, transit_component.size) + full_model = transit_component + baseline_trend + data = full_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=full_model, + transit=transit_component, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["transit_depth_for_residual_scatter"] == pytest.approx(0.016, abs=5e-4) + assert summary["residual_scatter_to_depth_ratio"] == pytest.approx( + summary["residual_scatter"] / summary["transit_depth_for_residual_scatter"] + ) + + def test_transit_qc_residual_flatness_summary_penalizes_residual_structure(): phase = np.linspace(-0.05, 0.05, 80) alternating_noise = 0.001 * np.where(np.arange(phase.size) % 2 == 0, -1.0, 1.0) @@ -2181,10 +2217,18 @@ def test_transit_qc_residual_flatness_summary_penalizes_residual_structure(): alternating_noise + 0.004 * np.sin(2.0 * np.pi * np.linspace(0.0, 1.0, phase.size)), phase, ) + smooth_bowl_summary = transit_qc_residual_flatness_summary( + alternating_noise + 0.003 * np.maximum(0.0, 1.0 - (phase / 0.02) ** 2), + phase, + ) heteroscedastic_summary = transit_qc_residual_flatness_summary( alternating_noise * np.r_[np.ones(40), np.full(40, 4.0)], phase, ) + one_sided_summary = transit_qc_residual_flatness_summary( + alternating_noise - 0.003, + phase, + ) assert flat_summary["available"] is True assert flat_summary["score"] > 0.9 @@ -2192,8 +2236,15 @@ def test_transit_qc_residual_flatness_summary_penalizes_residual_structure(): assert trend_summary["score"] < 0.5 assert curve_summary["score"] < flat_summary["score"] assert curve_summary["score"] < 0.6 + assert smooth_bowl_summary["score"] < flat_summary["score"] + assert smooth_bowl_summary["score"] < 0.5 + assert smooth_bowl_summary["dominant"] == "curvature/sinusoid" assert heteroscedastic_summary["score"] < flat_summary["score"] assert heteroscedastic_summary["score"] < 0.8 + assert one_sided_summary["score"] < flat_summary["score"] + assert one_sided_summary["score"] < 0.4 + assert one_sided_summary["dominant"] == "zero bias" + assert one_sided_summary["sign_imbalance"] > 0.8 def test_transit_qc_residual_flatness_summary_tolerates_one_quiet_patch(): diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 78f9c156..1d956140 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -269,9 +269,13 @@ def fake_lc_fitter( bounds, ) - assert len(captured["calls"]) == 2 + assert len(captured["calls"]) == 3 assert captured["calls"][1]["bounds"]["rprs"][1] == pytest.approx(0.5) + assert "rprs" not in captured["calls"][2]["bounds"] assert fit.rprs_posterior_refit_bounds[1] == pytest.approx(0.5) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.4) def test_rprs_posterior_retry_does_not_escape_configured_prior_range(monkeypatch): @@ -328,10 +332,13 @@ def fake_lc_fitter( bounds, ) - assert len(captured["calls"]) == 1 + assert len(captured["calls"]) == 2 assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.09, 0.11]) + assert "rprs" not in captured["calls"][1]["bounds"] assert fit.rprs_posterior_refit_applied is False - assert "configured prior-centered search range" in fit.rprs_posterior_refit_note + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.1) + assert "prior fallback" in fit.rprs_prior_fallback_note def test_rprs_restriction_uses_explicit_search_prior_instead_of_refined_prior(monkeypatch): @@ -497,6 +504,116 @@ def diagnostics(key): assert "prior fallback" in fit.rprs_prior_fallback_note +def test_pinned_rprs_after_retry_cap_reruns_with_prior_value(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR", True) + + def fake_transit(call_times, call_prior): + call_times = np.asarray(call_times, dtype=float) + depth = float(call_prior["rprs"]) ** 2 + return 1.0 - depth * (np.abs(call_times) <= 0.01) + + monkeypatch.setattr(exotic_module, "transit", fake_transit) + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + fixed_parameter_errors=None, + **kwargs, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + "fixed_parameter_errors": dict(fixed_parameter_errors or {}), + }) + model = fake_transit(call_times, call_prior) + fit = types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + transit=np.asarray(model, dtype=float), + model=np.asarray(model, dtype=float), + residuals=np.asarray(call_flux, dtype=float) - np.asarray(model, dtype=float), + airmass_model=np.ones_like(model), + parameters=dict(call_prior), + errors=dict(fixed_parameter_errors or {}), + fixed_parameter_errors=dict(fixed_parameter_errors or {}), + ) + fit.errors.setdefault("tmid", 0.001) + fit.errors.setdefault("ars", 0.1) + fit.errors.setdefault("inc", 0.1) + + def diagnostics(key): + if key == "rprs" and "rprs" in call_bounds: + return { + "clipped": True, + "edge": "upper", + "mode": call_bounds["rprs"][1], + "std": 0.006, + "bounds": [ + call_bounds["rprs"][0] + 0.01, + call_bounds["rprs"][1] + 0.01, + ], + "reason": "posterior peaks against the upper search bound.", + } + return { + "clipped": False, + "edge": None, + "mode": call_prior.get(key, np.nan), + "std": 0.001, + "bounds": call_bounds.get(key), + "reason": "posterior support is comfortably inside the sampled bounds.", + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + fit.call_index = call_index + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 9) + prior = {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0} + flux = fake_transit(times, prior) + np.array([0.0, 0.004, -0.003, 0.002, -0.004, 0.003, -0.002, 0.004, 0.0]) + fluxerr = np.full(times.shape, 0.003) + airmass = np.ones(times.shape) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + {"tmid": 0.0, "rprs": 0.11, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.09, 0.11], "tmid": [-0.01, 0.01], "ars": [8.0, 12.0], "inc": [84.0, 90.0]}, + max_rprs_retries=1, + max_ars_retries=0, + max_impact_parameter_retries=0, + search_restriction_prior={"rprs": 0.1, "ars": 10.0}, + ) + + assert len(captured["calls"]) == 3 + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.06, 0.16]) + assert "rprs" not in captured["calls"][2]["bounds"] + assert captured["calls"][2]["prior"]["rprs"] == pytest.approx(0.1) + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.1) + assert "prior fallback" in fit.rprs_prior_fallback_note + assert "then applied the Rp/R* prior fallback" in fit.rprs_posterior_refit_note + + def test_fast_ultranest_option_defaults_enabled_and_parses_false_values(): assert should_run_fast_ultranest_before_final_run(None) is True assert should_run_fast_ultranest_before_final_run("n") is False @@ -1149,7 +1266,7 @@ def fake_lc_fitter( for key, value in call_bounds.items() }, }) - return make_fit(diagnostics_sequence[call_index]) + return make_fit(diagnostics_sequence[min(call_index, len(diagnostics_sequence) - 1)]) monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) @@ -1170,9 +1287,10 @@ def fake_lc_fitter( ) assert RPRS_POSTERIOR_MAX_RETRIES_DEFAULT == 5 - assert len(captured["calls"]) == 6 + assert len(captured["calls"]) == 7 + assert "rprs" not in captured["calls"][-1]["bounds"] np.testing.assert_allclose( - np.asarray([call["bounds"]["rprs"] for call in captured["calls"]], dtype=float), + np.asarray([call["bounds"]["rprs"] for call in captured["calls"][:-1]], dtype=float), np.asarray([ [0.0, 0.125], [0.108, 0.208], @@ -1186,7 +1304,9 @@ def fake_lc_fitter( assert fit.rprs_posterior_refit_count == 5 assert fit.rprs_posterior_refit_edge == "upper" assert fit.rprs_posterior_refit_bounds == pytest.approx([0.156, 0.256]) - assert "after 5 retries" in fit.rprs_posterior_refit_note + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.1) + assert "prior fallback" in fit.rprs_prior_fallback_note def test_rprs_posterior_retry_expands_bounds_without_hitting_the_old_0p3_cap(monkeypatch): @@ -1328,13 +1448,15 @@ def fake_lc_fitter( bounds, ) - assert len(captured["calls"]) == 2 + assert len(captured["calls"]) == 3 assert captured["calls"][0]["prior"]["rprs"] == pytest.approx(0.35) assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([RPRS_SEARCH_BOUND_MIN, 0.35]) assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.175, 0.375]) + assert "rprs" not in captured["calls"][2]["bounds"] assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 1 + assert fit.rprs_prior_fallback_applied is True def test_rprs_posterior_retry_expands_lower_edge_down_to_zero(monkeypatch): diff --git a/tests/test_plots.py b/tests/test_plots.py index 9a0a9b21..6e94f0a7 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -641,4 +641,5 @@ def __init__(self): assert (tmp_path / "KTMF_QC_Target_2026-03-09.pdf").exists() assert any("KTMF\n3.07 / 5.00\nMARGINAL" in text for text in captured_text) assert any("0.11 / 0.89" in text for text in captured_text) - assert any("2.00" in text for text in captured_text) + assert not any("2.00" in text for text in captured_text) + assert not any("2.3437" in text for text in captured_text) From aed0dc9bf8a92270a720704955b25438ea43e01a Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 3 Jul 2026 08:50:39 +1000 Subject: [PATCH 073/116] seed not dependant on name for unknown planets --- exotic/exotic.py | 39 +++++++++++++++--------------- tests/test_exotic_proper_motion.py | 33 ++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index efb27dfc..296138b4 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -23279,25 +23279,6 @@ def _main_impl(): else: pDict = userpDict CandidatePlanetBool = False - # Seed random number generator (for run to run consistency) - if exotic_infoDict['random_seed']: - log_info(f"Setting random number seed to {exotic_infoDict['random_seed']}") - else: - exotic_infoDict['random_seed'] = int.from_bytes(hashlib.sha256(f"{pDict['pName']}:{pDict['midT']}".encode()).digest()[0:4], byteorder='little') - log_info(f"Generated random number seed {exotic_infoDict['random_seed']}") - np.random.seed(exotic_infoDict['random_seed']) - - if fitsortext == 1: - # Only do the dark correction if user selects this option - generalDark = process_dark_frames(exotic_infoDict['darks']) - generalBias = process_bias_frames(exotic_infoDict['biases']) - generalFlat = process_flat_frames(exotic_infoDict['flats'], generalBias) - - if exotic_infoDict['demosaic_fmt']: - demosaic_fmt = exotic_infoDict['demosaic_fmt'].upper() - if exotic_infoDict['demosaic_out']: - demosaic_out = exotic_infoDict['demosaic_out'] - demosaic_mult = calculate_demosaic_mult(demosaic_out) if file_cmd_opt == 2: if args.nasaexoarch: @@ -23320,6 +23301,26 @@ def _main_impl(): else: pDict = get_planetary_parameters(CandidatePlanetBool, userpDict, pdict=pDict) + # Seed random number generator (for run to run consistency) + if exotic_infoDict['random_seed']: + log_info(f"Setting random number seed to {exotic_infoDict['random_seed']}") + else: + exotic_infoDict['random_seed'] = int.from_bytes(hashlib.sha256(f"{pDict['pName']}:{pDict['midT']}".encode()).digest()[0:4], byteorder='little') + log_info(f"Generated random number seed {exotic_infoDict['random_seed']}") + np.random.seed(exotic_infoDict['random_seed']) + + if fitsortext == 1: + # Only do the dark correction if user selects this option + generalDark = process_dark_frames(exotic_infoDict['darks']) + generalBias = process_bias_frames(exotic_infoDict['biases']) + generalFlat = process_flat_frames(exotic_infoDict['flats'], generalBias) + + if exotic_infoDict['demosaic_fmt']: + demosaic_fmt = exotic_infoDict['demosaic_fmt'].upper() + if exotic_infoDict['demosaic_out']: + demosaic_out = exotic_infoDict['demosaic_out'] + demosaic_mult = calculate_demosaic_mult(demosaic_out) + # check for Nans + Zeros for k in pDict: if k == 'rprs' and (pDict[k] == 0 or np.isnan(pDict[k])): diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index a17d655e..bd9bda1d 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -6940,7 +6940,13 @@ def fake_lc_fitter( assert myfit.airmass_fit_skipped is True -def _run_main_until_vertical_flux_bound(monkeypatch, tmp_path, disable_vertical_flux_normalization=Ellipsis): +def _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + disable_vertical_flux_normalization=Ellipsis, + random_seed=123, + override=True, + nasa_result=None): import exotic.exotic as exotic_module class BoundReached(Exception): @@ -6996,7 +7002,7 @@ class BoundReached(Exception): "file_time": "BJD_TDB", "file_units": "flux", "airmass_already_corrected": False, - "random_seed": 123, + "random_seed": random_seed, "date": "2026-03-19", } if disable_vertical_flux_normalization is not Ellipsis: @@ -7009,7 +7015,7 @@ class BoundReached(Exception): reduce=None, prereduced=str(tmp_path / "inits.json"), photometry=None, - override=True, + override=override, nasaexoarch=False, non_interactive_run=True, use_nextastro_astrometry=False, @@ -7030,6 +7036,15 @@ def prereduced(self, planet): monkeypatch.setattr(exotic_module, "parse_args", lambda: args) monkeypatch.setattr(exotic_module, "Inputs", FakeInputs) + if nasa_result is not None: + class FakeNASAExoplanetArchive: + def __init__(self, planet): + self.planet = planet + + def planet_info(self): + return nasa_result + + monkeypatch.setattr(exotic_module, "NASAExoplanetArchive", FakeNASAExoplanetArchive) monkeypatch.setattr( exotic_module, "get_ld_values", @@ -7064,6 +7079,18 @@ def test_main_prereduced_respects_disable_vertical_flux_normalization_option(mon assert disabled is True +def test_main_prereduced_generates_seed_after_candidate_falls_back_to_inits(monkeypatch, tmp_path): + disabled = _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + random_seed=None, + override=False, + nasa_result=("TOI-3514.01", True, None), + ) + + assert disabled is False + + def test_cli_logs_unhandled_exception_once(monkeypatch): import exotic.exotic as exotic_module From 9d9271ebd1aa4fa2f3bee809971fbda8a414e48e Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 5 Jul 2026 08:27:17 +1000 Subject: [PATCH 074/116] Minor VSP plot upgrade --- exotic/exotic.py | 58 +++++++++++++++++++++++++++-- exotic/plots.py | 27 ++++++++------ tests/test_nextastro_variability.py | 9 ++++- tests/test_plots.py | 21 +++++++---- 4 files changed, 91 insertions(+), 24 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 296138b4..f5c9dbfb 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -13143,6 +13143,59 @@ def nextastro_photometry_band_candidates(obs_filter): return candidates +def aavso_vsp_band_for_filter(obs_filter): + filter_key = normalize_nextastro_filter_key(obs_filter) + direct_map = { + 'u': 'U', + 'johnsonu': 'U', + 'bu': 'U', + 'b': 'B', + 'johnsonb': 'B', + 'photographicb': 'B', + 'bb': 'B', + 'pb': 'B', + 'v': 'V', + 'johnsonv': 'V', + 'bv': 'V', + 'cv': 'V', + 'clearv': 'V', + 'clearunfilteredreducedtovsequence': 'V', + 'mobscv': 'V', + 'c': 'V', + 'clear': 'V', + 'lum': 'V', + 'luminance': 'V', + 'r': 'Rc', + 'rc': 'Rc', + 'cousinsr': 'Rc', + 'clearunfilteredreducedtorsequence': 'Rc', + 'cr': 'Rc', + 'i': 'Ic', + 'ic': 'Ic', + 'cousinsi': 'Ic', + 'su': 'SU', + 'sloanu': 'SU', + 'up': 'SU', + 'sg': 'SG', + 'sloang': 'SG', + 'sdssg': 'SG', + 'gp': 'SG', + 'sr': 'SR', + 'sloanr': 'SR', + 'sdssr': 'SR', + 'rp': 'SR', + 'si': 'SI', + 'sloani': 'SI', + 'sdssi': 'SI', + 'ip': 'SI', + 'sz': 'SZ', + 'sloanz': 'SZ', + 'sdssz': 'SZ', + 'zp': 'SZ', + } + return direct_map.get(filter_key, obs_filter) + + def nextastro_catalog_rows(catalog_response): if not isinstance(catalog_response, dict): return [] @@ -14071,10 +14124,7 @@ def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=No data = result.json() chart_id = data['chartid'] - if obs_filter == "CV": - obs_filter = "V" - elif obs_filter == "R": - obs_filter = "Rc" + obs_filter = aavso_vsp_band_for_filter(obs_filter) if data['photometry']: for star in data['photometry']: diff --git a/exotic/plots.py b/exotic/plots.py index ec2d62bc..21895e5d 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -581,21 +581,26 @@ def _finite_plot_float(value): def _stellar_variability_reference_label(vsp_param, comparison_label): comp_ra = _finite_plot_float(vsp_param.get('comp_ra')) comp_dec = _finite_plot_float(vsp_param.get('comp_dec')) + details = [] + comparison_label = str(comparison_label).strip() if comparison_label else "" + + if comparison_label and not comparison_label.lower().startswith("ra="): + details.append(f"Label: {comparison_label}") if comp_ra is not None and comp_dec is not None: - return f"RA={comp_ra:.6f}, Dec={comp_dec:.6f}" + details.append(f"RA={comp_ra:.6f}, Dec={comp_dec:.6f}") - return str(comparison_label) if comparison_label else "" + if not details and comparison_label: + details.append(comparison_label) + return " | ".join(details) -def _stellar_variability_aavso_assumption_label(vsp_param): - if not vsp_param.get('is_aavso_vsp', True): - return "" +def _stellar_variability_comparison_metadata_label(vsp_param): details = [] observed_filter = vsp_param.get('observed_filter') if observed_filter: - details.append(f"Observed filter: {observed_filter}") + details.append(f"Original filter: {observed_filter}") comparison_mag = magnitude_text( vsp_param.get('mag_band') or 'V', @@ -603,7 +608,7 @@ def _stellar_variability_aavso_assumption_label(vsp_param): vsp_param.get('cmag_err'), ) if comparison_mag is not None: - details.append(f"Assumed comparison: {comparison_mag}") + details.append(f"Comparison mag: {comparison_mag}") return " | ".join(details) @@ -631,10 +636,10 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): title_lines = [s_name] if reference_label: title_lines.append(reference_label) - assumption_label = _stellar_variability_aavso_assumption_label(first_param) - if assumption_label: - title_lines.append(assumption_label) - ax.set_title("\n".join(title_lines)) + metadata_label = _stellar_variability_comparison_metadata_label(first_param) + if metadata_label: + title_lines.append(metadata_label) + ax.set_title("\n".join(title_lines), fontsize=11) ax.set_ylabel(f"Magnitude ({band})") ax.set_xlabel("Time [JD]") fig.tight_layout() diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 392140ce..3d7cda63 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -333,6 +333,13 @@ def test_nextastro_photometry_catalog_match_rejects_separations_over_two_arcsec( assert match is None +def test_aavso_vsp_band_for_filter_uses_observed_filter_aliases(): + assert exotic_module.aavso_vsp_band_for_filter('MObs CV') == 'V' + assert exotic_module.aavso_vsp_band_for_filter('Clear (unfiltered) reduced to V sequence') == 'V' + assert exotic_module.aavso_vsp_band_for_filter('Cousins R') == 'Rc' + assert exotic_module.aavso_vsp_band_for_filter('Sloan g') == 'SG' + + def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], @@ -404,7 +411,7 @@ def world_to_pixel_values(self, ra_deg, dec_deg): vsp_comp_stars, chart_id = exotic_module.vsp_query( 'frame.fits', [100, 100], - 'CV', + 'MObs CV', 1.0, user_comp_stars=user_comp_stars, user_targ_star=[10, 10], diff --git a/tests/test_plots.py b/tests/test_plots.py index 6e94f0a7..52605dca 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -258,10 +258,11 @@ def spy_set_ylabel(self, label, *args, **kwargs): "NextAstro-123", ) - assert titles[-1] == "Host Star\nRA=10.100000, Dec=-20.200000" - assert "Comparison:" not in titles[-1] - assert "Observed filter" not in titles[-1] - assert "r=12.345 +/- 0.067" not in titles[-1] + assert titles[-1] == ( + "Host Star\n" + "Label: NextAstro-123 | RA=10.100000, Dec=-20.200000\n" + "Original filter: CV | Comparison mag: r=12.345 +/- 0.067" + ) assert ylabels[-1] == "Magnitude (r)" assert (tmp_path / "temp" / "Stellar_Variability.png").exists() @@ -296,8 +297,9 @@ def spy_set_title(self, label, *args, **kwargs): "000-BJX-718", ) - assert "Observed filter: CV" in titles[-1] - assert "Assumed comparison: V=12.345 +/- 0.067" in titles[-1] + assert "Label: 000-BJX-718 | RA=10.100000, Dec=-20.200000" in titles[-1] + assert "Original filter: CV" in titles[-1] + assert "Comparison mag: V=12.345 +/- 0.067" in titles[-1] def test_plot_stellar_variability_omits_invalid_reference_magnitudes(tmp_path, monkeypatch): @@ -330,8 +332,11 @@ def spy_set_title(self, label, *args, **kwargs): "NextAstro-123", ) - assert titles[-1] == "Host Star\nRA=10.100000, Dec=-20.200000" - assert "Observed filter" not in titles[-1] + assert titles[-1] == ( + "Host Star\n" + "Label: NextAstro-123 | RA=10.100000, Dec=-20.200000\n" + "Original filter: MObs CV" + ) assert "99.99" not in titles[-1] assert "V=" not in titles[-1] From 5308510450146bd73f41ea286efd56de33c178ae Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Wed, 8 Jul 2026 09:21:52 +1000 Subject: [PATCH 075/116] exposure times and exposure error tightening --- exotic/api/colab.py | 6 + exotic/exotic.py | 1406 ++++++++++++++++++++++++++-- exotic/exotic_gui.py | 13 + exotic/inputs.py | 84 ++ exotic/output_files.py | 8 + inits.json | 6 + tests/test_centroid_wcs.py | 11 + tests/test_exotic_proper_motion.py | 172 +++- tests/test_inputs.py | 58 ++ 9 files changed, 1686 insertions(+), 78 deletions(-) diff --git a/exotic/api/colab.py b/exotic/api/colab.py index 13a1662b..26b76127 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -384,6 +384,12 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", "use_adaptive_apertures": false, + "gain_electrons_per_adu": null, + "read_noise_electrons": null, + "dark_current_electrons_per_second_per_pixel": null, + "flat_field_fractional_error": null, + "telescope_aperture_m": null, + "scintillation_coefficient": null, "require_comp_star": "y" } } diff --git a/exotic/exotic.py b/exotic/exotic.py index f5c9dbfb..ba717f5e 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -278,6 +278,26 @@ FINAL_RESIDUAL_REJECTION_SIGMA = 3.0 FINAL_RESIDUAL_REJECTION_MAX_CLIP_ITERS = 10 FINAL_RESIDUAL_REJECTION_MAX_REFITS = 10 +NOISE_BUDGET_COMPONENT_KEYS = ( + 'source', + 'sky_aperture', + 'sky_estimate', + 'read', + 'dark', + 'flat', + 'scintillation', + 'total', +) +NOISE_BUDGET_SKY_MEDIAN_VARIANCE_FACTOR = np.pi / 2.0 +SCINTILLATION_COEFFICIENT_DEFAULT = 0.09 +PSF_EFFECTIVE_NOISE_AREA_FACTOR = 4.0 * np.pi +NOISE_GAIN_HEADER_KEYS = ('GAIN', 'EGAIN', 'EPERADU', 'E_PER_ADU', 'GAIN_EAD', 'CCDGAIN') +NOISE_READ_HEADER_KEYS = ('RDNOISE', 'READNOI', 'READNOIS', 'READNSE', 'RN_E', 'RON') +NOISE_DARK_HEADER_KEYS = ('DARKCUR', 'DARKCURR', 'DARKRATE', 'DCURR', 'DARK_EPS', 'PBDKCURR') +NOISE_FLAT_HEADER_KEYS = ('FLATERR', 'FLATFR', 'FLATFRAC', 'FFERR', 'FLATUNC') +NOISE_SCINTILLATION_HEADER_KEYS = ('SCINCOEF', 'SCINTC') +NOISE_APERTURE_HEADER_KEYS = ('TELAPER', 'APERTURE') +NOISE_APERTURE_MM_HEADER_KEYS = ('APR-DIA', 'APTDIA', 'APERTMM') COMPARISON_PREFLIGHT_FIELD_SCORE_RELATIVE_BAND = 0.25 COMPARISON_PREFLIGHT_FIELD_SCORE_ABSOLUTE_BAND = 2.5e-4 PARTIAL_COVERAGE_RPRS_POSTERIOR_MAX_RETRIES = 1 @@ -936,6 +956,9 @@ def annotate_selected_photometry_debug( comp_flux, raw_ratio, initial_sigma_keep_mask, + target_flux_error=None, + comp_flux_error=None, + relative_flux_error=None, prefit_raw_ratio_keep_mask=None, phase_clip_keep_mask_on_sigma_filtered=None, ): @@ -964,6 +987,12 @@ def annotate_selected_photometry_debug( 'target_flux': np.asarray(target_flux, dtype=float).copy(), 'comp_flux': np.asarray(comp_flux, dtype=float).copy(), 'raw_ratio': np.asarray(raw_ratio, dtype=float).copy(), + 'target_flux_error': np.asarray(target_flux_error, dtype=float).copy() + if target_flux_error is not None else np.full(sigma_keep_mask.shape, np.nan, dtype=float), + 'comp_flux_error': np.asarray(comp_flux_error, dtype=float).copy() + if comp_flux_error is not None else np.full(sigma_keep_mask.shape, np.nan, dtype=float), + 'relative_flux_error': np.asarray(relative_flux_error, dtype=float).copy() + if relative_flux_error is not None else np.full(sigma_keep_mask.shape, np.nan, dtype=float), 'initial_sigma_keep_mask': sigma_keep_mask.copy(), 'prefit_raw_ratio_keep_mask': raw_ratio_keep_mask.copy(), 'phase_clip_keep_mask_on_sigma_filtered': phase_keep_mask.copy(), @@ -3340,6 +3369,7 @@ def widen_rprs_bounds_to_data_uncertainty_window(bounds, prior): def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_flux, airmass, jd_times=None, adaptive_summary=None, + target_flux_error=None, comp_flux_error=None, expected_transit_depth=None): result = { 'applied': False, @@ -3349,6 +3379,9 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ 'debug_target_flux': np.array([], dtype=float), 'debug_comp_flux': np.array([], dtype=float), 'debug_raw_ratio': np.array([], dtype=float), + 'debug_target_flux_error': np.array([], dtype=float), + 'debug_comp_flux_error': np.array([], dtype=float), + 'debug_relative_flux_error': np.array([], dtype=float), 'initial_sigma_keep_mask': np.array([], dtype=bool), 'prefit_raw_ratio_keep_mask': np.array([], dtype=bool), 'time': np.array([], dtype=float), @@ -3358,6 +3391,8 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ 'jd_time': np.array([], dtype=float), 'target_flux': np.array([], dtype=float), 'comp_flux': np.array([], dtype=float), + 'target_flux_error': np.array([], dtype=float), + 'comp_flux_error': np.array([], dtype=float), 'source_indices': np.array([], dtype=int), } @@ -3366,6 +3401,8 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ target_flux, comp_flux, airmass, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, jd_times=jd_times, expected_transit_depth=expected_transit_depth, ) @@ -3375,6 +3412,9 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ 'debug_target_flux', 'debug_comp_flux', 'debug_raw_ratio', + 'debug_target_flux_error', + 'debug_comp_flux_error', + 'debug_relative_flux_error', 'initial_sigma_keep_mask', 'prefit_raw_ratio_keep_mask', ): @@ -3394,6 +3434,12 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ good_jd_times = np.asarray(prepared['jd_time'], dtype=float) good_target_flux = np.asarray(prepared['target_flux'], dtype=float) good_comp_flux = np.asarray(prepared['comp_flux'], dtype=float) + good_target_flux_error = np.asarray(prepared.get('target_flux_error', []), dtype=float) + good_comp_flux_error = np.asarray(prepared.get('comp_flux_error', []), dtype=float) + if good_target_flux_error.shape != good_target_flux.shape: + good_target_flux_error = np.full(good_target_flux.shape, np.nan, dtype=float) + if good_comp_flux_error.shape != good_comp_flux.shape: + good_comp_flux_error = np.full(good_comp_flux.shape, np.nan, dtype=float) source_indices = np.asarray(prepared['source_indices'], dtype=int) adaptive_clip_mask = np.zeros(good_times.shape[0], dtype=bool) @@ -3431,6 +3477,8 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ good_jd_times = good_jd_times[~adaptive_clip_mask] good_target_flux = good_target_flux[~adaptive_clip_mask] good_comp_flux = good_comp_flux[~adaptive_clip_mask] + good_target_flux_error = good_target_flux_error[~adaptive_clip_mask] + good_comp_flux_error = good_comp_flux_error[~adaptive_clip_mask] source_indices = source_indices[~adaptive_clip_mask] relative_flux_mask = relative_flux_filter_mask(good_flux) @@ -3450,6 +3498,8 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ 'jd_time': good_jd_times[relative_flux_mask], 'target_flux': good_target_flux[relative_flux_mask], 'comp_flux': good_comp_flux[relative_flux_mask], + 'target_flux_error': good_target_flux_error[relative_flux_mask], + 'comp_flux_error': good_comp_flux_error[relative_flux_mask], 'source_indices': source_indices[relative_flux_mask], }) return result @@ -3716,6 +3766,7 @@ def score_comparison_candidate_lightcurve_scout(prepared_series, eebls_summary, def build_comparison_candidate_preflight(times, jd_times, airmass, ld, p_dict, target_flux, comp_flux, + target_flux_error=None, comp_flux_error=None, adaptive_summary=None, use_eebls_to_initialize_tmid_and_bounds=True): prepared = prepare_comparison_candidate_full_reduction_series( times, @@ -3724,6 +3775,8 @@ def build_comparison_candidate_preflight(times, jd_times, airmass, ld, p_dict, t airmass, jd_times=jd_times, adaptive_summary=adaptive_summary, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) try: @@ -3890,6 +3943,8 @@ def match_time_subset_indices(full_times, subset_times, rtol=1e-10, atol=1e-10): def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, airmass, ld, p_dict, jd_times=None, + target_flux_error=None, + comp_flux_error=None, disable_vertical_flux_normalization=False, detrend_on_outoftransit_baseline=True, use_impactparameter_rather_than_inclination_to_fit=True, @@ -3911,6 +3966,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, 'good_jd_times': np.array([], dtype=float), 'good_target_flux': np.array([], dtype=float), 'good_comp_flux': np.array([], dtype=float), + 'good_target_flux_error': np.array([], dtype=float), + 'good_comp_flux_error': np.array([], dtype=float), 'source_indices': np.array([], dtype=int), 'data_highres': None, 'duration_samples': np.array([], dtype=float), @@ -3926,6 +3983,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, airmass, jd_times=jd_times, adaptive_summary=adaptive_summary, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) else: @@ -3945,6 +4004,12 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_jd_times = np.asarray(prepared['jd_time'], dtype=float) good_target_flux = np.asarray(prepared['target_flux'], dtype=float) good_comp_flux = np.asarray(prepared['comp_flux'], dtype=float) + good_target_flux_error = np.asarray(prepared.get('target_flux_error', []), dtype=float) + good_comp_flux_error = np.asarray(prepared.get('comp_flux_error', []), dtype=float) + if good_target_flux_error.shape != good_target_flux.shape: + good_target_flux_error = np.full(good_target_flux.shape, np.nan, dtype=float) + if good_comp_flux_error.shape != good_comp_flux.shape: + good_comp_flux_error = np.full(good_comp_flux.shape, np.nan, dtype=float) source_indices = np.asarray(prepared['source_indices'], dtype=int) prior = build_comparison_candidate_transit_prior(p_dict, ld) @@ -4037,6 +4102,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_jd_times = good_jd_times[~phase_clip_mask] good_target_flux = good_target_flux[~phase_clip_mask] good_comp_flux = good_comp_flux[~phase_clip_mask] + good_target_flux_error = good_target_flux_error[~phase_clip_mask] + good_comp_flux_error = good_comp_flux_error[~phase_clip_mask] source_indices = source_indices[~phase_clip_mask] full_good_times = np.asarray(good_times, dtype=float) @@ -4046,6 +4113,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, full_good_jd_times = np.asarray(good_jd_times, dtype=float) full_good_target_flux = np.asarray(good_target_flux, dtype=float) full_good_comp_flux = np.asarray(good_comp_flux, dtype=float) + full_good_target_flux_error = np.asarray(good_target_flux_error, dtype=float) + full_good_comp_flux_error = np.asarray(good_comp_flux_error, dtype=float) full_source_indices = np.asarray(source_indices, dtype=int) fast_binning = {'applied': False, 'note': None} @@ -4124,6 +4193,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_jd_times = good_jd_times[final_time_indices] good_target_flux = good_target_flux[final_time_indices] good_comp_flux = good_comp_flux[final_time_indices] + good_target_flux_error = good_target_flux_error[final_time_indices] + good_comp_flux_error = good_comp_flux_error[final_time_indices] source_indices = source_indices[final_time_indices] if run_final_residual_rejection and not fast_binning.get('applied'): @@ -4177,6 +4248,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, clipped_jd_times = good_jd_times[residual_keep_mask] clipped_target_flux = good_target_flux[residual_keep_mask] clipped_comp_flux = good_comp_flux[residual_keep_mask] + clipped_target_flux_error = good_target_flux_error[residual_keep_mask] + clipped_comp_flux_error = good_comp_flux_error[residual_keep_mask] clipped_source_indices = source_indices[residual_keep_mask] residual_refit_prior = dict(fit_prior) @@ -4268,6 +4341,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_jd_times = clipped_jd_times good_target_flux = clipped_target_flux good_comp_flux = clipped_comp_flux + good_target_flux_error = clipped_target_flux_error + good_comp_flux_error = clipped_comp_flux_error source_indices = clipped_source_indices else: residual_stop_reason = 'max_refits' @@ -4304,6 +4379,9 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, prepared['debug_comp_flux'], prepared['debug_raw_ratio'], prepared['initial_sigma_keep_mask'], + target_flux_error=prepared.get('debug_target_flux_error'), + comp_flux_error=prepared.get('debug_comp_flux_error'), + relative_flux_error=prepared.get('debug_relative_flux_error'), prefit_raw_ratio_keep_mask=prepared.get('prefit_raw_ratio_keep_mask'), phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, ) @@ -4319,6 +4397,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, 'good_jd_times': full_good_jd_times if fast_binning.get('applied') else np.asarray(good_jd_times, dtype=float), 'good_target_flux': full_good_target_flux if fast_binning.get('applied') else np.asarray(good_target_flux, dtype=float), 'good_comp_flux': full_good_comp_flux if fast_binning.get('applied') else np.asarray(good_comp_flux, dtype=float), + 'good_target_flux_error': full_good_target_flux_error if fast_binning.get('applied') else np.asarray(good_target_flux_error, dtype=float), + 'good_comp_flux_error': full_good_comp_flux_error if fast_binning.get('applied') else np.asarray(good_comp_flux_error, dtype=float), 'source_indices': full_source_indices if fast_binning.get('applied') else np.asarray(source_indices, dtype=int), 'fast_ultranest_binning': fast_binning, 'fast_fit_good_times': np.asarray(fit_times, dtype=float), @@ -4477,6 +4557,8 @@ def aligned_selected_array(key, dtype=float): target_flux_values = aligned_selected_array('good_target_flux') comp_flux_values = aligned_selected_array('good_comp_flux') + target_flux_error_values = aligned_selected_array('good_target_flux_error') + comp_flux_error_values = aligned_selected_array('good_comp_flux_error') source_indices = aligned_selected_array('source_indices', dtype=int) prior = build_full_resolution_final_prior_from_previous_fit(previous_fit, p_dict) @@ -4662,6 +4744,12 @@ def aligned_selected_array(key, dtype=float): retained_comp_flux_values = ( None if comp_flux_values is None else comp_flux_values[residual_keep_mask] ) + retained_target_flux_error_values = ( + None if target_flux_error_values is None else target_flux_error_values[residual_keep_mask] + ) + retained_comp_flux_error_values = ( + None if comp_flux_error_values is None else comp_flux_error_values[residual_keep_mask] + ) retained_source_indices = ( None if source_indices is None else source_indices[residual_keep_mask] ) @@ -4761,6 +4849,8 @@ def aligned_selected_array(key, dtype=float): jd_times = retained_jd_times target_flux_values = retained_target_flux_values comp_flux_values = retained_comp_flux_values + target_flux_error_values = retained_target_flux_error_values + comp_flux_error_values = retained_comp_flux_error_values source_indices = retained_source_indices pre_ultranest_coverage_assessment = residual_coverage_assessment else: @@ -4858,6 +4948,12 @@ def aligned_selected_array(key, dtype=float): if comp_flux_values is not None: selected_result['good_comp_flux'] = np.asarray(comp_flux_values, dtype=float) selected_result['cflux_fit'] = np.asarray(comp_flux_values, dtype=float) + if target_flux_error_values is not None: + selected_result['good_target_flux_error'] = np.asarray(target_flux_error_values, dtype=float) + selected_result['tflux_fit_error'] = np.asarray(target_flux_error_values, dtype=float) + if comp_flux_error_values is not None: + selected_result['good_comp_flux_error'] = np.asarray(comp_flux_error_values, dtype=float) + selected_result['cflux_fit_error'] = np.asarray(comp_flux_error_values, dtype=float) if source_indices is not None: selected_result['source_indices'] = np.asarray(source_indices, dtype=int) annotate_transit_detection_qc(fit) @@ -5112,6 +5208,18 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat target_flux = np.asarray(debug.get('target_flux'), dtype=float) comp_flux = np.asarray(debug.get('comp_flux'), dtype=float) raw_ratio = np.asarray(debug.get('raw_ratio'), dtype=float) + target_flux_error = np.asarray( + debug.get('target_flux_error', np.full(times.shape, np.nan)), + dtype=float, + ) + comp_flux_error = np.asarray( + debug.get('comp_flux_error', np.full(times.shape, np.nan)), + dtype=float, + ) + relative_flux_error = np.asarray( + debug.get('relative_flux_error', np.full(times.shape, np.nan)), + dtype=float, + ) initial_sigma_keep_mask = np.asarray(debug.get('initial_sigma_keep_mask'), dtype=bool) prefit_raw_ratio_keep_mask = np.asarray( debug.get('prefit_raw_ratio_keep_mask', np.ones(initial_sigma_keep_mask.shape)), @@ -5129,6 +5237,12 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat times.shape == target_flux.shape == comp_flux.shape == raw_ratio.shape == initial_sigma_keep_mask.shape ): return None + if target_flux_error.shape != times.shape: + target_flux_error = np.full(times.shape, np.nan, dtype=float) + if comp_flux_error.shape != times.shape: + comp_flux_error = np.full(times.shape, np.nan, dtype=float) + if relative_flux_error.shape != times.shape: + relative_flux_error = np.full(times.shape, np.nan, dtype=float) if prefit_raw_ratio_keep_mask.shape != initial_sigma_keep_mask.shape: prefit_raw_ratio_keep_mask = np.ones(initial_sigma_keep_mask.shape, dtype=bool) @@ -5154,6 +5268,9 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat target_flux, comp_flux, raw_ratio, + target_flux_error, + comp_flux_error, + relative_flux_error, initial_sigma_keep_mask.astype(int), prefit_raw_ratio_keep_mask.astype(int), phase_keep_full.astype(int), @@ -5165,11 +5282,12 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat delimiter=",", header=( "BJD_TDB,Target Flux,Comp Flux,Raw Ratio," + "Target Flux Error,Comp Flux Error,Relative Flux Error," "Kept After Initial Sigma Clip,Kept After Pre-Fit Raw Ratio Clip," "Kept After Phase Residual Clip" ), comments="", - fmt=["%.8f", "%.8f", "%.8f", "%.8f", "%d", "%d", "%d"], + fmt=["%.8f", "%.8f", "%.8f", "%.8f", "%.8f", "%.8f", "%.8f", "%d", "%d", "%d"], ) return output_path @@ -7169,6 +7287,84 @@ def psf_flux_series_from_rows(psf_rows, quality_mask=None): return flux +def initialize_psf_noise_data(frame_count, comp_star_count): + psf_noise_data = {'target': np.full(int(frame_count), np.nan, dtype=float)} + for component in NOISE_BUDGET_COMPONENT_KEYS: + psf_noise_data[f"target_noise_{component}"] = np.full(int(frame_count), np.nan, dtype=float) + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + psf_noise_data[ckey] = np.full(int(frame_count), np.nan, dtype=float) + for component in NOISE_BUDGET_COMPONENT_KEYS: + psf_noise_data[f"{ckey}_noise_{component}"] = np.full(int(frame_count), np.nan, dtype=float) + return psf_noise_data + + +def compute_psf_noise_budget_for_row(data, psf_row, star_index, noise_config=None, + exposure_s=np.nan, airmass=np.nan, fallback_sigma=np.nan, + fast_mode=False): + psf_row = np.asarray(psf_row, dtype=float).reshape(-1) + empty_budget = {component: np.nan for component in NOISE_BUDGET_COMPONENT_KEYS} + if psf_row.shape[0] < 5: + return empty_budget + xc, yc = psf_row[0], psf_row[1] + sigma_x, sigma_y = psf_row[3], psf_row[4] + if not ( + np.isfinite(xc) + and np.isfinite(yc) + and np.isfinite(sigma_x) + and np.isfinite(sigma_y) + and sigma_x > 0 + and sigma_y > 0 + ): + return empty_budget + + flux = psf_flux_series_from_rows(psf_row.reshape(1, -1))[0] + sigma = psf_sigma_from_fit(psf_row, fallback_sigma=fallback_sigma) + if not np.isfinite(sigma) or sigma <= 0: + sigma = max(float(np.sqrt(sigma_x * sigma_y)), 1.0) + effective_pixels = max(PSF_EFFECTIVE_NOISE_AREA_FACTOR * sigma_x * sigma_y, 1.0) + psf_aperture_radius = max(2.5 * sigma, 1.0) + psf_annulus_width = max(5.0 * sigma, 3.0) + try: + sky_geometry = resolve_sky_annulus_geometry( + psf_aperture_radius, + psf_annulus_width, + psf_sigma=sigma, + ) + _, sigmabg, n_sky = skybg_phot( + data, + star_index, + xc, + yc, + sky_geometry['inner_radius'], + sky_geometry['annulus_width'], + fast_mode=fast_mode, + ) + except Exception: + sigmabg = np.nan + n_sky = np.nan + + return compute_photometry_noise_budget( + flux, + sigmabg, + effective_pixels, + n_sky, + exposure_s=exposure_s, + airmass=airmass, + noise_config=noise_config, + ) + + +def store_psf_noise_budget(psf_noise_data, key, frame_index, budget): + if not isinstance(psf_noise_data, dict) or key not in psf_noise_data: + return + psf_noise_data[key][frame_index] = budget.get('total', np.nan) + for component in NOISE_BUDGET_COMPONENT_KEYS: + component_key = f"{key}_noise_{component}" + if component_key in psf_noise_data: + psf_noise_data[component_key][frame_index] = budget.get(component, np.nan) + + def psf_flux_data_source(psf_data, psf_flux_data=None): if isinstance(psf_flux_data, dict) and 'target' in psf_flux_data: return psf_flux_data @@ -11330,11 +11526,173 @@ def log_lightcurve_filter_diagnostics(diagnostics, header="Lightcurve frame reje log_info(f" {diagnostic_text}") +EXPOSURE_TIME_HEADER_KEYS = ("EXPTIME", "EFFEXPT", "EXPOSURE", "EXP", "REQTIME", "EXPREQ") +BJD_TDB_MID_EXPOSURE_HEADER_KEYS = ("BJD_TDB", "BJD_TBD", "BJD-TDB", "BJD-MID", "TDB-MID", "BJD", "TDB") +JD_MID_EXPOSURE_HEADER_KEYS = ("JD-MID",) +MJD_MID_EXPOSURE_HEADER_KEYS = ("MJD-MID",) +UTC_MID_EXPOSURE_HEADER_KEYS = ("DATE-AVG", "DATE-MID") +JD_START_EXPOSURE_HEADER_KEYS = ("JD-START", "JD", "JULIAN") +MJD_START_EXPOSURE_HEADER_KEYS = ("MJD-OBS", "MJD") +UTC_START_EXPOSURE_HEADER_KEYS = ("DATE-UTC", "DATE-BEG", "DATE-OBS", "UT-OBS") +UTC_END_EXPOSURE_HEADER_KEYS = ("DATE-END", "END-OBS") +EXPOSURE_VARIATION_REQUIRE_COMP_STAR_FRACTION = 0.01 + + +def header_scalar_value(value): + if isinstance(value, tuple) and value: + return value[0] + return value + + +def finite_header_float(value): + value = header_scalar_value(value) + if value is None: + return None + if isinstance(value, str) and value.strip().upper() in ("", "UNKNOWN", "N/A", "NA", "NULL", "NONE"): + return None + try: + numeric_value = float(str(value).strip()) + except (TypeError, ValueError): + return None + return numeric_value if np.isfinite(numeric_value) else None + + +def first_header_float(hdr, keys): + for key in keys: + if key not in hdr: + continue + numeric_value = finite_header_float(hdr[key]) + if numeric_value is not None: + return key, numeric_value + return None, None + + +def header_comment_text(hdr, key): + try: + return str(hdr.comments[key]) + except Exception: + return "" + + +def exposure_time_spread_fraction(exptimes): + values = np.asarray(exptimes, dtype=float) + values = values[np.isfinite(values)] + if values.size < 2: + return 0.0 + spread = float(np.nanmax(values) - np.nanmin(values)) + if spread <= 0: + return 0.0 + reference = float(np.nanmedian(values)) + if not np.isfinite(reference) or reference <= 0: + return np.inf + return spread / reference + + +def exposure_variation_requires_comp_star(exptimes, threshold_fraction=EXPOSURE_VARIATION_REQUIRE_COMP_STAR_FRACTION): + spread_fraction = exposure_time_spread_fraction(exptimes) + return (np.isfinite(spread_fraction) and spread_fraction > threshold_fraction) or np.isinf(spread_fraction) + + +def resolve_require_comp_star_for_exposure_times(config_value, exptimes): + require_comp_star = is_comp_star_required(config_value) + if exposure_variation_requires_comp_star(exptimes): + spread_percent = 100.0 * exposure_time_spread_fraction(exptimes) + if not require_comp_star: + log_info( + "Exposure times vary by more than 1% across retained frames " + f"({spread_percent:.2f}%); requiring a comparison star for this reduction.", + warn=True, + ) + return True + return require_comp_star + + +def parse_header_datetime_to_jd(hdr, key): + if key not in hdr: + return None + + value = header_scalar_value(hdr[key]) + if value is None: + return None + + value_text = str(value).strip() + if not value_text: + return None + + if key == "DATE-OBS" and "T" not in value_text and "TIME-OBS" in hdr: + value_text = f"{value_text}T{header_scalar_value(hdr['TIME-OBS'])}" + elif key == "UT-OBS" and "DATE-OBS" in hdr and "T" not in value_text: + date_text = str(header_scalar_value(hdr["DATE-OBS"])).strip().split("T")[0] + value_text = f"{date_text}T{value_text}" + + try: + dt = dup.parse(value_text) + return Time(dt).jd + except Exception: + return None + + +def first_header_datetime_jd(hdr, keys): + for key in keys: + jd_value = parse_header_datetime_to_jd(hdr, key) + if jd_value is not None and np.isfinite(jd_value): + return key, float(jd_value) + return None, None + + +def direct_bjd_tdb_mid_exposure(hdr): + return first_header_float(hdr, BJD_TDB_MID_EXPOSURE_HEADER_KEYS) + + +def utc_mid_exposure_jd(hdr): + key, jd_mid = first_header_float(hdr, JD_MID_EXPOSURE_HEADER_KEYS) + if jd_mid is not None: + return key, jd_mid + + key, mjd_mid = first_header_float(hdr, MJD_MID_EXPOSURE_HEADER_KEYS) + if mjd_mid is not None: + return key, mjd_mid + 2400000.5 + + return first_header_datetime_jd(hdr, UTC_MID_EXPOSURE_HEADER_KEYS) + + +def utc_start_exposure_jd(hdr): + key, jd_start = first_header_float(hdr, JD_START_EXPOSURE_HEADER_KEYS) + if jd_start is not None: + return key, jd_start + + key, mjd_start = first_header_float(hdr, MJD_START_EXPOSURE_HEADER_KEYS) + if mjd_start is not None: + return key, mjd_start + 2400000.5 + + return first_header_datetime_jd(hdr, UTC_START_EXPOSURE_HEADER_KEYS) + + +def utc_end_exposure_jd(hdr): + return first_header_datetime_jd(hdr, UTC_END_EXPOSURE_HEADER_KEYS) + + +def utc_exposure_midpoint_jd(hdr, exp): + key, jd_mid = utc_mid_exposure_jd(hdr) + if jd_mid is not None: + return key, jd_mid + + start_key, jd_start = utc_start_exposure_jd(hdr) + end_key, jd_end = utc_end_exposure_jd(hdr) + if jd_start is not None and jd_end is not None and jd_end >= jd_start: + return f"{start_key}/{end_key}", 0.5 * (jd_start + jd_end) + + if jd_start is None: + return None, None + + return start_key, jd_start + exp / (2.0 * 60.0 * 60.0 * 24.0) + + def exp_offset(hdr, time_unit, exp): """Returns exposure offset (in days) of more than 0 if headers reveals the time was estimated at the start of the exposure rather than the middle """ - if 'start' in hdr.comments[time_unit]: + if 'start' in header_comment_text(hdr, time_unit).lower(): return exp / (2.0 * 60.0 * 60.0 * 24.0) return 0.0 @@ -11370,9 +11728,8 @@ def julian_date(hdr, time_unit, exp): return julian_time + offset def get_exp_time(hdr): - exp_list = ["EXPTIME", "EXPOSURE", "EXP"] - exp_time = next((exptime for exptime in exp_list if exptime in hdr), None) - return hdr[exp_time] if exp_time is not None else 0.0 + _, exp_time = first_header_float(hdr, EXPOSURE_TIME_HEADER_KEYS) + return exp_time if exp_time is not None else 0.0 def img_time_jd(hdr): """Converts time from the header file to the Julian Date (JD, if needed) @@ -11387,17 +11744,9 @@ def img_time_jd(hdr): float Time of when the image was taken in the JD with exposure offset """ - time_list = ['UT-OBS', 'JULIAN', 'MJD-OBS', 'DATE-OBS'] - exp = get_exp_time(hdr) - hdr_time = next((time_unit for time_unit in time_list if time_unit in hdr), None) - - if hdr_time == 'MJD_OBS': - hdr_time = hdr_time if "epoch" not in hdr.comments[hdr_time] else 'DATE-OBS' - - if hdr_time in ['UT-OBS', 'DATE-OBS']: - return ut_date(hdr, hdr_time, exp) - return julian_date(hdr, hdr_time, exp) + _, jd_mid = utc_exposure_midpoint_jd(hdr, exp) + return jd_mid if jd_mid is not None else np.nan def img_time_bjd_tdb(hdr, p_dict, info_dict): @@ -11416,27 +11765,16 @@ def img_time_bjd_tdb(hdr, p_dict, info_dict): float Time of when the image was taken in BJD-TDB with exposure offset """ - # Check for BJD time first (preference) - time_list = ['BJD_TDB', 'BJD_TBD', 'BJD'] exp = get_exp_time(hdr) - hdr_time = next((time for time in time_list if time in hdr), None) - # Not found, get julian date + _, bjd_time = direct_bjd_tdb_mid_exposure(hdr) + if bjd_time is not None: + return bjd_time - if hdr_time is None: - time_list = ['UT-OBS', 'JULIAN', 'MJD-OBS', 'DATE-OBS'] - hdr_time = next((time for time in time_list if time in hdr), None) - if hdr_time == 'MJD_OBS': - hdr_time = hdr_time if "epoch" not in hdr.comments[hdr_time] else 'DATE-OBS' - if hdr_time in ['UT-OBS', 'DATE-OBS']: - jd_time = ut_date(hdr, hdr_time, exp) - else: - jd_time = julian_date(hdr, hdr_time, exp) - # And convert to BJD_TDB - bjd_time = convert_jd_to_bjd([jd_time], p_dict, info_dict)[0] - else: # Else, already BJD - convert and adjust for exposure - bjd_time = julian_date(hdr, hdr_time, exp) - return bjd_time + _, jd_time = utc_exposure_midpoint_jd(hdr, exp) + if jd_time is None: + return np.nan + return convert_jd_to_bjd([jd_time], p_dict, info_dict)[0] def air_mass(hdr, ra, dec, lat, long, elevation, time): """Scrapes or calculates the airmass at the time of when the image was taken. @@ -13741,6 +14079,105 @@ def nextastro_catalog_color(row, obs_filter): return None +def normalize_colour_index_label(value): + text = str(value or '').strip().upper().replace(' ', '') + aliases = { + 'B-V': 'B-V', + 'BV': 'B-V', + 'BP-RP': 'BP-RP', + 'BPRP': 'BP-RP', + 'GBP-GRP': 'BP-RP', + 'GAIABP-RP': 'BP-RP', + 'R-I': 'R-I', + 'RI': 'R-I', + 'G-R': 'G-R', + 'GR': 'G-R', + 'I-Z': 'I-Z', + 'IZ': 'I-Z', + 'U-G': 'U-G', + 'UG': 'U-G', + } + return aliases.get(text, text) + + +def colour_term_metadata_from_info(info_dict): + if not isinstance(info_dict, dict): + return {} + + def _number(*keys, nonnegative=False): + for key in keys: + value = _finite_float(info_dict.get(key)) + if value is None: + continue + if value <= -90.0: + continue + if nonnegative and value < 0.0: + continue + return float(value) + return None + + metadata = { + 'term': _number('colour_term', 'color_term', 'COLTERM'), + 'term_error': _number( + 'colour_term_error', + 'color_term_error', + 'COLTERR', + nonnegative=True, + ), + 'term_index': normalize_colour_index_label( + info_dict.get('colour_term_index') + or info_dict.get('color_term_index') + or info_dict.get('COLTIDX') + ), + 'bv_term': _number('colour_term_bv', 'color_term_bv', 'COLTBV'), + 'bv_error': _number( + 'colour_term_bv_error', + 'color_term_bv_error', + 'COLTBVER', + 'COLTBVERR', + nonnegative=True, + ), + 'bprp_term': _number('colour_term_bprp', 'color_term_bprp', 'COLTBPRP'), + 'bprp_error': _number( + 'colour_term_bprp_error', + 'color_term_bprp_error', + 'CBPRPERR', + 'COLTBPRPERR', + nonnegative=True, + ), + 'equation_filter': ( + info_dict.get('colour_equation_filter') + or info_dict.get('color_equation_filter') + or info_dict.get('COLEQFIL') + ), + } + return metadata + + +def colour_term_for_catalog_label(metadata, color_label): + if not isinstance(metadata, dict): + return None, None + normalized_label = normalize_colour_index_label(color_label) + if normalized_label == 'B-V': + term = metadata.get('bv_term') + error = metadata.get('bv_error') + if term is not None: + return term, error + if normalized_label == 'BP-RP': + term = metadata.get('bprp_term') + error = metadata.get('bprp_error') + if term is not None: + return term, error + + generic_term = metadata.get('term') + generic_index = normalize_colour_index_label(metadata.get('term_index')) + if generic_term is not None and (not generic_index or generic_index == normalized_label): + return generic_term, metadata.get('term_error') + if generic_term is not None and normalized_label not in {'B-V', 'BP-RP'}: + return generic_term, metadata.get('term_error') + return None, None + + def nextastro_catalog_nearest_color_row(catalog_response, ra, dec, obs_filter, max_separation_arcsec= AUTOMATIC_CALIBRATION_SELECTOR_COLOR_MATCH_RADIUS_ARCSEC): @@ -13780,7 +14217,8 @@ def select_automatic_optimal_calibration_stars( obs_filter, field_catalog, count=AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT, - min_comp_target_sep=REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS): + min_comp_target_sep=REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS, + colour_term_metadata=None): max_count = parse_automatic_calibration_selector_count(count) if image_data is None or field_catalog is None: return [], [] @@ -13869,6 +14307,16 @@ def select_automatic_optimal_calibration_stars( if match is None or color is None: continue color_delta = abs(color['color'] - target_color['color']) + colour_term, colour_term_error = colour_term_for_catalog_label( + colour_term_metadata, + color['label'], + ) + expected_colour_mismatch_mag = None + colour_term_uncertainty_mag = None + if colour_term is not None and np.isfinite(colour_term): + expected_colour_mismatch_mag = abs(float(colour_term)) * float(color_delta) + if colour_term_error is not None and np.isfinite(colour_term_error): + colour_term_uncertainty_mag = abs(float(color_delta)) * float(colour_term_error) candidates.append({ 'x': float(x_pos), 'y': float(y_pos), @@ -13881,12 +14329,30 @@ def select_automatic_optimal_calibration_stars( 'color_label': color['label'], 'target_color': target_color['color'], 'color_delta': float(color_delta), + 'colour_term': float(colour_term) if colour_term is not None else None, + 'colour_term_error': ( + float(colour_term_error) if colour_term_error is not None else None + ), + 'expected_colour_mismatch_mag': ( + float(expected_colour_mismatch_mag) + if expected_colour_mismatch_mag is not None + else None + ), + 'colour_term_uncertainty_mag': ( + float(colour_term_uncertainty_mag) + if colour_term_uncertainty_mag is not None + else None + ), 'target_flux': float(target_flux), }) candidates.sort( key=lambda candidate: ( - candidate['color_delta'], + ( + candidate['expected_colour_mismatch_mag'] + if candidate.get('expected_colour_mismatch_mag') is not None + else candidate['color_delta'] + ), abs(np.log(candidate['brightness_ratio'])), -candidate['flux'], ) @@ -13911,12 +14377,21 @@ def log_automatic_optimal_calibration_selection(comp_stars, candidates, requeste "and ranked by catalog color similarity to the target." ) for index, candidate in enumerate(candidates, start=1): + mismatch_text = "" + if candidate.get('expected_colour_mismatch_mag') is not None: + mismatch_text = ( + f", expected_colour_mismatch={candidate['expected_colour_mismatch_mag']:.5f} mag" + ) + if candidate.get('colour_term_uncertainty_mag') is not None: + mismatch_text += ( + f", colour_term_sigma={candidate['colour_term_uncertainty_mag']:.5f} mag" + ) log_info( f" Auto comp #{index}: pixels=[{candidate['x']:.2f}, {candidate['y']:.2f}], " f"flux_ratio={candidate['brightness_ratio']:.3f}, " f"{candidate['color_label']}={candidate['color']:.3f}, " f"target_{candidate['color_label']}={candidate['target_color']:.3f}, " - f"delta={candidate['color_delta']:.3f}." + f"delta={candidate['color_delta']:.3f}{mismatch_text}." ) @@ -16461,6 +16936,397 @@ def normalize_flux_series_to_approximate_unity( return normalized_flux, normalized_unc, float(baseline_level) +def is_blank_noise_budget_value(value): + if value is None: + return True + if isinstance(value, str): + return value.strip() == "" + return False + + +def coerce_noise_budget_scalar(value, *, require_positive=False, require_nonnegative=False): + if is_blank_noise_budget_value(value): + return np.nan + try: + scalar = float(value) + except (TypeError, ValueError): + return np.nan + if not np.isfinite(scalar): + return np.nan + if require_positive and scalar <= 0: + return np.nan + if require_nonnegative and scalar < 0: + return np.nan + return float(scalar) + + +def noise_budget_value_from_mapping(mapping, keys, *, require_positive=False, require_nonnegative=False): + if not isinstance(mapping, dict): + return np.nan + for key in keys: + if key not in mapping: + continue + value = coerce_noise_budget_scalar( + mapping.get(key), + require_positive=require_positive, + require_nonnegative=require_nonnegative, + ) + if np.isfinite(value): + return value + return np.nan + + +def noise_budget_value_from_header(header, keys, *, require_positive=False, require_nonnegative=False): + if header is None: + return np.nan + try: + header_keys = set(header.keys()) + except AttributeError: + return np.nan + upper_lookup = {str(key).upper(): key for key in header_keys} + for key in keys: + actual_key = upper_lookup.get(str(key).upper()) + if actual_key is None: + continue + value = coerce_noise_budget_scalar( + header.get(actual_key), + require_positive=require_positive, + require_nonnegative=require_nonnegative, + ) + if np.isfinite(value): + return value + return np.nan + + +def resolve_noise_budget_value(info_dict, aliases, header=None, header_keys=(), + *, require_positive=False, require_nonnegative=False): + value = noise_budget_value_from_mapping( + info_dict, + aliases, + require_positive=require_positive, + require_nonnegative=require_nonnegative, + ) + if np.isfinite(value): + return value, 'inits' + value = noise_budget_value_from_header( + header, + header_keys, + require_positive=require_positive, + require_nonnegative=require_nonnegative, + ) + if np.isfinite(value): + return value, 'fits_header' + return np.nan, None + + +def noise_budget_config_from_info(info_dict, header=None): + info_dict = info_dict if isinstance(info_dict, dict) else {} + gain, gain_source = resolve_noise_budget_value( + info_dict, + ( + 'gain_electrons_per_adu', + 'gain_e_per_adu', + 'gain', + 'Gain (e-/ADU)', + 'CCD Gain (e-/ADU)', + ), + header=header, + header_keys=NOISE_GAIN_HEADER_KEYS, + require_positive=True, + ) + if not np.isfinite(gain) or gain <= 0: + gain = 1.0 + gain_source = 'default' + + read_noise, read_source = resolve_noise_budget_value( + info_dict, + ( + 'read_noise_electrons', + 'read_noise_e', + 'read_noise', + 'Read Noise (e-)', + 'CCD Read Noise (e-)', + ), + header=header, + header_keys=NOISE_READ_HEADER_KEYS, + require_nonnegative=True, + ) + dark_current, dark_source = resolve_noise_budget_value( + info_dict, + ( + 'dark_current_electrons_per_second_per_pixel', + 'dark_current_e_per_s_pix', + 'dark_current', + 'Dark Current (e-/s/pix)', + ), + header=header, + header_keys=NOISE_DARK_HEADER_KEYS, + require_nonnegative=True, + ) + flat_fraction, flat_source = resolve_noise_budget_value( + info_dict, + ( + 'flat_field_fractional_error', + 'flat_field_fractional_noise', + 'flat_field_error_fraction', + 'Flat Field Fractional Error', + 'Flat-Field Fractional Error', + ), + header=header, + header_keys=NOISE_FLAT_HEADER_KEYS, + require_nonnegative=True, + ) + scintillation_coefficient, scint_source = resolve_noise_budget_value( + info_dict, + ( + 'scintillation_coefficient', + 'scintillation_noise_coefficient', + 'Scintillation Coefficient', + ), + header=header, + header_keys=NOISE_SCINTILLATION_HEADER_KEYS, + require_positive=True, + ) + telescope_aperture_m, aperture_source = resolve_noise_budget_value( + info_dict, + ( + 'telescope_aperture_m', + 'telescope_aperture_meters', + 'Telescope Aperture (m)', + ), + header=header, + header_keys=NOISE_APERTURE_HEADER_KEYS, + require_positive=True, + ) + telescope_aperture_cm, aperture_cm_source = resolve_noise_budget_value( + info_dict, + ( + 'telescope_aperture_cm', + 'Telescope Aperture (cm)', + ), + require_positive=True, + ) + telescope_aperture_mm, aperture_mm_source = resolve_noise_budget_value( + info_dict, + ( + 'telescope_aperture_mm', + 'Telescope Aperture (mm)', + ), + header=header, + header_keys=NOISE_APERTURE_MM_HEADER_KEYS, + require_positive=True, + ) + if np.isfinite(telescope_aperture_cm) and telescope_aperture_cm > 0: + telescope_aperture_m = telescope_aperture_cm / 100.0 + aperture_source = aperture_cm_source + elif np.isfinite(telescope_aperture_mm) and telescope_aperture_mm > 0: + telescope_aperture_m = telescope_aperture_mm / 1000.0 + aperture_source = aperture_mm_source + if not np.isfinite(scintillation_coefficient): + scintillation_coefficient = SCINTILLATION_COEFFICIENT_DEFAULT + scint_source = 'default' + + enabled_terms = ['source', 'sky_aperture', 'sky_estimate'] + source_by_term = {'gain': gain_source} + optional_terms = { + 'read': (read_noise, read_source), + 'dark': (dark_current, dark_source), + 'flat': (flat_fraction, flat_source), + } + for term, (value, source) in optional_terms.items(): + if np.isfinite(value) and value > 0: + enabled_terms.append(term) + source_by_term[term] = source + if np.isfinite(telescope_aperture_m) and telescope_aperture_m > 0: + enabled_terms.append('scintillation') + source_by_term['scintillation'] = aperture_source + source_by_term['scintillation_coefficient'] = scint_source + + return { + 'gain_e_per_adu': float(gain), + 'read_noise_electrons': read_noise, + 'dark_current_electrons_per_second_per_pixel': dark_current, + 'flat_field_fractional_error': flat_fraction, + 'scintillation_coefficient': scintillation_coefficient, + 'telescope_aperture_m': telescope_aperture_m, + 'elevation_m': coerce_noise_budget_scalar(info_dict.get('elev'), require_nonnegative=True), + 'enabled_terms': tuple(enabled_terms), + 'source_by_term': source_by_term, + } + + +def format_noise_budget_config_summary(config): + if not isinstance(config, dict): + return "source, sky_aperture, sky_estimate" + parts = [] + gain = config.get('gain_e_per_adu', 1.0) + parts.append(f"gain={gain:.4g} e-/ADU") + for term in ('read', 'dark', 'flat', 'scintillation'): + if term not in config.get('enabled_terms', ()): + continue + if term == 'read': + parts.append(f"read={config.get('read_noise_electrons'):.4g} e-") + elif term == 'dark': + parts.append( + f"dark={config.get('dark_current_electrons_per_second_per_pixel'):.4g} e-/s/pix" + ) + elif term == 'flat': + parts.append(f"flat={config.get('flat_field_fractional_error'):.4g} frac") + elif term == 'scintillation': + parts.append(f"scintillation D={config.get('telescope_aperture_m'):.4g} m") + return ", ".join(parts) + + +def empty_noise_budget_grids(shape): + return { + component: np.full(shape, np.nan, dtype=float) + for component in NOISE_BUDGET_COMPONENT_KEYS + } + + +def empty_noise_budget_series(length): + return { + component: np.full(int(length), np.nan, dtype=float) + for component in NOISE_BUDGET_COMPONENT_KEYS + } + + +def compute_scintillation_fraction(config, exposure_s=np.nan, airmass=np.nan): + if not isinstance(config, dict) or 'scintillation' not in config.get('enabled_terms', ()): + return np.nan + aperture_m = coerce_noise_budget_scalar(config.get('telescope_aperture_m'), require_positive=True) + exposure_s = coerce_noise_budget_scalar(exposure_s, require_positive=True) + if not np.isfinite(aperture_m) or not np.isfinite(exposure_s): + return np.nan + airmass = coerce_noise_budget_scalar(airmass, require_positive=True) + if not np.isfinite(airmass): + airmass = 1.0 + elevation_m = coerce_noise_budget_scalar(config.get('elevation_m'), require_nonnegative=True) + if not np.isfinite(elevation_m): + elevation_m = 0.0 + coefficient = coerce_noise_budget_scalar( + config.get('scintillation_coefficient', SCINTILLATION_COEFFICIENT_DEFAULT), + require_positive=True, + ) + if not np.isfinite(coefficient): + coefficient = SCINTILLATION_COEFFICIENT_DEFAULT + aperture_cm = aperture_m * 100.0 + return float( + coefficient + * aperture_cm ** (-2.0 / 3.0) + * airmass ** 1.75 + * np.exp(-elevation_m / 8000.0) + / np.sqrt(2.0 * exposure_s) + ) + + +def compute_photometry_noise_budget(flux_adu, sky_sigma_adu, aperture_pixels, sky_pixels, + exposure_s=np.nan, airmass=np.nan, noise_config=None): + config = noise_config if isinstance(noise_config, dict) else {} + gain = coerce_noise_budget_scalar(config.get('gain_e_per_adu', 1.0), require_positive=True) + if not np.isfinite(gain): + gain = 1.0 + flux_adu = coerce_noise_budget_scalar(flux_adu) + sky_sigma_adu = coerce_noise_budget_scalar(sky_sigma_adu, require_nonnegative=True) + aperture_pixels = coerce_noise_budget_scalar(aperture_pixels, require_positive=True) + sky_pixels = coerce_noise_budget_scalar(sky_pixels, require_positive=True) + exposure_s = coerce_noise_budget_scalar(exposure_s, require_positive=True) + + variances = {component: 0.0 for component in NOISE_BUDGET_COMPONENT_KEYS if component != 'total'} + if np.isfinite(flux_adu): + variances['source'] = max(float(flux_adu), 0.0) / gain + if np.isfinite(sky_sigma_adu) and np.isfinite(aperture_pixels): + variances['sky_aperture'] = aperture_pixels * sky_sigma_adu ** 2 + if np.isfinite(sky_pixels) and sky_pixels > 0: + variances['sky_estimate'] = ( + NOISE_BUDGET_SKY_MEDIAN_VARIANCE_FACTOR + * aperture_pixels ** 2 + * sky_sigma_adu ** 2 + / sky_pixels + ) + + read_noise = coerce_noise_budget_scalar(config.get('read_noise_electrons'), require_nonnegative=True) + if np.isfinite(read_noise) and np.isfinite(aperture_pixels): + variances['read'] = aperture_pixels * (read_noise / gain) ** 2 + + dark_current = coerce_noise_budget_scalar( + config.get('dark_current_electrons_per_second_per_pixel'), + require_nonnegative=True, + ) + if np.isfinite(dark_current) and np.isfinite(exposure_s) and np.isfinite(aperture_pixels): + variances['dark'] = aperture_pixels * dark_current * exposure_s / (gain ** 2) + + flux_abs = abs(float(flux_adu)) if np.isfinite(flux_adu) else np.nan + flat_fraction = coerce_noise_budget_scalar( + config.get('flat_field_fractional_error'), + require_nonnegative=True, + ) + if np.isfinite(flat_fraction) and np.isfinite(flux_abs): + variances['flat'] = (flat_fraction * flux_abs) ** 2 + + scintillation_fraction = compute_scintillation_fraction(config, exposure_s=exposure_s, airmass=airmass) + if np.isfinite(scintillation_fraction) and np.isfinite(flux_abs): + variances['scintillation'] = (scintillation_fraction * flux_abs) ** 2 + + total_variance = float( + np.nansum([ + variance + for variance in variances.values() + if np.isfinite(variance) and variance >= 0 + ]) + ) + budget = {} + for component in NOISE_BUDGET_COMPONENT_KEYS: + variance = total_variance if component == 'total' else variances.get(component, 0.0) + budget[component] = float(np.sqrt(max(variance, 0.0))) if np.isfinite(variance) else np.nan + return budget + + +def valid_flux_error_array(flux_error, shape): + if flux_error is None: + return None + try: + values = np.asarray(flux_error, dtype=float) + except (TypeError, ValueError): + return None + if values.shape != shape: + return None + return values + + +def relative_flux_uncertainty_from_star_errors(target_flux, comp_flux, + target_flux_error=None, comp_flux_error=None): + target_flux = np.asarray(target_flux, dtype=float) + comp_flux = np.asarray(comp_flux, dtype=float) + target_flux_error = valid_flux_error_array(target_flux_error, target_flux.shape) + comp_flux_error = valid_flux_error_array(comp_flux_error, comp_flux.shape) + + with np.errstate(invalid='ignore'): + target_fallback_error = np.sqrt(target_flux) + comp_fallback_error = np.sqrt(comp_flux) + + if target_flux_error is not None: + valid_target_error = np.isfinite(target_flux_error) & (target_flux_error > 0) + target_sigma = np.where(valid_target_error, target_flux_error, target_fallback_error) + else: + target_sigma = target_fallback_error + + if comp_flux_error is not None: + valid_comp_error = np.isfinite(comp_flux_error) & (comp_flux_error > 0) + comp_sigma = np.where(valid_comp_error, comp_flux_error, comp_fallback_error) + else: + comp_sigma = comp_fallback_error + + if np.allclose(comp_flux, 1.0): + return target_sigma + + with np.errstate(divide='ignore', invalid='ignore'): + return np.sqrt( + (target_sigma / comp_flux) ** 2 + + (comp_sigma * target_flux / comp_flux ** 2) ** 2 + ) + + def weighted_nanpercentile(values, weights, percentile): values = np.asarray(values, dtype=float).ravel() weights = np.asarray(weights, dtype=float).ravel() @@ -18422,13 +19288,17 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, use_impactparameter_rather_than_inclination_to_fit=True, plot_time_range=None, use_eebls_to_initialize_tmid_and_bounds=True, - compute_eebls_diagnostics=False): + compute_eebls_diagnostics=False, + target_flux_error=None, + comp_flux_error=None): plot_time_range = np.asarray(times if plot_time_range is None else plot_time_range, dtype=float) prepared = prepare_lightcurve_fit_input_series( times, tFlux, cFlux, airmass, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, jd_times=jd_times, expected_transit_depth=expected_transit_depth_from_planet_dict(pDict), ) @@ -18440,6 +19310,9 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, debug_target_flux = prepared['debug_target_flux'] debug_comp_flux = prepared['debug_comp_flux'] debug_raw_ratio = prepared['debug_raw_ratio'] + debug_target_flux_error = prepared.get('debug_target_flux_error') + debug_comp_flux_error = prepared.get('debug_comp_flux_error') + debug_relative_flux_error = prepared.get('debug_relative_flux_error') debug_initial_sigma_keep_mask = prepared['initial_sigma_keep_mask'] debug_prefit_raw_ratio_keep_mask = prepared['prefit_raw_ratio_keep_mask'] arrayFinalFlux = prepared['flux'] @@ -18647,6 +19520,9 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, debug_comp_flux, debug_raw_ratio, debug_initial_sigma_keep_mask, + target_flux_error=debug_target_flux_error, + comp_flux_error=debug_comp_flux_error, + relative_flux_error=debug_relative_flux_error, prefit_raw_ratio_keep_mask=debug_prefit_raw_ratio_keep_mask, phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, ) @@ -18661,6 +19537,8 @@ def diagnose_lightcurve_fit_inputs( tflux, cflux, airmass, + target_flux_error=None, + comp_flux_error=None, enforce_relative_flux_max=True, expected_transit_depth=None, ): @@ -18668,6 +19546,8 @@ def diagnose_lightcurve_fit_inputs( tflux = np.asarray(tflux, dtype=float) cflux = np.asarray(cflux, dtype=float) airmass = np.asarray(airmass, dtype=float) + target_flux_error = valid_flux_error_array(target_flux_error, tflux.shape) + comp_flux_error = valid_flux_error_array(comp_flux_error, cflux.shape) diagnostics = { 'input_point_count': int(times.shape[0]), @@ -18695,6 +19575,8 @@ def diagnose_lightcurve_fit_inputs( times_sorted = times[si] tflux_sorted = tflux[si] cflux_sorted = cflux[si] + target_flux_error_sorted = None if target_flux_error is None else target_flux_error[si] + comp_flux_error_sorted = None if comp_flux_error is None else comp_flux_error[si] with np.errstate(divide='ignore', invalid='ignore'): flux_ratio_sorted = np.divide(tflux_sorted, cflux_sorted) @@ -18725,6 +19607,10 @@ def diagnose_lightcurve_fit_inputs( times_sorted = times_sorted[relative_flux_mask] tflux_sorted = tflux_sorted[relative_flux_mask] cflux_sorted = cflux_sorted[relative_flux_mask] + if target_flux_error_sorted is not None: + target_flux_error_sorted = target_flux_error_sorted[relative_flux_mask] + if comp_flux_error_sorted is not None: + comp_flux_error_sorted = comp_flux_error_sorted[relative_flux_mask] flux_ratio_sorted = flux_ratio_sorted[relative_flux_mask] airmass_sorted = airmass[si][relative_flux_mask] if diagnostics['relative_flux_point_count'] <= 1: @@ -18785,13 +19671,10 @@ def diagnose_lightcurve_fit_inputs( arrayFinalFlux = flux_ratio_sorted[valid_mask] f1 = tflux_sorted[valid_mask] - sigf1 = f1 ** 0.5 f2 = cflux_sorted[valid_mask] - sigf2 = f2 ** 0.5 - if np.sum(cflux) == len(cflux): - arrayNormUnc = sigf1 - else: - arrayNormUnc = np.sqrt((sigf1 / f2) ** 2 + (sigf2 * f1 / f2 ** 2) ** 2) + f1_err = None if target_flux_error_sorted is None else target_flux_error_sorted[valid_mask] + f2_err = None if comp_flux_error_sorted is None else comp_flux_error_sorted[valid_mask] + arrayNormUnc = relative_flux_uncertainty_from_star_errors(f1, f2, f1_err, f2_err) arrayTimes = times_sorted[valid_mask] arrayAirmass = airmass_sorted[valid_mask] @@ -18836,6 +19719,8 @@ def prepare_lightcurve_fit_input_series( target_flux, comp_flux, airmass, + target_flux_error=None, + comp_flux_error=None, jd_times=None, expected_transit_depth=None, ): @@ -18843,6 +19728,8 @@ def prepare_lightcurve_fit_input_series( target_flux = np.asarray(target_flux, dtype=float) comp_flux = np.asarray(comp_flux, dtype=float) airmass = np.asarray(airmass, dtype=float) + target_flux_error = valid_flux_error_array(target_flux_error, target_flux.shape) + comp_flux_error = valid_flux_error_array(comp_flux_error, comp_flux.shape) jd_times_array = None if jd_times is None else np.asarray(jd_times, dtype=float) prepared = { @@ -18853,6 +19740,9 @@ def prepare_lightcurve_fit_input_series( 'debug_target_flux': np.array([], dtype=float), 'debug_comp_flux': np.array([], dtype=float), 'debug_raw_ratio': np.array([], dtype=float), + 'debug_target_flux_error': np.array([], dtype=float), + 'debug_comp_flux_error': np.array([], dtype=float), + 'debug_relative_flux_error': np.array([], dtype=float), 'initial_sigma_keep_mask': np.array([], dtype=bool), 'prefit_raw_ratio_keep_mask': np.array([], dtype=bool), 'time': np.array([], dtype=float), @@ -18862,6 +19752,8 @@ def prepare_lightcurve_fit_input_series( 'airmass': np.array([], dtype=float), 'target_flux': np.array([], dtype=float), 'comp_flux': np.array([], dtype=float), + 'target_flux_error': np.array([], dtype=float), + 'comp_flux_error': np.array([], dtype=float), 'source_indices': np.array([], dtype=int), 'skip_airmass_fit': False, 'approximate_baseline_level': np.nan, @@ -18883,6 +19775,8 @@ def prepare_lightcurve_fit_input_series( times_sorted = times[plot_indices] target_flux_sorted = target_flux[plot_indices] comp_flux_sorted = comp_flux[plot_indices] + target_flux_error_sorted = None if target_flux_error is None else target_flux_error[plot_indices] + comp_flux_error_sorted = None if comp_flux_error is None else comp_flux_error[plot_indices] source_indices = np.asarray(plot_indices, dtype=int) with np.errstate(divide='ignore', invalid='ignore'): flux_ratio_sorted = np.divide(target_flux_sorted, comp_flux_sorted) @@ -18900,6 +19794,10 @@ def prepare_lightcurve_fit_input_series( times_sorted = times_sorted[flux_ratio_mask] target_flux_sorted = target_flux_sorted[flux_ratio_mask] comp_flux_sorted = comp_flux_sorted[flux_ratio_mask] + if target_flux_error_sorted is not None: + target_flux_error_sorted = target_flux_error_sorted[flux_ratio_mask] + if comp_flux_error_sorted is not None: + comp_flux_error_sorted = comp_flux_error_sorted[flux_ratio_mask] flux_ratio_sorted = flux_ratio_sorted[flux_ratio_mask] source_indices = source_indices[flux_ratio_mask] if jd_times_array is None: @@ -18920,6 +19818,22 @@ def prepare_lightcurve_fit_input_series( debug_target_flux = np.asarray(target_flux_sorted, dtype=float).copy() debug_comp_flux = np.asarray(comp_flux_sorted, dtype=float).copy() debug_raw_ratio = np.asarray(flux_ratio_sorted, dtype=float).copy() + debug_target_flux_error = ( + np.asarray(target_flux_error_sorted, dtype=float).copy() + if target_flux_error_sorted is not None + else np.full(debug_times.shape, np.nan, dtype=float) + ) + debug_comp_flux_error = ( + np.asarray(comp_flux_error_sorted, dtype=float).copy() + if comp_flux_error_sorted is not None + else np.full(debug_times.shape, np.nan, dtype=float) + ) + debug_relative_flux_error = relative_flux_uncertainty_from_star_errors( + target_flux_sorted, + comp_flux_sorted, + target_flux_error_sorted, + comp_flux_error_sorted, + ) dt = np.mean(np.diff(times_sorted)) ndt = int(25. / 24. / 60. / dt) * 2 + 1 @@ -18971,12 +19885,14 @@ def prepare_lightcurve_fit_input_series( flux = flux_ratio_sorted[valid_mask] filtered_target_flux = target_flux_sorted[valid_mask] filtered_comp_flux = comp_flux_sorted[valid_mask] - if np.sum(comp_flux) == len(comp_flux): - unc = filtered_target_flux ** 0.5 - else: - sigf1 = filtered_target_flux ** 0.5 - sigf2 = filtered_comp_flux ** 0.5 - unc = np.sqrt((sigf1 / filtered_comp_flux) ** 2 + (sigf2 * filtered_target_flux / filtered_comp_flux ** 2) ** 2) + filtered_target_flux_error = None if target_flux_error_sorted is None else target_flux_error_sorted[valid_mask] + filtered_comp_flux_error = None if comp_flux_error_sorted is None else comp_flux_error_sorted[valid_mask] + unc = relative_flux_uncertainty_from_star_errors( + filtered_target_flux, + filtered_comp_flux, + filtered_target_flux_error, + filtered_comp_flux_error, + ) fit_times = times_sorted[valid_mask] fit_jd_times = jd_times_sorted[valid_mask] fit_airmass = airmass_sorted[valid_mask] @@ -18999,6 +19915,9 @@ def prepare_lightcurve_fit_input_series( 'debug_target_flux': debug_target_flux, 'debug_comp_flux': debug_comp_flux, 'debug_raw_ratio': debug_raw_ratio, + 'debug_target_flux_error': debug_target_flux_error, + 'debug_comp_flux_error': debug_comp_flux_error, + 'debug_relative_flux_error': debug_relative_flux_error, 'initial_sigma_keep_mask': initial_sigma_keep_mask, 'prefit_raw_ratio_keep_mask': prefit_raw_ratio_keep_mask, }) @@ -19016,6 +19935,9 @@ def prepare_lightcurve_fit_input_series( 'debug_target_flux': debug_target_flux, 'debug_comp_flux': debug_comp_flux, 'debug_raw_ratio': debug_raw_ratio, + 'debug_target_flux_error': debug_target_flux_error, + 'debug_comp_flux_error': debug_comp_flux_error, + 'debug_relative_flux_error': debug_relative_flux_error, 'initial_sigma_keep_mask': initial_sigma_keep_mask, 'prefit_raw_ratio_keep_mask': prefit_raw_ratio_keep_mask, 'time': fit_times[~nanmask], @@ -19025,6 +19947,10 @@ def prepare_lightcurve_fit_input_series( 'airmass': fit_airmass[~nanmask], 'target_flux': filtered_target_flux[~nanmask], 'comp_flux': filtered_comp_flux[~nanmask], + 'target_flux_error': np.full(filtered_target_flux.shape, np.nan, dtype=float)[~nanmask] + if filtered_target_flux_error is None else filtered_target_flux_error[~nanmask], + 'comp_flux_error': np.full(filtered_comp_flux.shape, np.nan, dtype=float)[~nanmask] + if filtered_comp_flux_error is None else filtered_comp_flux_error[~nanmask], 'source_indices': source_indices[~nanmask], 'skip_airmass_fit': should_skip_airmass_fit(fit_airmass[~nanmask]), 'approximate_baseline_level': approximate_baseline_level, @@ -20859,7 +21785,8 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p skip_low_comparison_coverage_rejection=False, use_impactparameter_rather_than_inclination_to_fit=True, use_eebls_to_initialize_tmid_and_bounds=True, - psf_flux_data=None): + psf_flux_data=None, + psf_noise_data=None): if photometry_info.get('best_fit_lc') is None or not comp_stars: return [] @@ -20871,6 +21798,11 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p if use_psf_photometry: psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) target_flux = psf_flux_series_from_rows(psf_flux_data['target']) + target_flux_error = ( + np.asarray(psf_noise_data.get('target'), dtype=float) + if isinstance(psf_noise_data, dict) and 'target' in psf_noise_data + else None + ) comp_flux_map = { f"comp{comp_index + 1}": psf_flux_series_from_rows( psf_flux_data[f"comp{comp_index + 1}"], @@ -20883,12 +21815,30 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p ) for comp_index in range(len(comp_stars)) } + comp_error_map = { + f"comp{comp_index + 1}": mask_series_with_quality( + psf_noise_data[f"comp{comp_index + 1}"], + psf_quality_mask_for_key( + psf_data, + f"comp{comp_index + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ), + ) + for comp_index in range(len(comp_stars)) + if isinstance(psf_noise_data, dict) and f"comp{comp_index + 1}" in psf_noise_data + } else: aperture_index = photometry_info.get('aperture_index') annulus_index = photometry_info.get('annulus_index') if aperture_index is None or annulus_index is None: return [] target_flux = np.asarray(aper_data['target'][:, aperture_index, annulus_index], dtype=float) + target_flux_error = ( + np.asarray(aper_data['target_unc'][:, aperture_index, annulus_index], dtype=float) + if 'target_unc' in aper_data + else None + ) comp_flux_map = { f"comp{comp_index + 1}": mask_series_with_quality( aper_data[f"comp{comp_index + 1}"][:, aperture_index, annulus_index], @@ -20896,6 +21846,14 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p ) for comp_index in range(len(comp_stars)) } + comp_error_map = { + f"comp{comp_index + 1}": mask_series_with_quality( + aper_data[f"comp{comp_index + 1}_unc"][:, aperture_index, annulus_index], + psf_quality_mask_for_key(psf_data, f"comp{comp_index + 1}", frame_count), + ) + for comp_index in range(len(comp_stars)) + if f"comp{comp_index + 1}_unc" in aper_data + } candidate_fit_summaries = [] selected_comp_star_num = photometry_info.get('comp_star_num') @@ -20920,9 +21878,17 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p if target_shape_mask.shape[0] != frame_count: target_shape_mask = np.ones(frame_count, dtype=bool) candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None + if target_flux_error is None + else mask_series_with_quality(target_flux_error, target_shape_mask) + ) + candidate_comp_flux_error = comp_error_map.get(ckey) fit_mask = target_shape_mask & robust_target_reference_flux_mask(candidate_target_flux, comp_flux_series) else: candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error + candidate_comp_flux_error = comp_error_map.get(ckey) fit_mask = valid_comparison_frame_mask(candidate_target_flux) & valid_comparison_frame_mask(comp_flux_series) coverage_count = coverage_summary[ckey]['coverage_count'] coverage_total_frame_count = coverage_summary[ckey]['coverage_total_frame_count'] @@ -20953,6 +21919,8 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p candidate_target_flux[fit_mask], comp_flux_series[fit_mask], airmass[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if candidate_comp_flux_error is None else candidate_comp_flux_error[fit_mask], enforce_relative_flux_max=False, expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) @@ -20965,6 +21933,8 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p ld, p_dict, jd_times[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if candidate_comp_flux_error is None else candidate_comp_flux_error[fit_mask], allow_mid_transit_range_warning=False, disable_vertical_flux_normalization=disable_vertical_flux_normalization, final_fit_mode='ns', @@ -21087,6 +22057,50 @@ def build_absolute_comp_ensemble_flux(comp_flux_map, active_keys, return ensemble * scale, member_keys +def build_absolute_comp_ensemble_uncertainty(comp_flux_map, comp_error_map, member_keys, + validity_mask_func=valid_comparison_frame_mask): + if not member_keys or not isinstance(comp_error_map, dict): + return None + + normalized_variances = [] + member_medians = [] + for key in member_keys: + if key not in comp_flux_map or key not in comp_error_map: + continue + flux_values = np.asarray(comp_flux_map[key], dtype=float) + error_values = np.asarray(comp_error_map[key], dtype=float) + if flux_values.shape != error_values.shape: + continue + valid_flux_mask = validity_mask_func(flux_values) + if np.count_nonzero(valid_flux_mask) < 5: + continue + member_median = float(bn.nanmedian(flux_values[valid_flux_mask])) + if not np.isfinite(member_median) or member_median <= 0: + continue + valid_error = valid_flux_mask & np.isfinite(error_values) & (error_values > 0) + normalized_variance = np.full(flux_values.shape, np.nan, dtype=float) + normalized_variance[valid_error] = (error_values[valid_error] / member_median) ** 2 + normalized_variances.append(normalized_variance) + member_medians.append(member_median) + + if not normalized_variances: + return None + + variance_stack = np.vstack(normalized_variances) + valid_count = np.count_nonzero(np.isfinite(variance_stack), axis=0) + summed_variance = np.nansum(variance_stack, axis=0) + ensemble_variance = np.full(variance_stack.shape[1], np.nan, dtype=float) + valid_frames = valid_count > 0 + ensemble_variance[valid_frames] = summed_variance[valid_frames] / (valid_count[valid_frames] ** 2) + scale = float(np.nanmedian(member_medians)) + if not np.isfinite(scale) or scale <= 0: + scale = 1.0 + ensemble_unc = np.full(variance_stack.shape[1], np.nan, dtype=float) + finite_var = np.isfinite(ensemble_variance) & (ensemble_variance >= 0) + ensemble_unc[finite_var] = np.sqrt(ensemble_variance[finite_var]) * scale + return ensemble_unc + + def comparison_star_coverage_summary(comp_flux_map, min_fraction=COMPARISON_STAR_MIN_COVERAGE_FRACTION, min_points=COMPARISON_STAR_MIN_VALID_FRAMES, @@ -21716,10 +22730,24 @@ def build_stability_iteration(active_keys, frame_keep_mask=None): } +def comparison_field_sort_value(value, zero_tolerance=1.0e-12): + if value is None: + return np.inf + try: + numeric_value = float(value) + except (TypeError, ValueError): + return np.inf + if not np.isfinite(numeric_value): + return np.inf + if abs(numeric_value) <= zero_tolerance: + return 0.0 + return numeric_value + + def comparison_field_sort_key(summary): return ( - np.inf if summary.get('field_score') is None else summary['field_score'], - np.inf if summary.get('best_comp_score') is None else summary['best_comp_score'], + comparison_field_sort_value(summary.get('field_score')), + comparison_field_sort_value(summary.get('best_comp_score')), ) @@ -21728,23 +22756,31 @@ def initialize_aperture_data_store(frame_count, aperture_count, annulus_count, c aper_data = { 'target': np.full(aper_shape, np.nan, dtype=float), 'target_bg': np.full(aper_shape, np.nan, dtype=float), + 'target_unc': np.full(aper_shape, np.nan, dtype=float), } + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"target_noise_{component}"] = np.full(aper_shape, np.nan, dtype=float) for comp_idx in range(comp_star_count): ckey = f"comp{comp_idx + 1}" aper_data[ckey] = np.full(aper_shape, np.nan, dtype=float) aper_data[f"{ckey}_bg"] = np.full(aper_shape, np.nan, dtype=float) + aper_data[f"{ckey}_unc"] = np.full(aper_shape, np.nan, dtype=float) + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"{ckey}_noise_{component}"] = np.full(aper_shape, np.nan, dtype=float) return aper_data def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=False, sigma_hint=np.nan, - aperture_correction_factors=None): + aperture_correction_factors=None, noise_config=None, exposure_s=np.nan, + airmass=np.nan, return_noise=False): flux_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) bg_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) + noise_grids = empty_noise_budget_grids(flux_grid.shape) if return_noise else None if np.isnan(xc) or np.isnan(yc): - return flux_grid, bg_grid + return (flux_grid, bg_grid, noise_grids) if return_noise else (flux_grid, bg_grid) mask_method = 'center' if fast_mode else 'exact' @@ -21768,7 +22804,7 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast annulus_width=float(annulus_width), psf_sigma=sigma_hint, ) - bgflux, _, _ = skybg_phot( + bgflux, sigmabg, n_sky = skybg_phot( data, star_index, xc, @@ -21779,6 +22815,8 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast ) else: bgflux = 0 + sigmabg = 0 + n_sky = 0 bg_grid[a_idx, an_idx] = bgflux @@ -21786,6 +22824,18 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast flux_grid[a_idx, an_idx] = 0 else: flux_grid[a_idx, an_idx] = raw_aperture_sum - bgflux * mask_area + if return_noise and noise_grids is not None: + budget = compute_photometry_noise_budget( + flux_grid[a_idx, an_idx], + sigmabg, + mask_area, + n_sky, + exposure_s=exposure_s, + airmass=airmass, + noise_config=noise_config, + ) + for component, grid in noise_grids.items(): + grid[a_idx, an_idx] = budget.get(component, np.nan) finally: _record_photometry_stage_timing('aperPhot', perf_counter() - stage_start) @@ -21795,13 +22845,17 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast valid_factors = np.isfinite(factors) & (factors > 0) if np.any(valid_factors): flux_grid[valid_factors, :] *= factors[valid_factors, None] + if return_noise and noise_grids is not None: + for grid in noise_grids.values(): + grid[valid_factors, :] *= factors[valid_factors, None] - return flux_grid, bg_grid + return (flux_grid, bg_grid, noise_grids) if return_noise else (flux_grid, bg_grid) def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_star_count, aper_data, apertures, annuli, fast_aperture_mask, adaptive_apertures=False, fallback_sigma=np.nan, - use_aperture_corrections_and_full_image_fwhm=False): + use_aperture_corrections_and_full_image_fwhm=False, + noise_config=None, exposure_s=np.nan, airmass=np.nan): target_sigma = psf_sigma_from_fit(psf_data['target'][frame_index], fallback_sigma=fallback_sigma) target_fwhm = psf_fwhm_from_sigma(target_sigma) field_star_psfs = np.empty((0, 7), dtype=float) @@ -21839,7 +22893,7 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta ) aperture_correction_factors = aperture_correction.get('correction_factors') - target_flux, target_bg = compute_star_aperture_grid( + target_flux, target_bg, target_noise = compute_star_aperture_grid( image_data, 0, psf_data['target'][frame_index, 0], @@ -21849,14 +22903,21 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta fast_mode=fast_aperture_mask, sigma_hint=frame_sigma, aperture_correction_factors=aperture_correction_factors, + noise_config=noise_config, + exposure_s=exposure_s, + airmass=airmass, + return_noise=True, ) aper_data['target'][frame_index] = target_flux aper_data['target_bg'][frame_index] = target_bg + aper_data['target_unc'][frame_index] = target_noise['total'] + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"target_noise_{component}"][frame_index] = target_noise[component] for comp_idx in range(comp_star_count): ckey = f"comp{comp_idx + 1}" comp_sigma = psf_sigma_from_fit(psf_data[ckey][frame_index], fallback_sigma=frame_sigma) - comp_flux, comp_bg = compute_star_aperture_grid( + comp_flux, comp_bg, comp_noise = compute_star_aperture_grid( image_data, comp_idx + 1, psf_data[ckey][frame_index, 0], @@ -21866,9 +22927,16 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta fast_mode=fast_aperture_mask, sigma_hint=comp_sigma, aperture_correction_factors=aperture_correction_factors, + noise_config=noise_config, + exposure_s=exposure_s, + airmass=airmass, + return_noise=True, ) aper_data[ckey][frame_index] = comp_flux aper_data[f"{ckey}_bg"][frame_index] = comp_bg + aper_data[f"{ckey}_unc"][frame_index] = comp_noise['total'] + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"{ckey}_noise_{component}"][frame_index] = comp_noise[component] return aperture_correction @@ -21906,6 +22974,7 @@ def auto_tune_aperture_sigma_grid(coarse_apertures_sigma, coarse_annuli_sigma, c psf_quality_masks=None): best_candidate = None best_score = np.inf + best_sort_key = (np.inf, np.inf) for a_idx, aperture_sigma in enumerate(coarse_apertures_sigma): for an_idx, annulus_sigma in enumerate(coarse_annuli_sigma): @@ -21925,8 +22994,10 @@ def auto_tune_aperture_sigma_grid(coarse_apertures_sigma, coarse_annuli_sigma, c skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, ) field_score = field_summary['field_score'] - if np.isfinite(field_score) and comparison_field_sort_key(field_summary) < (best_score, np.inf): + candidate_sort_key = comparison_field_sort_key(field_summary) + if np.isfinite(field_score) and candidate_sort_key < best_sort_key: best_score = field_score + best_sort_key = candidate_sort_key best_candidate = { 'aper_sigma': float(aperture_sigma), 'annulus_sigma': float(annulus_sigma), @@ -22109,6 +23180,7 @@ def ranked_comparison_calibration_summaries(comparison_calibration): def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p_dict, comparison_calibration, psf_data, aper_data, target_psf_flux, psf_flux_data=None, + psf_noise_data=None, plot_time_range=None, disable_vertical_flux_normalization=False, detrend_on_outoftransit_baseline=True, @@ -22150,8 +23222,18 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p if method == 'psf': target_flux = np.asarray(target_psf_flux, dtype=float) psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + target_flux_error = ( + np.asarray(psf_noise_data.get('target'), dtype=float) + if isinstance(psf_noise_data, dict) and 'target' in psf_noise_data + else None + ) else: target_flux = np.asarray(aper_data['target'][:, aperture_index, annulus_index], dtype=float) + target_flux_error = ( + np.asarray(aper_data['target_unc'][:, aperture_index, annulus_index], dtype=float) + if isinstance(aper_data, dict) and 'target_unc' in aper_data + else None + ) adaptive_summary = build_comparison_candidate_adaptive_summary( comparison_calibration, @@ -22205,15 +23287,48 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p for summary in ranked_summaries if summary.get('key') in psf_flux_data } + comp_error_map = { + summary['key']: mask_series_with_quality( + psf_noise_data[summary['key']], + np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + summary['key'], + frame_count, + psf_flux_data=psf_flux_data, + ), + ), + dtype=bool, + ), + ) + for summary in ranked_summaries + if ( + isinstance(psf_noise_data, dict) + and summary.get('key') in psf_noise_data + ) + } ensemble_flux, member_keys = build_absolute_comp_ensemble_flux( comp_flux_map, active_keys, validity_mask_func=robust_flux_floor_mask, ) + ensemble_flux_error = build_absolute_comp_ensemble_uncertainty( + comp_flux_map, + comp_error_map, + member_keys, + validity_mask_func=robust_flux_floor_mask, + ) target_shape_mask = target_psf_shape_quality_mask(target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data)) if target_shape_mask.shape[0] != frame_count: target_shape_mask = np.ones(frame_count, dtype=bool) candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None + if target_flux_error is None + else mask_series_with_quality(target_flux_error, target_shape_mask) + ) else: comp_flux_map = { summary['key']: mask_series_with_quality( @@ -22229,13 +23344,34 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p for summary in ranked_summaries if summary.get('key') in aper_data } + comp_error_map = { + summary['key']: mask_series_with_quality( + aper_data[f"{summary['key']}_unc"][:, aperture_index, annulus_index], + np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key(psf_data, summary['key'], frame_count), + ), + dtype=bool, + ), + ) + for summary in ranked_summaries + if summary.get('key') in aper_data and f"{summary['key']}_unc" in aper_data + } ensemble_flux, member_keys = build_absolute_comp_ensemble_flux( comp_flux_map, active_keys, validity_mask_func=valid_comparison_frame_mask, ) + ensemble_flux_error = build_absolute_comp_ensemble_uncertainty( + comp_flux_map, + comp_error_map, + member_keys, + validity_mask_func=valid_comparison_frame_mask, + ) target_shape_mask = np.ones(frame_count, dtype=bool) candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error if ensemble_flux is not None and member_keys: candidate_frame_keep_mask = np.ones(times.shape[0], dtype=bool) @@ -22261,6 +23397,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p candidate_target_flux[fit_mask], ensemble_flux[fit_mask], airmass[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if ensemble_flux_error is None else ensemble_flux_error[fit_mask], enforce_relative_flux_max=False, expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) @@ -22272,6 +23410,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p p_dict, candidate_target_flux[fit_mask], ensemble_flux[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if ensemble_flux_error is None else ensemble_flux_error[fit_mask], adaptive_summary=adaptive_summary, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, ) @@ -22296,6 +23436,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'ckey': None, 'target_flux': candidate_target_flux, 'comp_flux': ensemble_flux, + 'target_flux_error': candidate_target_flux_error, + 'comp_flux_error': ensemble_flux_error, 'fit_mask': fit_mask, 'candidate_frame_clip_diagnostic': None, 'fit_diagnostics': fit_diagnostics, @@ -22348,11 +23490,30 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p if target_shape_mask.shape[0] != frame_count: target_shape_mask = np.ones(frame_count, dtype=bool) candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None + if target_flux_error is None + else mask_series_with_quality(target_flux_error, target_shape_mask) + ) comp_flux = psf_flux_series_from_rows(psf_flux_data[ckey], comp_quality_mask) + comp_flux_error = ( + mask_series_with_quality(psf_noise_data[ckey], comp_quality_mask) + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data + else None + ) else: target_shape_mask = np.ones(frame_count, dtype=bool) candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error comp_flux = mask_series_with_quality(aper_data[ckey][:, aperture_index, annulus_index], comp_quality_mask) + comp_flux_error = ( + mask_series_with_quality( + aper_data[f"{ckey}_unc"][:, aperture_index, annulus_index], + comp_quality_mask, + ) + if f"{ckey}_unc" in aper_data + else None + ) candidate_frame_keep_mask = np.asarray( comp_summary.get('ensemble_frame_keep_mask', np.ones(times.shape[0], dtype=bool)), @@ -22390,6 +23551,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p candidate_target_flux[fit_mask], comp_flux[fit_mask], airmass[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], enforce_relative_flux_max=False, expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) @@ -22401,6 +23564,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p p_dict, candidate_target_flux[fit_mask], comp_flux[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], adaptive_summary=adaptive_summary, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, ) @@ -22410,6 +23575,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'ckey': ckey, 'target_flux': candidate_target_flux, 'comp_flux': comp_flux, + 'target_flux_error': candidate_target_flux_error, + 'comp_flux_error': comp_flux_error, 'fit_mask': fit_mask, 'candidate_frame_clip_diagnostic': candidate_frame_clip_diagnostic, 'fit_diagnostics': fit_diagnostics, @@ -22428,6 +23595,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ckey = plan['ckey'] candidate_target_flux = plan.get('target_flux', target_flux) comp_flux = plan['comp_flux'] + candidate_target_flux_error = plan.get('target_flux_error') + comp_flux_error = plan.get('comp_flux_error') fit_mask = plan['fit_mask'] candidate_frame_clip_diagnostic = plan.get('candidate_frame_clip_diagnostic') fit_diagnostics = plan['fit_diagnostics'] @@ -22451,6 +23620,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ld, p_dict, jd_times=jd_times[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], disable_vertical_flux_normalization=disable_vertical_flux_normalization, detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, use_impactparameter_rather_than_inclination_to_fit= @@ -22467,6 +23638,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p fit_result = final_reduction.get('fit') if final_reduction.get('applied') else None tflux_fit = final_reduction.get('good_target_flux') cflux_fit = final_reduction.get('good_comp_flux') + tflux_fit_error = final_reduction.get('good_target_flux_error') + cflux_fit_error = final_reduction.get('good_comp_flux_error') fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, fit_result, @@ -22553,8 +23726,12 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'good_unc': final_reduction.get('good_unc'), 'good_airmass': final_reduction.get('good_airmass'), 'good_jd_times': final_reduction.get('good_jd_times'), + 'good_target_flux_error': tflux_fit_error, + 'good_comp_flux_error': cflux_fit_error, 'tflux_fit': tflux_fit, 'cflux_fit': cflux_fit, + 'tflux_fit_error': tflux_fit_error, + 'cflux_fit_error': cflux_fit_error, 'source_indices': final_reduction.get('source_indices'), 'duration_samples': final_reduction.get('duration_samples'), 'data_highres': final_reduction.get('data_highres'), @@ -23406,7 +24583,7 @@ def _main_impl(): userpDict['ra'] = pDict['ra'] userpDict['dec'] = pDict['dec'] # time sort images - times, jd_times = [], [] + times, jd_times, header_exptimes = [], [], [] log_info(f"Reading FITS timestamps and converting to BJD_TDB for {len(inputfiles)} frame(s).") for file_index, file in enumerate(inputfiles): extension = 0 @@ -23419,6 +24596,7 @@ def _main_impl(): times.append(obsTime) plateStatus.setObsTime(obsTime) jd_times.append(img_time_jd(header)) + header_exptimes.append(get_exp_time(header)) completed = file_index + 1 if completed == len(inputfiles) or completed % 25 == 0: log_info(f"Timestamp conversion progress: {completed}/{len(inputfiles)}") @@ -23455,6 +24633,7 @@ def _main_impl(): si = np.argsort(times) times = np.array(times)[si] jd_times = np.array(jd_times)[si] + header_exptimes = np.array(header_exptimes, dtype=float)[si] inputfiles = np.array(inputfiles)[si] precheck_inputfile_count = int(len(inputfiles)) finite_plot_times = times[np.isfinite(times)] @@ -23482,6 +24661,7 @@ def _main_impl(): if dropped_wcs_files: times = times[wcs_keep_mask] jd_times = jd_times[wcs_keep_mask] + header_exptimes = header_exptimes[wcs_keep_mask] plateStatus.initializeFilenames(list(inputfiles)) post_wcs_inputfile_count = int(len(inputfiles)) target_wcs_precheck_inputfiles = np.array(inputfiles, copy=True) @@ -23501,6 +24681,7 @@ def _main_impl(): ) times = times[target_wcs_keep_mask] jd_times = jd_times[target_wcs_keep_mask] + header_exptimes = header_exptimes[target_wcs_keep_mask] finite_plot_times = times[np.isfinite(times)] full_plot_time_range = None if finite_plot_times.size: @@ -23549,6 +24730,7 @@ def _main_impl(): reference_fallback = pointing_reference_fallback or target_reference_fallback times = times[pointing_keep_mask] jd_times = jd_times[pointing_keep_mask] + header_exptimes = header_exptimes[pointing_keep_mask] finite_plot_times = times[np.isfinite(times)] full_plot_time_range = None if finite_plot_times.size: @@ -23639,6 +24821,7 @@ def _main_impl(): inputfiles = inputfiles[inc:] times = times[inc:] jd_times = jd_times[inc:] + header_exptimes = header_exptimes[inc:] pointing_alignment_transforms = {} plateStatus.setCurrentFilename(inputfiles[0]) header = get_first_image_header(inputfiles[0]) @@ -23768,6 +24951,7 @@ def _main_impl(): obs_filter=exotic_infoDict['filter'], field_catalog=nextastro_field_catalog, count=automatic_comp_count, + colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), ) log_automatic_optimal_calibration_selection( automatic_comp_stars, @@ -23832,7 +25016,12 @@ def _main_impl(): tar_comp_dist = {} vsp_num = [] comp_star_count = len(exotic_infoDict['comp_stars']) - require_comp_star = is_comp_star_required(exotic_infoDict.get('require_comp_star', 'y')) + psf_noise_data = initialize_psf_noise_data(len(inputfiles), comp_star_count) + frame_noise_configs = [] + require_comp_star = resolve_require_comp_star_for_exposure_times( + exotic_infoDict.get('require_comp_star', 'y'), + header_exptimes, + ) target_driven_comp_selection = is_target_driven_comp_selection_enabled( exotic_infoDict.get('target_driven_comp_selection', 'n') ) @@ -24026,6 +25215,15 @@ def _main_impl(): exotic_infoDict['elev'], jd_times[i])) exptimes.append(get_exp_time(image_header)) + frame_noise_config = noise_budget_config_from_info(exotic_infoDict, image_header) + frame_noise_configs.append(frame_noise_config) + if i == 0: + log_info( + "Photometry noise budget terms: " + f"{format_noise_budget_config_summary(frame_noise_config)}." + ) + frame_airmass = airMassList[-1] + frame_exposure_s = exptimes[-1] # IMAGES imageData = hdul[extension].data @@ -24140,6 +25338,21 @@ def _main_impl(): target_psf_flux_seed_row, 0, ) + store_psf_noise_budget( + psf_noise_data, + 'target', + i, + compute_psf_noise_budget_for_row( + imageData, + psf_flux_data['target'][i], + 0, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + fallback_sigma=sigma, + fast_mode=fast_aperture_mask, + ), + ) for comp_idx, comp_key in enumerate(comp_alignment_keys): comp_psf_flux_seed_row = psf_data[comp_key][i] if comp_key in psf_flux_seed_tracks: @@ -24149,6 +25362,21 @@ def _main_impl(): comp_psf_flux_seed_row, comp_idx + 1, ) + store_psf_noise_budget( + psf_noise_data, + comp_key, + i, + compute_psf_noise_budget_for_row( + imageData, + psf_flux_data[comp_key][i], + comp_idx + 1, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + fallback_sigma=sigma, + fast_mode=fast_aperture_mask, + ), + ) # aperture photometry if use_aperture_photometry and i == 0: @@ -24185,6 +25413,9 @@ def _main_impl(): adaptive_apertures=use_adaptive_apertures, fallback_sigma=sigma, use_aperture_corrections_and_full_image_fwhm=use_aperture_corrections_and_full_image_fwhm, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, ) if i == coarse_tune_frames - 1: @@ -24262,6 +25493,9 @@ def _main_impl(): use_aperture_corrections_and_full_image_fwhm=( use_aperture_corrections_and_full_image_fwhm ), + noise_config=frame_noise_configs[backfill_idx], + exposure_s=exptimes[backfill_idx], + airmass=airMassList[backfill_idx], ) finally: if loaded_from_disk: @@ -24293,6 +25527,9 @@ def _main_impl(): adaptive_apertures=use_adaptive_apertures, fallback_sigma=sigma, use_aperture_corrections_and_full_image_fwhm=use_aperture_corrections_and_full_image_fwhm, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, ) # close file + delete from memory @@ -24329,16 +25566,15 @@ def _main_impl(): airmass = np.array(airMassList)[goodmask] psf_data["target"] = psf_data["target"][goodmask] psf_flux_data["target"] = psf_flux_data["target"][goodmask] + for key in list(psf_noise_data.keys()): + psf_noise_data[key] = psf_noise_data[key][goodmask] if aper_data is not None: - aper_data["target"] = aper_data["target"][goodmask] - aper_data["target_bg"] = aper_data["target_bg"][goodmask] + for key in list(aper_data.keys()): + aper_data[key] = aper_data[key][goodmask] for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" psf_data[ckey] = psf_data[ckey][goodmask] psf_flux_data[ckey] = psf_flux_data[ckey][goodmask] - if aper_data is not None: - aper_data[ckey] = aper_data[ckey][goodmask] - aper_data[f"{ckey}_bg"] = aper_data[f"{ckey}_bg"][goodmask] psf_quality_diagnostics = [] if use_psf_photometry: @@ -24476,6 +25712,14 @@ def _main_impl(): 'comparison_ktmf_metric': np.nan, 'comparison_eebls_snr': np.nan, 'comparison_transit_delta_bic': np.nan, + 'noise_budget_summary': ( + format_noise_budget_config_summary(frame_noise_configs[0]) + if frame_noise_configs else None + ), + 'noise_budget_terms': ( + list(frame_noise_configs[0].get('enabled_terms', ())) + if frame_noise_configs else [] + ), } comparison_calibration = None @@ -24604,6 +25848,7 @@ def _main_impl(): aper_data, tFlux, psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, plot_time_range=full_plot_time_range, disable_vertical_flux_normalization=disable_vertical_flux_normalization, detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, @@ -24661,6 +25906,12 @@ def _main_impl(): myfit = selected_attempt['fit'] tFlux1 = selected_attempt['tflux_fit'] cFlux1 = selected_attempt['cflux_fit'] + tFlux1_error = selected_attempt.get('tflux_fit_error') + cFlux1_error = selected_attempt.get('cflux_fit_error') + if tFlux1_error is None or np.shape(tFlux1_error) != np.shape(tFlux1): + tFlux1_error = tFlux1 ** 0.5 + if cFlux1_error is None or np.shape(cFlux1_error) != np.shape(cFlux1): + cFlux1_error = cFlux1 ** 0.5 selected_source_indices = np.asarray( selected_attempt.get('source_indices', np.arange(len(tFlux1), dtype=int)), dtype=int, @@ -24757,6 +26008,8 @@ def _main_impl(): selected_fit_good_flux=selected_attempt.get('good_flux'), selected_fit_good_unc=selected_attempt.get('good_unc'), selected_fit_good_airmass=selected_attempt.get('good_airmass'), + selected_fit_good_target_flux_error=selected_attempt.get('tflux_fit_error'), + selected_fit_good_comp_flux_error=selected_attempt.get('cflux_fit_error'), selected_fit_duration_samples=selected_attempt.get('duration_samples'), selected_fit_data_highres=selected_attempt.get('data_highres'), selected_fit_final_output_dir=selected_attempt.get('final_output_dir'), @@ -24778,7 +26031,7 @@ def _main_impl(): selected_comparison_transit_qc_summary=selected_attempt.get('transit_qc_summary')) flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, - flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) + flux_unc_tar=tFlux1_error, flux_unc_ref=cFlux1_error) ref_centroid_x = np.full(selected_source_indices.shape, np.nan, dtype=float) ref_centroid_y = np.full(selected_source_indices.shape, np.nan, dtype=float) @@ -24803,6 +26056,8 @@ def _main_impl(): cFlux = psf_flux_series_from_rows(psf_flux_source[ckey]) vsp_fit, _, _ = fit_lightcurve( times, tFlux, cFlux, airmass, ld, pDict, jd_times, + target_flux_error=psf_noise_data.get('target'), + comp_flux_error=psf_noise_data.get(ckey), disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, @@ -24817,13 +26072,27 @@ def _main_impl(): best_a = comparison_calibration['a'] best_an = comparison_calibration['an'] best_target_flux = aper_data['target'][:, best_a, best_an] + best_target_flux_error = ( + aper_data['target_unc'][:, best_a, best_an] + if 'target_unc' in aper_data + else None + ) for j in vsp_num: ckey = f"comp{j + 1}" aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] + cFlux_error = ( + aper_data[f"{ckey}_unc"][aper_mask][:, best_a, best_an] + if f"{ckey}_unc" in aper_data + else None + ) vsp_fit, _, _ = fit_lightcurve( times[aper_mask], best_target_flux[aper_mask], cFlux, airmass[aper_mask], ld, pDict, jd_times[aper_mask], + target_flux_error=( + None if best_target_flux_error is None else best_target_flux_error[aper_mask] + ), + comp_flux_error=cFlux_error, disable_vertical_flux_normalization=disable_vertical_flux_normalization, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, @@ -24978,6 +26247,7 @@ def _main_impl(): use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, ) saved_candidate_fit_count = sum(1 for summary in candidate_fit_summaries if summary['fit'] is not None) failed_candidate_fit_count = len(candidate_fit_summaries) - saved_candidate_fit_count diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index fdaed4bd..2fbecad5 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -432,6 +432,7 @@ def save_input(): "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", + "Photometry Noise Budget": "Optional noise terms for raw-image photometry: gain_electrons_per_adu, read_noise_electrons, dark_current_electrons_per_second_per_pixel, flat_field_fractional_error, telescope_aperture_m, and scintillation_coefficient. Leave null to ignore an optional term.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -1612,6 +1613,12 @@ def save_input(): "restrict_a/Rs_range_percentage": 10.0, "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, + "gain_electrons_per_adu": null, + "read_noise_electrons": null, + "dark_current_electrons_per_second_per_pixel": null, + "flat_field_fractional_error": null, + "telescope_aperture_m": null, + "scintillation_coefficient": null, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } @@ -1670,6 +1677,12 @@ def save_input(): "use_prior_Rp/Rs_when_posterior_pinned": "y", "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, + "gain_electrons_per_adu": null, + "read_noise_electrons": null, + "dark_current_electrons_per_second_per_pixel": null, + "flat_field_fractional_error": null, + "telescope_aperture_m": null, + "scintillation_coefficient": null, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } diff --git a/exotic/inputs.py b/exotic/inputs.py index 15e8add6..4bf3bfe7 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -226,6 +226,12 @@ def __init__(self, init_opt): 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, 'use_aperture_corrections_and_full_image_fwhm': False, 'pointing_rejection_sigma': None, + 'gain_electrons_per_adu': None, + 'read_noise_electrons': None, + 'dark_current_electrons_per_second_per_pixel': None, + 'flat_field_fractional_error': None, + 'telescope_aperture_m': None, + 'scintillation_coefficient': None, 'skip_low_comparison_coverage_rejection': 'n', 'fit_lightcurve_to_every_comparison_candidate': 'n', 'ultranest_min_num_live_points': 200, @@ -567,6 +573,48 @@ def comp_params(self, init_file, planet_dict): 'automatic_optimal_calibration_selector_max_stars', 'Automatic Optimal Calibration Selector Max Stars', ), + 'colour_term': ( + 'colour_term', + 'color_term', + 'COLTERM', + ), + 'colour_term_error': ( + 'colour_term_error', + 'color_term_error', + 'COLTERR', + ), + 'colour_term_index': ( + 'colour_term_index', + 'color_term_index', + 'COLTIDX', + ), + 'colour_term_bv': ( + 'colour_term_bv', + 'color_term_bv', + 'COLTBV', + ), + 'colour_term_bv_error': ( + 'colour_term_bv_error', + 'color_term_bv_error', + 'COLTBVER', + 'COLTBVERR', + ), + 'colour_term_bprp': ( + 'colour_term_bprp', + 'color_term_bprp', + 'COLTBPRP', + ), + 'colour_term_bprp_error': ( + 'colour_term_bprp_error', + 'color_term_bprp_error', + 'CBPRPERR', + 'COLTBPRPERR', + ), + 'colour_equation_filter': ( + 'colour_equation_filter', + 'color_equation_filter', + 'COLEQFIL', + ), 'use_ensemble_photometry_rather_than_single_comp': ( 'use_ensemble_photometry_rather_than_single_comp', 'Use Ensemble Photometry Rather Than Single Comp? (y/n)', @@ -646,6 +694,42 @@ def comp_params(self, init_file, planet_dict): 'pointing_rejection_sigma', 'Pointing Rejection Sigma', ), + 'gain_electrons_per_adu': ( + 'gain_electrons_per_adu', + 'gain_e_per_adu', + 'Gain (e-/ADU)', + 'CCD Gain (e-/ADU)', + ), + 'read_noise_electrons': ( + 'read_noise_electrons', + 'read_noise_e', + 'read_noise', + 'Read Noise (e-)', + 'CCD Read Noise (e-)', + ), + 'dark_current_electrons_per_second_per_pixel': ( + 'dark_current_electrons_per_second_per_pixel', + 'dark_current_e_per_s_pix', + 'dark_current', + 'Dark Current (e-/s/pix)', + ), + 'flat_field_fractional_error': ( + 'flat_field_fractional_error', + 'flat_field_fractional_noise', + 'flat_field_error_fraction', + 'Flat Field Fractional Error', + 'Flat-Field Fractional Error', + ), + 'telescope_aperture_m': ( + 'telescope_aperture_m', + 'telescope_aperture_meters', + 'Telescope Aperture (m)', + ), + 'scintillation_coefficient': ( + 'scintillation_coefficient', + 'scintillation_noise_coefficient', + 'Scintillation Coefficient', + ), 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (arsec/pixel)'), 'exposure': 'Exposure Time (s)', diff --git a/exotic/output_files.py b/exotic/output_files.py index ceb66ec7..f03f8b7c 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1349,6 +1349,8 @@ def build_aavso_photometry_metadata(photometry_info): 'method': photometry_method_from_info(photometry_info), 'selected_comparison_star': photometry_info.get('comp_star_num'), 'selected_comparison_coordinates': photometry_info.get('comp_star_coords'), + 'noise_budget_summary': photometry_info.get('noise_budget_summary'), + 'noise_budget_terms': photometry_info.get('noise_budget_terms'), 'comparison_selection_basis': photometry_info.get('selection_basis'), 'comparison_selection_metric': photometry_info.get('selection_metric'), 'comparison_field_score': photometry_info.get('calibration_field_score'), @@ -1790,6 +1792,12 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num.update(format_fit_quality_final_params(fit_quality)) params_num.update(format_empirical_transit_uncertainty_final_params(empirical_uncertainty)) params_num.update(format_ktmf_decision_final_params(self.fit, photometry_info)) + if isinstance(photometry_info, dict) and photometry_info.get('noise_budget_summary'): + params_num["Photometry noise budget"] = str(photometry_info.get('noise_budget_summary')) + if photometry_info.get('noise_budget_terms'): + params_num["Photometry noise budget terms"] = ", ".join( + str(term) for term in photometry_info.get('noise_budget_terms') + ) if getattr(self.fit, 'airmass_fit_skipped', False): params_num["Airmass correction"] = getattr( self.fit, diff --git a/inits.json b/inits.json index a004a7f9..b32d8ea0 100644 --- a/inits.json +++ b/inits.json @@ -134,6 +134,12 @@ "use_aperture_photometry": "y", "use_adaptive_apertures": false, "use_aperture_corrections_and_full_image_fwhm": false, + "gain_electrons_per_adu": null, + "read_noise_electrons": null, + "dark_current_electrons_per_second_per_pixel": null, + "flat_field_fractional_error": null, + "telescope_aperture_m": null, + "scintillation_coefficient": null, "skip_low_comparison_coverage_rejection": "n", "fit_lightcurve_to_every_comparison_candidate": "n", "Use target-driven comp selection rather than comp-driven comp selection": "n", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 5ecdce3d..f4ac215b 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -1173,12 +1173,23 @@ def fake_color_match(_catalog, ra, dec, obs_filter, max_separation_arcsec=5.0): obs_filter="V", field_catalog=catalog, count=2, + colour_term_metadata={ + "term": 0.2, + "term_error": 0.01, + "term_index": "B-V", + }, ) assert comp_stars[0] == [220.0, 220.0] assert [candidate["color_delta"] for candidate in candidates] == sorted( candidate["color_delta"] for candidate in candidates ) + assert candidates[0]["expected_colour_mismatch_mag"] == pytest.approx( + 0.2 * candidates[0]["color_delta"] + ) + assert candidates[0]["colour_term_uncertainty_mag"] == pytest.approx( + 0.01 * candidates[0]["color_delta"] + ) assert all(0.5 <= candidate["brightness_ratio"] <= 2.0 for candidate in candidates) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index bd9bda1d..22d0c572 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -120,6 +120,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: comparison_candidate_fit_selection_reason, comparison_star_coverage_summary, comparison_star_stability_summary, + compute_photometry_noise_budget, configure_windows_multiprocessing_main_spec, deduplicate_comparison_star_coords, diagnose_lightcurve_fit_inputs, @@ -145,6 +146,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: log_comparison_calibration_fit_attempt_summaries, log_comparison_candidate_fit_summaries, log_target_fit_candidate_summaries, + noise_budget_config_from_info, normalize_flux_series_to_approximate_unity, phase_bin_sigma_clip, parse_deviation_from_expected_transit_in_qc_sigma, @@ -466,10 +468,11 @@ def test_save_selected_photometry_debug_series_writes_stage_masks(tmp_path): assert output_path.exists() rows = np.loadtxt(output_path, delimiter=",", skiprows=1) - assert rows.shape == (3, 7) - assert rows[:, 4].astype(int).tolist() == [1, 0, 1] - assert rows[:, 5].astype(int).tolist() == [1, 1, 1] - assert rows[:, 6].astype(int).tolist() == [1, 0, 0] + assert rows.shape == (3, 10) + assert np.isnan(rows[:, 4:7]).all() + assert rows[:, 7].astype(int).tolist() == [1, 0, 1] + assert rows[:, 8].astype(int).tolist() == [1, 1, 1] + assert rows[:, 9].astype(int).tolist() == [1, 0, 0] def test_finalize_comparison_candidate_phase_clips_before_nested_fit(monkeypatch): @@ -1173,6 +1176,64 @@ def test_is_comp_star_required_parses_values(): assert is_comp_star_required("n") is False +def test_mixed_exposure_times_force_comparison_star_requirement(): + import exotic.exotic as exotic_module + + assert exotic_module.exposure_time_spread_fraction([60.0, 60.3, 60.5]) < 0.01 + assert not exotic_module.exposure_variation_requires_comp_star([60.0, 60.3, 60.5]) + assert exotic_module.resolve_require_comp_star_for_exposure_times("n", [60.0, 60.3, 60.5]) is False + + assert exotic_module.exposure_time_spread_fraction([60.0, 61.0]) > 0.01 + assert exotic_module.exposure_variation_requires_comp_star([60.0, 61.0]) + assert exotic_module.resolve_require_comp_star_for_exposure_times("n", [60.0, 61.0]) is True + + +def test_img_time_bjd_tdb_prefers_direct_mid_exposure_bjd(monkeypatch): + import exotic.exotic as exotic_module + + header = exotic_module.fits.Header() + header["BJD_TDB"] = 2461152.1287422837 + header["DATE-AVG"] = "2026-04-22T15:05:23.333333" + header["DATE-UTC"] = "2026-04-22T15:05:08.333333" + header["EXPTIME"] = 30.0 + + monkeypatch.setattr( + exotic_module, + "convert_jd_to_bjd", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("conversion should not run")), + ) + + assert exotic_module.img_time_bjd_tdb(header, {}, {}) == pytest.approx(2461152.1287422837) + + +def test_img_time_bjd_tdb_uses_nina_date_avg_before_start_time(monkeypatch): + import exotic.exotic as exotic_module + + header = exotic_module.fits.Header() + header["DATE-AVG"] = "2026-04-22T15:05:23.333333" + header["DATE-UTC"] = "2026-04-22T15:05:08.333333" + header["EXPTIME"] = 10.0 + + converted_inputs = [] + + def fake_convert_jd_to_bjd(values, _p_dict, _info_dict): + converted_inputs.extend(values) + return np.asarray(values, dtype=float) + 0.25 + + monkeypatch.setattr(exotic_module, "convert_jd_to_bjd", fake_convert_jd_to_bjd) + + expected_midpoint_jd = exotic_module.Time("2026-04-22T15:05:23.333333", scale="utc").jd + expected_start_plus_exposure_jd = ( + exotic_module.Time("2026-04-22T15:05:08.333333", scale="utc").jd + + 5.0 / 86400.0 + ) + + assert exotic_module.img_time_jd(header) == pytest.approx(expected_midpoint_jd) + assert exotic_module.img_time_bjd_tdb(header, {}, {}) == pytest.approx(expected_midpoint_jd + 0.25) + assert converted_inputs == pytest.approx([expected_midpoint_jd]) + assert converted_inputs[0] != pytest.approx(expected_start_plus_exposure_jd, abs=1e-9) + + def test_is_target_driven_comp_selection_enabled_parses_values(): assert is_target_driven_comp_selection_enabled(None) is False assert is_target_driven_comp_selection_enabled("y") is True @@ -5227,7 +5288,7 @@ def fake_finalize( assert call_markers == [50, 40, 30] assert result["stopped_after_first_qc_pass"] is False - assert result["selection_metric"] == "ktmf" + assert result["selection_metric"] == "ktmf_combined_quality" assert result["selected_result"]["comp_index"] == 2 assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.80) @@ -5678,7 +5739,7 @@ def fake_save(save_dir, provisional_fit, final_fit, p_dict, observation_date, co assert call_markers == [50, 40] assert len(result["attempts"]) == 2 - assert result["selection_metric"] == "ktmf" + assert result["selection_metric"] == "ktmf_combined_quality" assert result["selected_result"]["comp_index"] == 1 assert [attempt["final_output_dir"] for attempt in result["attempts"]] == [ str(tmp_path / "comp1"), @@ -5706,7 +5767,7 @@ def fake_finalize( residual_level = 0.01 eebls_snr = 4.0 else: - residual_level = 0.02 + residual_level = 0.012 eebls_snr = 7.5 residuals = residual_level * np.array([-1.0, 1.0, -1.0, 1.0, -1.0, 1.0], dtype=float) @@ -5768,8 +5829,8 @@ def fake_finalize( assert result["selection_metric"] == "eebls_snr" assert result["selected_result"]["comp_index"] == 1 assert result["selected_result"]["eebls_snr"] == pytest.approx(7.5) - assert "highest EEBLS SNR" in result["selected_result"]["selection_reason"] - assert result["attempts"][0]["selection_reason"].startswith("not selected: EEBLS SNR") + assert "highest selection-pass EEBLS SNR" in result["selected_result"]["selection_reason"] + assert result["attempts"][0]["selection_reason"].startswith("not selected: selection-pass EEBLS SNR") def test_fit_ranked_comparison_calibration_candidates_logs_per_comp_run_reporting(monkeypatch): @@ -6486,7 +6547,7 @@ def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwa ) assert [attempt["rejected_by_transit_qc"] for attempt in result["attempts"]] == [True, True] - assert result["selection_metric"] == "ktmf" + assert result["selection_metric"] == "ktmf_combined_quality" assert result["selected_result"]["comp_index"] == 1 assert result["selected_result"]["selected_despite_transit_qc"] is True assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.80) @@ -6625,6 +6686,97 @@ def test_prepare_lightcurve_fit_input_series_normalizes_ratio_around_unity(): assert np.nanmedian(prepared["flux"]) == pytest.approx(1.0) +def test_prepare_lightcurve_fit_input_series_uses_per_star_flux_errors(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + target_flux = np.full(6, 400.0) + comp_flux = np.full(6, 100.0) + target_error = np.full(6, 20.0) + comp_error = np.full(6, 5.0) + + prepared = prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + np.linspace(1.0, 1.5, 6), + target_flux_error=target_error, + comp_flux_error=comp_error, + ) + + propagated_relative_error = np.sqrt((20.0 / 100.0) ** 2 + (5.0 * 400.0 / 100.0 ** 2) ** 2) + assert prepared["applied"] is True + assert np.nanmedian(prepared["debug_relative_flux_error"]) == pytest.approx(propagated_relative_error) + assert np.nanmedian(prepared["unc"]) == pytest.approx(propagated_relative_error / 4.0) + assert np.allclose(prepared["target_flux_error"], target_error) + assert np.allclose(prepared["comp_flux_error"], comp_error) + + +def test_compute_photometry_noise_budget_includes_optional_terms(): + config = { + "gain_e_per_adu": 2.0, + "read_noise_electrons": 4.0, + "dark_current_electrons_per_second_per_pixel": 0.1, + "flat_field_fractional_error": 0.01, + "telescope_aperture_m": 0.3, + "scintillation_coefficient": 0.09, + "elevation_m": 100.0, + "enabled_terms": ( + "source", + "sky_aperture", + "sky_estimate", + "read", + "dark", + "flat", + "scintillation", + ), + } + + budget = compute_photometry_noise_budget( + 10000.0, + 3.0, + 50.0, + 200.0, + exposure_s=60.0, + airmass=1.2, + noise_config=config, + ) + + assert budget["source"] == pytest.approx(np.sqrt(10000.0 / 2.0)) + assert budget["read"] == pytest.approx(np.sqrt(50.0 * (4.0 / 2.0) ** 2)) + assert budget["dark"] == pytest.approx(np.sqrt(50.0 * 0.1 * 60.0 / 2.0 ** 2)) + assert budget["flat"] == pytest.approx(100.0) + assert budget["scintillation"] > 0 + assert budget["total"] > budget["flat"] + + +def test_noise_budget_config_reads_inits_and_header_values(): + header = { + "GAIN": 1.5, + "RDNOISE": 7.0, + "DARKCURR": 0.02, + "FLATERR": 0.003, + "APR-DIA": 250.0, + } + config = noise_budget_config_from_info( + { + "read_noise_electrons": 5.0, + }, + header=header, + ) + + assert config["gain_e_per_adu"] == pytest.approx(1.5) + assert config["read_noise_electrons"] == pytest.approx(5.0) + assert config["dark_current_electrons_per_second_per_pixel"] == pytest.approx(0.02) + assert config["flat_field_fractional_error"] == pytest.approx(0.003) + assert config["telescope_aperture_m"] == pytest.approx(0.25) + assert "read" in config["enabled_terms"] + assert "flat" in config["enabled_terms"] + + def test_prepare_lightcurve_fit_input_series_clips_prefit_raw_ratio_outliers(monkeypatch): monkeypatch.setattr( "exotic.exotic.sigma_clip", diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 22e869c3..c01e32fe 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -929,6 +929,64 @@ def test_comp_params_reads_aperture_corrections_and_full_image_fwhm_from_optiona assert inputs.info_dict["use_aperture_corrections_and_full_image_fwhm"] is True +def test_comp_params_reads_noise_budget_terms_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "gain_electrons_per_adu": 1.7, + "read_noise_electrons": 5.2, + "dark_current_electrons_per_second_per_pixel": 0.03, + "flat_field_fractional_error": 0.004, + "telescope_aperture_m": 0.28, + "scintillation_coefficient": 0.09, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["gain_electrons_per_adu"] == pytest.approx(1.7) + assert inputs.info_dict["read_noise_electrons"] == pytest.approx(5.2) + assert inputs.info_dict["dark_current_electrons_per_second_per_pixel"] == pytest.approx(0.03) + assert inputs.info_dict["flat_field_fractional_error"] == pytest.approx(0.004) + assert inputs.info_dict["telescope_aperture_m"] == pytest.approx(0.28) + assert inputs.info_dict["scintillation_coefficient"] == pytest.approx(0.09) + + +def test_comp_params_reads_colour_term_metadata_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "colour_term": -0.045, + "colour_term_error": 0.004, + "colour_term_index": "r-i", + "colour_term_bv": -0.031, + "colour_term_bv_error": 0.003, + "colour_term_bprp": -0.028, + "colour_term_bprp_error": 0.002, + "colour_equation_filter": "rp", + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["colour_term"] == pytest.approx(-0.045) + assert inputs.info_dict["colour_term_error"] == pytest.approx(0.004) + assert inputs.info_dict["colour_term_index"] == "r-i" + assert inputs.info_dict["colour_term_bv"] == pytest.approx(-0.031) + assert inputs.info_dict["colour_term_bv_error"] == pytest.approx(0.003) + assert inputs.info_dict["colour_term_bprp"] == pytest.approx(-0.028) + assert inputs.info_dict["colour_term_bprp_error"] == pytest.approx(0.002) + assert inputs.info_dict["colour_equation_filter"] == "rp" + + class DummyResponse: def __init__(self, payload): self._payload = payload From 918a5438fff8560a3739dfdde40053a53e46b76f Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Wed, 8 Jul 2026 12:51:58 +1000 Subject: [PATCH 076/116] transit smearing and no-comp-flux-scaling Rob seems to think have no comp is a good idea... but it isn't with different exposure times so I've dealt with that. Also some adjustments for transit smearing with long exposures vs ingress/egresss --- exotic/api/elca.py | 264 +++++++++++++++- exotic/exotic.py | 395 ++++++++++++++++++++++-- tests/test_exotic_proper_motion.py | 105 ++++++- tests/test_lazy_pylightcurve_imports.py | 84 +++++ 4 files changed, 807 insertions(+), 41 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 12a59929..f0fdb421 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -78,6 +78,13 @@ ) BASELINE_MODEL_UNCERTAINTY_KEYS = ('a0', 'a1', 'a2') MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT = 2000 +EXPOSURE_SMEARING_EXPOSURE_TIME_KEY = '_exposure_time_days' +EXPOSURE_SMEARING_SUPERSAMPLE_KEY = '_exposure_smearing_supersample' +EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY = '_exposure_smearing_change_tolerance' +DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE = 7 +DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE = 1.0e-5 +MIN_EXPOSURE_SMEARING_SECONDS = 1.0 +EXPOSURE_SMEARING_TRANSIT_WINDOW_PADDING_FACTOR = 2.0 ULTRANEST_INFLATED_ERROR_REPLACEMENT_FACTOR = 3.0 ULTRANEST_LOCAL_UNCERTAINTY_MAX_DELTA_CHI2 = 9.0 @@ -140,7 +147,7 @@ def gaussian_weights(X, w=1, neighbors=50, feature_scale=1000): return gw, nearest.astype(int) -def transit(times, values): +def _instantaneous_transit(times, values): model = pytransit([values['u0'], values['u1'], values['u2'], values['u3']], values['rprs'], values['per'], values['ars'], values['ecc'], values['inc'], values['omega'], @@ -148,6 +155,164 @@ def transit(times, values): return model +def _exposure_time_days_for_times(values, shape): + try: + exposure_time_days = values.get(EXPOSURE_SMEARING_EXPOSURE_TIME_KEY) + except AttributeError: + return None + if exposure_time_days is None: + return None + + try: + exposure_time_days = np.asarray(exposure_time_days, dtype=float) + except (TypeError, ValueError): + return None + + if exposure_time_days.shape == (): + exposure_time_days = np.full(shape, float(exposure_time_days), dtype=float) + elif exposure_time_days.shape != shape: + return None + else: + exposure_time_days = np.array(exposure_time_days, dtype=float, copy=True) + + minimum_exposure_days = MIN_EXPOSURE_SMEARING_SECONDS / 86400.0 + finite_positive = np.isfinite(exposure_time_days) & (exposure_time_days > minimum_exposure_days) + if not np.any(finite_positive): + return None + + exposure_time_days[~finite_positive] = 0.0 + return exposure_time_days + + +def _exposure_smearing_supersample(values): + try: + sample_count = int(values.get(EXPOSURE_SMEARING_SUPERSAMPLE_KEY, DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE)) + except (AttributeError, TypeError, ValueError): + sample_count = DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE + sample_count = max(3, sample_count) + if sample_count % 2 == 0: + sample_count += 1 + return sample_count + + +def _exposure_smearing_change_tolerance(values): + try: + tolerance = float(values.get( + EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY, + DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE, + )) + except (AttributeError, TypeError, ValueError): + tolerance = DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE + if not np.isfinite(tolerance) or tolerance < 0: + tolerance = DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE + return tolerance + + +def _exposure_smearing_candidate_mask(times, exposure_time_days, values): + candidate = ( + np.isfinite(times) + & np.isfinite(exposure_time_days) + & (exposure_time_days > MIN_EXPOSURE_SMEARING_SECONDS / 86400.0) + ) + if not np.any(candidate): + return candidate + + try: + period = float(values['per']) + tmid = float(values['tmid']) + except (KeyError, TypeError, ValueError): + return candidate + if not np.isfinite(period) or period <= 0 or not np.isfinite(tmid): + return candidate + + duration = transit_duration(values) + if not np.isfinite(duration) or duration <= 0: + return candidate + + max_exposure = np.nanmax(exposure_time_days[candidate]) + if not np.isfinite(max_exposure) or max_exposure <= 0: + return np.zeros(times.shape, dtype=bool) + + half_window = ( + 0.5 * duration + + (0.5 + EXPOSURE_SMEARING_TRANSIT_WINDOW_PADDING_FACTOR) * max_exposure + ) + phase_days = get_phase(times[candidate], period, tmid) * period + narrowed = np.abs(phase_days) <= half_window + candidate_indices = np.flatnonzero(candidate) + candidate[candidate_indices] = narrowed + return candidate + + +def transit(times, values): + model = _instantaneous_transit(times, values) + exposure_time_days = _exposure_time_days_for_times(values, np.asarray(times).shape) + if exposure_time_days is None: + return model + + try: + times_array = np.asarray(times, dtype=float) + model_array = np.asarray(model, dtype=float) + except (TypeError, ValueError): + return model + if model_array.shape != times_array.shape: + return model + + flat_times = times_array.reshape(-1) + flat_model = model_array.reshape(-1).copy() + flat_exposure_time_days = exposure_time_days.reshape(-1) + candidate_mask = _exposure_smearing_candidate_mask( + flat_times, + flat_exposure_time_days, + values, + ) + candidate_indices = np.flatnonzero(candidate_mask) + if candidate_indices.size == 0: + return model_array + + candidate_times = flat_times[candidate_indices] + candidate_exposures = flat_exposure_time_days[candidate_indices] + half_exposures = 0.5 * candidate_exposures + try: + start_model = np.asarray( + _instantaneous_transit(candidate_times - half_exposures, values), + dtype=float, + ).reshape(-1) + end_model = np.asarray( + _instantaneous_transit(candidate_times + half_exposures, values), + dtype=float, + ).reshape(-1) + except Exception: + return model_array + if start_model.shape != candidate_times.shape or end_model.shape != candidate_times.shape: + return model_array + + center_model = flat_model[candidate_indices] + model_change = np.maximum.reduce(( + np.abs(start_model - center_model), + np.abs(end_model - center_model), + np.abs(end_model - start_model), + )) + active_indices = candidate_indices[model_change > _exposure_smearing_change_tolerance(values)] + if active_indices.size == 0: + return model_array + + sample_count = _exposure_smearing_supersample(values) + offsets = (np.arange(sample_count, dtype=float) + 0.5) / sample_count - 0.5 + active_times = flat_times[active_indices] + active_exposures = flat_exposure_time_days[active_indices] + sample_times = (active_times[:, None] + active_exposures[:, None] * offsets[None, :]).reshape(-1) + try: + sample_model = np.asarray(_instantaneous_transit(sample_times, values), dtype=float) + except Exception: + return model_array + if sample_model.size != active_indices.size * sample_count: + return model_array + + flat_model[active_indices] = sample_model.reshape(active_indices.size, sample_count).mean(axis=1) + return flat_model.reshape(model_array.shape) + + def impact_parameter_scale(values): ecc = values.get('ecc', 0.0) omega = np.deg2rad(values.get('omega', 0.0)) @@ -502,6 +667,29 @@ def binner(arr, n, err=''): return arr, err +def normalize_exposure_times_seconds_to_days(exposure_times_seconds, reference_shape): + if exposure_times_seconds is None: + return None + try: + exposure_times_seconds = np.asarray(exposure_times_seconds, dtype=float) + except (TypeError, ValueError): + return None + + if exposure_times_seconds.shape == (): + exposure_times_seconds = np.full(reference_shape, float(exposure_times_seconds), dtype=float) + elif exposure_times_seconds.shape != reference_shape: + return None + else: + exposure_times_seconds = np.array(exposure_times_seconds, dtype=float, copy=True) + + valid = np.isfinite(exposure_times_seconds) & (exposure_times_seconds > MIN_EXPOSURE_SMEARING_SECONDS) + if not np.any(valid): + return None + + exposure_times_seconds[~valid] = 0.0 + return exposure_times_seconds / 86400.0 + + class lc_fitter(object): def __init__( @@ -523,6 +711,9 @@ def __init__( fixed_parameter_errors=None, fixed_flux_baseline=False, ultranest_min_num_live_points=None, + exposure_times_seconds=None, + exposure_smearing_supersample=DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE, + exposure_smearing_change_tolerance=DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE, ): self.time = time self.data = data @@ -547,6 +738,28 @@ def __init__( ) self.fixed_flux_baseline = bool(fixed_flux_baseline) self.ultranest_min_num_live_points = ultranest_min_num_live_points + self.exposure_times_days = normalize_exposure_times_seconds_to_days( + exposure_times_seconds, + np.asarray(time).shape, + ) + self.exposure_smearing_supersample = _exposure_smearing_supersample({ + EXPOSURE_SMEARING_SUPERSAMPLE_KEY: exposure_smearing_supersample, + }) + self.exposure_smearing_change_tolerance = _exposure_smearing_change_tolerance({ + EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY: exposure_smearing_change_tolerance, + }) + self.exposure_smearing_available = self.exposure_times_days is not None + if self.exposure_smearing_available: + finite_exposures = self.exposure_times_days[ + np.isfinite(self.exposure_times_days) & (self.exposure_times_days > 0) + ] + self.exposure_smearing_median_seconds = ( + float(np.nanmedian(finite_exposures) * 86400.0) + if finite_exposures.size + else np.nan + ) + else: + self.exposure_smearing_median_seconds = np.nan self._ultranest_resume_context = None self.results = None self.sampled_keys = list(bounds.keys()) @@ -608,7 +821,7 @@ def _values_with_analytic_flux_baseline(self, values): return values try: - model = transit(self.time, values) + model = self._transit_model(self.time, values) model = np.asarray(model, dtype=float) * airmass_trend( values.get('a2', 0), self.airmass, @@ -670,6 +883,41 @@ def _get_plot_time_range(self): return plot_time_range return normalize_time_range(self.time) + def _exposure_times_for_model_times(self, times): + exposure_times_days = getattr(self, 'exposure_times_days', None) + if exposure_times_days is None: + return None + + times = np.asarray(times, dtype=float) + source_times = np.asarray(self.time, dtype=float) + if ( + times.shape == source_times.shape + and exposure_times_days.shape == source_times.shape + and np.allclose(times, source_times, rtol=0.0, atol=0.0) + ): + return exposure_times_days + + finite_exposures = exposure_times_days[ + np.isfinite(exposure_times_days) & (exposure_times_days > 0) + ] + if finite_exposures.size == 0: + return None + return float(np.nanmedian(finite_exposures)) + + def _values_with_exposure_smearing(self, values, times): + exposure_times_days = self._exposure_times_for_model_times(times) + if exposure_times_days is None: + return values + + smeared_values = copy.deepcopy(values) + smeared_values[EXPOSURE_SMEARING_EXPOSURE_TIME_KEY] = exposure_times_days + smeared_values[EXPOSURE_SMEARING_SUPERSAMPLE_KEY] = self.exposure_smearing_supersample + smeared_values[EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY] = self.exposure_smearing_change_tolerance + return smeared_values + + def _transit_model(self, times, values): + return transit(times, self._values_with_exposure_smearing(values, times)) + def _update_plot_geometry(self): plot_time_range = self._get_plot_time_range() self.phase = get_plot_phase(self.time, self.parameters['per'], self.parameters['tmid'], plot_time_range) @@ -679,7 +927,7 @@ def _update_plot_geometry(self): else: self.time_upsample = np.linspace(plot_time_range[0], plot_time_range[1], 1000) - self.transit_upsample = transit(self.time_upsample, self.parameters) + self.transit_upsample = self._transit_model(self.time_upsample, self.parameters) self.phase_upsample = get_plot_phase( self.time_upsample, self.parameters['per'], @@ -743,7 +991,7 @@ def _get_perturbed_transit_parameter_value(self, key, value): def _normalized_model_for_plot_times(self, times, values): values = self._values_with_analytic_flux_baseline(values) - model = np.asarray(transit(times, values), dtype=float) + model = np.asarray(self._transit_model(times, values), dtype=float) if np.ndim(self.airmass) == 2: return model @@ -3164,7 +3412,7 @@ def fit_LM(self): def lc2min_nneighbor(pars): for i in range(len(pars)): self.prior[freekeys[i]] = pars[i] - lightcurve = transit(self.time, self.prior) + lightcurve = self._transit_model(self.time, self.prior) detrended = self.data / lightcurve wf = weightedflux(detrended, self.gw, self.nearest) model = lightcurve * wf @@ -3173,7 +3421,7 @@ def lc2min_nneighbor(pars): def lc2min_airmass(pars): for i in range(len(pars)): self.prior[freekeys[i]] = pars[i] - model = transit(self.time, self.prior) + model = self._transit_model(self.time, self.prior) model *= airmass_trend( self.prior.get('a2', 0), self.airmass, @@ -3234,7 +3482,7 @@ def lc2min_airmass(pars): self.create_fit_variables() def create_fit_variables(self): - self.transit = transit(self.time, self.parameters) + self.transit = self._transit_model(self.time, self.parameters) self._apply_fixed_parameter_errors() self._update_plot_geometry() if np.ndim(self.airmass) != 2: @@ -3582,7 +3830,7 @@ def single_loglike(pars): duration_log_residual = np.log(duration / expected_duration) duration_loglike = -0.5 * (duration_log_residual / sigma_log_duration) ** 2 try: - model = np.asarray(transit(time, physical), dtype=float) + model = np.asarray(self._transit_model(time, physical), dtype=float) if sampled_a2_index is not None: model *= np.exp(float(pars[sampled_a2_index]) * centered_airmass) elif fixed_airmass_scale is not None: diff --git a/exotic/exotic.py b/exotic/exotic.py index ba717f5e..db0c6771 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -3060,7 +3060,11 @@ def estimate_transit_duration_samples_from_fit(fit, sample_count=1000, grid_size return None, np.array([], dtype=float) baseline_parameters = dict(parameters) - baseline_model = transit(transit_times, baseline_parameters) + fit_transit_model = getattr(fit, '_transit_model', None) + if callable(fit_transit_model): + baseline_model = fit_transit_model(transit_times, baseline_parameters) + else: + baseline_model = transit(transit_times, baseline_parameters) dt = float(np.nanmean(np.diff(transit_times))) if not np.isfinite(dt) or dt <= 0: return baseline_model, np.array([], dtype=float) @@ -3370,6 +3374,8 @@ def widen_rprs_bounds_to_data_uncertainty_window(bounds, prior): def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_flux, airmass, jd_times=None, adaptive_summary=None, target_flux_error=None, comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None, expected_transit_depth=None): result = { 'applied': False, @@ -3389,6 +3395,7 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ 'unc': np.array([], dtype=float), 'airmass': np.array([], dtype=float), 'jd_time': np.array([], dtype=float), + 'exposure_time_seconds': None, 'target_flux': np.array([], dtype=float), 'comp_flux': np.array([], dtype=float), 'target_flux_error': np.array([], dtype=float), @@ -3404,6 +3411,8 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ target_flux_error=target_flux_error, comp_flux_error=comp_flux_error, jd_times=jd_times, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, expected_transit_depth=expected_transit_depth, ) result['filter_diagnostics'] = prepared.get('filter_diagnostics', []) @@ -3432,6 +3441,8 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ good_unc = np.asarray(prepared['unc'], dtype=float) good_airmass = np.asarray(prepared['airmass'], dtype=float) good_jd_times = np.asarray(prepared['jd_time'], dtype=float) + good_exposure_times = prepared.get('exposure_time_seconds') + good_exposure_times = None if good_exposure_times is None else np.asarray(good_exposure_times, dtype=float) good_target_flux = np.asarray(prepared['target_flux'], dtype=float) good_comp_flux = np.asarray(prepared['comp_flux'], dtype=float) good_target_flux_error = np.asarray(prepared.get('target_flux_error', []), dtype=float) @@ -3475,6 +3486,8 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ good_unc = good_unc[~adaptive_clip_mask] good_airmass = good_airmass[~adaptive_clip_mask] good_jd_times = good_jd_times[~adaptive_clip_mask] + if good_exposure_times is not None: + good_exposure_times = good_exposure_times[~adaptive_clip_mask] good_target_flux = good_target_flux[~adaptive_clip_mask] good_comp_flux = good_comp_flux[~adaptive_clip_mask] good_target_flux_error = good_target_flux_error[~adaptive_clip_mask] @@ -3496,6 +3509,9 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ 'unc': good_unc[relative_flux_mask], 'airmass': good_airmass[relative_flux_mask], 'jd_time': good_jd_times[relative_flux_mask], + 'exposure_time_seconds': ( + None if good_exposure_times is None else good_exposure_times[relative_flux_mask] + ), 'target_flux': good_target_flux[relative_flux_mask], 'comp_flux': good_comp_flux[relative_flux_mask], 'target_flux_error': good_target_flux_error[relative_flux_mask], @@ -3767,6 +3783,8 @@ def score_comparison_candidate_lightcurve_scout(prepared_series, eebls_summary, def build_comparison_candidate_preflight(times, jd_times, airmass, ld, p_dict, target_flux, comp_flux, target_flux_error=None, comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None, adaptive_summary=None, use_eebls_to_initialize_tmid_and_bounds=True): prepared = prepare_comparison_candidate_full_reduction_series( times, @@ -3777,6 +3795,8 @@ def build_comparison_candidate_preflight(times, jd_times, airmass, ld, p_dict, t adaptive_summary=adaptive_summary, target_flux_error=target_flux_error, comp_flux_error=comp_flux_error, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) try: @@ -3945,6 +3965,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, jd_times=None, target_flux_error=None, comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None, disable_vertical_flux_normalization=False, detrend_on_outoftransit_baseline=True, use_impactparameter_rather_than_inclination_to_fit=True, @@ -3964,6 +3986,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, 'good_unc': np.array([], dtype=float), 'good_airmass': np.array([], dtype=float), 'good_jd_times': np.array([], dtype=float), + 'good_exposure_times_seconds': None, 'good_target_flux': np.array([], dtype=float), 'good_comp_flux': np.array([], dtype=float), 'good_target_flux_error': np.array([], dtype=float), @@ -3985,6 +4008,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, adaptive_summary=adaptive_summary, target_flux_error=target_flux_error, comp_flux_error=comp_flux_error, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), ) else: @@ -4002,6 +4027,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_unc = np.asarray(prepared['unc'], dtype=float) good_airmass = np.asarray(prepared['airmass'], dtype=float) good_jd_times = np.asarray(prepared['jd_time'], dtype=float) + good_exposure_times = prepared.get('exposure_time_seconds') + good_exposure_times = None if good_exposure_times is None else np.asarray(good_exposure_times, dtype=float) good_target_flux = np.asarray(prepared['target_flux'], dtype=float) good_comp_flux = np.asarray(prepared['comp_flux'], dtype=float) good_target_flux_error = np.asarray(prepared.get('target_flux_error', []), dtype=float) @@ -4066,6 +4093,13 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, ensure_pre_final_ultranest_baseline_bounds(prior, bounds, good_flux, fit_a2=True) debug_phase_clip_keep_mask = None + prefit_kwargs = { + 'jd_times': good_jd_times, + 'mode': 'lm', + 'use_impactparameter_rather_than_inclination_to_fit': + use_impactparameter_rather_than_inclination_to_fit, + } + add_exposure_times_to_lc_fitter_kwargs(prefit_kwargs, good_exposure_times) prefit = lc_fitter( good_times, good_flux, @@ -4073,9 +4107,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_airmass, prior, bounds, - jd_times=good_jd_times, - mode='lm', - use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + **prefit_kwargs, ) if ( run_final_fit_phase_residual_clip @@ -4100,6 +4132,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_unc = good_unc[~phase_clip_mask] good_airmass = good_airmass[~phase_clip_mask] good_jd_times = good_jd_times[~phase_clip_mask] + if good_exposure_times is not None: + good_exposure_times = good_exposure_times[~phase_clip_mask] good_target_flux = good_target_flux[~phase_clip_mask] good_comp_flux = good_comp_flux[~phase_clip_mask] good_target_flux_error = good_target_flux_error[~phase_clip_mask] @@ -4111,6 +4145,9 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, full_good_unc = np.asarray(good_unc, dtype=float) full_good_airmass = np.asarray(good_airmass, dtype=float) full_good_jd_times = np.asarray(good_jd_times, dtype=float) + full_good_exposure_times = ( + None if good_exposure_times is None else np.asarray(good_exposure_times, dtype=float) + ) full_good_target_flux = np.asarray(good_target_flux, dtype=float) full_good_comp_flux = np.asarray(good_comp_flux, dtype=float) full_good_target_flux_error = np.asarray(good_target_flux_error, dtype=float) @@ -4123,6 +4160,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, fit_unc = full_good_unc fit_airmass = full_good_airmass fit_jd_times = full_good_jd_times + fit_exposure_times = full_good_exposure_times if run_fast_ultranest_before_final_run: fast_binning = build_fast_ultranest_lightcurve_series( full_good_times, @@ -4130,6 +4168,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, full_good_unc, full_good_airmass, jd_times=full_good_jd_times, + exposure_times_seconds=full_good_exposure_times, ) if fast_binning.get('applied'): log_info(fast_binning['note']) @@ -4138,6 +4177,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, fit_unc = fast_binning['unc'] fit_airmass = fast_binning['airmass'] fit_jd_times = fast_binning['jd_times'] + fit_exposure_times = fast_binning.get('exposure_times_seconds') fit_prior = dict(prior) fit_bounds = clone_lightcurve_bounds(bounds) @@ -4159,6 +4199,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, fit_prior, fit_bounds, jd_times=fit_jd_times, + exposure_times_seconds=fit_exposure_times, skip_airmass_fit=skip_final_airmass_fit, airmass_skip_note=airmass_skip_note, disable_vertical_flux_normalization=disable_vertical_flux_normalization, @@ -4191,6 +4232,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_times = good_times[final_time_indices] good_airmass = good_airmass[final_time_indices] good_jd_times = good_jd_times[final_time_indices] + if good_exposure_times is not None: + good_exposure_times = good_exposure_times[final_time_indices] good_target_flux = good_target_flux[final_time_indices] good_comp_flux = good_comp_flux[final_time_indices] good_target_flux_error = good_target_flux_error[final_time_indices] @@ -4246,6 +4289,9 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, clipped_unc = good_unc[residual_keep_mask] clipped_airmass = good_airmass[residual_keep_mask] clipped_jd_times = good_jd_times[residual_keep_mask] + clipped_exposure_times = ( + None if good_exposure_times is None else good_exposure_times[residual_keep_mask] + ) clipped_target_flux = good_target_flux[residual_keep_mask] clipped_comp_flux = good_comp_flux[residual_keep_mask] clipped_target_flux_error = good_target_flux_error[residual_keep_mask] @@ -4281,6 +4327,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, residual_refit_prior, residual_refit_bounds, jd_times=clipped_jd_times, + exposure_times_seconds=clipped_exposure_times, skip_airmass_fit=skip_final_airmass_fit, airmass_skip_note=airmass_skip_note, disable_vertical_flux_normalization=disable_vertical_flux_normalization, @@ -4339,6 +4386,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, good_unc = clipped_unc good_airmass = clipped_airmass good_jd_times = clipped_jd_times + good_exposure_times = clipped_exposure_times good_target_flux = clipped_target_flux good_comp_flux = clipped_comp_flux good_target_flux_error = clipped_target_flux_error @@ -4395,6 +4443,11 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, 'good_unc': full_good_unc if fast_binning.get('applied') else np.asarray(good_unc, dtype=float), 'good_airmass': full_good_airmass if fast_binning.get('applied') else np.asarray(good_airmass, dtype=float), 'good_jd_times': full_good_jd_times if fast_binning.get('applied') else np.asarray(good_jd_times, dtype=float), + 'good_exposure_times_seconds': ( + full_good_exposure_times + if fast_binning.get('applied') + else None if good_exposure_times is None else np.asarray(good_exposure_times, dtype=float) + ), 'good_target_flux': full_good_target_flux if fast_binning.get('applied') else np.asarray(good_target_flux, dtype=float), 'good_comp_flux': full_good_comp_flux if fast_binning.get('applied') else np.asarray(good_comp_flux, dtype=float), 'good_target_flux_error': full_good_target_flux_error if fast_binning.get('applied') else np.asarray(good_target_flux_error, dtype=float), @@ -4406,6 +4459,9 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, 'fast_fit_good_unc': np.asarray(fitted_unc, dtype=float), 'fast_fit_good_airmass': np.asarray(fit_airmass, dtype=float), 'fast_fit_good_jd_times': None if fit_jd_times is None else np.asarray(fit_jd_times, dtype=float), + 'fast_fit_good_exposure_times_seconds': ( + None if fit_exposure_times is None else np.asarray(fit_exposure_times, dtype=float) + ), 'fast_fit_prior': fit_prior, 'fast_fit_bounds': fit_bounds, 'skip_airmass_fit': skip_final_airmass_fit, @@ -4529,6 +4585,8 @@ def refit_selected_fast_comparison_on_full_lightcurve( airmass = np.asarray(selected_result.get('good_airmass'), dtype=float) jd_times = selected_result.get('good_jd_times') jd_times = None if jd_times is None else np.asarray(jd_times, dtype=float) + exposure_times = selected_result.get('good_exposure_times_seconds') + exposure_times = None if exposure_times is None else np.asarray(exposure_times, dtype=float) if not (times.shape == flux_values.shape == flux_errors.shape == airmass.shape): log_info( "Warning: Could not run the full-resolution selected comparison-star final fit " @@ -4538,6 +4596,8 @@ def refit_selected_fast_comparison_on_full_lightcurve( return None if jd_times is not None and jd_times.shape != times.shape: jd_times = None + if exposure_times is not None and exposure_times.shape != times.shape: + exposure_times = None original_times = times.copy() base_filter_diagnostics = [ @@ -4674,6 +4734,7 @@ def aligned_selected_array(key, dtype=float): prior, bounds, jd_times=jd_times, + exposure_times_seconds=exposure_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, max_rprs_retries=0, max_ars_retries=0, @@ -4738,6 +4799,7 @@ def aligned_selected_array(key, dtype=float): retained_unc = fit_unc[residual_keep_mask] retained_airmass = airmass[residual_keep_mask] retained_jd_times = None if jd_times is None else jd_times[residual_keep_mask] + retained_exposure_times = None if exposure_times is None else exposure_times[residual_keep_mask] retained_target_flux_values = ( None if target_flux_values is None else target_flux_values[residual_keep_mask] ) @@ -4796,6 +4858,7 @@ def aligned_selected_array(key, dtype=float): residual_refit_prior, residual_refit_bounds, jd_times=retained_jd_times, + exposure_times_seconds=retained_exposure_times, use_impactparameter_rather_than_inclination_to_fit= use_impactparameter_rather_than_inclination_to_fit, max_rprs_retries=0, @@ -4847,6 +4910,7 @@ def aligned_selected_array(key, dtype=float): fit_unc = retained_unc airmass = retained_airmass jd_times = retained_jd_times + exposure_times = retained_exposure_times target_flux_values = retained_target_flux_values comp_flux_values = retained_comp_flux_values target_flux_error_values = retained_target_flux_error_values @@ -5373,6 +5437,17 @@ def callable_accepts_keyword(callable_obj, keyword): ) +def add_exposure_times_to_lc_fitter_kwargs(fit_kwargs, exposure_times_seconds): + if exposure_times_seconds is None or not callable_accepts_keyword(lc_fitter, 'exposure_times_seconds'): + return fit_kwargs + try: + exposure_times = np.asarray(exposure_times_seconds, dtype=float) + except (TypeError, ValueError): + return fit_kwargs + fit_kwargs['exposure_times_seconds'] = exposure_times + return fit_kwargs + + def get_configured_ultranest_min_num_live_points(): return parse_ultranest_min_num_live_points( os.environ.get( @@ -6235,6 +6310,7 @@ def run_nested_lightcurve_fit_with_rprs_posterior_retry( prior, bounds, jd_times=None, + exposure_times_seconds=None, use_impactparameter_rather_than_inclination_to_fit=True, max_rprs_retries=RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, duration_prior=None, @@ -6408,6 +6484,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): 'use_impactparameter_rather_than_inclination_to_fit': use_impactparameter_rather_than_inclination_to_fit, } + add_exposure_times_to_lc_fitter_kwargs(fit_kwargs, exposure_times_seconds) if isinstance(duration_prior, dict) and duration_prior.get('applied'): fit_kwargs['duration_prior'] = duration_prior if keep_ultranest_sampler and callable_accepts_keyword(lc_fitter, 'keep_ultranest_sampler'): @@ -8984,6 +9061,7 @@ def build_fast_ultranest_lightcurve_series( flux_errors, airmass, jd_times=None, + exposure_times_seconds=None, max_points=FAST_ULTRANEST_MAX_BINNED_POINTS, min_points_to_bin=FAST_ULTRANEST_MIN_POINTS_TO_BIN, ): @@ -8992,6 +9070,7 @@ def build_fast_ultranest_lightcurve_series( flux_errors = np.asarray(flux_errors, dtype=float) airmass = np.asarray(airmass, dtype=float) jd_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + exposure_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) base_result = { 'applied': False, @@ -9001,6 +9080,7 @@ def build_fast_ultranest_lightcurve_series( 'unc': flux_errors, 'airmass': airmass, 'jd_times': jd_array, + 'exposure_times_seconds': exposure_array, 'original_point_count': int(times.shape[0]), 'binned_point_count': int(times.shape[0]), 'bin_indices': None, @@ -9012,6 +9092,9 @@ def build_fast_ultranest_lightcurve_series( if jd_array is not None and jd_array.shape != times.shape: base_result['note'] = 'Skipped; JD timestamps were not aligned for fast UltraNest binning.' return base_result + if exposure_array is not None and exposure_array.shape != times.shape: + base_result['note'] = 'Skipped; exposure times were not aligned for fast UltraNest binning.' + return base_result point_count = int(times.shape[0]) if point_count <= int(min_points_to_bin): @@ -9032,6 +9115,8 @@ def build_fast_ultranest_lightcurve_series( ) if jd_array is not None: valid &= np.isfinite(jd_array) + if exposure_array is not None: + valid &= np.isfinite(exposure_array) if np.count_nonzero(valid) <= target_points: base_result['note'] = 'Skipped; too few finite points remained for fast UltraNest binning.' return base_result @@ -9047,6 +9132,7 @@ def build_fast_ultranest_lightcurve_series( binned_unc = [] binned_airmass = [] binned_jd = [] if jd_array is not None else None + binned_exposure = [] if exposure_array is not None else None for chunk in chunks: chunk_unc = flux_errors[chunk] weights = np.zeros(chunk_unc.shape, dtype=float) @@ -9062,6 +9148,8 @@ def build_fast_ultranest_lightcurve_series( binned_airmass.append(_weighted_mean_with_fallback(airmass[chunk], weights)) if jd_array is not None: binned_jd.append(_weighted_mean_with_fallback(jd_array[chunk], weights)) + if exposure_array is not None: + binned_exposure.append(_weighted_mean_with_fallback(exposure_array[chunk], weights)) binned_time = np.asarray(binned_time, dtype=float) binned_flux = np.asarray(binned_flux, dtype=float) @@ -9077,6 +9165,9 @@ def build_fast_ultranest_lightcurve_series( if binned_jd is not None: binned_jd = np.asarray(binned_jd, dtype=float) finite_binned &= np.isfinite(binned_jd) + if binned_exposure is not None: + binned_exposure = np.asarray(binned_exposure, dtype=float) + finite_binned &= np.isfinite(binned_exposure) if np.count_nonzero(finite_binned) < LIGHTCURVE_MIN_VALID_POINTS: base_result['note'] = 'Skipped; fast UltraNest binning produced too few finite bins.' @@ -9090,6 +9181,7 @@ def build_fast_ultranest_lightcurve_series( 'unc': binned_unc[finite_binned], 'airmass': binned_airmass[finite_binned], 'jd_times': None if binned_jd is None else binned_jd[finite_binned], + 'exposure_times_seconds': None if binned_exposure is None else binned_exposure[finite_binned], 'binned_point_count': int(np.count_nonzero(finite_binned)), 'bin_indices': [chunk.tolist() for i, chunk in enumerate(chunks) if finite_binned[i]], 'note': ( @@ -10283,6 +10375,7 @@ def build_final_fit_prefit_refinement_plan( bounds, fit, jd_times=None, + exposure_times_seconds=None, baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, ): times = np.asarray(times, dtype=float) @@ -10290,6 +10383,9 @@ def build_final_fit_prefit_refinement_plan( flux_errors = np.asarray(flux_errors, dtype=float) airmass = np.asarray(airmass, dtype=float) jd_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + exposure_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + if exposure_array is not None and exposure_array.shape != times.shape: + exposure_array = None original_tmid_bounds = clone_lightcurve_bounds(bounds).get('tmid') base_plan = { @@ -10308,6 +10404,7 @@ def build_final_fit_prefit_refinement_plan( 'unc': flux_errors, 'airmass': airmass, 'jd_times': jd_array, + 'exposure_times_seconds': exposure_array, 'prior': dict(prior), 'bounds': clone_lightcurve_bounds(bounds), } @@ -10413,6 +10510,7 @@ def build_final_fit_prefit_refinement_plan( refined_unc = flux_errors[keep_mask] refined_airmass = airmass[keep_mask] refined_jd_times = None if jd_array is None else jd_array[keep_mask] + refined_exposure_times = None if exposure_array is None else exposure_array[keep_mask] refined_prior = dict(prior) if isinstance(fit_parameters, dict): @@ -10438,6 +10536,7 @@ def build_final_fit_prefit_refinement_plan( 'unc': refined_unc, 'airmass': refined_airmass, 'jd_times': refined_jd_times, + 'exposure_times_seconds': refined_exposure_times, 'prior': refined_prior, 'bounds': refined_bounds, }) @@ -10459,6 +10558,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( prior, bounds, jd_times=None, + exposure_times_seconds=None, skip_airmass_fit=False, airmass_skip_note=None, disable_vertical_flux_normalization=False, @@ -10482,6 +10582,9 @@ def fit_final_lightcurve_with_oot_baseline_detrending( if isinstance(search_restriction_prior, dict) else dict(prior) if isinstance(prior, dict) else {} ) + exposure_times_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + if exposure_times_array is not None and exposure_times_array.shape != np.asarray(times).shape: + exposure_times_array = None search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( search_restriction_prior, times, @@ -10525,6 +10628,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( prior, bounds, jd_times=jd_times, + exposure_times_seconds=exposure_times_array, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, @@ -10551,6 +10655,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( effective_bounds, fit, jd_times=jd_times, + exposure_times_seconds=exposure_times_array, baseline_duration_multiplier=baseline_duration_multiplier, ) working_times = prefit_plan['times'] @@ -10558,6 +10663,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( working_unc = prefit_plan['unc'] working_airmass = prefit_plan['airmass'] working_jd_times = prefit_plan['jd_times'] + working_exposure_times = prefit_plan.get('exposure_times_seconds') working_prior = prefit_plan['prior'] working_bounds = prefit_plan['bounds'] @@ -10578,6 +10684,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( working_prior, working_bounds, jd_times=working_jd_times, + exposure_times_seconds=working_exposure_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, @@ -10674,6 +10781,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): baseline_constrained_prior, baseline_constrained_bounds, jd_times=working_jd_times, + exposure_times_seconds=working_exposure_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, @@ -10808,6 +10916,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): refit_prior, refit_bounds, jd_times=working_jd_times, + exposure_times_seconds=working_exposure_times, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, keep_ultranest_sampler=keep_ultranest_for_sparse_extension, @@ -11365,6 +11474,7 @@ def apply_lightcurve_mask(lightcurve, mask, sort_index=None): 'airmass', 'transit', 'jd_times', + 'exposure_times_days', 'phase', 'residuals', 'model', @@ -11600,10 +11710,10 @@ def resolve_require_comp_star_for_exposure_times(config_value, exptimes): if not require_comp_star: log_info( "Exposure times vary by more than 1% across retained frames " - f"({spread_percent:.2f}%); requiring a comparison star for this reduction.", + f"({spread_percent:.2f}%); target-only/no-comparison photometry will scale " + "source counts to a common exposure time before fitting.", warn=True, ) - return True return require_comp_star @@ -17327,6 +17437,69 @@ def relative_flux_uncertainty_from_star_errors(target_flux, comp_flux, ) +def exposure_scale_factors_to_max(exposure_times_seconds): + if exposure_times_seconds is None: + return None + try: + exposure_times = np.asarray(exposure_times_seconds, dtype=float) + except (TypeError, ValueError): + return None + if exposure_times.ndim != 1: + return None + + valid = np.isfinite(exposure_times) & (exposure_times > 0) + if not np.any(valid): + return None + + max_exposure = float(np.nanmax(exposure_times[valid])) + if not np.isfinite(max_exposure) or max_exposure <= 0: + return None + + factors = np.ones(exposure_times.shape, dtype=float) + factors[valid] = max_exposure / exposure_times[valid] + return factors + + +def source_flux_uncertainty_from_counts(flux_adu, gain_e_per_adu=None): + try: + gain = float(gain_e_per_adu) + except (TypeError, ValueError): + gain = 1.0 + if not np.isfinite(gain) or gain <= 0: + gain = 1.0 + + flux_adu = np.asarray(flux_adu, dtype=float) + with np.errstate(invalid='ignore'): + return np.sqrt(np.maximum(flux_adu, 0.0) / gain) + + +def scale_target_only_flux_to_common_exposure(target_flux, target_flux_error, comp_flux, + exposure_times_seconds=None, gain_e_per_adu=None): + target_flux = np.asarray(target_flux, dtype=float) + comp_flux = np.asarray(comp_flux, dtype=float) + target_flux_error = valid_flux_error_array(target_flux_error, target_flux.shape) + + if target_flux.ndim != 1 or comp_flux.shape != target_flux.shape: + return target_flux, target_flux_error + if not np.allclose(comp_flux, 1.0, equal_nan=False): + return target_flux, target_flux_error + + scale_factors = exposure_scale_factors_to_max(exposure_times_seconds) + if scale_factors is None or scale_factors.shape != target_flux.shape: + return target_flux, target_flux_error + if np.allclose(scale_factors, 1.0, rtol=1e-12, atol=1e-12): + return target_flux, target_flux_error + + scaled_flux = target_flux * scale_factors + fallback_error = source_flux_uncertainty_from_counts(target_flux, gain_e_per_adu) + if target_flux_error is None: + scaled_error = fallback_error * scale_factors + else: + valid_error = np.isfinite(target_flux_error) & (target_flux_error > 0) + scaled_error = np.where(valid_error, target_flux_error, fallback_error) * scale_factors + return scaled_flux, scaled_error + + def weighted_nanpercentile(values, weights, percentile): values = np.asarray(values, dtype=float).ravel() weights = np.asarray(weights, dtype=float).ravel() @@ -19290,7 +19463,9 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, use_eebls_to_initialize_tmid_and_bounds=True, compute_eebls_diagnostics=False, target_flux_error=None, - comp_flux_error=None): + comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None): plot_time_range = np.asarray(times if plot_time_range is None else plot_time_range, dtype=float) prepared = prepare_lightcurve_fit_input_series( times, @@ -19300,6 +19475,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, target_flux_error=target_flux_error, comp_flux_error=comp_flux_error, jd_times=jd_times, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, expected_transit_depth=expected_transit_depth_from_planet_dict(pDict), ) if not prepared.get('applied'): @@ -19321,6 +19498,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, arrayNormUnc = prepared['unc'] arrayTimes = prepared['time'] arrayJDTimes = prepared['jd_time'] + arrayExposureTimes = prepared.get('exposure_time_seconds') + arrayExposureTimes = None if arrayExposureTimes is None else np.asarray(arrayExposureTimes, dtype=float) arrayAirmass = prepared['airmass'] skip_airmass_fit = prepared['skip_airmass_fit'] @@ -19404,6 +19583,13 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, if np.isnan(arrayTimes).any() or np.isnan(arrayFinalFlux).any() or np.isnan(arrayNormUnc).any(): log_info("\nWarning: NANs in time, flux or error", warn=True) + fit_kwargs = { + 'jd_times': arrayJDTimes, + 'mode': 'lm', + 'use_impactparameter_rather_than_inclination_to_fit': + use_impactparameter_rather_than_inclination_to_fit, + } + add_exposure_times_to_lc_fitter_kwargs(fit_kwargs, arrayExposureTimes) myfit = lc_fitter( arrayTimes, arrayFinalFlux, @@ -19411,9 +19597,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, arrayAirmass, prior, mybounds, - jd_times=arrayJDTimes, - mode='lm', - use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + **fit_kwargs, ) myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) @@ -19441,10 +19625,19 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, arrayNormUnc = arrayNormUnc[~phase_clip_mask] arrayTimes = arrayTimes[~phase_clip_mask] arrayJDTimes = arrayJDTimes[~phase_clip_mask] + if arrayExposureTimes is not None: + arrayExposureTimes = arrayExposureTimes[~phase_clip_mask] arrayAirmass = arrayAirmass[~phase_clip_mask] f1 = f1[~phase_clip_mask] f2 = f2[~phase_clip_mask] + fit_kwargs = { + 'jd_times': arrayJDTimes, + 'mode': 'lm', + 'use_impactparameter_rather_than_inclination_to_fit': + use_impactparameter_rather_than_inclination_to_fit, + } + add_exposure_times_to_lc_fitter_kwargs(fit_kwargs, arrayExposureTimes) myfit = lc_fitter( arrayTimes, arrayFinalFlux, @@ -19452,9 +19645,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, arrayAirmass, prior, mybounds, - jd_times=arrayJDTimes, - mode='lm', - use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + **fit_kwargs, ) myfit = apply_plot_time_range(myfit, plot_time_range) annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) @@ -19491,6 +19682,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, nested_refinement['prior'], nested_refinement['bounds'], jd_times=arrayJDTimes, + exposure_times_seconds=arrayExposureTimes, use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, duration_prior=duration_prior, search_restriction_prior=search_restriction_prior, @@ -19722,6 +19914,8 @@ def prepare_lightcurve_fit_input_series( target_flux_error=None, comp_flux_error=None, jd_times=None, + exposure_times_seconds=None, + gain_e_per_adu=None, expected_transit_depth=None, ): times = np.asarray(times, dtype=float) @@ -19731,6 +19925,7 @@ def prepare_lightcurve_fit_input_series( target_flux_error = valid_flux_error_array(target_flux_error, target_flux.shape) comp_flux_error = valid_flux_error_array(comp_flux_error, comp_flux.shape) jd_times_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + exposure_times_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) prepared = { 'applied': False, @@ -19749,6 +19944,7 @@ def prepare_lightcurve_fit_input_series( 'flux': np.array([], dtype=float), 'unc': np.array([], dtype=float), 'jd_time': None, + 'exposure_time_seconds': None, 'airmass': np.array([], dtype=float), 'target_flux': np.array([], dtype=float), 'comp_flux': np.array([], dtype=float), @@ -19770,6 +19966,16 @@ def prepare_lightcurve_fit_input_series( if jd_times_array is not None and jd_times_array.shape != times.shape: jd_times_array = None + if exposure_times_array is not None and exposure_times_array.shape != times.shape: + exposure_times_array = None + + target_flux, target_flux_error = scale_target_only_flux_to_common_exposure( + target_flux, + target_flux_error, + comp_flux, + exposure_times_seconds=exposure_times_array, + gain_e_per_adu=gain_e_per_adu, + ) plot_indices = np.argsort(times) times_sorted = times[plot_indices] @@ -19777,6 +19983,7 @@ def prepare_lightcurve_fit_input_series( comp_flux_sorted = comp_flux[plot_indices] target_flux_error_sorted = None if target_flux_error is None else target_flux_error[plot_indices] comp_flux_error_sorted = None if comp_flux_error is None else comp_flux_error[plot_indices] + exposure_times_sorted = None if exposure_times_array is None else exposure_times_array[plot_indices] source_indices = np.asarray(plot_indices, dtype=int) with np.errstate(divide='ignore', invalid='ignore'): flux_ratio_sorted = np.divide(target_flux_sorted, comp_flux_sorted) @@ -19798,6 +20005,8 @@ def prepare_lightcurve_fit_input_series( target_flux_error_sorted = target_flux_error_sorted[flux_ratio_mask] if comp_flux_error_sorted is not None: comp_flux_error_sorted = comp_flux_error_sorted[flux_ratio_mask] + if exposure_times_sorted is not None: + exposure_times_sorted = exposure_times_sorted[flux_ratio_mask] flux_ratio_sorted = flux_ratio_sorted[flux_ratio_mask] source_indices = source_indices[flux_ratio_mask] if jd_times_array is None: @@ -19887,6 +20096,7 @@ def prepare_lightcurve_fit_input_series( filtered_comp_flux = comp_flux_sorted[valid_mask] filtered_target_flux_error = None if target_flux_error_sorted is None else target_flux_error_sorted[valid_mask] filtered_comp_flux_error = None if comp_flux_error_sorted is None else comp_flux_error_sorted[valid_mask] + filtered_exposure_times = None if exposure_times_sorted is None else exposure_times_sorted[valid_mask] unc = relative_flux_uncertainty_from_star_errors( filtered_target_flux, filtered_comp_flux, @@ -19944,6 +20154,7 @@ def prepare_lightcurve_fit_input_series( 'flux': normalized_flux, 'unc': normalized_unc, 'jd_time': fit_jd_times[~nanmask], + 'exposure_time_seconds': None if filtered_exposure_times is None else filtered_exposure_times[~nanmask], 'airmass': fit_airmass[~nanmask], 'target_flux': filtered_target_flux[~nanmask], 'comp_flux': filtered_comp_flux[~nanmask], @@ -20091,7 +20302,11 @@ def fitted_lightcurve_model_at(fit, times, airmass): return np.array([], dtype=float) try: - transit_model = np.asarray(transit(times, parameters), dtype=float) + fit_transit_model = getattr(fit, '_transit_model', None) + if callable(fit_transit_model): + transit_model = np.asarray(fit_transit_model(times, parameters), dtype=float) + else: + transit_model = np.asarray(transit(times, parameters), dtype=float) except Exception: return np.array([], dtype=float) if transit_model.shape != times.shape: @@ -20141,20 +20356,56 @@ def fitted_lightcurve_scatter_on_dataset(fit, times, flux_values, airmass): def evaluate_lightcurve_candidate(task): - ( - times, - tflux, - cflux, - airmass, - ld, - p_dict, - jd_times, - plot_time_range, - disable_vertical_flux_normalization, - use_impactparameter_rather_than_inclination_to_fit, - use_eebls_to_initialize_tmid_and_bounds, - compute_eebls_diagnostics, - ) = task + exposure_times_seconds = None + gain_e_per_adu = None + if len(task) == 14: + ( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + plot_time_range, + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, + exposure_times_seconds, + gain_e_per_adu, + ) = task + elif len(task) == 13: + ( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + plot_time_range, + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, + exposure_times_seconds, + ) = task + else: + ( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + plot_time_range, + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, + ) = task fit_diagnostics = diagnose_lightcurve_fit_inputs( times, tflux, @@ -20178,6 +20429,8 @@ def evaluate_lightcurve_candidate(task): plot_time_range=plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, compute_eebls_diagnostics=compute_eebls_diagnostics, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -20352,6 +20605,8 @@ def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, p use_impactparameter_rather_than_inclination_to_fit=True, use_eebls_to_initialize_tmid_and_bounds=True, compute_eebls_diagnostics=True, + exposure_times_seconds=None, + gain_e_per_adu=None, psf_flux_data=None): candidate_mask = np.asarray(candidate['mask'], dtype=bool) @@ -20390,6 +20645,8 @@ def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, p use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds, compute_eebls_diagnostics, + None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float)[candidate_mask], + gain_e_per_adu, ) @@ -20405,6 +20662,8 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co use_impactparameter_rather_than_inclination_to_fit=True, use_eebls_to_initialize_tmid_and_bounds=True, pick_comparison_by_eebls_snr=True, + exposure_times_seconds=None, + gain_e_per_adu=None, psf_flux_data=None): candidate_jobs = build_target_fit_candidate_jobs( psf_data, @@ -20454,6 +20713,8 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, compute_eebls_diagnostics=True, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, psf_flux_data=psf_flux_data, ) for candidate in evaluated_candidates @@ -21786,7 +22047,9 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p use_impactparameter_rather_than_inclination_to_fit=True, use_eebls_to_initialize_tmid_and_bounds=True, psf_flux_data=None, - psf_noise_data=None): + psf_noise_data=None, + exposure_times_seconds=None, + gain_e_per_adu=None): if photometry_info.get('best_fit_lc') is None or not comp_stars: return [] @@ -21942,6 +22205,12 @@ def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p plot_time_range=plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, compute_eebls_diagnostics=True, + exposure_times_seconds=( + None + if exposure_times_seconds is None + else np.asarray(exposure_times_seconds, dtype=float)[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, ) fit_diagnostics = ensure_lightcurve_fit_failure_reason( fit_diagnostics, @@ -23202,7 +23471,9 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p save_dir=None, planet_name=None, observation_date=None, - use_ensemble_photometry_rather_than_single_comp=False): + use_ensemble_photometry_rather_than_single_comp=False, + exposure_times_seconds=None, + gain_e_per_adu=None): ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) if not ranked_summaries: return { @@ -23219,6 +23490,9 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p frame_count = target_psf_flux.shape[0] else: frame_count = aper_data['target'].shape[0] + exposure_times_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + if exposure_times_array is not None and exposure_times_array.shape != times.shape: + exposure_times_array = None if method == 'psf': target_flux = np.asarray(target_psf_flux, dtype=float) psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) @@ -23412,6 +23686,10 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ensemble_flux[fit_mask], target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], comp_flux_error=None if ensemble_flux_error is None else ensemble_flux_error[fit_mask], + exposure_times_seconds=( + None if exposure_times_array is None else exposure_times_array[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, adaptive_summary=adaptive_summary, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, ) @@ -23566,6 +23844,10 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p comp_flux[fit_mask], target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + exposure_times_seconds=( + None if exposure_times_array is None else exposure_times_array[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, adaptive_summary=adaptive_summary, use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, ) @@ -23622,6 +23904,10 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p jd_times=jd_times[fit_mask], target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + exposure_times_seconds=( + None if exposure_times_array is None else exposure_times_array[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, disable_vertical_flux_normalization=disable_vertical_flux_normalization, detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, use_impactparameter_rather_than_inclination_to_fit= @@ -23726,6 +24012,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'good_unc': final_reduction.get('good_unc'), 'good_airmass': final_reduction.get('good_airmass'), 'good_jd_times': final_reduction.get('good_jd_times'), + 'good_exposure_times_seconds': final_reduction.get('good_exposure_times_seconds'), 'good_target_flux_error': tflux_fit_error, 'good_comp_flux_error': cflux_fit_error, 'tflux_fit': tflux_fit, @@ -25564,6 +25851,8 @@ def _main_impl(): times = times[goodmask] jd_times = jd_times[goodmask] airmass = np.array(airMassList)[goodmask] + exposure_times_seconds = np.asarray(exptimes, dtype=float)[goodmask] + exptimes = exposure_times_seconds.tolist() psf_data["target"] = psf_data["target"][goodmask] psf_flux_data["target"] = psf_flux_data["target"][goodmask] for key in list(psf_noise_data.keys()): @@ -25689,6 +25978,11 @@ def _main_impl(): 'flux_unc_tar': None, 'flux_unc_ref': None } + fallback_gain_e_per_adu = ( + frame_noise_configs[0].get('gain_e_per_adu') + if frame_noise_configs + else None + ) centroid_positions = { 'x_targ': None, @@ -25870,6 +26164,8 @@ def _main_impl(): observation_date=exotic_infoDict['date'], use_ensemble_photometry_rather_than_single_comp= use_ensemble_photometry_rather_than_single_comp, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, ) comparison_calibration['ranked_fit_comp_indices'] = [ summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] @@ -26008,6 +26304,9 @@ def _main_impl(): selected_fit_good_flux=selected_attempt.get('good_flux'), selected_fit_good_unc=selected_attempt.get('good_unc'), selected_fit_good_airmass=selected_attempt.get('good_airmass'), + selected_fit_good_exposure_times_seconds=selected_attempt.get( + 'good_exposure_times_seconds' + ), selected_fit_good_target_flux_error=selected_attempt.get('tflux_fit_error'), selected_fit_good_comp_flux_error=selected_attempt.get('cflux_fit_error'), selected_fit_duration_samples=selected_attempt.get('duration_samples'), @@ -26063,6 +26362,8 @@ def _main_impl(): use_impactparameter_rather_than_inclination_to_fit, plot_time_range=full_plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -26098,6 +26399,8 @@ def _main_impl(): use_impactparameter_rather_than_inclination_to_fit, plot_time_range=full_plot_time_range, use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + exposure_times_seconds=exposure_times_seconds[aper_mask], + gain_e_per_adu=fallback_gain_e_per_adu, ) ref_flux[j] = { 'myfit': vsp_fit, @@ -26248,6 +26551,8 @@ def _main_impl(): use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, psf_flux_data=psf_flux_source, psf_noise_data=psf_noise_data if use_psf_photometry else None, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, ) saved_candidate_fit_count = sum(1 for summary in candidate_fit_summaries if summary['fit'] is not None) failed_candidate_fit_count = len(candidate_fit_summaries) - saved_candidate_fit_count @@ -26353,6 +26658,18 @@ def _main_impl(): goodTimes = best_fit_lc.time goodAirmasses = best_fit_lc.airmass + goodExposureTimes = None + fit_exposure_days = getattr(best_fit_lc, 'exposure_times_days', None) + if fit_exposure_days is not None: + fit_exposure_days = np.asarray(fit_exposure_days, dtype=float) + if fit_exposure_days.shape == np.shape(goodTimes): + goodExposureTimes = fit_exposure_days * 86400.0 + selected_good_exposure_times = photometry_info.get('selected_fit_good_exposure_times_seconds') + if ( + selected_good_exposure_times is not None + and np.shape(selected_good_exposure_times) == np.shape(goodTimes) + ): + goodExposureTimes = np.asarray(selected_good_exposure_times, dtype=float) if reuse_selected_full_reduction_fit: selected_good_flux = photometry_info.get('selected_fit_good_flux') @@ -26414,6 +26731,8 @@ def _main_impl(): goodFluxes = goodFluxes[relative_flux_mask] goodNormUnc = goodNormUnc[relative_flux_mask] goodAirmasses = goodAirmasses[relative_flux_mask] + if goodExposureTimes is not None: + goodExposureTimes = goodExposureTimes[relative_flux_mask] centroid_positions.update(x_targ=centroid_positions['x_targ'][relative_flux_mask], y_targ=centroid_positions['y_targ'][relative_flux_mask], @@ -26585,6 +26904,15 @@ def _main_impl(): goodFluxes = goodFluxes[relative_flux_mask] goodNormUnc = goodNormUnc[relative_flux_mask] goodAirmasses = goodAirmasses[relative_flux_mask] + try: + prereduced_exposure = float(exotic_infoDict.get('exposure', np.nan)) + except (TypeError, ValueError): + prereduced_exposure = np.nan + goodExposureTimes = ( + np.full(goodTimes.shape, prereduced_exposure, dtype=float) + if np.isfinite(prereduced_exposure) and prereduced_exposure > 0 + else None + ) goodFluxes, goodNormUnc, _ = normalize_flux_series_to_approximate_unity( goodFluxes, goodNormUnc, @@ -26737,6 +27065,7 @@ def _main_impl(): goodAirmasses, prior, mybounds, + exposure_times_seconds=goodExposureTimes, skip_airmass_fit=skip_final_airmass_fit, airmass_skip_note=airmass_skip_note, disable_vertical_flux_normalization=disable_vertical_flux_normalization, @@ -26793,7 +27122,11 @@ def _main_impl(): # estimate transit duration pars = dict(**myfit.parameters) times = np.linspace(np.min(myfit.time), np.max(myfit.time), 1000) - data_highres = transit(times, pars) + fit_transit_model = getattr(myfit, '_transit_model', None) + if callable(fit_transit_model): + data_highres = fit_transit_model(times, pars) + else: + data_highres = transit(times, pars) dt = np.diff(times).mean() durs = [] for r in range(1000): diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 22d0c572..10fe90a3 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -1176,16 +1176,25 @@ def test_is_comp_star_required_parses_values(): assert is_comp_star_required("n") is False -def test_mixed_exposure_times_force_comparison_star_requirement(): +def test_mixed_exposure_times_keep_target_only_allowed_with_scaling_warning(monkeypatch): import exotic.exotic as exotic_module + messages = [] + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, **kwargs: messages.append((message, kwargs)), + ) + assert exotic_module.exposure_time_spread_fraction([60.0, 60.3, 60.5]) < 0.01 assert not exotic_module.exposure_variation_requires_comp_star([60.0, 60.3, 60.5]) assert exotic_module.resolve_require_comp_star_for_exposure_times("n", [60.0, 60.3, 60.5]) is False assert exotic_module.exposure_time_spread_fraction([60.0, 61.0]) > 0.01 assert exotic_module.exposure_variation_requires_comp_star([60.0, 61.0]) - assert exotic_module.resolve_require_comp_star_for_exposure_times("n", [60.0, 61.0]) is True + assert exotic_module.resolve_require_comp_star_for_exposure_times("n", [60.0, 61.0]) is False + assert exotic_module.resolve_require_comp_star_for_exposure_times("y", [60.0, 61.0]) is True + assert any("scale source counts to a common exposure time" in message for message, _ in messages) def test_img_time_bjd_tdb_prefers_direct_mid_exposure_bjd(monkeypatch): @@ -6715,6 +6724,36 @@ def test_prepare_lightcurve_fit_input_series_uses_per_star_flux_errors(monkeypat assert np.allclose(prepared["comp_flux_error"], comp_error) +def test_prepare_lightcurve_fit_input_series_scales_target_only_counts_to_max_exposure(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + target_flux = np.array([100.0, 200.0, 200.0, 200.0, 300.0, 300.0]) + comp_flux = np.ones(6) + exposure_times = np.array([30.0, 60.0, 60.0, 60.0, 60.0, 60.0]) + + prepared = prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + np.linspace(1.0, 1.5, 6), + exposure_times_seconds=exposure_times, + gain_e_per_adu=2.0, + ) + + expected_flux = np.array([200.0, 200.0, 200.0, 200.0, 300.0, 300.0]) + expected_error = np.sqrt(target_flux / 2.0) * np.array([2.0, 1.0, 1.0, 1.0, 1.0, 1.0]) + + assert prepared["applied"] is True + assert prepared["debug_target_flux"] == pytest.approx(expected_flux) + assert prepared["target_flux"] == pytest.approx(expected_flux) + assert prepared["target_flux_error"] == pytest.approx(expected_error) + assert prepared["debug_relative_flux_error"] == pytest.approx(expected_error) + + def test_compute_photometry_noise_budget_includes_optional_terms(): config = { "gain_e_per_adu": 2.0, @@ -7005,6 +7044,68 @@ def fake_lc_fitter( assert captured["flags"] == [False] +def test_fit_lightcurve_forwards_exposure_times_to_fitter(monkeypatch): + captured = {} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + exposure_times_seconds=None, + ): + captured["times"] = np.asarray(times, dtype=float) + captured["exposure_times_seconds"] = None if exposure_times_seconds is None else np.asarray( + exposure_times_seconds, + dtype=float, + ) + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + exposure_times = np.array([60.0, 60.0, 90.0, 90.0, 120.0, 120.0]) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + exposure_times_seconds=exposure_times, + ) + + assert captured["times"] == pytest.approx(times) + assert captured["exposure_times_seconds"] == pytest.approx(exposure_times) + + def test_build_initial_ars_bounds_prefers_published_uncertainty_when_available(monkeypatch): import exotic.exotic as exotic_module diff --git a/tests/test_lazy_pylightcurve_imports.py b/tests/test_lazy_pylightcurve_imports.py index 46310428..46e01785 100644 --- a/tests/test_lazy_pylightcurve_imports.py +++ b/tests/test_lazy_pylightcurve_imports.py @@ -90,6 +90,90 @@ def test_imports_eagerly_load_pylightcurve_without_noise(): assert "LOUD-STDERR" not in result.stderr +def test_transit_supersamples_points_with_significant_exposure_smearing(): + script = textwrap.dedent( + """ + import sys + import tempfile + import types + from pathlib import Path + + import numpy as np + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + package_dir = root / "pylightcurve" + model_dir = package_dir / "models" + model_dir.mkdir(parents=True) + + (package_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "exoplanet_lc.py").write_text( + "import numpy as np\\n" + "def transit(*args, **kwargs):\\n" + " times = np.asarray(args[-1], dtype=float)\\n" + " return 1.0 + times ** 2\\n" + "def eclipse_mid_time(*args, **kwargs): return 0.0\\n", + encoding="utf-8", + ) + + sys.path.insert(0, str(root)) + for name in list(sys.modules): + if name == "exotic.api.elca" or name.startswith("pylightcurve"): + sys.modules.pop(name) + + fake_ultranest = types.ModuleType("ultranest") + fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) + sys.modules["ultranest"] = fake_ultranest + fake_plotting = types.ModuleType("plotting") + fake_plotting.corner = lambda *args, **kwargs: None + sys.modules["plotting"] = fake_plotting + sys.modules["exotic.api.plotting"] = fake_plotting + fake_ultranest_utils = types.ModuleType("ultranest_utils") + fake_ultranest_utils.run_reactive_sampler = lambda *args, **kwargs: None + sys.modules["ultranest_utils"] = fake_ultranest_utils + sys.modules["exotic.api.ultranest_utils"] = fake_ultranest_utils + + import exotic.api.elca as elca + + values = { + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "rprs": 0.1, + "per": 1.0, + "ars": 10.0, + "ecc": 0.0, + "inc": 90.0, + "omega": 90.0, + "tmid": 0.0, + elca.EXPOSURE_SMEARING_EXPOSURE_TIME_KEY: np.array([3600.0 / 86400.0]), + elca.EXPOSURE_SMEARING_SUPERSAMPLE_KEY: 5, + elca.EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY: 0.0, + } + + result = elca.transit(np.array([0.0]), values) + offsets = (np.arange(5, dtype=float) + 0.5) / 5.0 - 0.5 + expected = np.mean(1.0 + ((3600.0 / 86400.0) * offsets) ** 2) + assert np.allclose(result, [expected]) + assert result[0] > 1.0 + print("smearing-ok") + """ + ) + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr or result.stdout + assert "smearing-ok" in result.stdout + + def test_import_exotic_avoids_unused_astroquery_modules(): script = textwrap.dedent( """ From 33bc0be0bfe432f3288721a53468a58101f8582a Mon Sep 17 00:00:00 2001 From: Opus Date: Thu, 9 Jul 2026 18:13:14 +0000 Subject: [PATCH 077/116] Fix inverted eccentricity dependence in analytic transit duration (#1383) The (1 - e^2)/(1 + e sin w) factor belongs in the impact parameter (Winn 2010 eq. 7) but was also used as the arcsin normalization, and the external velocity factor sqrt(1 - e^2)/(1 + e sin w) (eq. 16) was missing. Net error: (1 + e sin w)^2 / (1 - e^2)^{3/2}, up to ~60x for Kepler-1704 b (a 6 h transit computed as 15 days), with the direction flipping with the sign of sin w. Circular orbits were unaffected. Fixes all three sites (transit_depth.transit_duration_days, exotic.transit_qc_geometry_contact_duration, exotic.estimate_transit_duration_from_prior_geometry) and adds the unit tests the module lacked: exact-Winn agreement across an e/omega/inc grid, a physical direction check (periastron transit shorter, apastron longer), and reproduction of NASA Exoplanet Archive published durations for Kepler-1704 b, HD 17156 b, and HD 80606 b to within 5 percent. Co-Authored-By: Claude Fable 5 --- exotic/exotic.py | 10 +-- exotic/transit_depth.py | 5 +- tests/test_transit_duration.py | 111 +++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 tests/test_transit_duration.py diff --git a/exotic/exotic.py b/exotic/exotic.py index db0c6771..a5742073 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -1226,11 +1226,12 @@ def transit_qc_geometry_contact_duration(parameters, contact_radius): if not np.isfinite(chord_sq) or chord_sq <= 0: return np.nan - argument = np.sqrt(chord_sq) / (impact_scale * sin_inc) + argument = np.sqrt(chord_sq) / (ars * sin_inc) if not np.isfinite(argument): return np.nan argument = float(np.clip(argument, -1.0, 1.0)) - duration = (period / np.pi) * np.arcsin(argument) + eccentric_speed_factor = np.sqrt(1.0 - ecc ** 2) / denominator + duration = (period / np.pi) * np.arcsin(argument) * eccentric_speed_factor return float(duration) if np.isfinite(duration) and duration > 0 else np.nan @@ -8406,9 +8407,10 @@ def estimate_transit_duration_from_prior_geometry(prior): if not np.isfinite(chord_sq) or chord_sq <= 0 or not np.isfinite(impact_scale) or impact_scale <= 0: return np.nan - argument = np.sqrt(chord_sq) / (impact_scale * sin_inc) + argument = np.sqrt(chord_sq) / (ars * sin_inc) argument = float(np.clip(argument, -1.0, 1.0)) - duration = (period / np.pi) * np.arcsin(argument) + eccentric_speed_factor = np.sqrt(1.0 - ecc ** 2) / max(np.finfo(float).eps, 1.0 + ecc * np.sin(omega)) + duration = (period / np.pi) * np.arcsin(argument) * eccentric_speed_factor return float(duration) if np.isfinite(duration) and duration > 0 else np.nan diff --git a/exotic/transit_depth.py b/exotic/transit_depth.py index 021e05f3..e1a55be9 100644 --- a/exotic/transit_depth.py +++ b/exotic/transit_depth.py @@ -217,9 +217,10 @@ def transit_duration_days(parameters): if not np.isfinite(chord_sq) or chord_sq <= 0 or impact_scale <= 0: return np.nan - argument = math.sqrt(chord_sq) / (impact_scale * sin_inc) + argument = math.sqrt(chord_sq) / (ars * sin_inc) argument = float(np.clip(argument, -1.0, 1.0)) - duration = (period / math.pi) * math.asin(argument) + eccentric_speed_factor = math.sqrt(1.0 - ecc ** 2) / denominator + duration = (period / math.pi) * math.asin(argument) * eccentric_speed_factor return float(duration) if np.isfinite(duration) and duration > 0 else np.nan diff --git a/tests/test_transit_duration.py b/tests/test_transit_duration.py new file mode 100644 index 00000000..9ce1dc23 --- /dev/null +++ b/tests/test_transit_duration.py @@ -0,0 +1,111 @@ +"""Tests for the analytic transit-duration formula (issue #1383). + +The eccentric duration must follow Winn (2010), "Transits and Occultations", +arXiv:1001.2010: the impact parameter uses the eccentricity factor +(1 - e^2)/(1 + e sin w) (eq. 7), the arcsin argument is normalized by the +plain a/R* (eq. 14), and the whole expression is multiplied by the velocity +factor sqrt(1 - e^2)/(1 + e sin w) (eq. 16). Putting the eq.-7 factor inside +the arcsin instead inverts the eccentricity dependence: periastron-at-transit +(fastest planet, shortest transit) comes out longest, and the error grows as +(1 + e sin w)^2 / (1 - e^2)^{3/2}, reaching ~60x for Kepler-1704 b. +""" + +import math + +import numpy as np +import pytest + +from exotic.transit_depth import transit_duration_days + + +def winn_2010_duration_days(period, ars, inc_deg, rprs, ecc, omega_deg): + """Reference implementation: Winn (2010) eqs. 7, 14, 16.""" + inc = math.radians(inc_deg) + omega = math.radians(omega_deg) + denom = 1.0 + ecc * math.sin(omega) + b = ars * math.cos(inc) * (1.0 - ecc ** 2) / denom + chord_sq = (1.0 + rprs) ** 2 - b ** 2 + if chord_sq <= 0: + return float("nan") + argument = min(math.sqrt(chord_sq) / (ars * math.sin(inc)), 1.0) + velocity_factor = math.sqrt(1.0 - ecc ** 2) / denom + return (period / math.pi) * math.asin(argument) * velocity_factor + + +def duration_params(period, ars, inc, rprs, ecc, omega): + return { + "per": period, + "ars": ars, + "inc": inc, + "rprs": rprs, + "ecc": ecc, + "omega": omega, + } + + +def test_circular_matches_winn_exactly(): + params = duration_params(3.0, 10.0, 90.0, 0.1, 0.0, 90.0) + expected = winn_2010_duration_days(3.0, 10.0, 90.0, 0.1, 0.0, 90.0) + assert transit_duration_days(params) == pytest.approx(expected, rel=1e-12) + + +@pytest.mark.parametrize("ecc", [0.1, 0.3, 0.5, 0.7, 0.9]) +@pytest.mark.parametrize("omega", [0.0, 45.0, 90.0, 135.0, 180.0, 270.0]) +@pytest.mark.parametrize("inc", [90.0, 88.0, 85.0]) +def test_eccentric_matches_winn(ecc, omega, inc): + params = duration_params(3.0, 10.0, inc, 0.1, ecc, omega) + expected = winn_2010_duration_days(3.0, 10.0, inc, 0.1, ecc, omega) + result = transit_duration_days(params) + if math.isnan(expected): + assert math.isnan(result) + else: + assert result == pytest.approx(expected, rel=1e-9) + + +def test_periastron_transit_is_shorter_and_apastron_longer(): + circular = transit_duration_days(duration_params(3.0, 10.0, 90.0, 0.1, 0.0, 90.0)) + periastron = transit_duration_days(duration_params(3.0, 10.0, 90.0, 0.1, 0.5, 90.0)) + apastron = transit_duration_days(duration_params(3.0, 10.0, 90.0, 0.1, 0.5, 270.0)) + assert periastron < circular < apastron + + +@pytest.mark.parametrize( + "name, period, ars, inc, rprs, ecc, omega, published_hours", + [ + # NASA Exoplanet Archive `ps` default rows, pl_trandur in hours. + ("Kepler-1704 b", 988.88112, 256.4, 89.00, 0.0644, 0.920, 82.40, 6.007), + ("HD 17156 b", 21.2164294, 23.11, 86.51, 0.07412, 0.6772, 122.06, 3.1505), + ("HD 80606 b", 111.436765, 94.452, 89.24, 0.1009, 0.93183, -58.887, 11.98), + ], +) +def test_reproduces_published_durations(name, period, ars, inc, rprs, ecc, omega, published_hours): + params = duration_params(period, ars, inc, rprs, ecc, omega) + hours = transit_duration_days(params) * 24.0 + assert hours == pytest.approx(published_hours, rel=0.05), name + + +def _exotic_main_module(): + return pytest.importorskip( + "exotic.exotic", reason="exotic.exotic imports the full pipeline dependency stack" + ) + + +def test_qc_contact_duration_matches_winn(): + exotic_main = _exotic_main_module() + params = {"per": 3.0, "ars": 10.0, "inc": 89.0, "ecc": 0.5, "omega": 90.0} + result = exotic_main.transit_qc_geometry_contact_duration(params, 1.1) + inc = math.radians(89.0) + omega = math.radians(90.0) + denom = 1.0 + 0.5 * math.sin(omega) + b = 10.0 * math.cos(inc) * (1.0 - 0.25) / denom + argument = min(math.sqrt(1.1 ** 2 - b ** 2) / (10.0 * math.sin(inc)), 1.0) + expected = (3.0 / math.pi) * math.asin(argument) * (math.sqrt(0.75) / denom) + assert result == pytest.approx(expected, rel=1e-9) + + +def test_prior_geometry_duration_matches_winn(): + exotic_main = _exotic_main_module() + prior = {"per": 3.0, "ars": 10.0, "inc": 88.0, "rprs": 0.1, "ecc": 0.4, "omega": 120.0} + result = exotic_main.estimate_transit_duration_from_prior_geometry(prior) + expected = winn_2010_duration_days(3.0, 10.0, 88.0, 0.1, 0.4, 120.0) + assert result == pytest.approx(expected, rel=1e-9) From 8296ec56a7d34f946f0a7a1e2c39850a7e920066 Mon Sep 17 00:00:00 2001 From: mfitzasp Date: Sat, 11 Jul 2026 09:34:09 +1000 Subject: [PATCH 078/116] ldtk ftp fallback and missing a2 error --- exotic/api/gael_ld.py | 108 +++++++++++++++++++++++++++++++ exotic/exotic.py | 90 +++++++++++++++++++++++++- exotic/output_files.py | 43 ++++++++---- tests/test_exotic_rprs_retry.py | 98 ++++++++++++++++++++++++++++ tests/test_ldtk_http_fallback.py | 59 ++++++++++++++++- 5 files changed, 382 insertions(+), 16 deletions(-) diff --git a/exotic/api/gael_ld.py b/exotic/api/gael_ld.py index a408c53b..4f253a5e 100644 --- a/exotic/api/gael_ld.py +++ b/exotic/api/gael_ld.py @@ -42,8 +42,10 @@ import matplotlib.pyplot as plt import numpy as np import os +from html.parser import HTMLParser from pathlib import Path from urllib.parse import quote +from urllib.parse import unquote import requests @@ -55,6 +57,8 @@ ) _LDTK_HTTP_FALLBACK_ENV = "EXOTIC_LDTK_FALLBACK_BASE_URL" _LDTK_DOWNLOAD_TIMEOUT = (10, 120) +_LDTK_INDEX_TIMEOUT = (10, 60) +_LDTK_ORIGINAL_GET_SERVER_FILE_LIST = None _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES = None @@ -93,6 +97,90 @@ def _ldtk_http_fallback_url(base_url, client, ldtk_file): return f"{base_url}/{path}" +def _ldtk_http_fallback_index_url(base_url, *parts): + path = _quote_url_path(*parts) + if path: + return f"{base_url}/{path}/" + return f"{base_url}/" + + +class _HrefParser(HTMLParser): + def __init__(self): + super().__init__() + self.hrefs = [] + + def handle_starttag(self, tag, attrs): + if tag.lower() != "a": + return + for name, value in attrs: + if name.lower() == "href" and value: + self.hrefs.append(value) + return + + +def _http_index_names(url): + response = requests.get(url, timeout=_LDTK_INDEX_TIMEOUT) + try: + response.raise_for_status() + parser = _HrefParser() + parser.feed(response.text) + finally: + response.close() + + names = [] + for href in parser.hrefs: + href = unquote(href.split("?", 1)[0].split("#", 1)[0]).strip("/") + if not href or href in (".", "..") or "/" in href: + continue + names.append(href) + return names + + +def _get_ldtk_server_file_list_from_http_mirror(client, base_url): + root_url = _ldtk_http_fallback_index_url(base_url, client.edir) + zdirs = sorted(name for name in _http_index_names(root_url) if ".txt" not in name.lower()) + if not zdirs: + raise RuntimeError(f"No PHOENIX metallicity directories found at {root_url}") + + log.warning( + "Trying PHOENIX HTTP fallback index at %s for %d metallicity directories.", + base_url, + len(zdirs), + ) + + files_in_server = {} + for zdir in zdirs: + zdir_url = _ldtk_http_fallback_index_url(base_url, client.edir, zdir) + files_in_server[zdir] = sorted( + name for name in _http_index_names(zdir_url) if ".txt" not in name.lower() + ) + return files_in_server + + +def _get_ldtk_server_file_list_from_http(client): + base_urls = _ldtk_http_fallback_base_urls() + if not base_urls: + raise RuntimeError( + f"LDTk FTP file listing failed and {_LDTK_HTTP_FALLBACK_ENV} is empty, " + "so EXOTIC cannot try the HTTP PHOENIX fallback." + ) + + log.warning( + "LDTk FTP file listing failed; trying PHOENIX HTTP fallback mirrors in order: %s", + ", ".join(base_urls), + ) + + last_error = None + for base_url in base_urls: + try: + return _get_ldtk_server_file_list_from_http_mirror(client, base_url) + except Exception as mirror_error: + last_error = mirror_error + log.warning("PHOENIX HTTP fallback index failed at %s: %s", base_url, mirror_error) + + raise RuntimeError("All PHOENIX HTTP fallback indexes failed.") from last_error + + def _download_file(url, local_path): local_path = Path(local_path) local_path.parent.mkdir(parents=True, exist_ok=True) @@ -167,6 +255,7 @@ def _download_ldtk_uncached_files_from_http(client, force=False): def _install_ldtk_http_fallback(): + global _LDTK_ORIGINAL_GET_SERVER_FILE_LIST global _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES try: @@ -174,6 +263,25 @@ def _install_ldtk_http_fallback(): except Exception: return + if not getattr(Client.get_server_file_list, "_exotic_http_fallback", False): + _LDTK_ORIGINAL_GET_SERVER_FILE_LIST = Client.get_server_file_list + + def get_server_file_list_with_http_fallback(self): + try: + return _LDTK_ORIGINAL_GET_SERVER_FILE_LIST(self) + except Exception as ftp_error: + try: + log.warning("LDTk FTP file listing failed with %s", ftp_error) + return _get_ldtk_server_file_list_from_http(self) + except Exception as fallback_error: + raise RuntimeError( + "LDTk could not list PHOENIX files from the default FTP server " + "or the EXOTIC HTTP fallback." + ) from fallback_error + + get_server_file_list_with_http_fallback._exotic_http_fallback = True + Client.get_server_file_list = get_server_file_list_with_http_fallback + if getattr(Client.download_uncached_files, "_exotic_http_fallback", False): return diff --git a/exotic/exotic.py b/exotic/exotic.py index db0c6771..68f83830 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -3601,6 +3601,35 @@ def partial_transit_geometry_prior_assumption_fixed_error(key, prior, search_res return 0.0 +def estimate_fixed_airmass_coefficient_error(flux_values, flux_errors, airmass): + flux_values = np.asarray([] if flux_values is None else flux_values, dtype=float) + flux_errors = np.asarray([] if flux_errors is None else flux_errors, dtype=float) + airmass = np.asarray([] if airmass is None else airmass, dtype=float) + if not (flux_values.shape == flux_errors.shape == airmass.shape): + return None + + finite = ( + np.isfinite(flux_values) + & (flux_values > 0) + & np.isfinite(flux_errors) + & (flux_errors > 0) + & np.isfinite(airmass) + ) + if int(np.count_nonzero(finite)) < 2: + return None + + airmass_span_value = np.nanmax(airmass[finite]) - np.nanmin(airmass[finite]) + if not np.isfinite(airmass_span_value) or airmass_span_value <= 0: + return None + + relative_errors = flux_errors[finite] / np.maximum(flux_values[finite], np.finfo(float).eps) + relative_error = float(np.nanmedian(relative_errors)) + if not np.isfinite(relative_error) or relative_error < 0: + return None + + return float(relative_error / airmass_span_value) + + def partial_transit_geometry_prior_assumption_note(mode, assessment, sampled_parameters): observed_segment = assessment.get('observed_segment') or 'partial transit' pre_points = int(assessment.get('pre_ingress_points', 0) or 0) @@ -3637,6 +3666,7 @@ def apply_partial_transit_geometry_prior_assumption( bounds, assessment, flux_values=None, + flux_errors=None, airmass=None, fixed_parameter_errors=None, search_restriction_prior=None, @@ -3683,6 +3713,14 @@ def apply_partial_transit_geometry_prior_assumption( else: for key in ('a0', 'a1', 'a2'): local_bounds.pop(key, None) + if 'a2' in local_prior and 'a2' not in local_fixed_errors: + a2_error = estimate_fixed_airmass_coefficient_error( + flux_values, + flux_errors, + airmass, + ) + if a2_error is not None: + local_fixed_errors['a2'] = a2_error sampled_parameters = list(local_bounds.keys()) payload.update({ @@ -4504,6 +4542,7 @@ def coerce_fixed_baseline_error(value, default=None): def baseline_fixed_errors_from_fit(fit): + parameters = getattr(fit, 'parameters', {}) if fit is not None else {} errors = getattr(fit, 'errors', {}) if fit is not None else {} fixed_errors = {} if isinstance(errors, dict): @@ -4513,6 +4552,18 @@ def baseline_fixed_errors_from_fit(fit): fixed_errors[key] = value if 'a0' in fixed_errors and 'a1' not in fixed_errors: fixed_errors['a1'] = fixed_errors['a0'] + if ( + isinstance(parameters, dict) + and 'a2' in parameters + and 'a2' not in fixed_errors + ): + a2_error = estimate_fixed_airmass_coefficient_error( + getattr(fit, 'data', None), + getattr(fit, 'dataerr', None), + getattr(fit, 'airmass', None), + ) + if a2_error is not None: + fixed_errors['a2'] = a2_error return fixed_errors @@ -6464,6 +6515,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): local_bounds, pre_ultranest_coverage_assessment, flux_values=flux_values, + flux_errors=flux_errors, airmass=airmass, fixed_parameter_errors=effective_fixed_parameter_errors, search_restriction_prior=restriction_reference_prior, @@ -6522,6 +6574,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): current_bounds, pre_ultranest_coverage_assessment, flux_values=flux_values, + flux_errors=flux_errors, airmass=airmass, fixed_parameter_errors=base_fixed_parameter_errors, search_restriction_prior=restriction_reference_prior, @@ -10051,7 +10104,12 @@ def residuals(params): if covariance is not None and fit_a2 and covariance.shape[0] >= 2: a2_error = float(np.sqrt(max(covariance[1, 1], 0.0))) else: - a2_error = 0.0 if not fit_a2 else np.nan + a2_error = ( + estimate_fixed_airmass_coefficient_error(y, yerr, airmass[finite_mask]) + if not fit_a2 else np.nan + ) + if a2_error is None: + a2_error = 0.0 if not np.isfinite(a0_error) or a0_error <= 0: a0_error = float(np.nanmedian(yerr)) @@ -27245,8 +27303,34 @@ def _main_impl(): if getattr(myfit, 'airmass_fit_skipped', False): log_info(f" Airmass correction: {myfit.airmass_correction_note}") else: - log_info(f" Airmass coefficient 1: {round_to_2(myfit.parameters['a1'], myfit.errors['a1'])} +/- {round_to_2(myfit.errors['a1'])}") - log_info(f" Airmass coefficient 2: {round_to_2(myfit.parameters['a2'], myfit.errors['a2'])} +/- {round_to_2(myfit.errors['a2'])}") + fit_parameters = getattr(myfit, 'parameters', {}) or {} + fit_errors = getattr(myfit, 'errors', {}) or {} + airmass_scale_key = 'a1' if 'a1' in fit_parameters else 'a0' + if airmass_scale_key in fit_parameters: + airmass_scale_error = fit_errors.get(airmass_scale_key) + if airmass_scale_error is not None and np.isfinite(airmass_scale_error): + log_info( + f" Airmass coefficient 1: " + f"{round_to_2(fit_parameters[airmass_scale_key], airmass_scale_error)} " + f"+/- {round_to_2(airmass_scale_error)}" + ) + else: + log_info( + f" Airmass coefficient 1: " + f"{round_to_2(fit_parameters[airmass_scale_key])} (fixed; uncertainty unavailable)" + ) + if 'a2' in fit_parameters: + a2_error = fit_errors.get('a2') + if a2_error is not None and np.isfinite(a2_error): + log_info( + f" Airmass coefficient 2: " + f"{round_to_2(fit_parameters['a2'], a2_error)} +/- {round_to_2(a2_error)}" + ) + else: + log_info( + f" Airmass coefficient 2: " + f"{round_to_2(fit_parameters['a2'])} (fixed; uncertainty unavailable)" + ) if isinstance(transit_qc, dict) and transit_qc: residual_scatter = transit_qc.get('residual_scatter', np.nan) if np.isfinite(residual_scatter): diff --git a/exotic/output_files.py b/exotic/output_files.py index f03f8b7c..275208d9 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1806,19 +1806,38 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor ) else: if 'a0' in self.fit.parameters: - params_num["Baseline flux (a0)"] = ( - f"{round_to_2(self.fit.parameters['a0'], self.fit.errors['a0'])} +/- " - f"{round_to_2(self.fit.errors['a0'])}" - ) + a0_error = self.fit.errors.get('a0') if isinstance(self.fit.errors, dict) else None + if a0_error is not None and np.isfinite(a0_error): + params_num["Baseline flux (a0)"] = ( + f"{round_to_2(self.fit.parameters['a0'], a0_error)} +/- " + f"{round_to_2(a0_error)}" + ) + else: + params_num["Baseline flux (a0)"] = ( + f"{round_to_2(self.fit.parameters['a0'])} (fixed; uncertainty unavailable)" + ) else: - params_num["Flux normalization (a1)"] = ( - f"{round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- " - f"{round_to_2(self.fit.errors['a1'])}" - ) - params_num["Airmass coefficient 2 (a2)"] = ( - f"{round_to_2(self.fit.parameters['a2'], self.fit.errors['a2'])} +/- " - f"{round_to_2(self.fit.errors['a2'])}" - ) + a1_error = self.fit.errors.get('a1') if isinstance(self.fit.errors, dict) else None + if a1_error is not None and np.isfinite(a1_error): + params_num["Flux normalization (a1)"] = ( + f"{round_to_2(self.fit.parameters['a1'], a1_error)} +/- " + f"{round_to_2(a1_error)}" + ) + else: + params_num["Flux normalization (a1)"] = ( + f"{round_to_2(self.fit.parameters['a1'])} (fixed; uncertainty unavailable)" + ) + if 'a2' in self.fit.parameters: + a2_error = self.fit.errors.get('a2') if isinstance(self.fit.errors, dict) else None + if a2_error is not None and np.isfinite(a2_error): + params_num["Airmass coefficient 2 (a2)"] = ( + f"{round_to_2(self.fit.parameters['a2'], a2_error)} +/- " + f"{round_to_2(a2_error)}" + ) + else: + params_num["Airmass coefficient 2 (a2)"] = ( + f"{round_to_2(self.fit.parameters['a2'])} (fixed; uncertainty unavailable)" + ) if isinstance(transit_qc, dict) and transit_qc: qc_status = transit_qc.get('status') diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 1d956140..7d70b56b 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -983,6 +983,103 @@ def fake_run_nested( assert "a2" not in captured["bounds"] +def test_selected_fast_candidate_final_refit_estimates_missing_fixed_a2_error(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "selected_final_live_point_target", lambda *args, **kwargs: (200, None)) + captured = {} + + def fake_run_nested( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["fixed_parameter_errors"] = dict(kwargs.get("fixed_parameter_errors", {})) + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters=dict(prior), + errors=dict(kwargs.get("fixed_parameter_errors", {})), + residuals=np.zeros(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + duration_measured=0.04, + duration_expected=0.04, + transit_qc={"status": "pass", "summary": "ok"}, + transit_qc_status="pass", + ) + fit.get_parameter_posterior_samples = lambda key: np.linspace(0.0, 1.0, 1500) + return fit + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + parameters={ + "rprs": 0.1, + "ars": 10.0, + "per": 1.0, + "tmid": 0.5, + "inc": 89.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.03, + "a1": 1.03, + "a2": 0.12, + }, + errors={"a0": 0.02, "a1": 0.02, "rprs": 0.001, "tmid": 0.001, "ars": 0.1}, + data=np.ones(40, dtype=float), + dataerr=np.full(40, 0.01, dtype=float), + airmass=np.linspace(1.0, 1.4, 40), + bounds={ + "rprs": [0.05, 0.15], + "tmid": [0.49, 0.51], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + }, + ) + times = np.linspace(0.0, 1.0, 80) + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": np.ones(80), + "good_unc": np.full(80, 0.01), + "good_airmass": np.linspace(1.0, 1.3, 80), + "good_jd_times": 2460000.0 + times, + } + + returned = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + { + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "rprs": 0.1, + "aRs": 10.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + detrend_on_outoftransit_baseline=False, + ) + + assert returned is not None + assert captured["fixed_parameter_errors"]["a0"] == pytest.approx(0.02) + assert captured["fixed_parameter_errors"]["a2"] == pytest.approx(0.025) + + def test_selected_fast_candidate_final_refit_reruns_after_residual_rejection(monkeypatch): import exotic.exotic as exotic_module @@ -1755,6 +1852,7 @@ def fake_lc_fitter( assert captured["fixed_parameter_errors"]["rprs"] == pytest.approx(0.002) assert captured["fixed_parameter_errors"]["ars"] == pytest.approx(0.3) assert captured["fixed_parameter_errors"]["inc"] == pytest.approx(0.4) + assert captured["fixed_parameter_errors"]["a2"] == pytest.approx(0.025) assert fit.partial_transit_geometry_prior_assumption_applied is True assert fit.partial_transit_geometry_prior_assumption_mode == "tmid_only" assert fit.partial_transit_geometry_prior_assumption_sampled_parameters == ["tmid"] diff --git a/tests/test_ldtk_http_fallback.py b/tests/test_ldtk_http_fallback.py index 57010e7e..25dd4745 100644 --- a/tests/test_ldtk_http_fallback.py +++ b/tests/test_ldtk_http_fallback.py @@ -8,8 +8,9 @@ class DummyResponse: - def __init__(self, chunks): + def __init__(self, chunks=None, text=""): self._chunks = chunks + self.text = text self.closed = False def raise_for_status(self): @@ -113,3 +114,59 @@ def fake_fallback(self, force=False): assert Client.download_uncached_files(client, force=True) is False assert calls == [("ftp", True), ("http", True)] + + +def test_ldtk_file_list_wrapper_uses_http_fallback_when_ftp_listing_fails(monkeypatch): + from ldtk.client import Client + + class ClientStub: + edir = "SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011" + + calls = [] + requested_urls = [] + + def fake_original(self): + calls.append("ftp") + raise TimeoutError("ftp listing timed out") + + def fake_get(url, timeout): + requested_urls.append(url) + if url.endswith("PHOENIX-ACES-AGSS-COND-SPECINT-2011/"): + return DummyResponse(text=""" + ../ + README.txt + Z+0.5/ + Z-0.0/ + """) + if url.endswith("Z%2B0.5/"): + return DummyResponse(text=""" + ../ + + lte02300+0.00+0.5.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits + + """) + if url.endswith("Z-0.0/"): + return DummyResponse(text=""" + ../ + + lte02300-0.00-0.0.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits + + """) + raise AssertionError(f"unexpected URL: {url}") + + monkeypatch.setenv(gael_ld._LDTK_HTTP_FALLBACK_ENV, "https://mirror.example/PHOENIX") + monkeypatch.setattr(gael_ld, "_LDTK_ORIGINAL_GET_SERVER_FILE_LIST", fake_original) + monkeypatch.setattr(gael_ld.requests, "get", fake_get) + + files = Client.get_server_file_list(ClientStub()) + + assert calls == ["ftp"] + assert requested_urls == [ + "https://mirror.example/PHOENIX/SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011/", + "https://mirror.example/PHOENIX/SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011/Z%2B0.5/", + "https://mirror.example/PHOENIX/SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011/Z-0.0/", + ] + assert files == { + "Z+0.5": ["lte02300+0.00+0.5.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits"], + "Z-0.0": ["lte02300-0.00-0.0.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits"], + } From 118e3b056d7734c6aacc79b5bf6aebb90fce7227 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 13 Jul 2026 14:08:47 +1000 Subject: [PATCH 079/116] reject targets and comps by saturation. --- exotic/exotic.py | 1621 +++++++++++++++++++++++++-- exotic/exotic_gui.py | 20 + exotic/inputs.py | 26 + exotic/output_files.py | 87 +- exotic/plate_status.py | 10 + exotic/plots.py | 75 ++ inits.json | 8 + tests/test_exotic_proper_motion.py | 125 ++- tests/test_inputs.py | 70 ++ tests/test_nextastro_variability.py | 83 ++ tests/test_output_files.py | 46 + tests/test_plots.py | 36 + 12 files changed, 2121 insertions(+), 86 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 68f83830..ad640608 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -71,6 +71,7 @@ import traceback from concurrent.futures import ProcessPoolExecutor as _ProcessPoolExecutor, ThreadPoolExecutor, as_completed from time import sleep, perf_counter +from types import SimpleNamespace # Image alignment import import astroalign as aa aa.PIXEL_TOL = 1 @@ -244,6 +245,13 @@ RELATIVE_FLUX_MAX = 2.0 # Legacy threshold retained for compatibility; no longer used as a hard rejection cap. AIRMASS_FLAT_RANGE_THRESHOLD = 0.05 LIGHTCURVE_MIN_VALID_POINTS = 5 +STELLAR_VARIABILITY_ONLY_DEFAULT = False +REJECT_OVEREXPOSED_STARS_DEFAULT = True +SATURATION_VALUE_DEFAULT = 65535.0 +OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT = 0.9 +MICROOBSERVATORY_TELESCOP_SATURATION_VALUES = { + 'cecilia': 4096.0, +} COMPARISON_STAR_MIN_COVERAGE_FRACTION = 0.8 COMPARISON_STAR_MIN_VALID_FRAMES = 5 COMPARISON_STAR_COVERAGE_SIGMA = 3.0 # Legacy constant; coverage rejection is fraction-based. @@ -291,7 +299,7 @@ NOISE_BUDGET_SKY_MEDIAN_VARIANCE_FACTOR = np.pi / 2.0 SCINTILLATION_COEFFICIENT_DEFAULT = 0.09 PSF_EFFECTIVE_NOISE_AREA_FACTOR = 4.0 * np.pi -NOISE_GAIN_HEADER_KEYS = ('GAIN', 'EGAIN', 'EPERADU', 'E_PER_ADU', 'GAIN_EAD', 'CCDGAIN') +NOISE_GAIN_HEADER_KEYS = ('EGAIN', 'EPERADU', 'E_PER_ADU', 'GAIN_EAD', 'CCDGAIN', 'GAIN') NOISE_READ_HEADER_KEYS = ('RDNOISE', 'READNOI', 'READNOIS', 'READNSE', 'RN_E', 'RON') NOISE_DARK_HEADER_KEYS = ('DARKCUR', 'DARKCURR', 'DARKRATE', 'DCURR', 'DARK_EPS', 'PBDKCURR') NOISE_FLAT_HEADER_KEYS = ('FLATERR', 'FLATFR', 'FLATFRAC', 'FFERR', 'FLATUNC') @@ -3521,6 +3529,362 @@ def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_ return result +def stellar_variability_scatter_from_flux(flux_values): + flux_values = np.asarray(flux_values, dtype=float) + valid = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: + return np.nan + values = flux_values[valid] + center = bn.nanmedian(values) + if not np.isfinite(center): + return np.nan + scatter = robust_scatter(values - center) + return float(scatter) if np.isfinite(scatter) and scatter >= 0 else np.nan + + +def prepare_stellar_variability_only_direct_series(times, target_flux, comp_flux, airmass, + jd_times=None, target_flux_error=None, + comp_flux_error=None, + exposure_times_seconds=None, + inherited_diagnostics=None): + result = { + 'applied': False, + 'failure_reason': ( + "the raw comparison-candidate photometry did not yield a usable " + "stellar-variability light curve." + ), + 'filter_diagnostics': list(inherited_diagnostics or []), + 'time': np.array([], dtype=float), + 'flux': np.array([], dtype=float), + 'unc': np.array([], dtype=float), + 'airmass': np.array([], dtype=float), + 'jd_time': np.array([], dtype=float), + 'exposure_time_seconds': None, + 'target_flux': np.array([], dtype=float), + 'comp_flux': np.array([], dtype=float), + 'target_flux_error': np.array([], dtype=float), + 'comp_flux_error': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), + } + + times = np.asarray(times, dtype=float).reshape(-1) + target_flux = np.asarray(target_flux, dtype=float).reshape(-1) + comp_flux = np.asarray(comp_flux, dtype=float).reshape(-1) + airmass = np.asarray(airmass, dtype=float).reshape(-1) + if jd_times is None: + jd_times = times + jd_times = np.asarray(jd_times, dtype=float).reshape(-1) + if not (times.shape == target_flux.shape == comp_flux.shape == airmass.shape == jd_times.shape): + result['failure_reason'] = "stellar-variability input arrays did not have matching lengths." + return result + + if target_flux_error is None: + target_flux_error = np.sqrt(np.clip(np.abs(target_flux), 1.0, None)) + target_flux_error = np.asarray(target_flux_error, dtype=float).reshape(-1) + if target_flux_error.shape != target_flux.shape: + target_flux_error = np.sqrt(np.clip(np.abs(target_flux), 1.0, None)) + if comp_flux_error is None: + comp_flux_error = np.sqrt(np.clip(np.abs(comp_flux), 1.0, None)) + comp_flux_error = np.asarray(comp_flux_error, dtype=float).reshape(-1) + if comp_flux_error.shape != comp_flux.shape: + comp_flux_error = np.sqrt(np.clip(np.abs(comp_flux), 1.0, None)) + + exposure_times = None + if exposure_times_seconds is not None: + exposure_times = np.asarray(exposure_times_seconds, dtype=float).reshape(-1) + if exposure_times.shape != times.shape: + exposure_times = None + + valid = ( + np.isfinite(times) + & np.isfinite(target_flux) + & np.isfinite(comp_flux) + & np.isfinite(airmass) + & (target_flux > 0) + & (comp_flux > 0) + ) + diagnostic = build_time_rejection_diagnostic( + "Stellar-variability direct finite/positive filter", + times, + valid, + note=( + "Dropped non-finite or non-positive target/reference photometry while building a " + "stellar-variability-only light curve without transit fitting." + ), + ) + if diagnostic is not None: + result['filter_diagnostics'].append(diagnostic) + if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: + result['failure_reason'] = ( + "too few valid points remained after finite/positive filtering for " + "stellar-variability-only analysis." + ) + return result + + source_indices = np.arange(times.shape[0], dtype=int) + times = times[valid] + target_flux = target_flux[valid] + comp_flux = comp_flux[valid] + airmass = airmass[valid] + jd_times = jd_times[valid] + target_flux_error = target_flux_error[valid] + comp_flux_error = comp_flux_error[valid] + source_indices = source_indices[valid] + if exposure_times is not None: + exposure_times = exposure_times[valid] + + raw_ratio = target_flux / comp_flux + relative_unc = np.abs(raw_ratio) * np.sqrt( + (target_flux_error / target_flux) ** 2 + + (comp_flux_error / comp_flux) ** 2 + ) + positive_unc = relative_unc[np.isfinite(relative_unc) & (relative_unc > 0)] + fallback_unc = float(np.nanmedian(positive_unc)) if positive_unc.size else 1.0e-6 + relative_unc = np.where(np.isfinite(relative_unc) & (relative_unc > 0), relative_unc, fallback_unc) + norm_flux, norm_unc, _ = normalize_flux_series_to_approximate_unity(raw_ratio, relative_unc) + + relative_flux_mask = relative_flux_filter_mask(norm_flux) & np.isfinite(norm_unc) & (norm_unc > 0) + diagnostic = build_time_rejection_diagnostic( + "Stellar-variability direct relative-flux filter", + times, + relative_flux_mask, + note="Dropped invalid normalized target/reference flux values before stellar-variability analysis.", + ) + if diagnostic is not None: + result['filter_diagnostics'].append(diagnostic) + if np.count_nonzero(relative_flux_mask) < LIGHTCURVE_MIN_VALID_POINTS: + result['failure_reason'] = ( + "too few valid normalized target/reference points remained for " + "stellar-variability-only analysis." + ) + return result + + result.update({ + 'applied': True, + 'failure_reason': None, + 'time': times[relative_flux_mask], + 'flux': norm_flux[relative_flux_mask], + 'unc': norm_unc[relative_flux_mask], + 'airmass': airmass[relative_flux_mask], + 'jd_time': jd_times[relative_flux_mask], + 'exposure_time_seconds': ( + None if exposure_times is None else exposure_times[relative_flux_mask] + ), + 'target_flux': target_flux[relative_flux_mask], + 'comp_flux': comp_flux[relative_flux_mask], + 'target_flux_error': target_flux_error[relative_flux_mask], + 'comp_flux_error': comp_flux_error[relative_flux_mask], + 'source_indices': source_indices[relative_flux_mask], + }) + return result + + +def build_stellar_variability_only_lightcurve( + prepared, + p_dict, + filter_diagnostics=None, + comp_index=None, + comp_label=None, + comp_position=None, + method_label=None, + plot_time_range=None, +): + if prepared is None or not prepared.get('applied'): + return None + + base_times = np.asarray(prepared.get('time'), dtype=float) + oot_mask, exclusion_summary = stellar_variability_out_of_transit_mask(base_times, p_dict) + filter_diagnostics = list(filter_diagnostics or []) + exclusion_diagnostic = build_time_rejection_diagnostic( + "Stellar-variability transit-window exclusion", + base_times, + oot_mask, + note=exclusion_summary.get('note'), + ) + if exclusion_diagnostic is not None: + filter_diagnostics.append(exclusion_diagnostic) + + if np.count_nonzero(oot_mask) < LIGHTCURVE_MIN_VALID_POINTS: + return None + + time = base_times[oot_mask] + data = np.asarray(prepared.get('flux'), dtype=float)[oot_mask] + dataerr = np.asarray(prepared.get('unc'), dtype=float)[oot_mask] + airmass = np.asarray(prepared.get('airmass'), dtype=float)[oot_mask] + jd_times = np.asarray(prepared.get('jd_time'), dtype=float)[oot_mask] + source_indices = np.asarray(prepared.get('source_indices'), dtype=int)[oot_mask] + exposure_times_seconds = prepared.get('exposure_time_seconds') + if exposure_times_seconds is not None: + exposure_times_seconds = np.asarray(exposure_times_seconds, dtype=float)[oot_mask] + exposure_times_days = exposure_times_seconds / 86400.0 + else: + exposure_times_days = None + + finite = ( + np.isfinite(time) + & np.isfinite(data) + & np.isfinite(dataerr) + & (data > 0) + & (dataerr > 0) + & np.isfinite(airmass) + ) + if np.count_nonzero(finite) < LIGHTCURVE_MIN_VALID_POINTS: + return None + + time = time[finite] + data = data[finite] + dataerr = dataerr[finite] + airmass = airmass[finite] + jd_times = jd_times[finite] + source_indices = source_indices[finite] + if exposure_times_days is not None: + exposure_times_days = exposure_times_days[finite] + exposure_times_seconds = exposure_times_seconds[finite] + + phase = get_phase(time, p_dict.get('pPer', 1.0), p_dict.get('midT', np.nan)) + if np.any(np.isfinite(time)): + time_upsample = np.linspace(np.nanmin(time), np.nanmax(time), 1000) + else: + time_upsample = np.array([], dtype=float) + phase_upsample = get_phase(time_upsample, p_dict.get('pPer', 1.0), p_dict.get('midT', np.nan)) + flat_model = np.ones(time.shape, dtype=float) + flat_upsample = np.ones(time_upsample.shape, dtype=float) + scatter = stellar_variability_scatter_from_flux(data) + residuals = data - 1.0 + + parameters = { + 'tmid': p_dict.get('midT', np.nan), + 'per': p_dict.get('pPer', np.nan), + 'rprs': p_dict.get('rprs', np.nan), + 'ars': p_dict.get('aRs', np.nan), + 'inc': p_dict.get('inc', np.nan), + 'ecc': p_dict.get('ecc', 0.0), + 'omega': p_dict.get('omega', 0.0), + 'a0': 1.0, + 'a1': 1.0, + 'a2': 0.0, + } + errors = { + 'tmid': p_dict.get('midTUnc', np.nan), + 'per': p_dict.get('pPerUnc', np.nan), + 'rprs': p_dict.get('rprsUnc', np.nan), + 'ars': p_dict.get('aRsUnc', np.nan), + 'inc': p_dict.get('incUnc', np.nan), + 'a0': 0.0, + 'a1': 0.0, + 'a2': 0.0, + } + + fit = SimpleNamespace( + stellar_variability_only=True, + time=time, + jd_times=jd_times, + data=data, + dataerr=dataerr, + airmass=airmass, + airmass_model=np.ones(time.shape, dtype=float), + wf=np.ones(time.shape, dtype=float), + transit=flat_model, + model=flat_model, + detrended=data, + detrendederr=dataerr, + residuals=residuals, + phase=phase, + time_upsample=time_upsample, + phase_upsample=phase_upsample, + transit_upsample=flat_upsample, + exposure_times_days=exposure_times_days, + parameters=parameters, + errors=errors, + bounds={}, + sample_parameters={}, + sample_errors={}, + ns_type=None, + chi2=float(np.nansum((residuals / dataerr) ** 2)) if dataerr.size else np.nan, + frame_filter_diagnostics=filter_diagnostics, + stellar_variability_reference_comp_index=comp_index, + stellar_variability_reference_label=comp_label, + stellar_variability_reference_position=comp_position, + stellar_variability_method_label=method_label, + stellar_variability_scatter=scatter, + stellar_variability_transit_exclusion=exclusion_summary, + stellar_variability_source_indices=source_indices, + stellar_variability_target_flux=np.asarray(prepared.get('target_flux'), dtype=float)[oot_mask][finite], + stellar_variability_comp_flux=np.asarray(prepared.get('comp_flux'), dtype=float)[oot_mask][finite], + stellar_variability_target_flux_error=np.asarray(prepared.get('target_flux_error'), dtype=float)[oot_mask][finite], + stellar_variability_comp_flux_error=np.asarray(prepared.get('comp_flux_error'), dtype=float)[oot_mask][finite], + stellar_variability_exposure_times_seconds=exposure_times_seconds, + airmass_fit_skipped=True, + airmass_correction_note="Skipped in stellar-variability-only mode; no transit/systematics model was fit.", + transit_qc={ + 'status': 'SKIPPED', + 'summary': 'Stellar variability only mode skipped transit fitting.', + 'residual_scatter': scatter, + }, + ) + fit = apply_plot_time_range(fit, time if plot_time_range is None else plot_time_range) + return fit + + +def build_stellar_variability_only_lightcurve_from_fluxes( + times, + target_flux, + comp_flux, + airmass, + p_dict, + jd_times=None, + adaptive_summary=None, + target_flux_error=None, + comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None, + filter_diagnostics=None, + comp_index=None, + comp_label=None, + comp_position=None, + method_label=None, + plot_time_range=None, +): + prepared = prepare_comparison_candidate_full_reduction_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + adaptive_summary=adaptive_summary, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + if not prepared.get('applied'): + prepared = prepare_stellar_variability_only_direct_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, + exposure_times_seconds=exposure_times_seconds, + inherited_diagnostics=prepared.get('filter_diagnostics', []), + ) + diagnostics = list(filter_diagnostics or []) + diagnostics.extend(prepared.get('filter_diagnostics', [])) + fit = build_stellar_variability_only_lightcurve( + prepared, + p_dict, + filter_diagnostics=diagnostics, + comp_index=comp_index, + comp_label=comp_label, + comp_position=comp_position, + method_label=method_label, + plot_time_range=plot_time_range, + ) + return fit, prepared + + def comparison_candidate_coverage_priority(assessment): if not isinstance(assessment, dict) or not assessment.get('valid'): return 4 @@ -8259,6 +8623,108 @@ def should_use_aperture_corrections_and_full_image_fwhm(config_value): return False +def should_reject_overexposed_stars(config_value): + if config_value is None: + return REJECT_OVEREXPOSED_STARS_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'reject_overexposed_stars' value; keeping overexposed-star rejection enabled.", + warn=True, + ) + return REJECT_OVEREXPOSED_STARS_DEFAULT + + +def parse_saturation_value(config_value): + if config_value is None: + return SATURATION_VALUE_DEFAULT + if isinstance(config_value, str) and not config_value.strip(): + return SATURATION_VALUE_DEFAULT + try: + value = float(config_value) + except (TypeError, ValueError): + value = np.nan + if np.isfinite(value) and value > 0: + return float(value) + + log_info( + "Warning: Invalid 'saturation_value' value; using 65535 for overexposure rejection.", + warn=True, + ) + return SATURATION_VALUE_DEFAULT + + +def parse_saturation_value_adu(config_value): + return parse_saturation_value(config_value) + + +def header_value_case_insensitive(header, key): + if not header: + return None + try: + return header[key] + except Exception: + pass + key_lower = str(key).lower() + try: + items = header.items() + except Exception: + return None + for header_key, header_value in items: + if str(header_key).lower() == key_lower: + return header_value + return None + + +def microobservatory_saturation_value_from_header(header): + telescop = header_value_case_insensitive(header, 'TELESCOP') + telescop = header_scalar_value(telescop) + if telescop is None: + return None + return MICROOBSERVATORY_TELESCOP_SATURATION_VALUES.get(str(telescop).strip().lower()) + + +def saturation_value_from_header(header): + microobservatory_saturation = microobservatory_saturation_value_from_header(header) + if microobservatory_saturation is not None: + return microobservatory_saturation + + saturate = header_value_case_insensitive(header, 'SATURATE') + saturate = finite_header_float(saturate) + if saturate is not None and saturate > 0: + return float(saturate) + return None + + +def parse_overexposure_threshold_fraction(config_value): + if config_value is None: + return OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT + if isinstance(config_value, str) and not config_value.strip(): + return OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT + try: + value = float(config_value) + except (TypeError, ValueError): + value = np.nan + if np.isfinite(value) and 0 < value <= 1: + return float(value) + + log_info( + "Warning: Invalid 'overexposure_threshold_fraction' value; " + f"using {OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT:.1f} for overexposure rejection.", + warn=True, + ) + return OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT + + def should_ignore_header_wcs(config_value): if config_value is None: return False @@ -8385,6 +8851,27 @@ def is_vertical_flux_normalization_disabled(config_value): return False +def should_run_stellar_variability_only(config_value): + if config_value is None: + return STELLAR_VARIABILITY_ONLY_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'stellar_variability_only' value; using default false.", + warn=True, + ) + return STELLAR_VARIABILITY_ONLY_DEFAULT + + def is_out_of_transit_baseline_detrending_enabled(config_value): if config_value is None: return True @@ -8465,6 +8952,80 @@ def estimate_transit_duration_from_prior_geometry(prior): return float(duration) if np.isfinite(duration) and duration > 0 else np.nan +def stellar_variability_transit_prior_from_planet_dict(p_dict): + return { + 'per': p_dict.get('pPer'), + 'rprs': p_dict.get('rprs'), + 'ars': p_dict.get('aRs'), + 'inc': p_dict.get('inc'), + 'ecc': p_dict.get('ecc', 0.0), + 'omega': p_dict.get('omega', 0.0), + } + + +def stellar_variability_out_of_transit_mask(times, p_dict): + times = np.asarray(times, dtype=float) + keep = np.isfinite(times) + summary = { + 'applied': False, + 'input_point_count': int(np.count_nonzero(np.isfinite(times))), + 'kept_point_count': int(np.count_nonzero(keep)), + 'rejected_point_count': 0, + 'duration_days': np.nan, + 'start_ingress_to_end_egress_days': np.nan, + 'period_days': np.nan, + 'reference_tmid': np.nan, + 'note': "Transit-window exclusion was not applied.", + } + if times.size == 0: + summary['note'] = "Transit-window exclusion skipped; no light-curve points were available." + return keep, summary + + try: + period = float(p_dict.get('pPer')) + reference_tmid = float(p_dict.get('midT')) + except (TypeError, ValueError): + period = np.nan + reference_tmid = np.nan + + duration = estimate_transit_duration_from_prior_geometry( + stellar_variability_transit_prior_from_planet_dict(p_dict) + ) + summary.update({ + 'duration_days': duration, + 'start_ingress_to_end_egress_days': duration, + 'period_days': period, + 'reference_tmid': reference_tmid, + }) + if ( + not np.isfinite(period) + or period <= 0 + or not np.isfinite(reference_tmid) + or not np.isfinite(duration) + or duration <= 0 + ): + summary['note'] = ( + "Transit-window exclusion skipped; EXOTIC could not estimate a finite ingress-to-egress " + "window from the supplied planetary parameters." + ) + return keep, summary + + epochs = np.rint((times - reference_tmid) / period) + nearest_tmid = reference_tmid + epochs * period + in_transit = keep & (np.abs(times - nearest_tmid) <= 0.5 * duration) + keep = keep & ~in_transit + summary.update({ + 'applied': bool(np.any(in_transit)), + 'kept_point_count': int(np.count_nonzero(keep)), + 'rejected_point_count': int(np.count_nonzero(in_transit)), + 'note': ( + f"Excluded {int(np.count_nonzero(in_transit))} point(s) inside the predicted " + "start-ingress to end-egress transit window before stellar-variability analysis." + ), + }) + return keep, summary + + def _cacheable_duration_prior_scalar(value): try: numeric_value = float(value) @@ -11053,6 +11614,45 @@ def psf_sigma_from_fit(psf_row, fallback_sigma=np.nan): return np.nan +def overexposure_aperture_radius_from_psf_row(psf_row, fallback_sigma=np.nan): + sigma = psf_sigma_from_fit(psf_row, fallback_sigma=fallback_sigma) + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + return max(float(APERTURE_SIGMA_MAX) * float(sigma), 1.0) + + +def aperture_contains_overexposed_pixel(data, xc, yc, aperture_radius, threshold_value, + fast_mode=False): + if not ( + np.isfinite(xc) + and np.isfinite(yc) + and np.isfinite(aperture_radius) + and aperture_radius > 0 + and np.isfinite(threshold_value) + and threshold_value > 0 + ): + return False + + try: + aperture = CircularAperture(positions=[(float(xc), float(yc))], r=float(aperture_radius)) + mask_method = 'center' if fast_mode else 'exact' + mask = aperture.to_mask(method=mask_method)[0] + data_cutout = mask.cutout(data) + except Exception: + return False + + if data_cutout is None: + return False + + cutout = np.asarray(data_cutout, dtype=float) + weights = np.asarray(mask.data, dtype=float) + valid_pixels = np.isfinite(cutout) & np.isfinite(weights) & (weights > 0) + if not np.any(valid_pixels): + return False + + return bool(np.nanmax(cutout[valid_pixels]) > float(threshold_value)) + + def representative_psf_sigma(psf_rows, fallback_sigma=np.nan): try: sigmas = np.asarray(psf_rows[:, 3], dtype=float) + np.asarray(psf_rows[:, 4], dtype=float) @@ -11541,6 +12141,12 @@ def apply_lightcurve_mask(lightcurve, mask, sort_index=None): 'dataerr', 'airmass_model', 'wf', + 'stellar_variability_source_indices', + 'stellar_variability_target_flux', + 'stellar_variability_comp_flux', + 'stellar_variability_target_flux_error', + 'stellar_variability_comp_flux_error', + 'stellar_variability_exposure_times_seconds', ) for attr in array_attrs: @@ -21520,6 +22126,8 @@ def comparison_selection_metric_label(selection_metric): return "First QC PASS" if selection_metric == 'promising_partial': return "Promising Partial" + if selection_metric == 'stellar_variability_scatter': + return "Out-of-transit scatter" if selection_metric == 'comparison_field_rank': return "Comparison-Field Rank" if selection_metric == 'ktmf': @@ -23084,6 +23692,7 @@ def initialize_aperture_data_store(frame_count, aperture_count, annulus_count, c 'target': np.full(aper_shape, np.nan, dtype=float), 'target_bg': np.full(aper_shape, np.nan, dtype=float), 'target_unc': np.full(aper_shape, np.nan, dtype=float), + 'target_overexposed': np.zeros(frame_count, dtype=bool), } for component in NOISE_BUDGET_COMPONENT_KEYS: aper_data[f"target_noise_{component}"] = np.full(aper_shape, np.nan, dtype=float) @@ -23093,13 +23702,47 @@ def initialize_aperture_data_store(frame_count, aperture_count, annulus_count, c aper_data[ckey] = np.full(aper_shape, np.nan, dtype=float) aper_data[f"{ckey}_bg"] = np.full(aper_shape, np.nan, dtype=float) aper_data[f"{ckey}_unc"] = np.full(aper_shape, np.nan, dtype=float) + aper_data[f"{ckey}_overexposed"] = np.zeros(frame_count, dtype=bool) for component in NOISE_BUDGET_COMPONENT_KEYS: aper_data[f"{ckey}_noise_{component}"] = np.full(aper_shape, np.nan, dtype=float) return aper_data -def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=False, sigma_hint=np.nan, +def mask_aperture_star_frame(aper_data, key, frame_index): + if not isinstance(aper_data, dict) or key not in aper_data: + return + frame_index = int(frame_index) + for suffix in ('', '_bg', '_unc'): + data_key = f"{key}{suffix}" + if data_key in aper_data: + aper_data[data_key][frame_index, :, :] = np.nan + for component in NOISE_BUDGET_COMPONENT_KEYS: + noise_key = f"{key}_noise_{component}" + if noise_key in aper_data: + aper_data[noise_key][frame_index, :, :] = np.nan + overexposed_key = f"{key}_overexposed" + if overexposed_key in aper_data: + aper_data[overexposed_key][frame_index] = True + + +def apply_overexposure_masks_to_aperture_frame(aper_data, frame_index, target_overexposed, + comp_overexposed_masks=None): + if target_overexposed: + mask_aperture_star_frame(aper_data, 'target', frame_index) + + if not isinstance(comp_overexposed_masks, dict): + return + for key, mask in comp_overexposed_masks.items(): + try: + is_overexposed = bool(np.asarray(mask, dtype=bool)[int(frame_index)]) + except (IndexError, TypeError, ValueError): + is_overexposed = False + if is_overexposed: + mask_aperture_star_frame(aper_data, key, frame_index) + + +def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=False, sigma_hint=np.nan, aperture_correction_factors=None, noise_config=None, exposure_s=np.nan, airmass=np.nan, return_noise=False): flux_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) @@ -23376,7 +24019,8 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, skip_low_comparison_coverage_rejection=False, use_psf_photometry=True, use_aperture_photometry=True, - psf_flux_data=None): + psf_flux_data=None, + comp_overexposed_masks=None): candidate_summaries = [] comp_star_count = len(comp_stars) psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) @@ -23385,16 +24029,34 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, return None frame_count = len(airmass) + overexposure_masks = {} + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + mask = None + if isinstance(comp_overexposed_masks, dict) and ckey in comp_overexposed_masks: + mask = np.asarray(comp_overexposed_masks[ckey], dtype=bool) + elif isinstance(aper_data, dict) and f"{ckey}_overexposed" in aper_data: + mask = np.asarray(aper_data[f"{ckey}_overexposed"], dtype=bool) + if mask is not None and mask.shape[0] == frame_count: + overexposure_masks[ckey] = mask + else: + overexposure_masks[ckey] = np.zeros(frame_count, dtype=bool) centroid_psf_quality_masks = { - f"comp{comp_idx + 1}": psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count) + f"comp{comp_idx + 1}": ( + psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count) + & ~overexposure_masks[f"comp{comp_idx + 1}"] + ) for comp_idx in range(comp_star_count) } psf_quality_masks = { - f"comp{comp_idx + 1}": psf_quality_mask_for_key( - psf_data, - f"comp{comp_idx + 1}", - frame_count, - psf_flux_data=psf_flux_data, + f"comp{comp_idx + 1}": ( + psf_quality_mask_for_key( + psf_data, + f"comp{comp_idx + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ) + & ~overexposure_masks[f"comp{comp_idx + 1}"] ) for comp_idx in range(comp_star_count) } @@ -23469,6 +24131,11 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, comp_summary['psf_quality_rejected_count'] = int(np.count_nonzero(~quality_mask)) else: comp_summary['psf_quality_rejected_count'] = 0 + overexposure_mask = overexposure_masks.get(comp_summary['key']) + comp_summary['overexposure_rejected_count'] = ( + int(np.count_nonzero(overexposure_mask)) + if overexposure_mask is not None else 0 + ) comp_summary['selection_reason'] = comparison_calibration_selection_reason( comp_summary, best_candidate['best_comp_score'], @@ -23504,6 +24171,370 @@ def ranked_comparison_calibration_summaries(comparison_calibration): return ranked_summaries +def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, comparison_calibration, + psf_data, aper_data, target_psf_flux, + psf_flux_data=None, + psf_noise_data=None, + plot_time_range=None, + use_adaptive_apertures=False, + adaptive_aperture_values=None, + adaptive_annulus_values=None, + fallback_sigma=np.nan, + exposure_times_seconds=None, + gain_e_per_adu=None): + ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) + if not ranked_summaries: + return { + 'ranked_summaries': [], + 'attempts': [], + 'selected_result': None, + 'selection_metric': 'stellar_variability_scatter', + } + + method = comparison_calibration['method'] + method_label = comparison_calibration.get('method_label', method) + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + if method == 'psf': + frame_count = target_psf_flux.shape[0] + target_flux = np.asarray(target_psf_flux, dtype=float) + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + target_flux_error = ( + np.asarray(psf_noise_data.get('target'), dtype=float) + if isinstance(psf_noise_data, dict) and 'target' in psf_noise_data + else None + ) + else: + frame_count = aper_data['target'].shape[0] + target_flux = np.asarray(aper_data['target'][:, aperture_index, annulus_index], dtype=float) + target_flux_error = ( + np.asarray(aper_data['target_unc'][:, aperture_index, annulus_index], dtype=float) + if isinstance(aper_data, dict) and 'target_unc' in aper_data + else None + ) + + exposure_times_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + if exposure_times_array is not None and exposure_times_array.shape != times.shape: + exposure_times_array = None + + adaptive_summary = build_comparison_candidate_adaptive_summary( + comparison_calibration, + psf_data, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=adaptive_aperture_values, + adaptive_annulus_values=adaptive_annulus_values, + fallback_sigma=fallback_sigma, + ) + field_image_keep_mask = np.asarray( + comparison_calibration.get('field_image_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if field_image_keep_mask.shape != times.shape: + field_image_keep_mask = np.ones(times.shape[0], dtype=bool) + field_image_clip_diagnostic = None + if np.any(~field_image_keep_mask): + required_pairs = comparison_calibration.get('image_outlier_required_valid_pairs', 0) + sigma_threshold = comparison_calibration.get('image_outlier_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + field_image_clip_diagnostic = build_time_rejection_diagnostic( + "Comparison-field image clip", + times, + field_image_keep_mask, + note=( + "Dropped frames flagged after comparison-star suitability clipping because every " + f"valid pairwise comparison was more than {sigma_threshold:.2f} sigma from its flat-line median " + f"(min valid pair count={required_pairs})." + ), + ) + + attempts = [] + for field_rank, comp_summary in enumerate(ranked_summaries): + comp_index = comp_summary['comp_index'] + ckey = comp_summary.get('key', f"comp{comp_index + 1}") + comp_quality_mask = np.asarray( + comp_summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ), + ), + dtype=bool, + ) + if comp_quality_mask.shape[0] != frame_count: + comp_quality_mask = psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ) + + if method == 'psf': + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), + ) + if target_shape_mask.shape[0] != frame_count: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None if target_flux_error is None else mask_series_with_quality(target_flux_error, target_shape_mask) + ) + comp_flux = psf_flux_series_from_rows(psf_flux_data[ckey], comp_quality_mask) + comp_flux_error = ( + mask_series_with_quality(psf_noise_data[ckey], comp_quality_mask) + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data + else None + ) + else: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error + comp_flux = mask_series_with_quality(aper_data[ckey][:, aperture_index, annulus_index], comp_quality_mask) + comp_flux_error = ( + mask_series_with_quality( + aper_data[f"{ckey}_unc"][:, aperture_index, annulus_index], + comp_quality_mask, + ) + if f"{ckey}_unc" in aper_data + else None + ) + + candidate_frame_keep_mask = np.asarray( + comp_summary.get('ensemble_frame_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if candidate_frame_keep_mask.shape != times.shape: + candidate_frame_keep_mask = np.ones(times.shape[0], dtype=bool) + candidate_frame_clip_diagnostic = None + candidate_frame_diagnostic_keep_mask = candidate_frame_keep_mask | ~field_image_keep_mask + if np.any(~candidate_frame_diagnostic_keep_mask): + required_pairs = comp_summary.get( + 'ensemble_frame_required_valid_pairs', + COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS, + ) + sigma_threshold = comp_summary.get('ensemble_frame_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + candidate_frame_clip_diagnostic = build_time_rejection_diagnostic( + "Comparison-candidate ensemble clip", + times, + candidate_frame_diagnostic_keep_mask, + note=( + "Dropped frames where this comparison star disagreed with the comparison-star ensemble " + f"before target fitting; same-direction pairwise majority exceeded {sigma_threshold:.2f} sigma " + f"(min confirming pair count={required_pairs})." + ), + ) + + fit_mask = field_image_keep_mask & candidate_frame_keep_mask & comp_quality_mask & target_shape_mask + if method == 'psf': + fit_mask &= robust_target_reference_flux_mask(candidate_target_flux, comp_flux) + else: + fit_mask &= valid_comparison_frame_mask(candidate_target_flux) & valid_comparison_frame_mask(comp_flux) + + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + candidate_target_flux[fit_mask], + comp_flux[fit_mask], + airmass[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + external_filter_diagnostics = [] + if field_image_clip_diagnostic is not None: + external_filter_diagnostics.append(field_image_clip_diagnostic) + if candidate_frame_clip_diagnostic is not None: + external_filter_diagnostics.append(candidate_frame_clip_diagnostic) + + fit_result = None + prepared = None + if fit_diagnostics.get('failure_reason') is None: + fit_result, prepared = build_stellar_variability_only_lightcurve_from_fluxes( + times[fit_mask], + candidate_target_flux[fit_mask], + comp_flux[fit_mask], + airmass[fit_mask], + p_dict, + jd_times=jd_times[fit_mask], + adaptive_summary=adaptive_summary, + target_flux_error=( + None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask] + ), + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + exposure_times_seconds=( + None if exposure_times_array is None else exposure_times_array[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, + filter_diagnostics=external_filter_diagnostics, + comp_index=comp_index, + comp_label=comp_summary.get('label', f"Comp {comp_index + 1}"), + comp_position=comp_summary.get('position'), + method_label=method_label, + plot_time_range=plot_time_range, + ) + if fit_result is not None: + original_indices = np.flatnonzero(fit_mask) + source_indices = np.asarray(fit_result.stellar_variability_source_indices, dtype=int) + if source_indices.size and np.max(source_indices) < original_indices.size: + fit_result.stellar_variability_source_indices = original_indices[source_indices] + else: + fit_result.stellar_variability_source_indices = np.array([], dtype=int) + elif prepared is not None and prepared.get('applied'): + fit_diagnostics = dict(fit_diagnostics) + exclusion = stellar_variability_out_of_transit_mask(prepared.get('time'), p_dict)[1] + fit_diagnostics.update({ + 'failed_stage': 'stellar_variability_transit_window', + 'failure_reason': ( + "too few out-of-transit points remained after excluding the predicted " + "start-ingress to end-egress transit window." + ), + 'transit_window_rejected_point_count': exclusion.get('rejected_point_count', 0), + }) + else: + fit_diagnostics = ensure_lightcurve_fit_failure_reason( + fit_diagnostics, + fit_result, + failed_stage='stellar_variability_photometry', + failure_reason=( + "the raw comparison-candidate photometry did not yield a usable " + "out-of-transit stellar-variability light curve." + ), + ) + + source_indices = np.asarray( + getattr(fit_result, 'stellar_variability_source_indices', np.array([], dtype=int)), + dtype=int, + ) + tflux_fit = getattr(fit_result, 'stellar_variability_target_flux', np.array([], dtype=float)) + cflux_fit = getattr(fit_result, 'stellar_variability_comp_flux', np.array([], dtype=float)) + tflux_fit_error = getattr(fit_result, 'stellar_variability_target_flux_error', np.array([], dtype=float)) + cflux_fit_error = getattr(fit_result, 'stellar_variability_comp_flux_error', np.array([], dtype=float)) + scatter = getattr(fit_result, 'stellar_variability_scatter', np.nan) + exclusion_summary = getattr(fit_result, 'stellar_variability_transit_exclusion', {}) + + attempt = { + 'rank': field_rank, + 'field_rank': field_rank, + 'comp_index': comp_index, + 'ckey': ckey, + 'label': comp_summary.get('label', f"Comp {comp_index + 1}"), + 'position': comp_summary.get('position'), + 'aggregate_score': comp_summary.get('aggregate_score', np.inf), + 'coverage_count': comp_summary.get('coverage_count', 0), + 'coverage_total_frame_count': comp_summary.get('coverage_total_frame_count', 0), + 'coverage_reference_count': comp_summary.get('coverage_reference_count', np.nan), + 'coverage_min_required_count': comp_summary.get('coverage_min_required_count', 0), + 'coverage_rejected': comp_summary.get('coverage_rejected', False), + 'ensemble_frame_rejected_count': comp_summary.get('ensemble_frame_rejected_count', 0), + 'ensemble_frame_required_valid_pairs': comp_summary.get('ensemble_frame_required_valid_pairs', 0), + 'fit': fit_result, + 'full_reduction_fit': fit_result, + 'good_times': np.asarray(getattr(fit_result, 'time', np.array([], dtype=float)), dtype=float), + 'good_flux': np.asarray(getattr(fit_result, 'detrended', np.array([], dtype=float)), dtype=float), + 'good_unc': np.asarray(getattr(fit_result, 'detrendederr', np.array([], dtype=float)), dtype=float), + 'good_airmass': np.asarray(getattr(fit_result, 'airmass', np.array([], dtype=float)), dtype=float), + 'good_jd_times': np.asarray(getattr(fit_result, 'jd_times', np.array([], dtype=float)), dtype=float), + 'good_exposure_times_seconds': getattr( + fit_result, + 'stellar_variability_exposure_times_seconds', + None, + ), + 'good_target_flux_error': tflux_fit_error, + 'good_comp_flux_error': cflux_fit_error, + 'tflux_fit': tflux_fit, + 'cflux_fit': cflux_fit, + 'tflux_fit_error': tflux_fit_error, + 'cflux_fit_error': cflux_fit_error, + 'source_indices': source_indices, + 'duration_samples': np.array( + [exclusion_summary.get('duration_days', np.nan)], + dtype=float, + ), + 'data_highres': np.ones(1000, dtype=float), + 'fit_diagnostics': fit_diagnostics, + 'eebls_snr': np.nan, + 'transit_delta_bic': np.nan, + 'residual_scatter': scatter, + 'target_model_scatter_basis': 'out-of-transit normalized target/reference scatter', + 'projected_full_residual_scatter': scatter, + 'selection_scatter': scatter, + 'selection_scatter_basis': 'out-of-transit normalized target/reference scatter', + 'target_comp_scatter': target_comp_flux_scatter(tflux_fit, cflux_fit), + 'ktmf_metric': np.nan, + 'ktmf_contributions': [], + 'fit_point_count': int(np.asarray(tflux_fit).size), + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'parameter_summary': None, + 'transit_qc_status': 'SKIPPED', + 'transit_qc_summary': 'Stellar variability only mode skipped transit fitting.', + 'rejected_by_transit_qc': False, + 'selected': False, + 'selection_reason': None, + 'full_reduction_applied': fit_result is not None, + 'full_reduction_note': ( + "completed the stellar-variability-only reduction without fitting a transit model." + if fit_result is not None else None + ), + 'stellar_variability_transit_exclusion': exclusion_summary, + 'reuse_selected_full_reduction_fit': fit_result is not None, + } + attempts.append(attempt) + scatter_text = format_residual_scatter(scatter) + if fit_result is None: + log_info( + f" {attempt['label']}: no usable stellar-variability-only light curve " + f"({attempt['failure_reason']})." + ) + else: + rejected = exclusion_summary.get('rejected_point_count', 0) + log_info( + f" {attempt['label']}: out-of-transit scatter={scatter_text}; " + f"used {attempt['fit_point_count']} point(s), excluded {rejected} predicted in-transit point(s)." + ) + + eligible_attempts = [ + attempt for attempt in attempts + if attempt.get('fit') is not None and np.isfinite(attempt.get('selection_scatter', np.nan)) + ] + selected_result = None + if eligible_attempts: + selected_result = min( + eligible_attempts, + key=lambda attempt: ( + attempt.get('selection_scatter', np.inf), + attempt.get('aggregate_score', np.inf), + attempt.get('field_rank', np.inf), + ), + ) + selected_result['selected'] = True + selected_result['selection_reason'] = ( + "selected: lowest out-of-transit normalized target/reference scatter among " + "comparison-star calibration candidates" + ) + selected_scatter = selected_result.get('selection_scatter', np.nan) + for attempt in attempts: + if attempt is selected_result: + continue + if attempt.get('fit') is None: + continue + attempt['selection_reason'] = ( + "not selected: out-of-transit normalized target/reference scatter " + f"{format_residual_scatter(attempt.get('selection_scatter', np.nan))} was higher than " + f"the selected {format_residual_scatter(selected_scatter)}" + ) + + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': selected_result, + 'selection_metric': 'stellar_variability_scatter', + 'stopped_after_first_qc_pass': False, + 'stopped_after_promising_partial': False, + } + + def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p_dict, comparison_calibration, psf_data, aper_data, target_psf_flux, psf_flux_data=None, @@ -24706,6 +25737,15 @@ def _main_impl(): ), ) ) + stellar_variability_only = should_run_stellar_variability_only( + exotic_infoDict.get('stellar_variability_only', STELLAR_VARIABILITY_ONLY_DEFAULT) + ) + if stellar_variability_only: + use_eebls_tmid_initializer = False + pick_comparison_by_eebls_snr = False + run_fast_ultranest_before_final_run = False + run_final_residual_rejection = False + run_final_fit_phase_residual_clip = False use_legacy_psf_flux_mode = should_use_legacy_psf_flux_mode( exotic_infoDict.get( 'use_legacy_psf_flux', @@ -24825,6 +25865,12 @@ def _main_impl(): SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, ) ) + if stellar_variability_only: + use_sparse_posterior_live_point_retry = False + log_info( + "Stellar-variability-only mode enabled: EXOTIC will run the normal photometry and " + "comparison-star selection, discard predicted ingress-to-egress points, and skip transit fitting." + ) if use_sparse_posterior_live_point_retry: if run_fast_ultranest_before_final_run: log_info( @@ -25362,6 +26408,11 @@ def _main_impl(): vsp_num = [] comp_star_count = len(exotic_infoDict['comp_stars']) psf_noise_data = initialize_psf_noise_data(len(inputfiles), comp_star_count) + target_overexposed_frame_mask = np.zeros(len(inputfiles), dtype=bool) + comp_overexposed_masks = { + f"comp{comp_idx + 1}": np.zeros(len(inputfiles), dtype=bool) + for comp_idx in range(comp_star_count) + } frame_noise_configs = [] require_comp_star = resolve_require_comp_star_for_exposure_times( exotic_infoDict.get('require_comp_star', 'y'), @@ -25402,6 +26453,34 @@ def _main_impl(): use_aperture_corrections_and_full_image_fwhm = should_use_aperture_corrections_and_full_image_fwhm( exotic_infoDict.get('use_aperture_corrections_and_full_image_fwhm', False) ) + reject_overexposed_stars = should_reject_overexposed_stars( + exotic_infoDict.get('reject_overexposed_stars', REJECT_OVEREXPOSED_STARS_DEFAULT) + ) + configured_saturation_value = parse_saturation_value( + exotic_infoDict.get( + 'saturation_value', + exotic_infoDict.get('saturation_value_adu', SATURATION_VALUE_DEFAULT), + ) + ) + header_saturation_value = saturation_value_from_header(header) + saturation_value = configured_saturation_value + if ( + header_saturation_value is not None + and configured_saturation_value == SATURATION_VALUE_DEFAULT + ): + saturation_value = header_saturation_value + exotic_infoDict['saturation_value'] = saturation_value + log_info( + f"Using FITS header-derived saturation_value={saturation_value:.1f} " + "for overexposure rejection." + ) + overexposure_threshold_fraction = parse_overexposure_threshold_fraction( + exotic_infoDict.get( + 'overexposure_threshold_fraction', + OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT, + ) + ) + overexposure_threshold = saturation_value * overexposure_threshold_fraction if not use_psf_photometry and not use_aperture_photometry: log_info("Error: both PSF and aperture photometry are disabled in optional_info.", error=True) return @@ -25418,6 +26497,15 @@ def _main_impl(): "Ensemble comparison photometry enabled per optional_info setting; the final target " "light curve will use non-rejected comparison stars as a combined reference." ) + if reject_overexposed_stars: + log_info( + "Overexposed-star rejection enabled: target frames and comparison-star measurements " + f"with aperture pixels above {overexposure_threshold:.1f} will be rejected " + f"(saturation_value={saturation_value:.1f}, " + f"threshold_fraction={overexposure_threshold_fraction:.3f})." + ) + else: + log_info("Overexposed-star rejection disabled per optional_info setting.") if not use_deviation_from_expected_transit_in_qc: log_info("Expected-value transit QC deviation checks disabled per optional_info setting.") if target_driven_comp_selection: @@ -25669,6 +26757,56 @@ def _main_impl(): comp_alignment_keys, ) + if reject_overexposed_stars: + target_row = np.asarray(psf_data['target'][i], dtype=float) + target_radius = overexposure_aperture_radius_from_psf_row( + target_row, + fallback_sigma=sigma, + ) + if aperture_contains_overexposed_pixel( + imageData, + target_row[0] if target_row.size > 0 else np.nan, + target_row[1] if target_row.size > 1 else np.nan, + target_radius, + overexposure_threshold, + fast_mode=fast_aperture_mask, + ): + target_overexposed_frame_mask[i] = True + plateStatus.overexposedWarning( + 0, + target_row[0] if target_row.size > 0 else np.nan, + target_row[1] if target_row.size > 1 else np.nan, + overexposure_threshold, + ) + psf_data['target'][i, :] = np.nan + psf_flux_data['target'][i, :] = np.nan + hdul.close() + del hdul + del imageData + continue + + for comp_idx, comp_key in enumerate(comp_alignment_keys): + comp_row = np.asarray(psf_data[comp_key][i], dtype=float) + comp_radius = overexposure_aperture_radius_from_psf_row( + comp_row, + fallback_sigma=sigma, + ) + if aperture_contains_overexposed_pixel( + imageData, + comp_row[0] if comp_row.size > 0 else np.nan, + comp_row[1] if comp_row.size > 1 else np.nan, + comp_radius, + overexposure_threshold, + fast_mode=fast_aperture_mask, + ): + comp_overexposed_masks[comp_key][i] = True + plateStatus.overexposedWarning( + comp_idx + 1, + comp_row[0] if comp_row.size > 0 else np.nan, + comp_row[1] if comp_row.size > 1 else np.nan, + overexposure_threshold, + ) + if use_psf_photometry: psf_flux_row_fitter = ( fit_legacy_psf_photometry_flux_row @@ -25699,6 +26837,9 @@ def _main_impl(): ), ) for comp_idx, comp_key in enumerate(comp_alignment_keys): + if comp_overexposed_masks.get(comp_key, np.zeros(len(inputfiles), dtype=bool))[i]: + psf_flux_data[comp_key][i, :] = np.nan + continue comp_psf_flux_seed_row = psf_data[comp_key][i] if comp_key in psf_flux_seed_tracks: comp_psf_flux_seed_row = psf_flux_seed_tracks[comp_key][i] @@ -25762,6 +26903,12 @@ def _main_impl(): exposure_s=frame_exposure_s, airmass=frame_airmass, ) + apply_overexposure_masks_to_aperture_frame( + coarse_aper_data, + i, + target_overexposed_frame_mask[i], + comp_overexposed_masks, + ) if i == coarse_tune_frames - 1: subset_airmass = np.asarray(airMassList[:coarse_tune_frames], dtype=float) @@ -25809,6 +26956,15 @@ def _main_impl(): log_info(f"Backfilling refined aperture photometry for the first {coarse_tune_frames} frame(s).") for backfill_idx in range(coarse_tune_frames): + if target_overexposed_frame_mask[backfill_idx]: + apply_overexposure_masks_to_aperture_frame( + aper_data, + backfill_idx, + True, + comp_overexposed_masks, + ) + coarse_frame_cache[backfill_idx] = None + continue backfill_image = coarse_frame_cache[backfill_idx] loaded_from_disk = False if backfill_image is None: @@ -25842,6 +26998,12 @@ def _main_impl(): exposure_s=exptimes[backfill_idx], airmass=airMassList[backfill_idx], ) + apply_overexposure_masks_to_aperture_frame( + aper_data, + backfill_idx, + target_overexposed_frame_mask[backfill_idx], + comp_overexposed_masks, + ) finally: if loaded_from_disk: del backfill_image @@ -25876,6 +27038,12 @@ def _main_impl(): exposure_s=frame_exposure_s, airmass=frame_airmass, ) + apply_overexposure_masks_to_aperture_frame( + aper_data, + i, + target_overexposed_frame_mask[i], + comp_overexposed_masks, + ) # close file + delete from memory hdul.close() @@ -25890,6 +27058,26 @@ def _main_impl(): badmask = np.isnan(psf_data["target"][:, 0]) | (psf_data["target"][:, 0] == 0) if aper_data is not None: badmask = badmask | (aper_data["target"][:, 0, 0] == 0) | np.isnan(aper_data["target"][:, 0, 0]) + if reject_overexposed_stars and target_overexposed_frame_mask.shape == badmask.shape: + target_overexposure_diagnostic = build_time_rejection_diagnostic( + "Target overexposure filter", + times, + ~target_overexposed_frame_mask, + note=( + "Dropped frames before photometry selection because one or more target aperture pixels " + f"exceeded {overexposure_threshold:.1f} " + f"({overexposure_threshold_fraction:.3f} of saturation_value={saturation_value:.1f})." + ), + ) + if ( + target_overexposure_diagnostic is not None + and target_overexposure_diagnostic['dropped_point_count'] > 0 + ): + log_lightcurve_filter_diagnostics( + [target_overexposure_diagnostic], + header="Target overexposure frame rejections before photometry selection", + ) + badmask = badmask | target_overexposed_frame_mask goodmask = ~badmask global_frame_filter_diagnostic = build_time_rejection_diagnostic( "Target centroid/aperture validity filter", @@ -25918,6 +27106,11 @@ def _main_impl(): if aper_data is not None: for key in list(aper_data.keys()): aper_data[key] = aper_data[key][goodmask] + target_overexposed_frame_mask = target_overexposed_frame_mask[goodmask] + comp_overexposed_masks = { + key: np.asarray(mask, dtype=bool)[goodmask] + for key, mask in comp_overexposed_masks.items() + } for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" psf_data[ckey] = psf_data[ckey][goodmask] @@ -26087,6 +27280,7 @@ def _main_impl(): use_psf_photometry=use_psf_photometry, use_aperture_photometry=use_aperture_photometry, psf_flux_data=psf_flux_source, + comp_overexposed_masks=comp_overexposed_masks, ) if comparison_calibration is not None: @@ -26137,11 +27331,16 @@ def _main_impl(): psf_quality_text = ( f", psf_quality_rejects={summary['psf_quality_rejected_count']}" ) + overexposure_text = "" + if summary.get('overexposure_rejected_count', 0) > 0: + overexposure_text = ( + f", overexposure_rejects={summary['overexposure_rejected_count']}" + ) log_info( f" {summary['label']}{selected_label} ({position_text}): suitability={aggregate_text}, " f"ensemble={ensemble_text}, pairwise_median={pairwise_text}, " f"valid_pairs={summary['valid_pair_count']}, {coverage_text}" - f"{psf_quality_text}{ensemble_frame_text}, " + f"{psf_quality_text}{overexposure_text}{ensemble_frame_text}, " f"reason={summary['selection_reason']}" ) @@ -26189,42 +27388,67 @@ def _main_impl(): except Exception as e: log_info(f"Warning: Could not save comparison-star calibration outputs ({e}).", warn=True) - comparison_fit_search = fit_ranked_comparison_calibration_candidates( - times, - jd_times, - airmass, - ld, - pDict, - comparison_calibration, - psf_data, - aper_data, - tFlux, - psf_flux_data=psf_flux_source, - psf_noise_data=psf_noise_data if use_psf_photometry else None, - plot_time_range=full_plot_time_range, - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, - pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, - exit_at_first_qc_pass_solution=exit_at_first_qc_pass_solution, - final_fit_baseline_duration_multiplier=final_fit_baseline_duration_multiplier, - use_adaptive_apertures=use_adaptive_apertures, - adaptive_aperture_values=aperture_values, - adaptive_annulus_values=annulus_values, - fallback_sigma=sigma_display, - run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, - run_final_fit_phase_residual_clip=run_final_fit_phase_residual_clip, - run_final_residual_rejection=run_final_residual_rejection, - save_dir=exotic_infoDict['save'], - planet_name=pDict['pName'], - observation_date=exotic_infoDict['date'], - use_ensemble_photometry_rather_than_single_comp= - use_ensemble_photometry_rather_than_single_comp, - exposure_times_seconds=exposure_times_seconds, - gain_e_per_adu=fallback_gain_e_per_adu, - ) + if stellar_variability_only: + log_info( + "Stellar-variability-only mode: selecting comparison photometry by " + "out-of-transit target/reference scatter without fitting transit models." + ) + comparison_fit_search = select_stellar_variability_only_photometry( + times, + jd_times, + airmass, + pDict, + comparison_calibration, + psf_data, + aper_data, + tFlux, + psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + plot_time_range=full_plot_time_range, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=aperture_values, + adaptive_annulus_values=annulus_values, + fallback_sigma=sigma_display, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + ) + else: + comparison_fit_search = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld, + pDict, + comparison_calibration, + psf_data, + aper_data, + tFlux, + psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + plot_time_range=full_plot_time_range, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + exit_at_first_qc_pass_solution=exit_at_first_qc_pass_solution, + final_fit_baseline_duration_multiplier=final_fit_baseline_duration_multiplier, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=aperture_values, + adaptive_annulus_values=annulus_values, + fallback_sigma=sigma_display, + run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, + run_final_fit_phase_residual_clip=run_final_fit_phase_residual_clip, + run_final_residual_rejection=run_final_residual_rejection, + save_dir=exotic_infoDict['save'], + planet_name=pDict['pName'], + observation_date=exotic_infoDict['date'], + use_ensemble_photometry_rather_than_single_comp= + use_ensemble_photometry_rather_than_single_comp, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + ) comparison_calibration['ranked_fit_comp_indices'] = [ summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] ] @@ -26271,7 +27495,9 @@ def _main_impl(): dtype=int, ) selected_attempt_label = selected_attempt.get('label', 'comparison candidate') - if selected_attempt.get('search_stopped_after_qc_pass', False): + if stellar_variability_only: + selection_basis = 'stellar_variability_scatter' + elif selected_attempt.get('search_stopped_after_qc_pass', False): selection_basis = 'first_qc_pass' elif selected_attempt.get('search_stopped_after_promising_partial', False): selection_basis = 'promising_partial' @@ -26283,7 +27509,14 @@ def _main_impl(): selection_basis = 'comparison_field' else: selection_basis = 'comparison_field_retry' - if selection_basis == 'first_qc_pass': + if selection_basis == 'stellar_variability_scatter': + log_info( + "Stellar-variability-only comparison selection chose " + f"{selected_attempt_label} with {comparison_calibration['method_label']} " + "because it had the lowest out-of-transit target/reference scatter " + f"({format_residual_scatter(selected_attempt.get('selection_scatter', np.nan))})." + ) + elif selection_basis == 'first_qc_pass': log_info( "Comparison-star calibration target-fit selection chose " f"{selected_attempt_label} with {comparison_calibration['method_label']} " @@ -26372,6 +27605,7 @@ def _main_impl(): selected_fit_final_output_dir=selected_attempt.get('final_output_dir'), calibration_field_score=comparison_calibration['field_score'], selection_basis=selection_basis, + stellar_variability_only=stellar_variability_only, selection_metric=comparison_fit_search.get('selection_metric', 'ktmf'), selected_comparison_selection_reason=selected_attempt.get('selection_reason'), selected_comparison_attempt=compact_comparison_attempt_for_output(selected_attempt), @@ -26411,18 +27645,39 @@ def _main_impl(): for j in vsp_num: ckey = f"comp{j + 1}" cFlux = psf_flux_series_from_rows(psf_flux_source[ckey]) - vsp_fit, _, _ = fit_lightcurve( - times, tFlux, cFlux, airmass, ld, pDict, jd_times, - target_flux_error=psf_noise_data.get('target'), - comp_flux_error=psf_noise_data.get(ckey), - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - plot_time_range=full_plot_time_range, - use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, - exposure_times_seconds=exposure_times_seconds, - gain_e_per_adu=fallback_gain_e_per_adu, - ) + if stellar_variability_only: + vsp_fit, _ = build_stellar_variability_only_lightcurve_from_fluxes( + times, + tFlux, + cFlux, + airmass, + pDict, + jd_times=jd_times, + target_flux_error=psf_noise_data.get('target'), + comp_flux_error=psf_noise_data.get(ckey), + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + comp_index=j, + comp_label=f"Comp {j + 1}", + comp_position=exotic_infoDict['comp_stars'][j], + method_label=comparison_calibration['method_label'], + plot_time_range=full_plot_time_range, + ) + else: + vsp_fit, _, _ = fit_lightcurve( + times, tFlux, cFlux, airmass, ld, pDict, jd_times, + target_flux_error=psf_noise_data.get('target'), + comp_flux_error=psf_noise_data.get(ckey), + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + ) + if vsp_fit is None: + continue ref_flux[j] = { 'myfit': vsp_fit, 'pos': exotic_infoDict['comp_stars'][j] @@ -26445,21 +27700,51 @@ def _main_impl(): if f"{ckey}_unc" in aper_data else None ) - vsp_fit, _, _ = fit_lightcurve( - times[aper_mask], best_target_flux[aper_mask], cFlux, - airmass[aper_mask], ld, pDict, jd_times[aper_mask], - target_flux_error=( - None if best_target_flux_error is None else best_target_flux_error[aper_mask] - ), - comp_flux_error=cFlux_error, - disable_vertical_flux_normalization=disable_vertical_flux_normalization, - use_impactparameter_rather_than_inclination_to_fit= - use_impactparameter_rather_than_inclination_to_fit, - plot_time_range=full_plot_time_range, - use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, - exposure_times_seconds=exposure_times_seconds[aper_mask], - gain_e_per_adu=fallback_gain_e_per_adu, + aper_exposure_times = ( + None + if exposure_times_seconds is None + else exposure_times_seconds[aper_mask] ) + if stellar_variability_only: + vsp_fit, _ = build_stellar_variability_only_lightcurve_from_fluxes( + times[aper_mask], + best_target_flux[aper_mask], + cFlux, + airmass[aper_mask], + pDict, + jd_times=jd_times[aper_mask], + target_flux_error=( + None + if best_target_flux_error is None + else best_target_flux_error[aper_mask] + ), + comp_flux_error=cFlux_error, + exposure_times_seconds=aper_exposure_times, + gain_e_per_adu=fallback_gain_e_per_adu, + comp_index=j, + comp_label=f"Comp {j + 1}", + comp_position=exotic_infoDict['comp_stars'][j], + method_label=comparison_calibration['method_label'], + plot_time_range=full_plot_time_range, + ) + else: + vsp_fit, _, _ = fit_lightcurve( + times[aper_mask], best_target_flux[aper_mask], cFlux, + airmass[aper_mask], ld, pDict, jd_times[aper_mask], + target_flux_error=( + None if best_target_flux_error is None else best_target_flux_error[aper_mask] + ), + comp_flux_error=cFlux_error, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + exposure_times_seconds=aper_exposure_times, + gain_e_per_adu=fallback_gain_e_per_adu, + ) + if vsp_fit is None: + continue ref_flux[j] = { 'myfit': vsp_fit, 'pos': exotic_infoDict['comp_stars'][j] @@ -26530,14 +27815,19 @@ def _main_impl(): selected_method_label = selected_photometry_method_label(photometry_info) display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) adaptive_summary = photometry_info.get('adaptive_summary') + comparison_star_log_label = ( + "Stellar Variability Reference Star" + if stellar_variability_only + else "Transit Fit Comparison Star" + ) if photometry_info['min_aperture'] == 0: # psf if photometry_info.get('comp_star_num') == 'ensemble': - log_info("Transit Fit Comparison Star: ensemble") + log_info(f"{comparison_star_log_label}: ensemble") else: - log_info(f"Transit Fit Comparison Star: #{photometry_info['comp_star_num']}") + log_info(f"{comparison_star_log_label}: #{photometry_info['comp_star_num']}") log_info("Optimal Method: PSF photometry") elif photometry_info['min_aperture'] < 0: # no comp star - log_info("Transit Fit Comparison Star: None") + log_info(f"{comparison_star_log_label}: None") if adaptive_summary is not None: log_info(f"Optimal Aperture: {abs(display_aperture):.2f} +/- {adaptive_summary['aperture_std']:.2f} px") log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") @@ -26550,9 +27840,9 @@ def _main_impl(): log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") else: if photometry_info.get('comp_star_num') == 'ensemble': - log_info("Transit Fit Comparison Star: ensemble") + log_info(f"{comparison_star_log_label}: ensemble") else: - log_info(f"Transit Fit Comparison Star: #{photometry_info['comp_star_num']}") + log_info(f"{comparison_star_log_label}: #{photometry_info['comp_star_num']}") if adaptive_summary is not None: log_info(f"Optimal Aperture: {display_aperture:.2f} +/- {adaptive_summary['aperture_std']:.2f} px") log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") @@ -26590,7 +27880,13 @@ def _main_impl(): warn=True, ) - if fit_every_comparison_candidate and exotic_infoDict['comp_stars']: + if fit_every_comparison_candidate and stellar_variability_only: + log_info( + "Skipping fit_lightcurve_to_every_comparison_candidate because " + "stellar-variability-only mode does not fit transit models." + ) + + if fit_every_comparison_candidate and not stellar_variability_only and exotic_infoDict['comp_stars']: candidate_fit_summaries = fit_lightcurve_to_every_comparison_candidate( times, jd_times, @@ -26907,7 +28203,7 @@ def _main_impl(): wcs_file=wcs_file) else: log_info( - "Skipping AID magnitude output because no transit-fit comparison star was selected.", + "Skipping AID magnitude output because no reference comparison star was selected.", warn=True, ) @@ -26988,9 +28284,14 @@ def _main_impl(): return log_info("\n") - log_info("****************************************") - log_info("Fitting a Light Curve Model to Your Data") - log_info("****************************************\n") + if stellar_variability_only: + log_info("****************************************") + log_info("Preparing Stellar Variability Light Curve") + log_info("****************************************\n") + else: + log_info("****************************************") + log_info("Fitting a Light Curve Model to Your Data") + log_info("****************************************\n") reuse_selected_final_model = bool( fitsortext == 1 @@ -26998,6 +28299,158 @@ def _main_impl(): and photometry_info.get('best_fit_lc') is not None ) + if stellar_variability_only: + if reuse_selected_final_model: + myfit = photometry_info['best_fit_lc'] + goodTimes = np.asarray(getattr(myfit, 'time', goodTimes), dtype=float) + goodAirmasses = np.asarray(getattr(myfit, 'airmass', goodAirmasses), dtype=float) + reused_flux = photometry_info.get('selected_fit_good_flux') + reused_unc = photometry_info.get('selected_fit_good_unc') + if reused_flux is not None and np.shape(reused_flux) == np.shape(goodTimes): + goodFluxes = np.asarray(reused_flux, dtype=float) + else: + goodFluxes = np.asarray(getattr(myfit, 'detrended', goodFluxes), dtype=float) + if reused_unc is not None and np.shape(reused_unc) == np.shape(goodTimes): + goodNormUnc = np.asarray(reused_unc, dtype=float) + else: + goodNormUnc = np.asarray(getattr(myfit, 'detrendederr', goodNormUnc), dtype=float) + log_info( + "Using the selected comparison-star stellar-variability light curve for final outputs; " + "no transit model fit is being run." + ) + else: + prepared_variability = { + 'applied': True, + 'time': np.asarray(goodTimes, dtype=float), + 'flux': np.asarray(goodFluxes, dtype=float), + 'unc': np.asarray(goodNormUnc, dtype=float), + 'airmass': np.asarray(goodAirmasses, dtype=float), + 'jd_time': np.asarray(goodTimes, dtype=float), + 'exposure_time_seconds': goodExposureTimes, + 'target_flux': np.asarray(goodFluxes, dtype=float), + 'comp_flux': np.ones(np.shape(goodFluxes), dtype=float), + 'target_flux_error': np.asarray(goodNormUnc, dtype=float), + 'comp_flux_error': np.full(np.shape(goodFluxes), np.nan, dtype=float), + 'source_indices': np.arange(np.shape(goodFluxes)[0], dtype=int), + } + myfit = build_stellar_variability_only_lightcurve( + prepared_variability, + pDict, + method_label="pre-reduced light curve", + plot_time_range=full_plot_time_range, + ) + if myfit is None: + log_info( + "Error: stellar-variability-only mode could not build a usable " + "out-of-transit light curve from the supplied pre-reduced data.", + error=True, + ) + return + + if np.any(np.isfinite(myfit.time)): + times = np.linspace(np.nanmin(myfit.time), np.nanmax(myfit.time), 1000) + else: + times = np.array([], dtype=float) + data_highres = np.ones(times.shape, dtype=float) + exclusion = getattr(myfit, 'stellar_variability_transit_exclusion', {}) or {} + duration = exclusion.get('duration_days', np.nan) + durs = [duration] if np.isfinite(duration) else [] + + plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + + if fitsortext == 1: + observing_background_series = build_observing_background_series( + psf_data, + aper_data, + photometry_info, + len(exotic_infoDict['comp_stars']), + ) + plot_obs_stats(myfit, exotic_infoDict['comp_stars'], psf_data, obs_stats_sort_index, + obs_stats_keep_mask, pDict['pName'], + exotic_infoDict['save'], exotic_infoDict['date'], + relative_flux_mask=None, + background_series=observing_background_series) + + log_info("\n*********************************************************") + log_info("FINAL STELLAR VARIABILITY ANALYSIS\n") + log_info(" Analysis Mode: stellar variability only") + log_info(" Transit model fitting: skipped") + scatter = getattr(myfit, 'stellar_variability_scatter', np.nan) + if np.isfinite(scatter): + log_info(f" Out-of-transit scatter: {format_residual_scatter(scatter)}") + log_info(f" Light-curve point count: {len(myfit.time)}") + rejected_points = exclusion.get('rejected_point_count', 0) + log_info(f" Predicted in-transit points excluded: {rejected_points}") + if np.isfinite(duration): + log_info(f" Excluded transit-window duration [day]: {round_to_2(duration)}") + if fitsortext == 1: + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + if bestCompStar == 'ensemble': + log_info(" Stellar Variability Reference Star: ensemble") + elif bestCompStar is not None: + log_info(f" Stellar Variability Reference Star: #{bestCompStar} - {comp_coords}") + else: + log_info(" Stellar Variability Reference Star: None") + if photometry_info.get('min_aperture') == 0: + log_info(" Optimal Method: PSF photometry") + else: + log_info(f" Optimal Aperture: {abs(np.round(display_aperture, 2))}") + log_info(f" Optimal Annulus: {np.round(display_annulus, 2)}") + log_info("*********************************************************") + + if vsp_params: + AIDoutput_files = AIDOutputFiles(myfit, pDict, exotic_infoDict, auid, chart_id, vsp_params) + output_files = OutputFiles(myfit, pDict, exotic_infoDict, durs) + error_txt = "\n\tPlease report this issue on the Exoplanet Watch Slack Channel in #data-reductions." + + try: + phase = np.asarray(getattr(myfit, 'phase', get_phase(myfit.time, pDict['pPer'], pDict['midT']))) + output_files.final_lightcurve(phase) + except Exception as e: + log_info(f"\nError: Could not create FinalLightCurve.csv. {error_txt}\n\t{e}", error=True) + try: + if fitsortext == 1: + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + output_files.final_planetary_params(phot_opt=True, vsp_params=vsp_params, + comp_star=bestCompStar, comp_coords=comp_coords, + min_aper=np.round(display_aperture, 2), + min_annul=np.round(display_annulus, 2), + adaptive_summary=photometry_info.get('adaptive_summary'), + photometry_info=photometry_info, + publish_to_root=True) + else: + output_files.final_planetary_params( + phot_opt=False, + vsp_params=vsp_params, + publish_to_root=True, + ) + except Exception as e: + log_info(f"\nError: Could not create FinalParams.json. {error_txt}\n\t{e}", error=True) + try: + if fitsortext == 1: + output_files.plate_status(plateStatus) + except Exception as e: + log_info(f"\nError: Could not create plate_status.csv. {error_txt}\n\t{e}", error=True) + try: + if vsp_params: + AIDoutput_files.aavso() + except Exception as e: + log_info(f"\nError: Could not create AID_AAVSO.txt. {error_txt}\n\t{e}", error=True) + + log_info("Output Files Saved") + + log_info("\n************************") + log_info("End of Reduction Process") + log_info("************************") + + log_info("\n\n************************") + log_info("EXOTIC has successfully run!!!") + log_info("It is now safe to close this window.") + log_info("************************") + + log.debug("Stopped ...") + return + ########################## # NESTED SAMPLING FITTING ########################## diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 2fbecad5..633a1344 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -419,6 +419,7 @@ def save_input(): "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", @@ -432,6 +433,9 @@ def save_input(): "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", + "Reject Overexposed Stars": "Set optional_info 'reject_overexposed_stars' to true to reject overexposed target frames and overexposed comparison-star measurements. Default true.", + "Saturation Value": "Set optional_info 'saturation_value' to the detector saturation value in the same units as the image pixels. If omitted/default, EXOTIC uses FITS SATURATE when available, maps TELESCOP Cecilia to 4096, otherwise uses 65535.", + "Overexposure Threshold Fraction": "Set optional_info 'overexposure_threshold_fraction' to the fraction of saturation used for rejection. Default 0.9.", "Photometry Noise Budget": "Optional noise terms for raw-image photometry: gain_electrons_per_adu, read_noise_electrons, dark_current_electrons_per_second_per_pixel, flat_field_fractional_error, telescope_aperture_m, and scintillation_coefficient. Leave null to ignore an optional term.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", @@ -454,6 +458,7 @@ def save_input(): "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, + "stellar_variability_only": False, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, @@ -469,6 +474,9 @@ def save_input(): "restrict_a/Rs_range_percentage": 10.0, "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, + "reject_overexposed_stars": True, + "saturation_value": 65535, + "overexposure_threshold_fraction": 0.9, "Use target-driven comp selection rather than comp-driven comp selection": "n", "require_comp_star": "y" } @@ -1516,6 +1524,7 @@ def save_input(): "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", @@ -1529,6 +1538,9 @@ def save_input(): "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", + "Reject Overexposed Stars": "Set optional_info 'reject_overexposed_stars' to true to reject overexposed target frames and overexposed comparison-star measurements. Default true.", + "Saturation Value": "Set optional_info 'saturation_value' to the detector saturation value in the same units as the image pixels. If omitted/default, EXOTIC uses FITS SATURATE when available, maps TELESCOP Cecilia to 4096, otherwise uses 65535.", + "Overexposure Threshold Fraction": "Set optional_info 'overexposure_threshold_fraction' to the fraction of saturation used for rejection. Default 0.9.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", @@ -1598,6 +1610,7 @@ def save_input(): "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, + "stellar_variability_only": False, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, @@ -1613,6 +1626,9 @@ def save_input(): "restrict_a/Rs_range_percentage": 10.0, "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, + "reject_overexposed_stars": True, + "saturation_value": 65535, + "overexposure_threshold_fraction": 0.9, "gain_electrons_per_adu": null, "read_noise_electrons": null, "dark_current_electrons_per_second_per_pixel": null, @@ -1667,6 +1683,7 @@ def save_input(): "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, + "stellar_variability_only": False, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, @@ -1677,6 +1694,9 @@ def save_input(): "use_prior_Rp/Rs_when_posterior_pinned": "y", "use_sparse_posterior_live_point_retry": "y", "use_adaptive_apertures": False, + "reject_overexposed_stars": True, + "saturation_value": 65535, + "overexposure_threshold_fraction": 0.9, "gain_electrons_per_adu": null, "read_noise_electrons": null, "dark_current_electrons_per_second_per_pixel": null, diff --git a/exotic/inputs.py b/exotic/inputs.py index 4bf3bfe7..b2436cba 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -209,6 +209,7 @@ def __init__(self, init_opt): 'fast_aperture_mask': False, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', 'prefer_pixel_values_over_wcs_for_target': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, + 'stellar_variability_only': False, 'detrend_on_outoftransit_baseline': True, 'final_fit_baseline_duration_multiplier': 1.0, 'use_eebls_to_initialize_tmid_and_bounds': 'y', @@ -226,6 +227,9 @@ def __init__(self, init_opt): 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, 'use_aperture_corrections_and_full_image_fwhm': False, 'pointing_rejection_sigma': None, + 'reject_overexposed_stars': True, + 'saturation_value': 65535.0, + 'overexposure_threshold_fraction': 0.9, 'gain_electrons_per_adu': None, 'read_noise_electrons': None, 'dark_current_electrons_per_second_per_pixel': None, @@ -472,6 +476,12 @@ def comp_params(self, init_file, planet_dict): 'disable vertical flux normalization', 'Disable vertical flux normalization', ), + 'stellar_variability_only': ( + 'stellar_variability_only', + 'stellar variability only', + 'Stellar Variability Only', + 'Stellar Variability Only? (y/n)', + ), 'detect_bad_pixels_before_photometry': ( 'detect_bad_pixels_before_photometry', 'Detect Bad Pixels Before Photometry? (y/n)', @@ -555,6 +565,22 @@ def comp_params(self, init_file, planet_dict): 'Use Aperture Corrections and Full Image FWHM? (y/n)', 'Use Aperture Corrections And Full Image FWHM? (y/n)', ), + 'reject_overexposed_stars': ( + 'reject_overexposed_stars', + 'Reject Overexposed Stars? (y/n)', + 'Reject Overexposed Target and Comparison Stars? (y/n)', + ), + 'saturation_value': ( + 'saturation_value', + 'saturation_value_adu', + 'Saturation Value', + 'SATURATE', + ), + 'overexposure_threshold_fraction': ( + 'overexposure_threshold_fraction', + 'Overexposure Threshold Fraction', + 'Saturation Rejection Threshold Fraction', + ), 'skip_low_comparison_coverage_rejection': ( 'skip_low_comparison_coverage_rejection', 'Skip Low Comparison Coverage Rejection? (y/n)', diff --git a/exotic/output_files.py b/exotic/output_files.py index 275208d9..1a6a973b 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1643,6 +1643,90 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor extension="json", ) + if getattr(self.fit, 'stellar_variability_only', False): + exclusion = getattr(self.fit, 'stellar_variability_transit_exclusion', {}) or {} + scatter = getattr(self.fit, 'stellar_variability_scatter', np.nan) + params_num = { + "Analysis Mode": "Stellar variability only", + "Transit model fitting": "Skipped", + "Out-of-transit lightcurve point count": str(len(getattr(self.fit, 'time', []))), + "Predicted in-transit points excluded": str(exclusion.get('rejected_point_count', 0)), + } + duration = exclusion.get('duration_days', np.nan) + if np.isfinite(duration): + params_num["Excluded transit-window duration (day)"] = f"{duration:.8f}" + if np.isfinite(scatter): + params_num["Residual scatter around flat stellar-variability model"] = f"{scatter * 100.0:.4f} %" + note = exclusion.get('note') + if note: + params_num["Transit-window exclusion note"] = str(note) + if getattr(self.fit, 'airmass_fit_skipped', False): + params_num["Airmass correction"] = getattr( + self.fit, + 'airmass_correction_note', + "Skipped; no airmass correction applied.", + ) + if isinstance(photometry_info, dict) and photometry_info.get('noise_budget_summary'): + params_num["Photometry noise budget"] = str(photometry_info.get('noise_budget_summary')) + if photometry_info.get('noise_budget_terms'): + params_num["Photometry noise budget terms"] = ", ".join( + str(term) for term in photometry_info.get('noise_budget_terms') + ) + + if vsp_params: + params_num["Variable Reference Star"] = stellar_variability_reference_summary(vsp_params[0]) + params_num["Variable Reference Measurement"] = ( + f"Remeasured {len(vsp_params)} out-of-transit target/reference point(s) " + "against the stellar-variability reference catalog star; AID rows list the " + "BJD_TDB timestamps used." + ) + + if phot_opt: + if comp_star == 'ensemble': + reference_text = "ensemble" + else: + reference_text = ( + f"#{comp_star} - {comp_coords}" + if comp_star is not None and min_aper is not None and min_aper >= 0 + else str(comp_star) + ) + params_num["Stellar Variability Reference Star"] = reference_text + if min_aper == 0: + params_num["Optimal Method"] = "PSF photometry" + else: + if adaptive_summary: + params_num["Adaptive Aperture Scale"] = f"{adaptive_summary['aperture_sigma']:.2f} sigma" + params_num["Adaptive Annulus Scale"] = f"{adaptive_summary['annulus_sigma']:.2f} sigma" + params_num["Optimal Aperture"] = ( + f"{adaptive_summary['aperture_median']:.2f} +/- " + f"{adaptive_summary['aperture_std']:.2f} px" + ) + params_num["Aperture Range"] = ( + f"{adaptive_summary['aperture_min']:.2f} to " + f"{adaptive_summary['aperture_max']:.2f} px" + ) + params_num["Optimal Annulus"] = ( + f"{adaptive_summary['annulus_median']:.2f} +/- " + f"{adaptive_summary['annulus_std']:.2f} px" + ) + params_num["Annulus Range"] = ( + f"{adaptive_summary['annulus_min']:.2f} to " + f"{adaptive_summary['annulus_max']:.2f} px" + ) + else: + params_num["Optimal Aperture"] = f"{abs(min_aper)}" + params_num["Optimal Annulus"] = f"{min_annul}" + + final_params = {'FINAL STELLAR VARIABILITY PARAMETERS': params_num} + with params_file.open('w') as f: + dump(final_params, f, indent=4) + if publish_to_root: + root_params_file = self.dir / params_file.name + if root_params_file != params_file: + root_params_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(params_file, root_params_file) + return + transit_qc = getattr(self.fit, 'transit_qc', None) fit_quality = build_fit_quality_metadata(self.fit) empirical_uncertainty = fit_empirical_transit_uncertainty(self.fit, fit_quality=fit_quality) @@ -2306,7 +2390,7 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,ensemble_score,pairwise_median_score," "pairwise_max_score,self_score,valid_pair_count,coverage_count,coverage_peer_median," "coverage_min_required,coverage_rejected,suitability_outlier_rejected," - "psf_quality_rejected_count,ensemble_frame_rejected_count," + "psf_quality_rejected_count,overexposure_rejected_count,ensemble_frame_rejected_count," "ensemble_frame_required_valid_pairs\n") for summary in comp_summaries: @@ -2328,6 +2412,7 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label summary.get('coverage_rejected'), summary.get('suitability_outlier_rejected'), summary.get('psf_quality_rejected_count', 0), + summary.get('overexposure_rejected_count', 0), summary.get('ensemble_frame_rejected_count', 0), summary.get('ensemble_frame_required_valid_pairs', 0), ] diff --git a/exotic/plate_status.py b/exotic/plate_status.py index ff100a67..8c17bb80 100644 --- a/exotic/plate_status.py +++ b/exotic/plate_status.py @@ -8,6 +8,7 @@ def __init__(self, logfunc): self.logfunc = logfunc self.errorcodes.add("outofframe_target") self.errorcodes.add("lowflux_target") + self.errorcodes.add("overexposed_target") self.errorcodes.add("skybg_target") self.errorcodes.add("fits_error") self.errorcodes.add("alignment_error") @@ -24,6 +25,7 @@ def initializeComparisonStarCount(self, compCount: int): for i in range(compCount): self.errorcodes.add(f"outofframe_comp{i+1}") self.errorcodes.add(f"lowflux_comp{i+1}") + self.errorcodes.add(f"overexposed_comp{i+1}") self.errorcodes.add(f"skybg_comp{i+1}") # Sets current filename (for any reported errors) - sets starIndex=0 (target) def setCurrentFilename(self, filename: str): @@ -60,6 +62,14 @@ def lowFluxAmplitudeWarning(self, starIndex: int, xc: float, yc: float): else: self._logError(f"lowflux_comp{starIndex}", f"Measured flux for Comparison star #{starIndex} is low in file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") + # Report overexposure warning for star index (0=target, 1+=comp #N) + def overexposedWarning(self, starIndex: int, xc: float, yc: float, threshold: float): + if starIndex == 0: # Target star + self._logError("overexposed_target", + f"Target star is overexposed in file {self.filename}; aperture pixels near [{xc:.1f}, {yc:.1f}] exceeded {threshold:.1f}.") + else: + self._logError(f"overexposed_comp{starIndex}", + f"Comparison star #{starIndex} is overexposed in file {self.filename}; aperture pixels near [{xc:.1f}, {yc:.1f}] exceeded {threshold:.1f}.") # Report sky background warning for start ;index' (0=target, 1+=comp #N) def skyBackgroundWarning(self, starIndex: int, xc: float, yc: float): if starIndex == 0: # Target star diff --git a/exotic/plots.py b/exotic/plots.py index 21895e5d..8f82f9bb 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -922,6 +922,81 @@ def _plot_final_residual_rejected_points(ax_lc, ax_res, fit): def plot_final_lightcurve(fit, high_res, targ_name, save, date): + if getattr(fit, 'stellar_variability_only', False): + flux = np.asarray(getattr(fit, 'detrended', getattr(fit, 'data', [])), dtype=float) + flux_err = np.asarray(getattr(fit, 'detrendederr', getattr(fit, 'dataerr', [])), dtype=float) + obs_time = np.asarray(getattr(fit, 'time', []), dtype=float) + if obs_time.shape != flux.shape: + obs_time = np.arange(flux.shape[0], dtype=float) + finite = np.isfinite(obs_time) & np.isfinite(flux) + if flux_err.shape != flux.shape: + flux_err = np.full(flux.shape, np.nan, dtype=float) + if np.any(finite): + time_offset = float(np.nanmin(obs_time[finite])) + plot_time = obs_time - time_offset + x_label = f"Time [BJD_TDB - {time_offset:.5f}]" + else: + time_offset = 0.0 + plot_time = obs_time + x_label = "Point index" + + f, (ax_lc, ax_res) = plt.subplots( + 2, + 1, + figsize=(10, 7), + sharex=True, + gridspec_kw={'height_ratios': [3, 1]}, + ) + ax_lc.set_title(targ_name) + ax_lc.errorbar( + plot_time[finite], + flux[finite], + yerr=flux_err[finite], + fmt='ko', + ms=4, + elinewidth=1, + alpha=0.85, + label="Out-of-transit target/reference flux", + ) + if hasattr(fit, 'time_upsample') and hasattr(fit, 'transit_upsample'): + model_time = np.asarray(fit.time_upsample, dtype=float) - time_offset + model_flux = np.asarray(fit.transit_upsample, dtype=float) + elif np.any(finite): + model_time = np.linspace(np.nanmin(plot_time[finite]), np.nanmax(plot_time[finite]), 1000) + model_flux = np.ones(model_time.shape, dtype=float) + else: + model_time = np.array([], dtype=float) + model_flux = np.array([], dtype=float) + if model_time.size and model_flux.size: + model_order = np.argsort(model_time) + ax_lc.plot(model_time[model_order], model_flux[model_order], 'r', lw=2, label="Flat reference") + ax_lc.set_ylabel("Normalized Flux") + ax_lc.legend(loc='best') + + residual_percent = (flux - 1.0) * 100.0 + ax_res.axhline(0.0, color='r', lw=1.5) + ax_res.errorbar( + plot_time[finite], + residual_percent[finite], + yerr=flux_err[finite] * 100.0, + fmt='ko', + ms=4, + elinewidth=1, + alpha=0.85, + ) + ax_res.set_xlabel(x_label) + ax_res.set_ylabel("O-C [%]") + f.tight_layout() + + Path(save).mkdir(parents=True, exist_ok=True) + try: + f.savefig(Path(save) / _dated_plot_filename("FinalLightCurve", targ_name, date=date, extension="png"), bbox_inches="tight") + f.savefig(Path(save) / _dated_plot_filename("FinalLightCurve", targ_name, date=date, extension="pdf"), bbox_inches="tight") + except Exception: + pass + plt.close(f) + return + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): empirical_uncertainty = fit_empirical_transit_uncertainty(fit) diff --git a/inits.json b/inits.json index b32d8ea0..42a9223e 100644 --- a/inits.json +++ b/inits.json @@ -28,6 +28,7 @@ "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Leave blank/null or set to 0/off to disable. Default disabled.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", @@ -44,6 +45,9 @@ "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", "Aperture Corrections and Full Image FWHM": "Set optional_info 'use_aperture_corrections_and_full_image_fwhm' to true to estimate image FWHM from isolated field stars and apply isolated-star aperture corrections. Default false.", + "Reject Overexposed Stars": "Set optional_info 'reject_overexposed_stars' to true to reject overexposed target frames and overexposed comparison-star measurements. Default true.", + "Saturation Value": "Set optional_info 'saturation_value' to the detector saturation value in the same units as the image pixels. If omitted/default, EXOTIC uses FITS SATURATE when available, maps TELESCOP Cecilia to 4096, otherwise uses 65535.", + "Overexposure Threshold Fraction": "Set optional_info 'overexposure_threshold_fraction' to the fraction of saturation used for rejection. Default 0.9, so pixels above 0.9 * saturation_value are rejected.", "Skip Low Comparison Coverage Rejection": "Set optional_info 'skip_low_comparison_coverage_rejection' to y to disable EXOTIC's default rejection of comparison stars that are valid in far fewer frames than the rest of the comparison-star field. Default n.", "Fit Lightcurve to Every Comparison Candidate": "Set optional_info 'fit_lightcurve_to_every_comparison_candidate' to y to save one target lightcurve fit plot per comparison star into temp/ using the selected photometry setup. Default n.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", @@ -118,6 +122,7 @@ "pointing_rejection_sigma": null, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": false, + "stellar_variability_only": false, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "y", "detrend_on_outoftransit_baseline": true, @@ -134,6 +139,9 @@ "use_aperture_photometry": "y", "use_adaptive_apertures": false, "use_aperture_corrections_and_full_image_fwhm": false, + "reject_overexposed_stars": true, + "saturation_value": 65535, + "overexposure_threshold_fraction": 0.9, "gain_electrons_per_adu": null, "read_noise_electrons": null, "dark_current_electrons_per_second_per_pixel": null, diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 10fe90a3..237ad8f8 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -98,6 +98,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: GAUSSIAN_SIGMA_TO_FWHM, adaptive_aperture_outlier_mask, annotate_transit_qc_expected_values, + aperture_contains_overexposed_pixel, auto_tune_aperture_sigma_grid, build_aperture_correction_profile, build_initial_ars_bounds, @@ -148,6 +149,9 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: log_target_fit_candidate_summaries, noise_budget_config_from_info, normalize_flux_series_to_approximate_unity, + parse_overexposure_threshold_fraction, + parse_saturation_value, + saturation_value_from_header, phase_bin_sigma_clip, parse_deviation_from_expected_transit_in_qc_sigma, prepare_final_fit_lightcurve_series, @@ -181,6 +185,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: summarize_prior_transit_coverage, should_skip_airmass_fit, should_use_eebls_to_initialize_tmid_and_bounds, + should_reject_overexposed_stars, should_fit_lightcurve_to_every_comparison_candidate, should_detect_bad_pixels_before_photometry, should_use_aperture_photometry, @@ -1309,6 +1314,35 @@ def test_should_use_aperture_corrections_and_full_image_fwhm_parses_values(): assert should_use_aperture_corrections_and_full_image_fwhm(True) is True +def test_overexposure_rejection_config_parsers_default_and_override(): + assert should_reject_overexposed_stars(None) is True + assert should_reject_overexposed_stars("y") is True + assert should_reject_overexposed_stars("n") is False + assert should_reject_overexposed_stars(False) is False + + assert parse_saturation_value(None) == pytest.approx(65535.0) + assert parse_saturation_value("") == pytest.approx(65535.0) + assert parse_saturation_value("42000") == pytest.approx(42000.0) + assert parse_saturation_value(-1) == pytest.approx(65535.0) + assert parse_saturation_value("not-a-number") == pytest.approx(65535.0) + + assert parse_overexposure_threshold_fraction(None) == pytest.approx(0.9) + assert parse_overexposure_threshold_fraction("0.75") == pytest.approx(0.75) + assert parse_overexposure_threshold_fraction(1.0) == pytest.approx(1.0) + assert parse_overexposure_threshold_fraction(0) == pytest.approx(0.9) + assert parse_overexposure_threshold_fraction(1.5) == pytest.approx(0.9) + + +def test_saturation_value_from_header_uses_cecilia_microobservatory_value(): + assert saturation_value_from_header({"TELESCOP": "Cecilia "}) == pytest.approx(4096.0) + assert saturation_value_from_header({ + "TELESCOP": "Cecilia ", + "SATURATE": 65535.0, + }) == pytest.approx(4096.0) + assert saturation_value_from_header({"SATURATE": 76500.0}) == pytest.approx(76500.0) + assert saturation_value_from_header({}) is None + + def test_should_use_eebls_to_initialize_tmid_and_bounds_parses_values(): assert should_use_eebls_to_initialize_tmid_and_bounds(None) is True assert should_use_eebls_to_initialize_tmid_and_bounds("y") is True @@ -1694,6 +1728,40 @@ def test_compute_star_aperture_grid_applies_aperture_correction_factors(): np.testing.assert_allclose(corrected_flux[:, 0], raw_flux[:, 0] * np.array([2.0, 1.25])) +def test_aperture_contains_overexposed_pixel_checks_aperture_only(monkeypatch): + import exotic.exotic as exotic_module + + class FakeMask: + def __init__(self, xc, yc, radius): + self.x0 = int(np.floor(xc - radius)) + self.x1 = int(np.ceil(xc + radius)) + 1 + self.y0 = int(np.floor(yc - radius)) + self.y1 = int(np.ceil(yc + radius)) + 1 + y, x = np.mgrid[self.y0:self.y1, self.x0:self.x1] + self.data = (((x - xc) ** 2 + (y - yc) ** 2) <= radius ** 2).astype(float) + + def cutout(self, data): + return np.asarray(data)[self.y0:self.y1, self.x0:self.x1] + + class FakeCircularAperture: + def __init__(self, positions, r): + self.xc, self.yc = positions[0] + self.r = r + + def to_mask(self, method="exact"): + return [FakeMask(self.xc, self.yc, self.r)] + + monkeypatch.setattr(exotic_module, "CircularAperture", FakeCircularAperture) + + data = np.zeros((20, 20), dtype=float) + data[10, 10] = 90.0 + data[2, 2] = 100.0 + + assert aperture_contains_overexposed_pixel(data, 10.0, 10.0, 2.5, 80.0) is True + assert aperture_contains_overexposed_pixel(data, 10.0, 10.0, 2.5, 95.0) is False + assert aperture_contains_overexposed_pixel(data, 10.0, 10.0, 2.5, 90.0) is False + + def test_populate_aperture_data_skips_field_star_corrections_when_disabled(monkeypatch): import exotic.exotic as exotic_module @@ -2789,6 +2857,60 @@ def build_psf_rows(): assert np.isnan(comp1_summary["ensemble_ratio_series"][7]) +def test_select_comparison_calibrated_photometry_masks_overexposed_comp_measurements(): + frame_count = 24 + airmass = np.linspace(1.2, 1.0, frame_count) + + def build_psf_rows(): + rows = np.zeros((frame_count, 7), dtype=float) + rows[:, 0] = 10.0 + rows[:, 1] = 20.0 + rows[:, 2] = 200.0 + rows[:, 3] = 1.0 + rows[:, 4] = 1.0 + return rows + + psf_data = { + "target": build_psf_rows(), + "comp1": build_psf_rows(), + "comp2": build_psf_rows(), + } + aper_data = { + "target": np.full((frame_count, 1, 1), 1000.0), + "target_bg": np.full((frame_count, 1, 1), 10.0), + } + for key in ("comp1", "comp2"): + aper_data[key] = np.full((frame_count, 1, 1), 100.0) + aper_data[f"{key}_bg"] = np.full((frame_count, 1, 1), 10.0) + + comp_overexposed_masks = { + "comp1": np.zeros(frame_count, dtype=bool), + "comp2": np.zeros(frame_count, dtype=bool), + } + comp_overexposed_masks["comp2"][5] = True + aper_data["comp2"][5, 0, 0] = np.nan + + calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + apers=np.array([2.5]), + annuli=np.array([10.0]), + airmass=airmass, + comp_stars=[[10.0, 20.0], [30.0, 40.0]], + sigma=1.0, + use_psf_photometry=False, + use_aperture_photometry=True, + comp_overexposed_masks=comp_overexposed_masks, + ) + comp2_summary = calibration["comp_summaries"][1] + + assert comp2_summary["overexposure_rejected_count"] == 1 + assert comp2_summary["coverage_count"] == frame_count - 1 + assert np.isnan(comp2_summary["ensemble_ratio_series"][5]) + assert calibration["best_comp_index"] == 0 + assert calibration["comp_summaries"][0]["coverage_count"] == frame_count + + def test_cheap_lightcurve_prescore_treats_large_ratio_flag_as_noop(): tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) cflux = np.full(tflux.shape[0], 2.0) @@ -6794,7 +6916,8 @@ def test_compute_photometry_noise_budget_includes_optional_terms(): def test_noise_budget_config_reads_inits_and_header_values(): header = { - "GAIN": 1.5, + "GAIN": 99.0, + "EGAIN": 1.5, "RDNOISE": 7.0, "DARKCURR": 0.02, "FLATERR": 0.003, diff --git a/tests/test_inputs.py b/tests/test_inputs.py index c01e32fe..33fe4b96 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -76,6 +76,76 @@ def test_comp_params_defaults_aavso_comp_to_no(tmp_path): assert inputs.info_dict["aavso_comp"] == "n" +def test_comp_params_defaults_stellar_variability_only_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["stellar_variability_only"] is False + + +def test_comp_params_reads_stellar_variability_only_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "stellar_variability_only": True, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["stellar_variability_only"] is True + + +def test_comp_params_defaults_overexposure_rejection_options(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["reject_overexposed_stars"] is True + assert inputs.info_dict["saturation_value"] == pytest.approx(65535.0) + assert inputs.info_dict["overexposure_threshold_fraction"] == pytest.approx(0.9) + + +def test_comp_params_reads_overexposure_rejection_options_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "Reject Overexposed Stars? (y/n)": False, + "Saturation Value": 42000, + "Overexposure Threshold Fraction": 0.75, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["reject_overexposed_stars"] is False + assert inputs.info_dict["saturation_value"] == pytest.approx(42000.0) + assert inputs.info_dict["overexposure_threshold_fraction"] == pytest.approx(0.75) + + def test_complete_red_uses_fits_x_y_binning_when_pixel_bin_missing(tmp_path, monkeypatch): image_dir = tmp_path / "images" image_dir.mkdir() diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 3d7cda63..5fa341ad 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -1005,3 +1005,86 @@ def test_vsx_variable_parses_default_vsx_object_list(monkeypatch): is_variable = exotic_module.vsx_variable(ra=88.79292, dec=7.40706) assert is_variable is True + + +def _stellar_variability_only_planet_dict(): + return { + 'pName': 'Synthetic b', + 'sName': 'Synthetic', + 'pPer': 1.0, + 'pPerUnc': 0.0001, + 'midT': 10.0, + 'midTUnc': 0.0001, + 'rprs': 0.1, + 'rprsUnc': 0.001, + 'aRs': 12.0, + 'aRsUnc': 0.2, + 'inc': 89.0, + 'incUnc': 0.1, + 'ecc': 0.0, + 'omega': 90.0, + } + + +def test_stellar_variability_out_of_transit_mask_excludes_predicted_transit_window(): + p_dict = _stellar_variability_only_planet_dict() + duration = exotic_module.estimate_transit_duration_from_prior_geometry( + exotic_module.stellar_variability_transit_prior_from_planet_dict(p_dict) + ) + times = np.array([ + p_dict['midT'] - duration, + p_dict['midT'], + p_dict['midT'] + 0.49 * duration, + p_dict['midT'] + duration, + ]) + + keep_mask, summary = exotic_module.stellar_variability_out_of_transit_mask(times, p_dict) + + assert keep_mask.tolist() == [True, False, False, True] + assert summary['applied'] is True + assert summary['rejected_point_count'] == 2 + assert summary['duration_days'] == pytest.approx(duration) + + +def test_build_stellar_variability_only_lightcurve_discards_transit_points(monkeypatch): + p_dict = _stellar_variability_only_planet_dict() + duration = exotic_module.estimate_transit_duration_from_prior_geometry( + exotic_module.stellar_variability_transit_prior_from_planet_dict(p_dict) + ) + offsets = np.array([-3.0, -2.2, -1.4, -0.7, -0.1, 0.0, 0.1, 0.7, 1.4, 2.2, 3.0]) * duration + times = p_dict['midT'] + offsets + target_flux = np.full(times.shape, 10000.0) + comp_flux = np.full(times.shape, 10000.0) + flux_err = np.full(times.shape, 20.0) + airmass = np.ones(times.shape) + + monkeypatch.setattr( + exotic_module, + 'get_phase', + lambda t, per, tmid: ((np.asarray(t, dtype=float) - tmid) / per + 0.5) % 1.0 - 0.5, + ) + + fit, prepared = exotic_module.build_stellar_variability_only_lightcurve_from_fluxes( + times, + target_flux, + comp_flux, + airmass, + p_dict, + jd_times=times, + target_flux_error=flux_err, + comp_flux_error=flux_err, + exposure_times_seconds=np.full(times.shape, 60.0), + gain_e_per_adu=1.0, + comp_index=0, + comp_label="Comp 1", + comp_position=[1, 2], + method_label="PSF photometry", + ) + + assert prepared['applied'] is True + assert fit is not None + assert fit.stellar_variability_only is True + assert np.all(fit.transit == 1.0) + assert fit.stellar_variability_transit_exclusion['rejected_point_count'] == 3 + assert not np.any(np.isclose(fit.time, p_dict['midT'])) + assert len(fit.time) == times.size - 3 diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 05d7c781..9646a6c5 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -1,4 +1,5 @@ import json +from types import SimpleNamespace import numpy as np import pytest @@ -121,6 +122,50 @@ def test_prior_depth_respects_inclination_for_non_transiting_geometry(): assert summary["prior_observable_depth"] == pytest.approx(0.0) +def test_final_params_writes_stellar_variability_only_payload(tmp_path): + (tmp_path / "temp").mkdir() + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.arange(6, dtype=float), + stellar_variability_scatter=0.00123, + stellar_variability_transit_exclusion={ + 'rejected_point_count': 2, + 'duration_days': 0.083, + 'note': 'Excluded synthetic transit-window points.', + }, + airmass_fit_skipped=True, + airmass_correction_note=( + "Skipped in stellar-variability-only mode; no transit/systematics model was fit." + ), + ) + p_dict = {'pName': 'Syntheticb'} + i_dict = {'save': str(tmp_path), 'date': '2020-01-01'} + + OutputFiles(fit, p_dict, i_dict, [0.083]).final_planetary_params( + phot_opt=True, + vsp_params=None, + comp_star=2, + comp_coords=[10, 20], + min_aper=0, + min_annul=15, + photometry_info={'noise_budget_summary': 'gain only'}, + publish_to_root=True, + ) + + temp_file = next((tmp_path / "temp").glob("FinalParams_Syntheticb_2020-01-01.json")) + root_file = tmp_path / temp_file.name + payload = json.loads(temp_file.read_text()) + + params = payload["FINAL STELLAR VARIABILITY PARAMETERS"] + assert params["Analysis Mode"] == "Stellar variability only" + assert params["Transit model fitting"] == "Skipped" + assert params["Predicted in-transit points excluded"] == "2" + assert params["Residual scatter around flat stellar-variability model"] == "0.1230 %" + assert params["Stellar Variability Reference Star"] == "#2 - [10, 20]" + assert params["Optimal Method"] == "PSF photometry" + assert root_file.exists() + + def aavso_json_header(output_text, header_name): prefix = f"#{header_name}=" for line in output_text.splitlines(): @@ -479,6 +524,7 @@ def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): text = summary_path.read_text() assert "# Selected comparison star,1" in text assert "suitability_outlier_rejected" in text + assert "overexposure_rejected_count" in text assert "Comp 1,101,202,true" in text diff --git a/tests/test_plots.py b/tests/test_plots.py index 52605dca..d9a4ae94 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -1,6 +1,8 @@ import matplotlib matplotlib.use("Agg") +from types import SimpleNamespace + import numpy as np import matplotlib.pyplot as plt from matplotlib.axes import Axes @@ -68,6 +70,40 @@ def spy_plot(self, x, y, *args, **kwargs): assert (tmp_path / "temp" / "Observing_Statistics_target_2026-03-09.png").exists() +def test_stellar_variability_final_lightcurve_plots_by_time(tmp_path, monkeypatch): + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2461229.5, 2461229.6, 2461229.8]), + detrended=np.array([1.0, 1.01, 0.99]), + detrendederr=np.array([0.001, 0.001, 0.001]), + time_upsample=np.array([2461229.5, 2461229.8]), + transit_upsample=np.ones(2), + ) + captured_errorbar_x = [] + captured_labels = [] + + original_errorbar = Axes.errorbar + original_set_xlabel = Axes.set_xlabel + + def spy_errorbar(self, x, *args, **kwargs): + captured_errorbar_x.append(np.asarray(x, dtype=float)) + return original_errorbar(self, x, *args, **kwargs) + + def spy_set_xlabel(self, xlabel, *args, **kwargs): + captured_labels.append(xlabel) + return original_set_xlabel(self, xlabel, *args, **kwargs) + + monkeypatch.setattr(Axes, "errorbar", spy_errorbar) + monkeypatch.setattr(Axes, "set_xlabel", spy_set_xlabel) + + plot_final_lightcurve(fit, np.ones(2), "Target", str(tmp_path), "2026-07-08") + + np.testing.assert_allclose(captured_errorbar_x[0], np.array([0.0, 0.1, 0.3]), atol=1.0e-8) + assert captured_labels[-1].startswith("Time [BJD_TDB - 2461229.50000]") + assert captured_labels[-1] != "Orbital Phase" + assert (tmp_path / "FinalLightCurve_Target_2026-07-08.png").exists() + + def test_plot_obs_stats_uses_supplied_background_series(tmp_path, monkeypatch): fit = DummyFit() psf_rows = np.arange(35, dtype=float).reshape(5, 7) From cd59555def5f0e48b515e79f19001fe0c645c1fe Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 14 Jul 2026 09:23:26 +1000 Subject: [PATCH 080/116] stellar variability only options. --- exotic/exotic.py | 57 +++++++++-- exotic/output_files.py | 122 ++++++++++++++++++++-- exotic/plots.py | 152 +++++++++++++++++----------- tests/test_exotic_proper_motion.py | 33 ++++++ tests/test_nextastro_variability.py | 27 +++++ tests/test_output_files.py | 113 +++++++++++++++++++++ tests/test_plots.py | 117 +++++++++++++++++++-- 7 files changed, 540 insertions(+), 81 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index ad640608..fb004668 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -11804,8 +11804,8 @@ def build_observing_background_series(psf_data, aper_data, photometry_info, comp def resolve_frame_aperture_radii(apertures, annuli, adaptive_apertures=False, frame_sigma=np.nan, fallback_sigma=np.nan): - aperture_values = np.asarray(apertures, dtype=float) - annulus_values = np.asarray(annuli, dtype=float) + aperture_values = np.asarray(apertures, dtype=float).reshape(-1) + annulus_values = np.asarray(annuli, dtype=float).reshape(-1) if not adaptive_apertures: return aperture_values, annulus_values @@ -19126,6 +19126,13 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ 'allow_high_error_catalog_reference': allow_high_error_catalog_reference, }) + try: + lc_fit.stellar_variability_params = vsp_params + lc_fit.stellar_variability_target_name = s_name + lc_fit.stellar_variability_reference_label = display_label + except Exception: + pass + plot_stellar_variability(vsp_params, save, s_name, display_label) return vsp_params @@ -19289,7 +19296,12 @@ def derived_catalog_reference_for_selected_comp(fit_lc_refs, comp_stars, vsp_com if best_comp is None or best_comp not in fit_lc_refs: return None, None - selected_fit = fit_lc_refs[best_comp].get('myfit') + selected_ref = fit_lc_refs.get(best_comp) + if not isinstance(selected_ref, dict): + return None, None + selected_fit = selected_ref.get('myfit') + if selected_fit is None: + return None, None selected_pos = comp_stars[best_comp] selected_ra, selected_dec = None, None if comp_ra_dec is not None and best_comp < len(comp_ra_dec): @@ -19326,7 +19338,14 @@ def derived_catalog_reference_for_selected_comp(fit_lc_refs, comp_stars, vsp_com ): continue - ratio = aligned_reference_curve_ratio(selected_fit, fit_lc_refs[anchor_index].get('myfit')) + anchor_ref = fit_lc_refs.get(anchor_index) + if not isinstance(anchor_ref, dict): + continue + anchor_fit = anchor_ref.get('myfit') + if anchor_fit is None: + continue + + ratio = aligned_reference_curve_ratio(selected_fit, anchor_fit) if ratio.size == 0: continue @@ -23745,16 +23764,20 @@ def apply_overexposure_masks_to_aperture_frame(aper_data, frame_index, target_ov def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=False, sigma_hint=np.nan, aperture_correction_factors=None, noise_config=None, exposure_s=np.nan, airmass=np.nan, return_noise=False): + apertures = np.asarray(apertures, dtype=float).reshape(-1) + annuli = np.asarray(annuli, dtype=float).reshape(-1) flux_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) bg_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) noise_grids = empty_noise_budget_grids(flux_grid.shape) if return_noise else None - if np.isnan(xc) or np.isnan(yc): + if not (np.isfinite(xc) and np.isfinite(yc)): return (flux_grid, bg_grid, noise_grids) if return_noise else (flux_grid, bg_grid) mask_method = 'center' if fast_mode else 'exact' for a_idx, aperture_radius in enumerate(apertures): + if not np.isfinite(aperture_radius) or aperture_radius <= 0: + continue aperture = CircularAperture(positions=[(xc, yc)], r=float(aperture_radius)) mask = aperture.to_mask(method=mask_method)[0] data_cutout = mask.cutout(data) @@ -23766,6 +23789,8 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast raw_aperture_sum = (mask.data * data_cutout).sum() for an_idx, annulus_width in enumerate(annuli): + if not np.isfinite(annulus_width) or annulus_width < 0: + continue stage_start = perf_counter() try: if annulus_width > 0: @@ -28355,6 +28380,13 @@ def _main_impl(): exclusion = getattr(myfit, 'stellar_variability_transit_exclusion', {}) or {} duration = exclusion.get('duration_days', np.nan) durs = [duration] if np.isfinite(duration) else [] + if vsp_params: + try: + myfit.stellar_variability_params = vsp_params + myfit.stellar_variability_target_name = pDict.get('sName', pDict.get('pName')) + myfit.stellar_variability_reference_label = vsp_params[0].get('cname') + except Exception: + pass plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) @@ -28649,6 +28681,14 @@ def _main_impl(): tmask = data < 1 durs.append(tmask.sum() * dt) + if vsp_params: + try: + myfit.stellar_variability_params = vsp_params + myfit.stellar_variability_target_name = pDict.get('sName', pDict.get('pName')) + myfit.stellar_variability_reference_label = vsp_params[0].get('cname') + except Exception: + pass + plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) plot_prior_posterior_comparison(myfit, pDict, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) plot_ktmf_qc_metrics(myfit, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) @@ -28691,12 +28731,15 @@ def _main_impl(): empirical_uncertainty = getattr(myfit, 'empirical_transit_uncertainty', None) if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): empirical_uncertainty = fit_empirical_transit_uncertainty(myfit) + rprs_model_error = _finite_float(getattr(myfit, 'errors', {}).get('rprs', np.nan), default=np.nan) + rprs_prior_error = _finite_float(pDict.get('rprsUnc', np.nan), default=np.nan) + rprs_error_fallback = rprs_model_error if np.isfinite(rprs_model_error) else rprs_prior_error rprs_report_error = _finite_float( (empirical_uncertainty or {}).get('combined_rprs_uncertainty'), - default=myfit.errors['rprs'], + default=rprs_error_fallback, ) if not np.isfinite(rprs_report_error) or rprs_report_error < 0: - rprs_report_error = myfit.errors['rprs'] + rprs_report_error = rprs_error_fallback tmid_report_error = fit_parameter_model_data_uncertainty( myfit, 'tmid', diff --git a/exotic/output_files.py b/exotic/output_files.py index 1a6a973b..3a50ffe2 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -10,6 +10,7 @@ format_magnitude_error, format_magnitude, magnitude_text, + normalized_magnitude_error, round_to_2, rounded_magnitude_error, rounded_magnitude_value, @@ -21,6 +22,7 @@ format_magnitude_error, format_magnitude, magnitude_text, + normalized_magnitude_error, round_to_2, rounded_magnitude_error, rounded_magnitude_value, @@ -100,6 +102,27 @@ def finite_float(value, default=np.nan): return value if np.isfinite(value) else default +def apparent_magnitude_calibration_from_vsp_params(vsp_params): + rows = [] + for vsp_p in vsp_params or []: + mag = finite_float(vsp_p.get('mag')) + mag_err = normalized_magnitude_error(vsp_p.get('mag_err')) + if not np.isfinite(mag) or mag_err is None: + continue + rows.append((mag, mag_err, vsp_p.get('mag_band') or 'V')) + + if not rows: + return None + + magnitudes = np.array([row[0] for row in rows], dtype=float) + magnitude_errors = np.array([row[1] for row in rows], dtype=float) + return { + 'baseline_magnitude': float(np.nanmedian(magnitudes)), + 'baseline_error': float(np.nanmedian(magnitude_errors)), + 'band': rows[0][2], + } + + def aavso_json_safe(value): if isinstance(value, dict): return {str(key): aavso_json_safe(subvalue) for key, subvalue in value.items()} @@ -1561,6 +1584,28 @@ def fit_parameter_model_data_uncertainty(fit, parameter_name, empirical_uncertai return float(model_error * empirical_red_noise_error_scale(empirical_uncertainty)) +def fit_rprs_report_error(fit, empirical_uncertainty=None): + if empirical_uncertainty is None: + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if not isinstance(empirical_uncertainty, dict): + empirical_uncertainty = {} + + report_error = finite_float(empirical_uncertainty.get('combined_rprs_uncertainty')) + if np.isfinite(report_error) and report_error >= 0: + return report_error + + errors = getattr(fit, 'errors', {}) or {} + report_error = finite_float(errors.get('rprs')) + if np.isfinite(report_error) and report_error >= 0: + return report_error + + report_error = finite_float(getattr(fit, 'rprs_prior_fallback_data_uncertainty', np.nan)) + if np.isfinite(report_error) and report_error >= 0: + return report_error + + return np.nan + + def fit_impact_parameter_value_error(fit, errors_override=None): parameters = getattr(fit, 'parameters', {}) or {} errors = getattr(fit, 'errors', {}) or {} @@ -1624,14 +1669,68 @@ def final_lightcurve(self, phase): extension="csv", ) + if getattr(self.fit, 'stellar_variability_only', False): + vsp_params = getattr(self.fit, 'stellar_variability_params', None) or [] + with params_file.open('w') as f: + target_name = self.p_dict.get('sName', self.p_dict['pName']) + f.write(f"# FINAL STELLAR VARIABILITY TIMESERIES OF {target_name}\n") + f.write("# BJD_TDB,Magnitude,Uncertainty,Band,Airmass\n") + for vsp_p in vsp_params: + time_value = finite_float(vsp_p.get('time')) + mag_value = format_magnitude(vsp_p.get('mag'), default=None) + mag_error = format_magnitude_error(vsp_p.get('mag_err'), default=None) + if not np.isfinite(time_value) or mag_value is None or mag_error is None: + continue + band = vsp_p.get('mag_band') or self.i_dict.get('filter') or 'V' + airmass = finite_float(vsp_p.get('airmass')) + airmass_text = f"{airmass}" if np.isfinite(airmass) else "na" + f.write(f"{time_value}, {mag_value}, {mag_error}, {band}, {airmass_text}\n") + return + + magnitude_calibration = apparent_magnitude_calibration_from_vsp_params( + getattr(self.fit, 'stellar_variability_params', None) + ) + with params_file.open('w') as f: f.write(f"# FINAL TIMESERIES OF {self.p_dict['pName']}\n") - f.write("# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass\n") + if magnitude_calibration is None: + f.write("# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass\n") + else: + f.write( + "# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass," + "Apparent Magnitude,Magnitude Uncertainty,Band\n" + ) for bjd, phase, flux, fluxerr, model, am in zip(self.fit.time, phase, self.fit.detrended, self.fit.dataerr / self.fit.airmass_model, self.fit.transit, self.fit.airmass_model): - f.write(f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}\n") + row = f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}" + if magnitude_calibration is not None: + flux_value = finite_float(flux) + flux_error = finite_float(fluxerr) + if np.isfinite(flux_value) and flux_value > 0: + apparent_mag = ( + magnitude_calibration['baseline_magnitude'] + - (2.5 * np.log10(flux_value)) + ) + if np.isfinite(flux_error) and flux_error >= 0: + flux_mag_error = abs(2.5 * flux_error / (flux_value * np.log(10))) + apparent_mag_error = ( + magnitude_calibration['baseline_error'] ** 2 + + flux_mag_error ** 2 + ) ** 0.5 + else: + apparent_mag_error = magnitude_calibration['baseline_error'] + mag_text = format_magnitude(apparent_mag, default="na") + mag_error_text = format_magnitude_error(apparent_mag_error, default="na") + else: + mag_text = "na" + mag_error_text = "na" + row = ( + f"{row}, {mag_text}, {mag_error_text}, " + f"{magnitude_calibration['band']}" + ) + f.write(f"{row}\n") def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, min_annul=None, adaptive_summary=None, photometry_info=None, @@ -1747,9 +1846,12 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor qc_residual_scatter = float(abs(residuals.reshape(-1)[0]) / median_flux) headline_params = format_transit_qc_headline_final_params(transit_qc) - rprs_report_error = finite_float(empirical_uncertainty.get('combined_rprs_uncertainty')) + rprs_report_error = fit_rprs_report_error( + self.fit, + empirical_uncertainty=empirical_uncertainty, + ) if not np.isfinite(rprs_report_error) or rprs_report_error < 0: - rprs_report_error = self.fit.errors['rprs'] + rprs_report_error = finite_float(self.p_dict.get('rprsUnc')) tmid_report_error = fit_parameter_model_data_uncertainty( self.fit, 'tmid', @@ -2065,6 +2167,9 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, detrend_model = aavso_detrend_model(self.fit) qc_metadata = build_aavso_qc_metadata(self.fit) fit_quality_metadata = build_fit_quality_metadata(self.fit) + rprs_report_error = fit_rprs_report_error(self.fit) + if not np.isfinite(rprs_report_error) or rprs_report_error < 0: + rprs_report_error = finite_float(self.p_dict.get('rprsUnc')) ktmf_decision_metadata = build_ktmf_decision_metadata(self.fit, photometry_info) photometry_metadata = build_aavso_photometry_metadata(photometry_info) aperture_metadata = build_aavso_aperture_metadata(photometry_info) @@ -2124,7 +2229,7 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, f",u3={round_to_2(ld3[0], ld3[1])} +/- {round_to_2(ld3[1])}\n" f"#PRIORS-XC={dumps(priors_dict)}\n" # code yields f"#RESULTS=Tc={round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- {round_to_2(self.fit.errors['tmid'])}" - f",Rp/R*={round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- {round_to_2(self.fit.errors['rprs'])}" + f",Rp/R*={round_to_2(self.fit.parameters['rprs'], rprs_report_error)} +/- {round_to_2(rprs_report_error)}" f",inc={round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- {round_to_2(self.fit.errors['inc'])}" f",{aavso_airmass_terms[0][0]}={aavso_airmass_terms[0][1]} +/- {aavso_airmass_terms[0][2]}" f",{aavso_airmass_terms[1][0]}={aavso_airmass_terms[1][1]} +/- {aavso_airmass_terms[1][2]}\n" @@ -2223,6 +2328,9 @@ def aavso(self): def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): aavso_airmass_terms = aavso_airmass_results(fit) + rprs_report_error = fit_rprs_report_error(fit) + if not np.isfinite(rprs_report_error) or rprs_report_error < 0: + rprs_report_error = finite_float(planet_dict.get('rprsUnc')) priors = { 'Period': { 'value': str(round_to_2(planet_dict['pPer'], planet_dict['pPerUnc'])), @@ -2286,8 +2394,8 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): 'units': "BJD_TDB" }, 'Rp/R*': { - 'value': str(round_to_2(fit.parameters['rprs'], fit.errors['rprs'])), - 'uncertainty': str(round_to_2(fit.errors['rprs'])) + 'value': str(round_to_2(fit.parameters['rprs'], rprs_report_error)), + 'uncertainty': str(round_to_2(rprs_report_error)) }, 'inc': { 'value': str(round_to_2(fit.parameters['inc'], fit.errors['inc'])), diff --git a/exotic/plots.py b/exotic/plots.py index 8f82f9bb..d3849e36 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -649,6 +649,71 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): plt.close(fig) +def _stellar_variability_magnitude_series(vsp_params): + rows = [] + for vsp_p in vsp_params or []: + time_value = _finite_plot_float(vsp_p.get('time')) + mag_value = _finite_plot_float(vsp_p.get('mag')) + mag_err = normalized_magnitude_error(vsp_p.get('mag_err')) + if ( + time_value is None + or mag_value is None + or mag_err is None + or not is_usable_apparent_magnitude(mag_value) + ): + continue + rows.append((time_value, mag_value, mag_err, vsp_p)) + + if not rows: + return None + + rows.sort(key=lambda row: row[0]) + times = np.array([row[0] for row in rows], dtype=float) + magnitudes = np.array([row[1] for row in rows], dtype=float) + magnitude_errors = np.array([row[2] for row in rows], dtype=float) + return times, magnitudes, magnitude_errors, rows[0][3] + + +def _stellar_variability_apparent_magnitude_calibration(fit): + series = _stellar_variability_magnitude_series( + getattr(fit, 'stellar_variability_params', None) + ) + if series is None: + return None + _, magnitudes, _, first_param = series + finite = np.isfinite(magnitudes) + if not np.any(finite): + return None + return { + 'baseline_magnitude': float(np.nanmedian(magnitudes[finite])), + 'band': first_param.get('mag_band') or 'V', + } + + +def _add_apparent_magnitude_axis(ax_lc, fit): + calibration = _stellar_variability_apparent_magnitude_calibration(fit) + if calibration is None: + return False + baseline_magnitude = calibration['baseline_magnitude'] + + def flux_to_magnitude(flux): + flux = np.asarray(flux, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + return baseline_magnitude - (2.5 * np.log10(flux)) + + def magnitude_to_flux(magnitude): + magnitude = np.asarray(magnitude, dtype=float) + with np.errstate(over='ignore', invalid='ignore'): + return 10 ** ((baseline_magnitude - magnitude) / 2.5) + + secondary_axis = ax_lc.secondary_yaxis( + 'right', + functions=(flux_to_magnitude, magnitude_to_flux), + ) + secondary_axis.set_ylabel(f"Apparent Magnitude ({calibration['band']})") + return True + + # Observation statistics series selection def _select_plot_rows(rows, sort_index=None, sigma_mask=None, relative_flux_mask=None): rows = np.asarray(rows) @@ -923,69 +988,37 @@ def _plot_final_residual_rejected_points(ax_lc, ax_res, fit): def plot_final_lightcurve(fit, high_res, targ_name, save, date): if getattr(fit, 'stellar_variability_only', False): - flux = np.asarray(getattr(fit, 'detrended', getattr(fit, 'data', [])), dtype=float) - flux_err = np.asarray(getattr(fit, 'detrendederr', getattr(fit, 'dataerr', [])), dtype=float) - obs_time = np.asarray(getattr(fit, 'time', []), dtype=float) - if obs_time.shape != flux.shape: - obs_time = np.arange(flux.shape[0], dtype=float) - finite = np.isfinite(obs_time) & np.isfinite(flux) - if flux_err.shape != flux.shape: - flux_err = np.full(flux.shape, np.nan, dtype=float) - if np.any(finite): - time_offset = float(np.nanmin(obs_time[finite])) - plot_time = obs_time - time_offset - x_label = f"Time [BJD_TDB - {time_offset:.5f}]" - else: - time_offset = 0.0 - plot_time = obs_time - x_label = "Point index" - - f, (ax_lc, ax_res) = plt.subplots( - 2, - 1, - figsize=(10, 7), - sharex=True, - gridspec_kw={'height_ratios': [3, 1]}, + series = _stellar_variability_magnitude_series( + getattr(fit, 'stellar_variability_params', None) ) - ax_lc.set_title(targ_name) - ax_lc.errorbar( - plot_time[finite], - flux[finite], - yerr=flux_err[finite], - fmt='ko', - ms=4, - elinewidth=1, - alpha=0.85, - label="Out-of-transit target/reference flux", + if series is None: + return + + obs_time, magnitudes, magnitude_errors, first_param = series + f, ax_lc = plt.subplots(figsize=(8, 5)) + title_name = getattr(fit, 'stellar_variability_target_name', targ_name) + title_lines = [title_name] + reference_label = _stellar_variability_reference_label( + first_param, + getattr(fit, 'stellar_variability_reference_label', first_param.get('cname')), ) - if hasattr(fit, 'time_upsample') and hasattr(fit, 'transit_upsample'): - model_time = np.asarray(fit.time_upsample, dtype=float) - time_offset - model_flux = np.asarray(fit.transit_upsample, dtype=float) - elif np.any(finite): - model_time = np.linspace(np.nanmin(plot_time[finite]), np.nanmax(plot_time[finite]), 1000) - model_flux = np.ones(model_time.shape, dtype=float) - else: - model_time = np.array([], dtype=float) - model_flux = np.array([], dtype=float) - if model_time.size and model_flux.size: - model_order = np.argsort(model_time) - ax_lc.plot(model_time[model_order], model_flux[model_order], 'r', lw=2, label="Flat reference") - ax_lc.set_ylabel("Normalized Flux") - ax_lc.legend(loc='best') + if reference_label: + title_lines.append(reference_label) + metadata_label = _stellar_variability_comparison_metadata_label(first_param) + if metadata_label: + title_lines.append(metadata_label) - residual_percent = (flux - 1.0) * 100.0 - ax_res.axhline(0.0, color='r', lw=1.5) - ax_res.errorbar( - plot_time[finite], - residual_percent[finite], - yerr=flux_err[finite] * 100.0, - fmt='ko', - ms=4, - elinewidth=1, - alpha=0.85, + ax_lc.set_title("\n".join(title_lines), fontsize=11) + ax_lc.errorbar( + obs_time, + magnitudes, + yerr=magnitude_errors, + color="tomato", + fmt='.', ) - ax_res.set_xlabel(x_label) - ax_res.set_ylabel("O-C [%]") + band = first_param.get('mag_band') or 'V' + ax_lc.set_ylabel(f"Magnitude ({band})") + ax_lc.set_xlabel("Time [BJD_TDB]") f.tight_layout() Path(save).mkdir(parents=True, exist_ok=True) @@ -1022,6 +1055,7 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date): _plot_final_residual_rejected_points(ax_lc, ax_res, fit) if drew_data_scatter_band: ax_lc.legend(loc='best') + _add_apparent_magnitude_axis(ax_lc, fit) Path(save).mkdir(parents=True, exist_ok=True) try: diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 237ad8f8..693ec5f2 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -1728,6 +1728,24 @@ def test_compute_star_aperture_grid_applies_aperture_correction_factors(): np.testing.assert_allclose(corrected_flux[:, 0], raw_flux[:, 0] * np.array([2.0, 1.25])) +def test_compute_star_aperture_grid_ignores_invalid_aperture_geometry(): + image = np.ones((20, 20), dtype=float) + + flux, bg = compute_star_aperture_grid( + image, + 0, + 10.0, + 10.0, + np.array([np.nan]), + np.array([0.0]), + ) + + assert flux.shape == (1, 1) + assert bg.shape == (1, 1) + assert np.isnan(flux[0, 0]) + assert np.isnan(bg[0, 0]) + + def test_aperture_contains_overexposed_pixel_checks_aperture_only(monkeypatch): import exotic.exotic as exotic_module @@ -1816,6 +1834,21 @@ def test_resolve_frame_aperture_radii_scales_sigma_grid(): assert np.allclose(annuli, np.array([12.0, 15.0])) +def test_resolve_frame_aperture_radii_accepts_scalar_values(): + apertures, annuli = resolve_frame_aperture_radii( + 2.0, + 8.0, + adaptive_apertures=True, + frame_sigma=1.5, + fallback_sigma=1.0, + ) + + assert apertures.shape == (1,) + assert annuli.shape == (1,) + assert apertures[0] == pytest.approx(3.0) + assert annuli[0] == pytest.approx(12.0) + + def test_resolve_sky_annulus_geometry_enforces_fwhm_floor_and_min_sky_pixels(): geometry = resolve_sky_annulus_geometry(aperture_radius=1.5, annulus_width=2.0, psf_sigma=1.0) diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 5fa341ad..c78685ce 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -705,6 +705,33 @@ def pixel_to_world_values(self, x, y): assert params[0]['derived_reference_anchor_labels'] == ['NextAstro-67890'] +def test_derived_catalog_reference_skips_missing_anchor_fit(): + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + + label, star = exotic_module.derived_catalog_reference_for_selected_comp( + { + 0: {'myfit': DummyFit(), 'pos': [10, 10]}, + 1: None, + }, + [[10, 10], [20, 20]], + { + 'Anchor': { + 'pos': [20, 20], + 'mag': 12.0, + 'error': 0.03, + 'mag_band': 'V', + }, + }, + [1], + 0, + observed_filter='V', + ) + + assert label is None + assert star is None + + def test_stellar_variability_rejects_g_catalog_anchor_for_clearv(monkeypatch, tmp_path): logged = [] diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 9646a6c5..c3ca8091 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -11,6 +11,7 @@ PRIOR_OBSERVABLE_DEPTH_LABEL, AIDOutputFiles, OutputFiles, + aavso_dicts, fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, save_comp_star_calibration_summary, @@ -166,6 +167,65 @@ def test_final_params_writes_stellar_variability_only_payload(tmp_path): assert root_file.exists() +def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): + (tmp_path / "temp").mkdir() + fit = SimpleNamespace( + stellar_variability_only=True, + stellar_variability_params=[ + { + "time": 2461229.89899, + "mag": 13.7378, + "mag_err": 0.0042, + "mag_band": "r", + "airmass": 1.193135, + }, + { + "time": 2461229.90109, + "mag": 13.7401, + "mag_err": 0.0044, + "mag_band": "r", + "airmass": 1.1984942, + }, + ], + ) + p_dict = {'pName': 'WASP-194 b', 'sName': 'WASP-194'} + i_dict = {'save': str(tmp_path), 'date': '2026-07-08', 'filter': 'SR'} + + OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([])) + + output_text = next((tmp_path / "temp").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() + + assert "# FINAL STELLAR VARIABILITY TIMESERIES OF WASP-194" in output_text + assert "# BJD_TDB,Magnitude,Uncertainty,Band,Airmass" in output_text + assert "2461229.89899, 13.738, 0.004, r, 1.193135" in output_text + assert "Flux" not in output_text + + +def test_final_lightcurve_adds_transit_apparent_magnitude_columns_when_calibrated(tmp_path): + (tmp_path / "temp").mkdir() + fit = SimpleNamespace( + time=np.array([2461229.9, 2461229.91]), + detrended=np.array([1.0, 0.99]), + dataerr=np.array([0.001, 0.001]), + airmass_model=np.ones(2), + transit=np.array([1.0, 0.99]), + stellar_variability_params=[ + {"time": 2461229.9, "mag": 13.739, "mag_err": 0.001, "mag_band": "r"}, + {"time": 2461229.91, "mag": 13.741, "mag_err": 0.002, "mag_band": "r"}, + ], + ) + p_dict = {'pName': 'WASP-194 b', 'sName': 'WASP-194'} + i_dict = {'save': str(tmp_path), 'date': '2026-07-08', 'filter': 'SR'} + + OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([0.1, 0.2])) + + output_text = next((tmp_path / "temp").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() + + assert "Apparent Magnitude,Magnitude Uncertainty,Band" in output_text + assert "2461229.9, 0.1, 1.0, 0.001, 1.0, 1.0, 13.740" in output_text + assert output_text.rstrip().endswith(", r") + + def aavso_json_header(output_text, header_name): prefix = f"#{header_name}=" for line in output_text.splitlines(): @@ -583,6 +643,59 @@ def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path assert "V=12.345 +/- 0.067" in reference +def test_transit_outputs_use_rprs_fallback_uncertainty_when_model_error_missing(tmp_path): + fit = DummyFit() + fit.errors.pop("rprs") + fit.rprs_prior_fallback_applied = True + fit.rprs_prior_fallback_data_uncertainty = 0.005 + fit.rprs_prior_fallback_note = "Rp/R* fixed to prior." + (tmp_path / "temp").mkdir() + + p_dict = { + "pName": "HAT-P-32 b", + "pPer": 2.15, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.001, + "aRs": 12.0, + "aRsUnc": 0.4, + "inc": 88.5, + "incUnc": 0.2, + "ecc": 0.0, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "filter": "V", + "filter_desc": "Johnson V", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + final_params = json.loads( + (tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json").read_text(encoding="utf-8") + )["FINAL PLANETARY PARAMETERS"] + + assert "0.005" in final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] + + _, _, results = aavso_dicts( + p_dict, + fit, + i_dict, + [0.1], + (0.1, 0.01), + (0.2, 0.02), + (0.3, 0.03), + (0.4, 0.04), + ) + + assert results["Rp/R*"]["uncertainty"] == "0.005" + + def test_final_planetary_params_reports_transit_comparison_catalog_reference(tmp_path): fit = DummyFit() (tmp_path / "temp").mkdir() diff --git a/tests/test_plots.py b/tests/test_plots.py index d9a4ae94..9d9abc72 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -5,6 +5,7 @@ import numpy as np import matplotlib.pyplot as plt +import pytest from matplotlib.axes import Axes from exotic.plots import ( @@ -70,7 +71,7 @@ def spy_plot(self, x, y, *args, **kwargs): assert (tmp_path / "temp" / "Observing_Statistics_target_2026-03-09.png").exists() -def test_stellar_variability_final_lightcurve_plots_by_time(tmp_path, monkeypatch): +def test_stellar_variability_final_lightcurve_plots_calibrated_magnitude_by_time(tmp_path, monkeypatch): fit = SimpleNamespace( stellar_variability_only=True, time=np.array([2461229.5, 2461229.6, 2461229.8]), @@ -78,29 +79,80 @@ def test_stellar_variability_final_lightcurve_plots_by_time(tmp_path, monkeypatc detrendederr=np.array([0.001, 0.001, 0.001]), time_upsample=np.array([2461229.5, 2461229.8]), transit_upsample=np.ones(2), + stellar_variability_params=[ + { + "time": 2461229.5, + "mag": 13.738, + "mag_err": 0.004, + "cmag": 13.739, + "cmag_err": 0.001, + "mag_band": "r", + "observed_filter": "SR", + "comp_ra": 295.3085, + "comp_dec": 56.1606, + "cname": "RA=295.3085000 Dec=56.1606000", + }, + { + "time": 2461229.6, + "mag": 13.740, + "mag_err": 0.004, + "cmag": 13.739, + "cmag_err": 0.001, + "mag_band": "r", + "observed_filter": "SR", + "comp_ra": 295.3085, + "comp_dec": 56.1606, + "cname": "RA=295.3085000 Dec=56.1606000", + }, + { + "time": 2461229.8, + "mag": 13.735, + "mag_err": 0.004, + "cmag": 13.739, + "cmag_err": 0.001, + "mag_band": "r", + "observed_filter": "SR", + "comp_ra": 295.3085, + "comp_dec": 56.1606, + "cname": "RA=295.3085000 Dec=56.1606000", + }, + ], + stellar_variability_target_name="WASP-194", ) captured_errorbar_x = [] - captured_labels = [] + captured_errorbar_y = [] + captured_xlabels = [] + captured_ylabels = [] original_errorbar = Axes.errorbar original_set_xlabel = Axes.set_xlabel + original_set_ylabel = Axes.set_ylabel - def spy_errorbar(self, x, *args, **kwargs): + def spy_errorbar(self, x, y, *args, **kwargs): captured_errorbar_x.append(np.asarray(x, dtype=float)) - return original_errorbar(self, x, *args, **kwargs) + captured_errorbar_y.append(np.asarray(y, dtype=float)) + return original_errorbar(self, x, y, *args, **kwargs) def spy_set_xlabel(self, xlabel, *args, **kwargs): - captured_labels.append(xlabel) + captured_xlabels.append(xlabel) return original_set_xlabel(self, xlabel, *args, **kwargs) + def spy_set_ylabel(self, ylabel, *args, **kwargs): + captured_ylabels.append(ylabel) + return original_set_ylabel(self, ylabel, *args, **kwargs) + monkeypatch.setattr(Axes, "errorbar", spy_errorbar) monkeypatch.setattr(Axes, "set_xlabel", spy_set_xlabel) + monkeypatch.setattr(Axes, "set_ylabel", spy_set_ylabel) plot_final_lightcurve(fit, np.ones(2), "Target", str(tmp_path), "2026-07-08") - np.testing.assert_allclose(captured_errorbar_x[0], np.array([0.0, 0.1, 0.3]), atol=1.0e-8) - assert captured_labels[-1].startswith("Time [BJD_TDB - 2461229.50000]") - assert captured_labels[-1] != "Orbital Phase" + np.testing.assert_allclose(captured_errorbar_x[0], np.array([2461229.5, 2461229.6, 2461229.8])) + np.testing.assert_allclose(captured_errorbar_y[0], np.array([13.738, 13.740, 13.735])) + assert captured_xlabels[-1] == "Time [BJD_TDB]" + assert captured_xlabels[-1] != "Orbital Phase" + assert captured_ylabels[-1] == "Magnitude (r)" + assert "O-C [%]" not in captured_ylabels assert (tmp_path / "FinalLightCurve_Target_2026-07-08.png").exists() @@ -469,6 +521,55 @@ def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=Fal assert (tmp_path / "FinalLightCurve_Target_2026-03-09.pdf").exists() +def test_plot_final_lightcurve_adds_apparent_magnitude_axis_when_calibrated(tmp_path, monkeypatch): + class DummyFinalFit: + def __init__(self): + self.kwargs = None + self.phase_upsample = np.linspace(-0.05, 0.05, 5) + self.transit_upsample = np.ones(5) + self.stellar_variability_params = [ + {"time": 1.0, "mag": 13.739, "mag_err": 0.001, "mag_band": "r"}, + {"time": 2.0, "mag": 13.741, "mag_err": 0.002, "mag_band": "r"}, + ] + + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False): + self.kwargs = { + "show_flux_baseline_label": show_flux_baseline_label, + "show_model_uncertainty": show_model_uncertainty, + "show_baseline_uncertainty": show_baseline_uncertainty, + } + fig, axes = plt.subplots(2, 1) + return fig, axes + + secondary_calls = [] + secondary_labels = [] + + class FakeSecondaryAxis: + def set_ylabel(self, label): + secondary_labels.append(label) + + def spy_secondary_yaxis(self, location, functions=None, *args, **kwargs): + secondary_calls.append((location, functions)) + return FakeSecondaryAxis() + + monkeypatch.setattr(Axes, "secondary_yaxis", spy_secondary_yaxis) + + plot_final_lightcurve( + DummyFinalFit(), + high_res=np.ones(5), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + assert secondary_labels == ["Apparent Magnitude (r)"] + assert secondary_calls[0][0] == "right" + flux_to_mag, mag_to_flux = secondary_calls[0][1] + assert flux_to_mag(np.array([1.0])) == pytest.approx(np.array([13.740])) + assert mag_to_flux(np.array([13.740])) == pytest.approx(np.array([1.0])) + + def test_plot_final_lightcurve_draws_data_scatter_uncertainty_band(tmp_path, monkeypatch): captured = [] original_fill_between = Axes.fill_between From 4b8cd518547982018a711005757ed21c8d013e39 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 15 Jul 2026 17:34:32 +1000 Subject: [PATCH 081/116] Kalee's persistence bears variable fancy fruit Added in detecting and photometring identified VSX variables in the field. --- README.md | 8 + docs/README.md | 8 + exotic/exotic.py | 3051 +++++++++++++++++++++++++-- exotic/inputs.py | 16 + exotic/output_files.py | 97 +- inits.json | 8 +- tests/test_centroid_wcs.py | 37 +- tests/test_exotic_proper_motion.py | 16 + tests/test_inputs.py | 92 + tests/test_nextastro_variability.py | 963 ++++++++- tests/test_output_files.py | 102 + 11 files changed, 4210 insertions(+), 188 deletions(-) diff --git a/README.md b/README.md index 1f066ac5..9f4af8ed 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,10 @@ Get EXOTIC up and running faster with a json file. Please see the included file "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_aperture_corrections_and_full_image_fwhm": false, + "stellar_variability_only": false, + "use_ensemble_photometry_for_stellar_variability": true, + "photometer_fortuitous_variables": true, + "use_nextastro_vsx_cache_first": false, "skip_low_comparison_coverage_rejection": "n", "fit_lightcurve_to_every_comparison_candidate": "n", "detrend_on_outoftransit_baseline": true, @@ -182,6 +186,10 @@ Get EXOTIC up and running faster with a json file. Please see the included file } ``` +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against its own calibrated comparison ensemble. Exported light curves also retain only frames whose final ensemble-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or an ensemble member are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `fortuitous_variables/optimal_variables/` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `fortuitous_variables/rest_of_the_variables/` otherwise. Set the item to `false` to disable these products. + +`use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/rest classification. + ## Features and Pipeline Architecture - Automatic Plate Solution from http://nova.astrometry.net diff --git a/docs/README.md b/docs/README.md index 90cb05d8..28ef4d7e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -219,6 +219,10 @@ Get EXOTIC up and running faster with a json file. Please see the included file "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_aperture_corrections_and_full_image_fwhm": false, + "stellar_variability_only": false, + "use_ensemble_photometry_for_stellar_variability": true, + "photometer_fortuitous_variables": true, + "use_nextastro_vsx_cache_first": false, "detrend_on_outoftransit_baseline": true, "use_impactparameter_rather_than_inclination_to_fit": "y", "skip_low_comparison_coverage_rejection": "n", @@ -226,3 +230,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file } } ``` + +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against its own calibrated comparison ensemble. Exported light curves also retain only frames whose final ensemble-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or an ensemble member are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `fortuitous_variables/optimal_variables/` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `fortuitous_variables/rest_of_the_variables/` otherwise. Set the item to `false` to disable these products. + +`use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/rest classification. diff --git a/exotic/exotic.py b/exotic/exotic.py index fe1a737c..04ee545d 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -57,12 +57,14 @@ # standard imports import argparse +import csv import copy import faulthandler from functools import lru_cache import inspect import json import hashlib +from math import atan2, cos, radians, sin, sqrt import multiprocessing import os import shutil @@ -104,7 +106,7 @@ # scipy imports from scipy.optimize import least_squares from scipy.signal import savgol_filter -from scipy.ndimage import binary_erosion, gaussian_filter, maximum_filter, median_filter +from scipy.ndimage import binary_erosion, gaussian_filter, label as ndimage_label, maximum_filter, median_filter from skimage.registration import phase_cross_correlation from skimage.transform import SimilarityTransform # error handling for scraper @@ -246,6 +248,19 @@ AIRMASS_FLAT_RANGE_THRESHOLD = 0.05 LIGHTCURVE_MIN_VALID_POINTS = 5 STELLAR_VARIABILITY_ONLY_DEFAULT = False +STELLAR_VARIABILITY_ENSEMBLE_DEFAULT = True +STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS = 2 +STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS = 5 +STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_SIGMA = 3.0 +STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR = 1.0e-4 +STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR_FRACTION = 0.05 +STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG = 0.01 +PHOTOMETER_FORTUITOUS_VARIABLES_DEFAULT = True +USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT = False +FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR = 0.05 +FORTUITOUS_VARIABLE_OPTIMAL_MAX_PERIOD_DAYS = 10.0 +FORTUITOUS_VARIABLE_OPTIMAL_MIN_AMPLITUDE_MAG = 0.3 +FORTUITOUS_VARIABLE_VSX_MAGNITUDE_LIMIT = 20.0 REJECT_OVEREXPOSED_STARS_DEFAULT = True SATURATION_VALUE_DEFAULT = 65535.0 OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT = 0.9 @@ -378,6 +393,8 @@ NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS = 10 NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504} NEXTASTRO_PHOTOMETRY_API_URL = 'https://photometry.nextastro.org' +NEXTASTRO_VSX_QUERY_URL = f'{NEXTASTRO_PHOTOMETRY_API_URL}/vsx_query' +NEXTASTRO_VSX_QUERY_LIMIT = 200000 NEXTASTRO_PHOTOMETRY_COLUMNS = ( 'id', 'source_id', 'ra', 'dec', 'Bmag', 'err_Bmag', 'Vmag', 'err_Vmag', @@ -5434,6 +5451,13 @@ def aligned_selected_array(key, dtype=float): if comp_flux_error_values is not None: selected_result['good_comp_flux_error'] = np.asarray(comp_flux_error_values, dtype=float) selected_result['cflux_fit_error'] = np.asarray(comp_flux_error_values, dtype=float) + annotate_stellar_variability_raw_photometry( + fit, + selected_result.get('tflux_fit'), + selected_result.get('cflux_fit'), + target_flux_error=selected_result.get('tflux_fit_error'), + comp_flux_error=selected_result.get('cflux_fit_error'), + ) if source_indices is not None: selected_result['source_indices'] = np.asarray(source_indices, dtype=int) annotate_transit_detection_qc(fit) @@ -7974,6 +7998,30 @@ def should_use_ensemble_photometry_rather_than_single_comp(config_value): ) +def should_use_ensemble_photometry_for_stellar_variability(config_value): + return parse_bool_config_value( + config_value, + STELLAR_VARIABILITY_ENSEMBLE_DEFAULT, + 'use_ensemble_photometry_for_stellar_variability', + ) + + +def should_photometer_fortuitous_variables(config_value): + return parse_bool_config_value( + config_value, + PHOTOMETER_FORTUITOUS_VARIABLES_DEFAULT, + 'photometer_fortuitous_variables', + ) + + +def should_use_nextastro_vsx_cache_first(config_value): + return parse_bool_config_value( + config_value, + USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT, + 'use_nextastro_vsx_cache_first', + ) + + def parse_automatic_calibration_selector_count(config_value): if config_value is None or config_value == '': return AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT @@ -14006,6 +14054,581 @@ def extract_vsx_objects(payload): return [] +def vsx_object_value(vsx_object, *keys): + if not isinstance(vsx_object, dict): + return None + normalized = {str(key).strip().lower(): value for key, value in vsx_object.items()} + for key in keys: + value = normalized.get(str(key).strip().lower()) + if value not in (None, ''): + return value + return None + + +def vsx_numeric_value(value): + if value is None: + return None + if isinstance(value, (int, float, np.number)): + parsed = float(value) + return parsed if np.isfinite(parsed) else None + match = re.search( + r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?", + str(value).replace(',', ''), + ) + if match is None: + return None + parsed = _finite_float(match.group(0)) + return float(parsed) if parsed is not None else None + + +def vsx_object_ra_dec(vsx_object): + ra_value = vsx_object_value(vsx_object, 'RA2000', 'RA', 'ra_deg') + dec_value = vsx_object_value(vsx_object, 'Declination2000', 'Dec2000', 'DEC', 'dec_deg') + if ra_value is None or dec_value is None: + return None + + ra_text = str(ra_value).strip() + dec_text = str(dec_value).strip() + sexagesimal_ra = ':' in ra_text or len(ra_text.split()) > 1 + try: + if sexagesimal_ra: + coordinate = SkyCoord(ra_text, dec_text, unit=(u.hourangle, u.deg), frame='fk5') + return float(coordinate.ra.deg), float(coordinate.dec.deg) + ra_deg = float(ra_text) + dec_deg = float(dec_text) + if np.isfinite(ra_deg) and np.isfinite(dec_deg): + return ra_deg, dec_deg + except (TypeError, ValueError): + pass + try: + coordinate = SkyCoord(ra_text, dec_text, unit=(u.hourangle, u.deg), frame='fk5') + return float(coordinate.ra.deg), float(coordinate.dec.deg) + except Exception: + return None + + +def vsx_object_period_days(vsx_object): + return vsx_numeric_value(vsx_object_value(vsx_object, 'Period', 'period_days')) + + +def vsx_object_amplitude_mag(vsx_object): + direct_amplitude = vsx_numeric_value( + vsx_object_value(vsx_object, 'Amplitude', 'amplitude_mag') + ) + if direct_amplitude is not None and direct_amplitude >= 0: + return float(direct_amplitude) + + maximum_magnitude = vsx_numeric_value( + vsx_object_value(vsx_object, 'MaxMag', 'MaximumMagnitude', 'max_mag') + ) + minimum_magnitude = vsx_numeric_value( + vsx_object_value(vsx_object, 'MinMag', 'MinimumMagnitude', 'min_mag') + ) + if maximum_magnitude is None or minimum_magnitude is None: + return None + return float(abs(minimum_magnitude - maximum_magnitude)) + + +def fortuitous_variable_category(period_days, amplitude_mag): + period = _finite_float(period_days) + amplitude = _finite_float(amplitude_mag) + if ( + period is not None + and period > 0 + and period <= FORTUITOUS_VARIABLE_OPTIMAL_MAX_PERIOD_DAYS + and amplitude is not None + and amplitude >= FORTUITOUS_VARIABLE_OPTIMAL_MIN_AMPLITUDE_MAG + ): + return 'optimal_variables' + return 'rest_of_the_variables' + + +def nextastro_vsx_query_boxes(ra, dec, radius_degrees): + center_ra = float(ra) % 360.0 + center_dec = float(dec) + radius = max(0.0, float(radius_degrees)) + dec_min = max(-90.0, center_dec - radius) + dec_max = min(90.0, center_dec + radius) + cos_dec = abs(np.cos(np.deg2rad(center_dec))) + if cos_dec < 1.0e-12: + return [(0.0, 360.0, dec_min, dec_max)] + + ra_radius = min(180.0, radius / cos_dec) + if ra_radius >= 180.0: + return [(0.0, 360.0, dec_min, dec_max)] + ra_min = (center_ra - ra_radius) % 360.0 + ra_max = (center_ra + ra_radius) % 360.0 + if ra_min <= ra_max: + return [(ra_min, ra_max, dec_min, dec_max)] + return [ + (ra_min, 360.0, dec_min, dec_max), + (0.0, ra_max, dec_min, dec_max), + ] + + +def normalize_nextastro_vsx_row(row): + if not isinstance(row, dict): + return None + ra = _finite_float(vsx_object_value(row, 'ra_deg', 'RA2000', 'ra')) + dec = _finite_float(vsx_object_value(row, 'dec_deg', 'Declination2000', 'dec')) + if ra is None or dec is None: + return None + normalized_keys = {str(key).strip().lower() for key in row} + magnitude = vsx_object_value(row, 'max_mag', 'mag1', 'MaxMag') + magnitude_band = vsx_object_value(row, 'max_passband', 'mag1_band') + if magnitude not in (None, '') and magnitude_band not in (None, ''): + magnitude = f"{magnitude} {magnitude_band}" + minimum_magnitude = vsx_object_value(row, 'min_mag', 'MinMag') + minimum_band = vsx_object_value(row, 'min_passband') + if minimum_magnitude not in (None, '') and minimum_band not in (None, ''): + minimum_magnitude = f"{minimum_magnitude} {minimum_band}" + return { + **row, + 'Name': vsx_object_value(row, 'name', 'Name'), + 'OID': vsx_object_value(row, 'oid', 'OID'), + 'RA2000': float(ra), + 'Declination2000': float(dec), + 'VariabilityType': vsx_object_value(row, 'var_type', 'VariabilityType', 'Type'), + 'Period': vsx_object_value(row, 'period_days', 'Period'), + 'Amplitude': vsx_object_value(row, 'amplitude_mag', 'Amplitude'), + 'MaxMag': magnitude, + 'MinMag': minimum_magnitude, + 'Category': 'Variable', + '_vsx_source': 'nextastro_cache', + '_vsx_has_full_metadata': { + 'period_days', + 'amplitude_mag', + 'max_mag', + 'min_mag', + }.issubset(normalized_keys), + } + + +@retry(stop=stop_after_delay(30)) +def nextastro_vsx_field_query(ra, dec, radius_degrees): + rows = [] + for ra_min, ra_max, dec_min, dec_max in nextastro_vsx_query_boxes( + ra, + dec, + radius_degrees, + ): + payload = { + 'ra_min': float(ra_min), + 'ra_max': float(ra_max), + 'dec_min': float(dec_min), + 'dec_max': float(dec_max), + 'limit': NEXTASTRO_VSX_QUERY_LIMIT, + 'offset': 0, + 'include_table': True, + 'compact': False, + } + response = requests.post(NEXTASTRO_VSX_QUERY_URL, json=payload, timeout=30) + response.raise_for_status() + body = response.json() + if not isinstance(body, dict) or not isinstance(body.get('rows'), list): + raise RuntimeError("NextAstro VSX cache returned an unexpected response format.") + columns = body.get('columns') if isinstance(body.get('columns'), list) else [] + for raw_row in body['rows']: + if isinstance(raw_row, dict): + row = raw_row + elif isinstance(raw_row, (list, tuple)) and len(raw_row) == len(columns): + row = dict(zip(columns, raw_row)) + else: + continue + normalized = normalize_nextastro_vsx_row(row) + if normalized is not None: + rows.append(normalized) + + deduplicated = [] + seen = set() + for row in rows: + oid = vsx_object_value(row, 'OID', 'oid') + coordinates = vsx_object_ra_dec(row) + key = ( + str(oid).strip() if oid not in (None, '') else '', + round(coordinates[0], 7) if coordinates else None, + round(coordinates[1], 7) if coordinates else None, + ) + if key in seen: + continue + seen.add(key) + deduplicated.append(row) + return deduplicated + + +@retry(stop=stop_after_delay(30)) +def vsx_field_query(ra, dec, radius_degrees, maglimit=FORTUITOUS_VARIABLE_VSX_MAGNITUDE_LIMIT): + url = "https://www.aavso.org/vsx/index.php" + response = requests.get( + url, + params={ + 'view': 'api.list', + 'ra': float(ra), + 'dec': float(dec), + 'radius': float(radius_degrees), + 'tomag': float(maglimit), + 'format': 'json', + }, + timeout=30, + ) + response.raise_for_status() + return extract_vsx_objects(response.json()) + + +def angular_separation_arcsec(first_ra, first_dec, second_ra, second_dec): + first_ra_rad, first_dec_rad, second_ra_rad, second_dec_rad = np.deg2rad([ + first_ra, + first_dec, + second_ra, + second_dec, + ]) + delta_ra = second_ra_rad - first_ra_rad + delta_dec = second_dec_rad - first_dec_rad + haversine = ( + np.sin(delta_dec / 2.0) ** 2 + + np.cos(first_dec_rad) * np.cos(second_dec_rad) * np.sin(delta_ra / 2.0) ** 2 + ) + haversine = float(np.clip(haversine, 0.0, 1.0)) + return float(np.rad2deg(2.0 * np.arcsin(np.sqrt(haversine))) * 3600.0) + + +def enrich_nextastro_vsx_objects(nextastro_objects, aavso_objects, match_radius_arcsec=2.0): + aavso_by_oid = { + str(vsx_object_value(obj, 'OID', 'oid')).strip(): obj + for obj in aavso_objects + if vsx_object_value(obj, 'OID', 'oid') not in (None, '') + } + enriched = [] + for cached_object in nextastro_objects: + match = None + oid = vsx_object_value(cached_object, 'OID', 'oid') + if oid not in (None, ''): + match = aavso_by_oid.get(str(oid).strip()) + cached_coordinates = vsx_object_ra_dec(cached_object) + if match is None and cached_coordinates is not None: + nearest_distance = None + for aavso_object in aavso_objects: + aavso_coordinates = vsx_object_ra_dec(aavso_object) + if aavso_coordinates is None: + continue + distance = angular_separation_arcsec( + *cached_coordinates, + *aavso_coordinates, + ) + if distance <= float(match_radius_arcsec) and ( + nearest_distance is None or distance < nearest_distance + ): + match = aavso_object + nearest_distance = distance + if match is None: + enriched.append(cached_object) + else: + enriched.append({ + **cached_object, + **match, + '_vsx_source': 'nextastro_cache+aavso_metadata', + }) + return enriched + + +def vsx_field_query_with_preference( + ra, + dec, + radius_degrees, + maglimit=FORTUITOUS_VARIABLE_VSX_MAGNITUDE_LIMIT, + use_nextastro_vsx_cache_first=False): + if not use_nextastro_vsx_cache_first: + return vsx_field_query(ra, dec, radius_degrees, maglimit=maglimit) + + try: + cached_objects = nextastro_vsx_field_query(ra, dec, radius_degrees) + except Exception as exc: + log_info( + "Warning: NextAstro VSX cache-first lookup failed; falling back to AAVSO VSX " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + return vsx_field_query(ra, dec, radius_degrees, maglimit=maglimit) + + if not cached_objects: + log_info( + "NextAstro VSX cache-first lookup returned no field objects; " + "checking AAVSO VSX as a completeness fallback." + ) + return vsx_field_query(ra, dec, radius_degrees, maglimit=maglimit) + + log_info( + f"NextAstro VSX cache-first lookup returned {len(cached_objects)} field object(s)." + ) + if all(bool(obj.get('_vsx_has_full_metadata')) for obj in cached_objects): + log_info( + "NextAstro VSX cache supplied the full period/amplitude metadata schema; " + "skipping AAVSO metadata enrichment." + ) + return cached_objects + try: + aavso_objects = vsx_field_query(ra, dec, radius_degrees, maglimit=maglimit) + except Exception as exc: + log_info( + "Warning: AAVSO metadata enrichment failed; continuing with NextAstro VSX " + f"cache coordinates and types ({describe_retry_exception(exc)}).", + warn=True, + ) + return cached_objects + return enrich_nextastro_vsx_objects(cached_objects, aavso_objects) + + +def estimated_magnitude_error_from_reference_count_rate( + reference_image, + x_pos, + y_pos, + exposure_seconds=1.0, + gain_e_per_adu=None): + if reference_image is None: + return None + data = np.asarray(reference_image, dtype=float) + if data.ndim != 2: + return None + x_pos = _finite_float(x_pos) + y_pos = _finite_float(y_pos) + if x_pos is None or y_pos is None: + return None + + aperture_radius = float(REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS) + aperture = CircularAperture(positions=[(x_pos, y_pos)], r=aperture_radius) + aperture_mask = aperture.to_mask(method='exact')[0] + aperture_cutout = aperture_mask.cutout(data, fill_value=np.nan) + if aperture_cutout is None: + return None + aperture_cutout = np.asarray(aperture_cutout, dtype=float) + aperture_weights = np.asarray(aperture_mask.data, dtype=float) + aperture_valid = ( + np.isfinite(aperture_cutout) + & np.isfinite(aperture_weights) + & (aperture_weights > 0) + ) + if not np.any(aperture_valid): + return None + + annulus_geometry = resolve_sky_annulus_geometry( + aperture_radius, + 3.0 * aperture_radius, + ) + sky_background, sky_sigma, sky_pixels = skybg_phot( + data, + -1, + x_pos, + y_pos, + r=annulus_geometry['inner_radius'], + dr=annulus_geometry['annulus_width'], + ) + if ( + not np.isfinite(sky_background) + or not np.isfinite(sky_sigma) + or not np.isfinite(sky_pixels) + or sky_pixels <= 0 + ): + return None + + aperture_pixels = float(np.sum(aperture_weights[aperture_valid])) + aperture_sum = float(np.sum( + aperture_weights[aperture_valid] * aperture_cutout[aperture_valid] + )) + flux = aperture_sum - (float(sky_background) * aperture_pixels) + if not np.isfinite(flux) or flux <= 0: + return None + exposure = _finite_float(exposure_seconds, 1.0) + if exposure is None or exposure <= 0: + exposure = 1.0 + noise_budget = compute_photometry_noise_budget( + flux, + sky_sigma, + aperture_pixels, + sky_pixels, + exposure_s=exposure, + airmass=1.0, + noise_config={'gain_e_per_adu': gain_e_per_adu}, + ) + flux_error = _finite_float(noise_budget.get('total')) + if flux_error is None or not np.isfinite(flux_error) or flux_error < 0: + return None + magnitude_error = (2.5 / np.log(10.0)) * flux_error / flux + if not np.isfinite(magnitude_error): + return None + return { + 'aperture_flux_adu': float(flux), + 'count_rate_adu_per_second': float(flux / exposure), + 'estimated_magnitude_error': float(magnitude_error), + 'reference_sky_background_adu_per_pixel': float(sky_background), + 'reference_sky_sigma_adu': float(sky_sigma), + 'reference_aperture_pixels': float(aperture_pixels), + 'reference_sky_pixels': float(sky_pixels), + 'reference_flux_error_adu': float(flux_error), + 'reference_noise_components_adu': { + key: float(value) + for key, value in noise_budget.items() + if np.isfinite(value) + }, + } + + +def discover_fortuitous_vsx_variables( + wcs_file, + image_shape, + img_scale, + reference_image, + obs_filter, + target_pixel=None, + field_catalog=None, + exposure_seconds=1.0, + gain_e_per_adu=None, + saturation_threshold=None, + maximum_magnitude_error=FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + use_nextastro_vsx_cache_first=USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT): + if not wcs_file or reference_image is None or img_scale is None: + return [] + try: + image_height, image_width = image_shape[:2] + image_scale = float(img_scale) + wcs_header = search_wcs(wcs_file) + center_ra, center_dec = wcs_header.pixel_to_world_values( + float(image_width) / 2.0, + float(image_height) / 2.0, + ) + radius_degrees = ( + 0.5 * image_scale * float(np.hypot(image_width, image_height)) + + NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC + ) / 3600.0 + except Exception as exc: + log_info( + f"Warning: could not define the VSX field footprint for fortuitous photometry ({exc}).", + warn=True, + ) + return [] + + try: + vsx_objects = vsx_field_query_with_preference( + center_ra, + center_dec, + radius_degrees, + use_nextastro_vsx_cache_first=use_nextastro_vsx_cache_first, + ) + except Exception as exc: + log_info( + "Warning: full-field VSX lookup for fortuitous variables failed " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + return [] + + target_values = None + try: + candidate_target = np.asarray(target_pixel, dtype=float).reshape(-1) + if candidate_target.size >= 2 and np.all(np.isfinite(candidate_target[:2])): + target_values = candidate_target[:2] + except (TypeError, ValueError): + target_values = None + + variables = [] + for vsx_object in vsx_objects: + vsx_category = str(vsx_object_value(vsx_object, 'Category') or '').strip().lower() + if vsx_category and vsx_category != 'variable': + continue + coordinates = vsx_object_ra_dec(vsx_object) + if coordinates is None: + continue + ra_deg, dec_deg = coordinates + try: + x_pos, y_pos = wcs_header.world_to_pixel_values(ra_deg, dec_deg) + x_pos = float(np.asarray(x_pos).reshape(-1)[0]) + y_pos = float(np.asarray(y_pos).reshape(-1)[0]) + except Exception: + continue + if not pixel_within_image( + x_pos, + y_pos, + image_shape, + margin=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS + 2, + ): + continue + if target_values is not None: + target_distance = float(np.hypot(x_pos - target_values[0], y_pos - target_values[1])) + if target_distance <= REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS: + continue + if ( + _finite_float(saturation_threshold) is not None + and aperture_contains_overexposed_pixel( + reference_image, + x_pos, + y_pos, + REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, + float(saturation_threshold), + ) + ): + continue + + count_rate_estimate = estimated_magnitude_error_from_reference_count_rate( + reference_image, + x_pos, + y_pos, + exposure_seconds=exposure_seconds, + gain_e_per_adu=gain_e_per_adu, + ) + if count_rate_estimate is None: + continue + if count_rate_estimate['estimated_magnitude_error'] >= float(maximum_magnitude_error): + continue + + if any( + np.hypot(existing['x'] - x_pos, existing['y'] - y_pos) + <= REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS + for existing in variables + ): + continue + + catalog_match = None + if field_catalog is not None: + catalog_match = nextastro_photometry_catalog_match( + field_catalog, + ra_deg, + dec_deg, + obs_filter, + ) + period_days = vsx_object_period_days(vsx_object) + amplitude_mag = vsx_object_amplitude_mag(vsx_object) + name = str( + vsx_object_value(vsx_object, 'Name', 'name', 'Identifier') + or f"VSX J{ra_deg:.6f}{dec_deg:+.6f}" + ).strip() + variables.append({ + 'name': name, + 'auid': vsx_object_value(vsx_object, 'AUID', 'auid'), + 'variable_type': vsx_object_value(vsx_object, 'Type', 'VariabilityType', 'VarType'), + 'period_days': period_days, + 'amplitude_mag': amplitude_mag, + 'category': fortuitous_variable_category(period_days, amplitude_mag), + 'ra': float(ra_deg), + 'dec': float(dec_deg), + 'x': float(x_pos), + 'y': float(y_pos), + 'pos': [float(x_pos), float(y_pos)], + 'catalog_match': catalog_match, + **count_rate_estimate, + }) + + variables.sort(key=lambda variable: ( + 0 if variable['category'] == 'optimal_variables' else 1, + variable['estimated_magnitude_error'], + variable['name'], + )) + log_info( + "Fortuitous-variable VSX field search retained " + f"{len(variables)} star(s) with reference-frame estimated errors below " + f"{float(maximum_magnitude_error):.3f} mag." + ) + return variables + + def describe_retry_exception(err): if isinstance(err, RetryError): last_attempt = getattr(err, 'last_attempt', None) @@ -14348,9 +14971,19 @@ def row_nextastro_magnitude(row, band_candidates, max_error=CATALOG_REFERENCE_MA def sky_separation_arcsec(ra_a, dec_a, ra_b, dec_b): - first = SkyCoord(float(ra_a) * u.deg, float(dec_a) * u.deg, frame='fk5') - second = SkyCoord(float(ra_b) * u.deg, float(dec_b) * u.deg, frame='fk5') - return float(first.separation(second).arcsec) + ra_a_rad = radians(float(ra_a)) + dec_a_rad = radians(float(dec_a)) + ra_b_rad = radians(float(ra_b)) + dec_b_rad = radians(float(dec_b)) + delta_ra = ra_b_rad - ra_a_rad + delta_dec = dec_b_rad - dec_a_rad + haversine = ( + sin(delta_dec / 2.0) ** 2 + + cos(dec_a_rad) * cos(dec_b_rad) * sin(delta_ra / 2.0) ** 2 + ) + haversine = min(max(haversine, 0.0), 1.0) + separation_rad = 2.0 * atan2(sqrt(haversine), sqrt(1.0 - haversine)) + return float(separation_rad * 206264.80624709636) def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, @@ -14517,33 +15150,19 @@ def connected_component_sizes(mask): if mask.ndim != 2 or not np.any(mask): return None, [] - labels = np.full(mask.shape, -1, dtype=int) - component_sizes = [] - component_id = 0 - height, width = mask.shape - - for start_y, start_x in np.argwhere(mask): - if labels[start_y, start_x] != -1: - continue - - stack = [(int(start_y), int(start_x))] - labels[start_y, start_x] = component_id - size = 0 - while stack: - y_pos, x_pos = stack.pop() - size += 1 - for neighbor_y in range(max(0, y_pos - 1), min(height, y_pos + 2)): - for neighbor_x in range(max(0, x_pos - 1), min(width, x_pos + 2)): - if not mask[neighbor_y, neighbor_x]: - continue - if labels[neighbor_y, neighbor_x] != -1: - continue - labels[neighbor_y, neighbor_x] = component_id - stack.append((neighbor_y, neighbor_x)) - component_sizes.append(size) - component_id += 1 - - return labels, component_sizes + # Eight-connected labeling is equivalent to the former Python flood fill, + # but runs in compiled scipy code and avoids visiting every threshold pixel + # through nested Python loops on multi-megapixel reference frames. + labels, component_count = ndimage_label( + mask, + structure=np.ones((3, 3), dtype=np.uint8), + ) + if component_count <= 0: + return None, [] + component_sizes = np.bincount(labels.reshape(-1), minlength=component_count + 1)[1:] + labels = labels.astype(np.int32, copy=False) + labels -= 1 # Preserve the existing API: background=-1, components start at zero. + return labels, component_sizes.tolist() def detect_reference_fallback_bright_stars( @@ -14587,10 +15206,16 @@ def detect_reference_fallback_bright_stars( height, width = signal.shape margin = max(int(min_sep), int(aperture_radius) + 2) - flat_order = np.argsort(signal, axis=None)[::-1] - yy, xx = np.indices(signal.shape) - source_labels, source_sizes = connected_component_sizes(signal >= threshold) + threshold_mask = signal >= threshold + threshold_flat_indices = np.flatnonzero(threshold_mask) + if threshold_flat_indices.size == 0: + return [] + threshold_values = signal.reshape(-1)[threshold_flat_indices] + flat_order = threshold_flat_indices[np.argsort(threshold_values)[::-1]] + source_labels, source_sizes = connected_component_sizes(threshold_mask) stars = [] + separation_excluded = np.zeros(signal.shape, dtype=bool) + separation_radius = float(min_sep) for flat_index in flat_order: y_pos, x_pos = np.unravel_index(int(flat_index), signal.shape) @@ -14605,15 +15230,40 @@ def detect_reference_fallback_bright_stars( continue if x_pos < margin or y_pos < margin or x_pos >= (width - margin) or y_pos >= (height - margin): continue - if any((star['x'] - x_pos) ** 2 + (star['y'] - y_pos) ** 2 < (float(min_sep) ** 2) - for star in stars): + if separation_excluded[y_pos, x_pos]: continue - r2 = (xx - x_pos) ** 2 + (yy - y_pos) ** 2 - flux = float(signal[r2 <= (float(aperture_radius) ** 2)].sum()) + radius = float(aperture_radius) + x_min = max(0, int(np.floor(x_pos - radius))) + x_max = min(width, int(np.ceil(x_pos + radius)) + 1) + y_min = max(0, int(np.floor(y_pos - radius))) + y_max = min(height, int(np.ceil(y_pos + radius)) + 1) + local_signal = signal[y_min:y_max, x_min:x_max] + local_y, local_x = np.ogrid[y_min:y_max, x_min:x_max] + local_aperture = ( + (local_x - x_pos) ** 2 + (local_y - y_pos) ** 2 + <= radius ** 2 + ) + flux = float(local_signal[local_aperture].sum()) if flux <= 0: continue stars.append({'x': float(x_pos), 'y': float(y_pos), 'flux': flux}) + exclusion_x_min = max(0, int(np.floor(x_pos - separation_radius))) + exclusion_x_max = min(width, int(np.ceil(x_pos + separation_radius)) + 1) + exclusion_y_min = max(0, int(np.floor(y_pos - separation_radius))) + exclusion_y_max = min(height, int(np.ceil(y_pos + separation_radius)) + 1) + exclusion_y, exclusion_x = np.ogrid[ + exclusion_y_min:exclusion_y_max, + exclusion_x_min:exclusion_x_max, + ] + exclusion_circle = ( + (exclusion_x - x_pos) ** 2 + (exclusion_y - y_pos) ** 2 + < separation_radius ** 2 + ) + separation_excluded[ + exclusion_y_min:exclusion_y_max, + exclusion_x_min:exclusion_x_max, + ][exclusion_circle] = True if len(stars) >= max_stars: break @@ -14797,9 +15447,10 @@ def image_aperture_signal_flux(image_data, x_pos, y_pos, if not np.any(finite): return np.nan background = float(np.nanmedian(data[finite])) - yy, xx = np.indices(data.shape) - mask = (xx - x_pos) ** 2 + (yy - y_pos) ** 2 <= radius ** 2 - aperture_values = data[mask] + cutout = data[y_min:y_max, x_min:x_max] + local_y, local_x = np.ogrid[y_min:y_max, x_min:x_max] + mask = (local_x - x_pos) ** 2 + (local_y - y_pos) ** 2 <= radius ** 2 + aperture_values = cutout[mask] aperture_values = aperture_values[np.isfinite(aperture_values)] if aperture_values.size == 0: return np.nan @@ -14994,7 +15645,9 @@ def select_automatic_optimal_calibration_stars( field_catalog, count=AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT, min_comp_target_sep=REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS, - colour_term_metadata=None): + colour_term_metadata=None, + brightest_first=False, + saturation_threshold=None): max_count = parse_automatic_calibration_selector_count(count) if image_data is None or field_catalog is None: return [], [] @@ -15016,14 +15669,22 @@ def select_automatic_optimal_calibration_stars( height, width = image_shape[:2] target_xi = int(np.clip(round(target_x), 0, width - 1)) target_yi = int(np.clip(round(target_y), 0, height - 1)) - target_match = nextastro_catalog_nearest_color_row( - field_catalog, - ra_wcs[target_yi][target_xi], - dec_wcs[target_yi][target_xi], - obs_filter, - ) + if brightest_first: + target_match = nextastro_photometry_catalog_match( + field_catalog, + ra_wcs[target_yi][target_xi], + dec_wcs[target_yi][target_xi], + obs_filter, + ) + else: + target_match = nextastro_catalog_nearest_color_row( + field_catalog, + ra_wcs[target_yi][target_xi], + dec_wcs[target_yi][target_xi], + obs_filter, + ) target_color = nextastro_catalog_color((target_match or {}).get('catalog_row'), obs_filter) - if target_color is None: + if not brightest_first and target_color is None: log_info( "Warning: automatic calibration selector could not derive a target color from the " "NextAstro photometry catalog.", @@ -15031,6 +15692,10 @@ def select_automatic_optimal_calibration_stars( ) return [], [] + log_info( + "Scanning the reference frame for bright, non-saturated ensemble candidates." + ) + detection_started = perf_counter() detected_stars = detect_reference_fallback_bright_stars( image_data, max_stars=max( @@ -15039,9 +15704,15 @@ def select_automatic_optimal_calibration_stars( ), threshold_percentile=AUTOMATIC_CALIBRATION_SELECTOR_DETECTION_PERCENTILE, ) - comp_pool = filter_reference_fallback_stars_to_middle_fifty_percent( - dedupe_reference_fallback_stars(detected_stars), - image_shape, + log_info( + "Reference-frame ensemble candidate scan found " + f"{len(detected_stars)} source(s) in {perf_counter() - detection_started:.2f} seconds." + ) + detected_pool = dedupe_reference_fallback_stars(detected_stars) + comp_pool = ( + detected_pool + if brightest_first + else filter_reference_fallback_stars_to_middle_fifty_percent(detected_pool, image_shape) ) target_detection = nearest_reference_fallback_star_by_pixels( comp_pool, @@ -15066,10 +15737,25 @@ def select_automatic_optimal_calibration_stars( if (x_pos - target_x) ** 2 + (y_pos - target_y) ** 2 < min_sep2: continue brightness_ratio = float(star_flux / target_flux) - if not ( - AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MIN_RATIO - <= brightness_ratio - <= AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MAX_RATIO + if ( + not brightest_first + and not ( + AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MIN_RATIO + <= brightness_ratio + <= AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MAX_RATIO + ) + ): + continue + if ( + brightest_first + and _finite_float(saturation_threshold) is not None + and aperture_contains_overexposed_pixel( + image_data, + x_pos, + y_pos, + REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, + float(saturation_threshold), + ) ): continue xi = int(np.clip(round(x_pos), 0, width - 1)) @@ -15078,20 +15764,36 @@ def select_automatic_optimal_calibration_stars( comp_dec = _finite_float(dec_wcs[yi][xi]) if comp_ra is None or comp_dec is None: continue - match = nextastro_catalog_nearest_color_row(field_catalog, comp_ra, comp_dec, obs_filter) - color = nextastro_catalog_color((match or {}).get('catalog_row'), obs_filter) - if match is None or color is None: - continue - color_delta = abs(color['color'] - target_color['color']) + if brightest_first: + match = nextastro_photometry_catalog_match( + field_catalog, + comp_ra, + comp_dec, + obs_filter, + ) + color = nextastro_catalog_color((match or {}).get('catalog_row'), obs_filter) + if match is None: + continue + color_delta = ( + abs(color['color'] - target_color['color']) + if color is not None and target_color is not None + else np.nan + ) + else: + match = nextastro_catalog_nearest_color_row(field_catalog, comp_ra, comp_dec, obs_filter) + color = nextastro_catalog_color((match or {}).get('catalog_row'), obs_filter) + if match is None or color is None: + continue + color_delta = abs(color['color'] - target_color['color']) colour_term, colour_term_error = colour_term_for_catalog_label( colour_term_metadata, - color['label'], + (color or {}).get('label'), ) expected_colour_mismatch_mag = None colour_term_uncertainty_mag = None - if colour_term is not None and np.isfinite(colour_term): + if colour_term is not None and np.isfinite(colour_term) and np.isfinite(color_delta): expected_colour_mismatch_mag = abs(float(colour_term)) * float(color_delta) - if colour_term_error is not None and np.isfinite(colour_term_error): + if colour_term_error is not None and np.isfinite(colour_term_error) and np.isfinite(color_delta): colour_term_uncertainty_mag = abs(float(color_delta)) * float(colour_term_error) candidates.append({ 'x': float(x_pos), @@ -15101,10 +15803,16 @@ def select_automatic_optimal_calibration_stars( 'ra': comp_ra, 'dec': comp_dec, 'catalog_match': match, - 'color': color['color'], - 'color_label': color['label'], - 'target_color': target_color['color'], + 'color': (color or {}).get('color', np.nan), + 'color_label': (color or {}).get('label', ''), + 'target_color': (target_color or {}).get('color', np.nan), 'color_delta': float(color_delta), + 'catalog_magnitude': match.get('mag'), + 'catalog_magnitude_error': match.get('error'), + 'catalog_magnitude_band': match.get('mag_band'), + 'target_catalog_magnitude': (target_match or {}).get('mag'), + 'target_catalog_magnitude_error': (target_match or {}).get('error'), + 'target_catalog_magnitude_band': (target_match or {}).get('mag_band'), 'colour_term': float(colour_term) if colour_term is not None else None, 'colour_term_error': ( float(colour_term_error) if colour_term_error is not None else None @@ -15122,23 +15830,35 @@ def select_automatic_optimal_calibration_stars( 'target_flux': float(target_flux), }) - candidates.sort( - key=lambda candidate: ( - ( - candidate['expected_colour_mismatch_mag'] - if candidate.get('expected_colour_mismatch_mag') is not None - else candidate['color_delta'] - ), - abs(np.log(candidate['brightness_ratio'])), - -candidate['flux'], + if brightest_first: + candidates.sort( + key=lambda candidate: ( + -candidate['flux'], + _finite_float( + candidate.get('expected_colour_mismatch_mag'), + _finite_float(candidate.get('color_delta'), np.inf), + ), + ) + ) + else: + candidates.sort( + key=lambda candidate: ( + ( + candidate['expected_colour_mismatch_mag'] + if candidate.get('expected_colour_mismatch_mag') is not None + else candidate['color_delta'] + ), + abs(np.log(candidate['brightness_ratio'])), + -candidate['flux'], + ) ) - ) selected_candidates = candidates[:max_count] comp_stars = [[candidate['x'], candidate['y']] for candidate in selected_candidates] return comp_stars, selected_candidates -def log_automatic_optimal_calibration_selection(comp_stars, candidates, requested_count): +def log_automatic_optimal_calibration_selection(comp_stars, candidates, requested_count, + brightest_first=False): if not comp_stars: log_info( "Warning: automatic optimal calibration selector did not find any usable comparison stars; " @@ -15146,13 +15866,29 @@ def log_automatic_optimal_calibration_selection(comp_stars, candidates, requeste warn=True, ) return + if brightest_first: + candidate_text = "image-detected, non-saturated on the reference frame, and NextAstro matched" + ranking_text = "ranked brightest-first for the stellar-variability ensemble" + else: + candidate_text = "image-detected, flux-matched to 0.5-2.0x the target, and NextAstro matched" + ranking_text = "ranked by catalog color similarity to the target" log_info( "Automatic optimal calibration selector chose " f"{len(comp_stars)} comparison star(s) out of the requested {requested_count}. " - "Candidates were image-detected, flux-matched to 0.5-2.0x the target, NextAstro matched, " - "and ranked by catalog color similarity to the target." + f"Candidates were {candidate_text} and {ranking_text}." ) for index, candidate in enumerate(candidates, start=1): + if brightest_first: + catalog_band = candidate.get('catalog_magnitude_band') or 'magnitude' + log_info( + f" Stellar-variability ensemble candidate #{index}: " + f"pixels=[{candidate['x']:.2f}, {candidate['y']:.2f}], " + f"aperture flux={candidate['flux']:.3g}, " + f"catalog {catalog_band}=" + f"{candidate.get('catalog_magnitude', np.nan):.3f} +/- " + f"{candidate.get('catalog_magnitude_error', np.nan):.3f} mag." + ) + continue mismatch_text = "" if candidate.get('expected_colour_mismatch_mag') is not None: mismatch_text = ( @@ -15171,6 +15907,25 @@ def log_automatic_optimal_calibration_selection(comp_stars, candidates, requeste ) +def calibration_catalog_identity(star, label=None): + if not isinstance(star, dict): + return None + catalog_row = star.get('catalog_row') if isinstance(star.get('catalog_row'), dict) else {} + source_id = star.get('source_id', catalog_row.get('source_id')) + if source_id not in (None, ''): + return 'source_id', str(source_id).strip() + catalog_id = star.get('id', catalog_row.get('id')) + if catalog_id not in (None, ''): + return 'catalog_id', str(catalog_id).strip() + catalog_ra = _finite_float(star.get('catalog_ra', catalog_row.get('ra'))) + catalog_dec = _finite_float(star.get('catalog_dec', catalog_row.get('dec'))) + if catalog_ra is not None and catalog_dec is not None: + return 'catalog_position', round(catalog_ra, 7), round(catalog_dec, 7) + if str(label or '').startswith('NextAstro-'): + return 'catalog_label', str(label) + return None + + def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, existing_comp_stars=None, field_catalog=None): calibration_stars = dict(existing_comp_stars or {}) @@ -15179,6 +15934,11 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist for value in calibration_stars.values() if isinstance(value, dict) } + existing_catalog_identities = { + identity + for label, value in calibration_stars.items() + if (identity := calibration_catalog_identity(value, label)) is not None + } added_count = 0 for index, (comp_pos, comp_radec) in enumerate(zip(comp_stars, comp_ra_dec)): @@ -15210,9 +15970,17 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist ) continue - match.update({ - 'ra': comp_ra, - 'dec': comp_dec, + catalog_identity = calibration_catalog_identity(match) + if catalog_identity is not None and catalog_identity in existing_catalog_identities: + log_info( + f"Skipping duplicate NextAstro calibration for comparison star #{index + 1}: " + "the same catalog source is already represented in the comparison pool." + ) + continue + + match.update({ + 'ra': comp_ra, + 'dec': comp_dec, 'pos': list(comp_pos), 'catalog_source': 'NextAstro photometry catalog', 'is_aavso_vsp': False, @@ -15221,6 +15989,8 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist unique_label = unique_nextastro_calibration_label(calibration_stars, match) calibration_stars[unique_label] = match existing_positions.add(tuple(comp_pos)) + if catalog_identity is not None: + existing_catalog_identities.add(catalog_identity) added_count += 1 mag_text = magnitude_text(match['mag_band'], match['mag'], match['error']) log_info( @@ -19030,6 +19800,63 @@ def stellar_variability_label(comp_label, comp_star): return comp_label +def annotate_stellar_variability_raw_photometry(lc_fit, target_flux, comp_flux, + target_flux_error=None, comp_flux_error=None): + if lc_fit is None: + return lc_fit + + fit_shape = np.asarray(getattr(lc_fit, 'data', []), dtype=float).shape + target_flux = np.asarray(target_flux if target_flux is not None else [], dtype=float) + comp_flux = np.asarray(comp_flux if comp_flux is not None else [], dtype=float) + if target_flux.shape != fit_shape or comp_flux.shape != fit_shape: + return lc_fit + + def aligned_error(values): + if values is None: + return np.full(fit_shape, np.nan, dtype=float) + array = np.asarray(values, dtype=float) + if array.shape != fit_shape: + return np.full(fit_shape, np.nan, dtype=float) + return array + + lc_fit.stellar_variability_target_flux = target_flux.copy() + lc_fit.stellar_variability_comp_flux = comp_flux.copy() + lc_fit.stellar_variability_target_flux_error = aligned_error(target_flux_error).copy() + lc_fit.stellar_variability_comp_flux_error = aligned_error(comp_flux_error).copy() + return lc_fit + + +def stellar_variability_raw_photometry(lc_fit): + fit_shape = np.asarray(getattr(lc_fit, 'data', []), dtype=float).shape + target_flux = np.asarray( + getattr(lc_fit, 'stellar_variability_target_flux', []), + dtype=float, + ) + comp_flux = np.asarray( + getattr(lc_fit, 'stellar_variability_comp_flux', []), + dtype=float, + ) + if target_flux.shape != fit_shape or comp_flux.shape != fit_shape: + raise RuntimeError( + "Raw target and comparison fluxes are unavailable; an absolute differential magnitude " + "cannot be recovered from a normalized light curve." + ) + + target_flux_error = np.asarray( + getattr(lc_fit, 'stellar_variability_target_flux_error', []), + dtype=float, + ) + comp_flux_error = np.asarray( + getattr(lc_fit, 'stellar_variability_comp_flux_error', []), + dtype=float, + ) + if target_flux_error.shape != fit_shape: + target_flux_error = np.full(fit_shape, np.nan, dtype=float) + if comp_flux_error.shape != fit_shape: + comp_flux_error = np.full(fit_shape, np.nan, dtype=float) + return target_flux, comp_flux, target_flux_error, comp_flux_error + + def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_label, save, s_name, observed_filter=None): comp_mag = _finite_float(comp_star.get('mag')) @@ -19061,33 +19888,77 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ if not (fit_data.shape == fit_airmass_model.shape == fit_airmass.shape == fit_times.shape): raise RuntimeError("Lightcurve arrays have inconsistent shapes for stellar variability output.") + target_flux, comp_flux, target_flux_error, comp_flux_error = stellar_variability_raw_photometry(lc_fit) + if transit_model.shape == fit_data.shape: mask_ref = transit_model == 1 else: mask_ref = np.ones_like(fit_data, dtype=bool) - with np.errstate(divide='ignore', invalid='ignore'): - detrended_all = np.divide(fit_data, fit_airmass_model) if np.count_nonzero(mask_ref) == 0: - mask_ref = np.isfinite(detrended_all) + mask_ref = np.isfinite(fit_data) - detrended = detrended_all[mask_ref] + target_flux = target_flux[mask_ref] + comp_flux = comp_flux[mask_ref] + target_flux_error = target_flux_error[mask_ref] + comp_flux_error = comp_flux_error[mask_ref] + selected_airmass_model = fit_airmass_model[mask_ref] selected_times = fit_times[mask_ref] selected_airmass = fit_airmass[mask_ref] - oot_scatter = np.nanstd(detrended) with np.errstate(divide='ignore', invalid='ignore'): - target_mag = comp_mag - (2.5 * np.log10(detrended)) - target_mag_error = ( - comp_mag_error ** 2 - + (-2.5 * oot_scatter / (detrended * np.log(10))) ** 2 - ) ** 0.5 + raw_ratio = np.divide(target_flux, comp_flux) + valid_airmass_model = ( + np.isfinite(selected_airmass_model) + & (selected_airmass_model > 0) + ) + airmass_reference = ( + float(np.nanmedian(selected_airmass_model[valid_airmass_model])) + if np.any(valid_airmass_model) + else 1.0 + ) + if not np.isfinite(airmass_reference) or airmass_reference <= 0: + airmass_reference = 1.0 + relative_airmass_model = np.divide( + selected_airmass_model, + airmass_reference, + out=np.full(selected_airmass_model.shape, np.nan, dtype=float), + where=valid_airmass_model, + ) + with np.errstate(divide='ignore', invalid='ignore'): + calibrated_ratio = np.divide(raw_ratio, relative_airmass_model) + target_mag = comp_mag - (2.5 * np.log10(calibrated_ratio)) + magnitude_factor = 2.5 / np.log(10.0) + explicit_flux_error = magnitude_factor * np.sqrt( + (target_flux_error / target_flux) ** 2 + + (comp_flux_error / comp_flux) ** 2 + ) + + # The fitted relative-flux uncertainty is still a valid fractional-ratio + # uncertainty after normalization, so use it only when separate stellar + # flux errors were not retained by an older caller. + fit_data_error = np.asarray(getattr(lc_fit, 'dataerr', np.full(fit_data.shape, np.nan)), dtype=float) + if fit_data_error.shape == fit_data.shape: + with np.errstate(divide='ignore', invalid='ignore'): + fallback_flux_error = magnitude_factor * np.abs( + fit_data_error[mask_ref] / fit_data[mask_ref] + ) + else: + fallback_flux_error = np.full(target_mag.shape, np.nan, dtype=float) + flux_error = np.where( + np.isfinite(explicit_flux_error) & (explicit_flux_error >= 0), + explicit_flux_error, + fallback_flux_error, + ) + target_mag_error = np.hypot(comp_mag_error, flux_error) valid = ( np.isfinite(selected_times) & np.isfinite(selected_airmass) & np.isfinite(target_mag) & np.isfinite(target_mag_error) + & np.isfinite(calibrated_ratio) + & (calibrated_ratio > 0) & (target_mag <= MAX_APPARENT_MAGNITUDE) & (target_mag_error <= MAX_APPARENT_MAGNITUDE) ) @@ -19141,18 +20012,18 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ def stellar_variability_reference_series(lc_fit): fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) - fit_airmass_model = np.asarray( - getattr(lc_fit, 'airmass_model', np.ones_like(fit_data)), - dtype=float, - ) fit_times = np.asarray(getattr(lc_fit, 'jd_times', getattr(lc_fit, 'time', [])), dtype=float) transit_model = np.asarray(getattr(lc_fit, 'transit', np.ones_like(fit_data)), dtype=float) - if not (fit_data.shape == fit_airmass_model.shape == fit_times.shape): + if fit_data.shape != fit_times.shape: return np.array([], dtype=float), np.array([], dtype=float) + try: + target_flux, comp_flux, _, _ = stellar_variability_raw_photometry(lc_fit) + except RuntimeError: + return np.array([], dtype=float), np.array([], dtype=float) with np.errstate(divide='ignore', invalid='ignore'): - reference_curve = np.divide(fit_data, fit_airmass_model) + reference_curve = np.divide(target_flux, comp_flux) valid = ( np.isfinite(fit_times) @@ -19287,6 +20158,8 @@ def preferred_catalog_magnitude_band_for_filter(observed_filter): def catalog_band_priority(mag_band, observed_filter): + if observed_filter is None or str(observed_filter).strip() == '': + return 0 preferred_band = preferred_catalog_magnitude_band_for_filter(observed_filter) if preferred_band is None: return 0 @@ -20180,6 +21053,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, arrayFinalFlux = prepared['flux'] f1 = prepared['target_flux'] f2 = prepared['comp_flux'] + f1_error = np.asarray(prepared.get('target_flux_error'), dtype=float) + f2_error = np.asarray(prepared.get('comp_flux_error'), dtype=float) arrayNormUnc = prepared['unc'] arrayTimes = prepared['time'] arrayJDTimes = prepared['jd_time'] @@ -20315,6 +21190,8 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, arrayAirmass = arrayAirmass[~phase_clip_mask] f1 = f1[~phase_clip_mask] f2 = f2[~phase_clip_mask] + f1_error = f1_error[~phase_clip_mask] + f2_error = f2_error[~phase_clip_mask] fit_kwargs = { 'jd_times': arrayJDTimes, @@ -20405,6 +21282,13 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, ) annotate_transit_qc_expected_values(myfit, pDict) annotate_transit_detection_qc(myfit) + annotate_stellar_variability_raw_photometry( + myfit, + f1, + f2, + target_flux_error=f1_error, + comp_flux_error=f2_error, + ) return myfit, f1, f2 @@ -21581,6 +22465,40 @@ def deduplicate_comparison_star_coords(comp_stars, min_separation_pixels=COMPARI return normalized_coords, [] +def merge_automatic_comparison_star_coords(primary_stars, additional_stars, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + """Append automatic candidates without duplicating an already tracked sky source. + + Primary coordinates are preserved exactly because they can be intentional user + selections. The proximity rule only governs automatic additions to that list. + """ + merged, _ = deduplicate_comparison_star_coords(primary_stars) + messages = [] + try: + duplicate_radius = max(float(duplicate_radius_pixels), 0.0) + except (TypeError, ValueError): + duplicate_radius = REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS + + for coord in additional_stars or []: + try: + candidate = [float(coord[0]), float(coord[1])] + except (TypeError, ValueError, IndexError): + continue + duplicate_index = next(( + index for index, existing in enumerate(merged) + if np.hypot(candidate[0] - existing[0], candidate[1] - existing[1]) <= duplicate_radius + ), None) + if duplicate_index is not None: + messages.append( + "Skipped automatic comparison candidate " + f"[{candidate[0]:.2f}, {candidate[1]:.2f}] because it duplicates tracked " + f"comparison star #{duplicate_index + 1} within {duplicate_radius:.1f} pixels." + ) + continue + merged.append(candidate) + return merged, messages + + def format_comp_star_coverage_text(summary): coverage_text = ( f"{summary['coverage_count']} valid frame(s)" @@ -22149,6 +23067,8 @@ def comparison_selection_metric_label(selection_metric): return "Promising Partial" if selection_metric == 'stellar_variability_scatter': return "Out-of-transit scatter" + if selection_metric == 'stellar_variability_ensemble': + return "Calibrated stellar-variability ensemble" if selection_metric == 'comparison_field_rank': return "Comparison-Field Rank" if selection_metric == 'ktmf': @@ -24198,6 +25118,672 @@ def ranked_comparison_calibration_summaries(comparison_calibration): return ranked_summaries +def comparison_positions_match(first, second, tolerance_pixels=1.0e-6): + try: + first_values = np.asarray(first, dtype=float).reshape(-1) + second_values = np.asarray(second, dtype=float).reshape(-1) + except (TypeError, ValueError): + return False + return ( + first_values.size >= 2 + and second_values.size >= 2 + and np.all(np.isfinite(first_values[:2])) + and np.all(np.isfinite(second_values[:2])) + and np.allclose(first_values[:2], second_values[:2], rtol=0.0, atol=float(tolerance_pixels)) + ) + + +def stellar_variability_calibration_for_position(calibration_stars, position, observed_filter=None): + candidates = [] + for label, star in (calibration_stars or {}).items(): + if not isinstance(star, dict) or not comparison_positions_match(star.get('pos'), position): + continue + magnitude = _finite_float(star.get('mag')) + magnitude_error = normalized_magnitude_error(star.get('error')) + if not is_usable_apparent_magnitude(magnitude) or magnitude_error is None: + continue + candidates.append({ + 'label': label, + 'star': star, + 'magnitude': float(magnitude), + 'magnitude_error': float(magnitude_error), + 'band_priority': catalog_band_priority(star.get('mag_band'), observed_filter), + }) + if not candidates: + return None + candidates.sort(key=lambda candidate: ( + candidate['band_priority'], + candidate['magnitude_error'], + )) + return candidates[0] + + +def stellar_variability_catalog_profile(catalog_match, observed_filter=None): + if not isinstance(catalog_match, dict): + return {} + magnitude = _finite_float(catalog_match.get('mag')) + magnitude_error = normalized_magnitude_error(catalog_match.get('error')) + color = nextastro_catalog_color(catalog_match.get('catalog_row'), observed_filter) + return { + 'magnitude': float(magnitude) if magnitude is not None else None, + 'magnitude_error': float(magnitude_error) if magnitude_error is not None else None, + 'magnitude_band': catalog_match.get('mag_band'), + 'color': _finite_float((color or {}).get('color')), + 'color_label': (color or {}).get('label'), + 'catalog_ra': _finite_float(catalog_match.get('catalog_ra')), + 'catalog_dec': _finite_float(catalog_match.get('catalog_dec')), + 'source_id': catalog_match.get('source_id'), + 'catalog_id': catalog_match.get('id'), + } + + +def add_stellar_variability_member_similarity(candidate, target_profile, observed_filter=None): + enriched = dict(candidate) + member_color = nextastro_catalog_color( + enriched.get('star', {}).get('catalog_row'), + observed_filter, + ) + member_color_value = _finite_float((member_color or {}).get('color')) + target_color_value = _finite_float((target_profile or {}).get('color')) + member_color_label = (member_color or {}).get('label') + target_color_label = (target_profile or {}).get('color_label') + if ( + member_color_value is not None + and target_color_value is not None + and normalize_colour_index_label(member_color_label) + == normalize_colour_index_label(target_color_label) + ): + color_delta = abs(member_color_value - target_color_value) + else: + color_delta = None + + target_magnitude = _finite_float((target_profile or {}).get('magnitude')) + member_magnitude = _finite_float(enriched.get('magnitude')) + magnitude_delta = ( + abs(member_magnitude - target_magnitude) + if member_magnitude is not None and target_magnitude is not None + else None + ) + similarity_score = ( + float(np.hypot(color_delta, magnitude_delta)) + if color_delta is not None and magnitude_delta is not None + else None + ) + enriched.update({ + 'color': member_color_value, + 'color_label': member_color_label, + 'target_color': target_color_value, + 'target_color_label': target_color_label, + 'color_delta': color_delta, + 'target_magnitude': target_magnitude, + 'magnitude_delta': magnitude_delta, + 'color_magnitude_similarity_score': similarity_score, + }) + return enriched + + +def stellar_variability_ensemble_calibration_error_clip( + member_candidates, + sigma=STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_SIGMA, + min_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS): + candidates = list(member_candidates or []) + errors = np.asarray( + [candidate.get('magnitude_error', np.nan) for candidate in candidates], + dtype=float, + ) + keep = np.isfinite(errors) & (errors > 0) + threshold = np.nan + center = np.nan + scatter = np.nan + if np.count_nonzero(keep) < 3: + return keep, { + 'center': center, + 'scatter': scatter, + 'high_threshold': threshold, + 'minimum_high_threshold': ( + STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG + ), + } + + for _ in range(10): + kept_errors = errors[keep] + if kept_errors.size < 3: + break + center = float(bn.nanmedian(kept_errors)) + mad = float(bn.nanmedian(np.abs(kept_errors - center))) + robust_error_scatter = 1.4826 * mad if np.isfinite(mad) else np.nan + scatter_floor = max( + STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR, + abs(center) * STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR_FRACTION, + ) + scatter = max( + robust_error_scatter if np.isfinite(robust_error_scatter) else 0.0, + scatter_floor, + ) + threshold = max( + center + (float(sigma) * scatter), + STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG, + ) + updated_keep = keep & (errors <= threshold) + if np.count_nonzero(updated_keep) < int(min_members) or np.array_equal(updated_keep, keep): + break + keep = updated_keep + + return keep, { + 'center': center, + 'scatter': scatter, + 'high_threshold': threshold, + 'minimum_high_threshold': ( + STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG + ), + } + + +def select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter=None, + target_catalog_match=None, + max_members=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS): + target_profile = stellar_variability_catalog_profile(target_catalog_match, observed_filter) + candidates = [] + rejected = [] + represented_catalog_identities = set() + for summary in ranked_summaries or []: + ckey = summary.get('key') + position = summary.get('position') + if not ckey or ckey not in comp_flux_map: + rejected.append({'key': ckey, 'reason': 'no usable photometry series'}) + continue + calibration = stellar_variability_calibration_for_position( + calibration_stars, + position, + observed_filter=observed_filter, + ) + if calibration is None: + rejected.append({'key': ckey, 'reason': 'no usable catalog calibration'}) + continue + catalog_identity = calibration_catalog_identity( + calibration.get('star'), + calibration.get('label'), + ) + if catalog_identity is not None and catalog_identity in represented_catalog_identities: + rejected.append({ + 'key': ckey, + 'reason': 'duplicate catalog source already represented by another ensemble candidate', + }) + continue + + flux_values = np.asarray(comp_flux_map[ckey], dtype=float) + valid_flux = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(valid_flux) < LIGHTCURVE_MIN_VALID_POINTS: + rejected.append({'key': ckey, 'reason': 'too few finite positive flux measurements'}) + continue + median_flux = float(bn.nanmedian(flux_values[valid_flux])) + if not np.isfinite(median_flux) or median_flux <= 0: + rejected.append({'key': ckey, 'reason': 'invalid median brightness'}) + continue + + candidates.append(add_stellar_variability_member_similarity({ + 'key': ckey, + 'comp_index': summary.get('comp_index'), + 'label': summary.get('label', ckey), + 'position': position, + 'summary': summary, + 'median_flux': median_flux, + **calibration, + }, target_profile, observed_filter=observed_filter)) + if catalog_identity is not None: + represented_catalog_identities.add(catalog_identity) + + candidates.sort(key=lambda candidate: (-candidate['median_flux'], candidate.get('comp_index', np.inf))) + keep_mask, clip_summary = stellar_variability_ensemble_calibration_error_clip(candidates) + clipped_members = [] + for candidate, keep in zip(candidates, keep_mask): + if keep: + clipped_members.append(candidate) + else: + rejected.append({ + 'key': candidate.get('key'), + 'reason': ( + 'catalog magnitude uncertainty exceeded the stellar-variability ensemble ' + 'high-side sigma-clip threshold' + ), + 'magnitude_error': candidate.get('magnitude_error'), + }) + + try: + member_limit = max(int(max_members), STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS) + except (TypeError, ValueError): + member_limit = STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS + prelimit_member_count = len(clipped_members) + members = list(clipped_members) + if len(members) > member_limit: + members.sort(key=lambda candidate: ( + 0 if _finite_float(candidate.get('color_magnitude_similarity_score')) is not None else 1, + _finite_float(candidate.get('color_magnitude_similarity_score'), np.inf), + _finite_float(candidate.get('color_delta'), np.inf), + _finite_float(candidate.get('magnitude_delta'), np.inf), + -candidate.get('median_flux', 0.0), + candidate.get('comp_index', np.inf), + )) + excluded_by_limit = members[member_limit:] + members = members[:member_limit] + for candidate in excluded_by_limit: + rejected.append({ + 'key': candidate.get('key'), + 'reason': ( + f'not among the {member_limit} comparison stars closest to the target in ' + 'catalog color and magnitude' + ), + 'color_delta': candidate.get('color_delta'), + 'magnitude_delta': candidate.get('magnitude_delta'), + 'color_magnitude_similarity_score': candidate.get('color_magnitude_similarity_score'), + }) + for selection_rank, member in enumerate(members, start=1): + member['selection_rank'] = selection_rank + + return { + 'members': members, + 'rejected': rejected, + 'calibration_error_clip': clip_summary, + 'target_catalog_profile': target_profile, + 'prelimit_member_count': prelimit_member_count, + 'member_limit': member_limit, + } + + +def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flux_error, + comp_flux_map, comp_error_map, members): + target_flux = np.asarray(target_flux, dtype=float) + if target_flux.ndim != 1: + target_flux = target_flux.reshape(-1) + frame_count = target_flux.size + if target_flux_error is None: + target_flux_error = np.sqrt(np.clip(np.abs(target_flux), 1.0, None)) + target_flux_error = np.asarray(target_flux_error, dtype=float).reshape(-1) + if target_flux_error.shape != target_flux.shape: + target_flux_error = np.sqrt(np.clip(np.abs(target_flux), 1.0, None)) + + zero_points = [] + zero_point_errors = [] + magnitude_factor = 2.5 / np.log(10.0) + for member in members or []: + ckey = member.get('key') + if ckey not in comp_flux_map: + continue + comp_flux = np.asarray(comp_flux_map[ckey], dtype=float).reshape(-1) + if comp_flux.shape != target_flux.shape: + continue + comp_flux_error = None + if isinstance(comp_error_map, dict) and ckey in comp_error_map: + comp_flux_error = np.asarray(comp_error_map[ckey], dtype=float).reshape(-1) + if comp_flux_error is None or comp_flux_error.shape != comp_flux.shape: + comp_flux_error = np.sqrt(np.clip(np.abs(comp_flux), 1.0, None)) + + member_keep_mask = np.asarray( + member.get('summary', {}).get('ensemble_frame_keep_mask', np.ones(frame_count, dtype=bool)), + dtype=bool, + ) + if member_keep_mask.shape != target_flux.shape: + member_keep_mask = np.ones(frame_count, dtype=bool) + valid = ( + member_keep_mask + & np.isfinite(comp_flux) + & (comp_flux > 0) + & np.isfinite(comp_flux_error) + & (comp_flux_error >= 0) + ) + zero_point = np.full(target_flux.shape, np.nan, dtype=float) + zero_point_error = np.full(target_flux.shape, np.nan, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + zero_point[valid] = member['magnitude'] + (2.5 * np.log10(comp_flux[valid])) + instrumental_error = magnitude_factor * comp_flux_error[valid] / comp_flux[valid] + zero_point_error[valid] = np.hypot(member['magnitude_error'], instrumental_error) + zero_points.append(zero_point) + zero_point_errors.append(zero_point_error) + + empty = { + 'applied': False, + 'failure_reason': 'fewer than two calibrated comparison stars were usable for the ensemble.', + 'magnitude': np.full(target_flux.shape, np.nan, dtype=float), + 'magnitude_error': np.full(target_flux.shape, np.nan, dtype=float), + 'relative_flux': np.full(target_flux.shape, np.nan, dtype=float), + 'relative_flux_error': np.full(target_flux.shape, np.nan, dtype=float), + 'synthetic_reference_flux': np.full(target_flux.shape, np.nan, dtype=float), + 'synthetic_reference_flux_error': np.full(target_flux.shape, np.nan, dtype=float), + 'valid_member_count': np.zeros(target_flux.shape, dtype=int), + } + if len(zero_points) < STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + return empty + + zero_point_stack = np.vstack(zero_points) + zero_point_error_stack = np.vstack(zero_point_errors) + valid_member = ( + np.isfinite(zero_point_stack) + & np.isfinite(zero_point_error_stack) + & (zero_point_error_stack > 0) + ) + valid_member_count = np.count_nonzero(valid_member, axis=0) + weights = np.zeros(zero_point_error_stack.shape, dtype=float) + weights[valid_member] = 1.0 / (zero_point_error_stack[valid_member] ** 2) + weight_sum = np.sum(weights, axis=0) + weighted_zero_point_sum = np.nansum(weights * zero_point_stack, axis=0) + valid_target = ( + np.isfinite(target_flux) + & (target_flux > 0) + & np.isfinite(target_flux_error) + & (target_flux_error >= 0) + ) + valid = ( + valid_target + & (valid_member_count >= STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS) + & np.isfinite(weight_sum) + & (weight_sum > 0) + ) + if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: + empty['valid_member_count'] = valid_member_count + empty['failure_reason'] = 'too few frames retained at least two calibrated ensemble members.' + return empty + + magnitude = np.full(target_flux.shape, np.nan, dtype=float) + magnitude_error = np.full(target_flux.shape, np.nan, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + ensemble_zero_point = weighted_zero_point_sum[valid] / weight_sum[valid] + target_instrumental_error = magnitude_factor * target_flux_error[valid] / target_flux[valid] + magnitude[valid] = ensemble_zero_point - (2.5 * np.log10(target_flux[valid])) + magnitude_error[valid] = np.hypot(np.sqrt(1.0 / weight_sum[valid]), target_instrumental_error) + + baseline_magnitude = float(bn.nanmedian(magnitude[valid])) + relative_flux = np.full(target_flux.shape, np.nan, dtype=float) + relative_flux_error = np.full(target_flux.shape, np.nan, dtype=float) + synthetic_reference_flux = np.full(target_flux.shape, np.nan, dtype=float) + synthetic_reference_flux_error = np.full(target_flux.shape, np.nan, dtype=float) + with np.errstate(over='ignore', divide='ignore', invalid='ignore'): + relative_flux[valid] = 10.0 ** (-0.4 * (magnitude[valid] - baseline_magnitude)) + relative_flux_error[valid] = ( + relative_flux[valid] * (np.log(10.0) / 2.5) * magnitude_error[valid] + ) + synthetic_reference_flux[valid] = target_flux[valid] / relative_flux[valid] + target_fractional_error = target_flux_error[valid] / target_flux[valid] + ratio_fractional_error = relative_flux_error[valid] / relative_flux[valid] + reference_fractional_error = np.sqrt( + np.maximum((ratio_fractional_error ** 2) - (target_fractional_error ** 2), 0.0) + ) + synthetic_reference_flux_error[valid] = ( + synthetic_reference_flux[valid] * reference_fractional_error + ) + + return { + 'applied': True, + 'failure_reason': None, + 'magnitude': magnitude, + 'magnitude_error': magnitude_error, + 'relative_flux': relative_flux, + 'relative_flux_error': relative_flux_error, + 'synthetic_reference_flux': synthetic_reference_flux, + 'synthetic_reference_flux_error': synthetic_reference_flux_error, + 'valid_member_count': valid_member_count, + 'baseline_magnitude': baseline_magnitude, + } + + +def stellar_variability_json_safe(value): + if isinstance(value, dict): + return {str(key): stellar_variability_json_safe(subvalue) for key, subvalue in value.items()} + if isinstance(value, (list, tuple)): + return [stellar_variability_json_safe(item) for item in value] + if isinstance(value, np.ndarray): + return stellar_variability_json_safe(value.tolist()) + if isinstance(value, np.generic): + return stellar_variability_json_safe(value.item()) + if isinstance(value, Path): + return str(value) + if isinstance(value, float): + return float(value) if np.isfinite(value) else None + if isinstance(value, (bool, int, str)) or value is None: + return value + return str(value) + + +def stellar_variability_ensemble_member_json(member): + star = member.get('star', {}) if isinstance(member, dict) else {} + return { + 'selection_rank': member.get('selection_rank'), + 'key': member.get('key'), + 'label': member.get('label'), + 'pixel_position': member.get('position'), + 'ra_deg': star.get('ra', star.get('catalog_ra')), + 'dec_deg': star.get('dec', star.get('catalog_dec')), + 'catalog_source': star.get('catalog_source'), + 'catalog_source_id': star.get('source_id'), + 'catalog_id': star.get('id'), + 'catalog_magnitude': member.get('magnitude'), + 'catalog_magnitude_error': member.get('magnitude_error'), + 'catalog_magnitude_band': star.get('mag_band'), + 'catalog_color': member.get('color'), + 'catalog_color_label': member.get('color_label'), + 'target_catalog_color': member.get('target_color'), + 'target_catalog_color_label': member.get('target_color_label'), + 'color_delta': member.get('color_delta'), + 'target_catalog_magnitude': member.get('target_magnitude'), + 'magnitude_delta': member.get('magnitude_delta'), + 'color_magnitude_similarity_score': member.get('color_magnitude_similarity_score'), + 'median_flux_adu': member.get('median_flux'), + 'overexposure_rejected_frame_count': member.get('summary', {}).get( + 'overexposure_rejected_count', 0 + ), + } + + +def save_stellar_variability_ensemble_selection_json( + lc_fit, + save, + target_name, + observation_date=None, + target_metadata=None): + members = list(getattr(lc_fit, 'stellar_variability_ensemble_members', []) or []) + selection = getattr(lc_fit, 'stellar_variability_ensemble_selection', {}) or {} + if not members: + return None + target_profile = ( + selection.get('target_catalog_profile') + or getattr(lc_fit, 'stellar_variability_target_catalog_profile', {}) + or {} + ) + metadata = dict(target_metadata or {}) + metadata.setdefault('name', target_name) + metadata['catalog_profile'] = target_profile + payload = { + 'target': metadata, + 'ensemble': { + 'selection_rule': ( + 'After saturation, VSX, coverage, stability, and high catalog-error rejection, ' + 'use at most five stars ranked by joint catalog color and magnitude distance to the target.' + ), + 'minimum_members': STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + 'maximum_members': selection.get( + 'member_limit', STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS + ), + 'member_count_before_five_star_limit': selection.get( + 'prelimit_member_count', len(members) + ), + 'selected_member_count': len(members), + 'calibration_error_clip': selection.get( + 'calibration_error_clip', + getattr(lc_fit, 'stellar_variability_ensemble_calibration_error_clip', {}), + ), + 'members': [stellar_variability_ensemble_member_json(member) for member in members], + 'rejected_candidates': selection.get('rejected', []), + }, + } + output_dir = Path(save) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / safe_output_filename( + 'EnsembleSelection', + target_name, + filename_date_token(observation_date) if observation_date else 'undated', + extension='json', + ) + with output_path.open('w', encoding='utf-8') as handle: + json.dump(stellar_variability_json_safe(payload), handle, indent=2, sort_keys=True) + handle.write('\n') + return output_path + + +def save_stellar_variability_magnitude_csv(vsp_params, save, target_name, observation_date=None): + if not vsp_params: + return None + output_dir = Path(save) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / safe_output_filename( + 'StellarVariability', + target_name, + filename_date_token(observation_date) if observation_date else 'undated', + extension='csv', + ) + with output_path.open('w', encoding='utf-8', newline='') as handle: + writer = csv.writer(handle) + writer.writerow(['BJD_TDB', 'Airmass', 'Magnitude', 'Magnitude Error', 'Filter', 'Comparison']) + for row in vsp_params: + writer.writerow([ + row.get('time'), + row.get('airmass'), + row.get('mag'), + row.get('mag_err'), + row.get('observed_filter') or row.get('mag_band'), + row.get('cname'), + ]) + return output_path + + +def build_stellar_variability_ensemble_params_from_fit( + lc_fit, + save, + s_name, + observed_filter=None, + observation_date=None, + target_metadata=None): + magnitudes = np.asarray( + getattr(lc_fit, 'stellar_variability_ensemble_magnitudes', []), + dtype=float, + ) + magnitude_errors = np.asarray( + getattr(lc_fit, 'stellar_variability_ensemble_magnitude_errors', []), + dtype=float, + ) + times = np.asarray(getattr(lc_fit, 'jd_times', getattr(lc_fit, 'time', [])), dtype=float) + airmass = np.asarray(getattr(lc_fit, 'airmass', np.ones(times.shape)), dtype=float) + members = list(getattr(lc_fit, 'stellar_variability_ensemble_members', []) or []) + if not (magnitudes.shape == magnitude_errors.shape == times.shape == airmass.shape): + log_info("Warning: calibrated stellar-variability ensemble arrays had inconsistent shapes.", warn=True) + return [] + + valid = ( + np.isfinite(times) + & np.isfinite(airmass) + & np.isfinite(magnitudes) + & np.isfinite(magnitude_errors) + & (magnitude_errors > 0) + & (magnitudes <= MAX_APPARENT_MAGNITUDE) + ) + if not np.any(valid): + log_info("Warning: calibrated stellar-variability ensemble produced no finite magnitude rows.", warn=True) + return [] + + member_labels = [member.get('label') for member in members] + member_positions = [member.get('position') for member in members] + member_catalog_magnitudes = [member.get('magnitude') for member in members] + member_catalog_errors = [member.get('magnitude_error') for member in members] + member_catalog_sources = [member.get('star', {}).get('catalog_source') for member in members] + member_ra_degs = [] + member_dec_degs = [] + member_details = [] + for member in members: + star = member.get('star', {}) if isinstance(member, dict) else {} + ra_deg = _finite_float(star.get('ra', star.get('catalog_ra'))) + dec_deg = _finite_float(star.get('dec', star.get('catalog_dec'))) + member_ra_degs.append(ra_deg) + member_dec_degs.append(dec_deg) + member_details.append({ + 'selection_rank': member.get('selection_rank'), + 'key': member.get('key'), + 'label': member.get('label'), + 'ra_deg': ra_deg, + 'dec_deg': dec_deg, + 'pixel_position': member.get('position'), + 'catalog_magnitude': member.get('magnitude'), + 'catalog_magnitude_error': member.get('magnitude_error'), + 'catalog_magnitude_band': star.get('mag_band'), + 'catalog_source': star.get('catalog_source'), + 'catalog_source_id': star.get('source_id'), + 'catalog_id': star.get('id'), + }) + member_catalog_colors = [member.get('color') for member in members] + member_catalog_color_labels = [member.get('color_label') for member in members] + member_color_deltas = [member.get('color_delta') for member in members] + member_magnitude_deltas = [member.get('magnitude_delta') for member in members] + member_similarity_scores = [member.get('color_magnitude_similarity_score') for member in members] + display_label = f"ENSEMBLE ({len(members)} stars)" + vsp_params = [] + for time_value, airmass_value, magnitude, magnitude_error in zip( + times[valid], + airmass[valid], + magnitudes[valid], + magnitude_errors[valid], + ): + vsp_params.append({ + 'time': time_value, + 'airmass': airmass_value, + 'mag': magnitude, + 'mag_err': magnitude_error, + 'cname': display_label, + 'cmag': None, + 'cmag_err': None, + 'pos': member_positions, + 'comp_ra': None, + 'comp_dec': None, + 'catalog_ra': None, + 'catalog_dec': None, + 'catalog_source': 'Calibrated comparison-star ensemble', + 'is_aavso_vsp': False, + 'mag_band': observed_filter or 'V', + 'observed_filter': observed_filter, + 'ensemble_reference': True, + 'ensemble_member_count': len(members), + 'ensemble_member_labels': member_labels, + 'ensemble_member_positions': member_positions, + 'ensemble_member_catalog_magnitudes': member_catalog_magnitudes, + 'ensemble_member_catalog_errors': member_catalog_errors, + 'ensemble_member_catalog_sources': member_catalog_sources, + 'ensemble_member_ra_degs': member_ra_degs, + 'ensemble_member_dec_degs': member_dec_degs, + 'ensemble_members': member_details, + 'ensemble_member_catalog_colors': member_catalog_colors, + 'ensemble_member_catalog_color_labels': member_catalog_color_labels, + 'ensemble_member_color_deltas': member_color_deltas, + 'ensemble_member_magnitude_deltas': member_magnitude_deltas, + 'ensemble_member_similarity_scores': member_similarity_scores, + }) + + try: + lc_fit.stellar_variability_params = vsp_params + lc_fit.stellar_variability_target_name = s_name + lc_fit.stellar_variability_reference_label = display_label + except Exception: + pass + plot_stellar_variability(vsp_params, save, s_name, display_label) + save_stellar_variability_ensemble_selection_json( + lc_fit, + save, + s_name, + observation_date=observation_date, + target_metadata=target_metadata, + ) + return vsp_params + + def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, comparison_calibration, psf_data, aper_data, target_psf_flux, psf_flux_data=None, @@ -24208,7 +25794,11 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, adaptive_annulus_values=None, fallback_sigma=np.nan, exposure_times_seconds=None, - gain_e_per_adu=None): + gain_e_per_adu=None, + use_ensemble_photometry=True, + calibration_stars=None, + observed_filter=None, + target_catalog_match=None): ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) if not ranked_summaries: return { @@ -24274,32 +25864,342 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, ) attempts = [] - for field_rank, comp_summary in enumerate(ranked_summaries): - comp_index = comp_summary['comp_index'] - ckey = comp_summary.get('key', f"comp{comp_index + 1}") - comp_quality_mask = np.asarray( - comp_summary.get( - 'psf_quality_keep_mask', - psf_quality_mask_for_key( - psf_data, - ckey, - frame_count, - psf_flux_data=psf_flux_data if method == 'psf' else None, - ), - ), - dtype=bool, - ) - if comp_quality_mask.shape[0] != frame_count: - comp_quality_mask = psf_quality_mask_for_key( - psf_data, - ckey, - frame_count, - psf_flux_data=psf_flux_data if method == 'psf' else None, - ) - + if use_ensemble_photometry: if method == 'psf': - target_shape_mask = target_psf_shape_quality_mask( - target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + comp_flux_map = {} + comp_error_map = {} + for summary in ranked_summaries: + ckey = summary.get('key') + if not ckey or ckey not in psf_flux_data: + continue + quality_mask = np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data, + ), + ), + dtype=bool, + ) + comp_flux_map[ckey] = psf_flux_series_from_rows(psf_flux_data[ckey], quality_mask) + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data: + comp_error_map[ckey] = mask_series_with_quality(psf_noise_data[ckey], quality_mask) + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data) + ) + if target_shape_mask.shape != times.shape: + target_shape_mask = np.ones(times.shape[0], dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None + if target_flux_error is None + else mask_series_with_quality(target_flux_error, target_shape_mask) + ) + else: + comp_flux_map = {} + comp_error_map = {} + for summary in ranked_summaries: + ckey = summary.get('key') + if not ckey or ckey not in aper_data: + continue + quality_mask = np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key(psf_data, ckey, frame_count), + ), + dtype=bool, + ) + comp_flux_map[ckey] = mask_series_with_quality( + aper_data[ckey][:, aperture_index, annulus_index], + quality_mask, + ) + error_key = f"{ckey}_unc" + if error_key in aper_data: + comp_error_map[ckey] = mask_series_with_quality( + aper_data[error_key][:, aperture_index, annulus_index], + quality_mask, + ) + target_shape_mask = np.ones(times.shape[0], dtype=bool) + candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error + + member_selection = select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter=observed_filter, + target_catalog_match=target_catalog_match, + ) + ensemble_members = member_selection['members'] + clip_summary = member_selection['calibration_error_clip'] + for rejected_member in member_selection['rejected']: + log_info( + "Stellar-variability ensemble excluded " + f"{rejected_member.get('key') or 'comparison candidate'}: " + f"{rejected_member.get('reason')}." + ) + + if len(ensemble_members) >= STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + member_text = ", ".join( + f"{member['label']} (catalog sigma={member['magnitude_error']:.4f} mag)" + for member in ensemble_members + ) + threshold = clip_summary.get('high_threshold', np.nan) + threshold_text = ( + f"; high-side calibration-error clip threshold={threshold:.4f} mag" + if np.isfinite(threshold) + else "" + ) + log_info( + "Stellar-variability-only calibrated ensemble members: " + f"{member_text}{threshold_text}." + ) + if member_selection.get('prelimit_member_count', 0) > len(ensemble_members): + log_info( + "Stellar-variability ensemble had more than " + f"{member_selection.get('member_limit')} usable stars; retained the five closest " + "to the target in catalog color and magnitude." + ) + ensemble_series = build_stellar_variability_calibrated_ensemble_series( + candidate_target_flux, + candidate_target_flux_error, + comp_flux_map, + comp_error_map, + ensemble_members, + ) + if ensemble_series.get('applied'): + fit_mask = ( + field_image_keep_mask + & target_shape_mask + & np.isfinite(ensemble_series['relative_flux']) + & np.isfinite(ensemble_series['relative_flux_error']) + & (ensemble_series['relative_flux'] > 0) + & (ensemble_series['relative_flux_error'] > 0) + ) + availability_diagnostic = build_time_rejection_diagnostic( + "Stellar-variability calibrated ensemble availability filter", + times, + fit_mask, + note=( + "Kept frames with a finite target measurement and at least two unsaturated, " + "VSX-vetted, catalog-calibrated ensemble members." + ), + ) + filter_diagnostics = [] + if field_image_clip_diagnostic is not None: + filter_diagnostics.append(field_image_clip_diagnostic) + if availability_diagnostic is not None: + filter_diagnostics.append(availability_diagnostic) + + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + candidate_target_flux[fit_mask], + ensemble_series['synthetic_reference_flux'][fit_mask], + airmass[fit_mask], + target_flux_error=( + None + if candidate_target_flux_error is None + else candidate_target_flux_error[fit_mask] + ), + comp_flux_error=ensemble_series['synthetic_reference_flux_error'][fit_mask], + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + exposure_times_for_fit = ( + None + if exposure_times_array is None + else exposure_times_array[fit_mask] + ) + prepared = { + 'applied': True, + 'failure_reason': None, + 'time': times[fit_mask], + 'flux': ensemble_series['relative_flux'][fit_mask], + 'unc': ensemble_series['relative_flux_error'][fit_mask], + 'airmass': airmass[fit_mask], + 'jd_time': jd_times[fit_mask], + 'exposure_time_seconds': exposure_times_for_fit, + 'target_flux': candidate_target_flux[fit_mask], + 'comp_flux': ensemble_series['synthetic_reference_flux'][fit_mask], + 'target_flux_error': ( + np.sqrt(np.clip(np.abs(candidate_target_flux[fit_mask]), 1.0, None)) + if candidate_target_flux_error is None + else candidate_target_flux_error[fit_mask] + ), + 'comp_flux_error': ensemble_series['synthetic_reference_flux_error'][fit_mask], + 'source_indices': np.flatnonzero(fit_mask), + } + fit_result = build_stellar_variability_only_lightcurve( + prepared, + p_dict, + filter_diagnostics=filter_diagnostics, + comp_index=None, + comp_label=f"ENSEMBLE ({len(ensemble_members)} stars)", + comp_position=[member.get('position') for member in ensemble_members], + method_label=method_label, + plot_time_range=plot_time_range, + ) + if fit_result is not None: + selected_source_indices = np.asarray( + fit_result.stellar_variability_source_indices, + dtype=int, + ) + fit_result.stellar_variability_ensemble_members = ensemble_members + fit_result.stellar_variability_ensemble_magnitudes = ( + ensemble_series['magnitude'][selected_source_indices] + ) + fit_result.stellar_variability_ensemble_magnitude_errors = ( + ensemble_series['magnitude_error'][selected_source_indices] + ) + fit_result.stellar_variability_ensemble_valid_member_counts = ( + ensemble_series['valid_member_count'][selected_source_indices] + ) + fit_result.stellar_variability_ensemble_calibration_error_clip = clip_summary + fit_result.stellar_variability_ensemble_selection = member_selection + fit_result.stellar_variability_target_catalog_profile = member_selection.get( + 'target_catalog_profile', {} + ) + scatter = getattr(fit_result, 'stellar_variability_scatter', np.nan) + exclusion_summary = getattr(fit_result, 'stellar_variability_transit_exclusion', {}) + ensemble_summary = { + 'comp_index': None, + 'key': 'ensemble', + 'label': f"Calibrated comparison ensemble ({len(ensemble_members)} comps)", + 'position': None, + 'aggregate_score': comparison_calibration.get('field_score', np.inf), + 'coverage_count': int(np.count_nonzero(fit_mask)), + 'coverage_total_frame_count': int(frame_count), + 'coverage_reference_count': np.nan, + 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, + 'coverage_rejected': False, + 'ensemble_frame_rejected_count': int(np.count_nonzero(~fit_mask)), + 'ensemble_frame_required_valid_pairs': STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + 'ensemble_member_keys': [member['key'] for member in ensemble_members], + } + selected_result = { + 'rank': 0, + 'field_rank': 0, + 'comp_index': None, + 'ckey': 'ensemble', + 'label': ensemble_summary['label'], + 'position': None, + 'aggregate_score': ensemble_summary['aggregate_score'], + 'coverage_count': ensemble_summary['coverage_count'], + 'coverage_total_frame_count': frame_count, + 'coverage_reference_count': np.nan, + 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, + 'coverage_rejected': False, + 'ensemble_frame_rejected_count': ensemble_summary['ensemble_frame_rejected_count'], + 'ensemble_frame_required_valid_pairs': STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + 'ensemble_member_keys': ensemble_summary['ensemble_member_keys'], + 'fit': fit_result, + 'full_reduction_fit': fit_result, + 'good_times': np.asarray(fit_result.time, dtype=float), + 'good_flux': np.asarray(fit_result.detrended, dtype=float), + 'good_unc': np.asarray(fit_result.detrendederr, dtype=float), + 'good_airmass': np.asarray(fit_result.airmass, dtype=float), + 'good_jd_times': np.asarray(fit_result.jd_times, dtype=float), + 'good_exposure_times_seconds': getattr( + fit_result, + 'stellar_variability_exposure_times_seconds', + None, + ), + 'good_target_flux_error': fit_result.stellar_variability_target_flux_error, + 'good_comp_flux_error': fit_result.stellar_variability_comp_flux_error, + 'tflux_fit': fit_result.stellar_variability_target_flux, + 'cflux_fit': fit_result.stellar_variability_comp_flux, + 'tflux_fit_error': fit_result.stellar_variability_target_flux_error, + 'cflux_fit_error': fit_result.stellar_variability_comp_flux_error, + 'source_indices': selected_source_indices, + 'duration_samples': np.asarray( + [exclusion_summary.get('duration_days', np.nan)], + dtype=float, + ), + 'data_highres': np.ones(1000, dtype=float), + 'fit_diagnostics': fit_diagnostics, + 'eebls_snr': np.nan, + 'transit_delta_bic': np.nan, + 'residual_scatter': scatter, + 'target_model_scatter_basis': 'out-of-transit calibrated ensemble scatter', + 'projected_full_residual_scatter': scatter, + 'selection_scatter': scatter, + 'selection_scatter_basis': 'out-of-transit calibrated ensemble scatter', + 'target_comp_scatter': target_comp_flux_scatter( + fit_result.stellar_variability_target_flux, + fit_result.stellar_variability_comp_flux, + ), + 'ktmf_metric': np.nan, + 'ktmf_contributions': [], + 'fit_point_count': int(np.asarray(fit_result.time).size), + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'parameter_summary': None, + 'transit_qc_status': 'SKIPPED', + 'transit_qc_summary': 'Stellar variability only mode skipped transit fitting.', + 'rejected_by_transit_qc': False, + 'selected': True, + 'selection_reason': ( + 'selected: default calibrated ensemble of bright, unsaturated, VSX-vetted ' + 'comparison stars after high-side catalog-error clipping' + ), + 'full_reduction_applied': True, + 'full_reduction_note': ( + 'completed the stellar-variability-only calibrated ensemble reduction ' + 'without fitting a transit model.' + ), + 'stellar_variability_transit_exclusion': exclusion_summary, + 'reuse_selected_full_reduction_fit': True, + } + attempts.append(selected_result) + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': selected_result, + 'selection_metric': 'stellar_variability_ensemble', + 'stopped_after_first_qc_pass': False, + 'stopped_after_promising_partial': False, + } + log_info( + "Warning: the default stellar-variability calibrated ensemble did not yield a usable " + "out-of-transit light curve; falling back to single-comparison selection.", + warn=True, + ) + else: + log_info( + "Warning: fewer than two bright, unsaturated, VSX-vetted comparison stars had usable " + "catalog calibrations after calibration-error clipping; falling back to single-comparison " + "selection.", + warn=True, + ) + + for field_rank, comp_summary in enumerate(ranked_summaries): + comp_index = comp_summary['comp_index'] + ckey = comp_summary.get('key', f"comp{comp_index + 1}") + comp_quality_mask = np.asarray( + comp_summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ), + ), + dtype=bool, + ) + if comp_quality_mask.shape[0] != frame_count: + comp_quality_mask = psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ) + + if method == 'psf': + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), ) if target_shape_mask.shape[0] != frame_count: @@ -24562,6 +26462,483 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, } +def fortuitous_ensemble_flux_maps( + comparison_calibration, + psf_data, + aper_data, + psf_flux_data=None, + psf_noise_data=None): + ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) + method = comparison_calibration.get('method') + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + frame_count = np.asarray(comparison_calibration.get('field_image_keep_mask', [])).size + if frame_count == 0: + if method == 'psf': + frame_count = np.asarray(psf_data.get('target', [])).shape[0] + elif aper_data is not None: + frame_count = np.asarray(aper_data.get('target', [])).shape[0] + flux_map = {} + error_map = {} + psf_flux_source = psf_flux_data_source(psf_data, psf_flux_data) + for summary in ranked_summaries: + ckey = summary.get('key') + if not ckey: + continue + quality_mask = np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_source if method == 'psf' else None, + ), + ), + dtype=bool, + ) + if quality_mask.shape != (frame_count,): + quality_mask = np.ones(frame_count, dtype=bool) + if method == 'psf': + if ckey not in psf_flux_source: + continue + flux_map[ckey] = psf_flux_series_from_rows(psf_flux_source[ckey], quality_mask) + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data: + error_map[ckey] = mask_series_with_quality(psf_noise_data[ckey], quality_mask) + else: + if aper_data is None or ckey not in aper_data: + continue + flux_map[ckey] = mask_series_with_quality( + aper_data[ckey][:, aperture_index, annulus_index], + quality_mask, + ) + error_key = f"{ckey}_unc" + if error_key in aper_data: + error_map[ckey] = mask_series_with_quality( + aper_data[error_key][:, aperture_index, annulus_index], + quality_mask, + ) + return ranked_summaries, flux_map, error_map + + +def fortuitous_variable_target_series( + variable, + comparison_calibration, + psf_data, + aper_data, + psf_flux_data=None, + psf_noise_data=None, + comp_overexposed_masks=None): + ckey = variable.get('tracking_key') + method = comparison_calibration.get('method') + frame_count = np.asarray(comparison_calibration.get('field_image_keep_mask', [])).size + psf_flux_source = psf_flux_data_source(psf_data, psf_flux_data) + quality_mask = psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_source if method == 'psf' else None, + ) + quality_mask = np.asarray(quality_mask, dtype=bool) + if quality_mask.shape != (frame_count,): + quality_mask = np.ones(frame_count, dtype=bool) + if isinstance(comp_overexposed_masks, dict) and ckey in comp_overexposed_masks: + overexposed = np.asarray(comp_overexposed_masks[ckey], dtype=bool) + if overexposed.shape == quality_mask.shape: + quality_mask &= ~overexposed + + if method == 'psf': + if ckey not in psf_flux_source: + return None, None, quality_mask + flux = psf_flux_series_from_rows(psf_flux_source[ckey], quality_mask) + error = None + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data: + error = mask_series_with_quality(psf_noise_data[ckey], quality_mask) + return flux, error, quality_mask + + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + if aper_data is None or ckey not in aper_data: + return None, None, quality_mask + flux = mask_series_with_quality( + aper_data[ckey][:, aperture_index, annulus_index], + quality_mask, + ) + error_key = f"{ckey}_unc" + error = None + if error_key in aper_data: + error = mask_series_with_quality( + aper_data[error_key][:, aperture_index, annulus_index], + quality_mask, + ) + return flux, error, quality_mask + + +def fortuitous_variable_target_metadata(variable): + return { + 'name': variable.get('name'), + 'auid': variable.get('auid'), + 'variable_type': variable.get('variable_type'), + 'ra_deg': variable.get('ra'), + 'dec_deg': variable.get('dec'), + 'pixel_position': variable.get('pos'), + 'vsx_period_days': variable.get('period_days'), + 'vsx_amplitude_mag': variable.get('amplitude_mag'), + 'classification_folder': variable.get('category'), + 'classification_rule': ( + 'optimal_variables requires VSX period <= 10 days and amplitude >= 0.3 mag; ' + 'all other retained VSX stars use rest_of_the_variables.' + ), + 'reference_aperture_flux_adu': variable.get('aperture_flux_adu'), + 'reference_count_rate_adu_per_second': variable.get('count_rate_adu_per_second'), + 'reference_estimated_magnitude_error': variable.get('estimated_magnitude_error'), + 'reference_sky_background_adu_per_pixel': variable.get( + 'reference_sky_background_adu_per_pixel' + ), + 'reference_sky_sigma_adu': variable.get('reference_sky_sigma_adu'), + 'reference_aperture_pixels': variable.get('reference_aperture_pixels'), + 'reference_sky_pixels': variable.get('reference_sky_pixels'), + 'reference_flux_error_adu': variable.get('reference_flux_error_adu'), + 'reference_noise_components_adu': variable.get('reference_noise_components_adu'), + 'detection_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + 'output_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + 'output_magnitude_error_rule': ( + 'Only frames with a finite positive ensemble-calibrated magnitude error below ' + '0.05 mag are written.' + ), + 'saturation_rejection_scope': ( + 'Frames are rejected using this VSX target own overexposure mask; ' + 'the exoplanet target overexposure mask is not applied.' + ), + 'input_frame_count': variable.get('input_frame_count'), + 'target_overexposure_rejected_frame_count': variable.get( + 'target_overexposure_rejected_frame_count' + ), + 'target_quality_rejected_frame_count': variable.get( + 'target_quality_rejected_frame_count' + ), + 'output_magnitude_error_rejected_frame_count': variable.get( + 'output_magnitude_error_rejected_frame_count' + ), + 'output_magnitude_error_qualified_frame_count': variable.get( + 'output_magnitude_error_qualified_frame_count' + ), + 'output_magnitude_error_min': variable.get('output_magnitude_error_min'), + 'output_magnitude_error_median': variable.get('output_magnitude_error_median'), + 'output_magnitude_error_max': variable.get('output_magnitude_error_max'), + 'valid_output_frame_count': variable.get('valid_output_frame_count'), + } + + +def process_fortuitous_variables( + variables, + comparison_calibration, + calibration_stars, + times, + jd_times, + airmass, + psf_data, + aper_data, + info_dict, + psf_flux_data=None, + psf_noise_data=None, + comp_overexposed_masks=None, + exposure_times_seconds=None, + observed_filter=None): + if not variables or comparison_calibration is None: + return [] + times = np.asarray(times, dtype=float) + jd_times = np.asarray(jd_times, dtype=float) + airmass = np.asarray(airmass, dtype=float) + if not (times.shape == jd_times.shape == airmass.shape): + return [] + + ranked_summaries, comp_flux_map, comp_error_map = fortuitous_ensemble_flux_maps( + comparison_calibration, + psf_data, + aper_data, + psf_flux_data=psf_flux_data, + psf_noise_data=psf_noise_data, + ) + if len(ranked_summaries) < STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + log_info( + "Warning: fortuitous-variable photometry skipped because fewer than two independent " + "non-variable comparison candidates were usable.", + warn=True, + ) + return [] + + field_keep_mask = np.asarray( + comparison_calibration.get('field_image_keep_mask', np.ones(times.shape, dtype=bool)), + dtype=bool, + ) + if field_keep_mask.shape != times.shape: + field_keep_mask = np.ones(times.shape, dtype=bool) + exposure_array = None + if exposure_times_seconds is not None: + exposure_array = np.asarray(exposure_times_seconds, dtype=float) + if exposure_array.shape != times.shape: + exposure_array = None + + base_dir = Path(info_dict['save']) / 'fortuitous_variables' + results = [] + for variable in variables: + variable = dict(variable) + variable_name = variable.get('name') or 'VSX variable' + category = variable.get('category') or 'rest_of_the_variables' + variable['input_frame_count'] = int(times.size) + variable_overexposed_mask = np.zeros(times.shape, dtype=bool) + tracking_key = variable.get('tracking_key') + if isinstance(comp_overexposed_masks, dict) and tracking_key in comp_overexposed_masks: + candidate_mask = np.asarray(comp_overexposed_masks[tracking_key], dtype=bool) + if candidate_mask.shape == times.shape: + variable_overexposed_mask = candidate_mask + variable['target_overexposure_rejected_frame_count'] = int( + np.count_nonzero(variable_overexposed_mask) + ) + variable_dir = base_dir / category / safe_output_filename( + 'VSX', variable_name, extension='' + ) + variable_dir.mkdir(parents=True, exist_ok=True) + try: + target_flux, target_flux_error, target_quality_mask = fortuitous_variable_target_series( + variable, + comparison_calibration, + psf_data, + aper_data, + psf_flux_data=psf_flux_data, + psf_noise_data=psf_noise_data, + comp_overexposed_masks=comp_overexposed_masks, + ) + if target_flux is None: + raise ValueError('no usable tracked flux series') + variable['target_quality_rejected_frame_count'] = int( + np.count_nonzero(~np.asarray(target_quality_mask, dtype=bool)) + ) + + member_selection = select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter=observed_filter, + target_catalog_match=variable.get('catalog_match'), + ) + members = member_selection.get('members', []) + if len(members) < STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + raise ValueError('fewer than two independently calibrated ensemble members') + ensemble_series = build_stellar_variability_calibrated_ensemble_series( + target_flux, + target_flux_error, + comp_flux_map, + comp_error_map, + members, + ) + if not ensemble_series.get('applied'): + raise ValueError(ensemble_series.get('failure_reason') or 'ensemble combination failed') + + base_valid = ( + field_keep_mask + & target_quality_mask + & np.isfinite(ensemble_series['relative_flux']) + & (ensemble_series['relative_flux'] > 0) + & np.isfinite(ensemble_series['relative_flux_error']) + & (ensemble_series['relative_flux_error'] > 0) + ) + magnitude_errors = np.asarray(ensemble_series['magnitude_error'], dtype=float) + magnitude_error_valid = ( + np.isfinite(magnitude_errors) + & (magnitude_errors > 0) + & (magnitude_errors < FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR) + ) + valid = base_valid & magnitude_error_valid + eligible_magnitude_errors = magnitude_errors[base_valid & np.isfinite(magnitude_errors)] + variable['output_magnitude_error_rejected_frame_count'] = int( + np.count_nonzero(base_valid & ~magnitude_error_valid) + ) + variable['output_magnitude_error_qualified_frame_count'] = int(np.count_nonzero(valid)) + if eligible_magnitude_errors.size: + variable['output_magnitude_error_min'] = float(np.nanmin(eligible_magnitude_errors)) + variable['output_magnitude_error_median'] = float(np.nanmedian(eligible_magnitude_errors)) + variable['output_magnitude_error_max'] = float(np.nanmax(eligible_magnitude_errors)) + if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: + raise ValueError( + 'fewer than five ensemble-calibrated frames have internal magnitude error ' + f'below {FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR:.3f} mag' + ) + variable['valid_output_frame_count'] = int(np.count_nonzero(valid)) + target_error_values = target_flux_error + if target_error_values is None: + target_error_values = source_flux_uncertainty_from_counts(target_flux) + target_error_values = np.asarray(target_error_values, dtype=float) + prepared = { + 'applied': True, + 'failure_reason': None, + 'time': times[valid], + 'flux': ensemble_series['relative_flux'][valid], + 'unc': ensemble_series['relative_flux_error'][valid], + 'airmass': airmass[valid], + 'jd_time': jd_times[valid], + 'exposure_time_seconds': None if exposure_array is None else exposure_array[valid], + 'target_flux': np.asarray(target_flux, dtype=float)[valid], + 'comp_flux': ensemble_series['synthetic_reference_flux'][valid], + 'target_flux_error': target_error_values[valid], + 'comp_flux_error': ensemble_series['synthetic_reference_flux_error'][valid], + 'source_indices': np.flatnonzero(valid), + } + variable_period = _finite_float(variable.get('period_days'), 1.0) + if variable_period is None or variable_period <= 0: + variable_period = 1.0 + variable_prior = { + 'pPer': variable_period, + 'pPerUnc': np.nan, + 'midT': float(times[valid][0]), + 'midTUnc': np.nan, + 'rprs': np.nan, + 'rprsUnc': np.nan, + 'aRs': np.nan, + 'aRsUnc': np.nan, + 'inc': np.nan, + 'incUnc': np.nan, + 'ecc': 0.0, + 'omega': 0.0, + } + fit = build_stellar_variability_only_lightcurve( + prepared, + variable_prior, + comp_index=None, + comp_label=f"ENSEMBLE ({len(members)} stars)", + comp_position=[member.get('position') for member in members], + method_label=comparison_calibration.get('method_label'), + plot_time_range=times, + ) + if fit is None: + raise ValueError('stellar-variability light curve construction failed') + selected_indices = np.asarray(fit.stellar_variability_source_indices, dtype=int) + fit.stellar_variability_ensemble_members = members + fit.stellar_variability_ensemble_magnitudes = ensemble_series['magnitude'][selected_indices] + fit.stellar_variability_ensemble_magnitude_errors = ( + ensemble_series['magnitude_error'][selected_indices] + ) + fit.stellar_variability_ensemble_valid_member_counts = ( + ensemble_series['valid_member_count'][selected_indices] + ) + fit.stellar_variability_ensemble_calibration_error_clip = member_selection.get( + 'calibration_error_clip', {} + ) + fit.stellar_variability_ensemble_selection = member_selection + fit.stellar_variability_target_catalog_profile = member_selection.get( + 'target_catalog_profile', {} + ) + + target_metadata = fortuitous_variable_target_metadata(variable) + vsp_params = build_stellar_variability_ensemble_params_from_fit( + fit, + variable_dir, + variable_name, + observed_filter=observed_filter, + observation_date=info_dict.get('date'), + target_metadata=target_metadata, + ) + if not vsp_params: + raise ValueError('no calibrated magnitude rows were produced') + csv_path = save_stellar_variability_magnitude_csv( + vsp_params, + variable_dir, + variable_name, + observation_date=info_dict.get('date'), + ) + variable_info = dict(info_dict) + variable_info['save'] = str(variable_dir) + variable_planet = {'sName': variable_name, 'pName': variable_name} + AIDOutputFiles( + fit, + variable_planet, + variable_info, + variable.get('auid'), + None, + vsp_params, + ).aavso() + results.append({ + 'name': variable_name, + 'auid': variable.get('auid'), + 'category': category, + 'output_directory': str(variable_dir), + 'point_count': len(vsp_params), + 'ensemble_member_count': len(members), + 'input_frame_count': variable.get('input_frame_count'), + 'target_overexposure_rejected_frame_count': variable.get( + 'target_overexposure_rejected_frame_count' + ), + 'output_magnitude_error_rejected_frame_count': variable.get( + 'output_magnitude_error_rejected_frame_count' + ), + 'output_magnitude_error_qualified_frame_count': variable.get( + 'output_magnitude_error_qualified_frame_count' + ), + 'output_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + 'output_magnitude_error_max': variable.get('output_magnitude_error_max'), + 'magnitude_csv': str(csv_path) if csv_path else None, + 'status': 'completed', + }) + log_info( + f"Fortuitous-variable photometry completed for {variable_name}: " + f"{len(vsp_params)} point(s), {len(members)} ensemble member(s), outputs={variable_dir}." + ) + except Exception as exc: + results.append({ + 'name': variable_name, + 'auid': variable.get('auid'), + 'category': category, + 'output_directory': str(variable_dir), + 'input_frame_count': variable.get('input_frame_count'), + 'target_overexposure_rejected_frame_count': variable.get( + 'target_overexposure_rejected_frame_count' + ), + 'output_magnitude_error_rejected_frame_count': variable.get( + 'output_magnitude_error_rejected_frame_count' + ), + 'output_magnitude_error_qualified_frame_count': variable.get( + 'output_magnitude_error_qualified_frame_count' + ), + 'output_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + 'status': 'skipped', + 'reason': str(exc), + }) + status_path = variable_dir / safe_output_filename( + 'FortuitousVariableStatus', + variable_name, + filename_date_token(info_dict.get('date')), + extension='json', + ) + with status_path.open('w', encoding='utf-8') as handle: + json.dump( + stellar_variability_json_safe({ + 'target': fortuitous_variable_target_metadata(variable), + 'status': 'skipped', + 'reason': str(exc), + }), + handle, + indent=2, + sort_keys=True, + ) + handle.write('\n') + log_info( + f"Warning: fortuitous-variable photometry skipped {variable_name} ({exc}).", + warn=True, + ) + + base_dir.mkdir(parents=True, exist_ok=True) + manifest_path = base_dir / safe_output_filename( + 'FortuitousVariables', + filename_date_token(info_dict.get('date')), + extension='json', + ) + with manifest_path.open('w', encoding='utf-8') as handle: + json.dump(stellar_variability_json_safe({'variables': results}), handle, indent=2, sort_keys=True) + handle.write('\n') + return results + + def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p_dict, comparison_calibration, psf_data, aper_data, target_psf_flux, psf_flux_data=None, @@ -25767,6 +28144,26 @@ def _main_impl(): stellar_variability_only = should_run_stellar_variability_only( exotic_infoDict.get('stellar_variability_only', STELLAR_VARIABILITY_ONLY_DEFAULT) ) + use_ensemble_photometry_for_stellar_variability = ( + should_use_ensemble_photometry_for_stellar_variability( + exotic_infoDict.get( + 'use_ensemble_photometry_for_stellar_variability', + STELLAR_VARIABILITY_ENSEMBLE_DEFAULT, + ) + ) + ) + photometer_fortuitous_variables = should_photometer_fortuitous_variables( + exotic_infoDict.get( + 'photometer_fortuitous_variables', + PHOTOMETER_FORTUITOUS_VARIABLES_DEFAULT, + ) + ) + use_nextastro_vsx_cache_first = should_use_nextastro_vsx_cache_first( + exotic_infoDict.get( + 'use_nextastro_vsx_cache_first', + USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT, + ) + ) if stellar_variability_only: use_eebls_tmid_initializer = False pick_comparison_by_eebls_snr = False @@ -26257,6 +28654,12 @@ def _main_impl(): plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) ra_dec_tar, ra_dec_wcs = None, [] chart_id, vsp_comp_stars, vsp_list = None, {}, [] + nextastro_field_catalog = None + primary_target_catalog_match = None + science_comp_stars = [] + fortuitous_ensemble_stars = [] + fortuitous_variables = [] + fortuitous_calibration_stars = {} if wcs_file: if should_log_plate_solution_path(wcs_file): @@ -26319,7 +28722,6 @@ def _main_impl(): user_targ_star = [ exotic_UIprevTPX, exotic_UIprevTPY ]) vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] - nextastro_field_catalog = None try: nextastro_field_catalog = nextastro_photometry_catalog_for_wcs( wcs_file, @@ -26333,6 +28735,13 @@ def _main_impl(): f"({describe_retry_exception(exc)}). Will try per-comparison catalog lookups.", warn=True, ) + if nextastro_field_catalog is not None and ra_dec_tar is not None: + primary_target_catalog_match = nextastro_photometry_catalog_match( + nextastro_field_catalog, + ra_dec_tar[0], + ra_dec_tar[1], + exotic_infoDict['filter'], + ) if reference_fallback is not None: fallback_comp_stars, fallback_candidates = select_reference_fallback_comparison_stars( @@ -26354,12 +28763,41 @@ def _main_impl(): ) return - if should_use_automatic_optimal_calibration_selector( + automatic_calibration_selector_enabled = should_use_automatic_optimal_calibration_selector( exotic_infoDict.get('automatic_optimal_calibration_selector', 'n') - ): + ) + stellar_variability_ensemble_candidate_search = ( + stellar_variability_only + and use_ensemble_photometry_for_stellar_variability + ) + if automatic_calibration_selector_enabled or stellar_variability_ensemble_candidate_search: automatic_comp_count = parse_automatic_calibration_selector_count( exotic_infoDict.get('automatic_optimal_calibration_selector_count') ) + ensemble_candidate_saturation_threshold = None + if stellar_variability_ensemble_candidate_search: + configured_candidate_saturation = parse_saturation_value( + exotic_infoDict.get( + 'saturation_value', + exotic_infoDict.get('saturation_value_adu', SATURATION_VALUE_DEFAULT), + ) + ) + header_candidate_saturation = saturation_value_from_header(header) + candidate_saturation = configured_candidate_saturation + if ( + header_candidate_saturation is not None + and configured_candidate_saturation == SATURATION_VALUE_DEFAULT + ): + candidate_saturation = header_candidate_saturation + ensemble_candidate_saturation_threshold = ( + candidate_saturation + * parse_overexposure_threshold_fraction( + exotic_infoDict.get( + 'overexposure_threshold_fraction', + OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT, + ) + ) + ) automatic_comp_stars, automatic_candidates = select_automatic_optimal_calibration_stars( reference_image, reference_image.shape, @@ -26370,11 +28808,14 @@ def _main_impl(): field_catalog=nextastro_field_catalog, count=automatic_comp_count, colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), + brightest_first=stellar_variability_ensemble_candidate_search, + saturation_threshold=ensemble_candidate_saturation_threshold, ) log_automatic_optimal_calibration_selection( automatic_comp_stars, automatic_candidates, automatic_comp_count, + brightest_first=stellar_variability_ensemble_candidate_search, ) if automatic_comp_stars: exotic_infoDict['comp_stars'] = automatic_comp_stars @@ -26396,15 +28837,134 @@ def _main_impl(): for duplicate_message in duplicate_comp_messages: log_info(duplicate_message) - # Build RA/Dec for comp after list is finalized (avoid off by one issues, etc - ra_dec_wcs = build_comp_ra_dec(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) + science_comp_stars = [list(position) for position in exotic_infoDict['comp_stars']] + fortuitous_ensemble_stars = list(science_comp_stars) + configured_fortuitous_saturation = parse_saturation_value( + exotic_infoDict.get( + 'saturation_value', + exotic_infoDict.get('saturation_value_adu', SATURATION_VALUE_DEFAULT), + ) + ) + header_fortuitous_saturation = saturation_value_from_header(header) + fortuitous_saturation = configured_fortuitous_saturation + if ( + header_fortuitous_saturation is not None + and configured_fortuitous_saturation == SATURATION_VALUE_DEFAULT + ): + fortuitous_saturation = header_fortuitous_saturation + fortuitous_saturation_threshold = ( + fortuitous_saturation + * parse_overexposure_threshold_fraction( + exotic_infoDict.get( + 'overexposure_threshold_fraction', + OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT, + ) + ) + ) + + if photometer_fortuitous_variables: + fortuitous_variables = discover_fortuitous_vsx_variables( + wcs_file, + reference_image.shape, + img_scale, + reference_image, + exotic_infoDict['filter'], + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + field_catalog=nextastro_field_catalog, + exposure_seconds=(header_exptimes[0] if len(header_exptimes) else 1.0), + gain_e_per_adu=exotic_infoDict.get('gain_electrons_per_adu'), + saturation_threshold=fortuitous_saturation_threshold, + use_nextastro_vsx_cache_first=use_nextastro_vsx_cache_first, + ) + else: + log_info("Fortuitous-variable photometry disabled per optional_info setting.") + + if fortuitous_variables: + if stellar_variability_ensemble_candidate_search: + # The stellar-variability target path just selected and VSX-vetted the + # same brightest-first pool with the same count and saturation limit. + # Reuse it rather than performing an identical full-field image scan. + fortuitous_auto_stars = [] + log_info( + "Reusing the stellar-variability target comparison pool for fortuitous " + "VSX targets; skipping a duplicate automatic source scan." + ) + else: + fortuitous_comp_count = parse_automatic_calibration_selector_count( + exotic_infoDict.get('automatic_optimal_calibration_selector_count') + ) + fortuitous_auto_stars, _ = select_automatic_optimal_calibration_stars( + reference_image, + reference_image.shape, + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter=exotic_infoDict['filter'], + field_catalog=nextastro_field_catalog, + count=fortuitous_comp_count, + colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), + brightest_first=True, + saturation_threshold=fortuitous_saturation_threshold, + ) + check_for_variable_stars( + ra_wcs, + dec_wcs, + fortuitous_auto_stars, + use_nextastro_variability_server=args.use_nextastro_variability_server, + ) + variable_positions = [variable['pos'] for variable in fortuitous_variables] + fortuitous_auto_stars = [ + position for position in fortuitous_auto_stars + if not any( + np.hypot( + float(position[0]) - float(variable_position[0]), + float(position[1]) - float(variable_position[1]), + ) <= REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS + for variable_position in variable_positions + ) + ] + fortuitous_ensemble_stars, fortuitous_duplicate_messages = ( + merge_automatic_comparison_star_coords( + science_comp_stars, + fortuitous_auto_stars, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for duplicate_message in fortuitous_duplicate_messages: + log_info(duplicate_message) + log_info( + "Fortuitous-variable ensemble pool contains " + f"{len(fortuitous_ensemble_stars)} non-variable, non-saturated, " + "catalog-matched comparison candidate(s)." + ) + + ensemble_ra_dec = build_comp_ra_dec( + ra_wcs, + dec_wcs, + fortuitous_ensemble_stars, + ) vsp_comp_stars = merge_nextastro_calibration_stars( - exotic_infoDict['comp_stars'], - ra_dec_wcs, + science_comp_stars, + ensemble_ra_dec[:len(science_comp_stars)], exotic_infoDict['filter'], existing_comp_stars=vsp_comp_stars, field_catalog=nextastro_field_catalog, ) + fortuitous_calibration_stars = merge_nextastro_calibration_stars( + fortuitous_ensemble_stars, + ensemble_ra_dec, + exotic_infoDict['filter'], + existing_comp_stars=vsp_comp_stars, + field_catalog=nextastro_field_catalog, + ) + tracked_positions = [*fortuitous_ensemble_stars] + for variable in fortuitous_variables: + variable['tracking_key'] = f"comp{len(tracked_positions) + 1}" + tracked_positions.append(list(variable['pos'])) + exotic_infoDict['comp_stars'] = tracked_positions + # Build RA/Dec after the tracking list is finalized. Science comps remain first, + # followed by fortuitous-only ensemble candidates and then the VSX targets. + ra_dec_wcs = build_comp_ra_dec(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) else: @@ -26416,11 +28976,19 @@ def _main_impl(): error=True, ) return + if photometer_fortuitous_variables: + log_info( + "Warning: fortuitous-variable photometry requires a usable celestial WCS; " + "the full-field VSX search will be skipped for this reduction.", + warn=True, + ) exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( exotic_infoDict['comp_stars'] ) for duplicate_message in duplicate_comp_messages: log_info(duplicate_message) + science_comp_stars = [list(position) for position in exotic_infoDict['comp_stars']] + fortuitous_ensemble_stars = list(science_comp_stars) plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) # alloc psf fitting param @@ -26483,6 +29051,19 @@ def _main_impl(): reject_overexposed_stars = should_reject_overexposed_stars( exotic_infoDict.get('reject_overexposed_stars', REJECT_OVEREXPOSED_STARS_DEFAULT) ) + if ( + ( + (stellar_variability_only and use_ensemble_photometry_for_stellar_variability) + or (photometer_fortuitous_variables and bool(fortuitous_variables)) + ) + and not reject_overexposed_stars + ): + reject_overexposed_stars = True + log_info( + "Stellar-variability ensemble membership requires non-saturated stars; " + "overexposure rejection will remain enabled for this run.", + warn=True, + ) configured_saturation_value = parse_saturation_value( exotic_infoDict.get( 'saturation_value', @@ -26524,6 +29105,18 @@ def _main_impl(): "Ensemble comparison photometry enabled per optional_info setting; the final target " "light curve will use non-rejected comparison stars as a combined reference." ) + if stellar_variability_only: + if use_ensemble_photometry_for_stellar_variability: + log_info( + "Stellar-variability calibrated ensemble enabled (default): EXOTIC will combine " + "bright, unsaturated, VSX-vetted comparison stars after clipping high catalog " + "magnitude uncertainties." + ) + else: + log_info( + "Stellar-variability calibrated ensemble disabled per optional_info setting; " + "EXOTIC will select one comparison star by out-of-transit scatter." + ) if reject_overexposed_stars: log_info( "Overexposed-star rejection enabled: target frames and comparison-star measurements " @@ -26797,7 +29390,7 @@ def _main_impl(): target_radius, overexposure_threshold, fast_mode=fast_aperture_mask, - ): + ): target_overexposed_frame_mask[i] = True plateStatus.overexposedWarning( 0, @@ -26805,12 +29398,7 @@ def _main_impl(): target_row[1] if target_row.size > 1 else np.nan, overexposure_threshold, ) - psf_data['target'][i, :] = np.nan psf_flux_data['target'][i, :] = np.nan - hdul.close() - del hdul - del imageData - continue for comp_idx, comp_key in enumerate(comp_alignment_keys): comp_row = np.asarray(psf_data[comp_key][i], dtype=float) @@ -26840,29 +29428,32 @@ def _main_impl(): if use_legacy_psf_flux_mode else fit_psf_photometry_flux_row ) - target_psf_flux_seed_row = psf_data['target'][i] - if 'target' in psf_flux_seed_tracks: - target_psf_flux_seed_row = psf_flux_seed_tracks['target'][i] - psf_flux_data['target'][i] = psf_flux_row_fitter( - imageData, - target_psf_flux_seed_row, - 0, - ) - store_psf_noise_budget( - psf_noise_data, - 'target', - i, - compute_psf_noise_budget_for_row( + if target_overexposed_frame_mask[i]: + psf_flux_data['target'][i, :] = np.nan + else: + target_psf_flux_seed_row = psf_data['target'][i] + if 'target' in psf_flux_seed_tracks: + target_psf_flux_seed_row = psf_flux_seed_tracks['target'][i] + psf_flux_data['target'][i] = psf_flux_row_fitter( imageData, - psf_flux_data['target'][i], + target_psf_flux_seed_row, 0, - noise_config=frame_noise_config, - exposure_s=frame_exposure_s, - airmass=frame_airmass, - fallback_sigma=sigma, - fast_mode=fast_aperture_mask, - ), - ) + ) + store_psf_noise_budget( + psf_noise_data, + 'target', + i, + compute_psf_noise_budget_for_row( + imageData, + psf_flux_data['target'][i], + 0, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + fallback_sigma=sigma, + fast_mode=fast_aperture_mask, + ), + ) for comp_idx, comp_key in enumerate(comp_alignment_keys): if comp_overexposed_masks.get(comp_key, np.zeros(len(inputfiles), dtype=bool))[i]: psf_flux_data[comp_key][i, :] = np.nan @@ -27081,6 +29672,80 @@ def _main_impl(): log_photometry_timing_stats('Photometry timing summary (full reduction)') log_reduction_timing_overview('Reduction timing overview (full reduction)') + # Fortuitous VSX targets are independent science targets. Process them against + # the full image sequence before the exoplanet target validity/overexposure mask + # is applied below. Each VSX target series applies its own compN overexposure mask. + if photometer_fortuitous_variables and fortuitous_variables: + full_airmass = np.asarray(airMassList, dtype=float) + full_exposure_times_seconds = np.asarray(exptimes, dtype=float) + variable_sigma_rows = [ + np.asarray(psf_data.get(variable.get('tracking_key'), []), dtype=float) + for variable in fortuitous_variables + if variable.get('tracking_key') in psf_data + ] + variable_sigma_rows = [rows for rows in variable_sigma_rows if rows.ndim == 2 and rows.size] + if variable_sigma_rows: + fortuitous_sigma_display = representative_psf_sigma( + np.concatenate(variable_sigma_rows, axis=0), + fallback_sigma=sigma, + ) + else: + fortuitous_sigma_display = sigma + if not np.isfinite(fortuitous_sigma_display) or fortuitous_sigma_display <= 0: + fortuitous_sigma_display = 1.0 + + fortuitous_apers = apers + fortuitous_annuli = annuli + if aperture_values is not None and annulus_values is not None: + if use_adaptive_apertures: + fortuitous_apers = ( + np.asarray(aperture_values, dtype=float) * fortuitous_sigma_display + ) + fortuitous_annuli = ( + np.asarray(annulus_values, dtype=float) * fortuitous_sigma_display + ) + else: + fortuitous_apers = np.asarray(aperture_values, dtype=float) + fortuitous_annuli = np.asarray(annulus_values, dtype=float) + + fortuitous_psf_flux_source = ( + psf_flux_data if use_psf_photometry else psf_data + ) + fortuitous_comparison_calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + fortuitous_apers, + fortuitous_annuli, + full_airmass, + fortuitous_ensemble_stars, + fortuitous_sigma_display, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=use_psf_photometry, + use_aperture_photometry=use_aperture_photometry, + psf_flux_data=fortuitous_psf_flux_source, + comp_overexposed_masks=comp_overexposed_masks, + ) + exotic_infoDict['exposure'] = exp_time_med(exptimes) + process_fortuitous_variables( + fortuitous_variables, + fortuitous_comparison_calibration, + fortuitous_calibration_stars, + times, + jd_times, + full_airmass, + psf_data, + aper_data, + exotic_infoDict, + psf_flux_data=fortuitous_psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + comp_overexposed_masks=comp_overexposed_masks, + exposure_times_seconds=full_exposure_times_seconds, + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + ) + # filter bad images badmask = np.isnan(psf_data["target"][:, 0]) | (psf_data["target"][:, 0] == 0) if aper_data is not None: @@ -27301,7 +29966,7 @@ def _main_impl(): apers, annuli, airmass, - exotic_infoDict['comp_stars'], + science_comp_stars, sigma_display, skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, use_psf_photometry=use_psf_photometry, @@ -27310,6 +29975,10 @@ def _main_impl(): comp_overexposed_masks=comp_overexposed_masks, ) + # Fortuitous-only sources were appended solely so the shared image pass could measure them. + # Restore the science comparison list before normal target fitting and final metadata output. + exotic_infoDict['comp_stars'] = [list(position) for position in science_comp_stars] + if comparison_calibration is not None: log_info("\nCalibrating comparison stars before target fitting. Please wait.") log_info(f"Comparison-star field method: {comparison_calibration['method_label']}") @@ -27438,6 +30107,13 @@ def _main_impl(): fallback_sigma=sigma_display, exposure_times_seconds=exposure_times_seconds, gain_e_per_adu=fallback_gain_e_per_adu, + use_ensemble_photometry=use_ensemble_photometry_for_stellar_variability, + calibration_stars=vsp_comp_stars, + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + target_catalog_match=primary_target_catalog_match, ) else: comparison_fit_search = fit_ranked_comparison_calibration_candidates( @@ -27502,7 +30178,7 @@ def _main_impl(): selected_comp_coords = ( None if selected_is_ensemble - else exotic_infoDict['comp_stars'][selected_comp_index] + else science_comp_stars[selected_comp_index] ) selected_min_aperture = 0 if comparison_calibration['method'] == 'psf' else comparison_calibration['aper'] selected_min_annulus = comparison_calibration['annulus'] @@ -27522,7 +30198,9 @@ def _main_impl(): dtype=int, ) selected_attempt_label = selected_attempt.get('label', 'comparison candidate') - if stellar_variability_only: + if stellar_variability_only and selected_is_ensemble: + selection_basis = 'stellar_variability_ensemble' + elif stellar_variability_only: selection_basis = 'stellar_variability_scatter' elif selected_attempt.get('search_stopped_after_qc_pass', False): selection_basis = 'first_qc_pass' @@ -27536,7 +30214,15 @@ def _main_impl(): selection_basis = 'comparison_field' else: selection_basis = 'comparison_field_retry' - if selection_basis == 'stellar_variability_scatter': + if selection_basis == 'stellar_variability_ensemble': + ensemble_members = selected_attempt.get('ensemble_member_keys') or [] + log_info( + "Stellar-variability-only comparison selection chose the default calibrated " + f"ensemble with {len(ensemble_members)} member(s) using " + f"{comparison_calibration['method_label']}; members are bright, unsaturated, " + "VSX-vetted, and passed the high-side catalog-error clip." + ) + elif selection_basis == 'stellar_variability_scatter': log_info( "Stellar-variability-only comparison selection chose " f"{selected_attempt_label} with {comparison_calibration['method_label']} " @@ -27664,10 +30350,10 @@ def _main_impl(): if selected_comp_index is not None: ref_flux[selected_comp_index] = { 'myfit': myfit, - 'pos': exotic_infoDict['comp_stars'][selected_comp_index] + 'pos': science_comp_stars[selected_comp_index] } - if vsp_num: + if vsp_num and not (stellar_variability_only and selected_is_ensemble): if comparison_calibration['method'] == 'psf': for j in vsp_num: ckey = f"comp{j + 1}" @@ -27913,14 +30599,14 @@ def _main_impl(): "stellar-variability-only mode does not fit transit models." ) - if fit_every_comparison_candidate and not stellar_variability_only and exotic_infoDict['comp_stars']: + if fit_every_comparison_candidate and not stellar_variability_only and science_comp_stars: candidate_fit_summaries = fit_lightcurve_to_every_comparison_candidate( times, jd_times, airmass, ld, pDict, - exotic_infoDict['comp_stars'], + science_comp_stars, psf_data, aper_data, photometry_info, @@ -28217,14 +30903,31 @@ def _main_impl(): # Calculate the standard deviation of the normalized flux values # standardDev1 = np.std(goodFluxes) - if vsp_comp_stars: + if stellar_variability_only and bestCompStar == 'ensemble': + vsp_params = build_stellar_variability_ensemble_params_from_fit( + best_fit_lc, + exotic_infoDict['save'], + pDict['sName'], + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + observation_date=exotic_infoDict.get('date'), + target_metadata={ + 'name': pDict.get('sName'), + 'ra_deg': None if ra_dec_tar is None else ra_dec_tar[0], + 'dec_deg': None if ra_dec_tar is None else ra_dec_tar[1], + 'pixel_position': [exotic_UIprevTPX, exotic_UIprevTPY], + }, + ) + elif vsp_comp_stars: if isinstance(bestCompStar, int): - vsp_params = stellar_variability(ref_flux, best_fit_lc, exotic_infoDict['comp_stars'], + vsp_params = stellar_variability(ref_flux, best_fit_lc, science_comp_stars, vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], pDict['sName'], observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), - comp_ra_dec=ra_dec_wcs, + comp_ra_dec=ra_dec_wcs[:len(science_comp_stars)], field_catalog=nextastro_field_catalog, reference_image=reference_image, wcs_file=wcs_file) @@ -28397,9 +31100,9 @@ def _main_impl(): psf_data, aper_data, photometry_info, - len(exotic_infoDict['comp_stars']), + len(science_comp_stars), ) - plot_obs_stats(myfit, exotic_infoDict['comp_stars'], psf_data, obs_stats_sort_index, + plot_obs_stats(myfit, science_comp_stars, psf_data, obs_stats_sort_index, obs_stats_keep_mask, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], relative_flux_mask=None, @@ -28700,9 +31403,9 @@ def _main_impl(): psf_data, aper_data, photometry_info, - len(exotic_infoDict['comp_stars']), + len(science_comp_stars), ) - plot_obs_stats(myfit, exotic_infoDict['comp_stars'], psf_data, obs_stats_sort_index, + plot_obs_stats(myfit, science_comp_stars, psf_data, obs_stats_sort_index, obs_stats_keep_mask, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], relative_flux_mask=None, diff --git a/exotic/inputs.py b/exotic/inputs.py index b2436cba..2f3edf13 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -210,6 +210,9 @@ def __init__(self, init_opt): 'prefer_pixel_values_over_wcs_for_target': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'stellar_variability_only': False, + 'use_ensemble_photometry_for_stellar_variability': True, + 'photometer_fortuitous_variables': True, + 'use_nextastro_vsx_cache_first': False, 'detrend_on_outoftransit_baseline': True, 'final_fit_baseline_duration_multiplier': 1.0, 'use_eebls_to_initialize_tmid_and_bounds': 'y', @@ -482,6 +485,19 @@ def comp_params(self, init_file, planet_dict): 'Stellar Variability Only', 'Stellar Variability Only? (y/n)', ), + 'use_ensemble_photometry_for_stellar_variability': ( + 'use_ensemble_photometry_for_stellar_variability', + 'stellar_variability_use_ensemble', + 'Use Ensemble Photometry for Stellar Variability? (y/n)', + ), + 'photometer_fortuitous_variables': ( + 'photometer_fortuitous_variables', + 'Photometer Fortuitous Variables? (y/n)', + ), + 'use_nextastro_vsx_cache_first': ( + 'use_nextastro_vsx_cache_first', + 'Use NextAstro VSX Cache First? (y/n)', + ), 'detect_bad_pixels_before_photometry': ( 'detect_bad_pixels_before_photometry', 'Detect Bad Pixels Before Photometry? (y/n)', diff --git a/exotic/output_files.py b/exotic/output_files.py index 3a50ffe2..24281727 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -158,6 +158,14 @@ def stellar_variability_reference_summary(vsp_param): cname = vsp_param.get('cname', 'na') band = vsp_param.get('mag_band') or 'V' + if vsp_param.get('ensemble_reference'): + member_count = int(vsp_param.get('ensemble_member_count', 0) or 0) + labels = vsp_param.get('ensemble_member_labels') or [] + label_text = ", ".join(str(label) for label in labels) + return ( + f"Calibrated comparison-star ensemble ({member_count} stars)" + + (f": {label_text}" if label_text else "") + ) if vsp_param.get('is_aavso_vsp', True): return f"AAVSO Label: {cname}, Position: {vsp_param.get('pos')}" @@ -179,6 +187,12 @@ def stellar_variability_measurement_summary(vsp_params, transit_fit_comp_star=No if transit_fit_comp_star is None: return None + if vsp_params[0].get('ensemble_reference'): + return ( + f"Combined {point_count} out-of-transit target measurements against the calibrated " + "comparison-star ensemble; AID rows list the BJD_TDB timestamps used." + ) + return ( f"Remeasured {point_count} out-of-transit target/reference point(s) against the transit-fit " f"{'derived ' if vsp_params[0].get('derived_catalog_reference') else ''}catalog reference " @@ -215,6 +229,21 @@ def aid_comparison_metadata(vsp_param): 'magnitude_band': vsp_param.get('mag_band'), 'apparent_magnitude': rounded_magnitude_value(vsp_param.get('cmag')), 'apparent_magnitude_error': rounded_magnitude_error(vsp_param.get('cmag_err')), + 'ensemble_reference': bool(vsp_param.get('ensemble_reference', False)), + 'ensemble_member_count': vsp_param.get('ensemble_member_count'), + 'ensemble_member_labels': vsp_param.get('ensemble_member_labels'), + 'ensemble_member_positions': vsp_param.get('ensemble_member_positions'), + 'ensemble_member_catalog_magnitudes': vsp_param.get('ensemble_member_catalog_magnitudes'), + 'ensemble_member_catalog_errors': vsp_param.get('ensemble_member_catalog_errors'), + 'ensemble_member_catalog_sources': vsp_param.get('ensemble_member_catalog_sources'), + 'ensemble_member_ra_degs': vsp_param.get('ensemble_member_ra_degs'), + 'ensemble_member_dec_degs': vsp_param.get('ensemble_member_dec_degs'), + 'ensemble_members': vsp_param.get('ensemble_members'), + 'ensemble_member_catalog_colors': vsp_param.get('ensemble_member_catalog_colors'), + 'ensemble_member_catalog_color_labels': vsp_param.get('ensemble_member_catalog_color_labels'), + 'ensemble_member_color_deltas': vsp_param.get('ensemble_member_color_deltas'), + 'ensemble_member_magnitude_deltas': vsp_param.get('ensemble_member_magnitude_deltas'), + 'ensemble_member_similarity_scores': vsp_param.get('ensemble_member_similarity_scores'), } if anchor_labels is not None: metadata['derived_reference_anchor_labels'] = anchor_labels @@ -223,6 +252,52 @@ def aid_comparison_metadata(vsp_param): return aavso_json_safe(metadata) +def aid_ensemble_comparison_metadata(vsp_param): + if not vsp_param or not vsp_param.get('ensemble_reference'): + return {} + + members = vsp_param.get('ensemble_members') + if isinstance(members, np.ndarray): + members = members.tolist() + if not isinstance(members, (list, tuple)) or not members: + labels = list(vsp_param.get('ensemble_member_labels') or []) + positions = list(vsp_param.get('ensemble_member_positions') or []) + ra_degs = list(vsp_param.get('ensemble_member_ra_degs') or []) + dec_degs = list(vsp_param.get('ensemble_member_dec_degs') or []) + magnitudes = list(vsp_param.get('ensemble_member_catalog_magnitudes') or []) + magnitude_errors = list(vsp_param.get('ensemble_member_catalog_errors') or []) + catalog_sources = list(vsp_param.get('ensemble_member_catalog_sources') or []) + member_count = max( + int(vsp_param.get('ensemble_member_count', 0) or 0), + len(labels), + len(ra_degs), + len(dec_degs), + ) + + def value_at(values, index): + return values[index] if index < len(values) else None + + members = [ + { + 'label': value_at(labels, index), + 'ra_deg': value_at(ra_degs, index), + 'dec_deg': value_at(dec_degs, index), + 'pixel_position': value_at(positions, index), + 'catalog_magnitude': value_at(magnitudes, index), + 'catalog_magnitude_error': value_at(magnitude_errors, index), + 'catalog_source': value_at(catalog_sources, index), + } + for index in range(member_count) + ] + else: + members = list(members) + + return prune_aavso_metadata({ + 'member_count': int(vsp_param.get('ensemble_member_count', len(members)) or len(members)), + 'members': members, + }) + + def prune_aavso_metadata(value): if isinstance(value, dict): pruned = {} @@ -1774,11 +1849,17 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor if vsp_params: params_num["Variable Reference Star"] = stellar_variability_reference_summary(vsp_params[0]) - params_num["Variable Reference Measurement"] = ( - f"Remeasured {len(vsp_params)} out-of-transit target/reference point(s) " - "against the stellar-variability reference catalog star; AID rows list the " - "BJD_TDB timestamps used." - ) + if vsp_params[0].get('ensemble_reference'): + params_num["Variable Reference Measurement"] = ( + f"Combined {len(vsp_params)} out-of-transit target measurements against the " + "calibrated comparison-star ensemble; AID rows list the BJD_TDB timestamps used." + ) + else: + params_num["Variable Reference Measurement"] = ( + f"Remeasured {len(vsp_params)} out-of-transit target/reference point(s) " + "against the stellar-variability reference catalog star; AID rows list the " + "BJD_TDB timestamps used." + ) if phot_opt: if comp_star == 'ensemble': @@ -2284,6 +2365,7 @@ def __init__(self, fit, p_dict, i_dict, auid, chart_id, vsp_params): def aavso(self): first_vsp_param = self.vsp_params[0] if self.vsp_params else {} comparison_metadata = aid_comparison_metadata(first_vsp_param) + ensemble_comparison_metadata = aid_ensemble_comparison_metadata(first_vsp_param) variable_name = self.auid or self.p_dict.get('sName') or self.p_dict.get('pName') params_file = self.dir / safe_output_filename( @@ -2312,6 +2394,11 @@ def aavso(self): "aavso.org/data-usage-guidelines\n") if comparison_metadata: f.write(f"#COMPARISON-CATALOG-XC={dumps(comparison_metadata, sort_keys=True)}\n") + if ensemble_comparison_metadata: + f.write(format_aavso_json_header( + "ENSEMBLE-COMPARISONS-XC", + ensemble_comparison_metadata, + )) f.write("#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,GROUP,CHART,NOTES\n") for vsp_p in self.vsp_params: diff --git a/inits.json b/inits.json index 42a9223e..d40464f6 100644 --- a/inits.json +++ b/inits.json @@ -28,7 +28,10 @@ "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Leave blank/null or set to 0/off to disable. Default disabled.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", - "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", + "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, use the default calibrated comparison-star ensemble, and discard predicted ingress-to-egress transit-window points. Default false.", + "Stellar Variability Ensemble": "Set optional_info 'use_ensemble_photometry_for_stellar_variability' to false to disable the default calibrated ensemble in stellar_variability_only runs and restore single-comparison selection by out-of-transit scatter. The default ensemble automatically finds bright catalog-calibrated field-star candidates, removes saturated and VSX-variable stars, sigma-clips high catalog magnitude uncertainties, and when more than five remain uses the five closest to the target in catalog colour and magnitude. EnsembleSelection JSON records the target and comparison colours, magnitudes, errors, and selection deltas beside the final results.", + "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent ensemble photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under fortuitous_variables/optimal_variables for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under fortuitous_variables/rest_of_the_variables.", + "NextAstro VSX Cache First": "Set optional_info 'use_nextastro_vsx_cache_first' to true to query the NextAstro /vsx_query field cache before AAVSO VSX. The default is false. Empty or failed cache lookups fall back to AAVSO; legacy cache responses lacking the full period/amplitude schema are enriched from AAVSO.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", @@ -123,6 +126,9 @@ "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": false, "stellar_variability_only": false, + "use_ensemble_photometry_for_stellar_variability": true, + "photometer_fortuitous_variables": true, + "use_nextastro_vsx_cache_first": false, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "y", "detrend_on_outoftransit_baseline": true, diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index f4ac215b..070ab59f 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -1141,8 +1141,8 @@ def add_blob(x_pos, y_pos, value): image[y_pos, x_pos] = value add_blob(150, 150, 2000.0) - add_blob(220, 220, 1800.0) - add_blob(80, 80, 1700.0) + add_blob(220, 220, 1700.0) + add_blob(80, 80, 1800.0) add_blob(230, 80, 6000.0) ra_wcs = np.tile(np.arange(300, dtype=float), (300, 1)) @@ -1163,6 +1163,20 @@ def fake_color_match(_catalog, ra, dec, obs_filter, max_separation_arcsec=5.0): return {"catalog_row": {"Bmag": b_mag, "Vmag": v_mag}} monkeypatch.setattr(exotic_module, "nextastro_catalog_nearest_color_row", fake_color_match) + monkeypatch.setattr( + exotic_module, + "nextastro_photometry_catalog_match", + lambda catalog_response, ra, dec, obs_filter: ( + { + **fake_color_match(catalog_response, ra, dec, obs_filter), + "mag": 12.0, + "error": 0.01, + "mag_band": "V", + } + if fake_color_match(catalog_response, ra, dec, obs_filter) is not None + else None + ), + ) comp_stars, candidates = exotic_module.select_automatic_optimal_calibration_stars( image, @@ -1190,6 +1204,25 @@ def fake_color_match(_catalog, ra, dec, obs_filter, max_separation_arcsec=5.0): assert candidates[0]["colour_term_uncertainty_mag"] == pytest.approx( 0.01 * candidates[0]["color_delta"] ) + + brightest_comp_stars, brightest_candidates = exotic_module.select_automatic_optimal_calibration_stars( + image, + image.shape, + target_pixel=[150, 150], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter="V", + field_catalog=catalog, + count=2, + brightest_first=True, + saturation_threshold=5000.0, + ) + + assert brightest_comp_stars[0] == [80.0, 80.0] + assert [candidate["flux"] for candidate in brightest_candidates] == sorted( + [candidate["flux"] for candidate in brightest_candidates], + reverse=True, + ) assert all(0.5 <= candidate["brightness_ratio"] <= 2.0 for candidate in candidates) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 693ec5f2..abe3a26f 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -185,6 +185,8 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: summarize_prior_transit_coverage, should_skip_airmass_fit, should_use_eebls_to_initialize_tmid_and_bounds, + should_use_ensemble_photometry_for_stellar_variability, + should_photometer_fortuitous_variables, should_reject_overexposed_stars, should_fit_lightcurve_to_every_comparison_candidate, should_detect_bad_pixels_before_photometry, @@ -1266,6 +1268,20 @@ def test_should_fit_lightcurve_to_every_comparison_candidate_parses_values(): assert should_fit_lightcurve_to_every_comparison_candidate("n") is False +def test_stellar_variability_ensemble_config_defaults_on_and_supports_opt_out(): + assert should_use_ensemble_photometry_for_stellar_variability(None) is True + assert should_use_ensemble_photometry_for_stellar_variability("y") is True + assert should_use_ensemble_photometry_for_stellar_variability("n") is False + assert should_use_ensemble_photometry_for_stellar_variability(False) is False + + +def test_fortuitous_variable_photometry_config_defaults_on_and_supports_opt_out(): + assert should_photometer_fortuitous_variables(None) is True + assert should_photometer_fortuitous_variables("y") is True + assert should_photometer_fortuitous_variables("n") is False + assert should_photometer_fortuitous_variables(False) is False + + def test_should_detect_bad_pixels_before_photometry_parses_values(): assert should_detect_bad_pixels_before_photometry(None) is False assert should_detect_bad_pixels_before_photometry("y") is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 33fe4b96..5f3ea0f4 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -108,6 +108,98 @@ def test_comp_params_reads_stellar_variability_only_from_optional_info(tmp_path) assert inputs.info_dict["stellar_variability_only"] is True +def test_comp_params_defaults_stellar_variability_ensemble_to_true(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_ensemble_photometry_for_stellar_variability"] is True + + +def test_comp_params_reads_stellar_variability_ensemble_opt_out(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "use_ensemble_photometry_for_stellar_variability": False, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_ensemble_photometry_for_stellar_variability"] is False + + +def test_comp_params_defaults_fortuitous_variable_photometry_to_true(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["photometer_fortuitous_variables"] is True + + +def test_comp_params_reads_fortuitous_variable_photometry_opt_out(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"photometer_fortuitous_variables": False}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["photometer_fortuitous_variables"] is False + + +def test_comp_params_defaults_nextastro_vsx_cache_first_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_nextastro_vsx_cache_first"] is False + + +def test_comp_params_reads_nextastro_vsx_cache_first_opt_in(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_nextastro_vsx_cache_first": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_nextastro_vsx_cache_first"] is True + + def test_comp_params_defaults_overexposure_rejection_options(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index c78685ce..6e5f449e 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -201,6 +201,153 @@ def fake_post(url, data, headers, timeout): assert excinfo.value.last_attempt.attempt_number == 5 +def test_nextastro_vsx_query_boxes_split_ra_wrap(): + boxes = exotic_module.nextastro_vsx_query_boxes(359.9, 0.0, 0.2) + + np.testing.assert_allclose(boxes, [ + (359.7, 360.0, -0.2, 0.2), + (0.0, 0.1, -0.2, 0.2), + ]) + + +def test_nextastro_vsx_field_query_normalizes_rows(monkeypatch): + captured = {} + + def fake_post(url, json, timeout): + captured['url'] = url + captured['json'] = json + captured['timeout'] = timeout + return DummyResponse({ + 'columns': ['oid', 'name', 'ra_deg', 'dec_deg', 'var_type', 'mag1', 'mag1_band'], + 'count': 1, + 'row_format': 'objects', + 'rows': [{ + 'oid': 123, + 'name': 'Cached Variable', + 'ra_deg': 10.1, + 'dec_deg': -20.2, + 'var_type': 'EA', + 'mag1': 12.3, + 'mag1_band': 'V', + }], + }) + + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + + rows = exotic_module.nextastro_vsx_field_query(10.0, -20.0, 0.25) + + assert captured['url'].endswith('/vsx_query') + assert captured['timeout'] == 30 + assert captured['json']['compact'] is False + assert rows[0]['Name'] == 'Cached Variable' + assert rows[0]['OID'] == 123 + assert rows[0]['RA2000'] == pytest.approx(10.1) + assert rows[0]['Declination2000'] == pytest.approx(-20.2) + assert rows[0]['VariabilityType'] == 'EA' + assert rows[0]['MaxMag'] == '12.3 V' + assert rows[0]['_vsx_source'] == 'nextastro_cache' + assert rows[0]['_vsx_has_full_metadata'] is False + + +def test_vsx_field_query_cache_first_falls_back_when_cache_empty(monkeypatch): + calls = [] + monkeypatch.setattr(exotic_module, 'nextastro_vsx_field_query', lambda *args: []) + monkeypatch.setattr( + exotic_module, + 'vsx_field_query', + lambda *args, **kwargs: calls.append((args, kwargs)) or [{'Name': 'AAVSO Variable'}], + ) + + rows = exotic_module.vsx_field_query_with_preference( + 10.0, + -20.0, + 0.25, + use_nextastro_vsx_cache_first=True, + ) + + assert rows == [{'Name': 'AAVSO Variable'}] + assert len(calls) == 1 + + +def test_vsx_field_query_cache_first_enriches_period_and_amplitude(monkeypatch): + monkeypatch.setattr( + exotic_module, + 'nextastro_vsx_field_query', + lambda *args: [{ + 'OID': 123, + 'Name': 'Cached Name', + 'RA2000': 10.1, + 'Declination2000': -20.2, + 'VariabilityType': 'EA', + '_vsx_source': 'nextastro_cache', + }], + ) + monkeypatch.setattr( + exotic_module, + 'vsx_field_query', + lambda *args, **kwargs: [{ + 'OID': '123', + 'Name': 'AAVSO Name', + 'RA2000': '10.1000', + 'Declination2000': '-20.2000', + 'Period': '2.5', + 'MaxMag': '12.0 V', + 'MinMag': '12.4 V', + }], + ) + + rows = exotic_module.vsx_field_query_with_preference( + 10.0, + -20.0, + 0.25, + use_nextastro_vsx_cache_first=True, + ) + + assert rows[0]['Name'] == 'AAVSO Name' + assert rows[0]['Period'] == '2.5' + assert exotic_module.vsx_object_amplitude_mag(rows[0]) == pytest.approx(0.4) + assert rows[0]['_vsx_source'] == 'nextastro_cache+aavso_metadata' + + +def test_vsx_field_query_cache_first_uses_full_nextastro_metadata_without_aavso(monkeypatch): + cached_row = exotic_module.normalize_nextastro_vsx_row({ + 'oid': 123, + 'name': 'Cached Full Variable', + 'ra_deg': 10.1, + 'dec_deg': -20.2, + 'var_type': 'EA', + 'period_days': 2.5, + 'amplitude_mag': 0.4, + 'max_mag': 12.0, + 'max_passband': 'V', + 'min_mag': 12.4, + 'min_passband': 'V', + }) + monkeypatch.setattr( + exotic_module, + 'nextastro_vsx_field_query', + lambda *args: [cached_row], + ) + + def unexpected_aavso_call(*args, **kwargs): + raise AssertionError('AAVSO should not be called for the full NextAstro schema') + + monkeypatch.setattr(exotic_module, 'vsx_field_query', unexpected_aavso_call) + + rows = exotic_module.vsx_field_query_with_preference( + 10.0, + -20.0, + 0.25, + use_nextastro_vsx_cache_first=True, + ) + + assert rows[0]['_vsx_has_full_metadata'] is True + assert rows[0]['Period'] == 2.5 + assert rows[0]['Amplitude'] == 0.4 + assert rows[0]['MaxMag'] == '12.0 V' + assert rows[0]['MinMag'] == '12.4 V' + + def test_nextastro_photometry_catalog_match_prefers_requested_filter(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag', 'g', 'dg'], @@ -376,6 +523,32 @@ def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): assert calibration['observed_filter'] == 'V' +def test_merge_nextastro_calibration_stars_deduplicates_catalog_source_ids(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [{ + 'id': 9, + 'source_id': 12345, + 'ra': 10.0, + 'dec': -20.0, + 'Vmag': 11.2, + 'err_Vmag': 0.03, + }], + } + + calibration_stars = exotic_module.merge_nextastro_calibration_stars( + comp_stars=[[100, 200], [130, 230]], + comp_ra_dec=[(10.0, -20.0), (10.0001, -20.0001)], + obs_filter='V', + existing_comp_stars={}, + field_catalog=catalog, + ) + + assert list(calibration_stars) == ['NextAstro-12345'] + + def test_vsp_query_rejects_band_errors_over_limit(monkeypatch): class DummyWCS: def pixel_to_world_values(self, x_pixel, y_pixel): @@ -428,10 +601,15 @@ def test_build_stellar_variability_params_records_nextastro_reference(monkeypatc class DummyFit: data = np.array([1.0, 1.02, 0.98], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) airmass_model = np.ones(3, dtype=float) airmass = np.array([1.1, 1.2, 1.3], dtype=float) jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) transit = np.ones(3, dtype=float) + stellar_variability_target_flux = np.array([1000.0, 1020.0, 980.0], dtype=float) + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) def fake_plot(params, save, s_name, label): captured['params'] = params @@ -475,7 +653,7 @@ def fake_plot(params, save, s_name, label): assert params[0]['observed_filter'] == 'CV' -def test_build_stellar_variability_params_keeps_magnitude_errors_in_flux_ratio_units(monkeypatch, tmp_path): +def test_build_stellar_variability_params_uses_raw_ratio_and_per_exposure_errors(monkeypatch, tmp_path): comp_mag = 9.751 comp_mag_error = 0.018 target_mag = 13.1 @@ -483,11 +661,18 @@ def test_build_stellar_variability_params_keeps_magnitude_errors_in_flux_ratio_u detrended = flux_ratio * np.array([0.94, 1.0, 1.06], dtype=float) class DummyFit: - data = detrended - airmass_model = np.ones(3, dtype=float) + # The fitted series is intentionally normalized: the absolute target + # magnitude must come from the retained raw target/comparison fluxes. + data = detrended / np.nanmedian(detrended) + dataerr = np.full(3, 0.01, dtype=float) + airmass_model = np.array([0.94, 1.0, 1.06], dtype=float) airmass = np.array([1.1, 1.2, 1.3], dtype=float) jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) transit = np.ones(3, dtype=float) + stellar_variability_comp_flux = np.full(3, 100000.0, dtype=float) + stellar_variability_target_flux = stellar_variability_comp_flux * detrended + stellar_variability_target_flux_error = np.array([20.0, 21.0, 22.0], dtype=float) + stellar_variability_comp_flux_error = np.array([30.0, 31.0, 32.0], dtype=float) monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) @@ -510,17 +695,44 @@ class DummyFit: observed_filter='CV', ) - expected_scatter = np.nanstd(detrended) expected_mag_error = np.hypot( comp_mag_error, - 2.5 * expected_scatter / (flux_ratio * np.log(10)), + (2.5 / np.log(10.0)) * np.hypot( + DummyFit.stellar_variability_target_flux_error[1] + / DummyFit.stellar_variability_target_flux[1], + DummyFit.stellar_variability_comp_flux_error[1] + / DummyFit.stellar_variability_comp_flux[1], + ), ) - assert params[1]['mag'] == pytest.approx(target_mag) + np.testing.assert_allclose([row['mag'] for row in params], target_mag, atol=1.0e-10) assert params[1]['mag_err'] == pytest.approx(expected_mag_error) assert params[1]['mag_err'] < 0.08 +def test_build_stellar_variability_params_rejects_normalized_only_absolute_calibration( + monkeypatch, tmp_path): + class DummyFit: + data = np.array([0.99, 1.0, 1.01], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + + with pytest.raises(RuntimeError, match='cannot be recovered from a normalized light curve'): + exotic_module.build_stellar_variability_params_from_fit( + DummyFit(), + {'mag': 12.0, 'error': 0.02, 'mag_band': 'V'}, + [100, 200], + 'COMP', + tmp_path, + 'Host Star', + observed_filter='V', + ) + + def test_stellar_variability_requires_selected_transit_comparison(monkeypatch, tmp_path): logged = [] @@ -555,10 +767,15 @@ def test_stellar_variability_derives_selected_comparison_catalog_magnitude(monke class DummyFit: def __init__(self, data): self.data = np.array(data, dtype=float) + self.dataerr = np.full(3, 0.01, dtype=float) self.airmass_model = np.ones(3, dtype=float) self.airmass = np.ones(3, dtype=float) self.jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) self.transit = np.ones(3, dtype=float) + self.stellar_variability_target_flux = self.data * 1000.0 + self.stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + self.stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + self.stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) monkeypatch.setattr( @@ -606,10 +823,15 @@ def test_stellar_variability_uses_direct_catalog_when_derived_error_is_worse(mon class DummyFit: def __init__(self, data): self.data = np.array(data, dtype=float) + self.dataerr = np.full(4, 0.01, dtype=float) self.airmass_model = np.ones(4, dtype=float) self.airmass = np.ones(4, dtype=float) self.jd_times = np.array([2450000.1, 2450000.2, 2450000.3, 2450000.4], dtype=float) self.transit = np.ones(4, dtype=float) + self.stellar_variability_target_flux = self.data * 1000.0 + self.stellar_variability_comp_flux = np.full(4, 1000.0, dtype=float) + self.stellar_variability_target_flux_error = np.full(4, 2.0, dtype=float) + self.stellar_variability_comp_flux_error = np.full(4, 2.0, dtype=float) monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) @@ -652,10 +874,15 @@ def __init__(self, data): def test_stellar_variability_derives_catalog_magnitude_from_full_field(monkeypatch, tmp_path): class DummyFit: data = np.array([1.0, 1.01, 0.99], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) airmass_model = np.ones(3, dtype=float) airmass = np.ones(3, dtype=float) jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) transit = np.ones(3, dtype=float) + stellar_variability_target_flux = data * 1000.0 + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) class DummyWcs: def world_to_pixel_values(self, ra, dec): @@ -737,10 +964,15 @@ def test_stellar_variability_rejects_g_catalog_anchor_for_clearv(monkeypatch, tm class DummyFit: data = np.array([1.0, 1.01, 0.99], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) airmass_model = np.ones(3, dtype=float) airmass = np.ones(3, dtype=float) jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) transit = np.ones(3, dtype=float) + stellar_variability_target_flux = data * 1000.0 + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) class DummyWcs: def world_to_pixel_values(self, ra, dec): @@ -787,10 +1019,15 @@ def pixel_to_world_values(self, x, y): def test_stellar_variability_uses_v_catalog_anchor_for_clearv(monkeypatch, tmp_path): class DummyFit: data = np.array([1.0, 1.01, 0.99], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) airmass_model = np.ones(3, dtype=float) airmass = np.ones(3, dtype=float) jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) transit = np.ones(3, dtype=float) + stellar_variability_target_flux = data * 1000.0 + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) class DummyWcs: def world_to_pixel_values(self, ra, dec): @@ -1115,3 +1352,717 @@ def test_build_stellar_variability_only_lightcurve_discards_transit_points(monke assert fit.stellar_variability_transit_exclusion['rejected_point_count'] == 3 assert not np.any(np.isclose(fit.time, p_dict['midT'])) assert len(fit.time) == times.size - 3 + + +def test_stellar_variability_ensemble_masks_saturated_frames_and_error_clips_members(): + frame_count = 8 + ranked_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp3', 'comp_index': 2, 'label': 'Comp 3', 'position': [50, 60], + 'overexposure_rejected_count': 1, + }, + { + 'key': 'comp4', 'comp_index': 3, 'label': 'Comp 4', 'position': [70, 80], + 'overexposure_rejected_count': 0, + }, + ] + comp_flux_map = { + 'comp1': np.full(frame_count, 1000.0), + 'comp2': np.full(frame_count, 2000.0), + 'comp3': np.full(frame_count, 3000.0), + 'comp4': np.full(frame_count, 1500.0), + } + comp_flux_map['comp3'][0] = np.nan + calibration_stars = { + 'C1': {'pos': [10, 20], 'mag': 12.0, 'error': 0.010, 'mag_band': 'V'}, + 'C2': {'pos': [30, 40], 'mag': 12.5, 'error': 0.011, 'mag_band': 'V'}, + 'C3': {'pos': [50, 60], 'mag': 12.2, 'error': 0.010, 'mag_band': 'V'}, + 'C4': {'pos': [70, 80], 'mag': 12.3, 'error': 0.200, 'mag_band': 'V'}, + } + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + ) + + assert [member['key'] for member in selection['members']] == ['comp3', 'comp2', 'comp1'] + assert selection['members'][0]['summary']['overexposure_rejected_count'] == 1 + rejected_reasons = {item['key']: item['reason'] for item in selection['rejected']} + assert 'sigma-clip' in rejected_reasons['comp4'] + assert selection['calibration_error_clip']['high_threshold'] < 0.2 + assert selection['calibration_error_clip']['high_threshold'] >= 0.01 + assert selection['calibration_error_clip']['minimum_high_threshold'] == pytest.approx(0.01) + + +def test_stellar_variability_ensemble_error_clip_does_not_reject_below_point_zero_one_mag(): + candidates = [ + {'magnitude_error': error} + for error in (0.0010, 0.0011, 0.0012, 0.0090, 0.0110) + ] + + keep, summary = exotic_module.stellar_variability_ensemble_calibration_error_clip(candidates) + + assert keep.tolist() == [True, True, True, True, False] + assert summary['high_threshold'] == pytest.approx(0.01) + assert summary['minimum_high_threshold'] == pytest.approx(0.01) + + +def test_automatic_comparison_merge_deduplicates_only_added_sources(): + merged, messages = exotic_module.merge_automatic_comparison_star_coords( + [[10.0, 20.0], [11.0, 20.0]], + [[10.4, 20.3], [50.0, 60.0], [50.5, 60.2]], + duplicate_radius_pixels=2.0, + ) + + # Nearby primary/user selections remain intentional; automatic additions + # cannot repeat either a primary source or an earlier automatic source. + assert merged == [[10.0, 20.0], [11.0, 20.0], [50.0, 60.0]] + assert len(messages) == 2 + + +def test_stellar_variability_ensemble_caps_at_five_by_target_color_and_magnitude(): + frame_count = 8 + target_match = { + 'mag': 12.0, + 'error': 0.01, + 'mag_band': 'V', + 'catalog_row': {'Bmag': 12.5, 'Vmag': 12.0}, + } + candidate_values = [ + (10.0, -0.5), + (11.0, 0.0), + (12.1, 0.55), + (12.2, 0.60), + (11.9, 0.45), + (12.3, 0.40), + (12.0, 0.52), + ] + ranked_summaries = [] + calibration_stars = {} + comp_flux_map = {} + for index, (magnitude, color) in enumerate(candidate_values, start=1): + key = f'comp{index}' + position = [index * 10, index * 10 + 1] + ranked_summaries.append({ + 'key': key, + 'comp_index': index - 1, + 'label': f'Comp {index}', + 'position': position, + 'overexposure_rejected_count': 0, + }) + calibration_stars[f'C{index}'] = { + 'pos': position, + 'mag': magnitude, + 'error': 0.01, + 'mag_band': 'V', + 'catalog_row': {'Bmag': magnitude + color, 'Vmag': magnitude}, + } + comp_flux_map[key] = np.full(frame_count, 10000.0 - index * 100.0) + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + target_catalog_match=target_match, + ) + + assert selection['prelimit_member_count'] == 7 + assert selection['member_limit'] == 5 + assert [member['key'] for member in selection['members']] == [ + 'comp7', 'comp3', 'comp5', 'comp4', 'comp6', + ] + assert all(member['color_delta'] is not None for member in selection['members']) + assert all(member['magnitude_delta'] is not None for member in selection['members']) + limited_keys = { + rejected['key'] + for rejected in selection['rejected'] + if 'closest to the target' in rejected['reason'] + } + assert limited_keys == {'comp1', 'comp2'} + + +def test_stellar_variability_ensemble_rejects_duplicate_catalog_sources(): + frame_count = 8 + ranked_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp3', 'comp_index': 2, 'label': 'Comp 3', 'position': [50, 60], + 'overexposure_rejected_count': 0, + }, + ] + calibration_stars = { + 'NextAstro-111': { + 'pos': [10, 20], 'mag': 12.0, 'error': 0.01, 'mag_band': 'V', 'source_id': 111, + }, + 'NextAstro-111-2': { + 'pos': [30, 40], 'mag': 12.0, 'error': 0.01, 'mag_band': 'V', 'source_id': 111, + }, + 'NextAstro-222': { + 'pos': [50, 60], 'mag': 12.5, 'error': 0.01, 'mag_band': 'V', 'source_id': 222, + }, + } + comp_flux_map = { + 'comp1': np.full(frame_count, 3000.0), + 'comp2': np.full(frame_count, 2000.0), + 'comp3': np.full(frame_count, 1000.0), + } + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + ) + + assert [member['key'] for member in selection['members']] == ['comp1', 'comp3'] + duplicate = next(item for item in selection['rejected'] if item['key'] == 'comp2') + assert 'duplicate catalog source' in duplicate['reason'] + + +def test_discover_fortuitous_vsx_variables_filters_on_count_rate_error_and_classifies(monkeypatch): + class FakeWcs: + def pixel_to_world_values(self, x_value, y_value): + return x_value, y_value + + def world_to_pixel_values(self, ra_value, dec_value): + return ra_value, dec_value + + reference_image = np.zeros((80, 80), dtype=float) + reference_image[20, 20] = 10000.0 + reference_image[40, 40] = 100.0 + monkeypatch.setattr(exotic_module, 'search_wcs', lambda _path: FakeWcs()) + monkeypatch.setattr( + exotic_module, + 'vsx_field_query', + lambda *args, **kwargs: [ + { + 'Name': 'Bright VSX', 'AUID': '000-AAA-001', + 'RA2000': 20.0, 'Declination2000': 20.0, + 'Period': '5.0', 'MaxMag': '12.0 V', 'MinMag': '12.5 V', + }, + { + 'Name': 'Faint VSX', 'AUID': '000-AAA-002', + 'RA2000': 40.0, 'Declination2000': 40.0, + 'Period': '20.0', 'MaxMag': '15.0 V', 'MinMag': '15.2 V', + }, + ], + ) + + variables = exotic_module.discover_fortuitous_vsx_variables( + 'synthetic.wcs', + reference_image.shape, + 1.0, + reference_image, + 'V', + target_pixel=[60, 60], + exposure_seconds=60.0, + ) + + assert len(variables) == 1 + assert variables[0]['name'] == 'Bright VSX' + assert variables[0]['estimated_magnitude_error'] < 0.05 + assert variables[0]['category'] == 'optimal_variables' + assert variables[0]['period_days'] == pytest.approx(5.0) + assert variables[0]['amplitude_mag'] == pytest.approx(0.5) + + +def test_fortuitous_reference_error_estimate_includes_sky_noise(monkeypatch): + reference_image = np.full((80, 80), 1000.0, dtype=float) + reference_image[40, 40] += 5000.0 + monkeypatch.setattr( + exotic_module, + 'skybg_phot', + lambda *args, **kwargs: (1000.0, 100.0, 500.0), + ) + + estimate = exotic_module.estimated_magnitude_error_from_reference_count_rate( + reference_image, + 40.0, + 40.0, + exposure_seconds=60.0, + gain_e_per_adu=1.0, + ) + + source_only_error = ( + (2.5 / np.log(10.0)) + * exotic_module.source_flux_uncertainty_from_counts(5000.0, gain_e_per_adu=1.0) + / 5000.0 + ) + assert source_only_error < 0.05 + assert estimate['estimated_magnitude_error'] > 0.05 + assert estimate['reference_noise_components_adu']['sky_aperture'] > 0 + assert estimate['reference_noise_components_adu']['sky_estimate'] > 0 + + +def test_calibrated_stellar_variability_ensemble_combines_catalog_zero_points(): + frame_count = 7 + target_flux = np.full(frame_count, 1000.0) + target_error = np.full(frame_count, 1.0) + comp_flux_map = { + 'comp1': np.full(frame_count, 500.0), + 'comp2': np.full(frame_count, 250.0), + } + comp_error_map = { + 'comp1': np.full(frame_count, 1.0), + 'comp2': np.full(frame_count, 1.0), + } + members = [ + { + 'key': 'comp1', + 'magnitude': 12.0, + 'magnitude_error': 0.01, + 'summary': {'ensemble_frame_keep_mask': np.ones(frame_count, dtype=bool)}, + }, + { + 'key': 'comp2', + 'magnitude': 12.0 + 2.5 * np.log10(2.0), + 'magnitude_error': 0.01, + 'summary': {'ensemble_frame_keep_mask': np.ones(frame_count, dtype=bool)}, + }, + ] + + result = exotic_module.build_stellar_variability_calibrated_ensemble_series( + target_flux, + target_error, + comp_flux_map, + comp_error_map, + members, + ) + + expected_target_magnitude = 12.0 - 2.5 * np.log10(2.0) + assert result['applied'] is True + np.testing.assert_allclose(result['magnitude'], expected_target_magnitude, atol=1.0e-10) + np.testing.assert_allclose(result['relative_flux'], 1.0, atol=1.0e-10) + np.testing.assert_array_equal(result['valid_member_count'], np.full(frame_count, 2)) + assert np.all(result['magnitude_error'] > 0) + + +def test_build_stellar_variability_ensemble_params_preserves_member_metadata(monkeypatch, tmp_path): + captured = {} + monkeypatch.setattr( + exotic_module, + 'plot_stellar_variability', + lambda params, save, target, label: captured.update( + params=params, + save=save, + target=target, + label=label, + ), + ) + fit = types.SimpleNamespace( + jd_times=np.array([2460000.1, 2460000.2]), + airmass=np.array([1.1, 1.2]), + stellar_variability_ensemble_magnitudes=np.array([12.30, 12.31]), + stellar_variability_ensemble_magnitude_errors=np.array([0.01, 0.011]), + stellar_variability_ensemble_members=[ + { + 'label': 'C1', 'position': [10, 20], 'magnitude': 12.0, + 'magnitude_error': 0.01, + 'star': {'catalog_source': 'Catalog A', 'ra': 10.1, 'dec': -20.1}, + }, + { + 'label': 'C2', 'position': [30, 40], 'magnitude': 12.5, + 'magnitude_error': 0.011, + 'star': {'catalog_source': 'Catalog B', 'ra': 10.2, 'dec': -20.2}, + }, + ], + ) + + params = exotic_module.build_stellar_variability_ensemble_params_from_fit( + fit, + tmp_path, + 'Target Star', + observed_filter='V', + ) + + assert len(params) == 2 + assert params[0]['cname'] == 'ENSEMBLE (2 stars)' + assert params[0]['cmag'] is None + assert params[0]['ensemble_member_labels'] == ['C1', 'C2'] + assert params[0]['ensemble_member_catalog_errors'] == [0.01, 0.011] + assert params[0]['ensemble_member_ra_degs'] == [10.1, 10.2] + assert params[0]['ensemble_member_dec_degs'] == [-20.1, -20.2] + assert params[0]['ensemble_members'][0]['ra_deg'] == pytest.approx(10.1) + assert params[0]['ensemble_members'][1]['dec_deg'] == pytest.approx(-20.2) + assert fit.stellar_variability_params == params + assert captured['label'] == 'ENSEMBLE (2 stars)' + + +def test_stellar_variability_ensemble_selection_json_lists_color_and_magnitude(monkeypatch, tmp_path): + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + member = { + 'selection_rank': 1, + 'key': 'comp1', + 'label': 'C1', + 'position': [10, 20], + 'magnitude': 12.1, + 'magnitude_error': 0.01, + 'color': 0.55, + 'color_label': 'B-V', + 'target_color': 0.50, + 'target_color_label': 'B-V', + 'color_delta': 0.05, + 'target_magnitude': 12.0, + 'magnitude_delta': 0.1, + 'color_magnitude_similarity_score': np.hypot(0.05, 0.1), + 'median_flux': 5000.0, + 'star': { + 'ra': 10.1, + 'dec': -20.2, + 'mag_band': 'V', + 'catalog_source': 'Synthetic catalog', + }, + } + fit = types.SimpleNamespace( + jd_times=np.array([2460000.1]), + airmass=np.array([1.1]), + stellar_variability_ensemble_magnitudes=np.array([12.3]), + stellar_variability_ensemble_magnitude_errors=np.array([0.02]), + stellar_variability_ensemble_members=[member], + stellar_variability_ensemble_selection={ + 'members': [member], + 'rejected': [{'key': 'comp2', 'reason': 'not among the 5 closest'}], + 'member_limit': 5, + 'prelimit_member_count': 6, + 'calibration_error_clip': {'high_threshold': 0.03}, + 'target_catalog_profile': { + 'magnitude': 12.0, + 'magnitude_band': 'V', + 'color': 0.5, + 'color_label': 'B-V', + }, + }, + ) + + exotic_module.build_stellar_variability_ensemble_params_from_fit( + fit, + tmp_path, + 'Target Star', + observed_filter='V', + observation_date='2024-01-02', + ) + + output_path = next(tmp_path.glob('EnsembleSelection_TargetStar_2024-01-02.json')) + payload = json.loads(output_path.read_text(encoding='utf-8')) + assert payload['ensemble']['maximum_members'] == 5 + assert payload['ensemble']['member_count_before_five_star_limit'] == 6 + assert payload['target']['catalog_profile']['color'] == pytest.approx(0.5) + assert payload['ensemble']['members'][0]['color_delta'] == pytest.approx(0.05) + assert payload['ensemble']['members'][0]['magnitude_delta'] == pytest.approx(0.1) + + +def test_process_fortuitous_variable_writes_independent_ensemble_products(monkeypatch, tmp_path): + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + monkeypatch.setattr( + exotic_module, + 'psf_quality_mask_for_key', + lambda psf_data, key, frame_count, psf_flux_data=None: np.ones(frame_count, dtype=bool), + ) + frame_count = 12 + times = np.linspace(2460000.0, 2460000.1, frame_count) + quality_mask = np.ones(frame_count, dtype=bool) + comparison_calibration = { + 'method': 'aperture', + 'method_label': 'Aperture photometry', + 'a': 0, + 'an': 0, + 'field_image_keep_mask': quality_mask, + 'comp_summaries': [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'aggregate_score': 0.001, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'aggregate_score': 0.002, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + ], + } + psf_data = { + # The exoplanet target is unusable in every frame. Fortuitous-variable + # processing must remain independent of that target-specific mask. + 'target': np.full((frame_count, 7), np.nan), + 'comp1': np.ones((frame_count, 7)), + 'comp2': np.ones((frame_count, 7)), + 'comp3': np.ones((frame_count, 7)), + } + aper_data = { + 'target': np.full((frame_count, 1, 1), 1000.0), + 'comp1': np.full((frame_count, 1, 1), 500.0), + 'comp1_unc': np.full((frame_count, 1, 1), 1.0), + 'comp2': np.full((frame_count, 1, 1), 250.0), + 'comp2_unc': np.full((frame_count, 1, 1), 1.0), + 'comp3': ( + 800.0 * (1.0 + 0.02 * np.sin(np.linspace(0, 2 * np.pi, frame_count))) + )[:, None, None], + 'comp3_unc': np.full((frame_count, 1, 1), 1.0), + } + # One otherwise valid frame has an internal target error above 0.05 mag. + aper_data['comp3_unc'][2, 0, 0] = 80.0 + calibrations = { + 'C1': { + 'pos': [10, 20], 'mag': 12.0, 'error': 0.01, 'mag_band': 'V', + 'ra': 10.1, 'dec': -20.1, + 'catalog_source': 'Synthetic catalog', + 'catalog_row': {'Bmag': 12.5, 'Vmag': 12.0}, + }, + 'C2': { + 'pos': [30, 40], 'mag': 12.75, 'error': 0.011, 'mag_band': 'V', + 'ra': 10.2, 'dec': -20.2, + 'catalog_source': 'Synthetic catalog', + 'catalog_row': {'Bmag': 13.35, 'Vmag': 12.75}, + }, + } + variable = { + 'name': 'Synthetic VSX', + 'auid': '000-AAA-001', + 'variable_type': 'EA', + 'period_days': 5.0, + 'amplitude_mag': 0.5, + 'category': 'optimal_variables', + 'ra': 10.0, + 'dec': -20.0, + 'pos': [50, 60], + 'tracking_key': 'comp3', + 'aperture_flux_adu': 800.0, + 'count_rate_adu_per_second': 13.3, + 'estimated_magnitude_error': 0.04, + 'catalog_match': { + 'mag': 12.4, + 'error': 0.02, + 'mag_band': 'V', + 'catalog_row': {'Bmag': 12.95, 'Vmag': 12.4}, + }, + } + info_dict = { + 'save': str(tmp_path), + 'date': '2024-01-02', + 'aavso_num': 'RTZ', + 'camera': 'CCD', + 'filter': 'V', + 'lat': '+32.4', + 'long': '-110.7', + 'elev': 2600, + } + + variable_overexposed = np.zeros(frame_count, dtype=bool) + variable_overexposed[:2] = True + results = exotic_module.process_fortuitous_variables( + [variable], + comparison_calibration, + calibrations, + times, + times, + np.linspace(1.1, 1.3, frame_count), + psf_data, + aper_data, + info_dict, + comp_overexposed_masks={'comp3': variable_overexposed}, + exposure_times_seconds=np.full(frame_count, 60.0), + observed_filter='V', + ) + + assert results[0]['status'] == 'completed' + assert results[0]['input_frame_count'] == frame_count + assert results[0]['target_overexposure_rejected_frame_count'] == 2 + assert results[0]['output_magnitude_error_rejected_frame_count'] == 1 + assert results[0]['point_count'] == frame_count - 3 + variable_dir = tmp_path / 'fortuitous_variables' / 'optimal_variables' / 'VSX_SyntheticVSX' + assert next(variable_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')).is_file() + assert next(variable_dir.glob('EnsembleSelection_SyntheticVSX_2024-01-02.json')).is_file() + assert next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')).is_file() + manifest = json.loads( + next((tmp_path / 'fortuitous_variables').glob('FortuitousVariables_2024-01-02.json')).read_text( + encoding='utf-8' + ) + ) + assert manifest['variables'][0]['ensemble_member_count'] == 2 + assert manifest['variables'][0]['output_magnitude_error_rejected_frame_count'] == 1 + assert manifest['variables'][0]['output_magnitude_error_max'] > 0.05 + selection = json.loads( + next(variable_dir.glob('EnsembleSelection_SyntheticVSX_2024-01-02.json')).read_text( + encoding='utf-8' + ) + ) + assert selection['target']['input_frame_count'] == frame_count + assert selection['target']['target_overexposure_rejected_frame_count'] == 2 + assert selection['target']['output_magnitude_error_rejected_frame_count'] == 1 + assert selection['target']['valid_output_frame_count'] == frame_count - 3 + assert 'exoplanet target overexposure mask is not applied' in ( + selection['target']['saturation_rejection_scope'] + ) + aid_text = next(variable_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')).read_text( + encoding='utf-8' + ) + ensemble_header = next( + line for line in aid_text.splitlines() + if line.startswith('#ENSEMBLE-COMPARISONS-XC=') + ) + ensemble_metadata = json.loads(ensemble_header.split('=', 1)[1]) + assert ensemble_metadata['member_count'] == 2 + assert ensemble_metadata['members'][0]['ra_deg'] == pytest.approx(10.1) + assert ensemble_metadata['members'][0]['dec_deg'] == pytest.approx(-20.1) + assert ensemble_metadata['members'][1]['ra_deg'] == pytest.approx(10.2) + assert ensemble_metadata['members'][1]['dec_deg'] == pytest.approx(-20.2) + csv_path = next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')) + exported_errors = [ + float(row.split(',')[3]) + for row in csv_path.read_text(encoding='utf-8').splitlines()[1:] + if row.strip() + ] + assert exported_errors + assert max(exported_errors) < 0.05 + + +def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(): + frame_count = 12 + times = np.linspace(10.2, 10.3, frame_count) + target_flux = 1000.0 * (1.0 + np.linspace(-0.002, 0.002, frame_count)) + comp1_flux = np.full(frame_count, 500.0) + comp2_flux = np.full(frame_count, 250.0) + quality_mask = np.ones(frame_count, dtype=bool) + comp_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'aggregate_score': 0.001, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'aggregate_score': 0.002, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + ] + comparison_calibration = { + 'method': 'aperture', + 'method_label': 'Aperture photometry (aper=5px, annulus=10px)', + 'a': 0, + 'an': 0, + 'aper': 5.0, + 'annulus': 10.0, + 'field_score': 0.0015, + 'field_image_keep_mask': quality_mask, + 'comp_summaries': comp_summaries, + } + psf_data = { + 'target': np.ones((frame_count, 7), dtype=float), + 'comp1': np.ones((frame_count, 7), dtype=float), + 'comp2': np.ones((frame_count, 7), dtype=float), + } + aper_data = { + 'target': target_flux[:, None, None], + 'target_unc': np.full((frame_count, 1, 1), 1.0), + 'comp1': comp1_flux[:, None, None], + 'comp1_unc': np.full((frame_count, 1, 1), 1.0), + 'comp2': comp2_flux[:, None, None], + 'comp2_unc': np.full((frame_count, 1, 1), 1.0), + } + calibration_stars = { + 'C1': { + 'pos': [10, 20], 'mag': 12.0, 'error': 0.01, 'mag_band': 'V', + 'catalog_source': 'Synthetic catalog', + }, + 'C2': { + 'pos': [30, 40], 'mag': 12.0 + 2.5 * np.log10(2.0), + 'error': 0.011, 'mag_band': 'V', 'catalog_source': 'Synthetic catalog', + }, + } + + result = exotic_module.select_stellar_variability_only_photometry( + times, + times, + np.ones(frame_count), + _stellar_variability_only_planet_dict(), + comparison_calibration, + psf_data, + aper_data, + target_flux, + use_ensemble_photometry=True, + calibration_stars=calibration_stars, + observed_filter='V', + ) + + selected = result['selected_result'] + assert result['selection_metric'] == 'stellar_variability_ensemble' + assert selected['comp_index'] is None + assert selected['ensemble_member_keys'] == ['comp1', 'comp2'] + assert selected['fit'].stellar_variability_ensemble_members + assert len(selected['fit'].stellar_variability_ensemble_magnitudes) == len(selected['fit'].time) + + +def test_stellar_variability_selector_opt_out_restores_single_comp_selection(): + frame_count = 24 + times = np.linspace(10.2, 10.3, frame_count) + quality_mask = np.ones(frame_count, dtype=bool) + comparison_calibration = { + 'method': 'aperture', + 'method_label': 'Aperture photometry', + 'a': 0, + 'an': 0, + 'aper': 5.0, + 'annulus': 10.0, + 'field_score': 0.001, + 'field_image_keep_mask': quality_mask, + 'comp_summaries': [{ + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'aggregate_score': 0.001, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }], + } + target_flux = 1000.0 * (1.0 + np.linspace(-0.001, 0.001, frame_count)) + comp_flux = np.full(frame_count, 500.0) + psf_data = { + 'target': np.ones((frame_count, 7), dtype=float), + 'comp1': np.ones((frame_count, 7), dtype=float), + } + aper_data = { + 'target': target_flux[:, None, None], + 'target_unc': np.full((frame_count, 1, 1), 1.0), + 'comp1': comp_flux[:, None, None], + 'comp1_unc': np.full((frame_count, 1, 1), 1.0), + } + + result = exotic_module.select_stellar_variability_only_photometry( + times, + times, + np.ones(frame_count), + _stellar_variability_only_planet_dict(), + comparison_calibration, + psf_data, + aper_data, + target_flux, + use_ensemble_photometry=False, + ) + + assert result['selection_metric'] == 'stellar_variability_scatter' + assert result['selected_result']['comp_index'] == 0 diff --git a/tests/test_output_files.py b/tests/test_output_files.py index c3ca8091..60ca60b7 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -167,6 +167,45 @@ def test_final_params_writes_stellar_variability_only_payload(tmp_path): assert root_file.exists() +def test_final_params_describes_calibrated_stellar_variability_ensemble(tmp_path): + (tmp_path / "temp").mkdir() + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.arange(3, dtype=float), + stellar_variability_scatter=0.001, + stellar_variability_transit_exclusion={'rejected_point_count': 0}, + ) + vsp_params = [{ + 'time': 2460000.1, + 'mag': 12.3, + 'mag_err': 0.01, + 'cname': 'ENSEMBLE (2 stars)', + 'ensemble_reference': True, + 'ensemble_member_count': 2, + 'ensemble_member_labels': ['C1', 'C2'], + 'mag_band': 'V', + }] + p_dict = {'pName': 'Syntheticb'} + i_dict = {'save': str(tmp_path), 'date': '2020-01-01'} + + OutputFiles(fit, p_dict, i_dict, []).final_planetary_params( + phot_opt=True, + vsp_params=vsp_params, + comp_star='ensemble', + comp_coords=None, + min_aper=0, + min_annul=15, + ) + + output_path = next((tmp_path / "temp").glob("FinalParams_Syntheticb_2020-01-01.json")) + params = json.loads(output_path.read_text())["FINAL STELLAR VARIABILITY PARAMETERS"] + assert params["Stellar Variability Reference Star"] == "ensemble" + assert params["Variable Reference Star"] == ( + "Calibrated comparison-star ensemble (2 stars): C1, C2" + ) + assert "calibrated comparison-star ensemble" in params["Variable Reference Measurement"] + + def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): (tmp_path / "temp").mkdir() fit = SimpleNamespace( @@ -426,6 +465,69 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert "HAT-P-32,2450000.12345,12.340,0.050,V,NO,STD" in output_text +def test_aid_output_records_calibrated_ensemble_members(tmp_path): + fit = DummyFit() + p_dict = {"pName": "Target b", "sName": "Target"} + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.4", + "long": "-110.7", + "elev": 2600, + } + vsp_params = [{ + "time": 2450000.12345, + "mag": 12.34, + "mag_err": 0.02, + "airmass": 1.234, + "cname": "ENSEMBLE (2 stars)", + "cmag": None, + "cmag_err": None, + "catalog_source": "Calibrated comparison-star ensemble", + "is_aavso_vsp": False, + "mag_band": "V", + "ensemble_reference": True, + "ensemble_member_count": 2, + "ensemble_member_labels": ["C1", "C2"], + "ensemble_member_positions": [[10, 20], [30, 40]], + "ensemble_member_catalog_magnitudes": [12.0, 12.5], + "ensemble_member_catalog_errors": [0.01, 0.011], + "ensemble_member_catalog_sources": ["Catalog A", "Catalog B"], + "ensemble_member_ra_degs": [10.1, 10.2], + "ensemble_member_dec_degs": [-20.1, -20.2], + "ensemble_member_catalog_colors": [0.5, 0.6], + "ensemble_member_catalog_color_labels": ["B-V", "B-V"], + "ensemble_member_color_deltas": [0.02, 0.08], + "ensemble_member_magnitude_deltas": [0.1, 0.4], + "ensemble_member_similarity_scores": [0.102, 0.408], + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = (tmp_path / "AID_AAVSO_Target_2020-01-01.txt").read_text(encoding="utf-8") + metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") + ensemble_metadata = aavso_json_header(output_text, "ENSEMBLE-COMPARISONS-XC") + assert metadata["ensemble_reference"] is True + assert metadata["ensemble_member_count"] == 2 + assert metadata["ensemble_member_labels"] == ["C1", "C2"] + assert metadata["ensemble_member_ra_degs"] == [10.1, 10.2] + assert metadata["ensemble_member_dec_degs"] == [-20.1, -20.2] + assert metadata["ensemble_member_catalog_colors"] == [0.5, 0.6] + assert metadata["ensemble_member_color_deltas"] == [0.02, 0.08] + assert metadata["ensemble_member_magnitude_deltas"] == [0.1, 0.4] + assert ensemble_metadata["member_count"] == 2 + assert ensemble_metadata["members"][0]["label"] == "C1" + assert ensemble_metadata["members"][0]["ra_deg"] == pytest.approx(10.1) + assert ensemble_metadata["members"][0]["dec_deg"] == pytest.approx(-20.1) + assert ensemble_metadata["members"][1]["label"] == "C2" + assert ensemble_metadata["members"][1]["ra_deg"] == pytest.approx(10.2) + assert ensemble_metadata["members"][1]["dec_deg"] == pytest.approx(-20.2) + assert "Target,2450000.12345,12.340,0.020,V,NO,STD,ENSEMBLE (2 stars),na" in output_text + + def test_aid_output_samples_large_derived_anchor_label_lists(tmp_path): fit = DummyFit() p_dict = { From 61cc8325ace226405ba5614b78e263ccb62e5533 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 15 Jul 2026 21:46:51 +1000 Subject: [PATCH 082/116] A few small bugs. It is all Kalee's fault that I found them. --- exotic/api/filters.py | 1 + exotic/api/ultranest_utils.py | 15 +++- exotic/exotic.py | 32 ++++++-- tests/test_exotic_proper_motion.py | 124 +++++++++++++++++++++++++++++ tests/test_ld.py | 1 + tests/test_nonlinear_ld.py | 59 ++++++++++++++ tests/test_ultranest_utils.py | 43 ++++++++++ 7 files changed, 264 insertions(+), 11 deletions(-) create mode 100644 tests/test_nonlinear_ld.py diff --git a/exotic/api/filters.py b/exotic/api/filters.py index fc998414..82715ade 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -90,6 +90,7 @@ "bb": "Johnson B", "pb": "Photographic B", "bv": "Johnson V", + "G": "Photographic G", "pg": "Photographic G", "br": "Johnson R", "pr": "Photographic R", diff --git a/exotic/api/ultranest_utils.py b/exotic/api/ultranest_utils.py index c7cb3122..c9aa9006 100644 --- a/exotic/api/ultranest_utils.py +++ b/exotic/api/ultranest_utils.py @@ -488,17 +488,26 @@ def _mute_ultranest_logging(sampler): logger.disabled = disabled -def _traceback_mentions_ultranest_mlfriends(exc): +def _traceback_mentions_ultranest_mlfriends(exc, function_name=None): traceback = exc.__traceback__ while traceback is not None: filename = str(traceback.tb_frame.f_code.co_filename).replace("\\", "/") - if "ultranest/mlfriends" in filename: + frame_function = str(traceback.tb_frame.f_code.co_name).rsplit(".", 1)[-1] + if ( + "ultranest/mlfriends" in filename + and (function_name is None or frame_function == function_name) + ): return True traceback = traceback.tb_next return False def _is_ultranest_degenerate_region_error(exc): + if isinstance(exc, AssertionError): + # UltraNest 4.5.0 asserts here when a bootstrap region contains one + # unique point and its covariance is therefore non-finite. + return _traceback_mentions_ultranest_mlfriends(exc, "bounding_ellipsoid") + if not isinstance(exc, ValueError): return False @@ -513,7 +522,7 @@ def _is_ultranest_degenerate_region_error(exc): def _run_sampler_with_degenerate_region_guard(sampler, kwargs): try: return sampler.run(**kwargs) - except ValueError as exc: + except (AssertionError, ValueError) as exc: if not _is_ultranest_degenerate_region_error(exc): raise raise np.linalg.LinAlgError( diff --git a/exotic/exotic.py b/exotic/exotic.py index 04ee545d..89180ea2 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -4650,6 +4650,8 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, final_time_indices = match_time_subset_indices(good_times, final_fit_times) if final_time_indices is not None: good_times = good_times[final_time_indices] + good_flux = good_flux[final_time_indices] + good_unc = good_unc[final_time_indices] good_airmass = good_airmass[final_time_indices] good_jd_times = good_jd_times[final_time_indices] if good_exposure_times is not None: @@ -12911,7 +12913,7 @@ def user_entered_ld(ld, observed_filter): ld.set_ld(ld_[0], ld_[1], ld_[2], ld_[3]) -def nonlinear_ld(ld, info_dict): +def nonlinear_ld(ld, info_dict, non_interactive_run=False): user_entered = False observed_filter = { 'filter': info_dict['filter'], @@ -12926,6 +12928,12 @@ def nonlinear_ld(ld, info_dict): custom_range(ld, observed_filter) ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) else: + if non_interactive_run: + raise ValueError( + f"EXOTIC did not recognize the filter {info_dict.get('filter')!r}. " + "Non-interactive runs require a recognized standard filter or both wl_min and wl_max." + ) + opt = info_dict.get('ld_uncertainties') if isinstance(opt, str): @@ -12957,9 +12965,9 @@ def nonlinear_ld(ld, info_dict): info_dict['wl_max'] = ld.wl_max -def get_ld_values(planet_dict, info_dict): +def get_ld_values(planet_dict, info_dict, non_interactive_run=False): ld_obj = LimbDarkening(planet_dict) - nonlinear_ld(ld_obj, info_dict) + nonlinear_ld(ld_obj, info_dict, non_interactive_run=non_interactive_run) ld0 = ld_obj.ld0 ld1 = ld_obj.ld1 @@ -27930,9 +27938,9 @@ def parse_args(): "If the service returns an error, EXOTIC falls back to individual VSX checks.") parser.add_argument('--non-interactive-run', action='store_true', - help="Run without interactive prompts for target pixel-coordinate mismatch checks. " - "If a mismatch is detected, EXOTIC logs a warning and proceeds with the " - "user-provided coordinates.") + help="Run without interactive prompts for target pixel-coordinate mismatch checks " + "or unrecognized limb-darkening filters. Coordinate mismatches use an automatic " + "fallback; unrecognized filters abort unless wl_min and wl_max are provided.") parser.add_argument('--multiprocess-transformations', type=int, default=None, @@ -28438,7 +28446,11 @@ def _main_impl(): exotic_infoDict.setdefault('observed_filter', exotic_infoDict.get('filter')) log_info("Calculating limb-darkening coefficients.") - ld, ld0, ld1, ld2, ld3 = get_ld_values(pDict, exotic_infoDict) + ld, ld0, ld1, ld2, ld3 = get_ld_values( + pDict, + exotic_infoDict, + non_interactive_run=args.non_interactive_run, + ) log_info("Limb-darkening coefficients ready.") # check for EPW_MD5 checksum @@ -30942,7 +30954,11 @@ def _main_impl(): goodTimes, goodFluxes, goodNormUnc, goodAirmasses = [], [], [], [] bestCompStar, comp_coords = None, None exotic_infoDict.setdefault('observed_filter', exotic_infoDict.get('filter')) - ld, ld0, ld1, ld2, ld3 = get_ld_values(pDict, exotic_infoDict) + ld, ld0, ld1, ld2, ld3 = get_ld_values( + pDict, + exotic_infoDict, + non_interactive_run=args.non_interactive_run, + ) with exotic_infoDict['prered_file'].open('r') as f: for processed_data in f: diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index abe3a26f..bcb98624 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -646,6 +646,130 @@ def fake_final_fit(times, flux, unc, airmass, prior, bounds, jd_times=None, **kw ].tolist() == [True] * 10 +def test_finalize_comparison_candidate_keeps_flux_aligned_after_final_fit_subsets_times(monkeypatch): + fit_calls = [] + initial_subset_mask = np.ones(38, dtype=bool) + initial_subset_mask[[1, 3, 5, 7, 9, 11]] = False + + def fake_lc_fitter(times, flux, unc, airmass, prior, bounds, jd_times=None, mode=None, **kwargs): + return types.SimpleNamespace( + residuals=np.zeros(len(times), dtype=float), + phase=np.linspace(-0.5, 0.5, len(times)), + ) + + def fake_final_fit(times, flux, unc, airmass, prior, bounds, jd_times=None, **kwargs): + times = np.asarray(times, dtype=float) + flux = np.asarray(flux, dtype=float) + unc = np.asarray(unc, dtype=float) + airmass = np.asarray(airmass, dtype=float) + fit_calls.append({ + "times": times.copy(), + "flux": flux.copy(), + "unc": unc.copy(), + "airmass": airmass.copy(), + }) + + if len(fit_calls) == 1: + keep_mask = initial_subset_mask + residuals = np.zeros(np.count_nonzero(keep_mask), dtype=float) + residuals[10] = 1.0 + else: + keep_mask = np.ones(len(times), dtype=bool) + residuals = np.zeros(len(times), dtype=float) + + retained_times = times[keep_mask] + retained_flux = flux[keep_mask] + retained_unc = unc[keep_mask] + retained_airmass = airmass[keep_mask] + fit = types.SimpleNamespace( + time=retained_times, + airmass=retained_airmass, + data=retained_flux, + dataerr=retained_unc, + detrended=retained_flux, + detrendederr=retained_unc, + airmass_model=np.ones(len(retained_times), dtype=float), + transit=np.ones(len(retained_times), dtype=float), + phase=np.linspace(-0.5, 0.5, len(retained_times)), + residuals=residuals, + parameters={ + "tmid": 0.5, + "rprs": 0.1, + "ars": 10.0, + "inc": 89.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + }, + errors={ + "tmid": 0.001, + "rprs": 0.001, + "ars": 0.1, + "inc": 0.1, + "a0": 0.01, + "a1": 0.01, + "a2": 0.01, + }, + ) + return fit, retained_flux, retained_unc + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.fit_final_lightcurve_with_oot_baseline_detrending", fake_final_fit) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 1.0, 38) + result = finalize_comparison_candidate_full_reduction( + times, + np.full(38, 100.0, dtype=float), + np.full(38, 100.0, dtype=float), + np.linspace(1.0, 1.2, 38), + [0.1, 0.1, 0.1, 0.1], + { + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "aRs": 10.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + jd_times=2460000.0 + times, + run_fast_ultranest_before_final_run=False, + run_final_fit_phase_residual_clip=False, + run_final_residual_rejection=True, + use_eebls_to_initialize_tmid_and_bounds=False, + ) + + initially_retained_indices = np.flatnonzero(initial_subset_mask) + expected_source_indices = np.delete(initially_retained_indices, 10) + assert result["applied"] is True + assert [len(call["times"]) for call in fit_calls] == [38, 31] + assert all(call["times"].shape == call["flux"].shape == call["unc"].shape for call in fit_calls) + assert result["source_indices"].tolist() == expected_source_indices.tolist() + assert result["good_times"].tolist() == pytest.approx(times[expected_source_indices].tolist()) + assert all( + len(result[key]) == 31 + for key in ( + "good_times", + "good_flux", + "good_unc", + "good_airmass", + "good_jd_times", + "good_target_flux", + "good_comp_flux", + "good_target_flux_error", + "good_comp_flux_error", + "source_indices", + ) + ) + + def test_detrend_flux_on_out_of_transit_baseline_falls_back_to_prior_ephemeris(): times = np.linspace(-0.08, 0.08, 17) baseline = 1.0 + 0.25 * times diff --git a/tests/test_ld.py b/tests/test_ld.py index 5af579bf..71b5d368 100644 --- a/tests/test_ld.py +++ b/tests/test_ld.py @@ -235,6 +235,7 @@ def test_invalid_fwhm_range_2() -> None: def test_photographic_filter_aliases_in_filter_column() -> None: alias_cases = [ ("pb", "Photographic B", "PB", "391.6", "480.6"), + ("G", "Photographic G", "PG", "502.8", "586.8"), ("pg", "Photographic G", "PG", "502.8", "586.8"), ("pr", "Photographic R", "PR", "590.0", "810.0"), ] diff --git a/tests/test_nonlinear_ld.py b/tests/test_nonlinear_ld.py new file mode 100644 index 00000000..435650a4 --- /dev/null +++ b/tests/test_nonlinear_ld.py @@ -0,0 +1,59 @@ +import pytest + +import exotic.exotic as exotic_module + + +def test_nonlinear_ld_non_interactive_treats_g_as_photographic_g_without_prompt(monkeypatch): + ld = exotic_module.LimbDarkening({}) + monkeypatch.setattr(ld, "calculate_ld", lambda: None) + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("recognized G filter must not prompt"), + ) + info_dict = { + "filter": "G", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": "y", + } + + exotic_module.nonlinear_ld(ld, info_dict, non_interactive_run=True) + + assert info_dict["filter"] == "PG" + assert info_dict["filter_desc"] == "Photographic G" + assert info_dict["wl_min"] == 502.8 + assert info_dict["wl_max"] == 586.8 + + +class UnrecognizedFilterLimbDarkening: + fwhm_names_nonspecific = {} + + @staticmethod + def check_fwhm(_observed_filter): + return False + + @staticmethod + def check_standard(_observed_filter): + return False + + +def test_nonlinear_ld_non_interactive_rejects_unrecognized_filter_without_prompt(monkeypatch): + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("non-interactive limb-darkening selection must not prompt"), + ) + info_dict = { + "filter": "mystery-band", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": "y", + } + + with pytest.raises(ValueError, match="did not recognize the filter 'mystery-band'"): + exotic_module.nonlinear_ld( + UnrecognizedFilterLimbDarkening(), + info_dict, + non_interactive_run=True, + ) diff --git a/tests/test_ultranest_utils.py b/tests/test_ultranest_utils.py index 5c213222..69d4a07d 100644 --- a/tests/test_ultranest_utils.py +++ b/tests/test_ultranest_utils.py @@ -137,6 +137,49 @@ def run(self, **kwargs): assert isinstance(excinfo.value.__cause__, ValueError) +def test_run_reactive_sampler_translates_ultranest_bounding_ellipsoid_assertion(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def run(self, **kwargs): + exec( + compile( + "def bounding_ellipsoid():\n" + " raise AssertionError('(array(nan), array([[0.58720908]]))')\n" + "bounding_ellipsoid()", + "ultranest/mlfriends.pyx", + "exec", + ), + {}, + ) + + with pytest.raises(np.linalg.LinAlgError) as excinfo: + run_reactive_sampler(FakeSampler(), verbose=False) + + assert "degenerate sampling region" in str(excinfo.value) + assert isinstance(excinfo.value.__cause__, AssertionError) + + +def test_run_reactive_sampler_preserves_other_ultranest_mlfriends_assertion(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def run(self, **kwargs): + exec( + compile( + "def compute_enlargement():\n" + " raise AssertionError('not a bounding ellipsoid failure')\n" + "compute_enlargement()", + "ultranest/mlfriends.pyx", + "exec", + ), + {}, + ) + + with pytest.raises(AssertionError, match="not a bounding ellipsoid failure"): + run_reactive_sampler(FakeSampler(), verbose=False) + + def test_run_reactive_sampler_preserves_unrelated_value_error(monkeypatch): _reset_ultranest_env(monkeypatch) From 4fdd4b70e01cf81b059b106a08e792546cbb0b31 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 15 Jul 2026 23:18:51 +1000 Subject: [PATCH 083/116] Add gaussian Tmid facto to ktmf --- exotic/exotic.py | 215 +++++++++++++++++++++++++++++ exotic/output_files.py | 4 + exotic/plots.py | 1 + tests/test_exotic_proper_motion.py | 83 ++++++++++- tests/test_output_files.py | 22 ++- tests/test_plots.py | 5 + 6 files changed, 326 insertions(+), 4 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 89180ea2..f3316245 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -107,6 +107,7 @@ from scipy.optimize import least_squares from scipy.signal import savgol_filter from scipy.ndimage import binary_erosion, gaussian_filter, label as ndimage_label, maximum_filter, median_filter +from scipy.special import ndtri from skimage.registration import phase_cross_correlation from skimage.transform import SimilarityTransform # error handling for scraper @@ -443,10 +444,14 @@ TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT = True TRANSIT_QC_DEVIATION_SIGMA_DEFAULT = 5.0 TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION = 0.05 +TRANSIT_QC_TMID_GAUSSIANITY_MIN_EFFECTIVE_SAMPLES = 200 +TRANSIT_QC_TMID_GAUSSIANITY_BOOTSTRAP_DRAWS = 48 +TRANSIT_QC_TMID_GAUSSIANITY_BOOTSTRAP_MAX_SAMPLES = 2000 TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS = { 'deviation_from_expected_value': 2.0, 'residual_scatter': 0.7, 'residual_flatness': 1.0, + 'tmid_gaussianity': 1.0, 'duration_consistency': 0.75, 'eebls_depth_snr': 1.3, 'sampling': 0.7, @@ -2030,6 +2035,179 @@ def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=T return summary +def _transit_qc_weighted_quantiles(values, probabilities, weights=None): + values = np.asarray(values, dtype=float).reshape(-1) + probabilities = np.asarray(probabilities, dtype=float) + finite_mask = np.isfinite(values) + + finite_weights = None + if weights is not None: + weights = np.asarray(weights, dtype=float).reshape(-1) + if weights.shape == values.shape: + finite_mask &= np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + if finite_weights.size == 0 or np.sum(finite_weights) <= 0: + finite_weights = None + + finite_values = values[finite_mask] + if finite_values.size == 0: + return np.full(probabilities.shape, np.nan, dtype=float) + if finite_weights is None: + return np.nanpercentile(finite_values, 100.0 * probabilities) + + order = np.argsort(finite_values) + sorted_values = finite_values[order] + sorted_weights = finite_weights[order] + cumulative = np.cumsum(sorted_weights) + total = cumulative[-1] + if not np.isfinite(total) or total <= 0: + return np.nanpercentile(finite_values, 100.0 * probabilities) + + cumulative = (cumulative - 0.5 * sorted_weights) / total + cumulative = np.clip(cumulative, 0.0, 1.0) + return np.interp(probabilities, cumulative, sorted_values) + + +def _transit_qc_tmid_gaussianity_shape(values, weights=None): + probabilities = np.linspace(0.02, 0.98, 49) + posterior_quantiles = _transit_qc_weighted_quantiles(values, probabilities, weights) + q16, q50, q84 = _transit_qc_weighted_quantiles(values, [0.16, 0.50, 0.84], weights) + robust_sigma = float((q84 - q16) / 2.0) + minimum_scale = np.finfo(float).eps * max(1.0, abs(float(q50))) + if ( + not np.all(np.isfinite(posterior_quantiles)) + or not np.isfinite(q50) + or not np.isfinite(robust_sigma) + or robust_sigma <= minimum_scale + ): + return np.nan, np.nan, np.nan, robust_sigma + + standardized_quantiles = (posterior_quantiles - q50) / robust_sigma + gaussian_template = ndtri(probabilities) + # A uniform distribution has q84-q16 = 0.68 of its full width, so its + # robust sigma is 0.34 of that width after applying the same scaling. + flat_template = (probabilities - 0.5) / 0.34 + gaussian_distance = float(np.mean((standardized_quantiles - gaussian_template) ** 2)) + flat_distance = float(np.mean((standardized_quantiles - flat_template) ** 2)) + + if not np.isfinite(gaussian_distance) or not np.isfinite(flat_distance): + return np.nan, gaussian_distance, flat_distance, robust_sigma + if flat_distance <= np.finfo(float).eps: + score = 0.0 + else: + score = float(np.clip(1.0 - gaussian_distance / flat_distance, 0.0, 1.0)) + return score, gaussian_distance, flat_distance, robust_sigma + + +def transit_qc_tmid_gaussianity_summary(fit): + summary = { + 'available': False, + 'score': np.nan, + 'score_uncertainty': np.nan, + 'gaussian_distance': np.nan, + 'flat_distance': np.nan, + 'effective_sample_count': 0.0, + 'sample_count': 0, + 'robust_sigma': np.nan, + 'detail': 'Tmid posterior Gaussianity is unavailable.', + } + if fit is None: + return summary + + sampled_keys = getattr(fit, 'sampled_keys', None) + if sampled_keys is not None and 'tmid' not in list(sampled_keys): + summary['detail'] = 'Tmid was fixed rather than sampled; posterior Gaussianity is not scored.' + return summary + + sample_matrix, sample_weights = _fit_posterior_sample_matrix(fit, ['tmid']) + if sample_matrix.size == 0 or sample_matrix.shape[0] == 0: + summary['detail'] = 'Weighted Tmid posterior samples are unavailable.' + return summary + + values = np.asarray(sample_matrix[:, 0], dtype=float).reshape(-1) + finite_mask = np.isfinite(values) + parameter_weights = None + if sample_weights is not None: + sample_weights = np.asarray(sample_weights, dtype=float).reshape(-1) + if sample_weights.shape == values.shape: + finite_mask &= np.isfinite(sample_weights) & (sample_weights >= 0) + parameter_weights = sample_weights[finite_mask] + if parameter_weights.size == 0 or np.sum(parameter_weights) <= 0: + parameter_weights = None + + values = values[finite_mask] + sample_count = int(values.size) + effective_sample_count = _effective_sample_count(parameter_weights, sample_count) + summary['sample_count'] = sample_count + summary['effective_sample_count'] = float(effective_sample_count) + if sample_count < 2: + summary['detail'] = 'Too few finite Tmid posterior samples to assess Gaussianity.' + return summary + if effective_sample_count < TRANSIT_QC_TMID_GAUSSIANITY_MIN_EFFECTIVE_SAMPLES: + summary['detail'] = ( + 'Tmid posterior Gaussianity is not scored because effective samples ' + f'{effective_sample_count:.0f} < {TRANSIT_QC_TMID_GAUSSIANITY_MIN_EFFECTIVE_SAMPLES}.' + ) + return summary + + score, gaussian_distance, flat_distance, robust_sigma = _transit_qc_tmid_gaussianity_shape( + values, + parameter_weights, + ) + summary.update({ + 'score': score, + 'gaussian_distance': gaussian_distance, + 'flat_distance': flat_distance, + 'robust_sigma': robust_sigma, + }) + if not np.isfinite(score): + summary['detail'] = 'Tmid posterior Gaussianity is unavailable because its robust width is degenerate.' + return summary + + bootstrap_size = int(np.clip( + round(effective_sample_count), + TRANSIT_QC_TMID_GAUSSIANITY_MIN_EFFECTIVE_SAMPLES, + TRANSIT_QC_TMID_GAUSSIANITY_BOOTSTRAP_MAX_SAMPLES, + )) + choice_probabilities = None + if parameter_weights is not None: + choice_probabilities = parameter_weights / np.sum(parameter_weights) + rng = np.random.default_rng(24601) + bootstrap_indices = rng.choice( + sample_count, + size=(TRANSIT_QC_TMID_GAUSSIANITY_BOOTSTRAP_DRAWS, bootstrap_size), + replace=True, + p=choice_probabilities, + ) + bootstrap_scores = [] + for indices in bootstrap_indices: + bootstrap_score, _, _, _ = _transit_qc_tmid_gaussianity_shape(values[indices]) + if np.isfinite(bootstrap_score): + bootstrap_scores.append(float(bootstrap_score)) + if len(bootstrap_scores) > 1: + summary['score_uncertainty'] = float(np.std(bootstrap_scores, ddof=1)) + + if score >= 0.85: + interpretation = 'strongly Gaussian-like' + elif score >= 0.60: + interpretation = 'broadly Gaussian-like' + elif score >= 0.20: + interpretation = 'weakly Gaussian-like' + else: + interpretation = 'flat-like or strongly non-Gaussian' + uncertainty = summary['score_uncertainty'] + uncertainty_text = f' +/- {uncertainty:.2f}' if np.isfinite(uncertainty) else '' + summary.update({ + 'available': True, + 'detail': ( + f'{interpretation}; weighted-quantile score={score:.2f}{uncertainty_text}, ' + f'effective samples={effective_sample_count:.0f}, ' + f'Gaussian mismatch={gaussian_distance:.4f}, flat mismatch={flat_distance:.4f}' + ), + }) + return summary + + def compute_transit_qc_ktmf(summary): if not isinstance(summary, dict): return np.nan, [] @@ -2089,6 +2267,12 @@ def compute_transit_qc_ktmf(summary): ) residual_flatness_score = summary.get('residual_flatness_score', np.nan) residual_flatness_detail = summary.get('residual_flatness_detail') or "n/a" + tmid_gaussianity_score = summary.get('tmid_gaussianity_score', np.nan) + tmid_gaussianity_score_uncertainty = summary.get('tmid_gaussianity_score_uncertainty', np.nan) + tmid_gaussianity_detail = ( + summary.get('tmid_gaussianity_detail') + or "Tmid posterior Gaussianity is unavailable." + ) residual_scatter_score_uncertainty = np.nan point_count = summary.get('point_count', np.nan) if ( @@ -2181,6 +2365,13 @@ def compute_transit_qc_ktmf(summary): 'score_uncertainty': np.nan, 'detail': residual_flatness_detail, }, + { + 'key': 'tmid_gaussianity', + 'label': 'Tmid Posterior Gaussianity', + 'score': tmid_gaussianity_score, + 'score_uncertainty': tmid_gaussianity_score_uncertainty, + 'detail': tmid_gaussianity_detail, + }, { 'key': 'duration_consistency', 'label': 'Duration Consistency', @@ -2450,6 +2641,14 @@ def evaluate_transit_detection_qc(fit): 'residual_flatness_scatter_stability_score': np.nan, 'residual_flatness_dominant_metric': None, 'residual_flatness_detail': None, + 'tmid_gaussianity_score': np.nan, + 'tmid_gaussianity_score_uncertainty': np.nan, + 'tmid_gaussianity_gaussian_distance': np.nan, + 'tmid_gaussianity_flat_distance': np.nan, + 'tmid_gaussianity_effective_sample_count': 0.0, + 'tmid_gaussianity_sample_count': 0, + 'tmid_gaussianity_robust_sigma': np.nan, + 'tmid_gaussianity_detail': None, 'sampling_score': np.nan, 'sampling_detail': None, 'sampling_ingress_count': 0, @@ -2667,6 +2866,17 @@ def evaluate_transit_detection_qc(fit): 'sampling_total_duration': sampling_summary.get('total_duration', np.nan), 'sampling_ingress_duration': sampling_summary.get('ingress_duration', np.nan), }) + tmid_gaussianity = transit_qc_tmid_gaussianity_summary(fit) + summary.update({ + 'tmid_gaussianity_score': tmid_gaussianity.get('score', np.nan), + 'tmid_gaussianity_score_uncertainty': tmid_gaussianity.get('score_uncertainty', np.nan), + 'tmid_gaussianity_gaussian_distance': tmid_gaussianity.get('gaussian_distance', np.nan), + 'tmid_gaussianity_flat_distance': tmid_gaussianity.get('flat_distance', np.nan), + 'tmid_gaussianity_effective_sample_count': tmid_gaussianity.get('effective_sample_count', 0.0), + 'tmid_gaussianity_sample_count': tmid_gaussianity.get('sample_count', 0), + 'tmid_gaussianity_robust_sigma': tmid_gaussianity.get('robust_sigma', np.nan), + 'tmid_gaussianity_detail': tmid_gaussianity.get('detail'), + }) deviation_summary = evaluate_transit_qc_expected_value_deviation( fit, deviation_sigma_threshold, @@ -2848,6 +3058,11 @@ def annotate_transit_detection_qc(fit, summary=None): fit.transit_qc_duration_ratio = summary.get('duration_ratio') fit.transit_qc_eebls_depth_snr = summary.get('eebls_depth_snr') fit.transit_qc_residual_scatter = summary.get('residual_scatter') + fit.transit_qc_tmid_gaussianity_score = summary.get('tmid_gaussianity_score') + fit.transit_qc_tmid_gaussianity_score_uncertainty = summary.get( + 'tmid_gaussianity_score_uncertainty' + ) + fit.transit_qc_tmid_gaussianity_detail = summary.get('tmid_gaussianity_detail') fit.transit_qc_deviation_from_expected_value = summary.get('deviation_from_expected_value') fit.transit_qc_deviation_sigma_threshold = summary.get('deviation_sigma_threshold') fit.transit_qc_expected_tmid_value = summary.get('expected_tmid') diff --git a/exotic/output_files.py b/exotic/output_files.py index 24281727..82538561 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -812,6 +812,10 @@ def build_aavso_qc_metadata(fit): 'residual_flatness_trend_score', 'residual_flatness_curve_score', 'residual_flatness_scatter_stability_score', 'residual_flatness_dominant_metric', 'residual_flatness_detail', + 'tmid_gaussianity_score', 'tmid_gaussianity_score_uncertainty', + 'tmid_gaussianity_gaussian_distance', 'tmid_gaussianity_flat_distance', + 'tmid_gaussianity_effective_sample_count', 'tmid_gaussianity_sample_count', + 'tmid_gaussianity_robust_sigma', 'tmid_gaussianity_detail', 'rprs_sigma', 'duration_ratio', 'eebls_depth_snr', 'sampling_score', 'sampling_detail', 'sampling_ingress_count', 'sampling_egress_count', 'sampling_in_transit_count', diff --git a/exotic/plots.py b/exotic/plots.py index d3849e36..a3ae1650 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -1482,6 +1482,7 @@ def _short_ktmf_label(label): "Deviation From Expected Value": "Expected Rp/R*", "Residual Scatter Around Full Model Fit": "Residual scatter", "Residual Flatness": "Residual flatness", + "Tmid Posterior Gaussianity": "Tmid Gaussianity", "Duration Consistency": "Duration", "EEBLS Depth SNR": "EEBLS SNR", "Sampling / Cadence": "Sampling", diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index bcb98624..8a46fa81 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -115,6 +115,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: transit_qc_residual_scatter_score, transit_qc_residual_flatness_summary, transit_qc_sampling_summary, + transit_qc_tmid_gaussianity_summary, apply_comparison_star_suitability_outlier_rejection, comparison_calibration_selection_reason, comparison_candidate_triangle_plot_output_path, @@ -2434,6 +2435,9 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): "residual_scatter_to_depth_ratio": 0.5, "residual_flatness_score": 0.5, "residual_flatness_detail": "curve=0.50", + "tmid_gaussianity_score": 0.8, + "tmid_gaussianity_score_uncertainty": 0.04, + "tmid_gaussianity_detail": "strongly Gaussian-like", "rprs_sigma": 6.0, "duration_ratio": 1.0, "eebls_depth_snr": 8.0, @@ -2445,10 +2449,12 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): assert "Model Evidence" not in contributions_by_label assert "Delta BIC" not in contributions_by_label assert "Delta chi2" not in contributions_by_label - scale = 5.0 / (2.0 + 0.7 + 1.0 + 0.75 + 1.3) + scale = 5.0 / (2.0 + 0.7 + 1.0 + 1.0 + 0.75 + 1.3) assert contributions_by_label["Deviation From Expected Value"]["max_points"] == pytest.approx(2.0 * scale) assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) assert contributions_by_label["Residual Flatness"]["max_points"] == pytest.approx(1.0 * scale) + assert contributions_by_label["Tmid Posterior Gaussianity"]["max_points"] == pytest.approx(1.0 * scale) + assert contributions_by_label["Tmid Posterior Gaussianity"]["score_uncertainty"] == pytest.approx(0.04) assert "Rp/R* Significance" not in contributions_by_label assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75 * scale) assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) @@ -2460,12 +2466,77 @@ def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): 2.0 * 0.6 + 0.7 * 1.0 + 1.0 * 0.5 + + 1.0 * 0.8 + 0.75 * 1.0 + 1.3 * (1.0 - np.exp(-2.0)) ) assert ktmf_metric == pytest.approx(expected_ktmf) +class _TmidPosteriorFit: + def __init__(self, values, weights=None, sampled_keys=("tmid",)): + self.values = np.asarray(values, dtype=float) + self.weights = None if weights is None else np.asarray(weights, dtype=float) + self.sampled_keys = list(sampled_keys) + self.bounds = {"tmid": [float(np.min(self.values)), float(np.max(self.values))]} + + def _get_triangle_plot_samples(self): + return self.values[:, np.newaxis], np.zeros(self.values.size), self.weights + + +def test_tmid_posterior_gaussianity_distinguishes_gaussian_flat_skewed_and_multimodal_shapes(): + rng = np.random.default_rng(20260715) + center = 2460835.82621 + gaussian_values = center + rng.normal(0.0, 0.0015, 5000) + flat_values = np.linspace(center - 0.006, center + 0.006, 5000) + skewed_values = center + 0.002 * (rng.lognormal(-1.0, 0.5, 5000) - 0.42) + multimodal_values = center + np.concatenate([ + rng.normal(-0.003, 0.0005, 2500), + rng.normal(0.003, 0.0005, 2500), + ]) + + gaussian = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(gaussian_values)) + flat = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(flat_values)) + skewed = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(skewed_values)) + multimodal = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(multimodal_values)) + + assert gaussian["available"] is True + assert gaussian["score"] > 0.90 + assert np.isfinite(gaussian["score_uncertainty"]) + assert "strongly Gaussian-like" in gaussian["detail"] + assert flat["score"] < 0.05 + assert skewed["score"] < 0.60 + assert multimodal["score"] < 0.20 + + +def test_tmid_posterior_gaussianity_uses_ultranest_sample_weights(): + rng = np.random.default_rng(717) + center = 2460835.82621 + flat_values = np.linspace(center - 0.01, center + 0.01, 6000) + gaussian_values = center + rng.normal(0.0, 0.001, 2500) + values = np.concatenate([flat_values, gaussian_values]) + weights = np.concatenate([ + np.full(flat_values.size, 1e-8), + np.ones(gaussian_values.size), + ]) + + summary = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(values, weights=weights)) + + assert summary["available"] is True + assert summary["effective_sample_count"] == pytest.approx(2500.0, rel=1e-4) + assert summary["score"] > 0.85 + + +def test_tmid_posterior_gaussianity_is_unavailable_when_tmid_was_fixed(): + summary = transit_qc_tmid_gaussianity_summary( + _TmidPosteriorFit(np.linspace(0.0, 1.0, 500), sampled_keys=()) + ) + + assert summary["available"] is False + assert not np.isfinite(summary["score"]) + assert "fixed rather than sampled" in summary["detail"] + + def test_transit_qc_residual_scatter_score_full_credit_floor_and_zero_ceiling(): transit_depth = 0.02 assert transit_qc_residual_scatter_score(0.0, transit_depth) == pytest.approx(1.0) @@ -2634,7 +2705,7 @@ def test_compute_transit_qc_ktmf_omits_prior_assumed_rprs_component(): assert ktmf_metric == pytest.approx(expected_ktmf) -def test_compute_transit_qc_ktmf_uses_only_residual_and_eebls_for_prior_assumed_geometry(): +def test_compute_transit_qc_ktmf_adds_tmid_gaussianity_for_prior_assumed_geometry(): summary = { "geometry_prior_assumed": True, "geometry_prior_assumed_note": "Transit geometry was fixed to priors.", @@ -2647,6 +2718,9 @@ def test_compute_transit_qc_ktmf_uses_only_residual_and_eebls_for_prior_assumed_ "sampling_score": 1.0, "sampling_detail": "ingress=4, egress=4", "eebls_depth_snr": 8.0, + "tmid_gaussianity_score": 0.75, + "tmid_gaussianity_score_uncertainty": 0.05, + "tmid_gaussianity_detail": "broadly Gaussian-like", } ktmf_metric, contributions = compute_transit_qc_ktmf(summary) @@ -2657,12 +2731,15 @@ def test_compute_transit_qc_ktmf_uses_only_residual_and_eebls_for_prior_assumed_ assert contributions_by_label["Sampling / Cadence"]["available"] is False assert "fixed to priors" in contributions_by_label["Duration Consistency"]["detail"] - scale = 5.0 / (0.7 + 1.3) + scale = 5.0 / (0.7 + 1.0 + 1.3) assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) + assert contributions_by_label["Tmid Posterior Gaussianity"]["max_points"] == pytest.approx(1.0 * scale) + assert contributions_by_label["Tmid Posterior Gaussianity"]["score"] == pytest.approx(0.75) assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) expected_ktmf = scale * ( 0.7 * 1.0 + + 1.0 * 0.75 + 1.3 * (1.0 - np.exp(-2.0)) ) assert ktmf_metric == pytest.approx(expected_ktmf) diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 60ca60b7..3b8ce177 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -12,6 +12,7 @@ AIDOutputFiles, OutputFiles, aavso_dicts, + build_aavso_qc_metadata, fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, save_comp_star_calibration_summary, @@ -1238,6 +1239,10 @@ def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): "ktmf_metric": 4.63, "delta_bic": 18.4, "delta_chi2": 27.1, + "tmid_gaussianity_score": 0.94, + "tmid_gaussianity_score_uncertainty": 0.03, + "tmid_gaussianity_effective_sample_count": 1840.0, + "tmid_gaussianity_detail": "strongly Gaussian-like", "ktmf_contributions": [ { "label": "EEBLS Depth SNR", @@ -1246,7 +1251,16 @@ def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): "max_points": 0.80, "score": 0.93, "detail": "5.80", - } + }, + { + "label": "Tmid Posterior Gaussianity", + "available": True, + "points": 0.94, + "max_points": 1.00, + "score": 0.94, + "score_uncertainty": 0.03, + "detail": "strongly Gaussian-like", + }, ], } (tmp_path / "temp").mkdir() @@ -1321,6 +1335,12 @@ def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): assert "Comp 1" in final_params["KTMF comparison candidate 1"] assert "not selected: KTMF" in final_params["KTMF comparison candidate 1"] assert "Residual Scatter Around Full Model Fit" in final_params["KTMF selected comparison contribution 1"] + assert "Tmid Posterior Gaussianity" in final_params["KTMF target contribution 2"] + + qc_metadata = build_aavso_qc_metadata(fit) + assert qc_metadata["tmid_gaussianity_score"] == pytest.approx(0.94) + assert qc_metadata["tmid_gaussianity_score_uncertainty"] == pytest.approx(0.03) + assert qc_metadata["tmid_gaussianity_effective_sample_count"] == pytest.approx(1840.0) def test_final_planetary_params_reports_absolute_fit_quality(tmp_path): diff --git a/tests/test_plots.py b/tests/test_plots.py index 9d9abc72..8f81d02e 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -10,6 +10,7 @@ from exotic.plots import ( _format_parameter_value, + _short_ktmf_label, plot_fov, plot_adaptive_aperture_diagnostics, plot_comp_star_candidate_lightcurve_fits, @@ -785,3 +786,7 @@ def __init__(self): assert any("0.11 / 0.89" in text for text in captured_text) assert not any("2.00" in text for text in captured_text) assert not any("2.3437" in text for text in captured_text) + + +def test_ktmf_plot_shortens_tmid_posterior_gaussianity_label(): + assert _short_ktmf_label("Tmid Posterior Gaussianity") == "Tmid Gaussianity" From 1de7e83b7bb76f677ac668148ccaed34b002915e Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 16 Jul 2026 21:10:30 +1000 Subject: [PATCH 084/116] glitches, bugs, changes and speedups from the Gauntlet --- exotic/api/filters.py | 15 +- exotic/api/nea.py | 63 +- exotic/exotic.py | 2544 ++++++++++++++++++--- exotic/inputs.py | 5 + exotic/output_files.py | 35 +- exotic/plate_status.py | 191 +- inits.json | 4 +- tests/test_aperture_tuning_performance.py | 143 ++ tests/test_centroid_wcs.py | 112 + tests/test_ephemeris_validation.py | 92 + tests/test_exotic_proper_motion.py | 255 ++- tests/test_inputs.py | 36 +- tests/test_ld.py | 32 +- tests/test_nea_nextastro_fallback.py | 56 +- tests/test_nextastro_variability.py | 375 ++- tests/test_nonlinear_ld.py | 23 + tests/test_plate_status.py | 107 + tests/test_radec_non_interactive.py | 75 + tests/test_runtime_timing.py | 29 + 19 files changed, 3845 insertions(+), 347 deletions(-) create mode 100644 tests/test_aperture_tuning_performance.py create mode 100644 tests/test_ephemeris_validation.py create mode 100644 tests/test_plate_status.py create mode 100644 tests/test_radec_non_interactive.py create mode 100644 tests/test_runtime_timing.py diff --git a/exotic/api/filters.py b/exotic/api/filters.py index 82715ade..f740b098 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -52,8 +52,9 @@ "MObs CV": {"name": "CV", "fwhm": ("350.0", "850.0")}, "ClearV": {"name": "CV", "fwhm": ("350.0", "1000.0")}, - # Astrodon CBB; Source(s): George Silvis; https://astrodon.com/products/astrodon-exo-planet-filter/ - "Astrodon ExoPlanet-BB": {"name": "CBB", "fwhm": ("500.0", "1000.0")}, + # Clear with blue-blocking (CBB); wavelength source: + # https://astrodon.com/products/astrodon-exo-planet-filter/ + "CBB": {"name": "CBB", "fwhm": ("500.0", "1000.0")}, } # expose as fwhm and for convenience set 'desc' field equal to key fwhm = {k: v for k, v in __fwhm.items() if (v.update(desc=k),)} @@ -81,9 +82,10 @@ "Clear (unfiltered) reduced to V sequence": "MObs CV", "Clear (unfiltered) reduced to R sequence": "Cousins R", - "Clear with blue-blocking": "Astrodon ExoPlanet-BB", - "Astrodon-Exo": "Astrodon ExoPlanet-BB", - "Exop": "Astrodon ExoPlanet-BB", + "Clear with blue-blocking": "CBB", + "Astrodon ExoPlanet-BB": "CBB", + "Astrodon-Exo": "CBB", + "Exop": "CBB", # additional short aliases found in FILTER column values "bu": "Johnson U", @@ -115,7 +117,7 @@ "luminosity": "ClearV", "w": "ClearV", "pl": "ClearV", - "exo": "Astrodon ExoPlanet-BB", + "exo": "CBB", # OSC split-channel aliases "b1": "Photographic B", @@ -128,7 +130,6 @@ # standard filters w/o precisely defined FWHM values fwhm_names_nonspecific = { 'CR': "Clear (unfiltered) reduced to R sequence", - 'CBB': "Clear with blue-blocking", 'CV': "Clear (unfiltered) reduced to V sequence", 'TB': "DSLR Blue", 'TG': "DSLR Green", diff --git a/exotic/api/nea.py b/exotic/api/nea.py index 5ba69ede..a48ae20c 100644 --- a/exotic/api/nea.py +++ b/exotic/api/nea.py @@ -64,12 +64,54 @@ def result_if_max_retry_count(retry_state): pass +def _strip_observation_phase_suffix(name): + """Remove scheduler phase labels that are not part of a target name.""" + text = str(name or '').strip() + return re.sub(r'(?:\s+(?:ingress|egress))+\s*$', '', text, flags=re.IGNORECASE).strip() + + +def _collapse_number_planet_letter_spaces(name): + """Keep a trailing single planet letter attached to its numeric identifier.""" + return re.sub(r'(?<=\d)\s+(?=[a-z](?:\s|$))', '', str(name or '').strip()) + + +def planet_name_lookup_candidates(name): + """Return progressively smaller names for tolerant archive matching. + + The exact value is retained first. Observation-phase suffixes are then + removed, spaces between a number and a single planet letter are collapsed, + and finally each contiguous group of remaining space-separated terms is + offered from longest to shortest. + """ + candidates = [] + + def add(value): + value = str(value or '').strip() + if value and value not in candidates: + candidates.append(value) + + original = str(name or '').strip() + add(original) + without_phase = _strip_observation_phase_suffix(original) + add(without_phase) + collapsed = _collapse_number_planet_letter_spaces(without_phase) + add(collapsed) + + parts = collapsed.split() + for width in range(len(parts) - 1, 0, -1): + for start in range(0, len(parts) - width + 1): + add(' '.join(parts[start:start + width])) + + return candidates + + class NASAExoplanetArchive: - def __init__(self, planet=None, candidate=False): + def __init__(self, planet=None, candidate=False, non_interactive=False): self.planet = planet # self.candidate = candidate self.pl_dict = None + self.non_interactive = bool(non_interactive) # CONFIGURATIONS self.requests_timeout = 16, 512 # connection timeout, response timeout in secs. @@ -330,12 +372,12 @@ def _new_scrape(self, filename="eaConf.json"): if os.path.exists('pl_names.json'): with open("pl_names.json", "r") as f: planets = json.load(f) - planet_key = re.sub(r'[^a-zA-Z0-9]', '', self.planet.lower()) - - planet_exists = planets.get(planet_key, False) - - if planet_exists: - self.planet = planet_exists + for candidate_name in planet_name_lookup_candidates(self.planet): + planet_key = re.sub(r'[^a-zA-Z0-9]', '', candidate_name.lower()) + planet_exists = planets.get(planet_key, False) + if planet_exists: + self.planet = planet_exists + break print(f"\nLooking up {self.planet} on the NASA Exoplanet Archive. Please wait....") @@ -359,6 +401,13 @@ def _new_scrape(self, filename="eaConf.json"): f"\nAssuming {self.planet} is a planet candidate because {candidate_reason}.") return self.planet, True + if self.non_interactive: + raise RuntimeError( + f"Non-interactive run cancelled: target ({self.planet}) was not found in the NASA " + "Exoplanet Archive, so archive coordinates are unavailable. Check the Planet Name " + "in the initialization file or provide valid target RA and Dec coordinates." + ) + self.planet = input(f"Cannot find target ({self.planet}) in NASA Exoplanet Archive." f"\nPlease go to https://exoplanetarchive.ipac.caltech.edu to check naming and" "\nre-enter the planet's name or type 'candidate' if this is a planet candidate: ") diff --git a/exotic/exotic.py b/exotic/exotic.py index f3316245..6a15e24d 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -59,6 +59,7 @@ import argparse import csv import copy +from datetime import datetime import faulthandler from functools import lru_cache import inspect @@ -252,11 +253,19 @@ STELLAR_VARIABILITY_ENSEMBLE_DEFAULT = True STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS = 2 STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS = 5 +STELLAR_VARIABILITY_APERTURE_ESTIMATION_MAX_COMPARISONS = 5 STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_SIGMA = 3.0 STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR = 1.0e-4 STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR_FRACTION = 0.05 STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG = 0.01 +STELLAR_VARIABILITY_COMPARISON_GAP_MIN_RATIO = 5.0 +STELLAR_VARIABILITY_COMPARISON_GAP_MIN_SECONDS = 30.0 +STELLAR_VARIABILITY_COMPARISON_GAP_WINDOW_FRAMES = 50 +STELLAR_VARIABILITY_COMPARISON_GAP_MIN_POINTS = 10 +STELLAR_VARIABILITY_COMPARISON_GAP_MAX_STEP_MAG = 0.02 +STELLAR_VARIABILITY_COMPARISON_GAP_MIN_SIGNIFICANCE = 5.0 PHOTOMETER_FORTUITOUS_VARIABLES_DEFAULT = True +USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT = True USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT = False FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR = 0.05 FORTUITOUS_VARIABLE_OPTIMAL_MAX_PERIOD_DAYS = 10.0 @@ -388,6 +397,7 @@ PSF_FIT_SELECTION_MARGIN = 0.35 PSF_ALIGNMENT_TARGET_WIDTH_MAX_COMP_RATIO = 3.0 PSF_ALIGNMENT_CANDIDATE_SELECTION_MARGIN = 0.20 +LEGACY_ALIGNMENT_MAX_DIMENSION = 1600 TIME_REJECTION_RANGE_DISPLAY_LIMIT = 6 TIME_REJECTION_GROUP_GAP_CADENCE_MULTIPLIER = 2.5 NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS = 5 @@ -404,6 +414,7 @@ NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC = 30.0 NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 2.0 CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX = 0.05 +CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX = 0.10 REFERENCE_FALLBACK_COMPARISON_LIMIT = 10 REFERENCE_FALLBACK_DETECTION_MAX_STARS = 60 REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS = 12 @@ -7519,18 +7530,45 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): return fit +def format_clock_log_message(string, clock_time=None): + clock_time = datetime.now() if clock_time is None else clock_time + message = str(string) + leading_newlines = len(message) - len(message.lstrip('\r\n')) + prefix = message[:leading_newlines] + body = message[leading_newlines:] + return f"{prefix}[{clock_time.strftime('%H:%M')}] {body}" + + def log_info(string, warn=False, error=False): + timestamped_string = format_clock_log_message(string) if error: - print(f"\033[31m {string}\033[0m", flush=True) + print(f"\033[31m {timestamped_string}\033[0m", flush=True) elif warn: - print(f"\033[34m {string}\033[0m", flush=True) + print(f"\033[34m {timestamped_string}\033[0m", flush=True) else: - print(string, flush=True) - log.debug(string) + print(timestamped_string, flush=True) + log.debug(timestamped_string) _reset_runtime_traceback_watchdog() return True +class ReductionStageTimer: + def __init__(self, time_source=perf_counter): + self._time_source = time_source + self.started_at = float(time_source()) + self.previous_checkpoint = self.started_at + + def checkpoint(self, label): + now = float(self._time_source()) + elapsed_seconds = max(0.0, now - self.previous_checkpoint) + total_seconds = max(0.0, now - self.started_at) + log_info( + f"STEP TIMING | {label} | elapsed_s={elapsed_seconds:.2f} | total_s={total_seconds:.2f}" + ) + self.previous_checkpoint = now + return elapsed_seconds + + def _find_runtime_handler(handler_name): for handler in log.handlers: if getattr(handler, "_exotic_runtime_handler_name", None) == handler_name: @@ -8231,6 +8269,14 @@ def should_photometer_fortuitous_variables(config_value): ) +def should_use_single_comparison_for_fortuitous_variables(config_value): + return parse_bool_config_value( + config_value, + USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT, + 'use_single_comparison_for_fortuitous_variables', + ) + + def should_use_nextastro_vsx_cache_first(config_value): return parse_bool_config_value( config_value, @@ -8723,6 +8769,19 @@ def ProcessPoolExecutor(*args, **kwargs): return _ProcessPoolExecutor(*args, **kwargs) +def ImageProcessPoolExecutor(*args, **kwargs): + """Use real processes for CPU-bound image work, including on Windows. + + The general EXOTIC executor intentionally retains its Windows thread fallback + for GUI/fitter compatibility. Image alignment workers are module-level, + pickle-safe functions and benefit materially from bypassing the GIL, so use + a spawned process context for that narrower workload. + """ + if sys.platform == "win32": + kwargs.setdefault('mp_context', multiprocessing.get_context('spawn')) + return _ProcessPoolExecutor(*args, **kwargs) + + def _windows_python_spawn_executable(): candidates = [ getattr(sys, "_base_executable", None), @@ -12902,6 +12961,109 @@ def check_parameters(init_parameters, parameters): return False +REQUIRED_TRANSIT_EPHEMERIS_FIELDS = { + 'pPer': { + 'label': 'Orbital Period (days)', + 'uncertainty_key': 'pPerUnc', + }, + 'midT': { + 'label': 'Published Mid-Transit Time (BJD-UTC)', + 'uncertainty_key': 'midTUnc', + }, +} + + +def _positive_finite_ephemeris_value(value): + if isinstance(value, (bool, np.bool_)): + return None + try: + numeric_value = float(value) + except (TypeError, ValueError): + return None + if not np.isfinite(numeric_value) or numeric_value <= 0: + return None + return numeric_value + + +def invalid_required_transit_ephemeris_fields(planet_dict): + if not isinstance(planet_dict, dict): + return list(REQUIRED_TRANSIT_EPHEMERIS_FIELDS) + return [ + key + for key in REQUIRED_TRANSIT_EPHEMERIS_FIELDS + if _positive_finite_ephemeris_value(planet_dict.get(key)) is None + ] + + +def resolve_required_transit_ephemeris(planet_dict, archive_planet_dict=None, archive_lookup=None, + target_name=None): + """Fill missing period/Tmid values from NEA, then fail before reduction if either remains invalid.""" + resolved = dict(planet_dict) if isinstance(planet_dict, dict) else {} + target_name = target_name or resolved.get('pName') + invalid_fields = invalid_required_transit_ephemeris_fields(resolved) + archive_error = None + + if invalid_fields and not isinstance(archive_planet_dict, dict) and callable(archive_lookup): + try: + archive_planet_dict = archive_lookup() + except Exception as exc: + archive_error = exc + + if isinstance(archive_planet_dict, dict): + for key in invalid_fields: + archive_value = _positive_finite_ephemeris_value(archive_planet_dict.get(key)) + if archive_value is None: + continue + + field = REQUIRED_TRANSIT_EPHEMERIS_FIELDS[key] + original_value = resolved.get(key) + resolved[key] = archive_value + log_info( + f"Required ephemeris fallback for {target_name or 'the target'}: " + f"{field['label']} was missing or invalid ({original_value!r}); using NASA Exoplanet " + f"Archive value {archive_value}." + ) + + uncertainty_key = field['uncertainty_key'] + if _positive_finite_ephemeris_value(resolved.get(uncertainty_key)) is None: + archive_uncertainty = _positive_finite_ephemeris_value( + archive_planet_dict.get(uncertainty_key) + ) + if archive_uncertainty is not None: + resolved[uncertainty_key] = archive_uncertainty + + invalid_fields = invalid_required_transit_ephemeris_fields(resolved) + if invalid_fields: + invalid_descriptions = [ + f"{REQUIRED_TRANSIT_EPHEMERIS_FIELDS[key]['label']} ({key})={resolved.get(key)!r}" + for key in invalid_fields + ] + if archive_error is not None: + archive_note = ( + f" NASA Exoplanet Archive fallback failed with " + f"{type(archive_error).__name__}: {archive_error}." + ) + elif isinstance(archive_planet_dict, dict): + archive_note = " The NASA Exoplanet Archive did not provide usable replacement value(s)." + else: + archive_note = " NASA Exoplanet Archive parameters were unavailable." + + message = ( + f"Cannot start EXOTIC reduction for {target_name or 'the target'}: required planetary " + f"ephemeris is missing or invalid: {', '.join(invalid_descriptions)}. Orbital Period and " + f"Published Mid-Transit Time must both be finite numbers greater than zero." + f"{archive_note} Correct the initialization file or archive metadata before rerunning." + ) + log_info(message, error=True) + if archive_error is not None: + raise ValueError(message) from archive_error + raise ValueError(message) + + for key in REQUIRED_TRANSIT_EPHEMERIS_FIELDS: + resolved[key] = _positive_finite_ephemeris_value(resolved[key]) + return resolved + + # --------PLANETARY PARAMETERS UI------------------------------------------ # Get the user's confirmation of values that will later be used in lightcurve fit def get_planetary_parameters(candplanetbool, userpdict, pdict=None): @@ -12943,7 +13105,7 @@ def get_planetary_parameters(candplanetbool, userpdict, pdict=None): userpdict['ra'] = user_input(f"\nEnter the {planet_params[0]}: ", type_=str) if userpdict['dec'] is None: userpdict['dec'] = user_input(f"\nEnter the {planet_params[1]}: ", type_=str) - if type(userpdict['ra']) and type(userpdict['dec']) is str: + if isinstance(userpdict['ra'], str) or isinstance(userpdict['dec'], str): userpdict['ra'], userpdict['dec'] = radec_hours_to_degree(userpdict['ra'], userpdict['dec']) radeclist = ['ra', 'dec'] @@ -12972,7 +13134,7 @@ def get_planetary_parameters(candplanetbool, userpdict, pdict=None): userpdict['dec'] = user_input(f"Enter the {planet_params[1]}: ", type_=str) break - if type(userpdict['ra']) and type(userpdict['dec']) is str: + if isinstance(userpdict['ra'], str) or isinstance(userpdict['dec'], str): userpdict['ra'], userpdict['dec'] = radec_hours_to_degree(userpdict['ra'], userpdict['dec']) # Exoplanet confirmed in NASA Exoplanet Archive @@ -13037,27 +13199,69 @@ def get_planetary_parameters(candplanetbool, userpdict, pdict=None): # Conversion of Right Ascension and Declination: hours -> degrees -def radec_hours_to_degree(ra, dec): +def radec_hours_to_degree(ra, dec, non_interactive_run=False, archive_ra=None, archive_dec=None, + target_name=None): + def parse_coordinates(ra_input, dec_input): + ra_value = str(ra_input).strip() + dec_value = str(dec_input).strip() + + # Accept either sexagesimal RA strings (HH:MM:SS) or decimal RA degrees. + # A decimal-like value with no separators should be treated as degrees. + ra_unit = u.hourangle if any(sep in ra_value for sep in (':', ' ')) else u.deg + + # Declination can be provided as either sexagesimal or decimal degrees. + dec_unit = u.deg + if any(sep in dec_value for sep in (':', ' ')): + dec_value = dec_value.replace(':', ' ') + + if ra_unit is u.hourangle: + ra_value = ra_value.replace(':', ' ') + + coordinates = SkyCoord(ra=ra_value, dec=dec_value, unit=(ra_unit, dec_unit)) + ra_degrees = float(coordinates.ra.degree) + dec_degrees = float(coordinates.dec.degree) + if not np.isfinite(ra_degrees) or not np.isfinite(dec_degrees): + raise ValueError("RA and Dec must both be finite values") + return ra_degrees, dec_degrees + while True: try: - ra_value = str(ra).strip() - dec_value = str(dec).strip() - - # Accept either sexagesimal RA strings (HH:MM:SS) or decimal RA degrees. - # A decimal-like value with no separators should be treated as degrees. - ra_unit = u.hourangle if any(sep in ra_value for sep in (':', ' ')) else u.deg + return parse_coordinates(ra, dec) + except (TypeError, ValueError) as input_error: + if non_interactive_run: + target_description = f" for target {target_name}" if target_name else "" + archive_coordinates_supplied = archive_ra is not None or archive_dec is not None + if archive_ra is not None and archive_dec is not None: + try: + fallback_ra, fallback_dec = parse_coordinates(archive_ra, archive_dec) + except (TypeError, ValueError) as archive_error: + raise ValueError( + f"Non-interactive run cancelled{target_description}: initialization-file RA={ra!r} " + f"and Dec={dec!r} are invalid ({input_error}), and the NASA Exoplanet Archive " + f"coordinates RA={archive_ra!r} and Dec={archive_dec!r} are also unusable " + f"({archive_error}). Provide valid target coordinates in the initialization file." + ) from archive_error - # Declination can be provided as either sexagesimal or decimal degrees. - dec_unit = u.deg - if any(sep in dec_value for sep in (':', ' ')): - dec_value = dec_value.replace(':', ' ') + log_info( + f"Warning: initialization-file RA={ra!r} and Dec={dec!r} are invalid" + f"{target_description} ({input_error}). Using NASA Exoplanet Archive coordinates " + f"RA={fallback_ra:.8f} deg, Dec={fallback_dec:.8f} deg instead.", + warn=True, + ) + return fallback_ra, fallback_dec - if ra_unit is u.hourangle: - ra_value = ra_value.replace(':', ' ') + archive_reason = ( + f"the NASA Exoplanet Archive returned incomplete coordinates " + f"(RA={archive_ra!r}, Dec={archive_dec!r})" + if archive_coordinates_supplied + else "NASA Exoplanet Archive coordinates are unavailable" + ) + raise ValueError( + f"Non-interactive run cancelled{target_description}: initialization-file RA={ra!r} " + f"and Dec={dec!r} are invalid ({input_error}), and {archive_reason}. Provide valid target " + "coordinates in the initialization file." + ) from input_error - c = SkyCoord(ra=ra_value, dec=dec_value, unit=(ra_unit, dec_unit)) - return c.ra.degree, c.dec.degree - except ValueError: log_info("Error: The format entered for Right Ascension and/or Declination is not correct, " "please try again.", error=True) ra = input("Input the Right Ascension of target (HH:MM:SS): ") @@ -13551,7 +13755,7 @@ def collect_transform_frame_pointings(inputfiles, frame_loader=None, return_tran if getattr(image_data, "ndim", 0) != 2: continue - tform = transformation( + tform = downsampled_fallback_transformation( image_data, file_name, report_failure=False, @@ -15026,7 +15230,8 @@ def normalize_nextastro_filter_key(obs_filter): return re.sub(r"[^a-z0-9]", "", str(obs_filter or "").lower()) -def nextastro_photometry_band_candidates(obs_filter): +def nextastro_photometry_band_candidates(obs_filter, include_fallback=True): + raw_filter = str(obs_filter or '').strip() filter_key = normalize_nextastro_filter_key(obs_filter) direct_map = { 'u': [('umag', 'err_umag', 'u')], @@ -15052,11 +15257,8 @@ def nextastro_photometry_band_candidates(obs_filter): 'sg': [('g', 'dg', 'g')], 'sloang': [('g', 'dg', 'g')], 'sdssg': [('g', 'dg', 'g')], - 'photographicg': [('g', 'dg', 'g')], 'gp': [('g', 'dg', 'g')], 'g': [('g', 'dg', 'g')], - 'pg': [('g', 'dg', 'g')], - 'tg': [('g', 'dg', 'g')], 'sr': [('r', 'dr', 'r')], 'sloanr': [('r', 'dr', 'r')], 'sdssr': [('r', 'dr', 'r')], @@ -15098,8 +15300,13 @@ def nextastro_photometry_band_candidates(obs_filter): ('z', 'dz', 'z'), ('umag', 'err_umag', 'u'), ] - candidates = list(direct_map.get(filter_key, [])) - candidates.extend(candidate for candidate in fallback if candidate not in candidates) + # EXOTIC's exact uppercase ``G`` means Photographic G. It is not the + # NextAstro catalogue's Sloan-like ``g`` column, nor Gaia ``G`` + # (``phot_g_mean_mag``). Preserve case here because the normalized key + # intentionally cannot distinguish G from g. + candidates = [] if raw_filter == 'G' else list(direct_map.get(filter_key, [])) + if include_fallback: + candidates.extend(candidate for candidate in fallback if candidate not in candidates) return candidates @@ -15173,22 +15380,36 @@ def nextastro_catalog_rows(catalog_response): def row_nextastro_magnitude(row, band_candidates, max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX): + base_error_limit = _finite_float(max_error) + if base_error_limit is None: + base_error_limit = CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX for priority, (mag_column, error_column, band_label) in enumerate(band_candidates): + allow_bv_error_fallback = ( + str(band_label).strip().upper() in {'B', 'V'} + and base_error_limit == CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + ) + effective_error_limit = ( + CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX + if allow_bv_error_fallback + else base_error_limit + ) usable_magnitude = usable_catalog_reference_magnitude( row.get(mag_column), row.get(error_column), - max_error=max_error, + max_error=effective_error_limit, ) if usable_magnitude is None: continue magnitude, magnitude_error = usable_magnitude return { 'priority': priority, + 'magnitude_error_tier': int(magnitude_error > base_error_limit), 'mag': magnitude, 'error': magnitude_error, 'mag_band': band_label, 'mag_column': mag_column, 'mag_error_column': error_column, + 'uses_relaxed_bv_error_limit': bool(magnitude_error > base_error_limit), } return None @@ -15244,7 +15465,11 @@ def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, if not matches: return None - matches.sort(key=lambda match: (match['priority'], match['separation_arcsec'])) + matches.sort(key=lambda match: ( + match['priority'], + match.get('magnitude_error_tier', 0), + match['separation_arcsec'], + )) return matches[0] @@ -15995,7 +16220,10 @@ def select_automatic_optimal_calibration_stars( obs_filter, ) color = nextastro_catalog_color((match or {}).get('catalog_row'), obs_filter) - if match is None: + if ( + match is None + or catalog_band_priority(match.get('mag_band'), obs_filter) != 0 + ): continue color_delta = ( abs(color['color'] - target_color['color']) @@ -16192,6 +16420,14 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist warn=True, ) continue + if catalog_band_priority(match.get('mag_band'), obs_filter) != 0: + log_info( + "Warning: rejecting NextAstro photometry calibration for comparison star " + f"#{index + 1} because catalog band {match.get('mag_band')!r} does not match " + f"observed filter {obs_filter!r}.", + warn=True, + ) + continue catalog_identity = calibration_catalog_identity(match) if catalog_identity is not None and catalog_identity in existing_catalog_identities: @@ -16240,6 +16476,14 @@ def nextastro_prereduced_calibration_star(phot_comp_star, obs_filter): match = nextastro_photometry_for_coordinate(comp_ra, comp_dec, obs_filter) if match is None: return None, None + if catalog_band_priority(match.get('mag_band'), obs_filter) != 0: + log_info( + "Warning: rejecting NextAstro photometry calibration for the pre-reduced " + f"comparison star because catalog band {match.get('mag_band')!r} does not match " + f"observed filter {obs_filter!r}.", + warn=True, + ) + return None, None match.update({ 'ra': comp_ra, @@ -16708,6 +16952,42 @@ def transformation(image_data, file_name, roi=1, report_failure=True, reference_ plateStatus.alignmentError() return SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + +def downsampled_fallback_transformation(image_data, file_name, report_failure=True, reference_image=None, + max_dimension=LEGACY_ALIGNMENT_MAX_DIMENSION): + """Run the legacy image transform on a reduced image and return full-resolution coordinates.""" + if reference_image is None: + current_image = np.asarray(image_data[0]) + reference_image = np.asarray(image_data[1]) + else: + current_image = np.asarray(image_data) + reference_image = np.asarray(reference_image) + + largest_dimension = max(current_image.shape[:2] + reference_image.shape[:2]) + max_dimension = max(1, int(max_dimension)) + downsample_factor = max(1, int(np.ceil(float(largest_dimension) / max_dimension))) + if downsample_factor == 1: + return transformation( + current_image, + file_name, + report_failure=report_failure, + reference_image=reference_image, + ) + + reduced_current = current_image[::downsample_factor, ::downsample_factor] + reduced_reference = reference_image[::downsample_factor, ::downsample_factor] + reduced_tform = transformation( + reduced_current, + file_name, + report_failure=report_failure, + reference_image=reduced_reference, + ) + return SimilarityTransform( + scale=float(reduced_tform.scale), + rotation=float(reduced_tform.rotation), + translation=np.asarray(reduced_tform.translation, dtype=float) * downsample_factor, + ) + def load_image_data(file_name): hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) extension = 0 @@ -17108,7 +17388,12 @@ def transformation_task(i, file_name, reference_file): reference_image = load_image_data(reference_file) # Multiprocess pre-computation should not emit plate-status warnings; the # serial reduction path decides whether the fallback transform is needed. - return i, transformation(image_data, file_name, report_failure=False, reference_image=reference_image) + return i, downsampled_fallback_transformation( + image_data, + file_name, + report_failure=False, + reference_image=reference_image, + ) _TRANSFORM_REFERENCE_IMAGE = None @@ -17163,7 +17448,12 @@ def _transformation_pool_initializer(reference_file): def transformation_task_with_cached_reference(i, file_name): image_data = load_image_data(file_name) - return i, transformation(image_data, file_name, report_failure=False, reference_image=_TRANSFORM_REFERENCE_IMAGE) + return i, downsampled_fallback_transformation( + image_data, + file_name, + report_failure=False, + reference_image=_TRANSFORM_REFERENCE_IMAGE, + ) class _ParallelPlateStatusRecorder: @@ -17199,47 +17489,87 @@ def _alignment_pool_initializer(reference_file, generalDark, generalBias, genera 'demosaic_out': demosaic_out, 'demosaic_mult': demosaic_mult, 'bad_pixel_reference': bad_pixel_reference, + 'reference_file': str(reference_file), } - _TRANSFORM_REFERENCE_IMAGE = load_calibrated_reduction_image( - reference_file, - generalDark, - generalBias, - generalFlat, - demosaic_fmt, - demosaic_out, - demosaic_mult, - bad_pixel_reference=bad_pixel_reference, - ) + # WCS-first jobs do not need the large reference image. Load it lazily only + # inside a worker that is actually assigned a legacy alignment fallback. + _TRANSFORM_REFERENCE_IMAGE = None _TRANSFORM_REFERENCE_CACHE = None +def _alignment_worker_reference_image(): + global _TRANSFORM_REFERENCE_IMAGE + if _TRANSFORM_REFERENCE_IMAGE is None: + context = _ALIGNMENT_POOL_CONTEXT + _TRANSFORM_REFERENCE_IMAGE = load_calibrated_reduction_image( + context['reference_file'], + context.get('generalDark'), + context.get('generalBias'), + context.get('generalFlat'), + context.get('demosaic_fmt'), + context.get('demosaic_out'), + context.get('demosaic_mult'), + bad_pixel_reference=context.get('bad_pixel_reference'), + ) + return _TRANSFORM_REFERENCE_IMAGE + + def _load_alignment_worker_frame(file_name): context = _ALIGNMENT_POOL_CONTEXT - hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + use_memmap = can_memmap_aperture_tuning_cutouts( + generalDark=context.get('generalDark'), + generalBias=context.get('generalBias'), + generalFlat=context.get('generalFlat'), + demosaic_fmt=context.get('demosaic_fmt'), + bad_pixel_reference=context.get('bad_pixel_reference'), + ) + hdul = fits.open( + name=file_name, + memmap=use_memmap, + cache=False, + lazy_load_hdus=use_memmap, + ignore_missing_end=True, + ) extension = 0 image_header = hdul[extension].header while image_header["NAXIS"] == 0: extension += 1 image_header = hdul[extension].header + if use_memmap and not fits_header_supports_memmap(image_header): + hdul.close() + hdul = fits.open( + name=file_name, + memmap=False, + cache=False, + lazy_load_hdus=False, + ignore_missing_end=True, + ) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + use_memmap = False image_data = hdul[extension].data hdul.close() - image_data = apply_cals( - image_data, - context.get('generalDark'), - context.get('generalBias'), - context.get('generalFlat'), - 1, - ) - image_data = demosaic_img( - image_data, - context.get('demosaic_fmt'), - context.get('demosaic_out'), - context.get('demosaic_mult'), - 1, - ) - image_data = repair_bad_pixels_in_frame(image_data, context.get('bad_pixel_reference')) + if not use_memmap: + image_data = apply_cals( + image_data, + context.get('generalDark'), + context.get('generalBias'), + context.get('generalFlat'), + 1, + ) + image_data = demosaic_img( + image_data, + context.get('demosaic_fmt'), + context.get('demosaic_out'), + context.get('demosaic_mult'), + 1, + ) + image_data = repair_bad_pixels_in_frame(image_data, context.get('bad_pixel_reference')) return image_header, image_data @@ -17256,11 +17586,11 @@ def _pointing_precheck_alignment_task(task): 'transform': None, } - tform = transformation( + tform = downsampled_fallback_transformation( image_data, file_name, report_failure=False, - reference_image=_TRANSFORM_REFERENCE_IMAGE, + reference_image=_alignment_worker_reference_image(), ) mapped_anchor = np.asarray(tform(reference_anchor), dtype=float).reshape(-1, 2)[0] usable = bool(np.all(np.isfinite(mapped_anchor))) @@ -17433,17 +17763,17 @@ def _parallel_alignment_task(task): except Exception as exc: result['wcs_error'] = str(exc) - if precomputed_fallback_transform is not None or compute_fallback_transform or result['wcs'] is None: + if precomputed_fallback_transform is not None or compute_fallback_transform: if precomputed_fallback_transform is not None: tform = precomputed_fallback_transform elif i == 0: tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) else: - tform = transformation( + tform = downsampled_fallback_transformation( image_data, file_name, report_failure=False, - reference_image=_TRANSFORM_REFERENCE_IMAGE, + reference_image=_alignment_worker_reference_image(), ) transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) result['fallback'] = _fit_alignment_candidate_psfs( @@ -17698,6 +18028,112 @@ def apply_parallel_alignment_result(result, frame_index, psf_data, tar_comp_dist return selected_source +def wcs_alignment_candidate_is_acceptable(result, frame_index, psf_data, tar_comp_dist, comp_keys): + if not isinstance(result, dict) or result.get('wcs') is None: + return False + + _, _, diagnostics = select_alignment_candidate( + result, + frame_index, + psf_data, + tar_comp_dist, + comp_keys, + ) + return bool( + diagnostics['wcs_decision'].get('use_wcs_alignment') + and np.isfinite(diagnostics['wcs_score']) + ) + + +def classify_wcs_fallback_frames(results, target_and_comp_pixels): + """Return missing and rejected WCS frame indices without running legacy alignment.""" + target_and_comp_pixels = np.asarray(target_and_comp_pixels, dtype=float).reshape(-1, 2) + comp_keys = [f"comp{comp_idx + 1}" for comp_idx in range(max(0, len(target_and_comp_pixels) - 1))] + psf_data = {'target': np.zeros((len(results), 7), dtype=float)} + tar_comp_dist = {} + for comp_idx, comp_key in enumerate(comp_keys): + psf_data[comp_key] = np.zeros((len(results), 7), dtype=float) + tar_comp_dist[comp_key] = np.abs( + target_and_comp_pixels[comp_idx + 1] - target_and_comp_pixels[0] + ) + + missing_wcs_indices = [] + rejected_wcs_indices = [] + for frame_index, result in enumerate(results): + if result is None or result.get('wcs') is None: + missing_wcs_indices.append(frame_index) + elif wcs_alignment_candidate_is_acceptable( + result, + frame_index, + psf_data, + tar_comp_dist, + comp_keys, + ): + _store_alignment_candidate_psfs(result['wcs'], frame_index, psf_data, comp_keys) + if frame_index == 0: + _update_reference_comp_offsets(psf_data, tar_comp_dist, comp_keys) + continue + else: + rejected_wcs_indices.append(frame_index) + + # Keep the last accepted solution as the continuity reference while the + # rejected frame waits for its fallback result. + if frame_index > 0: + psf_data['target'][frame_index] = psf_data['target'][frame_index - 1] + for comp_key in comp_keys: + psf_data[comp_key][frame_index] = psf_data[comp_key][frame_index - 1] + + return missing_wcs_indices, rejected_wcs_indices + + +def _run_multiprocess_alignment_task_batch(tasks, max_processes, reference_file, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None, + bad_pixel_reference=None, progress_label='alignment'): + total_jobs = len(tasks) + if total_jobs == 0: + return {} + + batch_start = perf_counter() + max_workers = min(max_processes, os.cpu_count() or 1, total_jobs, MAX_MULTIPROCESS_TRANSFORM_WORKERS) + log_info( + f"Using multiprocessing for {progress_label} " + f"with {max_workers} worker(s) across {total_jobs} image(s)." + ) + + results = {} + with suppress_tk_cleanup_during_process_pool(): + with ImageProcessPoolExecutor( + max_workers=max_workers, + initializer=_alignment_pool_initializer, + initargs=( + str(reference_file), + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference, + ), + ) as executor: + futures = [executor.submit(_parallel_alignment_task, task) for task in tasks] + completed = 0 + for future in as_completed(futures): + result = future.result() + results[result['index']] = result + completed += 1 + if completed == total_jobs or completed % 10 == 0: + log_info(f"Multiprocessing {progress_label} progress: {completed}/{total_jobs}") + + elapsed_seconds = perf_counter() - batch_start + log_info( + f"Multiprocessing {progress_label} completed {total_jobs} image(s) in " + f"{elapsed_seconds:.2f}s ({1000.0 * elapsed_seconds / total_jobs:.1f} ms/image wall time)." + ) + return results + + def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_comp_pixels, target_and_comp_radec=None, ignore_header_wcs=False, generalDark=None, generalBias=None, generalFlat=None, @@ -17710,16 +18146,61 @@ def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_c if total_jobs == 0: return [] - max_workers = min(max_processes, os.cpu_count() or 1, total_jobs, MAX_MULTIPROCESS_TRANSFORM_WORKERS) - results = [None] * total_jobs + wcs_tasks = [] + for i, file_name in enumerate(inputfiles): + frame_fast_centroid = should_use_fast_centroid(i) if use_fast_centroid_cadence else False + target_fast_centroid = ( + should_use_fast_target_centroid(i, adaptive_apertures=use_adaptive_apertures) + if use_fast_centroid_cadence else False + ) + wcs_tasks.append(( + i, + str(file_name), + target_and_comp_pixels, + target_and_comp_radec, + ignore_header_wcs, + target_fast_centroid, + frame_fast_centroid, + False, + first_frame_uses_input_comp_pixels, + None, + )) - log_info( - "Using multiprocessing for alignment " - f"with {max_workers} worker(s) across {total_jobs} image(s)." + wcs_results_by_index = _run_multiprocess_alignment_task_batch( + wcs_tasks, + max_processes, + inputfiles[0], + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + progress_label='WCS-first alignment', ) + results = [wcs_results_by_index.get(i) for i in range(total_jobs)] + if not compute_fallback_transform: + return results - tasks = [] - for i, file_name in enumerate(inputfiles): + missing_wcs_indices, rejected_wcs_indices = classify_wcs_fallback_frames( + results, + target_and_comp_pixels, + ) + fallback_indices = sorted(missing_wcs_indices + rejected_wcs_indices) + log_info( + f"WCS-first alignment accepted {total_jobs - len(fallback_indices)}/{total_jobs} frame(s); " + f"queued {len(fallback_indices)} legacy fallback(s) " + f"(missing WCS: {len(missing_wcs_indices)}, rejected WCS: {len(rejected_wcs_indices)}). " + f"Legacy fallback images are downsampled to at most {LEGACY_ALIGNMENT_MAX_DIMENSION} pixels " + "on their longest side." + ) + if not fallback_indices: + return results + + fallback_tasks = [] + for i in fallback_indices: + file_name = inputfiles[i] precomputed_fallback_transform = None if precomputed_fallback_transforms: precomputed_fallback_transform = precomputed_fallback_transforms.get(str(file_name)) @@ -17728,41 +18209,36 @@ def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_c should_use_fast_target_centroid(i, adaptive_apertures=use_adaptive_apertures) if use_fast_centroid_cadence else False ) - tasks.append(( + fallback_tasks.append(( i, str(file_name), target_and_comp_pixels, target_and_comp_radec, - ignore_header_wcs, + True, target_fast_centroid, frame_fast_centroid, - compute_fallback_transform, + True, first_frame_uses_input_comp_pixels, precomputed_fallback_transform, )) - with ProcessPoolExecutor( - max_workers=max_workers, - initializer=_alignment_pool_initializer, - initargs=( - str(inputfiles[0]), - generalDark, - generalBias, - generalFlat, - demosaic_fmt, - demosaic_out, - demosaic_mult, - bad_pixel_reference, - ), - ) as executor: - futures = [executor.submit(_parallel_alignment_task, task) for task in tasks] - completed = 0 - for future in as_completed(futures): - result = future.result() - results[result['index']] = result - completed += 1 - if completed == total_jobs or completed % 10 == 0: - log_info(f"Multiprocessing alignment progress: {completed}/{total_jobs}") + fallback_results = _run_multiprocess_alignment_task_batch( + fallback_tasks, + max_processes, + inputfiles[0], + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + progress_label='legacy alignment fallback', + ) + for i in fallback_indices: + fallback_result = fallback_results.get(i) + if fallback_result is not None: + results[i]['fallback'] = fallback_result.get('fallback') return results @@ -17802,7 +18278,7 @@ def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_c APERTURE_AUTOTUNE_APER_HALF_WIDTH_SIGMA = 0.9 APERTURE_AUTOTUNE_ANNULUS_HALF_WIDTH_SIGMA = 2.0 APERTURE_AUTOTUNE_MIN_FRAMES = 8 -APERTURE_AUTOTUNE_MAX_FRAMES = 12 +APERTURE_AUTOTUNE_MAX_FRAMES = 24 # Refit full PSF moments periodically; use a faster moment estimator for most frames. CENTROID_FULL_FIT_CADENCE = 6 @@ -20086,6 +20562,12 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ comp_mag_error = normalized_magnitude_error(comp_star.get('error')) derived_catalog_reference = bool(comp_star.get('derived_catalog_reference', False)) allow_high_error_catalog_reference = bool(comp_star.get('allow_high_error_catalog_reference', False)) + allow_relaxed_bv_error = ( + bool(comp_star.get('uses_relaxed_bv_error_limit', False)) + and str(comp_star.get('mag_band') or '').strip().upper() in {'B', 'V'} + and comp_mag_error is not None + and comp_mag_error <= CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX + ) if ( comp_mag is None or comp_mag_error is None @@ -20093,11 +20575,18 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ comp_mag_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX and not derived_catalog_reference and not allow_high_error_catalog_reference + and not allow_relaxed_bv_error ) or not is_usable_apparent_magnitude(comp_mag) ): raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") observed_filter = observed_filter or comp_star.get('observed_filter') + if catalog_band_priority(comp_star.get('mag_band'), observed_filter) != 0: + raise RuntimeError( + "Comparison-star catalog magnitude band " + f"{comp_star.get('mag_band')!r} does not match observed filter " + f"{observed_filter!r}; cross-band absolute calibration is not permitted." + ) fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) fit_airmass_model = np.asarray( @@ -20105,7 +20594,9 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ dtype=float, ) fit_airmass = np.asarray(getattr(lc_fit, 'airmass', np.ones_like(fit_data)), dtype=float) - fit_times = np.asarray(getattr(lc_fit, 'jd_times', getattr(lc_fit, 'time', [])), dtype=float) + # Public magnitude products are labelled BJD_TDB; ``time`` is the + # barycentric series and ``jd_times`` retains the original FITS JD/UTC. + fit_times = np.asarray(getattr(lc_fit, 'time', getattr(lc_fit, 'jd_times', [])), dtype=float) transit_model = np.asarray(getattr(lc_fit, 'transit', np.ones_like(fit_data)), dtype=float) if not (fit_data.shape == fit_airmass_model.shape == fit_airmass.shape == fit_times.shape): @@ -20235,7 +20726,7 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ def stellar_variability_reference_series(lc_fit): fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) - fit_times = np.asarray(getattr(lc_fit, 'jd_times', getattr(lc_fit, 'time', [])), dtype=float) + fit_times = np.asarray(getattr(lc_fit, 'time', getattr(lc_fit, 'jd_times', [])), dtype=float) transit_model = np.asarray(getattr(lc_fit, 'transit', np.ones_like(fit_data)), dtype=float) if fit_data.shape != fit_times.shape: @@ -20374,7 +20865,7 @@ def combine_catalog_reference_estimates(derived_estimates, selected_pos, observe def preferred_catalog_magnitude_band_for_filter(observed_filter): - candidates = nextastro_photometry_band_candidates(observed_filter) + candidates = nextastro_photometry_band_candidates(observed_filter, include_fallback=False) if not candidates: return None return candidates[0][2] @@ -20385,7 +20876,7 @@ def catalog_band_priority(mag_band, observed_filter): return 0 preferred_band = preferred_catalog_magnitude_band_for_filter(observed_filter) if preferred_band is None: - return 0 + return 1 return 0 if str(mag_band or '').strip().lower() == str(preferred_band).strip().lower() else 1 @@ -21119,22 +21610,33 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet use_multiprocess_transform_precompute, ) - cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None - if cached_tform is not None: - tform = cached_tform - elif i == 0: - tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) - else: - tform = transformation(imageData, fileName, reference_image=firstImage) + if not wcs_alignment_candidate_is_acceptable( + alignment_result, + i, + psf_data, + tar_comp_dist, + ['comp'], + ): + cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None + if cached_tform is not None: + tform = cached_tform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + else: + tform = downsampled_fallback_transformation( + imageData, + fileName, + reference_image=firstImage, + ) - transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) - alignment_result['fallback'] = _fit_alignment_candidate_psfs( - imageData, - transformed_coords, - target_fast_centroid, - frame_fast_centroid, - previous_psf_rows=previous_psf_rows, - ) + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + alignment_result['fallback'] = _fit_alignment_candidate_psfs( + imageData, + transformed_coords, + target_fast_centroid, + frame_fast_centroid, + previous_psf_rows=previous_psf_rows, + ) apply_parallel_alignment_result( alignment_result, i, @@ -22722,19 +23224,88 @@ def merge_automatic_comparison_star_coords(primary_stars, additional_stars, return merged, messages -def format_comp_star_coverage_text(summary): - coverage_text = ( - f"{summary['coverage_count']} valid frame(s)" - f" out of {summary.get('coverage_total_frame_count', 'n/a')} total" - f"; min_required={summary.get('coverage_min_required_count', 0)}" - ) - coverage_median = summary.get('coverage_reference_count', np.nan) - if np.isfinite(coverage_median): - coverage_text += f"; peer_median={coverage_median:.1f}" - return coverage_text - - -def format_comp_star_coverage_rejection_detail(summary): +def fortuitous_variable_overlap(position, fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + """Return the closest full-field VSX variable matching a tracked pixel position.""" + try: + candidate = np.asarray(position, dtype=float).reshape(-1) + if candidate.size < 2 or not np.all(np.isfinite(candidate[:2])): + return None + duplicate_radius = max(float(duplicate_radius_pixels), 0.0) + except (TypeError, ValueError): + return None + + closest_match = None + for variable in fortuitous_variables or []: + try: + variable_position = np.asarray( + variable.get('pos', [variable.get('x'), variable.get('y')]), + dtype=float, + ).reshape(-1) + if variable_position.size < 2 or not np.all(np.isfinite(variable_position[:2])): + continue + except (AttributeError, TypeError, ValueError): + continue + + distance = float(np.hypot( + candidate[0] - variable_position[0], + candidate[1] - variable_position[1], + )) + if distance > duplicate_radius: + continue + if closest_match is None or distance < closest_match['distance_pixels']: + closest_match = { + 'variable': variable, + 'variable_name': str(variable.get('name') or 'unnamed VSX variable'), + 'variable_position': [float(variable_position[0]), float(variable_position[1])], + 'distance_pixels': distance, + } + return closest_match + + +def filter_comparison_stars_against_fortuitous_variables( + comparison_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + """Remove every science comparison later identified by the full-field VSX search.""" + retained = [] + rejected = [] + for index, position in enumerate(comparison_stars or []): + try: + normalized_position = [float(position[0]), float(position[1])] + except (TypeError, ValueError, IndexError): + continue + + overlap = fortuitous_variable_overlap( + normalized_position, + fortuitous_variables, + duplicate_radius_pixels=duplicate_radius_pixels, + ) + if overlap is None: + retained.append(normalized_position) + continue + + rejected.append({ + 'comparison_index': index, + 'position': normalized_position, + **overlap, + }) + return retained, rejected + + +def format_comp_star_coverage_text(summary): + coverage_text = ( + f"{summary['coverage_count']} valid frame(s)" + f" out of {summary.get('coverage_total_frame_count', 'n/a')} total" + f"; min_required={summary.get('coverage_min_required_count', 0)}" + ) + coverage_median = summary.get('coverage_reference_count', np.nan) + if np.isfinite(coverage_median): + coverage_text += f"; peer_median={coverage_median:.1f}" + return coverage_text + + +def format_comp_star_coverage_rejection_detail(summary): coverage_count = int(summary.get('coverage_count', 0) or 0) threshold_values = [] for threshold in ( @@ -24873,6 +25444,62 @@ def initialize_aperture_data_store(frame_count, aperture_count, annulus_count, c return aper_data +def aperture_estimation_comparison_stars(science_comp_stars, stellar_variability_only=False): + """Choose from comparison stars after the caller's VSX-variable rejection pass.""" + candidates = [list(position) for position in (science_comp_stars or [])] + if stellar_variability_only: + return candidates[:STELLAR_VARIABILITY_APERTURE_ESTIMATION_MAX_COMPARISONS] + return candidates + + +def collapse_aperture_data_to_selected_grid_cell(aper_data, aperture_index, annulus_index): + if not isinstance(aper_data, dict): + return None + + aperture_index = int(aperture_index) + annulus_index = int(annulus_index) + selected = {} + for key, values in aper_data.items(): + array = np.asarray(values) + if array.ndim == 3: + if not ( + 0 <= aperture_index < array.shape[1] + and 0 <= annulus_index < array.shape[2] + ): + raise IndexError( + f"Selected aperture grid cell [{aperture_index}, {annulus_index}] " + f"is outside {key} shape {array.shape}." + ) + selected[key] = np.array( + array[:, aperture_index:aperture_index + 1, annulus_index:annulus_index + 1], + copy=True, + ) + else: + selected[key] = np.array(array, copy=True) + return selected + + +def aperture_frame_sigma_from_psf_data(psf_data, frame_index, fallback_sigma=np.nan, + comparison_indices=None): + if comparison_indices is None: + return psf_sigma_from_fit( + psf_data['target'][frame_index], + fallback_sigma=fallback_sigma, + ) + + comparison_sigmas = [] + for comp_idx in comparison_indices: + ckey = f"comp{int(comp_idx) + 1}" + if ckey not in psf_data: + continue + comp_sigma = psf_sigma_from_fit(psf_data[ckey][frame_index], fallback_sigma=np.nan) + if np.isfinite(comp_sigma) and comp_sigma > 0: + comparison_sigmas.append(float(comp_sigma)) + if comparison_sigmas: + return float(np.median(comparison_sigmas)) + return finite_positive_or_nan(fallback_sigma) + + def mask_aperture_star_frame(aper_data, key, frame_index): if not isinstance(aper_data, dict) or key not in aper_data: return @@ -24995,18 +25622,25 @@ def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_star_count, aper_data, apertures, annuli, fast_aperture_mask, adaptive_apertures=False, fallback_sigma=np.nan, use_aperture_corrections_and_full_image_fwhm=False, - noise_config=None, exposure_s=np.nan, airmass=np.nan): - target_sigma = psf_sigma_from_fit(psf_data['target'][frame_index], fallback_sigma=fallback_sigma) - target_fwhm = psf_fwhm_from_sigma(target_sigma) + noise_config=None, exposure_s=np.nan, airmass=np.nan, + comp_indices=None, include_target=True, + frame_sigma_comp_indices=None): + frame_seed_sigma = aperture_frame_sigma_from_psf_data( + psf_data, + frame_index, + fallback_sigma=fallback_sigma, + comparison_indices=frame_sigma_comp_indices, + ) + frame_seed_fwhm = psf_fwhm_from_sigma(frame_seed_sigma) field_star_psfs = np.empty((0, 7), dtype=float) - image_fwhm = target_fwhm + image_fwhm = frame_seed_fwhm if use_aperture_corrections_and_full_image_fwhm: - field_star_psfs = estimate_isolated_field_star_psfs(image_data, fwhm_hint=target_fwhm) - image_fwhm = image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=target_fwhm) + field_star_psfs = estimate_isolated_field_star_psfs(image_data, fwhm_hint=frame_seed_fwhm) + image_fwhm = image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=frame_seed_fwhm) frame_sigma = ( image_fwhm / GAUSSIAN_SIGMA_TO_FWHM if np.isfinite(image_fwhm) and image_fwhm > 0 - else target_sigma + else frame_seed_sigma ) frame_apertures, frame_annuli = resolve_frame_aperture_radii( apertures, @@ -25033,28 +25667,38 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta ) aperture_correction_factors = aperture_correction.get('correction_factors') - target_flux, target_bg, target_noise = compute_star_aperture_grid( - image_data, - 0, - psf_data['target'][frame_index, 0], - psf_data['target'][frame_index, 1], - frame_apertures, - frame_annuli, - fast_mode=fast_aperture_mask, - sigma_hint=frame_sigma, - aperture_correction_factors=aperture_correction_factors, - noise_config=noise_config, - exposure_s=exposure_s, - airmass=airmass, - return_noise=True, - ) - aper_data['target'][frame_index] = target_flux - aper_data['target_bg'][frame_index] = target_bg - aper_data['target_unc'][frame_index] = target_noise['total'] - for component in NOISE_BUDGET_COMPONENT_KEYS: - aper_data[f"target_noise_{component}"][frame_index] = target_noise[component] + if include_target: + target_flux, target_bg, target_noise = compute_star_aperture_grid( + image_data, + 0, + psf_data['target'][frame_index, 0], + psf_data['target'][frame_index, 1], + frame_apertures, + frame_annuli, + fast_mode=fast_aperture_mask, + sigma_hint=frame_sigma, + aperture_correction_factors=aperture_correction_factors, + noise_config=noise_config, + exposure_s=exposure_s, + airmass=airmass, + return_noise=True, + ) + aper_data['target'][frame_index] = target_flux + aper_data['target_bg'][frame_index] = target_bg + aper_data['target_unc'][frame_index] = target_noise['total'] + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"target_noise_{component}"][frame_index] = target_noise[component] - for comp_idx in range(comp_star_count): + if comp_indices is None: + selected_comp_indices = range(comp_star_count) + else: + selected_comp_indices = sorted({ + int(comp_idx) + for comp_idx in comp_indices + if 0 <= int(comp_idx) < int(comp_star_count) + }) + + for comp_idx in selected_comp_indices: ckey = f"comp{comp_idx + 1}" comp_sigma = psf_sigma_from_fit(psf_data[ckey][frame_index], fallback_sigma=frame_sigma) comp_flux, comp_bg, comp_noise = compute_star_aperture_grid( @@ -25084,6 +25728,22 @@ def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_sta def load_calibrated_reduction_image(file_name, generalDark, generalBias, generalFlat, demosaic_fmt, demosaic_out, demosaic_mult, bad_pixel_reference=None): + _, image_data = load_calibrated_reduction_frame( + file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + return image_data + + +def load_calibrated_reduction_frame(file_name, generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult, + bad_pixel_reference=None): hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) extension = 0 image_header = hdul[extension].header @@ -25097,7 +25757,257 @@ def load_calibrated_reduction_image(file_name, generalDark, generalBias, general image_data = apply_cals(image_data, generalDark, generalBias, generalFlat, 1) image_data = demosaic_img(image_data, demosaic_fmt, demosaic_out, demosaic_mult, 1) image_data = repair_bad_pixels_in_frame(image_data, bad_pixel_reference) - return image_data + return image_header, image_data + + +def evenly_spaced_aperture_tuning_indices(frame_count, max_frames=APERTURE_AUTOTUNE_MAX_FRAMES, + min_frames=APERTURE_AUTOTUNE_MIN_FRAMES): + """Select representative frames from the beginning through the end of a run.""" + frame_count = max(0, int(frame_count)) + if frame_count == 0: + return np.array([], dtype=int) + + requested = min(frame_count, max(1, int(max_frames))) + if frame_count >= int(min_frames): + requested = max(int(min_frames), requested) + return np.linspace(0, frame_count - 1, requested, dtype=int) + + +def centered_numpy_cutout(data, xc, yc, radius): + """Copy only the square slice needed for local aperture measurements.""" + data = np.asarray(data) + if data.ndim != 2 or not (np.isfinite(xc) and np.isfinite(yc) and np.isfinite(radius)): + return None, np.nan, np.nan + + radius = max(float(radius), 1.0) + x0 = max(0, int(np.floor(float(xc) - radius))) + x1 = min(data.shape[1], int(np.ceil(float(xc) + radius)) + 1) + y0 = max(0, int(np.floor(float(yc) - radius))) + y1 = min(data.shape[0], int(np.ceil(float(yc) + radius)) + 1) + if x1 <= x0 or y1 <= y0: + return None, np.nan, np.nan + return np.array(data[y0:y1, x0:x1], copy=True), float(xc) - x0, float(yc) - y0 + + +def can_memmap_aperture_tuning_cutouts(generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, bad_pixel_reference=None): + calibration_arrays = (generalDark, generalBias, generalFlat) + has_calibration = any( + value is not None and np.asarray(value).size > 0 + for value in calibration_arrays + ) + return bool( + not has_calibration + and not demosaic_fmt + and bad_pixel_reference is None + ) + + +def fits_header_supports_memmap(header): + """Astropy cannot expose scaled FITS image arrays through a memory map.""" + try: + bscale = float(header.get('BSCALE', 1.0)) + bzero = float(header.get('BZERO', 0.0)) + except (TypeError, ValueError): + return False + return bool(bscale == 1.0 and bzero == 0.0) + + +def _open_memmapped_reduction_frame(file_name): + hdul = fits.open( + name=file_name, + memmap=True, + cache=False, + lazy_load_hdus=True, + ignore_missing_end=True, + ) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + if not fits_header_supports_memmap(image_header): + hdul.close() + raise ValueError("Scaled FITS image requires the non-memmap reduction path.") + return hdul, image_header, hdul[extension].data + + +def build_aperture_tuning_cutouts(inputfiles, frame_indices, psf_data, comparison_indices, + adaptive_apertures, reference_sigma, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None, + bad_pixel_reference=None, p_dict=None, info_dict=None, + jd_times=None, reject_overexposed=False, + overexposure_threshold=np.nan, fast_aperture_mask=False): + """Read each tuning frame once and retain compact star-local NumPy slices.""" + comparison_indices = tuple(int(index) for index in comparison_indices) + frame_indices = np.asarray(frame_indices, dtype=int) + frames = [] + sample_airmass = [] + sample_overexposed_masks = { + f"comp{comp_idx + 1}": np.zeros(len(frame_indices), dtype=bool) + for comp_idx in comparison_indices + } + use_memmap = can_memmap_aperture_tuning_cutouts( + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + bad_pixel_reference=bad_pixel_reference, + ) + + for sample_index, frame_index in enumerate(frame_indices): + memmap_hdul = None + if use_memmap: + try: + memmap_hdul, header, image_data = _open_memmapped_reduction_frame( + inputfiles[frame_index] + ) + except (OSError, ValueError, TypeError): + if memmap_hdul is not None: + memmap_hdul.close() + memmap_hdul = None + header, image_data = load_calibrated_reduction_frame( + inputfiles[frame_index], + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + else: + header, image_data = load_calibrated_reduction_frame( + inputfiles[frame_index], + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + frame_sigma = aperture_frame_sigma_from_psf_data( + psf_data, + frame_index, + fallback_sigma=reference_sigma, + comparison_indices=comparison_indices, + ) + if not np.isfinite(frame_sigma) or frame_sigma <= 0: + frame_sigma = finite_positive_or_nan(reference_sigma) + if not np.isfinite(frame_sigma) or frame_sigma <= 0: + frame_sigma = 1.0 + + max_aperture, max_annulus = resolve_frame_aperture_radii( + [APERTURE_SIGMA_MAX if adaptive_apertures else APERTURE_SIGMA_MAX * reference_sigma], + [ANNULUS_SIGMA_MAX if adaptive_apertures else ANNULUS_SIGMA_MAX * reference_sigma], + adaptive_apertures=adaptive_apertures, + frame_sigma=frame_sigma, + fallback_sigma=reference_sigma, + ) + max_geometry = resolve_sky_annulus_geometry( + float(max_aperture[0]), + float(max_annulus[0]), + psf_sigma=frame_sigma, + ) + cutout_radius = float( + max_geometry['inner_radius'] + max_geometry['annulus_width'] + 2.0 + ) + + stars = {} + for comp_idx in comparison_indices: + ckey = f"comp{comp_idx + 1}" + row = np.asarray(psf_data[ckey][frame_index], dtype=float) + xc = row[0] if row.size > 0 else np.nan + yc = row[1] if row.size > 1 else np.nan + cutout, local_xc, local_yc = centered_numpy_cutout( + image_data, + xc, + yc, + cutout_radius, + ) + stars[ckey] = { + 'data': cutout, + 'xc': local_xc, + 'yc': local_yc, + 'sigma': psf_sigma_from_fit(row, fallback_sigma=frame_sigma), + } + if reject_overexposed: + sample_overexposed_masks[ckey][sample_index] = aperture_contains_overexposed_pixel( + image_data, + xc, + yc, + overexposure_aperture_radius_from_psf_row(row, fallback_sigma=frame_sigma), + overexposure_threshold, + fast_mode=fast_aperture_mask, + ) + + if p_dict is not None and info_dict is not None and jd_times is not None: + sample_airmass.append(air_mass( + header, + p_dict['ra'], + p_dict['dec'], + info_dict['lat'], + info_dict['long'], + info_dict['elev'], + jd_times[frame_index], + )) + else: + sample_airmass.append(np.nan) + frames.append({'frame_sigma': frame_sigma, 'stars': stars}) + del image_data + if memmap_hdul is not None: + memmap_hdul.close() + + return frames, np.asarray(sample_airmass, dtype=float), sample_overexposed_masks + + +def populate_aperture_tuning_data_from_cutouts(cutout_frames, apertures, annuli, + comparison_indices, adaptive_apertures, + reference_sigma, fast_aperture_mask=False): + """Evaluate an aperture grid using only compact per-star image slices.""" + comparison_indices = tuple(int(index) for index in comparison_indices) + comp_star_count = max(comparison_indices, default=-1) + 1 + aper_data = initialize_aperture_data_store( + len(cutout_frames), + len(apertures), + len(annuli), + comp_star_count, + ) + for frame_index, frame in enumerate(cutout_frames): + frame_sigma = finite_positive_or_nan(frame.get('frame_sigma')) + if not np.isfinite(frame_sigma) or frame_sigma <= 0: + frame_sigma = finite_positive_or_nan(reference_sigma) + if not np.isfinite(frame_sigma) or frame_sigma <= 0: + frame_sigma = 1.0 + frame_apertures, frame_annuli = resolve_frame_aperture_radii( + apertures, + annuli, + adaptive_apertures=adaptive_apertures, + frame_sigma=frame_sigma, + fallback_sigma=reference_sigma, + ) + for comp_idx in comparison_indices: + ckey = f"comp{comp_idx + 1}" + star = frame['stars'].get(ckey, {}) + cutout = star.get('data') + if cutout is None: + continue + flux, background = compute_star_aperture_grid( + cutout, + comp_idx + 1, + star.get('xc', np.nan), + star.get('yc', np.nan), + frame_apertures, + frame_annuli, + fast_mode=fast_aperture_mask, + sigma_hint=star.get('sigma', frame_sigma), + return_noise=False, + ) + aper_data[ckey][frame_index] = flux + aper_data[f"{ckey}_bg"][frame_index] = background + return aper_data def _refined_sigma_grid(center, lower_bound, upper_bound, half_width, points): @@ -25365,12 +26275,14 @@ def stellar_variability_calibration_for_position(calibration_stars, position, ob magnitude_error = normalized_magnitude_error(star.get('error')) if not is_usable_apparent_magnitude(magnitude) or magnitude_error is None: continue + if catalog_band_priority(star.get('mag_band'), observed_filter) != 0: + continue candidates.append({ 'label': label, 'star': star, 'magnitude': float(magnitude), 'magnitude_error': float(magnitude_error), - 'band_priority': catalog_band_priority(star.get('mag_band'), observed_filter), + 'band_priority': 0, }) if not candidates: return None @@ -25502,13 +26414,173 @@ def stellar_variability_ensemble_calibration_error_clip( } +def stellar_variability_acquisition_gap_boundaries( + times, + minimum_gap_ratio=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_RATIO, + minimum_gap_seconds=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_SECONDS, + minimum_side_points=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_POINTS): + values = np.asarray(times, dtype=float).reshape(-1) + if values.size < (2 * int(minimum_side_points)): + return [] + differences = np.diff(values) + positive = differences[np.isfinite(differences) & (differences > 0)] + if positive.size == 0: + return [] + cadence_days = float(bn.nanmedian(positive)) + threshold_days = max( + float(minimum_gap_ratio) * cadence_days, + float(minimum_gap_seconds) / 86400.0, + ) + boundaries = [] + for boundary_index in np.flatnonzero( + np.isfinite(differences) & (differences >= threshold_days)) + 1: + if ( + boundary_index < int(minimum_side_points) + or values.size - boundary_index < int(minimum_side_points) + ): + continue + boundaries.append({ + 'source_index': int(boundary_index), + 'pre_time': float(values[boundary_index - 1]), + 'post_time': float(values[boundary_index]), + 'gap_seconds': float(differences[boundary_index - 1] * 86400.0), + 'median_cadence_seconds': float(cadence_days * 86400.0), + }) + return boundaries + + +def _robust_median_scatter_count(values): + finite = np.asarray(values, dtype=float) + finite = finite[np.isfinite(finite)] + if finite.size == 0: + return np.nan, np.nan, 0 + median = float(bn.nanmedian(finite)) + scatter = float(1.4826 * bn.nanmedian(np.abs(finite - median))) + return median, scatter, int(finite.size) + + +def stellar_variability_comparison_gap_stability( + candidates, + comp_flux_map, + times, + window_frames=STELLAR_VARIABILITY_COMPARISON_GAP_WINDOW_FRAMES, + minimum_points=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_POINTS, + maximum_step_magnitude=STELLAR_VARIABILITY_COMPARISON_GAP_MAX_STEP_MAG, + minimum_significance=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_SIGNIFICANCE): + candidates = list(candidates or []) + boundaries = stellar_variability_acquisition_gap_boundaries( + times, + minimum_side_points=minimum_points, + ) + summary = { + 'applied': False, + 'reason': None, + 'boundaries': boundaries, + 'window_frames': int(window_frames), + 'minimum_points_per_side': int(minimum_points), + 'maximum_allowed_step_magnitude': float(maximum_step_magnitude), + 'minimum_rejection_significance': float(minimum_significance), + 'candidates': {}, + } + if len(candidates) < 3: + summary['reason'] = 'fewer than three independently calibrated comparison candidates' + return summary + if not boundaries: + summary['reason'] = 'no acquisition gap exceeded the cadence-based threshold' + return summary + + times_array = np.asarray(times, dtype=float).reshape(-1) + instrumental_columns = [] + usable_candidates = [] + for candidate in candidates: + key = candidate.get('key') + flux = np.asarray(comp_flux_map.get(key, []), dtype=float).reshape(-1) + if flux.shape != times_array.shape: + continue + instrumental = np.full(flux.shape, np.nan, dtype=float) + valid = np.isfinite(flux) & (flux > 0) + with np.errstate(divide='ignore', invalid='ignore'): + instrumental[valid] = -2.5 * np.log10(flux[valid]) + center = bn.nanmedian(instrumental) + if not np.isfinite(center): + continue + instrumental_columns.append(instrumental - center) + usable_candidates.append(candidate) + + if len(usable_candidates) < 3: + summary['reason'] = 'fewer than three comparison candidates had usable flux series' + return summary + + instrumental_stack = np.column_stack(instrumental_columns) + window = max(int(window_frames), int(minimum_points)) + for candidate_index, candidate in enumerate(usable_candidates): + other_stack = np.delete(instrumental_stack, candidate_index, axis=1) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=RuntimeWarning) + leave_one_out_reference = np.nanmedian(other_stack, axis=1) + residual = instrumental_stack[:, candidate_index] - leave_one_out_reference + boundary_results = [] + rejected = False + maximum_absolute_step = 0.0 + maximum_step_significance = 0.0 + for boundary in boundaries: + boundary_index = boundary['source_index'] + pre = residual[max(0, boundary_index - window):boundary_index] + post = residual[boundary_index:min(residual.size, boundary_index + window)] + pre_median, pre_scatter, pre_count = _robust_median_scatter_count(pre) + post_median, post_scatter, post_count = _robust_median_scatter_count(post) + if pre_count < int(minimum_points) or post_count < int(minimum_points): + continue + step = float(post_median - pre_median) + uncertainty = float(np.hypot( + pre_scatter / np.sqrt(pre_count), + post_scatter / np.sqrt(post_count), + )) + significance = ( + float(abs(step) / uncertainty) + if np.isfinite(uncertainty) and uncertainty > 0 + else (float('inf') if step != 0 else 0.0) + ) + step_rejected = ( + abs(step) > float(maximum_step_magnitude) + and significance >= float(minimum_significance) + ) + rejected = rejected or step_rejected + maximum_absolute_step = max(maximum_absolute_step, abs(step)) + maximum_step_significance = max(maximum_step_significance, significance) + boundary_results.append({ + **boundary, + 'step_magnitude': step, + 'step_uncertainty_magnitude': uncertainty, + 'step_significance': significance, + 'pre_point_count': pre_count, + 'post_point_count': post_count, + 'rejected': step_rejected, + }) + summary['candidates'][candidate.get('key')] = { + 'label': candidate.get('label'), + 'position': candidate.get('position'), + 'catalog_source': candidate.get('star', {}).get('catalog_source'), + 'catalog_magnitude_band': candidate.get('star', {}).get('mag_band'), + 'maximum_absolute_step_magnitude': maximum_absolute_step, + 'maximum_step_significance': maximum_step_significance, + 'rejected': rejected, + 'boundary_results': boundary_results, + } + + summary['applied'] = True + return summary + + def select_stellar_variability_ensemble_members( ranked_summaries, calibration_stars, comp_flux_map, observed_filter=None, target_catalog_match=None, - max_members=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS): + max_members=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + min_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + times=None): target_profile = stellar_variability_catalog_profile(target_catalog_match, observed_filter) candidates = [] rejected = [] @@ -25560,8 +26632,57 @@ def select_stellar_variability_ensemble_members( if catalog_identity is not None: represented_catalog_identities.add(catalog_identity) + gap_stability = stellar_variability_comparison_gap_stability( + candidates, + comp_flux_map, + times, + ) if times is not None else { + 'applied': False, + 'reason': 'observation times were unavailable', + 'boundaries': [], + 'candidates': {}, + } + if gap_stability.get('applied'): + stable_candidates = [] + for candidate in candidates: + diagnostic = gap_stability.get('candidates', {}).get(candidate.get('key'), {}) + candidate['gap_stability'] = diagnostic + candidate['gap_stability_max_abs_step_mag'] = diagnostic.get( + 'maximum_absolute_step_magnitude' + ) + candidate['gap_stability_max_significance'] = diagnostic.get( + 'maximum_step_significance' + ) + if diagnostic.get('rejected'): + rejected.append({ + 'key': candidate.get('key'), + 'label': candidate.get('label'), + 'position': candidate.get('position'), + 'catalog_source': candidate.get('star', {}).get('catalog_source'), + 'catalog_magnitude_band': candidate.get('star', {}).get('mag_band'), + 'reason': ( + 'comparison changed discontinuously relative to the leave-one-out comparison ' + 'ensemble across an acquisition gap' + ), + 'maximum_absolute_step_magnitude': diagnostic.get( + 'maximum_absolute_step_magnitude' + ), + 'maximum_step_significance': diagnostic.get('maximum_step_significance'), + 'boundary_results': diagnostic.get('boundary_results', []), + }) + continue + stable_candidates.append(candidate) + candidates = stable_candidates + candidates.sort(key=lambda candidate: (-candidate['median_flux'], candidate.get('comp_index', np.inf))) - keep_mask, clip_summary = stellar_variability_ensemble_calibration_error_clip(candidates) + try: + minimum_members = max(1, int(min_members)) + except (TypeError, ValueError): + minimum_members = STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS + keep_mask, clip_summary = stellar_variability_ensemble_calibration_error_clip( + candidates, + min_members=minimum_members, + ) clipped_members = [] for candidate, keep in zip(candidates, keep_mask): if keep: @@ -25577,13 +26698,14 @@ def select_stellar_variability_ensemble_members( }) try: - member_limit = max(int(max_members), STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS) + member_limit = max(int(max_members), minimum_members) except (TypeError, ValueError): member_limit = STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS prelimit_member_count = len(clipped_members) members = list(clipped_members) if len(members) > member_limit: members.sort(key=lambda candidate: ( + _finite_float(candidate.get('gap_stability_max_abs_step_mag'), np.inf), 0 if _finite_float(candidate.get('color_magnitude_similarity_score')) is not None else 1, _finite_float(candidate.get('color_magnitude_similarity_score'), np.inf), _finite_float(candidate.get('color_delta'), np.inf), @@ -25614,11 +26736,13 @@ def select_stellar_variability_ensemble_members( 'target_catalog_profile': target_profile, 'prelimit_member_count': prelimit_member_count, 'member_limit': member_limit, + 'gap_stability': gap_stability, } def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flux_error, - comp_flux_map, comp_error_map, members): + comp_flux_map, comp_error_map, members, + minimum_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS): target_flux = np.asarray(target_flux, dtype=float) if target_flux.ndim != 1: target_flux = target_flux.reshape(-1) @@ -25667,9 +26791,16 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu zero_points.append(zero_point) zero_point_errors.append(zero_point_error) + try: + required_members = max(1, int(minimum_members)) + except (TypeError, ValueError): + required_members = STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS + member_text = "comparison star" if required_members == 1 else "comparison stars" empty = { 'applied': False, - 'failure_reason': 'fewer than two calibrated comparison stars were usable for the ensemble.', + 'failure_reason': ( + f'fewer than {required_members} calibrated {member_text} were usable for the reference.' + ), 'magnitude': np.full(target_flux.shape, np.nan, dtype=float), 'magnitude_error': np.full(target_flux.shape, np.nan, dtype=float), 'relative_flux': np.full(target_flux.shape, np.nan, dtype=float), @@ -25678,7 +26809,7 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu 'synthetic_reference_flux_error': np.full(target_flux.shape, np.nan, dtype=float), 'valid_member_count': np.zeros(target_flux.shape, dtype=int), } - if len(zero_points) < STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + if len(zero_points) < required_members: return empty zero_point_stack = np.vstack(zero_points) @@ -25701,13 +26832,15 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu ) valid = ( valid_target - & (valid_member_count >= STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS) + & (valid_member_count >= required_members) & np.isfinite(weight_sum) & (weight_sum > 0) ) if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: empty['valid_member_count'] = valid_member_count - empty['failure_reason'] = 'too few frames retained at least two calibrated ensemble members.' + empty['failure_reason'] = ( + f'too few frames retained at least {required_members} calibrated {member_text}.' + ) return empty magnitude = np.full(target_flux.shape, np.nan, dtype=float) @@ -25797,6 +26930,7 @@ def stellar_variability_ensemble_member_json(member): 'overexposure_rejected_frame_count': member.get('summary', {}).get( 'overexposure_rejected_count', 0 ), + 'gap_stability': member.get('gap_stability'), } @@ -25839,6 +26973,7 @@ def save_stellar_variability_ensemble_selection_json( ), 'members': [stellar_variability_ensemble_member_json(member) for member in members], 'rejected_candidates': selection.get('rejected', []), + 'comparison_gap_stability': selection.get('gap_stability', {}), }, } output_dir = Path(save) @@ -25896,7 +27031,10 @@ def build_stellar_variability_ensemble_params_from_fit( getattr(lc_fit, 'stellar_variability_ensemble_magnitude_errors', []), dtype=float, ) - times = np.asarray(getattr(lc_fit, 'jd_times', getattr(lc_fit, 'time', [])), dtype=float) + # Public stellar-variability products are explicitly labelled BJD_TDB. The + # light-curve ``time`` array carries BJD_TDB, while ``jd_times`` preserves + # the original FITS JD/UTC timestamps for frame-level diagnostics. + times = np.asarray(getattr(lc_fit, 'time', getattr(lc_fit, 'jd_times', [])), dtype=float) airmass = np.asarray(getattr(lc_fit, 'airmass', np.ones(times.shape)), dtype=float) members = list(getattr(lc_fit, 'stellar_variability_ensemble_members', []) or []) if not (magnitudes.shape == magnitude_errors.shape == times.shape == airmass.shape): @@ -26155,13 +27293,14 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, comp_flux_map, observed_filter=observed_filter, target_catalog_match=target_catalog_match, + times=times, ) ensemble_members = member_selection['members'] clip_summary = member_selection['calibration_error_clip'] for rejected_member in member_selection['rejected']: log_info( "Stellar-variability ensemble excluded " - f"{rejected_member.get('key') or 'comparison candidate'}: " + f"{rejected_member.get('label') or rejected_member.get('key') or 'comparison candidate'}: " f"{rejected_member.get('reason')}." ) @@ -26798,6 +27937,16 @@ def fortuitous_variable_target_series( def fortuitous_variable_target_metadata(variable): + reference_mode = variable.get('reference_mode', 'ensemble') + reference_description = ( + 'single-comparison calibrated' + if reference_mode == 'single_comparison' + else 'ensemble-calibrated' + ) + output_error_limit = _finite_float( + variable.get('output_magnitude_error_limit'), + FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + ) return { 'name': variable.get('name'), 'auid': variable.get('auid'), @@ -26824,11 +27973,13 @@ def fortuitous_variable_target_metadata(variable): 'reference_flux_error_adu': variable.get('reference_flux_error_adu'), 'reference_noise_components_adu': variable.get('reference_noise_components_adu'), 'detection_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, - 'output_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + 'output_magnitude_error_limit': output_error_limit, 'output_magnitude_error_rule': ( - 'Only frames with a finite positive ensemble-calibrated magnitude error below ' - '0.05 mag are written.' + f'Only frames with a finite positive {reference_description} magnitude error below ' + f'{output_error_limit:.2f} mag are written.' ), + 'reference_mode': reference_mode, + 'comparison_label': variable.get('comparison_label'), 'saturation_rejection_scope': ( 'Frames are rejected using this VSX target own overexposure mask; ' 'the exoplanet target overexposure mask is not applied.' @@ -26853,6 +28004,33 @@ def fortuitous_variable_target_metadata(variable): } +def fortuitous_output_magnitude_error_limit(members): + for member in members or []: + star = member.get('star', {}) if isinstance(member, dict) else {} + if ( + bool(star.get('uses_relaxed_bv_error_limit', False)) + and str(star.get('mag_band') or '').strip().upper() in {'B', 'V'} + ): + return CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX + return FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR + + +def clear_previous_fortuitous_variable_products(variable_dir): + output_dir = Path(variable_dir) + for prefix in ( + 'AID_AAVSO_', + 'StellarVariability_', + 'EnsembleSelection_', + 'FortuitousVariableStatus_', + ): + for path in output_dir.glob(f'{prefix}*'): + if path.is_file(): + path.unlink() + plot_path = output_dir / 'temp' / 'Stellar_Variability.png' + if plot_path.is_file(): + plot_path.unlink() + + def process_fortuitous_variables( variables, comparison_calibration, @@ -26867,7 +28045,8 @@ def process_fortuitous_variables( psf_noise_data=None, comp_overexposed_masks=None, exposure_times_seconds=None, - observed_filter=None): + observed_filter=None, + use_single_comparison=USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT): if not variables or comparison_calibration is None: return [] times = np.asarray(times, dtype=float) @@ -26875,6 +28054,11 @@ def process_fortuitous_variables( airmass = np.asarray(airmass, dtype=float) if not (times.shape == jd_times.shape == airmass.shape): return [] + use_single_comparison = bool(use_single_comparison) + required_comparison_members = ( + 1 if use_single_comparison else STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS + ) + reference_mode = 'single_comparison' if use_single_comparison else 'ensemble' ranked_summaries, comp_flux_map, comp_error_map = fortuitous_ensemble_flux_maps( comparison_calibration, @@ -26883,10 +28067,11 @@ def process_fortuitous_variables( psf_flux_data=psf_flux_data, psf_noise_data=psf_noise_data, ) - if len(ranked_summaries) < STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + if len(ranked_summaries) < required_comparison_members: log_info( - "Warning: fortuitous-variable photometry skipped because fewer than two independent " - "non-variable comparison candidates were usable.", + "Warning: fortuitous-variable photometry skipped because fewer than " + f"{required_comparison_members} independent non-variable comparison candidate(s) " + "were usable.", warn=True, ) return [] @@ -26904,9 +28089,16 @@ def process_fortuitous_variables( exposure_array = None base_dir = Path(info_dict['save']) / 'fortuitous_variables' + base_dir.mkdir(parents=True, exist_ok=True) + for stale_combined_aid in base_dir.glob('AID_AAVSO_FortuitousVariables_*.txt'): + if stale_combined_aid.is_file(): + stale_combined_aid.unlink() results = [] + combined_vsp_params = [] + logged_comparison_gap_rejections = set() for variable in variables: variable = dict(variable) + variable['reference_mode'] = reference_mode variable_name = variable.get('name') or 'VSX variable' category = variable.get('category') or 'rest_of_the_variables' variable['input_frame_count'] = int(times.size) @@ -26923,6 +28115,7 @@ def process_fortuitous_variables( 'VSX', variable_name, extension='' ) variable_dir.mkdir(parents=True, exist_ok=True) + clear_previous_fortuitous_variable_products(variable_dir) try: target_flux, target_flux_error, target_quality_mask = fortuitous_variable_target_series( variable, @@ -26945,16 +28138,48 @@ def process_fortuitous_variables( comp_flux_map, observed_filter=observed_filter, target_catalog_match=variable.get('catalog_match'), - ) + max_members=( + 1 if use_single_comparison else STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS + ), + min_members=required_comparison_members, + times=times, + ) + variable['comparison_gap_stability'] = member_selection.get('gap_stability', {}) + variable['comparison_gap_rejected_candidates'] = [ + candidate + for candidate in member_selection.get('rejected', []) + if candidate.get('maximum_absolute_step_magnitude') is not None + ] + for rejected_candidate in variable['comparison_gap_rejected_candidates']: + rejection_identity = ( + rejected_candidate.get('key'), + rejected_candidate.get('maximum_absolute_step_magnitude'), + ) + if rejection_identity in logged_comparison_gap_rejections: + continue + logged_comparison_gap_rejections.add(rejection_identity) + log_info( + "Fortuitous-variable comparison rejected across acquisition gap: " + f"{rejected_candidate.get('label') or rejected_candidate.get('key')} " + f"at {rejected_candidate.get('position')}, " + f"step={rejected_candidate.get('maximum_absolute_step_magnitude'):.4f} mag, " + f"significance={rejected_candidate.get('maximum_step_significance'):.2f} sigma." + ) members = member_selection.get('members', []) - if len(members) < STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: - raise ValueError('fewer than two independently calibrated ensemble members') + if len(members) < required_comparison_members: + raise ValueError( + f'fewer than {required_comparison_members} independently calibrated ' + 'comparison member(s)' + ) + output_magnitude_error_limit = fortuitous_output_magnitude_error_limit(members) + variable['output_magnitude_error_limit'] = output_magnitude_error_limit ensemble_series = build_stellar_variability_calibrated_ensemble_series( target_flux, target_flux_error, comp_flux_map, comp_error_map, members, + minimum_members=required_comparison_members, ) if not ensemble_series.get('applied'): raise ValueError(ensemble_series.get('failure_reason') or 'ensemble combination failed') @@ -26971,7 +28196,7 @@ def process_fortuitous_variables( magnitude_error_valid = ( np.isfinite(magnitude_errors) & (magnitude_errors > 0) - & (magnitude_errors < FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR) + & (magnitude_errors <= output_magnitude_error_limit) ) valid = base_valid & magnitude_error_valid eligible_magnitude_errors = magnitude_errors[base_valid & np.isfinite(magnitude_errors)] @@ -26985,14 +28210,44 @@ def process_fortuitous_variables( variable['output_magnitude_error_max'] = float(np.nanmax(eligible_magnitude_errors)) if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: raise ValueError( - 'fewer than five ensemble-calibrated frames have internal magnitude error ' - f'below {FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR:.3f} mag' + f'fewer than five {reference_mode.replace("_", "-")}-calibrated frames ' + 'have internal magnitude error ' + f'at or below {output_magnitude_error_limit:.3f} mag' ) variable['valid_output_frame_count'] = int(np.count_nonzero(valid)) target_error_values = target_flux_error if target_error_values is None: target_error_values = source_flux_uncertainty_from_counts(target_flux) target_error_values = np.asarray(target_error_values, dtype=float) + if use_single_comparison: + selected_member = members[0] + selected_comparison_key = selected_member.get('key') + selected_comparison_flux = np.asarray( + comp_flux_map[selected_comparison_key], + dtype=float, + ) + selected_comparison_error = None + if selected_comparison_key in comp_error_map: + selected_comparison_error = np.asarray( + comp_error_map[selected_comparison_key], + dtype=float, + ) + if ( + selected_comparison_error is None + or selected_comparison_error.shape != selected_comparison_flux.shape + ): + selected_comparison_error = source_flux_uncertainty_from_counts( + selected_comparison_flux + ) + reference_label = selected_member.get('label') or selected_comparison_key + reference_position = selected_member.get('position') + variable['comparison_label'] = reference_label + else: + selected_member = None + selected_comparison_flux = ensemble_series['synthetic_reference_flux'] + selected_comparison_error = ensemble_series['synthetic_reference_flux_error'] + reference_label = f"ENSEMBLE ({len(members)} stars)" + reference_position = [member.get('position') for member in members] prepared = { 'applied': True, 'failure_reason': None, @@ -27003,9 +28258,9 @@ def process_fortuitous_variables( 'jd_time': jd_times[valid], 'exposure_time_seconds': None if exposure_array is None else exposure_array[valid], 'target_flux': np.asarray(target_flux, dtype=float)[valid], - 'comp_flux': ensemble_series['synthetic_reference_flux'][valid], + 'comp_flux': selected_comparison_flux[valid], 'target_flux_error': target_error_values[valid], - 'comp_flux_error': ensemble_series['synthetic_reference_flux_error'][valid], + 'comp_flux_error': selected_comparison_error[valid], 'source_indices': np.flatnonzero(valid), } variable_period = _finite_float(variable.get('period_days'), 1.0) @@ -27029,39 +28284,50 @@ def process_fortuitous_variables( prepared, variable_prior, comp_index=None, - comp_label=f"ENSEMBLE ({len(members)} stars)", - comp_position=[member.get('position') for member in members], + comp_label=reference_label, + comp_position=reference_position, method_label=comparison_calibration.get('method_label'), plot_time_range=times, ) - if fit is None: - raise ValueError('stellar-variability light curve construction failed') - selected_indices = np.asarray(fit.stellar_variability_source_indices, dtype=int) - fit.stellar_variability_ensemble_members = members - fit.stellar_variability_ensemble_magnitudes = ensemble_series['magnitude'][selected_indices] - fit.stellar_variability_ensemble_magnitude_errors = ( - ensemble_series['magnitude_error'][selected_indices] - ) - fit.stellar_variability_ensemble_valid_member_counts = ( - ensemble_series['valid_member_count'][selected_indices] - ) - fit.stellar_variability_ensemble_calibration_error_clip = member_selection.get( - 'calibration_error_clip', {} - ) - fit.stellar_variability_ensemble_selection = member_selection - fit.stellar_variability_target_catalog_profile = member_selection.get( - 'target_catalog_profile', {} - ) - - target_metadata = fortuitous_variable_target_metadata(variable) - vsp_params = build_stellar_variability_ensemble_params_from_fit( - fit, - variable_dir, - variable_name, - observed_filter=observed_filter, - observation_date=info_dict.get('date'), - target_metadata=target_metadata, - ) + if fit is None: + raise ValueError('stellar-variability light curve construction failed') + if use_single_comparison: + vsp_params = build_stellar_variability_params_from_fit( + fit, + selected_member.get('star', {}), + selected_member.get('position'), + reference_label, + variable_dir, + variable_name, + observed_filter=observed_filter, + ) + else: + selected_indices = np.asarray(fit.stellar_variability_source_indices, dtype=int) + fit.stellar_variability_ensemble_members = members + fit.stellar_variability_ensemble_magnitudes = ensemble_series['magnitude'][selected_indices] + fit.stellar_variability_ensemble_magnitude_errors = ( + ensemble_series['magnitude_error'][selected_indices] + ) + fit.stellar_variability_ensemble_valid_member_counts = ( + ensemble_series['valid_member_count'][selected_indices] + ) + fit.stellar_variability_ensemble_calibration_error_clip = member_selection.get( + 'calibration_error_clip', {} + ) + fit.stellar_variability_ensemble_selection = member_selection + fit.stellar_variability_target_catalog_profile = member_selection.get( + 'target_catalog_profile', {} + ) + + target_metadata = fortuitous_variable_target_metadata(variable) + vsp_params = build_stellar_variability_ensemble_params_from_fit( + fit, + variable_dir, + variable_name, + observed_filter=observed_filter, + observation_date=info_dict.get('date'), + target_metadata=target_metadata, + ) if not vsp_params: raise ValueError('no calibrated magnitude rows were produced') csv_path = save_stellar_variability_magnitude_csv( @@ -27081,12 +28347,27 @@ def process_fortuitous_variables( None, vsp_params, ).aavso() + aid_variable_name = variable.get('auid') or variable_name + combined_vsp_params.extend([ + {**vsp_param, '_aid_name': aid_variable_name} + for vsp_param in vsp_params + ]) results.append({ 'name': variable_name, 'auid': variable.get('auid'), 'category': category, 'output_directory': str(variable_dir), 'point_count': len(vsp_params), + 'reference_mode': reference_mode, + 'comparison_label': reference_label, + 'selected_comparison_gap_stability': ( + selected_member.get('gap_stability') if selected_member is not None else None + ), + 'comparison_gap_stability': variable.get('comparison_gap_stability'), + 'comparison_gap_rejected_candidates': variable.get( + 'comparison_gap_rejected_candidates', [] + ), + 'comparison_member_count': len(members), 'ensemble_member_count': len(members), 'input_frame_count': variable.get('input_frame_count'), 'target_overexposure_rejected_frame_count': variable.get( @@ -27098,14 +28379,18 @@ def process_fortuitous_variables( 'output_magnitude_error_qualified_frame_count': variable.get( 'output_magnitude_error_qualified_frame_count' ), - 'output_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + 'output_magnitude_error_limit': variable.get( + 'output_magnitude_error_limit', + FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + ), 'output_magnitude_error_max': variable.get('output_magnitude_error_max'), 'magnitude_csv': str(csv_path) if csv_path else None, 'status': 'completed', }) log_info( f"Fortuitous-variable photometry completed for {variable_name}: " - f"{len(vsp_params)} point(s), {len(members)} ensemble member(s), outputs={variable_dir}." + f"{len(vsp_params)} point(s), reference={reference_label} " + f"({reference_mode}), outputs={variable_dir}." ) except Exception as exc: results.append({ @@ -27113,6 +28398,12 @@ def process_fortuitous_variables( 'auid': variable.get('auid'), 'category': category, 'output_directory': str(variable_dir), + 'reference_mode': reference_mode, + 'comparison_label': variable.get('comparison_label'), + 'comparison_gap_stability': variable.get('comparison_gap_stability'), + 'comparison_gap_rejected_candidates': variable.get( + 'comparison_gap_rejected_candidates', [] + ), 'input_frame_count': variable.get('input_frame_count'), 'target_overexposure_rejected_frame_count': variable.get( 'target_overexposure_rejected_frame_count' @@ -27123,7 +28414,10 @@ def process_fortuitous_variables( 'output_magnitude_error_qualified_frame_count': variable.get( 'output_magnitude_error_qualified_frame_count' ), - 'output_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + 'output_magnitude_error_limit': variable.get( + 'output_magnitude_error_limit', + FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + ), 'status': 'skipped', 'reason': str(exc), }) @@ -27151,13 +28445,47 @@ def process_fortuitous_variables( ) base_dir.mkdir(parents=True, exist_ok=True) + combined_aid_path = None + combined_aid_error = None + if combined_vsp_params: + try: + combined_info = dict(info_dict) + combined_info['save'] = str(base_dir) + combined_target = { + 'sName': 'FortuitousVariables', + 'pName': 'FortuitousVariables', + } + combined_aid_path = AIDOutputFiles( + None, + combined_target, + combined_info, + None, + None, + combined_vsp_params, + ).combined_aavso() + log_info( + f"Combined fortuitous-variable AID file written with " + f"{len(combined_vsp_params)} row(s): {combined_aid_path}." + ) + except Exception as exc: + combined_aid_error = str(exc) + log_info( + f"Warning: could not create the combined fortuitous-variable AID file ({exc}).", + warn=True, + ) manifest_path = base_dir / safe_output_filename( 'FortuitousVariables', filename_date_token(info_dict.get('date')), extension='json', ) with manifest_path.open('w', encoding='utf-8') as handle: - json.dump(stellar_variability_json_safe({'variables': results}), handle, indent=2, sort_keys=True) + manifest_payload = { + 'variables': results, + 'combined_aid': str(combined_aid_path) if combined_aid_path else None, + } + if combined_aid_error: + manifest_payload['combined_aid_error'] = combined_aid_error + json.dump(stellar_variability_json_safe(manifest_payload), handle, indent=2, sort_keys=True) handle.write('\n') return results @@ -28153,9 +29481,11 @@ def parse_args(): "If the service returns an error, EXOTIC falls back to individual VSX checks.") parser.add_argument('--non-interactive-run', action='store_true', - help="Run without interactive prompts for target pixel-coordinate mismatch checks " - "or unrecognized limb-darkening filters. Coordinate mismatches use an automatic " - "fallback; unrecognized filters abort unless wl_min and wl_max are provided.") + help="Avoid interactive prompts for invalid target RA/Dec values, target pixel-coordinate " + "mismatches, and unrecognized limb-darkening filters. Invalid initialization-file " + "RA/Dec values use NASA Exoplanet Archive coordinates when available, otherwise " + "the run aborts. Pixel mismatches use an automatic fallback; unrecognized filters " + "abort unless wl_min and wl_max are provided.") parser.add_argument('--multiprocess-transformations', type=int, default=None, @@ -28261,6 +29591,7 @@ def _main_impl(): # ----USER INPUTS---------------------------------------------------------- else: + reduction_stage_timer = ReductionStageTimer() log_info("\n**************************") log_info("Complete Reduction Routine") log_info("**************************") @@ -28381,6 +29712,14 @@ def _main_impl(): PHOTOMETER_FORTUITOUS_VARIABLES_DEFAULT, ) ) + use_single_comparison_for_fortuitous_variables = ( + should_use_single_comparison_for_fortuitous_variables( + exotic_infoDict.get( + 'use_single_comparison_for_fortuitous_variables', + USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT, + ) + ) + ) use_nextastro_vsx_cache_first = should_use_nextastro_vsx_cache_first( exotic_infoDict.get( 'use_nextastro_vsx_cache_first', @@ -28538,9 +29877,15 @@ def _main_impl(): # Make a temp directory of helpful files Path(Path(exotic_infoDict['save']) / "temp").mkdir(exist_ok=True) + archive_planet_dict = None if not args.override: - nea_obj = NASAExoplanetArchive(planet=userpDict['pName']) + nea_obj = NASAExoplanetArchive( + planet=userpDict['pName'], + non_interactive=args.non_interactive_run, + ) userpDict['pName'], CandidatePlanetBool, pDict = nea_obj.planet_info() + if isinstance(pDict, dict): + archive_planet_dict = dict(pDict) else: pDict = userpDict CandidatePlanetBool = False @@ -28549,13 +29894,55 @@ def _main_impl(): if args.nasaexoarch: pass elif args.override: - if type(pDict['ra']) and type(pDict['dec']) is str: - pDict['ra'], pDict['dec'] = radec_hours_to_degree(pDict['ra'], pDict['dec']) + try: + pDict['ra'], pDict['dec'] = radec_hours_to_degree( + pDict.get('ra'), + pDict.get('dec'), + non_interactive_run=args.non_interactive_run, + target_name=pDict.get('pName'), + ) + except ValueError as coordinate_error: + if not args.non_interactive_run: + raise + + try: + coordinate_nea_obj = NASAExoplanetArchive( + planet=pDict.get('pName'), + non_interactive=True, + ) + _, _, coordinate_pdict = coordinate_nea_obj.planet_info() + except Exception as archive_error: + raise ValueError( + f"Non-interactive run cancelled for target {pDict.get('pName')}: the " + f"initialization-file coordinates are invalid ({coordinate_error}), and the " + f"NASA Exoplanet Archive coordinate lookup failed ({archive_error})." + ) from archive_error + + archive_ra = coordinate_pdict.get('ra') if isinstance(coordinate_pdict, dict) else None + archive_dec = coordinate_pdict.get('dec') if isinstance(coordinate_pdict, dict) else None + if isinstance(coordinate_pdict, dict): + archive_planet_dict = dict(coordinate_pdict) + pDict['ra'], pDict['dec'] = radec_hours_to_degree( + pDict.get('ra'), + pDict.get('dec'), + non_interactive_run=True, + archive_ra=archive_ra, + archive_dec=archive_dec, + target_name=pDict.get('pName'), + ) else: diff = False - if type(userpDict['ra']) and type(userpDict['dec']) is str: - userpDict['ra'], userpDict['dec'] = radec_hours_to_degree(userpDict['ra'], userpDict['dec']) + archive_ra = pDict.get('ra') if isinstance(pDict, dict) else None + archive_dec = pDict.get('dec') if isinstance(pDict, dict) else None + userpDict['ra'], userpDict['dec'] = radec_hours_to_degree( + userpDict.get('ra'), + userpDict.get('dec'), + non_interactive_run=args.non_interactive_run, + archive_ra=archive_ra, + archive_dec=archive_dec, + target_name=userpDict.get('pName'), + ) if not CandidatePlanetBool: diff = check_parameters(userpDict, pDict) @@ -28566,6 +29953,31 @@ def _main_impl(): else: pDict = get_planetary_parameters(CandidatePlanetBool, userpDict, pdict=pDict) + def lookup_archive_ephemeris(): + lookup_name = ( + pDict.get('pName') + if isinstance(pDict, dict) + else userpDict.get('pName') + ) + _, archive_candidate, archive_parameters = NASAExoplanetArchive( + planet=lookup_name, + non_interactive=True, + ).planet_info() + if archive_candidate or not isinstance(archive_parameters, dict): + return None + return archive_parameters + + pDict = resolve_required_transit_ephemeris( + pDict, + archive_planet_dict=archive_planet_dict, + archive_lookup=lookup_archive_ephemeris if args.override else None, + target_name=( + pDict.get('pName') + if isinstance(pDict, dict) + else userpDict.get('pName') + ), + ) + # Seed random number generator (for run to run consistency) if exotic_infoDict['random_seed']: log_info(f"Setting random number seed to {exotic_infoDict['random_seed']}") @@ -28607,6 +30019,7 @@ def _main_impl(): log_info("\n**************************" "\nStarting Reduction Process" "\n**************************\n") + reduction_stage_timer.checkpoint("Initialization, configuration, and calibration masters") ######################################### # FLUX DATA EXTRACTION AND MANIPULATION @@ -28667,6 +30080,7 @@ def _main_impl(): non_interactive_run=args.non_interactive_run, ) log_info("Limb-darkening coefficients ready.") + reduction_stage_timer.checkpoint("FITS validation, timestamp conversion, and limb darkening") # check for EPW_MD5 checksum if 'EPW_MD5' in header: @@ -29090,6 +30504,16 @@ def _main_impl(): ) if photometer_fortuitous_variables: + if use_single_comparison_for_fortuitous_variables: + log_info( + "Fortuitous-variable single-comparison mode enabled (default): each " + "retained VSX target will use one unsaturated, non-variable, " + "catalog-calibrated comparison star." + ) + else: + log_info( + "Fortuitous-variable calibrated ensemble mode enabled per optional_info setting." + ) fortuitous_variables = discover_fortuitous_vsx_variables( wcs_file, reference_image.shape, @@ -29107,6 +30531,37 @@ def _main_impl(): log_info("Fortuitous-variable photometry disabled per optional_info setting.") if fortuitous_variables: + science_comp_stars, variable_comparison_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + science_comp_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + if variable_comparison_rejections: + for rejection in variable_comparison_rejections: + log_info( + "Removed science comparison star " + f"#{rejection['comparison_index'] + 1} at " + f"[{rejection['position'][0]:.1f}, {rejection['position'][1]:.1f}] because " + f"the full-field VSX search identified the same source as " + f"{rejection['variable_name']} " + f"({rejection['distance_pixels']:.2f} pixel separation).", + warn=True, + ) + exotic_infoDict['comp_stars'] = [ + list(position) for position in science_comp_stars + ] + vsp_comp_stars = { + key: star + for key, star in vsp_comp_stars.items() + if fortuitous_variable_overlap( + star.get('pos'), + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) is None + } + if stellar_variability_ensemble_candidate_search: # The stellar-variability target path just selected and VSX-vetted the # same brightest-first pool with the same count and saturation limit. @@ -29139,17 +30594,21 @@ def _main_impl(): fortuitous_auto_stars, use_nextastro_variability_server=args.use_nextastro_variability_server, ) - variable_positions = [variable['pos'] for variable in fortuitous_variables] - fortuitous_auto_stars = [ - position for position in fortuitous_auto_stars - if not any( - np.hypot( - float(position[0]) - float(variable_position[0]), - float(position[1]) - float(variable_position[1]), - ) <= REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS - for variable_position in variable_positions + fortuitous_auto_stars, automatic_variable_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + fortuitous_auto_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for rejection in automatic_variable_rejections: + log_info( + "Removed automatic fortuitous-variable comparison candidate at " + f"[{rejection['position'][0]:.1f}, {rejection['position'][1]:.1f}] because " + f"the full-field VSX search identified {rejection['variable_name']} " + f"at the same source ({rejection['distance_pixels']:.2f} pixel separation).", + warn=True, ) - ] fortuitous_ensemble_stars, fortuitous_duplicate_messages = ( merge_automatic_comparison_star_coords( science_comp_stars, @@ -29229,6 +30688,26 @@ def _main_impl(): tar_comp_dist = {} vsp_num = [] comp_star_count = len(exotic_infoDict['comp_stars']) + tracked_vsx_labels = { + str(variable.get('tracking_key')): f"Tracked VSX variable {variable.get('name', 'unknown')}" + for variable in fortuitous_variables + if variable.get('tracking_key') + } + plateStatus.setComparisonStarLabels({ + int(comp_key[4:]): label + for comp_key, label in tracked_vsx_labels.items() + if comp_key.startswith('comp') and comp_key[4:].isdigit() + }) + aperture_estimation_stars = aperture_estimation_comparison_stars( + science_comp_stars, + stellar_variability_only=stellar_variability_only, + ) + aperture_estimation_comp_count = len(aperture_estimation_stars) + aperture_estimation_comp_indices = tuple(range(aperture_estimation_comp_count)) + aperture_estimation_includes_target = not stellar_variability_only + aperture_frame_sigma_comp_indices = ( + aperture_estimation_comp_indices if stellar_variability_only else None + ) psf_noise_data = initialize_psf_noise_data(len(inputfiles), comp_star_count) target_overexposed_frame_mask = np.zeros(len(inputfiles), dtype=bool) comp_overexposed_masks = { @@ -29379,12 +30858,17 @@ def _main_impl(): tar_comp_dist[ckey] = np.zeros(2) coarse_tune_frames = 0 + coarse_tune_frame_indices = np.array([], dtype=int) coarse_apertures_sigma = None coarse_annuli_sigma = None if use_aperture_photometry: coarse_tune_frames = min(len(inputfiles), APERTURE_AUTOTUNE_MAX_FRAMES) if len(inputfiles) >= APERTURE_AUTOTUNE_MIN_FRAMES: coarse_tune_frames = max(APERTURE_AUTOTUNE_MIN_FRAMES, coarse_tune_frames) + coarse_tune_frame_indices = evenly_spaced_aperture_tuning_indices( + len(inputfiles), + max_frames=coarse_tune_frames, + ) coarse_apertures_sigma = np.linspace( APERTURE_SIGMA_MIN, APERTURE_SIGMA_MAX, @@ -29398,7 +30882,7 @@ def _main_impl(): log_info( "Automatic aperture tuning enabled: " f"coarse_grid={len(coarse_apertures_sigma)}x{len(coarse_annuli_sigma)}, " - f"coarse_frames={coarse_tune_frames}." + f"coarse_frames={coarse_tune_frames}, sampling=evenly_spaced_full_sequence." ) sigma = np.nan @@ -29441,11 +30925,39 @@ def _main_impl(): ) if use_aperture_corrections_and_full_image_fwhm: log_info("Aperture corrections and full-image FWHM estimation enabled per optional_info setting.") + if stellar_variability_only: + if use_ensemble_photometry_for_stellar_variability: + estimator_text = ( + f"{aperture_estimation_comp_count} bright, reference-frame non-saturated, " + "VSX-vetted non-variable comparison star(s)" + ) + else: + estimator_text = ( + f"the first {aperture_estimation_comp_count} supplied science comparison star(s), " + "with existing saturation and PSF-quality masks" + ) + log_info( + f"Stellar-variability aperture estimation will use only {estimator_text}. " + "The variable science target and additional tracked stars will be measured once " + "with the selected aperture." + ) + elif comp_star_count > aperture_estimation_comp_count: + log_info( + "Aperture-grid estimation will use only the science target and " + f"{aperture_estimation_comp_count} science comparison star(s); " + f"{comp_star_count - aperture_estimation_comp_count} fortuitous-only tracked " + "star(s) will reuse the selected aperture." + ) + + reduction_stage_timer.checkpoint("Frame prechecks, WCS, catalogs, and comparison preparation") # open files, calibrate, align, photometry reset_transform_timing_stats() reset_photometry_timing_stats() multiprocess_alignment_results = None + multiprocess_alignment_results_applied = False + aperture_preselected_from_sample = False + aperture_tuning_sample_score = np.nan use_multiprocess_alignment = ( args.multiprocess_transformations is not None and args.multiprocess_transformations > 0 ) @@ -29474,11 +30986,189 @@ def _main_impl(): compute_fallback_transform=True, precomputed_fallback_transforms=pointing_alignment_transforms, ) + for alignment_index, alignment_result in enumerate(multiprocess_alignment_results): + apply_parallel_alignment_result( + alignment_result, + alignment_index, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + multiprocess_alignment_results_applied = True + + if ( + use_aperture_photometry + and multiprocess_alignment_results_applied + and aperture_estimation_comp_count > 0 + and not use_aperture_corrections_and_full_image_fwhm + ): + aperture_tuning_start = perf_counter() + sigma = aperture_frame_sigma_from_psf_data( + psf_data, + 0, + comparison_indices=aperture_frame_sigma_comp_indices, + ) + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + + memmap_cutouts = can_memmap_aperture_tuning_cutouts( + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + bad_pixel_reference=bad_pixel_reference, + ) + log_info( + "Aperture tuning sample: " + f"{len(coarse_tune_frame_indices)} evenly spaced frame(s) spanning " + f"1-{len(inputfiles)}; image access=" + + ("FITS memmap star cutouts" if memmap_cutouts else "calibrated full-frame fallback") + + "." + ) + tuning_cutouts, tuning_airmass, tuning_overexposed_masks = build_aperture_tuning_cutouts( + inputfiles, + coarse_tune_frame_indices, + psf_data, + aperture_estimation_comp_indices, + use_adaptive_apertures, + sigma, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + p_dict=pDict, + info_dict=exotic_infoDict, + jd_times=jd_times, + reject_overexposed=reject_overexposed_stars, + overexposure_threshold=overexposure_threshold, + fast_aperture_mask=fast_aperture_mask, + ) + tuning_psf_data = { + key: np.asarray(values)[coarse_tune_frame_indices] + for key, values in psf_data.items() + } + coarse_aper_data = populate_aperture_tuning_data_from_cutouts( + tuning_cutouts, + coarse_apertures_sigma if use_adaptive_apertures else coarse_apertures_sigma * sigma, + coarse_annuli_sigma if use_adaptive_apertures else coarse_annuli_sigma * sigma, + aperture_estimation_comp_indices, + use_adaptive_apertures, + sigma, + fast_aperture_mask=fast_aperture_mask, + ) + for tuning_frame_index in range(len(tuning_cutouts)): + apply_overexposure_masks_to_aperture_frame( + coarse_aper_data, + tuning_frame_index, + False, + tuning_overexposed_masks, + ) + tuning_psf_quality_masks = { + f"comp{comp_idx + 1}": psf_quality_mask_for_key( + tuning_psf_data, + f"comp{comp_idx + 1}", + len(tuning_cutouts), + ) + for comp_idx in range(aperture_estimation_comp_count) + } + refined_apertures_sigma, refined_annuli_sigma, best_coarse_candidate, best_coarse_score = auto_tune_aperture_sigma_grid( + coarse_apertures_sigma, + coarse_annuli_sigma, + coarse_aper_data, + aperture_estimation_comp_count, + tuning_airmass, + require_comp_star=require_comp_star, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + psf_quality_masks=tuning_psf_quality_masks, + ) + refined_tuning_data = populate_aperture_tuning_data_from_cutouts( + tuning_cutouts, + refined_apertures_sigma if use_adaptive_apertures else refined_apertures_sigma * sigma, + refined_annuli_sigma if use_adaptive_apertures else refined_annuli_sigma * sigma, + aperture_estimation_comp_indices, + use_adaptive_apertures, + sigma, + fast_aperture_mask=fast_aperture_mask, + ) + for tuning_frame_index in range(len(tuning_cutouts)): + apply_overexposure_masks_to_aperture_frame( + refined_tuning_data, + tuning_frame_index, + False, + tuning_overexposed_masks, + ) + tuning_selection = select_comparison_calibrated_photometry( + tuning_psf_data, + refined_tuning_data, + refined_apertures_sigma * sigma, + refined_annuli_sigma * sigma, + tuning_airmass, + aperture_estimation_stars, + sigma, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=False, + use_aperture_photometry=True, + comp_overexposed_masks=tuning_overexposed_masks, + ) + if tuning_selection is not None: + selected_aperture_index = int(tuning_selection['a']) + selected_annulus_index = int(tuning_selection['an']) + selected_aperture_sigma = float(refined_apertures_sigma[selected_aperture_index]) + selected_annulus_sigma = float(refined_annuli_sigma[selected_annulus_index]) + if use_adaptive_apertures: + aperture_values = np.asarray([selected_aperture_sigma], dtype=float) + annulus_values = np.asarray([selected_annulus_sigma], dtype=float) + else: + aperture_values = np.asarray([selected_aperture_sigma * sigma], dtype=float) + annulus_values = np.asarray([selected_annulus_sigma * sigma], dtype=float) + apers = np.asarray([selected_aperture_sigma * sigma], dtype=float) + annuli = np.asarray([selected_annulus_sigma * sigma], dtype=float) + aper_data = initialize_aperture_data_store( + len(inputfiles), + 1, + 1, + comp_star_count, + ) + aperture_grid_tuned = True + aperture_preselected_from_sample = True + aperture_tuning_sample_score = float(tuning_selection['field_score']) + best_aper_fwhm = selected_aperture_sigma / GAUSSIAN_SIGMA_TO_FWHM + log_info( + "Distributed aperture tuning selected " + f"aper={selected_aperture_sigma:.2f} sigma/{best_aper_fwhm:.2f} FWHM, " + f"annulus={selected_annulus_sigma:.2f} sigma, " + f"sample_field_score={aperture_tuning_sample_score * 100.0:.4f}%. " + "The selected 1x1 aperture will now be measured for every tracked star on every frame." + ) + del tuning_cutouts + log_info( + "Distributed aperture tuning completed in " + f"{perf_counter() - aperture_tuning_start:.2f}s." + ) + reset_photometry_timing_stats() use_multiprocess_transform_precompute = False fallback_transforms = pointing_alignment_transforms + use_memmap_initial_photometry = can_memmap_aperture_tuning_cutouts( + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + bad_pixel_reference=bad_pixel_reference, + ) + if use_memmap_initial_photometry: + log_info( + "Initial photometry image access: FITS memmap enabled; only tracked-star pixel " + "neighborhoods will be paged in." + ) + initial_photometry_start = perf_counter() for i, fileName in enumerate(inputfiles): plateStatus.setCurrentFilename(fileName) - hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, + frame_uses_memmap = use_memmap_initial_photometry + hdul = fits.open(name=fileName, memmap=frame_uses_memmap, cache=False, + lazy_load_hdus=frame_uses_memmap, ignore_missing_end=True) # Final reductions should always use the full centroid fit so the # centroid series does not inherit the fast moment-estimator cadence. @@ -29491,6 +31181,22 @@ def _main_impl(): extension += 1 image_header = hdul[extension].header + if frame_uses_memmap and not fits_header_supports_memmap(image_header): + hdul.close() + frame_uses_memmap = False + hdul = fits.open( + name=fileName, + memmap=False, + cache=False, + lazy_load_hdus=False, + ignore_missing_end=True, + ) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + airMassList.append(air_mass(image_header, pDict['ra'], pDict['dec'], exotic_infoDict['lat'], exotic_infoDict['long'], exotic_infoDict['elev'], jd_times[i])) @@ -29509,22 +31215,24 @@ def _main_impl(): imageData = hdul[extension].data # CALS - imageData = apply_cals(imageData, generalDark, generalBias, generalFlat, i) - # Demosaic, if needed - imageData = demosaic_img(imageData, demosaic_fmt, demosaic_out, demosaic_mult, i) - imageData = repair_bad_pixels_in_frame(imageData, bad_pixel_reference) + if not frame_uses_memmap: + imageData = apply_cals(imageData, generalDark, generalBias, generalFlat, i) + # Demosaic, if needed + imageData = demosaic_img(imageData, demosaic_fmt, demosaic_out, demosaic_mult, i) + imageData = repair_bad_pixels_in_frame(imageData, bad_pixel_reference) - if i == 0: + if i == 0 and multiprocess_alignment_results is None: firstImage = np.copy(imageData) if multiprocess_alignment_results is not None: - apply_parallel_alignment_result( - multiprocess_alignment_results[i], - i, - psf_data, - tar_comp_dist, - comp_alignment_keys, - ) + if not multiprocess_alignment_results_applied: + apply_parallel_alignment_result( + multiprocess_alignment_results[i], + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) else: alignment_result = { 'index': i, @@ -29580,22 +31288,33 @@ def _main_impl(): use_multiprocess_transform_precompute, ) - cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None - if cached_tform is not None: - tform = cached_tform - elif i == 0: - tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) - else: - tform = transformation(imageData, fileName, reference_image=firstImage) + if not wcs_alignment_candidate_is_acceptable( + alignment_result, + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ): + cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None + if cached_tform is not None: + tform = cached_tform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + else: + tform = downsampled_fallback_transformation( + imageData, + fileName, + reference_image=firstImage, + ) - transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) - alignment_result['fallback'] = _fit_alignment_candidate_psfs( - imageData, - transformed_coords, - target_fast_centroid, - frame_fast_centroid, - previous_psf_rows=previous_psf_rows, - ) + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + alignment_result['fallback'] = _fit_alignment_candidate_psfs( + imageData, + transformed_coords, + target_fast_centroid, + frame_fast_centroid, + previous_psf_rows=previous_psf_rows, + ) apply_parallel_alignment_result( alignment_result, i, @@ -29647,6 +31366,7 @@ def _main_impl(): comp_row[0] if comp_row.size > 0 else np.nan, comp_row[1] if comp_row.size > 1 else np.nan, overexposure_threshold, + starLabel=tracked_vsx_labels.get(comp_key), ) if use_psf_photometry: @@ -29710,8 +31430,12 @@ def _main_impl(): ) # aperture photometry - if use_aperture_photometry and i == 0: - sigma = psf_sigma_from_fit(psf_data['target'][0]) + if use_aperture_photometry and i == 0 and not aperture_preselected_from_sample: + sigma = aperture_frame_sigma_from_psf_data( + psf_data, + 0, + comparison_indices=aperture_frame_sigma_comp_indices, + ) if use_aperture_corrections_and_full_image_fwhm: image_fwhm = estimate_image_fwhm_from_isolated_stars( imageData, @@ -29730,7 +31454,33 @@ def _main_impl(): coarse_aperture_values = coarse_apertures_sigma * sigma coarse_annulus_values = coarse_annuli_sigma * sigma - if use_aperture_photometry and i < coarse_tune_frames: + if use_aperture_photometry and aperture_preselected_from_sample: + populate_aperture_data_for_frame( + imageData, + i, + psf_data, + comp_star_count, + aper_data, + aperture_values, + annulus_values, + fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=False, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + comp_indices=range(comp_star_count), + include_target=True, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, + ) + apply_overexposure_masks_to_aperture_frame( + aper_data, + i, + target_overexposed_frame_mask[i], + comp_overexposed_masks, + ) + elif use_aperture_photometry and i < coarse_tune_frames: coarse_frame_cache[i] = np.array(imageData, copy=True) populate_aperture_data_for_frame( imageData, @@ -29747,6 +31497,9 @@ def _main_impl(): noise_config=frame_noise_config, exposure_s=frame_exposure_s, airmass=frame_airmass, + comp_indices=aperture_estimation_comp_indices, + include_target=aperture_estimation_includes_target, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, ) apply_overexposure_masks_to_aperture_frame( coarse_aper_data, @@ -29763,13 +31516,13 @@ def _main_impl(): f"comp{comp_idx + 1}", coarse_tune_frames, ) - for comp_idx in range(comp_star_count) + for comp_idx in range(aperture_estimation_comp_count) } refined_apertures_sigma, refined_annuli_sigma, best_coarse_candidate, best_coarse_score = auto_tune_aperture_sigma_grid( coarse_apertures_sigma, coarse_annuli_sigma, coarse_aper_data, - comp_star_count, + aperture_estimation_comp_count, subset_airmass, require_comp_star=require_comp_star, skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, @@ -29801,7 +31554,10 @@ def _main_impl(): log_info(f"Backfilling refined aperture photometry for the first {coarse_tune_frames} frame(s).") for backfill_idx in range(coarse_tune_frames): - if target_overexposed_frame_mask[backfill_idx]: + if ( + aperture_estimation_includes_target + and target_overexposed_frame_mask[backfill_idx] + ): apply_overexposure_masks_to_aperture_frame( aper_data, backfill_idx, @@ -29842,6 +31598,9 @@ def _main_impl(): noise_config=frame_noise_configs[backfill_idx], exposure_s=exptimes[backfill_idx], airmass=airMassList[backfill_idx], + comp_indices=aperture_estimation_comp_indices, + include_target=aperture_estimation_includes_target, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, ) apply_overexposure_masks_to_aperture_frame( aper_data, @@ -29882,6 +31641,9 @@ def _main_impl(): noise_config=frame_noise_config, exposure_s=frame_exposure_s, airmass=frame_airmass, + comp_indices=aperture_estimation_comp_indices, + include_target=aperture_estimation_includes_target, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, ) apply_overexposure_masks_to_aperture_frame( aper_data, @@ -29895,9 +31657,211 @@ def _main_impl(): del hdul del imageData + completed_photometry_frames = i + 1 + if completed_photometry_frames == len(inputfiles) or completed_photometry_frames % 50 == 0: + log_info( + "Initial photometry progress: " + f"{completed_photometry_frames}/{len(inputfiles)} frame(s)." + ) + + log_info( + "Initial selected-aperture/PSF photometry completed in " + f"{perf_counter() - initial_photometry_start:.2f}s." + ) + plateStatus.logAggregatedWarningSummary() log_transform_timing_stats('Transformation timing summary (full reduction)') log_photometry_timing_stats('Photometry timing summary (full reduction)') log_reduction_timing_overview('Reduction timing overview (full reduction)') + reduction_stage_timer.checkpoint("WCS alignment, centroiding, and initial frame photometry") + + frozen_aperture_data = None + frozen_aperture_values = None + frozen_annulus_values = None + frozen_apers = None + frozen_annuli = None + frozen_backfill_comp_indices = tuple(range(aperture_estimation_comp_count, comp_star_count)) + if ( + use_aperture_photometry + and aper_data is not None + and aperture_values is not None + and annulus_values is not None + and (frozen_backfill_comp_indices or stellar_variability_only) + ): + full_airmass = np.asarray(airMassList, dtype=float) + aperture_reference_sigmas = np.asarray([ + aperture_frame_sigma_from_psf_data( + psf_data, + frame_index, + comparison_indices=aperture_frame_sigma_comp_indices, + ) + for frame_index in range(len(inputfiles)) + ], dtype=float) + aperture_reference_sigmas = aperture_reference_sigmas[ + np.isfinite(aperture_reference_sigmas) & (aperture_reference_sigmas > 0) + ] + aperture_reference_sigma = ( + float(np.median(aperture_reference_sigmas)) + if aperture_reference_sigmas.size + else finite_positive_or_nan(sigma) + ) + if not np.isfinite(aperture_reference_sigma) or aperture_reference_sigma <= 0: + aperture_reference_sigma = 1.0 + if use_adaptive_apertures: + aperture_grid_apers = np.asarray(aperture_values, dtype=float) * aperture_reference_sigma + aperture_grid_annuli = np.asarray(annulus_values, dtype=float) * aperture_reference_sigma + else: + aperture_grid_apers = np.asarray(aperture_values, dtype=float) + aperture_grid_annuli = np.asarray(annulus_values, dtype=float) + + aperture_estimation_calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + aperture_grid_apers, + aperture_grid_annuli, + full_airmass, + aperture_estimation_stars, + aperture_reference_sigma, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=False, + use_aperture_photometry=True, + comp_overexposed_masks=comp_overexposed_masks, + ) + if aperture_estimation_calibration is None: + log_info( + "Warning: science comparison stars did not yield a usable aperture-grid " + "selection, so additional tracked stars cannot be aperture-photometered " + "with a frozen science aperture. PSF photometry remains available when enabled.", + warn=True, + ) + else: + selected_aperture_index = int(aperture_estimation_calibration['a']) + selected_annulus_index = int(aperture_estimation_calibration['an']) + frozen_aperture_values = np.asarray([ + np.asarray(aperture_values, dtype=float)[selected_aperture_index] + ]) + frozen_annulus_values = np.asarray([ + np.asarray(annulus_values, dtype=float)[selected_annulus_index] + ]) + frozen_apers = np.asarray([aperture_estimation_calibration['aper']], dtype=float) + frozen_annuli = np.asarray([aperture_estimation_calibration['annulus']], dtype=float) + frozen_aperture_data = collapse_aperture_data_to_selected_grid_cell( + aper_data, + selected_aperture_index, + selected_annulus_index, + ) + estimator_description = ( + "the first five bright, reference-frame non-saturated, VSX-vetted " + "non-variable comparison stars" + if ( + stellar_variability_only + and use_ensemble_photometry_for_stellar_variability + and aperture_estimation_comp_count == 5 + ) + else f"{aperture_estimation_comp_count} science comparison star(s)" + ) + if aperture_preselected_from_sample: + score_delta = ( + aperture_estimation_calibration['field_score'] - aperture_tuning_sample_score + if np.isfinite(aperture_tuning_sample_score) + else np.nan + ) + delta_text = ( + f", delta={score_delta * 100.0:+.4f}%" + if np.isfinite(score_delta) + else "" + ) + log_info( + "Full-sequence aperture validation: " + f"aper={frozen_apers[0]:.2f}px, annulus={frozen_annuli[0]:.2f}px, " + f"field_score={aperture_estimation_calibration['field_score'] * 100.0:.4f}%" + f"{delta_text}. Target and all {comp_star_count} tracked star(s) were already " + "measured in the initial pass; frozen-aperture reread skipped." + ) + else: + log_info( + "Frozen aperture selected from the science reduction grid using " + f"{estimator_description}: aper={frozen_apers[0]:.2f}px, " + f"annulus={frozen_annuli[0]:.2f}px. " + + ( + "Measuring the variable science target once and " + if stellar_variability_only + else "Measuring " + ) + + f"{len(frozen_backfill_comp_indices)} additional tracked star(s) once " + "with this setup." + ) + + reset_photometry_timing_stats() + frozen_backfill_total = len(inputfiles) + for frozen_frame_index, frozen_file_name in enumerate(inputfiles): + active_comp_indices = [ + comp_idx + for comp_idx in frozen_backfill_comp_indices + if not comp_overexposed_masks.get( + f"comp{comp_idx + 1}", + np.zeros(frozen_backfill_total, dtype=bool), + )[frozen_frame_index] + ] + measure_frozen_target = bool( + stellar_variability_only + and not target_overexposed_frame_mask[frozen_frame_index] + ) + if active_comp_indices or measure_frozen_target: + frozen_image = load_calibrated_reduction_image( + frozen_file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + try: + populate_aperture_data_for_frame( + frozen_image, + frozen_frame_index, + psf_data, + comp_star_count, + frozen_aperture_data, + frozen_aperture_values, + frozen_annulus_values, + fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=( + use_aperture_corrections_and_full_image_fwhm + ), + noise_config=frame_noise_configs[frozen_frame_index], + exposure_s=exptimes[frozen_frame_index], + airmass=airMassList[frozen_frame_index], + comp_indices=active_comp_indices, + include_target=measure_frozen_target, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, + ) + finally: + del frozen_image + apply_overexposure_masks_to_aperture_frame( + frozen_aperture_data, + frozen_frame_index, + bool( + stellar_variability_only + and target_overexposed_frame_mask[frozen_frame_index] + ), + comp_overexposed_masks, + ) + completed_frozen_frames = frozen_frame_index + 1 + if ( + completed_frozen_frames == frozen_backfill_total + or completed_frozen_frames % 50 == 0 + ): + log_info( + "Frozen-aperture photometry progress: " + f"{completed_frozen_frames}/{frozen_backfill_total}" + ) + log_photometry_timing_stats( + 'Photometry timing summary (frozen-aperture additional stars)' + ) # Fortuitous VSX targets are independent science targets. Process them against # the full image sequence before the exoplanet target validity/overexposure mask @@ -29921,9 +31885,18 @@ def _main_impl(): if not np.isfinite(fortuitous_sigma_display) or fortuitous_sigma_display <= 0: fortuitous_sigma_display = 1.0 - fortuitous_apers = apers - fortuitous_annuli = annuli - if aperture_values is not None and annulus_values is not None: + fortuitous_aperture_data = frozen_aperture_data + fortuitous_apers = frozen_apers + fortuitous_annuli = frozen_annuli + if fortuitous_aperture_data is None: + fortuitous_aperture_data = aper_data + fortuitous_apers = apers + fortuitous_annuli = annuli + if ( + frozen_aperture_data is None + and aperture_values is not None + and annulus_values is not None + ): if use_adaptive_apertures: fortuitous_apers = ( np.asarray(aperture_values, dtype=float) * fortuitous_sigma_display @@ -29940,7 +31913,7 @@ def _main_impl(): ) fortuitous_comparison_calibration = select_comparison_calibrated_photometry( psf_data, - aper_data, + fortuitous_aperture_data, fortuitous_apers, fortuitous_annuli, full_airmass, @@ -29961,7 +31934,7 @@ def _main_impl(): jd_times, full_airmass, psf_data, - aper_data, + fortuitous_aperture_data, exotic_infoDict, psf_flux_data=fortuitous_psf_flux_source, psf_noise_data=psf_noise_data if use_psf_photometry else None, @@ -29971,8 +31944,18 @@ def _main_impl(): 'observed_filter', exotic_infoDict.get('filter'), ), + use_single_comparison=use_single_comparison_for_fortuitous_variables, ) + reduction_stage_timer.checkpoint("Aperture finalization and fortuitous-variable photometry") + + if stellar_variability_only and frozen_aperture_data is not None: + aper_data = frozen_aperture_data + apers = frozen_apers + annuli = frozen_annuli + aperture_values = frozen_aperture_values + annulus_values = frozen_annulus_values + # filter bad images badmask = np.isnan(psf_data["target"][:, 0]) | (psf_data["target"][:, 0] == 0) if aper_data is not None: @@ -30202,7 +32185,7 @@ def _main_impl(): comp_overexposed_masks=comp_overexposed_masks, ) - # Fortuitous-only sources were appended solely so the shared image pass could measure them. + # Fortuitous-only sources remain in the shared centroid/PSF tracks and frozen-aperture store. # Restore the science comparison list before normal target fitting and final metadata output. exotic_infoDict['comp_stars'] = [list(position) for position in science_comp_stars] @@ -30794,6 +32777,7 @@ def _main_impl(): log_info(f"Optimal Aperture: {np.round(display_aperture, 2)}") log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") log_info("*********************************************\n") + reduction_stage_timer.checkpoint("Comparison calibration and target light-curve selection") best_fit_lc = photometry_info['best_fit_lc'] bestCompStar = photometry_info['comp_star_num'] @@ -31339,6 +33323,8 @@ def _main_impl(): relative_flux_mask=None, background_series=observing_background_series) + reduction_stage_timer.checkpoint("Final stellar-variability light curve, plots, and diagnostics") + log_info("\n*********************************************************") log_info("FINAL STELLAR VARIABILITY ANALYSIS\n") log_info(" Analysis Mode: stellar variability only") @@ -31406,6 +33392,7 @@ def _main_impl(): log_info(f"\nError: Could not create AID_AAVSO.txt. {error_txt}\n\t{e}", error=True) log_info("Output Files Saved") + reduction_stage_timer.checkpoint("Output file generation") log_info("\n************************") log_info("End of Reduction Process") @@ -31928,6 +33915,7 @@ def _main_impl(): log_info(f"\nError: Could not create plate_status.csv. {error_txt}\n\t{e}", error=True) log_info("Output Files Saved") + reduction_stage_timer.checkpoint("Final transit analysis and output file generation") log_info("\n************************") log_info("End of Reduction Process") diff --git a/exotic/inputs.py b/exotic/inputs.py index 2f3edf13..1c94d4ac 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -212,6 +212,7 @@ def __init__(self, init_opt): 'stellar_variability_only': False, 'use_ensemble_photometry_for_stellar_variability': True, 'photometer_fortuitous_variables': True, + 'use_single_comparison_for_fortuitous_variables': True, 'use_nextastro_vsx_cache_first': False, 'detrend_on_outoftransit_baseline': True, 'final_fit_baseline_duration_multiplier': 1.0, @@ -494,6 +495,10 @@ def comp_params(self, init_file, planet_dict): 'photometer_fortuitous_variables', 'Photometer Fortuitous Variables? (y/n)', ), + 'use_single_comparison_for_fortuitous_variables': ( + 'use_single_comparison_for_fortuitous_variables', + 'Use Single Comparison for Fortuitous Variables? (y/n)', + ), 'use_nextastro_vsx_cache_first': ( 'use_nextastro_vsx_cache_first', 'Use NextAstro VSX Cache First? (y/n)', diff --git a/exotic/output_files.py b/exotic/output_files.py index 82538561..808da73f 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -2366,18 +2366,20 @@ def __init__(self, fit, p_dict, i_dict, auid, chart_id, vsp_params): self.dir = Path(self.i_dict['save']) self.vsp_params = vsp_params - def aavso(self): - first_vsp_param = self.vsp_params[0] if self.vsp_params else {} - comparison_metadata = aid_comparison_metadata(first_vsp_param) - ensemble_comparison_metadata = aid_ensemble_comparison_metadata(first_vsp_param) - variable_name = self.auid or self.p_dict.get('sName') or self.p_dict.get('pName') - - params_file = self.dir / safe_output_filename( + def _aavso_path(self): + return self.dir / safe_output_filename( "AID_AAVSO", self.p_dict['sName'], filename_date_token(self.i_dict['date']), extension="txt", ) + + def _write_aavso(self, params_file, use_row_names=False, include_comparison_metadata=True): + first_vsp_param = self.vsp_params[0] if self.vsp_params else {} + comparison_metadata = aid_comparison_metadata(first_vsp_param) + ensemble_comparison_metadata = aid_ensemble_comparison_metadata(first_vsp_param) + default_variable_name = self.auid or self.p_dict.get('sName') or self.p_dict.get('pName') + with params_file.open('w', encoding="utf-8") as f: f.write("#TYPE=EXTENDED\n" # fixed f"#OBSCODE={self.i_dict['aavso_num']}\n" # UI @@ -2396,9 +2398,9 @@ def aavso(self): "Space Telescope Science Institute.\n" "# Use of this data is governed by the AAVSO Data Usage Guidelines: " "aavso.org/data-usage-guidelines\n") - if comparison_metadata: + if include_comparison_metadata and comparison_metadata: f.write(f"#COMPARISON-CATALOG-XC={dumps(comparison_metadata, sort_keys=True)}\n") - if ensemble_comparison_metadata: + if include_comparison_metadata and ensemble_comparison_metadata: f.write(format_aavso_json_header( "ENSEMBLE-COMPARISONS-XC", ensemble_comparison_metadata, @@ -2406,6 +2408,9 @@ def aavso(self): f.write("#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,GROUP,CHART,NOTES\n") for vsp_p in self.vsp_params: + variable_name = default_variable_name + if use_row_names: + variable_name = vsp_p.get('_aid_name') or variable_name mag = format_magnitude(vsp_p.get('mag'), default=None) if mag is None: continue @@ -2415,6 +2420,18 @@ def aavso(self): f.write(f"{variable_name},{round(vsp_p['time'], 5)},{mag},{mag_err}," f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{cmag},na,na," f"{round(vsp_p['airmass'], 7)},na,{chart_id},na\n") + return params_file + + def aavso(self): + return self._write_aavso(self._aavso_path()) + + def combined_aavso(self): + """Write one AID file containing rows for multiple named variables.""" + return self._write_aavso( + self._aavso_path(), + use_row_names=True, + include_comparison_metadata=False, + ) def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): diff --git a/exotic/plate_status.py b/exotic/plate_status.py index 8c17bb80..eb257752 100644 --- a/exotic/plate_status.py +++ b/exotic/plate_status.py @@ -1,11 +1,23 @@ class PlateStatus: + WARNING_PROGRESS_INTERVAL = 100 + WARNING_CONDITION_TEXT = { + "outofframe": "outside the image", + "lowflux": "low flux", + "overexposed": "overexposed", + "skybg": "a sky-background failure", + } + def __init__(self, logfunc): self.statusByFilename = dict() self.filenameList = [] self.filename = "N/A" self.errorcodes = set() self.logfunc = logfunc + self.comparisonStarLabels = {} + self.aggregatedWarnings = {} + self.lastAggregatedWarningSummary = {} + self.aggregationNoticeLogged = False self.errorcodes.add("outofframe_target") self.errorcodes.add("lowflux_target") self.errorcodes.add("overexposed_target") @@ -26,7 +38,33 @@ def initializeComparisonStarCount(self, compCount: int): self.errorcodes.add(f"outofframe_comp{i+1}") self.errorcodes.add(f"lowflux_comp{i+1}") self.errorcodes.add(f"overexposed_comp{i+1}") - self.errorcodes.add(f"skybg_comp{i+1}") + self.errorcodes.add(f"skybg_comp{i+1}") + + def setComparisonStarLabels(self, labels=None): + normalized_labels = {} + for star_index, label in (labels or {}).items(): + try: + normalized_index = int(star_index) + except (TypeError, ValueError): + continue + if normalized_index <= 0 or not isinstance(label, str) or not label.strip(): + continue + normalized_labels[normalized_index] = label.strip() + self.comparisonStarLabels = normalized_labels + return self + + def _starLabel(self, starIndex: int): + if starIndex == 0: + return "Target star" + return self.comparisonStarLabels.get(starIndex, f"Comparison star #{starIndex}") + + def _displayFilename(self): + return str(self.filename).replace('\\', '/').rsplit('/', 1)[-1] + + def _conditionCountText(self, condition: str, count: int): + condition_text = self.WARNING_CONDITION_TEXT.get(condition, condition) + return f"{condition_text} in {count} frame(s)" + # Sets current filename (for any reported errors) - sets starIndex=0 (target) def setCurrentFilename(self, filename: str): filename = str(filename) @@ -35,7 +73,8 @@ def setCurrentFilename(self, filename: str): self.filename = filename return self # Log an error - def _logError(self, errorcode: str, message: str) -> None: + def _logError(self, errorcode: str, message: str, starIndex: int = None, + condition: str = None, starLabel: str = None) -> None: if self.filename not in self.statusByFilename: self.statusByFilename[self.filename] = {} rec = self.statusByFilename[self.filename] @@ -44,40 +83,133 @@ def _logError(self, errorcode: str, message: str) -> None: # Mark error on this file rec[errorcode] = True self.errorcodes.add(errorcode) - # And log new warning - self.logfunc(message, warn=True) + if starIndex is None or condition is None: + self.logfunc(message, warn=True) + return + + label = ( + starLabel.strip() + if isinstance(starLabel, str) and starLabel.strip() + else self._starLabel(starIndex) + ) + aggregate = self.aggregatedWarnings.setdefault(errorcode, { + 'star_index': starIndex, + 'label': label, + 'condition': condition, + 'count': 0, + }) + aggregate['label'] = label + aggregate['count'] += 1 + warning_count = aggregate['count'] + + if warning_count == 1: + self.logfunc(message, warn=True) + if not self.aggregationNoticeLogged: + self.logfunc( + "Plate-status detail: repeated frame-level star warnings are aggregated after " + "their first occurrence; running counts are reported every " + f"{self.WARNING_PROGRESS_INTERVAL} frames and exact per-frame flags are preserved " + "in the PlateStatus CSV." + ) + self.aggregationNoticeLogged = True + elif warning_count % self.WARNING_PROGRESS_INTERVAL == 0: + self.logfunc( + "Plate-status warning update: " + f"{label}: {self._conditionCountText(condition, warning_count)} so far.", + warn=True, + ) + + def logAggregatedWarningSummary(self): + current_counts = { + errorcode: aggregate['count'] + for errorcode, aggregate in self.aggregatedWarnings.items() + } + if current_counts == self.lastAggregatedWarningSummary: + return + + repeated = [ + aggregate for aggregate in self.aggregatedWarnings.values() + if aggregate['count'] > 1 + ] + self.lastAggregatedWarningSummary = current_counts + if not repeated: + return + + grouped = {} + for aggregate in repeated: + group = grouped.setdefault(aggregate['star_index'], { + 'label': aggregate['label'], + 'conditions': [], + 'total': 0, + }) + group['label'] = aggregate['label'] + group['conditions'].append( + self._conditionCountText(aggregate['condition'], aggregate['count']) + ) + group['total'] += aggregate['count'] + + self.logfunc( + "Plate-status warning summary: repeated frame-level diagnostics were aggregated; " + "exact per-frame flags are preserved in the PlateStatus CSV." + ) + for group in sorted( + grouped.values(), + key=lambda item: (-item['total'], item['label']), + ): + self.logfunc(f">-- {group['label']}: {'; '.join(group['conditions'])}.") + # Report out of frame warning for start ;index' (0=target, 1+=comp #N) def outOfFrameWarning(self, starIndex): - if starIndex == 0: # Target star - self._logError("outofframe_target", - f"Target star beyond edge of file {self.filename}") - else: - self._logError(f"outofframe_comp{starIndex}", - f"Comparison star #{starIndex} star beyond edge of file {self.filename}") + label = self._starLabel(starIndex) + errorcode = "outofframe_target" if starIndex == 0 else f"outofframe_comp{starIndex}" + self._logError( + errorcode, + f"{label} is beyond the edge of file {self._displayFilename()}", + starIndex=starIndex, + condition="outofframe", + starLabel=label, + ) # Report low flux amplitude warning for start ;index' (0=target, 1+=comp #N) def lowFluxAmplitudeWarning(self, starIndex: int, xc: float, yc: float): - if starIndex == 0: # Target star - self._logError("lowflux_target", - f"Measured flux for Target star is low in file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") - else: - self._logError(f"lowflux_comp{starIndex}", - f"Measured flux for Comparison star #{starIndex} is low in file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") + label = self._starLabel(starIndex) + errorcode = "lowflux_target" if starIndex == 0 else f"lowflux_comp{starIndex}" + self._logError( + errorcode, + f"Measured flux for {label} is low in file {self._displayFilename()} - " + f"are you sure there is a star at [{xc:.1f}, {yc:.1f}]?", + starIndex=starIndex, + condition="lowflux", + starLabel=label, + ) # Report overexposure warning for star index (0=target, 1+=comp #N) - def overexposedWarning(self, starIndex: int, xc: float, yc: float, threshold: float): - if starIndex == 0: # Target star - self._logError("overexposed_target", - f"Target star is overexposed in file {self.filename}; aperture pixels near [{xc:.1f}, {yc:.1f}] exceeded {threshold:.1f}.") - else: - self._logError(f"overexposed_comp{starIndex}", - f"Comparison star #{starIndex} is overexposed in file {self.filename}; aperture pixels near [{xc:.1f}, {yc:.1f}] exceeded {threshold:.1f}.") + def overexposedWarning(self, starIndex: int, xc: float, yc: float, threshold: float, + starLabel: str = None): + label = ( + starLabel.strip() + if isinstance(starLabel, str) and starLabel.strip() + else self._starLabel(starIndex) + ) + errorcode = "overexposed_target" if starIndex == 0 else f"overexposed_comp{starIndex}" + self._logError( + errorcode, + f"{label} is overexposed in file {self._displayFilename()}; aperture pixels near " + f"[{xc:.1f}, {yc:.1f}] exceeded {threshold:.1f}.", + starIndex=starIndex, + condition="overexposed", + starLabel=label, + ) # Report sky background warning for start ;index' (0=target, 1+=comp #N) def skyBackgroundWarning(self, starIndex: int, xc: float, yc: float): - if starIndex == 0: # Target star - self._logError("skybg_target", - f"Sky background error for Target star for file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") - else: - self._logError(f"skybg_comp{starIndex}", - f"Sky background error for Comparison star #{starIndex} for file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") + label = self._starLabel(starIndex) + errorcode = "skybg_target" if starIndex == 0 else f"skybg_comp{starIndex}" + self._logError( + errorcode, + f"Sky background error for {label} in file {self._displayFilename()} - " + f"are you sure there is a star at [{xc:.1f}, {yc:.1f}]?", + starIndex=starIndex, + condition="skybg", + starLabel=label, + ) # Reort file format error def fitsFormatError(self, e: OSError): self._logError("fits_error", @@ -94,6 +226,7 @@ def alignmentError(self): f"File {self.filename} failed to align with first file") # Write plate status to CSV file def writePlateStatus(self, file: str): + self.logAggregatedWarningSummary() with open(file, 'w') as f: cols = list(self.errorcodes) cols.sort() diff --git a/inits.json b/inits.json index d40464f6..6141ef57 100644 --- a/inits.json +++ b/inits.json @@ -30,7 +30,8 @@ "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, use the default calibrated comparison-star ensemble, and discard predicted ingress-to-egress transit-window points. Default false.", "Stellar Variability Ensemble": "Set optional_info 'use_ensemble_photometry_for_stellar_variability' to false to disable the default calibrated ensemble in stellar_variability_only runs and restore single-comparison selection by out-of-transit scatter. The default ensemble automatically finds bright catalog-calibrated field-star candidates, removes saturated and VSX-variable stars, sigma-clips high catalog magnitude uncertainties, and when more than five remain uses the five closest to the target in catalog colour and magnitude. EnsembleSelection JSON records the target and comparison colours, magnitudes, errors, and selection deltas beside the final results.", - "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent ensemble photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under fortuitous_variables/optimal_variables for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under fortuitous_variables/rest_of_the_variables.", + "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent calibrated photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under fortuitous_variables/optimal_variables for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under fortuitous_variables/rest_of_the_variables.", + "Fortuitous Variable Single Comparison": "Set optional_info 'use_single_comparison_for_fortuitous_variables' to false to use the calibrated comparison-star ensemble for fortuitous VSX targets. The default true selects one unsaturated, non-variable, catalog-calibrated comparison star closest to each VSX target in catalog colour and magnitude.", "NextAstro VSX Cache First": "Set optional_info 'use_nextastro_vsx_cache_first' to true to query the NextAstro /vsx_query field cache before AAVSO VSX. The default is false. Empty or failed cache lookups fall back to AAVSO; legacy cache responses lacking the full period/amplitude schema are enriched from AAVSO.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", @@ -128,6 +129,7 @@ "stellar_variability_only": false, "use_ensemble_photometry_for_stellar_variability": true, "photometer_fortuitous_variables": true, + "use_single_comparison_for_fortuitous_variables": true, "use_nextastro_vsx_cache_first": false, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "y", diff --git a/tests/test_aperture_tuning_performance.py b/tests/test_aperture_tuning_performance.py new file mode 100644 index 00000000..c87aaff7 --- /dev/null +++ b/tests/test_aperture_tuning_performance.py @@ -0,0 +1,143 @@ +import numpy as np +from astropy.io import fits + +import exotic.exotic as exotic_module + + +def test_evenly_spaced_aperture_tuning_indices_span_full_sequence(): + indices = exotic_module.evenly_spaced_aperture_tuning_indices(524, max_frames=24) + + assert len(indices) == 24 + assert indices[0] == 0 + assert indices[-1] == 523 + assert np.all(np.diff(indices) >= 22) + assert np.all(np.diff(indices) <= 23) + + +def test_centered_numpy_cutout_uses_local_slice_coordinates(): + image = np.arange(100, dtype=float).reshape(10, 10) + + cutout, local_x, local_y = exotic_module.centered_numpy_cutout( + image, + xc=5.25, + yc=4.75, + radius=2.0, + ) + + np.testing.assert_array_equal(cutout, image[2:8, 3:9]) + assert local_x == 2.25 + assert local_y == 2.75 + + +def test_memmap_cutouts_require_identity_frame_processing(): + empty = np.empty((0, 0)) + + assert exotic_module.can_memmap_aperture_tuning_cutouts( + generalDark=empty, + generalBias=empty, + generalFlat=empty, + demosaic_fmt=None, + bad_pixel_reference=None, + ) + assert not exotic_module.can_memmap_aperture_tuning_cutouts( + generalDark=np.ones((2, 2)), + ) + assert not exotic_module.can_memmap_aperture_tuning_cutouts( + demosaic_fmt="RGGB", + ) + assert not exotic_module.can_memmap_aperture_tuning_cutouts( + bad_pixel_reference={"coord_x": np.array([1]), "coord_y": np.array([1])}, + ) + + +def test_fits_header_memmap_guard_rejects_scaled_images(): + assert exotic_module.fits_header_supports_memmap({"BITPIX": -32}) + assert not exotic_module.fits_header_supports_memmap({"BSCALE": 2.0}) + assert not exotic_module.fits_header_supports_memmap({"BZERO": 32768}) + + +def test_alignment_worker_memmap_path_skips_full_frame_calibration(tmp_path, monkeypatch): + frame_path = tmp_path / "frame.fits" + expected = np.arange(100, dtype=np.float32).reshape(10, 10) + fits.PrimaryHDU(expected).writeto(frame_path) + empty = np.empty((0, 0)) + exotic_module._ALIGNMENT_POOL_CONTEXT = { + "generalDark": empty, + "generalBias": empty, + "generalFlat": empty, + "demosaic_fmt": None, + "demosaic_out": None, + "demosaic_mult": None, + "bad_pixel_reference": None, + } + monkeypatch.setattr( + exotic_module, + "apply_cals", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("identity memmap path must not calibrate the full image") + ), + ) + + header, image = exotic_module._load_alignment_worker_frame(str(frame_path)) + + assert header["NAXIS1"] == 10 + np.testing.assert_array_equal(image[2:5, 3:7], expected[2:5, 3:7]) + + +def test_aperture_tuning_cutout_grid_matches_direct_local_measurement(): + y, x = np.indices((41, 41), dtype=float) + image = 100.0 + 5000.0 * np.exp(-((x - 20.2) ** 2 + (y - 19.8) ** 2) / (2.0 * 2.0 ** 2)) + frame = { + "frame_sigma": 2.0, + "stars": { + "comp1": { + "data": image, + "xc": 20.2, + "yc": 19.8, + "sigma": 2.0, + } + }, + } + apertures = np.array([2.5, 3.0]) + annuli = np.array([6.0, 8.0]) + + result = exotic_module.populate_aperture_tuning_data_from_cutouts( + [frame], + apertures, + annuli, + comparison_indices=(0,), + adaptive_apertures=True, + reference_sigma=2.0, + ) + expected_flux, expected_bg = exotic_module.compute_star_aperture_grid( + image, + 1, + 20.2, + 19.8, + apertures * 2.0, + annuli * 2.0, + sigma_hint=2.0, + ) + + np.testing.assert_allclose(result["comp1"][0], expected_flux) + np.testing.assert_allclose(result["comp1_bg"][0], expected_bg) + + +def test_image_process_pool_uses_spawn_context_on_windows(monkeypatch): + captured = {} + + class FakeProcessPool: + def __init__(self, *args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + + fake_context = object() + monkeypatch.setattr(exotic_module.sys, "platform", "win32") + monkeypatch.setattr(exotic_module.multiprocessing, "get_context", lambda mode: fake_context) + monkeypatch.setattr(exotic_module, "_ProcessPoolExecutor", FakeProcessPool) + + executor = exotic_module.ImageProcessPoolExecutor(max_workers=3) + + assert isinstance(executor, FakeProcessPool) + assert captured["kwargs"]["max_workers"] == 3 + assert captured["kwargs"]["mp_context"] is fake_context diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 070ab59f..8b8621ff 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -776,6 +776,118 @@ def test_parallel_alignment_task_uses_precomputed_fallback_transform(monkeypatch assert np.allclose(result["fallback"]["coords"], [[6.0, 1.0], [8.0, 3.0]]) +def test_classify_wcs_fallback_frames_queues_only_missing_or_rejected_candidates(): + def candidate(target_xy, comp_xy): + coords = np.array([target_xy, comp_xy], dtype=float) + return { + "coords": coords, + "projected_off_frame": False, + "psf_rows": { + "target": np.array([*target_xy, 100.0, 2.0, 2.0, 0.0, 50.0]), + "comp1": np.array([*comp_xy, 90.0, 2.0, 2.0, 0.0, 50.0]), + }, + "warnings": [], + } + + results = [ + {"index": 0, "file_name": "frame0.fits", "wcs": candidate((10.0, 10.0), (20.0, 10.0)), "fallback": None}, + {"index": 1, "file_name": "frame1.fits", "wcs": candidate((11.0, 10.0), (50.0, 50.0)), "fallback": None}, + {"index": 2, "file_name": "frame2.fits", "wcs": None, "fallback": None}, + {"index": 3, "file_name": "frame3.fits", "wcs": candidate((13.0, 10.0), (23.0, 10.0)), "fallback": None}, + ] + + missing, rejected = exotic_module.classify_wcs_fallback_frames( + results, + np.array([[10.0, 10.0], [20.0, 10.0]]), + ) + + assert missing == [2] + assert rejected == [1] + + +def test_build_multiprocess_alignment_results_runs_legacy_batch_only_for_wcs_failures(monkeypatch): + def candidate(target_xy, comp_xy): + coords = np.array([target_xy, comp_xy], dtype=float) + return { + "coords": coords, + "projected_off_frame": False, + "psf_rows": { + "target": np.array([*target_xy, 100.0, 2.0, 2.0, 0.0, 50.0]), + "comp1": np.array([*comp_xy, 90.0, 2.0, 2.0, 0.0, 50.0]), + }, + "warnings": [], + } + + batches = [] + + def fake_run_batch(tasks, *_args, **_kwargs): + batches.append(tasks) + if len(batches) == 1: + return { + 0: {"index": 0, "file_name": "frame0.fits", "wcs": candidate((10.0, 10.0), (20.0, 10.0)), "fallback": None}, + 1: {"index": 1, "file_name": "frame1.fits", "wcs": candidate((11.0, 10.0), (21.0, 10.0)), "fallback": None}, + 2: {"index": 2, "file_name": "frame2.fits", "wcs": candidate((12.0, 10.0), (50.0, 50.0)), "fallback": None}, + 3: {"index": 3, "file_name": "frame3.fits", "wcs": None, "fallback": None}, + } + + return { + task[0]: { + "index": task[0], + "file_name": task[1], + "wcs": None, + "fallback": candidate((10.0 + task[0], 10.0), (20.0 + task[0], 10.0)), + } + for task in tasks + } + + monkeypatch.setattr(exotic_module, "_run_multiprocess_alignment_task_batch", fake_run_batch) + + results = exotic_module.build_multiprocess_alignment_results( + np.array(["frame0.fits", "frame1.fits", "frame2.fits", "frame3.fits"]), + 4, + np.array([[10.0, 10.0], [20.0, 10.0]]), + target_and_comp_radec=np.array([[1.0, 2.0], [1.1, 2.1]]), + compute_fallback_transform=True, + ) + + assert [task[0] for task in batches[0]] == [0, 1, 2, 3] + assert all(task[7] is False for task in batches[0]) + assert [task[0] for task in batches[1]] == [2, 3] + assert all(task[4] is True and task[7] is True for task in batches[1]) + assert results[0]["fallback"] is None + assert results[1]["fallback"] is None + assert results[2]["fallback"] is not None + assert results[3]["fallback"] is not None + + +def test_downsampled_fallback_transformation_restores_full_resolution_translation(monkeypatch): + calls = [] + + def fake_transformation(image_data, _file_name, **kwargs): + calls.append((image_data.shape, kwargs["reference_image"].shape)) + return exotic_module.SimilarityTransform( + scale=1.01, + rotation=0.02, + translation=[2.0, -3.0], + ) + + monkeypatch.setattr(exotic_module, "transformation", fake_transformation) + image = np.ones((8, 12), dtype=float) + reference = np.ones((8, 12), dtype=float) + + result = exotic_module.downsampled_fallback_transformation( + image, + "frame.fits", + reference_image=reference, + max_dimension=6, + ) + + assert calls == [((4, 6), (4, 6))] + assert result.scale == pytest.approx(1.01) + assert result.rotation == pytest.approx(0.02) + assert np.allclose(result.translation, [4.0, -6.0]) + + def test_fit_alignment_candidate_psfs_serializes_plate_status_swap(monkeypatch): sentinel_status = types.SimpleNamespace(name="original-plate-status") started = threading.Event() diff --git a/tests/test_ephemeris_validation.py b/tests/test_ephemeris_validation.py new file mode 100644 index 00000000..9154efe8 --- /dev/null +++ b/tests/test_ephemeris_validation.py @@ -0,0 +1,92 @@ +import numpy as np +import pytest + +from exotic.exotic import resolve_required_transit_ephemeris + + +def test_required_ephemeris_keeps_valid_initialization_values_without_archive_lookup(): + lookup_called = False + + def archive_lookup(): + nonlocal lookup_called + lookup_called = True + return {'pPer': 9.0, 'midT': 2469999.0} + + result = resolve_required_transit_ephemeris( + {'pName': 'Example b', 'pPer': 2.5, 'midT': 2460000.25}, + archive_lookup=archive_lookup, + ) + + assert result['pPer'] == 2.5 + assert result['midT'] == 2460000.25 + assert lookup_called is False + + +@pytest.mark.parametrize( + ('initial_values', 'expected_period', 'expected_tmid'), + [ + ({'pPer': None, 'midT': 2460000.25}, 2.5, 2460000.25), + ({'pPer': 0.0, 'midT': 2460000.25}, 2.5, 2460000.25), + ({'pPer': np.nan, 'midT': 2460000.25}, 2.5, 2460000.25), + ({'pPer': True, 'midT': 2460000.25}, 2.5, 2460000.25), + ({'pPer': 2.5, 'midT': None}, 2.5, 2460000.25), + ({'pPer': 2.5, 'midT': 0.0}, 2.5, 2460000.25), + ({'pPer': 2.5, 'midT': np.nan}, 2.5, 2460000.25), + ], +) +def test_required_ephemeris_fills_only_invalid_values_from_archive( + initial_values, expected_period, expected_tmid): + result = resolve_required_transit_ephemeris( + {'pName': 'Example b', **initial_values}, + archive_planet_dict={ + 'pPer': 2.5, + 'pPerUnc': 0.001, + 'midT': 2460000.25, + 'midTUnc': 0.002, + }, + ) + + assert result['pPer'] == expected_period + assert result['midT'] == expected_tmid + + +def test_required_ephemeris_copies_archive_uncertainty_with_fallback_value(): + result = resolve_required_transit_ephemeris( + { + 'pName': 'Example b', + 'pPer': None, + 'pPerUnc': None, + 'midT': 2460000.25, + 'midTUnc': 0.005, + }, + archive_planet_dict={ + 'pPer': 2.5, + 'pPerUnc': 0.001, + 'midT': 2461111.0, + 'midTUnc': 0.002, + }, + ) + + assert result['pPer'] == 2.5 + assert result['pPerUnc'] == 0.001 + assert result['midT'] == 2460000.25 + assert result['midTUnc'] == 0.005 + + +def test_required_ephemeris_fails_before_reduction_when_archive_values_are_unusable(): + with pytest.raises(ValueError, match=r"Cannot start EXOTIC reduction.*pPer.*midT"): + resolve_required_transit_ephemeris( + {'pName': 'Example b', 'pPer': None, 'midT': 0.0}, + archive_planet_dict={'pPer': np.nan, 'midT': None}, + ) + + +def test_required_ephemeris_reports_archive_lookup_failure(): + def archive_lookup(): + raise RuntimeError('archive unavailable') + + with pytest.raises(ValueError, match=r"NASA Exoplanet Archive fallback failed.*archive unavailable"): + resolve_required_transit_ephemeris( + {'pName': 'Example b', 'pPer': None, 'midT': 2460000.25}, + archive_lookup=archive_lookup, + ) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 8a46fa81..da93820b 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -128,6 +128,9 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: diagnose_lightcurve_fit_inputs, detrend_flux_on_out_of_transit_baseline, alignment_candidate_quality_score, + aperture_estimation_comparison_stars, + aperture_frame_sigma_from_psf_data, + collapse_aperture_data_to_selected_grid_cell, ensure_lightcurve_fit_failure_reason, evaluate_lightcurve_candidate, evaluate_transit_detection_qc, @@ -1956,6 +1959,178 @@ def fail_field_star_estimate(*_args, **_kwargs): assert np.isfinite(aper_data["target"][0, 0, 0]) +def test_stellar_variability_aperture_estimation_uses_first_five_vetted_comparisons(): + science_comp_stars = [[float(index), float(index + 100)] for index in range(8)] + + assert aperture_estimation_comparison_stars( + science_comp_stars, + stellar_variability_only=False, + ) == science_comp_stars + assert aperture_estimation_comparison_stars( + science_comp_stars, + stellar_variability_only=True, + ) == science_comp_stars[:5] + + +def test_stellar_variability_aperture_grid_excludes_variable_target_and_uses_comp_seeing(monkeypatch): + import exotic.exotic as exotic_module + + measured = [] + + def fake_compute_star_aperture_grid( + _data, + star_index, + _xc, + _yc, + apertures, + annuli, + **_kwargs, + ): + aperture_values = np.asarray(apertures, dtype=float).reshape(-1) + annulus_values = np.asarray(annuli, dtype=float).reshape(-1) + measured.append((star_index, aperture_values.copy(), annulus_values.copy())) + shape = (len(aperture_values), len(annulus_values)) + flux = np.full(shape, 100.0 + star_index, dtype=float) + background = np.full(shape, 10.0 + star_index, dtype=float) + noise = { + component: np.ones(shape, dtype=float) + for component in exotic_module.NOISE_BUDGET_COMPONENT_KEYS + } + return flux, background, noise + + monkeypatch.setattr(exotic_module, "compute_star_aperture_grid", fake_compute_star_aperture_grid) + psf_data = { + # The VSX science target deliberately has very different seeing. It must not + # determine the stellar-variability aperture grid. + "target": np.array([[10.0, 10.0, 100.0, 9.0, 9.0, 0.0, 0.0]]), + "comp1": np.array([[12.0, 10.0, 90.0, 2.0, 2.0, 0.0, 0.0]]), + "comp2": np.array([[14.0, 10.0, 80.0, 4.0, 4.0, 0.0, 0.0]]), + } + assert aperture_frame_sigma_from_psf_data( + psf_data, + 0, + comparison_indices=[0, 1], + ) == pytest.approx(3.0) + + full_grid = initialize_aperture_data_store(1, 1, 1, 2) + populate_aperture_data_for_frame( + np.zeros((25, 25), dtype=float), + 0, + psf_data, + 2, + full_grid, + np.array([2.0]), + np.array([8.0]), + fast_aperture_mask=False, + adaptive_apertures=True, + comp_indices=[0, 1], + include_target=False, + frame_sigma_comp_indices=[0, 1], + ) + + assert [star_index for star_index, _apers, _annuli in measured] == [1, 2] + assert all(apers[0] == pytest.approx(6.0) for _index, apers, _annuli in measured) + assert np.all(np.isnan(full_grid["target"])) + + frozen = collapse_aperture_data_to_selected_grid_cell(full_grid, 0, 0) + measured.clear() + populate_aperture_data_for_frame( + np.zeros((25, 25), dtype=float), + 0, + psf_data, + 2, + frozen, + np.array([2.0]), + np.array([8.0]), + fast_aperture_mask=False, + adaptive_apertures=True, + comp_indices=[], + include_target=True, + frame_sigma_comp_indices=[0, 1], + ) + + assert len(measured) == 1 + assert measured[0][0] == 0 + assert measured[0][1][0] == pytest.approx(6.0) + assert frozen["target"][0, 0, 0] == pytest.approx(100.0) + + +def test_frozen_aperture_path_only_grids_estimators_then_backfills_additional_stars(monkeypatch): + import exotic.exotic as exotic_module + + measured_star_indices = [] + + def fake_compute_star_aperture_grid( + _data, + star_index, + _xc, + _yc, + apertures, + annuli, + **_kwargs, + ): + measured_star_indices.append(star_index) + shape = (len(np.asarray(apertures).reshape(-1)), len(np.asarray(annuli).reshape(-1))) + flux = np.full(shape, 100.0 + star_index, dtype=float) + background = np.full(shape, 10.0 + star_index, dtype=float) + noise = { + component: np.full(shape, 1.0 + star_index, dtype=float) + for component in exotic_module.NOISE_BUDGET_COMPONENT_KEYS + } + return flux, background, noise + + monkeypatch.setattr(exotic_module, "compute_star_aperture_grid", fake_compute_star_aperture_grid) + psf_data = { + "target": np.array([[10.0, 10.0, 100.0, 2.0, 2.0, 0.0, 0.0]]), + "comp1": np.array([[12.0, 10.0, 90.0, 2.0, 2.0, 0.0, 0.0]]), + "comp2": np.array([[14.0, 10.0, 80.0, 2.0, 2.0, 0.0, 0.0]]), + "comp3": np.array([[16.0, 10.0, 70.0, 2.0, 2.0, 0.0, 0.0]]), + "comp4": np.array([[18.0, 10.0, 60.0, 2.0, 2.0, 0.0, 0.0]]), + } + full_grid = initialize_aperture_data_store(1, 2, 2, 4) + populate_aperture_data_for_frame( + np.zeros((25, 25), dtype=float), + 0, + psf_data, + 4, + full_grid, + np.array([3.0, 4.0]), + np.array([8.0, 10.0]), + fast_aperture_mask=False, + comp_indices=[0, 1], + ) + + assert measured_star_indices == [0, 1, 2] + assert np.all(np.isfinite(full_grid["target"])) + assert np.all(np.isfinite(full_grid["comp1"])) + assert np.all(np.isfinite(full_grid["comp2"])) + assert np.all(np.isnan(full_grid["comp3"])) + assert np.all(np.isnan(full_grid["comp4"])) + + frozen = collapse_aperture_data_to_selected_grid_cell(full_grid, 1, 0) + measured_star_indices.clear() + populate_aperture_data_for_frame( + np.zeros((25, 25), dtype=float), + 0, + psf_data, + 4, + frozen, + np.array([4.0]), + np.array([8.0]), + fast_aperture_mask=False, + comp_indices=[2, 3], + include_target=False, + ) + + assert measured_star_indices == [3, 4] + assert frozen["target"].shape == (1, 1, 1) + assert frozen["target"][0, 0, 0] == pytest.approx(100.0) + assert frozen["comp1"][0, 0, 0] == pytest.approx(101.0) + assert frozen["comp2"][0, 0, 0] == pytest.approx(102.0) + assert frozen["comp3"][0, 0, 0] == pytest.approx(103.0) + assert frozen["comp4"][0, 0, 0] == pytest.approx(104.0) + + def test_should_use_fast_target_centroid_disables_fast_sigma_path_for_adaptive_runs(): assert should_use_fast_target_centroid(1, adaptive_apertures=False) is True assert should_use_fast_target_centroid(6, adaptive_apertures=False) is False @@ -7569,10 +7744,15 @@ def fake_lc_fitter( def _run_main_until_vertical_flux_bound( monkeypatch, tmp_path, - disable_vertical_flux_normalization=Ellipsis, - random_seed=123, - override=True, - nasa_result=None): + disable_vertical_flux_normalization=Ellipsis, + random_seed=123, + override=True, + nasa_result=None, + target_ra=10.0, + target_dec=20.0, + ephemeris_overrides=None, + expected_ephemeris=None, + expected_error=None): import exotic.exotic as exotic_module class BoundReached(Exception): @@ -7593,8 +7773,8 @@ class BoundReached(Exception): ) user_pdict = { - "ra": 10.0, - "dec": 20.0, + "ra": target_ra, + "dec": target_dec, "pName": "Test Planet b", "sName": "Test Star", "pPer": 1.0, @@ -7622,6 +7802,8 @@ class BoundReached(Exception): "pm_ra": 0.0, "pm_dec": 0.0, } + if ephemeris_overrides: + user_pdict.update(ephemeris_overrides) exotic_info = { "save": tmp_path, "prered_file": prered_file, @@ -7664,8 +7846,9 @@ def prereduced(self, planet): monkeypatch.setattr(exotic_module, "Inputs", FakeInputs) if nasa_result is not None: class FakeNASAExoplanetArchive: - def __init__(self, planet): + def __init__(self, planet, non_interactive=False): self.planet = planet + self.non_interactive = non_interactive def planet_info(self): return nasa_result @@ -7679,6 +7862,9 @@ def planet_info(self): def fake_apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled): captured["disabled"] = disabled + if expected_ephemeris is not None: + assert prior['per'] == pytest.approx(expected_ephemeris['pPer']) + assert prior['tmid'] == pytest.approx(expected_ephemeris['midT']) raise BoundReached() monkeypatch.setattr( @@ -7687,6 +7873,11 @@ def fake_apply_vertical_flux_normalization_bound(prior, bounds, flux_values, dis fake_apply_vertical_flux_normalization_bound, ) + if expected_error is not None: + with pytest.raises(ValueError, match=expected_error): + exotic_module.main() + return None + with pytest.raises(BoundReached): exotic_module.main() @@ -7705,6 +7896,56 @@ def test_main_prereduced_respects_disable_vertical_flux_normalization_option(mon assert disabled is True +def test_main_prereduced_override_invalid_coordinates_use_nasa_fallback_without_prompt(monkeypatch, tmp_path): + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive coordinate resolution must not prompt"), + ) + + disabled = _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + override=True, + nasa_result=("Test Planet b", False, {"ra": 123.456, "dec": -45.678}), + target_ra="not-an-ra", + target_dec="not-a-dec", + ) + + assert disabled is False + + +def test_main_prereduced_override_missing_ephemeris_uses_nasa_fallback(monkeypatch, tmp_path): + archive_parameters = { + 'pPer': 2.5, + 'pPerUnc': 0.001, + 'midT': 2450000.25, + 'midTUnc': 0.002, + } + + disabled = _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + override=True, + nasa_result=("Test Planet b", False, archive_parameters), + ephemeris_overrides={'pPer': None, 'midT': 0.0}, + expected_ephemeris=archive_parameters, + ) + + assert disabled is False + + +def test_main_prereduced_stops_before_fitting_when_required_ephemeris_cannot_be_resolved( + monkeypatch, tmp_path): + _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + override=True, + nasa_result=("Test Planet b", False, {'pPer': np.nan, 'midT': None}), + ephemeris_overrides={'pPer': None, 'midT': 0.0}, + expected_error=r"Cannot start EXOTIC reduction.*pPer.*midT", + ) + + def test_main_prereduced_generates_seed_after_candidate_falls_back_to_inits(monkeypatch, tmp_path): disabled = _run_main_until_vertical_flux_bound( monkeypatch, diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 5f3ea0f4..934d8406 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -170,6 +170,38 @@ def test_comp_params_reads_fortuitous_variable_photometry_opt_out(tmp_path): assert inputs.info_dict["photometer_fortuitous_variables"] is False +def test_comp_params_defaults_fortuitous_variables_to_single_comparison(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_single_comparison_for_fortuitous_variables"] is True + + +def test_comp_params_reads_fortuitous_single_comparison_opt_out(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "use_single_comparison_for_fortuitous_variables": False, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_single_comparison_for_fortuitous_variables"] is False + + def test_comp_params_defaults_nextastro_vsx_cache_first_to_false(tmp_path): init_data = { "user_info": {}, @@ -1540,7 +1572,7 @@ def test_parse_aavso_prereduced_overrides_uses_known_filter_lookup_when_filter_x overrides = parse_aavso_prereduced_overrides(pre_reduced_file) assert overrides["filter"] == "CBB" - assert overrides["filter_desc"] == "Astrodon ExoPlanet-BB" + assert overrides["filter_desc"] == "CBB" assert overrides["wl_min"] == "500.0" assert overrides["wl_max"] == "1000.0" @@ -1557,7 +1589,7 @@ def test_parse_aavso_prereduced_overrides_uses_astrodon_exo_alias_lookup(tmp_pat overrides = parse_aavso_prereduced_overrides(pre_reduced_file) assert overrides["filter"] == "Astrodon-Exo" - assert overrides["filter_desc"] == "Astrodon ExoPlanet-BB" + assert overrides["filter_desc"] == "CBB" assert overrides["wl_min"] == "500.0" assert overrides["wl_max"] == "1000.0" diff --git a/tests/test_ld.py b/tests/test_ld.py index 71b5d368..5b0e732e 100644 --- a/tests/test_ld.py +++ b/tests/test_ld.py @@ -251,6 +251,33 @@ def test_photographic_filter_aliases_in_filter_column() -> None: } +def test_cbb_filter_uses_neutral_canonical_name() -> None: + observed_filter = {'filter': "CBB", 'name': None, 'wl_min': None, 'wl_max': None} + + setting_filter_values(observed_filter) + + assert observed_filter == { + 'filter': "CBB", + 'name': "CBB", + 'wl_min': "500.0", + 'wl_max': "1000.0", + } + + +def test_cbb_brand_names_remain_accepted_as_input_aliases() -> None: + for alias in ("Astrodon ExoPlanet-BB", "Astrodon-Exo", "Exop", "exo"): + observed_filter = {'filter': alias, 'name': None, 'wl_min': None, 'wl_max': None} + + setting_filter_values(observed_filter) + + assert observed_filter == { + 'filter': "CBB", + 'name': "CBB", + 'wl_min': "500.0", + 'wl_max': "1000.0", + } + + def test_additional_standard_filter_aliases_in_filter_column() -> None: alias_cases = [ ("bu", "Johnson U", "U", "333.8", "398.8"), @@ -269,8 +296,9 @@ def test_additional_standard_filter_aliases_in_filter_column() -> None: ("clearV", "MObs CV", "CV", "350.0", "850.0"), ("w", "MObs CV", "CV", "350.0", "850.0"), ("pl", "MObs CV", "CV", "350.0", "850.0"), - ("exo", "Astrodon ExoPlanet-BB", "CBB", "500.0", "1000.0"), - ("Astrodon-Exo", "Astrodon ExoPlanet-BB", "CBB", "500.0", "1000.0"), + ("exo", "CBB", "CBB", "500.0", "1000.0"), + ("Astrodon ExoPlanet-BB", "CBB", "CBB", "500.0", "1000.0"), + ("Astrodon-Exo", "CBB", "CBB", "500.0", "1000.0"), ] for alias, expected_filter, expected_name, expected_min, expected_max in alias_cases: diff --git a/tests/test_nea_nextastro_fallback.py b/tests/test_nea_nextastro_fallback.py index 8c44c50e..2b413928 100644 --- a/tests/test_nea_nextastro_fallback.py +++ b/tests/test_nea_nextastro_fallback.py @@ -2,7 +2,7 @@ import pytest import requests -from exotic.api.nea import NASAExoplanetArchive +from exotic.api.nea import NASAExoplanetArchive, planet_name_lookup_candidates class DummyResponse: @@ -97,3 +97,57 @@ def test_new_scrape_auto_marks_candidate_like_names_without_prompt(monkeypatch, output = capsys.readouterr().out assert f"Cannot find target ({planet_name}) in NASA Exoplanet Archive." in output assert f"Assuming {planet_name} is a planet candidate because {reason}." in output + + +def test_new_scrape_non_interactive_unknown_target_aborts_without_prompt(monkeypatch, tmp_path): + planet_name = 'Definitely Not A Planet b' + nea = NASAExoplanetArchive(planet_name, non_interactive=True) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(nea, 'planet_names', lambda filename="pl_names.json": None) + monkeypatch.setattr(nea, '_tap_query', lambda *args, **kwargs: pandas.DataFrame()) + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive NASA lookup must not prompt"), + ) + + with pytest.raises( + RuntimeError, + match=r"Non-interactive run cancelled: target \(Definitely Not A Planet b\) was not found", + ): + nea._new_scrape() + + +def test_planet_name_lookup_candidates_strip_phase_and_preserve_planet_letter(): + candidates = planet_name_lookup_candidates('field WASP-164 b ingress') + + assert 'field WASP-164 b' in candidates + assert 'field WASP-164b' in candidates + assert 'WASP-164b' in candidates + assert 'ingress' not in candidates + + +@pytest.mark.parametrize('phase', ['ingress', 'EGRESS']) +def test_new_scrape_resolves_phase_labeled_name_from_planet_cache( + monkeypatch, + tmp_path, + phase, +): + nea = NASAExoplanetArchive(f'WASP-164 b {phase}', non_interactive=True) + monkeypatch.chdir(tmp_path) + (tmp_path / 'pl_names.json').write_text( + '{"wasp164b": "WASP-164 b"}', + encoding='utf-8', + ) + + class LookupResolved(Exception): + pass + + def stop_after_name_resolution(*args, **kwargs): + assert nea.planet == 'WASP-164 b' + raise LookupResolved + + monkeypatch.setattr(nea, '_tap_query', stop_after_name_resolution) + + with pytest.raises(LookupResolved): + nea._new_scrape() diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 6e5f449e..b082e622 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -8,6 +8,8 @@ import pytest from tenacity import Future, RetryError +from exotic.plate_status import PlateStatus + fake_barycorrpy = types.ModuleType('barycorrpy') fake_utc_tdb = types.ModuleType('barycorrpy.utc_tdb') fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None @@ -408,7 +410,7 @@ def test_nextastro_photometry_catalog_match_floors_zero_magnitude_error(): assert match['error'] == pytest.approx(0.001) -def test_nextastro_photometry_catalog_match_rejects_high_magnitude_error(): +def test_nextastro_photometry_catalog_match_accepts_relaxed_v_magnitude_error(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], 'count': 1, @@ -427,7 +429,9 @@ def test_nextastro_photometry_catalog_match_rejects_high_magnitude_error(): match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') - assert match is None + assert match['mag_band'] == 'V' + assert match['error'] == pytest.approx(0.051) + assert match['uses_relaxed_bv_error_limit'] is True def test_nextastro_photometry_catalog_match_ignores_over_30_magnitudes(): @@ -487,6 +491,89 @@ def test_aavso_vsp_band_for_filter_uses_observed_filter_aliases(): assert exotic_module.aavso_vsp_band_for_filter('Sloan g') == 'SG' +def test_nextastro_catalog_band_tokens_keep_bessell_sloan_and_gaia_distinct(): + assert exotic_module.nextastro_photometry_band_candidates('bv', include_fallback=False) == [ + ('Vmag', 'err_Vmag', 'V') + ] + assert exotic_module.nextastro_photometry_band_candidates('bb', include_fallback=False) == [ + ('Bmag', 'err_Bmag', 'B') + ] + assert exotic_module.nextastro_photometry_band_candidates( + 'Sloan g', include_fallback=False + ) == [('g', 'dg', 'g')] + assert exotic_module.nextastro_photometry_band_candidates('G', include_fallback=False) == [] + assert exotic_module.nextastro_photometry_band_candidates('Gaia G', include_fallback=False) == [] + assert exotic_module.catalog_band_priority('V', 'bv') == 0 + assert exotic_module.catalog_band_priority('B', 'bb') == 0 + assert exotic_module.catalog_band_priority('g', 'Sloan g') == 0 + assert exotic_module.catalog_band_priority('g', 'G') == 1 + assert exotic_module.catalog_band_priority('g', 'Gaia G') == 1 + + +def test_nextastro_catalog_match_uses_relaxed_error_only_as_bv_fallback(): + catalog = { + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, 'ra': 10.00001, 'dec': 20.0, + 'Vmag': 12.1, 'err_Vmag': 0.07, + }, + { + 'id': 2, 'ra': 10.00010, 'dec': 20.0, + 'Vmag': 12.2, 'err_Vmag': 0.03, + }, + ], + } + + preferred = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'V') + assert preferred['id'] == 2 + assert preferred['uses_relaxed_bv_error_limit'] is False + + relaxed_v = exotic_module.nextastro_photometry_catalog_match( + {'row_format': 'objects', 'rows': [catalog['rows'][0]]}, + 10.0, + 20.0, + 'bv', + ) + assert relaxed_v['id'] == 1 + assert relaxed_v['error'] == pytest.approx(0.07) + assert relaxed_v['uses_relaxed_bv_error_limit'] is True + + relaxed_b = exotic_module.nextastro_photometry_catalog_match( + { + 'row_format': 'objects', + 'rows': [{'id': 3, 'ra': 10.0, 'dec': 20.0, 'Bmag': 13.0, 'err_Bmag': 0.10}], + }, + 10.0, + 20.0, + 'bb', + ) + assert relaxed_b['id'] == 3 + assert relaxed_b['uses_relaxed_bv_error_limit'] is True + + rejected_g = exotic_module.nextastro_photometry_catalog_match( + { + 'row_format': 'objects', + 'rows': [{'id': 4, 'ra': 10.0, 'dec': 20.0, 'g': 13.0, 'dg': 0.051}], + }, + 10.0, + 20.0, + 'Sloan g', + ) + assert rejected_g is None + + rejected_v = exotic_module.nextastro_photometry_catalog_match( + { + 'row_format': 'objects', + 'rows': [{'id': 5, 'ra': 10.0, 'dec': 20.0, 'Vmag': 13.0, 'err_Vmag': 0.101}], + }, + 10.0, + 20.0, + 'V', + ) + assert rejected_v is None + + def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], @@ -653,6 +740,36 @@ def fake_plot(params, save, s_name, label): assert params[0]['observed_filter'] == 'CV' +def test_build_stellar_variability_params_rejects_cross_band_calibration(tmp_path): + class DummyFit: + data = np.ones(3, dtype=float) + dataerr = np.full(3, 0.01, dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.array([1.1, 1.2, 1.3], dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + stellar_variability_target_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + with pytest.raises(RuntimeError, match='cross-band absolute calibration is not permitted'): + exotic_module.build_stellar_variability_params_from_fit( + DummyFit(), + { + 'mag': 11.615, + 'error': 0.001, + 'mag_band': 'g', + 'observed_filter': 'V', + }, + [100, 200], + 'NextAstro-invalid-g-reference', + tmp_path, + 'Host Star', + observed_filter='V', + ) + + def test_build_stellar_variability_params_uses_raw_ratio_and_per_exposure_errors(monkeypatch, tmp_path): comp_mag = 9.751 comp_mag_error = 0.018 @@ -1404,6 +1521,125 @@ def test_stellar_variability_ensemble_masks_saturated_frames_and_error_clips_mem assert selection['calibration_error_clip']['minimum_high_threshold'] == pytest.approx(0.01) +def test_stellar_variability_ensemble_skips_cross_band_catalog_reference(): + frame_count = 8 + ranked_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + ] + calibration_stars = { + 'C1': {'pos': [10, 20], 'mag': 11.0, 'error': 0.001, 'mag_band': 'g'}, + 'C2': {'pos': [30, 40], 'mag': 12.5, 'error': 0.011, 'mag_band': 'V'}, + } + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + { + 'comp1': np.full(frame_count, 2000.0), + 'comp2': np.full(frame_count, 1000.0), + }, + observed_filter='V', + min_members=1, + max_members=1, + ) + + assert [member['key'] for member in selection['members']] == ['comp2'] + assert selection['members'][0]['star']['mag_band'] == 'V' + rejected = {item['key']: item['reason'] for item in selection['rejected']} + assert rejected['comp1'] == 'no usable catalog calibration' + + +def test_stellar_variability_rejects_comparison_that_steps_across_acquisition_gap(): + frame_count = 180 + cadence_days = 6.0 / 86400.0 + times = 2460000.0 + (np.arange(frame_count, dtype=float) * cadence_days) + times[90:] += 120.0 / 86400.0 + ranked_summaries = [ + { + 'key': f'comp{index}', + 'comp_index': index - 1, + 'label': f'Comp {index}', + 'position': [10 * index, 20 * index], + 'overexposure_rejected_count': 0, + } + for index in range(1, 5) + ] + calibration_stars = { + f'C{index}': { + 'pos': [10 * index, 20 * index], + 'mag': 12.0 + (0.1 * index), + 'error': 0.01, + 'mag_band': 'V', + 'catalog_source': 'Synthetic catalog', + } + for index in range(1, 5) + } + phase = np.linspace(0, 4 * np.pi, frame_count) + common = 1000.0 * (1.0 + (0.01 * np.sin(phase))) + comp_flux_map = { + f'comp{index}': common * (1.0 + (0.001 * index * np.cos(phase))) + for index in range(1, 5) + } + # A +0.06 mag instrumental discontinuity in comparison 1. + comp_flux_map['comp1'] = comp_flux_map['comp1'].copy() + comp_flux_map['comp1'][90:] *= 10.0 ** (-0.4 * 0.06) + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + min_members=1, + max_members=1, + times=times, + ) + + assert selection['gap_stability']['applied'] is True + assert selection['gap_stability']['boundaries'][0]['source_index'] == 90 + assert selection['gap_stability']['candidates']['comp1']['rejected'] is True + assert selection['gap_stability']['candidates']['comp1'][ + 'maximum_absolute_step_magnitude' + ] == pytest.approx(0.06, abs=0.003) + assert selection['gap_stability']['candidates']['comp1']['label'] == 'C1' + assert selection['gap_stability']['candidates']['comp1']['position'] == [10, 20] + assert selection['gap_stability']['candidates']['comp1']['catalog_magnitude_band'] == 'V' + assert selection['gap_stability']['candidates']['comp1']['catalog_source'] == 'Synthetic catalog' + assert selection['members'][0]['key'] != 'comp1' + rejected = {item['key']: item for item in selection['rejected']} + assert rejected['comp1']['label'] == 'C1' + assert rejected['comp1']['position'] == [10, 20] + assert 'changed discontinuously' in rejected['comp1']['reason'] + + +def test_fortuitous_output_error_limit_relaxes_only_for_flagged_bv_catalog_reference(): + assert exotic_module.fortuitous_output_magnitude_error_limit([ + {'star': {'mag_band': 'V', 'uses_relaxed_bv_error_limit': True}} + ]) == pytest.approx(0.10) + assert exotic_module.fortuitous_output_magnitude_error_limit([ + {'star': {'mag_band': 'B', 'uses_relaxed_bv_error_limit': True}} + ]) == pytest.approx(0.10) + assert exotic_module.fortuitous_output_magnitude_error_limit([ + {'star': {'mag_band': 'g', 'uses_relaxed_bv_error_limit': True}} + ]) == pytest.approx(0.05) + assert exotic_module.fortuitous_output_magnitude_error_limit([ + {'star': {'mag_band': 'V', 'uses_relaxed_bv_error_limit': False}} + ]) == pytest.approx(0.05) + + metadata = exotic_module.fortuitous_variable_target_metadata({ + 'name': 'Synthetic', + 'reference_mode': 'single_comparison', + 'output_magnitude_error_limit': 0.10, + }) + assert metadata['detection_magnitude_error_limit'] == pytest.approx(0.05) + assert metadata['output_magnitude_error_limit'] == pytest.approx(0.10) + + def test_stellar_variability_ensemble_error_clip_does_not_reject_below_point_zero_one_mag(): candidates = [ {'magnitude_error': error} @@ -1430,6 +1666,58 @@ def test_automatic_comparison_merge_deduplicates_only_added_sources(): assert len(messages) == 2 +def test_full_field_vsx_variables_are_removed_from_science_comparisons(): + comparison_stars = [ + [4969.0, 1695.0], + [5221.0, 2714.0], + [3923.0, 1362.0], + ] + fortuitous_variables = [ + {'name': 'DI Her', 'pos': [4971.55, 1695.42]}, + { + 'name': 'ASASSN-V J185327.35+241158.6', + 'x': 3926.38, + 'y': 1360.36, + }, + ] + + retained, rejected = exotic_module.filter_comparison_stars_against_fortuitous_variables( + comparison_stars, + fortuitous_variables, + duplicate_radius_pixels=10.0, + ) + + assert retained == [[5221.0, 2714.0]] + assert [item['comparison_index'] for item in rejected] == [0, 2] + assert [item['variable_name'] for item in rejected] == [ + 'DI Her', + 'ASASSN-V J185327.35+241158.6', + ] + assert rejected[0]['distance_pixels'] == pytest.approx(2.5840, abs=1.0e-3) + assert rejected[1]['distance_pixels'] == pytest.approx(3.7563, abs=1.0e-3) + + +def test_tracked_vsx_overexposure_warning_does_not_call_variable_a_comparison_star(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.setCurrentFilename('frame.fits') + + status.overexposedWarning( + 12, + 4973.2, + 1676.6, + 58981.5, + starLabel='Tracked VSX variable DI Her', + ) + + assert messages[0] == ( + 'Tracked VSX variable DI Her is overexposed in file frame.fits; ' + 'aperture pixels near [4973.2, 1676.6] exceeded 58981.5.' + ) + assert 'repeated frame-level star warnings are aggregated' in messages[1] + assert 'Comparison star' not in messages[0] + + def test_stellar_variability_ensemble_caps_at_five_by_target_color_and_magnitude(): frame_count = 8 target_match = { @@ -1668,6 +1956,7 @@ def test_build_stellar_variability_ensemble_params_preserves_member_metadata(mon ), ) fit = types.SimpleNamespace( + time=np.array([2460000.105, 2460000.205]), jd_times=np.array([2460000.1, 2460000.2]), airmass=np.array([1.1, 1.2]), stellar_variability_ensemble_magnitudes=np.array([12.30, 12.31]), @@ -1694,6 +1983,7 @@ def test_build_stellar_variability_ensemble_params_preserves_member_metadata(mon ) assert len(params) == 2 + assert [row['time'] for row in params] == pytest.approx([2460000.105, 2460000.205]) assert params[0]['cname'] == 'ENSEMBLE (2 stars)' assert params[0]['cmag'] is None assert params[0]['ensemble_member_labels'] == ['C1', 'C2'] @@ -1769,7 +2059,7 @@ def test_stellar_variability_ensemble_selection_json_lists_color_and_magnitude(m assert payload['ensemble']['members'][0]['magnitude_delta'] == pytest.approx(0.1) -def test_process_fortuitous_variable_writes_independent_ensemble_products(monkeypatch, tmp_path): +def test_process_fortuitous_variables_write_independent_and_combined_aid_products(monkeypatch, tmp_path): monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) monkeypatch.setattr( exotic_module, @@ -1809,6 +2099,7 @@ def test_process_fortuitous_variable_writes_independent_ensemble_products(monkey 'comp1': np.ones((frame_count, 7)), 'comp2': np.ones((frame_count, 7)), 'comp3': np.ones((frame_count, 7)), + 'comp4': np.ones((frame_count, 7)), } aper_data = { 'target': np.full((frame_count, 1, 1), 1000.0), @@ -1820,6 +2111,10 @@ def test_process_fortuitous_variable_writes_independent_ensemble_products(monkey 800.0 * (1.0 + 0.02 * np.sin(np.linspace(0, 2 * np.pi, frame_count))) )[:, None, None], 'comp3_unc': np.full((frame_count, 1, 1), 1.0), + 'comp4': ( + 700.0 * (1.0 + 0.01 * np.cos(np.linspace(0, 2 * np.pi, frame_count))) + )[:, None, None], + 'comp4_unc': np.full((frame_count, 1, 1), 1.0), } # One otherwise valid frame has an internal target error above 0.05 mag. aper_data['comp3_unc'][2, 0, 0] = 80.0 @@ -1858,6 +2153,15 @@ def test_process_fortuitous_variable_writes_independent_ensemble_products(monkey 'catalog_row': {'Bmag': 12.95, 'Vmag': 12.4}, }, } + second_variable = { + **variable, + 'name': 'Synthetic VSX 2', + 'auid': '000-AAA-002', + 'pos': [70, 80], + 'tracking_key': 'comp4', + 'aperture_flux_adu': 700.0, + 'count_rate_adu_per_second': 11.7, + } info_dict = { 'save': str(tmp_path), 'date': '2024-01-02', @@ -1872,7 +2176,7 @@ def test_process_fortuitous_variable_writes_independent_ensemble_products(monkey variable_overexposed = np.zeros(frame_count, dtype=bool) variable_overexposed[:2] = True results = exotic_module.process_fortuitous_variables( - [variable], + [variable, second_variable], comparison_calibration, calibrations, times, @@ -1884,6 +2188,7 @@ def test_process_fortuitous_variable_writes_independent_ensemble_products(monkey comp_overexposed_masks={'comp3': variable_overexposed}, exposure_times_seconds=np.full(frame_count, 60.0), observed_filter='V', + use_single_comparison=False, ) assert results[0]['status'] == 'completed' @@ -1893,14 +2198,36 @@ def test_process_fortuitous_variable_writes_independent_ensemble_products(monkey assert results[0]['point_count'] == frame_count - 3 variable_dir = tmp_path / 'fortuitous_variables' / 'optimal_variables' / 'VSX_SyntheticVSX' assert next(variable_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')).is_file() + second_variable_dir = ( + tmp_path / 'fortuitous_variables' / 'optimal_variables' / 'VSX_SyntheticVSX2' + ) + assert next(second_variable_dir.glob('AID_AAVSO_SyntheticVSX2_2024-01-02.txt')).is_file() assert next(variable_dir.glob('EnsembleSelection_SyntheticVSX_2024-01-02.json')).is_file() assert next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')).is_file() + combined_aid_path = ( + tmp_path / 'fortuitous_variables' / 'AID_AAVSO_FortuitousVariables_2024-01-02.txt' + ) + combined_aid_text = combined_aid_path.read_text(encoding='utf-8') + combined_aid_rows = [ + line for line in combined_aid_text.splitlines() + if line and not line.startswith('#') + ] + assert combined_aid_text.count('#TYPE=EXTENDED') == 1 + assert '#ENSEMBLE-COMPARISONS-XC=' not in combined_aid_text + assert len(combined_aid_rows) == sum(result['point_count'] for result in results) + assert {row.split(',', 1)[0] for row in combined_aid_rows} == { + '000-AAA-001', + '000-AAA-002', + } manifest = json.loads( next((tmp_path / 'fortuitous_variables').glob('FortuitousVariables_2024-01-02.json')).read_text( encoding='utf-8' ) ) + assert manifest['combined_aid'] == str(combined_aid_path) assert manifest['variables'][0]['ensemble_member_count'] == 2 + assert manifest['variables'][0]['comparison_gap_stability']['applied'] is False + assert manifest['variables'][0]['comparison_gap_rejected_candidates'] == [] assert manifest['variables'][0]['output_magnitude_error_rejected_frame_count'] == 1 assert manifest['variables'][0]['output_magnitude_error_max'] > 0.05 selection = json.loads( @@ -1937,6 +2264,46 @@ def test_process_fortuitous_variable_writes_independent_ensemble_products(monkey assert exported_errors assert max(exported_errors) < 0.05 + single_root = tmp_path / 'single' + single_info = {**info_dict, 'save': str(single_root)} + single_results = exotic_module.process_fortuitous_variables( + [variable], + comparison_calibration, + calibrations, + times, + times - 0.005, + np.linspace(1.1, 1.3, frame_count), + psf_data, + aper_data, + single_info, + comp_overexposed_masks={'comp3': variable_overexposed}, + exposure_times_seconds=np.full(frame_count, 60.0), + observed_filter='V', + ) + + assert single_results[0]['status'] == 'completed' + assert single_results[0]['reference_mode'] == 'single_comparison' + assert single_results[0]['comparison_member_count'] == 1 + assert single_results[0]['comparison_label'] == 'C2' + single_dir = ( + single_root / 'fortuitous_variables' / 'optimal_variables' / 'VSX_SyntheticVSX' + ) + assert not list(single_dir.glob('EnsembleSelection_*.json')) + single_csv = next(single_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')) + single_csv_rows = [ + line for line in single_csv.read_text(encoding='utf-8').splitlines()[1:] + if line.strip() + ] + assert float(single_csv_rows[0].split(',')[0]) == pytest.approx(times[3]) + single_aid = next(single_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')) + single_aid_text = single_aid.read_text(encoding='utf-8') + assert '#ENSEMBLE-COMPARISONS-XC=' not in single_aid_text + single_aid_row = next( + line for line in single_aid_text.splitlines() + if line and not line.startswith('#') + ) + assert single_aid_row.split(',')[7] == 'C2' + def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(): frame_count = 12 diff --git a/tests/test_nonlinear_ld.py b/tests/test_nonlinear_ld.py index 435650a4..9ec70fa7 100644 --- a/tests/test_nonlinear_ld.py +++ b/tests/test_nonlinear_ld.py @@ -26,6 +26,29 @@ def test_nonlinear_ld_non_interactive_treats_g_as_photographic_g_without_prompt( assert info_dict["wl_max"] == 586.8 +def test_nonlinear_ld_non_interactive_uses_neutral_cbb_name_without_prompt(monkeypatch): + ld = exotic_module.LimbDarkening({}) + monkeypatch.setattr(ld, "calculate_ld", lambda: None) + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("recognized CBB filter must not prompt"), + ) + info_dict = { + "filter": "CBB", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": "y", + } + + exotic_module.nonlinear_ld(ld, info_dict, non_interactive_run=True) + + assert info_dict["filter"] == "CBB" + assert info_dict["filter_desc"] == "CBB" + assert info_dict["wl_min"] == 500.0 + assert info_dict["wl_max"] == 1000.0 + + class UnrecognizedFilterLimbDarkening: fwhm_names_nonspecific = {} diff --git a/tests/test_plate_status.py b/tests/test_plate_status.py new file mode 100644 index 00000000..d8eb6260 --- /dev/null +++ b/tests/test_plate_status.py @@ -0,0 +1,107 @@ +import csv + +from exotic.plate_status import PlateStatus + + +def test_out_of_frame_warning_reports_only_fits_basename(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.setCurrentFilename('/mnt/data/session/TIC 13510052901-R-20240403-030-063313_out.fits') + + status.outOfFrameWarning(11) + + assert messages[0] == ( + 'Comparison star #11 is beyond the edge of file ' + 'TIC 13510052901-R-20240403-030-063313_out.fits' + ) + assert 'repeated frame-level star warnings are aggregated' in messages[1] + + +def test_out_of_frame_warning_preserves_fits_fz_basename(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.setCurrentFilename(r'C:\data\session\compressed-frame.fits.fz') + + status.outOfFrameWarning(0) + + assert messages[0] == 'Target star is beyond the edge of file compressed-frame.fits.fz' + + +def test_tracked_vsx_label_is_used_for_every_star_warning_type(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.setComparisonStarLabels({18: 'Tracked VSX variable DI Her'}) + + status.setCurrentFilename('/mnt/data/frame-1.fits.fz') + status.outOfFrameWarning(18) + status.setCurrentFilename('/mnt/data/frame-2.fits.fz') + status.lowFluxAmplitudeWarning(18, 123.4, 234.5) + status.setCurrentFilename('/mnt/data/frame-3.fits.fz') + status.overexposedWarning(18, 124.4, 235.5, 58981.5) + status.setCurrentFilename('/mnt/data/frame-4.fits.fz') + status.skyBackgroundWarning(18, 125.4, 236.5) + + warning_messages = [message for message in messages if 'file frame-' in message] + assert len(warning_messages) == 4 + assert all('Tracked VSX variable DI Her' in message for message in warning_messages) + assert all('Comparison star' not in message for message in warning_messages) + assert all('/mnt/data/' not in message for message in warning_messages) + + +def test_repeated_frame_warnings_are_aggregated_with_progress_and_summary(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + + for frame_index in range(205): + status.setCurrentFilename(f'/mnt/data/frame-{frame_index:04d}.fits') + status.overexposedWarning(1, 100.0, 200.0, 50000.0) + + status.logAggregatedWarningSummary() + + assert sum( + 'Comparison star #1 is overexposed in file' in message + for message in messages + ) == 1 + assert any( + 'Comparison star #1: overexposed in 100 frame(s) so far.' in message + for message in messages + ) + assert any( + 'Comparison star #1: overexposed in 200 frame(s) so far.' in message + for message in messages + ) + assert messages[-1] == '>-- Comparison star #1: overexposed in 205 frame(s).' + assert sum( + 'overexposed_comp1' in frame_status + for frame_status in status.statusByFilename.values() + ) == 205 + + message_count = len(messages) + status.logAggregatedWarningSummary() + assert len(messages) == message_count + + +def test_aggregated_warnings_preserve_exact_per_frame_csv_flags(tmp_path): + messages = [] + filenames = [f'/mnt/data/frame-{frame_index:04d}.fits' for frame_index in range(3)] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.initializeFilenames(filenames) + status.initializeComparisonStarCount(1) + + for filename in filenames: + status.setCurrentFilename(filename) + status.overexposedWarning(1, 100.0, 200.0, 50000.0) + + output_path = tmp_path / 'PlateStatus.csv' + status.writePlateStatus(output_path) + + with output_path.open(newline='') as handle: + rows = list(csv.reader(handle)) + header = rows[0] + overexposed_column = header.index('overexposed_comp1') + assert [row[overexposed_column] for row in rows[1:]] == ['True', 'True', 'True'] + assert sum( + 'Comparison star #1 is overexposed in file' in message + for message in messages + ) == 1 + assert messages[-1] == '>-- Comparison star #1: overexposed in 3 frame(s).' diff --git a/tests/test_radec_non_interactive.py b/tests/test_radec_non_interactive.py new file mode 100644 index 00000000..51f71ef9 --- /dev/null +++ b/tests/test_radec_non_interactive.py @@ -0,0 +1,75 @@ +import pytest + +from exotic import exotic as exotic_module + + +def test_invalid_target_coordinates_use_nasa_archive_fallback_without_prompt(monkeypatch): + messages = [] + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive coordinate resolution must not prompt"), + ) + monkeypatch.setattr( + exotic_module, + 'log_info', + lambda message, **kwargs: messages.append((message, kwargs)), + ) + + ra, dec = exotic_module.radec_hours_to_degree( + 'not-an-ra', + '+20:00:00', + non_interactive_run=True, + archive_ra=123.456, + archive_dec=-45.678, + target_name='Example b', + ) + + assert ra == pytest.approx(123.456) + assert dec == pytest.approx(-45.678) + assert len(messages) == 1 + assert "Using NASA Exoplanet Archive coordinates" in messages[0][0] + assert messages[0][1] == {'warn': True} + + +def test_invalid_target_and_archive_coordinates_abort_without_prompt(monkeypatch): + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive coordinate resolution must not prompt"), + ) + + with pytest.raises( + ValueError, + match=( + r"Non-interactive run cancelled for target Example b: .*" + r"NASA Exoplanet Archive coordinates .* are also unusable" + ), + ): + exotic_module.radec_hours_to_degree( + 'not-an-ra', + '+20:00:00', + non_interactive_run=True, + archive_ra='also-not-an-ra', + archive_dec='also-not-a-dec', + target_name='Example b', + ) + + +def test_invalid_target_coordinates_abort_when_archive_coordinates_unavailable(monkeypatch): + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive coordinate resolution must not prompt"), + ) + + with pytest.raises( + ValueError, + match=( + r"Non-interactive run cancelled for target Example b: .*" + r"NASA Exoplanet Archive coordinates are unavailable" + ), + ): + exotic_module.radec_hours_to_degree( + 'not-an-ra', + '+20:00:00', + non_interactive_run=True, + target_name='Example b', + ) diff --git a/tests/test_runtime_timing.py b/tests/test_runtime_timing.py new file mode 100644 index 00000000..a5fc70a3 --- /dev/null +++ b/tests/test_runtime_timing.py @@ -0,0 +1,29 @@ +from datetime import datetime + +import pytest + +from exotic import exotic as exotic_module + + +def test_format_clock_log_message_uses_hour_and_minute_and_preserves_leading_newlines(): + message = exotic_module.format_clock_log_message( + "\n\nStarting reduction", + clock_time=datetime(2026, 7, 16, 7, 5, 42), + ) + + assert message == "\n\n[07:05] Starting reduction" + + +def test_reduction_stage_timer_logs_step_and_total_elapsed_seconds(monkeypatch): + clock_values = iter([100.0, 102.5, 109.0]) + logged = [] + monkeypatch.setattr(exotic_module, "log_info", logged.append) + + timer = exotic_module.ReductionStageTimer(time_source=lambda: next(clock_values)) + + assert timer.checkpoint("first stage") == pytest.approx(2.5) + assert timer.checkpoint("second stage") == pytest.approx(6.5) + assert logged == [ + "STEP TIMING | first stage | elapsed_s=2.50 | total_s=2.50", + "STEP TIMING | second stage | elapsed_s=6.50 | total_s=9.00", + ] From 179aa5fda7361d326eb0c7a32d9118fd0f2cd38a Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 17 Jul 2026 09:29:00 +1000 Subject: [PATCH 085/116] clean up directory structure. --- exotic/api/plate_solution.py | 4 +- exotic/exotic.py | 175 ++++++++++++++++++++++++----- exotic/exotic_gui.py | 4 +- exotic/output_files.py | 53 ++++++++- exotic/plots.py | 52 +++++---- inits.json | 4 +- tests/test_centroid_wcs.py | 4 +- tests/test_exotic_proper_motion.py | 22 ++-- tests/test_exotic_rprs_retry.py | 87 ++++++++++++++ tests/test_nextastro_astrometry.py | 8 +- tests/test_output_files.py | 102 ++++++++++------- tests/test_plots.py | 36 +++--- 12 files changed, 416 insertions(+), 135 deletions(-) diff --git a/exotic/api/plate_solution.py b/exotic/api/plate_solution.py index 4598f1f9..c0845a71 100644 --- a/exotic/api/plate_solution.py +++ b/exotic/api/plate_solution.py @@ -105,7 +105,7 @@ def plate_solution(self): job_url = self._get_url(f"jobs/{job_id}") download_url = self.api_url.replace("/api/", f"/wcs_file/{job_id}/") - wcs_file = Path(self.directory) / "temp" / "wcs.fits" + wcs_file = Path(self.directory) / "working_artifacts" / "wcs.fits" wcs_file = self._job_status(job_url, wcs_file, download_url) if not wcs_file: return self._fail('Job Status') @@ -226,7 +226,7 @@ def plate_solution(self): if not wcs_header: return self._fail('NextAstro solve status') - wcs_file = Path(self.directory) / "temp" / "wcs.fits" + wcs_file = Path(self.directory) / "working_artifacts" / "wcs.fits" hdu = PrimaryHDU(data=getdata(filename=self.file), header=wcs_header) hdu.writeto(wcs_file, overwrite=True) self._emit_debug("WCS file creation successful.") diff --git a/exotic/exotic.py b/exotic/exotic.py index 6a15e24d..a81ef2aa 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -3162,7 +3162,8 @@ def archive_exception_payload(action, exc): def failed_comparison_archive_dir(save_dir, comp_index): - base_dir = Path(save_dir) + base_dir = Path(save_dir) / "Diagnostics" + base_dir.mkdir(parents=True, exist_ok=True) candidate_dir = base_dir / f"comp_{comp_index + 1}_failed" if not candidate_dir.exists(): return candidate_dir @@ -3176,13 +3177,13 @@ def failed_comparison_archive_dir(save_dir, comp_index): def comparison_candidate_output_dir(save_dir, comp_index): - return Path(save_dir) / f"comp{comp_index + 1}" + return Path(save_dir) / "Diagnostics" / f"comp{comp_index + 1}" def triangle_plot_output_path(save_dir, planet_name, observation_date): return ( Path(save_dir) - / "temp" + / "Diagnostics" / safe_output_filename("Triangle", planet_name, filename_date_token(observation_date), extension="png") ) @@ -3190,6 +3191,7 @@ def triangle_plot_output_path(save_dir, planet_name, observation_date): def final_triangle_plot_output_path(save_dir, planet_name, observation_date): return ( Path(save_dir) + / "Diagnostics" / safe_output_filename("FinalTriangle", planet_name, filename_date_token(observation_date), extension="png") ) @@ -3197,6 +3199,7 @@ def final_triangle_plot_output_path(save_dir, planet_name, observation_date): def zoomed_final_triangle_plot_output_path(save_dir, planet_name, observation_date): return ( Path(save_dir) + / "Diagnostics" / safe_output_filename("ZoomedTrianglePlot", planet_name, filename_date_token(observation_date), extension="png") ) @@ -3204,7 +3207,7 @@ def zoomed_final_triangle_plot_output_path(save_dir, planet_name, observation_da def comparison_candidate_triangle_plot_output_path(save_dir, planet_name, observation_date, comp_index): return ( Path(save_dir) - / "temp" + / "working_artifacts" / safe_output_filename( f"Comp{int(comp_index) + 1}_Triangle", planet_name, @@ -5703,7 +5706,7 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, return None candidate_dir = comparison_candidate_output_dir(save_dir, comp_index) - temp_dir = candidate_dir / "temp" + temp_dir = candidate_dir / "working_artifacts" temp_dir.mkdir(parents=True, exist_ok=True) archive_errors = [] @@ -5852,7 +5855,7 @@ def archive_failed_comparison_fit(save_dir, planet_name, observation_date, attem return None archive_dir = failed_comparison_archive_dir(save_dir, comp_index) - temp_dir = archive_dir / "temp" + temp_dir = archive_dir / "working_artifacts" temp_dir.mkdir(parents=True, exist_ok=True) fit = attempt.get('fit') @@ -5985,7 +5988,7 @@ def save_selected_photometry_debug_series(save_dir, planet_name, observation_dat phase_clip_keep_mask = np.ones(prefit_kept_indices.size, dtype=bool) phase_keep_full[prefit_kept_indices] = phase_clip_keep_mask - output_dir = Path(save_dir) / "temp" + output_dir = Path(save_dir) / "working_artifacts" output_dir.mkdir(parents=True, exist_ok=True) output_path = output_dir / safe_output_filename( "SelectedPhotometryRawRatio", @@ -7477,6 +7480,117 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): fit = fallback_fit final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + ars_final_diagnostics = None + if callable(final_diagnostics_getter) and 'ars' in current_bounds: + ars_final_diagnostics = final_diagnostics_getter('ars') + latest_diagnostics['ars'] = ars_final_diagnostics + elif latest_diagnostics.get('ars') is not None: + ars_final_diagnostics = latest_diagnostics['ars'] + + ars_restriction_bounds = configured_prior_centered_bounds_for_key( + 'ars', + restriction_reference_prior, + ) + ars_pinned_at_prior_restriction = ( + isinstance(ars_final_diagnostics, dict) + and ars_final_diagnostics.get('clipped') + and 'ars' in current_bounds + and ars_restriction_bounds is not None + and bounds_are_close(current_bounds.get('ars'), ars_restriction_bounds) + ) + if ars_pinned_at_prior_restriction: + prior_ars = restriction_reference_prior.get( + 'ars', + prior.get('ars') if isinstance(prior, dict) else np.nan, + ) + try: + prior_ars = float(prior_ars) + except (TypeError, ValueError): + prior_ars = np.nan + + if np.isfinite(prior_ars) and prior_ars > ARS_SEARCH_BOUND_MIN: + original_fit = fit + original_bounds = clone_lightcurve_bounds(current_bounds) + original_ars_value = (getattr(original_fit, 'parameters', {}) or {}).get('ars', np.nan) + fixed_prior = dict(current_prior) + fixed_prior['ars'] = prior_ars + fixed_bounds = clone_lightcurve_bounds(current_bounds) + fixed_bounds.pop('ars', None) + + fixed_error_override = {} + existing_errors = getattr(original_fit, 'errors', {}) or {} + if 'rprs' not in fixed_bounds: + fixed_rprs_error = existing_errors.get('rprs', np.nan) + try: + fixed_rprs_error = float(fixed_rprs_error) + except (TypeError, ValueError): + fixed_rprs_error = np.nan + if np.isfinite(fixed_rprs_error) and fixed_rprs_error >= 0: + fixed_error_override['rprs'] = float(fixed_rprs_error) + prior_ars_error = restriction_reference_prior.get('ars_unc', np.nan) + try: + prior_ars_error = float(prior_ars_error) + except (TypeError, ValueError): + prior_ars_error = np.nan + if np.isfinite(prior_ars_error) and prior_ars_error >= 0: + fixed_error_override['ars'] = prior_ars_error + + log_info( + "a/Rs posterior remains pinned against the " + f"{ars_final_diagnostics.get('edge', 'active')} edge of the configured " + f"prior-centered range [{ars_restriction_bounds[0]:.6f}, " + f"{ars_restriction_bounds[1]:.6f}]; rerunning UltraNest with a/Rs fixed " + f"to the input prior ({prior_ars:.6f})." + ) + fallback_fit = build_fit( + fixed_prior, + fixed_bounds, + fixed_parameter_errors_override=fixed_error_override, + ) + fallback_parameters = getattr(fallback_fit, 'parameters', None) + if isinstance(fallback_parameters, dict): + fallback_parameters['ars'] = prior_ars + fallback_errors = getattr(fallback_fit, 'errors', None) + if not isinstance(fallback_errors, dict): + fallback_fit.errors = {} + fallback_errors = fallback_fit.errors + if np.isfinite(prior_ars_error) and prior_ars_error >= 0: + fallback_errors['ars'] = prior_ars_error + fixed_errors = getattr(fallback_fit, 'fixed_parameter_errors', None) + if not isinstance(fixed_errors, dict): + fallback_fit.fixed_parameter_errors = {} + fixed_errors = fallback_fit.fixed_parameter_errors + fixed_errors['ars'] = prior_ars_error + + for attribute_name, attribute_value in getattr(original_fit, '__dict__', {}).items(): + if attribute_name.startswith('rprs_prior_fallback_'): + setattr(fallback_fit, attribute_name, attribute_value) + + fallback_note = ( + "Applied a/Rs prior fallback because the sampled posterior remained pinned against " + f"the {ars_final_diagnostics.get('edge', 'active')} edge of the configured " + f"prior-centered range [{ars_restriction_bounds[0]:.6f}, " + f"{ars_restriction_bounds[1]:.6f}]. EXOTIC reran UltraNest with a/Rs fixed " + f"to the input prior ({prior_ars:.6f})" + ) + if np.isfinite(prior_ars_error) and prior_ars_error >= 0: + fallback_note += f" with input uncertainty {prior_ars_error:.6f}." + else: + fallback_note += "." + fallback_fit.ars_prior_fallback_applied = True + fallback_fit.ars_prior_fallback_prior_value = prior_ars + fallback_fit.ars_prior_fallback_prior_uncertainty = prior_ars_error + fallback_fit.ars_prior_fallback_original_fit_value = original_ars_value + fallback_fit.ars_prior_fallback_original_bounds = original_bounds.get('ars') + fallback_fit.ars_prior_fallback_edge = ars_final_diagnostics.get('edge') + fallback_fit.ars_prior_fallback_original_diagnostics = dict(ars_final_diagnostics) + fallback_fit.ars_prior_fallback_note = fallback_note + retry_notes['ars'] = fallback_note + current_prior = fixed_prior + current_bounds = fixed_bounds + fit = fallback_fit + final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + annotate_posterior_refit_final_bounds(fit, current_bounds) for config in retry_configs: key = config['key'] @@ -7487,10 +7601,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): final_diagnostics = None available = config.get('available') config_available = not callable(available) or available(fit, current_bounds) - prior_fallback_applied = ( - key == 'rprs' - and bool(getattr(fit, 'rprs_prior_fallback_applied', False)) - ) + prior_fallback_applied = bool(getattr(fit, f'{key}_prior_fallback_applied', False)) if prior_fallback_applied: final_diagnostics = None elif callable(final_diagnostics_getter) and bounds_key in current_bounds and config_available: @@ -7499,12 +7610,12 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): final_diagnostics = latest_diagnostics[key] if prior_fallback_applied: - fallback_note = getattr(fit, 'rprs_prior_fallback_note', None) or retry_notes.get(key) + fallback_note = getattr(fit, f'{key}_prior_fallback_note', None) or retry_notes.get(key) note = fallback_note if history: note = ( f"Applied {len(history)} automatic {label} posterior range refit(s), " - "then applied the Rp/R* prior fallback." + f"then applied the {label} prior fallback." ) elif history: note = f"Applied {len(history)} automatic {label} posterior range refit(s)." @@ -17296,7 +17407,7 @@ def build_persistent_bad_pixel_map(inputfiles, frame_loader, save_directory=None counts_path = None mask_path = None if save_directory is not None: - temp_dir = Path(save_directory) / "temp" + temp_dir = Path(save_directory) / "working_artifacts" temp_dir.mkdir(parents=True, exist_ok=True) counts_path = temp_dir / BAD_PIXEL_COUNTS_FILENAME mask_path = temp_dir / BAD_PIXEL_MASK_FILENAME @@ -17310,7 +17421,7 @@ def build_persistent_bad_pixel_map(inputfiles, frame_loader, save_directory=None f"({required_count}+ detections)." ) if counts_path is not None and mask_path is not None: - summary += f" Saved {counts_path.name} and {mask_path.name} to temp/." + summary += f" Saved {counts_path.name} and {mask_path.name} to working_artifacts/." log_info(summary) return { @@ -19029,8 +19140,14 @@ def fcn2min(pars): def _psf_seed_track_candidate_paths(seed_track_directory, key): root = Path(seed_track_directory).expanduser() search_dirs = [root] - if root.name.lower() != 'temp': - search_dirs.append(root / 'temp') + if root.name.lower() not in {'working_artifacts', 'temp'}: + search_dirs.extend((root / 'working_artifacts', root / 'temp')) + + expanded_search_dirs = [] + for directory in search_dirs: + expanded_search_dirs.append(directory) + expanded_search_dirs.append(directory / 'psf_flux_data') + search_dirs = expanded_search_dirs if key == 'target': names = ( @@ -28026,7 +28143,7 @@ def clear_previous_fortuitous_variable_products(variable_dir): for path in output_dir.glob(f'{prefix}*'): if path.is_file(): path.unlink() - plot_path = output_dir / 'temp' / 'Stellar_Variability.png' + plot_path = output_dir / 'working_artifacts' / 'Stellar_Variability.png' if plot_path.is_file(): plot_path.unlink() @@ -29874,8 +29991,8 @@ def _main_impl(): ) log_ultranest_mpi_status() - # Make a temp directory of helpful files - Path(Path(exotic_infoDict['save']) / "temp").mkdir(exist_ok=True) + # Keep non-final reduction products separate from the primary results. + Path(Path(exotic_infoDict['save']) / "working_artifacts").mkdir(exist_ok=True) archive_planet_dict = None if not args.override: @@ -32097,13 +32214,16 @@ def lookup_archive_ephemeris(): exotic_infoDict['exposure'] = exp_time_med(exptimes) # save PSF data to disk using savetxt - np.savetxt(Path(exotic_infoDict['save']) / "temp" / "psf_data_target.txt", psf_data["target"], + working_artifacts_dir = Path(exotic_infoDict['save']) / "working_artifacts" + psf_flux_artifacts_dir = working_artifacts_dir / "psf_flux_data" + np.savetxt(working_artifacts_dir / "psf_data_target.txt", psf_data["target"], header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f") # x-cent, y-cent, amplitude, sigma-x, sigma-y, rotation, offset if use_psf_photometry: + psf_flux_artifacts_dir.mkdir(parents=True, exist_ok=True) np.savetxt( - Path(exotic_infoDict['save']) / "temp" / "psf_flux_data_target.txt", + psf_flux_artifacts_dir / "psf_flux_data_target.txt", psf_flux_data["target"], header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f", @@ -32111,7 +32231,7 @@ def lookup_archive_ephemeris(): for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" np.savetxt( - Path(exotic_infoDict['save']) / "temp" / f"psf_flux_data_{ckey}.txt", + psf_flux_artifacts_dir / f"psf_flux_data_{ckey}.txt", psf_flux_data[ckey], header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f", @@ -32845,7 +32965,7 @@ def lookup_archive_ephemeris(): selected_method_label, ) log_info( - f"Saved {saved_candidate_fit_count} comparison-candidate lightcurve fit plot(s) to temp/." + f"Saved {saved_candidate_fit_count} comparison-candidate lightcurve fit plot(s) to working_artifacts/." ) if failed_candidate_fit_count: log_info( @@ -32856,7 +32976,7 @@ def lookup_archive_ephemeris(): # save psf_data to disk for best comparison star if isinstance(bestCompStar, int): - np.savetxt(Path(exotic_infoDict['save']) / "temp" / "psf_data_comp.txt", psf_data[f"comp{bestCompStar}"], + np.savetxt(working_artifacts_dir / "psf_data_comp.txt", psf_data[f"comp{bestCompStar}"], header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f") @@ -33613,8 +33733,9 @@ def lookup_archive_ephemeris(): pass plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) - plot_prior_posterior_comparison(myfit, pDict, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) - plot_ktmf_qc_metrics(myfit, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + diagnostics_dir = Path(exotic_infoDict['save']) / "Diagnostics" + plot_prior_posterior_comparison(myfit, pDict, pDict['pName'], diagnostics_dir, exotic_infoDict['date']) + plot_ktmf_qc_metrics(myfit, pDict['pName'], diagnostics_dir, exotic_infoDict['date']) if fitsortext == 1: observing_background_series = build_observing_background_series( diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 633a1344..162c517d 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -420,7 +420,7 @@ def save_input(): "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", - "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", @@ -1525,7 +1525,7 @@ def save_input(): "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", - "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", diff --git a/exotic/output_files.py b/exotic/output_files.py index 808da73f..d435ebbe 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -252,6 +252,42 @@ def aid_comparison_metadata(vsp_param): return aavso_json_safe(metadata) +def aid_comparison_coordinate_headers(vsp_params, indexed=False): + """Return standards-safe comparison coordinates with RA and Dec on separate lines.""" + if isinstance(vsp_params, dict): + vsp_params = [vsp_params] + + coordinates = [] + seen = set() + for vsp_param in vsp_params or []: + if not isinstance(vsp_param, dict) or vsp_param.get('ensemble_reference'): + continue + comp_ra = finite_float(vsp_param.get('comp_ra')) + comp_dec = finite_float(vsp_param.get('comp_dec')) + if not np.isfinite(comp_ra) or not np.isfinite(comp_dec): + continue + comparison_name = format_aavso_header_value(vsp_param.get('cname')) + identity = (comparison_name, round(float(comp_ra), 10), round(float(comp_dec), 10)) + if identity in seen: + continue + seen.add(identity) + coordinates.append((comparison_name, float(comp_ra), float(comp_dec))) + + if not coordinates: + return "" + if not indexed and len(coordinates) == 1: + _, comp_ra, comp_dec = coordinates[0] + return f"#COMPARISON_RA={comp_ra:.7f}\n#COMPARISON_DEC={comp_dec:.7f}\n" + + headers = [] + for index, (comparison_name, comp_ra, comp_dec) in enumerate(coordinates, start=1): + if comparison_name: + headers.append(f"#COMPARISON_{index}_NAME={comparison_name}") + headers.append(f"#COMPARISON_{index}_RA={comp_ra:.7f}") + headers.append(f"#COMPARISON_{index}_DEC={comp_dec:.7f}") + return "\n".join(headers) + "\n" + + def aid_ensemble_comparison_metadata(vsp_param): if not vsp_param or not vsp_param.get('ensemble_reference'): return {} @@ -1741,7 +1777,7 @@ def __init__(self, fit, p_dict, i_dict, durs): self.dir = Path(self.i_dict['save']) def final_lightcurve(self, phase): - params_file = self.dir / "temp" / safe_output_filename( + params_file = self.dir / "working_artifacts" / safe_output_filename( "FinalLightCurve", self.p_dict['pName'], filename_date_token(self.i_dict['date']), @@ -1814,7 +1850,7 @@ def final_lightcurve(self, phase): def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, min_annul=None, adaptive_summary=None, photometry_info=None, publish_to_root=False): - params_file = self.dir / "temp" / safe_output_filename( + params_file = self.dir / "working_artifacts" / safe_output_filename( "FinalParams", self.p_dict['pName'], filename_date_token(self.i_dict['date']), @@ -2046,6 +2082,9 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor geometry_prior_note = getattr(self.fit, 'partial_transit_geometry_prior_assumption_note', None) if geometry_prior_note: params_num["Prior-assumed partial-transit geometry note"] = str(geometry_prior_note) + ars_prior_fallback_note = getattr(self.fit, 'ars_prior_fallback_note', None) + if ars_prior_fallback_note: + params_num["a/Rs prior fallback note"] = str(ars_prior_fallback_note) oot_baseline_parameter_note = getattr(self.fit, 'oot_baseline_parameter_fit_note', None) if oot_baseline_parameter_note: params_num["Out-of-transit baseline parameter-fit note"] = str(oot_baseline_parameter_note) @@ -2348,7 +2387,7 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, f"{round(self.fit.dataerr[aavsoC], 7)},{round(airmasses[aavsoC], 7)}," f"{round(detrend_model[aavsoC], 7)}\n") def plate_status(self, plate_status: PlateStatus): - plate_status_file = self.dir / "temp" / safe_output_filename( + plate_status_file = self.dir / "working_artifacts" / safe_output_filename( "PlateStatus", self.p_dict['pName'], filename_date_token(self.i_dict['date']), @@ -2378,6 +2417,10 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta first_vsp_param = self.vsp_params[0] if self.vsp_params else {} comparison_metadata = aid_comparison_metadata(first_vsp_param) ensemble_comparison_metadata = aid_ensemble_comparison_metadata(first_vsp_param) + comparison_coordinate_headers = aid_comparison_coordinate_headers( + self.vsp_params, + indexed=use_row_names, + ) default_variable_name = self.auid or self.p_dict.get('sName') or self.p_dict.get('pName') with params_file.open('w', encoding="utf-8") as f: @@ -2400,6 +2443,8 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta "aavso.org/data-usage-guidelines\n") if include_comparison_metadata and comparison_metadata: f.write(f"#COMPARISON-CATALOG-XC={dumps(comparison_metadata, sort_keys=True)}\n") + if comparison_coordinate_headers: + f.write(comparison_coordinate_headers) if include_comparison_metadata and ensemble_comparison_metadata: f.write(format_aavso_json_header( "ENSEMBLE-COMPARISONS-XC", @@ -2586,7 +2631,7 @@ def format_aavso_header_value(value): def save_comp_star_calibration_summary(save_dir, target_name, date, method_label, field_score, comp_summaries, best_comp_index): - temp_dir = Path(save_dir) / "temp" + temp_dir = Path(save_dir) / "working_artifacts" temp_dir.mkdir(parents=True, exist_ok=True) summary_file = temp_dir / safe_output_filename( "CompStarCalibrationSummary", diff --git a/exotic/plots.py b/exotic/plots.py index a3ae1650..c94f6ff7 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -46,6 +46,12 @@ def _dated_plot_filename(prefix, *parts, date, extension): return safe_output_filename(prefix, *parts, filename_date_token(date), extension=extension) +def _working_artifacts_dir(save): + output_dir = Path(save) / "working_artifacts" + output_dir.mkdir(parents=True, exist_ok=True) + return output_dir + + # Plots of the centroid positions as a function of time def plot_centroids(x_targ, y_targ, x_ref, y_ref, times, target_name, save, date): fig, axs = plt.subplots(3, 2, figsize=(12, 10)) @@ -83,7 +89,7 @@ def plot_centroids(x_targ, y_targ, x_ref, y_ref, times, target_name, save, date) axs[2, 1].plot(times[e] - np.nanmin(times), abs(y_targ[e] - y_ref[e]), 'k.') plt.tight_layout() - plt.savefig(Path(save) / "temp" / _dated_plot_filename( + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename( "CentroidPositions&Distances", target_name, date=date, @@ -186,17 +192,17 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc apos = '\'' Path(save).mkdir(parents=True, exist_ok=True) - Path(save, "temp").mkdir(parents=True, exist_ok=True) + _working_artifacts_dir(save) stretch_name = str(stretch.__class__).split('.')[-1].split(apos)[0] - plt.savefig(Path(save) / "temp" / _dated_plot_filename( + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename( "FOV", targ_name, stretch_name, date=date, extension="pdf", ), bbox_inches='tight') - plt.savefig(Path(save) / "temp" / _dated_plot_filename( + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename( "FOV", targ_name, stretch_name, @@ -212,7 +218,7 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Flux [ADU]") plt.errorbar(times, targ, yerr=targ_unc, linestyle='None', fmt='-o') - plt.savefig(Path(save) / "temp" / _dated_plot_filename("TargetRawFlux", targ_name, date=date, extension="pdf")) + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename("TargetRawFlux", targ_name, date=date, extension="pdf")) plt.close() plt.figure() @@ -220,7 +226,7 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Flux [ADU]") plt.errorbar(times, ref, yerr=ref_unc, linestyle='None', fmt='-o') - plt.savefig(Path(save) / "temp" / _dated_plot_filename("CompRawFlux", targ_name, date=date, extension="pdf")) + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename("CompRawFlux", targ_name, date=date, extension="pdf")) plt.close() # Plots final reduced light curve (after the 3 sigma clip) @@ -229,11 +235,11 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Normalized Flux") plt.errorbar(times, norm_flux, yerr=norm_unc, linestyle='None', fmt='-bo') - plt.savefig(Path(save) / "temp" / _dated_plot_filename("NormalizedFluxTime", targ_name, date=date, extension="pdf")) + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename("NormalizedFluxTime", targ_name, date=date, extension="pdf")) plt.close() # Save normalized flux to text file prior to NS - params_file = Path(save) / "temp" / _dated_plot_filename("NormalizedFlux", targ_name, date=date, extension="txt") + params_file = _working_artifacts_dir(save) / _dated_plot_filename("NormalizedFlux", targ_name, date=date, extension="txt") with params_file.open('w') as f: f.write("BJD,Norm Flux,Norm Err,AM\n") @@ -246,8 +252,7 @@ def plot_comp_star_pairwise_matrix(pairwise_matrix, best_comp_index, targ_name, if matrix.size == 0: return - temp_dir = Path(save) / "temp" - temp_dir.mkdir(parents=True, exist_ok=True) + temp_dir = _working_artifacts_dir(save) fig, ax = plt.subplots(figsize=(max(6, matrix.shape[0] * 1.3), max(5, matrix.shape[0] * 1.1))) plot_matrix = np.ma.masked_invalid(matrix * 100.0) @@ -284,8 +289,7 @@ def plot_comp_star_calibration_series(times, comp_summaries, targ_name, save, da return times = np.asarray(times, dtype=float) - temp_dir = Path(save) / "temp" - temp_dir.mkdir(parents=True, exist_ok=True) + temp_dir = _working_artifacts_dir(save) colors = plt.cm.tab10(np.linspace(0.0, 1.0, 10)) fig_height = max(3.2, 2.4 * len(comp_summaries)) @@ -309,8 +313,7 @@ def plot_individual_comp_star_calibration_series(times, comp_summaries, targ_nam return times = np.asarray(times, dtype=float) - temp_dir = Path(save) / "temp" - temp_dir.mkdir(parents=True, exist_ok=True) + temp_dir = _working_artifacts_dir(save) colors = plt.cm.tab10(np.linspace(0.0, 1.0, 10)) for summary in comp_summaries: @@ -341,8 +344,7 @@ def plot_comp_star_candidate_lightcurve_fits(candidate_fit_summaries, targ_name, if not candidate_fit_summaries: return - temp_dir = Path(save) / "temp" - temp_dir.mkdir(parents=True, exist_ok=True) + temp_dir = _working_artifacts_dir(save) for summary in candidate_fit_summaries: fit = summary.get('fit') @@ -449,8 +451,7 @@ def plot_comp_star_suitability(comp_summaries, targ_name, save, date, method_lab if not comp_summaries: return - temp_dir = Path(save) / "temp" - temp_dir.mkdir(parents=True, exist_ok=True) + temp_dir = _working_artifacts_dir(save) labels = [summary['label'] for summary in comp_summaries] positions = np.arange(len(labels)) @@ -505,8 +506,7 @@ def plot_adaptive_aperture_diagnostics(times, aperture_series, annulus_series, f valid_fwhm = np.isfinite(fwhm_series) valid_airmass = np.isfinite(airmass) - temp_dir = Path(save) / "temp" - temp_dir.mkdir(parents=True, exist_ok=True) + temp_dir = _working_artifacts_dir(save) fig, axes = plt.subplots(2, 2, figsize=(12, 8.5)) fig.suptitle( @@ -566,7 +566,7 @@ def plot_variable_residuals(save): plt.ylabel("Residuals (flux)") plt.xlabel("Time [JD]") plt.legend() - plt.savefig(Path(save) / "temp" / f"Variable_Residuals.png") + plt.savefig(_working_artifacts_dir(save) / "Variable_Residuals.png") plt.close() @@ -588,12 +588,12 @@ def _stellar_variability_reference_label(vsp_param, comparison_label): details.append(f"Label: {comparison_label}") if comp_ra is not None and comp_dec is not None: - details.append(f"RA={comp_ra:.6f}, Dec={comp_dec:.6f}") + details.extend((f"RA={comp_ra:.6f}", f"Dec={comp_dec:.6f}")) if not details and comparison_label: details.append(comparison_label) - return " | ".join(details) + return "\n".join(details) def _stellar_variability_comparison_metadata_label(vsp_param): @@ -643,8 +643,7 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): ax.set_ylabel(f"Magnitude ({band})") ax.set_xlabel("Time [JD]") fig.tight_layout() - output_dir = Path(save) / "temp" - output_dir.mkdir(parents=True, exist_ok=True) + output_dir = _working_artifacts_dir(save) fig.savefig(output_dir / f"Stellar_Variability.png") plt.close(fig) @@ -738,8 +737,7 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relati background_series=None): fit_time = np.asarray(fit.time) fit_airmass = np.asarray(fit.airmass) - temp_dir = Path(save) / "temp" - temp_dir.mkdir(parents=True, exist_ok=True) + temp_dir = _working_artifacts_dir(save) for i in range(len(comp_stars) + 1): if i == 0: diff --git a/inits.json b/inits.json index 6141ef57..6747e5ce 100644 --- a/inits.json +++ b/inits.json @@ -33,7 +33,7 @@ "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent calibrated photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under fortuitous_variables/optimal_variables for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under fortuitous_variables/rest_of_the_variables.", "Fortuitous Variable Single Comparison": "Set optional_info 'use_single_comparison_for_fortuitous_variables' to false to use the calibrated comparison-star ensemble for fortuitous VSX targets. The default true selects one unsaturated, non-variable, catalog-calibrated comparison star closest to each VSX target in catalog colour and magnitude.", "NextAstro VSX Cache First": "Set optional_info 'use_nextastro_vsx_cache_first' to true to query the NextAstro /vsx_query field cache before AAVSO VSX. The default is false. Empty or failed cache lookups fall back to AAVSO; legacy cache responses lacking the full period/amplitude schema are enriched from AAVSO.", - "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into temp/, and median-8 repair those pixels before centroiding and photometry. Default n.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", @@ -53,7 +53,7 @@ "Saturation Value": "Set optional_info 'saturation_value' to the detector saturation value in the same units as the image pixels. If omitted/default, EXOTIC uses FITS SATURATE when available, maps TELESCOP Cecilia to 4096, otherwise uses 65535.", "Overexposure Threshold Fraction": "Set optional_info 'overexposure_threshold_fraction' to the fraction of saturation used for rejection. Default 0.9, so pixels above 0.9 * saturation_value are rejected.", "Skip Low Comparison Coverage Rejection": "Set optional_info 'skip_low_comparison_coverage_rejection' to y to disable EXOTIC's default rejection of comparison stars that are valid in far fewer frames than the rest of the comparison-star field. Default n.", - "Fit Lightcurve to Every Comparison Candidate": "Set optional_info 'fit_lightcurve_to_every_comparison_candidate' to y to save one target lightcurve fit plot per comparison star into temp/ using the selected photometry setup. Default n.", + "Fit Lightcurve to Every Comparison Candidate": "Set optional_info 'fit_lightcurve_to_every_comparison_candidate' to y to save one target lightcurve fit plot per comparison star into working_artifacts/ using the selected photometry setup. Default n.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 8b8621ff..e2b9ec01 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -137,8 +137,8 @@ def test_build_persistent_bad_pixel_map_thresholds_recurrence_and_saves_outputs( assert reference["mask"][2, 3] assert not reference["mask"][6, 5] - count_image = fits.getdata(tmp_path / "temp" / "BadPixelDetectionCounts.fits") - mask_image = fits.getdata(tmp_path / "temp" / "BadPixelMask.fits").astype(bool) + count_image = fits.getdata(tmp_path / "working_artifacts" / "BadPixelDetectionCounts.fits") + mask_image = fits.getdata(tmp_path / "working_artifacts" / "BadPixelMask.fits").astype(bool) assert count_image[2, 3] == 4 assert count_image[6, 5] == 3 diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index da93820b..eaa66b63 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -225,7 +225,7 @@ def plot_triangle(self): observation_date = "2024-12-14" source_dir = tmp_path / "comp6" final_dir = tmp_path / "final" - source_temp = source_dir / "temp" + source_temp = source_dir / "working_artifacts" source_temp.mkdir(parents=True) source_plot = source_temp / "Triangle_TOI-1728b_2024-12-14.png" source_plot.write_bytes(b"stale-selected-comp-6") @@ -239,9 +239,9 @@ def plot_triangle(self): source_dir=source_dir, ) - assert output_path == final_dir / "FinalTriangle_TOI-1728b_2024-12-14.png" + assert output_path == final_dir / "Diagnostics" / "FinalTriangle_TOI-1728b_2024-12-14.png" assert output_path.read_bytes() == b"regenerated-final" - assert (final_dir / "temp" / "Triangle_TOI-1728b_2024-12-14.png").read_bytes() == b"regenerated-final" + assert (final_dir / "Diagnostics" / "Triangle_TOI-1728b_2024-12-14.png").read_bytes() == b"regenerated-final" assert fit.called is True @@ -254,7 +254,7 @@ def test_comparison_candidate_triangle_plot_uses_candidate_specific_name(tmp_pat ) assert output_path.name == "Comp7_Triangle_WASP-80b_2025-06-22.png" - assert output_path.parent == tmp_path / "comp7" / "temp" + assert output_path.parent == tmp_path / "comp7" / "working_artifacts" def test_comparison_candidate_triangle_plot_uses_date_only_from_timestamp(tmp_path): @@ -275,7 +275,7 @@ def test_zoomed_final_triangle_plot_uses_named_artifact(tmp_path): "2026-05-06T19:51:13.964-0700", ) - assert output_path == tmp_path / "final" / "ZoomedTrianglePlot_XO-1-b_2026-05-06.png" + assert output_path == tmp_path / "final" / "Diagnostics" / "ZoomedTrianglePlot_XO-1-b_2026-05-06.png" def test_save_final_triangle_plot_creates_zoomed_companion_when_supported(tmp_path): @@ -341,12 +341,12 @@ def plot_triangle(self): assert fit.called is True assert output_path.read_bytes() == b"regenerated" - assert output_path.parent == tmp_path / "final" + assert output_path.parent == tmp_path / "final" / "Diagnostics" assert output_path.name == "FinalTriangle_TOI-1728b_2024-12-14.png" assert ( tmp_path / "final" - / "temp" + / "Diagnostics" / "Triangle_TOI-1728b_2024-12-14.png" ).read_bytes() == b"regenerated" @@ -7022,7 +7022,13 @@ def fake_save(save_dir, provisional_fit, final_fit, p_dict, observation_date, co assert result["attempts"][0]["fit_diagnostics"]["failed_stage"] == "transit_qc" failed_run_dir = result["attempts"][0]["failed_run_dir"] assert failed_run_dir is not None - assert (tmp_path / "comp_1_failed" / "temp" / "FailedFitSummary_HAT-P-32b_2026-04-28.json").exists() + assert ( + tmp_path + / "Diagnostics" + / "comp_1_failed" + / "working_artifacts" + / "FailedFitSummary_HAT-P-32b_2026-04-28.json" + ).exists() assert Path(failed_run_dir).exists() diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 7d70b56b..1c6232ef 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -1708,6 +1708,93 @@ def fake_lc_fitter( assert fit.ars_posterior_refit_bounds == pytest.approx([12.60, 16.30]) +def test_ars_posterior_pinned_at_configured_restriction_falls_back_to_prior(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + fixed_parameter_errors=None, + **kwargs, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": dict(call_bounds), + "fixed_parameter_errors": dict(fixed_parameter_errors or {}), + }) + parameters = dict(call_prior) + if "ars" in call_bounds: + parameters["ars"] = 10.99 + fit = types.SimpleNamespace( + parameters=parameters, + errors=dict(fixed_parameter_errors or {}), + sampled_keys=list(call_bounds), + sample_bounds=dict(call_bounds), + ) + + def diagnostics(key): + if key == "ars": + return { + "clipped": True, + "edge": "upper", + "mode": 10.99, + "std": 0.20, + "bounds": [9.0, 12.0], + } + return { + "clipped": False, + "edge": None, + "mode": parameters.get(key, np.nan), + "std": 0.01, + "bounds": call_bounds.get(key), + "reason": "posterior is not clipped", + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.0, 0.25], + "ars": [9.0, 11.0], + "tmid": [-0.01, 0.01], + "inc": [84.0, 90.0], + "a2": [-3.0, 3.0], + }, + search_restriction_prior={ + "rprs": 0.1, + "ars": 10.0, + "ars_unc": 0.4, + "inc": 89.0, + }, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["ars"] == pytest.approx([9.0, 11.0]) + assert "ars" not in captured["calls"][1]["bounds"] + assert captured["calls"][1]["prior"]["ars"] == pytest.approx(10.0) + assert captured["calls"][1]["fixed_parameter_errors"]["ars"] == pytest.approx(0.4) + assert fit.parameters["ars"] == pytest.approx(10.0) + assert fit.errors["ars"] == pytest.approx(0.4) + assert fit.ars_prior_fallback_applied is True + assert "configured prior-centered range" in fit.ars_prior_fallback_note + + def test_partial_coverage_suppresses_open_geometry_posterior_retries(monkeypatch): import exotic.exotic as exotic_module diff --git a/tests/test_nextastro_astrometry.py b/tests/test_nextastro_astrometry.py index 29165f60..a3eb7b7a 100644 --- a/tests/test_nextastro_astrometry.py +++ b/tests/test_nextastro_astrometry.py @@ -56,7 +56,7 @@ def test_generate_source_list(tmp_path): def test_plate_solution_writes_wcs_file(tmp_path, monkeypatch): fits_path = _create_test_fits(tmp_path) - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() def fake_post(url, data, headers, timeout): payload = _decode_request_body(data, headers) @@ -102,7 +102,7 @@ def fake_get(url, timeout): solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path, ra=210.8023, dec=54.3489, pixel_scale=1.23) wcs_file = solver.plate_solution() - assert wcs_file == tmp_path / 'temp' / 'wcs.fits' + assert wcs_file == tmp_path / 'working_artifacts' / 'wcs.fits' header = getheader(wcs_file) assert header['CTYPE1'] == 'RA---TAN' assert header['CTYPE2'] == 'DEC--TAN' @@ -110,7 +110,7 @@ def fake_get(url, timeout): def test_plate_solution_logs_json_via_message_logger_when_fail_warnings_suppressed(tmp_path, monkeypatch): fits_path = _create_test_fits(tmp_path) - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() logged = [] monkeypatch.setattr( @@ -145,7 +145,7 @@ def test_plate_solution_logs_json_via_message_logger_when_fail_warnings_suppress wcs_file = solver.plate_solution() - assert wcs_file == tmp_path / 'temp' / 'wcs.fits' + assert wcs_file == tmp_path / 'working_artifacts' / 'wcs.fits' assert any('NextAstro astrometry request JSON:' in message for message in logged) assert any('NextAstro astrometry request compression:' in message for message in logged) assert any('NextAstro astrometry submission response JSON:' in message for message in logged) diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 3b8ce177..aafe1a26 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -11,6 +11,7 @@ PRIOR_OBSERVABLE_DEPTH_LABEL, AIDOutputFiles, OutputFiles, + aid_comparison_coordinate_headers, aavso_dicts, build_aavso_qc_metadata, fit_empirical_transit_uncertainty, @@ -125,7 +126,7 @@ def test_prior_depth_respects_inclination_for_non_transiting_geometry(): def test_final_params_writes_stellar_variability_only_payload(tmp_path): - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() fit = SimpleNamespace( stellar_variability_only=True, time=np.arange(6, dtype=float), @@ -154,7 +155,7 @@ def test_final_params_writes_stellar_variability_only_payload(tmp_path): publish_to_root=True, ) - temp_file = next((tmp_path / "temp").glob("FinalParams_Syntheticb_2020-01-01.json")) + temp_file = next((tmp_path / "working_artifacts").glob("FinalParams_Syntheticb_2020-01-01.json")) root_file = tmp_path / temp_file.name payload = json.loads(temp_file.read_text()) @@ -169,7 +170,7 @@ def test_final_params_writes_stellar_variability_only_payload(tmp_path): def test_final_params_describes_calibrated_stellar_variability_ensemble(tmp_path): - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() fit = SimpleNamespace( stellar_variability_only=True, time=np.arange(3, dtype=float), @@ -198,7 +199,7 @@ def test_final_params_describes_calibrated_stellar_variability_ensemble(tmp_path min_annul=15, ) - output_path = next((tmp_path / "temp").glob("FinalParams_Syntheticb_2020-01-01.json")) + output_path = next((tmp_path / "working_artifacts").glob("FinalParams_Syntheticb_2020-01-01.json")) params = json.loads(output_path.read_text())["FINAL STELLAR VARIABILITY PARAMETERS"] assert params["Stellar Variability Reference Star"] == "ensemble" assert params["Variable Reference Star"] == ( @@ -208,7 +209,7 @@ def test_final_params_describes_calibrated_stellar_variability_ensemble(tmp_path def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() fit = SimpleNamespace( stellar_variability_only=True, stellar_variability_params=[ @@ -233,7 +234,7 @@ def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([])) - output_text = next((tmp_path / "temp").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() + output_text = next((tmp_path / "working_artifacts").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() assert "# FINAL STELLAR VARIABILITY TIMESERIES OF WASP-194" in output_text assert "# BJD_TDB,Magnitude,Uncertainty,Band,Airmass" in output_text @@ -242,7 +243,7 @@ def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): def test_final_lightcurve_adds_transit_apparent_magnitude_columns_when_calibrated(tmp_path): - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() fit = SimpleNamespace( time=np.array([2461229.9, 2461229.91]), detrended=np.array([1.0, 0.99]), @@ -259,7 +260,7 @@ def test_final_lightcurve_adds_transit_apparent_magnitude_columns_when_calibrate OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([0.1, 0.2])) - output_text = next((tmp_path / "temp").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() + output_text = next((tmp_path / "working_artifacts").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() assert "Apparent Magnitude,Magnitude Uncertainty,Band" in output_text assert "2461229.9, 0.1, 1.0, 0.001, 1.0, 1.0, 13.740" in output_text @@ -415,6 +416,26 @@ def test_aavso_output_omits_obsname_header_when_blank(tmp_path): assert "#GAIAPMDEC=" not in output_text +def test_aid_comparison_coordinate_headers_index_unique_comparisons_on_separate_lines(): + headers = aid_comparison_coordinate_headers( + [ + {"cname": "Comp A", "comp_ra": 10.1, "comp_dec": -20.2}, + {"cname": "Comp A", "comp_ra": 10.1, "comp_dec": -20.2}, + {"cname": "Comp B", "comp_ra": 11.3, "comp_dec": -21.4}, + ], + indexed=True, + ) + + assert headers.splitlines() == [ + "#COMPARISON_1_NAME=Comp A", + "#COMPARISON_1_RA=10.1000000", + "#COMPARISON_1_DEC=-20.2000000", + "#COMPARISON_2_NAME=Comp B", + "#COMPARISON_2_RA=11.3000000", + "#COMPARISON_2_DEC=-21.4000000", + ] + + def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): fit = DummyFit() p_dict = { @@ -462,6 +483,7 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert metadata["comparison_dec_deg"] == pytest.approx(-20.2) assert metadata["apparent_magnitude"] == pytest.approx(12.1) assert metadata["apparent_magnitude_error"] == pytest.approx(0.03) + assert "#COMPARISON_RA=10.1000000\n#COMPARISON_DEC=-20.2000000\n" in output_text assert "#DATE=BJD_TDB" in output_text assert "HAT-P-32,2450000.12345,12.340,0.050,V,NO,STD" in output_text @@ -695,7 +717,7 @@ def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): fit = DummyFit() fit.airmass_fit_skipped = True fit.airmass_correction_note = "Skipped (airmass span 0.0400 <= 0.05); no airmass correction applied." - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -705,7 +727,7 @@ def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" output_text = output_file.read_text(encoding="utf-8") assert "Airmass correction" in output_text @@ -715,7 +737,7 @@ def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path): fit = DummyFit() - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -736,7 +758,7 @@ def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path vsp_params=vsp_params, ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] reference = final_params["Variable Reference Star"] @@ -752,7 +774,7 @@ def test_transit_outputs_use_rprs_fallback_uncertainty_when_model_error_missing( fit.rprs_prior_fallback_applied = True fit.rprs_prior_fallback_data_uncertainty = 0.005 fit.rprs_prior_fallback_note = "Rp/R* fixed to prior." - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = { "pName": "HAT-P-32 b", @@ -780,7 +802,7 @@ def test_transit_outputs_use_rprs_fallback_uncertainty_when_model_error_missing( vsp_params=[], ) final_params = json.loads( - (tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json").read_text(encoding="utf-8") + (tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json").read_text(encoding="utf-8") )["FINAL PLANETARY PARAMETERS"] assert "0.005" in final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] @@ -801,7 +823,7 @@ def test_transit_outputs_use_rprs_fallback_uncertainty_when_model_error_missing( def test_final_planetary_params_reports_transit_comparison_catalog_reference(tmp_path): fit = DummyFit() - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -835,7 +857,7 @@ def test_final_planetary_params_reports_transit_comparison_catalog_reference(tmp min_annul=10.15, ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] assert final_params["Transit Fit Comparison Star"] == "#1 - [616, 113]" @@ -848,7 +870,7 @@ def test_final_planetary_params_reports_transit_comparison_catalog_reference(tmp def test_final_planetary_params_suppresses_variable_reference_without_transit_comparison(tmp_path): fit = DummyFit() - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -871,7 +893,7 @@ def test_final_planetary_params_suppresses_variable_reference_without_transit_co min_annul=10.15, ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] assert final_params["Transit Fit Comparison Star"] == "None" @@ -881,7 +903,7 @@ def test_final_planetary_params_suppresses_variable_reference_without_transit_co def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclination(tmp_path): fit = DummyFit() - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -891,7 +913,7 @@ def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclinati vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" output_data = json.loads(output_file.read_text(encoding="utf-8")) final_params = output_data["FINAL PLANETARY PARAMETERS"] keys = list(final_params) @@ -908,7 +930,7 @@ def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclinati def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertainties(tmp_path): fit = DummyFit() - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = { "pName": "HAT-P-32 b", @@ -924,7 +946,7 @@ def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertaintie vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] assert final_params["Mid-Transit Time (Tmid)"].endswith("+/- 0.0001 BJD_TDB") @@ -1012,7 +1034,7 @@ def test_final_planetary_params_reports_model_and_red_noise_uncertainties(tmp_pa fit.residuals = fit.data - fit.model fit.dataerr = np.full_like(fit.model, 0.01) fit.airmass_model = np.ones_like(fit.model) - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -1022,7 +1044,7 @@ def test_final_planetary_params_reports_model_and_red_noise_uncertainties(tmp_pa vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] assert final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] == ( @@ -1083,7 +1105,7 @@ def test_final_planetary_params_reports_prior_fallback_data_only_uncertainty(tmp fit.residuals = fit.data - fit.model fit.dataerr = np.full_like(fit.model, 0.01) fit.airmass_model = np.ones_like(fit.model) - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -1093,7 +1115,7 @@ def test_final_planetary_params_reports_prior_fallback_data_only_uncertainty(tmp vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] assert final_params["Rp/R* uncertainty basis"] == "prior_assumed_data_only" @@ -1106,7 +1128,7 @@ def test_final_planetary_params_reports_prior_fallback_data_only_uncertainty(tmp def test_final_planetary_params_can_publish_accepted_copy_to_root(tmp_path): fit = DummyFit() - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -1117,7 +1139,7 @@ def test_final_planetary_params_can_publish_accepted_copy_to_root(tmp_path): publish_to_root=True, ) - temp_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + temp_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" root_file = tmp_path / "FinalParams_HAT-P-32b_2020-01-01.json" assert temp_file.exists() @@ -1127,7 +1149,7 @@ def test_final_planetary_params_can_publish_accepted_copy_to_root(tmp_path): def test_final_planetary_params_reports_adaptive_aperture_summary(tmp_path): fit = DummyFit() - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -1154,7 +1176,7 @@ def test_final_planetary_params_reports_adaptive_aperture_summary(tmp_path): adaptive_summary=adaptive_summary, ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" output_text = output_file.read_text(encoding="utf-8") assert "Adaptive Aperture Scale" in output_text @@ -1205,7 +1227,7 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): ], "notes": ["The transit model is strongly preferred over the flat/null model."], } - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -1215,7 +1237,7 @@ def test_final_planetary_params_reports_transit_qc_summary(tmp_path): vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" output_text = output_file.read_text(encoding="utf-8") assert "Transit detection QC" in output_text @@ -1263,7 +1285,7 @@ def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): }, ], } - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -1321,7 +1343,7 @@ def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): photometry_info=photometry_info, ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" output_data = json.loads(output_file.read_text(encoding="utf-8")) final_params = output_data["FINAL PLANETARY PARAMETERS"] final_param_keys = list(final_params) @@ -1352,7 +1374,7 @@ def test_final_planetary_params_reports_absolute_fit_quality(tmp_path): fit.time = np.arange(6, dtype=float) fit.airmass_model = np.ones(6, dtype=float) fit.bounds = {"tmid": [0, 1], "rprs": [0, 1], "a1": [0, 2]} - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -1362,7 +1384,7 @@ def test_final_planetary_params_reports_absolute_fit_quality(tmp_path): vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" output_data = json.loads(output_file.read_text(encoding="utf-8")) final_params = output_data["FINAL PLANETARY PARAMETERS"] @@ -1380,7 +1402,7 @@ def test_final_planetary_params_reports_prior_assumed_geometry_note(tmp_path): fit.partial_transit_geometry_prior_assumption_note = ( "Applied prior-assumed transit geometry for a one-sided partial light curve." ) - (tmp_path / "temp").mkdir() + (tmp_path / "working_artifacts").mkdir() p_dict = {"pName": "HAT-P-32 b"} i_dict = {"save": str(tmp_path), "date": "2020-01-01"} @@ -1390,7 +1412,7 @@ def test_final_planetary_params_reports_prior_assumed_geometry_note(tmp_path): vsp_params=[], ) - output_file = tmp_path / "temp" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] assert ( @@ -1651,8 +1673,8 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): "frame_count": 10, "required_count": 4, "minimum_fraction": 0.3, - "counts_path": tmp_path / "temp" / "BadPixelDetectionCounts.fits", - "mask_path": tmp_path / "temp" / "BadPixelMask.fits", + "counts_path": tmp_path / "working_artifacts" / "BadPixelDetectionCounts.fits", + "mask_path": tmp_path / "working_artifacts" / "BadPixelMask.fits", }, ) diff --git a/tests/test_plots.py b/tests/test_plots.py index 8f81d02e..43da8a61 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -69,7 +69,7 @@ def spy_plot(self, x, y, *args, **kwargs): assert captured np.testing.assert_array_equal(captured[0][0], fit.time) np.testing.assert_array_equal(captured[0][1], np.array([14.0, 7.0, 21.0])) - assert (tmp_path / "temp" / "Observing_Statistics_target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "Observing_Statistics_target_2026-03-09.png").exists() def test_stellar_variability_final_lightcurve_plots_calibrated_magnitude_by_time(tmp_path, monkeypatch): @@ -207,8 +207,8 @@ def test_plot_adaptive_aperture_diagnostics_writes_outputs(tmp_path): annulus_sigma=9.0, ) - assert (tmp_path / "temp" / "AdaptiveApertureDiagnostics_Target_2026-03-09.png").exists() - assert (tmp_path / "temp" / "AdaptiveApertureDiagnostics_Target_2026-03-09.pdf").exists() + assert (tmp_path / "working_artifacts" / "AdaptiveApertureDiagnostics_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "AdaptiveApertureDiagnostics_Target_2026-03-09.pdf").exists() def test_plot_fov_psf_legend_omits_aperture_annulus_text(tmp_path, monkeypatch): @@ -270,10 +270,10 @@ def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): method_label="PSF photometry", ) - assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp1_Target_2026-03-09.png").exists() - assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp1_Target_2026-03-09.pdf").exists() - assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.png").exists() - assert (tmp_path / "temp" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.pdf").exists() + assert (tmp_path / "working_artifacts" / "CompStarCalibrationCurve_Comp1_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarCalibrationCurve_Comp1_Target_2026-03-09.pdf").exists() + assert (tmp_path / "working_artifacts" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.pdf").exists() def test_plot_individual_comp_star_calibration_series_masks_rejected_frame_lines(tmp_path, monkeypatch): @@ -349,11 +349,13 @@ def spy_set_ylabel(self, label, *args, **kwargs): assert titles[-1] == ( "Host Star\n" - "Label: NextAstro-123 | RA=10.100000, Dec=-20.200000\n" + "Label: NextAstro-123\n" + "RA=10.100000\n" + "Dec=-20.200000\n" "Original filter: CV | Comparison mag: r=12.345 +/- 0.067" ) assert ylabels[-1] == "Magnitude (r)" - assert (tmp_path / "temp" / "Stellar_Variability.png").exists() + assert (tmp_path / "working_artifacts" / "Stellar_Variability.png").exists() def test_plot_stellar_variability_labels_aavso_filter_and_assumed_comparison(tmp_path, monkeypatch): @@ -386,7 +388,7 @@ def spy_set_title(self, label, *args, **kwargs): "000-BJX-718", ) - assert "Label: 000-BJX-718 | RA=10.100000, Dec=-20.200000" in titles[-1] + assert "Label: 000-BJX-718\nRA=10.100000\nDec=-20.200000" in titles[-1] assert "Original filter: CV" in titles[-1] assert "Comparison mag: V=12.345 +/- 0.067" in titles[-1] @@ -423,7 +425,7 @@ def spy_set_title(self, label, *args, **kwargs): assert titles[-1] == ( "Host Star\n" - "Label: NextAstro-123 | RA=10.100000, Dec=-20.200000\n" + "Label: NextAstro-123\nRA=10.100000\nDec=-20.200000\n" "Original filter: MObs CV" ) assert "99.99" not in titles[-1] @@ -447,7 +449,7 @@ def test_plot_stellar_variability_skips_over_30_measurements(tmp_path): "Comp", ) - assert not (tmp_path / "temp" / "Stellar_Variability.png").exists() + assert not (tmp_path / "working_artifacts" / "Stellar_Variability.png").exists() def test_plot_comp_star_candidate_lightcurve_fits_writes_outputs(tmp_path): @@ -479,11 +481,11 @@ def plot_bestfit(self, phase=False, show_flux_baseline_label=True): assert selected_fit.kwargs == {"phase": False, "show_flux_baseline_label": False} assert other_fit.kwargs == {"phase": False, "show_flux_baseline_label": False} - assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.png").exists() - assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.pdf").exists() - assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.png").exists() - assert (tmp_path / "temp" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.pdf").exists() - assert not (tmp_path / "temp" / "CompStarLightCurveFit_Comp3_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.pdf").exists() + assert (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.pdf").exists() + assert not (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp3_Target_2026-03-09.png").exists() def test_plot_final_lightcurve_requests_uncertainty_bands_without_baseline_label(tmp_path): From 09ee08ae47d6011fa7585248070e4dba722f0967 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 17 Jul 2026 09:54:02 +1000 Subject: [PATCH 086/116] expand search range for TOI and TIC objects ... because their parameters tend to be less well definied/studied. --- exotic/exotic.py | 105 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 8 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index a81ef2aa..da3f7744 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -369,6 +369,8 @@ ARS_RANGE_RESTRICTION_ENABLED = ARS_RANGE_RESTRICTION_DEFAULT ARS_RANGE_RESTRICTION_PERCENTAGE = ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT ARS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 +TOI_TIC_ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT = 30.0 +TOI_TIC_ARS_POSTERIOR_MAX_RETRIES_DEFAULT = 8 ARS_RETRY_MIN_HALF_WIDTH = 0.0 IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES_DEFAULT = 5 INCLINATION_SEARCH_BOUND_MIN = 0.0 @@ -3398,6 +3400,8 @@ def build_search_restriction_prior_from_planet_dict(p_dict): if not isinstance(p_dict, dict): return {} return { + 'pName': p_dict.get('pName'), + 'sName': p_dict.get('sName'), 'rprs': p_dict.get('rprs'), 'rprs_unc': p_dict.get('rprsUnc'), 'ars': p_dict.get('aRs'), @@ -4726,10 +4730,12 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, "no airmass correction applied." ) + search_restriction_prior = build_search_restriction_prior_from_planet_dict(p_dict) bounds = build_initial_transit_bounds( prior, [lower, upper], ars_unc=p_dict.get('aRsUnc'), + search_restriction_prior=search_restriction_prior, ) apply_vertical_flux_normalization_bound( prior, @@ -4864,7 +4870,7 @@ def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, keep_ultranest_sampler_for_deferred_extension=not bool(fast_binning.get('applied')), fix_baseline_terms_for_final=not bool(fast_binning.get('applied')), pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, - search_restriction_prior=build_search_restriction_prior_from_planet_dict(p_dict), + search_restriction_prior=search_restriction_prior, ) annotate_fast_ultranest_binning(final_fit, fast_binning) if final_fit is None: @@ -5308,7 +5314,11 @@ def aligned_selected_array(key, dtype=float): half_width = 0.01 bounds[key] = [float(tmid) - half_width, float(tmid) + half_width] elif key == 'ars': - bounds[key] = build_initial_ars_bounds(prior.get('ars', p_dict.get('aRs')), p_dict.get('aRsUnc')) + bounds[key] = build_initial_ars_bounds( + prior.get('ars', p_dict.get('aRs')), + p_dict.get('aRsUnc'), + search_restriction_prior=search_restriction_prior, + ) elif key == 'inc': inc = float(prior.get('inc', p_dict.get('inc', 89.0))) bounds[key] = [inc - 5.0, min(90.0, inc + 5.0)] @@ -6494,6 +6504,50 @@ def prior_centered_parameter_bounds( return [float(lower_bound), float(upper_bound)] +def target_name_is_toi_or_tic(value): + if value is None: + return False + return re.match(r'^\s*(?:TOI|TIC)(?:\s*[-_]?\s*)\d', str(value), flags=re.IGNORECASE) is not None + + +def is_toi_or_tic_target(target): + if isinstance(target, dict): + return any( + target_name_is_toi_or_tic(target.get(key)) + for key in ('pName', 'sName', 'planet_name', 'host_name') + ) + return target_name_is_toi_or_tic(target) + + +def ars_range_restriction_percentage_for_prior(prior): + percentage = float(ARS_RANGE_RESTRICTION_PERCENTAGE) + if ( + is_toi_or_tic_target(prior) + and np.isclose( + percentage, + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + rtol=0.0, + atol=1e-12, + ) + ): + return float(TOI_TIC_ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT) + return percentage + + +def ars_posterior_retry_limit_for_prior(prior, requested_max_retries): + try: + requested_max_retries = int(max(0, requested_max_retries)) + except (TypeError, ValueError): + requested_max_retries = ARS_POSTERIOR_MAX_RETRIES_DEFAULT + + if ( + is_toi_or_tic_target(prior) + and requested_max_retries >= ARS_POSTERIOR_MAX_RETRIES_DEFAULT + ): + return max(requested_max_retries, TOI_TIC_ARS_POSTERIOR_MAX_RETRIES_DEFAULT) + return requested_max_retries + + def configured_prior_centered_bounds_for_key(key, prior): if not isinstance(prior, dict): return None @@ -6506,7 +6560,7 @@ def configured_prior_centered_bounds_for_key(key, prior): return None return prior_centered_parameter_bounds( prior.get('ars'), - ARS_RANGE_RESTRICTION_PERCENTAGE, + ars_range_restriction_percentage_for_prior(prior), ARS_SEARCH_BOUND_MIN, ) return None @@ -6595,6 +6649,7 @@ def build_initial_ars_bounds( ars_unc=None, sigma_multiplier=INITIAL_ARS_BOUND_SIGMA_MULTIPLIER, fallback_relative_half_width=INITIAL_ARS_BOUND_FALLBACK_RELATIVE_HALF_WIDTH, + search_restriction_prior=None, ): try: ars = float(ars) @@ -6620,9 +6675,16 @@ def build_initial_ars_bounds( upper_bound = float(lower_bound + max(np.finfo(float).eps, ARS_SEARCH_BOUND_MIN)) bounds = [float(lower_bound), float(upper_bound)] + restriction_prior = ( + dict(search_restriction_prior) + if isinstance(search_restriction_prior, dict) + else {} + ) + restriction_prior['ars'] = ars + restriction_prior['ars_unc'] = ars_unc restricted_bounds = intersect_parameter_bounds( bounds, - configured_prior_centered_bounds_for_key('ars', {'ars': ars}), + configured_prior_centered_bounds_for_key('ars', restriction_prior), ) return restricted_bounds if restricted_bounds is not None else bounds @@ -6633,6 +6695,7 @@ def build_initial_transit_bounds( ars_unc=None, inclination_half_width=5.0, rprs_data_uncertainty=None, + search_restriction_prior=None, ): lower, upper = [float(value) for value in np.asarray(tmid_bounds, dtype=float).reshape(-1)[:2]] # Keep ars ahead of inc so the internal impact-parameter parameterization @@ -6643,7 +6706,11 @@ def build_initial_transit_bounds( rprs_data_uncertainty=rprs_data_uncertainty, ), 'tmid': [lower, upper], - 'ars': build_initial_ars_bounds(prior['ars'], ars_unc=ars_unc), + 'ars': build_initial_ars_bounds( + prior['ars'], + ars_unc=ars_unc, + search_restriction_prior=search_restriction_prior, + ), 'inc': [prior['inc'] - inclination_half_width, min(90, prior['inc'] + inclination_half_width)], } @@ -7052,6 +7119,10 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co return new_lower < previous_lower - 1e-12 or new_upper > previous_upper + 1e-12 partial_retry_limits = partial_transit_geometry_retry_limits(pre_ultranest_coverage_assessment) + effective_max_ars_retries = ars_posterior_retry_limit_for_prior( + restriction_reference_prior, + max_ars_retries, + ) retry_configs = [ { 'key': 'rprs', @@ -7082,10 +7153,10 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co 'propose_bounds': identity_retry_bounds, 'expands_bounds': normal_retry_expands, 'max_retries': min( - max_ars_retries, + effective_max_ars_retries, partial_retry_limits['max_retries']['ars'], - ) if partial_retry_limits['active'] else max_ars_retries, - 'requested_max_retries': max_ars_retries, + ) if partial_retry_limits['active'] else effective_max_ars_retries, + 'requested_max_retries': effective_max_ars_retries, 'min_bound': ARS_SEARCH_BOUND_MIN, 'max_bound': None, 'prior_mode_key': 'ars', @@ -21969,6 +22040,7 @@ def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, [lower, upper], ars_unc=pDict.get('aRsUnc'), rprs_data_uncertainty=search_restriction_prior.get('rprs_data_uncertainty'), + search_restriction_prior=search_restriction_prior, ) apply_vertical_flux_normalization_bound( prior, @@ -30095,6 +30167,22 @@ def lookup_archive_ephemeris(): ), ) + target_search_restriction_prior = build_search_restriction_prior_from_planet_dict(pDict) + if restrict_ars_range and is_toi_or_tic_target(target_search_restriction_prior): + effective_ars_percentage = ars_range_restriction_percentage_for_prior( + target_search_restriction_prior + ) + effective_ars_retries = ars_posterior_retry_limit_for_prior( + target_search_restriction_prior, + ARS_POSTERIOR_MAX_RETRIES_DEFAULT, + ) + log_info( + "TOI/TIC a/Rs search policy enabled: the initial range uses five times the " + "quoted a/Rs uncertainty, the prior-centered ceiling is " + f"+/- {effective_ars_percentage:.1f}%, and edge-pinned posteriors may receive " + f"up to {effective_ars_retries} automatic a/Rs refit(s)." + ) + # Seed random number generator (for run to run consistency) if exotic_infoDict['random_seed']: log_info(f"Setting random number seed to {exotic_infoDict['random_seed']}") @@ -33610,6 +33698,7 @@ def lookup_archive_ephemeris(): [lower, upper], ars_unc=pDict.get('aRsUnc'), rprs_data_uncertainty=search_restriction_prior.get('rprs_data_uncertainty'), + search_restriction_prior=search_restriction_prior, ) apply_vertical_flux_normalization_bound( prior, From cd250050326ecbd65942756ef0aa3b59c5cfb123 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 17 Jul 2026 10:26:47 +1000 Subject: [PATCH 087/116] better exotic bounds for TOI and TIC --- exotic/exotic.py | 150 ++++++++++++++++++----- tests/test_exotic_proper_motion.py | 6 +- tests/test_exotic_rprs_retry.py | 184 +++++++++++++++++++++++++++-- 3 files changed, 299 insertions(+), 41 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index da3f7744..09904ea0 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -6534,6 +6534,32 @@ def ars_range_restriction_percentage_for_prior(prior): return percentage +def ars_initial_range_percentage_for_prior( + prior, + sigma_multiplier=INITIAL_ARS_BOUND_SIGMA_MULTIPLIER, + fallback_relative_half_width=INITIAL_ARS_BOUND_FALLBACK_RELATIVE_HALF_WIDTH, +): + base_percentage = ars_range_restriction_percentage_for_prior(prior) + if not isinstance(prior, dict): + return float(base_percentage) + + try: + ars = float(prior.get('ars')) + ars_unc = float(prior.get('ars_unc')) + sigma_multiplier = float(sigma_multiplier) + except (TypeError, ValueError): + ars = np.nan + ars_unc = np.nan + sigma_multiplier = INITIAL_ARS_BOUND_SIGMA_MULTIPLIER + + if np.isfinite(ars) and ars > ARS_SEARCH_BOUND_MIN and np.isfinite(ars_unc) and ars_unc > 0: + uncertainty_percentage = 100.0 * sigma_multiplier * ars_unc / ars + else: + uncertainty_percentage = 100.0 * float(fallback_relative_half_width) + + return float(max(base_percentage, uncertainty_percentage)) + + def ars_posterior_retry_limit_for_prior(prior, requested_max_retries): try: requested_max_retries = int(max(0, requested_max_retries)) @@ -6560,7 +6586,7 @@ def configured_prior_centered_bounds_for_key(key, prior): return None return prior_centered_parameter_bounds( prior.get('ars'), - ars_range_restriction_percentage_for_prior(prior), + ars_initial_range_percentage_for_prior(prior), ARS_SEARCH_BOUND_MIN, ) return None @@ -6586,11 +6612,13 @@ def intersect_parameter_bounds(bounds, restriction): return [float(lower_bound), float(upper_bound)] -def apply_configured_prior_search_restrictions(bounds, prior): +def apply_configured_prior_search_restrictions(bounds, prior, allow_ars_expansion=False): restricted = widen_rprs_bounds_to_data_uncertainty_window(bounds, prior) for key in ('rprs', 'ars'): if key not in restricted: continue + if key == 'ars' and allow_ars_expansion: + continue restriction = configured_prior_centered_bounds_for_key(key, prior) intersection = intersect_parameter_bounds(restricted[key], restriction) if intersection is not None: @@ -6664,10 +6692,27 @@ def build_initial_ars_bounds( if not np.isfinite(ars) or ars <= ARS_SEARCH_BOUND_MIN: return [float(ARS_SEARCH_BOUND_MIN), float(ARS_SEARCH_BOUND_FALLBACK_MAX)] + restriction_prior = ( + dict(search_restriction_prior) + if isinstance(search_restriction_prior, dict) + else {} + ) + restriction_prior['ars'] = ars + restriction_prior['ars_unc'] = ars_unc + if np.isfinite(ars_unc) and ars_unc > 0: half_width = float(max(ARS_SEARCH_BOUND_MIN, sigma_multiplier * ars_unc)) else: half_width = float(max(ARS_SEARCH_BOUND_MIN, fallback_relative_half_width * ars)) + if ARS_RANGE_RESTRICTION_ENABLED: + minimum_initial_half_width = ( + ars * ars_initial_range_percentage_for_prior( + restriction_prior, + sigma_multiplier=sigma_multiplier, + fallback_relative_half_width=fallback_relative_half_width, + ) / 100.0 + ) + half_width = float(max(half_width, minimum_initial_half_width)) lower_bound = max(float(ARS_SEARCH_BOUND_MIN), float(ars - half_width)) upper_bound = float(ars + half_width) @@ -6675,13 +6720,6 @@ def build_initial_ars_bounds( upper_bound = float(lower_bound + max(np.finfo(float).eps, ARS_SEARCH_BOUND_MIN)) bounds = [float(lower_bound), float(upper_bound)] - restriction_prior = ( - dict(search_restriction_prior) - if isinstance(search_restriction_prior, dict) - else {} - ) - restriction_prior['ars'] = ars - restriction_prior['ars_unc'] = ars_unc restricted_bounds = intersect_parameter_bounds( bounds, configured_prior_centered_bounds_for_key('ars', restriction_prior), @@ -7189,9 +7227,18 @@ def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, co else {} ) - def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): + def build_fit( + local_prior, + local_bounds, + fixed_parameter_errors_override=None, + allow_ars_expansion=False, + ): local_bounds = sanitize_retry_search_bounds( - apply_configured_prior_search_restrictions(local_bounds, restriction_reference_prior) + apply_configured_prior_search_restrictions( + local_bounds, + restriction_reference_prior, + allow_ars_expansion=allow_ars_expansion, + ) ) effective_fixed_parameter_errors = ( dict(base_fixed_parameter_errors) @@ -7276,6 +7323,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): retry_notes = {config['key']: None for config in retry_configs} latest_diagnostics = {config['key']: None for config in retry_configs} blocked_retry_keys = set() + ars_range_expansion_active = False fit = build_fit(current_prior, current_bounds) while True: @@ -7349,6 +7397,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): continue previous_bounds = current_bounds.get(bounds_key) + allow_ars_expansion = ars_range_expansion_active or key == 'ars' clamped_bounds = retry_config['sanitize_bounds']({bounds_key: [new_lower, new_upper]}).get( bounds_key, [new_lower, new_upper], @@ -7361,6 +7410,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): clamped_bounds = apply_configured_prior_search_restrictions( {bounds_key: clamped_bounds}, restriction_reference_prior, + allow_ars_expansion=allow_ars_expansion, ).get(bounds_key, clamped_bounds) clamped_bounds = retry_config['sanitize_bounds']({bounds_key: clamped_bounds}).get(bounds_key, clamped_bounds) new_lower, new_upper = [float(value) for value in clamped_bounds] @@ -7376,9 +7426,13 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): if not expands_sampled_range: maximum_bound = retry_config['max_bound'] - restricted_bounds = configured_prior_centered_bounds_for_key( - bounds_key, - restriction_reference_prior, + restricted_bounds = ( + None + if bounds_key == 'ars' and allow_ars_expansion + else configured_prior_centered_bounds_for_key( + bounds_key, + restriction_reference_prior, + ) ) if ( restricted_bounds is not None @@ -7422,7 +7476,11 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): updated_bounds = clone_lightcurve_bounds(current_bounds) updated_bounds[bounds_key] = [new_lower, new_upper] updated_bounds = sanitize_retry_search_bounds( - apply_configured_prior_search_restrictions(updated_bounds, restriction_reference_prior) + apply_configured_prior_search_restrictions( + updated_bounds, + restriction_reference_prior, + allow_ars_expansion=allow_ars_expansion, + ) ) updated_prior = dict(current_prior) @@ -7438,7 +7496,12 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): current_prior = updated_prior current_bounds = updated_bounds - fit = build_fit(current_prior, current_bounds) + ars_range_expansion_active = allow_ars_expansion + fit = build_fit( + current_prior, + current_bounds, + allow_ars_expansion=ars_range_expansion_active, + ) final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) rprs_final_diagnostics = None @@ -7497,6 +7560,7 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): fixed_prior, fixed_bounds, fixed_parameter_errors_override=fixed_error_override, + allow_ars_expansion=ars_range_expansion_active, ) fallback_parameters = getattr(fallback_fit, 'parameters', None) if isinstance(fallback_parameters, dict): @@ -7569,7 +7633,24 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): and ars_restriction_bounds is not None and bounds_are_close(current_bounds.get('ars'), ars_restriction_bounds) ) - if ars_pinned_at_prior_restriction: + ars_retries_exhausted = ( + isinstance(ars_final_diagnostics, dict) + and ars_final_diagnostics.get('clipped') + and 'ars' in current_bounds + and len(retry_histories['ars']) >= effective_max_ars_retries + ) + ars_retry_blocked = ( + isinstance(ars_final_diagnostics, dict) + and ars_final_diagnostics.get('clipped') + and 'ars' in current_bounds + and 'ars' in blocked_retry_keys + ) + ars_prior_fallback_required = ( + ars_pinned_at_prior_restriction + or ars_retries_exhausted + or ars_retry_blocked + ) + if ars_prior_fallback_required: prior_ars = restriction_reference_prior.get( 'ars', prior.get('ars') if isinstance(prior, dict) else np.nan, @@ -7606,12 +7687,21 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): if np.isfinite(prior_ars_error) and prior_ars_error >= 0: fixed_error_override['ars'] = prior_ars_error + if ars_retries_exhausted: + fallback_trigger = ( + f"after {len(retry_histories['ars'])} automatic range expansion(s)" + ) + elif ars_retry_blocked: + fallback_trigger = "after the automatic range expansion could not widen the sampled bounds" + else: + fallback_trigger = ( + "at the edge of the initial prior-centered range " + f"[{ars_restriction_bounds[0]:.6f}, {ars_restriction_bounds[1]:.6f}]" + ) log_info( "a/Rs posterior remains pinned against the " - f"{ars_final_diagnostics.get('edge', 'active')} edge of the configured " - f"prior-centered range [{ars_restriction_bounds[0]:.6f}, " - f"{ars_restriction_bounds[1]:.6f}]; rerunning UltraNest with a/Rs fixed " - f"to the input prior ({prior_ars:.6f})." + f"{ars_final_diagnostics.get('edge', 'active')} edge {fallback_trigger}; " + f"rerunning UltraNest with a/Rs fixed to the input prior ({prior_ars:.6f})." ) fallback_fit = build_fit( fixed_prior, @@ -7639,9 +7729,8 @@ def build_fit(local_prior, local_bounds, fixed_parameter_errors_override=None): fallback_note = ( "Applied a/Rs prior fallback because the sampled posterior remained pinned against " - f"the {ars_final_diagnostics.get('edge', 'active')} edge of the configured " - f"prior-centered range [{ars_restriction_bounds[0]:.6f}, " - f"{ars_restriction_bounds[1]:.6f}]. EXOTIC reran UltraNest with a/Rs fixed " + f"the {ars_final_diagnostics.get('edge', 'active')} edge {fallback_trigger}. " + "EXOTIC reran UltraNest with a/Rs fixed " f"to the input prior ({prior_ars:.6f})" ) if np.isfinite(prior_ars_error) and prior_ars_error >= 0: @@ -29999,8 +30088,9 @@ def _main_impl(): log_info("Rp/R* pinned-posterior prior fallback disabled per optional_info setting.") if restrict_ars_range: log_info( - "a/Rs prior-centered search restriction enabled: " - f"+/- {restrict_ars_percentage:.1f}% around the input prior." + "a/Rs initial prior-centered search range enabled: at least " + f"+/- {restrict_ars_percentage:.1f}% around the input prior, widened when " + "five times the quoted a/Rs uncertainty is larger." ) else: log_info("a/Rs prior-centered search restriction disabled.") @@ -30169,7 +30259,7 @@ def lookup_archive_ephemeris(): target_search_restriction_prior = build_search_restriction_prior_from_planet_dict(pDict) if restrict_ars_range and is_toi_or_tic_target(target_search_restriction_prior): - effective_ars_percentage = ars_range_restriction_percentage_for_prior( + effective_ars_percentage = ars_initial_range_percentage_for_prior( target_search_restriction_prior ) effective_ars_retries = ars_posterior_retry_limit_for_prior( @@ -30177,9 +30267,9 @@ def lookup_archive_ephemeris(): ARS_POSTERIOR_MAX_RETRIES_DEFAULT, ) log_info( - "TOI/TIC a/Rs search policy enabled: the initial range uses five times the " - "quoted a/Rs uncertainty, the prior-centered ceiling is " - f"+/- {effective_ars_percentage:.1f}%, and edge-pinned posteriors may receive " + "TOI/TIC a/Rs search policy enabled: the initial half-width is the larger of " + f"30% and five times the quoted uncertainty ({effective_ars_percentage:.1f}% for this target), " + "and edge-pinned posteriors may expand beyond that initial range with " f"up to {effective_ars_retries} automatic a/Rs refit(s)." ) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index eaa66b63..1a373abf 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -7666,8 +7666,8 @@ def test_build_initial_ars_bounds_prefers_published_uncertainty_when_available(m monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) - assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([14.5, 15.5]) - assert build_initial_ars_bounds(15.0, None) == pytest.approx([13.5, 16.5]) + assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([13.5, 16.5]) + assert build_initial_ars_bounds(15.0, None) == pytest.approx([11.25, 18.75]) monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", False) assert build_initial_ars_bounds(15.0, None) == pytest.approx([11.25, 18.75]) @@ -7742,7 +7742,7 @@ def fake_lc_fitter( assert myfit is not None assert list(captured["bounds"])[:4] == ["rprs", "tmid", "ars", "inc"] - assert captured["bounds"]["ars"] == pytest.approx([14.5, 15.5]) + assert captured["bounds"]["ars"] == pytest.approx([13.5, 16.5]) assert "a2" not in captured["bounds"] assert myfit.airmass_fit_skipped is True diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 1c6232ef..1df10338 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -91,6 +91,11 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: RPRS_SEARCH_BOUND_MAX, RPRS_SEARCH_BOUND_MIN, SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + TOI_TIC_ARS_POSTERIOR_MAX_RETRIES_DEFAULT, + TOI_TIC_ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ars_initial_range_percentage_for_prior, + ars_posterior_retry_limit_for_prior, + ars_range_restriction_percentage_for_prior, build_initial_ars_bounds, build_fast_ultranest_lightcurve_series, build_expected_transit_coverage_assessment, @@ -104,6 +109,8 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: configure_prior_rprs_fallback_on_pinned_posterior, configure_rprs_search_bound_max, configure_rprs_range_restriction, + configured_prior_centered_bounds_for_key, + is_toi_or_tic_target, should_use_legacy_psf_flux_mode, should_run_final_fit_phase_residual_clip, should_run_final_residual_rejection, @@ -157,14 +164,92 @@ def test_build_initial_rprs_bounds_widens_prior_window_for_data_uncertainty(monk assert build_initial_rprs_bounds(0.1, rprs_data_uncertainty=0.02) == pytest.approx([0.04, 0.16]) -def test_build_initial_ars_bounds_restricts_fallback_window_to_prior_centered_range(monkeypatch): +def test_build_initial_ars_bounds_uses_larger_of_percentage_and_uncertainty_window(monkeypatch): import exotic.exotic as exotic_module monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) - assert build_initial_ars_bounds(15.0, None) == pytest.approx([13.5, 16.5]) - assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([14.5, 15.5]) + assert build_initial_ars_bounds(15.0, None) == pytest.approx([11.25, 18.75]) + assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([13.5, 16.5]) + + +@pytest.mark.parametrize( + "target_name", + ["TOI-2969 b", "toi 2969.01", "TIC 123456789", "TIC-123456789"], +) +def test_toi_tic_target_detection_accepts_catalog_candidate_names(target_name): + assert is_toi_or_tic_target({"pName": target_name}) is True + + +@pytest.mark.parametrize("target_name", ["WASP-194 b", "HAT-P-32 b", "TOIL-1 b", None]) +def test_toi_tic_target_detection_rejects_established_or_unrelated_names(target_name): + assert is_toi_or_tic_target({"pName": target_name}) is False + + +def test_toi_tic_ars_policy_uses_thirty_percent_ceiling_and_more_retries(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr( + exotic_module, + "ARS_RANGE_RESTRICTION_PERCENTAGE", + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ) + candidate_prior = { + "pName": "TOI-2969 b", + "sName": "TOI-2969", + "ars": 8.0, + "ars_unc": 0.4, + } + established_prior = { + "pName": "WASP-194 b", + "sName": "WASP-194", + "ars": 8.0, + "ars_unc": 0.4, + } + + assert ars_range_restriction_percentage_for_prior(candidate_prior) == pytest.approx( + TOI_TIC_ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT + ) + assert ars_initial_range_percentage_for_prior(candidate_prior) == pytest.approx(30.0) + assert configured_prior_centered_bounds_for_key("ars", candidate_prior) == pytest.approx([5.6, 10.4]) + assert configured_prior_centered_bounds_for_key("ars", established_prior) == pytest.approx([6.0, 10.0]) + assert build_initial_ars_bounds( + 8.0, + 0.4, + search_restriction_prior=candidate_prior, + ) == pytest.approx([5.6, 10.4]) + assert build_initial_ars_bounds( + 8.0, + 0.4, + search_restriction_prior=established_prior, + ) == pytest.approx([6.0, 10.0]) + high_uncertainty_candidate_prior = { + **candidate_prior, + "ars_unc": 0.8, + } + assert ars_initial_range_percentage_for_prior(high_uncertainty_candidate_prior) == pytest.approx(50.0) + assert build_initial_ars_bounds( + 8.0, + 0.8, + search_restriction_prior=high_uncertainty_candidate_prior, + ) == pytest.approx([4.0, 12.0]) + assert ars_posterior_retry_limit_for_prior(candidate_prior, 5) == ( + TOI_TIC_ARS_POSTERIOR_MAX_RETRIES_DEFAULT + ) + assert ars_posterior_retry_limit_for_prior(established_prior, 5) == 5 + assert ars_posterior_retry_limit_for_prior(candidate_prior, 2) == 2 + + +def test_toi_tic_ars_policy_preserves_explicit_nondefault_restriction(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 12.5) + + assert ars_range_restriction_percentage_for_prior( + {"pName": "TIC 123456789", "ars": 8.0} + ) == pytest.approx(12.5) def test_build_initial_rprs_bounds_clamps_to_configured_search_ceiling(monkeypatch): @@ -1708,6 +1793,88 @@ def fake_lc_fitter( assert fit.ars_posterior_refit_bounds == pytest.approx([12.60, 16.30]) +def test_toi_tic_ars_posterior_can_retry_beyond_established_target_limit(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr( + exotic_module, + "ARS_RANGE_RESTRICTION_PERCENTAGE", + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ) + captured_calls = [] + proposed_bounds = [ + [6.8, 13.2], + [6.6, 13.4], + [6.4, 13.6], + [6.2, 13.8], + [6.0, 14.0], + [5.8, 14.2], + ] + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + **kwargs, + ): + call_index = len(captured_calls) + captured_calls.append({ + "prior": dict(call_prior), + "bounds": {key: list(value) for key, value in call_bounds.items()}, + }) + clipped = call_index < len(proposed_bounds) + ars_bounds = proposed_bounds[call_index] if clipped else list(call_bounds["ars"]) + fit = types.SimpleNamespace( + parameters={"rprs": 0.1, "ars": 10.0, "tmid": 0.0, "inc": 89.0, "a2": 0.0}, + ) + + def diagnostics(key): + if key == "rprs": + return {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]} + return { + "clipped": clipped, + "edge": "upper" if clipped else None, + "mode": 10.0, + "std": 0.2, + "bounds": ars_bounds, + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.05, 0.15], + "ars": [7.0, 13.0], + "tmid": [-0.01, 0.01], + "inc": [84.0, 90.0], + "a2": [-3.0, 3.0], + }, + search_restriction_prior={ + "pName": "TOI-2969 b", + "sName": "TOI-2969", + "ars": 10.0, + "ars_unc": 0.2, + }, + ) + + assert len(captured_calls) == 7 + assert captured_calls[0]["bounds"]["ars"] == pytest.approx([7.0, 13.0]) + assert captured_calls[-1]["bounds"]["ars"] == pytest.approx([5.8, 14.2]) + assert fit.ars_posterior_refit_count == 6 + + def test_ars_posterior_pinned_at_configured_restriction_falls_back_to_prior(monkeypatch): import exotic.exotic as exotic_module @@ -1784,15 +1951,16 @@ def diagnostics(key): }, ) - assert len(captured["calls"]) == 2 + assert len(captured["calls"]) == 3 assert captured["calls"][0]["bounds"]["ars"] == pytest.approx([9.0, 11.0]) - assert "ars" not in captured["calls"][1]["bounds"] - assert captured["calls"][1]["prior"]["ars"] == pytest.approx(10.0) - assert captured["calls"][1]["fixed_parameter_errors"]["ars"] == pytest.approx(0.4) + assert captured["calls"][1]["bounds"]["ars"] == pytest.approx([9.0, 12.0]) + assert "ars" not in captured["calls"][2]["bounds"] + assert captured["calls"][2]["prior"]["ars"] == pytest.approx(10.0) + assert captured["calls"][2]["fixed_parameter_errors"]["ars"] == pytest.approx(0.4) assert fit.parameters["ars"] == pytest.approx(10.0) assert fit.errors["ars"] == pytest.approx(0.4) assert fit.ars_prior_fallback_applied is True - assert "configured prior-centered range" in fit.ars_prior_fallback_note + assert "could not widen the sampled bounds" in fit.ars_prior_fallback_note def test_partial_coverage_suppresses_open_geometry_posterior_retries(monkeypatch): From e11f88fa5c7a43dbcbca0465d36a44cf9cb99ae8 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 17 Jul 2026 13:17:55 +1000 Subject: [PATCH 088/116] lazy load necessary fits we moved to using memmaps, but it wanted an image in memory for the plots. --- exotic/exotic.py | 31 +++++++++++++++++++++++ tests/test_aperture_tuning_performance.py | 30 ++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/exotic/exotic.py b/exotic/exotic.py index 09904ea0..0f14a463 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -26038,6 +26038,25 @@ def load_calibrated_reduction_frame(file_name, generalDark, generalBias, general return image_header, image_data +def ensure_first_reduction_image_for_fov(first_image, first_file_name, + generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult, + bad_pixel_reference=None): + """Lazily load the first calibrated frame when multiprocessing did not retain it.""" + if first_image is not None: + return first_image + return load_calibrated_reduction_image( + first_file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + + def evenly_spaced_aperture_tuning_indices(frame_count, max_frames=APERTURE_AUTOTUNE_MAX_FRAMES, min_frames=APERTURE_AUTOTUNE_MIN_FRAMES): """Select representative frames from the beginning through the end of a run.""" @@ -31459,6 +31478,7 @@ def lookup_archive_ephemeris(): "neighborhoods will be paged in." ) initial_photometry_start = perf_counter() + firstImage = None for i, fileName in enumerate(inputfiles): plateStatus.setCurrentFilename(fileName) frame_uses_memmap = use_memmap_initial_photometry @@ -33359,6 +33379,17 @@ def lookup_archive_ephemeris(): and np.isfinite(centroid_positions['y_ref'][0]) ) if reference_centroid_available: + firstImage = ensure_first_reduction_image_for_fov( + firstImage, + inputfiles[0], + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) plot_fov(fov_aperture, fov_annulus, sigma_display, centroid_positions['x_targ'][0], centroid_positions['y_targ'][0], centroid_positions['x_ref'][0], centroid_positions['y_ref'][0], diff --git a/tests/test_aperture_tuning_performance.py b/tests/test_aperture_tuning_performance.py index c87aaff7..789f9005 100644 --- a/tests/test_aperture_tuning_performance.py +++ b/tests/test_aperture_tuning_performance.py @@ -56,6 +56,36 @@ def test_fits_header_memmap_guard_rejects_scaled_images(): assert not exotic_module.fits_header_supports_memmap({"BZERO": 32768}) +def test_missing_first_image_is_lazy_loaded_for_multiprocess_fov_plot(monkeypatch): + expected = np.arange(16, dtype=float).reshape(4, 4) + captured = {} + + def fake_load(file_name, *args, **kwargs): + captured["file_name"] = file_name + captured["bad_pixel_reference"] = kwargs["bad_pixel_reference"] + return expected + + monkeypatch.setattr(exotic_module, "load_calibrated_reduction_image", fake_load) + + retained = exotic_module.ensure_first_reduction_image_for_fov( + None, + "first.fits", + None, + None, + None, + None, + None, + None, + bad_pixel_reference="bad-pixel-map", + ) + + assert retained is expected + assert captured == { + "file_name": "first.fits", + "bad_pixel_reference": "bad-pixel-map", + } + + def test_alignment_worker_memmap_path_skips_full_frame_calibration(tmp_path, monkeypatch): frame_path = tmp_path / "frame.fits" expected = np.arange(100, dtype=np.float32).reshape(10, 10) From 7e446229db0af9be46e49991db6d18db069f53e8 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 17 Jul 2026 13:28:14 +1000 Subject: [PATCH 089/116] better photometry endpoint usage --- exotic/exotic.py | 249 ++++++++++++++++++++++++++-- tests/test_nextastro_variability.py | 161 ++++++++++++++++++ 2 files changed, 395 insertions(+), 15 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 0f14a463..69216f4c 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -407,6 +407,8 @@ NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504} NEXTASTRO_PHOTOMETRY_API_URL = 'https://photometry.nextastro.org' NEXTASTRO_VSX_QUERY_URL = f'{NEXTASTRO_PHOTOMETRY_API_URL}/vsx_query' +NEXTASTRO_PHOTOMETRY_SINGLE_OBJECT_URL = f'{NEXTASTRO_PHOTOMETRY_API_URL}/single_object' +NEXTASTRO_PHOTOMETRY_OBJECTS_QUERY_URL = f'{NEXTASTRO_PHOTOMETRY_API_URL}/objects_query' NEXTASTRO_VSX_QUERY_LIMIT = 200000 NEXTASTRO_PHOTOMETRY_COLUMNS = ( 'id', 'source_id', 'ra', 'dec', @@ -15772,6 +15774,192 @@ def nextastro_photometry_cone_query(ra, dec, radius_arcsec, columns=None): return body +def _validate_nextastro_object_match(result, context): + if not isinstance(result, dict): + raise RuntimeError( + f"NextAstro photometry catalog returned an invalid {context} result." + ) + match = result.get('match') + separation = result.get('separation_arcsec') + if match is not None and not isinstance(match, dict): + raise RuntimeError( + f"NextAstro photometry catalog returned an invalid {context} match." + ) + if match is None: + if separation is not None: + raise RuntimeError( + f"NextAstro photometry catalog returned a separation without a {context} match." + ) + return + if _finite_float(separation) is None or float(separation) < 0: + raise RuntimeError( + f"NextAstro photometry catalog returned an invalid {context} separation." + ) + + +@retry( + stop=stop_after_attempt(NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS), + wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), + retry=retry_if_exception(should_retry_nextastro_variability_error), +) +def nextastro_photometry_single_object_query(ra, dec, radius_arcsec, columns=None): + payload = { + 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), + 'ra': float(ra), + 'dec': float(dec), + 'radius_arcsec': float(radius_arcsec), + } + log_info(f"NextAstro single-object photometry request JSON: {json.dumps(payload)}") + result = requests.post( + NEXTASTRO_PHOTOMETRY_SINGLE_OBJECT_URL, + json=payload, + timeout=30, + ) + if result.status_code != 200: + raise RuntimeError( + f"NextAstro single-object photometry lookup returned HTTP {result.status_code}." + ) + + body = result.json() + if not isinstance(body, dict) or not isinstance(body.get('columns'), list): + raise RuntimeError( + "NextAstro single-object photometry lookup returned an unexpected response format." + ) + _validate_nextastro_object_match(body, 'single-object') + log_info( + "NextAstro single-object photometry response JSON: " + f"{json.dumps({'matched': body.get('match') is not None, 'columns': body.get('columns')})}" + ) + return body + + +@retry( + stop=stop_after_attempt(NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS), + wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), + retry=retry_if_exception(should_retry_nextastro_variability_error), +) +def nextastro_photometry_objects_query(coordinates, radius_arcsec, columns=None): + objects = [ + {'key': str(index), 'ra': float(ra), 'dec': float(dec)} + for index, (ra, dec) in enumerate(coordinates) + ] + if not objects: + return { + 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), + 'count': 0, + 'results': [], + } + payload = { + 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), + 'objects': objects, + 'radius_arcsec': float(radius_arcsec), + } + log_info( + "NextAstro multi-object photometry request JSON: " + f"{json.dumps({'objects': objects, 'radius_arcsec': payload['radius_arcsec']})}" + ) + result = requests.post( + NEXTASTRO_PHOTOMETRY_OBJECTS_QUERY_URL, + json=payload, + timeout=30, + ) + if result.status_code != 200: + raise RuntimeError( + f"NextAstro multi-object photometry lookup returned HTTP {result.status_code}." + ) + + body = result.json() + results = body.get('results') if isinstance(body, dict) else None + if ( + not isinstance(body, dict) + or not isinstance(body.get('columns'), list) + or not isinstance(body.get('count'), int) + or not isinstance(results, list) + or len(results) != len(objects) + ): + raise RuntimeError( + "NextAstro multi-object photometry lookup returned an unexpected response format." + ) + for index, (requested, object_result) in enumerate(zip(objects, results)): + _validate_nextastro_object_match(object_result, f'multi-object #{index + 1}') + if object_result.get('key') != requested['key']: + raise RuntimeError( + "NextAstro multi-object photometry lookup returned results out of order." + ) + response_ra = _finite_float(object_result.get('ra')) + response_dec = _finite_float(object_result.get('dec')) + if ( + response_ra is None + or response_dec is None + or sky_separation_arcsec( + requested['ra'], requested['dec'], response_ra, response_dec + ) > 0.01 + ): + raise RuntimeError( + "NextAstro multi-object photometry lookup returned mismatched coordinates." + ) + matched_count = sum(item.get('match') is not None for item in results) + if body['count'] != matched_count: + raise RuntimeError( + "NextAstro multi-object photometry lookup returned an inconsistent match count." + ) + log_info( + "NextAstro multi-object photometry response JSON: " + f"{json.dumps({'count': body['count'], 'requested': len(objects), 'columns': body['columns']})}" + ) + return body + + +def nextastro_photometry_match_from_object_result( + object_result, + ra, + dec, + obs_filter, + radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + if not isinstance(object_result, dict) or object_result.get('match') is None: + return None + return nextastro_photometry_catalog_match( + { + 'row_format': 'objects', + 'rows': [object_result['match']], + }, + ra, + dec, + obs_filter, + max_separation_arcsec=radius_arcsec, + ) + + +def nextastro_photometry_for_coordinates( + coordinates, + obs_filter, + radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + coordinates = list(coordinates) + if not coordinates: + return [] + if len(coordinates) == 1: + ra, dec = coordinates[0] + return [ + nextastro_photometry_for_coordinate( + ra, + dec, + obs_filter, + radius_arcsec=radius_arcsec, + ) + ] + response = nextastro_photometry_objects_query(coordinates, radius_arcsec) + return [ + nextastro_photometry_match_from_object_result( + object_result, + ra, + dec, + obs_filter, + radius_arcsec=radius_arcsec, + ) + for (ra, dec), object_result in zip(coordinates, response['results']) + ] + + def nextastro_photometry_catalog_for_wcs(wcs_file, axis, img_scale, obs_filter): if not wcs_file or img_scale is None: return None @@ -15792,13 +15980,17 @@ def nextastro_photometry_catalog_for_wcs(wcs_file, axis, img_scale, obs_filter): def nextastro_photometry_for_coordinate(ra, dec, obs_filter, radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): - catalog_response = nextastro_photometry_cone_query(ra, dec, radius_arcsec) - return nextastro_photometry_catalog_match( - catalog_response, + object_result = nextastro_photometry_single_object_query( + ra, + dec, + radius_arcsec, + ) + return nextastro_photometry_match_from_object_result( + object_result, ra, dec, obs_filter, - max_separation_arcsec=radius_arcsec, + radius_arcsec=radius_arcsec, ) @@ -16662,7 +16854,7 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist if (identity := calibration_catalog_identity(value, label)) is not None } - added_count = 0 + candidates = [] for index, (comp_pos, comp_radec) in enumerate(zip(comp_stars, comp_ra_dec)): if tuple(comp_pos) in existing_positions: continue @@ -16674,17 +16866,44 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist match = None if field_catalog is not None: match = nextastro_photometry_catalog_match(field_catalog, comp_ra, comp_dec, obs_filter) + candidates.append({ + 'index': index, + 'comp_pos': comp_pos, + 'ra': comp_ra, + 'dec': comp_dec, + 'match': match, + }) + + unresolved = [candidate for candidate in candidates if candidate['match'] is None] + remote_lookup_failed = False + if unresolved: + try: + remote_matches = nextastro_photometry_for_coordinates( + [(candidate['ra'], candidate['dec']) for candidate in unresolved], + obs_filter, + ) + for candidate, match in zip(unresolved, remote_matches): + candidate['match'] = match + except Exception as exc: + remote_lookup_failed = True + log_info( + "Warning: NextAstro object photometry catalog lookup failed for " + f"{len(unresolved)} comparison star(s) ({describe_retry_exception(exc)}).", + warn=True, + ) + + added_count = 0 + for candidate in candidates: + index = candidate['index'] + comp_pos = candidate['comp_pos'] + comp_ra = candidate['ra'] + comp_dec = candidate['dec'] + match = candidate['match'] + if tuple(comp_pos) in existing_positions: + continue if match is None: - try: - match = nextastro_photometry_for_coordinate(comp_ra, comp_dec, obs_filter) - except Exception as exc: - log_info( - f"Warning: NextAstro photometry catalog lookup failed for comparison star #{index + 1} " - f"({describe_retry_exception(exc)}).", - warn=True, - ) + if remote_lookup_failed: continue - if match is None: log_info( f"Warning: NextAstro photometry catalog did not find a usable magnitude for " f"comparison star #{index + 1}.", @@ -30687,7 +30906,7 @@ def lookup_archive_ephemeris(): except Exception as exc: log_info( "\nWarning: NextAstro full-field photometry catalog lookup failed " - f"({describe_retry_exception(exc)}). Will try per-comparison catalog lookups.", + f"({describe_retry_exception(exc)}). Will try a batched comparison-star lookup.", warn=True, ) if nextastro_field_catalog is not None and ra_dec_tar is not None: diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index b082e622..cdccc4a0 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -574,6 +574,167 @@ def test_nextastro_catalog_match_uses_relaxed_error_only_as_bv_fallback(): assert rejected_v is None +def test_nextastro_photometry_for_coordinate_uses_single_object_endpoint(monkeypatch): + captured = {} + + def fake_post(url, json, timeout): + captured['url'] = url + captured['json'] = json + captured['timeout'] = timeout + return DummyResponse({ + 'columns': list(exotic_module.NEXTASTRO_PHOTOMETRY_COLUMNS), + 'match': { + 'id': 9, + 'source_id': 12345, + 'ra': 10.00001, + 'dec': -20.00001, + 'Vmag': 11.2, + 'err_Vmag': 0.03, + }, + 'separation_arcsec': 0.05, + }) + + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + match = exotic_module.nextastro_photometry_for_coordinate(10.0, -20.0, 'V') + + assert captured['url'] == 'https://photometry.nextastro.org/single_object' + assert captured['json']['ra'] == pytest.approx(10.0) + assert captured['json']['dec'] == pytest.approx(-20.0) + assert captured['json']['radius_arcsec'] == pytest.approx(2.0) + assert captured['json']['columns'] == list(exotic_module.NEXTASTRO_PHOTOMETRY_COLUMNS) + assert captured['timeout'] == 30 + assert match['source_id'] == 12345 + assert match['mag'] == pytest.approx(11.2) + assert match['mag_band'] == 'V' + + +def test_nextastro_photometry_for_coordinates_uses_objects_query_endpoint(monkeypatch): + captured = {} + + def fake_post(url, json, timeout): + captured['url'] = url + captured['json'] = json + captured['timeout'] = timeout + return DummyResponse({ + 'columns': list(exotic_module.NEXTASTRO_PHOTOMETRY_COLUMNS), + 'count': 1, + 'results': [ + { + 'key': '0', + 'ra': 10.0, + 'dec': -20.0, + 'match': { + 'id': 9, + 'source_id': 12345, + 'ra': 10.00001, + 'dec': -20.00001, + 'Vmag': 11.2, + 'err_Vmag': 0.03, + }, + 'separation_arcsec': 0.05, + }, + { + 'key': '1', + 'ra': 11.0, + 'dec': -21.0, + 'match': None, + 'separation_arcsec': None, + }, + ], + }) + + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + matches = exotic_module.nextastro_photometry_for_coordinates( + [(10.0, -20.0), (11.0, -21.0)], + 'V', + ) + + assert captured['url'] == 'https://photometry.nextastro.org/objects_query' + assert captured['json']['objects'] == [ + {'key': '0', 'ra': 10.0, 'dec': -20.0}, + {'key': '1', 'ra': 11.0, 'dec': -21.0}, + ] + assert captured['json']['radius_arcsec'] == pytest.approx(2.0) + assert captured['timeout'] == 30 + assert matches[0]['source_id'] == 12345 + assert matches[0]['mag_band'] == 'V' + assert matches[1] is None + + +def test_nextastro_photometry_for_coordinates_routes_single_target_to_single_object(monkeypatch): + calls = [] + expected_match = {'source_id': 12345, 'mag': 11.2, 'error': 0.03, 'mag_band': 'V'} + + def fake_single(ra, dec, obs_filter, radius_arcsec=2.0): + calls.append((ra, dec, obs_filter, radius_arcsec)) + return expected_match + + monkeypatch.setattr(exotic_module, 'nextastro_photometry_for_coordinate', fake_single) + monkeypatch.setattr( + exotic_module, + 'nextastro_photometry_objects_query', + lambda *args, **kwargs: pytest.fail('one target should not use /objects_query'), + ) + + matches = exotic_module.nextastro_photometry_for_coordinates([(10.0, -20.0)], 'V') + + assert calls == [(10.0, -20.0, 'V', 2.0)] + assert matches == [expected_match] + + +def test_merge_nextastro_calibration_stars_batches_missing_field_matches(monkeypatch): + calls = [] + + def fake_matches(coordinates, obs_filter, radius_arcsec=2.0): + calls.append((coordinates, obs_filter, radius_arcsec)) + return [ + { + 'source_id': 101, + 'id': 1, + 'mag': 11.2, + 'error': 0.03, + 'mag_band': 'V', + 'catalog_ra': 10.0, + 'catalog_dec': -20.0, + 'separation_arcsec': 0.1, + 'catalog_row': {'source_id': 101, 'ra': 10.0, 'dec': -20.0}, + }, + { + 'source_id': 202, + 'id': 2, + 'mag': 12.1, + 'error': 0.04, + 'mag_band': 'V', + 'catalog_ra': 11.0, + 'catalog_dec': -21.0, + 'separation_arcsec': 0.2, + 'catalog_row': {'source_id': 202, 'ra': 11.0, 'dec': -21.0}, + }, + ] + + monkeypatch.setattr( + exotic_module, + 'nextastro_photometry_for_coordinates', + fake_matches, + ) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + calibration_stars = exotic_module.merge_nextastro_calibration_stars( + comp_stars=[[100, 200], [130, 230]], + comp_ra_dec=[(10.0, -20.0), (11.0, -21.0)], + obs_filter='V', + existing_comp_stars={}, + field_catalog=None, + ) + + assert calls == [([(10.0, -20.0), (11.0, -21.0)], 'V', 2.0)] + assert list(calibration_stars) == ['NextAstro-101', 'NextAstro-202'] + + def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], From 78f74d4be79ab4034836860a41854313441902ce Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 17 Jul 2026 20:10:12 +1000 Subject: [PATCH 090/116] EXOTIC used 1 instead of 0 pixel origin for WCS --- exotic/exotic.py | 425 ++++++++++++++++++++++++---- inits.json | 1 + tests/test_centroid_wcs.py | 19 ++ tests/test_nextastro_variability.py | 358 +++++++++++++++++++++++ 4 files changed, 750 insertions(+), 53 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 69216f4c..c1182ff3 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -419,6 +419,7 @@ NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 2.0 CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX = 0.05 CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX = 0.10 +VSP_COMPARISON_MATCH_TOLERANCE_PIXELS = 3.0 REFERENCE_FALLBACK_COMPARISON_LIMIT = 10 REFERENCE_FALLBACK_DETECTION_MAX_STARS = 60 REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS = 12 @@ -14333,7 +14334,10 @@ def get_ra_dec(header, image_shape=None): xaxis = np.arange(width) yaxis = np.arange(height) x, y = np.meshgrid(xaxis, yaxis) - return wcs_header.all_pix2world(x, y, 1) + # Image arrays and every pixel coordinate used by EXOTIC are zero-based. + # Passing origin=1 here displaced the sky-coordinate grid by one pixel in + # both axes and made precise catalog matches fail on coarse image scales. + return wcs_header.all_pix2world(x, y, 0) def deg_to_pix(exp_ra, exp_dec, ra_list, dec_list): @@ -17076,19 +17080,21 @@ def demosaic_img(image_data, demosaic_fmt, demosaic_out, demosaic_mult, i): image_data = (new_image_data @ demosaic_mult).astype(img_dtype) return image_data -def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=None, user_targ_star=None): +def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=None, + user_targ_star=None, max_new_comp_stars=2): if user_comp_stars is None: user_comp_stars = [] + try: + max_new_comp_stars = max(0, int(max_new_comp_stars)) + except (TypeError, ValueError): + max_new_comp_stars = 2 + vsp_comp_stars_info = {} vsp_star_count = 0 observed_filter = obs_filter - # Build combined list for comps and target - there are known cases when AAVsO comps have planets (XO-2 N) - # Plus, we don't want comp too close to target - targ_and_comp_stars = user_comp_stars[:] - if user_targ_star is not None: - targ_and_comp_stars.append(user_targ_star) + initial_user_comp_stars = [list(position) for position in user_comp_stars] wcs_hdr = search_wcs(file) fov = (img_scale * max(axis)) / 60 @@ -17104,51 +17110,205 @@ def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=No obs_filter = aavso_vsp_band_for_filter(obs_filter) + vsp_candidates = [] if data['photometry']: for star in data['photometry']: ra_deg, dec_deg = radec_hours_to_degree(star['ra'], star['dec']) ra_pix, dec_pix = wcs_hdr.world_to_pixel_values(ra_deg, dec_deg) - if (ra_pix < axis[0] and dec_pix < axis[1]) and (ra_pix > 1 and dec_pix > 1): - vsp_star = [int(ra_pix.min()), int(dec_pix.min())] - exist, vsp_star = check_comp_star_exists(targ_and_comp_stars, vsp_star) + ra_pixel = float(np.asarray(ra_pix, dtype=float).reshape(-1)[0]) + dec_pixel = float(np.asarray(dec_pix, dtype=float).reshape(-1)[0]) + if not ( + 1 < ra_pixel < axis[0] + and 1 < dec_pixel < axis[1] + and obs_filter in [band['band'] for band in star['bands']] + ): + continue + star_info = next(band for band in star['bands'] if band['band'] == obs_filter) + usable_magnitude = usable_catalog_reference_magnitude( + star_info.get('mag'), + star_info.get('error'), + ) + if usable_magnitude is None: + continue + star_mag, star_mag_error = usable_magnitude + vsp_candidates.append({ + 'label': star['auid'], + 'pixel_position': [ra_pixel, dec_pixel], + 'star': { + 'mag': star_mag, + 'error': star_mag_error, + 'ra': ra_deg, + 'dec': dec_deg, + 'catalog_ra': ra_deg, + 'catalog_dec': dec_deg, + 'mag_band': obs_filter, + 'observed_filter': observed_filter, + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + }) - if obs_filter in [band['band'] for band in star['bands']]: - star_info = next(band for band in star['bands'] if band['band'] == obs_filter) - usable_magnitude = usable_catalog_reference_magnitude( - star_info.get('mag'), - star_info.get('error'), - ) - if usable_magnitude is None: - continue - star_mag, star_mag_error = usable_magnitude - - vsp_comp_stars_info[star['auid']] = { - 'pos': vsp_star, - 'mag': star_mag, - 'error': star_mag_error, - 'ra': ra_deg, - 'dec': dec_deg, - 'catalog_ra': ra_deg, - 'catalog_dec': dec_deg, - 'mag_band': obs_filter, - 'observed_filter': observed_filter, - 'catalog_source': 'AAVSO VSP', - 'is_aavso_vsp': True, - } + # Match all supplied coordinates before applying the new-star cap. Greedy + # nearest-pair assignment makes the association one-to-one and prevents two + # nearby VSP sources from lending different magnitudes to the same measured star. + match_pairs = [] + for candidate_index, candidate in enumerate(vsp_candidates): + for user_index, user_position in enumerate(initial_user_comp_stars): + distance = comparison_star_pixel_distance( + candidate['pixel_position'], + user_position, + ) + if distance <= VSP_COMPARISON_MATCH_TOLERANCE_PIXELS: + match_pairs.append((distance, candidate_index, user_index)) + matched_candidates = {} + matched_user_indices = set() + for _, candidate_index, user_index in sorted(match_pairs): + if candidate_index in matched_candidates or user_index in matched_user_indices: + continue + matched_candidates[candidate_index] = list(initial_user_comp_stars[user_index]) + matched_user_indices.add(user_index) + + occupied_positions = [*initial_user_comp_stars] + if user_targ_star is not None: + occupied_positions.append(list(user_targ_star)) + matched_supplied_count = 0 + for candidate_index, candidate in enumerate(vsp_candidates): + candidate_position = candidate['pixel_position'] + if ( + user_targ_star is not None + and comparison_star_pixel_distance(candidate_position, user_targ_star) + <= VSP_COMPARISON_MATCH_TOLERANCE_PIXELS + ): + continue - if not exist: - vsp_star_count = add_vsp_star(vsp_star_count, user_comp_stars, vsp_star) + if candidate_index in matched_candidates: + vsp_star = matched_candidates[candidate_index] + matched_supplied_count += 1 + else: + # A candidate close to a supplied coordinate that was already assigned a + # nearer VSP source is ambiguous, so do not add it as a separate star. + if any( + comparison_star_pixel_distance(candidate_position, user_position) + <= VSP_COMPARISON_MATCH_TOLERANCE_PIXELS + for user_position in initial_user_comp_stars + ): + continue + if vsp_star_count >= max_new_comp_stars: + continue + vsp_star = [int(round(candidate_position[0])), int(round(candidate_position[1]))] + if any( + comparison_star_pixel_distance(vsp_star, occupied_position) + <= VSP_COMPARISON_MATCH_TOLERANCE_PIXELS + for occupied_position in occupied_positions + ): + continue + vsp_star_count = add_vsp_star(vsp_star_count, user_comp_stars, vsp_star) + occupied_positions.append(vsp_star) - if len(vsp_comp_stars_info) > 1: - break + vsp_comp_stars_info[candidate['label']] = { + **candidate['star'], + 'pos': vsp_star, + } - if not vsp_star_count: + if not vsp_comp_stars_info: log_info("\nNo comparison stars were gathered from AAVSO.\n") + if matched_supplied_count: + log_info( + f"\nMatched {matched_supplied_count} supplied comparison star coordinate(s) " + "one-to-one with AAVSO VSP photometry.\n" + ) return vsp_comp_stars_info, chart_id +def catalog_calibration_is_usable_for_filter(star, observed_filter, max_error=None): + if not isinstance(star, dict): + return False + magnitude = _finite_float(star.get('mag')) + magnitude_error = normalized_magnitude_error(star.get('error')) + if ( + not is_usable_apparent_magnitude(magnitude) + or magnitude_error is None + or catalog_band_priority(star.get('mag_band'), observed_filter) != 0 + ): + return False + effective_max_error = _finite_float(max_error) + return effective_max_error is None or magnitude_error <= effective_max_error + + +def merge_aavso_vsp_v_calibration_fallback( + file, axis, obs_filter, img_scale, calibration_stars, user_comp_stars, + user_targ_star=None, max_new_comp_stars=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS): + """Query VSP when a V-family observation has no usable direct V calibration. + + Existing AAVSO VSP calibrations mean the field has already been queried. The + returned mapping contains the unified input-plus-VSP calibration pool, while + the second mapping contains only the VSP results from this fallback query. + """ + unified_calibrations = dict(calibration_stars or {}) + preferred_band = preferred_catalog_magnitude_band_for_filter(obs_filter) + if str(preferred_band or '').strip().upper() != 'V': + return unified_calibrations, {}, None, False + + usable_direct_v = any( + star.get('catalog_source') == 'NextAstro photometry catalog' + and catalog_calibration_is_usable_for_filter( + star, + obs_filter, + max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX, + ) + for star in unified_calibrations.values() + if isinstance(star, dict) + ) + if usable_direct_v: + return unified_calibrations, {}, None, False + + usable_vsp_v = any( + (star.get('is_aavso_vsp') or star.get('catalog_source') == 'AAVSO VSP') + and catalog_calibration_is_usable_for_filter( + star, + obs_filter, + max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX, + ) + for star in unified_calibrations.values() + if isinstance(star, dict) + ) + if usable_vsp_v: + return unified_calibrations, {}, None, False + + log_info( + "No usable direct V-band comparison calibration was returned by the NextAstro " + "photometry server; querying AAVSO VSP for this V-family observation." + ) + try: + vsp_calibrations, chart_id = vsp_query( + file, + axis, + obs_filter, + img_scale, + user_comp_stars=user_comp_stars, + user_targ_star=user_targ_star, + max_new_comp_stars=max_new_comp_stars, + ) + except Exception as exc: + log_info( + "Warning: automatic AAVSO VSP V-band calibration fallback failed " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + return unified_calibrations, {}, None, True + + for label, star in vsp_calibrations.items(): + unified_calibrations[label] = star + if vsp_calibrations: + log_info( + f"Added {len(vsp_calibrations)} AAVSO VSP V-band calibration(s) to the " + "comparison-star calibration pool." + ) + return unified_calibrations, vsp_calibrations, chart_id, True + + def add_vsp_star(vsp_star_count, user_comp_stars, vsp_star): user_comp_stars.append(vsp_star) log_info(f"\nAdded Comparison Star #{len(user_comp_stars)}, coordinates {vsp_star} from AAVSO") @@ -17156,32 +17316,43 @@ def add_vsp_star(vsp_star_count, user_comp_stars, vsp_star): return vsp_star_count + 1 -def check_comp_star_exists(user_stars, vsp_star, tol=10): - """Checks if a comparison star from VSP exists in the user-entered - comparison star list +def comparison_star_pixel_distance(first_position, second_position): + try: + first = np.asarray(first_position, dtype=float).reshape(-1) + second = np.asarray(second_position, dtype=float).reshape(-1) + except (TypeError, ValueError): + return np.inf + if first.size < 2 or second.size < 2 or not np.all(np.isfinite([*first[:2], *second[:2]])): + return np.inf + return float(np.hypot(first[0] - second[0], first[1] - second[1])) + + +def check_comp_star_exists(user_stars, vsp_star, tol=VSP_COMPARISON_MATCH_TOLERANCE_PIXELS): + """Return the nearest user-entered comparison within ``tol`` pixels. Parameters ---------- user_stars : list - A header file that may include the airmass or altitude from when the image was taken + User-entered comparison-star pixel coordinates. vsp_star : list - Right Ascension + VSP star pixel coordinates. tol : float - Declination + Maximum Euclidean pixel separation. Returns ------- bool True if VSP star exists in user entered stars, otherwise False list - Pixel coordinate of either the user entered star (exists), otherwise pixel coordinates - of VSP + The matching user coordinate, otherwise the original VSP coordinate. """ - for user_star in user_stars: - pixel_distance = [abs(star1 - star2) for star1, star2 in zip(user_star, vsp_star)] - - if all(i <= tol for i in pixel_distance): - return True, user_star + matches = [ + (comparison_star_pixel_distance(user_star, vsp_star), user_star) + for user_star in user_stars + ] + matches = [match for match in matches if match[0] <= float(tol)] + if matches: + return True, min(matches, key=lambda match: match[0])[1] return False, vsp_star @@ -30832,6 +31003,8 @@ def lookup_archive_ephemeris(): primary_target_catalog_match = None science_comp_stars = [] fortuitous_ensemble_stars = [] + fortuitous_auto_stars = [] + fortuitous_auto_scan_performed = False fortuitous_variables = [] fortuitous_calibration_stars = {} @@ -31100,11 +31273,13 @@ def lookup_archive_ephemeris(): # same brightest-first pool with the same count and saturation limit. # Reuse it rather than performing an identical full-field image scan. fortuitous_auto_stars = [] + fortuitous_auto_scan_performed = True log_info( "Reusing the stellar-variability target comparison pool for fortuitous " "VSX targets; skipping a duplicate automatic source scan." ) else: + fortuitous_auto_scan_performed = True fortuitous_comp_count = parse_automatic_calibration_selector_count( exotic_infoDict.get('automatic_optimal_calibration_selector_count') ) @@ -31169,6 +31344,79 @@ def lookup_archive_ephemeris(): existing_comp_stars=vsp_comp_stars, field_catalog=nextastro_field_catalog, ) + usable_science_nextastro_v = any( + star.get('catalog_source') == 'NextAstro photometry catalog' + and catalog_calibration_is_usable_for_filter( + star, + exotic_infoDict['filter'], + max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX, + ) + for star in vsp_comp_stars.values() + if isinstance(star, dict) + ) + if ( + str( + preferred_catalog_magnitude_band_for_filter(exotic_infoDict['filter']) or '' + ).upper() == 'V' + and not usable_science_nextastro_v + and not fortuitous_auto_scan_performed + ): + fortuitous_auto_scan_performed = True + fortuitous_comp_count = parse_automatic_calibration_selector_count( + exotic_infoDict.get('automatic_optimal_calibration_selector_count') + ) + fortuitous_auto_stars, _ = select_automatic_optimal_calibration_stars( + reference_image, + reference_image.shape, + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter=exotic_infoDict['filter'], + field_catalog=nextastro_field_catalog, + count=fortuitous_comp_count, + colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), + brightest_first=True, + saturation_threshold=fortuitous_saturation_threshold, + ) + check_for_variable_stars( + ra_wcs, + dec_wcs, + fortuitous_auto_stars, + use_nextastro_variability_server=args.use_nextastro_variability_server, + ) + fortuitous_auto_stars, automatic_variable_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + fortuitous_auto_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for rejection in automatic_variable_rejections: + log_info( + "Removed automatic NextAstro V calibration candidate at " + f"[{rejection['position'][0]:.1f}, {rejection['position'][1]:.1f}] because " + f"the full-field VSX search identified {rejection['variable_name']} at the " + f"same source ({rejection['distance_pixels']:.2f} pixel separation).", + warn=True, + ) + fortuitous_ensemble_stars, duplicate_messages = ( + merge_automatic_comparison_star_coords( + science_comp_stars, + fortuitous_auto_stars, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for duplicate_message in duplicate_messages: + log_info(duplicate_message) + ensemble_ra_dec = build_comp_ra_dec( + ra_wcs, + dec_wcs, + fortuitous_ensemble_stars, + ) + log_info( + "Full-field NextAstro V calibration search expanded the tracked comparison " + f"pool to {len(fortuitous_ensemble_stars)} star(s) before considering AAVSO VSP." + ) fortuitous_calibration_stars = merge_nextastro_calibration_stars( fortuitous_ensemble_stars, ensemble_ra_dec, @@ -31176,6 +31424,77 @@ def lookup_archive_ephemeris(): existing_comp_stars=vsp_comp_stars, field_catalog=nextastro_field_catalog, ) + _, fallback_vsp_stars, fallback_chart_id, fallback_vsp_queried = ( + merge_aavso_vsp_v_calibration_fallback( + wcs_file, + [header['NAXIS1'], header['NAXIS2']], + exotic_infoDict['filter'], + img_scale, + fortuitous_calibration_stars, + science_comp_stars, + user_targ_star=[exotic_UIprevTPX, exotic_UIprevTPY], + ) + ) + if fallback_chart_id is not None: + chart_id = fallback_chart_id + if fallback_vsp_queried and fallback_vsp_stars: + fallback_vsp_stars = { + label: star + for label, star in fallback_vsp_stars.items() + if fortuitous_variable_overlap( + star.get('pos'), + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) is None + } + science_comp_stars, fallback_variable_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + science_comp_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for rejection in fallback_variable_rejections: + log_info( + "Removed AAVSO VSP comparison star at " + f"[{rejection['position'][0]:.1f}, {rejection['position'][1]:.1f}] because " + f"the full-field VSX search identified {rejection['variable_name']} at the " + f"same source ({rejection['distance_pixels']:.2f} pixel separation).", + warn=True, + ) + vsp_comp_stars.update(fallback_vsp_stars) + fortuitous_ensemble_stars, fallback_duplicate_messages = ( + merge_automatic_comparison_star_coords( + science_comp_stars, + fortuitous_auto_stars, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for duplicate_message in fallback_duplicate_messages: + log_info(duplicate_message) + ensemble_ra_dec = build_comp_ra_dec( + ra_wcs, + dec_wcs, + fortuitous_ensemble_stars, + ) + vsp_comp_stars = merge_nextastro_calibration_stars( + science_comp_stars, + ensemble_ra_dec[:len(science_comp_stars)], + exotic_infoDict['filter'], + existing_comp_stars=vsp_comp_stars, + field_catalog=nextastro_field_catalog, + ) + fortuitous_calibration_stars = merge_nextastro_calibration_stars( + fortuitous_ensemble_stars, + ensemble_ra_dec, + exotic_infoDict['filter'], + existing_comp_stars=vsp_comp_stars, + field_catalog=nextastro_field_catalog, + ) + # The target variability plot can use any tracked, VSX-vetted + # catalog calibration, including full-field NextAstro candidates + # that were added for fortuitous-variable photometry. + vsp_comp_stars = dict(fortuitous_calibration_stars) tracked_positions = [*fortuitous_ensemble_stars] for variable in fortuitous_variables: variable['tracking_key'] = f"comp{len(tracked_positions) + 1}" @@ -33681,12 +34000,12 @@ def lookup_archive_ephemeris(): ) elif vsp_comp_stars: if isinstance(bestCompStar, int): - vsp_params = stellar_variability(ref_flux, best_fit_lc, science_comp_stars, + vsp_params = stellar_variability(ref_flux, best_fit_lc, fortuitous_ensemble_stars, vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], pDict['sName'], observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), - comp_ra_dec=ra_dec_wcs[:len(science_comp_stars)], + comp_ra_dec=ra_dec_wcs[:len(fortuitous_ensemble_stars)], field_catalog=nextastro_field_catalog, reference_image=reference_image, wcs_file=wcs_file) diff --git a/inits.json b/inits.json index 6747e5ce..6e459a3d 100644 --- a/inits.json +++ b/inits.json @@ -32,6 +32,7 @@ "Stellar Variability Ensemble": "Set optional_info 'use_ensemble_photometry_for_stellar_variability' to false to disable the default calibrated ensemble in stellar_variability_only runs and restore single-comparison selection by out-of-transit scatter. The default ensemble automatically finds bright catalog-calibrated field-star candidates, removes saturated and VSX-variable stars, sigma-clips high catalog magnitude uncertainties, and when more than five remain uses the five closest to the target in catalog colour and magnitude. EnsembleSelection JSON records the target and comparison colours, magnitudes, errors, and selection deltas beside the final results.", "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent calibrated photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under fortuitous_variables/optimal_variables for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under fortuitous_variables/rest_of_the_variables.", "Fortuitous Variable Single Comparison": "Set optional_info 'use_single_comparison_for_fortuitous_variables' to false to use the calibrated comparison-star ensemble for fortuitous VSX targets. The default true selects one unsaturated, non-variable, catalog-calibrated comparison star closest to each VSX target in catalog colour and magnitude.", + "Automatic AAVSO V Calibration Fallback": "For V-family observations, including Clear, CV, and bv, EXOTIC always requires V-band comparison magnitudes. If the NextAstro photometry server supplies no usable V calibration, EXOTIC automatically queries AAVSO VSP and adds matched or discovered V-sequence stars to the same single-comparison or ensemble calibration pool, even when 'Add Comparison Stars from AAVSO?' is n.", "NextAstro VSX Cache First": "Set optional_info 'use_nextastro_vsx_cache_first' to true to query the NextAstro /vsx_query field cache before AAVSO VSX. The default is false. Empty or failed cache lookups fall back to AAVSO; legacy cache responses lacking the full period/amplitude schema are enriched from AAVSO.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index e2b9ec01..bee4ae03 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -432,6 +432,25 @@ def test_get_ra_dec_uses_image_shape_when_header_lacks_naxis(): assert dec_list.shape == (100, 120) +def test_get_ra_dec_matches_zero_based_astropy_pixel_coordinates(): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [210.0, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + header = wcs.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 120 + header["NAXIS2"] = 100 + + ra_list, dec_list = exotic_module.get_ra_dec(header) + + for x_pixel, y_pixel in [(0, 0), (59, 49), (119, 99), (23, 71)]: + expected_ra, expected_dec = wcs.pixel_to_world_values(x_pixel, y_pixel) + assert ra_list[y_pixel, x_pixel] == pytest.approx(expected_ra, abs=1.0e-12) + assert dec_list[y_pixel, x_pixel] == pytest.approx(expected_dec, abs=1.0e-12) + + def test_get_first_image_header_skips_empty_primary_hdu(tmp_path): wcs_path = _write_extension_wcs_fits(tmp_path) diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index cdccc4a0..2f78564b 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -844,6 +844,204 @@ def world_to_pixel_values(self, ra_deg, dec_deg): assert user_comp_stars == [[40, 50]] +def test_vsp_query_keeps_late_supplied_matches_after_new_star_limit(monkeypatch): + class DummyWCS: + def pixel_to_world_values(self, x_pixel, y_pixel): + return 10.0, 20.0 + + def world_to_pixel_values(self, ra_deg, dec_deg): + return np.array([ra_deg]), np.array([dec_deg]) + + payload = { + 'chartid': 'X-LIMIT', + 'photometry': [ + { + 'auid': 'NEW-1', + 'ra': '20', + 'dec': '20', + 'bands': [{'band': 'V', 'mag': 11.0, 'error': 0.01}], + }, + { + 'auid': 'NEW-2', + 'ra': '40', + 'dec': '40', + 'bands': [{'band': 'V', 'mag': 12.0, 'error': 0.01}], + }, + { + 'auid': 'SUPPLIED', + 'ra': '80', + 'dec': '80', + 'bands': [{'band': 'V', 'mag': 13.0, 'error': 0.02}], + }, + ], + } + user_comp_stars = [[80, 80]] + + monkeypatch.setattr(exotic_module, 'search_wcs', lambda file: DummyWCS()) + monkeypatch.setattr( + exotic_module, + 'radec_hours_to_degree', + lambda ra, dec: (float(ra), float(dec)), + ) + monkeypatch.setattr(exotic_module.requests, 'get', lambda url: DummyResponse(payload)) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + vsp_comp_stars, chart_id = exotic_module.vsp_query( + 'frame.fits', + [100, 100], + 'Clear', + 1.0, + user_comp_stars=user_comp_stars, + max_new_comp_stars=1, + ) + + assert chart_id == 'X-LIMIT' + assert list(vsp_comp_stars) == ['NEW-1', 'SUPPLIED'] + assert vsp_comp_stars['SUPPLIED']['pos'] == [80, 80] + assert user_comp_stars == [[80, 80], [20, 20]] + + +def test_vsp_query_assigns_only_nearest_catalog_source_to_supplied_coordinate(monkeypatch): + class DummyWCS: + def pixel_to_world_values(self, x_pixel, y_pixel): + return 10.0, 20.0 + + def world_to_pixel_values(self, ra_deg, dec_deg): + return np.array([ra_deg]), np.array([dec_deg]) + + payload = { + 'chartid': 'X-NEAREST', + 'photometry': [ + { + 'auid': 'FARTHER', + 'ra': '47', + 'dec': '50', + 'bands': [{'band': 'V', 'mag': 11.0, 'error': 0.01}], + }, + { + 'auid': 'NEAREST', + 'ra': '51', + 'dec': '50', + 'bands': [{'band': 'V', 'mag': 12.0, 'error': 0.02}], + }, + ], + } + user_comp_stars = [[50, 50]] + + monkeypatch.setattr(exotic_module, 'search_wcs', lambda file: DummyWCS()) + monkeypatch.setattr( + exotic_module, + 'radec_hours_to_degree', + lambda ra, dec: (float(ra), float(dec)), + ) + monkeypatch.setattr(exotic_module.requests, 'get', lambda url: DummyResponse(payload)) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + vsp_comp_stars, _ = exotic_module.vsp_query( + 'frame.fits', + [100, 100], + 'Clear', + 1.0, + user_comp_stars=user_comp_stars, + max_new_comp_stars=0, + ) + + assert list(vsp_comp_stars) == ['NEAREST'] + assert vsp_comp_stars['NEAREST']['pos'] == [50, 50] + assert user_comp_stars == [[50, 50]] + + +def test_clear_v_calibration_fallback_merges_aavso_with_existing_pool(monkeypatch): + calls = [] + supplied_positions = [[100, 200]] + + def fake_vsp_query(file, axis, obs_filter, img_scale, **kwargs): + calls.append((file, axis, obs_filter, img_scale, kwargs)) + kwargs['user_comp_stars'].append([300, 400]) + return { + '000-BPW-929': { + 'pos': [100, 200], + 'mag': 11.85, + 'error': 0.046, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + '000-BMX-191': { + 'pos': [300, 400], + 'mag': 12.121, + 'error': 0.005, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + }, 'X42753ZU' + + monkeypatch.setattr(exotic_module, 'vsp_query', fake_vsp_query) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + combined, fallback_stars, chart_id, queried = ( + exotic_module.merge_aavso_vsp_v_calibration_fallback( + 'frame.fits', + [512, 512], + 'Clear', + 1.2, + { + 'NextAstro-g-only': { + 'pos': [100, 200], + 'mag': 11.7, + 'error': 0.01, + 'mag_band': 'g', + 'catalog_source': 'NextAstro photometry catalog', + }, + }, + supplied_positions, + user_targ_star=[250, 250], + ) + ) + + assert queried is True + assert chart_id == 'X42753ZU' + assert len(calls) == 1 + assert calls[0][2] == 'Clear' + assert calls[0][4]['max_new_comp_stars'] == 5 + assert supplied_positions == [[100, 200], [300, 400]] + assert set(fallback_stars) == {'000-BPW-929', '000-BMX-191'} + assert set(combined) == {'NextAstro-g-only', '000-BPW-929', '000-BMX-191'} + + +def test_clear_v_calibration_fallback_skips_vsp_when_nextastro_has_usable_v(monkeypatch): + def unexpected_vsp_query(*args, **kwargs): + raise AssertionError('VSP must not be queried when NextAstro supplied usable V') + + monkeypatch.setattr(exotic_module, 'vsp_query', unexpected_vsp_query) + + existing = { + 'NextAstro-123': { + 'pos': [100, 200], + 'mag': 11.7, + 'error': 0.02, + 'mag_band': 'V', + 'catalog_source': 'NextAstro photometry catalog', + }, + } + combined, fallback_stars, chart_id, queried = ( + exotic_module.merge_aavso_vsp_v_calibration_fallback( + 'frame.fits', + [512, 512], + 'Clear', + 1.2, + existing, + [[100, 200]], + ) + ) + + assert combined == existing + assert fallback_stars == {} + assert chart_id is None + assert queried is False + + def test_build_stellar_variability_params_records_nextastro_reference(monkeypatch, tmp_path): captured = {} @@ -1237,6 +1435,63 @@ class DummyFit: assert star is None +def test_derived_catalog_reference_ensembles_multiple_aavso_v_anchors(): + class DummyFit: + def __init__(self, reference_curve): + reference_curve = np.asarray(reference_curve, dtype=float) + self.data = reference_curve + self.time = np.array([1.0, 2.0, 3.0], dtype=float) + self.transit = np.ones(3, dtype=float) + self.stellar_variability_target_flux = reference_curve * 1000.0 + self.stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + self.stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + self.stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + selected_mag = 11.0 + first_anchor_mag = 12.0 + second_anchor_mag = 13.0 + first_ratio = 10.0 ** ((first_anchor_mag - selected_mag) / 2.5) + second_ratio = 10.0 ** ((second_anchor_mag - selected_mag) / 2.5) + + label, star = exotic_module.derived_catalog_reference_for_selected_comp( + { + 0: {'myfit': DummyFit(np.ones(3)), 'pos': [10, 10]}, + 1: {'myfit': DummyFit(np.full(3, first_ratio)), 'pos': [20, 20]}, + 2: {'myfit': DummyFit(np.full(3, second_ratio)), 'pos': [30, 30]}, + }, + [[10, 10], [20, 20], [30, 30]], + { + 'AAVSO-1': { + 'pos': [20, 20], + 'mag': first_anchor_mag, + 'error': 0.02, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + 'AAVSO-2': { + 'pos': [30, 30], + 'mag': second_anchor_mag, + 'error': 0.04, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + }, + [1, 2], + 0, + observed_filter='Clear', + ) + + assert label == 'Derived Comp 1' + assert star['mag'] == pytest.approx(selected_mag) + assert star['error'] == pytest.approx((1.0 / (1.0 / 0.02 ** 2 + 1.0 / 0.04 ** 2)) ** 0.5) + assert star['mag_band'] == 'V' + assert star['derived_catalog_reference'] is True + assert star['derived_reference_anchor_count'] == 2 + assert star['derived_reference_anchor_labels'] == ['AAVSO-1', 'AAVSO-2'] + + def test_stellar_variability_rejects_g_catalog_anchor_for_clearv(monkeypatch, tmp_path): logged = [] @@ -1716,6 +1971,109 @@ def test_stellar_variability_ensemble_skips_cross_band_catalog_reference(): assert rejected['comp1'] == 'no usable catalog calibration' +def test_stellar_variability_ensemble_combines_nextastro_and_aavso_v_members(): + frame_count = 8 + ranked_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + ] + calibration_stars = { + 'NextAstro-123': { + 'pos': [10, 20], + 'mag': 11.8, + 'error': 0.02, + 'mag_band': 'V', + 'catalog_source': 'NextAstro photometry catalog', + 'source_id': 123, + }, + '000-BMX-191': { + 'pos': [30, 40], + 'mag': 12.121, + 'error': 0.005, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + 'catalog_ra': 18.0, + 'catalog_dec': 35.0, + }, + } + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + { + 'comp1': np.full(frame_count, 2000.0), + 'comp2': np.full(frame_count, 1000.0), + }, + observed_filter='Clear', + ) + + assert [member['key'] for member in selection['members']] == ['comp1', 'comp2'] + assert [member['label'] for member in selection['members']] == [ + 'NextAstro-123', + '000-BMX-191', + ] + assert [member['star']['catalog_source'] for member in selection['members']] == [ + 'NextAstro photometry catalog', + 'AAVSO VSP', + ] + + +def test_stellar_variability_single_mode_can_select_aavso_v_fallback_member(): + frame_count = 8 + selection = exotic_module.select_stellar_variability_ensemble_members( + [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + ], + { + '000-BPW-929': { + 'pos': [10, 20], + 'mag': 11.85, + 'error': 0.046, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + 'catalog_ra': 18.0, + 'catalog_dec': 35.0, + }, + '000-BMX-191': { + 'pos': [30, 40], + 'mag': 12.121, + 'error': 0.005, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + 'catalog_ra': 18.1, + 'catalog_dec': 35.1, + }, + }, + { + 'comp1': np.full(frame_count, 2000.0), + 'comp2': np.full(frame_count, 1000.0), + }, + observed_filter='Clear', + min_members=1, + max_members=1, + ) + + assert [member['key'] for member in selection['members']] == ['comp1'] + assert selection['members'][0]['label'] == '000-BPW-929' + assert selection['members'][0]['star']['catalog_source'] == 'AAVSO VSP' + + def test_stellar_variability_rejects_comparison_that_steps_across_acquisition_gap(): frame_count = 180 cadence_days = 6.0 / 86400.0 From 29e29f0230bae798b45c495666b91657486880c8 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 18 Jul 2026 09:35:39 +1000 Subject: [PATCH 091/116] Shorten directory names and fix wrong filter fallbacks --- README.md | 4 +- docs/README.md | 4 +- exotic/exotic.py | 613 ++++++++++++++++++++++------ exotic/output_files.py | 5 +- exotic/plots.py | 4 +- inits.json | 2 +- tests/test_centroid_wcs.py | 14 +- tests/test_nextastro_variability.py | 530 ++++++++++++++++++++++-- tests/test_output_files.py | 5 +- tests/test_plots.py | 11 +- 10 files changed, 1026 insertions(+), 166 deletions(-) diff --git a/README.md b/README.md index 9f4af8ed..ab221c9e 100644 --- a/README.md +++ b/README.md @@ -186,9 +186,9 @@ Get EXOTIC up and running faster with a json file. Please see the included file } ``` -`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against its own calibrated comparison ensemble. Exported light curves also retain only frames whose final ensemble-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or an ensemble member are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `fortuitous_variables/optimal_variables/` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `fortuitous_variables/rest_of_the_variables/` otherwise. Set the item to `false` to disable these products. +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against its own calibrated comparison ensemble. Exported light curves also retain only frames whose final ensemble-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or an ensemble member are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set the item to `false` to disable these products. -`use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/rest classification. +`use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/normal classification. ## Features and Pipeline Architecture diff --git a/docs/README.md b/docs/README.md index 28ef4d7e..c44eeed2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -231,6 +231,6 @@ Get EXOTIC up and running faster with a json file. Please see the included file } ``` -`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against its own calibrated comparison ensemble. Exported light curves also retain only frames whose final ensemble-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or an ensemble member are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `fortuitous_variables/optimal_variables/` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `fortuitous_variables/rest_of_the_variables/` otherwise. Set the item to `false` to disable these products. +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against its own calibrated comparison ensemble. Exported light curves also retain only frames whose final ensemble-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or an ensemble member are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set the item to `false` to disable these products. -`use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/rest classification. +`use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/normal classification. diff --git a/exotic/exotic.py b/exotic/exotic.py index c1182ff3..5ad00359 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -121,9 +121,9 @@ except ImportError: # package import from api.elca import lc_fitter, transit, get_phase try: # output files - from inputs import Inputs, comparison_star_coords + from inputs import Inputs, NEXTASTRO_GAIA_DISTPM_ENDPOINT, comparison_star_coords except ImportError: # package import - from .inputs import Inputs, comparison_star_coords + from .inputs import Inputs, NEXTASTRO_GAIA_DISTPM_ENDPOINT, comparison_star_coords try: # ld from .api.ld import LimbDarkening, ld_re_punct_p except ImportError: # package import @@ -415,8 +415,11 @@ 'Bmag', 'err_Bmag', 'Vmag', 'err_Vmag', 'umag', 'err_umag', 'g', 'dg', 'r', 'dr', 'i', 'di', 'z', 'dz', ) +NEXTASTRO_PHOTOMETRY_IDENTITY_COLUMNS = ('id', 'source_id', 'ra', 'dec') NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC = 30.0 NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 2.0 +NEXTASTRO_GAIA_COLOR_LOOKUP_TIMEOUT_SECONDS = 10 +NEXTASTRO_GAIA_COLOR_LOOKUP_MAX_PER_SELECTOR = 25 CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX = 0.05 CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX = 0.10 VSP_COMPARISON_MATCH_TOLERANCE_PIXELS = 3.0 @@ -14844,7 +14847,7 @@ def fortuitous_variable_category(period_days, amplitude_mag): and amplitude >= FORTUITOUS_VARIABLE_OPTIMAL_MIN_AMPLITUDE_MAG ): return 'optimal_variables' - return 'rest_of_the_variables' + return 'normal' def nextastro_vsx_query_boxes(ra, dec, radius_degrees): @@ -15507,9 +15510,57 @@ def normalize_nextastro_filter_key(obs_filter): return re.sub(r"[^a-z0-9]", "", str(obs_filter or "").lower()) -def nextastro_photometry_band_candidates(obs_filter, include_fallback=True): +def observed_filter_uses_clear_v_calibration(obs_filter): raw_filter = str(obs_filter or '').strip() + clear_v_filter_keys = { + 'cv', + 'clearv', + 'clearunfilteredreducedtovsequence', + 'mobscv', + 'c', + 'clear', + 'lum', + 'luminance', + 'w', + 'pl', + 'photographicg', + 'gaiag', + 'pg', + 'g1', + 'g2', + } + # Exact uppercase G is EXOTIC's short alias for Photographic G. Lowercase + # g remains the distinct Sloan-like catalogue band. + return ( + raw_filter == 'G' + or normalize_nextastro_filter_key(raw_filter) in clear_v_filter_keys + ) + + +def nextastro_catalog_match_radius_arcsec(img_scale=None): + """Allow at least one image pixel when matching pixel-derived sky positions.""" + pixel_scale_arcsec = _finite_float(img_scale) + if pixel_scale_arcsec is None or pixel_scale_arcsec <= 0: + return NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC + return max(NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC, float(pixel_scale_arcsec)) + + +def reported_stellar_variability_band(observed_filter, fallback_band=None): + """Keep the catalogue anchor band separate from the measured passband.""" + if observed_filter_uses_clear_v_calibration(observed_filter): + return 'ClearV' + return fallback_band or observed_filter or 'V' + + +def nextastro_photometry_band_candidates(obs_filter): + """Return the one explicitly configured catalogue calibration band. + + Absolute calibration never falls through to another band when that + configured magnitude or uncertainty is unavailable. + """ filter_key = normalize_nextastro_filter_key(obs_filter) + if observed_filter_uses_clear_v_calibration(obs_filter): + return [('Vmag', 'err_Vmag', 'V')] direct_map = { 'u': [('umag', 'err_umag', 'u')], 'johnsonu': [('umag', 'err_umag', 'u')], @@ -15568,26 +15619,27 @@ def nextastro_photometry_band_candidates(obs_filter, include_fallback=True): 'zs': [('z', 'dz', 'z')], } - fallback = [ - ('Vmag', 'err_Vmag', 'V'), - ('g', 'dg', 'g'), - ('r', 'dr', 'r'), - ('i', 'di', 'i'), - ('Bmag', 'err_Bmag', 'B'), - ('z', 'dz', 'z'), - ('umag', 'err_umag', 'u'), - ] - # EXOTIC's exact uppercase ``G`` means Photographic G. It is not the - # NextAstro catalogue's Sloan-like ``g`` column, nor Gaia ``G`` - # (``phot_g_mean_mag``). Preserve case here because the normalized key - # intentionally cannot distinguish G from g. - candidates = [] if raw_filter == 'G' else list(direct_map.get(filter_key, [])) - if include_fallback: - candidates.extend(candidate for candidate in fallback if candidate not in candidates) - return candidates + return list(direct_map.get(filter_key, [])) + + +def nextastro_photometry_lookup_columns(obs_filter): + band_candidates = nextastro_photometry_band_candidates(obs_filter) + if not band_candidates: + return None + magnitude_column, error_column, _ = band_candidates[0] + return { + 'columns': [ + *NEXTASTRO_PHOTOMETRY_IDENTITY_COLUMNS, + magnitude_column, + error_column, + ], + 'required_columns': [magnitude_column, error_column], + } def aavso_vsp_band_for_filter(obs_filter): + if observed_filter_uses_clear_v_calibration(obs_filter): + return 'V' filter_key = normalize_nextastro_filter_key(obs_filter) direct_map = { 'u': 'U', @@ -15711,12 +15763,8 @@ def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC, max_magnitude_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX): effective_max_separation_arcsec = _finite_float(max_separation_arcsec) - if effective_max_separation_arcsec is None: + if effective_max_separation_arcsec is None or effective_max_separation_arcsec <= 0: effective_max_separation_arcsec = NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC - effective_max_separation_arcsec = min( - effective_max_separation_arcsec, - NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC, - ) band_candidates = nextastro_photometry_band_candidates(obs_filter) matches = [] for row in nextastro_catalog_rows(catalog_response): @@ -15806,13 +15854,16 @@ def _validate_nextastro_object_match(result, context): wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), retry=retry_if_exception(should_retry_nextastro_variability_error), ) -def nextastro_photometry_single_object_query(ra, dec, radius_arcsec, columns=None): +def nextastro_photometry_single_object_query( + ra, dec, radius_arcsec, columns=None, required_columns=None): payload = { 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), 'ra': float(ra), 'dec': float(dec), 'radius_arcsec': float(radius_arcsec), } + if required_columns: + payload['required_columns'] = list(required_columns) log_info(f"NextAstro single-object photometry request JSON: {json.dumps(payload)}") result = requests.post( NEXTASTRO_PHOTOMETRY_SINGLE_OBJECT_URL, @@ -15842,7 +15893,8 @@ def nextastro_photometry_single_object_query(ra, dec, radius_arcsec, columns=Non wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), retry=retry_if_exception(should_retry_nextastro_variability_error), ) -def nextastro_photometry_objects_query(coordinates, radius_arcsec, columns=None): +def nextastro_photometry_objects_query( + coordinates, radius_arcsec, columns=None, required_columns=None): objects = [ {'key': str(index), 'ra': float(ra), 'dec': float(dec)} for index, (ra, dec) in enumerate(coordinates) @@ -15858,6 +15910,8 @@ def nextastro_photometry_objects_query(coordinates, radius_arcsec, columns=None) 'objects': objects, 'radius_arcsec': float(radius_arcsec), } + if required_columns: + payload['required_columns'] = list(required_columns) log_info( "NextAstro multi-object photometry request JSON: " f"{json.dumps({'objects': objects, 'radius_arcsec': payload['radius_arcsec']})}" @@ -15941,6 +15995,9 @@ def nextastro_photometry_for_coordinates( coordinates = list(coordinates) if not coordinates: return [] + lookup_columns = nextastro_photometry_lookup_columns(obs_filter) + if lookup_columns is None: + return [None] * len(coordinates) if len(coordinates) == 1: ra, dec = coordinates[0] return [ @@ -15951,7 +16008,11 @@ def nextastro_photometry_for_coordinates( radius_arcsec=radius_arcsec, ) ] - response = nextastro_photometry_objects_query(coordinates, radius_arcsec) + response = nextastro_photometry_objects_query( + coordinates, + radius_arcsec, + **lookup_columns, + ) return [ nextastro_photometry_match_from_object_result( object_result, @@ -15984,10 +16045,14 @@ def nextastro_photometry_catalog_for_wcs(wcs_file, axis, img_scale, obs_filter): def nextastro_photometry_for_coordinate(ra, dec, obs_filter, radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + lookup_columns = nextastro_photometry_lookup_columns(obs_filter) + if lookup_columns is None: + return None object_result = nextastro_photometry_single_object_query( ra, dec, radius_arcsec, + **lookup_columns, ) return nextastro_photometry_match_from_object_result( object_result, @@ -16375,13 +16440,15 @@ def image_aperture_signal_flux(image_data, x_pos, y_pos, def nextastro_color_candidate_pairs(obs_filter): filter_key = normalize_nextastro_filter_key(obs_filter) - if filter_key in ('u', 'johnsonu', 'su', 'up'): + if observed_filter_uses_clear_v_calibration(obs_filter): + preferred = [('Bmag', 'Vmag', 'B-V')] + elif filter_key in ('u', 'johnsonu', 'su', 'up'): preferred = [('umag', 'g', 'u-g')] elif filter_key in ('b', 'johnsonb', 'photographicb', 'bb', 'pb'): preferred = [('Bmag', 'Vmag', 'B-V')] elif filter_key in ('v', 'johnsonv', 'bv', 'cv', 'clearv', 'c', 'clear', 'lum', 'luminance'): preferred = [('Bmag', 'Vmag', 'B-V')] - elif filter_key in ('sg', 'sloang', 'sdssg', 'photographicg', 'gp', 'g', 'pg', 'tg'): + elif filter_key in ('sg', 'sloang', 'sdssg', 'gp', 'g', 'tg'): preferred = [('g', 'r', 'g-r')] elif filter_key in ('sr', 'sloanr', 'sdssr', 'johnsonr', 'cousinsr', 'rp', 'r', 'rc', 'rj', 'pr', 'tr', 'cr'): preferred = [('r', 'i', 'r-i')] @@ -16392,22 +16459,95 @@ def nextastro_color_candidate_pairs(obs_filter): else: preferred = [] - fallback = [ + universal_fallbacks = [ ('Bmag', 'Vmag', 'B-V'), - ('g', 'r', 'g-r'), - ('r', 'i', 'r-i'), - ('i', 'z', 'i-z'), - ('umag', 'g', 'u-g'), + ('phot_bp_mean_mag', 'phot_rp_mean_mag', 'BP-RP'), ] - pairs = list(preferred) - pairs.extend(pair for pair in fallback if pair not in pairs) - return pairs + return preferred + [pair for pair in universal_fallbacks if pair not in preferred] -def nextastro_catalog_color(row, obs_filter): +def nextastro_catalog_bp_rp(row): + for direct_key in ( + 'bp_rp', 'BP_RP', 'BP-RP', 'BPRP', 'gaia_bp_rp', 'GAIA_BP_RP', 'phot_bp_rp' + ): + direct_value = _finite_float(row.get(direct_key)) + if direct_value is not None: + return direct_value, direct_key, None + + for bp_key, rp_key in ( + ('phot_bp_mean_mag', 'phot_rp_mean_mag'), + ('PHOT_BP_MEAN_MAG', 'PHOT_RP_MEAN_MAG'), + ('GAIA_BP', 'GAIA_RP'), + ('BP_MAG', 'RP_MAG'), + ('BP', 'RP'), + ): + bp_magnitude = _finite_float(row.get(bp_key)) + rp_magnitude = _finite_float(row.get(rp_key)) + if bp_magnitude is not None and rp_magnitude is not None: + return float(bp_magnitude - rp_magnitude), bp_key, rp_key + return None + + +@lru_cache(maxsize=2048) +def _cached_nextastro_gaia_bp_rp(ra, dec, max_separation_arcsec): + response = requests.get( + NEXTASTRO_GAIA_DISTPM_ENDPOINT, + params={'ra': float(ra), 'dec': float(dec)}, + timeout=NEXTASTRO_GAIA_COLOR_LOOKUP_TIMEOUT_SECONDS, + ) + if response.status_code != 200: + raise RuntimeError(f"NextAstro Gaia lookup returned HTTP {response.status_code}.") + body = response.json() + gaia = body.get('gaia') if isinstance(body, dict) else None + if not isinstance(gaia, dict): + return None + separation = _finite_float(gaia.get('separation_arcsec')) + if separation is None or separation > float(max_separation_arcsec): + return None + bp_rp = nextastro_catalog_bp_rp(gaia) + if bp_rp is None: + return None + color, first_column, second_column = bp_rp + return { + 'color': float(color), + 'label': 'BP-RP', + 'first_column': first_column, + 'second_column': second_column, + 'catalog_source': 'NextAstro Gaia DR3', + 'gaia_source_id': gaia.get('source_id'), + 'gaia_separation_arcsec': separation, + } + + +def nextastro_gaia_bp_rp_for_coordinate( + ra, dec, max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + parsed_ra = _finite_float(ra) + parsed_dec = _finite_float(dec) + parsed_radius = _finite_float(max_separation_arcsec) + if parsed_ra is None or parsed_dec is None or parsed_radius is None or parsed_radius <= 0: + return None + return _cached_nextastro_gaia_bp_rp( + round(parsed_ra, 7), + round(parsed_dec, 7), + round(parsed_radius, 3), + ) + + +def nextastro_catalog_color_from_pairs(row, pairs): if not isinstance(row, dict): return None - for first_column, second_column, label in nextastro_color_candidate_pairs(obs_filter): + for first_column, second_column, label in pairs: + if label == 'BP-RP': + bp_rp = nextastro_catalog_bp_rp(row) + if bp_rp is None: + continue + color, first_column, second_column = bp_rp + return { + 'color': float(color), + 'label': label, + 'first_column': first_column, + 'second_column': second_column, + } first = _finite_float(row.get(first_column)) second = _finite_float(row.get(second_column)) if first is None or second is None: @@ -16421,6 +16561,13 @@ def nextastro_catalog_color(row, obs_filter): return None +def nextastro_catalog_color(row, obs_filter): + return nextastro_catalog_color_from_pairs( + row, + nextastro_color_candidate_pairs(obs_filter), + ) + + def normalize_colour_index_label(value): text = str(value or '').strip().upper().replace(' ', '') aliases = { @@ -16522,32 +16669,40 @@ def colour_term_for_catalog_label(metadata, color_label): def nextastro_catalog_nearest_color_row(catalog_response, ra, dec, obs_filter, max_separation_arcsec= - AUTOMATIC_CALIBRATION_SELECTOR_COLOR_MATCH_RADIUS_ARCSEC): - best_match = None - best_separation = None + AUTOMATIC_CALIBRATION_SELECTOR_COLOR_MATCH_RADIUS_ARCSEC, + gaia_lookup_state=None, + gaia_match_radius_arcsec= + NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + nearby_rows = [] for row in nextastro_catalog_rows(catalog_response): row_ra = _finite_float(row.get('ra')) row_dec = _finite_float(row.get('dec')) if row_ra is None or row_dec is None: continue - color = nextastro_catalog_color(row, obs_filter) - if color is None: - continue separation = sky_separation_arcsec(ra, dec, row_ra, row_dec) if separation > float(max_separation_arcsec): continue - if best_separation is None or separation < best_separation: - best_match = { - 'catalog_row': row, - 'catalog_ra': row_ra, - 'catalog_dec': row_dec, - 'source_id': row.get('source_id'), - 'id': row.get('id'), - 'separation_arcsec': separation, - 'color': color, - } - best_separation = separation - return best_match + nearby_rows.append((separation, row, row_ra, row_dec)) + + for separation, row, row_ra, row_dec in sorted(nearby_rows, key=lambda item: item[0]): + color = nextastro_catalog_color_with_gaia_fallback( + row, + obs_filter, + lookup_state=gaia_lookup_state, + max_separation_arcsec=gaia_match_radius_arcsec, + ) + if color is None: + continue + return { + 'catalog_row': row, + 'catalog_ra': row_ra, + 'catalog_dec': row_dec, + 'source_id': row.get('source_id'), + 'id': row.get('id'), + 'separation_arcsec': separation, + 'color': color, + } + return None def select_automatic_optimal_calibration_stars( @@ -16562,7 +16717,8 @@ def select_automatic_optimal_calibration_stars( min_comp_target_sep=REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS, colour_term_metadata=None, brightest_first=False, - saturation_threshold=None): + saturation_threshold=None, + catalog_match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): max_count = parse_automatic_calibration_selector_count(count) if image_data is None or field_catalog is None: return [], [] @@ -16584,12 +16740,27 @@ def select_automatic_optimal_calibration_stars( height, width = image_shape[:2] target_xi = int(np.clip(round(target_x), 0, width - 1)) target_yi = int(np.clip(round(target_y), 0, height - 1)) + gaia_lookup_state = { + 'remaining': min( + NEXTASTRO_GAIA_COLOR_LOOKUP_MAX_PER_SELECTOR, + max(1, max_count * 2 + 1), + ), + 'attempted': 0, + 'matched': 0, + } if brightest_first: target_match = nextastro_photometry_catalog_match( field_catalog, ra_wcs[target_yi][target_xi], dec_wcs[target_yi][target_xi], obs_filter, + max_separation_arcsec=catalog_match_radius_arcsec, + ) + target_color = nextastro_catalog_color_with_gaia_fallback( + (target_match or {}).get('catalog_row'), + obs_filter, + lookup_state=gaia_lookup_state, + max_separation_arcsec=catalog_match_radius_arcsec, ) else: target_match = nextastro_catalog_nearest_color_row( @@ -16597,12 +16768,15 @@ def select_automatic_optimal_calibration_stars( ra_wcs[target_yi][target_xi], dec_wcs[target_yi][target_xi], obs_filter, + gaia_lookup_state=gaia_lookup_state, + gaia_match_radius_arcsec=catalog_match_radius_arcsec, ) - target_color = nextastro_catalog_color((target_match or {}).get('catalog_row'), obs_filter) + target_color = (target_match or {}).get('color') if not brightest_first and target_color is None: + log_nextastro_gaia_color_lookup_summary(gaia_lookup_state) log_info( "Warning: automatic calibration selector could not derive a target color from the " - "NextAstro photometry catalog.", + "NextAstro photometry catalog or Gaia DR3.", warn=True, ) return [], [] @@ -16685,8 +16859,14 @@ def select_automatic_optimal_calibration_stars( comp_ra, comp_dec, obs_filter, + max_separation_arcsec=catalog_match_radius_arcsec, + ) + color = nextastro_catalog_color_with_gaia_fallback( + (match or {}).get('catalog_row'), + obs_filter, + lookup_state=gaia_lookup_state, + max_separation_arcsec=catalog_match_radius_arcsec, ) - color = nextastro_catalog_color((match or {}).get('catalog_row'), obs_filter) if ( match is None or catalog_band_priority(match.get('mag_band'), obs_filter) != 0 @@ -16698,8 +16878,15 @@ def select_automatic_optimal_calibration_stars( else np.nan ) else: - match = nextastro_catalog_nearest_color_row(field_catalog, comp_ra, comp_dec, obs_filter) - color = nextastro_catalog_color((match or {}).get('catalog_row'), obs_filter) + match = nextastro_catalog_nearest_color_row( + field_catalog, + comp_ra, + comp_dec, + obs_filter, + gaia_lookup_state=gaia_lookup_state, + gaia_match_radius_arcsec=catalog_match_radius_arcsec, + ) + color = (match or {}).get('color') if match is None or color is None: continue color_delta = abs(color['color'] - target_color['color']) @@ -16772,6 +16959,7 @@ def select_automatic_optimal_calibration_stars( ) selected_candidates = candidates[:max_count] comp_stars = [[candidate['x'], candidate['y']] for candidate in selected_candidates] + log_nextastro_gaia_color_lookup_summary(gaia_lookup_state) return comp_stars, selected_candidates @@ -16844,8 +17032,75 @@ def calibration_catalog_identity(star, label=None): return None -def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, existing_comp_stars=None, - field_catalog=None): +def nextastro_catalog_color_with_gaia_fallback( + row, + obs_filter, + lookup_state=None, + max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + local_color = nextastro_catalog_color(row, obs_filter) + if local_color is not None or not isinstance(row, dict): + return local_color + if lookup_state is None or lookup_state.get('remaining', 0) <= 0: + return None + if lookup_state.get('error') is not None: + return None + + row_ra = _finite_float(row.get('ra')) + row_dec = _finite_float(row.get('dec')) + if row_ra is None or row_dec is None: + return None + + lookup_state['remaining'] -= 1 + lookup_state['attempted'] = lookup_state.get('attempted', 0) + 1 + try: + color = nextastro_gaia_bp_rp_for_coordinate( + row_ra, + row_dec, + max_separation_arcsec=max_separation_arcsec, + ) + except Exception as exc: + lookup_state['error'] = describe_retry_exception(exc) + return None + if color is not None: + lookup_state['matched'] = lookup_state.get('matched', 0) + 1 + return color + return None + + +def log_nextastro_gaia_color_lookup_summary(lookup_state): + if not isinstance(lookup_state, dict) or lookup_state.get('reported'): + return + lookup_state['reported'] = True + attempted = int(lookup_state.get('attempted', 0)) + matched = int(lookup_state.get('matched', 0)) + if attempted <= 0: + return + if matched: + log_info( + "Gaia DR3 BP-RP fallback supplied color data for " + f"{matched} of {attempted} queried star(s)." + ) + if lookup_state.get('error'): + log_info( + "Warning: Gaia DR3 BP-RP fallback became unavailable after " + f"{attempted} request(s): {lookup_state['error']}", + warn=True, + ) + elif lookup_state.get('remaining', 0) <= 0: + log_info( + "Warning: Gaia DR3 BP-RP fallback reached its per-selection request limit; " + "remaining candidates were evaluated only with photometry-catalog colors.", + warn=True, + ) + + +def merge_nextastro_calibration_stars( + comp_stars, + comp_ra_dec, + obs_filter, + existing_comp_stars=None, + field_catalog=None, + match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): calibration_stars = dict(existing_comp_stars or {}) existing_positions = { tuple(value.get('pos', [])) @@ -16869,7 +17124,13 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist match = None if field_catalog is not None: - match = nextastro_photometry_catalog_match(field_catalog, comp_ra, comp_dec, obs_filter) + match = nextastro_photometry_catalog_match( + field_catalog, + comp_ra, + comp_dec, + obs_filter, + max_separation_arcsec=match_radius_arcsec, + ) candidates.append({ 'index': index, 'comp_pos': comp_pos, @@ -16885,6 +17146,7 @@ def merge_nextastro_calibration_stars(comp_stars, comp_ra_dec, obs_filter, exist remote_matches = nextastro_photometry_for_coordinates( [(candidate['ra'], candidate['dec']) for candidate in unresolved], obs_filter, + radius_arcsec=match_radius_arcsec, ) for candidate, match in zip(unresolved, remote_matches): candidate['match'] = match @@ -17327,6 +17589,17 @@ def comparison_star_pixel_distance(first_position, second_position): return float(np.hypot(first[0] - second[0], first[1] - second[1])) +def tracked_comparison_position(tracked_comparison_stars, comp_index): + """Return a position using the stable full tracking-list index space.""" + index = int(comp_index) + if index < 0 or index >= len(tracked_comparison_stars): + raise IndexError( + f"Tracked comparison index {index} is outside the " + f"{len(tracked_comparison_stars)}-star calibration pool." + ) + return list(tracked_comparison_stars[index]) + + def check_comp_star_exists(user_stars, vsp_star, tol=VSP_COMPARISON_MATCH_TOLERANCE_PIXELS): """Return the nearest user-entered comparison within ``tol`` pixels. @@ -21248,12 +21521,17 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ ): raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") observed_filter = observed_filter or comp_star.get('observed_filter') - if catalog_band_priority(comp_star.get('mag_band'), observed_filter) != 0: + catalog_mag_band = comp_star.get('mag_band', 'V') + if catalog_band_priority(catalog_mag_band, observed_filter) != 0: raise RuntimeError( "Comparison-star catalog magnitude band " - f"{comp_star.get('mag_band')!r} does not match observed filter " + f"{catalog_mag_band!r} does not match observed filter " f"{observed_filter!r}; cross-band absolute calibration is not permitted." ) + measurement_mag_band = reported_stellar_variability_band( + observed_filter, + fallback_band=catalog_mag_band, + ) fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) fit_airmass_model = np.asarray( @@ -21369,7 +21647,8 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ 'catalog_dec': comp_star.get('catalog_dec'), 'catalog_source': comp_star.get('catalog_source', 'AAVSO VSP'), 'is_aavso_vsp': bool(comp_star.get('is_aavso_vsp', True)), - 'mag_band': comp_star.get('mag_band', 'V'), + 'mag_band': measurement_mag_band, + 'catalog_mag_band': catalog_mag_band, 'observed_filter': observed_filter, 'source_id': comp_star.get('source_id'), 'catalog_id': comp_star.get('id'), @@ -21442,8 +21721,13 @@ def aligned_reference_curve_ratio(selected_fit, anchor_fit): ] -def build_direct_selected_catalog_candidate(comp_stars, comp_ra_dec, field_catalog, best_comp, - observed_filter=None): +def build_direct_selected_catalog_candidate( + comp_stars, + comp_ra_dec, + field_catalog, + best_comp, + observed_filter=None, + match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): if best_comp is None or best_comp < 0 or best_comp >= len(comp_stars): return None if field_catalog is None or not comp_ra_dec or best_comp >= len(comp_ra_dec): @@ -21459,8 +21743,28 @@ def build_direct_selected_catalog_candidate(comp_stars, comp_ra_dec, field_catal comp_ra, comp_dec, observed_filter, + max_separation_arcsec=match_radius_arcsec, max_magnitude_error=MAX_APPARENT_MAGNITUDE, ) + effective_match_radius = _finite_float(match_radius_arcsec) + if ( + match is None + and effective_match_radius is not None + and effective_match_radius > NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC + ): + try: + match = nextastro_photometry_for_coordinate( + comp_ra, + comp_dec, + observed_filter, + radius_arcsec=effective_match_radius, + ) + except Exception as exc: + log_info( + "Warning: scale-aware direct catalog lookup for the selected comparison star " + f"failed ({describe_retry_exception(exc)}).", + warn=True, + ) if match is None: return None if catalog_band_priority(match.get('mag_band'), observed_filter) != 0: @@ -21532,7 +21836,7 @@ def combine_catalog_reference_estimates(derived_estimates, selected_pos, observe def preferred_catalog_magnitude_band_for_filter(observed_filter): - candidates = nextastro_photometry_band_candidates(observed_filter, include_fallback=False) + candidates = nextastro_photometry_band_candidates(observed_filter) if not candidates: return None return candidates[0][2] @@ -21774,8 +22078,14 @@ def choose_selected_comp_catalog_reference_candidate(candidates, observed_filter ] if not usable: return None + source_priority = { + 'direct_catalog': 0, + 'provided_comp_derived': 1, + 'field_derived': 2, + } usable.sort(key=lambda candidate: ( catalog_band_priority((candidate.get('star') or {}).get('mag_band'), observed_filter), + source_priority.get(candidate.get('source'), 3), catalog_reference_candidate_error(candidate), )) return usable[0] @@ -21783,9 +22093,8 @@ def choose_selected_comp_catalog_reference_candidate(candidates, observed_filter def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vsp_ind, best_comp, save, s_name, observed_filter=None, comp_ra_dec=None, field_catalog=None, reference_image=None, - wcs_file=None): - info_comps = {} - + wcs_file=None, + catalog_match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): try: if best_comp is None: log_info( @@ -21794,7 +22103,7 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs ) return [] comp_pos = comp_stars[best_comp] - info_comps[best_comp] = calculate_variablility(fit_lc_refs[best_comp]['myfit'], fit_lc_best) + selected_comp_fit = fit_lc_refs[best_comp]['myfit'] except Exception as e: log_info(f"Error selecting or calculating variability for comparison star: {e}", warn=True) return [] @@ -21816,6 +22125,7 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs field_catalog, best_comp, observed_filter=observed_filter, + match_radius_arcsec=catalog_match_radius_arcsec, ) if direct_relaxed is not None: candidates.append(direct_relaxed) @@ -21892,9 +22202,8 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs ) try: - info_comp = info_comps[comp_stars.index(comp_pos)] return build_stellar_variability_params_from_fit( - info_comp['fit_lc'], + selected_comp_fit, comp_star, comp_pos, vsp_auid_comp, @@ -26980,12 +27289,17 @@ def stellar_variability_calibration_for_position(calibration_stars, position, ob return candidates[0] -def stellar_variability_catalog_profile(catalog_match, observed_filter=None): +def stellar_variability_catalog_profile( + catalog_match, observed_filter=None, gaia_lookup_state=None): if not isinstance(catalog_match, dict): return {} magnitude = _finite_float(catalog_match.get('mag')) magnitude_error = normalized_magnitude_error(catalog_match.get('error')) - color = nextastro_catalog_color(catalog_match.get('catalog_row'), observed_filter) + color = nextastro_catalog_color_with_gaia_fallback( + catalog_match.get('catalog_row'), + observed_filter, + lookup_state=gaia_lookup_state, + ) return { 'magnitude': float(magnitude) if magnitude is not None else None, 'magnitude_error': float(magnitude_error) if magnitude_error is not None else None, @@ -26999,11 +27313,13 @@ def stellar_variability_catalog_profile(catalog_match, observed_filter=None): } -def add_stellar_variability_member_similarity(candidate, target_profile, observed_filter=None): +def add_stellar_variability_member_similarity( + candidate, target_profile, observed_filter=None, gaia_lookup_state=None): enriched = dict(candidate) - member_color = nextastro_catalog_color( + member_color = nextastro_catalog_color_with_gaia_fallback( enriched.get('star', {}).get('catalog_row'), observed_filter, + lookup_state=gaia_lookup_state, ) member_color_value = _finite_float((member_color or {}).get('color')) target_color_value = _finite_float((target_profile or {}).get('color')) @@ -27268,7 +27584,16 @@ def select_stellar_variability_ensemble_members( max_members=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, min_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, times=None): - target_profile = stellar_variability_catalog_profile(target_catalog_match, observed_filter) + gaia_lookup_state = { + 'remaining': NEXTASTRO_GAIA_COLOR_LOOKUP_MAX_PER_SELECTOR, + 'attempted': 0, + 'matched': 0, + } + target_profile = stellar_variability_catalog_profile( + target_catalog_match, + observed_filter, + gaia_lookup_state=gaia_lookup_state, + ) candidates = [] rejected = [] represented_catalog_identities = set() @@ -27307,15 +27632,20 @@ def select_stellar_variability_ensemble_members( rejected.append({'key': ckey, 'reason': 'invalid median brightness'}) continue - candidates.append(add_stellar_variability_member_similarity({ - 'key': ckey, - 'comp_index': summary.get('comp_index'), - 'label': summary.get('label', ckey), - 'position': position, - 'summary': summary, - 'median_flux': median_flux, - **calibration, - }, target_profile, observed_filter=observed_filter)) + candidates.append(add_stellar_variability_member_similarity( + { + 'key': ckey, + 'comp_index': summary.get('comp_index'), + 'label': summary.get('label', ckey), + 'position': position, + 'summary': summary, + 'median_flux': median_flux, + **calibration, + }, + target_profile, + observed_filter=observed_filter, + gaia_lookup_state=gaia_lookup_state, + )) if catalog_identity is not None: represented_catalog_identities.add(catalog_identity) @@ -27416,6 +27746,7 @@ def select_stellar_variability_ensemble_members( for selection_rank, member in enumerate(members, start=1): member['selection_rank'] = selection_rank + log_nextastro_gaia_color_lookup_summary(gaia_lookup_state) return { 'members': members, 'rejected': rejected, @@ -27697,7 +28028,7 @@ def save_stellar_variability_magnitude_csv(vsp_params, save, target_name, observ row.get('airmass'), row.get('mag'), row.get('mag_err'), - row.get('observed_filter') or row.get('mag_band'), + row.get('mag_band') or row.get('observed_filter'), row.get('cname'), ]) return output_path @@ -27796,7 +28127,11 @@ def build_stellar_variability_ensemble_params_from_fit( 'catalog_dec': None, 'catalog_source': 'Calibrated comparison-star ensemble', 'is_aavso_vsp': False, - 'mag_band': observed_filter or 'V', + 'mag_band': reported_stellar_variability_band( + observed_filter, + fallback_band=observed_filter or 'V', + ), + 'catalog_mag_band': preferred_catalog_magnitude_band_for_filter(observed_filter), 'observed_filter': observed_filter, 'ensemble_reference': True, 'ensemble_member_count': len(members), @@ -28646,7 +28981,7 @@ def fortuitous_variable_target_metadata(variable): 'classification_folder': variable.get('category'), 'classification_rule': ( 'optimal_variables requires VSX period <= 10 days and amplitude >= 0.3 mag; ' - 'all other retained VSX stars use rest_of_the_variables.' + 'all other retained VSX stars use normal.' ), 'reference_aperture_flux_adu': variable.get('aperture_flux_adu'), 'reference_count_rate_adu_per_second': variable.get('count_rate_adu_per_second'), @@ -28716,6 +29051,17 @@ def clear_previous_fortuitous_variable_products(variable_dir): plot_path = output_dir / 'working_artifacts' / 'Stellar_Variability.png' if plot_path.is_file(): plot_path.unlink() + working_artifacts_dir = output_dir / 'working_artifacts' + if working_artifacts_dir.is_dir(): + try: + working_artifacts_dir.rmdir() + except OSError: + pass + if output_dir.is_dir(): + try: + output_dir.rmdir() + except OSError: + pass def process_fortuitous_variables( @@ -28775,7 +29121,7 @@ def process_fortuitous_variables( if exposure_array.shape != times.shape: exposure_array = None - base_dir = Path(info_dict['save']) / 'fortuitous_variables' + base_dir = Path(info_dict['save']) / 'variables' base_dir.mkdir(parents=True, exist_ok=True) for stale_combined_aid in base_dir.glob('AID_AAVSO_FortuitousVariables_*.txt'): if stale_combined_aid.is_file(): @@ -28787,7 +29133,10 @@ def process_fortuitous_variables( variable = dict(variable) variable['reference_mode'] = reference_mode variable_name = variable.get('name') or 'VSX variable' - category = variable.get('category') or 'rest_of_the_variables' + category = variable.get('category') or 'normal' + if category == 'rest_of_the_variables': + category = 'normal' + variable['category'] = category variable['input_frame_count'] = int(times.size) variable_overexposed_mask = np.zeros(times.shape, dtype=bool) tracking_key = variable.get('tracking_key') @@ -28798,10 +29147,7 @@ def process_fortuitous_variables( variable['target_overexposure_rejected_frame_count'] = int( np.count_nonzero(variable_overexposed_mask) ) - variable_dir = base_dir / category / safe_output_filename( - 'VSX', variable_name, extension='' - ) - variable_dir.mkdir(parents=True, exist_ok=True) + variable_dir = base_dir / category / safe_output_filename(variable_name, extension='') clear_previous_fortuitous_variable_products(variable_dir) try: target_flux, target_flux_error, target_quality_mask = fortuitous_variable_target_series( @@ -28978,6 +29324,7 @@ def process_fortuitous_variables( ) if fit is None: raise ValueError('stellar-variability light curve construction failed') + variable_dir.mkdir(parents=True, exist_ok=True) if use_single_comparison: vsp_params = build_stellar_variability_params_from_fit( fit, @@ -29080,11 +29427,12 @@ def process_fortuitous_variables( f"({reference_mode}), outputs={variable_dir}." ) except Exception as exc: + clear_previous_fortuitous_variable_products(variable_dir) results.append({ 'name': variable_name, 'auid': variable.get('auid'), 'category': category, - 'output_directory': str(variable_dir), + 'output_directory': None, 'reference_mode': reference_mode, 'comparison_label': variable.get('comparison_label'), 'comparison_gap_stability': variable.get('comparison_gap_stability'), @@ -29108,24 +29456,6 @@ def process_fortuitous_variables( 'status': 'skipped', 'reason': str(exc), }) - status_path = variable_dir / safe_output_filename( - 'FortuitousVariableStatus', - variable_name, - filename_date_token(info_dict.get('date')), - extension='json', - ) - with status_path.open('w', encoding='utf-8') as handle: - json.dump( - stellar_variability_json_safe({ - 'target': fortuitous_variable_target_metadata(variable), - 'status': 'skipped', - 'reason': str(exc), - }), - handle, - indent=2, - sort_keys=True, - ) - handle.write('\n') log_info( f"Warning: fortuitous-variable photometry skipped {variable_name} ({exc}).", warn=True, @@ -30996,6 +31326,12 @@ def lookup_archive_ephemeris(): ra=hint_ra, dec=hint_dec, pixel_scale=exotic_infoDict.get('pixel_scale'), ignore_header_wcs=ignore_header_wcs) img_scale_str, img_scale = get_img_scale(header, wcs_file, exotic_infoDict['pixel_scale']) + photometry_catalog_match_radius_arcsec = nextastro_catalog_match_radius_arcsec(img_scale) + log_info( + "NextAstro photometry matches for image-derived positions will use a " + f"{photometry_catalog_match_radius_arcsec:.2f} arcsec radius " + f"(max of {NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC:.1f} arcsec and one image pixel)." + ) plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) ra_dec_tar, ra_dec_wcs = None, [] chart_id, vsp_comp_stars, vsp_list = None, {}, [] @@ -31088,6 +31424,7 @@ def lookup_archive_ephemeris(): ra_dec_tar[0], ra_dec_tar[1], exotic_infoDict['filter'], + max_separation_arcsec=photometry_catalog_match_radius_arcsec, ) if reference_fallback is not None: @@ -31157,6 +31494,7 @@ def lookup_archive_ephemeris(): colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), brightest_first=stellar_variability_ensemble_candidate_search, saturation_threshold=ensemble_candidate_saturation_threshold, + catalog_match_radius_arcsec=photometry_catalog_match_radius_arcsec, ) log_automatic_optimal_calibration_selection( automatic_comp_stars, @@ -31295,6 +31633,7 @@ def lookup_archive_ephemeris(): colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), brightest_first=True, saturation_threshold=fortuitous_saturation_threshold, + catalog_match_radius_arcsec=photometry_catalog_match_radius_arcsec, ) check_for_variable_stars( ra_wcs, @@ -31343,6 +31682,7 @@ def lookup_archive_ephemeris(): exotic_infoDict['filter'], existing_comp_stars=vsp_comp_stars, field_catalog=nextastro_field_catalog, + match_radius_arcsec=photometry_catalog_match_radius_arcsec, ) usable_science_nextastro_v = any( star.get('catalog_source') == 'NextAstro photometry catalog' @@ -31377,6 +31717,7 @@ def lookup_archive_ephemeris(): colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), brightest_first=True, saturation_threshold=fortuitous_saturation_threshold, + catalog_match_radius_arcsec=photometry_catalog_match_radius_arcsec, ) check_for_variable_stars( ra_wcs, @@ -31423,6 +31764,7 @@ def lookup_archive_ephemeris(): exotic_infoDict['filter'], existing_comp_stars=vsp_comp_stars, field_catalog=nextastro_field_catalog, + match_radius_arcsec=photometry_catalog_match_radius_arcsec, ) _, fallback_vsp_stars, fallback_chart_id, fallback_vsp_queried = ( merge_aavso_vsp_v_calibration_fallback( @@ -31483,6 +31825,7 @@ def lookup_archive_ephemeris(): exotic_infoDict['filter'], existing_comp_stars=vsp_comp_stars, field_catalog=nextastro_field_catalog, + match_radius_arcsec=photometry_catalog_match_radius_arcsec, ) fortuitous_calibration_stars = merge_nextastro_calibration_stars( fortuitous_ensemble_stars, @@ -31490,6 +31833,7 @@ def lookup_archive_ephemeris(): exotic_infoDict['filter'], existing_comp_stars=vsp_comp_stars, field_catalog=nextastro_field_catalog, + match_radius_arcsec=photometry_catalog_match_radius_arcsec, ) # The target variability plot can use any tracked, VSX-vetted # catalog calibration, including full-field NextAstro candidates @@ -33413,6 +33757,19 @@ def lookup_archive_ephemeris(): x_ref=ref_centroid_x, y_ref=ref_centroid_y) + # A selected full-resolution retry can replace the original + # fit object after its raw photometry annotations were made. + # Reattach the final aligned flux arrays here so absolute + # stellar-variability magnitudes never have to be inferred + # from the normalized light curve. + annotate_stellar_variability_raw_photometry( + myfit, + tFlux1, + cFlux1, + target_flux_error=tFlux1_error, + comp_flux_error=cFlux1_error, + ) + if selected_comp_index is not None: ref_flux[selected_comp_index] = { 'myfit': myfit, @@ -33438,7 +33795,7 @@ def lookup_archive_ephemeris(): gain_e_per_adu=fallback_gain_e_per_adu, comp_index=j, comp_label=f"Comp {j + 1}", - comp_position=exotic_infoDict['comp_stars'][j], + comp_position=tracked_comparison_position(fortuitous_ensemble_stars, j), method_label=comparison_calibration['method_label'], plot_time_range=full_plot_time_range, ) @@ -33459,7 +33816,7 @@ def lookup_archive_ephemeris(): continue ref_flux[j] = { 'myfit': vsp_fit, - 'pos': exotic_infoDict['comp_stars'][j] + 'pos': tracked_comparison_position(fortuitous_ensemble_stars, j) } else: best_a = comparison_calibration['a'] @@ -33502,7 +33859,7 @@ def lookup_archive_ephemeris(): gain_e_per_adu=fallback_gain_e_per_adu, comp_index=j, comp_label=f"Comp {j + 1}", - comp_position=exotic_infoDict['comp_stars'][j], + comp_position=tracked_comparison_position(fortuitous_ensemble_stars, j), method_label=comparison_calibration['method_label'], plot_time_range=full_plot_time_range, ) @@ -33526,7 +33883,7 @@ def lookup_archive_ephemeris(): continue ref_flux[j] = { 'myfit': vsp_fit, - 'pos': exotic_infoDict['comp_stars'][j] + 'pos': tracked_comparison_position(fortuitous_ensemble_stars, j) } else: if fit_attempts: @@ -34008,7 +34365,9 @@ def lookup_archive_ephemeris(): comp_ra_dec=ra_dec_wcs[:len(fortuitous_ensemble_stars)], field_catalog=nextastro_field_catalog, reference_image=reference_image, - wcs_file=wcs_file) + wcs_file=wcs_file, + catalog_match_radius_arcsec= + photometry_catalog_match_radius_arcsec) else: log_info( "Skipping AID magnitude output because no reference comparison star was selected.", diff --git a/exotic/output_files.py b/exotic/output_files.py index d435ebbe..d35d309c 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -157,7 +157,7 @@ def stellar_variability_reference_summary(vsp_param): return "na" cname = vsp_param.get('cname', 'na') - band = vsp_param.get('mag_band') or 'V' + band = vsp_param.get('catalog_mag_band') or vsp_param.get('mag_band') or 'V' if vsp_param.get('ensemble_reference'): member_count = int(vsp_param.get('ensemble_member_count', 0) or 0) labels = vsp_param.get('ensemble_member_labels') or [] @@ -226,7 +226,8 @@ def aid_comparison_metadata(vsp_param): 'catalog_match_separation_arcsec': vsp_param.get('separation_arcsec'), 'derived_catalog_reference': bool(vsp_param.get('derived_catalog_reference', False)), 'derived_reference_anchor_count': vsp_param.get('derived_reference_anchor_count'), - 'magnitude_band': vsp_param.get('mag_band'), + 'magnitude_band': vsp_param.get('catalog_mag_band') or vsp_param.get('mag_band'), + 'reported_measurement_band': vsp_param.get('mag_band'), 'apparent_magnitude': rounded_magnitude_value(vsp_param.get('cmag')), 'apparent_magnitude_error': rounded_magnitude_error(vsp_param.get('cmag_err')), 'ensemble_reference': bool(vsp_param.get('ensemble_reference', False)), diff --git a/exotic/plots.py b/exotic/plots.py index c94f6ff7..97cc8b63 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -603,7 +603,7 @@ def _stellar_variability_comparison_metadata_label(vsp_param): details.append(f"Original filter: {observed_filter}") comparison_mag = magnitude_text( - vsp_param.get('mag_band') or 'V', + vsp_param.get('catalog_mag_band') or vsp_param.get('mag_band') or 'V', vsp_param.get('cmag'), vsp_param.get('cmag_err'), ) @@ -644,7 +644,7 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): ax.set_xlabel("Time [JD]") fig.tight_layout() output_dir = _working_artifacts_dir(save) - fig.savefig(output_dir / f"Stellar_Variability.png") + fig.savefig(output_dir / f"Stellar_Variability.png", bbox_inches="tight") plt.close(fig) diff --git a/inits.json b/inits.json index 6e459a3d..06467452 100644 --- a/inits.json +++ b/inits.json @@ -30,7 +30,7 @@ "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, use the default calibrated comparison-star ensemble, and discard predicted ingress-to-egress transit-window points. Default false.", "Stellar Variability Ensemble": "Set optional_info 'use_ensemble_photometry_for_stellar_variability' to false to disable the default calibrated ensemble in stellar_variability_only runs and restore single-comparison selection by out-of-transit scatter. The default ensemble automatically finds bright catalog-calibrated field-star candidates, removes saturated and VSX-variable stars, sigma-clips high catalog magnitude uncertainties, and when more than five remain uses the five closest to the target in catalog colour and magnitude. EnsembleSelection JSON records the target and comparison colours, magnitudes, errors, and selection deltas beside the final results.", - "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent calibrated photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under fortuitous_variables/optimal_variables for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under fortuitous_variables/rest_of_the_variables.", + "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent calibrated photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under variables/optimal_variables/ for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under variables/normal/. Skipped variables are recorded only in the shared variables manifest and do not receive an object directory.", "Fortuitous Variable Single Comparison": "Set optional_info 'use_single_comparison_for_fortuitous_variables' to false to use the calibrated comparison-star ensemble for fortuitous VSX targets. The default true selects one unsaturated, non-variable, catalog-calibrated comparison star closest to each VSX target in catalog colour and magnitude.", "Automatic AAVSO V Calibration Fallback": "For V-family observations, including Clear, CV, and bv, EXOTIC always requires V-band comparison magnitudes. If the NextAstro photometry server supplies no usable V calibration, EXOTIC automatically queries AAVSO VSP and adds matched or discovered V-sequence stars to the same single-comparison or ensemble calibration pool, even when 'Add Comparison Stars from AAVSO?' is n.", "NextAstro VSX Cache First": "Set optional_info 'use_nextastro_vsx_cache_first' to true to query the NextAstro /vsx_query field cache before AAVSO VSX. The default is false. Empty or failed cache lookups fall back to AAVSO; legacy cache responses lacking the full period/amplitude schema are enriched from AAVSO.", diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index bee4ae03..cbdfcb1b 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -1280,7 +1280,7 @@ def add_blob(x_pos, y_pos, value): dec_wcs = np.tile(np.arange(300, dtype=float)[:, None], (1, 300)) catalog = {"rows": []} - def fake_color_match(_catalog, ra, dec, obs_filter, max_separation_arcsec=5.0): + def fake_color_match(_catalog, ra, dec, obs_filter, max_separation_arcsec=5.0, **kwargs): colors = { (150, 150): (12.0, 11.4), (220, 220): (13.0, 12.41), @@ -1291,13 +1291,21 @@ def fake_color_match(_catalog, ra, dec, obs_filter, max_separation_arcsec=5.0): if key not in colors: return None b_mag, v_mag = colors[key] - return {"catalog_row": {"Bmag": b_mag, "Vmag": v_mag}} + return { + "catalog_row": {"Bmag": b_mag, "Vmag": v_mag}, + "color": { + "color": b_mag - v_mag, + "label": "B-V", + "first_column": "Bmag", + "second_column": "Vmag", + }, + } monkeypatch.setattr(exotic_module, "nextastro_catalog_nearest_color_row", fake_color_match) monkeypatch.setattr( exotic_module, "nextastro_photometry_catalog_match", - lambda catalog_response, ra, dec, obs_filter: ( + lambda catalog_response, ra, dec, obs_filter, **kwargs: ( { **fake_color_match(catalog_response, ra, dec, obs_filter), "mag": 12.0, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 2f78564b..5d34ea33 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -456,7 +456,7 @@ def test_nextastro_photometry_catalog_match_ignores_over_30_magnitudes(): assert match is None -def test_nextastro_photometry_catalog_match_rejects_separations_over_two_arcsec(): +def test_nextastro_photometry_catalog_match_honors_scale_aware_radius(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], 'count': 1, @@ -473,41 +473,349 @@ def test_nextastro_photometry_catalog_match_rejects_separations_over_two_arcsec( ], } - match = exotic_module.nextastro_photometry_catalog_match( + default_match = exotic_module.nextastro_photometry_catalog_match( + catalog, + 10.0, + 20.0, + 'CV', + ) + scale_aware_match = exotic_module.nextastro_photometry_catalog_match( catalog, 10.0, 20.0, 'CV', - max_separation_arcsec=30.0, + max_separation_arcsec=5.2, ) - assert match is None + assert default_match is None + assert scale_aware_match['source_id'] == 111 + assert 2.0 < scale_aware_match['separation_arcsec'] < 5.2 + + +def test_nextastro_catalog_match_radius_uses_one_pixel_with_two_arcsec_floor(): + assert exotic_module.nextastro_catalog_match_radius_arcsec(None) == pytest.approx(2.0) + assert exotic_module.nextastro_catalog_match_radius_arcsec(1.4) == pytest.approx(2.0) + assert exotic_module.nextastro_catalog_match_radius_arcsec(5.153485) == pytest.approx(5.153485) + + +def test_direct_selected_catalog_candidate_uses_targeted_scale_aware_lookup(monkeypatch): + calls = [] + + def fake_lookup(ra, dec, observed_filter, radius_arcsec=2.0): + calls.append((ra, dec, observed_filter, radius_arcsec)) + return { + 'id': 185212647, + 'source_id': None, + 'mag': 10.038, + 'error': 0.027, + 'mag_band': 'V', + 'catalog_ra': 294.68751, + 'catalog_dec': 31.360037, + 'separation_arcsec': 2.643594, + } + + monkeypatch.setattr(exotic_module, 'nextastro_photometry_for_coordinate', fake_lookup) + + candidate = exotic_module.build_direct_selected_catalog_candidate( + [[232.0, 348.0]], + [(294.68834158880503, 31.36022406806484)], + {'row_format': 'objects', 'rows': []}, + 0, + observed_filter='CV', + match_radius_arcsec=5.153485, + ) + + assert calls == [(294.68834158880503, 31.36022406806484, 'CV', 5.153485)] + assert candidate['source'] == 'direct_catalog' + assert candidate['star']['mag'] == pytest.approx(10.038) + assert candidate['star']['mag_band'] == 'V' + assert candidate['star']['pos'] == [232.0, 348.0] + + +def test_reported_stellar_variability_band_distinguishes_clearv_from_catalog_v(): + for observed_filter in ( + 'MObs CV', + 'CV', + 'Clear', + 'Luminance', + 'Photographic G', + 'Gaia G', + 'G', + 'G1', + 'G2', + ): + assert exotic_module.reported_stellar_variability_band( + observed_filter, 'V' + ) == 'ClearV' + assert exotic_module.reported_stellar_variability_band('bv', 'V') == 'V' + + +def test_selected_comparison_direct_catalog_match_precedes_derived_fallback(): + selected = exotic_module.choose_selected_comp_catalog_reference_candidate( + [ + { + 'source': 'field_derived', + 'error': 0.010, + 'star': {'mag_band': 'V', 'error': 0.010}, + }, + { + 'source': 'direct_catalog', + 'error': 0.027, + 'star': {'mag_band': 'V', 'error': 0.027}, + }, + ], + observed_filter='CV', + ) + + assert selected['source'] == 'direct_catalog' def test_aavso_vsp_band_for_filter_uses_observed_filter_aliases(): assert exotic_module.aavso_vsp_band_for_filter('MObs CV') == 'V' assert exotic_module.aavso_vsp_band_for_filter('Clear (unfiltered) reduced to V sequence') == 'V' + assert exotic_module.aavso_vsp_band_for_filter('Photographic G') == 'V' + assert exotic_module.aavso_vsp_band_for_filter('Gaia G') == 'V' assert exotic_module.aavso_vsp_band_for_filter('Cousins R') == 'Rc' assert exotic_module.aavso_vsp_band_for_filter('Sloan g') == 'SG' -def test_nextastro_catalog_band_tokens_keep_bessell_sloan_and_gaia_distinct(): - assert exotic_module.nextastro_photometry_band_candidates('bv', include_fallback=False) == [ +def test_nextastro_catalog_band_tokens_keep_bessell_sloan_and_clearv_distinct(): + assert exotic_module.nextastro_photometry_band_candidates('bv') == [ ('Vmag', 'err_Vmag', 'V') ] - assert exotic_module.nextastro_photometry_band_candidates('bb', include_fallback=False) == [ + assert exotic_module.nextastro_photometry_band_candidates('bb') == [ ('Bmag', 'err_Bmag', 'B') ] - assert exotic_module.nextastro_photometry_band_candidates( - 'Sloan g', include_fallback=False - ) == [('g', 'dg', 'g')] - assert exotic_module.nextastro_photometry_band_candidates('G', include_fallback=False) == [] - assert exotic_module.nextastro_photometry_band_candidates('Gaia G', include_fallback=False) == [] + assert exotic_module.nextastro_photometry_band_candidates('Sloan g') == [('g', 'dg', 'g')] + for observed_filter in ('G', 'Photographic G', 'Gaia G', 'G1', 'G2'): + assert exotic_module.nextastro_photometry_band_candidates(observed_filter) == [ + ('Vmag', 'err_Vmag', 'V') + ] assert exotic_module.catalog_band_priority('V', 'bv') == 0 assert exotic_module.catalog_band_priority('B', 'bb') == 0 assert exotic_module.catalog_band_priority('g', 'Sloan g') == 0 assert exotic_module.catalog_band_priority('g', 'G') == 1 assert exotic_module.catalog_band_priority('g', 'Gaia G') == 1 + assert exotic_module.catalog_band_priority('V', 'G') == 0 + assert exotic_module.catalog_band_priority('V', 'Gaia G') == 0 + + +@pytest.mark.parametrize( + ('observed_filter', 'expected_labels'), + [ + ('u', ['u-g', 'B-V', 'BP-RP']), + ('Johnson U', ['u-g', 'B-V', 'BP-RP']), + ('B', ['B-V', 'BP-RP']), + ('Photographic B', ['B-V', 'BP-RP']), + ('V', ['B-V', 'BP-RP']), + ('Sloan g', ['g-r', 'B-V', 'BP-RP']), + ('Sloan r', ['r-i', 'B-V', 'BP-RP']), + ('Cousins R', ['r-i', 'B-V', 'BP-RP']), + ('Sloan i', ['r-i', 'B-V', 'BP-RP']), + ('Cousins I', ['r-i', 'B-V', 'BP-RP']), + ('Sloan z', ['i-z', 'B-V', 'BP-RP']), + ('CV', ['B-V', 'BP-RP']), + ('Clear', ['B-V', 'BP-RP']), + ('Luminance', ['B-V', 'BP-RP']), + ('Photographic G', ['B-V', 'BP-RP']), + ('Gaia G', ['B-V', 'BP-RP']), + ('G', ['B-V', 'BP-RP']), + ('G1', ['B-V', 'BP-RP']), + ('G2', ['B-V', 'BP-RP']), + ('Unknown', ['B-V', 'BP-RP']), + ], +) +def test_all_filters_use_filter_specific_color_then_universal_fallbacks( + observed_filter, expected_labels): + pairs = exotic_module.nextastro_color_candidate_pairs(observed_filter) + assert [pair[2] for pair in pairs] == expected_labels + + +@pytest.mark.parametrize( + ('observed_filter', 'primary_label', 'primary_columns'), + [ + ('u', 'u-g', ('umag',)), + ('B', 'B-V', ()), + ('V', 'B-V', ()), + ('Sloan g', 'g-r', ('g',)), + ('Sloan r', 'r-i', ('r',)), + ('Sloan i', 'r-i', ('r',)), + ('Sloan z', 'i-z', ('i',)), + ('Clear', 'B-V', ()), + ('Photographic G', 'B-V', ()), + ('Gaia G', 'B-V', ()), + ], +) +def test_all_filters_fall_back_from_specific_color_to_bv_then_bp_rp( + observed_filter, primary_label, primary_columns): + row = { + 'Bmag': 13.0, + 'Vmag': 12.5, + 'umag': 13.8, + 'g': 12.8, + 'r': 12.2, + 'i': 12.0, + 'z': 11.8, + 'bp_rp': 1.1, + } + assert exotic_module.nextastro_catalog_color(row, observed_filter)['label'] == primary_label + + for column in primary_columns: + row[column] = None + assert exotic_module.nextastro_catalog_color(row, observed_filter)['label'] == 'B-V' + + row['Bmag'] = None + assert exotic_module.nextastro_catalog_color(row, observed_filter) == { + 'color': pytest.approx(1.1), + 'label': 'BP-RP', + 'first_column': 'bp_rp', + 'second_column': None, + } + + +def test_clearv_catalog_color_derives_bp_rp_from_gaia_magnitudes(): + color = exotic_module.nextastro_catalog_color( + { + 'Vmag': 12.5, + 'phot_bp_mean_mag': 13.4, + 'phot_rp_mean_mag': 12.1, + }, + 'Gaia G', + ) + + assert color == { + 'color': pytest.approx(1.3), + 'label': 'BP-RP', + 'first_column': 'phot_bp_mean_mag', + 'second_column': 'phot_rp_mean_mag', + } + + +def test_nextastro_gaia_bp_rp_lookup_is_cached(monkeypatch): + captured = [] + + def fake_get(url, params, timeout): + captured.append((url, params, timeout)) + return DummyResponse({ + 'gaia': { + 'source_id': 123456, + 'separation_arcsec': 0.2, + 'phot_bp_mean_mag': 13.4, + 'phot_rp_mean_mag': 12.1, + 'bp_rp': 1.3, + }, + }) + + exotic_module._cached_nextastro_gaia_bp_rp.cache_clear() + monkeypatch.setattr(exotic_module.requests, 'get', fake_get) + + first = exotic_module.nextastro_gaia_bp_rp_for_coordinate(10.12345678, -20.25) + second = exotic_module.nextastro_gaia_bp_rp_for_coordinate(10.12345678, -20.25) + + assert first == second + assert first['color'] == pytest.approx(1.3) + assert first['label'] == 'BP-RP' + assert first['catalog_source'] == 'NextAstro Gaia DR3' + assert captured == [( + exotic_module.NEXTASTRO_GAIA_DISTPM_ENDPOINT, + {'ra': 10.1234568, 'dec': -20.25}, + exotic_module.NEXTASTRO_GAIA_COLOR_LOOKUP_TIMEOUT_SECONDS, + )] + + +def test_local_catalog_color_does_not_query_gaia(monkeypatch): + monkeypatch.setattr( + exotic_module, + 'nextastro_gaia_bp_rp_for_coordinate', + lambda *args, **kwargs: pytest.fail('Gaia should not be queried when B-V is available.'), + ) + state = {'remaining': 3, 'attempted': 0, 'matched': 0} + + color = exotic_module.nextastro_catalog_color_with_gaia_fallback( + {'ra': 10.0, 'dec': 20.0, 'Bmag': 13.0, 'Vmag': 12.5}, + 'V', + lookup_state=state, + ) + + assert color['label'] == 'B-V' + assert color['color'] == pytest.approx(0.5) + assert state == {'remaining': 3, 'attempted': 0, 'matched': 0} + + +def test_nearest_catalog_color_row_uses_gaia_bp_rp_last_resort(monkeypatch): + calls = [] + + def fake_gaia_lookup(ra, dec, max_separation_arcsec): + calls.append((ra, dec, max_separation_arcsec)) + return {'color': 1.25, 'label': 'BP-RP'} + + monkeypatch.setattr(exotic_module, 'nextastro_gaia_bp_rp_for_coordinate', fake_gaia_lookup) + state = {'remaining': 3, 'attempted': 0, 'matched': 0} + catalog = { + 'rows': [ + {'source_id': 42, 'ra': 10.00001, 'dec': 20.0, 'Vmag': 12.5}, + ], + } + + match = exotic_module.nextastro_catalog_nearest_color_row( + catalog, + 10.0, + 20.0, + 'V', + gaia_lookup_state=state, + gaia_match_radius_arcsec=1.5, + ) + + assert match['source_id'] == 42 + assert match['color'] == {'color': 1.25, 'label': 'BP-RP'} + assert calls == [(10.00001, 20.0, 1.5)] + assert state == {'remaining': 2, 'attempted': 1, 'matched': 1} + + +@pytest.mark.parametrize( + ('observed_filter', 'magnitude_column', 'error_column'), + [ + ('u', 'umag', 'err_umag'), + ('B', 'Bmag', 'err_Bmag'), + ('V', 'Vmag', 'err_Vmag'), + ('Sloan g', 'g', 'dg'), + ('Sloan r', 'r', 'dr'), + ('Sloan i', 'i', 'di'), + ('Sloan z', 'z', 'dz'), + ], +) +def test_nextastro_catalog_match_never_falls_back_to_another_band( + observed_filter, magnitude_column, error_column): + row = { + 'id': 1, + 'ra': 10.0, + 'dec': 20.0, + 'Bmag': 12.1, + 'err_Bmag': 0.01, + 'Vmag': 12.2, + 'err_Vmag': 0.01, + 'umag': 12.3, + 'err_umag': 0.01, + 'g': 12.4, + 'dg': 0.01, + 'r': 12.5, + 'dr': 0.01, + 'i': 12.6, + 'di': 0.01, + 'z': 12.7, + 'dz': 0.01, + } + row[magnitude_column] = None + row[error_column] = None + + match = exotic_module.nextastro_photometry_catalog_match( + {'row_format': 'objects', 'rows': [row]}, + 10.0, + 20.0, + observed_filter, + ) + + assert match is None def test_nextastro_catalog_match_uses_relaxed_error_only_as_bv_fallback(): @@ -603,7 +911,10 @@ def fake_post(url, json, timeout): assert captured['json']['ra'] == pytest.approx(10.0) assert captured['json']['dec'] == pytest.approx(-20.0) assert captured['json']['radius_arcsec'] == pytest.approx(2.0) - assert captured['json']['columns'] == list(exotic_module.NEXTASTRO_PHOTOMETRY_COLUMNS) + assert captured['json']['columns'] == [ + 'id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag', + ] + assert captured['json']['required_columns'] == ['Vmag', 'err_Vmag'] assert captured['timeout'] == 30 assert match['source_id'] == 12345 assert match['mag'] == pytest.approx(11.2) @@ -659,6 +970,10 @@ def fake_post(url, json, timeout): {'key': '1', 'ra': 11.0, 'dec': -21.0}, ] assert captured['json']['radius_arcsec'] == pytest.approx(2.0) + assert captured['json']['columns'] == [ + 'id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag', + ] + assert captured['json']['required_columns'] == ['Vmag', 'err_Vmag'] assert captured['timeout'] == 30 assert matches[0]['source_id'] == 12345 assert matches[0]['mag_band'] == 'V' @@ -771,6 +1086,33 @@ def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): assert calibration['observed_filter'] == 'V' +def test_merge_nextastro_calibration_stars_uses_image_scale_match_radius(): + catalog = { + 'row_format': 'objects', + 'rows': [{ + 'id': 9, + 'source_id': 12345, + 'ra': 10.001, + 'dec': 20.0, + 'Vmag': 10.038, + 'err_Vmag': 0.027, + }], + } + + calibration_stars = exotic_module.merge_nextastro_calibration_stars( + comp_stars=[[232, 348]], + comp_ra_dec=[(10.0, 20.0)], + obs_filter='MObs CV', + field_catalog=catalog, + match_radius_arcsec=5.153485, + ) + + assert list(calibration_stars) == ['NextAstro-12345'] + assert calibration_stars['NextAstro-12345']['mag'] == pytest.approx(10.038) + assert calibration_stars['NextAstro-12345']['mag_band'] == 'V' + assert calibration_stars['NextAstro-12345']['separation_arcsec'] > 2.0 + + def test_merge_nextastro_calibration_stars_deduplicates_catalog_source_ids(): catalog = { 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], @@ -951,6 +1293,28 @@ def world_to_pixel_values(self, ra_deg, dec_deg): assert user_comp_stars == [[50, 50]] +def test_tracked_comparison_position_keeps_full_field_anchor_index_after_science_reset(): + science_comp_stars = [[217.0, 210.0], [408.0, 261.0]] + tracked_calibration_stars = [ + *science_comp_stars, + [415.0, 203.0], + [449.0, 267.0], + ] + + # The target fit restores the shorter science list, while catalog-anchor + # indices retain the full tracking-list index space. + restored_science_comp_stars = list(science_comp_stars) + assert len(restored_science_comp_stars) == 2 + assert exotic_module.tracked_comparison_position( + tracked_calibration_stars, + 2, + ) == [415.0, 203.0] + assert exotic_module.tracked_comparison_position( + tracked_calibration_stars, + 3, + ) == [449.0, 267.0] + + def test_clear_v_calibration_fallback_merges_aavso_with_existing_pool(monkeypatch): calls = [] supplied_positions = [[100, 200]] @@ -1042,7 +1406,12 @@ def unexpected_vsp_query(*args, **kwargs): assert queried is False -def test_build_stellar_variability_params_records_nextastro_reference(monkeypatch, tmp_path): +@pytest.mark.parametrize( + 'observed_filter', + ['CV', 'Clear', 'Luminance', 'Photographic G', 'Gaia G'], +) +def test_build_stellar_variability_params_records_nextastro_reference( + monkeypatch, tmp_path, observed_filter): captured = {} class DummyFit: @@ -1085,7 +1454,7 @@ def fake_plot(params, save, s_name, label): 'NextAstro-123', tmp_path, 'Host Star', - observed_filter='CV', + observed_filter=observed_filter, ) assert captured['label'] == 'RA=10.1000000 Dec=-20.2000000' @@ -1096,7 +1465,9 @@ def fake_plot(params, save, s_name, label): assert params[0]['comp_dec'] == pytest.approx(-20.2) assert params[0]['cmag'] == pytest.approx(12.0) assert params[0]['cmag_err'] == pytest.approx(0.05) - assert params[0]['observed_filter'] == 'CV' + assert params[0]['observed_filter'] == observed_filter + assert params[0]['mag_band'] == 'ClearV' + assert params[0]['catalog_mag_band'] == 'V' def test_build_stellar_variability_params_rejects_cross_band_calibration(tmp_path): @@ -1186,6 +1557,32 @@ class DummyFit: assert params[1]['mag_err'] < 0.08 +def test_annotate_stellar_variability_raw_photometry_restores_final_selected_fit_fluxes(): + class DummyFit: + data = np.array([0.99, 1.0, 1.01], dtype=float) + + target_flux = np.array([9900.0, 10000.0, 10100.0], dtype=float) + comp_flux = np.full(3, 20000.0, dtype=float) + target_error = np.array([10.0, 11.0, 12.0], dtype=float) + comp_error = np.array([20.0, 21.0, 22.0], dtype=float) + fit = DummyFit() + + exotic_module.annotate_stellar_variability_raw_photometry( + fit, + target_flux, + comp_flux, + target_flux_error=target_error, + comp_flux_error=comp_error, + ) + + retained = exotic_module.stellar_variability_raw_photometry(fit) + for actual, expected in zip( + retained, + (target_flux, comp_flux, target_error, comp_error), + ): + np.testing.assert_array_equal(actual, expected) + + def test_build_stellar_variability_params_rejects_normalized_only_absolute_calibration( monkeypatch, tmp_path): class DummyFit: @@ -1293,7 +1690,7 @@ def __init__(self, data): assert any('derived catalog magnitude' in message for message in logged) -def test_stellar_variability_uses_direct_catalog_when_derived_error_is_worse(monkeypatch, tmp_path): +def test_stellar_variability_uses_direct_catalog_without_shared_fit_oot_points(monkeypatch, tmp_path): logged = [] class DummyFit: @@ -1314,12 +1711,14 @@ def __init__(self, data): selected_fit = DummyFit([1.0, 1.0, 1.0, 1.0]) noisy_anchor_fit = DummyFit([2.0, 0.8, 2.2, 0.7]) + best_fit = DummyFit([1.0, 1.0, 1.0, 1.0]) + best_fit.transit = np.zeros(4, dtype=float) params = exotic_module.stellar_variability( { 0: {'myfit': selected_fit, 'pos': [100, 200]}, 1: {'myfit': noisy_anchor_fit, 'pos': [300, 400]}, }, - DummyFit([1.0, 1.0, 1.0, 1.0]), + best_fit, [[100, 200], [300, 400]], {'ANCHOR': {'pos': [300, 400], 'mag': 12.0, 'error': 0.02, 'mag_band': 'g'}}, [1], @@ -1602,7 +2001,8 @@ def pixel_to_world_values(self, x, y): ) assert len(params) == 3 - assert params[0]['mag_band'] == 'V' + assert params[0]['mag_band'] == 'ClearV' + assert params[0]['catalog_mag_band'] == 'V' assert params[0]['cmag_err'] == pytest.approx(0.03) @@ -2299,6 +2699,54 @@ def test_stellar_variability_ensemble_caps_at_five_by_target_color_and_magnitude assert limited_keys == {'comp1', 'comp2'} +def test_stellar_variability_ensemble_uses_gaia_bp_rp_when_local_colors_are_missing( + monkeypatch): + calls = [] + + def fake_gaia_lookup(ra, dec, max_separation_arcsec): + calls.append((ra, dec, max_separation_arcsec)) + return { + 'color': 1.2 if ra == 10.0 else 1.25, + 'label': 'BP-RP', + } + + monkeypatch.setattr(exotic_module, 'nextastro_gaia_bp_rp_for_coordinate', fake_gaia_lookup) + selection = exotic_module.select_stellar_variability_ensemble_members( + [{ + 'key': 'comp1', + 'comp_index': 0, + 'label': 'Comp 1', + 'position': [10, 20], + 'overexposure_rejected_count': 0, + }], + { + 'NextAstro-1': { + 'pos': [10, 20], + 'mag': 12.1, + 'error': 0.01, + 'mag_band': 'V', + 'catalog_row': {'ra': 11.0, 'dec': 20.0, 'Vmag': 12.1}, + }, + }, + {'comp1': np.full(8, 1000.0)}, + observed_filter='V', + target_catalog_match={ + 'mag': 12.0, + 'error': 0.01, + 'mag_band': 'V', + 'catalog_row': {'ra': 10.0, 'dec': 20.0, 'Vmag': 12.0}, + }, + min_members=1, + max_members=1, + ) + + assert selection['target_catalog_profile']['color_label'] == 'BP-RP' + assert selection['target_catalog_profile']['color'] == pytest.approx(1.2) + assert selection['members'][0]['color_label'] == 'BP-RP' + assert selection['members'][0]['color_delta'] == pytest.approx(0.05) + assert calls == [(10.0, 20.0, 2.0), (11.0, 20.0, 2.0)] + + def test_stellar_variability_ensemble_rejects_duplicate_catalog_sources(): frame_count = 8 ranked_summaries = [ @@ -2389,6 +2837,7 @@ def world_to_pixel_values(self, ra_value, dec_value): assert variables[0]['category'] == 'optimal_variables' assert variables[0]['period_days'] == pytest.approx(5.0) assert variables[0]['amplitude_mag'] == pytest.approx(0.5) + assert exotic_module.fortuitous_variable_category(20.0, 0.2) == 'normal' def test_fortuitous_reference_error_estimate_includes_sky_noise(monkeypatch): @@ -2498,13 +2947,15 @@ def test_build_stellar_variability_ensemble_params_preserves_member_metadata(mon fit, tmp_path, 'Target Star', - observed_filter='V', + observed_filter='MObs CV', ) assert len(params) == 2 assert [row['time'] for row in params] == pytest.approx([2460000.105, 2460000.205]) assert params[0]['cname'] == 'ENSEMBLE (2 stars)' assert params[0]['cmag'] is None + assert params[0]['mag_band'] == 'ClearV' + assert params[0]['catalog_mag_band'] == 'V' assert params[0]['ensemble_member_labels'] == ['C1', 'C2'] assert params[0]['ensemble_member_catalog_errors'] == [0.01, 0.011] assert params[0]['ensemble_member_ra_degs'] == [10.1, 10.2] @@ -2715,16 +3166,16 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product assert results[0]['target_overexposure_rejected_frame_count'] == 2 assert results[0]['output_magnitude_error_rejected_frame_count'] == 1 assert results[0]['point_count'] == frame_count - 3 - variable_dir = tmp_path / 'fortuitous_variables' / 'optimal_variables' / 'VSX_SyntheticVSX' + variable_dir = tmp_path / 'variables' / 'optimal_variables' / 'SyntheticVSX' assert next(variable_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')).is_file() second_variable_dir = ( - tmp_path / 'fortuitous_variables' / 'optimal_variables' / 'VSX_SyntheticVSX2' + tmp_path / 'variables' / 'optimal_variables' / 'SyntheticVSX2' ) assert next(second_variable_dir.glob('AID_AAVSO_SyntheticVSX2_2024-01-02.txt')).is_file() assert next(variable_dir.glob('EnsembleSelection_SyntheticVSX_2024-01-02.json')).is_file() assert next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')).is_file() combined_aid_path = ( - tmp_path / 'fortuitous_variables' / 'AID_AAVSO_FortuitousVariables_2024-01-02.txt' + tmp_path / 'variables' / 'AID_AAVSO_FortuitousVariables_2024-01-02.txt' ) combined_aid_text = combined_aid_path.read_text(encoding='utf-8') combined_aid_rows = [ @@ -2739,7 +3190,7 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product '000-AAA-002', } manifest = json.loads( - next((tmp_path / 'fortuitous_variables').glob('FortuitousVariables_2024-01-02.json')).read_text( + next((tmp_path / 'variables').glob('FortuitousVariables_2024-01-02.json')).read_text( encoding='utf-8' ) ) @@ -2805,7 +3256,7 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product assert single_results[0]['comparison_member_count'] == 1 assert single_results[0]['comparison_label'] == 'C2' single_dir = ( - single_root / 'fortuitous_variables' / 'optimal_variables' / 'VSX_SyntheticVSX' + single_root / 'variables' / 'optimal_variables' / 'SyntheticVSX' ) assert not list(single_dir.glob('EnsembleSelection_*.json')) single_csv = next(single_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')) @@ -2823,6 +3274,35 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product ) assert single_aid_row.split(',')[7] == 'C2' + failed_root = tmp_path / 'failed' + failed_results = exotic_module.process_fortuitous_variables( + [variable], + comparison_calibration, + {}, + times, + times, + np.linspace(1.1, 1.3, frame_count), + psf_data, + aper_data, + {**info_dict, 'save': str(failed_root)}, + comp_overexposed_masks={'comp3': variable_overexposed}, + exposure_times_seconds=np.full(frame_count, 60.0), + observed_filter='V', + ) + + assert failed_results[0]['status'] == 'skipped' + assert failed_results[0]['output_directory'] is None + assert not (failed_root / 'variables' / 'optimal_variables' / 'SyntheticVSX').exists() + failed_manifest = json.loads( + next((failed_root / 'variables').glob('FortuitousVariables_2024-01-02.json')).read_text( + encoding='utf-8' + ) + ) + assert failed_manifest['variables'][0]['status'] == 'skipped' + assert 'fewer than 1 independently calibrated comparison member' in ( + failed_manifest['variables'][0]['reason'] + ) + def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(): frame_count = 12 diff --git a/tests/test_output_files.py b/tests/test_output_files.py index aafe1a26..88100a27 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -467,7 +467,8 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): "catalog_dec": -20.20001, "catalog_source": "NextAstro photometry catalog", "is_aavso_vsp": False, - "mag_band": "V", + "mag_band": "ClearV", + "catalog_mag_band": "V", "source_id": 12345, "separation_arcsec": 0.2, }] @@ -483,6 +484,8 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert metadata["comparison_dec_deg"] == pytest.approx(-20.2) assert metadata["apparent_magnitude"] == pytest.approx(12.1) assert metadata["apparent_magnitude_error"] == pytest.approx(0.03) + assert metadata["magnitude_band"] == "V" + assert metadata["reported_measurement_band"] == "ClearV" assert "#COMPARISON_RA=10.1000000\n#COMPARISON_DEC=-20.2000000\n" in output_text assert "#DATE=BJD_TDB" in output_text assert "HAT-P-32,2450000.12345,12.340,0.050,V,NO,STD" in output_text diff --git a/tests/test_plots.py b/tests/test_plots.py index 43da8a61..e18a8fbf 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -360,13 +360,20 @@ def spy_set_ylabel(self, label, *args, **kwargs): def test_plot_stellar_variability_labels_aavso_filter_and_assumed_comparison(tmp_path, monkeypatch): titles = [] + ylabels = [] original_set_title = Axes.set_title + original_set_ylabel = Axes.set_ylabel def spy_set_title(self, label, *args, **kwargs): titles.append(label) return original_set_title(self, label, *args, **kwargs) + def spy_set_ylabel(self, label, *args, **kwargs): + ylabels.append(label) + return original_set_ylabel(self, label, *args, **kwargs) + monkeypatch.setattr(Axes, "set_title", spy_set_title) + monkeypatch.setattr(Axes, "set_ylabel", spy_set_ylabel) plot_stellar_variability( [ @@ -378,7 +385,8 @@ def spy_set_title(self, label, *args, **kwargs): "cmag_err": 0.067, "comp_ra": 10.1, "comp_dec": -20.2, - "mag_band": "V", + "mag_band": "ClearV", + "catalog_mag_band": "V", "observed_filter": "CV", "is_aavso_vsp": True, } @@ -391,6 +399,7 @@ def spy_set_title(self, label, *args, **kwargs): assert "Label: 000-BJX-718\nRA=10.100000\nDec=-20.200000" in titles[-1] assert "Original filter: CV" in titles[-1] assert "Comparison mag: V=12.345 +/- 0.067" in titles[-1] + assert ylabels[-1] == "Magnitude (ClearV)" def test_plot_stellar_variability_omits_invalid_reference_magnitudes(tmp_path, monkeypatch): From 6f4a5687b5994248e68240a30ef6e8039080aae9 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sat, 18 Jul 2026 14:06:54 +1000 Subject: [PATCH 092/116] save stellarvariability to root. --- exotic/plots.py | 4 +++- tests/test_plots.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/exotic/plots.py b/exotic/plots.py index 97cc8b63..d3152359 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -644,7 +644,9 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): ax.set_xlabel("Time [JD]") fig.tight_layout() output_dir = _working_artifacts_dir(save) - fig.savefig(output_dir / f"Stellar_Variability.png", bbox_inches="tight") + output_path = Path(save) / "Stellar_Variability.png" + fig.savefig(output_dir / "Stellar_Variability.png", bbox_inches="tight") + fig.savefig(output_path, bbox_inches="tight") plt.close(fig) diff --git a/tests/test_plots.py b/tests/test_plots.py index e18a8fbf..c1b55835 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -356,6 +356,7 @@ def spy_set_ylabel(self, label, *args, **kwargs): ) assert ylabels[-1] == "Magnitude (r)" assert (tmp_path / "working_artifacts" / "Stellar_Variability.png").exists() + assert (tmp_path / "Stellar_Variability.png").exists() def test_plot_stellar_variability_labels_aavso_filter_and_assumed_comparison(tmp_path, monkeypatch): @@ -459,6 +460,7 @@ def test_plot_stellar_variability_skips_over_30_measurements(tmp_path): ) assert not (tmp_path / "working_artifacts" / "Stellar_Variability.png").exists() + assert not (tmp_path / "Stellar_Variability.png").exists() def test_plot_comp_star_candidate_lightcurve_fits_writes_outputs(tmp_path): From d34248dfd58b094ce141c4be1296a9682c1a482a Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 21 Jul 2026 06:58:27 +1000 Subject: [PATCH 093/116] Add CV to the filter list --- exotic/api/filters.py | 28 ++++++++++++++-------------- exotic/exotic.py | 2 +- tests/test_inputs.py | 20 ++++++++++++++------ tests/test_ld.py | 31 +++++++++++++++++++------------ tests/test_nonlinear_ld.py | 23 +++++++++++++++++++++++ 5 files changed, 71 insertions(+), 33 deletions(-) diff --git a/exotic/api/filters.py b/exotic/api/filters.py index f740b098..14ec7e6b 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -48,9 +48,8 @@ "PanSTARRS Y": {"name": "Y", "fwhm": ("946.4", "1054.4")}, "PanSTARRS w": {"name": "N/A", "fwhm": ("404.2", "845.8")}, - # MObs Clear Filter; Source(s): Martin Fowler - "MObs CV": {"name": "CV", "fwhm": ("350.0", "850.0")}, - "ClearV": {"name": "CV", "fwhm": ("350.0", "1000.0")}, + # ClearV filter + "CV": {"name": "CV", "fwhm": ("350.0", "1000.0")}, # Clear with blue-blocking (CBB); wavelength source: # https://astrodon.com/products/astrodon-exo-planet-filter/ @@ -79,7 +78,7 @@ "LCO Pan-STARRS Y": "PanSTARRS Y", "LCO Pan-STARRS w": "PanSTARRS w", - "Clear (unfiltered) reduced to V sequence": "MObs CV", + "Clear (unfiltered) reduced to V sequence": "CV", "Clear (unfiltered) reduced to R sequence": "Cousins R", "Clear with blue-blocking": "CBB", @@ -108,15 +107,17 @@ "sy": "Stromgren y", "hb": "Stromgren Hbw", "zs": "PanSTARRS z-short", - "clearV": "ClearV", - "C": "MObs CV", - "clear": "ClearV", - "lum": "ClearV", - "Lum": "ClearV", - "Luminosity": "ClearV", - "luminosity": "ClearV", - "w": "ClearV", - "pl": "ClearV", + "MObs CV": "CV", + "ClearV": "CV", + "clearV": "CV", + "C": "CV", + "clear": "CV", + "lum": "CV", + "Lum": "CV", + "Luminosity": "CV", + "luminosity": "CV", + "w": "CV", + "pl": "CV", "exo": "CBB", # OSC split-channel aliases @@ -130,7 +131,6 @@ # standard filters w/o precisely defined FWHM values fwhm_names_nonspecific = { 'CR': "Clear (unfiltered) reduced to R sequence", - 'CV': "Clear (unfiltered) reduced to V sequence", 'TB': "DSLR Blue", 'TG': "DSLR Green", 'TR': "DSLR Red", diff --git a/exotic/exotic.py b/exotic/exotic.py index 5ad00359..8e54eeb1 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -31100,7 +31100,7 @@ def lookup_archive_ephemeris(): exotic_infoDict['second_obs'] += ",MOBS" else: exotic_infoDict['second_obs'] = "MOBS" - exotic_infoDict['filter'] = "MObs CV" + exotic_infoDict['filter'] = "CV" exotic_infoDict['elev'] = 1268 exotic_infoDict['lat'] = 31.675467 exotic_infoDict['long'] = -110.951376 diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 934d8406..88a97aeb 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -1598,8 +1598,16 @@ def test_lookup_aavso_filter_metadata_uses_c_alias_for_cv_filter() -> None: filter_metadata = inputs_module.lookup_aavso_filter_metadata("C") assert filter_metadata["name"] == "CV" - assert filter_metadata["desc"] == "MObs CV" - assert filter_metadata["fwhm"] == ("350.0", "850.0") + assert filter_metadata["desc"] == "CV" + assert filter_metadata["fwhm"] == ("350.0", "1000.0") + + +def test_lookup_aavso_filter_metadata_interprets_cv_as_clearv() -> None: + filter_metadata = inputs_module.lookup_aavso_filter_metadata("CV") + + assert filter_metadata["name"] == "CV" + assert filter_metadata["desc"] == "CV" + assert filter_metadata["fwhm"] == ("350.0", "1000.0") def test_lookup_aavso_filter_metadata_uses_luminosity_aliases_for_clearv_filter() -> None: @@ -1607,7 +1615,7 @@ def test_lookup_aavso_filter_metadata_uses_luminosity_aliases_for_clearv_filter( filter_metadata = inputs_module.lookup_aavso_filter_metadata(alias) assert filter_metadata["name"] == "CV" - assert filter_metadata["desc"] == "ClearV" + assert filter_metadata["desc"] == "CV" assert filter_metadata["fwhm"] == ("350.0", "1000.0") @@ -1640,9 +1648,9 @@ def test_parse_aavso_prereduced_overrides_uses_c_alias_for_cv_filter_lookup(tmp_ overrides = parse_aavso_prereduced_overrides(pre_reduced_file) assert overrides["filter"] == "C" - assert overrides["filter_desc"] == "MObs CV" + assert overrides["filter_desc"] == "CV" assert overrides["wl_min"] == "350.0" - assert overrides["wl_max"] == "850.0" + assert overrides["wl_max"] == "1000.0" def test_parse_aavso_prereduced_overrides_uses_luminosity_alias_for_clearv_lookup(tmp_path): @@ -1657,7 +1665,7 @@ def test_parse_aavso_prereduced_overrides_uses_luminosity_alias_for_clearv_looku overrides = parse_aavso_prereduced_overrides(pre_reduced_file) assert overrides["filter"] == "Luminosity" - assert overrides["filter_desc"] == "ClearV" + assert overrides["filter_desc"] == "CV" assert overrides["wl_min"] == "350.0" assert overrides["wl_max"] == "1000.0" diff --git a/tests/test_ld.py b/tests/test_ld.py index 5b0e732e..fd0ab1c2 100644 --- a/tests/test_ld.py +++ b/tests/test_ld.py @@ -75,7 +75,7 @@ def test_existing_standard_filter_alias_name() -> None: assert observed_filter == expected_filter -def test_existing_mobs_standard_filter_name() -> None: +def test_legacy_mobs_filter_name_maps_to_cv() -> None: observed_filter = { 'filter': "MObs CV", 'name': None, @@ -84,17 +84,17 @@ def test_existing_mobs_standard_filter_name() -> None: } expected_filter = { - 'filter': "MObs CV", + 'filter': "CV", 'name': 'CV', 'wl_min': '350.0', - 'wl_max': '850.0' + 'wl_max': '1000.0' } setting_filter_values(observed_filter) assert observed_filter == expected_filter -def test_custom_nonspecific_standard_filter_abbreviation_1() -> None: +def test_cv_standard_filter_abbreviation_uses_clearv() -> None: observed_filter = { 'filter': "CV", 'name': None, @@ -104,7 +104,13 @@ def test_custom_nonspecific_standard_filter_abbreviation_1() -> None: ld_obj = LimbDarkening(stellar_params) - assert ld_obj.check_standard(observed_filter) == False + assert ld_obj.check_standard(observed_filter) is True + assert observed_filter == { + 'filter': "CV", + 'name': "CV", + 'wl_min': "350.0", + 'wl_max': "1000.0", + } def test_custom_nonspecific_standard_filter_abbreviation_2() -> None: observed_filter = { @@ -149,19 +155,19 @@ def test_existing_standard_filter_fwhm() -> None: assert observed_filter == expected_filter -def test_existing_mobs_standard_filter_mobs() -> None: +def test_existing_clearv_standard_filter_wavelengths() -> None: observed_filter = { 'filter': None, 'name': None, 'wl_min': '350.0', - 'wl_max': '850.0' + 'wl_max': '1000.0' } expected_filter = { - 'filter': "MObs CV", + 'filter': "CV", 'name': 'CV', 'wl_min': '350.0', - 'wl_max': '850.0' + 'wl_max': '1000.0' } setting_filter_values(observed_filter) @@ -293,9 +299,10 @@ def test_additional_standard_filter_aliases_in_filter_column() -> None: ("sy", "Stromgren y", "STY", "536.7", "559.3"), ("hb", "Stromgren Hbw", "STHBW", "481.5", "496.5"), ("zs", "PanSTARRS z-short", "ZS", "826.0", "920.0"), - ("clearV", "MObs CV", "CV", "350.0", "850.0"), - ("w", "MObs CV", "CV", "350.0", "850.0"), - ("pl", "MObs CV", "CV", "350.0", "850.0"), + ("CV", "CV", "CV", "350.0", "1000.0"), + ("clearV", "CV", "CV", "350.0", "1000.0"), + ("w", "CV", "CV", "350.0", "1000.0"), + ("pl", "CV", "CV", "350.0", "1000.0"), ("exo", "CBB", "CBB", "500.0", "1000.0"), ("Astrodon ExoPlanet-BB", "CBB", "CBB", "500.0", "1000.0"), ("Astrodon-Exo", "CBB", "CBB", "500.0", "1000.0"), diff --git a/tests/test_nonlinear_ld.py b/tests/test_nonlinear_ld.py index 9ec70fa7..df13aaba 100644 --- a/tests/test_nonlinear_ld.py +++ b/tests/test_nonlinear_ld.py @@ -49,6 +49,29 @@ def test_nonlinear_ld_non_interactive_uses_neutral_cbb_name_without_prompt(monke assert info_dict["wl_max"] == 1000.0 +def test_nonlinear_ld_non_interactive_treats_cv_as_clearv_without_prompt(monkeypatch): + ld = exotic_module.LimbDarkening({}) + monkeypatch.setattr(ld, "calculate_ld", lambda: None) + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("recognized CV filter must not prompt"), + ) + info_dict = { + "filter": "CV", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": "y", + } + + exotic_module.nonlinear_ld(ld, info_dict, non_interactive_run=True) + + assert info_dict["filter"] == "CV" + assert info_dict["filter_desc"] == "CV" + assert info_dict["wl_min"] == 350.0 + assert info_dict["wl_max"] == 1000.0 + + class UnrecognizedFilterLimbDarkening: fwhm_names_nonspecific = {} From 7c76d425160e8e582ee26d53d29eaae91a4d552b Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 23 Jul 2026 09:49:17 +1000 Subject: [PATCH 094/116] plot title clarity --- exotic/plots.py | 2 +- tests/test_plots.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/exotic/plots.py b/exotic/plots.py index d3152359..8640f630 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -588,7 +588,7 @@ def _stellar_variability_reference_label(vsp_param, comparison_label): details.append(f"Label: {comparison_label}") if comp_ra is not None and comp_dec is not None: - details.extend((f"RA={comp_ra:.6f}", f"Dec={comp_dec:.6f}")) + details.extend((f"Comparison RA={comp_ra:.6f}", f"Dec={comp_dec:.6f}")) if not details and comparison_label: details.append(comparison_label) diff --git a/tests/test_plots.py b/tests/test_plots.py index c1b55835..82c417ae 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -350,7 +350,7 @@ def spy_set_ylabel(self, label, *args, **kwargs): assert titles[-1] == ( "Host Star\n" "Label: NextAstro-123\n" - "RA=10.100000\n" + "Comparison RA=10.100000\n" "Dec=-20.200000\n" "Original filter: CV | Comparison mag: r=12.345 +/- 0.067" ) @@ -397,7 +397,11 @@ def spy_set_ylabel(self, label, *args, **kwargs): "000-BJX-718", ) - assert "Label: 000-BJX-718\nRA=10.100000\nDec=-20.200000" in titles[-1] + assert ( + "Label: 000-BJX-718\n" + "Comparison RA=10.100000\n" + "Dec=-20.200000" + ) in titles[-1] assert "Original filter: CV" in titles[-1] assert "Comparison mag: V=12.345 +/- 0.067" in titles[-1] assert ylabels[-1] == "Magnitude (ClearV)" @@ -435,7 +439,7 @@ def spy_set_title(self, label, *args, **kwargs): assert titles[-1] == ( "Host Star\n" - "Label: NextAstro-123\nRA=10.100000\nDec=-20.200000\n" + "Label: NextAstro-123\nComparison RA=10.100000\nDec=-20.200000\n" "Original filter: MObs CV" ) assert "99.99" not in titles[-1] From b37ab7374c097b403f5d208f8d604f6ea6d0ef9a Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 23 Jul 2026 11:15:34 +1000 Subject: [PATCH 095/116] ensembel fix --- exotic/exotic.py | 39 ++++++++++------ exotic/plots.py | 2 + tests/test_nextastro_variability.py | 72 +++++++++++++++++++++++++++++ tests/test_plots.py | 16 +++++++ 4 files changed, 116 insertions(+), 13 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 8e54eeb1..e8c6d8ba 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -15549,7 +15549,10 @@ def reported_stellar_variability_band(observed_filter, fallback_band=None): """Keep the catalogue anchor band separate from the measured passband.""" if observed_filter_uses_clear_v_calibration(observed_filter): return 'ClearV' - return fallback_band or observed_filter or 'V' + reported_band = fallback_band or observed_filter or 'V' + if str(reported_band).strip().lower() == 'r': + return 'rp' + return reported_band def nextastro_photometry_band_candidates(obs_filter): @@ -27809,10 +27812,16 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu zero_points.append(zero_point) zero_point_errors.append(zero_point_error) + selected_member_count = len(members or []) try: - required_members = max(1, int(minimum_members)) + minimum_required_members = max(1, int(minimum_members)) except (TypeError, ValueError): - required_members = STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS + minimum_required_members = STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS + # An ensemble has a fixed membership. Allowing the contributing subset to + # change from frame to frame changes the photometric zero point and can + # create false variability. Every selected member must therefore be valid + # for a frame, or that frame is rejected. + required_members = max(minimum_required_members, selected_member_count) member_text = "comparison star" if required_members == 1 else "comparison stars" empty = { 'applied': False, @@ -27975,9 +27984,11 @@ def save_stellar_variability_ensemble_selection_json( 'ensemble': { 'selection_rule': ( 'After saturation, VSX, coverage, stability, and high catalog-error rejection, ' - 'use at most five stars ranked by joint catalog color and magnitude distance to the target.' + 'use at most five stars ranked by joint catalog color and magnitude distance to the target; ' + 'retain a frame only when every selected ensemble member is valid.' ), 'minimum_members': STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + 'per_frame_required_members': len(members), 'maximum_members': selection.get( 'member_limit', STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS ), @@ -28105,6 +28116,11 @@ def build_stellar_variability_ensemble_params_from_fit( member_magnitude_deltas = [member.get('magnitude_delta') for member in members] member_similarity_scores = [member.get('color_magnitude_similarity_score') for member in members] display_label = f"ENSEMBLE ({len(members)} stars)" + catalog_mag_band = preferred_catalog_magnitude_band_for_filter(observed_filter) + measurement_mag_band = reported_stellar_variability_band( + observed_filter, + fallback_band=catalog_mag_band, + ) vsp_params = [] for time_value, airmass_value, magnitude, magnitude_error in zip( times[valid], @@ -28127,11 +28143,8 @@ def build_stellar_variability_ensemble_params_from_fit( 'catalog_dec': None, 'catalog_source': 'Calibrated comparison-star ensemble', 'is_aavso_vsp': False, - 'mag_band': reported_stellar_variability_band( - observed_filter, - fallback_band=observed_filter or 'V', - ), - 'catalog_mag_band': preferred_catalog_magnitude_band_for_filter(observed_filter), + 'mag_band': measurement_mag_band, + 'catalog_mag_band': catalog_mag_band, 'observed_filter': observed_filter, 'ensemble_reference': True, 'ensemble_member_count': len(members), @@ -28368,8 +28381,8 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, times, fit_mask, note=( - "Kept frames with a finite target measurement and at least two unsaturated, " - "VSX-vetted, catalog-calibrated ensemble members." + "Kept frames with a finite target measurement and every selected unsaturated, " + "VSX-vetted, catalog-calibrated ensemble member." ), ) filter_diagnostics = [] @@ -28460,7 +28473,7 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, 'coverage_rejected': False, 'ensemble_frame_rejected_count': int(np.count_nonzero(~fit_mask)), - 'ensemble_frame_required_valid_pairs': STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + 'ensemble_frame_required_valid_pairs': len(ensemble_members), 'ensemble_member_keys': [member['key'] for member in ensemble_members], } selected_result = { @@ -28477,7 +28490,7 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, 'coverage_rejected': False, 'ensemble_frame_rejected_count': ensemble_summary['ensemble_frame_rejected_count'], - 'ensemble_frame_required_valid_pairs': STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + 'ensemble_frame_required_valid_pairs': len(ensemble_members), 'ensemble_member_keys': ensemble_summary['ensemble_member_keys'], 'fit': fit_result, 'full_reduction_fit': fit_result, diff --git a/exotic/plots.py b/exotic/plots.py index 8640f630..f616f7b0 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -641,6 +641,7 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): title_lines.append(metadata_label) ax.set_title("\n".join(title_lines), fontsize=11) ax.set_ylabel(f"Magnitude ({band})") + ax.invert_yaxis() ax.set_xlabel("Time [JD]") fig.tight_layout() output_dir = _working_artifacts_dir(save) @@ -1018,6 +1019,7 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date): ) band = first_param.get('mag_band') or 'V' ax_lc.set_ylabel(f"Magnitude ({band})") + ax_lc.invert_yaxis() ax_lc.set_xlabel("Time [BJD_TDB]") f.tight_layout() diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 5d34ea33..506708bc 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -548,6 +548,10 @@ def test_reported_stellar_variability_band_distinguishes_clearv_from_catalog_v() observed_filter, 'V' ) == 'ClearV' assert exotic_module.reported_stellar_variability_band('bv', 'V') == 'V' + for observed_filter in ('R', 'SR', 'rp'): + assert exotic_module.reported_stellar_variability_band( + observed_filter, 'r' + ) == 'rp' def test_selected_comparison_direct_catalog_match_precedes_derived_fallback(): @@ -2911,6 +2915,62 @@ def test_calibrated_stellar_variability_ensemble_combines_catalog_zero_points(): assert np.all(result['magnitude_error'] > 0) +def test_calibrated_stellar_variability_ensemble_rejects_frame_missing_any_member(): + frame_count = 7 + target_flux = np.full(frame_count, 1000.0) + target_error = np.full(frame_count, 1.0) + comp_flux_map = { + 'comp1': np.full(frame_count, 500.0), + 'comp2': np.full(frame_count, 250.0), + 'comp3': np.full(frame_count, 400.0), + } + comp_error_map = { + key: np.full(frame_count, 1.0) + for key in comp_flux_map + } + members = [ + { + 'key': 'comp1', + 'magnitude': 12.0, + 'magnitude_error': 0.01, + 'summary': {'ensemble_frame_keep_mask': np.ones(frame_count, dtype=bool)}, + }, + { + 'key': 'comp2', + 'magnitude': 12.0 + 2.5 * np.log10(2.0), + 'magnitude_error': 0.01, + 'summary': {'ensemble_frame_keep_mask': np.ones(frame_count, dtype=bool)}, + }, + { + 'key': 'comp3', + 'magnitude': 12.0 + 2.5 * np.log10(1.25), + 'magnitude_error': 0.01, + 'summary': { + 'ensemble_frame_keep_mask': np.array( + [True, True, True, False, True, True, True], + dtype=bool, + ), + }, + }, + ] + + result = exotic_module.build_stellar_variability_calibrated_ensemble_series( + target_flux, + target_error, + comp_flux_map, + comp_error_map, + members, + minimum_members=2, + ) + + assert result['applied'] is True + assert result['valid_member_count'][3] == 2 + assert np.isnan(result['magnitude'][3]) + assert np.isnan(result['relative_flux'][3]) + finite_indices = np.flatnonzero(np.isfinite(result['magnitude'])) + np.testing.assert_array_equal(finite_indices, [0, 1, 2, 4, 5, 6]) + + def test_build_stellar_variability_ensemble_params_preserves_member_metadata(monkeypatch, tmp_path): captured = {} monkeypatch.setattr( @@ -2965,6 +3025,16 @@ def test_build_stellar_variability_ensemble_params_preserves_member_metadata(mon assert fit.stellar_variability_params == params assert captured['label'] == 'ENSEMBLE (2 stars)' + r_params = exotic_module.build_stellar_variability_ensemble_params_from_fit( + fit, + tmp_path, + 'Target Star', + observed_filter='R', + ) + + assert r_params[0]['mag_band'] == 'rp' + assert r_params[0]['catalog_mag_band'] == 'r' + def test_stellar_variability_ensemble_selection_json_lists_color_and_magnitude(monkeypatch, tmp_path): monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) @@ -3024,6 +3094,8 @@ def test_stellar_variability_ensemble_selection_json_lists_color_and_magnitude(m payload = json.loads(output_path.read_text(encoding='utf-8')) assert payload['ensemble']['maximum_members'] == 5 assert payload['ensemble']['member_count_before_five_star_limit'] == 6 + assert payload['ensemble']['per_frame_required_members'] == 1 + assert 'every selected ensemble member is valid' in payload['ensemble']['selection_rule'] assert payload['target']['catalog_profile']['color'] == pytest.approx(0.5) assert payload['ensemble']['members'][0]['color_delta'] == pytest.approx(0.05) assert payload['ensemble']['members'][0]['magnitude_delta'] == pytest.approx(0.1) diff --git a/tests/test_plots.py b/tests/test_plots.py index 82c417ae..5076576f 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -124,10 +124,12 @@ def test_stellar_variability_final_lightcurve_plots_calibrated_magnitude_by_time captured_errorbar_y = [] captured_xlabels = [] captured_ylabels = [] + inverted_axes = [] original_errorbar = Axes.errorbar original_set_xlabel = Axes.set_xlabel original_set_ylabel = Axes.set_ylabel + original_invert_yaxis = Axes.invert_yaxis def spy_errorbar(self, x, y, *args, **kwargs): captured_errorbar_x.append(np.asarray(x, dtype=float)) @@ -142,9 +144,14 @@ def spy_set_ylabel(self, ylabel, *args, **kwargs): captured_ylabels.append(ylabel) return original_set_ylabel(self, ylabel, *args, **kwargs) + def spy_invert_yaxis(self, *args, **kwargs): + inverted_axes.append(self) + return original_invert_yaxis(self, *args, **kwargs) + monkeypatch.setattr(Axes, "errorbar", spy_errorbar) monkeypatch.setattr(Axes, "set_xlabel", spy_set_xlabel) monkeypatch.setattr(Axes, "set_ylabel", spy_set_ylabel) + monkeypatch.setattr(Axes, "invert_yaxis", spy_invert_yaxis) plot_final_lightcurve(fit, np.ones(2), "Target", str(tmp_path), "2026-07-08") @@ -153,6 +160,7 @@ def spy_set_ylabel(self, ylabel, *args, **kwargs): assert captured_xlabels[-1] == "Time [BJD_TDB]" assert captured_xlabels[-1] != "Orbital Phase" assert captured_ylabels[-1] == "Magnitude (r)" + assert len(inverted_axes) == 1 assert "O-C [%]" not in captured_ylabels assert (tmp_path / "FinalLightCurve_Target_2026-07-08.png").exists() @@ -313,8 +321,10 @@ def spy_plot(self, x, y, *args, **kwargs): def test_plot_stellar_variability_labels_reference_coordinates(tmp_path, monkeypatch): titles = [] ylabels = [] + inverted_axes = [] original_set_title = Axes.set_title original_set_ylabel = Axes.set_ylabel + original_invert_yaxis = Axes.invert_yaxis def spy_set_title(self, label, *args, **kwargs): titles.append(label) @@ -324,8 +334,13 @@ def spy_set_ylabel(self, label, *args, **kwargs): ylabels.append(label) return original_set_ylabel(self, label, *args, **kwargs) + def spy_invert_yaxis(self, *args, **kwargs): + inverted_axes.append(self) + return original_invert_yaxis(self, *args, **kwargs) + monkeypatch.setattr(Axes, "set_title", spy_set_title) monkeypatch.setattr(Axes, "set_ylabel", spy_set_ylabel) + monkeypatch.setattr(Axes, "invert_yaxis", spy_invert_yaxis) plot_stellar_variability( [ @@ -355,6 +370,7 @@ def spy_set_ylabel(self, label, *args, **kwargs): "Original filter: CV | Comparison mag: r=12.345 +/- 0.067" ) assert ylabels[-1] == "Magnitude (r)" + assert len(inverted_axes) == 1 assert (tmp_path / "working_artifacts" / "Stellar_Variability.png").exists() assert (tmp_path / "Stellar_Variability.png").exists() From 7dd88b98f965fc0337187fcf796afd889f949633 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Fri, 24 Jul 2026 08:58:54 +1000 Subject: [PATCH 096/116] 4 decimal places --- exotic/output_files.py | 4 ++-- tests/test_output_files.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/exotic/output_files.py b/exotic/output_files.py index d35d309c..75e46948 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -2457,10 +2457,10 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta variable_name = default_variable_name if use_row_names: variable_name = vsp_p.get('_aid_name') or variable_name - mag = format_magnitude(vsp_p.get('mag'), default=None) + mag = format_magnitude(vsp_p.get('mag'), default=None, digits=4) if mag is None: continue - mag_err = format_magnitude_error(vsp_p.get('mag_err')) + mag_err = format_magnitude_error(vsp_p.get('mag_err'), digits=4) cmag = format_magnitude(vsp_p.get('cmag')) chart_id = self.chart_id or vsp_p.get('chart_id') or 'na' f.write(f"{variable_name},{round(vsp_p['time'], 5)},{mag},{mag_err}," diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 88100a27..3780fcbe 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -454,8 +454,8 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): } vsp_params = [{ "time": 2450000.12345, - "mag": 12.34, - "mag_err": 0.05, + "mag": 12.34567, + "mag_err": 0.012345, "airmass": 1.234, "cname": "RA=10.1000000 Dec=-20.2000000", "cmag": 12.1, @@ -488,7 +488,7 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert metadata["reported_measurement_band"] == "ClearV" assert "#COMPARISON_RA=10.1000000\n#COMPARISON_DEC=-20.2000000\n" in output_text assert "#DATE=BJD_TDB" in output_text - assert "HAT-P-32,2450000.12345,12.340,0.050,V,NO,STD" in output_text + assert "HAT-P-32,2450000.12345,12.3457,0.0123,V,NO,STD" in output_text def test_aid_output_records_calibrated_ensemble_members(tmp_path): @@ -551,7 +551,7 @@ def test_aid_output_records_calibrated_ensemble_members(tmp_path): assert ensemble_metadata["members"][1]["label"] == "C2" assert ensemble_metadata["members"][1]["ra_deg"] == pytest.approx(10.2) assert ensemble_metadata["members"][1]["dec_deg"] == pytest.approx(-20.2) - assert "Target,2450000.12345,12.340,0.020,V,NO,STD,ENSEMBLE (2 stars),na" in output_text + assert "Target,2450000.12345,12.3400,0.0200,V,NO,STD,ENSEMBLE (2 stars),na" in output_text def test_aid_output_samples_large_derived_anchor_label_lists(tmp_path): @@ -634,7 +634,7 @@ def test_aid_output_floors_reported_magnitude_errors(tmp_path): metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") assert metadata["apparent_magnitude_error"] == pytest.approx(0.001) - assert "HAT-P-32,2450000.12345,12.340,0.001,V,NO,STD" in output_text + assert "HAT-P-32,2450000.12345,12.3400,0.0010,V,NO,STD" in output_text def test_aid_output_skips_over_30_magnitude_rows(tmp_path): From 34973f0d0e19208bb5b8040dbd0fece8a9789394 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 28 Jul 2026 09:25:34 +1000 Subject: [PATCH 097/116] better ensemble options and readme --- README.md | 23 ++- docs/README.md | 23 ++- .../for_exotic_py_candidate_inits_maker.py | 6 + exotic/api/colab.py | 2 + exotic/exotic.py | 169 ++++++++++++++++-- exotic/exotic_gui.py | 10 ++ exotic/inputs.py | 10 ++ inits.json | 6 +- tests/test_exotic_proper_motion.py | 86 +++++++++ tests/test_inputs.py | 41 +++++ tests/test_nextastro_variability.py | 18 ++ 11 files changed, 372 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ab221c9e..ef43b006 100644 --- a/README.md +++ b/README.md @@ -165,9 +165,13 @@ Get EXOTIC up and running faster with a json file. Please see the included file "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_aperture_corrections_and_full_image_fwhm": false, + "use_ensemble_photometry_rather_than_single_comp": false, "stellar_variability_only": false, "use_ensemble_photometry_for_stellar_variability": true, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "photometer_fortuitous_variables": true, + "use_single_comparison_for_fortuitous_variables": true, "use_nextastro_vsx_cache_first": false, "skip_low_comparison_coverage_rejection": "n", "fit_lightcurve_to_every_comparison_candidate": "n", @@ -186,7 +190,24 @@ Get EXOTIC up and running faster with a json file. Please see the included file } ``` -`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against its own calibrated comparison ensemble. Exported light curves also retain only frames whose final ensemble-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or an ensemble member are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set the item to `false` to disable these products. +### Comparison-star mode tags + +Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended; EXOTIC also accepts equivalent values such as `"y"` and `"n"`. + +| Reduction | Requested comparison mode | `optional_info` settings | +|---|---|---| +| Transit fit | Single comparison star (default) | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": false` | +| Transit fit | Comparison-star ensemble | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": true`, `"maximum_number_of_ensemble_comparisons_for_transit": 5` | +| Transit fit | No comparison star | There is no tag that forces this mode. `"require_comp_star": false` only removes the requirement for a comparison star; it does not force target-only photometry. The current comparison-calibration FITS path still selects a single comparison or an ensemble. | +| Stellar-variability-only run | Single comparison star | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": false` | +| Stellar-variability-only run | Calibrated comparison-star ensemble (default) | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": true`, `"maximum_number_of_ensemble_comparisons_for_stellar_variability": 5` | +| Stellar-variability-only run | No comparison star | Not supported for raw-FITS absolute variability photometry; a single calibrated comparison or calibrated ensemble is required. A pre-reduced relative light curve can be supplied without raw comparison-star photometry, but it is not selected by a comparison-mode tag. | + +The two ensemble limits are independent. `"maximum_number_of_ensemble_comparisons_for_transit"` caps only the transit-fit ensemble. `"maximum_number_of_ensemble_comparisons_for_stellar_variability"` caps both stellar-variability-only and fortuitous-variable ensembles. Each defaults to `5`, must be an integer of at least `2`, and has no configured upper limit. Increase either value to permit a much larger ensemble; EXOTIC will enlarge automatic candidate discovery for the corresponding ensemble where applicable, then use up to that number of surviving comparisons. Very large ensembles require more photometry work. Stellar-variability and fortuitous-variable ensembles can also retain fewer frames because every selected member must have a usable measurement in a retained frame. + +Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable photometry has no no-comparison mode. + +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. `use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/normal classification. diff --git a/docs/README.md b/docs/README.md index c44eeed2..3f335727 100644 --- a/docs/README.md +++ b/docs/README.md @@ -219,9 +219,13 @@ Get EXOTIC up and running faster with a json file. Please see the included file "use_psf_photometry": "y", "use_aperture_photometry": "y", "use_aperture_corrections_and_full_image_fwhm": false, + "use_ensemble_photometry_rather_than_single_comp": false, "stellar_variability_only": false, "use_ensemble_photometry_for_stellar_variability": true, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "photometer_fortuitous_variables": true, + "use_single_comparison_for_fortuitous_variables": true, "use_nextastro_vsx_cache_first": false, "detrend_on_outoftransit_baseline": true, "use_impactparameter_rather_than_inclination_to_fit": "y", @@ -231,6 +235,23 @@ Get EXOTIC up and running faster with a json file. Please see the included file } ``` -`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against its own calibrated comparison ensemble. Exported light curves also retain only frames whose final ensemble-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or an ensemble member are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set the item to `false` to disable these products. +### Comparison-star mode tags + +Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended; EXOTIC also accepts equivalent values such as `"y"` and `"n"`. + +| Reduction | Requested comparison mode | `optional_info` settings | +|---|---|---| +| Transit fit | Single comparison star (default) | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": false` | +| Transit fit | Comparison-star ensemble | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": true`, `"maximum_number_of_ensemble_comparisons_for_transit": 5` | +| Transit fit | No comparison star | There is no tag that forces this mode. `"require_comp_star": false` only removes the requirement for a comparison star; it does not force target-only photometry. The current comparison-calibration FITS path still selects a single comparison or an ensemble. | +| Stellar-variability-only run | Single comparison star | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": false` | +| Stellar-variability-only run | Calibrated comparison-star ensemble (default) | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": true`, `"maximum_number_of_ensemble_comparisons_for_stellar_variability": 5` | +| Stellar-variability-only run | No comparison star | Not supported for raw-FITS absolute variability photometry; a single calibrated comparison or calibrated ensemble is required. A pre-reduced relative light curve can be supplied without raw comparison-star photometry, but it is not selected by a comparison-mode tag. | + +The two ensemble limits are independent. `"maximum_number_of_ensemble_comparisons_for_transit"` caps only the transit-fit ensemble. `"maximum_number_of_ensemble_comparisons_for_stellar_variability"` caps both stellar-variability-only and fortuitous-variable ensembles. Each defaults to `5`, must be an integer of at least `2`, and has no configured upper limit. Increase either value to permit a much larger ensemble; EXOTIC will enlarge automatic candidate discovery for the corresponding ensemble where applicable, then use up to that number of surviving comparisons. Very large ensembles require more photometry work. Stellar-variability and fortuitous-variable ensembles can also retain fewer frames because every selected member must have a usable measurement in a retained frame. + +Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable photometry has no no-comparison mode. + +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. `use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/normal classification. diff --git a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py index 3e9f9654..b33abd9f 100644 --- a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py +++ b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py @@ -556,6 +556,12 @@ def create_inits_file(parameters, file_name): "Filter Minimum Wavelength (nm)": parameters.get("Filter Minimum Wavelength (nm)", None), "Filter Maximum Wavelength (nm)": parameters.get("Filter Maximum Wavelength (nm)", None), "disable vertical flux normalization": parameters.get("disable vertical flux normalization", False), + "maximum_number_of_ensemble_comparisons_for_transit": parameters.get( + "maximum_number_of_ensemble_comparisons_for_transit", 5 + ), + "maximum_number_of_ensemble_comparisons_for_stellar_variability": parameters.get( + "maximum_number_of_ensemble_comparisons_for_stellar_variability", 5 + ), "require_comp_star": parameters.get("require_comp_star", "y") } } diff --git a/exotic/api/colab.py b/exotic/api/colab.py index 26b76127..cb5b67f6 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -383,6 +383,8 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "use_eebls_to_initialize_tmid_and_bounds": "y", "pick_comparison_by_eebls_snr": "y", "use_impactparameter_rather_than_inclination_to_fit": "y", + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "use_adaptive_apertures": false, "gain_electrons_per_adu": null, "read_noise_electrons": null, diff --git a/exotic/exotic.py b/exotic/exotic.py index e8c6d8ba..226be24a 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -252,6 +252,7 @@ STELLAR_VARIABILITY_ONLY_DEFAULT = False STELLAR_VARIABILITY_ENSEMBLE_DEFAULT = True STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS = 2 +TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT = 5 STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS = 5 STELLAR_VARIABILITY_APERTURE_ESTIMATION_MAX_COMPARISONS = 5 STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_SIGMA = 3.0 @@ -8538,6 +8539,43 @@ def should_use_ensemble_photometry_for_stellar_variability(config_value): ) +def parse_ensemble_comparison_limit(config_value, config_key, default_value): + if config_value is None or config_value == '': + return default_value + try: + count = int(float(config_value)) + except (TypeError, ValueError): + log_info( + f"Warning: Invalid '{config_key}' value; defaulting to {default_value}.", + warn=True, + ) + return default_value + if count < STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + log_info( + f"'{config_key}' must be at least {STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS}; " + f"defaulting to {default_value}.", + warn=True, + ) + return default_value + return count + + +def parse_maximum_number_of_ensemble_comparisons_for_transit(config_value): + return parse_ensemble_comparison_limit( + config_value, + 'maximum_number_of_ensemble_comparisons_for_transit', + TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT, + ) + + +def parse_maximum_number_of_ensemble_comparisons_for_stellar_variability(config_value): + return parse_ensemble_comparison_limit( + config_value, + 'maximum_number_of_ensemble_comparisons_for_stellar_variability', + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + ) + + def should_photometer_fortuitous_variables(config_value): return parse_bool_config_value( config_value, @@ -17504,7 +17542,8 @@ def catalog_calibration_is_usable_for_filter(star, observed_filter, max_error=No def merge_aavso_vsp_v_calibration_fallback( file, axis, obs_filter, img_scale, calibration_stars, user_comp_stars, - user_targ_star=None, max_new_comp_stars=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS): + user_targ_star=None, + max_new_comp_stars=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS): """Query VSP when a V-family observation has no usable direct V calibration. Existing AAVSO VSP calibrations mean the field has already been queried. The @@ -27979,22 +28018,27 @@ def save_stellar_variability_ensemble_selection_json( metadata = dict(target_metadata or {}) metadata.setdefault('name', target_name) metadata['catalog_profile'] = target_profile + maximum_members = selection.get( + 'member_limit', + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + ) + prelimit_member_count = selection.get('prelimit_member_count', len(members)) payload = { 'target': metadata, 'ensemble': { 'selection_rule': ( 'After saturation, VSX, coverage, stability, and high catalog-error rejection, ' - 'use at most five stars ranked by joint catalog color and magnitude distance to the target; ' + f'use at most {maximum_members} stars ranked by joint catalog color and magnitude ' + 'distance to the target; ' 'retain a frame only when every selected ensemble member is valid.' ), 'minimum_members': STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, 'per_frame_required_members': len(members), - 'maximum_members': selection.get( - 'member_limit', STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS - ), - 'member_count_before_five_star_limit': selection.get( - 'prelimit_member_count', len(members) - ), + 'maximum_members': maximum_members, + 'member_count_before_limit': prelimit_member_count, + # Retained for consumers of EnsembleSelection JSON written before + # Retained after the fixed five-member limit became configurable. + 'member_count_before_five_star_limit': prelimit_member_count, 'selected_member_count': len(members), 'calibration_error_clip': selection.get( 'calibration_error_clip', @@ -28192,6 +28236,8 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, exposure_times_seconds=None, gain_e_per_adu=None, use_ensemble_photometry=True, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, calibration_stars=None, observed_filter=None, target_catalog_match=None): @@ -28328,6 +28374,7 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, comp_flux_map, observed_filter=observed_filter, target_catalog_match=target_catalog_match, + max_members=maximum_number_of_ensemble_comparisons_for_stellar_variability, times=times, ) ensemble_members = member_selection['members'] @@ -28357,7 +28404,8 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, if member_selection.get('prelimit_member_count', 0) > len(ensemble_members): log_info( "Stellar-variability ensemble had more than " - f"{member_selection.get('member_limit')} usable stars; retained the five closest " + f"{member_selection.get('member_limit')} usable stars; retained the configured maximum " + f"of {len(ensemble_members)} closest " "to the target in catalog color and magnitude." ) ensemble_series = build_stellar_variability_calibrated_ensemble_series( @@ -29092,7 +29140,9 @@ def process_fortuitous_variables( comp_overexposed_masks=None, exposure_times_seconds=None, observed_filter=None, - use_single_comparison=USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT): + use_single_comparison=USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS): if not variables or comparison_calibration is None: return [] times = np.asarray(times, dtype=float) @@ -29185,7 +29235,9 @@ def process_fortuitous_variables( observed_filter=observed_filter, target_catalog_match=variable.get('catalog_match'), max_members=( - 1 if use_single_comparison else STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS + 1 + if use_single_comparison + else maximum_number_of_ensemble_comparisons_for_stellar_variability ), min_members=required_comparison_members, times=times, @@ -29520,6 +29572,20 @@ def process_fortuitous_variables( return results +def limited_ensemble_comparison_keys( + ranked_summaries, + maximum_number_of_ensemble_comparisons_for_transit): + ensemble_limit = parse_maximum_number_of_ensemble_comparisons_for_transit( + maximum_number_of_ensemble_comparisons_for_transit + ) + keys = [ + summary.get('key') + for summary in ranked_summaries or [] + if summary.get('key') + ] + return keys[:ensemble_limit] + + def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p_dict, comparison_calibration, psf_data, aper_data, target_psf_flux, psf_flux_data=None, @@ -29546,6 +29612,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p planet_name=None, observation_date=None, use_ensemble_photometry_rather_than_single_comp=False, + maximum_number_of_ensemble_comparisons_for_transit= + TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT, exposure_times_seconds=None, gain_e_per_adu=None): ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) @@ -29614,7 +29682,13 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p preflight_plans = [] if use_ensemble_photometry_rather_than_single_comp: - active_keys = [summary.get('key') for summary in ranked_summaries if summary.get('key')] + ensemble_limit = parse_maximum_number_of_ensemble_comparisons_for_transit( + maximum_number_of_ensemble_comparisons_for_transit + ) + active_keys = limited_ensemble_comparison_keys( + ranked_summaries, + ensemble_limit, + ) if method == 'psf': comp_flux_map = { summary['key']: psf_flux_series_from_rows( @@ -29798,7 +29872,8 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p log_info( "Ensemble comparison photometry enabled: target fit will use " f"{len(member_keys)} non-rejected comparison star(s) as a median normalized ensemble " - "rather than fitting each comparison star independently." + "rather than fitting each comparison star independently " + f"(configured maximum={ensemble_limit})." ) else: log_info( @@ -30736,6 +30811,27 @@ def _main_impl(): ) ) ) + use_ensemble_photometry_rather_than_single_comp = ( + should_use_ensemble_photometry_rather_than_single_comp( + exotic_infoDict.get('use_ensemble_photometry_rather_than_single_comp', 'n') + ) + ) + maximum_number_of_ensemble_comparisons_for_transit = ( + parse_maximum_number_of_ensemble_comparisons_for_transit( + exotic_infoDict.get( + 'maximum_number_of_ensemble_comparisons_for_transit', + TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT, + ) + ) + ) + maximum_number_of_ensemble_comparisons_for_stellar_variability = ( + parse_maximum_number_of_ensemble_comparisons_for_stellar_variability( + exotic_infoDict.get( + 'maximum_number_of_ensemble_comparisons_for_stellar_variability', + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + ) + ) + ) photometer_fortuitous_variables = should_photometer_fortuitous_variables( exotic_infoDict.get( 'photometer_fortuitous_variables', @@ -31471,6 +31567,19 @@ def lookup_archive_ephemeris(): automatic_comp_count = parse_automatic_calibration_selector_count( exotic_infoDict.get('automatic_optimal_calibration_selector_count') ) + if stellar_variability_ensemble_candidate_search: + automatic_comp_count = max( + automatic_comp_count, + maximum_number_of_ensemble_comparisons_for_stellar_variability, + ) + elif ( + automatic_calibration_selector_enabled + and use_ensemble_photometry_rather_than_single_comp + ): + automatic_comp_count = max( + automatic_comp_count, + maximum_number_of_ensemble_comparisons_for_transit, + ) ensemble_candidate_saturation_threshold = None if stellar_variability_ensemble_candidate_search: configured_candidate_saturation = parse_saturation_value( @@ -31569,7 +31678,10 @@ def lookup_archive_ephemeris(): ) else: log_info( - "Fortuitous-variable calibrated ensemble mode enabled per optional_info setting." + "Fortuitous-variable calibrated ensemble mode enabled per optional_info setting; " + "up to " + f"{maximum_number_of_ensemble_comparisons_for_stellar_variability} " + "comparisons will be used." ) fortuitous_variables = discover_fortuitous_vsx_variables( wcs_file, @@ -31634,6 +31746,11 @@ def lookup_archive_ephemeris(): fortuitous_comp_count = parse_automatic_calibration_selector_count( exotic_infoDict.get('automatic_optimal_calibration_selector_count') ) + if not use_single_comparison_for_fortuitous_variables: + fortuitous_comp_count = max( + fortuitous_comp_count, + maximum_number_of_ensemble_comparisons_for_stellar_variability, + ) fortuitous_auto_stars, _ = select_automatic_optimal_calibration_stars( reference_image, reference_image.shape, @@ -31718,6 +31835,11 @@ def lookup_archive_ephemeris(): fortuitous_comp_count = parse_automatic_calibration_selector_count( exotic_infoDict.get('automatic_optimal_calibration_selector_count') ) + if not use_single_comparison_for_fortuitous_variables: + fortuitous_comp_count = max( + fortuitous_comp_count, + maximum_number_of_ensemble_comparisons_for_stellar_variability, + ) fortuitous_auto_stars, _ = select_automatic_optimal_calibration_stars( reference_image, reference_image.shape, @@ -31788,6 +31910,8 @@ def lookup_archive_ephemeris(): fortuitous_calibration_stars, science_comp_stars, user_targ_star=[exotic_UIprevTPX, exotic_UIprevTPY], + max_new_comp_stars= + maximum_number_of_ensemble_comparisons_for_stellar_variability, ) ) if fallback_chart_id is not None: @@ -31939,9 +32063,6 @@ def lookup_archive_ephemeris(): fit_every_comparison_candidate = should_fit_lightcurve_to_every_comparison_candidate( exotic_infoDict.get('fit_lightcurve_to_every_comparison_candidate', 'n') ) - use_ensemble_photometry_rather_than_single_comp = should_use_ensemble_photometry_rather_than_single_comp( - exotic_infoDict.get('use_ensemble_photometry_rather_than_single_comp', 'n') - ) use_deviation_from_expected_transit_in_qc = should_use_deviation_from_expected_transit_in_qc( exotic_infoDict.get('use_deviation_from_expected_transit_in_qc', True) ) @@ -32018,14 +32139,18 @@ def lookup_archive_ephemeris(): if use_ensemble_photometry_rather_than_single_comp: log_info( "Ensemble comparison photometry enabled per optional_info setting; the final target " - "light curve will use non-rejected comparison stars as a combined reference." + "light curve will use non-rejected comparison stars as a combined reference, up to " + "maximum_number_of_ensemble_comparisons_for_transit=" + f"{maximum_number_of_ensemble_comparisons_for_transit}." ) if stellar_variability_only: if use_ensemble_photometry_for_stellar_variability: log_info( "Stellar-variability calibrated ensemble enabled (default): EXOTIC will combine " "bright, unsaturated, VSX-vetted comparison stars after clipping high catalog " - "magnitude uncertainties." + "magnitude uncertainties, up to " + "maximum_number_of_ensemble_comparisons_for_stellar_variability=" + f"{maximum_number_of_ensemble_comparisons_for_stellar_variability}." ) else: log_info( @@ -33155,6 +33280,8 @@ def lookup_archive_ephemeris(): exotic_infoDict.get('filter'), ), use_single_comparison=use_single_comparison_for_fortuitous_variables, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + maximum_number_of_ensemble_comparisons_for_stellar_variability, ) reduction_stage_timer.checkpoint("Aperture finalization and fortuitous-variable photometry") @@ -33531,6 +33658,8 @@ def lookup_archive_ephemeris(): exposure_times_seconds=exposure_times_seconds, gain_e_per_adu=fallback_gain_e_per_adu, use_ensemble_photometry=use_ensemble_photometry_for_stellar_variability, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + maximum_number_of_ensemble_comparisons_for_stellar_variability, calibration_stars=vsp_comp_stars, observed_filter=exotic_infoDict.get( 'observed_filter', @@ -33572,6 +33701,8 @@ def lookup_archive_ephemeris(): observation_date=exotic_infoDict['date'], use_ensemble_photometry_rather_than_single_comp= use_ensemble_photometry_rather_than_single_comp, + maximum_number_of_ensemble_comparisons_for_transit= + maximum_number_of_ensemble_comparisons_for_transit, exposure_times_seconds=exposure_times_seconds, gain_e_per_adu=fallback_gain_e_per_adu, ) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 162c517d..e2e705da 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -420,6 +420,8 @@ def save_input(): "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", + "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; minimum 2; no configured upper limit.", + "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; minimum 2; no configured upper limit.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", @@ -459,6 +461,8 @@ def save_input(): "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "stellar_variability_only": False, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, @@ -1525,6 +1529,8 @@ def save_input(): "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", + "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; minimum 2; no configured upper limit.", + "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; minimum 2; no configured upper limit.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", @@ -1611,6 +1617,8 @@ def save_input(): "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "stellar_variability_only": False, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, @@ -1684,6 +1692,8 @@ def save_input(): "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "stellar_variability_only": False, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "detect_bad_pixels_before_photometry": "n", "multiprocess_bad_pixel_precheck": "n", "detrend_on_outoftransit_baseline": True, diff --git a/exotic/inputs.py b/exotic/inputs.py index 1c94d4ac..e7a04a0d 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -211,6 +211,8 @@ def __init__(self, init_opt): 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'stellar_variability_only': False, 'use_ensemble_photometry_for_stellar_variability': True, + 'maximum_number_of_ensemble_comparisons_for_transit': 5, + 'maximum_number_of_ensemble_comparisons_for_stellar_variability': 5, 'photometer_fortuitous_variables': True, 'use_single_comparison_for_fortuitous_variables': True, 'use_nextastro_vsx_cache_first': False, @@ -491,6 +493,14 @@ def comp_params(self, init_file, planet_dict): 'stellar_variability_use_ensemble', 'Use Ensemble Photometry for Stellar Variability? (y/n)', ), + 'maximum_number_of_ensemble_comparisons_for_transit': ( + 'maximum_number_of_ensemble_comparisons_for_transit', + 'Maximum Number of Ensemble Comparisons for Transit', + ), + 'maximum_number_of_ensemble_comparisons_for_stellar_variability': ( + 'maximum_number_of_ensemble_comparisons_for_stellar_variability', + 'Maximum Number of Ensemble Comparisons for Stellar Variability', + ), 'photometer_fortuitous_variables': ( 'photometer_fortuitous_variables', 'Photometer Fortuitous Variables? (y/n)', diff --git a/inits.json b/inits.json index 06467452..56aeb6a8 100644 --- a/inits.json +++ b/inits.json @@ -29,7 +29,9 @@ "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, use the default calibrated comparison-star ensemble, and discard predicted ingress-to-egress transit-window points. Default false.", - "Stellar Variability Ensemble": "Set optional_info 'use_ensemble_photometry_for_stellar_variability' to false to disable the default calibrated ensemble in stellar_variability_only runs and restore single-comparison selection by out-of-transit scatter. The default ensemble automatically finds bright catalog-calibrated field-star candidates, removes saturated and VSX-variable stars, sigma-clips high catalog magnitude uncertainties, and when more than five remain uses the five closest to the target in catalog colour and magnitude. EnsembleSelection JSON records the target and comparison colours, magnitudes, errors, and selection deltas beside the final results.", + "Stellar Variability Ensemble": "Set optional_info 'use_ensemble_photometry_for_stellar_variability' to false to disable the default calibrated ensemble in stellar_variability_only runs and restore single-comparison selection by out-of-transit scatter. The default ensemble automatically finds bright catalog-calibrated field-star candidates, removes saturated and VSX-variable stars, sigma-clips high catalog magnitude uncertainties, and retains up to maximum_number_of_ensemble_comparisons_for_stellar_variability stars closest to the target in catalog colour and magnitude. EnsembleSelection JSON records the target and comparison colours, magnitudes, errors, and selection deltas beside the final results.", + "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; values must be integers of at least 2, with no configured upper limit.", + "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; values must be integers of at least 2, with no configured upper limit. Large variability ensembles increase photometry work and can reject more frames because every selected ensemble member must be usable in a retained frame.", "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent calibrated photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under variables/optimal_variables/ for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under variables/normal/. Skipped variables are recorded only in the shared variables manifest and do not receive an object directory.", "Fortuitous Variable Single Comparison": "Set optional_info 'use_single_comparison_for_fortuitous_variables' to false to use the calibrated comparison-star ensemble for fortuitous VSX targets. The default true selects one unsaturated, non-variable, catalog-calibrated comparison star closest to each VSX target in catalog colour and magnitude.", "Automatic AAVSO V Calibration Fallback": "For V-family observations, including Clear, CV, and bv, EXOTIC always requires V-band comparison magnitudes. If the NextAstro photometry server supplies no usable V calibration, EXOTIC automatically queries AAVSO VSP and adds matched or discovered V-sequence stars to the same single-comparison or ensemble calibration pool, even when 'Add Comparison Stars from AAVSO?' is n.", @@ -129,6 +131,8 @@ "disable vertical flux normalization": false, "stellar_variability_only": false, "use_ensemble_photometry_for_stellar_variability": true, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "photometer_fortuitous_variables": true, "use_single_comparison_for_fortuitous_variables": true, "use_nextastro_vsx_cache_first": false, diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 1a373abf..bad8223f 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -148,6 +148,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: is_comp_star_required, is_out_of_transit_baseline_detrending_enabled, is_target_driven_comp_selection_enabled, + limited_ensemble_comparison_keys, log_comparison_calibration_fit_attempt_summaries, log_comparison_candidate_fit_summaries, log_target_fit_candidate_summaries, @@ -158,6 +159,8 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: saturation_value_from_header, phase_bin_sigma_clip, parse_deviation_from_expected_transit_in_qc_sigma, + parse_maximum_number_of_ensemble_comparisons_for_stellar_variability, + parse_maximum_number_of_ensemble_comparisons_for_transit, prepare_final_fit_lightcurve_series, prepare_lightcurve_fit_input_series, psf_frame_quality_components, @@ -1403,6 +1406,89 @@ def test_stellar_variability_ensemble_config_defaults_on_and_supports_opt_out(): assert should_use_ensemble_photometry_for_stellar_variability(False) is False +def test_independent_ensemble_comparison_limits_default_to_five_and_have_no_upper_cap(): + assert parse_maximum_number_of_ensemble_comparisons_for_transit(None) == 5 + assert parse_maximum_number_of_ensemble_comparisons_for_transit(1) == 5 + assert parse_maximum_number_of_ensemble_comparisons_for_transit("250") == 250 + assert parse_maximum_number_of_ensemble_comparisons_for_stellar_variability(None) == 5 + assert parse_maximum_number_of_ensemble_comparisons_for_stellar_variability(2) == 2 + assert parse_maximum_number_of_ensemble_comparisons_for_stellar_variability("125") == 125 + + +def test_limited_ensemble_comparison_keys_uses_configured_maximum(): + ranked_summaries = [ + {'key': f'comp{index}'} + for index in range(1, 13) + ] + + assert limited_ensemble_comparison_keys(ranked_summaries, None) == [ + f'comp{index}' for index in range(1, 6) + ] + assert limited_ensemble_comparison_keys(ranked_summaries, 12) == [ + f'comp{index}' for index in range(1, 13) + ] + + +def test_transit_ensemble_fit_uses_configured_maximum(monkeypatch): + frame_count = 6 + quality_mask = np.ones(frame_count, dtype=bool) + ranked_summaries = [ + { + 'key': f'comp{index}', + 'comp_index': index - 1, + 'aggregate_score': index / 1000.0, + 'coverage_rejected': False, + 'suitability_outlier_rejected': False, + 'psf_quality_keep_mask': quality_mask, + } + for index in range(1, 13) + ] + comparison_calibration = { + 'method': 'aperture', + 'a': 0, + 'an': 0, + 'aper': 5.0, + 'annulus': 12.0, + 'comp_summaries': ranked_summaries, + } + aper_data = { + 'target': np.full((frame_count, 1, 1), 1000.0), + **{ + f'comp{index}': np.full((frame_count, 1, 1), 100.0 + index) + for index in range(1, 13) + }, + } + captured = {} + + def capture_active_keys(comp_flux_map, active_keys, validity_mask_func): + captured['active_keys'] = list(active_keys) + raise RuntimeError('captured configured transit ensemble') + + monkeypatch.setattr( + 'exotic.exotic.build_absolute_comp_ensemble_flux', + capture_active_keys, + ) + + with pytest.raises(RuntimeError, match='captured configured transit ensemble'): + fit_ranked_comparison_calibration_candidates( + np.linspace(0.0, 0.05, frame_count), + np.linspace(2460000.0, 2460000.05, frame_count), + np.linspace(1.0, 1.2, frame_count), + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(frame_count), + use_ensemble_photometry_rather_than_single_comp=True, + maximum_number_of_ensemble_comparisons_for_transit=9, + ) + + assert captured['active_keys'] == [ + f'comp{index}' for index in range(1, 10) + ] + + def test_fortuitous_variable_photometry_config_defaults_on_and_supports_opt_out(): assert should_photometer_fortuitous_variables(None) is True assert should_photometer_fortuitous_variables("y") is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 88a97aeb..4853aab3 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -140,6 +140,47 @@ def test_comp_params_reads_stellar_variability_ensemble_opt_out(tmp_path): assert inputs.info_dict["use_ensemble_photometry_for_stellar_variability"] is False +def test_comp_params_defaults_independent_ensemble_comparison_limits_to_five(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["maximum_number_of_ensemble_comparisons_for_transit"] == 5 + assert ( + inputs.info_dict["maximum_number_of_ensemble_comparisons_for_stellar_variability"] + == 5 + ) + + +def test_comp_params_reads_different_transit_and_variability_ensemble_limits(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "maximum_number_of_ensemble_comparisons_for_transit": 250, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 125, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["maximum_number_of_ensemble_comparisons_for_transit"] == 250 + assert ( + inputs.info_dict["maximum_number_of_ensemble_comparisons_for_stellar_variability"] + == 125 + ) + + def test_comp_params_defaults_fortuitous_variable_photometry_to_true(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 506708bc..bc783cb0 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -2702,6 +2702,22 @@ def test_stellar_variability_ensemble_caps_at_five_by_target_color_and_magnitude } assert limited_keys == {'comp1', 'comp2'} + expanded_selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + target_catalog_match=target_match, + max_members=7, + ) + + assert expanded_selection['member_limit'] == 7 + assert len(expanded_selection['members']) == 7 + assert not any( + 'closest to the target' in rejected['reason'] + for rejected in expanded_selection['rejected'] + ) + def test_stellar_variability_ensemble_uses_gaia_bp_rp_when_local_colors_are_missing( monkeypatch): @@ -3231,6 +3247,7 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product exposure_times_seconds=np.full(frame_count, 60.0), observed_filter='V', use_single_comparison=False, + maximum_number_of_ensemble_comparisons_for_stellar_variability=2, ) assert results[0]['status'] == 'completed' @@ -3444,6 +3461,7 @@ def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(): aper_data, target_flux, use_ensemble_photometry=True, + maximum_number_of_ensemble_comparisons_for_stellar_variability=2, calibration_stars=calibration_stars, observed_filter='V', ) From 3a4986f1f89007bb155d5bc9453df517d3b2885e Mon Sep 17 00:00:00 2001 From: Opus Date: Tue, 28 Jul 2026 18:46:52 +0000 Subject: [PATCH 098/116] Fix epoch selection: report the transit nearest the data, not at-or-before its end estimate_ephemeris_tmid_and_bounds used floor(phases).max(), which selects the transit at or before the last valid frame. For ingress-only partial transits, where the true mid falls after the last surviving frame, that centers the Tmid search bounds one full period early; the periodic transit model fits the data perfectly at the wrong epoch and the reported absolute Tmid is exactly one period early, while every phase-folded plot looks correct. Fixed-window observations that cut egress hit exactly this geometry, and whether the bug fires depends on which edge frames survive quality rejection. round(median(phases)) selects the epoch nearest the bulk of the data: identical for full transits and one-sided baselines near the mid, correct for the ingress-only case, and insensitive to edge-frame rejection. Regression test uses the geometry of the 2026-07-27 TOI-1516 b MicroObservatory night where two independent WBoM reductions reported the same Tmid one period early (issue #1387). Co-Authored-By: Claude Fable 5 --- exotic/exotic.py | 7 ++++++- tests/test_exotic_proper_motion.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 226be24a..373029f2 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -9865,7 +9865,12 @@ def estimate_ephemeris_tmid_and_bounds( return summary phases = (valid_times - prior_tmid) / period - cycle_index = float(np.floor(phases).max()) + # Select the transit epoch nearest the bulk of the data. floor(phases).max() + # picked the transit at-or-before the last valid frame, which reports Tmid one + # full period early whenever the true mid falls after the last surviving frame + # (ingress-only partial transits, the common fixed-window case); the periodic + # transit model then fits perfectly at the wrong epoch. See issue #1387. + cycle_index = float(np.round(np.median(phases))) tmid = float(prior_tmid + cycle_index * period) propagated_uncertainty = np.sqrt(midt_unc ** 2 + (cycle_index * per_unc) ** 2) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index bad8223f..51fb961f 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -1848,6 +1848,35 @@ def test_estimate_ephemeris_tmid_and_bounds_keeps_wider_bounds_for_one_sided_run assert summary["half_width"] == pytest.approx(0.25 * prior["per"]) +def test_estimate_ephemeris_tmid_and_bounds_selects_nearest_epoch_for_ingress_only_runs(): + # Regression for issue #1387: an ingress-only partial transit whose true mid + # falls minutes AFTER the last surviving frame. floor(phases).max() snapped to + # the previous cycle and reported Tmid one full period early; the epoch nearest + # the data is the correct one. Geometry taken from the 2026-07-27 TOI-1516 b + # MicroObservatory night that surfaced the bug (two independent reductions + # reported 2461246.93, one period before the actual night of the frames). + prior_tmid = 2458765.325 + period = 2.056014 + times = np.linspace(2461248.8938, 2461248.9847, 34) + expected_mid = prior_tmid + 1208 * period # 2461248.9899, ~7 min after times.max() + + summary = estimate_ephemeris_tmid_and_bounds( + times, + prior_tmid, + period, + midt_unc=0.00023, + per_unc=2.1e-6, + expected_duration=0.1177, + sigma_multiplier=25.0, + ) + + assert summary["cycle_index"] == pytest.approx(1208.0) + assert summary["tmid"] == pytest.approx(expected_mid, abs=1e-6) + # The search bounds must be able to reach the true mid. + assert summary["bounds"][0] <= expected_mid <= summary["bounds"][1] + assert summary["observations_bracket_expected_transit"] is False + + def test_is_adaptive_aperture_mode_enabled_parses_values(): assert is_adaptive_aperture_mode_enabled(None) is False assert is_adaptive_aperture_mode_enabled("y") is True From cb551045f675c27d3310cba0351d0ef297aac341 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 29 Jul 2026 09:05:09 +1000 Subject: [PATCH 099/116] OOT variability working for ensemble now. --- exotic/exotic.py | 120 +++++++++++++++++++++++++++- tests/test_nextastro_variability.py | 35 +++++++- 2 files changed, 151 insertions(+), 4 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 373029f2..0fdc94d0 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -28229,6 +28229,59 @@ def build_stellar_variability_ensemble_params_from_fit( return vsp_params +def build_stellar_variability_params_from_photometry_selection( + selection, + calibration_stars, + save, + s_name, + observed_filter=None, + observation_date=None, + target_metadata=None): + selected_result = (selection or {}).get('selected_result') + if not selected_result or selected_result.get('fit') is None: + log_info( + "Warning: calibrated stellar-variability photometry did not select a usable " + "out-of-transit reference, so AID magnitude output could not be created.", + warn=True, + ) + return [] + + selected_fit = selected_result['fit'] + if selected_result.get('comp_index') is None: + return build_stellar_variability_ensemble_params_from_fit( + selected_fit, + save, + s_name, + observed_filter=observed_filter, + observation_date=observation_date, + target_metadata=target_metadata, + ) + + selected_position = selected_result.get('position') + calibration = stellar_variability_calibration_for_position( + calibration_stars, + selected_position, + observed_filter=observed_filter, + ) + if calibration is None: + log_info( + "Warning: the fallback stellar-variability comparison star had no usable " + "same-band catalog calibration, so AID magnitude output could not be created.", + warn=True, + ) + return [] + + return build_stellar_variability_params_from_fit( + selected_fit, + calibration['star'], + selected_position, + calibration['label'], + save, + s_name, + observed_filter=observed_filter, + ) + + def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, comparison_calibration, psf_data, aper_data, target_psf_flux, psf_flux_data=None, @@ -33515,6 +33568,7 @@ def lookup_archive_ephemeris(): } comparison_calibration = None + stellar_variability_output_selection = None comparison_calibration = select_comparison_calibrated_photometry( psf_data, aper_data, @@ -33711,6 +33765,39 @@ def lookup_archive_ephemeris(): exposure_times_seconds=exposure_times_seconds, gain_e_per_adu=fallback_gain_e_per_adu, ) + if use_ensemble_photometry_for_stellar_variability and vsp_comp_stars: + log_info( + "Preparing an independent calibrated comparison-star ensemble for " + "out-of-transit stellar-variability AID output." + ) + stellar_variability_output_selection = select_stellar_variability_only_photometry( + times, + jd_times, + airmass, + pDict, + comparison_calibration, + psf_data, + aper_data, + tFlux, + psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + plot_time_range=full_plot_time_range, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=aperture_values, + adaptive_annulus_values=annulus_values, + fallback_sigma=sigma_display, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + use_ensemble_photometry=True, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + maximum_number_of_ensemble_comparisons_for_stellar_variability, + calibration_stars=vsp_comp_stars, + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + target_catalog_match=primary_target_catalog_match, + ) comparison_calibration['ranked_fit_comp_indices'] = [ summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] ] @@ -34487,7 +34574,34 @@ def lookup_archive_ephemeris(): # Calculate the standard deviation of the normalized flux values # standardDev1 = np.std(goodFluxes) - if stellar_variability_only and bestCompStar == 'ensemble': + if stellar_variability_output_selection is not None: + try: + vsp_params = build_stellar_variability_params_from_photometry_selection( + stellar_variability_output_selection, + vsp_comp_stars, + exotic_infoDict['save'], + pDict['sName'], + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + observation_date=exotic_infoDict.get('date'), + target_metadata={ + 'name': pDict.get('sName'), + 'ra_deg': None if ra_dec_tar is None else ra_dec_tar[0], + 'dec_deg': None if ra_dec_tar is None else ra_dec_tar[1], + 'pixel_position': [exotic_UIprevTPX, exotic_UIprevTPY], + }, + ) + except Exception as exc: + log_info( + "Warning: could not create calibrated ensemble stellar-variability " + f"AID rows ({describe_retry_exception(exc)}).", + warn=True, + ) + vsp_params = [] + + if not vsp_params and stellar_variability_only and bestCompStar == 'ensemble': vsp_params = build_stellar_variability_ensemble_params_from_fit( best_fit_lc, exotic_infoDict['save'], @@ -34504,7 +34618,7 @@ def lookup_archive_ephemeris(): 'pixel_position': [exotic_UIprevTPX, exotic_UIprevTPY], }, ) - elif vsp_comp_stars: + elif not vsp_params and vsp_comp_stars: if isinstance(bestCompStar, int): vsp_params = stellar_variability(ref_flux, best_fit_lc, fortuitous_ensemble_stars, vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], @@ -34517,7 +34631,7 @@ def lookup_archive_ephemeris(): wcs_file=wcs_file, catalog_match_radius_arcsec= photometry_catalog_match_radius_arcsec) - else: + elif stellar_variability_output_selection is None: log_info( "Skipping AID magnitude output because no reference comparison star was selected.", warn=True, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index bc783cb0..b614fc1d 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -3393,7 +3393,8 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product ) -def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(): +def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(monkeypatch, tmp_path): + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) frame_count = 12 times = np.linspace(10.2, 10.3, frame_count) target_flux = 1000.0 * (1.0 + np.linspace(-0.002, 0.002, frame_count)) @@ -3473,6 +3474,38 @@ def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(): assert selected['fit'].stellar_variability_ensemble_members assert len(selected['fit'].stellar_variability_ensemble_magnitudes) == len(selected['fit'].time) + vsp_params = exotic_module.build_stellar_variability_params_from_photometry_selection( + result, + calibration_stars, + tmp_path, + 'Synthetic', + observed_filter='V', + observation_date='2024-01-02', + ) + aid_path = exotic_module.AIDOutputFiles( + selected['fit'], + {'sName': 'Synthetic', 'pName': 'Synthetic b'}, + { + 'save': tmp_path, + 'date': '2024-01-02', + 'aavso_num': 'TEST', + 'camera': 'CCD', + 'lat': 0.0, + 'long': 0.0, + 'elev': 0.0, + 'filter': 'V', + }, + 'AUID-TEST', + None, + vsp_params, + ).aavso() + + assert len(vsp_params) == frame_count + assert aid_path.is_file() + aid_text = aid_path.read_text(encoding='utf-8') + assert '#ENSEMBLE-COMPARISONS-XC=' in aid_text + assert 'AUID-TEST,' in aid_text + def test_stellar_variability_selector_opt_out_restores_single_comp_selection(): frame_count = 24 From 4398bb88ca394a8a8974362f605570957d7548a6 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 30 Jul 2026 12:16:23 +1000 Subject: [PATCH 100/116] Ultranest efficiency optimization #72 The one where we re-use old valid points in a re-weighted fashion. --- exotic/api/elca.py | 556 +++++++++++++++++++++- exotic/exotic.py | 55 ++- scripts/expanded_prior_real_data_trial.py | 368 ++++++++++++++ tests/test_elca_baseline.py | 271 +++++++++++ tests/test_exotic_proper_motion.py | 8 +- tests/test_exotic_rprs_retry.py | 101 +++- 6 files changed, 1335 insertions(+), 24 deletions(-) create mode 100644 scripts/expanded_prior_real_data_trial.py diff --git a/exotic/api/elca.py b/exotic/api/elca.py index f0fdb421..7f034fa8 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -54,6 +54,7 @@ from scipy import spatial from scipy.optimize import least_squares from scipy.signal import savgol_filter +from scipy.special import ndtr, ndtri from ultranest import ReactiveNestedSampler try: @@ -87,6 +88,9 @@ EXPOSURE_SMEARING_TRANSIT_WINDOW_PADDING_FACTOR = 2.0 ULTRANEST_INFLATED_ERROR_REPLACEMENT_FACTOR = 3.0 ULTRANEST_LOCAL_UNCERTAINTY_MAX_DELTA_CHI2 = 9.0 +ULTRANEST_EXPANDED_PRIOR_WARMSTART_FULL_PRIOR_FRACTION = 0.5 +ULTRANEST_EXPANDED_PRIOR_WARMSTART_MINIMUM_SAMPLE_COUNT = 32 +ULTRANEST_EXPANDED_PRIOR_WARMSTART_MAXIMUM_SAMPLE_COUNT = 20000 def _pylightcurve_import_watchdog_seconds(): try: @@ -714,6 +718,7 @@ def __init__( exposure_times_seconds=None, exposure_smearing_supersample=DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE, exposure_smearing_change_tolerance=DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE, + ultranest_warmstart_source=None, ): self.time = time self.data = data @@ -738,6 +743,16 @@ def __init__( ) self.fixed_flux_baseline = bool(fixed_flux_baseline) self.ultranest_min_num_live_points = ultranest_min_num_live_points + self.ultranest_warmstart_source = ultranest_warmstart_source + self.ultranest_expanded_prior_warmstart_attempted = False + self.ultranest_expanded_prior_warmstart_applied = False + self.ultranest_expanded_prior_warmstart_note = None + self.ultranest_expanded_prior_warmstart_source_sample_count = 0 + self.ultranest_expanded_prior_warmstart_effective_sample_size = 0.0 + self.ultranest_expanded_prior_warmstart_expanded_keys = [] + self.ultranest_expanded_prior_warmstart_full_prior_fraction = ( + ULTRANEST_EXPANDED_PRIOR_WARMSTART_FULL_PRIOR_FRACTION + ) self.exposure_times_days = normalize_exposure_times_seconds_to_days( exposure_times_seconds, np.asarray(time).shape, @@ -1724,6 +1739,440 @@ def _sample_point_from_unit_cube(self, upars, bound_keys=None): return sample_point + def _unit_cube_from_sample_points(self, sample_points, bound_keys=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + points = np.asarray(sample_points, dtype=float) + scalar_input = points.ndim == 1 + points_2d = np.atleast_2d(points) + if points_2d.shape[1] != len(bound_keys): + raise ValueError( + "Sample-point dimensionality does not match the expanded-prior bounds." + ) + + boundarray = np.array([self.bounds[key] for key in bound_keys], dtype=float) + widths = boundarray[:, 1] - boundarray[:, 0] + if ( + not np.all(np.isfinite(boundarray)) + or not np.all(np.isfinite(widths)) + or np.any(widths <= 0) + ): + raise ValueError("Expanded-prior bounds are not finite and increasing.") + + unit_points = (points_2d - boundarray[:, 0]) / widths + if self._uses_internal_impact_parameter() and 'inc' in bound_keys: + inc_index = bound_keys.index('inc') + upper_bounds = self._get_impact_parameter_upper_bounds_for_sample_points( + points_2d, + bound_keys, + ) + valid_upper = np.isfinite(upper_bounds) & (upper_bounds > 0) + if not np.all(valid_upper): + raise ValueError( + "Impact-parameter upper bounds are invalid for warm-start samples." + ) + unit_points[:, inc_index] = points_2d[:, inc_index] / upper_bounds + + return unit_points[0] if scalar_input else unit_points + + @staticmethod + def _warmstart_values_match(left, right): + if left is None or right is None: + return left is None and right is None + if isinstance(left, dict) or isinstance(right, dict): + if not isinstance(left, dict) or not isinstance(right, dict): + return False + if set(left) != set(right): + return False + return all( + lc_fitter._warmstart_values_match(left[key], right[key]) + for key in left + ) + try: + left_array = np.asarray(left) + right_array = np.asarray(right) + if left_array.shape != right_array.shape: + return False + if ( + np.issubdtype(left_array.dtype, np.number) + and np.issubdtype(right_array.dtype, np.number) + ): + return bool(np.array_equal( + left_array.astype(float), + right_array.astype(float), + equal_nan=True, + )) + return bool(np.array_equal(left_array, right_array)) + except (TypeError, ValueError): + return left == right + + def _expanded_prior_warmstart_compatibility(self, source, bound_keys, sampled_keys): + if source is None: + return None, "No previous UltraNest fit was supplied." + if getattr(source, 'ns_type', None) != 'ultranest': + return None, "The previous fit is not an UltraNest result." + + source_bound_keys = list(getattr(source, 'bounds', {}).keys()) + source_sampled_keys = list(getattr(source, 'sampled_keys', [])) + if source_bound_keys != list(bound_keys) or source_sampled_keys != list(sampled_keys): + return None, "The sampled parameterization changed between UltraNest fits." + + expanded_keys = [] + tolerance = 1e-12 + for key in bound_keys: + try: + source_lower, source_upper = [ + float(value) + for value in np.asarray(source.bounds[key], dtype=float).reshape(-1)[:2] + ] + target_lower, target_upper = [ + float(value) + for value in np.asarray(self.bounds[key], dtype=float).reshape(-1)[:2] + ] + except (KeyError, TypeError, ValueError): + return None, f"The {key} bounds are unavailable for warm-start validation." + if ( + target_lower > source_lower + tolerance + or target_upper < source_upper - tolerance + ): + return None, f"The {key} bounds contracted instead of forming a true superset." + if ( + target_lower < source_lower - tolerance + or target_upper > source_upper + tolerance + ): + expanded_keys.append(key) + + if not expanded_keys: + return None, "The prior bounds did not expand." + + likelihood_attributes = ( + 'time', + 'data', + 'dataerr', + 'airmass', + 'exposure_times_days', + 'baseline_fit_mask', + 'duration_prior', + 'fixed_flux_baseline', + 'use_impactparameter_rather_than_inclination_to_fit', + ) + for attribute_name in likelihood_attributes: + if not self._warmstart_values_match( + getattr(source, attribute_name, None), + getattr(self, attribute_name, None), + ): + return None, ( + f"The likelihood input {attribute_name} changed between UltraNest fits." + ) + + free_keys = set(bound_keys) + source_prior = getattr(source, 'prior', {}) + if not isinstance(source_prior, dict) or not isinstance(self.prior, dict): + return None, "The fixed model parameters are unavailable for warm-start validation." + fixed_keys = (set(source_prior) | set(self.prior)) - free_keys + for key in fixed_keys: + if not self._warmstart_values_match( + source_prior.get(key), + self.prior.get(key), + ): + return None, f"The fixed model parameter {key} changed between UltraNest fits." + + try: + weighted_samples = source.results['weighted_samples'] + source_points = np.asarray(weighted_samples['points'], dtype=float) + source_weights = np.asarray(weighted_samples['weights'], dtype=float) + except (AttributeError, KeyError, TypeError, ValueError): + return None, "The previous weighted posterior samples are unavailable." + + parameter_count = len(sampled_keys) + if ( + source_points.ndim != 2 + or source_points.shape[1] < parameter_count + or source_weights.ndim != 1 + or source_weights.shape[0] != source_points.shape[0] + ): + return None, "The previous weighted posterior sample arrays are malformed." + + source_points = source_points[:, :parameter_count] + valid = ( + np.all(np.isfinite(source_points), axis=1) + & np.isfinite(source_weights) + & (source_weights > 0) + ) + source_points = source_points[valid] + source_weights = source_weights[valid] + minimum_count = max( + ULTRANEST_EXPANDED_PRIOR_WARMSTART_MINIMUM_SAMPLE_COUNT, + 4 * max(1, parameter_count), + ) + if source_points.shape[0] < minimum_count: + return None, ( + f"Only {source_points.shape[0]} valid previous posterior samples are available; " + f"at least {minimum_count} are required." + ) + + weight_sum = float(np.sum(source_weights)) + if not np.isfinite(weight_sum) or weight_sum <= 0: + return None, "The previous posterior sample weights are invalid." + source_weights = source_weights / weight_sum + effective_sample_size = float(1.0 / np.sum(source_weights ** 2)) + minimum_effective_count = max(16, 2 * max(1, parameter_count)) + if not np.isfinite(effective_sample_size) or effective_sample_size < minimum_effective_count: + return None, ( + f"The previous posterior effective sample size is only " + f"{effective_sample_size:.1f}; at least {minimum_effective_count} is required." + ) + + return { + 'expanded_keys': expanded_keys, + 'points': source_points, + 'weights': source_weights, + 'effective_sample_size': effective_sample_size, + }, None + + @staticmethod + def _bounded_warmstart_posterior_sample(points, weights, maximum_count): + if points.shape[0] <= maximum_count: + return points, weights + + cumulative = np.cumsum(weights) + cumulative[-1] = 1.0 + quantiles = (np.arange(maximum_count, dtype=float) + 0.5) / maximum_count + selected = np.searchsorted(cumulative, quantiles, side='left') + selected = np.clip(selected, 0, points.shape[0] - 1) + return points[selected], np.full(maximum_count, 1.0 / maximum_count) + + def _build_expanded_prior_warmstart_problem( + self, + bound_keys, + sampled_keys, + loglike, + prior_transform, + ): + source = getattr(self, 'ultranest_warmstart_source', None) + compatibility, reason = self._expanded_prior_warmstart_compatibility( + source, + bound_keys, + sampled_keys, + ) + if compatibility is None: + return None, reason + + points, weights = self._bounded_warmstart_posterior_sample( + compatibility['points'], + compatibility['weights'], + ULTRANEST_EXPANDED_PRIOR_WARMSTART_MAXIMUM_SAMPLE_COUNT, + ) + unit_points = np.asarray( + self._unit_cube_from_sample_points(points, bound_keys), + dtype=float, + ) + tolerance = 1e-10 + inside = ( + np.all(np.isfinite(unit_points), axis=1) + & np.all(unit_points >= -tolerance, axis=1) + & np.all(unit_points <= 1.0 + tolerance, axis=1) + ) + unit_points = unit_points[inside] + weights = weights[inside] + if unit_points.shape[0] < ULTRANEST_EXPANDED_PRIOR_WARMSTART_MINIMUM_SAMPLE_COUNT: + return None, ( + "Too few previous posterior samples remain inside the expanded prior." + ) + + weights = weights / np.sum(weights) + unit_points = np.clip(unit_points, 1e-12, 1.0 - 1e-12) + full_prior_fraction = float(np.clip( + ULTRANEST_EXPANDED_PRIOR_WARMSTART_FULL_PRIOR_FRACTION, + 1e-6, + 1.0 - 1e-6, + )) + parameter_count = len(sampled_keys) + weighted_mean = np.sum(unit_points * weights[:, None], axis=0) + weighted_variance = np.sum( + weights[:, None] * (unit_points - weighted_mean) ** 2, + axis=0, + ) + # A modest scale floor avoids a singular hot proposal when the old + # posterior is extremely narrow. The 50% uniform component below is + # the stronger defense and guarantees full expanded-prior support. + hot_scale = np.clip(np.sqrt(np.maximum(weighted_variance, 0.0)), 0.02, 0.5) + hot_mean = np.clip(weighted_mean, 1e-8, 1.0 - 1e-8) + hot_alpha = -hot_mean / hot_scale + hot_beta = (1.0 - hot_mean) / hot_scale + hot_cdf_lower = ndtr(hot_alpha) + hot_cdf_width = np.maximum( + ndtr(hot_beta) - hot_cdf_lower, + np.finfo(float).tiny, + ) + log_hot_normalization = np.log(hot_cdf_width) + log_two_pi_half = 0.5 * np.log(2.0 * np.pi) + log_full_fraction = np.log(full_prior_fraction) + log_hot_fraction = np.log1p(-full_prior_fraction) + + def hot_transform(unit_values): + probabilities = hot_cdf_lower + unit_values * hot_cdf_width + probabilities = np.clip( + probabilities, + np.finfo(float).eps, + 1.0 - np.finfo(float).eps, + ) + return np.clip( + hot_mean + hot_scale * ndtri(probabilities), + 0.0, + 1.0, + ) + + def hot_log_density(unit_values): + standardized = (unit_values - hot_mean) / hot_scale + return np.sum( + -0.5 * standardized ** 2 + - np.log(hot_scale) + - log_two_pi_half + - log_hot_normalization, + axis=1, + ) + + # Defensive importance proposal: + # + # q_mix(u) = f * Uniform(u) + (1-f) * q_hot(u) + # + # The latent selector samples one of those components, while every + # point receives the common correction pi(u)/q_mix(u). Unlike using a + # branch-specific correction, this remains evidence-correct even if + # nested sampling prunes the component that contributes negligibly in + # a particular likelihood region. Half of all proposal mass still + # comes directly from the complete expanded prior. + def defensive_transform(unit_values): + values = np.asarray(unit_values, dtype=float) + scalar_input = values.ndim == 1 + values_2d = np.atleast_2d(values) + if values_2d.shape[1] != parameter_count + 1: + raise ValueError( + "Expanded-prior warm-start unit points have the wrong dimensionality." + ) + + proposal_unit = np.empty( + (values_2d.shape[0], parameter_count), + dtype=float, + ) + full_prior_mask = values_2d[:, -1] < full_prior_fraction + proposal_unit[full_prior_mask] = values_2d[ + full_prior_mask, :parameter_count + ] + hot_mask = ~full_prior_mask + if np.any(hot_mask): + proposal_unit[hot_mask] = hot_transform( + values_2d[hot_mask, :parameter_count] + ) + + log_q_hot = hot_log_density(proposal_unit) + log_q_mix = np.logaddexp( + log_full_fraction, + log_hot_fraction + log_q_hot, + ) + transformed = np.empty( + (values_2d.shape[0], parameter_count + 1), + dtype=float, + ) + transformed[:, :parameter_count] = prior_transform(proposal_unit) + transformed[:, -1] = -log_q_mix + + return transformed[0] if scalar_input else transformed + + def defensive_loglike(parameters): + values = np.asarray(parameters, dtype=float) + physical = values[..., :parameter_count] + correction = values[..., parameter_count] + return loglike(physical) + correction + + self.ultranest_expanded_prior_warmstart_source_sample_count = int( + unit_points.shape[0] + ) + self.ultranest_expanded_prior_warmstart_effective_sample_size = float( + compatibility['effective_sample_size'] + ) + self.ultranest_expanded_prior_warmstart_expanded_keys = list( + compatibility['expanded_keys'] + ) + return { + 'param_names': list(sampled_keys) + ['aux_logweight'], + 'loglike': defensive_loglike, + 'transform': defensive_transform, + 'vectorized': True, + 'full_prior_fraction': full_prior_fraction, + }, None + + @staticmethod + def _expanded_prior_warmstart_result_is_usable(results, parameter_count): + try: + maximum_likelihood = np.asarray( + results['maximum_likelihood']['point'], + dtype=float, + ) + weighted_points = np.asarray( + results['weighted_samples']['points'], + dtype=float, + ) + weighted_logl = np.asarray( + results['weighted_samples']['logl'], + dtype=float, + ) + except (KeyError, TypeError, ValueError): + return False + return bool( + maximum_likelihood.ndim == 1 + and maximum_likelihood.size >= parameter_count + and np.all(np.isfinite(maximum_likelihood[:parameter_count])) + and weighted_points.ndim == 2 + and weighted_points.shape[0] > 0 + and weighted_points.shape[1] >= parameter_count + and weighted_logl.shape == (weighted_points.shape[0],) + and np.any(np.isfinite(weighted_logl)) + ) + + @staticmethod + def _restore_expanded_prior_physical_likelihoods(results, loglike, parameter_count): + weighted_samples = results['weighted_samples'] + points = np.asarray(weighted_samples['points'], dtype=float) + auxiliary_logl = np.asarray(weighted_samples['logl'], dtype=float) + physical_logl = np.asarray( + loglike(points[:, :parameter_count]), + dtype=float, + ) + if physical_logl.shape != (points.shape[0],) or not np.any(np.isfinite(physical_logl)): + raise ValueError( + "the corrected warm-start samples could not be evaluated under the physical likelihood" + ) + + weighted_samples['auxiliary_logl'] = auxiliary_logl.copy() + weighted_samples['logl'] = physical_logl + if points.shape[1] > parameter_count: + weighted_samples['auxiliary_points'] = points[:, parameter_count:].copy() + weighted_samples['points'] = points[:, :parameter_count].copy() + maximum_index = int(np.nanargmax(physical_logl)) + maximum_likelihood = results.setdefault('maximum_likelihood', {}) + # The auxiliary likelihood contains a proposal correction and is not + # the physical maximum-likelihood criterion reported by EXOTIC. + maximum_likelihood['auxiliary_point'] = points[maximum_index].copy() + maximum_likelihood['point'] = points[maximum_index, :parameter_count].copy() + maximum_likelihood['logl'] = float(physical_logl[maximum_index]) + + equal_weight_samples = np.asarray(results.get('samples'), dtype=float) + if ( + equal_weight_samples.ndim == 2 + and equal_weight_samples.shape[1] >= parameter_count + ): + if equal_weight_samples.shape[1] > parameter_count: + results['auxiliary_samples'] = equal_weight_samples[:, parameter_count:].copy() + results['samples'] = equal_weight_samples[:, :parameter_count].copy() + + posterior = results.get('posterior') + if isinstance(posterior, dict): + for key, value in list(posterior.items()): + value_array = np.asarray(value) + if value_array.ndim == 1 and value_array.size == points.shape[1]: + posterior[key] = value_array[:parameter_count].copy() + def _physical_values_from_sample_point(self, sample_point, bound_keys=None, sampled_keys=None): bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) sampled_keys = self._get_sampled_keys(bound_keys) if sampled_keys is None else list(sampled_keys) @@ -1762,6 +2211,9 @@ def _get_ultranest_weighted_sample_arrays(self): return None, None if logl.shape[0] != points.shape[0]: return None, None + sampled_key_count = len(getattr(self, 'sampled_keys', [])) + if sampled_key_count and points.shape[1] >= sampled_key_count: + points = points[:, :sampled_key_count] return points, logl def _loglike_neighborhood_uncertainty(self, parameter_index, center, minimum_count=8, points=None, logl=None): @@ -2180,6 +2632,9 @@ def _get_triangle_plot_samples(self): weighted_samples = self.results['weighted_samples'] points = np.asarray(weighted_samples['points'], dtype=float) logl = np.asarray(weighted_samples['logl'], dtype=float) + sampled_key_count = len(getattr(self, 'sampled_keys', [])) + if sampled_key_count and points.shape[1] >= sampled_key_count: + points = points[:, :sampled_key_count] weights = self._get_triangle_plot_sample_weights( weighted_samples.get('weights'), points.shape[0], @@ -2205,10 +2660,18 @@ def _get_triangle_plot_sample_weights(self, weights, sample_count): return weights def get_parameter_posterior_samples(self, key): + sample_points = None try: - sample_points, _, _ = self._get_triangle_plot_samples() + equal_weight_samples = np.asarray(self.results.get('samples'), dtype=float) + if equal_weight_samples.ndim == 2 and equal_weight_samples.shape[0] > 0: + sample_points = equal_weight_samples except Exception: - return np.array([], dtype=float) + sample_points = None + if sample_points is None: + try: + sample_points, _, _ = self._get_triangle_plot_samples() + except Exception: + return np.array([], dtype=float) sample_points = np.asarray(sample_points, dtype=float) if sample_points.ndim != 2 or sample_points.shape[0] == 0: @@ -3677,6 +4140,17 @@ def extend_ultranest_fit(self, min_num_live_points=None, max_ncalls=None): run_kwargs=run_kwargs, verbose=self.verbose, ) + if getattr(self, 'ultranest_expanded_prior_warmstart_applied', False): + if not self._expanded_prior_warmstart_result_is_usable( + self.results, + len(context['sampled_keys']), + ): + return False + self._restore_expanded_prior_physical_likelihoods( + self.results, + context['loglike'], + len(context['sampled_keys']), + ) self._finalize_ultranest_fit_results( context['bound_keys'], context['sampled_keys'], @@ -3916,17 +4390,82 @@ def prior_transform(upars): return sample_points self.ns_type = 'ultranest' - test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform, vectorized=True) + warmstart_problem, warmstart_skip_reason = self._build_expanded_prior_warmstart_problem( + bound_keys, + sampled_keys, + loglike, + prior_transform, + ) + self.ultranest_expanded_prior_warmstart_attempted = ( + getattr(self, 'ultranest_warmstart_source', None) is not None + ) + if warmstart_problem is not None: + test = ReactiveNestedSampler( + warmstart_problem['param_names'], + warmstart_problem['loglike'], + warmstart_problem['transform'], + vectorized=warmstart_problem['vectorized'], + ) + self.ultranest_expanded_prior_warmstart_applied = True + self.ultranest_expanded_prior_warmstart_note = ( + "Applied a corrected expanded-prior UltraNest warm start using " + f"{self.ultranest_expanded_prior_warmstart_source_sample_count} previous " + "weighted posterior sample(s), with " + f"{100.0 * warmstart_problem['full_prior_fraction']:.0f}% of the auxiliary " + "prior reserved for direct exploration of the full expanded prior." + ) + else: + test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform, vectorized=True) + self.ultranest_expanded_prior_warmstart_applied = False + self.ultranest_expanded_prior_warmstart_note = warmstart_skip_reason run_kwargs = {"max_ncalls": int(self.max_ncalls)} if self.ultranest_min_num_live_points is not None: run_kwargs["min_num_live_points"] = int(self.ultranest_min_num_live_points) - self.results = run_reactive_sampler( - test, - run_kwargs=run_kwargs, - verbose=self.verbose, - ) + try: + self.results = run_reactive_sampler( + test, + run_kwargs=run_kwargs, + verbose=self.verbose, + ) + if ( + warmstart_problem is not None + and not self._expanded_prior_warmstart_result_is_usable( + self.results, + len(sampled_keys), + ) + ): + raise ValueError( + "the corrected warm-start result did not contain usable posterior samples" + ) + if warmstart_problem is not None: + self._restore_expanded_prior_physical_likelihoods( + self.results, + loglike, + len(sampled_keys), + ) + except Exception as exc: + if warmstart_problem is None: + raise + self.ultranest_expanded_prior_warmstart_applied = False + self.ultranest_expanded_prior_warmstart_note = ( + "Corrected expanded-prior warm start failed; reran from the full expanded " + f"prior instead ({type(exc).__name__}: {exc})." + ) + if self.verbose: + print(f"WARNING: {self.ultranest_expanded_prior_warmstart_note}") + test = ReactiveNestedSampler( + sampled_keys, + loglike, + prior_transform, + vectorized=True, + ) + self.results = run_reactive_sampler( + test, + run_kwargs=run_kwargs, + verbose=self.verbose, + ) if self.keep_ultranest_sampler: self._ultranest_resume_context = { @@ -3934,6 +4473,7 @@ def prior_transform(upars): 'bound_keys': list(bound_keys), 'sampled_keys': list(sampled_keys), 'physical_from_sample_point': physical_from_sample_point, + 'loglike': loglike, } else: self._ultranest_resume_context = None diff --git a/exotic/exotic.py b/exotic/exotic.py index 0fdc94d0..53d04199 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -7239,6 +7239,7 @@ def build_fit( local_bounds, fixed_parameter_errors_override=None, allow_ars_expansion=False, + ultranest_warmstart_source=None, ): local_bounds = sanitize_retry_search_bounds( apply_configured_prior_search_restrictions( @@ -7297,6 +7298,11 @@ def build_fit( and callable_accepts_keyword(lc_fitter, 'ultranest_min_num_live_points') ): fit_kwargs['ultranest_min_num_live_points'] = ultranest_min_num_live_points + if ( + ultranest_warmstart_source is not None + and callable_accepts_keyword(lc_fitter, 'ultranest_warmstart_source') + ): + fit_kwargs['ultranest_warmstart_source'] = ultranest_warmstart_source fit = lc_fitter( times, flux_values, @@ -7420,10 +7426,40 @@ def build_fit( allow_ars_expansion=allow_ars_expansion, ).get(bounds_key, clamped_bounds) clamped_bounds = retry_config['sanitize_bounds']({bounds_key: clamped_bounds}).get(bounds_key, clamped_bounds) + if previous_bounds is not None: + previous_lower, previous_upper = [ + float(value) + for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2] + ] + clamped_bounds = [ + min(previous_lower, float(clamped_bounds[0])), + max(previous_upper, float(clamped_bounds[1])), + ] + clamped_bounds = retry_config['sanitize_bounds']( + {bounds_key: clamped_bounds} + ).get(bounds_key, clamped_bounds) + clamped_bounds = apply_configured_prior_search_restrictions( + {bounds_key: clamped_bounds}, + restriction_reference_prior, + allow_ars_expansion=allow_ars_expansion, + ).get(bounds_key, clamped_bounds) + clamped_bounds = retry_config['sanitize_bounds']( + {bounds_key: clamped_bounds} + ).get(bounds_key, clamped_bounds) new_lower, new_upper = [float(value) for value in clamped_bounds] if previous_bounds is not None: - previous_lower, previous_upper = [float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2]] clipped_edge = diagnostics.get('edge') + preserves_previous_range = ( + new_lower <= previous_lower + 1e-12 + and new_upper >= previous_upper - 1e-12 + ) + if not preserves_previous_range: + retry_notes[key] = ( + f"Skipped; the automatic {label} retry could not preserve the complete " + "previous sampled range while expanding the prior." + ) + blocked_retry_keys.add(key) + continue expands_sampled_range = retry_config.get('expands_bounds', normal_retry_expands)( previous_bounds, [new_lower, new_upper], @@ -7504,11 +7540,28 @@ def build_fit( current_prior = updated_prior current_bounds = updated_bounds ars_range_expansion_active = allow_ars_expansion + previous_fit = fit fit = build_fit( current_prior, current_bounds, allow_ars_expansion=ars_range_expansion_active, + ultranest_warmstart_source=previous_fit, ) + warmstart_note = getattr( + fit, + 'ultranest_expanded_prior_warmstart_note', + None, + ) + if getattr(fit, 'ultranest_expanded_prior_warmstart_applied', False): + log_info(warmstart_note) + elif ( + getattr(fit, 'ultranest_expanded_prior_warmstart_attempted', False) + and warmstart_note + ): + log_info( + f"Warning: expanded-prior UltraNest warm start was not used. {warmstart_note}", + warn=True, + ) final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) rprs_final_diagnostics = None diff --git a/scripts/expanded_prior_real_data_trial.py b/scripts/expanded_prior_real_data_trial.py new file mode 100644 index 00000000..6a47b1c4 --- /dev/null +++ b/scripts/expanded_prior_real_data_trial.py @@ -0,0 +1,368 @@ +"""Exercise corrected UltraNest expanded-prior warm starts on saved real light curves. + +This is a manual validation helper. It fits each data set with a deliberately +narrow Rp/R* prior, expands that prior, and compares the corrected warm-started +fit with an independent cold fit over the same expanded prior. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import time +from pathlib import Path + +import numpy as np + +from exotic.api.elca import lc_fitter + + +DATASETS = ( + { + "name": "WASP-43b full transit", + "path": Path(r"D:\WASP43b_codex_current_fix_run\temp\NormalizedFlux_WASP-43 b_2026-03-20.txt"), + "format": "normalized", + "prior": { + "rprs": 0.1594, + "tmid": 2461120.68976, + "ars": 4.86, + "per": 0.813475, + "inc": 82.6, + "ecc": 0.0, + "omega": 90.0, + }, + "initial_rprs_bounds": [0.155, 0.165], + "expanded_rprs_bounds": [0.05, 0.30], + "tmid_bounds": [2461120.686, 2461120.694], + }, + { + "name": "TrES-5b full transit", + "path": Path( + r"D:\TrES5b_20260716_baron_rp\TrES5b_20260716_baron_rp" + r"\20260718_112527\Diagnostics\comp7\working_artifacts" + r"\FinalLightCurve_TrES-5b_2026-07-16.csv" + ), + "format": "final_lightcurve", + "prior": { + "rprs": 0.143, + "tmid": 2461238.83817, + "ars": 6.1, + "per": 1.48224686, + "inc": 84.27, + "ecc": 0.0, + "omega": 0.0, + }, + "initial_rprs_bounds": [0.140, 0.146], + "expanded_rprs_bounds": [0.05, 0.30], + "tmid_bounds": [2461238.834, 2461238.842], + }, + { + "name": "KELT-20b one-sided partial transit", + "path": Path( + r"D:\KELT-20\20260718_003832_codex_full_test" + r"\working_artifacts\NormalizedFlux_KELT-20b_2026-07-15.txt" + ), + "format": "normalized", + "prior": { + "rprs": 0.1144, + "tmid": 2461237.776, + "ars": 7.42, + "per": 3.4741085, + "inc": 86.12, + "ecc": 0.0, + "omega": 0.0, + }, + "initial_rprs_bounds": [0.110, 0.120], + "expanded_rprs_bounds": [0.02, 0.30], + "tmid_bounds": [2461237.736, 2461237.816], + }, +) + + +def load_normalized(path: Path) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + with path.open(newline="", encoding="utf-8-sig") as stream: + rows = list(csv.DictReader(stream)) + time_values = np.asarray([float(row["BJD"]) for row in rows], dtype=float) + flux = np.asarray([float(row["Norm Flux"]) for row in rows], dtype=float) + error = np.asarray([float(row["Norm Err"]) for row in rows], dtype=float) + airmass = np.asarray([float(row["AM"]) for row in rows], dtype=float) + return time_values, flux, error, airmass + + +def load_final_lightcurve(path: Path) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + values = np.loadtxt(path, delimiter=",", comments="#") + return values[:, 0], values[:, 2], values[:, 3], values[:, 5] + + +def load_dataset(config: dict) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + if config["format"] == "normalized": + return load_normalized(config["path"]) + return load_final_lightcurve(config["path"]) + + +def complete_prior(values: dict) -> dict: + return { + **values, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + } + + +def summarize_fit(fit: lc_fitter, elapsed_seconds: float) -> dict: + samples = np.asarray(fit.results["samples"], dtype=float) + rprs_index = list(fit.sampled_keys).index("rprs") + rprs_samples = samples[:, rprs_index] + q16, median, q84 = np.quantile(rprs_samples, [0.16, 0.5, 0.84]) + results = fit.results + return { + "elapsed_seconds": elapsed_seconds, + "ncall": int(results.get("ncall", 0)), + "posterior_sample_count": int(samples.shape[0]), + "rprs_q16": float(q16), + "rprs_median": float(median), + "rprs_q84": float(q84), + "rprs_stdev": float(np.std(rprs_samples, ddof=1)), + "rprs_maximum_likelihood": float(fit.parameters["rprs"]), + "logz": float(results.get("logz", math.nan)), + "logzerr": float(results.get("logzerr", math.nan)), + "warmstart_attempted": bool( + getattr(fit, "ultranest_expanded_prior_warmstart_attempted", False) + ), + "warmstart_applied": bool( + getattr(fit, "ultranest_expanded_prior_warmstart_applied", False) + ), + "warmstart_note": getattr( + fit, "ultranest_expanded_prior_warmstart_note", None + ), + "warmstart_source_sample_count": int( + getattr( + fit, + "ultranest_expanded_prior_warmstart_source_sample_count", + 0, + ) + ), + "warmstart_source_effective_sample_size": float( + getattr( + fit, + "ultranest_expanded_prior_warmstart_effective_sample_size", + 0.0, + ) + ), + "warmstart_expanded_keys": list( + getattr( + fit, + "ultranest_expanded_prior_warmstart_expanded_keys", + [], + ) + ), + "warmstart_full_prior_fraction": float( + getattr( + fit, + "ultranest_expanded_prior_warmstart_full_prior_fraction", + math.nan, + ) + ), + } + + +def run_fit( + *, + time_values: np.ndarray, + flux: np.ndarray, + error: np.ndarray, + airmass: np.ndarray, + prior: dict, + bounds: list[float], + tmid_bounds: list[float], + seed: int, + live_points: int, + warmstart_source: lc_fitter | None = None, +) -> tuple[lc_fitter, dict]: + np.random.seed(seed) + started = time.perf_counter() + fit = lc_fitter( + time_values, + flux, + error, + airmass, + prior, + {"rprs": list(bounds), "tmid": list(tmid_bounds)}, + mode="ns", + jd_times=time_values, + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + ultranest_min_num_live_points=live_points, + ultranest_warmstart_source=warmstart_source, + ) + elapsed = time.perf_counter() - started + return fit, summarize_fit(fit, elapsed) + + +def run_dataset(config: dict, live_points: int, seed: int) -> dict: + time_values, flux, error, airmass = load_dataset(config) + finite = ( + np.isfinite(time_values) + & np.isfinite(flux) + & np.isfinite(error) + & np.isfinite(airmass) + & (error > 0) + ) + time_values = time_values[finite] + flux = flux[finite] + error = error[finite] + airmass = airmass[finite] + prior = complete_prior(config["prior"]) + + initial_fit, initial_summary = run_fit( + time_values=time_values, + flux=flux, + error=error, + airmass=airmass, + prior=prior, + bounds=config["initial_rprs_bounds"], + tmid_bounds=config["tmid_bounds"], + seed=seed, + live_points=live_points, + ) + warm_fit, warm_summary = run_fit( + time_values=time_values, + flux=flux, + error=error, + airmass=airmass, + prior=prior, + bounds=config["expanded_rprs_bounds"], + tmid_bounds=config["tmid_bounds"], + seed=seed + 1, + live_points=live_points, + warmstart_source=initial_fit, + ) + _, cold_summary = run_fit( + time_values=time_values, + flux=flux, + error=error, + airmass=airmass, + prior=prior, + bounds=config["expanded_rprs_bounds"], + tmid_bounds=config["tmid_bounds"], + seed=seed + 2, + live_points=live_points, + ) + + combined_sigma = math.hypot( + warm_summary["rprs_stdev"], cold_summary["rprs_stdev"] + ) + posterior_z = ( + abs(warm_summary["rprs_median"] - cold_summary["rprs_median"]) + / combined_sigma + if combined_sigma > 0 + else math.inf + ) + logz_sigma = math.hypot( + warm_summary["logzerr"], cold_summary["logzerr"] + ) + logz_z = ( + abs(warm_summary["logz"] - cold_summary["logz"]) / logz_sigma + if np.isfinite(logz_sigma) and logz_sigma > 0 + else math.nan + ) + old_lower, old_upper = config["initial_rprs_bounds"] + warm_outside_old = ( + warm_summary["rprs_median"] < old_lower + or warm_summary["rprs_median"] > old_upper + ) + cold_outside_old = ( + cold_summary["rprs_median"] < old_lower + or cold_summary["rprs_median"] > old_upper + ) + passed = ( + warm_summary["warmstart_applied"] + and warm_summary["warmstart_expanded_keys"] == ["rprs"] + and posterior_z <= 1.0 + and (not np.isfinite(logz_z) or logz_z <= 3.0) + and warm_outside_old == cold_outside_old + ) + return { + "name": config["name"], + "source_path": str(config["path"]), + "point_count": int(time_values.size), + "time_min_bjd_tdb": float(np.min(time_values)), + "time_max_bjd_tdb": float(np.max(time_values)), + "initial_rprs_bounds": list(config["initial_rprs_bounds"]), + "expanded_rprs_bounds": list(config["expanded_rprs_bounds"]), + "tmid_bounds": list(config["tmid_bounds"]), + "initial": initial_summary, + "expanded_warm": warm_summary, + "expanded_cold": cold_summary, + "comparison": { + "posterior_median_difference_sigma": float(posterior_z), + "logz_difference_sigma": float(logz_z), + "warm_median_outside_initial_bounds": bool(warm_outside_old), + "cold_median_outside_initial_bounds": bool(cold_outside_old), + }, + "passed": bool(passed), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--live-points", type=int, default=100) + parser.add_argument("--seed", type=int, default=4729) + parser.add_argument( + "--dataset-index", + type=int, + action="append", + help="Zero-based dataset index to run; repeat to select more than one.", + ) + args = parser.parse_args() + + args.output.parent.mkdir(parents=True, exist_ok=True) + trial_started = time.perf_counter() + results = [] + selected_indices = ( + list(range(len(DATASETS))) + if args.dataset_index is None + else args.dataset_index + ) + for index in selected_indices: + if index < 0 or index >= len(DATASETS): + parser.error(f"--dataset-index must be between 0 and {len(DATASETS) - 1}") + config = DATASETS[index] + print(f"TRIAL START: {config['name']}", flush=True) + result = run_dataset( + config, + live_points=max(40, args.live_points), + seed=args.seed + index * 100, + ) + results.append(result) + print( + "TRIAL DONE: " + f"{config['name']} | pass={result['passed']} | " + f"warm={result['expanded_warm']['warmstart_applied']} | " + f"posterior_z={result['comparison']['posterior_median_difference_sigma']:.3f} | " + f"logz_z={result['comparison']['logz_difference_sigma']:.3f}", + flush=True, + ) + + payload = { + "live_points": max(40, args.live_points), + "seed": args.seed, + "elapsed_seconds": time.perf_counter() - trial_started, + "all_passed": all(result["passed"] for result in results), + "datasets": results, + } + args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8") + print(f"RESULTS: {args.output}", flush=True) + print(f"ALL PASSED: {payload['all_passed']}", flush=True) + return 0 if payload["all_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 5f74ccc1..820cfbf9 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -68,6 +68,71 @@ def make_prior(): } +def make_expanded_prior_warmstart_source( + prior, + time, + data, + dataerr, + airmass, + sample_points, +): + sample_points = np.asarray(sample_points, dtype=float) + sample_count = sample_points.shape[0] + return types.SimpleNamespace( + ns_type="ultranest", + bounds={ + "rprs": [0.08, 0.12], + "tmid": [-0.005, 0.005], + }, + sampled_keys=["rprs", "tmid"], + prior=prior.copy(), + time=np.asarray(time, dtype=float), + data=np.asarray(data, dtype=float), + dataerr=np.asarray(dataerr, dtype=float), + airmass=np.asarray(airmass, dtype=float), + exposure_times_days=None, + baseline_fit_mask=None, + duration_prior=None, + fixed_flux_baseline=False, + use_impactparameter_rather_than_inclination_to_fit=False, + results={ + "weighted_samples": { + "points": sample_points, + "weights": np.full(sample_count, 1.0 / sample_count), + "logl": np.linspace(-5.0, -1.0, sample_count), + }, + }, + ) + + +def make_dummy_nested_result(sample_points, auxiliary=False): + sample_points = np.asarray(sample_points, dtype=float) + if auxiliary: + result_points = np.column_stack([ + sample_points, + np.zeros(sample_points.shape[0], dtype=float), + ]) + else: + result_points = sample_points + parameter_count = result_points.shape[1] + maximum_likelihood = np.zeros(parameter_count, dtype=float) + maximum_likelihood[0] = 0.1 + return { + "maximum_likelihood": {"point": maximum_likelihood}, + "posterior": { + "stdev": np.full(parameter_count, 0.001), + "errlo": np.full(parameter_count, -0.001), + "errup": np.full(parameter_count, 0.001), + }, + "weighted_samples": { + "points": result_points, + "weights": np.full(result_points.shape[0], 1.0 / result_points.shape[0]), + "logl": np.linspace(-5.0, -1.0, result_points.shape[0]), + }, + "samples": result_points.copy(), + } + + def test_lc_fitter_recovers_explicit_a0_baseline(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) prior = make_prior() @@ -917,6 +982,212 @@ def test_unit_cube_transform_vectorizes_simple_bounds(monkeypatch, tmp_path): ) +def test_unit_cube_inverse_maps_expanded_prior_samples_back_to_unit_cube(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = False + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.05, 0.15], + "tmid": [-0.01, 0.01], + } + + unit_points = np.array([ + [0.30, 0.25], + [0.70, 0.75], + ]) + sample_points = fit._sample_point_from_unit_cube(unit_points) + + np.testing.assert_allclose( + fit._unit_cube_from_sample_points(sample_points), + unit_points, + ) + + +def test_expanded_prior_warmstart_uses_corrected_guarded_auxiliary_problem(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 81) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + source_points = np.column_stack([ + np.linspace(0.085, 0.115, 64), + np.linspace(-0.004, 0.004, 64), + ]) + source = make_expanded_prior_warmstart_source( + prior, + time, + data, + dataerr, + airmass, + source_points, + ) + captured = {} + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + captured["sampler"] = self + + def fake_run_reactive_sampler(sampler, *args, **kwargs): + unit_values = np.array([ + [0.4, 0.6, 0.25], + [0.4, 0.6, 0.75], + ]) + transformed = np.asarray(sampler.args[2](unit_values), dtype=float) + captured["transformed"] = transformed + captured["physical_loglike"] = np.asarray( + [ + sampler.args[1](np.append(row[:2], 0.0)) + for row in transformed + ], + dtype=float, + ) + captured["corrected_loglike"] = np.asarray( + sampler.args[1](transformed), + dtype=float, + ) + return make_dummy_nested_result(source_points, auxiliary=True) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr(elca, "run_reactive_sampler", fake_run_reactive_sampler) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + { + "rprs": [0.05, 0.15], + "tmid": [-0.005, 0.005], + }, + mode="ns", + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + ultranest_warmstart_source=source, + ) + + assert captured["sampler"].args[0] == ["rprs", "tmid", "aux_logweight"] + np.testing.assert_allclose(captured["transformed"][0, :2], [0.09, 0.001]) + assert np.all(np.isfinite(captured["transformed"])) + assert captured["transformed"][1, 0] > captured["transformed"][0, 0] + np.testing.assert_allclose( + captured["corrected_loglike"] - captured["physical_loglike"], + captured["transformed"][:, 2], + ) + assert fit.ultranest_expanded_prior_warmstart_attempted is True + assert fit.ultranest_expanded_prior_warmstart_applied is True + assert fit.ultranest_expanded_prior_warmstart_expanded_keys == ["rprs"] + assert fit.ultranest_expanded_prior_warmstart_source_sample_count == 64 + assert fit._get_triangle_plot_samples()[0].shape[1] == 2 + + +def test_expanded_prior_warmstart_failure_falls_back_to_clean_sampler(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 81) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + source_points = np.column_stack([ + np.linspace(0.085, 0.115, 64), + np.linspace(-0.004, 0.004, 64), + ]) + source = make_expanded_prior_warmstart_source( + prior, + time, + data, + dataerr, + airmass, + source_points, + ) + samplers = [] + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + samplers.append(self) + + def fake_run_reactive_sampler(sampler, *args, **kwargs): + if len(sampler.args[0]) == 3: + raise RuntimeError("synthetic corrected-warmstart failure") + return make_dummy_nested_result(source_points, auxiliary=False) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr(elca, "run_reactive_sampler", fake_run_reactive_sampler) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + { + "rprs": [0.05, 0.15], + "tmid": [-0.005, 0.005], + }, + mode="ns", + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + ultranest_warmstart_source=source, + ) + + assert [sampler.args[0] for sampler in samplers] == [ + ["rprs", "tmid", "aux_logweight"], + ["rprs", "tmid"], + ] + assert fit.ultranest_expanded_prior_warmstart_attempted is True + assert fit.ultranest_expanded_prior_warmstart_applied is False + assert "reran from the full expanded prior" in fit.ultranest_expanded_prior_warmstart_note + assert fit.parameters["rprs"] == pytest.approx(0.1) + + +def test_expanded_prior_warmstart_restores_physical_likelihood_for_best_fit(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + results = { + "maximum_likelihood": { + "point": np.array([0.09, 0.0, 5.0]), + "logl": 5.0, + }, + "weighted_samples": { + "points": np.array([ + [0.09, 0.0, 5.0], + [0.11, 0.0, -5.0], + ]), + "logl": np.array([5.0, -5.0]), + }, + } + + def physical_loglike(points): + points = np.asarray(points, dtype=float) + return -((points[:, 0] - 0.11) / 0.01) ** 2 + + elca.lc_fitter._restore_expanded_prior_physical_likelihoods( + results, + physical_loglike, + 2, + ) + + np.testing.assert_allclose( + results["weighted_samples"]["auxiliary_logl"], + np.array([5.0, -5.0]), + ) + np.testing.assert_allclose( + results["weighted_samples"]["logl"], + np.array([-4.0, 0.0]), + ) + assert results["weighted_samples"]["points"].shape == (2, 2) + assert results["weighted_samples"]["auxiliary_points"].shape == (2, 1) + assert results["maximum_likelihood"]["point"].shape == (2,) + assert results["maximum_likelihood"]["point"][0] == pytest.approx(0.11) + assert results["maximum_likelihood"]["logl"] == pytest.approx(0.0) + + def test_nested_fit_can_keep_inclination_parameterization_when_requested(monkeypatch, tmp_path): elca = load_elca_with_stubs(monkeypatch, tmp_path) fit = elca.lc_fitter.__new__(elca.lc_fitter) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 51fb961f..0fc062b4 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -3926,11 +3926,11 @@ def fake_lc_fitter( assert len(captured["calls"]) == 2 assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.125]) assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.158) - assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.108, 0.208]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.0, 0.208]) assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 1 assert fit.rprs_posterior_refit_edge == "upper" - assert fit.rprs_posterior_refit_bounds == pytest.approx([0.108, 0.208]) + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.0, 0.208]) def test_fit_final_lightcurve_carries_retry_bounds_into_oot_baseline_refit(monkeypatch): @@ -4021,8 +4021,8 @@ def fake_lc_fitter( assert len(captured["calls"]) == 3 assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.125]) - assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.108, 0.208]) - assert captured["calls"][2]["bounds"]["rprs"] == pytest.approx([0.108, 0.208]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.0, 0.208]) + assert captured["calls"][2]["bounds"]["rprs"] == pytest.approx([0.0, 0.208]) assert np.allclose(captured["calls"][2]["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) assert fit.oot_baseline_detrending_applied is True diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index 1df10338..c8cc0bb6 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -355,6 +355,7 @@ def fake_lc_fitter( ) assert len(captured["calls"]) == 3 + assert captured["calls"][1]["bounds"]["rprs"][0] == pytest.approx(0.0) assert captured["calls"][1]["bounds"]["rprs"][1] == pytest.approx(0.5) assert "rprs" not in captured["calls"][2]["bounds"] assert fit.rprs_posterior_refit_bounds[1] == pytest.approx(0.5) @@ -363,6 +364,84 @@ def fake_lc_fitter( assert fit.parameters["rprs"] == pytest.approx(0.4) +def test_expanded_prior_retry_passes_previous_fit_as_corrected_warmstart_source(monkeypatch): + import exotic.exotic as exotic_module + + calls = [] + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ultranest_warmstart_source=None, + ): + call_index = len(calls) + fit = types.SimpleNamespace( + parameters={ + "tmid": 0.0, + "rprs": 0.19 if call_index == 0 else 0.21, + "inc": 89.0, + "a2": 0.0, + }, + ) + + def diagnostics(key): + if key != "rprs": + return None + if call_index == 0: + return { + "clipped": True, + "edge": "upper", + "mode": 0.19, + "std": 0.02, + "bounds": [0.15, 0.30], + } + return { + "clipped": False, + "edge": None, + "mode": 0.21, + "std": 0.02, + "bounds": list(call_bounds["rprs"]), + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + calls.append({ + "fit": fit, + "warmstart_source": ultranest_warmstart_source, + "bounds": {key: list(value) for key, value in call_bounds.items()}, + }) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.0, 0.2], + "tmid": [-0.01, 0.01], + "inc": [84.0, 90.0], + "a2": [-3.0, 3.0], + }, + use_prior_rprs_when_posterior_pinned=False, + ) + + assert len(calls) == 2 + assert calls[0]["warmstart_source"] is None + assert calls[1]["warmstart_source"] is calls[0]["fit"] + assert calls[1]["bounds"]["rprs"] == pytest.approx([0.0, 0.30]) + assert fit.rprs_posterior_refit_applied is True + + def test_rprs_posterior_retry_does_not_escape_configured_prior_range(monkeypatch): import exotic.exotic as exotic_module @@ -1475,17 +1554,17 @@ def fake_lc_fitter( np.asarray([call["bounds"]["rprs"] for call in captured["calls"][:-1]], dtype=float), np.asarray([ [0.0, 0.125], - [0.108, 0.208], - [0.132, 0.232], - [0.144, 0.244], - [0.151, 0.251], - [0.156, 0.256], + [0.0, 0.208], + [0.0, 0.232], + [0.0, 0.244], + [0.0, 0.251], + [0.0, 0.256], ], dtype=float), ) assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 5 assert fit.rprs_posterior_refit_edge == "upper" - assert fit.rprs_posterior_refit_bounds == pytest.approx([0.156, 0.256]) + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.0, 0.256]) assert fit.rprs_prior_fallback_applied is True assert fit.parameters["rprs"] == pytest.approx(0.1) assert "prior fallback" in fit.rprs_prior_fallback_note @@ -1561,10 +1640,10 @@ def fake_lc_fitter( assert len(captured["calls"]) == 2 assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.25]) assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) - assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.175, 0.375]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.0, 0.375]) assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 1 - assert fit.rprs_posterior_refit_bounds == pytest.approx([0.175, 0.375]) + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.0, 0.375]) def test_rprs_posterior_retry_can_continue_above_the_old_maximum_exoplanet_range(monkeypatch): @@ -1634,7 +1713,7 @@ def fake_lc_fitter( assert captured["calls"][0]["prior"]["rprs"] == pytest.approx(0.35) assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([RPRS_SEARCH_BOUND_MIN, 0.35]) assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) - assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.175, 0.375]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.0, 0.375]) assert "rprs" not in captured["calls"][2]["bounds"] assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 1 @@ -1704,11 +1783,11 @@ def fake_lc_fitter( assert len(captured["calls"]) == 2 assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.025, 0.300]) assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.030) - assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.000, 0.120]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.000, 0.300]) assert fit.rprs_posterior_refit_applied is True assert fit.rprs_posterior_refit_count == 1 assert fit.rprs_posterior_refit_edge == "lower" - assert fit.rprs_posterior_refit_bounds == pytest.approx([0.000, 0.120]) + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.000, 0.300]) def test_ars_posterior_retry_expands_bounds_when_upper_edge_is_truncated(monkeypatch): From 312372434547c29ac2c48cc3ae638002e6bd109a Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 2 Aug 2026 10:53:30 +1000 Subject: [PATCH 101/116] Comp RA&Dec and use_only_these_comps commit --- README.md | 14 +- docs/README.md | 16 +- docs/system_prompt.txt | 1 + .../for_exotic_py_candidate_inits_maker.py | 9 +- exotic/api/colab.py | 3 + exotic/exotic.py | 1111 ++++++++++++++--- exotic/exotic_gui.py | 16 +- exotic/inputs.py | 92 +- exotic/output_files.py | 233 ++++ exotic/plots.py | 108 +- inits.json | 7 +- tests/test_exotic_proper_motion.py | 83 ++ tests/test_inputs.py | 81 ++ tests/test_nextastro_variability.py | 105 +- tests/test_output_files.py | 55 + tests/test_plots.py | 34 + 16 files changed, 1750 insertions(+), 218 deletions(-) diff --git a/README.md b/README.md index ef43b006..d3b519a0 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,8 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Plate Solution? (y/n)": "y", "Target Star X & Y Pixel": [424, 286], - "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263], [], [], [], [], [], [], [], []] + "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263], [], [], [], [], [], [], [], []], + "Comparison Star(s) RA & Dec": null }, "planetary_parameters": { "Target Star RA": "02:04:10", @@ -168,6 +169,8 @@ Get EXOTIC up and running faster with a json file. Please see the included file "use_ensemble_photometry_rather_than_single_comp": false, "stellar_variability_only": false, "use_ensemble_photometry_for_stellar_variability": true, + "require_apparent_magnitudes": true, + "use_exactly_the_comps_provided": false, "maximum_number_of_ensemble_comparisons_for_transit": 5, "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "photometer_fortuitous_variables": true, @@ -194,10 +197,13 @@ Get EXOTIC up and running faster with a json file. Please see the included file Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended; EXOTIC also accepts equivalent values such as `"y"` and `"n"`. +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Celestial coordinates require a usable WCS and are projected onto the selected reference image before photometry; after projection they are treated identically to supplied X/Y positions. + | Reduction | Requested comparison mode | `optional_info` settings | |---|---|---| | Transit fit | Single comparison star (default) | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": false` | | Transit fit | Comparison-star ensemble | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": true`, `"maximum_number_of_ensemble_comparisons_for_transit": 5` | +| Transit or variability run | Exactly the supplied comparison(s) | `"use_exactly_the_comps_provided": true`. Comparisons may be supplied as X/Y or RA/Dec. One supplied comparison is used alone; two or more are all used as one fixed ensemble. Automatic replacement, addition, VSX/stability vetting, ranking, and ensemble-size limiting are bypassed. | | Transit fit | No comparison star | There is no tag that forces this mode. `"require_comp_star": false` only removes the requirement for a comparison star; it does not force target-only photometry. The current comparison-calibration FITS path still selects a single comparison or an ensemble. | | Stellar-variability-only run | Single comparison star | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": false` | | Stellar-variability-only run | Calibrated comparison-star ensemble (default) | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": true`, `"maximum_number_of_ensemble_comparisons_for_stellar_variability": 5` | @@ -205,7 +211,11 @@ Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` The two ensemble limits are independent. `"maximum_number_of_ensemble_comparisons_for_transit"` caps only the transit-fit ensemble. `"maximum_number_of_ensemble_comparisons_for_stellar_variability"` caps both stellar-variability-only and fortuitous-variable ensembles. Each defaults to `5`, must be an integer of at least `2`, and has no configured upper limit. Increase either value to permit a much larger ensemble; EXOTIC will enlarge automatic candidate discovery for the corresponding ensemble where applicable, then use up to that number of surviving comparisons. Very large ensembles require more photometry work. Stellar-variability and fortuitous-variable ensembles can also retain fewer frames because every selected member must have a usable measurement in a retained frame. -Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable photometry has no no-comparison mode. +Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble, except when `"use_exactly_the_comps_provided"` is true. Exact-comparison mode fails explicitly if the supplied reference cannot be measured; it never silently substitutes or drops a supplied comparison. This makes the same reference star or ensemble reproducible across multiple runs. + +Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. + +For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. `photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. diff --git a/docs/README.md b/docs/README.md index 3f335727..a404d20e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -122,7 +122,7 @@ The scatter in the residuals of the lightcurve fit is: 0.5414 % - Plate solve my images - select if you want EXOTIC to calibrate the right ascenscion and declination of your pixels in your image via Astrometry.net; it is recommended that this option is selected - Align my images - select this option for EXOTIC to align all of your images to provide better tracking of your stars in your images; it is recommended that this option is selected - Target Star X & Y Pixel Position - the pixel location of your target exoplanet host star in [x-position, y-position] format - - Comparison Star(s) X & Y Pixel Position - the pixel location of your comparision star(s) in [x-position, y-position] format; it is recommended that you input at least 2 comparision stars and EXOTIC will automatically select the "best" comparision by the one that produces the least amount of scatter in your data + - Comparison Star(s) Position - provide either X/Y pixel pairs or RA/Dec pairs, but not both. RA/Dec accepts decimal degrees or sexagesimal strings, requires a usable WCS, and is projected onto the selected reference image before being treated exactly like X/Y input - *NOTE:* In the screenshot below, Rob has already entered all of the information for you for the sample data (with the exception that you'll need to point to the correct directory for your FITS files and your EXOTIC Output) ![EXOTIC Input Observation Information](https://github.com/rzellem/EXOTIC/blob/develop/docs/images/exotic_inputobs.png) @@ -182,7 +182,8 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Plate Solution? (y/n)": "n", "Target Star X & Y Pixel": [424, 286], - "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263]] + "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263]], + "Comparison Star(s) RA & Dec": null }, "planetary_parameters": { "Target Star RA": "02:04:10", @@ -222,6 +223,8 @@ Get EXOTIC up and running faster with a json file. Please see the included file "use_ensemble_photometry_rather_than_single_comp": false, "stellar_variability_only": false, "use_ensemble_photometry_for_stellar_variability": true, + "require_apparent_magnitudes": true, + "use_exactly_the_comps_provided": false, "maximum_number_of_ensemble_comparisons_for_transit": 5, "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "photometer_fortuitous_variables": true, @@ -243,6 +246,7 @@ Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` |---|---|---| | Transit fit | Single comparison star (default) | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": false` | | Transit fit | Comparison-star ensemble | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": true`, `"maximum_number_of_ensemble_comparisons_for_transit": 5` | +| Transit or variability run | Exactly the supplied comparison(s) | `"use_exactly_the_comps_provided": true`. Comparisons may be supplied as X/Y or RA/Dec. One supplied comparison is used alone; two or more are all used as one fixed ensemble. Automatic replacement, addition, VSX/stability vetting, ranking, and ensemble-size limiting are bypassed. | | Transit fit | No comparison star | There is no tag that forces this mode. `"require_comp_star": false` only removes the requirement for a comparison star; it does not force target-only photometry. The current comparison-calibration FITS path still selects a single comparison or an ensemble. | | Stellar-variability-only run | Single comparison star | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": false` | | Stellar-variability-only run | Calibrated comparison-star ensemble (default) | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": true`, `"maximum_number_of_ensemble_comparisons_for_stellar_variability": 5` | @@ -250,7 +254,13 @@ Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` The two ensemble limits are independent. `"maximum_number_of_ensemble_comparisons_for_transit"` caps only the transit-fit ensemble. `"maximum_number_of_ensemble_comparisons_for_stellar_variability"` caps both stellar-variability-only and fortuitous-variable ensembles. Each defaults to `5`, must be an integer of at least `2`, and has no configured upper limit. Increase either value to permit a much larger ensemble; EXOTIC will enlarge automatic candidate discovery for the corresponding ensemble where applicable, then use up to that number of surviving comparisons. Very large ensembles require more photometry work. Stellar-variability and fortuitous-variable ensembles can also retain fewer frames because every selected member must have a usable measurement in a retained frame. -Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable photometry has no no-comparison mode. +Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble, except when `"use_exactly_the_comps_provided"` is true. Exact-comparison mode fails explicitly if the supplied reference cannot be measured; it never silently substitutes or drops a supplied comparison. This makes the same reference star or ensemble reproducible across multiple runs. + +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Celestial coordinates require a usable WCS and are projected onto the selected reference image before photometry; after projection they are treated identically to supplied X/Y positions. + +Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. + +For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. `photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. diff --git a/docs/system_prompt.txt b/docs/system_prompt.txt index e79bc8d9..32bccef1 100644 --- a/docs/system_prompt.txt +++ b/docs/system_prompt.txt @@ -897,6 +897,7 @@ Example `inits.json` file: "Target Star X & Y Pixel": "[424, 286]", "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, "Demosaic Output": null diff --git a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py index b33abd9f..e000c14b 100644 --- a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py +++ b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py @@ -549,7 +549,8 @@ def create_inits_file(parameters, file_name): "Observing Notes": parameters.get("Observing Notes", "N/A"), "Plate Solution? (y/n)": parameters.get("Plate Solution? (y/n)", None), "Target Star X & Y Pixel": parameters.get("Target Star X & Y Pixel", None), - "Comparison Star(s) X & Y Pixel": parameters.get("Comparison Star(s) X & Y Pixel", None) + "Comparison Star(s) X & Y Pixel": parameters.get("Comparison Star(s) X & Y Pixel", None), + "Comparison Star(s) RA & Dec": parameters.get("Comparison Star(s) RA & Dec", None) }, "optional_info": { "Pixel Scale (Ex: 5.21 arcsecs/pixel)": parameters.get("Pixel Scale (Ex: 5.21 arcsecs/pixel)", None), @@ -562,6 +563,12 @@ def create_inits_file(parameters, file_name): "maximum_number_of_ensemble_comparisons_for_stellar_variability": parameters.get( "maximum_number_of_ensemble_comparisons_for_stellar_variability", 5 ), + "require_apparent_magnitudes": parameters.get( + "require_apparent_magnitudes", True + ), + "use_exactly_the_comps_provided": parameters.get( + "use_exactly_the_comps_provided", False + ), "require_comp_star": parameters.get("require_comp_star", "y") } } diff --git a/exotic/api/colab.py b/exotic/api/colab.py index cb5b67f6..37ae779b 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -369,6 +369,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Target Star X & Y Pixel": %s, "Comparison Star(s) X & Y Pixel": %s, + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, "Demosaic Output": null @@ -385,6 +386,8 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "use_impactparameter_rather_than_inclination_to_fit": "y", "maximum_number_of_ensemble_comparisons_for_transit": 5, "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, + "require_apparent_magnitudes": true, + "use_exactly_the_comps_provided": false, "use_adaptive_apertures": false, "gain_electrons_per_adu": null, "read_noise_electrons": null, diff --git a/exotic/exotic.py b/exotic/exotic.py index 53d04199..8bd5d2bd 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -163,6 +163,7 @@ format_parameter_with_error, formatted_transit_depth_parameters, save_comp_star_calibration_summary, + write_differential_magnitude_csv, ) except ImportError: # package import from .output_files import ( @@ -175,6 +176,7 @@ format_parameter_with_error, formatted_transit_depth_parameters, save_comp_star_calibration_summary, + write_differential_magnitude_csv, ) try: from transit_depth import fit_transit_depth_summary @@ -187,14 +189,14 @@ try: # plots from plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ plot_prior_posterior_comparison, plot_ktmf_qc_metrics, \ - plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ + plot_stellar_variability, plot_differential_magnitude, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ plot_comp_star_calibration_series, plot_individual_comp_star_calibration_series, \ plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ plot_adaptive_aperture_diagnostics except ImportError: # package import from .plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ plot_prior_posterior_comparison, plot_ktmf_qc_metrics, \ - plot_stellar_variability, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ + plot_stellar_variability, plot_differential_magnitude, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ plot_comp_star_calibration_series, plot_individual_comp_star_calibration_series, \ plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ plot_adaptive_aperture_diagnostics @@ -251,6 +253,8 @@ LIGHTCURVE_MIN_VALID_POINTS = 5 STELLAR_VARIABILITY_ONLY_DEFAULT = False STELLAR_VARIABILITY_ENSEMBLE_DEFAULT = True +REQUIRE_APPARENT_MAGNITUDES_DEFAULT = True +USE_EXACTLY_PROVIDED_COMPARISONS_DEFAULT = False STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS = 2 TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT = 5 STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS = 5 @@ -4073,7 +4077,10 @@ def build_stellar_variability_only_lightcurve( stellar_variability_comp_flux_error=np.asarray(prepared.get('comp_flux_error'), dtype=float)[oot_mask][finite], stellar_variability_exposure_times_seconds=exposure_times_seconds, airmass_fit_skipped=True, - airmass_correction_note="Skipped in stellar-variability-only mode; no transit/systematics model was fit.", + airmass_correction_note=( + "Intentionally not applied to stellar variability; the raw target/reference ratio is " + "preserved so real time-dependent variability is not fitted away." + ), transit_qc={ 'status': 'SKIPPED', 'summary': 'Stellar variability only mode skipped transit fitting.', @@ -5802,7 +5809,13 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, if data_highres is None: data_highres, _ = estimate_transit_duration_samples_from_fit(final_fit, sample_count=1) if data_highres is not None: - plot_final_lightcurve(final_fit, data_highres, p_dict['pName'], candidate_info_dict['save'], observation_date) + plot_final_lightcurve( + final_fit, + data_highres, + p_dict['pName'], + candidate_info_dict['save'], + observation_date, + ) plot_prior_posterior_comparison(final_fit, p_dict, p_dict['pName'], candidate_info_dict['save'], observation_date) plot_ktmf_qc_metrics(final_fit, p_dict['pName'], candidate_info_dict['save'], observation_date) except Exception as exc: @@ -5814,6 +5827,8 @@ def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, output_files = OutputFiles(final_fit, p_dict, candidate_info_dict, duration_samples) try: phase = get_phase(final_fit.time, p_dict['pPer'], final_fit.parameters['tmid']) + output_files.differential_magnitude() + output_files.stellar_variability_differential_magnitude() output_files.final_lightcurve(phase) except Exception as exc: archive_errors.append(archive_exception_payload( @@ -14483,6 +14498,25 @@ def project_ra_dec_to_wcs_pixel(ra, dec, wcs_header): return x_pixel, y_pixel +def project_comparison_radec_to_pixels(comp_stars_radec, wcs_header, image_shape): + """Project supplied celestial comparison coordinates onto a reference image.""" + projected = [] + for index, (ra_deg, dec_deg) in enumerate(comp_stars_radec or [], start=1): + try: + x_pixel, y_pixel = project_ra_dec_to_wcs_pixel(ra_deg, dec_deg, wcs_header) + except Exception as exc: + raise ValueError( + f"comparison star {index} RA/Dec could not be projected by the reference-image WCS" + ) from exc + if not pixel_within_image(x_pixel, y_pixel, image_shape): + raise ValueError( + f"comparison star {index} at RA={ra_deg:.8f}, Dec={dec_deg:.8f} " + "projects outside the reference image" + ) + projected.append([x_pixel, y_pixel]) + return projected + + def _representative_obs_time(obs_times): if obs_times is None: return None @@ -21597,7 +21631,17 @@ def stellar_variability_raw_photometry(lc_fit): def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_label, save, s_name, - observed_filter=None): + observed_filter=None, observation_date=None): + # Differential photometry is independent of catalogue calibration. Emit + # it before validating the comparison magnitude so a failed apparent- + # magnitude attempt can never suppress the raw stellar-variability output. + save_stellar_variability_differential_products( + lc_fit, + save, + s_name, + observation_date=observation_date, + observed_filter=observed_filter, + ) comp_mag = _finite_float(comp_star.get('mag')) comp_mag_error = normalized_magnitude_error(comp_star.get('error')) derived_catalog_reference = bool(comp_star.get('derived_catalog_reference', False)) @@ -21634,17 +21678,13 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ ) fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) - fit_airmass_model = np.asarray( - getattr(lc_fit, 'airmass_model', np.ones_like(fit_data)), - dtype=float, - ) fit_airmass = np.asarray(getattr(lc_fit, 'airmass', np.ones_like(fit_data)), dtype=float) # Public magnitude products are labelled BJD_TDB; ``time`` is the # barycentric series and ``jd_times`` retains the original FITS JD/UTC. fit_times = np.asarray(getattr(lc_fit, 'time', getattr(lc_fit, 'jd_times', [])), dtype=float) transit_model = np.asarray(getattr(lc_fit, 'transit', np.ones_like(fit_data)), dtype=float) - if not (fit_data.shape == fit_airmass_model.shape == fit_airmass.shape == fit_times.shape): + if not (fit_data.shape == fit_airmass.shape == fit_times.shape): raise RuntimeError("Lightcurve arrays have inconsistent shapes for stellar variability output.") target_flux, comp_flux, target_flux_error, comp_flux_error = stellar_variability_raw_photometry(lc_fit) @@ -21661,31 +21701,17 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ comp_flux = comp_flux[mask_ref] target_flux_error = target_flux_error[mask_ref] comp_flux_error = comp_flux_error[mask_ref] - selected_airmass_model = fit_airmass_model[mask_ref] selected_times = fit_times[mask_ref] selected_airmass = fit_airmass[mask_ref] with np.errstate(divide='ignore', invalid='ignore'): raw_ratio = np.divide(target_flux, comp_flux) - valid_airmass_model = ( - np.isfinite(selected_airmass_model) - & (selected_airmass_model > 0) - ) - airmass_reference = ( - float(np.nanmedian(selected_airmass_model[valid_airmass_model])) - if np.any(valid_airmass_model) - else 1.0 - ) - if not np.isfinite(airmass_reference) or airmass_reference <= 0: - airmass_reference = 1.0 - relative_airmass_model = np.divide( - selected_airmass_model, - airmass_reference, - out=np.full(selected_airmass_model.shape, np.nan, dtype=float), - where=valid_airmass_model, - ) + # Do not apply the transit fit's airmass trend to stellar-variability + # measurements. A real stellar change that is correlated with time is + # also correlated with airmass during a normal observing sequence, so the + # detrending model could suppress the astrophysical signal. + calibrated_ratio = raw_ratio with np.errstate(divide='ignore', invalid='ignore'): - calibrated_ratio = np.divide(raw_ratio, relative_airmass_model) target_mag = comp_mag - (2.5 * np.log10(calibrated_ratio)) magnitude_factor = 2.5 / np.log(10.0) explicit_flux_error = magnitude_factor * np.sqrt( @@ -22193,7 +22219,7 @@ def choose_selected_comp_catalog_reference_candidate(candidates, observed_filter def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vsp_ind, best_comp, save, s_name, observed_filter=None, comp_ra_dec=None, field_catalog=None, reference_image=None, - wcs_file=None, + wcs_file=None, observation_date=None, catalog_match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): try: if best_comp is None: @@ -22310,6 +22336,7 @@ def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vs save, s_name, observed_filter=observed_filter, + observation_date=observation_date, ) except KeyError as e: log_info(f"Key error in processing stellar variability: {e}", warn=True) @@ -25848,6 +25875,87 @@ def build_absolute_comp_ensemble_uncertainty(comp_flux_map, comp_error_map, memb return ensemble_unc +def build_relative_comparison_ensemble_series(target_flux, target_flux_error, + comp_flux_map, comp_error_map, member_keys, + validity_mask_func=valid_comparison_frame_mask): + """Combine exactly the requested reference members without catalogue magnitudes.""" + requested_keys = [key for key in member_keys or [] if key] + ensemble_flux, used_keys = build_absolute_comp_ensemble_flux( + comp_flux_map, + requested_keys, + validity_mask_func=validity_mask_func, + ) + if ensemble_flux is None or used_keys != requested_keys: + return { + 'applied': False, + 'failure_reason': 'not every requested comparison had a usable flux series', + 'member_keys': used_keys, + } + ensemble_error = build_absolute_comp_ensemble_uncertainty( + comp_flux_map, + comp_error_map, + used_keys, + validity_mask_func=validity_mask_func, + ) + + target_flux = np.asarray(target_flux, dtype=float) + if target_flux_error is None: + target_flux_error = source_flux_uncertainty_from_counts(target_flux) + target_flux_error = np.asarray(target_flux_error, dtype=float) + if target_flux_error.shape != target_flux.shape: + target_flux_error = source_flux_uncertainty_from_counts(target_flux) + if ensemble_error is None or np.asarray(ensemble_error).shape != ensemble_flux.shape: + ensemble_error = source_flux_uncertainty_from_counts(ensemble_flux) + ensemble_error = np.asarray(ensemble_error, dtype=float) + + all_members_valid = np.ones(target_flux.shape, dtype=bool) + for key in requested_keys: + member_flux = np.asarray(comp_flux_map[key], dtype=float) + if member_flux.shape != target_flux.shape: + return { + 'applied': False, + 'failure_reason': f'{key} had a mismatched flux-series shape', + 'member_keys': used_keys, + } + all_members_valid &= validity_mask_func(member_flux) + + valid = ( + all_members_valid + & np.isfinite(target_flux) + & (target_flux > 0) + & np.isfinite(ensemble_flux) + & (ensemble_flux > 0) + & np.isfinite(target_flux_error) + & (target_flux_error >= 0) + & np.isfinite(ensemble_error) + & (ensemble_error >= 0) + ) + relative_flux = np.full(target_flux.shape, np.nan, dtype=float) + relative_flux_error = np.full(target_flux.shape, np.nan, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + relative_flux[valid] = target_flux[valid] / ensemble_flux[valid] + relative_flux_error[valid] = relative_flux[valid] * np.sqrt( + (target_flux_error[valid] / target_flux[valid]) ** 2 + + (ensemble_error[valid] / ensemble_flux[valid]) ** 2 + ) + valid &= np.isfinite(relative_flux) & (relative_flux > 0) + valid &= np.isfinite(relative_flux_error) & (relative_flux_error > 0) + return { + 'applied': bool(np.count_nonzero(valid) >= LIGHTCURVE_MIN_VALID_POINTS), + 'failure_reason': ( + None + if np.count_nonzero(valid) >= LIGHTCURVE_MIN_VALID_POINTS + else 'fewer than five frames contained every requested comparison member' + ), + 'member_keys': used_keys, + 'reference_flux': ensemble_flux, + 'reference_flux_error': ensemble_error, + 'relative_flux': relative_flux, + 'relative_flux_error': relative_flux_error, + 'valid_mask': valid, + } + + def comparison_star_coverage_summary(comp_flux_map, min_fraction=COMPARISON_STAR_MIN_COVERAGE_FRACTION, min_points=COMPARISON_STAR_MIN_VALID_FRAMES, @@ -26227,7 +26335,8 @@ def comparison_star_candidate_frame_outlier_summary( def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_rejection=False, - validity_mask_func=valid_comparison_frame_mask): + validity_mask_func=valid_comparison_frame_mask, + bypass_vetting=False): if not comp_flux_map: return { 'pairwise_matrix': np.empty((0, 0), dtype=float), @@ -26255,7 +26364,7 @@ def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_ } coverage_summary = comparison_star_coverage_summary( comp_flux_map, - skip_rejection=skip_low_coverage_rejection, + skip_rejection=(skip_low_coverage_rejection or bypass_vetting), validity_mask_func=validity_mask_func, ) coverage_qualified_keys = [ @@ -26353,10 +26462,10 @@ def build_stability_iteration(active_keys, frame_keep_mask=None): return pairwise_matrix, comp_summaries - active_keys = list(coverage_qualified_keys) + active_keys = list(comp_keys if bypass_vetting else coverage_qualified_keys) rejected_outlier_keys = set() rejection_metadata = {} - for _ in range(COMPARISON_STAR_SUITABILITY_MAX_ITERS): + for _ in range(0 if bypass_vetting else COMPARISON_STAR_SUITABILITY_MAX_ITERS): _, iteration_summaries = build_stability_iteration(active_keys) active_indices = [comp_keys.index(key) for key in active_keys] outlier_summary = apply_comparison_star_suitability_outlier_rejection( @@ -26383,10 +26492,22 @@ def build_stability_iteration(active_keys, frame_keep_mask=None): rejected_outlier_keys.update(newly_rejected_keys) active_keys = [key for key in active_keys if key not in rejected_outlier_keys] - image_outlier_summary = comparison_star_image_outlier_summary( - normalized_flux_map, - active_keys, - ) + if bypass_vetting: + series_length = np.asarray(next(iter(normalized_flux_map.values()))).shape[0] + image_outlier_summary = { + 'frame_keep_mask': np.ones(series_length, dtype=bool), + 'rejected_frame_count': 0, + 'sigma': COMPARISON_IMAGE_OUTLIER_SIGMA, + 'required_valid_pair_count': 0, + 'available_pair_count': 0, + 'valid_pair_counts': np.zeros(series_length, dtype=int), + 'outlier_pair_counts': np.zeros(series_length, dtype=int), + } + else: + image_outlier_summary = comparison_star_image_outlier_summary( + normalized_flux_map, + active_keys, + ) field_image_keep_mask = image_outlier_summary['frame_keep_mask'] pairwise_matrix, comp_summaries = build_stability_iteration( active_keys, @@ -26424,12 +26545,24 @@ def build_stability_iteration(active_keys, frame_keep_mask=None): summary['suitability_scatter'] = final_scatter summary['suitability_high_threshold'] = final_high_threshold - candidate_frame_summary = comparison_star_candidate_frame_outlier_summary( - normalized_flux_map, - summary['key'], - active_keys, - field_image_keep_mask=field_image_keep_mask, - ) + if bypass_vetting: + candidate_frame_summary = { + 'frame_keep_mask': np.ones(field_image_keep_mask.shape, dtype=bool), + 'rejected_frame_indices': [], + 'rejected_frame_count': 0, + 'valid_pair_counts': np.zeros(field_image_keep_mask.shape, dtype=int), + 'outlier_pair_counts': np.zeros(field_image_keep_mask.shape, dtype=int), + 'required_valid_pair_count': 0, + 'available_pair_count': 0, + 'sigma': COMPARISON_IMAGE_OUTLIER_SIGMA, + } + else: + candidate_frame_summary = comparison_star_candidate_frame_outlier_summary( + normalized_flux_map, + summary['key'], + active_keys, + field_image_keep_mask=field_image_keep_mask, + ) summary['ensemble_frame_keep_mask'] = candidate_frame_summary['frame_keep_mask'] summary['ensemble_frame_rejected_indices'] = candidate_frame_summary['rejected_frame_indices'] summary['ensemble_frame_rejected_count'] = candidate_frame_summary['rejected_frame_count'] @@ -27196,7 +27329,8 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, use_psf_photometry=True, use_aperture_photometry=True, psf_flux_data=None, - comp_overexposed_masks=None): + comp_overexposed_masks=None, + use_exactly_the_comps_provided=False): candidate_summaries = [] comp_star_count = len(comp_stars) psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) @@ -27250,6 +27384,7 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, airmass, skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, validity_mask_func=robust_flux_floor_mask, + bypass_vetting=use_exactly_the_comps_provided, ) psf_summary.update({ 'method': 'psf', @@ -27274,6 +27409,7 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, comp_flux_map, airmass, skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, + bypass_vetting=use_exactly_the_comps_provided, ) candidate_summary.update({ 'method': 'aperture', @@ -27288,11 +27424,21 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, candidate for candidate in candidate_summaries if np.isfinite(candidate['field_score']) and candidate['best_comp_index'] is not None ] - if not finite_candidates: + if finite_candidates: + finite_candidates.sort(key=comparison_field_sort_key) + best_candidate = finite_candidates[0] + elif use_exactly_the_comps_provided and candidate_summaries: + # Stability is diagnostic-only in exact mode. Keep a photometry + # method available even when the supplied stars have no finite field + # score; the later light-curve construction will either measure every + # requested star or fail explicitly without substitution. + best_candidate = candidate_summaries[0] + if best_candidate.get('best_comp_index') is None: + best_candidate['best_comp_index'] = 0 + best_candidate['best_comp_score'] = np.inf + else: return None - finite_candidates.sort(key=comparison_field_sort_key) - best_candidate = finite_candidates[0] best_comp_index = best_candidate['best_comp_index'] method_label = comparison_method_label(best_candidate) comp_summaries = [] @@ -27323,27 +27469,30 @@ def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, return best_candidate -def ranked_comparison_calibration_summaries(comparison_calibration): +def ranked_comparison_calibration_summaries(comparison_calibration, include_unvetted=False): if comparison_calibration is None: return [] ranked_summaries = [] for summary in comparison_calibration.get('comp_summaries', []): aggregate_score = summary.get('aggregate_score', np.inf) - if summary.get('coverage_rejected'): + if not include_unvetted and summary.get('coverage_rejected'): continue - if summary.get('suitability_outlier_rejected'): + if not include_unvetted and summary.get('suitability_outlier_rejected'): continue - if not np.isfinite(aggregate_score): + if not include_unvetted and not np.isfinite(aggregate_score): continue ranked_summaries.append(summary) - ranked_summaries.sort( - key=lambda summary: ( - summary.get('aggregate_score', np.inf), - summary.get('comp_index', np.inf), + if include_unvetted: + ranked_summaries.sort(key=lambda summary: summary.get('comp_index', np.inf)) + else: + ranked_summaries.sort( + key=lambda summary: ( + summary.get('aggregate_score', np.inf), + summary.get('comp_index', np.inf), + ) ) - ) return ranked_summaries @@ -28147,6 +28296,52 @@ def save_stellar_variability_magnitude_csv(vsp_params, save, target_name, observ return output_path +def save_stellar_variability_differential_products( + lc_fit, + save, + target_name, + observation_date=None, + observed_filter=None): + """Save raw target/reference variability products without airmass detrending.""" + csv_path = None + try: + csv_path = write_differential_magnitude_csv( + lc_fit, + save, + target_name, + observation_date=observation_date, + observed_filter=observed_filter, + out_of_transit_only=True, + apply_airmass_correction=False, + filename_prefix='StellarVariabilityDifferentialMagnitude', + ) + except Exception as exc: + log_info( + "Warning: could not save the stellar-variability differential-magnitude CSV " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + try: + plot_differential_magnitude( + lc_fit, + target_name, + save, + observation_date or 'undated', + observed_filter=observed_filter, + out_of_transit_only=True, + apply_airmass_correction=False, + filename_prefix='StellarVariabilityDifferentialMagnitude', + save_stellar_variability_alias=True, + ) + except Exception as exc: + log_info( + "Warning: could not save the stellar-variability differential-magnitude plot " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + return csv_path + + def build_stellar_variability_ensemble_params_from_fit( lc_fit, save, @@ -28154,6 +28349,13 @@ def build_stellar_variability_ensemble_params_from_fit( observed_filter=None, observation_date=None, target_metadata=None): + save_stellar_variability_differential_products( + lc_fit, + save, + s_name, + observation_date=observation_date, + observed_filter=observed_filter, + ) magnitudes = np.asarray( getattr(lc_fit, 'stellar_variability_ensemble_magnitudes', []), dtype=float, @@ -28282,6 +28484,22 @@ def build_stellar_variability_ensemble_params_from_fit( return vsp_params +def should_require_apparent_magnitudes(config_value): + return parse_bool_config_value( + config_value, + REQUIRE_APPARENT_MAGNITUDES_DEFAULT, + 'require_apparent_magnitudes', + ) + + +def should_use_exactly_the_comps_provided(config_value): + return parse_bool_config_value( + config_value, + USE_EXACTLY_PROVIDED_COMPARISONS_DEFAULT, + 'use_exactly_the_comps_provided', + ) + + def build_stellar_variability_params_from_photometry_selection( selection, calibration_stars, @@ -28332,6 +28550,7 @@ def build_stellar_variability_params_from_photometry_selection( save, s_name, observed_filter=observed_filter, + observation_date=observation_date, ) @@ -28351,8 +28570,13 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, calibration_stars=None, observed_filter=None, - target_catalog_match=None): - ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) + target_catalog_match=None, + require_apparent_magnitudes=True, + use_exactly_the_comps_provided=False): + ranked_summaries = ranked_comparison_calibration_summaries( + comparison_calibration, + include_unvetted=use_exactly_the_comps_provided, + ) if not ranked_summaries: return { 'ranked_summaries': [], @@ -28479,16 +28703,112 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, candidate_target_flux = target_flux candidate_target_flux_error = target_flux_error - member_selection = select_stellar_variability_ensemble_members( - ranked_summaries, - calibration_stars, - comp_flux_map, - observed_filter=observed_filter, - target_catalog_match=target_catalog_match, - max_members=maximum_number_of_ensemble_comparisons_for_stellar_variability, - times=times, + fixed_reference_mode = ( + use_exactly_the_comps_provided or not require_apparent_magnitudes ) - ensemble_members = member_selection['members'] + prebuilt_ensemble_series = None + if fixed_reference_mode: + if use_exactly_the_comps_provided: + selected_summaries = list(ranked_summaries) + else: + selected_summaries = list(ranked_summaries)[ + :parse_maximum_number_of_ensemble_comparisons_for_stellar_variability( + maximum_number_of_ensemble_comparisons_for_stellar_variability + ) + ] + ensemble_members = [] + calibrated_members = [] + for summary in selected_summaries: + member = { + 'key': summary.get('key'), + 'comp_index': summary.get('comp_index'), + 'label': summary.get('label', summary.get('key')), + 'position': summary.get('position'), + 'summary': summary, + } + ensemble_members.append(member) + calibration = stellar_variability_calibration_for_position( + calibration_stars, + summary.get('position'), + observed_filter=observed_filter, + ) + if calibration is not None: + calibrated_members.append({**member, **calibration}) + + requested_keys = [member['key'] for member in ensemble_members if member.get('key')] + if len(calibrated_members) == len(ensemble_members) and ensemble_members: + prebuilt_ensemble_series = build_stellar_variability_calibrated_ensemble_series( + candidate_target_flux, + candidate_target_flux_error, + comp_flux_map, + comp_error_map, + calibrated_members, + minimum_members=len(calibrated_members), + ) + if prebuilt_ensemble_series.get('applied'): + ensemble_members = calibrated_members + if not prebuilt_ensemble_series or not prebuilt_ensemble_series.get('applied'): + relative_series = build_relative_comparison_ensemble_series( + candidate_target_flux, + candidate_target_flux_error, + comp_flux_map, + comp_error_map, + requested_keys, + validity_mask_func=( + robust_flux_floor_mask if method == 'psf' else valid_comparison_frame_mask + ), + ) + if relative_series.get('applied'): + valid_member_count = np.where( + relative_series['valid_mask'], + len(requested_keys), + 0, + ) + prebuilt_ensemble_series = { + 'applied': True, + 'failure_reason': None, + 'magnitude': np.full(candidate_target_flux.shape, np.nan, dtype=float), + 'magnitude_error': np.full(candidate_target_flux.shape, np.nan, dtype=float), + 'relative_flux': relative_series['relative_flux'], + 'relative_flux_error': relative_series['relative_flux_error'], + 'synthetic_reference_flux': relative_series['reference_flux'], + 'synthetic_reference_flux_error': relative_series['reference_flux_error'], + 'valid_member_count': valid_member_count, + 'baseline_magnitude': np.nan, + } + else: + prebuilt_ensemble_series = relative_series + member_selection = { + 'members': ensemble_members, + 'rejected': [], + 'calibration_error_clip': {}, + 'target_catalog_profile': {}, + 'prelimit_member_count': len(ensemble_members), + 'member_limit': len(ensemble_members), + 'gap_stability': { + 'applied': False, + 'reason': 'disabled for a fixed, unvetted comparison reference', + 'boundaries': [], + 'candidates': {}, + }, + 'fixed_reference': True, + 'apparent_calibration_available': bool( + prebuilt_ensemble_series + and prebuilt_ensemble_series.get('applied') + and np.any(np.isfinite(prebuilt_ensemble_series.get('magnitude', []))) + ), + } + else: + member_selection = select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter=observed_filter, + target_catalog_match=target_catalog_match, + max_members=maximum_number_of_ensemble_comparisons_for_stellar_variability, + times=times, + ) + ensemble_members = member_selection['members'] clip_summary = member_selection['calibration_error_clip'] for rejected_member in member_selection['rejected']: log_info( @@ -28499,7 +28819,11 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, if len(ensemble_members) >= STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: member_text = ", ".join( - f"{member['label']} (catalog sigma={member['magnitude_error']:.4f} mag)" + ( + f"{member['label']} (catalog sigma={member['magnitude_error']:.4f} mag)" + if member.get('magnitude_error') is not None + else f"{member['label']} (differential only)" + ) for member in ensemble_members ) threshold = clip_summary.get('high_threshold', np.nan) @@ -28509,7 +28833,7 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, else "" ) log_info( - "Stellar-variability-only calibrated ensemble members: " + "Stellar-variability-only comparison ensemble members: " f"{member_text}{threshold_text}." ) if member_selection.get('prelimit_member_count', 0) > len(ensemble_members): @@ -28519,7 +28843,7 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, f"of {len(ensemble_members)} closest " "to the target in catalog color and magnitude." ) - ensemble_series = build_stellar_variability_calibrated_ensemble_series( + ensemble_series = prebuilt_ensemble_series or build_stellar_variability_calibrated_ensemble_series( candidate_target_flux, candidate_target_flux_error, comp_flux_map, @@ -28536,12 +28860,12 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, & (ensemble_series['relative_flux_error'] > 0) ) availability_diagnostic = build_time_rejection_diagnostic( - "Stellar-variability calibrated ensemble availability filter", + "Stellar-variability comparison ensemble availability filter", times, fit_mask, note=( - "Kept frames with a finite target measurement and every selected unsaturated, " - "VSX-vetted, catalog-calibrated ensemble member." + "Kept frames with a finite target measurement and every selected comparison " + "ensemble member." ), ) filter_diagnostics = [] @@ -28623,7 +28947,7 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, ensemble_summary = { 'comp_index': None, 'key': 'ensemble', - 'label': f"Calibrated comparison ensemble ({len(ensemble_members)} comps)", + 'label': f"Comparison ensemble ({len(ensemble_members)} comps)", 'position': None, 'aggregate_score': comparison_calibration.get('field_score', np.inf), 'coverage_count': int(np.count_nonzero(fit_mask)), @@ -28679,10 +29003,10 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, 'eebls_snr': np.nan, 'transit_delta_bic': np.nan, 'residual_scatter': scatter, - 'target_model_scatter_basis': 'out-of-transit calibrated ensemble scatter', + 'target_model_scatter_basis': 'out-of-transit comparison ensemble scatter', 'projected_full_residual_scatter': scatter, 'selection_scatter': scatter, - 'selection_scatter_basis': 'out-of-transit calibrated ensemble scatter', + 'selection_scatter_basis': 'out-of-transit comparison ensemble scatter', 'target_comp_scatter': target_comp_flux_scatter( fit_result.stellar_variability_target_flux, fit_result.stellar_variability_comp_flux, @@ -28697,12 +29021,18 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, 'rejected_by_transit_qc': False, 'selected': True, 'selection_reason': ( - 'selected: default calibrated ensemble of bright, unsaturated, VSX-vetted ' - 'comparison stars after high-side catalog-error clipping' + 'selected: exact supplied comparison ensemble without star vetting' + if use_exactly_the_comps_provided + else ( + 'selected: fixed differential comparison ensemble' + if fixed_reference_mode + else 'selected: default calibrated ensemble of bright, unsaturated, ' + 'VSX-vetted comparison stars after high-side catalog-error clipping' + ) ), 'full_reduction_applied': True, 'full_reduction_note': ( - 'completed the stellar-variability-only calibrated ensemble reduction ' + 'completed the stellar-variability-only comparison ensemble reduction ' 'without fitting a transit model.' ), 'stellar_variability_transit_exclusion': exclusion_summary, @@ -28713,16 +29043,44 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, 'ranked_summaries': ranked_summaries, 'attempts': attempts, 'selected_result': selected_result, - 'selection_metric': 'stellar_variability_ensemble', + 'selection_metric': ( + 'exact_stellar_variability_ensemble' + if use_exactly_the_comps_provided + else 'stellar_variability_ensemble' + ), 'stopped_after_first_qc_pass': False, 'stopped_after_promising_partial': False, } + if use_exactly_the_comps_provided: + log_info( + "Error: the exact supplied stellar-variability ensemble did not yield a usable " + "light curve; exact-comparison mode will not drop a member or select another star.", + error=True, + ) + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': None, + 'selection_metric': 'exact_stellar_variability_ensemble', + } log_info( - "Warning: the default stellar-variability calibrated ensemble did not yield a usable " + "Warning: the stellar-variability comparison ensemble did not yield a usable " "out-of-transit light curve; falling back to single-comparison selection.", warn=True, ) else: + if use_exactly_the_comps_provided: + log_info( + "Error: the exact supplied comparison ensemble contained fewer than two usable " + "members; exact-comparison mode will not fall back to one star.", + error=True, + ) + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': None, + 'selection_metric': 'exact_stellar_variability_ensemble', + } log_info( "Warning: fewer than two bright, unsaturated, VSX-vetted comparison stars had usable " "catalog calibrations after calibration-error clipping; falling back to single-comparison " @@ -29214,6 +29572,7 @@ def clear_previous_fortuitous_variable_products(variable_dir): for prefix in ( 'AID_AAVSO_', 'StellarVariability_', + 'DifferentialMagnitude_', 'EnsembleSelection_', 'FortuitousVariableStatus_', ): @@ -29223,6 +29582,12 @@ def clear_previous_fortuitous_variable_products(variable_dir): plot_path = output_dir / 'working_artifacts' / 'Stellar_Variability.png' if plot_path.is_file(): plot_path.unlink() + for differential_plot in ( + output_dir / 'Stellar_Variability_DifferentialMagnitude.png', + output_dir / 'working_artifacts' / 'Stellar_Variability_DifferentialMagnitude.png', + ): + if differential_plot.is_file(): + differential_plot.unlink() working_artifacts_dir = output_dir / 'working_artifacts' if working_artifacts_dir.is_dir(): try: @@ -29375,21 +29740,103 @@ def process_fortuitous_variables( f"significance={rejected_candidate.get('maximum_step_significance'):.2f} sigma." ) members = member_selection.get('members', []) - if len(members) < required_comparison_members: - raise ValueError( - f'fewer than {required_comparison_members} independently calibrated ' - 'comparison member(s)' + apparent_reference_available = len(members) >= required_comparison_members + if not apparent_reference_available: + # A catalogue magnitude is optional for differential + # photometry. Retain the normal vetted/ranked candidate order + # and build the requested single or ensemble reference from + # instrumental flux alone. + raw_members = [] + member_limit = ( + 1 + if use_single_comparison + else parse_maximum_number_of_ensemble_comparisons_for_stellar_variability( + maximum_number_of_ensemble_comparisons_for_stellar_variability + ) ) + for summary in ranked_summaries: + key = summary.get('key') + flux = np.asarray(comp_flux_map.get(key, []), dtype=float) + if key not in comp_flux_map or np.count_nonzero( + np.isfinite(flux) & (flux > 0) + ) < LIGHTCURVE_MIN_VALID_POINTS: + continue + raw_members.append({ + 'key': key, + 'comp_index': summary.get('comp_index'), + 'label': summary.get('label', key), + 'position': summary.get('position'), + 'summary': summary, + }) + if len(raw_members) >= member_limit: + break + members = raw_members + if len(members) < required_comparison_members: + raise ValueError( + f'fewer than {required_comparison_members} usable differential ' + 'comparison member(s)' + ) + member_selection = { + **member_selection, + 'members': members, + 'fixed_reference': True, + 'apparent_calibration_available': False, + } output_magnitude_error_limit = fortuitous_output_magnitude_error_limit(members) variable['output_magnitude_error_limit'] = output_magnitude_error_limit - ensemble_series = build_stellar_variability_calibrated_ensemble_series( - target_flux, - target_flux_error, - comp_flux_map, - comp_error_map, - members, - minimum_members=required_comparison_members, - ) + if apparent_reference_available: + ensemble_series = build_stellar_variability_calibrated_ensemble_series( + target_flux, + target_flux_error, + comp_flux_map, + comp_error_map, + members, + minimum_members=required_comparison_members, + ) + else: + relative_series = build_relative_comparison_ensemble_series( + target_flux, + target_flux_error, + comp_flux_map, + comp_error_map, + [member.get('key') for member in members], + ) + relative_flux = np.asarray( + relative_series.get('relative_flux', np.full(target_flux.shape, np.nan)), + dtype=float, + ) + relative_flux_error = np.asarray( + relative_series.get( + 'relative_flux_error', + np.full(target_flux.shape, np.nan), + ), + dtype=float, + ) + with np.errstate(divide='ignore', invalid='ignore'): + instrumental_magnitude_error = ( + (2.5 / np.log(10.0)) + * np.abs(relative_flux_error / relative_flux) + ) + ensemble_series = { + 'applied': relative_series.get('applied', False), + 'failure_reason': relative_series.get('failure_reason'), + 'relative_flux': relative_flux, + 'relative_flux_error': relative_flux_error, + 'synthetic_reference_flux': relative_series.get('reference_flux'), + 'synthetic_reference_flux_error': relative_series.get( + 'reference_flux_error' + ), + 'magnitude': np.full(target_flux.shape, np.nan, dtype=float), + 'magnitude_error': instrumental_magnitude_error, + 'valid_member_count': np.where( + relative_series.get( + 'valid_mask', + np.zeros(target_flux.shape, dtype=bool), + ), + len(members), + 0, + ), + } if not ensemble_series.get('applied'): raise ValueError(ensemble_series.get('failure_reason') or 'ensemble combination failed') @@ -29501,73 +29948,110 @@ def process_fortuitous_variables( if fit is None: raise ValueError('stellar-variability light curve construction failed') variable_dir.mkdir(parents=True, exist_ok=True) - if use_single_comparison: - vsp_params = build_stellar_variability_params_from_fit( - fit, - selected_member.get('star', {}), - selected_member.get('position'), - reference_label, - variable_dir, - variable_name, - observed_filter=observed_filter, - ) - else: - selected_indices = np.asarray(fit.stellar_variability_source_indices, dtype=int) - fit.stellar_variability_ensemble_members = members - fit.stellar_variability_ensemble_magnitudes = ensemble_series['magnitude'][selected_indices] - fit.stellar_variability_ensemble_magnitude_errors = ( - ensemble_series['magnitude_error'][selected_indices] - ) - fit.stellar_variability_ensemble_valid_member_counts = ( - ensemble_series['valid_member_count'][selected_indices] - ) - fit.stellar_variability_ensemble_calibration_error_clip = member_selection.get( - 'calibration_error_clip', {} - ) - fit.stellar_variability_ensemble_selection = member_selection - fit.stellar_variability_target_catalog_profile = member_selection.get( - 'target_catalog_profile', {} - ) - - target_metadata = fortuitous_variable_target_metadata(variable) - vsp_params = build_stellar_variability_ensemble_params_from_fit( - fit, - variable_dir, - variable_name, - observed_filter=observed_filter, - observation_date=info_dict.get('date'), - target_metadata=target_metadata, - ) - if not vsp_params: - raise ValueError('no calibrated magnitude rows were produced') - csv_path = save_stellar_variability_magnitude_csv( - vsp_params, + differential_csv_path = write_differential_magnitude_csv( + fit, variable_dir, variable_name, observation_date=info_dict.get('date'), + observed_filter=observed_filter, ) + plot_differential_magnitude( + fit, + variable_name, + variable_dir, + info_dict.get('date'), + observed_filter=observed_filter, + ) + + apparent_output_error = None + vsp_params = [] + csv_path = None + try: + if use_single_comparison: + vsp_params = build_stellar_variability_params_from_fit( + fit, + selected_member.get('star', {}), + selected_member.get('position'), + reference_label, + variable_dir, + variable_name, + observed_filter=observed_filter, + observation_date=info_dict.get('date'), + ) + else: + selected_indices = np.asarray(fit.stellar_variability_source_indices, dtype=int) + fit.stellar_variability_ensemble_members = members + fit.stellar_variability_ensemble_magnitudes = ensemble_series['magnitude'][selected_indices] + fit.stellar_variability_ensemble_magnitude_errors = ( + ensemble_series['magnitude_error'][selected_indices] + ) + fit.stellar_variability_ensemble_valid_member_counts = ( + ensemble_series['valid_member_count'][selected_indices] + ) + fit.stellar_variability_ensemble_calibration_error_clip = member_selection.get( + 'calibration_error_clip', {} + ) + fit.stellar_variability_ensemble_selection = member_selection + fit.stellar_variability_target_catalog_profile = member_selection.get( + 'target_catalog_profile', {} + ) + + target_metadata = fortuitous_variable_target_metadata(variable) + vsp_params = build_stellar_variability_ensemble_params_from_fit( + fit, + variable_dir, + variable_name, + observed_filter=observed_filter, + observation_date=info_dict.get('date'), + target_metadata=target_metadata, + ) + if vsp_params: + csv_path = save_stellar_variability_magnitude_csv( + vsp_params, + variable_dir, + variable_name, + observation_date=info_dict.get('date'), + ) + except Exception as exc: + apparent_output_error = str(exc) + vsp_params = [] + log_info( + f"Warning: apparent-magnitude output was unavailable for {variable_name} " + f"({exc}); differential-magnitude products were retained.", + warn=True, + ) variable_info = dict(info_dict) variable_info['save'] = str(variable_dir) variable_planet = {'sName': variable_name, 'pName': variable_name} - AIDOutputFiles( - fit, - variable_planet, - variable_info, - variable.get('auid'), - None, - vsp_params, - ).aavso() - aid_variable_name = variable.get('auid') or variable_name - combined_vsp_params.extend([ - {**vsp_param, '_aid_name': aid_variable_name} - for vsp_param in vsp_params - ]) + if vsp_params: + try: + AIDOutputFiles( + fit, + variable_planet, + variable_info, + variable.get('auid'), + None, + vsp_params, + ).aavso() + aid_variable_name = variable.get('auid') or variable_name + combined_vsp_params.extend([ + {**vsp_param, '_aid_name': aid_variable_name} + for vsp_param in vsp_params + ]) + except Exception as exc: + apparent_output_error = str(exc) + log_info( + f"Warning: AID apparent-magnitude output failed for {variable_name} " + f"({exc}); differential-magnitude products were retained.", + warn=True, + ) results.append({ 'name': variable_name, 'auid': variable.get('auid'), 'category': category, 'output_directory': str(variable_dir), - 'point_count': len(vsp_params), + 'point_count': int(len(fit.time)), + 'apparent_magnitude_point_count': len(vsp_params), 'reference_mode': reference_mode, 'comparison_label': reference_label, 'selected_comparison_gap_stability': ( @@ -29595,11 +30079,16 @@ def process_fortuitous_variables( ), 'output_magnitude_error_max': variable.get('output_magnitude_error_max'), 'magnitude_csv': str(csv_path) if csv_path else None, + 'differential_magnitude_csv': ( + str(differential_csv_path) if differential_csv_path else None + ), + 'apparent_magnitude_error': apparent_output_error, 'status': 'completed', }) log_info( f"Fortuitous-variable photometry completed for {variable_name}: " - f"{len(vsp_params)} point(s), reference={reference_label} " + f"{len(fit.time)} differential point(s), {len(vsp_params)} apparent point(s), " + f"reference={reference_label} " f"({reference_mode}), outputs={variable_dir}." ) except Exception as exc: @@ -29726,8 +30215,12 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p maximum_number_of_ensemble_comparisons_for_transit= TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT, exposure_times_seconds=None, - gain_e_per_adu=None): - ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) + gain_e_per_adu=None, + use_exactly_the_comps_provided=False): + ranked_summaries = ranked_comparison_calibration_summaries( + comparison_calibration, + include_unvetted=use_exactly_the_comps_provided, + ) if not ranked_summaries: return { 'ranked_summaries': [], @@ -29793,13 +30286,17 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p preflight_plans = [] if use_ensemble_photometry_rather_than_single_comp: - ensemble_limit = parse_maximum_number_of_ensemble_comparisons_for_transit( - maximum_number_of_ensemble_comparisons_for_transit - ) - active_keys = limited_ensemble_comparison_keys( - ranked_summaries, - ensemble_limit, - ) + if use_exactly_the_comps_provided: + active_keys = [summary.get('key') for summary in ranked_summaries if summary.get('key')] + ensemble_limit = len(active_keys) + else: + ensemble_limit = parse_maximum_number_of_ensemble_comparisons_for_transit( + maximum_number_of_ensemble_comparisons_for_transit + ) + active_keys = limited_ensemble_comparison_keys( + ranked_summaries, + ensemble_limit, + ) if method == 'psf': comp_flux_map = { summary['key']: psf_flux_series_from_rows( @@ -29906,7 +30403,11 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p candidate_target_flux = target_flux candidate_target_flux_error = target_flux_error - if ensemble_flux is not None and member_keys: + exact_members_complete = ( + not use_exactly_the_comps_provided + or member_keys == active_keys + ) + if ensemble_flux is not None and member_keys and exact_members_complete: candidate_frame_keep_mask = np.ones(times.shape[0], dtype=bool) for summary in ranked_summaries: if summary.get('key') not in set(member_keys): @@ -29987,11 +30488,33 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p f"(configured maximum={ensemble_limit})." ) else: - log_info( - "Warning: ensemble comparison photometry was enabled, but no usable non-rejected " - "comparison-star ensemble could be built; falling back to ranked single-comp fits.", - warn=True, - ) + if use_exactly_the_comps_provided: + log_info( + "Error: the fixed comparison ensemble could not be built with every supplied " + "comparison star; exact-comparison mode will not drop a member or fall back to " + "a different reference.", + error=True, + ) + else: + log_info( + "Warning: ensemble comparison photometry was enabled, but no usable non-rejected " + "comparison-star ensemble could be built; falling back to ranked single-comp fits.", + warn=True, + ) + + if ( + use_exactly_the_comps_provided + and use_ensemble_photometry_rather_than_single_comp + and not preflight_plans + ): + return { + 'ranked_summaries': ranked_summaries, + 'attempts': [], + 'selected_result': None, + 'selection_metric': 'exact_comparison_ensemble', + 'stopped_after_first_qc_pass': False, + 'stopped_after_promising_partial': False, + } if not preflight_plans: ranked_summaries_to_fit = ranked_summaries @@ -30639,6 +31162,19 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p if final_output_dir is not None: selected_result['final_output_dir'] = str(final_output_dir) + if use_exactly_the_comps_provided: + selection_metric = ( + 'exact_comparison_ensemble' + if use_ensemble_photometry_rather_than_single_comp + else 'exact_single_comparison' + ) + if selected_result is not None: + selected_result['selection_reason'] = ( + 'selected: used every supplied comparison as one fixed ensemble without vetting' + if use_ensemble_photometry_rather_than_single_comp + else 'selected: used the one supplied comparison without vetting or alternatives' + ) + for attempt in attempts: if selected_result is not None and attempt is selected_result: continue @@ -30914,6 +31450,18 @@ def _main_impl(): stellar_variability_only = should_run_stellar_variability_only( exotic_infoDict.get('stellar_variability_only', STELLAR_VARIABILITY_ONLY_DEFAULT) ) + require_apparent_magnitudes = should_require_apparent_magnitudes( + exotic_infoDict.get( + 'require_apparent_magnitudes', + REQUIRE_APPARENT_MAGNITUDES_DEFAULT, + ) + ) + use_exactly_the_comps_provided = should_use_exactly_the_comps_provided( + exotic_infoDict.get( + 'use_exactly_the_comps_provided', + USE_EXACTLY_PROVIDED_COMPARISONS_DEFAULT, + ) + ) use_ensemble_photometry_for_stellar_variability = ( should_use_ensemble_photometry_for_stellar_variability( exotic_infoDict.get( @@ -30943,6 +31491,29 @@ def _main_impl(): ) ) ) + provided_comparison_radec = [ + list(coords) for coords in (exotic_infoDict.get('comp_stars_radec') or []) + ] + comparisons_supplied_as_radec = bool(provided_comparison_radec) + provided_comparison_count = ( + len(provided_comparison_radec) + if comparisons_supplied_as_radec + else len(exotic_infoDict.get('comp_stars') or []) + ) + if use_exactly_the_comps_provided: + if provided_comparison_count == 0 and fitsortext == 1: + log_info( + "Error: 'use_exactly_the_comps_provided' is enabled, but no comparison " + "coordinates were supplied.", + error=True, + ) + return + fixed_ensemble = provided_comparison_count > 1 + use_ensemble_photometry_rather_than_single_comp = fixed_ensemble + use_ensemble_photometry_for_stellar_variability = fixed_ensemble + if fixed_ensemble: + maximum_number_of_ensemble_comparisons_for_transit = provided_comparison_count + maximum_number_of_ensemble_comparisons_for_stellar_variability = provided_comparison_count photometer_fortuitous_variables = should_photometer_fortuitous_variables( exotic_infoDict.get( 'photometer_fortuitous_variables', @@ -31571,6 +32142,26 @@ def lookup_archive_ephemeris(): wcs_header = get_first_image_header(wcs_file) ra_wcs, dec_wcs = get_ra_dec(wcs_header, image_shape=reference_image.shape) + if comparisons_supplied_as_radec: + try: + exotic_infoDict['comp_stars'] = project_comparison_radec_to_pixels( + provided_comparison_radec, + wcs_header, + reference_image.shape, + ) + except ValueError as exc: + log_info( + "Error: supplied comparison-star RA/Dec coordinates are unusable: " + f"{exc}.", + error=True, + ) + return + plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) + log_info( + f"Projected {len(exotic_infoDict['comp_stars'])} supplied comparison-star " + "RA/Dec coordinate(s) onto the selected reference image." + ) + if reference_fallback is not None: target_projection = estimate_target_pixel_from_ra_dec( pDict, @@ -31609,7 +32200,15 @@ def lookup_archive_ephemeris(): auid = vsx_auid(ra_dec_tar[0], ra_dec_tar[1]) - if reference_fallback is not None: + if reference_fallback is not None and not comparisons_supplied_as_radec: + if use_exactly_the_comps_provided: + log_info( + "Error: the reference image containing the exact supplied comparison-star " + "pixels was rejected. EXOTIC will not replace or reproject those comparisons " + "while 'use_exactly_the_comps_provided' is enabled.", + error=True, + ) + return old_comp_count = len(exotic_infoDict['comp_stars']) exotic_infoDict['comp_stars'] = [] log_info( @@ -31622,7 +32221,10 @@ def lookup_archive_ephemeris(): vsp_comp_stars, chart_id = vsp_query(wcs_file,[header['NAXIS1'], header['NAXIS2']], exotic_infoDict['filter'], img_scale, user_comp_stars=exotic_infoDict['comp_stars'], - user_targ_star = [ exotic_UIprevTPX, exotic_UIprevTPY ]) + user_targ_star = [ exotic_UIprevTPX, exotic_UIprevTPY ], + max_new_comp_stars=( + 0 if use_exactly_the_comps_provided else 2 + )) vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] try: @@ -31674,7 +32276,10 @@ def lookup_archive_ephemeris(): stellar_variability_only and use_ensemble_photometry_for_stellar_variability ) - if automatic_calibration_selector_enabled or stellar_variability_ensemble_candidate_search: + if ( + not use_exactly_the_comps_provided + and (automatic_calibration_selector_enabled or stellar_variability_ensemble_candidate_search) + ): automatic_comp_count = parse_automatic_calibration_selector_count( exotic_infoDict.get('automatic_optimal_calibration_selector_count') ) @@ -31739,8 +32344,9 @@ def lookup_archive_ephemeris(): exotic_infoDict['comp_stars'] = automatic_comp_stars vsp_comp_stars = {} - check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], - use_nextastro_variability_server=args.use_nextastro_variability_server) + if not use_exactly_the_comps_provided: + check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], + use_nextastro_variability_server=args.use_nextastro_variability_server) while not exotic_infoDict['comp_stars']: log_info("\nThere are no comparison stars left as all of them were indicated as variable stars." @@ -31749,9 +32355,12 @@ def lookup_archive_ephemeris(): check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], use_nextastro_variability_server=args.use_nextastro_variability_server) - exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( - exotic_infoDict['comp_stars'] - ) + if use_exactly_the_comps_provided: + duplicate_comp_messages = [] + else: + exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( + exotic_infoDict['comp_stars'] + ) for duplicate_message in duplicate_comp_messages: log_info(duplicate_message) @@ -31811,13 +32420,16 @@ def lookup_archive_ephemeris(): log_info("Fortuitous-variable photometry disabled per optional_info setting.") if fortuitous_variables: - science_comp_stars, variable_comparison_rejections = ( - filter_comparison_stars_against_fortuitous_variables( - science_comp_stars, - fortuitous_variables, - duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + if use_exactly_the_comps_provided: + variable_comparison_rejections = [] + else: + science_comp_stars, variable_comparison_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + science_comp_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) ) - ) if variable_comparison_rejections: for rejection in variable_comparison_rejections: log_info( @@ -32022,7 +32634,11 @@ def lookup_archive_ephemeris(): science_comp_stars, user_targ_star=[exotic_UIprevTPX, exotic_UIprevTPY], max_new_comp_stars= - maximum_number_of_ensemble_comparisons_for_stellar_variability, + ( + 0 + if use_exactly_the_comps_provided + else maximum_number_of_ensemble_comparisons_for_stellar_variability + ), ) ) if fallback_chart_id is not None: @@ -32037,13 +32653,16 @@ def lookup_archive_ephemeris(): duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, ) is None } - science_comp_stars, fallback_variable_rejections = ( - filter_comparison_stars_against_fortuitous_variables( - science_comp_stars, - fortuitous_variables, - duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + if use_exactly_the_comps_provided: + fallback_variable_rejections = [] + else: + science_comp_stars, fallback_variable_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + science_comp_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) ) - ) for rejection in fallback_variable_rejections: log_info( "Removed AAVSO VSP comparison star at " @@ -32095,9 +32714,27 @@ def lookup_archive_ephemeris(): # Build RA/Dec after the tracking list is finalized. Science comps remain first, # followed by fortuitous-only ensemble candidates and then the VSX targets. ra_dec_wcs = build_comp_ra_dec(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) + if ( + comparisons_supplied_as_radec + and use_exactly_the_comps_provided + and len(ra_dec_wcs) >= len(provided_comparison_radec) + ): + # Preserve the user's celestial coordinates exactly for + # frame-to-frame WCS tracking and output metadata. The + # projected pixels remain the reference-frame centroids. + ra_dec_wcs[:len(provided_comparison_radec)] = [ + list(coords) for coords in provided_comparison_radec + ] vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) else: + if comparisons_supplied_as_radec: + log_info( + "Error: 'Comparison Star(s) RA & Dec' requires a usable celestial WCS on the " + "selected reference image so the supplied stars can be projected into pixels.", + error=True, + ) + return if reference_fallback is not None: log_info( "Error: the original reference image was removed, but the new reference image does not " @@ -32112,9 +32749,12 @@ def lookup_archive_ephemeris(): "the full-field VSX search will be skipped for this reduction.", warn=True, ) - exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( - exotic_infoDict['comp_stars'] - ) + if use_exactly_the_comps_provided: + duplicate_comp_messages = [] + else: + exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( + exotic_infoDict['comp_stars'] + ) for duplicate_message in duplicate_comp_messages: log_info(duplicate_message) science_comp_stars = [list(position) for position in exotic_infoDict['comp_stars']] @@ -32247,14 +32887,27 @@ def lookup_archive_ephemeris(): log_info("EEBLS transit initializer disabled per optional_info setting.") if not pick_comparison_by_eebls_snr: log_info("Comparison-star selection by EEBLS SNR disabled per optional_info setting.") - if use_ensemble_photometry_rather_than_single_comp: + if use_exactly_the_comps_provided: + exact_mode = "single comparison" if len(science_comp_stars) == 1 else "fixed ensemble" + log_info( + "Exact supplied-comparison mode enabled: EXOTIC will use the " + f"{len(science_comp_stars)} supplied comparison coordinate(s) as a {exact_mode}, " + "without automatic replacement, VSX rejection, stability vetting, ranking, or " + "ensemble-size limiting." + ) + elif use_ensemble_photometry_rather_than_single_comp: log_info( "Ensemble comparison photometry enabled per optional_info setting; the final target " "light curve will use non-rejected comparison stars as a combined reference, up to " "maximum_number_of_ensemble_comparisons_for_transit=" f"{maximum_number_of_ensemble_comparisons_for_transit}." ) - if stellar_variability_only: + if not require_apparent_magnitudes: + log_info( + "Apparent magnitudes are not required per optional_info; differential-magnitude " + "products will still be written, and any available apparent calibration remains optional." + ) + if stellar_variability_only and not use_exactly_the_comps_provided: if use_ensemble_photometry_for_stellar_variability: log_info( "Stellar-variability calibrated ensemble enabled (default): EXOTIC will combine " @@ -33630,11 +34283,16 @@ def lookup_archive_ephemeris(): airmass, science_comp_stars, sigma_display, - skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + skip_low_comparison_coverage_rejection=( + skip_low_comp_coverage_rejection or use_exactly_the_comps_provided + ), use_psf_photometry=use_psf_photometry, use_aperture_photometry=use_aperture_photometry, psf_flux_data=psf_flux_source, - comp_overexposed_masks=comp_overexposed_masks, + comp_overexposed_masks=( + None if use_exactly_the_comps_provided else comp_overexposed_masks + ), + use_exactly_the_comps_provided=use_exactly_the_comps_provided, ) # Fortuitous-only sources remain in the shared centroid/PSF tracks and frozen-aperture store. @@ -33778,6 +34436,8 @@ def lookup_archive_ephemeris(): exotic_infoDict.get('filter'), ), target_catalog_match=primary_target_catalog_match, + require_apparent_magnitudes=require_apparent_magnitudes, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, ) else: comparison_fit_search = fit_ranked_comparison_calibration_candidates( @@ -33817,6 +34477,7 @@ def lookup_archive_ephemeris(): maximum_number_of_ensemble_comparisons_for_transit, exposure_times_seconds=exposure_times_seconds, gain_e_per_adu=fallback_gain_e_per_adu, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, ) if use_ensemble_photometry_for_stellar_variability and vsp_comp_stars: log_info( @@ -33850,6 +34511,8 @@ def lookup_archive_ephemeris(): exotic_infoDict.get('filter'), ), target_catalog_match=primary_target_catalog_match, + require_apparent_magnitudes=require_apparent_magnitudes, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, ) comparison_calibration['ranked_fit_comp_indices'] = [ summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] @@ -33898,9 +34561,21 @@ def lookup_archive_ephemeris(): ) selected_attempt_label = selected_attempt.get('label', 'comparison candidate') if stellar_variability_only and selected_is_ensemble: - selection_basis = 'stellar_variability_ensemble' + selection_basis = ( + 'exact_stellar_variability_ensemble' + if use_exactly_the_comps_provided + else 'stellar_variability_ensemble' + ) elif stellar_variability_only: - selection_basis = 'stellar_variability_scatter' + selection_basis = ( + 'exact_single_comparison' + if use_exactly_the_comps_provided + else 'stellar_variability_scatter' + ) + elif use_exactly_the_comps_provided and selected_is_ensemble: + selection_basis = 'exact_comparison_ensemble' + elif use_exactly_the_comps_provided: + selection_basis = 'exact_single_comparison' elif selected_attempt.get('search_stopped_after_qc_pass', False): selection_basis = 'first_qc_pass' elif selected_attempt.get('search_stopped_after_promising_partial', False): @@ -33913,7 +34588,14 @@ def lookup_archive_ephemeris(): selection_basis = 'comparison_field' else: selection_basis = 'comparison_field_retry' - if selection_basis == 'stellar_variability_ensemble': + if selection_basis == 'exact_stellar_variability_ensemble': + ensemble_members = selected_attempt.get('ensemble_member_keys') or [] + log_info( + "Stellar-variability-only comparison selection used the exact supplied " + f"ensemble with {len(ensemble_members)} member(s); no supplied comparison " + "was vetted out or replaced." + ) + elif selection_basis == 'stellar_variability_ensemble': ensemble_members = selected_attempt.get('ensemble_member_keys') or [] log_info( "Stellar-variability-only comparison selection chose the default calibrated " @@ -33921,6 +34603,11 @@ def lookup_archive_ephemeris(): f"{comparison_calibration['method_label']}; members are bright, unsaturated, " "VSX-vetted, and passed the high-side catalog-error clip." ) + elif selection_basis == 'exact_single_comparison': + log_info( + "Comparison selection used the one exact supplied comparison star; " + "no alternative comparison was evaluated." + ) elif selection_basis == 'stellar_variability_scatter': log_info( "Stellar-variability-only comparison selection chose " @@ -33973,6 +34660,12 @@ def lookup_archive_ephemeris(): f"{fallback_metric_value}) so final outputs are still produced.", warn=True, ) + elif selection_basis == 'exact_comparison_ensemble': + log_info( + "Comparison-star calibration target-fit selection used every supplied " + "comparison star as one fixed ensemble because " + "'use_exactly_the_comps_provided' is enabled." + ) elif selection_basis == 'comparison_ensemble': log_info( "Comparison-star calibration target-fit selection chose the comparison-star ensemble " @@ -34058,6 +34751,7 @@ def lookup_archive_ephemeris(): target_flux_error=tFlux1_error, comp_flux_error=cFlux1_error, ) + myfit.differential_magnitude_reference_label = selected_attempt_label if selected_comp_index is not None: ref_flux[selected_comp_index] = { @@ -34678,6 +35372,7 @@ def lookup_archive_ephemeris(): pDict['sName'], observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), + observation_date=exotic_infoDict.get('date'), comp_ra_dec=ra_dec_wcs[:len(fortuitous_ensemble_stars)], field_catalog=nextastro_field_catalog, reference_image=reference_image, @@ -34850,7 +35545,14 @@ def lookup_archive_ephemeris(): except Exception: pass - plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + plot_final_lightcurve( + myfit, + data_highres, + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), + ) if fitsortext == 1: observing_background_series = build_observing_background_series( @@ -34901,6 +35603,8 @@ def lookup_archive_ephemeris(): try: phase = np.asarray(getattr(myfit, 'phase', get_phase(myfit.time, pDict['pPer'], pDict['midT']))) + output_files.differential_magnitude() + output_files.stellar_variability_differential_magnitude() output_files.final_lightcurve(phase) except Exception as e: log_info(f"\nError: Could not create FinalLightCurve.csv. {error_txt}\n\t{e}", error=True) @@ -35112,6 +35816,7 @@ def lookup_archive_ephemeris(): exotic_infoDict['save'], pDict['sName'], observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), + observation_date=exotic_infoDict.get('date'), ) if not auid: auid = vsx_auid(pDict['ra'], pDict['dec']) @@ -35155,7 +35860,14 @@ def lookup_archive_ephemeris(): except Exception: pass - plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + plot_final_lightcurve( + myfit, + data_highres, + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), + ) diagnostics_dir = Path(exotic_infoDict['save']) / "Diagnostics" plot_prior_posterior_comparison(myfit, pDict, pDict['pName'], diagnostics_dir, exotic_infoDict['date']) plot_ktmf_qc_metrics(myfit, pDict['pName'], diagnostics_dir, exotic_infoDict['date']) @@ -35360,6 +36072,8 @@ def lookup_archive_ephemeris(): try: phase = get_phase(myfit.time, pDict['pPer'], myfit.parameters['tmid']) + output_files.differential_magnitude() + output_files.stellar_variability_differential_magnitude() output_files.final_lightcurve(phase) except Exception as e: log_info(f"\nError: Could not create FinalLightCurve.csv. {error_txt}\n\t{e}", error=True) @@ -35412,6 +36126,7 @@ def lookup_archive_ephemeris(): 'plate_solution_option': exotic_infoDict.get('plate_opt'), 'target_input_pixel': exotic_infoDict.get('tar_coords'), 'comparison_input_pixels': exotic_infoDict.get('comp_stars'), + 'comparison_input_ra_dec_deg': provided_comparison_radec, 'target_ra_dec_deg': ra_dec_tar, 'comparison_ra_dec_deg': ra_dec_wcs, 'catalog_ra_dec_deg': [pDict.get('ra'), pDict.get('dec')], diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index e2e705da..d04634ad 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -404,7 +404,7 @@ def save_input(): "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", - "Comparison Star(s) Guide": "Up to 10 comparison stars can be added following the format given below.", + "Comparison Star(s) Guide": "Provide comparison stars either as X/Y pixels or as RA/Dec coordinates, but not both. RA/Dec requires a usable reference-image WCS.", "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Plate Solution": "For your image to be given a plate solution, type y.", @@ -420,6 +420,8 @@ def save_input(): "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", + "Apparent Magnitudes Required": "Set optional_info 'require_apparent_magnitudes' to false when catalogue-calibrated apparent magnitudes are not required. Differential-magnitude products remain independent of catalogue calibration. Stellar-variability magnitudes use the raw target/reference ratio with no airmass correction. Default true.", + "Use Exactly Supplied Comparisons": "Set optional_info 'use_exactly_the_comps_provided' to true to use only the supplied X/Y or RA/Dec comparison coordinates with no replacement, addition, vetting, ranking, or ensemble-size limit. One comparison is used alone; multiple comparisons are all used as a fixed ensemble. Default false.", "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; minimum 2; no configured upper limit.", "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; minimum 2; no configured upper limit.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", @@ -449,6 +451,7 @@ def save_input(): "Target Star X & Y Pixel": input_data['targetpos'], "Comparison Star(s) X & Y Pixel": [input_data['comppos']], + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, # TODO add GUI input for these "Demosaic Output": null } @@ -461,6 +464,8 @@ def save_input(): "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "stellar_variability_only": False, + "require_apparent_magnitudes": True, + "use_exactly_the_comps_provided": False, "maximum_number_of_ensemble_comparisons_for_transit": 5, "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "detect_bad_pixels_before_photometry": "n", @@ -1513,7 +1518,7 @@ def save_input(): "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", - "Comparison Star(s) Guide": "Up to 10 comparison stars can be added following the format given below.", + "Comparison Star(s) Guide": "Provide comparison stars either as X/Y pixels or as RA/Dec coordinates, but not both. RA/Dec requires a usable reference-image WCS.", "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Plate Solution": "For your image to be given a plate solution, type y.", @@ -1529,6 +1534,8 @@ def save_input(): "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", + "Apparent Magnitudes Required": "Set optional_info 'require_apparent_magnitudes' to false when catalogue-calibrated apparent magnitudes are not required. Differential-magnitude products remain independent of catalogue calibration. Stellar-variability magnitudes use the raw target/reference ratio with no airmass correction. Default true.", + "Use Exactly Supplied Comparisons": "Set optional_info 'use_exactly_the_comps_provided' to true to use only the supplied X/Y or RA/Dec comparison coordinates with no replacement, addition, vetting, ranking, or ensemble-size limit. One comparison is used alone; multiple comparisons are all used as a fixed ensemble. Default false.", "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; minimum 2; no configured upper limit.", "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; minimum 2; no configured upper limit.", "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", @@ -1582,6 +1589,7 @@ def save_input(): "Target Star X & Y Pixel": (input_data['targetpos']), "Comparison Star(s) X & Y Pixel": (input_data['comppos']), + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, # TODO add GUI input for these "Demosaic Output": null @@ -1617,6 +1625,8 @@ def save_input(): "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "stellar_variability_only": False, + "require_apparent_magnitudes": True, + "use_exactly_the_comps_provided": False, "maximum_number_of_ensemble_comparisons_for_transit": 5, "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "detect_bad_pixels_before_photometry": "n", @@ -1692,6 +1702,8 @@ def save_input(): "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, "stellar_variability_only": False, + "require_apparent_magnitudes": True, + "use_exactly_the_comps_provided": False, "maximum_number_of_ensemble_comparisons_for_transit": 5, "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "detect_bad_pixels_before_photometry": "n", diff --git a/exotic/inputs.py b/exotic/inputs.py index e7a04a0d..d085f687 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -133,6 +133,63 @@ def radec_to_decimal_degrees(ra, dec): return coords.ra.degree, coords.dec.degree +def comparison_star_coords_provided(comp_stars): + """Return whether an X/Y comparison-star input contains a coordinate pair.""" + if isinstance(comp_stars, (list, tuple)): + if len(comp_stars) == 2 and not any(isinstance(value, (list, tuple, dict)) for value in comp_stars): + return not any(is_blank_value(value) for value in comp_stars) + return any( + isinstance(star, (list, tuple)) + and len(star) == 2 + and not any(is_blank_value(value) for value in star) + for star in comp_stars + ) + if isinstance(comp_stars, str): + return len(re.findall(r"[-+]?(?:\d*\.?\d+)", comp_stars)) >= 2 + return False + + +def comparison_star_radec_coords(comp_stars): + """Normalize one or more comparison-star RA/Dec pairs to decimal degrees.""" + if is_blank_value(comp_stars): + return [] + + if isinstance(comp_stars, str): + try: + comp_stars = json.loads(comp_stars) + except json.JSONDecodeError as exc: + raise ValueError( + "Comparison Star(s) RA & Dec must be a JSON list of [RA, Dec] pairs." + ) from exc + + if ( + isinstance(comp_stars, (list, tuple)) + and len(comp_stars) == 2 + and not any(isinstance(value, (list, tuple, dict)) for value in comp_stars) + ): + comp_stars = [comp_stars] + + if not isinstance(comp_stars, (list, tuple)): + raise ValueError("Comparison Star(s) RA & Dec must be a list of [RA, Dec] pairs.") + + normalized = [] + for index, star in enumerate(comp_stars, start=1): + if is_blank_value(star) or star == [] or star == (): + continue + if not isinstance(star, (list, tuple)) or len(star) != 2: + raise ValueError( + f"Comparison star {index} RA/Dec must contain exactly two values: [RA, Dec]." + ) + try: + ra_deg, dec_deg = radec_to_decimal_degrees(star[0], star[1]) + except (TypeError, ValueError) as exc: + raise ValueError(f"Comparison star {index} has invalid RA/Dec coordinates: {exc}") from exc + if ra_deg is None or dec_deg is None: + raise ValueError(f"Comparison star {index} has blank RA or Dec coordinates.") + normalized.append([float(ra_deg), float(dec_deg)]) + return normalized + + def fetch_nextastro_gaia_distpm(ra_deg, dec_deg): response = requests.get( NEXTASTRO_GAIA_DISTPM_ENDPOINT, @@ -202,6 +259,7 @@ def __init__(self, init_opt): 'aavso_num': None, 'second_obs': None, 'obs_name': '', 'date': None, 'lat': None, 'long': None, 'elev': None, 'camera': None, 'pixel_bin': None, 'filter': None, 'notes': None, 'plate_opt': None, 'aavso_comp': None, 'tar_coords': None, 'comp_stars': None, + 'comp_stars_radec': None, 'prered_file': None, 'file_units': None, 'file_time': None, 'phot_comp_star': None, 'wl_min': None, 'wl_max': None, 'pixel_scale': None, 'exposure': None, 'dist': None, 'pm_ra': None, 'pm_dec': None, 'airmass_already_corrected': False, @@ -211,6 +269,8 @@ def __init__(self, init_opt): 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'stellar_variability_only': False, 'use_ensemble_photometry_for_stellar_variability': True, + 'require_apparent_magnitudes': True, + 'use_exactly_the_comps_provided': False, 'maximum_number_of_ensemble_comparisons_for_transit': 5, 'maximum_number_of_ensemble_comparisons_for_stellar_variability': 5, 'photometer_fortuitous_variables': True, @@ -276,7 +336,12 @@ def complete_red(self, planet): elif key == 'tar_coords': self.info_dict[key] = self.params[key](self.info_dict[key], planet) elif key == 'comp_stars': - self.info_dict[key] = self.params[key](self.info_dict[key], False) + if self.info_dict.get('comp_stars_radec'): + # Celestial comparison coordinates are projected into pixels + # after the reference image has a usable WCS. + self.info_dict[key] = [] + else: + self.info_dict[key] = self.params[key](self.info_dict[key], False) elif key == 'images': pass elif key in ('lat', 'long'): @@ -413,7 +478,13 @@ def comp_params(self, init_file, planet_dict): 'pixel_bin': 'Pixel Binning', 'filter': 'Filter Name (aavso.org/filters)', 'notes': 'Observing Notes', 'plate_opt': 'Plate Solution? (y/n)', 'aavso_comp': 'Add Comparison Stars from AAVSO? (y/n)', - 'tar_coords': 'Target Star X & Y Pixel', 'comp_stars': 'Comparison Star(s) X & Y Pixel', + 'tar_coords': 'Target Star X & Y Pixel', + 'comp_stars': 'Comparison Star(s) X & Y Pixel', + 'comp_stars_radec': ( + 'Comparison Star(s) RA & Dec', + 'Comparison Star(s) RA and Dec', + 'Comparison Star(s) RA & Dec (degrees)', + ), } planet_params = { 'ra': 'Target Star RA', 'dec': 'Target Star Dec', 'pName': "Planet Name", 'sName': "Host Star Name", @@ -493,6 +564,14 @@ def comp_params(self, init_file, planet_dict): 'stellar_variability_use_ensemble', 'Use Ensemble Photometry for Stellar Variability? (y/n)', ), + 'require_apparent_magnitudes': ( + 'require_apparent_magnitudes', + 'Require Apparent Magnitudes? (y/n)', + ), + 'use_exactly_the_comps_provided': ( + 'use_exactly_the_comps_provided', + 'Use Exactly the Comparisons Provided? (y/n)', + ), 'maximum_number_of_ensemble_comparisons_for_transit': ( 'maximum_number_of_ensemble_comparisons_for_transit', 'Maximum Number of Ensemble Comparisons for Transit', @@ -794,6 +873,15 @@ def comp_params(self, init_file, planet_dict): } self.info_dict = init_params(user_info, self.info_dict, data['user_info']) + self.info_dict['comp_stars_radec'] = comparison_star_radec_coords( + self.info_dict.get('comp_stars_radec') + ) + if self.info_dict['comp_stars_radec'] and comparison_star_coords_provided( + self.info_dict.get('comp_stars')): + raise ValueError( + "Provide comparison stars using either 'Comparison Star(s) X & Y Pixel' " + "or 'Comparison Star(s) RA & Dec', not both." + ) if self.info_dict['aavso_comp'] is None: self.info_dict['aavso_comp'] = 'n' self.info_dict = init_params(opt_info, self.info_dict, data['optional_info']) diff --git a/exotic/output_files.py b/exotic/output_files.py index 75e46948..12bc52df 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -123,6 +123,195 @@ def apparent_magnitude_calibration_from_vsp_params(vsp_params): } +def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, + apply_airmass_correction=None): + """Return target-minus-reference instrumental magnitudes. + + Unlike apparent magnitudes, this series needs no catalogue magnitude. Raw + target/reference fluxes are preferred so the instrumental zero point is + preserved; pre-reduced light curves fall back to their relative flux. + Stellar-variability fits intentionally remain uncorrected for airmass so a + real time-dependent stellar signal is not fitted away. + """ + fit_data = np.asarray(getattr(fit, 'data', []), dtype=float).reshape(-1) + fit_times = np.asarray( + getattr(fit, 'time', getattr(fit, 'jd_times', [])), + dtype=float, + ).reshape(-1) + if fit_data.size == 0 or fit_times.shape != fit_data.shape: + return None + + target_flux = np.asarray( + getattr(fit, 'stellar_variability_target_flux', []), + dtype=float, + ).reshape(-1) + reference_flux = np.asarray( + getattr(fit, 'stellar_variability_comp_flux', []), + dtype=float, + ).reshape(-1) + target_error = np.asarray( + getattr(fit, 'stellar_variability_target_flux_error', []), + dtype=float, + ).reshape(-1) + reference_error = np.asarray( + getattr(fit, 'stellar_variability_comp_flux_error', []), + dtype=float, + ).reshape(-1) + + has_raw_photometry = ( + target_flux.shape == fit_data.shape + and reference_flux.shape == fit_data.shape + ) + if not has_raw_photometry: + target_flux = np.asarray(getattr(fit, 'detrended', fit_data), dtype=float).reshape(-1) + if target_flux.shape != fit_data.shape: + target_flux = fit_data.copy() + reference_flux = np.ones(fit_data.shape, dtype=float) + target_error = np.asarray( + getattr(fit, 'detrendederr', getattr(fit, 'dataerr', [])), + dtype=float, + ).reshape(-1) + reference_error = np.zeros(fit_data.shape, dtype=float) + + if target_error.shape != fit_data.shape: + target_error = np.full(fit_data.shape, np.nan, dtype=float) + if reference_error.shape != fit_data.shape: + reference_error = np.full(fit_data.shape, np.nan, dtype=float) + + if apply_airmass_correction is None: + apply_airmass_correction = not bool( + getattr(fit, 'stellar_variability_only', False) + ) + # ``fit.detrended`` is already corrected. Only divide an explicitly + # retained raw target/reference ratio by the fitted airmass model. + apply_airmass_correction = bool(apply_airmass_correction and has_raw_photometry) + relative_airmass_model = np.ones(fit_data.shape, dtype=float) + if apply_airmass_correction: + airmass_model = np.asarray( + getattr(fit, 'airmass_model', np.ones(fit_data.shape)), + dtype=float, + ).reshape(-1) + if airmass_model.shape != fit_data.shape: + airmass_model = np.ones(fit_data.shape, dtype=float) + valid_airmass_model = np.isfinite(airmass_model) & (airmass_model > 0) + airmass_reference = ( + float(np.nanmedian(airmass_model[valid_airmass_model])) + if np.any(valid_airmass_model) + else 1.0 + ) + if not np.isfinite(airmass_reference) or airmass_reference <= 0: + airmass_reference = 1.0 + relative_airmass_model = np.divide( + airmass_model, + airmass_reference, + out=np.full(fit_data.shape, np.nan, dtype=float), + where=valid_airmass_model, + ) + + with np.errstate(divide='ignore', invalid='ignore'): + raw_ratio = np.divide(target_flux, reference_flux) + corrected_ratio = np.divide(raw_ratio, relative_airmass_model) + differential_magnitude = -2.5 * np.log10(corrected_ratio) + magnitude_factor = 2.5 / np.log(10.0) + explicit_error = magnitude_factor * np.sqrt( + (target_error / target_flux) ** 2 + + (reference_error / reference_flux) ** 2 + ) + + fit_error = np.asarray(getattr(fit, 'dataerr', []), dtype=float).reshape(-1) + if fit_error.shape == fit_data.shape: + with np.errstate(divide='ignore', invalid='ignore'): + fallback_error = magnitude_factor * np.abs(fit_error / fit_data) + else: + fallback_error = np.full(fit_data.shape, np.nan, dtype=float) + differential_error = np.where( + np.isfinite(explicit_error) & (explicit_error >= 0), + explicit_error, + fallback_error, + ) + + airmass = np.asarray( + getattr(fit, 'airmass', np.full(fit_data.shape, np.nan)), + dtype=float, + ).reshape(-1) + if airmass.shape != fit_data.shape: + airmass = np.full(fit_data.shape, np.nan, dtype=float) + + keep = ( + np.isfinite(fit_times) + & np.isfinite(corrected_ratio) + & (corrected_ratio > 0) + & np.isfinite(differential_magnitude) + ) + if out_of_transit_only: + transit_model = np.asarray( + getattr(fit, 'transit', np.ones(fit_data.shape)), + dtype=float, + ).reshape(-1) + if transit_model.shape == fit_data.shape and np.any(transit_model == 1): + keep &= transit_model == 1 + + if not np.any(keep): + return None + return { + 'time': fit_times[keep], + 'airmass': airmass[keep], + 'magnitude': differential_magnitude[keep], + 'magnitude_error': differential_error[keep], + 'source_mask': keep, + 'airmass_corrected': apply_airmass_correction, + } + + +def write_differential_magnitude_csv(fit, save, target_name, observation_date=None, + observed_filter=None, out_of_transit_only=False, + apply_airmass_correction=None, + filename_prefix='DifferentialMagnitude'): + series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=apply_airmass_correction, + ) + if series is None: + return None + + output_dir = Path(save) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / safe_output_filename( + filename_prefix, + target_name, + filename_date_token(observation_date) if observation_date else 'undated', + extension='csv', + ) + comparison = ( + getattr(fit, 'stellar_variability_reference_label', None) + or getattr(fit, 'differential_magnitude_reference_label', None) + or 'selected comparison reference' + ) + with output_path.open('w', encoding='utf-8') as handle: + handle.write( + '# AIRMASS_CORRECTION=' + f"{'YES' if series['airmass_corrected'] else 'NO'}\n" + ) + handle.write( + '# BJD_TDB,Airmass,Differential Magnitude,' + 'Differential Magnitude Uncertainty,Filter,Comparison\n' + ) + for time_value, airmass, magnitude, magnitude_error in zip( + series['time'], + series['airmass'], + series['magnitude'], + series['magnitude_error'], + ): + airmass_text = f"{airmass}" if np.isfinite(airmass) else 'na' + error_text = f"{magnitude_error:.6f}" if np.isfinite(magnitude_error) else 'na' + handle.write( + f"{time_value}, {airmass_text}, {magnitude:.6f}, {error_text}, " + f"{observed_filter or 'na'}, {comparison}\n" + ) + return output_path + + def aavso_json_safe(value): if isinstance(value, dict): return {str(key): aavso_json_safe(subvalue) for key, subvalue in value.items()} @@ -1848,6 +2037,50 @@ def final_lightcurve(self, phase): ) f.write(f"{row}\n") + def differential_magnitude(self): + target_name = self.p_dict.get('sName') or self.p_dict.get('pName') or 'target' + return write_differential_magnitude_csv( + self.fit, + self.dir, + target_name, + observation_date=self.i_dict.get('date'), + observed_filter=( + self.i_dict.get('observed_filter') + or self.i_dict.get('filter') + ), + out_of_transit_only=False, + ) + + def stellar_variability_differential_magnitude(self): + """Write the raw, out-of-transit stellar-variability counterpart.""" + if getattr(self.fit, 'stellar_variability_only', False): + return None + fit_shape = np.asarray(getattr(self.fit, 'data', []), dtype=float).shape + target_shape = np.asarray( + getattr(self.fit, 'stellar_variability_target_flux', []), + dtype=float, + ).shape + reference_shape = np.asarray( + getattr(self.fit, 'stellar_variability_comp_flux', []), + dtype=float, + ).shape + if target_shape != fit_shape or reference_shape != fit_shape: + return None + target_name = self.p_dict.get('sName') or self.p_dict.get('pName') or 'target' + return write_differential_magnitude_csv( + self.fit, + self.dir, + target_name, + observation_date=self.i_dict.get('date'), + observed_filter=( + self.i_dict.get('observed_filter') + or self.i_dict.get('filter') + ), + out_of_transit_only=True, + apply_airmass_correction=False, + filename_prefix='StellarVariabilityDifferentialMagnitude', + ) + def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, min_annul=None, adaptive_summary=None, photometry_info=None, publish_to_root=False): diff --git a/exotic/plots.py b/exotic/plots.py index f616f7b0..501b3735 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -26,6 +26,7 @@ try: from output_files import ( + differential_magnitude_series_from_fit, empirical_red_noise_error_scale, fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, @@ -33,6 +34,7 @@ ) except ImportError: from .output_files import ( + differential_magnitude_series_from_fit, empirical_red_noise_error_scale, fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, @@ -636,6 +638,7 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): title_lines = [s_name] if reference_label: title_lines.append(reference_label) + title_lines.append('No airmass correction applied to stellar variability') metadata_label = _stellar_variability_comparison_metadata_label(first_param) if metadata_label: title_lines.append(metadata_label) @@ -716,6 +719,78 @@ def magnitude_to_flux(magnitude): return True +def plot_differential_magnitude(fit, target_name, save, date, observed_filter=None, + out_of_transit_only=False, apply_airmass_correction=None, + filename_prefix='DifferentialMagnitude', + save_stellar_variability_alias=False): + series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=apply_airmass_correction, + ) + if series is None: + return None + + fig, ax = plt.subplots(figsize=(8, 5)) + finite_error = np.isfinite(series['magnitude_error']) & (series['magnitude_error'] >= 0) + if np.any(finite_error): + ax.errorbar( + series['time'][finite_error], + series['magnitude'][finite_error], + yerr=series['magnitude_error'][finite_error], + color='royalblue', + fmt='.', + ) + if np.any(~finite_error): + ax.plot( + series['time'][~finite_error], + series['magnitude'][~finite_error], + '.', + color='royalblue', + ) + correction_label = ( + 'Airmass-corrected target/reference ratio' + if series['airmass_corrected'] + else 'Raw target/reference ratio; no airmass correction' + ) + ax.set_title(f"{target_name}\n{correction_label}") + band_label = f" ({observed_filter})" if observed_filter else '' + ax.set_ylabel(f"Differential Magnitude{band_label}") + ax.invert_yaxis() + ax.set_xlabel("Time [BJD_TDB]") + fig.tight_layout() + + output_dir = Path(save) + output_dir.mkdir(parents=True, exist_ok=True) + png_path = output_dir / _dated_plot_filename( + filename_prefix, + target_name, + date=date, + extension='png', + ) + pdf_path = output_dir / _dated_plot_filename( + filename_prefix, + target_name, + date=date, + extension='pdf', + ) + fig.savefig(png_path, bbox_inches='tight') + fig.savefig(pdf_path, bbox_inches='tight') + + if save_stellar_variability_alias or getattr(fit, 'stellar_variability_only', False): + artifacts_dir = _working_artifacts_dir(save) + fig.savefig( + artifacts_dir / 'Stellar_Variability_DifferentialMagnitude.png', + bbox_inches='tight', + ) + fig.savefig( + output_dir / 'Stellar_Variability_DifferentialMagnitude.png', + bbox_inches='tight', + ) + plt.close(fig) + return png_path + + # Observation statistics series selection def _select_plot_rows(rows, sort_index=None, sigma_mask=None, relative_flux_mask=None): rows = np.asarray(rows) @@ -987,7 +1062,37 @@ def _plot_final_residual_rejected_points(ax_lc, ax_res, fit): ) -def plot_final_lightcurve(fit, high_res, targ_name, save, date): +def plot_final_lightcurve(fit, high_res, targ_name, save, date, observed_filter=None): + plot_differential_magnitude( + fit, + getattr(fit, 'stellar_variability_target_name', targ_name), + save, + date, + observed_filter=observed_filter, + ) + fit_shape = np.asarray(getattr(fit, 'data', []), dtype=float).shape + has_raw_stellar_photometry = ( + np.asarray( + getattr(fit, 'stellar_variability_target_flux', []), + dtype=float, + ).shape == fit_shape + and np.asarray( + getattr(fit, 'stellar_variability_comp_flux', []), + dtype=float, + ).shape == fit_shape + ) + if not getattr(fit, 'stellar_variability_only', False) and has_raw_stellar_photometry: + plot_differential_magnitude( + fit, + getattr(fit, 'stellar_variability_target_name', targ_name), + save, + date, + observed_filter=observed_filter, + out_of_transit_only=True, + apply_airmass_correction=False, + filename_prefix='StellarVariabilityDifferentialMagnitude', + save_stellar_variability_alias=True, + ) if getattr(fit, 'stellar_variability_only', False): series = _stellar_variability_magnitude_series( getattr(fit, 'stellar_variability_params', None) @@ -1005,6 +1110,7 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date): ) if reference_label: title_lines.append(reference_label) + title_lines.append('No airmass correction applied to stellar variability') metadata_label = _stellar_variability_comparison_metadata_label(first_param) if metadata_label: title_lines.append(metadata_label) diff --git a/inits.json b/inits.json index 56aeb6a8..8b7ea175 100644 --- a/inits.json +++ b/inits.json @@ -8,7 +8,7 @@ "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", - "Comparison Star(s) Guide": "Up to 10 comparison stars can be added following the format given below.", + "Comparison Star(s) Guide": "Provide comparison stars either as X/Y pixels or as RA/Dec coordinates, but not both. RA/Dec accepts decimal degrees or sexagesimal strings and requires a usable reference-image WCS.", "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Camera Type (1)": "If you are using a CMOS, please enter CCD in 'Camera Type (CCD or DSLR)' and then note", @@ -30,6 +30,8 @@ "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, use the default calibrated comparison-star ensemble, and discard predicted ingress-to-egress transit-window points. Default false.", "Stellar Variability Ensemble": "Set optional_info 'use_ensemble_photometry_for_stellar_variability' to false to disable the default calibrated ensemble in stellar_variability_only runs and restore single-comparison selection by out-of-transit scatter. The default ensemble automatically finds bright catalog-calibrated field-star candidates, removes saturated and VSX-variable stars, sigma-clips high catalog magnitude uncertainties, and retains up to maximum_number_of_ensemble_comparisons_for_stellar_variability stars closest to the target in catalog colour and magnitude. EnsembleSelection JSON records the target and comparison colours, magnitudes, errors, and selection deltas beside the final results.", + "Apparent Magnitudes Required": "Set optional_info 'require_apparent_magnitudes' to false when catalogue-calibrated apparent magnitudes are not required. EXOTIC still attempts apparent outputs when calibration is available, but differential-magnitude CSV and plot products never depend on a catalogue magnitude. Stellar-variability magnitudes use the raw target/reference ratio with no airmass correction. Default true.", + "Use Exactly Supplied Comparisons": "Set optional_info 'use_exactly_the_comps_provided' to true to use only the comparison coordinates supplied in user_info, whether supplied as X/Y pixels or RA/Dec, without automatic replacement, VSX rejection, stability vetting, ranking, or ensemble-size limiting. One supplied comparison is used alone; two or more are all used together as one fixed ensemble. Default false.", "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; values must be integers of at least 2, with no configured upper limit.", "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; values must be integers of at least 2, with no configured upper limit. Large variability ensembles increase photometry work and can reject more frames because every selected ensemble member must be usable in a retained frame.", "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent calibrated photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under variables/optimal_variables/ for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under variables/normal/. Skipped variables are recorded only in the shared variables manifest and do not receive an object directory.", @@ -83,6 +85,7 @@ "Add Comparison Stars from AAVSO? (y/n)": "n", "Target Star X & Y Pixel": "[424, 286]", "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, "Demosaic Output": null }, @@ -131,6 +134,8 @@ "disable vertical flux normalization": false, "stellar_variability_only": false, "use_ensemble_photometry_for_stellar_variability": true, + "require_apparent_magnitudes": true, + "use_exactly_the_comps_provided": false, "maximum_number_of_ensemble_comparisons_for_transit": 5, "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, "photometer_fortuitous_variables": true, diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 0fc062b4..28124970 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -5,6 +5,7 @@ from pathlib import Path import numpy as np import pytest +from astropy.wcs import WCS def _module_available(name: str) -> bool: @@ -163,6 +164,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: parse_maximum_number_of_ensemble_comparisons_for_transit, prepare_final_fit_lightcurve_series, prepare_lightcurve_fit_input_series, + project_comparison_radec_to_pixels, psf_frame_quality_components, psf_frame_quality_mask, psf_solution_quality_score, @@ -191,6 +193,8 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: summarize_adaptive_aperture_usage, summarize_prior_transit_coverage, should_skip_airmass_fit, + should_require_apparent_magnitudes, + should_use_exactly_the_comps_provided, should_use_eebls_to_initialize_tmid_and_bounds, should_use_ensemble_photometry_for_stellar_variability, should_photometer_fortuitous_variables, @@ -1406,6 +1410,35 @@ def test_stellar_variability_ensemble_config_defaults_on_and_supports_opt_out(): assert should_use_ensemble_photometry_for_stellar_variability(False) is False +def test_apparent_and_exact_comparison_config_defaults_and_values(): + assert should_require_apparent_magnitudes(None) is True + assert should_require_apparent_magnitudes("n") is False + assert should_use_exactly_the_comps_provided(None) is False + assert should_use_exactly_the_comps_provided("y") is True + + +def test_project_comparison_radec_to_reference_pixels_and_reject_out_of_frame(): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [50.0, 50.0] + wcs.wcs.cdelt = np.array([-0.001, 0.001]) + wcs.wcs.crval = [31.04125, 46.68972] + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + + projected = project_comparison_radec_to_pixels( + [[31.04125, 46.68972]], + wcs.to_header(), + (100, 100), + ) + + assert projected[0] == pytest.approx([49.0, 49.0]) + with pytest.raises(ValueError, match="projects outside the reference image"): + project_comparison_radec_to_pixels( + [[32.04125, 46.68972]], + wcs.to_header(), + (100, 100), + ) + + def test_independent_ensemble_comparison_limits_default_to_five_and_have_no_upper_cap(): assert parse_maximum_number_of_ensemble_comparisons_for_transit(None) == 5 assert parse_maximum_number_of_ensemble_comparisons_for_transit(1) == 5 @@ -3084,6 +3117,28 @@ def test_comparison_star_stability_summary_penalizes_variable_candidates(): assert summary["comp_summaries"][2]["aggregate_score"] > summary["comp_summaries"][0]["aggregate_score"] +def test_exact_comparison_mode_bypasses_star_and_frame_vetting(): + airmass = np.linspace(1.0, 1.5, 8) + summary = comparison_star_stability_summary( + { + "comp1": np.array([100.0, 101.0, 100.0, 101.0, 100.0, 101.0, 100.0, 101.0]), + "comp2": np.array([80.0, 80.0, 80.0, 160.0, 80.0, 80.0, 80.0, 80.0]), + "comp3": np.array([60.0, 60.0, 60.0, 60.0, 60.0, 60.0, np.nan, np.nan]), + }, + airmass, + bypass_vetting=True, + ) + + assert [row["key"] for row in summary["comp_summaries"]] == [ + "comp1", + "comp2", + "comp3", + ] + assert not any(row["coverage_rejected"] for row in summary["comp_summaries"]) + assert not any(row["suitability_outlier_rejected"] for row in summary["comp_summaries"]) + assert np.all(summary["field_image_keep_mask"]) + + def test_apply_comparison_star_suitability_outlier_rejection_rejects_high_tail(): comp_summaries = [ {"label": "Comp 1", "aggregate_score": 0.139668, "coverage_rejected": False}, @@ -3397,6 +3452,34 @@ def build_psf_rows(): assert np.isnan(comp1_summary["ensemble_ratio_series"][7]) +def test_exact_comparison_calibration_does_not_reject_an_infinite_stability_score(): + frame_count = 6 + psf_rows = np.ones((frame_count, 7), dtype=float) + psf_rows[:, 3:5] = 1.0 + calibration = select_comparison_calibrated_photometry( + { + "target": psf_rows.copy(), + "comp1": psf_rows.copy(), + }, + { + "target": np.full((frame_count, 1, 1), 1000.0), + "comp1": np.full((frame_count, 1, 1), np.nan), + }, + apers=np.array([2.5]), + annuli=np.array([10.0]), + airmass=np.linspace(1.0, 1.5, frame_count), + comp_stars=[[10.0, 20.0]], + sigma=1.0, + use_psf_photometry=False, + use_aperture_photometry=True, + use_exactly_the_comps_provided=True, + ) + + assert calibration is not None + assert calibration["best_comp_index"] == 0 + assert calibration["comp_summaries"][0]["coverage_rejected"] is False + + def test_select_comparison_calibrated_photometry_masks_overexposed_comp_measurements(): frame_count = 24 airmass = np.linspace(1.2, 1.0, frame_count) diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 4853aab3..fc4dcf6b 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -29,6 +29,52 @@ def test_comparison_star_coords_accepts_more_than_ten_manual_comps(): assert inputs_module.comparison_star_coords(comp_stars, rt_bool=False) == comp_stars +def test_comparison_star_radec_coords_accepts_decimal_and_sexagesimal_pairs(): + coords = inputs_module.comparison_star_radec_coords([ + [31.04125, 46.68972], + ["02:04:09.90", "+46:41:23.0"], + [], + ]) + + assert coords[0] == pytest.approx([31.04125, 46.68972]) + assert coords[1][0] == pytest.approx(31.04125) + assert coords[1][1] == pytest.approx(46.6897222222) + + +def test_comp_params_accepts_comparison_radec_without_pixel_coordinates(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) X & Y Pixel": [], + "Comparison Star(s) RA & Dec": [[31.04125, 46.68972]], + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert np.allclose(inputs.info_dict["comp_stars_radec"], [[31.04125, 46.68972]]) + + +def test_comp_params_rejects_simultaneous_pixel_and_radec_comparisons(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) X & Y Pixel": [[465, 183]], + "Comparison Star(s) RA & Dec": [[31.04125, 46.68972]], + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + with pytest.raises(ValueError, match="either .*X & Y Pixel.*or .*RA & Dec.*not both"): + Inputs(init_opt="y").comp_params(init_file, {}) + + def test_comp_params_accepts_verbose_camera_key(tmp_path): init_data = { "user_info": { @@ -108,6 +154,41 @@ def test_comp_params_reads_stellar_variability_only_from_optional_info(tmp_path) assert inputs.info_dict["stellar_variability_only"] is True +def test_comp_params_defaults_apparent_and_exact_comparison_options(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["require_apparent_magnitudes"] is True + assert inputs.info_dict["use_exactly_the_comps_provided"] is False + + +def test_comp_params_reads_apparent_and_exact_comparison_options(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "require_apparent_magnitudes": False, + "use_exactly_the_comps_provided": True, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["require_apparent_magnitudes"] is False + assert inputs.info_dict["use_exactly_the_comps_provided"] is True + + def test_comp_params_defaults_stellar_variability_ensemble_to_true(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index b614fc1d..e697574a 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -1501,8 +1501,17 @@ class DummyFit: tmp_path, 'Host Star', observed_filter='V', + observation_date='2026-08-02', ) + differential_csv = next( + tmp_path.glob('StellarVariabilityDifferentialMagnitude_HostStar_2026-08-02.csv') + ) + assert '# AIRMASS_CORRECTION=NO' in differential_csv.read_text(encoding='utf-8') + assert ( + tmp_path / 'StellarVariabilityDifferentialMagnitude_HostStar_2026-08-02.png' + ).exists() + def test_build_stellar_variability_params_uses_raw_ratio_and_per_exposure_errors(monkeypatch, tmp_path): comp_mag = 9.751 @@ -1556,7 +1565,13 @@ class DummyFit: ), ) - np.testing.assert_allclose([row['mag'] for row in params], target_mag, atol=1.0e-10) + expected_raw_magnitudes = comp_mag - (2.5 * np.log10(detrended)) + np.testing.assert_allclose( + [row['mag'] for row in params], + expected_raw_magnitudes, + atol=1.0e-10, + ) + assert params[0]['mag'] != pytest.approx(target_mag) assert params[1]['mag_err'] == pytest.approx(expected_mag_error) assert params[1]['mag_err'] < 0.08 @@ -3379,18 +3394,24 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product observed_filter='V', ) - assert failed_results[0]['status'] == 'skipped' - assert failed_results[0]['output_directory'] is None - assert not (failed_root / 'variables' / 'optimal_variables' / 'SyntheticVSX').exists() + assert failed_results[0]['status'] == 'completed' + assert failed_results[0]['apparent_magnitude_point_count'] == 0 + assert failed_results[0]['apparent_magnitude_error'] + differential_only_dir = ( + failed_root / 'variables' / 'optimal_variables' / 'SyntheticVSX' + ) + assert differential_only_dir.exists() + assert next(differential_only_dir.glob('DifferentialMagnitude_*.csv')).is_file() + assert not list(differential_only_dir.glob('StellarVariability_*.csv')) + assert not list(differential_only_dir.glob('AID_AAVSO_*.txt')) failed_manifest = json.loads( next((failed_root / 'variables').glob('FortuitousVariables_2024-01-02.json')).read_text( encoding='utf-8' ) ) - assert failed_manifest['variables'][0]['status'] == 'skipped' - assert 'fewer than 1 independently calibrated comparison member' in ( - failed_manifest['variables'][0]['reason'] - ) + assert failed_manifest['variables'][0]['status'] == 'completed' + assert failed_manifest['variables'][0]['apparent_magnitude_point_count'] == 0 + assert failed_manifest['variables'][0]['differential_magnitude_csv'] def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(monkeypatch, tmp_path): @@ -3507,6 +3528,74 @@ def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(monkey assert 'AUID-TEST,' in aid_text +def test_stellar_variability_exact_comparisons_use_every_supplied_member_without_catalogue(): + frame_count = 8 + times = np.linspace(10.2, 10.3, frame_count) + quality_mask = np.ones(frame_count, dtype=bool) + target_flux = np.linspace(990.0, 1010.0, frame_count) + comparison_calibration = { + 'method': 'aperture', + 'method_label': 'Aperture photometry', + 'a': 0, + 'an': 0, + 'field_score': np.inf, + 'field_image_keep_mask': quality_mask, + 'comp_summaries': [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'aggregate_score': np.inf, 'coverage_rejected': True, + 'suitability_outlier_rejected': True, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'aggregate_score': np.inf, 'coverage_rejected': True, + 'suitability_outlier_rejected': True, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + ], + } + psf_data = { + 'target': np.ones((frame_count, 7), dtype=float), + 'comp1': np.ones((frame_count, 7), dtype=float), + 'comp2': np.ones((frame_count, 7), dtype=float), + } + aper_data = { + 'target': target_flux[:, None, None], + 'target_unc': np.ones((frame_count, 1, 1)), + 'comp1': np.full((frame_count, 1, 1), 500.0), + 'comp1_unc': np.ones((frame_count, 1, 1)), + 'comp2': np.full((frame_count, 1, 1), 250.0), + 'comp2_unc': np.ones((frame_count, 1, 1)), + } + + result = exotic_module.select_stellar_variability_only_photometry( + times, + times, + np.linspace(1.0, 1.5, frame_count), + _stellar_variability_only_planet_dict(), + comparison_calibration, + psf_data, + aper_data, + target_flux, + use_ensemble_photometry=True, + calibration_stars={}, + observed_filter='V', + require_apparent_magnitudes=False, + use_exactly_the_comps_provided=True, + ) + + selected = result['selected_result'] + assert result['selection_metric'] == 'exact_stellar_variability_ensemble' + assert selected['ensemble_member_keys'] == ['comp1', 'comp2'] + assert [ + member['key'] for member in selected['fit'].stellar_variability_ensemble_members + ] == ['comp1', 'comp2'] + assert np.all(np.isnan(selected['fit'].stellar_variability_ensemble_magnitudes)) + + def test_stellar_variability_selector_opt_out_restores_single_comp_selection(): frame_count = 24 times = np.linspace(10.2, 10.3, frame_count) diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 3780fcbe..f783f54a 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -16,7 +16,9 @@ build_aavso_qc_metadata, fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, + differential_magnitude_series_from_fit, save_comp_star_calibration_summary, + write_differential_magnitude_csv, ) from exotic.transit_depth import ( fit_transit_depth_summary, @@ -25,6 +27,59 @@ ) +def test_stellar_variability_differential_magnitudes_never_apply_airmass_correction(): + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2460000.1, 2460000.2, 2460000.3]), + data=np.ones(3), + dataerr=np.full(3, 0.01), + airmass=np.array([1.1, 1.3, 1.5]), + airmass_model=np.array([0.8, 1.0, 1.2]), + transit=np.ones(3), + stellar_variability_target_flux=np.array([80.0, 100.0, 120.0]), + stellar_variability_comp_flux=np.full(3, 100.0), + stellar_variability_target_flux_error=np.ones(3), + stellar_variability_comp_flux_error=np.ones(3), + ) + + series = differential_magnitude_series_from_fit(fit) + + np.testing.assert_allclose( + series['magnitude'], + -2.5 * np.log10(fit.stellar_variability_target_flux / fit.stellar_variability_comp_flux), + ) + assert series['airmass_corrected'] is False + + +def test_differential_csv_does_not_require_apparent_magnitude_calibration(tmp_path): + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2460000.1, 2460000.2]), + data=np.ones(2), + dataerr=np.full(2, 0.01), + airmass=np.array([1.1, 1.2]), + airmass_model=np.array([0.9, 1.1]), + transit=np.ones(2), + stellar_variability_target_flux=np.array([500.0, 550.0]), + stellar_variability_comp_flux=np.array([1000.0, 1000.0]), + stellar_variability_target_flux_error=np.full(2, 2.0), + stellar_variability_comp_flux_error=np.full(2, 3.0), + ) + + output_path = write_differential_magnitude_csv( + fit, + tmp_path, + 'Variable Star', + observation_date='2026-08-02', + observed_filter='V', + ) + + output_text = output_path.read_text(encoding='utf-8') + assert '# AIRMASS_CORRECTION=NO' in output_text + assert 'Differential Magnitude' in output_text + assert 'Apparent' not in output_text + + class DummyFit: def __init__(self): self.parameters = { diff --git a/tests/test_plots.py b/tests/test_plots.py index 5076576f..ea4d63f8 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -19,6 +19,7 @@ plot_ktmf_qc_metrics, plot_obs_stats, plot_prior_posterior_comparison, + plot_differential_magnitude, plot_stellar_variability, ) @@ -165,6 +166,37 @@ def spy_invert_yaxis(self, *args, **kwargs): assert (tmp_path / "FinalLightCurve_Target_2026-07-08.png").exists() +def test_stellar_variability_differential_plot_survives_without_apparent_magnitudes(tmp_path): + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2461229.5, 2461229.6, 2461229.8]), + data=np.ones(3), + dataerr=np.full(3, 0.001), + airmass=np.array([1.1, 1.2, 1.3]), + airmass_model=np.array([0.8, 1.0, 1.2]), + transit=np.ones(3), + stellar_variability_target_flux=np.array([900.0, 1000.0, 1100.0]), + stellar_variability_comp_flux=np.full(3, 1000.0), + stellar_variability_target_flux_error=np.ones(3), + stellar_variability_comp_flux_error=np.ones(3), + stellar_variability_params=[], + ) + + output_path = plot_differential_magnitude( + fit, + 'Variable Star', + tmp_path, + '2026-08-02', + observed_filter='V', + ) + + assert output_path.exists() + assert (tmp_path / 'Stellar_Variability_DifferentialMagnitude.png').exists() + assert ( + tmp_path / 'working_artifacts' / 'Stellar_Variability_DifferentialMagnitude.png' + ).exists() + + def test_plot_obs_stats_uses_supplied_background_series(tmp_path, monkeypatch): fit = DummyFit() psf_rows = np.arange(35, dtype=float).reshape(5, 7) @@ -367,6 +399,7 @@ def spy_invert_yaxis(self, *args, **kwargs): "Label: NextAstro-123\n" "Comparison RA=10.100000\n" "Dec=-20.200000\n" + "No airmass correction applied to stellar variability\n" "Original filter: CV | Comparison mag: r=12.345 +/- 0.067" ) assert ylabels[-1] == "Magnitude (r)" @@ -456,6 +489,7 @@ def spy_set_title(self, label, *args, **kwargs): assert titles[-1] == ( "Host Star\n" "Label: NextAstro-123\nComparison RA=10.100000\nDec=-20.200000\n" + "No airmass correction applied to stellar variability\n" "Original filter: MObs CV" ) assert "99.99" not in titles[-1] From 3850f2dd3d203384488dc2ad4f12f4caf5429bf6 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 2 Aug 2026 22:57:07 +1000 Subject: [PATCH 102/116] Just some ID when requesting astrometry --- exotic/api/plate_solution.py | 12 ++++++++++++ tests/test_nextastro_astrometry.py | 3 +++ 2 files changed, 15 insertions(+) diff --git a/exotic/api/plate_solution.py b/exotic/api/plate_solution.py index c0845a71..c1d1294d 100644 --- a/exotic/api/plate_solution.py +++ b/exotic/api/plate_solution.py @@ -51,6 +51,14 @@ except ImportError: from http_compression import build_compressed_json_request +try: + from ..version import __version__ +except ImportError: + try: + from version import __version__ + except ImportError: + __version__ = "unknown" + _R_MAX_STOPS_LOW = 7 _R_MAX_STOPS = 10 _R_MAX_SECS = 37 @@ -59,6 +67,7 @@ _NEXTASTRO_STATUS_MAX_POLLS = 60 _NEXTASTRO_STATUS_POLL_SEC = 2 _NEXTASTRO_IN_PROGRESS_STATUSES = {'queued', 'running'} +_NEXTASTRO_SOFTWARE_NAME = f"EXOTIC/{__version__}" def is_false(value): @@ -277,6 +286,7 @@ def _generate_source_list(self): "x": bright_sources["x"], "y": bright_sources["y"], "flux": bright_sources["flux"], + "origin": "exotic", "pixel_indexing": "0-based" } @@ -302,6 +312,7 @@ def _fallback_source_list(self, image_data, median, std): "x": bright_sources["x"], "y": bright_sources["y"], "flux": bright_sources["flux"], + "origin": "exotic", "pixel_indexing": "0-based" } @@ -370,6 +381,7 @@ def _submit_solve_request(self, source_list): payload["hints"] = hints request_body, headers, content_encoding, raw_size, compressed_size = build_compressed_json_request(payload) + headers["X-NextAstro-Software"] = _NEXTASTRO_SOFTWARE_NAME self._emit_debug(f"NextAstro astrometry request JSON: {self._json_message(payload)}") self._emit_debug( diff --git a/tests/test_nextastro_astrometry.py b/tests/test_nextastro_astrometry.py index a3eb7b7a..3ea5afa4 100644 --- a/tests/test_nextastro_astrometry.py +++ b/tests/test_nextastro_astrometry.py @@ -50,6 +50,7 @@ def test_generate_source_list(tmp_path): assert source_list is not None assert source_list["pixel_indexing"] == "0-based" + assert source_list["origin"] == "exotic" assert len(source_list["x"]) > 0 assert len(source_list["x"]) == len(source_list["y"]) == len(source_list["flux"]) @@ -63,6 +64,8 @@ def fake_post(url, data, headers, timeout): assert url.endswith('/solve') assert headers["Content-Type"] == "application/json" assert headers["Content-Encoding"] in {"gzip", "zstd"} + assert headers["X-NextAstro-Software"].startswith("EXOTIC/") + assert payload["sources"]["origin"] == "exotic" assert payload['image'] == {'width': 120, 'height': 100} assert payload['hints']['ra_deg'] == 210.8023 assert payload['hints']['dec_deg'] == 54.3489 From dce494d954247c74ba2c9efb5d0f753fda2cf50f Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 3 Aug 2026 08:33:52 +1000 Subject: [PATCH 103/116] RA, Dec in sexigesimal. Stop expanding comps in exact mode. --- README.md | 4 +- docs/README.md | 4 +- exotic/exotic.py | 123 ++++++++- exotic/output_files.py | 393 ++++++++++++++++++++++++---- tests/test_exotic_proper_motion.py | 73 ++++++ tests/test_inputs.py | 24 ++ tests/test_nextastro_variability.py | 27 +- tests/test_output_files.py | 127 ++++++++- 8 files changed, 717 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index d3b519a0..201d19c1 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended; EXOTIC also accepts equivalent values such as `"y"` and `"n"`. -Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Celestial coordinates require a usable WCS and are projected onto the selected reference image before photometry; after projection they are treated identically to supplied X/Y positions. +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Sexagesimal values must be quoted because they are JSON strings; forms such as `[[02:04:09.90, +46:41:23.0]]` are not valid JSON. Celestial coordinates require a usable WCS and are projected onto the selected reference image before photometry; after projection they are treated identically to supplied X/Y positions. | Reduction | Requested comparison mode | `optional_info` settings | |---|---|---| @@ -215,6 +215,8 @@ Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. +Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. + For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. `photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. diff --git a/docs/README.md b/docs/README.md index a404d20e..f4134640 100644 --- a/docs/README.md +++ b/docs/README.md @@ -256,10 +256,12 @@ The two ensemble limits are independent. `"maximum_number_of_ensemble_comparison Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble, except when `"use_exactly_the_comps_provided"` is true. Exact-comparison mode fails explicitly if the supplied reference cannot be measured; it never silently substitutes or drops a supplied comparison. This makes the same reference star or ensemble reproducible across multiple runs. -Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Celestial coordinates require a usable WCS and are projected onto the selected reference image before photometry; after projection they are treated identically to supplied X/Y positions. +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Sexagesimal values must be quoted because they are JSON strings; forms such as `[[02:04:09.90, +46:41:23.0]]` are not valid JSON. Celestial coordinates require a usable WCS and are projected onto the selected reference image before photometry; after projection they are treated identically to supplied X/Y positions. Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. +Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. + For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. `photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. diff --git a/exotic/exotic.py b/exotic/exotic.py index 8bd5d2bd..066b0f8d 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -157,6 +157,7 @@ OutputFiles, AIDOutputFiles, empirical_red_noise_error_scale, + differential_magnitude_series_from_fit, fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, fit_parameter_model_data_uncertainty, @@ -170,6 +171,7 @@ OutputFiles, AIDOutputFiles, empirical_red_noise_error_scale, + differential_magnitude_series_from_fit, fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, fit_parameter_model_data_uncertainty, @@ -21712,7 +21714,8 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ # detrending model could suppress the astrophysical signal. calibrated_ratio = raw_ratio with np.errstate(divide='ignore', invalid='ignore'): - target_mag = comp_mag - (2.5 * np.log10(calibrated_ratio)) + differential_mag = -2.5 * np.log10(calibrated_ratio) + target_mag = comp_mag + differential_mag magnitude_factor = 2.5 / np.log(10.0) explicit_flux_error = magnitude_factor * np.sqrt( (target_flux_error / target_flux) ** 2 @@ -21752,17 +21755,21 @@ def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_ display_label = stellar_variability_label(comp_label, comp_star) vsp_params = [] - for time_value, airmass_value, mag_value, mag_error_value in zip( + for time_value, airmass_value, mag_value, mag_error_value, differential_value, differential_error in zip( selected_times[valid], selected_airmass[valid], target_mag[valid], target_mag_error[valid], + differential_mag[valid], + flux_error[valid], ): vsp_params.append({ 'time': time_value, 'airmass': airmass_value, 'mag': mag_value, 'mag_err': mag_error_value, + 'differential_mag': differential_value, + 'differential_mag_err': differential_error, 'cname': display_label, 'cmag': comp_mag, 'cmag_err': comp_mag_error, @@ -24328,6 +24335,20 @@ def merge_automatic_comparison_star_coords(primary_stars, additional_stars, return merged, messages +def build_tracked_comparison_pool(science_comp_stars, automatic_comp_stars, + use_exactly_the_comps_provided=False, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + """Build the tracked comparison pool without expanding an exact supplied set.""" + supplied_stars = [list(position) for position in (science_comp_stars or [])] + if use_exactly_the_comps_provided: + return supplied_stars, [] + return merge_automatic_comparison_star_coords( + supplied_stars, + automatic_comp_stars, + duplicate_radius_pixels=duplicate_radius_pixels, + ) + + def fortuitous_variable_overlap(position, fortuitous_variables, duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): """Return the closest full-field VSX variable matching a tracked pixel position.""" @@ -28283,13 +28304,32 @@ def save_stellar_variability_magnitude_csv(vsp_params, save, target_name, observ ) with output_path.open('w', encoding='utf-8', newline='') as handle: writer = csv.writer(handle) - writer.writerow(['BJD_TDB', 'Airmass', 'Magnitude', 'Magnitude Error', 'Filter', 'Comparison']) + writer.writerow([ + 'BJD_TDB', + 'Airmass', + 'Apparent Magnitude', + 'Apparent Magnitude Error', + 'Differential Magnitude', + 'Differential Magnitude Error', + 'Filter', + 'Comparison', + ]) for row in vsp_params: + differential_mag = row.get( + 'differential_mag', + row.get('differential_magnitude'), + ) + differential_mag_err = row.get( + 'differential_mag_err', + row.get('differential_magnitude_error'), + ) writer.writerow([ row.get('time'), row.get('airmass'), row.get('mag'), row.get('mag_err'), + differential_mag, + differential_mag_err, row.get('mag_band') or row.get('observed_filter'), row.get('cname'), ]) @@ -28425,18 +28465,38 @@ def build_stellar_variability_ensemble_params_from_fit( observed_filter, fallback_band=catalog_mag_band, ) + differential_magnitudes = np.full(times.shape, np.nan, dtype=float) + differential_magnitude_errors = np.full(times.shape, np.nan, dtype=float) + differential_series = differential_magnitude_series_from_fit( + lc_fit, + apply_airmass_correction=False, + ) + if differential_series is not None: + differential_source_mask = np.asarray( + differential_series.get('source_mask', []), + dtype=bool, + ) + if differential_source_mask.shape == times.shape: + differential_magnitudes[differential_source_mask] = differential_series['magnitude'] + differential_magnitude_errors[differential_source_mask] = ( + differential_series['magnitude_error'] + ) vsp_params = [] - for time_value, airmass_value, magnitude, magnitude_error in zip( + for time_value, airmass_value, magnitude, magnitude_error, differential_mag, differential_mag_err in zip( times[valid], airmass[valid], magnitudes[valid], magnitude_errors[valid], + differential_magnitudes[valid], + differential_magnitude_errors[valid], ): vsp_params.append({ 'time': time_value, 'airmass': airmass_value, 'mag': magnitude, 'mag_err': magnitude_error, + 'differential_mag': differential_mag, + 'differential_mag_err': differential_mag_err, 'cname': display_label, 'cmag': None, 'cmag_err': None, @@ -31495,6 +31555,11 @@ def _main_impl(): list(coords) for coords in (exotic_infoDict.get('comp_stars_radec') or []) ] comparisons_supplied_as_radec = bool(provided_comparison_radec) + provided_comparison_pixels = ( + [] + if comparisons_supplied_as_radec + else [list(coords) for coords in (exotic_infoDict.get('comp_stars') or [])] + ) provided_comparison_count = ( len(provided_comparison_radec) if comparisons_supplied_as_radec @@ -32156,6 +32221,9 @@ def lookup_archive_ephemeris(): error=True, ) return + provided_comparison_pixels = [ + list(coords) for coords in exotic_infoDict['comp_stars'] + ] plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) log_info( f"Projected {len(exotic_infoDict['comp_stars'])} supplied comparison-star " @@ -32454,7 +32522,15 @@ def lookup_archive_ephemeris(): ) is None } - if stellar_variability_ensemble_candidate_search: + if use_exactly_the_comps_provided: + fortuitous_auto_stars = [] + fortuitous_auto_scan_performed = True + log_info( + "Exact supplied-comparison mode: fortuitous-variable photometry will reuse " + "only the supplied comparison stars; skipping automatic comparison-pool " + "expansion." + ) + elif stellar_variability_ensemble_candidate_search: # The stellar-variability target path just selected and VSX-vetted the # same brightest-first pool with the same count and saturation limit. # Reuse it rather than performing an identical full-field image scan. @@ -32510,9 +32586,10 @@ def lookup_archive_ephemeris(): warn=True, ) fortuitous_ensemble_stars, fortuitous_duplicate_messages = ( - merge_automatic_comparison_star_coords( + build_tracked_comparison_pool( science_comp_stars, fortuitous_auto_stars, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, ) ) @@ -32553,6 +32630,7 @@ def lookup_archive_ephemeris(): ).upper() == 'V' and not usable_science_nextastro_v and not fortuitous_auto_scan_performed + and not use_exactly_the_comps_provided ): fortuitous_auto_scan_performed = True fortuitous_comp_count = parse_automatic_calibration_selector_count( @@ -32599,9 +32677,10 @@ def lookup_archive_ephemeris(): warn=True, ) fortuitous_ensemble_stars, duplicate_messages = ( - merge_automatic_comparison_star_coords( + build_tracked_comparison_pool( science_comp_stars, fortuitous_auto_stars, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, ) ) @@ -32888,6 +32967,23 @@ def lookup_archive_ephemeris(): if not pick_comparison_by_eebls_snr: log_info("Comparison-star selection by EEBLS SNR disabled per optional_info setting.") if use_exactly_the_comps_provided: + exact_set_preserved = ( + len(science_comp_stars) == len(provided_comparison_pixels) + and np.allclose( + np.asarray(science_comp_stars, dtype=float), + np.asarray(provided_comparison_pixels, dtype=float), + rtol=0.0, + atol=1e-9, + ) + ) + if not exact_set_preserved: + log_info( + "Error: the supplied comparison-star set changed before photometry; " + "exact-comparison mode will not continue with an added, removed, or " + "substituted comparison.", + error=True, + ) + return exact_mode = "single comparison" if len(science_comp_stars) == 1 else "fixed ensemble" log_info( "Exact supplied-comparison mode enabled: EXOTIC will use the " @@ -32895,6 +32991,19 @@ def lookup_archive_ephemeris(): "without automatic replacement, VSX rejection, stability vetting, ranking, or " "ensemble-size limiting." ) + for comp_index, pixel_position in enumerate(science_comp_stars, start=1): + if comparisons_supplied_as_radec: + ra_deg, dec_deg = provided_comparison_radec[comp_index - 1] + log_info( + f" Exact supplied comp #{comp_index}: " + f"RA={ra_deg:.8f} deg, Dec={dec_deg:.8f} deg -> " + f"pixels=[{pixel_position[0]:.3f}, {pixel_position[1]:.3f}]" + ) + else: + log_info( + f" Exact supplied comp #{comp_index}: " + f"pixels=[{pixel_position[0]:.3f}, {pixel_position[1]:.3f}]" + ) elif use_ensemble_photometry_rather_than_single_comp: log_info( "Ensemble comparison photometry enabled per optional_info setting; the final target " diff --git a/exotic/output_files.py b/exotic/output_files.py index 12bc52df..e6f8b947 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -104,6 +104,8 @@ def finite_float(value, default=np.nan): def apparent_magnitude_calibration_from_vsp_params(vsp_params): rows = [] + zero_points = [] + zero_point_errors = [] for vsp_p in vsp_params or []: mag = finite_float(vsp_p.get('mag')) mag_err = normalized_magnitude_error(vsp_p.get('mag_err')) @@ -111,6 +113,33 @@ def apparent_magnitude_calibration_from_vsp_params(vsp_params): continue rows.append((mag, mag_err, vsp_p.get('mag_band') or 'V')) + differential_mag, differential_err = differential_magnitude_from_vsp_param( + vsp_p + ) + if np.isfinite(differential_mag): + comparison_mag = finite_float(vsp_p.get('cmag')) + comparison_mag_err = normalized_magnitude_error(vsp_p.get('cmag_err')) + zero_points.append( + comparison_mag if np.isfinite(comparison_mag) else mag - differential_mag + ) + if comparison_mag_err is not None: + zero_point_errors.append(comparison_mag_err) + elif np.isfinite(differential_err): + zero_point_errors.append( + np.sqrt(max(mag_err ** 2 - differential_err ** 2, 0.0)) + ) + else: + zero_point_errors.append(mag_err) + continue + + comparison_mag = finite_float(vsp_p.get('cmag')) + comparison_mag_err = normalized_magnitude_error(vsp_p.get('cmag_err')) + if np.isfinite(comparison_mag): + zero_points.append(comparison_mag) + zero_point_errors.append( + comparison_mag_err if comparison_mag_err is not None else mag_err + ) + if not rows: return None @@ -120,9 +149,51 @@ def apparent_magnitude_calibration_from_vsp_params(vsp_params): 'baseline_magnitude': float(np.nanmedian(magnitudes)), 'baseline_error': float(np.nanmedian(magnitude_errors)), 'band': rows[0][2], + 'zero_point_magnitude': ( + float(np.nanmedian(zero_points)) if zero_points else np.nan + ), + 'zero_point_error': ( + float(np.nanmedian(zero_point_errors)) if zero_point_errors else np.nan + ), } +def differential_magnitude_from_vsp_param(vsp_param): + """Return target-minus-reference magnitude and its flux-only uncertainty.""" + differential_mag = finite_float( + vsp_param.get( + 'differential_mag', + vsp_param.get('differential_magnitude'), + ) + ) + differential_err = finite_float( + vsp_param.get( + 'differential_mag_err', + vsp_param.get('differential_magnitude_error'), + ) + ) + if np.isfinite(differential_err) and differential_err < 0: + differential_err = np.nan + + if np.isfinite(differential_mag): + return differential_mag, differential_err + + apparent_mag = finite_float(vsp_param.get('mag')) + comparison_mag = finite_float(vsp_param.get('cmag')) + if not (np.isfinite(apparent_mag) and np.isfinite(comparison_mag)): + return np.nan, np.nan + + apparent_err = normalized_magnitude_error(vsp_param.get('mag_err')) + comparison_err = normalized_magnitude_error(vsp_param.get('cmag_err')) + if apparent_err is not None and comparison_err is not None: + differential_err = np.sqrt(max(apparent_err ** 2 - comparison_err ** 2, 0.0)) + elif apparent_err is not None: + differential_err = apparent_err + else: + differential_err = np.nan + return apparent_mag - comparison_mag, differential_err + + def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, apply_airmass_correction=None): """Return target-minus-reference instrumental magnitudes. @@ -133,7 +204,10 @@ def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, Stellar-variability fits intentionally remain uncorrected for airmass so a real time-dependent stellar signal is not fitted away. """ - fit_data = np.asarray(getattr(fit, 'data', []), dtype=float).reshape(-1) + fit_data = np.asarray( + getattr(fit, 'data', getattr(fit, 'detrended', [])), + dtype=float, + ).reshape(-1) fit_times = np.asarray( getattr(fit, 'time', getattr(fit, 'jd_times', [])), dtype=float, @@ -260,9 +334,119 @@ def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, 'magnitude_error': differential_error[keep], 'source_mask': keep, 'airmass_corrected': apply_airmass_correction, + 'has_raw_photometry': has_raw_photometry, } +def magnitude_series_from_fit(fit, out_of_transit_only=False, + apply_airmass_correction=None): + """Return aligned differential and, when calibrated, apparent magnitudes.""" + fit_data = np.asarray( + getattr(fit, 'data', getattr(fit, 'detrended', [])), + dtype=float, + ).reshape(-1) + result = { + 'differential_magnitude': np.full(fit_data.shape, np.nan, dtype=float), + 'differential_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'apparent_magnitude': np.full(fit_data.shape, np.nan, dtype=float), + 'apparent_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'band': None, + 'airmass_corrected': False, + 'has_raw_photometry': False, + 'apparent_calibrated': False, + } + series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=apply_airmass_correction, + ) + if series is None: + return result + + source_mask = np.asarray(series['source_mask'], dtype=bool) + if source_mask.shape != fit_data.shape: + return result + result['differential_magnitude'][source_mask] = series['magnitude'] + result['differential_magnitude_error'][source_mask] = series['magnitude_error'] + result['airmass_corrected'] = bool(series['airmass_corrected']) + result['has_raw_photometry'] = bool(series['has_raw_photometry']) + + calibration = apparent_magnitude_calibration_from_vsp_params( + getattr(fit, 'stellar_variability_params', None) + ) + if calibration is None: + return result + + magnitude_offset = np.nan + calibration_error = calibration['baseline_error'] + if np.isfinite(calibration['zero_point_error']): + calibration_error = calibration['zero_point_error'] + + differential_magnitude = result['differential_magnitude'] + differential_error = result['differential_magnitude_error'] + fit_times = np.asarray( + getattr(fit, 'time', getattr(fit, 'jd_times', [])), + dtype=float, + ).reshape(-1) + matched_offsets = [] + if result['has_raw_photometry'] and fit_times.shape == differential_magnitude.shape: + for vsp_param in getattr(fit, 'stellar_variability_params', None) or []: + apparent_mag = finite_float(vsp_param.get('mag')) + apparent_time = finite_float(vsp_param.get('time')) + if not (np.isfinite(apparent_mag) and np.isfinite(apparent_time)): + continue + matches = np.flatnonzero(np.isclose( + fit_times, + apparent_time, + rtol=0.0, + atol=1.0e-7, + )) + if matches.size == 0: + continue + matched_differential = differential_magnitude[matches[0]] + if np.isfinite(matched_differential): + matched_offsets.append(apparent_mag - matched_differential) + if not result['has_raw_photometry']: + magnitude_offset = calibration['baseline_magnitude'] + elif matched_offsets: + magnitude_offset = float(np.nanmedian(matched_offsets)) + else: + reference_mask = np.isfinite(differential_magnitude) + transit_model = np.asarray( + getattr(fit, 'transit', np.ones(fit_data.shape)), + dtype=float, + ).reshape(-1) + if transit_model.shape == fit_data.shape and np.any(transit_model == 1): + reference_mask &= transit_model == 1 + if np.any(reference_mask): + magnitude_offset = ( + calibration['baseline_magnitude'] + - float(np.nanmedian(differential_magnitude[reference_mask])) + ) + if not np.isfinite(magnitude_offset): + return result + + finite_differential = np.isfinite(differential_magnitude) + result['apparent_magnitude'][finite_differential] = ( + differential_magnitude[finite_differential] + magnitude_offset + ) + if np.isfinite(calibration_error): + finite_error = finite_differential & np.isfinite(differential_error) + result['apparent_magnitude_error'][finite_error] = np.hypot( + differential_error[finite_error], + calibration_error, + ) + missing_error = finite_differential & ~np.isfinite(differential_error) + result['apparent_magnitude_error'][missing_error] = calibration_error + else: + result['apparent_magnitude_error'][finite_differential] = ( + differential_error[finite_differential] + ) + result['band'] = calibration['band'] + result['apparent_calibrated'] = True + return result + + def write_differential_magnitude_csv(fit, save, target_name, observation_date=None, observed_filter=None, out_of_transit_only=False, apply_airmass_correction=None, @@ -543,8 +727,8 @@ def prune_aavso_metadata(value): return aavso_json_safe(value) -def format_aavso_json_header(name, payload): - payload = prune_aavso_metadata(payload) +def format_aavso_json_header(name, payload, preserve_nulls=False): + payload = aavso_json_safe(payload) if preserve_nulls else prune_aavso_metadata(payload) if not payload: return "" return f"#{name}={dumps(payload, sort_keys=True)}\n" @@ -1979,62 +2163,96 @@ def final_lightcurve(self, phase): with params_file.open('w') as f: target_name = self.p_dict.get('sName', self.p_dict['pName']) f.write(f"# FINAL STELLAR VARIABILITY TIMESERIES OF {target_name}\n") - f.write("# BJD_TDB,Magnitude,Uncertainty,Band,Airmass\n") + reference_label = ( + getattr(self.fit, 'stellar_variability_reference_label', None) + or (vsp_params[0].get('cname') if vsp_params else None) + or 'selected comparison reference' + ) + f.write(f"# DIFFERENTIAL_MAGNITUDE_REFERENCE={reference_label}\n") + f.write("# DIFFERENTIAL_MAGNITUDE_AIRMASS_CORRECTED=NO\n") + f.write( + "# BJD_TDB,Apparent Magnitude,Apparent Magnitude Uncertainty," + "Differential Magnitude,Differential Magnitude Uncertainty,Band,Airmass\n" + ) for vsp_p in vsp_params: time_value = finite_float(vsp_p.get('time')) mag_value = format_magnitude(vsp_p.get('mag'), default=None) mag_error = format_magnitude_error(vsp_p.get('mag_err'), default=None) if not np.isfinite(time_value) or mag_value is None or mag_error is None: continue + differential_mag, differential_error = differential_magnitude_from_vsp_param( + vsp_p + ) + differential_mag_text = format_magnitude( + differential_mag, + default="na", + digits=6, + ) + differential_error_text = format_magnitude_error( + differential_error, + default="na", + digits=6, + ) band = vsp_p.get('mag_band') or self.i_dict.get('filter') or 'V' airmass = finite_float(vsp_p.get('airmass')) airmass_text = f"{airmass}" if np.isfinite(airmass) else "na" - f.write(f"{time_value}, {mag_value}, {mag_error}, {band}, {airmass_text}\n") + f.write( + f"{time_value}, {mag_value}, {mag_error}, {differential_mag_text}, " + f"{differential_error_text}, {band}, {airmass_text}\n" + ) return - magnitude_calibration = apparent_magnitude_calibration_from_vsp_params( - getattr(self.fit, 'stellar_variability_params', None) - ) + magnitude_series = magnitude_series_from_fit(self.fit) + band = magnitude_series['band'] or self.i_dict.get('filter') or 'na' with params_file.open('w') as f: f.write(f"# FINAL TIMESERIES OF {self.p_dict['pName']}\n") - if magnitude_calibration is None: - f.write("# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass\n") - else: - f.write( - "# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass," - "Apparent Magnitude,Magnitude Uncertainty,Band\n" - ) + reference_label = ( + getattr(self.fit, 'differential_magnitude_reference_label', None) + or getattr(self.fit, 'stellar_variability_reference_label', None) + or 'selected comparison reference' + ) + f.write(f"# DIFFERENTIAL_MAGNITUDE_REFERENCE={reference_label}\n") + f.write( + "# DIFFERENTIAL_MAGNITUDE_AIRMASS_CORRECTED=" + f"{'YES' if magnitude_series['airmass_corrected'] else 'NO'}\n" + ) + f.write( + "# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass," + "Differential Magnitude,Differential Magnitude Uncertainty," + "Apparent Magnitude,Apparent Magnitude Uncertainty,Band\n" + ) - for bjd, phase, flux, fluxerr, model, am in zip(self.fit.time, phase, self.fit.detrended, - self.fit.dataerr / self.fit.airmass_model, - self.fit.transit, self.fit.airmass_model): + for row_index, (bjd, phase, flux, fluxerr, model, am) in enumerate(zip( + self.fit.time, + phase, + self.fit.detrended, + self.fit.dataerr / self.fit.airmass_model, + self.fit.transit, + self.fit.airmass_model)): row = f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}" - if magnitude_calibration is not None: - flux_value = finite_float(flux) - flux_error = finite_float(fluxerr) - if np.isfinite(flux_value) and flux_value > 0: - apparent_mag = ( - magnitude_calibration['baseline_magnitude'] - - (2.5 * np.log10(flux_value)) - ) - if np.isfinite(flux_error) and flux_error >= 0: - flux_mag_error = abs(2.5 * flux_error / (flux_value * np.log(10))) - apparent_mag_error = ( - magnitude_calibration['baseline_error'] ** 2 - + flux_mag_error ** 2 - ) ** 0.5 - else: - apparent_mag_error = magnitude_calibration['baseline_error'] - mag_text = format_magnitude(apparent_mag, default="na") - mag_error_text = format_magnitude_error(apparent_mag_error, default="na") - else: - mag_text = "na" - mag_error_text = "na" - row = ( - f"{row}, {mag_text}, {mag_error_text}, " - f"{magnitude_calibration['band']}" - ) + differential_mag_text = format_magnitude( + magnitude_series['differential_magnitude'][row_index], + default="na", + digits=6, + ) + differential_error_text = format_magnitude_error( + magnitude_series['differential_magnitude_error'][row_index], + default="na", + digits=6, + ) + apparent_mag_text = format_magnitude( + magnitude_series['apparent_magnitude'][row_index], + default="na", + ) + apparent_error_text = format_magnitude_error( + magnitude_series['apparent_magnitude_error'][row_index], + default="na", + ) + row = ( + f"{row}, {differential_mag_text}, {differential_error_text}, " + f"{apparent_mag_text}, {apparent_error_text}, {band}" + ) f.write(f"{row}\n") def differential_magnitude(self): @@ -2534,6 +2752,7 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, frame_filtering_metadata = build_aavso_frame_filtering_metadata(self.fit, frame_filtering_info) astrometry_metadata = build_aavso_astrometry_metadata(astrometry_info, comp_star) bad_pixel_metadata = build_aavso_bad_pixel_metadata(bad_pixel_info) + magnitude_series = magnitude_series_from_fit(self.fit) obs_name = format_aavso_header_value(self.i_dict.get('obs_name')) obs_name_header = f"#OBSNAME={obs_name}\n" if obs_name else "" gaia_dist = format_aavso_header_value(self.p_dict.get('dist')) @@ -2600,6 +2819,41 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, f.write(format_aavso_json_header("FRAME_FILTERING-XC", frame_filtering_metadata)) f.write(format_aavso_json_header("ASTROMETRY-XC", astrometry_metadata)) f.write(format_aavso_json_header("BAD_PIXEL-XC", bad_pixel_metadata)) + f.write(format_aavso_json_header("MAGNITUDE_FIELDS-XC", { + 'apparent_magnitude': 'catalogue-calibrated target magnitude', + 'apparent_magnitude_error': 'flux and catalogue calibration uncertainty', + 'differential_magnitude': ( + 'target minus selected comparison reference; ' + '-2.5 log10(target_flux/reference_flux)' + ), + 'differential_magnitude_error': 'flux-only uncertainty', + 'per_point_header': 'MAGNITUDE-XC', + 'band': magnitude_series['band'] or self.i_dict.get('filter'), + 'airmass_corrected': magnitude_series['airmass_corrected'], + 'apparent_calibrated': magnitude_series['apparent_calibrated'], + 'comparison_reference': ( + getattr(self.fit, 'differential_magnitude_reference_label', None) + or getattr(self.fit, 'stellar_variability_reference_label', None) + or 'selected comparison reference' + ), + })) + for magnitude_index in range(0, len(self.fit.time)): + f.write(format_aavso_json_header("MAGNITUDE-XC", { + 'date_bjd_tdb': finite_float(self.fit.time[magnitude_index]), + 'differential_magnitude': finite_float( + magnitude_series['differential_magnitude'][magnitude_index] + ), + 'differential_magnitude_error': finite_float( + magnitude_series['differential_magnitude_error'][magnitude_index] + ), + 'apparent_magnitude': finite_float( + magnitude_series['apparent_magnitude'][magnitude_index] + ), + 'apparent_magnitude_error': finite_float( + magnitude_series['apparent_magnitude_error'][magnitude_index] + ), + 'band': magnitude_series['band'] or self.i_dict.get('filter'), + }, preserve_nulls=True)) if epw_md5: f.write(f"#EPW_MD5-XC={dumps({'epw_checkout_md5': epw_md5})}\n") @@ -2651,6 +2905,14 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta first_vsp_param = self.vsp_params[0] if self.vsp_params else {} comparison_metadata = aid_comparison_metadata(first_vsp_param) ensemble_comparison_metadata = aid_ensemble_comparison_metadata(first_vsp_param) + fallback_differential_series = differential_magnitude_series_from_fit( + self.fit, + apply_airmass_correction=False, + ) + fallback_times = np.asarray( + [] if fallback_differential_series is None else fallback_differential_series['time'], + dtype=float, + ) comparison_coordinate_headers = aid_comparison_coordinate_headers( self.vsp_params, indexed=use_row_names, @@ -2684,6 +2946,17 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta "ENSEMBLE-COMPARISONS-XC", ensemble_comparison_metadata, )) + f.write(format_aavso_json_header("MAGNITUDE_FIELDS-XC", { + 'apparent_magnitude': 'MAG', + 'apparent_magnitude_error': 'MERR', + 'differential_magnitude': 'NOTES subfield DIFFMAG', + 'differential_magnitude_error': 'NOTES subfield DIFFERR', + 'differential_magnitude_definition': ( + 'target minus selected comparison reference; ' + '-2.5 log10(target_flux/reference_flux)' + ), + 'airmass_corrected': False, + })) f.write("#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,GROUP,CHART,NOTES\n") for vsp_p in self.vsp_params: @@ -2696,9 +2969,41 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta mag_err = format_magnitude_error(vsp_p.get('mag_err'), digits=4) cmag = format_magnitude(vsp_p.get('cmag')) chart_id = self.chart_id or vsp_p.get('chart_id') or 'na' + differential_mag, differential_error = differential_magnitude_from_vsp_param( + vsp_p + ) + if not np.isfinite(differential_mag) and fallback_times.size: + row_time = finite_float(vsp_p.get('time')) + time_matches = np.flatnonzero(np.isclose( + fallback_times, + row_time, + rtol=0.0, + atol=5.0e-5, + )) + if time_matches.size: + fallback_index = time_matches[0] + differential_mag = fallback_differential_series['magnitude'][fallback_index] + differential_error = ( + fallback_differential_series['magnitude_error'][fallback_index] + ) + differential_mag_text = format_magnitude( + differential_mag, + default=None, + digits=6, + ) + differential_error_text = format_magnitude_error( + differential_error, + default=None, + digits=6, + ) + notes = 'na' + if differential_mag_text is not None: + notes = f"|DIFFMAG={differential_mag_text}" + if differential_error_text is not None: + notes += f"|DIFFERR={differential_error_text}" f.write(f"{variable_name},{round(vsp_p['time'], 5)},{mag},{mag_err}," f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{cmag},na,na," - f"{round(vsp_p['airmass'], 7)},na,{chart_id},na\n") + f"{round(vsp_p['airmass'], 7)},na,{chart_id},{notes}\n") return params_file def aavso(self): diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 28124970..75e12126 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -131,6 +131,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: alignment_candidate_quality_score, aperture_estimation_comparison_stars, aperture_frame_sigma_from_psf_data, + build_tracked_comparison_pool, collapse_aperture_data_to_selected_grid_cell, ensure_lightcurve_fit_failure_reason, evaluate_lightcurve_candidate, @@ -1522,6 +1523,78 @@ def capture_active_keys(comp_flux_map, active_keys, validity_mask_func): ] +def test_exact_transit_ensemble_uses_every_supplied_comparison(monkeypatch): + frame_count = 6 + quality_mask = np.ones(frame_count, dtype=bool) + comparison_calibration = { + 'method': 'aperture', + 'a': 0, + 'an': 0, + 'aper': 5.0, + 'annulus': 12.0, + 'comp_summaries': [ + { + 'key': f'comp{index}', + 'comp_index': index - 1, + 'aggregate_score': index / 1000.0, + 'coverage_rejected': False, + 'suitability_outlier_rejected': False, + 'psf_quality_keep_mask': quality_mask, + } + for index in range(1, 6) + ], + } + aper_data = { + 'target': np.full((frame_count, 1, 1), 1000.0), + **{ + f'comp{index}': np.full((frame_count, 1, 1), 100.0 + index) + for index in range(1, 6) + }, + } + captured = {} + + def capture_active_keys(comp_flux_map, active_keys, validity_mask_func): + captured['active_keys'] = list(active_keys) + raise RuntimeError('captured exact transit ensemble') + + monkeypatch.setattr( + 'exotic.exotic.build_absolute_comp_ensemble_flux', + capture_active_keys, + ) + + with pytest.raises(RuntimeError, match='captured exact transit ensemble'): + fit_ranked_comparison_calibration_candidates( + np.linspace(0.0, 0.05, frame_count), + np.linspace(2460000.0, 2460000.05, frame_count), + np.linspace(1.0, 1.2, frame_count), + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(frame_count), + use_ensemble_photometry_rather_than_single_comp=True, + maximum_number_of_ensemble_comparisons_for_transit=1, + use_exactly_the_comps_provided=True, + ) + + assert captured['active_keys'] == [f'comp{index}' for index in range(1, 6)] + + +def test_exact_comparison_mode_does_not_expand_tracked_pool(): + supplied = [[10.0, 20.0], [30.0, 40.0]] + automatic = [[50.0, 60.0], [70.0, 80.0]] + + tracked, messages = build_tracked_comparison_pool( + supplied, + automatic, + use_exactly_the_comps_provided=True, + ) + + assert tracked == supplied + assert messages == [] + + def test_fortuitous_variable_photometry_config_defaults_on_and_supports_opt_out(): assert should_photometer_fortuitous_variables(None) is True assert should_photometer_fortuitous_variables("y") is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py index fc4dcf6b..b2001553 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -59,6 +59,30 @@ def test_comp_params_accepts_comparison_radec_without_pixel_coordinates(tmp_path assert np.allclose(inputs.info_dict["comp_stars_radec"], [[31.04125, 46.68972]]) +def test_comp_params_accepts_sexagesimal_comparison_radec_in_exact_mode(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) X & Y Pixel": [], + "Comparison Star(s) RA & Dec": [["18:37:32.87", "+18:45:39.4"]], + }, + "optional_info": { + "use_exactly_the_comps_provided": True, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert np.allclose( + inputs.info_dict["comp_stars_radec"], + [[279.3869583333, 18.7609444444]], + ) + assert inputs.info_dict["use_exactly_the_comps_provided"] is True + + def test_comp_params_rejects_simultaneous_pixel_and_radec_comparisons(tmp_path): init_data = { "user_info": { diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index e697574a..96ec4d95 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -1571,8 +1571,16 @@ class DummyFit: expected_raw_magnitudes, atol=1.0e-10, ) + np.testing.assert_allclose( + [row['differential_mag'] for row in params], + -2.5 * np.log10(detrended), + atol=1.0e-10, + ) assert params[0]['mag'] != pytest.approx(target_mag) assert params[1]['mag_err'] == pytest.approx(expected_mag_error) + assert params[1]['differential_mag_err'] == pytest.approx( + np.sqrt(expected_mag_error ** 2 - comp_mag_error ** 2) + ) assert params[1]['mag_err'] < 0.08 @@ -3017,7 +3025,16 @@ def test_build_stellar_variability_ensemble_params_preserves_member_metadata(mon fit = types.SimpleNamespace( time=np.array([2460000.105, 2460000.205]), jd_times=np.array([2460000.1, 2460000.2]), + data=np.array([1.0, 1.1]), + dataerr=np.array([0.01, 0.011]), airmass=np.array([1.1, 1.2]), + airmass_model=np.ones(2), + transit=np.ones(2), + stellar_variability_only=True, + stellar_variability_target_flux=np.array([500.0, 550.0]), + stellar_variability_comp_flux=np.full(2, 1000.0), + stellar_variability_target_flux_error=np.full(2, 2.0), + stellar_variability_comp_flux_error=np.full(2, 3.0), stellar_variability_ensemble_magnitudes=np.array([12.30, 12.31]), stellar_variability_ensemble_magnitude_errors=np.array([0.01, 0.011]), stellar_variability_ensemble_members=[ @@ -3047,6 +3064,9 @@ def test_build_stellar_variability_ensemble_params_preserves_member_metadata(mon assert params[0]['cmag'] is None assert params[0]['mag_band'] == 'ClearV' assert params[0]['catalog_mag_band'] == 'V' + assert params[0]['differential_mag'] == pytest.approx(-2.5 * np.log10(0.5)) + assert params[1]['differential_mag'] == pytest.approx(-2.5 * np.log10(0.55)) + assert params[0]['differential_mag_err'] > 0 assert params[0]['ensemble_member_labels'] == ['C1', 'C2'] assert params[0]['ensemble_member_catalog_errors'] == [0.01, 0.011] assert params[0]['ensemble_member_ra_degs'] == [10.1, 10.2] @@ -3319,6 +3339,8 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product aid_text = next(variable_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')).read_text( encoding='utf-8' ) + assert '|DIFFMAG=' in aid_text + assert '|DIFFERR=' in aid_text ensemble_header = next( line for line in aid_text.splitlines() if line.startswith('#ENSEMBLE-COMPARISONS-XC=') @@ -3330,9 +3352,12 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product assert ensemble_metadata['members'][1]['ra_deg'] == pytest.approx(10.2) assert ensemble_metadata['members'][1]['dec_deg'] == pytest.approx(-20.2) csv_path = next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')) + csv_text = csv_path.read_text(encoding='utf-8') + assert 'Apparent Magnitude' in csv_text.splitlines()[0] + assert 'Differential Magnitude' in csv_text.splitlines()[0] exported_errors = [ float(row.split(',')[3]) - for row in csv_path.read_text(encoding='utf-8').splitlines()[1:] + for row in csv_text.splitlines()[1:] if row.strip() ] assert exported_errors diff --git a/tests/test_output_files.py b/tests/test_output_files.py index f783f54a..40e7746c 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -17,6 +17,7 @@ fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, differential_magnitude_series_from_fit, + magnitude_series_from_fit, save_comp_star_calibration_summary, write_differential_magnitude_csv, ) @@ -272,6 +273,8 @@ def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): "time": 2461229.89899, "mag": 13.7378, "mag_err": 0.0042, + "differential_mag": 1.2378, + "differential_mag_err": 0.0021, "mag_band": "r", "airmass": 1.193135, }, @@ -279,6 +282,8 @@ def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): "time": 2461229.90109, "mag": 13.7401, "mag_err": 0.0044, + "differential_mag": 1.2401, + "differential_mag_err": 0.0022, "mag_band": "r", "airmass": 1.1984942, }, @@ -292,8 +297,9 @@ def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): output_text = next((tmp_path / "working_artifacts").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() assert "# FINAL STELLAR VARIABILITY TIMESERIES OF WASP-194" in output_text - assert "# BJD_TDB,Magnitude,Uncertainty,Band,Airmass" in output_text - assert "2461229.89899, 13.738, 0.004, r, 1.193135" in output_text + assert "Apparent Magnitude,Apparent Magnitude Uncertainty" in output_text + assert "Differential Magnitude,Differential Magnitude Uncertainty" in output_text + assert "2461229.89899, 13.738, 0.004, 1.237800, 0.002100, r, 1.193135" in output_text assert "Flux" not in output_text @@ -317,11 +323,81 @@ def test_final_lightcurve_adds_transit_apparent_magnitude_columns_when_calibrate output_text = next((tmp_path / "working_artifacts").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() - assert "Apparent Magnitude,Magnitude Uncertainty,Band" in output_text - assert "2461229.9, 0.1, 1.0, 0.001, 1.0, 1.0, 13.740" in output_text + assert "Differential Magnitude,Differential Magnitude Uncertainty" in output_text + assert "Apparent Magnitude,Apparent Magnitude Uncertainty,Band" in output_text + assert "2461229.9, 0.1, 1.0, 0.001, 1.0, 1.0, -0.000000, 0.001086, 13.740" in output_text assert output_text.rstrip().endswith(", r") +def test_final_lightcurve_keeps_differential_magnitude_when_apparent_calibration_is_unavailable(tmp_path): + (tmp_path / "working_artifacts").mkdir() + fit = SimpleNamespace( + time=np.array([2461229.9]), + data=np.array([0.8]), + detrended=np.array([0.8]), + dataerr=np.array([0.008]), + airmass_model=np.ones(1), + transit=np.ones(1), + ) + p_dict = {'pName': 'Uncalibrated b', 'sName': 'Uncalibrated'} + i_dict = {'save': str(tmp_path), 'date': '2026-07-08', 'filter': 'V'} + + OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([0.1])) + + output_text = next( + (tmp_path / "working_artifacts").glob("FinalLightCurve_Uncalibratedb_2026-07-08.csv") + ).read_text() + expected_differential = -2.5 * np.log10(0.8) + assert f"{expected_differential:.6f}" in output_text + assert ", na, na, V" in output_text + + +def test_magnitude_series_preserves_raw_ratio_for_later_apparent_recalibration(): + target_flux = np.array([500.0, 550.0]) + reference_flux = np.full(2, 1000.0) + target_error = np.full(2, 2.0) + reference_error = np.full(2, 3.0) + differential_mag = -2.5 * np.log10(target_flux / reference_flux) + magnitude_factor = 2.5 / np.log(10.0) + differential_error = magnitude_factor * np.sqrt( + (target_error / target_flux) ** 2 + + (reference_error / reference_flux) ** 2 + ) + fit = SimpleNamespace( + time=np.array([2461229.9, 2461229.91]), + data=np.array([1.0, 1.1]), + dataerr=np.full(2, 0.001), + detrended=np.array([1.0, 1.1]), + airmass=np.array([1.1, 1.2]), + airmass_model=np.ones(2), + transit=np.ones(2), + stellar_variability_target_flux=target_flux, + stellar_variability_comp_flux=reference_flux, + stellar_variability_target_flux_error=target_error, + stellar_variability_comp_flux_error=reference_error, + stellar_variability_params=[{ + "time": 2461229.9, + "mag": 12.0 + differential_mag[0], + "mag_err": np.hypot(0.02, differential_error[0]), + "differential_mag": differential_mag[0], + "differential_mag_err": differential_error[0], + "cmag": 12.0, + "cmag_err": 0.02, + "mag_band": "V", + }], + ) + + series = magnitude_series_from_fit(fit, apply_airmass_correction=False) + + np.testing.assert_allclose(series['differential_magnitude'], differential_mag) + np.testing.assert_allclose(series['differential_magnitude_error'], differential_error) + np.testing.assert_allclose(series['apparent_magnitude'], 12.0 + differential_mag) + np.testing.assert_allclose( + series['apparent_magnitude_error'], + np.hypot(0.02, differential_error), + ) + + def aavso_json_header(output_text, header_name): prefix = f"#{header_name}=" for line in output_text.splitlines(): @@ -357,6 +433,24 @@ def test_observable_depth_is_separate_from_area_depth_for_grazing_geometry(): def test_aavso_output_includes_observatory_location_headers(tmp_path): fit = DummyFit() + fit.stellar_variability_target_flux = np.array([500.0]) + fit.stellar_variability_comp_flux = np.array([1000.0]) + fit.stellar_variability_target_flux_error = np.array([2.0]) + fit.stellar_variability_comp_flux_error = np.array([3.0]) + differential_mag = float(-2.5 * np.log10(0.5)) + differential_error = float( + (2.5 / np.log(10.0)) * np.hypot(2.0 / 500.0, 3.0 / 1000.0) + ) + fit.stellar_variability_params = [{ + "time": fit.time[0], + "mag": 12.0 + differential_mag, + "mag_err": np.hypot(0.02, differential_error), + "differential_mag": differential_mag, + "differential_mag_err": differential_error, + "cmag": 12.0, + "cmag_err": 0.02, + "mag_band": "V", + }] p_dict = { "pName": "HAT-P-32 b", "sName": "HAT-P-32", @@ -413,6 +507,16 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): assert "#GAIADIST=245.7" in output_text assert "#GAIAPMRA=14.25" in output_text assert "#GAIAPMDEC=-9.5" in output_text + magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") + magnitude_row = aavso_json_header(output_text, "MAGNITUDE-XC") + assert magnitude_fields["apparent_calibrated"] is True + assert magnitude_fields["differential_magnitude"].startswith("target minus") + assert magnitude_row["differential_magnitude"] == pytest.approx(differential_mag) + assert magnitude_row["differential_magnitude_error"] == pytest.approx(differential_error) + assert magnitude_row["apparent_magnitude"] == pytest.approx(12.0 + differential_mag) + assert magnitude_row["apparent_magnitude_error"] == pytest.approx( + np.hypot(0.02, differential_error) + ) def test_aavso_output_omits_obsname_header_when_blank(tmp_path): @@ -469,6 +573,12 @@ def test_aavso_output_omits_obsname_header_when_blank(tmp_path): assert "#GAIADIST=" not in output_text assert "#GAIAPMRA=" not in output_text assert "#GAIAPMDEC=" not in output_text + magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") + magnitude_row = aavso_json_header(output_text, "MAGNITUDE-XC") + assert magnitude_fields["apparent_calibrated"] is False + assert magnitude_row["differential_magnitude"] == pytest.approx(0.0) + assert magnitude_row["apparent_magnitude"] is None + assert magnitude_row["apparent_magnitude_error"] is None def test_aid_comparison_coordinate_headers_index_unique_comparisons_on_separate_lines(): @@ -511,6 +621,8 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): "time": 2450000.12345, "mag": 12.34567, "mag_err": 0.012345, + "differential_mag": 0.24567, + "differential_mag_err": 0.006789, "airmass": 1.234, "cname": "RA=10.1000000 Dec=-20.2000000", "cmag": 12.1, @@ -544,6 +656,10 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert "#COMPARISON_RA=10.1000000\n#COMPARISON_DEC=-20.2000000\n" in output_text assert "#DATE=BJD_TDB" in output_text assert "HAT-P-32,2450000.12345,12.3457,0.0123,V,NO,STD" in output_text + assert "|DIFFMAG=0.245670|DIFFERR=0.006789" in output_text + magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") + assert magnitude_fields["apparent_magnitude"] == "MAG" + assert magnitude_fields["differential_magnitude"] == "NOTES subfield DIFFMAG" def test_aid_output_records_calibrated_ensemble_members(tmp_path): @@ -563,6 +679,8 @@ def test_aid_output_records_calibrated_ensemble_members(tmp_path): "time": 2450000.12345, "mag": 12.34, "mag_err": 0.02, + "differential_mag": 1.234567, + "differential_mag_err": 0.00789, "airmass": 1.234, "cname": "ENSEMBLE (2 stars)", "cmag": None, @@ -607,6 +725,7 @@ def test_aid_output_records_calibrated_ensemble_members(tmp_path): assert ensemble_metadata["members"][1]["ra_deg"] == pytest.approx(10.2) assert ensemble_metadata["members"][1]["dec_deg"] == pytest.approx(-20.2) assert "Target,2450000.12345,12.3400,0.0200,V,NO,STD,ENSEMBLE (2 stars),na" in output_text + assert "|DIFFMAG=1.234567|DIFFERR=0.007890" in output_text def test_aid_output_samples_large_derived_anchor_label_lists(tmp_path): From 5a38aeee43210b2c1900305877bfc77a412a9c5c Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 4 Aug 2026 07:42:49 +1000 Subject: [PATCH 104/116] differential bug --- exotic/exotic.py | 146 +++++++++++++++++++++++++++- exotic/inputs.py | 4 + exotic/output_files.py | 49 +++++++++- exotic/plots.py | 15 +-- tests/test_inputs.py | 38 ++++++++ tests/test_nextastro_variability.py | 47 +++++++++ tests/test_output_files.py | 55 +++++++++++ tests/test_plots.py | 14 ++- 8 files changed, 356 insertions(+), 12 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 066b0f8d..56131f8e 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -21575,6 +21575,34 @@ def stellar_variability_label(comp_label, comp_star): return comp_label +def annotate_differential_magnitude_raw_photometry(lc_fit, target_flux, reference_flux, + target_flux_error=None, + reference_flux_error=None): + """Retain the unnormalized flux pair used by differential-magnitude outputs.""" + if lc_fit is None: + return False + + fit_shape = np.asarray(getattr(lc_fit, 'data', []), dtype=float).shape + target_flux = np.asarray(target_flux if target_flux is not None else [], dtype=float) + reference_flux = np.asarray(reference_flux if reference_flux is not None else [], dtype=float) + if target_flux.shape != fit_shape or reference_flux.shape != fit_shape: + return False + + def aligned_error(values): + if values is None: + return np.full(fit_shape, np.nan, dtype=float) + array = np.asarray(values, dtype=float) + if array.shape != fit_shape: + return np.full(fit_shape, np.nan, dtype=float) + return array + + lc_fit.differential_magnitude_target_flux = target_flux.copy() + lc_fit.differential_magnitude_reference_flux = reference_flux.copy() + lc_fit.differential_magnitude_target_flux_error = aligned_error(target_flux_error).copy() + lc_fit.differential_magnitude_reference_flux_error = aligned_error(reference_flux_error).copy() + return True + + def annotate_stellar_variability_raw_photometry(lc_fit, target_flux, comp_flux, target_flux_error=None, comp_flux_error=None): if lc_fit is None: @@ -28030,7 +28058,8 @@ def select_stellar_variability_ensemble_members( def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flux_error, comp_flux_map, comp_error_map, members, - minimum_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS): + minimum_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + validity_mask_func=valid_comparison_frame_mask): target_flux = np.asarray(target_flux, dtype=float) if target_flux.ndim != 1: target_flux = target_flux.reshape(-1) @@ -28043,6 +28072,8 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu zero_points = [] zero_point_errors = [] + raw_member_keys = [] + raw_comp_error_map = {} magnitude_factor = 2.5 / np.log(10.0) for member in members or []: ckey = member.get('key') @@ -28056,6 +28087,8 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu comp_flux_error = np.asarray(comp_error_map[ckey], dtype=float).reshape(-1) if comp_flux_error is None or comp_flux_error.shape != comp_flux.shape: comp_flux_error = np.sqrt(np.clip(np.abs(comp_flux), 1.0, None)) + raw_member_keys.append(ckey) + raw_comp_error_map[ckey] = comp_flux_error member_keep_mask = np.asarray( member.get('summary', {}).get('ensemble_frame_keep_mask', np.ones(frame_count, dtype=bool)), @@ -28079,6 +28112,29 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu zero_points.append(zero_point) zero_point_errors.append(zero_point_error) + raw_reference_flux = np.full(target_flux.shape, np.nan, dtype=float) + raw_reference_flux_error = np.full(target_flux.shape, np.nan, dtype=float) + built_raw_reference, built_raw_member_keys = build_absolute_comp_ensemble_flux( + comp_flux_map, + raw_member_keys, + validity_mask_func=validity_mask_func, + ) + if built_raw_reference is not None and built_raw_member_keys == raw_member_keys: + built_raw_reference = np.asarray(built_raw_reference, dtype=float) + if built_raw_reference.shape == target_flux.shape: + raw_reference_flux = built_raw_reference + built_raw_reference_error = build_absolute_comp_ensemble_uncertainty( + comp_flux_map, + raw_comp_error_map, + raw_member_keys, + validity_mask_func=validity_mask_func, + ) + if ( + built_raw_reference_error is not None + and np.asarray(built_raw_reference_error).shape == target_flux.shape + ): + raw_reference_flux_error = np.asarray(built_raw_reference_error, dtype=float) + selected_member_count = len(members or []) try: minimum_required_members = max(1, int(minimum_members)) @@ -28101,6 +28157,8 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu 'relative_flux_error': np.full(target_flux.shape, np.nan, dtype=float), 'synthetic_reference_flux': np.full(target_flux.shape, np.nan, dtype=float), 'synthetic_reference_flux_error': np.full(target_flux.shape, np.nan, dtype=float), + 'raw_reference_flux': raw_reference_flux, + 'raw_reference_flux_error': raw_reference_flux_error, 'valid_member_count': np.zeros(target_flux.shape, dtype=int), } if len(zero_points) < required_members: @@ -28174,11 +28232,73 @@ def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flu 'relative_flux_error': relative_flux_error, 'synthetic_reference_flux': synthetic_reference_flux, 'synthetic_reference_flux_error': synthetic_reference_flux_error, + 'raw_reference_flux': raw_reference_flux, + 'raw_reference_flux_error': raw_reference_flux_error, 'valid_member_count': valid_member_count, 'baseline_magnitude': baseline_magnitude, } +def annotate_stellar_variability_ensemble_differential_photometry(lc_fit, ensemble_series): + """Attach the real ensemble flux scale without replacing normalized fitting inputs.""" + fit_shape = np.asarray(getattr(lc_fit, 'data', []), dtype=float).shape + source_indices = np.asarray( + getattr(lc_fit, 'stellar_variability_source_indices', []), + dtype=int, + ) + raw_reference_flux = np.asarray( + ensemble_series.get('raw_reference_flux', []), + dtype=float, + ) + raw_reference_flux_error = np.asarray( + ensemble_series.get('raw_reference_flux_error', []), + dtype=float, + ) + target_flux = np.asarray( + getattr(lc_fit, 'stellar_variability_target_flux', []), + dtype=float, + ) + target_flux_error = np.asarray( + getattr(lc_fit, 'stellar_variability_target_flux_error', []), + dtype=float, + ) + + if source_indices.shape != fit_shape or target_flux.shape != fit_shape: + raise RuntimeError( + "Calibrated comparison-ensemble fit did not retain aligned target fluxes for " + "differential-magnitude output." + ) + if ( + raw_reference_flux.ndim != 1 + or source_indices.size == 0 + or np.any(source_indices < 0) + or np.any(source_indices >= raw_reference_flux.size) + ): + raise RuntimeError( + "Raw comparison-ensemble reference flux is unavailable; refusing to report a " + "median-normalized light curve as differential magnitude." + ) + selected_reference_flux = raw_reference_flux[source_indices] + selected_reference_error = ( + raw_reference_flux_error[source_indices] + if raw_reference_flux_error.shape == raw_reference_flux.shape + else np.full(fit_shape, np.nan, dtype=float) + ) + attached = annotate_differential_magnitude_raw_photometry( + lc_fit, + target_flux, + selected_reference_flux, + target_flux_error=target_flux_error, + reference_flux_error=selected_reference_error, + ) + if not attached: + raise RuntimeError( + "Raw target and comparison-ensemble fluxes could not be aligned for " + "differential-magnitude output." + ) + return lc_fit + + def stellar_variability_json_safe(value): if isinstance(value, dict): return {str(key): stellar_variability_json_safe(subvalue) for key, subvalue in value.items()} @@ -28804,6 +28924,9 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, comp_error_map, calibrated_members, minimum_members=len(calibrated_members), + validity_mask_func=( + robust_flux_floor_mask if method == 'psf' else valid_comparison_frame_mask + ), ) if prebuilt_ensemble_series.get('applied'): ensemble_members = calibrated_members @@ -28833,6 +28956,8 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, 'relative_flux_error': relative_series['relative_flux_error'], 'synthetic_reference_flux': relative_series['reference_flux'], 'synthetic_reference_flux_error': relative_series['reference_flux_error'], + 'raw_reference_flux': relative_series['reference_flux'], + 'raw_reference_flux_error': relative_series['reference_flux_error'], 'valid_member_count': valid_member_count, 'baseline_magnitude': np.nan, } @@ -28909,6 +29034,9 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, comp_flux_map, comp_error_map, ensemble_members, + validity_mask_func=( + robust_flux_floor_mask if method == 'psf' else valid_comparison_frame_mask + ), ) if ensemble_series.get('applied'): fit_mask = ( @@ -28983,6 +29111,10 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, plot_time_range=plot_time_range, ) if fit_result is not None: + annotate_stellar_variability_ensemble_differential_photometry( + fit_result, + ensemble_series, + ) selected_source_indices = np.asarray( fit_result.stellar_variability_source_indices, dtype=int, @@ -29852,6 +29984,11 @@ def process_fortuitous_variables( comp_error_map, members, minimum_members=required_comparison_members, + validity_mask_func=( + robust_flux_floor_mask + if comparison_calibration.get('method') == 'psf' + else valid_comparison_frame_mask + ), ) else: relative_series = build_relative_comparison_ensemble_series( @@ -29886,6 +30023,8 @@ def process_fortuitous_variables( 'synthetic_reference_flux_error': relative_series.get( 'reference_flux_error' ), + 'raw_reference_flux': relative_series.get('reference_flux'), + 'raw_reference_flux_error': relative_series.get('reference_flux_error'), 'magnitude': np.full(target_flux.shape, np.nan, dtype=float), 'magnitude_error': instrumental_magnitude_error, 'valid_member_count': np.where( @@ -30007,6 +30146,11 @@ def process_fortuitous_variables( ) if fit is None: raise ValueError('stellar-variability light curve construction failed') + if not use_single_comparison: + annotate_stellar_variability_ensemble_differential_photometry( + fit, + ensemble_series, + ) variable_dir.mkdir(parents=True, exist_ok=True) differential_csv_path = write_differential_magnitude_csv( fit, diff --git a/exotic/inputs.py b/exotic/inputs.py index d085f687..a2f48759 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -485,6 +485,10 @@ def comp_params(self, init_file, planet_dict): 'Comparison Star(s) RA and Dec', 'Comparison Star(s) RA & Dec (degrees)', ), + # Accept existing init files that placed this option beside the + # comparison coordinates. optional_info is parsed later and wins + # when both locations are populated. + 'use_exactly_the_comps_provided': 'use_exactly_the_comps_provided', } planet_params = { 'ra': 'Target Star RA', 'dec': 'Target Star Dec', 'pName': "Planet Name", 'sName': "Host Star Name", diff --git a/exotic/output_files.py b/exotic/output_files.py index e6f8b947..46960786 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -216,19 +216,35 @@ def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, return None target_flux = np.asarray( - getattr(fit, 'stellar_variability_target_flux', []), + getattr( + fit, + 'differential_magnitude_target_flux', + getattr(fit, 'stellar_variability_target_flux', []), + ), dtype=float, ).reshape(-1) reference_flux = np.asarray( - getattr(fit, 'stellar_variability_comp_flux', []), + getattr( + fit, + 'differential_magnitude_reference_flux', + getattr(fit, 'stellar_variability_comp_flux', []), + ), dtype=float, ).reshape(-1) target_error = np.asarray( - getattr(fit, 'stellar_variability_target_flux_error', []), + getattr( + fit, + 'differential_magnitude_target_flux_error', + getattr(fit, 'stellar_variability_target_flux_error', []), + ), dtype=float, ).reshape(-1) reference_error = np.asarray( - getattr(fit, 'stellar_variability_comp_flux_error', []), + getattr( + fit, + 'differential_magnitude_reference_flux_error', + getattr(fit, 'stellar_variability_comp_flux_error', []), + ), dtype=float, ).reshape(-1) @@ -377,6 +393,31 @@ def magnitude_series_from_fit(fit, out_of_transit_only=False, if calibration is None: return result + # A calibrated comparison ensemble already carries its independently + # derived apparent-magnitude series. Do not reconstruct that series by + # adding a constant to the raw instrumental differential magnitudes: the + # calibrated ensemble and the raw median-scaled ensemble intentionally use + # different reference constructions and can have different time trends. + calibrated_magnitude = np.asarray( + getattr(fit, 'stellar_variability_ensemble_magnitudes', []), + dtype=float, + ).reshape(-1) + calibrated_error = np.asarray( + getattr(fit, 'stellar_variability_ensemble_magnitude_errors', []), + dtype=float, + ).reshape(-1) + if calibrated_magnitude.shape == fit_data.shape: + calibrated_mask = source_mask & np.isfinite(calibrated_magnitude) + result['apparent_magnitude'][calibrated_mask] = calibrated_magnitude[calibrated_mask] + if calibrated_error.shape == fit_data.shape: + calibrated_error_mask = calibrated_mask & np.isfinite(calibrated_error) + result['apparent_magnitude_error'][calibrated_error_mask] = ( + calibrated_error[calibrated_error_mask] + ) + result['band'] = calibration['band'] + result['apparent_calibrated'] = bool(np.any(calibrated_mask)) + return result + magnitude_offset = np.nan calibration_error = calibration['baseline_error'] if np.isfinite(calibration['zero_point_error']): diff --git a/exotic/plots.py b/exotic/plots.py index 501b3735..75381ced 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -748,12 +748,15 @@ def plot_differential_magnitude(fit, target_name, save, date, observed_filter=No '.', color='royalblue', ) - correction_label = ( - 'Airmass-corrected target/reference ratio' - if series['airmass_corrected'] - else 'Raw target/reference ratio; no airmass correction' - ) - ax.set_title(f"{target_name}\n{correction_label}") + if getattr(fit, 'stellar_variability_only', False) or save_stellar_variability_alias: + ax.set_title(target_name) + else: + correction_label = ( + 'Airmass-corrected target/reference ratio' + if series['airmass_corrected'] + else 'Raw target/reference ratio' + ) + ax.set_title(f"{target_name}\n{correction_label}") band_label = f" ({observed_filter})" if observed_filter else '' ax.set_ylabel(f"Differential Magnitude{band_label}") ax.invert_yaxis() diff --git a/tests/test_inputs.py b/tests/test_inputs.py index b2001553..ec37bd33 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -213,6 +213,44 @@ def test_comp_params_reads_apparent_and_exact_comparison_options(tmp_path): assert inputs.info_dict["use_exactly_the_comps_provided"] is True +def test_comp_params_accepts_exact_comparison_option_beside_user_coordinates(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) RA & Dec": [[279.3869583, 18.7609444]], + "use_exactly_the_comps_provided": True, + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_exactly_the_comps_provided"] is True + + +def test_optional_info_exact_comparison_option_overrides_user_info_alias(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) RA & Dec": [[279.3869583, 18.7609444]], + "use_exactly_the_comps_provided": True, + }, + "optional_info": { + "use_exactly_the_comps_provided": False, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_exactly_the_comps_provided"] is False + + def test_comp_params_defaults_stellar_variability_ensemble_to_true(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 96ec4d95..2a0f7b8f 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -2950,6 +2950,8 @@ def test_calibrated_stellar_variability_ensemble_combines_catalog_zero_points(): assert result['applied'] is True np.testing.assert_allclose(result['magnitude'], expected_target_magnitude, atol=1.0e-10) np.testing.assert_allclose(result['relative_flux'], 1.0, atol=1.0e-10) + np.testing.assert_allclose(result['raw_reference_flux'], 375.0, atol=1.0e-10) + assert np.all(np.isfinite(result['raw_reference_flux_error'])) np.testing.assert_array_equal(result['valid_member_count'], np.full(frame_count, 2)) assert np.all(result['magnitude_error'] > 0) @@ -3519,6 +3521,41 @@ def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(monkey assert selected['ensemble_member_keys'] == ['comp1', 'comp2'] assert selected['fit'].stellar_variability_ensemble_members assert len(selected['fit'].stellar_variability_ensemble_magnitudes) == len(selected['fit'].time) + np.testing.assert_allclose( + selected['fit'].differential_magnitude_reference_flux, + 375.0, + atol=1.0e-10, + ) + differential_series = exotic_module.differential_magnitude_series_from_fit( + selected['fit'], + apply_airmass_correction=False, + ) + expected_differential = -2.5 * np.log10(target_flux / 375.0) + np.testing.assert_allclose( + differential_series['magnitude'], + expected_differential, + atol=1.0e-10, + ) + assert abs(float(np.nanmedian(differential_series['magnitude']))) > 0.5 + + # Final output preparation may refresh the normalized fitting photometry. + # That must never overwrite the separately retained raw ensemble reference. + exotic_module.annotate_stellar_variability_raw_photometry( + selected['fit'], + target_flux, + target_flux, + target_flux_error=np.ones(frame_count), + comp_flux_error=np.ones(frame_count), + ) + differential_after_fit_refresh = exotic_module.differential_magnitude_series_from_fit( + selected['fit'], + apply_airmass_correction=False, + ) + np.testing.assert_allclose( + differential_after_fit_refresh['magnitude'], + expected_differential, + atol=1.0e-10, + ) vsp_params = exotic_module.build_stellar_variability_params_from_photometry_selection( result, @@ -3547,6 +3584,16 @@ def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(monkey ).aavso() assert len(vsp_params) == frame_count + np.testing.assert_allclose( + [row['mag'] for row in vsp_params], + selected['fit'].stellar_variability_ensemble_magnitudes, + atol=1.0e-10, + ) + np.testing.assert_allclose( + [row['differential_mag'] for row in vsp_params], + expected_differential, + atol=1.0e-10, + ) assert aid_path.is_file() aid_text = aid_path.read_text(encoding='utf-8') assert '#ENSEMBLE-COMPARISONS-XC=' in aid_text diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 40e7746c..067489c2 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -398,6 +398,61 @@ def test_magnitude_series_preserves_raw_ratio_for_later_apparent_recalibration() ) +def test_calibrated_ensemble_keeps_apparent_magnitudes_independent_of_raw_differential(): + target_flux = np.array([1000.0, 1010.0]) + raw_reference_flux = np.full(2, 375.0) + calibrated_magnitude = np.array([11.25, 11.27]) + calibrated_error = np.array([0.02, 0.021]) + expected_differential = -2.5 * np.log10(target_flux / raw_reference_flux) + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2461229.9, 2461229.91]), + data=np.ones(2), + detrended=np.ones(2), + dataerr=np.full(2, 0.001), + airmass=np.array([1.1, 1.2]), + airmass_model=np.ones(2), + transit=np.ones(2), + # The calibrated ensemble's normalized fitting reference remains + # separate from the raw instrumental reference used for DIFFMAG. + stellar_variability_target_flux=target_flux, + stellar_variability_comp_flux=target_flux.copy(), + stellar_variability_target_flux_error=np.ones(2), + stellar_variability_comp_flux_error=np.ones(2), + differential_magnitude_target_flux=target_flux, + differential_magnitude_reference_flux=raw_reference_flux, + differential_magnitude_target_flux_error=np.ones(2), + differential_magnitude_reference_flux_error=np.ones(2), + stellar_variability_ensemble_magnitudes=calibrated_magnitude, + stellar_variability_ensemble_magnitude_errors=calibrated_error, + stellar_variability_params=[ + { + 'time': 2461229.9, + 'mag': calibrated_magnitude[0], + 'mag_err': calibrated_error[0], + 'differential_mag': expected_differential[0], + 'differential_mag_err': 0.002, + 'mag_band': 'V', + }, + { + 'time': 2461229.91, + 'mag': calibrated_magnitude[1], + 'mag_err': calibrated_error[1], + 'differential_mag': expected_differential[1], + 'differential_mag_err': 0.002, + 'mag_band': 'V', + }, + ], + ) + + series = magnitude_series_from_fit(fit, apply_airmass_correction=False) + + np.testing.assert_allclose(series['differential_magnitude'], expected_differential) + np.testing.assert_allclose(series['apparent_magnitude'], calibrated_magnitude) + np.testing.assert_allclose(series['apparent_magnitude_error'], calibrated_error) + assert series['apparent_calibrated'] is True + + def aavso_json_header(output_text, header_name): prefix = f"#{header_name}=" for line in output_text.splitlines(): diff --git a/tests/test_plots.py b/tests/test_plots.py index ea4d63f8..0bd76a1b 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -166,7 +166,18 @@ def spy_invert_yaxis(self, *args, **kwargs): assert (tmp_path / "FinalLightCurve_Target_2026-07-08.png").exists() -def test_stellar_variability_differential_plot_survives_without_apparent_magnitudes(tmp_path): +def test_stellar_variability_differential_plot_survives_without_apparent_magnitudes( + tmp_path, monkeypatch): + from matplotlib.axes import Axes + + captured_titles = [] + original_set_title = Axes.set_title + + def spy_set_title(self, title, *args, **kwargs): + captured_titles.append(title) + return original_set_title(self, title, *args, **kwargs) + + monkeypatch.setattr(Axes, "set_title", spy_set_title) fit = SimpleNamespace( stellar_variability_only=True, time=np.array([2461229.5, 2461229.6, 2461229.8]), @@ -195,6 +206,7 @@ def test_stellar_variability_differential_plot_survives_without_apparent_magnitu assert ( tmp_path / 'working_artifacts' / 'Stellar_Variability_DifferentialMagnitude.png' ).exists() + assert captured_titles[-1] == 'Variable Star' def test_plot_obs_stats_uses_supplied_background_series(tmp_path, monkeypatch): From edfea3afe3d20b1fceb1b7552a97fe41d0bb8f5b Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 4 Aug 2026 09:29:52 +1000 Subject: [PATCH 105/116] restrict to 4 decimal places (again) --- exotic/output_files.py | 54 +++++++++++++++++++++++++------------- exotic/plots.py | 2 -- exotic/utils.py | 2 +- tests/test_output_files.py | 25 ++++++++++-------- tests/test_plots.py | 12 ++++----- 5 files changed, 56 insertions(+), 39 deletions(-) diff --git a/exotic/output_files.py b/exotic/output_files.py index 46960786..eecc681e 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -6,6 +6,7 @@ try: from utils import ( + MAGNITUDE_DECIMAL_PLACES, filename_date_token, format_magnitude_error, format_magnitude, @@ -18,6 +19,7 @@ ) except ImportError: from .utils import ( + MAGNITUDE_DECIMAL_PLACES, filename_date_token, format_magnitude_error, format_magnitude, @@ -529,9 +531,14 @@ def write_differential_magnitude_csv(fit, save, target_name, observation_date=No series['magnitude_error'], ): airmass_text = f"{airmass}" if np.isfinite(airmass) else 'na' - error_text = f"{magnitude_error:.6f}" if np.isfinite(magnitude_error) else 'na' + error_text = ( + f"{magnitude_error:.{MAGNITUDE_DECIMAL_PLACES}f}" + if np.isfinite(magnitude_error) + else 'na' + ) handle.write( - f"{time_value}, {airmass_text}, {magnitude:.6f}, {error_text}, " + f"{time_value}, {airmass_text}, " + f"{magnitude:.{MAGNITUDE_DECIMAL_PLACES}f}, {error_text}, " f"{observed_filter or 'na'}, {comparison}\n" ) return output_path @@ -2227,12 +2234,12 @@ def final_lightcurve(self, phase): differential_mag_text = format_magnitude( differential_mag, default="na", - digits=6, + digits=MAGNITUDE_DECIMAL_PLACES, ) differential_error_text = format_magnitude_error( differential_error, default="na", - digits=6, + digits=MAGNITUDE_DECIMAL_PLACES, ) band = vsp_p.get('mag_band') or self.i_dict.get('filter') or 'V' airmass = finite_float(vsp_p.get('airmass')) @@ -2275,12 +2282,12 @@ def final_lightcurve(self, phase): differential_mag_text = format_magnitude( magnitude_series['differential_magnitude'][row_index], default="na", - digits=6, + digits=MAGNITUDE_DECIMAL_PLACES, ) differential_error_text = format_magnitude_error( magnitude_series['differential_magnitude_error'][row_index], default="na", - digits=6, + digits=MAGNITUDE_DECIMAL_PLACES, ) apparent_mag_text = format_magnitude( magnitude_series['apparent_magnitude'][row_index], @@ -2881,17 +2888,21 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, for magnitude_index in range(0, len(self.fit.time)): f.write(format_aavso_json_header("MAGNITUDE-XC", { 'date_bjd_tdb': finite_float(self.fit.time[magnitude_index]), - 'differential_magnitude': finite_float( - magnitude_series['differential_magnitude'][magnitude_index] + 'differential_magnitude': rounded_magnitude_value( + magnitude_series['differential_magnitude'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, ), - 'differential_magnitude_error': finite_float( - magnitude_series['differential_magnitude_error'][magnitude_index] + 'differential_magnitude_error': rounded_magnitude_error( + magnitude_series['differential_magnitude_error'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, ), - 'apparent_magnitude': finite_float( - magnitude_series['apparent_magnitude'][magnitude_index] + 'apparent_magnitude': rounded_magnitude_value( + magnitude_series['apparent_magnitude'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, ), - 'apparent_magnitude_error': finite_float( - magnitude_series['apparent_magnitude_error'][magnitude_index] + 'apparent_magnitude_error': rounded_magnitude_error( + magnitude_series['apparent_magnitude_error'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, ), 'band': magnitude_series['band'] or self.i_dict.get('filter'), }, preserve_nulls=True)) @@ -3004,10 +3015,17 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta variable_name = default_variable_name if use_row_names: variable_name = vsp_p.get('_aid_name') or variable_name - mag = format_magnitude(vsp_p.get('mag'), default=None, digits=4) + mag = format_magnitude( + vsp_p.get('mag'), + default=None, + digits=MAGNITUDE_DECIMAL_PLACES, + ) if mag is None: continue - mag_err = format_magnitude_error(vsp_p.get('mag_err'), digits=4) + mag_err = format_magnitude_error( + vsp_p.get('mag_err'), + digits=MAGNITUDE_DECIMAL_PLACES, + ) cmag = format_magnitude(vsp_p.get('cmag')) chart_id = self.chart_id or vsp_p.get('chart_id') or 'na' differential_mag, differential_error = differential_magnitude_from_vsp_param( @@ -3030,12 +3048,12 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta differential_mag_text = format_magnitude( differential_mag, default=None, - digits=6, + digits=MAGNITUDE_DECIMAL_PLACES, ) differential_error_text = format_magnitude_error( differential_error, default=None, - digits=6, + digits=MAGNITUDE_DECIMAL_PLACES, ) notes = 'na' if differential_mag_text is not None: diff --git a/exotic/plots.py b/exotic/plots.py index 75381ced..976084a2 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -638,7 +638,6 @@ def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): title_lines = [s_name] if reference_label: title_lines.append(reference_label) - title_lines.append('No airmass correction applied to stellar variability') metadata_label = _stellar_variability_comparison_metadata_label(first_param) if metadata_label: title_lines.append(metadata_label) @@ -1113,7 +1112,6 @@ def plot_final_lightcurve(fit, high_res, targ_name, save, date, observed_filter= ) if reference_label: title_lines.append(reference_label) - title_lines.append('No airmass correction applied to stellar variability') metadata_label = _stellar_variability_comparison_metadata_label(first_param) if metadata_label: title_lines.append(metadata_label) diff --git a/exotic/utils.py b/exotic/utils.py index 2adadd21..1b877e13 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -25,7 +25,7 @@ _WINDOWS_ILLEGAL_FILENAME_CHARS_RE = re.compile(r'[<>:"/\\|?*\x00-\x1f\x7f]') _FILENAME_WHITESPACE_RE = re.compile(r'\s+') MAX_APPARENT_MAGNITUDE = 30.0 -MAGNITUDE_DECIMAL_PLACES = 3 +MAGNITUDE_DECIMAL_PLACES = 4 MINIMUM_MAGNITUDE_ERROR = 0.001 diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 067489c2..f6e28fb2 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -79,6 +79,8 @@ def test_differential_csv_does_not_require_apparent_magnitude_calibration(tmp_pa assert '# AIRMASS_CORRECTION=NO' in output_text assert 'Differential Magnitude' in output_text assert 'Apparent' not in output_text + assert ', 0.7526, 0.0054, V, ' in output_text + assert ', 0.6491, 0.0051, V, ' in output_text class DummyFit: @@ -299,7 +301,7 @@ def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): assert "# FINAL STELLAR VARIABILITY TIMESERIES OF WASP-194" in output_text assert "Apparent Magnitude,Apparent Magnitude Uncertainty" in output_text assert "Differential Magnitude,Differential Magnitude Uncertainty" in output_text - assert "2461229.89899, 13.738, 0.004, 1.237800, 0.002100, r, 1.193135" in output_text + assert "2461229.89899, 13.7378, 0.0042, 1.2378, 0.0021, r, 1.193135" in output_text assert "Flux" not in output_text @@ -325,7 +327,7 @@ def test_final_lightcurve_adds_transit_apparent_magnitude_columns_when_calibrate assert "Differential Magnitude,Differential Magnitude Uncertainty" in output_text assert "Apparent Magnitude,Apparent Magnitude Uncertainty,Band" in output_text - assert "2461229.9, 0.1, 1.0, 0.001, 1.0, 1.0, -0.000000, 0.001086, 13.740" in output_text + assert "2461229.9, 0.1, 1.0, 0.001, 1.0, 1.0, -0.0000, 0.0011, 13.7400" in output_text assert output_text.rstrip().endswith(", r") @@ -348,7 +350,7 @@ def test_final_lightcurve_keeps_differential_magnitude_when_apparent_calibration (tmp_path / "working_artifacts").glob("FinalLightCurve_Uncalibratedb_2026-07-08.csv") ).read_text() expected_differential = -2.5 * np.log10(0.8) - assert f"{expected_differential:.6f}" in output_text + assert f"{expected_differential:.4f}" in output_text assert ", na, na, V" in output_text @@ -566,11 +568,12 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): magnitude_row = aavso_json_header(output_text, "MAGNITUDE-XC") assert magnitude_fields["apparent_calibrated"] is True assert magnitude_fields["differential_magnitude"].startswith("target minus") - assert magnitude_row["differential_magnitude"] == pytest.approx(differential_mag) - assert magnitude_row["differential_magnitude_error"] == pytest.approx(differential_error) - assert magnitude_row["apparent_magnitude"] == pytest.approx(12.0 + differential_mag) - assert magnitude_row["apparent_magnitude_error"] == pytest.approx( - np.hypot(0.02, differential_error) + assert magnitude_row["differential_magnitude"] == round(differential_mag, 4) + assert magnitude_row["differential_magnitude_error"] == round(differential_error, 4) + assert magnitude_row["apparent_magnitude"] == round(12.0 + differential_mag, 4) + assert magnitude_row["apparent_magnitude_error"] == round( + np.hypot(0.02, differential_error), + 4, ) @@ -711,7 +714,7 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert "#COMPARISON_RA=10.1000000\n#COMPARISON_DEC=-20.2000000\n" in output_text assert "#DATE=BJD_TDB" in output_text assert "HAT-P-32,2450000.12345,12.3457,0.0123,V,NO,STD" in output_text - assert "|DIFFMAG=0.245670|DIFFERR=0.006789" in output_text + assert "|DIFFMAG=0.2457|DIFFERR=0.0068" in output_text magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") assert magnitude_fields["apparent_magnitude"] == "MAG" assert magnitude_fields["differential_magnitude"] == "NOTES subfield DIFFMAG" @@ -780,7 +783,7 @@ def test_aid_output_records_calibrated_ensemble_members(tmp_path): assert ensemble_metadata["members"][1]["ra_deg"] == pytest.approx(10.2) assert ensemble_metadata["members"][1]["dec_deg"] == pytest.approx(-20.2) assert "Target,2450000.12345,12.3400,0.0200,V,NO,STD,ENSEMBLE (2 stars),na" in output_text - assert "|DIFFMAG=1.234567|DIFFERR=0.007890" in output_text + assert "|DIFFMAG=1.2346|DIFFERR=0.0079" in output_text def test_aid_output_samples_large_derived_anchor_label_lists(tmp_path): @@ -997,7 +1000,7 @@ def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path assert "NextAstro photometry catalog" in reference assert "RA=10.1000000" in reference assert "Dec=-20.2000000" in reference - assert "V=12.345 +/- 0.067" in reference + assert "V=12.3450 +/- 0.0670" in reference def test_transit_outputs_use_rprs_fallback_uncertainty_when_model_error_missing(tmp_path): diff --git a/tests/test_plots.py b/tests/test_plots.py index 0bd76a1b..0b73360d 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -156,12 +156,12 @@ def spy_invert_yaxis(self, *args, **kwargs): plot_final_lightcurve(fit, np.ones(2), "Target", str(tmp_path), "2026-07-08") - np.testing.assert_allclose(captured_errorbar_x[0], np.array([2461229.5, 2461229.6, 2461229.8])) - np.testing.assert_allclose(captured_errorbar_y[0], np.array([13.738, 13.740, 13.735])) + np.testing.assert_allclose(captured_errorbar_x[-1], np.array([2461229.5, 2461229.6, 2461229.8])) + np.testing.assert_allclose(captured_errorbar_y[-1], np.array([13.738, 13.740, 13.735])) assert captured_xlabels[-1] == "Time [BJD_TDB]" assert captured_xlabels[-1] != "Orbital Phase" assert captured_ylabels[-1] == "Magnitude (r)" - assert len(inverted_axes) == 1 + assert len(inverted_axes) == 2 assert "O-C [%]" not in captured_ylabels assert (tmp_path / "FinalLightCurve_Target_2026-07-08.png").exists() @@ -411,8 +411,7 @@ def spy_invert_yaxis(self, *args, **kwargs): "Label: NextAstro-123\n" "Comparison RA=10.100000\n" "Dec=-20.200000\n" - "No airmass correction applied to stellar variability\n" - "Original filter: CV | Comparison mag: r=12.345 +/- 0.067" + "Original filter: CV | Comparison mag: r=12.3450 +/- 0.0670" ) assert ylabels[-1] == "Magnitude (r)" assert len(inverted_axes) == 1 @@ -464,7 +463,7 @@ def spy_set_ylabel(self, label, *args, **kwargs): "Dec=-20.200000" ) in titles[-1] assert "Original filter: CV" in titles[-1] - assert "Comparison mag: V=12.345 +/- 0.067" in titles[-1] + assert "Comparison mag: V=12.3450 +/- 0.0670" in titles[-1] assert ylabels[-1] == "Magnitude (ClearV)" @@ -501,7 +500,6 @@ def spy_set_title(self, label, *args, **kwargs): assert titles[-1] == ( "Host Star\n" "Label: NextAstro-123\nComparison RA=10.100000\nDec=-20.200000\n" - "No airmass correction applied to stellar variability\n" "Original filter: MObs CV" ) assert "99.99" not in titles[-1] From c32814d69c2ec34173330e5da3794c3dd7441f51 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 4 Aug 2026 11:22:34 +1000 Subject: [PATCH 106/116] | vs , delimiter glitch --- exotic/output_files.py | 19 ++++++++++--------- tests/test_nextastro_variability.py | 12 ++++++++++-- tests/test_output_files.py | 16 +++++++++++++--- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/exotic/output_files.py b/exotic/output_files.py index eecc681e..5276338a 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -3001,8 +3001,8 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta f.write(format_aavso_json_header("MAGNITUDE_FIELDS-XC", { 'apparent_magnitude': 'MAG', 'apparent_magnitude_error': 'MERR', - 'differential_magnitude': 'NOTES subfield DIFFMAG', - 'differential_magnitude_error': 'NOTES subfield DIFFERR', + 'differential_magnitude': 'DIFFMAG', + 'differential_magnitude_error': 'DIFFERR', 'differential_magnitude_definition': ( 'target minus selected comparison reference; ' '-2.5 log10(target_flux/reference_flux)' @@ -3010,7 +3010,10 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta 'airmass_corrected': False, })) - f.write("#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,GROUP,CHART,NOTES\n") + f.write( + "#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG," + "AMASS,GROUP,CHART,NOTES,DIFFMAG,DIFFERR\n" + ) for vsp_p in self.vsp_params: variable_name = default_variable_name if use_row_names: @@ -3055,14 +3058,12 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta default=None, digits=MAGNITUDE_DECIMAL_PLACES, ) - notes = 'na' - if differential_mag_text is not None: - notes = f"|DIFFMAG={differential_mag_text}" - if differential_error_text is not None: - notes += f"|DIFFERR={differential_error_text}" + differential_mag_text = differential_mag_text or 'na' + differential_error_text = differential_error_text or 'na' f.write(f"{variable_name},{round(vsp_p['time'], 5)},{mag},{mag_err}," f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{cmag},na,na," - f"{round(vsp_p['airmass'], 7)},na,{chart_id},{notes}\n") + f"{round(vsp_p['airmass'], 7)},na,{chart_id},na," + f"{differential_mag_text},{differential_error_text}\n") return params_file def aavso(self): diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 2a0f7b8f..d2853623 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -3341,8 +3341,16 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product aid_text = next(variable_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')).read_text( encoding='utf-8' ) - assert '|DIFFMAG=' in aid_text - assert '|DIFFERR=' in aid_text + assert ( + '#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,' + 'GROUP,CHART,NOTES,DIFFMAG,DIFFERR' + ) in aid_text + assert '|DIFFMAG=' not in aid_text + assert '|DIFFERR=' not in aid_text + aid_data_row = next(line for line in aid_text.splitlines() if not line.startswith('#')) + assert aid_data_row.split(',')[-3] == 'na' + assert aid_data_row.split(',')[-2] != 'na' + assert aid_data_row.split(',')[-1] != 'na' ensemble_header = next( line for line in aid_text.splitlines() if line.startswith('#ENSEMBLE-COMPARISONS-XC=') diff --git a/tests/test_output_files.py b/tests/test_output_files.py index f6e28fb2..fb0d7d1d 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -714,10 +714,20 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): assert "#COMPARISON_RA=10.1000000\n#COMPARISON_DEC=-20.2000000\n" in output_text assert "#DATE=BJD_TDB" in output_text assert "HAT-P-32,2450000.12345,12.3457,0.0123,V,NO,STD" in output_text - assert "|DIFFMAG=0.2457|DIFFERR=0.0068" in output_text + assert ( + "#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS," + "GROUP,CHART,NOTES,DIFFMAG,DIFFERR\n" + ) in output_text + aid_header = next(line for line in output_text.splitlines() if line.startswith("#NAME,")) + aid_data_row = next(line for line in output_text.splitlines() if not line.startswith("#")) + assert len(aid_header.split(",")) == len(aid_data_row.split(",")) == 17 + assert aid_data_row.split(",")[-3:] == ["na", "0.2457", "0.0068"] + assert "|DIFFMAG=" not in output_text + assert "|DIFFERR=" not in output_text magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") assert magnitude_fields["apparent_magnitude"] == "MAG" - assert magnitude_fields["differential_magnitude"] == "NOTES subfield DIFFMAG" + assert magnitude_fields["differential_magnitude"] == "DIFFMAG" + assert magnitude_fields["differential_magnitude_error"] == "DIFFERR" def test_aid_output_records_calibrated_ensemble_members(tmp_path): @@ -783,7 +793,7 @@ def test_aid_output_records_calibrated_ensemble_members(tmp_path): assert ensemble_metadata["members"][1]["ra_deg"] == pytest.approx(10.2) assert ensemble_metadata["members"][1]["dec_deg"] == pytest.approx(-20.2) assert "Target,2450000.12345,12.3400,0.0200,V,NO,STD,ENSEMBLE (2 stars),na" in output_text - assert "|DIFFMAG=1.2346|DIFFERR=0.0079" in output_text + assert output_text.rstrip().endswith(",na,1.2346,0.0079") def test_aid_output_samples_large_derived_anchor_label_lists(tmp_path): From f47819a2efcbfdfdca23d361ce1638abe02a008d Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Tue, 4 Aug 2026 14:01:23 +1000 Subject: [PATCH 107/116] Collate AAVSO Files together --- README.md | 4 ++-- docs/README.md | 4 ++-- exotic/api/output_aavso.py | 8 ++++---- exotic/exotic.py | 19 +++++++++++++++--- exotic/output_files.py | 6 ++++-- exotic/utils.py | 10 ++++++++++ tests/test_nextastro_variability.py | 22 +++++++++++++-------- tests/test_output_files.py | 30 ++++++++++++++++++++--------- 8 files changed, 73 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 201d19c1..c6f6170c 100644 --- a/README.md +++ b/README.md @@ -215,11 +215,11 @@ Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. -Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. +Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. -`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise; each AAVSO AID file is placed in that variable directory's `AAVSO_Files` subfolder. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. `use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/normal classification. diff --git a/docs/README.md b/docs/README.md index f4134640..1c08c398 100644 --- a/docs/README.md +++ b/docs/README.md @@ -260,10 +260,10 @@ Comparison stars may be supplied in `user_info` using either `"Comparison Star(s Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. -Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. +Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. -`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, AAVSO AID, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise; each AAVSO AID file is placed in that variable directory's `AAVSO_Files` subfolder. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. `use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/normal classification. diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 3163fa6b..c22472ac 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -43,9 +43,9 @@ import re try: - from .utils import round_to_2, safe_output_filename + from ..utils import aavso_output_directory, round_to_2, safe_output_filename except ImportError: - from utils import round_to_2, safe_output_filename + from utils import aavso_output_directory, round_to_2, safe_output_filename try: from .version import __version__ except ImportError: @@ -228,7 +228,7 @@ def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): hash_id = hash_object.hexdigest()[:32] #params_file = self.dir / f"TESS_{hash_id}_{self.plname}_{tmidstr}_AAVSO.txt" - params_file = self.dir / safe_output_filename( + params_file = aavso_output_directory(self.dir) / safe_output_filename( tmidstr, hash_id, self.plname, @@ -301,7 +301,7 @@ def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" - params_file = self.dir / safe_output_filename( + params_file = aavso_output_directory(self.dir) / safe_output_filename( "TESS", tmidstr, self.p_dict['pl_name'], diff --git a/exotic/exotic.py b/exotic/exotic.py index 56131f8e..412d85f5 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -204,6 +204,7 @@ plot_adaptive_aperture_diagnostics try: # tools from utils import ( + AAVSO_OUTPUT_FOLDER_NAME, MAX_APPARENT_MAGNITUDE, filename_date_token, is_usable_apparent_magnitude, @@ -215,6 +216,7 @@ ) except ImportError: # package import from .utils import ( + AAVSO_OUTPUT_FOLDER_NAME, MAX_APPARENT_MAGNITUDE, filename_date_token, is_usable_apparent_magnitude, @@ -29771,6 +29773,15 @@ def clear_previous_fortuitous_variable_products(variable_dir): for path in output_dir.glob(f'{prefix}*'): if path.is_file(): path.unlink() + aavso_dir = output_dir / AAVSO_OUTPUT_FOLDER_NAME + if aavso_dir.is_dir(): + for path in aavso_dir.glob('AID_AAVSO_*'): + if path.is_file(): + path.unlink() + try: + aavso_dir.rmdir() + except OSError: + pass plot_path = output_dir / 'working_artifacts' / 'Stellar_Variability.png' if plot_path.is_file(): plot_path.unlink() @@ -29854,9 +29865,11 @@ def process_fortuitous_variables( base_dir = Path(info_dict['save']) / 'variables' base_dir.mkdir(parents=True, exist_ok=True) - for stale_combined_aid in base_dir.glob('AID_AAVSO_FortuitousVariables_*.txt'): - if stale_combined_aid.is_file(): - stale_combined_aid.unlink() + for combined_aavso_dir in (base_dir, base_dir / AAVSO_OUTPUT_FOLDER_NAME): + for stale_combined_aid in combined_aavso_dir.glob( + 'AID_AAVSO_FortuitousVariables_*.txt'): + if stale_combined_aid.is_file(): + stale_combined_aid.unlink() results = [] combined_vsp_params = [] logged_comparison_gap_rejections = set() diff --git a/exotic/output_files.py b/exotic/output_files.py index 5276338a..1a335234 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -6,6 +6,7 @@ try: from utils import ( + aavso_output_directory, MAGNITUDE_DECIMAL_PLACES, filename_date_token, format_magnitude_error, @@ -19,6 +20,7 @@ ) except ImportError: from .utils import ( + aavso_output_directory, MAGNITUDE_DECIMAL_PLACES, filename_date_token, format_magnitude_error, @@ -2810,7 +2812,7 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" - params_file = self.dir / safe_output_filename( + params_file = aavso_output_directory(self.dir) / safe_output_filename( "AAVSO", self.p_dict['pName'], filename_date_token(self.i_dict['date']), @@ -2946,7 +2948,7 @@ def __init__(self, fit, p_dict, i_dict, auid, chart_id, vsp_params): self.vsp_params = vsp_params def _aavso_path(self): - return self.dir / safe_output_filename( + return aavso_output_directory(self.dir) / safe_output_filename( "AID_AAVSO", self.p_dict['sName'], filename_date_token(self.i_dict['date']), diff --git a/exotic/utils.py b/exotic/utils.py index 1b877e13..ba4b7aa4 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -1,5 +1,6 @@ import logging from math import isfinite +from pathlib import Path import re import requests from numpy import floor, log10 @@ -27,6 +28,15 @@ MAX_APPARENT_MAGNITUDE = 30.0 MAGNITUDE_DECIMAL_PLACES = 4 MINIMUM_MAGNITUDE_ERROR = 0.001 +AAVSO_OUTPUT_FOLDER_NAME = 'AAVSO_Files' + + +def aavso_output_directory(root): + """Return the dedicated AAVSO output directory, creating it when needed.""" + + output_directory = Path(root) / AAVSO_OUTPUT_FOLDER_NAME + output_directory.mkdir(parents=True, exist_ok=True) + return output_directory def _clean_filename_text(value): diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index d2853623..5ffd99d8 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -3293,15 +3293,19 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product assert results[0]['output_magnitude_error_rejected_frame_count'] == 1 assert results[0]['point_count'] == frame_count - 3 variable_dir = tmp_path / 'variables' / 'optimal_variables' / 'SyntheticVSX' - assert next(variable_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')).is_file() + assert next( + (variable_dir / 'AAVSO_Files').glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt') + ).is_file() second_variable_dir = ( tmp_path / 'variables' / 'optimal_variables' / 'SyntheticVSX2' ) - assert next(second_variable_dir.glob('AID_AAVSO_SyntheticVSX2_2024-01-02.txt')).is_file() + assert next( + (second_variable_dir / 'AAVSO_Files').glob('AID_AAVSO_SyntheticVSX2_2024-01-02.txt') + ).is_file() assert next(variable_dir.glob('EnsembleSelection_SyntheticVSX_2024-01-02.json')).is_file() assert next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')).is_file() combined_aid_path = ( - tmp_path / 'variables' / 'AID_AAVSO_FortuitousVariables_2024-01-02.txt' + tmp_path / 'variables' / 'AAVSO_Files' / 'AID_AAVSO_FortuitousVariables_2024-01-02.txt' ) combined_aid_text = combined_aid_path.read_text(encoding='utf-8') combined_aid_rows = [ @@ -3338,9 +3342,9 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product assert 'exoplanet target overexposure mask is not applied' in ( selection['target']['saturation_rejection_scope'] ) - aid_text = next(variable_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')).read_text( - encoding='utf-8' - ) + aid_text = next( + (variable_dir / 'AAVSO_Files').glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt') + ).read_text(encoding='utf-8') assert ( '#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,' 'GROUP,CHART,NOTES,DIFFMAG,DIFFERR' @@ -3404,7 +3408,9 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product if line.strip() ] assert float(single_csv_rows[0].split(',')[0]) == pytest.approx(times[3]) - single_aid = next(single_dir.glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt')) + single_aid = next( + (single_dir / 'AAVSO_Files').glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt') + ) single_aid_text = single_aid.read_text(encoding='utf-8') assert '#ENSEMBLE-COMPARISONS-XC=' not in single_aid_text single_aid_row = next( @@ -3438,7 +3444,7 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product assert differential_only_dir.exists() assert next(differential_only_dir.glob('DifferentialMagnitude_*.csv')).is_file() assert not list(differential_only_dir.glob('StellarVariability_*.csv')) - assert not list(differential_only_dir.glob('AID_AAVSO_*.txt')) + assert not list((differential_only_dir / 'AAVSO_Files').glob('AID_AAVSO_*.txt')) failed_manifest = json.loads( next((failed_root / 'variables').glob('FortuitousVariables_2024-01-02.json')).read_text( encoding='utf-8' diff --git a/tests/test_output_files.py b/tests/test_output_files.py index fb0d7d1d..40df48a6 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -553,7 +553,7 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): None, ) - output_file = tmp_path / "AAVSO_HAT-P-32b_2020-01-01.txt" + output_file = tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" output_text = output_file.read_text(encoding="utf-8") assert "#OBSDATE=2020-01-01" in output_text @@ -624,7 +624,7 @@ def test_aavso_output_omits_obsname_header_when_blank(tmp_path): None, ) - output_file = tmp_path / "AAVSO_HAT-P-32b_2020-01-01.txt" + output_file = tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" output_text = output_file.read_text(encoding="utf-8") assert "#OBSNAME=" not in output_text @@ -700,7 +700,9 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() - output_text = (tmp_path / "AID_AAVSO_HAT-P-32_2020-01-01.txt").read_text(encoding="utf-8") + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" + ).read_text(encoding="utf-8") metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") assert metadata["source"] == "NextAstro photometry catalog" @@ -774,7 +776,9 @@ def test_aid_output_records_calibrated_ensemble_members(tmp_path): AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() - output_text = (tmp_path / "AID_AAVSO_Target_2020-01-01.txt").read_text(encoding="utf-8") + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_Target_2020-01-01.txt" + ).read_text(encoding="utf-8") metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") ensemble_metadata = aavso_json_header(output_text, "ENSEMBLE-COMPARISONS-XC") assert metadata["ensemble_reference"] is True @@ -832,7 +836,9 @@ def test_aid_output_samples_large_derived_anchor_label_lists(tmp_path): AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() - output_text = (tmp_path / "AID_AAVSO_HAT-P-32_2020-01-01.txt").read_text(encoding="utf-8") + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" + ).read_text(encoding="utf-8") metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") assert metadata["derived_reference_anchor_count"] == 20 @@ -872,7 +878,9 @@ def test_aid_output_floors_reported_magnitude_errors(tmp_path): AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() - output_text = (tmp_path / "AID_AAVSO_HAT-P-32_2020-01-01.txt").read_text(encoding="utf-8") + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" + ).read_text(encoding="utf-8") metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") assert metadata["apparent_magnitude_error"] == pytest.approx(0.001) @@ -908,7 +916,9 @@ def test_aid_output_skips_over_30_magnitude_rows(tmp_path): AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() - output_text = (tmp_path / "AID_AAVSO_HAT-P-32_2020-01-01.txt").read_text(encoding="utf-8") + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" + ).read_text(encoding="utf-8") assert "HAT-P-32,2450000.12345" not in output_text @@ -1716,7 +1726,7 @@ def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_p None, ) - output_file = tmp_path / "AAVSO_HAT-P-32b_2020-01-01.txt" + output_file = tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" output_text = output_file.read_text(encoding="utf-8") assert "Am1=0 +/- 0" in output_text @@ -1923,7 +1933,9 @@ def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): }, ) - output_text = (tmp_path / "AAVSO_HAT-P-32b_2020-01-01.txt").read_text(encoding="utf-8") + output_text = ( + tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" + ).read_text(encoding="utf-8") results = aavso_json_header(output_text, "RESULTS-XC") assert "a/R*" in results From ccef030b48d466af608740cc4693c1b07e14ac03 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 5 Aug 2026 06:51:49 +1000 Subject: [PATCH 108/116] ensemble finders + AAVSO collation --- README.md | 2 +- docs/README.md | 2 +- exotic/exotic.py | 72 +++++++++++++++++++++++--- exotic/output_files.py | 80 ++++++++++++++++++++++++++++- exotic/plots.py | 66 +++++++++++++++++++----- tests/test_nextastro_variability.py | 23 +++++++++ tests/test_output_files.py | 34 ++++++++++++ tests/test_plots.py | 36 +++++++++++++ 8 files changed, 291 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c6f6170c..1580d401 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. -Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. +Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. diff --git a/docs/README.md b/docs/README.md index 1c08c398..3fe6986e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -260,7 +260,7 @@ Comparison stars may be supplied in `user_info` using either `"Comparison Star(s Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. -Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. +Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. diff --git a/exotic/exotic.py b/exotic/exotic.py index 412d85f5..dd7179e6 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -17738,6 +17738,38 @@ def tracked_comparison_position(tracked_comparison_stars, comp_index): return list(tracked_comparison_stars[index]) +def selected_comparison_finder_entries(comparison_stars, comp_index=None, + ensemble_member_keys=None): + """Return labelled pixel positions for the selected single or ensemble reference.""" + + if ensemble_member_keys: + keys = list(ensemble_member_keys) + elif comp_index is not None: + keys = [f'comp{int(comp_index) + 1}'] + else: + keys = [] + + entries = [] + for key in keys: + match = re.fullmatch(r'comp(\d+)', str(key).strip(), flags=re.IGNORECASE) + if match is None: + continue + index = int(match.group(1)) - 1 + try: + position = tracked_comparison_position(comparison_stars, index) + position_values = np.asarray(position, dtype=float).reshape(-1) + except (IndexError, TypeError, ValueError): + continue + if position_values.size < 2 or not np.all(np.isfinite(position_values[:2])): + continue + entries.append({ + 'key': f'comp{index + 1}', + 'label': f'Comp {index + 1}', + 'position': [float(position_values[0]), float(position_values[1])], + }) + return entries + + def check_comp_star_exists(user_stars, vsp_star, tol=VSP_COMPARISON_MATCH_TOLERANCE_PIXELS): """Return the nearest user-entered comparison within ``tol`` pixels. @@ -31004,6 +31036,7 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p 'coverage_rejected': comp_summary.get('coverage_rejected', False), 'ensemble_frame_rejected_count': comp_summary.get('ensemble_frame_rejected_count', 0), 'ensemble_frame_required_valid_pairs': comp_summary.get('ensemble_frame_required_valid_pairs', 0), + 'ensemble_member_keys': list(comp_summary.get('ensemble_member_keys') or []), 'fit': selection_fit, 'provisional_fit': None, 'full_reduction_fit': final_reduction.get('fit'), @@ -34808,6 +34841,11 @@ def lookup_archive_ephemeris(): if selected_is_ensemble else science_comp_stars[selected_comp_index] ) + finder_entries = selected_comparison_finder_entries( + science_comp_stars, + comp_index=selected_comp_index, + ensemble_member_keys=selected_attempt.get('ensemble_member_keys'), + ) selected_min_aperture = 0 if comparison_calibration['method'] == 'psf' else comparison_calibration['aper'] selected_min_annulus = comparison_calibration['annulus'] selected_a = None if comparison_calibration['method'] == 'psf' else comparison_calibration['a'] @@ -34953,6 +34991,7 @@ def lookup_archive_ephemeris(): 'ensemble' if selected_is_ensemble else selected_comp_index + 1 ), comp_star_coords=selected_comp_coords, + finder_comparison_entries=finder_entries, min_aperture=selected_min_aperture, min_annulus=selected_min_annulus, aperture_index=selected_a, @@ -35522,7 +35561,16 @@ def lookup_archive_ephemeris(): np.isfinite(centroid_positions['x_ref'][0]) and np.isfinite(centroid_positions['y_ref'][0]) ) - if reference_centroid_available: + finder_entries = list(photometry_info.get('finder_comparison_entries') or []) + if not finder_entries and reference_centroid_available: + finder_entries = [{ + 'label': 'Comp Star', + 'position': [ + float(centroid_positions['x_ref'][0]), + float(centroid_positions['y_ref'][0]), + ], + }] + if finder_entries: firstImage = ensure_first_reduction_image_for_fov( firstImage, inputfiles[0], @@ -35534,21 +35582,33 @@ def lookup_archive_ephemeris(): demosaic_mult, bad_pixel_reference=bad_pixel_reference, ) + finder_positions = [entry['position'] for entry in finder_entries] + finder_labels = [entry['label'] for entry in finder_entries] + finder_target_position = [ + float(centroid_positions['x_targ'][0]), + float(centroid_positions['y_targ'][0]), + ] + first_target_centroid = np.asarray(psf_data['target'][0, :2], dtype=float) + if np.all(np.isfinite(first_target_centroid)): + finder_target_position = first_target_centroid.tolist() plot_fov(fov_aperture, fov_annulus, sigma_display, - centroid_positions['x_targ'][0], centroid_positions['y_targ'][0], - centroid_positions['x_ref'][0], centroid_positions['y_ref'][0], + finder_target_position[0], finder_target_position[1], + finder_positions[0][0], finder_positions[0][1], firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov, sky_inner_radius=fov_sky_geometry['inner_radius'], - sky_outer_radius=fov_sky_geometry['outer_radius']) + sky_outer_radius=fov_sky_geometry['outer_radius'], + comparison_positions=finder_positions, + comparison_labels=finder_labels) + if reference_centroid_available: plot_centroids(centroid_positions['x_targ'], centroid_positions['y_targ'], centroid_positions['x_ref'], centroid_positions['y_ref'], goodTimes, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) else: log_info( - "Skipping reference-star FOV and centroid plots because the selected reference is " - "a comparison-star ensemble rather than a single star." + "Skipping reference-star centroid plots because the selected reference does not " + "have one single-star centroid series." ) plot_flux(goodTimes, flux_values['flux_tar'], flux_values['flux_unc_tar'], diff --git a/exotic/output_files.py b/exotic/output_files.py index 1a335234..18c831b1 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -62,6 +62,64 @@ ) +AAVSO_FINDER_STRETCH_NAMES = ( + 'LinearStretch', + 'SquaredStretch', + 'SqrtStretch', + 'LogStretch', +) + + +def copy_aavso_supporting_artifacts(save, target_name, observation_date): + """Copy final lightcurve, finder, triangle, and QC products into ``AAVSO_Files``.""" + + output_dir = Path(save) + working_artifacts_dir = output_dir / 'working_artifacts' + diagnostics_dir = output_dir / 'Diagnostics' + date_token = filename_date_token(observation_date) + source_paths = [ + output_dir / safe_output_filename( + 'FinalLightCurve', target_name, date_token, extension=extension + ) + for extension in ('png', 'pdf') + ] + source_paths.append( + working_artifacts_dir / safe_output_filename( + 'FinalLightCurve', target_name, date_token, extension='csv' + ) + ) + for stretch_name in AAVSO_FINDER_STRETCH_NAMES: + source_paths.extend( + working_artifacts_dir / safe_output_filename( + 'FOV', target_name, stretch_name, date_token, extension=extension + ) + for extension in ('png', 'pdf') + ) + for prefix, extensions in ( + ('FinalTriangle', ('png',)), + ('Triangle', ('png',)), + ('ZoomedTrianglePlot', ('png',)), + ('KTMF_QC', ('png', 'pdf')), + ('PriorPosteriorComparison', ('png', 'pdf')), + ): + source_paths.extend( + diagnostics_dir / safe_output_filename( + prefix, target_name, date_token, extension=extension + ) + for extension in extensions + ) + + aavso_dir = aavso_output_directory(output_dir) + copied_paths = [] + for source_path in source_paths: + if not source_path.is_file(): + continue + destination_path = aavso_dir / source_path.name + shutil.copy2(source_path, destination_path) + copied_paths.append(destination_path) + return copied_paths + + def aavso_airmass_results(fit): if getattr(fit, 'airmass_fit_skipped', False): return ( @@ -2928,6 +2986,12 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, f.write(f"{round(self.fit.time[aavsoC], 8)},{round(self.fit.data[aavsoC], 7)}," f"{round(self.fit.dataerr[aavsoC], 7)},{round(airmasses[aavsoC], 7)}," f"{round(detrend_model[aavsoC], 7)}\n") + copy_aavso_supporting_artifacts( + self.dir, + self.p_dict['pName'], + self.i_dict['date'], + ) + def plate_status(self, plate_status: PlateStatus): plate_status_file = self.dir / "working_artifacts" / safe_output_filename( "PlateStatus", @@ -3069,15 +3133,27 @@ def _write_aavso(self, params_file, use_row_names=False, include_comparison_meta return params_file def aavso(self): - return self._write_aavso(self._aavso_path()) + params_file = self._write_aavso(self._aavso_path()) + copy_aavso_supporting_artifacts( + self.dir, + self.p_dict.get('pName') or self.p_dict.get('sName'), + self.i_dict['date'], + ) + return params_file def combined_aavso(self): """Write one AID file containing rows for multiple named variables.""" - return self._write_aavso( + params_file = self._write_aavso( self._aavso_path(), use_row_names=True, include_comparison_metadata=False, ) + copy_aavso_supporting_artifacts( + self.dir, + self.p_dict.get('pName') or self.p_dict.get('sName'), + self.i_dict['date'], + ) + return params_file def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): diff --git a/exotic/plots.py b/exotic/plots.py index 976084a2..92ec5e2e 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -100,13 +100,38 @@ def plot_centroids(x_targ, y_targ, x_ref, y_ref, times, target_name, save, date) plt.close() def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_scale, targ_name, save, date, - opt_method, min_aper_fov, min_annulus_fov, sky_inner_radius=None, sky_outer_radius=None): + opt_method, min_aper_fov, min_annulus_fov, sky_inner_radius=None, sky_outer_radius=None, + comparison_positions=None, comparison_labels=None): - ref_circle, ref_circle_sky = None, None - picframe = 10. * (aper + 15. * sigma) + if comparison_positions is None: + comparison_positions = [[x_ref, y_ref]] + valid_comparison_positions = [] + for position in comparison_positions: + try: + position_values = np.asarray(position, dtype=float).reshape(-1) + except (TypeError, ValueError): + continue + if position_values.size < 2 or not np.all(np.isfinite(position_values[:2])): + continue + valid_comparison_positions.append(( + float(position_values[0]), + float(position_values[1]), + )) + if aper < 0: + valid_comparison_positions = [] + + labels = list(comparison_labels or []) + if len(labels) != len(valid_comparison_positions): + if len(valid_comparison_positions) == 1: + labels = ['Comp Star'] + else: + labels = [f'Comp {index + 1}' for index in range(len(valid_comparison_positions))] - pltx = [max([0, min([x_targ, x_ref]) - picframe]), min([np.shape(image)[1], max([x_targ, x_ref]) + picframe])] - plty = [max([0, min([y_targ, y_ref]) - picframe]), min([np.shape(image)[0], max([y_targ, y_ref]) + picframe])] + picframe = 10. * (aper + 15. * sigma) + plotted_x = [x_targ, *(position[0] for position in valid_comparison_positions)] + plotted_y = [y_targ, *(position[1] for position in valid_comparison_positions)] + pltx = [max(0, min(plotted_x) - picframe), min(np.shape(image)[1], max(plotted_x) + picframe)] + plty = [max(0, min(plotted_y) - picframe), min(np.shape(image)[0], max(plotted_y) + picframe)] for stretch in [LinearStretch(), SquaredStretch(), SqrtStretch(), LogStretch()]: fig, ax = plt.subplots() @@ -135,12 +160,6 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc target_circle_sky_inner = plt.Circle((x_targ, y_targ), local_sky_inner_radius, color=outer_circle_color, fill=False, ls='--') target_circle_sky_outer = plt.Circle((x_targ, y_targ), local_sky_outer_radius, color=outer_circle_color, fill=False, ls='-') - # IF EXOTIC is using a comparison star, create its circles - if aper >= 0: - ref_circle = plt.Circle((x_ref, y_ref), aper, color=outer_circle_color, fill=False, ls='-') - ref_circle_sky_inner = plt.Circle((x_ref, y_ref), local_sky_inner_radius, color=outer_circle_color, fill=False, ls='--') - ref_circle_sky = plt.Circle((x_ref, y_ref), local_sky_outer_radius, color=outer_circle_color, fill=False, ls='-') - interval = ZScaleInterval() vmin, vmax = interval.get_limits(image) @@ -155,12 +174,31 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc ax.text(x_targ + local_sky_outer_radius + 5, y_targ, targ_name, color='w', fontsize=10, path_effects=[path_effects.withStroke(linewidth=2, foreground='black')]) - if aper >= 0: #EXOTIC is using a comparison star + for (comparison_x, comparison_y), comparison_label in zip( + valid_comparison_positions, labels): + ref_circle = plt.Circle( + (comparison_x, comparison_y), aper, + color=outer_circle_color, fill=False, ls='-' + ) + ref_circle_sky_inner = plt.Circle( + (comparison_x, comparison_y), local_sky_inner_radius, + color=outer_circle_color, fill=False, ls='--' + ) + ref_circle_sky = plt.Circle( + (comparison_x, comparison_y), local_sky_outer_radius, + color=outer_circle_color, fill=False, ls='-' + ) ax.add_artist(ref_circle) ax.add_artist(ref_circle_sky_inner) ax.add_artist(ref_circle_sky) - ax.text(x_ref + local_sky_outer_radius + 5, y_ref, 'Comp Star', color='w', fontsize=10, - path_effects=[path_effects.withStroke(linewidth=2, foreground='black')]) + ax.text( + comparison_x + local_sky_outer_radius + 5, + comparison_y, + comparison_label, + color='w', + fontsize=10, + path_effects=[path_effects.withStroke(linewidth=2, foreground='black')], + ) handles = [] if opt_method == "PSF": diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 5ffd99d8..1a06a9be 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -1319,6 +1319,29 @@ def test_tracked_comparison_position_keeps_full_field_anchor_index_after_science ) == [449.0, 267.0] +def test_selected_comparison_finder_entries_include_every_ensemble_member(): + entries = exotic_module.selected_comparison_finder_entries( + [[100.0, 200.0], [300.0, 400.0], [500.0, 600.0]], + ensemble_member_keys=['comp3', 'comp1'], + ) + + assert entries == [ + {'key': 'comp3', 'label': 'Comp 3', 'position': [500.0, 600.0]}, + {'key': 'comp1', 'label': 'Comp 1', 'position': [100.0, 200.0]}, + ] + + +def test_selected_comparison_finder_entries_do_not_depend_on_aavso_metadata(): + entries = exotic_module.selected_comparison_finder_entries( + [[217.0, 210.0]], + comp_index=0, + ) + + assert entries == [ + {'key': 'comp1', 'label': 'Comp 1', 'position': [217.0, 210.0]}, + ] + + def test_clear_v_calibration_fallback_merges_aavso_with_existing_pool(monkeypatch): calls = [] supplied_positions = [[100, 200]] diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 40df48a6..a889c1ae 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -542,6 +542,24 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): "wl_min": None, "wl_max": None, } + final_plot_source = tmp_path / "FinalLightCurve_HAT-P-32b_2020-01-01.png" + final_plot_source.write_bytes(b"final lightcurve") + diagnostics_dir = tmp_path / "Diagnostics" + diagnostics_dir.mkdir() + diagnostic_sources = [ + diagnostics_dir / filename + for filename in ( + "FinalTriangle_HAT-P-32b_2020-01-01.png", + "Triangle_HAT-P-32b_2020-01-01.png", + "ZoomedTrianglePlot_HAT-P-32b_2020-01-01.png", + "KTMF_QC_HAT-P-32b_2020-01-01.png", + "KTMF_QC_HAT-P-32b_2020-01-01.pdf", + "PriorPosteriorComparison_HAT-P-32b_2020-01-01.png", + "PriorPosteriorComparison_HAT-P-32b_2020-01-01.pdf", + ) + ] + for diagnostic_source in diagnostic_sources: + diagnostic_source.write_bytes(diagnostic_source.name.encode("utf-8")) OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( {"ra": "", "dec": "", "x": "493", "y": "202"}, @@ -555,6 +573,13 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): output_file = tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" output_text = output_file.read_text(encoding="utf-8") + assert ( + tmp_path / "AAVSO_Files" / final_plot_source.name + ).read_bytes() == b"final lightcurve" + for diagnostic_source in diagnostic_sources: + assert ( + tmp_path / "AAVSO_Files" / diagnostic_source.name + ).read_bytes() == diagnostic_source.name.encode("utf-8") assert "#OBSDATE=2020-01-01" in output_text assert "#OBSNAME=Whipple Observatory" in output_text @@ -697,12 +722,21 @@ def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): "source_id": 12345, "separation_arcsec": 0.2, }] + working_artifacts_dir = tmp_path / "working_artifacts" + working_artifacts_dir.mkdir() + finder_source = ( + working_artifacts_dir / "FOV_HAT-P-32b_LinearStretch_2020-01-01.png" + ) + finder_source.write_bytes(b"finder chart") AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() output_text = ( tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" ).read_text(encoding="utf-8") + assert ( + tmp_path / "AAVSO_Files" / finder_source.name + ).read_bytes() == b"finder chart" metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") assert metadata["source"] == "NextAstro photometry catalog" diff --git a/tests/test_plots.py b/tests/test_plots.py index 0b73360d..d3963c29 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -297,6 +297,42 @@ def spy_legend(*args, **kwargs): assert set(labels) == {"PSF Photometry"} +def test_plot_fov_marks_every_ensemble_comparison(tmp_path, monkeypatch): + plotted_labels = [] + original_text = Axes.text + + def spy_text(self, x, y, text, *args, **kwargs): + plotted_labels.append(text) + return original_text(self, x, y, text, *args, **kwargs) + + monkeypatch.setattr(Axes, "text", spy_text) + + plot_fov( + aper=8.0, + annulus=20.0, + sigma=2.0, + x_targ=50.0, + y_targ=60.0, + x_ref=90.0, + y_ref=100.0, + image=np.ones((220, 220)), + image_scale="Image scale in arcsec/pixel: 0.53", + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + opt_method="Aperture", + min_aper_fov=8.0, + min_annulus_fov=20.0, + comparison_positions=[[90.0, 100.0], [130.0, 140.0], [170.0, 180.0]], + comparison_labels=["Comp 1", "Comp 3", "Comp 4"], + ) + + assert {"Target", "Comp 1", "Comp 3", "Comp 4"}.issubset(plotted_labels) + assert ( + tmp_path / "working_artifacts" / "FOV_Target_LinearStretch_2026-03-09.png" + ).is_file() + + def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): plot_individual_comp_star_calibration_series( times=np.array([1.0, 2.0, 3.0]), From abe35d580cb4d1205d199b297e44389f38d8e067 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Sun, 9 Aug 2026 12:50:20 +1000 Subject: [PATCH 109/116] fallback/retry VSP rather than crash. No unnecessary align when WCS. --- README.md | 31 +- docs/README.md | 23 +- docs/system_prompt.txt | 5 +- exotic/api/colab.py | 1 + exotic/exotic.py | 501 ++++++++++++++++---- exotic/exotic_gui.py | 11 +- exotic/inputs.py | 26 +- exotic/utils.py | 28 ++ inits.json | 35 +- output/pdf/EXOTIC_inits_default_options.pdf | Bin 0 -> 14849 bytes tests/test_boolean_config.py | 63 +++ tests/test_centroid_wcs.py | 137 +++++- tests/test_exotic_proper_motion.py | 84 +++- tests/test_inputs.py | 45 ++ tests/test_nextastro_variability.py | 121 ++++- tests/test_nonlinear_ld.py | 79 +++ tests/test_utils.py | 13 + 17 files changed, 1053 insertions(+), 150 deletions(-) create mode 100644 output/pdf/EXOTIC_inits_default_options.pdf create mode 100644 tests/test_boolean_config.py diff --git a/README.md b/README.md index 1580d401..2afa2467 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Name (aavso.org/filters)": "V", "Observing Notes": "Weather, seeing was nice.", - "Plate Solution? (y/n)": "y", + "Plate Solution? (y/n)": true, "Target Star X & Y Pixel": [424, 286], "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263], [], [], [], [], [], [], [], []], @@ -162,9 +162,10 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Maximum Wavelength (nm)": null, "Fast Aperture Mask (y/n)": false, - "prefer_pixel_values_over_wcs_for_target": "n", - "use_psf_photometry": "y", - "use_aperture_photometry": "y", + "allow_pixel_alignment_fallback": false, + "prefer_pixel_values_over_wcs_for_target": false, + "use_psf_photometry": true, + "use_aperture_photometry": true, "use_aperture_corrections_and_full_image_fwhm": false, "use_ensemble_photometry_rather_than_single_comp": false, "stellar_variability_only": false, @@ -176,15 +177,15 @@ Get EXOTIC up and running faster with a json file. Please see the included file "photometer_fortuitous_variables": true, "use_single_comparison_for_fortuitous_variables": true, "use_nextastro_vsx_cache_first": false, - "skip_low_comparison_coverage_rejection": "n", - "fit_lightcurve_to_every_comparison_candidate": "n", + "skip_low_comparison_coverage_rejection": false, + "fit_lightcurve_to_every_comparison_candidate": false, "detrend_on_outoftransit_baseline": true, "final_fit_baseline_duration_multiplier": 1.0, - "use_eebls_to_initialize_tmid_and_bounds": "y", - "pick_comparison_by_eebls_snr": "y", - "use_impactparameter_rather_than_inclination_to_fit": "y", - "Use target-driven comp selection rather than comp-driven comp selection": "n", - "require_comp_star": "y", + "use_eebls_to_initialize_tmid_and_bounds": true, + "pick_comparison_by_eebls_snr": true, + "use_impactparameter_rather_than_inclination_to_fit": true, + "Use target-driven comp selection rather than comp-driven comp selection": false, + "require_comp_star": true, "Pixel Scale (Ex: 5.21 arcsecs/pixel)": null, @@ -195,9 +196,11 @@ Get EXOTIC up and running faster with a json file. Please see the included file ### Comparison-star mode tags -Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended; EXOTIC also accepts equivalent values such as `"y"` and `"n"`. +Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended. Every initialization boolean also accepts numeric `1`/`0` and case-insensitive strings `"y"`/`"n"`, `"yes"`/`"no"`, `"true"`/`"false"`, and `"on"`/`"off"`. -Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Sexagesimal values must be quoted because they are JSON strings; forms such as `[[02:04:09.90, +46:41:23.0]]` are not valid JSON. Celestial coordinates require a usable WCS and are projected onto the selected reference image before photometry; after projection they are treated identically to supplied X/Y positions. +Raw-image reductions are WCS-authoritative by default. For every frame with celestial WCS, EXOTIC projects the target and comparison-star sky coordinates through that frame's own FITS header and starts centroiding at those projected pixels. It does not register the image to a reference frame with Astroalign. Set `"allow_pixel_alignment_fallback": true` only when you explicitly want legacy pixel-based registration for a frame whose WCS-derived candidate is unusable. The existing `"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"` option explicitly enables pixel alignment for the entire run. + +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Sexagesimal values must be quoted because they are JSON strings; forms such as `[[02:04:09.90, +46:41:23.0]]` are not valid JSON. Supplied X/Y positions are converted to sky coordinates with the reference frame's WCS; during photometry those sky coordinates are projected independently through every retained frame's own WCS header. | Reduction | Requested comparison mode | `optional_info` settings | |---|---|---| @@ -215,7 +218,7 @@ Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. -Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. +Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. Each reduction writes its run log from startup through shutdown to a unique `Diagnostics/EXOTIC_RunLog__pid.log`, so separate or midnight-spanning runs do not overwrite or split one another. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. diff --git a/docs/README.md b/docs/README.md index 3fe6986e..54bdc14a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -179,7 +179,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Name (aavso.org/filters)": "V", "Observing Notes": "Weather, seeing was nice.", - "Plate Solution? (y/n)": "n", + "Plate Solution? (y/n)": false, "Target Star X & Y Pixel": [424, 286], "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263]], @@ -216,9 +216,10 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, "Fast Aperture Mask (y/n)": false, - "prefer_pixel_values_over_wcs_for_target": "n", - "use_psf_photometry": "y", - "use_aperture_photometry": "y", + "allow_pixel_alignment_fallback": false, + "prefer_pixel_values_over_wcs_for_target": false, + "use_psf_photometry": true, + "use_aperture_photometry": true, "use_aperture_corrections_and_full_image_fwhm": false, "use_ensemble_photometry_rather_than_single_comp": false, "stellar_variability_only": false, @@ -231,16 +232,20 @@ Get EXOTIC up and running faster with a json file. Please see the included file "use_single_comparison_for_fortuitous_variables": true, "use_nextastro_vsx_cache_first": false, "detrend_on_outoftransit_baseline": true, - "use_impactparameter_rather_than_inclination_to_fit": "y", - "skip_low_comparison_coverage_rejection": "n", - "require_comp_star": "y" + "use_impactparameter_rather_than_inclination_to_fit": true, + "skip_low_comparison_coverage_rejection": false, + "require_comp_star": true } } ``` ### Comparison-star mode tags -Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended; EXOTIC also accepts equivalent values such as `"y"` and `"n"`. +Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended. Every initialization boolean also accepts numeric `1`/`0` and case-insensitive strings `"y"`/`"n"`, `"yes"`/`"no"`, `"true"`/`"false"`, and `"on"`/`"off"`. + +Raw-image reductions are WCS-authoritative by default. For every frame with celestial WCS, EXOTIC projects the target and comparison-star sky coordinates through that frame's own FITS header and starts centroiding at those projected pixels. It does not register the image to a reference frame with Astroalign. Set `"allow_pixel_alignment_fallback": true` only when you explicitly want legacy pixel-based registration for a frame whose WCS-derived candidate is unusable. The existing `"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"` option explicitly enables pixel alignment for the entire run. + +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. Supplied X/Y positions are converted to sky coordinates with the reference frame's WCS; during photometry those sky coordinates are projected independently through every retained frame's own WCS header. | Reduction | Requested comparison mode | `optional_info` settings | |---|---|---| @@ -260,7 +265,7 @@ Comparison stars may be supplied in `user_info` using either `"Comparison Star(s Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. -Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. +Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. Each reduction writes its run log from startup through shutdown to a unique `Diagnostics/EXOTIC_RunLog__pid.log`, so separate or midnight-spanning runs do not overwrite or split one another. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. diff --git a/docs/system_prompt.txt b/docs/system_prompt.txt index 32bccef1..461b488a 100644 --- a/docs/system_prompt.txt +++ b/docs/system_prompt.txt @@ -870,6 +870,7 @@ Example `inits.json` file: "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", + "Boolean Values": "All boolean settings accept JSON true/false, numeric 1/0, or case-insensitive strings y/n. The equivalent strings yes/no and on/off are also accepted.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." }, @@ -892,8 +893,8 @@ Example `inits.json` file: "Filter Name (aavso.org/filters)": "CV", "Observing Notes": "Weather, seeing was nice.", - "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "n", + "Plate Solution? (y/n)": true, + "Add Comparison Stars from AAVSO? (y/n)": false, "Target Star X & Y Pixel": "[424, 286]", "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", diff --git a/exotic/api/colab.py b/exotic/api/colab.py index 37ae779b..1ccb8dbd 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -379,6 +379,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Filter Minimum Wavelength (nm)": %s, "Filter Maximum Wavelength (nm)": %s, "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "allow_pixel_alignment_fallback": false, "bad_wcs_threshold_percent": 3.0, "detrend_on_outoftransit_baseline": true, "use_eebls_to_initialize_tmid_and_bounds": "y", diff --git a/exotic/exotic.py b/exotic/exotic.py index dd7179e6..6fbd472a 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -94,7 +94,7 @@ import imreg_dft as ird from pathlib import Path import logging -from logging.handlers import TimedRotatingFileHandler +import tempfile from matplotlib.animation import FuncAnimation # Pyplot imports import bottleneck as bn @@ -206,6 +206,7 @@ from utils import ( AAVSO_OUTPUT_FOLDER_NAME, MAX_APPARENT_MAGNITUDE, + coerce_boolean_config_value, filename_date_token, is_usable_apparent_magnitude, magnitude_text, @@ -218,6 +219,7 @@ from .utils import ( AAVSO_OUTPUT_FOLDER_NAME, MAX_APPARENT_MAGNITUDE, + coerce_boolean_config_value, filename_date_token, is_usable_apparent_magnitude, magnitude_text, @@ -248,6 +250,8 @@ _BJD_FALLBACK_WARNING_LOGGED = False _RUNTIME_FILE_HANDLER_NAME = "exotic-runtime-file" _RUNTIME_CONSOLE_HANDLER_NAME = "exotic-runtime-console" +_RUNTIME_LOG_BASENAME = None +_RUNTIME_LOG_PATH = None _RUNTIME_TRACEBACK_WATCHDOG_SECONDS_ENV = "EXOTIC_RUNTIME_TRACEBACK_WATCHDOG_SECONDS" _RUNTIME_TRACEBACK_WATCHDOG_DEFAULT_SECONDS = 1800.0 _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = False @@ -432,6 +436,9 @@ CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX = 0.05 CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX = 0.10 VSP_COMPARISON_MATCH_TOLERANCE_PIXELS = 3.0 +AAVSO_VSP_REQUEST_TIMEOUT_SECONDS = 30 +AAVSO_VSP_RETRY_DELAY_SECONDS = 60 +AAVSO_VSP_MAX_RETRIES = 5 REFERENCE_FALLBACK_COMPARISON_LIMIT = 10 REFERENCE_FALLBACK_DETECTION_MAX_STARS = 60 REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS = 12 @@ -7972,28 +7979,160 @@ def cancel_runtime_traceback_watchdog(): _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = False -def configure_runtime_logging(): - global _RUNTIME_LOGGING_CONFIGURED +def _runtime_output_directory_from_command_line(argv=None): + """Return the configured output directory when an init file is on the command line.""" + command_line = list(sys.argv[1:] if argv is None else argv) + init_options = { + '-red', '--reduce', '-pre', '--prereduced', '-phot', '--photometry', '-rt', '--realtime', + } + init_path = None + + for index, argument in enumerate(command_line): + if argument in init_options: + if index + 1 < len(command_line) and command_line[index + 1]: + init_path = command_line[index + 1] + break + for option in init_options: + option_prefix = f"{option}=" + if argument.startswith(option_prefix): + init_path = argument[len(option_prefix):] + break + if init_path is not None: + break + + if not init_path: + return None + + try: + with open(Path(init_path).expanduser(), encoding='utf-8') as init_file: + init_data = json.load(init_file) + output_directory = init_data.get('user_info', {}).get('Directory to Save Plots') + except (OSError, TypeError, ValueError): + return None + + return output_directory or None + + +def _new_runtime_log_basename(): + run_timestamp = datetime.now().strftime("%Y%m%dT%H%M%S_%f") + return f"EXOTIC_RunLog_{run_timestamp}_pid{os.getpid()}.log" + + +def _runtime_log_directory(output_dir=None): + if output_dir: + return Path(output_dir).expanduser().resolve() / "Diagnostics" + return Path(tempfile.gettempdir()).resolve() / "exotic-runtime-logs" + + +def _available_runtime_log_path(directory, basename): + candidate = directory / basename + if not candidate.exists(): + return candidate + + stem = Path(basename).stem + suffix = Path(basename).suffix + duplicate_number = 2 + while True: + candidate = directory / f"{stem}_{duplicate_number}{suffix}" + if not candidate.exists(): + return candidate + duplicate_number += 1 + + +def _runtime_file_formatter(): + return logging.Formatter( + "%(asctime)s.%(msecs)03d [%(threadName)-12.12s] %(levelname)-5.5s " + "%(funcName)s:%(lineno)d - %(message)s", + "%Y-%m-%dT%H:%M:%S", + ) + + +def _open_runtime_file_handler(log_path): + file_handler = logging.FileHandler(filename=log_path, mode='a', encoding='utf-8') + file_handler._exotic_runtime_handler_name = _RUNTIME_FILE_HANDLER_NAME + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(_runtime_file_formatter()) + log.addHandler(file_handler) + return file_handler + + +def _close_runtime_file_handler(): + file_handler = _find_runtime_handler(_RUNTIME_FILE_HANDLER_NAME) + if file_handler is None: + return + + log.removeHandler(file_handler) + try: + file_handler.flush() + finally: + file_handler.close() + + +def close_runtime_logging(): + global _RUNTIME_LOGGING_CONFIGURED, _RUNTIME_LOG_BASENAME, _RUNTIME_LOG_PATH + + _close_runtime_file_handler() + _RUNTIME_LOGGING_CONFIGURED = False + _RUNTIME_LOG_BASENAME = None + _RUNTIME_LOG_PATH = None + + +def configure_runtime_logging(output_dir=None, start_new_run=False): + global _RUNTIME_LOGGING_CONFIGURED, _RUNTIME_LOG_BASENAME, _RUNTIME_LOG_PATH logging.root.setLevel(logging.DEBUG) log.setLevel(logging.DEBUG) - if _find_runtime_handler(_RUNTIME_FILE_HANDLER_NAME) is None: + if start_new_run: + _close_runtime_file_handler() + _RUNTIME_LOG_BASENAME = _new_runtime_log_basename() + _RUNTIME_LOG_PATH = None + if output_dir is None: + output_dir = _runtime_output_directory_from_command_line() + elif _RUNTIME_LOG_BASENAME is None: + _RUNTIME_LOG_BASENAME = _new_runtime_log_basename() + + try: + requested_log_directory = _runtime_log_directory(output_dir) + requested_log_directory.mkdir(parents=True, exist_ok=True) + except Exception as exc: + print(f"Warning: Could not initialize the EXOTIC run log directory ({exc}).") + requested_log_directory = _runtime_log_directory() + requested_log_directory.mkdir(parents=True, exist_ok=True) + output_dir = None + + file_handler = _find_runtime_handler(_RUNTIME_FILE_HANDLER_NAME) + if file_handler is not None and output_dir: + current_log_path = Path(file_handler.baseFilename).resolve() + requested_parent = requested_log_directory.resolve() + if current_log_path.parent != requested_parent: + destination = _available_runtime_log_path(requested_parent, _RUNTIME_LOG_BASENAME) + log.debug(f"Relocating EXOTIC run log to {destination}") + _close_runtime_file_handler() + try: + shutil.move(str(current_log_path), str(destination)) + except Exception as exc: + print(f"Warning: Could not move the EXOTIC run log into Diagnostics ({exc}).") + destination = current_log_path + try: + file_handler = _open_runtime_file_handler(destination) + _RUNTIME_LOG_PATH = destination + _RUNTIME_LOG_BASENAME = destination.name + print(f"EXOTIC run log: {destination}", flush=True) + except Exception as exc: + file_handler = None + print(f"Warning: Could not reopen the EXOTIC run log ({exc}).") + + if file_handler is None: + log_path = _available_runtime_log_path(requested_log_directory, _RUNTIME_LOG_BASENAME) try: - file_handler = TimedRotatingFileHandler(filename="exotic.log", when="midnight", backupCount=2) + file_handler = _open_runtime_file_handler(log_path) except Exception as exc: - print(f"Warning: Could not initialize exotic.log ({exc}).") + print(f"Warning: Could not initialize the EXOTIC run log ({exc}).") else: - file_handler._exotic_runtime_handler_name = _RUNTIME_FILE_HANDLER_NAME - file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter( - logging.Formatter( - "%(asctime)s.%(msecs)03d [%(threadName)-12.12s] %(levelname)-5.5s " - "%(funcName)s:%(lineno)d - %(message)s", - "%Y-%m-%dT%H:%M:%S", - ) - ) - log.addHandler(file_handler) + _RUNTIME_LOG_PATH = log_path + _RUNTIME_LOG_BASENAME = log_path.name + print(f"EXOTIC run log: {log_path}", flush=True) console_handler = _find_runtime_handler(_RUNTIME_CONSOLE_HANDLER_NAME) if console_handler is None: @@ -8992,16 +9131,9 @@ def configure_rprs_search_bound_max(config_value): def parse_bool_config_value(config_value, default, option_name): if config_value is None: return default - if isinstance(config_value, bool): - return config_value - if isinstance(config_value, (int, float)): - return bool(config_value) - if isinstance(config_value, str): - normalized = config_value.strip().lower() - if normalized in ('y', 'yes', 'true', '1', 'on'): - return True - if normalized in ('n', 'no', 'false', '0', 'off', ''): - return False + parsed = coerce_boolean_config_value(config_value) + if parsed is not None: + return parsed default_text = "enabled" if default else "disabled" log_info( @@ -9456,6 +9588,28 @@ def should_ignore_header_wcs(config_value): return False +def should_allow_pixel_alignment_fallback(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'allow_pixel_alignment_fallback' value; " + "pixel-based image alignment will remain disabled.", + warn=True, + ) + return False + + def should_prefer_pixel_values_over_wcs_for_target(config_value): if config_value is None: return False @@ -13745,16 +13899,19 @@ def nonlinear_ld(ld, info_dict, non_interactive_run=False): "Non-interactive runs require a recognized standard filter or both wl_min and wl_max." ) - opt = info_dict.get('ld_uncertainties') - - if isinstance(opt, str): - opt = opt.lower().strip() + raw_opt = info_dict.get('ld_uncertainties') + opt = ( + None + if raw_opt is None or (isinstance(raw_opt, str) and not raw_opt.strip()) + else coerce_boolean_config_value(raw_opt) + ) - if opt not in ('y', 'n'): + if opt is None: opt = user_input("\nWould you like EXOTIC to calculate your limb darkening parameters " "with uncertainties? (y/n):", type_=str, values=['y', 'n']) + opt = coerce_boolean_config_value(opt) - if opt == 'y': + if opt: opt = user_input("Please enter 1 to use a standard filter or 2 for a customized filter:", type_=int, values=[1, 2]) if opt == 1: @@ -14212,6 +14369,7 @@ def sigma_clip_pointing_positions(positions, sigma=3.0, max_iters=5): def filter_pointing_outlier_frames(inputfiles, pointing_rejection_sigma=None, ignore_header_wcs=False, + allow_pixel_alignment_fallback=False, frame_loader=None, return_alignment_transforms=False, multiprocess_transformations=None, generalDark=None, generalBias=None, generalFlat=None, @@ -14244,6 +14402,8 @@ def format_result(result_inputfiles, result_keep_mask, dropped_files): usable_mask = None mode_label = None + pixel_alignment_enabled = bool(ignore_header_wcs or allow_pixel_alignment_fallback) + if not ignore_header_wcs: wcs_positions, wcs_usable_mask = collect_wcs_frame_center_pointings(inputfiles) usable_wcs_count = int(np.count_nonzero(wcs_usable_mask)) @@ -14251,13 +14411,21 @@ def format_result(result_inputfiles, result_keep_mask, dropped_files): positions = wcs_positions usable_mask = wcs_usable_mask mode_label = "WCS" - elif usable_wcs_count > 0: + elif usable_wcs_count > 0 and pixel_alignment_enabled: log_info( f"Pointing precheck: usable WCS-derived pointing centers found for " f"{usable_wcs_count}/{len(inputfiles)} frame(s); falling back to alignment-derived positions." ) - else: + elif pixel_alignment_enabled: log_info("Pointing precheck: no usable WCS-derived pointing centers found; using alignment-derived positions.") + else: + positions = wcs_positions + usable_mask = wcs_usable_mask + mode_label = "WCS" + log_info( + f"Pointing precheck: usable WCS-derived pointing centers found for " + f"{usable_wcs_count}/{len(inputfiles)} frame(s). Pixel alignment fallback is disabled." + ) if positions is None: positions, usable_mask, alignment_transforms = collect_transform_frame_pointings( @@ -14306,7 +14474,8 @@ def format_result(result_inputfiles, result_keep_mask, dropped_files): return format_result(retained_files, keep_mask, dropped_files) -def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_missing_fraction=None): +def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_missing_fraction=None, + allow_pixel_alignment_fallback=False): inputfiles = np.array(inputfiles) keep_mask = np.ones(len(inputfiles), dtype=bool) if ignore_header_wcs or len(inputfiles) == 0: @@ -14321,6 +14490,16 @@ def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_mi if missing_count == 0: return inputfiles, keep_mask, [] + if not allow_pixel_alignment_fallback: + retained_files = inputfiles[keep_mask] + log_info( + f"WCS-authoritative precheck: {len(retained_files)}/{total_files} files have celestial WCS. " + f"Dropping all {missing_count} file(s) without celestial WCS because pixel alignment fallback " + "is disabled." + ) + log_missing_celestial_wcs_preview(missing_wcs_files) + return retained_files, keep_mask, missing_wcs_files + missing_fraction = missing_count / total_files if missing_count < total_files and missing_fraction < max_missing_fraction: retained_files = inputfiles[keep_mask] @@ -14336,7 +14515,8 @@ def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_mi return inputfiles, np.ones(total_files, dtype=bool), [] -def should_use_multiprocess_transform_precompute(inputfiles, requested_processes, ignore_header_wcs=False): +def should_use_multiprocess_transform_precompute(inputfiles, requested_processes, ignore_header_wcs=False, + allow_pixel_alignment_fallback=False): if requested_processes is None or requested_processes <= 0: return False @@ -14344,6 +14524,10 @@ def should_use_multiprocess_transform_precompute(inputfiles, requested_processes log_info("Header WCS ignore override enabled. Keeping multiprocessing transformation precompute.") return True + if not allow_pixel_alignment_fallback: + log_info("Pixel alignment fallback is disabled. Skipping multiprocessing transformation precompute.") + return False + all_have_celestial_wcs, missing_wcs_files = evaluate_celestial_wcs_coverage(inputfiles) if all_have_celestial_wcs: log_info("All input FITS files have celestial WCS in their headers. " @@ -17479,6 +17663,50 @@ def demosaic_img(image_data, demosaic_fmt, demosaic_out, demosaic_mult, i): image_data = (new_image_data @ demosaic_mult).astype(img_dtype) return image_data +class AAVSOVSPUnavailableError(RuntimeError): + """Raised after the AAVSO VSP endpoint exhausts its response retries.""" + + +def fetch_aavso_vsp_chart(url): + """Fetch and validate a VSP chart, retrying transient/unusable responses.""" + total_attempts = AAVSO_VSP_MAX_RETRIES + 1 + for attempt_number in range(1, total_attempts + 1): + try: + response = requests.get(url, timeout=AAVSO_VSP_REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + data = response.json() + if not isinstance(data, dict): + raise ValueError( + f"AAVSO VSP returned {type(data).__name__} instead of a JSON object." + ) + missing_fields = [field for field in ('chartid', 'photometry') if field not in data] + if missing_fields: + raise ValueError( + "AAVSO VSP JSON response is missing required field(s): " + + ", ".join(missing_fields) + ) + if data['photometry'] is not None and not isinstance(data['photometry'], list): + raise ValueError("AAVSO VSP JSON field 'photometry' is not a list.") + return data + except (requests.RequestException, ValueError) as exc: + if attempt_number >= total_attempts: + raise AAVSOVSPUnavailableError( + f"AAVSO VSP returned no usable response after {total_attempts} attempts " + f"({AAVSO_VSP_MAX_RETRIES} retries): {describe_retry_exception(exc)}" + ) from exc + + retries_remaining = total_attempts - attempt_number + log_info( + "\nWarning: AAVSO VSP request failed " + f"on attempt {attempt_number}/{total_attempts} " + f"({describe_retry_exception(exc)}). Retrying in " + f"{AAVSO_VSP_RETRY_DELAY_SECONDS} seconds; " + f"{retries_remaining} retr{'y' if retries_remaining == 1 else 'ies'} remain.", + warn=True, + ) + sleep(AAVSO_VSP_RETRY_DELAY_SECONDS) + + def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=None, user_targ_star=None, max_new_comp_stars=2): if user_comp_stars is None: @@ -17503,8 +17731,7 @@ def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=No maglimit = 12 url = f"https://www.aavso.org/apps/vsp/api/chart/?format=json&ra={ra:5f}&dec={dec:5f}&fov={fov}&maglimit={maglimit}" - result = requests.get(url) - data = result.json() + data = fetch_aavso_vsp_chart(url) chart_id = data['chartid'] obs_filter = aavso_vsp_band_for_filter(obs_filter) @@ -17639,7 +17866,8 @@ def catalog_calibration_is_usable_for_filter(star, observed_filter, max_error=No def merge_aavso_vsp_v_calibration_fallback( file, axis, obs_filter, img_scale, calibration_stars, user_comp_stars, user_targ_star=None, - max_new_comp_stars=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS): + max_new_comp_stars=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + vsp_query_available=True): """Query VSP when a V-family observation has no usable direct V calibration. Existing AAVSO VSP calibrations mean the field has already been queried. The @@ -17677,6 +17905,14 @@ def merge_aavso_vsp_v_calibration_fallback( if usable_vsp_v: return unified_calibrations, {}, None, False + if not vsp_query_available: + log_info( + "Skipping the AAVSO VSP V-band calibration fallback because the earlier " + "VSP request already exhausted all retries.", + warn=True, + ) + return unified_calibrations, {}, None, False + log_info( "No usable direct V-band comparison calibration was returned by the NextAstro " "photometry server; querying AAVSO VSP for this V-family observation." @@ -18848,10 +19084,6 @@ def _parallel_alignment_task(task): pix_x = np.asarray(pix_x, dtype=float).reshape(-1) pix_y = np.asarray(pix_y, dtype=float).reshape(-1) projected_coords = np.column_stack((pix_x, pix_y)) - if i == 0: - projected_coords[0] = target_and_comp_pixels[0] - if first_frame_uses_input_comp_pixels: - projected_coords = np.array(target_and_comp_pixels, dtype=float, copy=True) wcs_candidate = _fit_alignment_candidate_psfs( image_data, @@ -19149,6 +19381,33 @@ def wcs_alignment_candidate_is_acceptable(result, frame_index, psf_data, tar_com ) +def log_wcs_authoritative_candidate_diagnostics(result, frame_index, psf_data, tar_comp_dist, comp_keys): + file_name = _display_filename(result.get('file_name')) if isinstance(result, dict) else '' + if not isinstance(result, dict) or result.get('wcs') is None: + detail = result.get('wcs_error') if isinstance(result, dict) else None + suffix = f" ({detail})" if detail else "" + log.debug( + f"WCS-authoritative frame has no usable WCS candidate for {file_name}{suffix}; " + "pixel alignment fallback is disabled." + ) + return + + _, _, diagnostics = select_alignment_candidate( + result, + frame_index, + psf_data, + tar_comp_dist, + comp_keys, + ) + decision = diagnostics.get('wcs_decision', {}) + log.debug( + "WCS-authoritative mode retained the frame-WCS-derived candidate without pixel alignment " + f"for {file_name}: reason={decision.get('reason', 'unknown')}, " + f"geometry={decision.get('geometry_match_count', 0)}/" + f"{decision.get('geometry_test_count', 0)}, wcs_score={diagnostics.get('wcs_score')}." + ) + + def classify_wcs_fallback_frames(results, target_and_comp_pixels): """Return missing and rejected WCS frame indices without running legacy alignment.""" target_and_comp_pixels = np.asarray(target_and_comp_pixels, dtype=float).reshape(-1, 2) @@ -19243,7 +19502,7 @@ def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_c generalDark=None, generalBias=None, generalFlat=None, demosaic_fmt=None, demosaic_out=None, demosaic_mult=None, bad_pixel_reference=None, use_fast_centroid_cadence=False, - use_adaptive_apertures=False, compute_fallback_transform=True, + use_adaptive_apertures=False, compute_fallback_transform=False, first_frame_uses_input_comp_pixels=False, precomputed_fallback_transforms=None): total_jobs = len(inputfiles) @@ -19281,7 +19540,7 @@ def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_c demosaic_out=demosaic_out, demosaic_mult=demosaic_mult, bad_pixel_reference=bad_pixel_reference, - progress_label='WCS-first alignment', + progress_label='WCS coordinate projection', ) results = [wcs_results_by_index.get(i) for i in range(total_jobs)] if not compute_fallback_transform: @@ -19419,16 +19678,20 @@ def build_multiprocess_transformations(inputfiles, max_processes): return transforms -def log_alignment_progress(i, total_jobs, file_name, use_multiprocess_progress): +def log_alignment_progress(i, total_jobs, file_name, use_multiprocess_progress, + pixel_alignment_enabled=False): if use_multiprocess_progress: completed = i + 1 if completed == total_jobs or completed % 10 == 0: - log_info(f"Multiprocessing alignment progress: {completed}/{total_jobs}") + mode = "pixel alignment" if pixel_alignment_enabled else "WCS coordinate projection" + log_info(f"Multiprocessing {mode} progress: {completed}/{total_jobs}") return display_file_name = _display_filename(file_name) - sys.stdout.write(f"Aligning frame {i + 1} of {total_jobs} : {display_file_name}\n") - log.debug(f"Aligning frame {i + 1} of {total_jobs} : {display_file_name}\n") + action = "Pixel-aligning" if pixel_alignment_enabled else "WCS-locating stars in" + message = f"{action} frame {i + 1} of {total_jobs} : {display_file_name}\n" + sys.stdout.write(message) + log.debug(message) sys.stdout.flush() @@ -22445,6 +22708,19 @@ def save_comp_ra_dec(wcs_file, ra_file, dec_file, comp_coords): def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astrometry=False, multiprocess_transformations=None): timeList, airMassList, exptimes, norm_flux = [], [], [], [] ignore_header_wcs = should_ignore_header_wcs(info_dict.get('ignore_header_wcs')) + allow_pixel_alignment_fallback = should_allow_pixel_alignment_fallback( + info_dict.get('allow_pixel_alignment_fallback', False) + ) + pixel_alignment_enabled = bool(ignore_header_wcs or allow_pixel_alignment_fallback) + if ignore_header_wcs: + log_info("Pixel alignment enabled explicitly: header WCS will be ignored for manual alignment.") + elif allow_pixel_alignment_fallback: + log_info("WCS-first coordinate projection enabled with explicit pixel alignment fallback.") + else: + log_info( + "WCS-authoritative coordinate mode enabled: each frame's header WCS will supply star pixel " + "positions; Astroalign/pixel alignment fallback is disabled." + ) bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction(info_dict.get('bad_wcs_threshold_percent')) pointing_rejection_sigma = get_pointing_rejection_sigma(info_dict.get('pointing_rejection_sigma')) detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( @@ -22478,10 +22754,17 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet inputfiles, ignore_header_wcs=ignore_header_wcs, max_missing_fraction=bad_wcs_threshold_fraction, + allow_pixel_alignment_fallback=allow_pixel_alignment_fallback, ) if dropped_wcs_files: times = times[wcs_keep_mask] plateStatus.initializeFilenames(list(inputfiles)) + if len(inputfiles) == 0: + log_info( + "Error: no input frame has celestial WCS and pixel alignment fallback is disabled.", + error=True, + ) + return target_wcs_precheck_inputfiles = np.array(inputfiles, copy=True) target_wcs_reference_file = inputfiles[0] if len(inputfiles) else None inputfiles, target_wcs_keep_mask, dropped_target_wcs_files = filter_wcs_target_out_of_frame_frames( @@ -22514,6 +22797,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet inputfiles, pointing_rejection_sigma=pointing_rejection_sigma, ignore_header_wcs=ignore_header_wcs, + allow_pixel_alignment_fallback=allow_pixel_alignment_fallback, return_alignment_transforms=True, multiprocess_transformations=multiprocess_transformations, ) @@ -22687,7 +22971,7 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet bad_pixel_reference=bad_pixel_reference, use_fast_centroid_cadence=True, use_adaptive_apertures=use_adaptive_apertures, - compute_fallback_transform=True, + compute_fallback_transform=pixel_alignment_enabled, first_frame_uses_input_comp_pixels=True, precomputed_fallback_transforms=pointing_alignment_transforms, ) @@ -22749,23 +23033,19 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet if has_wcs_alignment and target_and_comp_radec is not None: try: - if i == 0: - projected_coords = np.array(target_and_comp_pixels, dtype=float, copy=True) - else: - pix_x, pix_y = wcs_hdr.world_to_pixel_values( - target_and_comp_radec[:, 0], - target_and_comp_radec[:, 1], - ) - pix_x = np.asarray(pix_x, dtype=float).reshape(-1) - pix_y = np.asarray(pix_y, dtype=float).reshape(-1) - projected_coords = np.column_stack((pix_x, pix_y)) + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + projected_coords = np.column_stack((pix_x, pix_y)) wcs_candidate = _fit_alignment_candidate_psfs( imageData, projected_coords, target_fast_centroid, frame_fast_centroid, - previous_psf_rows=previous_psf_rows, ) wcs_candidate['projected_off_frame'] = any_projected_coord_out_of_frame( projected_coords, @@ -22780,15 +23060,25 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet len(inputfiles), fileName, use_multiprocess_transform_precompute, + pixel_alignment_enabled=pixel_alignment_enabled, ) - if not wcs_alignment_candidate_is_acceptable( + wcs_candidate_acceptable = wcs_alignment_candidate_is_acceptable( alignment_result, i, psf_data, tar_comp_dist, ['comp'], - ): + ) + if not pixel_alignment_enabled and not wcs_candidate_acceptable: + log_wcs_authoritative_candidate_diagnostics( + alignment_result, + i, + psf_data, + tar_comp_dist, + ['comp'], + ) + if pixel_alignment_enabled and not wcs_candidate_acceptable: cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None if cached_tform is not None: tform = cached_tform @@ -31568,6 +31858,7 @@ def _main_impl(): init_path, userpDict = inputs_obj.search_init(args.realtime, userpDict) exotic_infoDict, userpDict['pName'] = inputs_obj.real_time(userpDict['pName']) + configure_runtime_logging(output_dir=exotic_infoDict.get('save')) while True: carry_on = user_input(f"\nType continue after the first image has been taken and saved: ", type_=str) @@ -31650,6 +31941,7 @@ def _main_impl(): header_motion_value = exotic_infoDict.get(motion_key) if header_motion_value is not None: userpDict[motion_key] = header_motion_value + configure_runtime_logging(output_dir=exotic_infoDict.get('save')) disable_vertical_flux_normalization = is_vertical_flux_normalization_disabled( exotic_infoDict.get('disable_vertical_flux_normalization', False) ) @@ -32177,6 +32469,19 @@ def lookup_archive_ephemeris(): if finite_plot_times.size: full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) + allow_pixel_alignment_fallback = should_allow_pixel_alignment_fallback( + exotic_infoDict.get('allow_pixel_alignment_fallback', False) + ) + pixel_alignment_enabled = bool(ignore_header_wcs or allow_pixel_alignment_fallback) + if ignore_header_wcs: + log_info("Pixel alignment enabled explicitly: header WCS will be ignored for manual alignment.") + elif allow_pixel_alignment_fallback: + log_info("WCS-first coordinate projection enabled with explicit pixel alignment fallback.") + else: + log_info( + "WCS-authoritative coordinate mode enabled: each frame's header WCS will supply star pixel " + "positions; Astroalign/pixel alignment fallback is disabled." + ) bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction( exotic_infoDict.get('bad_wcs_threshold_percent') ) @@ -32193,12 +32498,19 @@ def lookup_archive_ephemeris(): inputfiles, ignore_header_wcs=ignore_header_wcs, max_missing_fraction=bad_wcs_threshold_fraction, + allow_pixel_alignment_fallback=allow_pixel_alignment_fallback, ) if dropped_wcs_files: times = times[wcs_keep_mask] jd_times = jd_times[wcs_keep_mask] header_exptimes = header_exptimes[wcs_keep_mask] plateStatus.initializeFilenames(list(inputfiles)) + if len(inputfiles) == 0: + log_info( + "Error: no input frame has celestial WCS and pixel alignment fallback is disabled.", + error=True, + ) + return post_wcs_inputfile_count = int(len(inputfiles)) target_wcs_precheck_inputfiles = np.array(inputfiles, copy=True) target_wcs_reference_file = inputfiles[0] if len(inputfiles) else None @@ -32239,6 +32551,7 @@ def lookup_archive_ephemeris(): inputfiles, pointing_rejection_sigma=pointing_rejection_sigma, ignore_header_wcs=ignore_header_wcs, + allow_pixel_alignment_fallback=allow_pixel_alignment_fallback, frame_loader=lambda file_name: load_calibrated_reduction_image( file_name, generalDark, @@ -32319,7 +32632,7 @@ def lookup_archive_ephemeris(): # fit target in the first image and use it to determine aperture and annulus range inc = 0 - if reference_fallback is None: + if pixel_alignment_enabled and reference_fallback is None: for ifile in inputfiles: plateStatus.setCurrentFilename(ifile) if bad_pixel_reference is not None: @@ -32345,12 +32658,17 @@ def lookup_archive_ephemeris(): inc += 1 finally: del first_image - else: + elif reference_fallback is not None: log_info( "Skipping the old-pixel target precheck because the original reference image was " "removed; the target will be projected from RA/Dec after the new reference WCS is ready.", warn=True, ) + else: + log.debug( + "Skipping the old-pixel target precheck in WCS-authoritative mode; target coordinates " + "will be projected independently from each frame's header WCS." + ) if inc > 0: log_info(f"Skipping first {inc} files - Target star not found") @@ -32381,6 +32699,7 @@ def lookup_archive_ephemeris(): plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) ra_dec_tar, ra_dec_wcs = None, [] chart_id, vsp_comp_stars, vsp_list = None, {}, [] + aavso_vsp_query_failed = False nextastro_field_catalog = None primary_target_catalog_match = None science_comp_stars = [] @@ -32476,14 +32795,25 @@ def lookup_archive_ephemeris(): ) if exotic_infoDict['aavso_comp'] == 'y' and reference_fallback is None: - vsp_comp_stars, chart_id = vsp_query(wcs_file,[header['NAXIS1'], header['NAXIS2']], - exotic_infoDict['filter'], img_scale, - user_comp_stars=exotic_infoDict['comp_stars'], - user_targ_star = [ exotic_UIprevTPX, exotic_UIprevTPY ], - max_new_comp_stars=( - 0 if use_exactly_the_comps_provided else 2 - )) - vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] + try: + vsp_comp_stars, chart_id = vsp_query( + wcs_file, + [header['NAXIS1'], header['NAXIS2']], + exotic_infoDict['filter'], + img_scale, + user_comp_stars=exotic_infoDict['comp_stars'], + user_targ_star=[exotic_UIprevTPX, exotic_UIprevTPY], + max_new_comp_stars=(0 if use_exactly_the_comps_provided else 2), + ) + vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] + except AAVSOVSPUnavailableError as exc: + aavso_vsp_query_failed = True + log_info( + "\nWarning: AAVSO VSP comparison-star lookup remains unavailable " + f"after five retries ({describe_retry_exception(exc)}). Continuing " + "without VSP data so the existing NextAstro/catalog fallback can run.", + warn=True, + ) try: nextastro_field_catalog = nextastro_photometry_catalog_for_wcs( @@ -32908,6 +33238,7 @@ def lookup_archive_ephemeris(): if use_exactly_the_comps_provided else maximum_number_of_ensemble_comparisons_for_stellar_variability ), + vsp_query_available=not aavso_vsp_query_failed, ) ) if fallback_chart_id is not None: @@ -33380,7 +33711,7 @@ def lookup_archive_ephemeris(): bad_pixel_reference=bad_pixel_reference, use_fast_centroid_cadence=False, use_adaptive_apertures=use_adaptive_apertures, - compute_fallback_transform=True, + compute_fallback_transform=pixel_alignment_enabled, precomputed_fallback_transforms=pointing_alignment_transforms, ) for alignment_index, alignment_result in enumerate(multiprocess_alignment_results): @@ -33661,15 +33992,11 @@ def lookup_archive_ephemeris(): pix_x = np.asarray(pix_x, dtype=float).reshape(-1) pix_y = np.asarray(pix_y, dtype=float).reshape(-1) projected_coords = np.column_stack((pix_x, pix_y)) - if i == 0: - projected_coords[0] = target_and_comp_pixels[0] - wcs_candidate = _fit_alignment_candidate_psfs( imageData, projected_coords, target_fast_centroid, frame_fast_centroid, - previous_psf_rows=previous_psf_rows, ) wcs_candidate['projected_off_frame'] = any_projected_coord_out_of_frame( projected_coords, @@ -33684,15 +34011,25 @@ def lookup_archive_ephemeris(): len(inputfiles), fileName, use_multiprocess_transform_precompute, + pixel_alignment_enabled=pixel_alignment_enabled, ) - if not wcs_alignment_candidate_is_acceptable( + wcs_candidate_acceptable = wcs_alignment_candidate_is_acceptable( alignment_result, i, psf_data, tar_comp_dist, comp_alignment_keys, - ): + ) + if not pixel_alignment_enabled and not wcs_candidate_acceptable: + log_wcs_authoritative_candidate_diagnostics( + alignment_result, + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + if pixel_alignment_enabled and not wcs_candidate_acceptable: cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None if cached_tform is not None: tform = cached_tform @@ -36518,7 +36855,7 @@ def main(): global _UNHANDLED_EXCEPTION_LOGGED _UNHANDLED_EXCEPTION_LOGGED = False - configure_runtime_logging() + configure_runtime_logging(start_new_run=True) install_exception_hooks() try: @@ -36530,13 +36867,13 @@ def main(): raise finally: cancel_runtime_traceback_watchdog() + close_runtime_logging() def cli(): global _UNHANDLED_EXCEPTION_LOGGED _UNHANDLED_EXCEPTION_LOGGED = False - configure_runtime_logging() install_exception_hooks() try: diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index d04634ad..f69ae2f0 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -416,7 +416,8 @@ def save_input(): "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", - "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", + "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to true only to permit legacy Astroalign image registration when a frame's WCS-derived star locations are unusable. Default false; WCS headers are authoritative.", + "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them for legacy alignment. With the default WCS-authoritative mode, every image without celestial WCS is dropped. Default 3.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", @@ -443,6 +444,7 @@ def save_input(): "Photometry Noise Budget": "Optional noise terms for raw-image photometry: gain_electrons_per_adu, read_noise_electrons, dark_current_electrons_per_second_per_pixel, flat_field_fractional_error, telescope_aperture_m, and scintillation_coefficient. Leave null to ignore an optional term.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", + "Boolean Values": "All boolean settings accept JSON true/false, numeric 1/0, or case-insensitive strings y/n. The equivalent strings yes/no and on/off are also accepted.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." } @@ -460,6 +462,7 @@ def save_input(): } new_inits['optional_info'] = { "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "allow_pixel_alignment_fallback": False, "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, @@ -1530,7 +1533,8 @@ def save_input(): "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", - "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", + "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to true only to permit legacy Astroalign image registration when a frame's WCS-derived star locations are unusable. Default false; WCS headers are authoritative.", + "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them for legacy alignment. With the default WCS-authoritative mode, every image without celestial WCS is dropped. Default 3.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", @@ -1556,6 +1560,7 @@ def save_input(): "Overexposure Threshold Fraction": "Set optional_info 'overexposure_threshold_fraction' to the fraction of saturation used for rejection. Default 0.9.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", + "Boolean Values": "All boolean settings accept JSON true/false, numeric 1/0, or case-insensitive strings y/n. The equivalent strings yes/no and on/off are also accepted.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." } @@ -1621,6 +1626,7 @@ def save_input(): "Filter Maximum Wavelength (nm)": input_data.get('filtermax', null), "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "allow_pixel_alignment_fallback": False, "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, @@ -1698,6 +1704,7 @@ def save_input(): "Exposure Time (s)": input_data['exp'], "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "allow_pixel_alignment_fallback": False, "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, diff --git a/exotic/inputs.py b/exotic/inputs.py index a2f48759..778dfada 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -11,10 +11,10 @@ import re try: - from utils import user_input, init_params, typecast_check, \ + from utils import coerce_boolean_config_value, user_input, init_params, typecast_check, \ process_lat_long, find, open_elevation except ImportError: - from .utils import user_input, init_params, typecast_check, \ + from .utils import coerce_boolean_config_value, user_input, init_params, typecast_check, \ process_lat_long, find, open_elevation try: from animate import animate_toggle @@ -265,6 +265,7 @@ def __init__(self, init_opt): 'dist': None, 'pm_ra': None, 'pm_dec': None, 'airmass_already_corrected': False, 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, 'fast_aperture_mask': False, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', + 'allow_pixel_alignment_fallback': False, 'prefer_pixel_values_over_wcs_for_target': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'stellar_variability_only': False, @@ -548,6 +549,11 @@ def comp_params(self, init_file, planet_dict): 'Ignore WCS in header and do manual alignment', 'ignore_header_wcs', ), + 'allow_pixel_alignment_fallback': ( + 'allow_pixel_alignment_fallback', + 'Allow Pixel Alignment Fallback', + 'Allow Pixel Alignment Fallback? (y/n)', + ), 'prefer_pixel_values_over_wcs_for_target': ( 'prefer_pixel_values_over_wcs_for_target', 'Prefer Pixel Coordinates to WCS Coordinates if there is a conflict', @@ -1209,23 +1215,23 @@ def obs_notes(notes): def plate_solution_opt(opt): - if opt: - opt = opt.lower().strip() - if opt not in ('y', 'n'): + parsed = None if is_blank_value(opt) else coerce_boolean_config_value(opt) + if parsed is None: opt = user_input("\nWould you like to upload the your image for a plate solution?" "\nThis will allow EXOTIC to translate your image's pixels into coordinates on the sky." "\nDISCLAIMER: One of your imaging files will be publicly viewable on " "nova.astrometry.net. (y/n): ", type_=str, values=['y', 'n']) - return opt + parsed = coerce_boolean_config_value(opt) + return 'y' if parsed else 'n' def aavso_comp(opt): - if opt: - opt = opt.lower().strip() - if opt not in ('y', 'n'): + parsed = None if is_blank_value(opt) else coerce_boolean_config_value(opt) + if parsed is None: opt = user_input("\nWould you like Comparison Stars added automatically from AAVSO? (y/n): ", type_=str, values=['y', 'n']) - return opt + parsed = coerce_boolean_config_value(opt) + return 'y' if parsed else 'n' def target_star_coords(coords, planet): diff --git a/exotic/utils.py b/exotic/utils.py index ba4b7aa4..ed75af6d 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -29,6 +29,34 @@ MAGNITUDE_DECIMAL_PLACES = 4 MINIMUM_MAGNITUDE_ERROR = 0.001 AAVSO_OUTPUT_FOLDER_NAME = 'AAVSO_Files' +BOOLEAN_CONFIG_TRUE_STRINGS = frozenset(('y', 'yes', 'true', '1', 'on')) +BOOLEAN_CONFIG_FALSE_STRINGS = frozenset(('n', 'no', 'false', '0', 'off', '')) + + +def coerce_boolean_config_value(value): + """Return a configured boolean, or ``None`` when the value is not boolean-like. + + JSON booleans and numeric 1/0 are accepted directly. String values are + case-insensitive and accept y/n, yes/no, true/false, 1/0, and on/off. + """ + + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + if not isfinite(value): + return None + if value == 1: + return True + if value == 0: + return False + return None + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in BOOLEAN_CONFIG_TRUE_STRINGS: + return True + if normalized in BOOLEAN_CONFIG_FALSE_STRINGS: + return False + return None def aavso_output_directory(root): diff --git a/inits.json b/inits.json index 8b7ea175..cd95406f 100644 --- a/inits.json +++ b/inits.json @@ -61,6 +61,7 @@ "Fit Lightcurve to Every Comparison Candidate": "Set optional_info 'fit_lightcurve_to_every_comparison_candidate' to y to save one target lightcurve fit plot per comparison star into working_artifacts/ using the selected photometry setup. Default n.", "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", + "Boolean Values": "All boolean settings accept JSON true/false, numeric 1/0, or case-insensitive strings y/n. The equivalent strings yes/no and on/off are also accepted.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." }, @@ -81,8 +82,8 @@ "Pixel Binning": "1x1", "Filter Name (aavso.org/filters)": "CV", "Observing Notes": "Weather, seeing was nice.", - "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "n", + "Plate Solution? (y/n)": true, + "Add Comparison Stars from AAVSO? (y/n)": false, "Target Star X & Y Pixel": "[424, 286]", "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", "Comparison Star(s) RA & Dec": null, @@ -127,10 +128,10 @@ "Filter Maximum Wavelength (nm)": null, "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Fast Aperture Mask (y/n)": false, - "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "Ignore WCS in Header and Do Manual Alignment? (y/n)": false, "bad_wcs_threshold_percent": 3.0, "pointing_rejection_sigma": null, - "prefer_pixel_values_over_wcs_for_target": "n", + "prefer_pixel_values_over_wcs_for_target": false, "disable vertical flux normalization": false, "stellar_variability_only": false, "use_ensemble_photometry_for_stellar_variability": true, @@ -141,20 +142,20 @@ "photometer_fortuitous_variables": true, "use_single_comparison_for_fortuitous_variables": true, "use_nextastro_vsx_cache_first": false, - "detect_bad_pixels_before_photometry": "n", - "multiprocess_bad_pixel_precheck": "y", + "detect_bad_pixels_before_photometry": false, + "multiprocess_bad_pixel_precheck": true, "detrend_on_outoftransit_baseline": true, "final_fit_baseline_duration_multiplier": 1.0, - "use_eebls_to_initialize_tmid_and_bounds": "y", - "pick_comparison_by_eebls_snr": "y", + "use_eebls_to_initialize_tmid_and_bounds": true, + "pick_comparison_by_eebls_snr": true, "use_deviation_from_expected_transit_in_qc": true, "deviation_from_expected_transit_in_qc_sigma": 5.0, - "use_impactparameter_rather_than_inclination_to_fit": "y", + "use_impactparameter_rather_than_inclination_to_fit": true, "minimum number of live points for ultranest": 200, - "run fast ultranest before final run": "y", - "use_sparse_posterior_live_point_retry": "y", - "use_psf_photometry": "y", - "use_aperture_photometry": "y", + "run fast ultranest before final run": true, + "use_sparse_posterior_live_point_retry": true, + "use_psf_photometry": true, + "use_aperture_photometry": true, "use_adaptive_apertures": false, "use_aperture_corrections_and_full_image_fwhm": false, "reject_overexposed_stars": true, @@ -166,10 +167,10 @@ "flat_field_fractional_error": null, "telescope_aperture_m": null, "scintillation_coefficient": null, - "skip_low_comparison_coverage_rejection": "n", - "fit_lightcurve_to_every_comparison_candidate": "n", - "Use target-driven comp selection rather than comp-driven comp selection": "n", - "require_comp_star": "y", + "skip_low_comparison_coverage_rejection": false, + "fit_lightcurve_to_every_comparison_candidate": false, + "Use target-driven comp selection rather than comp-driven comp selection": false, + "require_comp_star": true, "Image Scale (Ex: 5.21 arcsecs/pixel)": null, "Pixel Scale (arsec/pixel)": null, "Exposure Time (s)": 60.0 diff --git a/output/pdf/EXOTIC_inits_default_options.pdf b/output/pdf/EXOTIC_inits_default_options.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4e025a7deff106aae6c1a042426c97667c437964 GIT binary patch literal 14849 zcmdVB$=gQx_x$`IN$Ne{$v1O6^bhi<_LJE0AENjNp^!iI{Bik1DD@xeuXofB{p;=zN&3Tk zy1%;q>htff^saADN%BF7p?Cim?-M!KA7b!y5;uAVKZF@QyU%y@9}TnX!^?*!e~8uB z{`^~-KN$J{{Q1`<|0N&mx2b-8#Q)D!)n0OU|1*s3~dD(wvBI-Xm z5%b$b{ha2UFYh1n&z1fmG|!E$z2ski4tcNI57S`qI>)d-JWb*}-{60aKZJhGe({q0 zE1QN1kh;`A-w?-~LCEx$hq&f5<=d|1?yt zLEW#D>wn}wp!Jt3okn(c&HnAD`eXeDslTT9r73fKi#;bt+kx-r*XUoopTTx+jQT&( zGx|i=d-Om5B;@B`uJ>=gIR7K`p7~F1^e>V7z21MutNHBb=Kb6IC)e~pSNol-`}}*2 z=yT5r|81x@9>0_4kL>x^@BQ;LPk$}X`&ZZOnx;AU6Uz1e{#{@H$%p@ty>AW{Y`!c} zz9fgAzo_lM=)1otiofizzewhvV!QkWK!0K4pN##Tg})=H65T)fvgOFrUG+yJo{_!$ zVDJm&?>YDW`j8Jb^`5~e{2@7#{x#J5lk@oThiVsqLmOC))}^I?-dlqSwq0ME@8GqL z!0L`#`?Df>+Zj9eh%3Ex=bfeV7wKl3p7()6T_yvhC@SbfJG=2z2R7$xPdHsVY4}ke z+X4U(0*n-l^`KdhI(H%da`SKNdexgE0yk=7sS94f*DOx-WrtE1dLLCD27+`}G*C|G zV?`>!+bYzb!B%?2F6VVcbsvoD8^m%qPhX#PRjfugm{!L|A=y%eZohUc6yR(2W!&LL zS+LRd+L?fGrGCS|_wZ?TcJ+0?eq|Tc(P+&+1grp_>VxxGs^59Ink{T63CSo%q}XJ- zV{SWsT;O|uR38Py*M5Hlw==66@K=r6-2?->h|TRr5n`9^>C?k1Bb-?j^*Yd?eG*PU z*zY|ze(8eT*3orH)?lo1YORJbL^I<}t$LiON-{3PM3-=DCs0=zwf+pNNl%j^9iSIF zMfxPZqc5)8?S=L2cgJYnc>cQMOit*)z5a1eO%|Poaz_sj+VEL68qz(#akyhkmP%LZbh)gm)=KEGk6Eu9DCmo3)BiE9iN`*>936K9SD;cK9;p;Hq+*btwA zv%yYRUHENj`|ED)l(ImwbmFZP?j@_zkY^W8Mh+pk{v^W@cEHL>&)wYWr8DSm4l0Tj z)ekssd=_BP;f1?DG5Y=Z(plorL^xEjvnn7}QjvA7n3&lJ@?=stdG<7Iz5<*+%x!|=?I)p}Zps#pwY4Rq41{!w ztn2-U1s)9gvbw)p3Mt$}0j!TD?ifjpFM%7?_;GGX@z!|k>4%IX!noU8RPdg@xj)bY zQ>{6N**HG%vJD**O>I!S5^*1_pxSHO3=TW-R_Jl7?P4H3W&gUz(TTb^%53d6T6W7@ zsUhsSw=+8XQ==|i`0^0I!h{Osug6wnezGG)uHH_?hy#0Kt1w8vO6#@$gi9XA*n`U6 zI*Yoyw4rt-7yL*>zy1Y<$f(5y;zzu;XA+P^v~`gZo)|5R4a@abgF*Rqd87-h_?2SB zlo#7~eaGL)!sK-zCK}n@OzY@eZx60Z@@!d4v#^`{rH4ZfILVpLw@m6z4?(@vq(;+& zmZlD9#O54)X87&O_1d=bu#damViZ-1;Waoxm5*@?sd06Pjs4!`WS`2~=CK*}s)xqC zQUYd$OKs~bvuQ}W+;Tg!i=CE z(9xkQje0Kp(b`ZB)mBg2sH?BJl;tXnqp~w!pfK}p`zXM>>usTSC}TKKUt?NwWW8x) zx6omy!p`(5T^Z*>rgU-Ua@BdEMD}(VeQFr%cM-H)dXdBGnh@4co*_y_P`fTa#myjF z5=ls#&tdhupMk<^ek1V1W?q%DO>X!XnwwD7NN1kFh=ySe0P2y7$ZTc{X+$+$zhd8j z5io*quw8~Ei)%){#F?jzS7=xFM>BQ{u+ zYINRyhzg8|L!ZqZs^a{)-UEZ(1woPN!nBkE3+atryK}VCL^ame(!`ateV4dw+`)5$ zpTUuZFADe-4E7IIaM*nPHY-)9k}zFOJGW7GnsiS6Vs;U9v`TG_Cpo0%*A}H$ubz2( ztb8#3a78~r6gS~cuyyJoz2?09UZ_}N^^Q2QGW#>7C;$poW$5ordziQH4S<1zwK`An zJKVVNZq0RxiM;`Mf`gT04+X#+lV-cOvI{VioOz zlVUDh%6Ew`i?fS-n@BZ}xR|Z}G~NmzdwIT=Z-_oZ{`_Sh2NH4bOppBI*tHlRXlod8 zT6cC5Jo97T+kb}M#t(O0Av_(buU7fVB7iL4mZDOgm9@+)?h?@ej09ci-rVnE+h;tu z+C>OS!NGDqxShvHLy_0#MPHo7Cw3gQWF^Qvab{clFb2m`He=3{ zmbY@9#vPIZ^`wuh{596c)}~>7FPt&jl7Mj>qxCMUGbJ&(`~6$T7|-V5Ii5{>&33Is zD9G}GJn4#GHC=T5nIDCD%R?lXUOLOkr~rQp%8BY-*SEg2T0Z>{>=%JkL#LKDp*%Ju zP(s8DIuN)vJq8pLE?h>YP`*JGJ#1a6^+$zmODjs4{2909=S<|95GzphqVEn?=+275 zxpX^O4OK~P5;IMpS9JCmXQPGEN>6a=2xtR)N0Q*-gzIx#n+FIfV{*CPE;;VqIZM0c zMH$Uojcg`-`;HtZ^n66jXW0uM9A%hocthJP^+n`(*a!*Q?y;%VSntL!ZT~rk#f~MU zi82evvM~9e)h&7H)HdiHV7$%bb*94iSgCKb60|^W)9Ho~kHw?0eu>v7d8oK9JDRsI zi0Fl&gjY2H+`n!>V}L6cFs$O)ye>@9oz{+5WY?uh3AWn?usP03yF;05n#bx(llE}BATXsogtgh<)5>5DzRGjee`Sq6Jj!|fLFJf$!F zdaax-Mp$AVfe!1KRtt^dVSR=7(v(o2ejg>xMgw^UAD_bK z#OEW!1wmSuqz)K=(Vi8R>HTEbL$|v%D^fZDiTA2-L
>L7M7+f;95V$sFFv-Qu1 zcW&mMRiRtEv0TId3_pMOEdH1AldJ!3P3HKiN!^TFm6nNjh$=vIEK@rjuyDL88lQwi z&|)H%>f7aot(HEWRj-(Nn!YAJf;$(ES0r86uYAkz^zBwh^`@`qD@YIU!mZHBWoh8m zNq_rXZgxb!#jw`>P6OKuwjRdYF+9cRoNB{cG23UaDZHBmXPv`7M^FHZY9A+w(*AKf zPbcqI_hgm!6jI==8_J$KcPj5XYJAAfcR*Ey&257%I$L2(?P0cf);Q$iCOw|30$7(D4{E*U zV7;6*{G;5Mn%hxU7)S%L>`m*M13vp}z2#g&eS{pMGh8Xw?f3P%uijy_Vq-^q#cfL? zyAqgbAeRFiO4QN=OxwLGrUX@G6V)5dU0>Op%gU>bub&)M*_HM*x8QOmIFP@$@PsDn z8+tbhZHdP%{$3q~z`5S0#(MhRjaM8c@f}HtX+=IKck>`%A}+T+9blv1TLF4^P0Y|iW!|r*;i|`({@+g zQ_ATxh+K0Fe%`jd%AVb|x+%2kk81;H0|d{_=d`Sd`vvL0w}+#MOoOf$@?*f8Y;eM) z+aOg_Lbw3#?_V8pDhP^ANwFmpd%-rt4WZQ8U(yA|SO zz;80+vw;<*` z9~Z)R3oI6T?HX-)Ne|6oUVNzXimPK0IjOhcVkH=^)F6avIIcJ*xYS!Gd{K=(wqur0 zFtHL*Zm$pV<+5I4g+Xb#tB7IeP&EfJeEeepI%tf41D?qU$WTHn$D+?hxi{Q_ZEf zJ>ArRY*cw3tl`b5AV_`H*nOQ#K$f!xicOGSZP;RN(;?PpZWifD#&OA1x#wBmMJpL! z<>&S5nE;Nk>aZQ$sigjRyv;Jiy??(xUp$n?>ZZ1ux-+0%42PW;N$1Lm-+mLB^a9C2 z(QbQHLMVZyy8(;NNUpyEi=?%Wt}fwjXy@{giXpK{h1@@t8-73lY9g&w9L+Dix8>%R zGsDmCr#NS46Rt^tJ+ zY8kU$tCPyTw4C47-1ONvuZR2sj{*XApQ{;XlW<`V?OH^y_Ykde z|N7K!-)!cQ#wos4t%eKl>tTjG`_0^pUv|h^JYT45Y`Wlv1M9h5=1NE3JvedaWtVpL7|`22f)Kfqbg;BV!)*9>+-ns9Ccgh%b|1y4jnjiBy;p z#;M~sB2??v#=Dg%&7JXgBSYX?O)HFW$E+}&xt48mL^B?YY7PC}A;t^mmX;+YU&GHe>BY^o08KyLZE9u1K^O8MKjv_KR5II)vpdOfpoq-);@# zD^`pKufFZpVWoD$nr)6|>)OYhQ9sib`U^`Qp7g%L&jt%`EDYykDN&yRwBl zWuI<@ad&ZuPk5#@@k@X~w)hsKySvLBtm)B zi4F$wQDU@<11@P%+3DmeV-ZgVm@cZ9+pjCz8x0;s2b#5NX`R{?6N1N3`pcCrS4!%H zyc1LAtaPhaAE#7$zPIvmI&n+Ntii^5{Mo)`Bk7-eLgGd6?W<5%7*9oUm@f9q^5guV zJpXA&i$PiIPs>tA?X^4CQ@FrzWgwDN?L?opN*T(&v+te1CDkW8m76F~9^dFkBcz_P zH@~xah=iY_uL=CpnfZ|mJ7us}muTE@?6vycCL~Xg`289ub8(t7v*KD9HB8ctuvs;d zkckpb9WEBXl)kH8p3cqb&SfzhdHf@|;oAp`4GO%DnX{uK7D0A1U1h6HHMgOHR4y$c zUJ~i(zG&3o7$+9uPOWbC$3uloz#Y&mC*uSVzEShm=PZ3+S$PNf(wQW!?mj)#lMd-k zM}eIiSQD-~x;~XP*+gK$;IXhz8H>R-x~zDSo(y51ymz;g^R0koMQ^Z_{o1g2q5*rE zHWyh8-5w=mz|?}$ZQq;~8)hUeUUI!ss=t;lzI-hRFbn7fG;LObTe)wjPDM6mI|Mm( zl)4+gsYa33Hd+gbpkr`bmdYl+5BvpOghIYB=*q2qTP>!G#|A#Y{W~<0#gW(+A?49X z#-4t5j5|aQZvY{7p=bN)WNsFhD(+dll;J&Y7z_I_XbU--`^}(T9JRt0C=7$z@w*Bw z|D@B>TvMDOHv2x|xP0sM*K6dw+YZ`QY_7YLaXH%PaXfA*pTVd!pV%*g8!wEmGs@t8 zyZhSDp96fAly`uCc#JeF4`2x$a~2QD!t6$ISZ$E1fC@gCnn>dH$v%P-b2AX?ZX%Lo zHeL;$nOHX%)<|BHnGpMW?i;}$g~r0HGoCnyFOOYAY8)8l1u_SJ)IQLYC6WgZA$Lmj z9^gSSU#)s4#A4trRnv%7M-@KX%QItF#N=TxGS1;D?9L;1_T4n{Eea-Ej)ADyr|z?F z?nSoUWpF6lqnD;}(jfi*yWKx@@3X*(rAOd%kK7^V4))vifST6)U^soV6D!SwD(6 zVqET!zo{!1d!upJ(?gCzH>w~FChV*<7;!g)5yV=X?M+eR3f$!nac3gvQt?tWLomLT zS~K`%?bn+s_$GT(z6bO1&92{PtPf}@e=@^SVy~cfcx#CF)3DZPf0#6$50=Vnl zj^j%#t8}Oo(&7oHUPd)`R+a_rtu7A7b?}w;&}7*h(j{OEoP%-6)e)&X+K4#6LFZ5} zjThh7bX-8Eo6FYdX6xLhx)wqgoQzrghV$<0Mz7GjzxA~Y&H~Hb+Q42l?Ls$F@0PHr zJ`Ki(Rz{2{LkfM>(LL)Qmzv*#BzpBajtlh#^@-YacUD)kflbg1+DzB(^;W!5;)}a= zHP#alWSZ_x-FRSJT#2Qi;^8qqc21`Sk-TF4x_+)muci+=qm6-X0##t9xgpywgL$Bk z>4CcbCClgO)1^x%GBn6wSBA~Y@!Y;MT@88j1qxsTl7C9xc^2exKLStE<2rU1S1!(0 zy*!wYiIw369S+)l&16nxIIGoqagg%GI9bi1GP{?DjWfSFkNej9LF}7JMh1kyrT2O5 zxZ7cyiRSS(T{Lw6i@K&;q+xq7DvHhA11!G)@ztw9$;Q3lbLzD{AeVk?m<-@r4e+ zs7n}cE6aqkt%>5Yg)tPo3?E-PxxLe_lo6}=pc0scHA@lT;WE&txBI@jJ~W+9S!x() zVwQ6MhewMK>o!ob+Rz!F%1(JFNmQH5hO1gb1skAh4m*qXtoVT??L2u`&dS`Mg-u2| zjuxvt%tVP7+ZAg7cBFlEvNkiaRyLp0&jPlzU|h^&bJlGKHw4U~^;V%+iAaBimIBE1 z7oV>9K(x0o9Y(;#2Gs3fVOh|9UVQMvLqI>%@YL)Sy{tFQBIOnOaO*QG4NmRNXrDPN zrj`3puBY6j10PB2l``%3C^vD1`r>Y0*G@AT9rxQek4zdZaKHCPT1GAO!#W%x^i0TZ zn%42JEL0Re&A7Mr%td)?tkl_Nf{97rO8;LB#Gitp1 zfXFB)r((=2a@J^A!XR7JlI0hyD}1w5?~~upr?)k^+2*|64YHbBF_fusEM!J=zFTQ@ zT^5F$)0J$`HxUWlnqyMV9ib@N{5I(6VtfzE>zU_v+QC`I^38F*pI(piW~uI!V`$wP zrSa`)KN{1Mf!~h96Bv4yCXkIuYd0)OD<=6aVm9`GvVipDfagj6=Kd>r68FkU}EpdeCVE0*xE@P= z1Q6F!NR<+CaJHwAFVh`nCcEg7!<}ZpEuV8|^a?gFya3NnBR|`q<#}UN(WrcD&m(~+ z>0?+n4*29$UTwB3oRB9vOb7jV(*S$a^3?AEo#Q6dPIkUeNYO$s zU3_aqZ>ZZvi)n$WSC93>D8&n^A09?WX=-vt^jMt(4ZP*}JC%pPbsJ+ro-cOLc!iWcJtnlVQkr)!7k z(V!%-uNkyH?u@a8M$UmeM+3f%+-SM4tj))8Q#)p@9@b{2c4?+mj^#ceEEdO;TjIGS)11DDe>xoWcb!Iz%75)%G<@d+B?dN z22Qt3y3DUDX`5#>Re^TlXph?5w17X$wqM3J3RSemO>e~e_Wl-2`%Arsi@I1e#@UTN zQp1p-kbc$Xk4aAyk`EKF*ReF`nws1X8eXgf*51)Td z_rzM13~nV>BpT5^sd%=)R<)T7!ijm4uo-nNzYX;jXaX|F<2yr##hou%EqDcy)}i!r zql98A)v76}^QD2s<;CX0LBJh+w?A?b=$c{-0v)It1;acBKOVu?vx_EktJ0VE?{qu9 zHiNfw>oYPrD6JjzhLV+$dP#@Y_Yf8L)o77kF2xu4hU6En7s6BNs4~4~4^@x%_>_GI z%bVlh?%R8~HP!d!G2ILcG?aT5qke)n!BXc$w+`e{Tn~EiQJ{!g&p%qp#!=&oO)uvB zqi#qm_pxwp!aOo=7knjPVX^S__l@kEa?>g`6RxJV7h<+7 z2Yqww&W79FFs>y-b_aAy8`Z%kvzdPpAyqBL9&{)Rk#4(Pl#ofM_72!JvMUL` za+>VZsaaw=4bvyVP(R;bYOt&WhP9_?_3c7v5lQa%g)RYPHX+TH47%R9Z7nPk%#+O? zA9^{oneU6L!ps`hba_%PEDv?TJX#weXmm~4_tEcc($7V`oo8Dc?UC^;S<=ZW?lY-v zP8Jj=b*IUe*)=6|^#UZy+0W>hfn@OXIn^tNHAtxThqDiFN2;JfaghtuD0mkr!!~-< zb~)VG5z_!%l6^If_dF_E*SEnxj2XqU*HVb;kVXsmxBK$jZzvoa8n^AVF1vEp*6qzC6ai!$V>=BK z^7OQ|ezb-slvnc1^= z(#EC+(az(r!`wBp7|FPy4ScWgby|XbgA(F8UobO6L4`$8VHGPCx^y7sX~h-1tqPxV zaqnAdeLig|=_2Q=RW%eR@JmJqJ7bAxI*6DJh^oj zuJd?*kJVU%3$Z~0T&qrUMA`WOp?59O7^l+^!+VS-TRavGwEAh9cm`Ui+Ic!OMnCk* zrQXOh3xJl;CMT_n0sy!6P6@$*y4<$&lx(>2thUcjZ8-F59{_pmntFbQHH(+6#*RmW zKH{eG;@Sk^THPPkls(k8wP-d)mZO0^Vcw5g=_;M~YPR53NnLMrhPJzJCEHKm#fjUO zX${(g9_wtaP8CLqU$c~2($lPIMexxWC&OH;FIlr(9-{lI7I3xmb;PUJb-ytz?PrdZ zO^%K6JkQi{&yRj*G~NU z`+5HFHu6^`#NV~m$^4A+cWo5)kMGh{o;2}y{aE@R`|votNaOe`u@M8vocv^0{N|Z{MXFjl7iLZ`&B2$*X4mwvEe+2L8US!P0-f zeq=uHZ|nETuF>7g{;YZ&$v=M_M4Dsx+ST{jJfHJQ(WJw$PR(UWk9F#d%l>j7|DSps z|A2lgz6}0(@nyC8<3@Qg%-@w^{y~hMe)98Y1cu2a+>5Hy%8fBb*gF0Ekz literal 0 HcmV?d00001 diff --git a/tests/test_boolean_config.py b/tests/test_boolean_config.py new file mode 100644 index 00000000..3f1b1d6e --- /dev/null +++ b/tests/test_boolean_config.py @@ -0,0 +1,63 @@ +import pytest + +import exotic.exotic as exotic_module + + +BOOLEAN_RUNTIME_PARSERS = ( + exotic_module.is_fast_aperture_mask_enabled, + exotic_module.is_comp_star_required, + exotic_module.is_target_driven_comp_selection_enabled, + exotic_module.should_skip_low_comparison_coverage_rejection, + exotic_module.should_fit_lightcurve_to_every_comparison_candidate, + exotic_module.should_use_automatic_optimal_calibration_selector, + exotic_module.should_use_ensemble_photometry_rather_than_single_comp, + exotic_module.should_use_ensemble_photometry_for_stellar_variability, + exotic_module.should_photometer_fortuitous_variables, + exotic_module.should_use_single_comparison_for_fortuitous_variables, + exotic_module.should_use_nextastro_vsx_cache_first, + exotic_module.should_use_sparse_posterior_live_point_retry, + exotic_module.should_run_fast_ultranest_before_final_run, + exotic_module.should_run_final_residual_rejection, + exotic_module.should_use_legacy_psf_flux_mode, + exotic_module.should_run_final_fit_phase_residual_clip, + exotic_module.should_pick_comparison_by_eebls_snr, + exotic_module.should_use_deviation_from_expected_transit_in_qc, + exotic_module.should_exit_at_first_qc_pass_solution, + exotic_module.should_restrict_rprs_range, + exotic_module.should_use_prior_rprs_when_posterior_pinned, + exotic_module.should_restrict_ars_range, + exotic_module.should_use_psf_photometry, + exotic_module.should_use_aperture_photometry, + exotic_module.should_use_eebls_to_initialize_tmid_and_bounds, + exotic_module.should_detect_bad_pixels_before_photometry, + exotic_module.is_adaptive_aperture_mode_enabled, + exotic_module.should_use_aperture_corrections_and_full_image_fwhm, + exotic_module.should_reject_overexposed_stars, + exotic_module.should_ignore_header_wcs, + exotic_module.should_prefer_pixel_values_over_wcs_for_target, + exotic_module.is_vertical_flux_normalization_disabled, + exotic_module.should_run_stellar_variability_only, + exotic_module.is_out_of_transit_baseline_detrending_enabled, + exotic_module.should_use_impactparameter_rather_than_inclination_to_fit, + exotic_module.should_require_apparent_magnitudes, + exotic_module.should_use_exactly_the_comps_provided, +) + + +TRUE_VARIANTS = (True, 1, "1", "y", "Y", "yes", "TRUE", "on") +FALSE_VARIANTS = (False, 0, "0", "n", "N", "no", "FALSE", "off") + + +@pytest.mark.parametrize("parser", BOOLEAN_RUNTIME_PARSERS, ids=lambda parser: parser.__name__) +def test_runtime_boolean_parser_accepts_every_supported_form(parser): + for value in TRUE_VARIANTS: + assert parser(value) is True + for value in FALSE_VARIANTS: + assert parser(value) is False + + +def test_multiprocess_bad_pixel_boolean_forms_are_consistent(): + for value in TRUE_VARIANTS: + assert exotic_module.get_multiprocess_bad_pixel_precheck_processes(value) >= 1 + for value in FALSE_VARIANTS: + assert exotic_module.get_multiprocess_bad_pixel_precheck_processes(value) is None diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index cbdfcb1b..51bd37c4 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -472,8 +472,18 @@ def test_get_img_scale_uses_first_image_extension_header(tmp_path): def test_should_ignore_header_wcs_defaults_to_false(): assert exotic_module.should_ignore_header_wcs(None) is False - assert exotic_module.should_ignore_header_wcs("n") is False - assert exotic_module.should_ignore_header_wcs("y") is True + for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): + assert exotic_module.should_ignore_header_wcs(value) is True + for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): + assert exotic_module.should_ignore_header_wcs(value) is False + + +def test_should_allow_pixel_alignment_fallback_defaults_to_false(): + assert exotic_module.should_allow_pixel_alignment_fallback(None) is False + for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): + assert exotic_module.should_allow_pixel_alignment_fallback(value) is True + for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): + assert exotic_module.should_allow_pixel_alignment_fallback(value) is False def test_get_bad_wcs_threshold_fraction_defaults_to_three_percent(): @@ -531,7 +541,7 @@ def test_format_plate_solution_reference_uses_basename_only(): ) -def test_log_alignment_progress_prints_basename(monkeypatch): +def test_log_alignment_progress_reports_wcs_location_and_basename(monkeypatch): stdout = io.StringIO() debug_messages = [] @@ -545,8 +555,25 @@ def test_log_alignment_progress_prints_basename(monkeypatch): False, ) - assert stdout.getvalue() == "Aligning frame 145 of 220 : frame_145.fits.fz\n" - assert debug_messages == ["Aligning frame 145 of 220 : frame_145.fits.fz\n"] + expected = "WCS-locating stars in frame 145 of 220 : frame_145.fits.fz\n" + assert stdout.getvalue() == expected + assert debug_messages == [expected] + + +def test_log_alignment_progress_only_says_pixel_aligning_when_enabled(monkeypatch): + stdout = io.StringIO() + monkeypatch.setattr(exotic_module.sys, "stdout", stdout) + monkeypatch.setattr(exotic_module.log, "debug", lambda _message: None) + + exotic_module.log_alignment_progress( + 0, + 2, + "frame_1.fits", + False, + pixel_alignment_enabled=True, + ) + + assert stdout.getvalue() == "Pixel-aligning frame 1 of 2 : frame_1.fits\n" def test_collect_transform_frame_pointings_logs_alignment_progress(monkeypatch): @@ -879,6 +906,82 @@ def fake_run_batch(tasks, *_args, **_kwargs): assert results[3]["fallback"] is not None +def test_build_multiprocess_alignment_results_does_not_fallback_by_default(monkeypatch): + batches = [] + + def fake_run_batch(tasks, *_args, **_kwargs): + batches.append(tasks) + return { + task[0]: { + "index": task[0], + "file_name": task[1], + "wcs": None, + "fallback": None, + } + for task in tasks + } + + monkeypatch.setattr(exotic_module, "_run_multiprocess_alignment_task_batch", fake_run_batch) + + results = exotic_module.build_multiprocess_alignment_results( + np.array(["frame0.fits", "frame1.fits"]), + 2, + np.array([[10.0, 10.0], [20.0, 10.0]]), + target_and_comp_radec=np.array([[1.0, 2.0], [1.1, 2.1]]), + ) + + assert len(batches) == 1 + assert all(task[7] is False for task in batches[0]) + assert all(result["fallback"] is None for result in results) + + +def test_parallel_wcs_task_uses_first_frames_own_wcs_projection(monkeypatch): + projected = np.array([[101.25, 202.5], [303.75, 404.5]], dtype=float) + captured = {} + + class FakeWcs: + is_celestial = True + + @staticmethod + def world_to_pixel_values(_ra, _dec): + return projected[:, 0], projected[:, 1] + + monkeypatch.setattr( + exotic_module, + "_load_alignment_worker_frame", + lambda _file_name: (object(), np.ones((512, 512), dtype=float)), + ) + monkeypatch.setattr(exotic_module, "search_wcs_from_header", lambda _header: FakeWcs()) + + def fake_fit(_image, predicted_coords, *_args, **_kwargs): + captured["coords"] = np.array(predicted_coords, dtype=float, copy=True) + return { + "coords": np.array(predicted_coords, dtype=float, copy=True), + "psf_rows": { + "target": np.array([101.25, 202.5, 1.0, 1.0, 1.0, 0.0, 0.0]), + "comp1": np.array([303.75, 404.5, 1.0, 1.0, 1.0, 0.0, 0.0]), + }, + "warnings": [], + } + + monkeypatch.setattr(exotic_module, "_fit_alignment_candidate_psfs", fake_fit) + + exotic_module._parallel_alignment_task(( + 0, + "frame0.fits", + np.array([[10.0, 20.0], [30.0, 40.0]]), + np.array([[1.0, 2.0], [1.1, 2.1]]), + False, + False, + False, + False, + True, + None, + )) + + np.testing.assert_allclose(captured["coords"], projected) + + def test_downsampled_fallback_transformation_restores_full_resolution_translation(monkeypatch): calls = [] @@ -963,7 +1066,7 @@ def test_filter_sparse_missing_wcs_frames_drops_files_below_three_percent(monkey assert dropped == [missing_frame] -def test_filter_sparse_missing_wcs_frames_keeps_files_at_three_percent_or_higher(monkeypatch): +def test_filter_sparse_missing_wcs_frames_drops_all_missing_wcs_by_default(monkeypatch): frames = [f"frame_{i}.fits" for i in range(33)] missing_frame = frames[5] @@ -976,6 +1079,27 @@ def test_filter_sparse_missing_wcs_frames_keeps_files_at_three_percent_or_higher filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames(frames) + assert filtered.tolist() == [frame for frame in frames if frame != missing_frame] + assert keep_mask.tolist() == [frame != missing_frame for frame in frames] + assert dropped == [missing_frame] + + +def test_filter_sparse_missing_wcs_frames_can_keep_missing_wcs_when_pixel_fallback_is_explicit(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(33)] + missing_frame = frames[5] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: str(file_name)) + monkeypatch.setattr( + exotic_module, + "search_wcs_from_header", + lambda header: types.SimpleNamespace(is_celestial=header != missing_frame), + ) + + filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames( + frames, + allow_pixel_alignment_fallback=True, + ) + assert filtered.tolist() == frames assert keep_mask.tolist() == [True] * len(frames) assert dropped == [] @@ -1143,6 +1267,7 @@ def fake_collect_transform_frame_pointings(inputfiles, frame_loader=None, return filtered, keep_mask, dropped, cached_transforms = exotic_module.filter_pointing_outlier_frames( frames, pointing_rejection_sigma=3.0, + allow_pixel_alignment_fallback=True, return_alignment_transforms=True, ) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 75e12126..4f072bc6 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -1,5 +1,6 @@ import importlib import importlib.util +import json import sys import types from pathlib import Path @@ -8240,7 +8241,7 @@ def test_cli_logs_unhandled_exception_once(monkeypatch): logged = [] - monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda: None) + monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda *args, **kwargs: None) monkeypatch.setattr(exotic_module, "install_exception_hooks", lambda: None) monkeypatch.setattr(exotic_module, "main", lambda: (_ for _ in ()).throw(RuntimeError("boom"))) @@ -8278,30 +8279,103 @@ def fake_import_module(module_name): assert exotic._load_runtime_callable("main")() == "nested-main" -def test_configure_runtime_logging_rebinds_console_handler_to_current_stdout(monkeypatch): +def test_configure_runtime_logging_rebinds_console_handler_to_current_stdout(monkeypatch, tmp_path): import io import exotic.exotic as exotic_module original_handlers = list(exotic_module.log.handlers) original_configured = exotic_module._RUNTIME_LOGGING_CONFIGURED + original_basename = exotic_module._RUNTIME_LOG_BASENAME + original_path = exotic_module._RUNTIME_LOG_PATH try: exotic_module.log.handlers = [] exotic_module._RUNTIME_LOGGING_CONFIGURED = False + exotic_module._RUNTIME_LOG_BASENAME = None + exotic_module._RUNTIME_LOG_PATH = None + monkeypatch.setattr(exotic_module, "_reset_runtime_traceback_watchdog", lambda: None) first_stdout = io.StringIO() monkeypatch.setattr(exotic_module.sys, "stdout", first_stdout) - exotic_module.configure_runtime_logging() + exotic_module.configure_runtime_logging(output_dir=tmp_path, start_new_run=True) handler = exotic_module._find_runtime_handler(exotic_module._RUNTIME_CONSOLE_HANDLER_NAME) assert handler.stream is first_stdout second_stdout = io.StringIO() monkeypatch.setattr(exotic_module.sys, "stdout", second_stdout) - exotic_module.configure_runtime_logging() + exotic_module.configure_runtime_logging(output_dir=tmp_path) assert handler.stream is second_stdout finally: + exotic_module._close_runtime_file_handler() exotic_module.log.handlers = original_handlers exotic_module._RUNTIME_LOGGING_CONFIGURED = original_configured + exotic_module._RUNTIME_LOG_BASENAME = original_basename + exotic_module._RUNTIME_LOG_PATH = original_path + + +def test_runtime_output_directory_is_read_from_command_line_init_file(tmp_path): + import exotic.exotic as exotic_module + + output_dir = tmp_path / "run output" + init_path = tmp_path / "inits.json" + init_path.write_text(json.dumps({ + "user_info": {"Directory to Save Plots": str(output_dir)}, + }), encoding="utf-8") + + assert exotic_module._runtime_output_directory_from_command_line( + ["-red", str(init_path), "-ov"] + ) == str(output_dir) + assert exotic_module._runtime_output_directory_from_command_line( + [f"--reduce={init_path}"] + ) == str(output_dir) + + +def test_runtime_logging_relocates_startup_content_and_keeps_runs_unique(monkeypatch, tmp_path): + import exotic.exotic as exotic_module + + original_handlers = list(exotic_module.log.handlers) + original_configured = exotic_module._RUNTIME_LOGGING_CONFIGURED + original_basename = exotic_module._RUNTIME_LOG_BASENAME + original_path = exotic_module._RUNTIME_LOG_PATH + + try: + exotic_module.log.handlers = [] + exotic_module._RUNTIME_LOGGING_CONFIGURED = False + exotic_module._RUNTIME_LOG_BASENAME = None + exotic_module._RUNTIME_LOG_PATH = None + monkeypatch.setattr(exotic_module.tempfile, "gettempdir", lambda: str(tmp_path / "staging")) + monkeypatch.setattr(exotic_module, "_reset_runtime_traceback_watchdog", lambda: None) + + exotic_module.configure_runtime_logging(start_new_run=True) + staged_log = Path(exotic_module._RUNTIME_LOG_PATH) + exotic_module.log_info("startup message before the save directory was known") + + output_dir = tmp_path / "output" + exotic_module.configure_runtime_logging(output_dir=output_dir) + first_log = Path(exotic_module._RUNTIME_LOG_PATH) + exotic_module.log_info("message after the save directory was known") + exotic_module.close_runtime_logging() + + assert not staged_log.exists() + assert first_log.parent == output_dir.resolve() / "Diagnostics" + first_content = first_log.read_text(encoding="utf-8") + assert "startup message before the save directory was known" in first_content + assert "message after the save directory was known" in first_content + + exotic_module.configure_runtime_logging(output_dir=output_dir, start_new_run=True) + second_log = Path(exotic_module._RUNTIME_LOG_PATH) + exotic_module.log_info("second run message") + exotic_module.close_runtime_logging() + + assert second_log != first_log + assert len(list((output_dir / "Diagnostics").glob("EXOTIC_RunLog_*.log"))) == 2 + assert "second run message" in second_log.read_text(encoding="utf-8") + finally: + exotic_module._close_runtime_file_handler() + exotic_module.log.handlers = original_handlers + exotic_module._RUNTIME_LOGGING_CONFIGURED = original_configured + exotic_module._RUNTIME_LOG_BASENAME = original_basename + exotic_module._RUNTIME_LOG_PATH = original_path def test_log_exception_with_fallback_writes_traceback_to_current_stdout(monkeypatch, capsys): @@ -8328,7 +8402,7 @@ def test_log_exception_with_fallback_writes_traceback_to_current_stdout(monkeypa def test_main_logs_direct_call_exceptions_to_current_stdout(monkeypatch, capsys): import exotic.exotic as exotic_module - monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda: None) + monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda *args, **kwargs: None) monkeypatch.setattr(exotic_module, "install_exception_hooks", lambda: None) monkeypatch.setattr(exotic_module, "_logger_has_current_stdout_handler", lambda logger: False) monkeypatch.setattr(exotic_module, "_main_impl", lambda: (_ for _ in ()).throw(RuntimeError("boom"))) diff --git a/tests/test_inputs.py b/tests/test_inputs.py index ec37bd33..b99dfbfd 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -23,6 +23,21 @@ def test_camera_keeps_dslr_as_dslr(): assert camera("canon dslr") == "DSLR" +@pytest.mark.parametrize("parser", [inputs_module.plate_solution_opt, inputs_module.aavso_comp]) +def test_user_info_boolean_options_accept_supported_forms_without_prompt(monkeypatch, parser): + monkeypatch.setattr( + inputs_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("valid boolean initialization value must not prompt"), + ) + + for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): + assert parser(value) == "y" + + for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): + assert parser(value) == "n" + + def test_comparison_star_coords_accepts_more_than_ten_manual_comps(): comp_stars = [[float(index), float(index + 1)] for index in range(12)] @@ -515,6 +530,21 @@ def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "n" +def test_comp_params_defaults_allow_pixel_alignment_fallback_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["allow_pixel_alignment_fallback"] is False + + def test_comp_params_defaults_prefer_pixel_values_over_wcs_for_target_to_no(tmp_path): init_data = { "user_info": {}, @@ -893,6 +923,21 @@ def test_comp_params_reads_ignore_header_wcs_from_optional_info(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "y" +def test_comp_params_reads_allow_pixel_alignment_fallback_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"allow_pixel_alignment_fallback": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["allow_pixel_alignment_fallback"] is True + + def test_comp_params_reads_prefer_pixel_values_over_wcs_for_target_from_optional_info(tmp_path): init_data = { "user_info": {}, diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 1a06a9be..a22accdc 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -1143,6 +1143,77 @@ def test_merge_nextastro_calibration_stars_deduplicates_catalog_source_ids(): assert list(calibration_stars) == ['NextAstro-12345'] +def test_fetch_aavso_vsp_chart_retries_malformed_json_five_times_then_succeeds(monkeypatch): + payload = {'chartid': 'X-RETRY', 'photometry': []} + responses = [None] * exotic_module.AAVSO_VSP_MAX_RETRIES + [DummyResponse(payload)] + request_timeouts = [] + sleep_delays = [] + log_messages = [] + + class InvalidJSONResponse: + def raise_for_status(self): + return None + + def json(self): + return json.loads('') + + def fake_get(url, timeout): + request_timeouts.append(timeout) + response = responses.pop(0) + return InvalidJSONResponse() if response is None else response + + monkeypatch.setattr(exotic_module.requests, 'get', fake_get) + monkeypatch.setattr(exotic_module, 'sleep', sleep_delays.append) + monkeypatch.setattr( + exotic_module, + 'log_info', + lambda message, **kwargs: log_messages.append((message, kwargs)), + ) + + assert exotic_module.fetch_aavso_vsp_chart('https://example.invalid/vsp') == payload + assert request_timeouts == [ + exotic_module.AAVSO_VSP_REQUEST_TIMEOUT_SECONDS + ] * (exotic_module.AAVSO_VSP_MAX_RETRIES + 1) + assert sleep_delays == [ + exotic_module.AAVSO_VSP_RETRY_DELAY_SECONDS + ] * exotic_module.AAVSO_VSP_MAX_RETRIES + assert 'attempt 1/6' in log_messages[0][0] + assert 'attempt 5/6' in log_messages[-1][0] + assert all(kwargs.get('warn') is True for _, kwargs in log_messages) + + +def test_fetch_aavso_vsp_chart_raises_after_five_failed_retries(monkeypatch): + request_count = 0 + sleep_delays = [] + + class InvalidJSONResponse: + def raise_for_status(self): + return None + + def json(self): + return json.loads('') + + def fake_get(url, timeout): + nonlocal request_count + request_count += 1 + assert timeout == exotic_module.AAVSO_VSP_REQUEST_TIMEOUT_SECONDS + return InvalidJSONResponse() + + monkeypatch.setattr(exotic_module.requests, 'get', fake_get) + monkeypatch.setattr(exotic_module, 'sleep', sleep_delays.append) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + with pytest.raises(exotic_module.AAVSOVSPUnavailableError) as exc_info: + exotic_module.fetch_aavso_vsp_chart('https://example.invalid/vsp') + + assert request_count == exotic_module.AAVSO_VSP_MAX_RETRIES + 1 + assert sleep_delays == [ + exotic_module.AAVSO_VSP_RETRY_DELAY_SECONDS + ] * exotic_module.AAVSO_VSP_MAX_RETRIES + assert 'after 6 attempts (5 retries)' in str(exc_info.value) + assert 'JSONDecodeError' in str(exc_info.value) + + def test_vsp_query_rejects_band_errors_over_limit(monkeypatch): class DummyWCS: def pixel_to_world_values(self, x_pixel, y_pixel): @@ -1172,7 +1243,11 @@ def world_to_pixel_values(self, ra_deg, dec_deg): monkeypatch.setattr(exotic_module, 'search_wcs', lambda file: DummyWCS()) monkeypatch.setattr(exotic_module, 'radec_hours_to_degree', lambda ra, dec: (10.0, 20.0)) - monkeypatch.setattr(exotic_module.requests, 'get', lambda url: DummyResponse(payload)) + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda url, timeout: DummyResponse(payload), + ) monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) vsp_comp_stars, chart_id = exotic_module.vsp_query( @@ -1229,7 +1304,11 @@ def world_to_pixel_values(self, ra_deg, dec_deg): 'radec_hours_to_degree', lambda ra, dec: (float(ra), float(dec)), ) - monkeypatch.setattr(exotic_module.requests, 'get', lambda url: DummyResponse(payload)) + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda url, timeout: DummyResponse(payload), + ) monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) vsp_comp_stars, chart_id = exotic_module.vsp_query( @@ -1280,7 +1359,11 @@ def world_to_pixel_values(self, ra_deg, dec_deg): 'radec_hours_to_degree', lambda ra, dec: (float(ra), float(dec)), ) - monkeypatch.setattr(exotic_module.requests, 'get', lambda url: DummyResponse(payload)) + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda url, timeout: DummyResponse(payload), + ) monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) vsp_comp_stars, _ = exotic_module.vsp_query( @@ -1433,6 +1516,38 @@ def unexpected_vsp_query(*args, **kwargs): assert queried is False +def test_clear_v_calibration_fallback_does_not_repeat_exhausted_vsp_query(monkeypatch): + def unexpected_vsp_query(*args, **kwargs): + raise AssertionError('An exhausted VSP request must not start another retry cycle') + + log_messages = [] + monkeypatch.setattr(exotic_module, 'vsp_query', unexpected_vsp_query) + monkeypatch.setattr( + exotic_module, + 'log_info', + lambda message, **kwargs: log_messages.append((message, kwargs)), + ) + + combined, fallback_stars, chart_id, queried = ( + exotic_module.merge_aavso_vsp_v_calibration_fallback( + 'frame.fits', + [512, 512], + 'Clear', + 1.2, + {}, + [[100, 200]], + vsp_query_available=False, + ) + ) + + assert combined == {} + assert fallback_stars == {} + assert chart_id is None + assert queried is False + assert 'already exhausted all retries' in log_messages[-1][0] + assert log_messages[-1][1].get('warn') is True + + @pytest.mark.parametrize( 'observed_filter', ['CV', 'Clear', 'Luminance', 'Photographic G', 'Gaia G'], diff --git a/tests/test_nonlinear_ld.py b/tests/test_nonlinear_ld.py index df13aaba..79f493e8 100644 --- a/tests/test_nonlinear_ld.py +++ b/tests/test_nonlinear_ld.py @@ -84,6 +84,85 @@ def check_standard(_observed_filter): return False +class BooleanOptionLimbDarkening(UnrecognizedFilterLimbDarkening): + filter_name = None + filter_desc = None + wl_min = None + wl_max = None + + def calculate_ld(self): + return None + + +def set_boolean_option_filter(ld, label): + ld.filter_name = label + ld.filter_desc = label + ld.wl_min = 400.0 + ld.wl_max = 700.0 + + +@pytest.mark.parametrize("config_value", [True, 1, "1", "y", "Y", "yes", "TRUE", "on"]) +def test_nonlinear_ld_boolean_option_accepts_true_forms(monkeypatch, config_value): + ld = BooleanOptionLimbDarkening() + monkeypatch.setattr( + exotic_module, + "user_input", + lambda prompt, **_kwargs: 1 if "enter 1" in prompt.lower() else pytest.fail( + "valid true boolean must not trigger the y/n prompt" + ), + ) + monkeypatch.setattr( + exotic_module, + "standard_filter", + lambda selected_ld, _observed_filter: set_boolean_option_filter(selected_ld, "standard"), + ) + monkeypatch.setattr( + exotic_module, + "user_entered_ld", + lambda *_args, **_kwargs: pytest.fail("true must select calculated limb darkening"), + ) + info_dict = { + "filter": "mystery-band", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": config_value, + } + + exotic_module.nonlinear_ld(ld, info_dict) + + assert info_dict["filter"] == "standard" + + +@pytest.mark.parametrize("config_value", [False, 0, "0", "n", "N", "no", "FALSE", "off"]) +def test_nonlinear_ld_boolean_option_accepts_false_forms(monkeypatch, config_value): + ld = BooleanOptionLimbDarkening() + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("valid false boolean must not prompt"), + ) + monkeypatch.setattr( + exotic_module, + "standard_filter", + lambda *_args, **_kwargs: pytest.fail("false must select user-entered limb darkening"), + ) + monkeypatch.setattr( + exotic_module, + "user_entered_ld", + lambda selected_ld, _observed_filter: set_boolean_option_filter(selected_ld, "manual"), + ) + info_dict = { + "filter": "mystery-band", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": config_value, + } + + exotic_module.nonlinear_ld(ld, info_dict) + + assert info_dict["filter"] == "manual" + + def test_nonlinear_ld_non_interactive_rejects_unrecognized_filter_without_prompt(monkeypatch): monkeypatch.setattr( exotic_module, diff --git a/tests/test_utils.py b/tests/test_utils.py index fea9d693..6aeac6ee 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,6 +2,19 @@ from unittest.mock import patch +def test_coerce_boolean_config_value_accepts_all_supported_forms(): + for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): + assert coerce_boolean_config_value(value) is True + + for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): + assert coerce_boolean_config_value(value) is False + + +def test_coerce_boolean_config_value_rejects_non_boolean_values(): + for value in (None, 2, -1, "sometimes", [], {}): + assert coerce_boolean_config_value(value) is None + + def test_filename_date_token_uses_date_only_for_iso_timestamp(): assert filename_date_token("2026-05-06T19:51:13.964-0700") == "2026-05-06" assert filename_date_token("20260506T195113") == "2026-05-06" From d295d8c4b65954b4aadfca642f644c90c8119abf Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 10 Aug 2026 00:11:52 +1000 Subject: [PATCH 110/116] Interpret N,S,E,W in long/lat headers --- exotic/api/colab.py | 48 ++++++++++++++++++++++-------- exotic/exotic.py | 21 +++++++++---- exotic/utils.py | 61 ++++++++++++++++++++++++++------------ tests/test_centroid_wcs.py | 19 ++++++++++++ tests/test_utils.py | 31 +++++++++++++++++++ 5 files changed, 143 insertions(+), 37 deletions(-) diff --git a/exotic/api/colab.py b/exotic/api/colab.py index 1ccb8dbd..fc93c1c2 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -182,20 +182,42 @@ def get_val(hdr, ks): ######################################################### def process_lat_long(val, key): - m = re.search(r"\'?([+-]?\d+)[\s\:](\d+)[\s\:](\d+\.?\d*)", val) - if m: - deg, min, sec = float(m.group(1)), float(m.group(2)), float(m.group(3)) - if deg < 0: - v = deg - (((60*min) + sec)/3600) - else: - v = deg + (((60*min) + sec)/3600) - return(add_sign(v)) - m = re.search(r"^'?([+-]?\d+\.\d+)", val) - if m: - v = float(m.group(1)) - return(add_sign(v)) - else: + text = str(val).strip() + coordinate_type = str(key).strip().lower() + valid_hemispheres = { + "latitude": {"N", "S"}, + "longitude": {"E", "W"}, + }.get(coordinate_type) + hemisphere = None + trailing_hemisphere = re.search(r"([NSEW])\s*$", text) + leading_hemisphere = re.match(r"\s*([NSEW])(?=\s|[+-]?\d)", text) + hemisphere_match = trailing_hemisphere or leading_hemisphere + if hemisphere_match: + hemisphere = hemisphere_match.group(1) + if valid_hemispheres is not None and hemisphere not in valid_hemispheres: + print(f"Cannot match value {val}, which is meant to be {key}.") + return None + start, end = hemisphere_match.span(1) + text = f"{text[:start]}{text[end:]}".strip() + + number_tokens = re.findall(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)", text) + if not 1 <= len(number_tokens) <= 3: + print(f"Cannot match value {val}, which is meant to be {key}.") + return None + if (len(number_tokens) == 1 and hemisphere is None and + "." not in number_tokens[0] and number_tokens[0][0] not in "+-"): print(f"Cannot match value {val}, which is meant to be {key}.") + return None + + degrees = float(number_tokens[0]) + minutes = abs(float(number_tokens[1])) if len(number_tokens) >= 2 else 0.0 + seconds = abs(float(number_tokens[2])) if len(number_tokens) >= 3 else 0.0 + magnitude = abs(degrees) + minutes / 60.0 + seconds / 3600.0 + if hemisphere: + sign = -1.0 if hemisphere in {"S", "W"} else 1.0 + else: + sign = -1.0 if number_tokens[0].startswith("-") else 1.0 + return(add_sign(sign * magnitude)) ######################################################### diff --git a/exotic/exotic.py b/exotic/exotic.py index 6fbd472a..d3cb872c 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -13182,6 +13182,9 @@ def log_lightcurve_filter_diagnostics(diagnostics, header="Lightcurve frame reje UTC_START_EXPOSURE_HEADER_KEYS = ("DATE-UTC", "DATE-BEG", "DATE-OBS", "UT-OBS") UTC_END_EXPOSURE_HEADER_KEYS = ("DATE-END", "END-OBS") EXPOSURE_VARIATION_REQUIRE_COMP_STAR_FRACTION = 0.01 +HEADER_NUMERIC_TOKEN_RE = re.compile( + r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?" +) def header_scalar_value(value): @@ -13190,7 +13193,7 @@ def header_scalar_value(value): return value -def finite_header_float(value): +def finite_header_float(value, allow_unit_text=False): value = header_scalar_value(value) if value is None: return None @@ -13199,15 +13202,23 @@ def finite_header_float(value): try: numeric_value = float(str(value).strip()) except (TypeError, ValueError): - return None + if not allow_unit_text: + return None + numeric_match = HEADER_NUMERIC_TOKEN_RE.search(str(value)) + if numeric_match is None: + return None + try: + numeric_value = float(numeric_match.group(0)) + except (TypeError, ValueError): + return None return numeric_value if np.isfinite(numeric_value) else None -def first_header_float(hdr, keys): +def first_header_float(hdr, keys, allow_unit_text=False): for key in keys: if key not in hdr: continue - numeric_value = finite_header_float(hdr[key]) + numeric_value = finite_header_float(hdr[key], allow_unit_text=allow_unit_text) if numeric_value is not None: return key, numeric_value return None, None @@ -13374,7 +13385,7 @@ def julian_date(hdr, time_unit, exp): return julian_time + offset def get_exp_time(hdr): - _, exp_time = first_header_float(hdr, EXPOSURE_TIME_HEADER_KEYS) + _, exp_time = first_header_float(hdr, EXPOSURE_TIME_HEADER_KEYS, allow_unit_text=True) return exp_time if exp_time is not None else 0.0 def img_time_jd(hdr): diff --git a/exotic/utils.py b/exotic/utils.py index ed75af6d..19b4149e 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -443,8 +443,9 @@ def process_lat_long(val, key): Parameters ---------- val : str - either a longitude or latitude coordinate, with a preceding + or -, - expressed in _either_ HH:MM:SS or degree values. ex: +152.51 or +37:2:24. + Either a longitude or latitude coordinate expressed in HH:MM:SS or + decimal degrees. It may use a leading + or - or a FITS-style N/S/E/W + hemisphere letter. Examples: +152.51, +37:2:24, or 16 30 39.7 W. key : str expects "longitude" or "latitude" @@ -454,26 +455,48 @@ def process_lat_long(val, key): longitude or latitude expressed in degree coordinates with a preceding + or -. Six digits of precision after the decimal. ex: +152.510000 """ - m = re.search(r"\'?([+-]?\d+)[\s:](\d+)[\s:](\d+\.?\d*)", val) or \ - re.search(r"\'?([+-]?\d+)[\s:](\d+\.\d*)", val) - if m: - try: - deg, min, sec = float(m.group(1)), float(m.group(2)), float(m.group(3)) - except IndexError: - deg, min, sec = float(m.group(1)), float(m.group(2)), 0 - if deg < 0: - v = deg - (((60 * min) + sec) / 3600) - else: - v = deg + (((60 * min) + sec) / 3600) - return add_sign(v) + text = str(val).strip() + coordinate_type = str(key).strip().lower() + valid_hemispheres = { + "latitude": {"N", "S"}, + "longitude": {"E", "W"}, + }.get(coordinate_type) + hemisphere = None + + # FITS writers commonly append a hemisphere letter to an otherwise + # unsigned decimal or sexagesimal coordinate. A hemisphere overrides a + # redundant leading sign so that ``-16 30 W`` is not double-negated. + trailing_hemisphere = re.search(r"([NSEW])\s*$", text) + leading_hemisphere = re.match(r"\s*([NSEW])(?=\s|[+-]?\d)", text) + hemisphere_match = trailing_hemisphere or leading_hemisphere + if hemisphere_match: + hemisphere = hemisphere_match.group(1) + if valid_hemispheres is not None and hemisphere not in valid_hemispheres: + print(f"Cannot match value {val}, which is meant to be {key}.") + return None + start, end = hemisphere_match.span(1) + text = f"{text[:start]}{text[end:]}".strip() - m = re.search(r"^'?([+-]?\d+\.\d+)", val) + number_tokens = re.findall(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)", text) + if not 1 <= len(number_tokens) <= 3: + print(f"Cannot match value {val}, which is meant to be {key}.") + return None + if len(number_tokens) == 1 and hemisphere is None \ + and "." not in number_tokens[0] and number_tokens[0][0] not in "+-": + # Preserve the historical rejection of an unsigned integer while + # accepting one when a hemisphere supplies the otherwise missing sign. + print(f"Cannot match value {val}, which is meant to be {key}.") + return None - if m: - v = float(m.group(1)) - return add_sign(v) + degrees = float(number_tokens[0]) + minutes = abs(float(number_tokens[1])) if len(number_tokens) >= 2 else 0.0 + seconds = abs(float(number_tokens[2])) if len(number_tokens) >= 3 else 0.0 + magnitude = abs(degrees) + minutes / 60.0 + seconds / 3600.0 + if hemisphere: + sign = -1.0 if hemisphere in {"S", "W"} else 1.0 else: - print(f"Cannot match value {val}, which is meant to be {key}.") + sign = -1.0 if number_tokens[0].startswith("-") else 1.0 + return add_sign(sign * magnitude) # Credit: Kalee Tock diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index 51bd37c4..dbe69e1a 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -83,6 +83,25 @@ def __call__(self, *args, **kwargs): from exotic import exotic as exotic_module +@pytest.mark.parametrize( + ("header_value", "expected"), + ( + ("120", 120.0), + ("120.0s", 120.0), + ("120.0 sec", 120.0), + ("120.0 secs", 120.0), + ("120.0 seconds", 120.0), + ("exposure 1.2e2 seconds", 120.0), + ), +) +def test_get_exp_time_accepts_numeric_strings_with_unit_text(header_value, expected): + assert exotic_module.get_exp_time({"EXPTIME": header_value}) == pytest.approx(expected) + + +def test_get_exp_time_rejects_unit_text_without_a_number(): + assert exotic_module.get_exp_time({"EXPTIME": "seconds"}) == 0.0 + + def _gaussian_image(shape=(80, 80), center=(40.0, 35.0), amplitude=5000.0, sigma=2.0, background=100.0): y, x = np.indices(shape, dtype=float) cx, cy = center diff --git a/tests/test_utils.py b/tests/test_utils.py index 6aeac6ee..994f71e0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,8 @@ from exotic.utils import * from unittest.mock import patch +import pytest + def test_coerce_boolean_config_value_accepts_all_supported_forms(): for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): @@ -491,6 +493,23 @@ def test_process_lat_long_dms_inputs(self): assert self._EXPECTED_LONGITUDE_RESULT == process_lat_long("+152:30:36", "longitude") assert self._EXPECTED_LATITUDE_RESULT == process_lat_long("+37:2:24", "latitude") + @pytest.mark.parametrize( + ("value", "coordinate_type", "expected"), + ( + ("28 17 58.8 N", "latitude", 28.2996666667), + ("28 17 58.8 S", "latitude", -28.2996666667), + ("16 30 39.7 E", "longitude", 16.5110277778), + ("16 30 39.7 W", "longitude", -16.5110277778), + ("-16 30 39.7 W", "longitude", -16.5110277778), + ("S28:17:58.8", "latitude", -28.2996666667), + ), + ) + def test_process_lat_long_hemisphere_inputs(self, value, coordinate_type, expected): + assert float(process_lat_long(value, coordinate_type)) == pytest.approx(expected) + + def test_process_lat_long_rejects_wrong_hemisphere_for_axis(self): + assert process_lat_long("28 17 58.8 W", "latitude") is None + @patch("builtins.print") def test_bad_inputs(self, mock_print): result = process_lat_long("foo", "longitude") @@ -615,6 +634,18 @@ def test_generic_hdr(self, mock_pll): assert result == hdr["LAT"] # NOTE: actual return value is "+34.560000" but I mocked this call + def test_generic_hdr_interprets_coordinate_hemispheres(self): + hdr = { + "SITELAT": "28 17 58.8 S", + "SITELONG": "16 30 39.7 W", + } + + latitude_result = find(hdr, ['LATITUDE', 'LAT', 'SITELAT']) + longitude_result = find(hdr, ['LONGITUD', 'LONG', 'LONGITUDE', 'SITELONG']) + + assert float(latitude_result) == pytest.approx(-28.2996666667) + assert float(longitude_result) == pytest.approx(-16.5110277778) + @patch("exotic.utils.get_val") def test_ks_zero_not_expected(self, mock_get_val): # NOTE: returns whatever is returned in `val = get_val()` From 9b90ebfbc517dab7a49d2b2b989259ff3f3eaa17 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 12 Aug 2026 08:25:08 +1000 Subject: [PATCH 111/116] Calob logging bug --- exotic/exotic.py | 6 +++- tests/test_exotic_proper_motion.py | 48 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index d3cb872c..0baebb09 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -8080,8 +8080,12 @@ def close_runtime_logging(): def configure_runtime_logging(output_dir=None, start_new_run=False): global _RUNTIME_LOGGING_CONFIGURED, _RUNTIME_LOG_BASENAME, _RUNTIME_LOG_PATH - logging.root.setLevel(logging.DEBUG) log.setLevel(logging.DEBUG) + # EXOTIC owns both of the handlers it needs below. Do not also propagate + # records into environment-owned root handlers: notebook runtimes such as + # Colab can leave one of those handlers attached to a disconnected output + # transport while the current sys.stdout remains usable. + log.propagate = False if start_new_run: _close_runtime_file_handler() diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 4f072bc6..dc9ee964 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -8313,6 +8313,54 @@ def test_configure_runtime_logging_rebinds_console_handler_to_current_stdout(mon exotic_module._RUNTIME_LOG_PATH = original_path +def test_configure_runtime_logging_does_not_use_environment_root_handlers(monkeypatch, tmp_path, capsys): + import io + import logging + import exotic.exotic as exotic_module + + class DisconnectedColabStream(io.StringIO): + def write(self, _value): + raise OSError(107, "Transport endpoint is not connected") + + def flush(self): + raise OSError(107, "Transport endpoint is not connected") + + original_handlers = list(exotic_module.log.handlers) + original_propagate = exotic_module.log.propagate + original_configured = exotic_module._RUNTIME_LOGGING_CONFIGURED + original_basename = exotic_module._RUNTIME_LOG_BASENAME + original_path = exotic_module._RUNTIME_LOG_PATH + root_logger = logging.getLogger() + original_root_handlers = list(root_logger.handlers) + original_root_level = root_logger.level + + try: + exotic_module.log.handlers = [] + exotic_module.log.propagate = True + exotic_module._RUNTIME_LOGGING_CONFIGURED = False + exotic_module._RUNTIME_LOG_BASENAME = None + exotic_module._RUNTIME_LOG_PATH = None + root_logger.handlers = [logging.StreamHandler(DisconnectedColabStream())] + root_logger.setLevel(logging.WARNING) + monkeypatch.setattr(exotic_module, "_reset_runtime_traceback_watchdog", lambda: None) + + exotic_module.configure_runtime_logging(output_dir=tmp_path, start_new_run=True) + exotic_module.log.debug("frame progress written only to EXOTIC's file handler") + + assert exotic_module.log.propagate is False + assert root_logger.level == logging.WARNING + assert "Logging error" not in capsys.readouterr().err + finally: + exotic_module._close_runtime_file_handler() + exotic_module.log.handlers = original_handlers + exotic_module.log.propagate = original_propagate + exotic_module._RUNTIME_LOGGING_CONFIGURED = original_configured + exotic_module._RUNTIME_LOG_BASENAME = original_basename + exotic_module._RUNTIME_LOG_PATH = original_path + root_logger.handlers = original_root_handlers + root_logger.setLevel(original_root_level) + + def test_runtime_output_directory_is_read_from_command_line_init_file(tmp_path): import exotic.exotic as exotic_module From 1d39debb4a40cff28c4d6e437f9c44ac9a062ef9 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Wed, 12 Aug 2026 17:55:44 +1000 Subject: [PATCH 112/116] colab error suppression --- exotic/exotic.py | 90 ++++++++++++++++++++++++++---- tests/test_exotic_proper_motion.py | 79 ++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 12 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index 0baebb09..2a756ce5 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -8047,8 +8047,63 @@ def _runtime_file_formatter(): ) +class FailSoftRuntimeFileHandler(logging.FileHandler): + """Keep notebook output usable when a mounted run-log stream disconnects.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._exotic_stream_warning_emitted = False + + def _discard_disconnected_stream(self): + stream = self.stream + self.stream = None + if stream is not None: + try: + stream.close() + except OSError: + pass + + if self._exotic_stream_warning_emitted: + return + self._exotic_stream_warning_emitted = True + try: + print( + "Warning: The EXOTIC run log stream disconnected; console output will continue " + "and EXOTIC will retry the log file automatically.", + file=sys.stdout, + flush=True, + ) + except Exception: + pass + + def emit(self, record): + try: + super().emit(record) + except OSError: + # FileHandler._open() happens outside StreamHandler.emit()'s error + # guard, so mounted-drive failures while reopening need handling here. + self._discard_disconnected_stream() + + def flush(self): + self.acquire() + try: + if self.stream is not None: + try: + self.stream.flush() + except OSError: + self._discard_disconnected_stream() + finally: + self.release() + + def handleError(self, record): + if isinstance(sys.exc_info()[1], OSError): + self._discard_disconnected_stream() + return + super().handleError(record) + + def _open_runtime_file_handler(log_path): - file_handler = logging.FileHandler(filename=log_path, mode='a', encoding='utf-8') + file_handler = FailSoftRuntimeFileHandler(filename=log_path, mode='a', encoding='utf-8') file_handler._exotic_runtime_handler_name = _RUNTIME_FILE_HANDLER_NAME file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(_runtime_file_formatter()) @@ -36870,19 +36925,30 @@ def main(): global _UNHANDLED_EXCEPTION_LOGGED _UNHANDLED_EXCEPTION_LOGGED = False - configure_runtime_logging(start_new_run=True) - install_exception_hooks() - + previous_logging_raise_exceptions = logging.raiseExceptions + # Python's logging package prints its own ``--- Logging error ---`` + # traceback when any handler fails and this development flag is true. + # Notebook transports and mounted Drive files can disconnect independently + # of the reduction, so suppress all such internal logging tracebacks for the + # duration of the run. Genuine EXOTIC exceptions are still reported by the + # explicit exception handling below. + logging.raiseExceptions = False try: - return _main_impl() - except (KeyboardInterrupt, SystemExit): - raise - except Exception as exc: - _handle_unhandled_exception(type(exc), exc, exc.__traceback__) - raise + configure_runtime_logging(start_new_run=True) + install_exception_hooks() + + try: + return _main_impl() + except (KeyboardInterrupt, SystemExit): + raise + except Exception as exc: + _handle_unhandled_exception(type(exc), exc, exc.__traceback__) + raise + finally: + cancel_runtime_traceback_watchdog() + close_runtime_logging() finally: - cancel_runtime_traceback_watchdog() - close_runtime_logging() + logging.raiseExceptions = previous_logging_raise_exceptions def cli(): diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index dc9ee964..b5141c63 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -8361,6 +8361,45 @@ def flush(self): root_logger.setLevel(original_root_level) +def test_runtime_file_handler_suppresses_disconnected_mount_and_reopens(monkeypatch, tmp_path, capsys): + import io + import logging + import exotic.exotic as exotic_module + + class DisconnectedDriveStream(io.StringIO): + def write(self, _value): + raise OSError(107, "Transport endpoint is not connected") + + def flush(self): + raise OSError(107, "Transport endpoint is not connected") + + def close(self): + pass + + log_path = tmp_path / "EXOTIC_RunLog_test.log" + handler = exotic_module.FailSoftRuntimeFileHandler(log_path, mode="a", encoding="utf-8") + handler.setFormatter(logging.Formatter("%(message)s")) + handler.stream = DisconnectedDriveStream() + recovered_stream = io.StringIO() + monkeypatch.setattr(handler, "_open", lambda: recovered_stream) + + try: + handler.emit(logging.LogRecord("exotic", logging.DEBUG, __file__, 1, "frame 18", (), None)) + first_output = capsys.readouterr() + assert "Logging error" not in first_output.err + assert "run log stream disconnected" in first_output.out + assert handler.stream is None + + handler.emit(logging.LogRecord("exotic", logging.DEBUG, __file__, 1, "frame 19", (), None)) + second_output = capsys.readouterr() + assert "Logging error" not in second_output.err + assert "run log stream disconnected" not in second_output.out + assert recovered_stream.getvalue() == "frame 19\n" + finally: + handler.stream = None + handler.close() + + def test_runtime_output_directory_is_read_from_command_line_init_file(tmp_path): import exotic.exotic as exotic_module @@ -8461,3 +8500,43 @@ def test_main_logs_direct_call_exceptions_to_current_stdout(monkeypatch, capsys) output = capsys.readouterr().out assert "Unhandled exception during EXOTIC run" in output assert "RuntimeError: boom" in output + + +def test_main_suppresses_all_internal_logging_error_tracebacks(monkeypatch, capsys): + import io + import logging + import exotic.exotic as exotic_module + + class DisconnectedColabStream(io.StringIO): + def write(self, _value): + raise OSError(107, "Transport endpoint is not connected") + + def flush(self): + raise OSError(107, "Transport endpoint is not connected") + + environment_logger = logging.getLogger("test.disconnected_colab_handler") + environment_logger.handlers = [logging.StreamHandler(DisconnectedColabStream())] + environment_logger.propagate = False + original_raise_exceptions = logging.raiseExceptions + + monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, "install_exception_hooks", lambda: None) + monkeypatch.setattr(exotic_module, "cancel_runtime_traceback_watchdog", lambda: None) + monkeypatch.setattr(exotic_module, "close_runtime_logging", lambda: None) + + def report_through_disconnected_handler(): + environment_logger.error("frame progress") + return "completed" + + monkeypatch.setattr(exotic_module, "_main_impl", report_through_disconnected_handler) + + try: + logging.raiseExceptions = True + assert exotic_module.main() == "completed" + output = capsys.readouterr() + assert "--- Logging error ---" not in output.err + assert "Transport endpoint is not connected" not in output.err + assert logging.raiseExceptions is True + finally: + environment_logger.handlers = [] + logging.raiseExceptions = original_raise_exceptions From fbad14ad48184bb4b1bf7be158efe59ec985bf31 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 13 Aug 2026 09:48:14 +1000 Subject: [PATCH 113/116] Single WCS =/= all images have WCS bug. --- README.md | 4 +- docs/README.md | 4 +- exotic/api/colab.py | 2 +- exotic/api/output_aavso.py | 18 ++- exotic/exotic.py | 209 ++++++++++++++++++++++++++--- exotic/exotic_gui.py | 14 +- exotic/inputs.py | 2 +- exotic/output_files.py | 4 +- exotic/utils.py | 8 ++ inits.json | 4 +- tests/test_centroid_wcs.py | 28 ++-- tests/test_exotic_proper_motion.py | 105 +++++++++++++++ tests/test_inputs.py | 4 +- tests/test_output_files.py | 3 +- tests/test_utils.py | 14 ++ 15 files changed, 370 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 2afa2467..9b71595a 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Maximum Wavelength (nm)": null, "Fast Aperture Mask (y/n)": false, - "allow_pixel_alignment_fallback": false, + "allow_pixel_alignment_fallback": true, "prefer_pixel_values_over_wcs_for_target": false, "use_psf_photometry": true, "use_aperture_photometry": true, @@ -198,7 +198,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended. Every initialization boolean also accepts numeric `1`/`0` and case-insensitive strings `"y"`/`"n"`, `"yes"`/`"no"`, `"true"`/`"false"`, and `"on"`/`"off"`. -Raw-image reductions are WCS-authoritative by default. For every frame with celestial WCS, EXOTIC projects the target and comparison-star sky coordinates through that frame's own FITS header and starts centroiding at those projected pixels. It does not register the image to a reference frame with Astroalign. Set `"allow_pixel_alignment_fallback": true` only when you explicitly want legacy pixel-based registration for a frame whose WCS-derived candidate is unusable. The existing `"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"` option explicitly enables pixel alignment for the entire run. +Raw-image reductions prefer per-frame WCS when WCS coverage is consistent across the dataset. With the default `"allow_pixel_alignment_fallback": true`, EXOTIC uses `"bad_wcs_threshold_percent"` to choose the safe path: sparse missing-WCS frames below the threshold are dropped and the retained sequence remains WCS-based; when the missing-WCS fraction reaches or exceeds the threshold, all frames are retained and legacy pixel alignment is available for frames without usable WCS. Set `"allow_pixel_alignment_fallback": false` to require WCS-only processing and drop every frame without celestial WCS. The existing `"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"` option explicitly enables pixel alignment for the entire run. Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Sexagesimal values must be quoted because they are JSON strings; forms such as `[[02:04:09.90, +46:41:23.0]]` are not valid JSON. Supplied X/Y positions are converted to sky coordinates with the reference frame's WCS; during photometry those sky coordinates are projected independently through every retained frame's own WCS header. diff --git a/docs/README.md b/docs/README.md index 54bdc14a..0f444fcc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -216,7 +216,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, "Fast Aperture Mask (y/n)": false, - "allow_pixel_alignment_fallback": false, + "allow_pixel_alignment_fallback": true, "prefer_pixel_values_over_wcs_for_target": false, "use_psf_photometry": true, "use_aperture_photometry": true, @@ -243,7 +243,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended. Every initialization boolean also accepts numeric `1`/`0` and case-insensitive strings `"y"`/`"n"`, `"yes"`/`"no"`, `"true"`/`"false"`, and `"on"`/`"off"`. -Raw-image reductions are WCS-authoritative by default. For every frame with celestial WCS, EXOTIC projects the target and comparison-star sky coordinates through that frame's own FITS header and starts centroiding at those projected pixels. It does not register the image to a reference frame with Astroalign. Set `"allow_pixel_alignment_fallback": true` only when you explicitly want legacy pixel-based registration for a frame whose WCS-derived candidate is unusable. The existing `"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"` option explicitly enables pixel alignment for the entire run. +Raw-image reductions prefer per-frame WCS when WCS coverage is consistent across the dataset. With the default `"allow_pixel_alignment_fallback": true`, EXOTIC uses `"bad_wcs_threshold_percent"` to choose the safe path: sparse missing-WCS frames below the threshold are dropped and the retained sequence remains WCS-based; when the missing-WCS fraction reaches or exceeds the threshold, all frames are retained and legacy pixel alignment is available for frames without usable WCS. Set `"allow_pixel_alignment_fallback": false` to require WCS-only processing and drop every frame without celestial WCS. The existing `"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"` option explicitly enables pixel alignment for the entire run. Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. Supplied X/Y positions are converted to sky coordinates with the reference frame's WCS; during photometry those sky coordinates are projected independently through every retained frame's own WCS header. diff --git a/exotic/api/colab.py b/exotic/api/colab.py index fc93c1c2..1e35d67f 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -401,7 +401,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Filter Minimum Wavelength (nm)": %s, "Filter Maximum Wavelength (nm)": %s, "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, - "allow_pixel_alignment_fallback": false, + "allow_pixel_alignment_fallback": true, "bad_wcs_threshold_percent": 3.0, "detrend_on_outoftransit_baseline": true, "use_eebls_to_initialize_tmid_and_bounds": "y", diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index c22472ac..23b935a4 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -43,9 +43,19 @@ import re try: - from ..utils import aavso_output_directory, round_to_2, safe_output_filename + from ..utils import ( + aavso_output_directory, + format_aavso_exoplanet_name, + round_to_2, + safe_output_filename, + ) except ImportError: - from utils import aavso_output_directory, round_to_2, safe_output_filename + from utils import ( + aavso_output_directory, + format_aavso_exoplanet_name, + round_to_2, + safe_output_filename, + ) try: from .version import __version__ except ImportError: @@ -246,7 +256,7 @@ def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): "#DATE_TYPE=BJD_TDB\n" # fixed f"#OBSTYPE=CCD\n" f"#STAR_NAME={self.p_dict['hostname']}\n" # code yields - f"#EXOPLANET_NAME={self.p_dict['pl_name']}\n" # code yields + f"#EXOPLANET_NAME={format_aavso_exoplanet_name(self.p_dict['pl_name'])}\n" # code yields f"#BINNING=1x1\n" # uhhh i just put One. f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI f"{gaia_dist_header}" @@ -318,7 +328,7 @@ def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): "#DATE_TYPE=BJD_TDB\n" # fixed f"#OBSTYPE=CCD\n" f"#STAR_NAME={self.p_dict['hostname']}\n" # code yields - f"#EXOPLANET_NAME={self.p_dict['pl_name']}\n" # code yields + f"#EXOPLANET_NAME={format_aavso_exoplanet_name(self.p_dict['pl_name'])}\n" # code yields f"#BINNING=1x1\n" # uhhh i just put One. f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI f"{gaia_dist_header}" diff --git a/exotic/exotic.py b/exotic/exotic.py index 2a756ce5..edb1b44f 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -9649,7 +9649,7 @@ def should_ignore_header_wcs(config_value): def should_allow_pixel_alignment_fallback(config_value): if config_value is None: - return False + return True if isinstance(config_value, bool): return config_value if isinstance(config_value, (int, float)): @@ -9663,10 +9663,10 @@ def should_allow_pixel_alignment_fallback(config_value): log_info( "Warning: Invalid 'allow_pixel_alignment_fallback' value; " - "pixel-based image alignment will remain disabled.", + "allowing pixel-based image alignment when WCS coverage is incomplete.", warn=True, ) - return False + return True def should_prefer_pixel_values_over_wcs_for_target(config_value): @@ -14545,7 +14545,7 @@ def format_result(result_inputfiles, result_keep_mask, dropped_files): def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_missing_fraction=None, - allow_pixel_alignment_fallback=False): + allow_pixel_alignment_fallback=True): inputfiles = np.array(inputfiles) keep_mask = np.ones(len(inputfiles), dtype=bool) if ignore_header_wcs or len(inputfiles) == 0: @@ -14582,11 +14582,18 @@ def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_mi log_missing_celestial_wcs_preview(missing_wcs_files) return retained_files, keep_mask, missing_wcs_files + threshold_percent = max_missing_fraction * 100.0 + log_info( + f"WCS precheck: {total_files - missing_count}/{total_files} files have celestial WCS. " + f"Retaining all {total_files} frame(s) and enabling pixel alignment fallback because the " + f"missing-WCS fraction is at or above the {threshold_percent:g}% threshold." + ) + log_missing_celestial_wcs_preview(missing_wcs_files) return inputfiles, np.ones(total_files, dtype=bool), [] def should_use_multiprocess_transform_precompute(inputfiles, requested_processes, ignore_header_wcs=False, - allow_pixel_alignment_fallback=False): + allow_pixel_alignment_fallback=True): if requested_processes is None or requested_processes <= 0: return False @@ -22779,13 +22786,16 @@ def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astromet timeList, airMassList, exptimes, norm_flux = [], [], [], [] ignore_header_wcs = should_ignore_header_wcs(info_dict.get('ignore_header_wcs')) allow_pixel_alignment_fallback = should_allow_pixel_alignment_fallback( - info_dict.get('allow_pixel_alignment_fallback', False) + info_dict.get('allow_pixel_alignment_fallback', True) ) pixel_alignment_enabled = bool(ignore_header_wcs or allow_pixel_alignment_fallback) if ignore_header_wcs: log_info("Pixel alignment enabled explicitly: header WCS will be ignored for manual alignment.") elif allow_pixel_alignment_fallback: - log_info("WCS-first coordinate projection enabled with explicit pixel alignment fallback.") + log_info( + "WCS coverage-aware coordinate mode enabled: per-frame WCS is preferred when coverage is " + "consistent, with pixel alignment fallback available for incomplete-WCS datasets." + ) else: log_info( "WCS-authoritative coordinate mode enabled: each frame's header WCS will supply star pixel " @@ -24523,6 +24533,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co 'selected_eebls_snr': np.nan, 'flux_tar': None, 'flux_ref': None, + 'selected_source_indices': None, } fit_tasks = [ @@ -24577,6 +24588,7 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co best_cflux = None selection_metric = 'ktmf' selected_eebls_snr = np.nan + selected_source_indices = None if successful_candidates: has_ktmf = any(np.isfinite(item[0].get('ktmf_metric', np.nan)) for item in successful_candidates) has_eebls = any(np.isfinite(item[0].get('eebls_snr', np.nan)) for item in successful_candidates) @@ -24656,6 +24668,18 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co best_ktmf_metric = selected_summary.get('ktmf_metric', np.nan) best_transit_delta_bic = selected_summary.get('transit_delta_bic', np.nan) selected_eebls_snr = selected_summary.get('eebls_snr', np.nan) + candidate_source_indices = np.flatnonzero(np.asarray(best_candidate['mask'], dtype=bool)) + fitted_times = np.asarray(getattr(best_fit_lc, 'time', []), dtype=float) + candidate_times = np.asarray(times, dtype=float)[candidate_source_indices] + fitted_time_indices = ( + match_time_subset_indices(candidate_times, fitted_times) + if fitted_times.size + else None + ) + if fitted_time_indices is not None: + selected_source_indices = candidate_source_indices[fitted_time_indices] + elif best_tflux is not None and len(best_tflux) == len(candidate_source_indices): + selected_source_indices = candidate_source_indices best_identity = target_fit_candidate_identity(best_candidate) for summary in candidate_summaries: summary['selected'] = target_fit_candidate_identity(summary) == best_identity @@ -24673,9 +24697,90 @@ def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, co 'selected_eebls_snr': selected_eebls_snr, 'flux_tar': best_tflux, 'flux_ref': best_cflux, + 'selected_source_indices': selected_source_indices, } +def apply_raw_target_photometry_selection(target_driven_search, photometry_info, flux_values, + centroid_positions, psf_data): + best_candidate = target_driven_search.get('best_candidate') + best_fit_lc = target_driven_search.get('best_fit_lc') + if ( + best_candidate is None + or best_fit_lc is None + or best_candidate.get('comp_index') is not None + or best_candidate.get('method') != 'aperture' + ): + return False + + aperture = float(best_candidate.get('aper', np.nan)) + annulus = float(best_candidate.get('annulus', np.nan)) + if not np.isfinite(aperture) or aperture <= 0 or not np.isfinite(annulus): + return False + + target_flux = np.asarray(target_driven_search.get('flux_tar'), dtype=float) + reference_flux = np.asarray(target_driven_search.get('flux_ref'), dtype=float) + if target_flux.ndim != 1 or reference_flux.shape != target_flux.shape or target_flux.size == 0: + return False + + source_indices = target_driven_search.get('selected_source_indices') + if source_indices is None: + source_indices = np.arange(target_flux.size, dtype=int) + source_indices = np.asarray(source_indices, dtype=int) + target_rows = np.asarray(psf_data.get('target', [])) + if ( + source_indices.shape != target_flux.shape + or target_rows.ndim < 2 + or target_rows.shape[1] < 2 + or np.any(source_indices < 0) + or np.any(source_indices >= target_rows.shape[0]) + ): + return False + + selected_summary = next( + ( + summary for summary in target_driven_search.get('candidate_summaries', []) + if summary.get('selected') + ), + {}, + ) + photometry_info.update( + best_fit_lc=best_fit_lc, + comp_star_num=None, + comp_star_coords=None, + finder_comparison_entries=[], + min_aperture=-abs(aperture), + min_annulus=annulus, + aperture_index=best_candidate.get('a'), + annulus_index=best_candidate.get('an'), + selected_source_indices=source_indices, + selection_basis='raw_target_flux_fallback', + selection_metric=target_driven_search.get('selection_metric', 'ktmf'), + comparison_ktmf_metric=target_driven_search.get('selected_ktmf_metric', np.nan), + comparison_eebls_snr=target_driven_search.get('selected_eebls_snr', np.nan), + comparison_transit_delta_bic=target_driven_search.get('selected_transit_delta_bic', np.nan), + selected_comparison_fit_point_count=selected_summary.get('fit_point_count'), + selected_comparison_transit_qc_status=selected_summary.get('transit_qc_status'), + selected_comparison_transit_qc_summary=selected_summary.get('transit_qc_summary'), + ) + target_uncertainty = np.sqrt(np.clip(target_flux, 0.0, None)) + flux_values.update( + flux_tar=target_flux, + flux_ref=reference_flux, + flux_unc_tar=target_uncertainty, + flux_unc_ref=np.zeros(reference_flux.shape, dtype=float), + ) + target_x = target_rows[source_indices, 0] + target_y = target_rows[source_indices, 1] + centroid_positions.update( + x_targ=target_x, + y_targ=target_y, + x_ref=np.full(target_x.shape, np.nan, dtype=float), + y_ref=np.full(target_y.shape, np.nan, dtype=float), + ) + return True + + def selected_photometry_method_label(photometry_info): min_aperture = photometry_info.get('min_aperture') min_annulus = photometry_info.get('min_annulus') @@ -32540,13 +32645,16 @@ def lookup_archive_ephemeris(): full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) allow_pixel_alignment_fallback = should_allow_pixel_alignment_fallback( - exotic_infoDict.get('allow_pixel_alignment_fallback', False) + exotic_infoDict.get('allow_pixel_alignment_fallback', True) ) pixel_alignment_enabled = bool(ignore_header_wcs or allow_pixel_alignment_fallback) if ignore_header_wcs: log_info("Pixel alignment enabled explicitly: header WCS will be ignored for manual alignment.") elif allow_pixel_alignment_fallback: - log_info("WCS-first coordinate projection enabled with explicit pixel alignment fallback.") + log_info( + "WCS coverage-aware coordinate mode enabled: per-frame WCS is preferred when coverage is " + "consistent, with pixel alignment fallback available for incomplete-WCS datasets." + ) else: log_info( "WCS-authoritative coordinate mode enabled: each frame's header WCS will supply star pixel " @@ -35590,21 +35698,88 @@ def lookup_archive_ephemeris(): ) attempted_count = len(fit_attempts) ranked_count = len(comparison_fit_search['ranked_summaries']) - log_info( - "Error: Comparison-star calibration exhausted " + failure_message = ( + "Comparison-star calibration exhausted " f"{attempted_count}/{ranked_count} ranked comparison star(s) for " f"{comparison_calibration['method_label']} without a usable fully reduced target fit " - f"(last attempt: Comp {failed_comp_index + 1}; reason: {failure_reason}).", - error=True, + f"(last attempt: Comp {failed_comp_index + 1}; reason: {failure_reason})." ) else: - log_info( - "Error: Comparison-star calibration did not produce any coverage-qualified " - "comparison stars to fully reduce against the target fit.", - error=True, + failure_message = ( + "Comparison-star calibration did not produce any coverage-qualified " + "comparison stars to fully reduce against the target fit." ) + if require_comp_star: + log_info(f"Error: {failure_message}", error=True) + return + log_info( + f"Warning: {failure_message} Falling back to raw target-flux aperture photometry " + "because require_comp_star is disabled.", + warn=True, + ) + + if photometry_info['best_fit_lc'] is None and not require_comp_star: + if not use_aperture_photometry or aper_data is None or apers is None or annuli is None: + log_info( + "Error: require_comp_star is disabled, but raw target-flux fallback requires usable " + "aperture photometry and no aperture grid is available.", + error=True, + ) return + log_info( + "\nNo usable comparison-star reduction was selected. Evaluating raw target-flux " + "aperture candidates because require_comp_star is disabled." + ) + raw_target_search = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + pDict, + [], + psf_data, + aper_data, + apers, + annuli, + sigma_display, + require_comp_star=False, + plot_time_range=full_plot_time_range, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + skip_low_comparison_coverage_rejection=True, + use_psf_photometry=False, + use_aperture_photometry=True, + multiprocess_lightcurve_fits=args.multiprocess_lightcurve_fits, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + psf_flux_data=psf_flux_source, + ) + if not apply_raw_target_photometry_selection( + raw_target_search, + photometry_info, + flux_values, + centroid_positions, + psf_data, + ): + candidate_summaries = raw_target_search.get('candidate_summaries', []) + if candidate_summaries: + log_target_fit_candidate_summaries(candidate_summaries) + log_info( + "Error: require_comp_star is disabled, but no raw target-flux aperture candidate " + "completed a usable reduction.", + error=True, + ) + return + log_info( + "Selected raw target-flux aperture photometry with no comparison star because " + "require_comp_star is disabled.", + warn=True, + ) + update_photometry_adaptive_summary( photometry_info, use_adaptive_apertures, diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index f69ae2f0..ad8293db 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -416,8 +416,8 @@ def save_input(): "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", - "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to true only to permit legacy Astroalign image registration when a frame's WCS-derived star locations are unusable. Default false; WCS headers are authoritative.", - "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them for legacy alignment. With the default WCS-authoritative mode, every image without celestial WCS is dropped. Default 3.", + "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to false to require WCS-only processing and drop every frame without celestial WCS. Default true; EXOTIC prefers WCS when coverage is consistent and otherwise retains the sequence for legacy alignment.", + "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS while still using WCS-only processing. Below the threshold, missing-WCS frames are dropped; at or above it, all frames are retained for alignment fallback. Default 3.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", @@ -462,7 +462,7 @@ def save_input(): } new_inits['optional_info'] = { "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", - "allow_pixel_alignment_fallback": False, + "allow_pixel_alignment_fallback": True, "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, @@ -1533,8 +1533,8 @@ def save_input(): "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", - "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to true only to permit legacy Astroalign image registration when a frame's WCS-derived star locations are unusable. Default false; WCS headers are authoritative.", - "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them for legacy alignment. With the default WCS-authoritative mode, every image without celestial WCS is dropped. Default 3.", + "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to false to require WCS-only processing and drop every frame without celestial WCS. Default true; EXOTIC prefers WCS when coverage is consistent and otherwise retains the sequence for legacy alignment.", + "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS while still using WCS-only processing. Below the threshold, missing-WCS frames are dropped; at or above it, all frames are retained for alignment fallback. Default 3.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", @@ -1626,7 +1626,7 @@ def save_input(): "Filter Maximum Wavelength (nm)": input_data.get('filtermax', null), "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", - "allow_pixel_alignment_fallback": False, + "allow_pixel_alignment_fallback": True, "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, @@ -1704,7 +1704,7 @@ def save_input(): "Exposure Time (s)": input_data['exp'], "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", - "allow_pixel_alignment_fallback": False, + "allow_pixel_alignment_fallback": True, "bad_wcs_threshold_percent": 3.0, "prefer_pixel_values_over_wcs_for_target": "n", "disable vertical flux normalization": False, diff --git a/exotic/inputs.py b/exotic/inputs.py index 778dfada..67e07384 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -265,7 +265,7 @@ def __init__(self, init_opt): 'dist': None, 'pm_ra': None, 'pm_dec': None, 'airmass_already_corrected': False, 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, 'fast_aperture_mask': False, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', - 'allow_pixel_alignment_fallback': False, + 'allow_pixel_alignment_fallback': True, 'prefer_pixel_values_over_wcs_for_target': 'n', 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, 'stellar_variability_only': False, diff --git a/exotic/output_files.py b/exotic/output_files.py index 18c831b1..be06eb06 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -9,6 +9,7 @@ aavso_output_directory, MAGNITUDE_DECIMAL_PLACES, filename_date_token, + format_aavso_exoplanet_name, format_magnitude_error, format_magnitude, magnitude_text, @@ -23,6 +24,7 @@ aavso_output_directory, MAGNITUDE_DECIMAL_PLACES, filename_date_token, + format_aavso_exoplanet_name, format_magnitude_error, format_magnitude, magnitude_text, @@ -2888,7 +2890,7 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, f"{obs_name_header}" f"#OBSTYPE={self.i_dict['camera']}\n" f"#STAR_NAME={self.p_dict['sName']}\n" # code yields - f"#EXOPLANET_NAME={self.p_dict['pName']}\n" # code yields + f"#EXOPLANET_NAME={format_aavso_exoplanet_name(self.p_dict['pName'])}\n" # code yields f"#BINNING={self.i_dict['pixel_bin']}\n" # user input f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI f"#OBSLAT={format_aavso_header_value(self.i_dict.get('lat'))}\n" diff --git a/exotic/utils.py b/exotic/utils.py index 19b4149e..1d7e3366 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -25,6 +25,7 @@ } _WINDOWS_ILLEGAL_FILENAME_CHARS_RE = re.compile(r'[<>:"/\\|?*\x00-\x1f\x7f]') _FILENAME_WHITESPACE_RE = re.compile(r'\s+') +_COMPACT_EXOPLANET_SUFFIX_RE = re.compile(r'(?<=[0-9A-Z])([b-z])$') MAX_APPARENT_MAGNITUDE = 30.0 MAGNITUDE_DECIMAL_PLACES = 4 MINIMUM_MAGNITUDE_ERROR = 0.001 @@ -67,6 +68,13 @@ def aavso_output_directory(root): return output_directory +def format_aavso_exoplanet_name(value): + """Separate a compact trailing planet letter for the AAVSO header.""" + + name = str(value or '').strip() + return _COMPACT_EXOPLANET_SUFFIX_RE.sub(r' \1', name) + + def _clean_filename_text(value): cleaned = _WINDOWS_ILLEGAL_FILENAME_CHARS_RE.sub('-', str(value or '')) cleaned = _FILENAME_WHITESPACE_RE.sub('', cleaned) diff --git a/inits.json b/inits.json index cd95406f..47307c64 100644 --- a/inits.json +++ b/inits.json @@ -24,7 +24,8 @@ "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", "Fast Aperture Mask": "Default false/exact mode for fractional-pixel aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to true to opt into center-based masks for speed.", "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", - "Bad WCS Threshold Percent": "Set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS before EXOTIC keeps them and falls back to legacy alignment. If the missing-WCS fraction is below this threshold, those images are dropped. Default 3.", + "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to false to require WCS-only processing and drop every frame without celestial WCS. Default true; EXOTIC prefers WCS when coverage is consistent and otherwise retains the sequence for legacy alignment.", + "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS while still using WCS-only processing. Below the threshold, missing-WCS frames are dropped; at or above it, all frames are retained for alignment fallback. Default 3.", "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Leave blank/null or set to 0/off to disable. Default disabled.", "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", @@ -129,6 +130,7 @@ "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, "Fast Aperture Mask (y/n)": false, "Ignore WCS in Header and Do Manual Alignment? (y/n)": false, + "allow_pixel_alignment_fallback": true, "bad_wcs_threshold_percent": 3.0, "pointing_rejection_sigma": null, "prefer_pixel_values_over_wcs_for_target": false, diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py index dbe69e1a..2266558a 100644 --- a/tests/test_centroid_wcs.py +++ b/tests/test_centroid_wcs.py @@ -497,8 +497,8 @@ def test_should_ignore_header_wcs_defaults_to_false(): assert exotic_module.should_ignore_header_wcs(value) is False -def test_should_allow_pixel_alignment_fallback_defaults_to_false(): - assert exotic_module.should_allow_pixel_alignment_fallback(None) is False +def test_should_allow_pixel_alignment_fallback_defaults_to_true(): + assert exotic_module.should_allow_pixel_alignment_fallback(None) is True for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): assert exotic_module.should_allow_pixel_alignment_fallback(value) is True for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): @@ -1085,25 +1085,25 @@ def test_filter_sparse_missing_wcs_frames_drops_files_below_three_percent(monkey assert dropped == [missing_frame] -def test_filter_sparse_missing_wcs_frames_drops_all_missing_wcs_by_default(monkeypatch): - frames = [f"frame_{i}.fits" for i in range(33)] - missing_frame = frames[5] +def test_filter_sparse_missing_wcs_frames_keeps_full_sequence_when_most_frames_lack_wcs(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(46)] + only_wcs_frame = frames[0] monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: str(file_name)) monkeypatch.setattr( exotic_module, "search_wcs_from_header", - lambda header: types.SimpleNamespace(is_celestial=header != missing_frame), + lambda header: types.SimpleNamespace(is_celestial=header == only_wcs_frame), ) filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames(frames) - assert filtered.tolist() == [frame for frame in frames if frame != missing_frame] - assert keep_mask.tolist() == [frame != missing_frame for frame in frames] - assert dropped == [missing_frame] + assert filtered.tolist() == frames + assert keep_mask.tolist() == [True] * len(frames) + assert dropped == [] -def test_filter_sparse_missing_wcs_frames_can_keep_missing_wcs_when_pixel_fallback_is_explicit(monkeypatch): +def test_filter_sparse_missing_wcs_frames_can_drop_missing_wcs_when_pixel_fallback_is_disabled(monkeypatch): frames = [f"frame_{i}.fits" for i in range(33)] missing_frame = frames[5] @@ -1116,12 +1116,12 @@ def test_filter_sparse_missing_wcs_frames_can_keep_missing_wcs_when_pixel_fallba filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames( frames, - allow_pixel_alignment_fallback=True, + allow_pixel_alignment_fallback=False, ) - assert filtered.tolist() == frames - assert keep_mask.tolist() == [True] * len(frames) - assert dropped == [] + assert filtered.tolist() == [frame for frame in frames if frame != missing_frame] + assert keep_mask.tolist() == [frame != missing_frame for frame in frames] + assert dropped == [missing_frame] def test_filter_wcs_target_out_of_frame_frames_drops_only_projected_misses(monkeypatch): diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index b5141c63..7f88d0ae 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -132,6 +132,7 @@ def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: alignment_candidate_quality_score, aperture_estimation_comparison_stars, aperture_frame_sigma_from_psf_data, + apply_raw_target_photometry_selection, build_tracked_comparison_pool, collapse_aperture_data_to_selected_grid_cell, ensure_lightcurve_fit_failure_reason, @@ -7063,6 +7064,110 @@ def fake_evaluate(task): assert result["selected_transit_delta_bic"] == pytest.approx(18.0) +def test_apply_raw_target_photometry_selection_sets_no_comparison_aperture_sentinel(): + fit = types.SimpleNamespace(time=np.linspace(0.0, 0.05, 6)) + target_flux = np.linspace(1000.0, 1010.0, 6) + target_driven_search = { + "best_candidate": { + "method": "aperture", + "a": 1, + "an": 2, + "aper": 5.0, + "annulus": 12.0, + "comp_index": None, + }, + "best_fit_lc": fit, + "selected_ktmf_metric": 3.5, + "selected_transit_delta_bic": 12.0, + "selection_metric": "ktmf", + "selected_eebls_snr": 7.0, + "flux_tar": target_flux, + "flux_ref": np.ones(6), + "selected_source_indices": np.arange(6), + "candidate_summaries": [{"selected": True, "fit_point_count": 6}], + } + photometry_info = {"min_aperture": None, "comp_star_num": None} + flux_values = {} + centroid_positions = {} + psf_data = { + "target": np.column_stack([ + np.linspace(10.0, 15.0, 6), + np.linspace(20.0, 25.0, 6), + ]) + } + + applied = apply_raw_target_photometry_selection( + target_driven_search, + photometry_info, + flux_values, + centroid_positions, + psf_data, + ) + + assert applied is True + assert photometry_info["best_fit_lc"] is fit + assert photometry_info["comp_star_num"] is None + assert photometry_info["min_aperture"] == pytest.approx(-5.0) + assert photometry_info["min_annulus"] == pytest.approx(12.0) + assert photometry_info["selection_basis"] == "raw_target_flux_fallback" + assert flux_values["flux_tar"] == pytest.approx(target_flux) + assert flux_values["flux_ref"] == pytest.approx(np.ones(6)) + assert flux_values["flux_unc_ref"] == pytest.approx(np.zeros(6)) + assert np.isnan(centroid_positions["x_ref"]).all() + assert np.isnan(centroid_positions["y_ref"]).all() + + +def test_run_target_driven_photometry_search_can_select_raw_target_without_comparison(monkeypatch): + times = np.linspace(0.0, 0.05, 6) + target_flux = np.linspace(1000.0, 1010.0, 6) + + def fake_evaluate(task): + candidate_times, candidate_target_flux, candidate_reference_flux, *_ = task + fit = types.SimpleNamespace( + time=np.asarray(candidate_times, dtype=float), + residuals=np.full(6, 0.01), + data=np.ones(6), + ) + return { + "myfit": fit, + "accepted": True, + "ktmf_metric": 3.0, + "fit_point_count": 6, + }, np.asarray(candidate_target_flux), np.asarray(candidate_reference_flux) + + monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) + psf_data = { + "target": np.column_stack([ + np.linspace(10.0, 15.0, 6), + np.linspace(20.0, 25.0, 6), + ]) + } + aper_data = {"target": target_flux.reshape(6, 1, 1)} + + result = run_target_driven_photometry_search( + times, + 2460000.0 + times, + np.linspace(1.0, 1.5, 6), + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comp_stars=[], + psf_data=psf_data, + aper_data=aper_data, + apers=np.array([5.0]), + annuli=np.array([12.0]), + sigma=1.0, + require_comp_star=False, + use_psf_photometry=False, + use_aperture_photometry=True, + ) + + assert result["best_candidate"]["comp_index"] is None + assert result["best_candidate"]["method"] == "aperture" + assert result["flux_tar"] == pytest.approx(target_flux) + assert result["flux_ref"] == pytest.approx(np.ones(6)) + assert result["selected_source_indices"] == pytest.approx(np.arange(6)) + + def test_run_target_driven_photometry_search_can_prefer_highest_eebls_snr(monkeypatch): class DummyFit: def __init__(self, residual_level, eebls_snr, delta_bic): diff --git a/tests/test_inputs.py b/tests/test_inputs.py index b99dfbfd..41bcb21e 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -530,7 +530,7 @@ def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): assert inputs.info_dict["ignore_header_wcs"] == "n" -def test_comp_params_defaults_allow_pixel_alignment_fallback_to_false(tmp_path): +def test_comp_params_defaults_allow_pixel_alignment_fallback_to_true(tmp_path): init_data = { "user_info": {}, "optional_info": {}, @@ -542,7 +542,7 @@ def test_comp_params_defaults_allow_pixel_alignment_fallback_to_false(tmp_path): inputs = Inputs(init_opt="y") inputs.comp_params(init_file, {}) - assert inputs.info_dict["allow_pixel_alignment_fallback"] is False + assert inputs.info_dict["allow_pixel_alignment_fallback"] is True def test_comp_params_defaults_prefer_pixel_values_over_wcs_for_target_to_no(tmp_path): diff --git a/tests/test_output_files.py b/tests/test_output_files.py index a889c1ae..456e673a 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -509,7 +509,7 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): "mag_band": "V", }] p_dict = { - "pName": "HAT-P-32 b", + "pName": "HAT-P-32b", "sName": "HAT-P-32", "pPer": 2.1500082, "pPerUnc": 1.3e-07, @@ -582,6 +582,7 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): ).read_bytes() == diagnostic_source.name.encode("utf-8") assert "#OBSDATE=2020-01-01" in output_text + assert "#EXOPLANET_NAME=HAT-P-32 b" in output_text assert "#OBSNAME=Whipple Observatory" in output_text assert "#OBSLAT=+32.41638889" in output_text assert "#OBSLON=-110.73444444" in output_text diff --git a/tests/test_utils.py b/tests/test_utils.py index 994f71e0..8427b350 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -46,6 +46,20 @@ def test_safe_output_filename_removes_spaces_from_planet_names(): assert " " not in filename +@pytest.mark.parametrize( + ("planet_name", "expected"), + ( + ("TOI-4010b", "TOI-4010 b"), + ("Kepler-11c", "Kepler-11 c"), + ("HD 41004Ag", "HD 41004A g"), + ("TOI-4010 b", "TOI-4010 b"), + ("Candidate", "Candidate"), + ), +) +def test_format_aavso_exoplanet_name_separates_planet_suffix(planet_name, expected): + assert format_aavso_exoplanet_name(planet_name) == expected + + def test_sanitize_filename_component_cleans_fallback(): filename = sanitize_filename_component(" ", fallback="bad fallback") From fd0eb7b323381399c8cca316ed5798f178269c62 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 13 Aug 2026 10:59:06 +1000 Subject: [PATCH 114/116] Tightening up "ensemble" language --- exotic/exotic.py | 43 +++++++++++++++++------------ exotic/output_files.py | 6 ++-- exotic/plots.py | 6 ++-- tests/test_exotic_proper_motion.py | 4 +-- tests/test_nextastro_variability.py | 6 ++++ tests/test_output_files.py | 3 ++ tests/test_plots.py | 4 +-- 7 files changed, 44 insertions(+), 28 deletions(-) diff --git a/exotic/exotic.py b/exotic/exotic.py index edb1b44f..37128e0b 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -17180,7 +17180,8 @@ def select_automatic_optimal_calibration_stars( return [], [] log_info( - "Scanning the reference frame for bright, non-saturated ensemble candidates." + "Scanning the reference frame for bright, non-saturated stellar-variability " + "ensemble candidates." ) detection_started = perf_counter() detected_stars = detect_reference_fallback_bright_stars( @@ -17192,7 +17193,7 @@ def select_automatic_optimal_calibration_stars( threshold_percentile=AUTOMATIC_CALIBRATION_SELECTOR_DETECTION_PERCENTILE, ) log_info( - "Reference-frame ensemble candidate scan found " + "Reference-frame stellar-variability ensemble candidate scan found " f"{len(detected_stars)} source(s) in {perf_counter() - detection_started:.2f} seconds." ) detected_pool = dedupe_reference_fallback_stars(detected_stars) @@ -25456,7 +25457,7 @@ def log_comparison_candidate_evaluation_start(comp_summary, rank, ranked_count, ) if ensemble_frame_rejected_count > 0: log_info( - " Candidate ensemble clipping rejects " + " Candidate intercomparison clipping rejects " f"{ensemble_frame_rejected_count} comparison-unstable frame(s) before target fitting." ) log_info(" Preparing comparison-candidate light curve for the full reduction.") @@ -29517,7 +29518,8 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, else "" ) log_info( - "Stellar-variability-only comparison ensemble members: " + f"Using a {len(ensemble_members)}-star comparison ensemble for " + "stellar-variability products only: " f"{member_text}{threshold_text}." ) if member_selection.get('prelimit_member_count', 0) > len(ensemble_members): @@ -29848,11 +29850,12 @@ def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, ) sigma_threshold = comp_summary.get('ensemble_frame_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) candidate_frame_clip_diagnostic = build_time_rejection_diagnostic( - "Comparison-candidate ensemble clip", + "Comparison-candidate intercomparison clip", times, candidate_frame_diagnostic_keep_mask, note=( - "Dropped frames where this comparison star disagreed with the comparison-star ensemble " + "Dropped frames where this comparison star disagreed with the peer-star " + "intercomparison reference " f"before target fitting; same-direction pairwise majority exceeded {sigma_threshold:.2f} sigma " f"(min confirming pair count={required_pairs})." ), @@ -31305,11 +31308,12 @@ def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p ) sigma_threshold = comp_summary.get('ensemble_frame_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) candidate_frame_clip_diagnostic = build_time_rejection_diagnostic( - "Comparison-candidate ensemble clip", + "Comparison-candidate intercomparison clip", times, candidate_frame_diagnostic_keep_mask, note=( - "Dropped frames where this comparison star disagreed with the comparison-star ensemble " + "Dropped frames where this comparison star disagreed with the peer-star " + "intercomparison reference " f"before target fitting; same-direction pairwise majority exceeded {sigma_threshold:.2f} sigma " f"(min confirming pair count={required_pairs})." ), @@ -33294,7 +33298,7 @@ def lookup_archive_ephemeris(): for duplicate_message in fortuitous_duplicate_messages: log_info(duplicate_message) log_info( - "Fortuitous-variable ensemble pool contains " + "Fortuitous-variable pool contains " f"{len(fortuitous_ensemble_stars)} non-variable, non-saturated, " "catalog-matched comparison candidate(s)." ) @@ -34987,7 +34991,8 @@ def lookup_archive_ephemeris(): times, quality_keep_mask, note=( - "Dropped this star's frame-level photometry before comparison ensemble scoring " + "Dropped this star's frame-level photometry before comparison-star " + "intercomparison scoring " f"based on robust PSF diagnostics ({reason_text})." ), )) @@ -35142,7 +35147,7 @@ def lookup_archive_ephemeris(): ) for summary in comparison_calibration['comp_summaries']: aggregate_text = "n/a" if not np.isfinite(summary['aggregate_score']) else f"{summary['aggregate_score'] * 100.0:.4f}%" - ensemble_text = "n/a" if not np.isfinite(summary['ensemble_score']) else f"{summary['ensemble_score'] * 100.0:.4f}%" + intercomparison_text = "n/a" if not np.isfinite(summary['ensemble_score']) else f"{summary['ensemble_score'] * 100.0:.4f}%" pairwise_text = "n/a" if not np.isfinite(summary['pairwise_median_score']) else f"{summary['pairwise_median_score'] * 100.0:.4f}%" selected_label = " [selected]" if summary['selected'] else "" position_text = format_comp_star_position(summary['position']) @@ -35151,10 +35156,10 @@ def lookup_archive_ephemeris(): coverage_text += " [rejected: low coverage]" if summary.get('suitability_outlier_rejected'): coverage_text += " [rejected: high suitability outlier]" - ensemble_frame_text = "" + intercomparison_frame_text = "" if summary.get('ensemble_frame_rejected_count', 0) > 0: - ensemble_frame_text = ( - f", ensemble_frame_rejects={summary['ensemble_frame_rejected_count']}" + intercomparison_frame_text = ( + f", intercomparison_frame_rejects={summary['ensemble_frame_rejected_count']}" ) psf_quality_text = "" if summary.get('psf_quality_rejected_count', 0) > 0: @@ -35168,9 +35173,9 @@ def lookup_archive_ephemeris(): ) log_info( f" {summary['label']}{selected_label} ({position_text}): suitability={aggregate_text}, " - f"ensemble={ensemble_text}, pairwise_median={pairwise_text}, " + f"intercomparison={intercomparison_text}, pairwise_median={pairwise_text}, " f"valid_pairs={summary['valid_pair_count']}, {coverage_text}" - f"{psf_quality_text}{overexposure_text}{ensemble_frame_text}, " + f"{psf_quality_text}{overexposure_text}{intercomparison_frame_text}, " f"reason={summary['selection_reason']}" ) @@ -35295,8 +35300,10 @@ def lookup_archive_ephemeris(): ) if use_ensemble_photometry_for_stellar_variability and vsp_comp_stars: log_info( - "Preparing an independent calibrated comparison-star ensemble for " - "out-of-transit stellar-variability AID output." + "Stellar-variability products only: selecting an independent calibrated " + "comparison-star ensemble of up to " + f"{maximum_number_of_ensemble_comparisons_for_stellar_variability} stars for " + "out-of-transit AID output. This ensemble is not used by the transit fit." ) stellar_variability_output_selection = select_stellar_variability_only_photometry( times, diff --git a/exotic/output_files.py b/exotic/output_files.py index be06eb06..bcd05403 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -3327,11 +3327,11 @@ def save_comp_star_calibration_summary(save_dir, target_name, date, method_label else: handle.write("# Field suitability score,\n") handle.write(f"# Selected comparison star,{'' if best_comp_index is None else best_comp_index + 1}\n") - handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,ensemble_score,pairwise_median_score," + handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,intercomparison_score,pairwise_median_score," "pairwise_max_score,self_score,valid_pair_count,coverage_count,coverage_peer_median," "coverage_min_required,coverage_rejected,suitability_outlier_rejected," - "psf_quality_rejected_count,overexposure_rejected_count,ensemble_frame_rejected_count," - "ensemble_frame_required_valid_pairs\n") + "psf_quality_rejected_count,overexposure_rejected_count,intercomparison_frame_rejected_count," + "intercomparison_frame_required_valid_pairs\n") for summary in comp_summaries: position = summary.get('position') or [None, None] diff --git a/exotic/plots.py b/exotic/plots.py index 92ec5e2e..96ae9293 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -469,12 +469,12 @@ def _draw_comp_star_calibration_axis(axis, times, summary, colors): line_valid = ensemble_time_valid & np.isfinite(line_ratio) if np.any(line_valid): axis.plot(times[ensemble_time_valid], line_ratio[ensemble_time_valid], color='black', lw=1.8, - label='Ensemble') + label='Intercomparison') if has_ensemble_keep_mask: rejected = ensemble_valid & ~ensemble_keep_mask if np.any(rejected): axis.scatter(times[rejected], ensemble_ratio[rejected], marker='x', s=42, - color='red', linewidths=1.4, label='Ensemble clip') + color='red', linewidths=1.4, label='Intercomparison clip') selected_text = " selected" if summary.get('selected') else "" aggregate = summary.get('aggregate_score', np.nan) @@ -502,7 +502,7 @@ def plot_comp_star_suitability(comp_summaries, targ_name, save, date, method_lab fig, ax = plt.subplots(figsize=(max(7, 1.5 * len(labels)), 5)) width = 0.25 ax.bar(positions - width, aggregate, width=width, label='Suitability') - ax.bar(positions, ensemble, width=width, label='Vs ensemble') + ax.bar(positions, ensemble, width=width, label='Intercomparison') ax.bar(positions + width, pairwise, width=width, label='Pairwise median') for position, summary in zip(positions, comp_summaries): diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 7f88d0ae..c5adef98 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -6377,7 +6377,7 @@ def fake_finalize( assert np.nanmedian(result["selected_result"]["cflux_fit"]) == pytest.approx(2.0 * np.pi * 240.0) -def test_fit_ranked_comparison_calibration_candidates_applies_candidate_ensemble_clip(monkeypatch): +def test_fit_ranked_comparison_calibration_candidates_applies_candidate_intercomparison_clip(monkeypatch): observed_lengths = [] def fake_diagnostics(times, *args, **kwargs): @@ -6473,7 +6473,7 @@ def fake_finalize( assert observed_lengths == [("diagnostics", 5), ("finalize", 5)] diagnostic = result["attempts"][0]["fit"].frame_filter_diagnostics[0] - assert diagnostic["stage"] == "Comparison-candidate ensemble clip" + assert diagnostic["stage"] == "Comparison-candidate intercomparison clip" assert diagnostic["dropped_point_count"] == 1 assert result["attempts"][0]["fit_point_count"] == 5 diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index a22accdc..36bf44d1 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -3595,6 +3595,8 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(monkeypatch, tmp_path): monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + logged = [] + monkeypatch.setattr(exotic_module, 'log_info', lambda message, **kwargs: logged.append(message)) frame_count = 12 times = np.linspace(10.2, 10.3, frame_count) target_flux = 1000.0 * (1.0 + np.linspace(-0.002, 0.002, frame_count)) @@ -3672,6 +3674,10 @@ def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(monkey assert selected['comp_index'] is None assert selected['ensemble_member_keys'] == ['comp1', 'comp2'] assert selected['fit'].stellar_variability_ensemble_members + assert any( + 'Using a 2-star comparison ensemble for stellar-variability products only:' in message + for message in logged + ) assert len(selected['fit'].stellar_variability_ensemble_magnitudes) == len(selected['fit'].time) np.testing.assert_allclose( selected['fit'].differential_magnitude_reference_flux, diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 456e673a..e13ba368 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -998,6 +998,9 @@ def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): text = summary_path.read_text() assert "# Selected comparison star,1" in text + assert "intercomparison_score" in text + assert "intercomparison_frame_rejected_count" in text + assert "ensemble_score" not in text assert "suitability_outlier_rejected" in text assert "overexposure_rejected_count" in text assert "Comp 1,101,202,true" in text diff --git a/tests/test_plots.py b/tests/test_plots.py index d3963c29..f6e537b7 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -370,7 +370,7 @@ def test_plot_individual_comp_star_calibration_series_masks_rejected_frame_lines def spy_plot(self, x, y, *args, **kwargs): label = kwargs.get("label") - if label in {"vs 2", "Ensemble"}: + if label in {"vs 2", "Intercomparison"}: captured_lines[label] = (np.asarray(x), np.asarray(y)) return original_plot(self, x, y, *args, **kwargs) @@ -395,7 +395,7 @@ def spy_plot(self, x, y, *args, **kwargs): ) assert np.isnan(captured_lines["vs 2"][1][1]) - assert np.isnan(captured_lines["Ensemble"][1][1]) + assert np.isnan(captured_lines["Intercomparison"][1][1]) def test_plot_stellar_variability_labels_reference_coordinates(tmp_path, monkeypatch): From 9100cd3184ae06058f65c2a8b0cadaf8d112bfac Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Thu, 13 Aug 2026 11:40:40 +1000 Subject: [PATCH 115/116] Clearing up sig. figs and coefficient reporting --- exotic/api/elca.py | 33 +-- exotic/api/output_aavso.py | 129 ++++++------ exotic/exotic.py | 128 ++++++++++-- exotic/output_files.py | 312 ++++++++++++++++++----------- exotic/plots.py | 21 +- exotic/utils.py | 62 ++++++ tests/test_elca_baseline.py | 2 +- tests/test_exotic_proper_motion.py | 18 ++ tests/test_exotic_rprs_retry.py | 6 + tests/test_output_files.py | 112 ++++++++++- tests/test_utils.py | 17 ++ 11 files changed, 600 insertions(+), 240 deletions(-) diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 7f034fa8..9e75443b 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -57,6 +57,11 @@ from scipy.special import ndtr, ndtri from ultranest import ReactiveNestedSampler +try: + from ..utils import format_value_and_uncertainty, format_value_with_uncertainty +except ImportError: + from utils import format_value_and_uncertainty, format_value_with_uncertainty + try: from plotting import corner except ImportError: @@ -599,21 +604,6 @@ def round_to_2(*args): return round(x, roundval) -def _decimal_places_for_two_sigfig_error(error): - try: - error = float(error) - except (TypeError, ValueError): - return None - - if not np.isfinite(error): - return None - if error == 0: - return 2 - - exponent = int(np.floor(np.log10(abs(error)))) - return max(0, 1 - exponent) - - def format_value_error_for_plot(value, error): """Format value/error text with a two-significant-figure uncertainty.""" try: @@ -626,14 +616,13 @@ def format_value_error_for_plot(value, error): except (TypeError, ValueError): error = np.nan - decimal_places = _decimal_places_for_two_sigfig_error(error) - if decimal_places is None: + if not np.isfinite(error) or error < 0: value_text = f"{value:.6f}".rstrip('0').rstrip('.') if np.isfinite(value) else "n/a" return value_text, "n/a" - value_text = f"{value:.{decimal_places}f}" if np.isfinite(value) else "n/a" - error_text = f"{error:.{decimal_places}f}" if np.isfinite(error) else "n/a" - return value_text, error_text + if not np.isfinite(value): + return "n/a", format_value_and_uncertainty(0, error)[1] + return format_value_and_uncertainty(value, error) # average data into bins of dt from start to finish @@ -2752,7 +2741,7 @@ def _format_triangle_plot_parameter_title(self, value, error): return "n/a" if not np.isfinite(error) or error < 0: return str(round_to_2(value)) - return f"{round_to_2(value, error)} +/- {round_to_2(error)}" + return format_value_with_uncertainty(value, error) def _weighted_quantiles(self, values, quantiles, weights=None): values = np.asarray(values, dtype=float) @@ -3327,7 +3316,7 @@ def _format_triangle_plot_geometry_value(self, value, error, suffix=''): return f"n/a{suffix}" if error is None or not np.isfinite(error) or error < 0: return f"{round_to_2(value)}{suffix}" - return f"{round_to_2(value, error)} +/- {round_to_2(error)}{suffix}" + return f"{format_value_with_uncertainty(value, error)}{suffix}" def _get_triangle_plot_geometry_summary(self, sampled_keys, sample_points): bound_keys = list(self.bounds.keys()) diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 23b935a4..9a8081b5 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -46,6 +46,8 @@ from ..utils import ( aavso_output_directory, format_aavso_exoplanet_name, + format_value_and_uncertainty, + format_value_with_uncertainty, round_to_2, safe_output_filename, ) @@ -53,6 +55,8 @@ from utils import ( aavso_output_directory, format_aavso_exoplanet_name, + format_value_and_uncertainty, + format_value_with_uncertainty, round_to_2, safe_output_filename, ) @@ -94,7 +98,7 @@ def _format_depth(value, error): except (TypeError, ValueError): error = math.nan if math.isfinite(error) and error >= 0: - return f"{round_to_2(value, error)} +/- {round_to_2(error)} [%]" + return f"{format_value_with_uncertainty(value, error)} [%]" return f"{round_to_2(value)} +/- n/a [%]" @@ -132,12 +136,21 @@ def _depth_result_entry(value, error): error = float(error) except (TypeError, ValueError): error = math.nan - entry = { - 'value': str(round_to_2(value, error)) if math.isfinite(error) else str(round_to_2(value)), - 'units': "percent", - } - if math.isfinite(error): - entry['uncertainty'] = str(round_to_2(error)) + if math.isfinite(error) and error >= 0: + value_text, uncertainty_text = format_value_and_uncertainty(value, error) + else: + value_text, uncertainty_text = str(round_to_2(value)), None + entry = {'value': value_text, 'units': "percent"} + if uncertainty_text is not None: + entry['uncertainty'] = uncertainty_text + return entry + + +def _result_entry(value, error, units=None): + value_text, uncertainty_text = format_value_and_uncertainty(value, error) + entry = {'value': value_text, 'uncertainty': uncertainty_text} + if units: + entry['units'] = units return entry @@ -177,16 +190,22 @@ def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min ) params_num = { - "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " - f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB", - "Ratio of Planet to Stellar Radius (Rp/Rs)": f"{round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- " - f"{round_to_2(self.fit.errors['rprs'])}", - "Semi Major Axis/Star Radius (a/Rs)": f"{round_to_2(self.fit.parameters['ars'], self.fit.errors['ars'])} +/- " - f"{round_to_2(self.fit.errors['ars'])} ", - "Airmass coefficient 1 (a1)": f"{round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- " - f"{round_to_2(self.fit.errors['a1'])}", - "Airmass coefficient 2 (a2)": f"{round_to_2(self.fit.parameters['a2'], self.fit.errors['a2'])} +/- " - f"{round_to_2(self.fit.errors['a2'])}", + "Mid-Transit Time (Tmid)": ( + f"{format_value_with_uncertainty(self.fit.parameters['tmid'], self.fit.errors['tmid'])} " + "BJD_TDB" + ), + "Ratio of Planet to Stellar Radius (Rp/Rs)": format_value_with_uncertainty( + self.fit.parameters['rprs'], self.fit.errors['rprs'] + ), + "Semi Major Axis/Star Radius (a/Rs)": ( + f"{format_value_with_uncertainty(self.fit.parameters['ars'], self.fit.errors['ars'])} " + ), + "Airmass coefficient 1 (a1)": format_value_with_uncertainty( + self.fit.parameters['a1'], self.fit.errors['a1'] + ), + "Airmass coefficient 2 (a2)": format_value_with_uncertainty( + self.fit.parameters['a2'], self.fit.errors['a2'] + ), "Scatter in the residuals of the lightcurve fit is": f"{round_to_2(100. * std(self.fit.residuals / median(self.fit.data)))} %", } depth_params = _depth_final_params(self.fit, self.p_dict) @@ -216,8 +235,9 @@ def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min phot_ext["Optimal Annulus"] = f"{min_annul}" params_num.update(phot_ext) - params_num["Transit Duration (day)"] = (f"{round_to_2(mean(self.durs), std(self.durs))} +/- " - f"{round_to_2(std(self.durs))}") + params_num["Transit Duration (day)"] = format_value_with_uncertainty( + mean(self.durs), std(self.durs) + ) final_params = {'FINAL PLANETARY PARAMETERS': params_num} with params_file.open('w') as f: @@ -268,17 +288,17 @@ def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): "#MEASUREMENT_TYPE=Rnflux\n" # fixed f"#FILTER=I\n" f"#FILTER-XC={dumps(filter_dict)}\n" - f"#PRIORS=Period={round_to_2(self.p_dict['pl_orbper'], self.p_dict['pl_orbpererr1'])} +/- {round_to_2(self.p_dict['pl_orbpererr1'])}" - f",a/R*={round_to_2(self.p_dict['pl_ratdor'], self.p_dict['pl_ratdorerr1'])} +/- {round_to_2(self.p_dict['pl_ratdorerr1'])}" - f",inc={round_to_2(self.p_dict['pl_orbincl'], self.p_dict['pl_orbinclerr1'])} +/- {round_to_2(self.p_dict['pl_orbinclerr1'])}" + f"#PRIORS=Period={format_value_with_uncertainty(self.p_dict['pl_orbper'], self.p_dict['pl_orbpererr1'])}" + f",a/R*={format_value_with_uncertainty(self.p_dict['pl_ratdor'], self.p_dict['pl_ratdorerr1'])}" + f",inc={format_value_with_uncertainty(self.p_dict['pl_orbincl'], self.p_dict['pl_orbinclerr1'])}" f",ecc={round_to_2(self.p_dict['pl_orbeccen'])}" f",u0={round_to_2(ld0)}" f",u1={round_to_2(ld1)}" f",u2={round_to_2(ld2)}" f",u3={round_to_2(ld3)}\n" f"#PRIORS-XC={dumps(priors_dict)}\n" # code yields - f"#RESULTS=Tc={round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- {round_to_2(self.fit.errors['tmid'])}" - f",Rp/R*={round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- {round_to_2(self.fit.errors['rprs'])}" + f"#RESULTS=Tc={format_value_with_uncertainty(self.fit.parameters['tmid'], self.fit.errors['tmid'])}" + f",Rp/R*={format_value_with_uncertainty(self.fit.parameters['rprs'], self.fit.errors['rprs'])}" f",Am1=0" f",Am2=0\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields @@ -340,17 +360,17 @@ def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): "#MEASUREMENT_TYPE=Rnflux\n" # fixed f"#FILTER=I\n" f"#FILTER-XC={dumps(filter_dict)}\n" - f"#PRIORS=Period={round_to_2(self.p_dict['pl_orbper'], self.p_dict['pl_orbpererr1'])} +/- {round_to_2(self.p_dict['pl_orbpererr1'])}" - f",a/R*={round_to_2(self.p_dict['pl_ratdor'], self.p_dict['pl_ratdorerr1'])} +/- {round_to_2(self.p_dict['pl_ratdorerr1'])}" - f",inc={round_to_2(self.p_dict['pl_orbincl'], self.p_dict['pl_orbinclerr1'])} +/- {round_to_2(self.p_dict['pl_orbinclerr1'])}" + f"#PRIORS=Period={format_value_with_uncertainty(self.p_dict['pl_orbper'], self.p_dict['pl_orbpererr1'])}" + f",a/R*={format_value_with_uncertainty(self.p_dict['pl_ratdor'], self.p_dict['pl_ratdorerr1'])}" + f",inc={format_value_with_uncertainty(self.p_dict['pl_orbincl'], self.p_dict['pl_orbinclerr1'])}" f",ecc={round_to_2(self.p_dict['pl_orbeccen'])}" f",u0={round_to_2(ld0)}" f",u1={round_to_2(ld1)}" f",u2={round_to_2(ld2)}" f",u3={round_to_2(ld3)}\n" f"#PRIORS-XC={dumps(priors_dict)}\n" # code yields - f"#RESULTS=Tc={round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- {round_to_2(self.fit.errors['tmid'])}" - f",Rp/R*={round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- {round_to_2(self.fit.errors['rprs'])}" + f"#RESULTS=Tc={format_value_with_uncertainty(self.fit.parameters['tmid'], self.fit.errors['tmid'])}" + f",Rp/R*={format_value_with_uncertainty(self.fit.parameters['rprs'], self.fit.errors['rprs'])}" f",Am1=0" f",Am2=0\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields @@ -375,20 +395,13 @@ def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): def aavso_dicts(planet_dict, fit, i_dict, durs, ld0, ld1, ld2, ld3): priors = { - 'Period': { - 'value': str(round_to_2(planet_dict['pl_orbper'], planet_dict['pl_orbpererr1'])), - 'uncertainty': str(round_to_2(planet_dict['pl_orbpererr1'])) if planet_dict['pl_orbpererr1'] else None, - 'units': "days" - }, - 'a/R*': { - 'value': str(round_to_2(planet_dict['pl_ratdor'], planet_dict['pl_ratdorerr1'])), - 'uncertainty': str(round_to_2(planet_dict['pl_ratdorerr1'])) if planet_dict['pl_ratdorerr1'] else planet_dict['pl_ratdorerr1'], - }, - 'inc': { - 'value': str(round_to_2(planet_dict['pl_orbincl'], planet_dict['pl_orbinclerr1'])), - 'uncertainty': str(round_to_2(planet_dict['pl_orbinclerr1'])) if planet_dict['pl_orbinclerr1'] else planet_dict['pl_orbinclerr1'], - 'units': "degrees" - }, + 'Period': _result_entry( + planet_dict['pl_orbper'], planet_dict['pl_orbpererr1'], units="days" + ), + 'a/R*': _result_entry(planet_dict['pl_ratdor'], planet_dict['pl_ratdorerr1']), + 'inc': _result_entry( + planet_dict['pl_orbincl'], planet_dict['pl_orbinclerr1'], units="degrees" + ), 'ecc': { 'value': str(round_to_2(planet_dict['pl_orbeccen'])), 'uncertainty': None, @@ -426,15 +439,8 @@ def aavso_dicts(planet_dict, fit, i_dict, durs, ld0, ld1, ld2, ld3): } results = { - 'Tc': { - 'value': str(round_to_2(fit.parameters['tmid'], fit.errors['tmid'])), - 'uncertainty': str(round_to_2(fit.errors['tmid'])), - 'units': "BJD_TDB" - }, - 'Rp/R*': { - 'value': str(round_to_2(fit.parameters['rprs'], fit.errors['rprs'])), - 'uncertainty': str(round_to_2(fit.errors['rprs'])) - }, + 'Tc': _result_entry(fit.parameters['tmid'], fit.errors['tmid'], units="BJD_TDB"), + 'Rp/R*': _result_entry(fit.parameters['rprs'], fit.errors['rprs']), 'Am1': { 'value': 0, 'uncertainty': None @@ -443,27 +449,18 @@ def aavso_dicts(planet_dict, fit, i_dict, durs, ld0, ld1, ld2, ld3): 'value': 0, 'uncertainty': None }, - 'Duration': { - 'value': str(round_to_2(mean(durs))), - 'uncertainty': str(round_to_2(std(durs))), - 'units': "days" - } + 'Duration': _result_entry(mean(durs), std(durs), units="days"), } # try to add a/Rs if it exists if 'ars' in fit.errors: - results['a/R*'] = { - 'value': str(round_to_2(fit.parameters['ars'], fit.errors['ars'])), - 'uncertainty': str(round_to_2(fit.errors['ars'])), - } + results['a/R*'] = _result_entry(fit.parameters['ars'], fit.errors['ars']) # check for inclination if 'inc' in fit.errors: - results['inc'] = { - 'value': str(round_to_2(fit.parameters['inc'], fit.errors['inc'])), - 'uncertainty': str(round_to_2(fit.errors['inc'])), - 'units': "degrees" - } + results['inc'] = _result_entry( + fit.parameters['inc'], fit.errors['inc'], units="degrees" + ) limb_darkening = (ld0, ld1, ld2, ld3) depth_summary = fit_transit_depth_summary( diff --git a/exotic/exotic.py b/exotic/exotic.py index 37128e0b..2e821592 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -156,6 +156,8 @@ from output_files import ( OutputFiles, AIDOutputFiles, + baseline_fixed_after_detrending, + pre_detrending_baseline_report, empirical_red_noise_error_scale, differential_magnitude_series_from_fit, fit_empirical_transit_uncertainty, @@ -170,6 +172,8 @@ from .output_files import ( OutputFiles, AIDOutputFiles, + baseline_fixed_after_detrending, + pre_detrending_baseline_report, empirical_red_noise_error_scale, differential_magnitude_series_from_fit, fit_empirical_transit_uncertainty, @@ -208,6 +212,7 @@ MAX_APPARENT_MAGNITUDE, coerce_boolean_config_value, filename_date_token, + format_value_with_uncertainty, is_usable_apparent_magnitude, magnitude_text, normalized_magnitude_error, @@ -221,6 +226,7 @@ MAX_APPARENT_MAGNITUDE, coerce_boolean_config_value, filename_date_token, + format_value_with_uncertainty, is_usable_apparent_magnitude, magnitude_text, normalized_magnitude_error, @@ -570,6 +576,27 @@ def annotate_out_of_transit_baseline_parameter_fit( fit.oot_baseline_parameter_fit_a2_error = a2_error +def annotate_pre_detrending_baseline_coefficients( + fit, + source=None, + scale_parameter=None, + scale_value=None, + scale_error=None, + a2_value=None, + a2_error=None, +): + """Retain the measured baseline coefficients that preceded detrending.""" + if fit is None: + return + + fit.pre_detrending_baseline_source = source + fit.pre_detrending_baseline_scale_parameter = scale_parameter + fit.pre_detrending_baseline_scale_value = scale_value + fit.pre_detrending_baseline_scale_error = scale_error + fit.pre_detrending_baseline_a2_value = a2_value + fit.pre_detrending_baseline_a2_error = a2_error + + def annotate_partial_transit_geometry_prior_assumption(fit, payload): if fit is None: return @@ -5655,6 +5682,21 @@ def aligned_selected_array(key, dtype=float): a2=prior.get('a2'), a2_error=fixed_errors.get('a2'), ) + if detrend_result.get('applied'): + previous_parameters = getattr(previous_fit, 'parameters', {}) + previous_parameters = previous_parameters if isinstance(previous_parameters, dict) else {} + previous_errors = getattr(previous_fit, 'errors', {}) + previous_errors = previous_errors if isinstance(previous_errors, dict) else {} + scale_parameter = 'a1' if 'a1' in previous_parameters else 'a0' + annotate_pre_detrending_baseline_coefficients( + fit, + source="selected fast UltraNest fit before out-of-transit linear baseline detrending", + scale_parameter=scale_parameter, + scale_value=previous_parameters.get(scale_parameter), + scale_error=previous_errors.get(scale_parameter, fixed_errors.get(scale_parameter)), + a2_value=previous_parameters.get('a2'), + a2_error=previous_errors.get('a2', fixed_errors.get('a2')), + ) annotate_out_of_transit_baseline_detrending( fit, bool(detrend_result.get('applied')), @@ -12296,6 +12338,7 @@ def fit_final_lightcurve_with_oot_baseline_detrending( baseline_fixed_errors = {} baseline_constrained_prior = dict(working_prior) baseline_constrained_bounds = clone_lightcurve_bounds(working_bounds) + pre_detrending_baseline = None if baseline_parameter_result.get('applied'): log_info("Prepared out-of-transit airmass/baseline parameter constraints for a fallback final transit refit.") log_info(baseline_parameter_result['note']) @@ -12310,6 +12353,14 @@ def fit_final_lightcurve_with_oot_baseline_detrending( baseline_constrained_bounds.pop('a0', None) baseline_constrained_bounds.pop('a1', None) baseline_constrained_bounds.pop('a2', None) + pre_detrending_baseline = { + 'source': "out-of-transit airmass/baseline parameter fit before linear baseline detrending", + 'scale_parameter': 'a0', + 'scale_value': baseline_parameter_result.get('a0'), + 'scale_error': baseline_parameter_result.get('a0_error'), + 'a2_value': baseline_parameter_result.get('a2'), + 'a2_error': baseline_parameter_result.get('a2_error'), + } else: annotate_out_of_transit_baseline_parameter_fit( fit, @@ -12513,11 +12564,13 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): note=baseline_parameter_fit_note, pre_points=baseline_parameter_result.get('pre_points', 0), post_points=baseline_parameter_result.get('post_points', 0), - a0=baseline_parameter_result.get('a0') if baseline_parameter_fit_used else None, - a0_error=baseline_parameter_result.get('a0_error') if baseline_parameter_fit_used else None, - a2=baseline_parameter_result.get('a2') if baseline_parameter_fit_used else None, - a2_error=baseline_parameter_result.get('a2_error') if baseline_parameter_fit_used else None, + a0=baseline_parameter_result.get('a0') if baseline_parameter_result.get('applied') else None, + a0_error=baseline_parameter_result.get('a0_error') if baseline_parameter_result.get('applied') else None, + a2=baseline_parameter_result.get('a2') if baseline_parameter_result.get('applied') else None, + a2_error=baseline_parameter_result.get('a2_error') if baseline_parameter_result.get('applied') else None, ) + if pre_detrending_baseline is not None: + annotate_pre_detrending_baseline_coefficients(refit, **pre_detrending_baseline) annotate_transit_detection_qc(refit) if extend_sparse_posterior_live_points: refit = extend_sparse_posterior_live_points_if_needed( @@ -26015,7 +26068,7 @@ def format_fit_parameter_with_uncertainty(value, error=None, scale=1.0, suffix=" return f"{round_to_2(scaled_value)}{suffix}" scaled_error = float(error) * abs(scale) - return f"{round_to_2(scaled_value, scaled_error)} +/- {round_to_2(scaled_error)}{suffix}" + return f"{format_value_with_uncertainty(scaled_value, scaled_error)}{suffix}" def summarize_lightcurve_fit_parameters(fit): @@ -36855,8 +36908,14 @@ def lookup_archive_ephemeris(): ) if not np.isfinite(ars_report_error) or ars_report_error < 0: ars_report_error = myfit.errors.get('ars', np.nan) - log_info(f" Mid-Transit Time [BJD_TDB]: {round_to_2(myfit.parameters['tmid'], tmid_report_error)} +/- {round_to_2(tmid_report_error)}") - log_info(f" Radius Ratio (Planet/Star) [Rp/R*]: {round_to_2(myfit.parameters['rprs'], rprs_report_error)} +/- {round_to_2(rprs_report_error)}") + log_info( + " Mid-Transit Time [BJD_TDB]: " + f"{format_value_with_uncertainty(myfit.parameters['tmid'], tmid_report_error)}" + ) + log_info( + " Radius Ratio (Planet/Star) [Rp/R*]: " + f"{format_value_with_uncertainty(myfit.parameters['rprs'], rprs_report_error)}" + ) rprs_prior_fallback_note = getattr(myfit, 'rprs_prior_fallback_note', None) if rprs_prior_fallback_note: log_info(f" Rp/R* fallback note: {rprs_prior_fallback_note}") @@ -36866,7 +36925,10 @@ def lookup_archive_ephemeris(): empirical_uncertainty=empirical_uncertainty, ).items(): log_info(f" {depth_label}: {depth_text}") - log_info(f" Orbital Inclination [inc]: {round_to_2(myfit.parameters['inc'], inc_report_error)} +/- {round_to_2(inc_report_error)}") + log_info( + " Orbital Inclination [inc]: " + f"{format_value_with_uncertainty(myfit.parameters['inc'], inc_report_error)}" + ) ars_text = format_parameter_with_error(myfit.parameters.get('ars'), ars_report_error) if ars_text is not None: log_info(f" Ratio of Distance to Stellar Radius [a/Rs]: {ars_text}") @@ -36896,13 +36958,42 @@ def lookup_archive_ephemeris(): fit_parameters = getattr(myfit, 'parameters', {}) or {} fit_errors = getattr(myfit, 'errors', {}) or {} airmass_scale_key = 'a1' if 'a1' in fit_parameters else 'a0' + fixed_after_detrending = baseline_fixed_after_detrending(myfit) + pre_detrending_report = pre_detrending_baseline_report(myfit) + if fixed_after_detrending and pre_detrending_report is not None: + if pre_detrending_report.get('source'): + log_info( + " Pre-detrending baseline source: " + f"{pre_detrending_report['source']}" + ) + if pre_detrending_report.get('scale_text'): + scale_label = ( + "baseline flux (a0)" + if pre_detrending_report['scale_parameter'] == 'a0' + else "airmass coefficient 1 (a1)" + ) + log_info( + f" Pre-detrending {scale_label}: " + f"{pre_detrending_report['scale_text']}" + ) + if pre_detrending_report.get('a2_text'): + log_info( + " Pre-detrending airmass coefficient 2 (a2): " + f"{pre_detrending_report['a2_text']}" + ) + log_info(" Final detrended-fit baseline coefficients:") if airmass_scale_key in fit_parameters: airmass_scale_error = fit_errors.get(airmass_scale_key) - if airmass_scale_error is not None and np.isfinite(airmass_scale_error): + if fixed_after_detrending: log_info( f" Airmass coefficient 1: " - f"{round_to_2(fit_parameters[airmass_scale_key], airmass_scale_error)} " - f"+/- {round_to_2(airmass_scale_error)}" + f"{round_to_2(fit_parameters[airmass_scale_key])} " + "(fixed after out-of-transit baseline detrending)" + ) + elif airmass_scale_error is not None and np.isfinite(airmass_scale_error): + log_info( + f" Airmass coefficient 1: " + f"{format_value_with_uncertainty(fit_parameters[airmass_scale_key], airmass_scale_error)}" ) else: log_info( @@ -36911,10 +37002,16 @@ def lookup_archive_ephemeris(): ) if 'a2' in fit_parameters: a2_error = fit_errors.get('a2') - if a2_error is not None and np.isfinite(a2_error): + if fixed_after_detrending: + log_info( + f" Airmass coefficient 2: " + f"{round_to_2(fit_parameters['a2'])} " + "(fixed after out-of-transit baseline detrending)" + ) + elif a2_error is not None and np.isfinite(a2_error): log_info( f" Airmass coefficient 2: " - f"{round_to_2(fit_parameters['a2'], a2_error)} +/- {round_to_2(a2_error)}" + f"{format_value_with_uncertainty(fit_parameters['a2'], a2_error)}" ) else: log_info( @@ -36960,7 +37057,10 @@ def lookup_archive_ephemeris(): else: log_info(f" Optimal Aperture: {abs(np.round(display_aperture, 2))}") log_info(f" Optimal Annulus: {np.round(display_annulus, 2)}") - log_info(f" Transit Duration [day]: {round_to_2(np.mean(durs), np.std(durs))} +/- {round_to_2(np.std(durs))}") + log_info( + " Transit Duration [day]: " + f"{format_value_with_uncertainty(np.mean(durs), np.std(durs))}" + ) log_info("*********************************************************") ########## diff --git a/exotic/output_files.py b/exotic/output_files.py index bcd05403..f6a0943f 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -13,6 +13,9 @@ format_magnitude_error, format_magnitude, magnitude_text, + format_uncertainty, + format_value_and_uncertainty, + format_value_with_uncertainty, normalized_magnitude_error, round_to_2, rounded_magnitude_error, @@ -28,6 +31,9 @@ format_magnitude_error, format_magnitude, magnitude_text, + format_uncertainty, + format_value_and_uncertainty, + format_value_with_uncertainty, normalized_magnitude_error, round_to_2, rounded_magnitude_error, @@ -129,25 +135,43 @@ def aavso_airmass_results(fit): ('Am2', '0', '0'), ) + if baseline_fixed_after_detrending(fit): + first_key = 'A0' if 'a0' in fit.parameters else 'Am1' + first_value = fit.parameters.get('a0', fit.parameters.get('a1', 1.0)) + return ( + (first_key, str(round_to_2(first_value)), '0'), + ('Am2', str(round_to_2(fit.parameters.get('a2', 0.0))), '0'), + ) + if 'a0' in fit.parameters: + value_text, uncertainty_text = format_value_and_uncertainty( + fit.parameters['a0'], fit.errors['a0'] + ) first_result = ( 'A0', - str(round_to_2(fit.parameters['a0'], fit.errors['a0'])), - str(round_to_2(fit.errors['a0'])), + value_text, + uncertainty_text, ) else: + value_text, uncertainty_text = format_value_and_uncertainty( + fit.parameters['a1'], fit.errors['a1'] + ) first_result = ( 'Am1', - str(round_to_2(fit.parameters['a1'], fit.errors['a1'])), - str(round_to_2(fit.errors['a1'])), + value_text, + uncertainty_text, ) + a2_value_text, a2_uncertainty_text = format_value_and_uncertainty( + fit.parameters.get('a2', 0), fit.errors.get('a2', 0) + ) + return ( first_result, ( 'Am2', - str(round_to_2(fit.parameters.get('a2', 0), fit.errors.get('a2', 0))), - str(round_to_2(fit.errors.get('a2', 0))), + a2_value_text, + a2_uncertainty_text, ), ) @@ -158,6 +182,52 @@ def aavso_detrend_model(fit): return np.asarray(fit.airmass_model, dtype=float) +def baseline_fixed_after_detrending(fit): + """Return whether the reported neutral baseline was fixed after detrending.""" + + return bool(getattr(fit, 'oot_baseline_detrending_applied', False)) + + +def fixed_detrended_baseline_text(value): + return f"{round_to_2(value)} (fixed after out-of-transit baseline detrending)" + + +def pre_detrending_baseline_report(fit): + """Return formatted measured baseline coefficients retained before detrending.""" + scale_parameter = getattr(fit, 'pre_detrending_baseline_scale_parameter', None) + scale_value = getattr(fit, 'pre_detrending_baseline_scale_value', None) + scale_error = getattr(fit, 'pre_detrending_baseline_scale_error', None) + a2_value = getattr(fit, 'pre_detrending_baseline_a2_value', None) + a2_error = getattr(fit, 'pre_detrending_baseline_a2_error', None) + + def formatted_measurement(value, error): + try: + value_is_finite = np.isfinite(value) + except TypeError: + value_is_finite = False + if not value_is_finite: + return None + try: + error_is_finite = np.isfinite(error) and float(error) >= 0 + except (TypeError, ValueError): + error_is_finite = False + if error_is_finite: + return format_value_with_uncertainty(value, error) + return f"{round_to_2(value)} (uncertainty unavailable)" + + scale_text = formatted_measurement(scale_value, scale_error) + a2_text = formatted_measurement(a2_value, a2_error) + if scale_text is None and a2_text is None: + return None + + return { + 'source': getattr(fit, 'pre_detrending_baseline_source', None), + 'scale_parameter': scale_parameter if scale_parameter in {'a0', 'a1'} else 'a1', + 'scale_text': scale_text, + 'a2_text': a2_text, + } + + def finite_float(value, default=np.nan): try: value = float(value) @@ -850,11 +920,14 @@ def aavso_result_entry(value, uncertainty=None, units=None): if not np.isfinite(value): return None - entry = { - 'value': str(round_to_2(value, uncertainty)) if np.isfinite(uncertainty) else str(round_to_2(value)), - } - if np.isfinite(uncertainty): - entry['uncertainty'] = str(round_to_2(uncertainty)) + if np.isfinite(uncertainty) and uncertainty >= 0: + value_text, uncertainty_text = format_value_and_uncertainty(value, uncertainty) + else: + value_text, uncertainty_text = str(round_to_2(value)), None + + entry = {'value': value_text} + if np.isfinite(uncertainty) and uncertainty >= 0: + entry['uncertainty'] = uncertainty_text if units: entry['units'] = units return entry @@ -1828,35 +1901,33 @@ def format_empirical_transit_uncertainty_final_params(empirical_uncertainty): and model_rprs_uncertainty >= 0 ): params["Ratio of Planet to Stellar Radius (Rp/R*) model-fit uncertainty"] = ( - f"{round_to_2(rprs, model_rprs_uncertainty)} +/- {round_to_2(model_rprs_uncertainty)}" + format_value_with_uncertainty(rprs, model_rprs_uncertainty) ) if np.isfinite(rprs) and np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: if rprs_prior_fallback: params["Ratio of Planet to Stellar Radius (Rp/R*) prior-assumed data-only uncertainty"] = ( - f"{round_to_2(rprs, data_rprs_uncertainty)} +/- {round_to_2(data_rprs_uncertainty)}" + format_value_with_uncertainty(rprs, data_rprs_uncertainty) ) else: params["Ratio of Planet to Stellar Radius (Rp/R*) data-fit red-noise uncertainty"] = ( - f"{round_to_2(rprs, data_rprs_uncertainty)} +/- {round_to_2(data_rprs_uncertainty)}" + format_value_with_uncertainty(rprs, data_rprs_uncertainty) ) if np.isfinite(rprs) and np.isfinite(combined_rprs_uncertainty) and combined_rprs_uncertainty >= 0: if rprs_prior_fallback: params["Ratio of Planet to Stellar Radius (Rp/R*) data-only uncertainty used for primary value"] = ( - f"{round_to_2(rprs, combined_rprs_uncertainty)} +/- " - f"{round_to_2(combined_rprs_uncertainty)}" + format_value_with_uncertainty(rprs, combined_rprs_uncertainty) ) else: params["Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty"] = ( - f"{round_to_2(rprs, combined_rprs_uncertainty)} +/- " - f"{round_to_2(combined_rprs_uncertainty)}" + format_value_with_uncertainty(rprs, combined_rprs_uncertainty) ) if np.isfinite(conservative_rprs_uncertainty): params["Conservative Rp/R* uncertainty to quote"] = ( - f"+/- {round_to_2(conservative_rprs_uncertainty)}" + f"+/- {format_uncertainty(conservative_rprs_uncertainty)}" ) if np.isfinite(rprs) and np.isfinite(data_rprs_standard_error) and data_rprs_standard_error >= 0: params["Ratio of Planet to Stellar Radius (Rp/R*) data-fit standard-error estimate"] = ( - f"{round_to_2(rprs, data_rprs_standard_error)} +/- {round_to_2(data_rprs_standard_error)}" + format_value_with_uncertainty(rprs, data_rprs_standard_error) ) if ( not rprs_prior_fallback @@ -1865,8 +1936,7 @@ def format_empirical_transit_uncertainty_final_params(empirical_uncertainty): and combined_rprs_standard_error >= 0 ): params["Ratio of Planet to Stellar Radius (Rp/R*) model+standard-error estimate"] = ( - f"{round_to_2(rprs, combined_rprs_standard_error)} +/- " - f"{round_to_2(combined_rprs_standard_error)}" + format_value_with_uncertainty(rprs, combined_rprs_standard_error) ) if ( np.isfinite(rprs) @@ -1874,8 +1944,7 @@ def format_empirical_transit_uncertainty_final_params(empirical_uncertainty): and data_rprs_flux_scatter_uncertainty >= 0 ): params["Ratio of Planet to Stellar Radius (Rp/R*) flux-scatter equivalent"] = ( - f"{round_to_2(rprs, data_rprs_flux_scatter_uncertainty)} +/- " - f"{round_to_2(data_rprs_flux_scatter_uncertainty)}" + format_value_with_uncertainty(rprs, data_rprs_flux_scatter_uncertainty) ) if np.isfinite(depth_uncertainty_percent): params["Transit depth red-noise uncertainty"] = ( @@ -2080,7 +2149,7 @@ def format_parameter_with_error(value, error): if not np.isfinite(value): return None if np.isfinite(error) and error >= 0: - return f"{round_to_2(value, error)} +/- {round_to_2(error)}" + return format_value_with_uncertainty(value, error) return f"{round_to_2(value)} +/- n/a" @@ -2557,12 +2626,15 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor if not np.isfinite(ars_report_error) or ars_report_error < 0: ars_report_error = self.fit.errors.get('ars', np.nan) core_params = { - "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], tmid_report_error)} +/- " - f"{round_to_2(tmid_report_error)} BJD_TDB", - "Ratio of Planet to Stellar Radius (Rp/R*)": f"{round_to_2(self.fit.parameters['rprs'], rprs_report_error)} +/- " - f"{round_to_2(rprs_report_error)}", - "Orbital Inclination (inc)": f"{round_to_2(self.fit.parameters['inc'], inc_report_error)} +/- " - f"{round_to_2(inc_report_error)} ", + "Mid-Transit Time (Tmid)": ( + f"{format_value_with_uncertainty(self.fit.parameters['tmid'], tmid_report_error)} BJD_TDB" + ), + "Ratio of Planet to Stellar Radius (Rp/R*)": format_value_with_uncertainty( + self.fit.parameters['rprs'], rprs_report_error + ), + "Orbital Inclination (inc)": ( + f"{format_value_with_uncertainty(self.fit.parameters['inc'], inc_report_error)} " + ), } depth_params = formatted_transit_depth_parameters( self.fit, @@ -2604,16 +2676,15 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num["Impact Parameter (b)"] = impact_text if empirical_uncertainty.get('available'): tmid_model_text = ( - f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " - f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB" + f"{format_value_with_uncertainty(self.fit.parameters['tmid'], self.fit.errors['tmid'])} " + "BJD_TDB" ) tmid_combined_text = core_params["Mid-Transit Time (Tmid)"] params_num["Mid-Transit Time (Tmid) model-fit uncertainty"] = tmid_model_text params_num["Mid-Transit Time (Tmid) model+red-noise uncertainty"] = tmid_combined_text inc_model_text = ( - f"{round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- " - f"{round_to_2(self.fit.errors['inc'])} " + f"{format_value_with_uncertainty(self.fit.parameters['inc'], self.fit.errors['inc'])} " ) params_num["Orbital Inclination (inc) model-fit uncertainty"] = inc_model_text params_num["Orbital Inclination (inc) model+red-noise uncertainty"] = core_params[ @@ -2676,13 +2747,43 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor 'airmass_correction_note', "Skipped; no airmass correction applied.", ) + elif baseline_fixed_after_detrending(self.fit): + pre_detrending_report = pre_detrending_baseline_report(self.fit) + if pre_detrending_report is not None: + if pre_detrending_report.get('source'): + params_num["Pre-detrending baseline source"] = str( + pre_detrending_report['source'] + ) + if pre_detrending_report.get('scale_text'): + scale_parameter = pre_detrending_report['scale_parameter'] + scale_label = ( + "Pre-detrending baseline flux (a0)" + if scale_parameter == 'a0' + else "Pre-detrending airmass coefficient 1 (a1)" + ) + params_num[scale_label] = pre_detrending_report['scale_text'] + if pre_detrending_report.get('a2_text'): + params_num["Pre-detrending airmass coefficient 2 (a2)"] = ( + pre_detrending_report['a2_text'] + ) + if 'a0' in self.fit.parameters: + params_num["Baseline flux (a0)"] = fixed_detrended_baseline_text( + self.fit.parameters['a0'] + ) + else: + params_num["Flux normalization (a1)"] = fixed_detrended_baseline_text( + self.fit.parameters['a1'] + ) + if 'a2' in self.fit.parameters: + params_num["Airmass coefficient 2 (a2)"] = fixed_detrended_baseline_text( + self.fit.parameters['a2'] + ) else: if 'a0' in self.fit.parameters: a0_error = self.fit.errors.get('a0') if isinstance(self.fit.errors, dict) else None if a0_error is not None and np.isfinite(a0_error): - params_num["Baseline flux (a0)"] = ( - f"{round_to_2(self.fit.parameters['a0'], a0_error)} +/- " - f"{round_to_2(a0_error)}" + params_num["Baseline flux (a0)"] = format_value_with_uncertainty( + self.fit.parameters['a0'], a0_error ) else: params_num["Baseline flux (a0)"] = ( @@ -2691,9 +2792,8 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor else: a1_error = self.fit.errors.get('a1') if isinstance(self.fit.errors, dict) else None if a1_error is not None and np.isfinite(a1_error): - params_num["Flux normalization (a1)"] = ( - f"{round_to_2(self.fit.parameters['a1'], a1_error)} +/- " - f"{round_to_2(a1_error)}" + params_num["Flux normalization (a1)"] = format_value_with_uncertainty( + self.fit.parameters['a1'], a1_error ) else: params_num["Flux normalization (a1)"] = ( @@ -2702,9 +2802,8 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor if 'a2' in self.fit.parameters: a2_error = self.fit.errors.get('a2') if isinstance(self.fit.errors, dict) else None if a2_error is not None and np.isfinite(a2_error): - params_num["Airmass coefficient 2 (a2)"] = ( - f"{round_to_2(self.fit.parameters['a2'], a2_error)} +/- " - f"{round_to_2(a2_error)}" + params_num["Airmass coefficient 2 (a2)"] = format_value_with_uncertainty( + self.fit.parameters['a2'], a2_error ) else: params_num["Airmass coefficient 2 (a2)"] = ( @@ -2753,23 +2852,23 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor params_num["Expected-value Rp/R* deviation"] = f"{qc_rprs_deviation_sigma:.2f} sigma" if np.isfinite(qc_rprs_deviation_fit_unc): params_num["Expected-value Rp/R* fit uncertainty used"] = ( - f"+/- {round_to_2(qc_rprs_deviation_fit_unc)}" + f"+/- {format_uncertainty(qc_rprs_deviation_fit_unc)}" ) if np.isfinite(qc_rprs_deviation_model_unc): params_num["Expected-value Rp/R* model-fit uncertainty"] = ( - f"+/- {round_to_2(qc_rprs_deviation_model_unc)}" + f"+/- {format_uncertainty(qc_rprs_deviation_model_unc)}" ) if np.isfinite(qc_rprs_deviation_data_unc): params_num["Expected-value Rp/R* data-fit red-noise uncertainty"] = ( - f"+/- {round_to_2(qc_rprs_deviation_data_unc)}" + f"+/- {format_uncertainty(qc_rprs_deviation_data_unc)}" ) if np.isfinite(qc_rprs_deviation_expected_unc): params_num["Expected-value Rp/R* prior uncertainty"] = ( - f"+/- {round_to_2(qc_rprs_deviation_expected_unc)}" + f"+/- {format_uncertainty(qc_rprs_deviation_expected_unc)}" ) if np.isfinite(qc_rprs_deviation_comparison_unc): params_num["Expected-value Rp/R* total comparison uncertainty"] = ( - f"+/- {round_to_2(qc_rprs_deviation_comparison_unc)}" + f"+/- {format_uncertainty(qc_rprs_deviation_comparison_unc)}" ) if np.isfinite(qc_ktmf): params_num["KTMF"] = f"{qc_ktmf:.2f} / 5.00" @@ -2832,8 +2931,9 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor phot_ext["Optimal Annulus"] = f"{min_annul}" params_num.update(phot_ext) - params_num["Transit Duration (day)"] = (f"{round_to_2(mean(self.durs), std(self.durs))} +/- " - f"{round_to_2(std(self.durs))}") + params_num["Transit Duration (day)"] = format_value_with_uncertainty( + mean(self.durs), std(self.durs) + ) final_params = {'FINAL PLANETARY PARAMETERS': params_num} with params_file.open('w') as f: @@ -2871,6 +2971,28 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, gaia_dist_header = f"#GAIADIST={gaia_dist}\n" if gaia_dist else "" gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" + prior_period_text = format_value_with_uncertainty( + self.p_dict['pPer'], self.p_dict['pPerUnc'] + ) + prior_rprs_text = format_value_with_uncertainty( + self.p_dict['rprs'], self.p_dict['rprsUnc'] + ) + prior_ars_text = format_value_with_uncertainty( + self.p_dict['aRs'], self.p_dict['aRsUnc'] + ) + prior_inc_text = format_value_with_uncertainty( + self.p_dict['inc'], self.p_dict['incUnc'] + ) + prior_ld_texts = [format_value_with_uncertainty(*ld) for ld in (ld0, ld1, ld2, ld3)] + result_tmid_text = format_value_with_uncertainty( + self.fit.parameters['tmid'], self.fit.errors['tmid'] + ) + result_rprs_text = format_value_with_uncertainty( + self.fit.parameters['rprs'], rprs_report_error + ) + result_inc_text = format_value_with_uncertainty( + self.fit.parameters['inc'], self.fit.errors['inc'] + ) params_file = aavso_output_directory(self.dir) / safe_output_filename( "AAVSO", @@ -2905,19 +3027,19 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, "#MEASUREMENT_TYPE=Rnflux\n" # fixed f"#FILTER={self.i_dict['filter']}\n" f"#FILTER-XC={dumps(filter_dict)}\n" - f"#PRIORS=Period={round_to_2(self.p_dict['pPer'], self.p_dict['pPerUnc'])} +/- {round_to_2(self.p_dict['pPerUnc'])}" - f",Rp/R*={round_to_2(self.p_dict['rprs'], self.p_dict['rprsUnc'])} +/- {round_to_2(self.p_dict['rprsUnc'])}" - f",a/R*={round_to_2(self.p_dict['aRs'], self.p_dict['aRsUnc'])} +/- {round_to_2(self.p_dict['aRsUnc'])}" - f",inc={round_to_2(self.p_dict['inc'], self.p_dict['incUnc'])} +/- {round_to_2(self.p_dict['incUnc'])}" + f"#PRIORS=Period={prior_period_text}" + f",Rp/R*={prior_rprs_text}" + f",a/R*={prior_ars_text}" + f",inc={prior_inc_text}" f",ecc={round_to_2(self.p_dict['ecc'])}" - f",u0={round_to_2(ld0[0], ld0[1])} +/- {round_to_2(ld0[1])}" - f",u1={round_to_2(ld1[0], ld1[1])} +/- {round_to_2(ld1[1])}" - f",u2={round_to_2(ld2[0], ld2[1])} +/- {round_to_2(ld2[1])}" - f",u3={round_to_2(ld3[0], ld3[1])} +/- {round_to_2(ld3[1])}\n" + f",u0={prior_ld_texts[0]}" + f",u1={prior_ld_texts[1]}" + f",u2={prior_ld_texts[2]}" + f",u3={prior_ld_texts[3]}\n" f"#PRIORS-XC={dumps(priors_dict)}\n" # code yields - f"#RESULTS=Tc={round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- {round_to_2(self.fit.errors['tmid'])}" - f",Rp/R*={round_to_2(self.fit.parameters['rprs'], rprs_report_error)} +/- {round_to_2(rprs_report_error)}" - f",inc={round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- {round_to_2(self.fit.errors['inc'])}" + f"#RESULTS=Tc={result_tmid_text}" + f",Rp/R*={result_rprs_text}" + f",inc={result_inc_text}" f",{aavso_airmass_terms[0][0]}={aavso_airmass_terms[0][1]} +/- {aavso_airmass_terms[0][2]}" f",{aavso_airmass_terms[1][0]}={aavso_airmass_terms[1][1]} +/- {aavso_airmass_terms[1][2]}\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields @@ -3164,44 +3286,18 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): if not np.isfinite(rprs_report_error) or rprs_report_error < 0: rprs_report_error = finite_float(planet_dict.get('rprsUnc')) priors = { - 'Period': { - 'value': str(round_to_2(planet_dict['pPer'], planet_dict['pPerUnc'])), - 'uncertainty': str(round_to_2(planet_dict['pPerUnc'])) if planet_dict['pPerUnc'] else planet_dict['pPerUnc'], - 'units': "days" - }, - 'Rp/R*': { - 'value': str(round_to_2(planet_dict['rprs'], planet_dict['rprsUnc'])), - 'uncertainty': str(round_to_2(planet_dict['rprsUnc'])) if planet_dict['rprsUnc'] else planet_dict['rprsUnc'], - }, - 'a/R*': { - 'value': str(round_to_2(planet_dict['aRs'], planet_dict['aRsUnc'])), - 'uncertainty': str(round_to_2(planet_dict['aRsUnc'])) if planet_dict['aRsUnc'] else planet_dict['aRsUnc'], - }, - 'inc': { - 'value': str(round_to_2(planet_dict['inc'], planet_dict['incUnc'])), - 'uncertainty': str(round_to_2(planet_dict['incUnc'])) if planet_dict['incUnc'] else planet_dict['incUnc'], - 'units': "degrees" - }, + 'Period': aavso_result_entry(planet_dict['pPer'], planet_dict['pPerUnc'], units="days"), + 'Rp/R*': aavso_result_entry(planet_dict['rprs'], planet_dict['rprsUnc']), + 'a/R*': aavso_result_entry(planet_dict['aRs'], planet_dict['aRsUnc']), + 'inc': aavso_result_entry(planet_dict['inc'], planet_dict['incUnc'], units="degrees"), 'ecc': { 'value': str(round_to_2(planet_dict['ecc'])), 'uncertainty': None, }, - 'u0': { - 'value': str(round_to_2(ld0[0], ld0[1])), - 'uncertainty': str(round_to_2(ld0[1])) - }, - 'u1': { - 'value': str(round_to_2(ld1[0], ld1[1])), - 'uncertainty': str(round_to_2(ld1[1])) - }, - 'u2': { - 'value': str(round_to_2(ld2[0], ld2[1])), - 'uncertainty': str(round_to_2(ld2[1])) - }, - 'u3': { - 'value': str(round_to_2(ld3[0], ld3[1])), - 'uncertainty': str(round_to_2(ld3[1])) - } + 'u0': aavso_result_entry(*ld0), + 'u1': aavso_result_entry(*ld1), + 'u2': aavso_result_entry(*ld2), + 'u3': aavso_result_entry(*ld3), } filter_type = { @@ -3220,28 +3316,16 @@ def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): } results = { - 'Tc': { - 'value': str(round_to_2(fit.parameters['tmid'], fit.errors['tmid'])), - 'uncertainty': str(round_to_2(fit.errors['tmid'])), - 'units': "BJD_TDB" - }, - 'Rp/R*': { - 'value': str(round_to_2(fit.parameters['rprs'], rprs_report_error)), - 'uncertainty': str(round_to_2(rprs_report_error)) - }, - 'inc': { - 'value': str(round_to_2(fit.parameters['inc'], fit.errors['inc'])), - 'uncertainty': str(round_to_2(fit.errors['inc'])), - }, + 'Tc': aavso_result_entry( + fit.parameters['tmid'], fit.errors['tmid'], units="BJD_TDB" + ), + 'Rp/R*': aavso_result_entry(fit.parameters['rprs'], rprs_report_error), + 'inc': aavso_result_entry(fit.parameters['inc'], fit.errors['inc']), 'Am2': { 'value': aavso_airmass_terms[1][1], 'uncertainty': aavso_airmass_terms[1][2] }, - 'Duration': { - 'value': str(round_to_2(mean(durs))), - 'uncertainty': str(round_to_2(std(durs))), - 'units': "days" - } + 'Duration': aavso_result_entry(mean(durs), std(durs), units="days"), } results[aavso_airmass_terms[0][0]] = { diff --git a/exotic/plots.py b/exotic/plots.py index 96ae9293..eaca0f6a 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -10,6 +10,7 @@ try: from utils import ( filename_date_token, + format_value_with_uncertainty, is_usable_apparent_magnitude, magnitude_text, normalized_magnitude_error, @@ -18,6 +19,7 @@ except ImportError: from .utils import ( filename_date_token, + format_value_with_uncertainty, is_usable_apparent_magnitude, magnitude_text, normalized_magnitude_error, @@ -1231,15 +1233,6 @@ def _plot_positive_error(value): return value -def _decimal_places_for_two_sigfig_error(error): - error = _plot_positive_error(error) - if not np.isfinite(error) or error == 0: - return None - - exponent = int(np.floor(np.log10(abs(error)))) - return max(0, 1 - exponent) - - def _format_parameter_value(value, error=None, unit="", split_error=False): value = _plot_scalar(value) if not np.isfinite(value): @@ -1251,14 +1244,10 @@ def _format_parameter_value(value, error=None, unit="", split_error=False): error = _plot_positive_error(error) if np.isfinite(error): - decimal_places = _decimal_places_for_two_sigfig_error(error) - if decimal_places is None: - decimal_places = 0 - value_text = f"{value:.{decimal_places}f}" - error_text = f"{error:.{decimal_places}f}" + formatted = format_value_with_uncertainty(value, error) if split_error: - return f"{value_text}\n+/- {error_text}{suffix}" - return f"{value_text} +/- {error_text}{suffix}" + formatted = formatted.replace(" +/- ", "\n+/- ", 1) + return f"{formatted}{suffix}" return f"{value:.6f}".rstrip('0').rstrip('.') + suffix diff --git a/exotic/utils.py b/exotic/utils.py index 1d7e3366..ffad4da5 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -372,6 +372,68 @@ def round_to_2(*args): return round(x, roundval) +def _two_significant_figure_decimal_places(uncertainty): + """Return the decimal place needed to show an uncertainty with two sig figs.""" + + uncertainty = float(uncertainty) + if not isfinite(uncertainty) or uncertainty < 0: + raise ValueError("uncertainty must be a finite, non-negative number") + if uncertainty == 0: + return 2 + + exponent = int(floor(log10(abs(uncertainty)))) + decimal_places = 1 - exponent + + # A carry can change the exponent (for example, 0.00999 -> 0.010). + rounded_uncertainty = round(uncertainty, decimal_places) + if rounded_uncertainty: + rounded_exponent = int(floor(log10(abs(rounded_uncertainty)))) + decimal_places = 1 - rounded_exponent + return decimal_places + + +def _format_at_decimal_place(value, decimal_places): + """Format a number at a decimal place, including insignificant zeroes.""" + + value = float(value) + if not isfinite(value): + raise ValueError("value must be a finite number") + if decimal_places >= 0: + return f"{value:.{decimal_places}f}" + return f"{round(value, decimal_places):.0f}" + + +def format_value_and_uncertainty(value, uncertainty): + """Return value/error text with a two-significant-figure uncertainty. + + Both strings end at the same decimal place. Unlike ``round_to_2``, this is + a reporting helper: it deliberately retains trailing zeroes which carry + precision information. + """ + + decimal_places = _two_significant_figure_decimal_places(uncertainty) + return ( + _format_at_decimal_place(value, decimal_places), + format_uncertainty(uncertainty), + ) + + +def format_uncertainty(uncertainty): + """Format an uncertainty with exactly two significant figures.""" + + decimal_places = _two_significant_figure_decimal_places(uncertainty) + if decimal_places < 0: + return f"{float(uncertainty):.1e}" + return _format_at_decimal_place(uncertainty, decimal_places) + + +def format_value_with_uncertainty(value, uncertainty): + """Return ``value +/- uncertainty`` using matched two-sig-fig precision.""" + + value_text, uncertainty_text = format_value_and_uncertainty(value, uncertainty) + return f"{value_text} +/- {uncertainty_text}" + + # Credit: Kalee Tock def get_val(hdr, ks): """ diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py index 820cfbf9..53966fe9 100644 --- a/tests/test_elca_baseline.py +++ b/tests/test_elca_baseline.py @@ -1963,7 +1963,7 @@ def test_triangle_payload_titles_follow_weighted_posterior_display_estimate(monk payload = fit._get_triangle_plot_payload() - assert payload["titles"][0] == "0.119 +/- 0.0089" + assert payload["titles"][0] == "0.1190 +/- 0.0089" assert payload["truths"][0] == pytest.approx(0.1186, abs=5e-4) np.testing.assert_allclose(payload["display_weights"], weights) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index c5adef98..5d72facb 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -3841,6 +3841,24 @@ def fake_run_nested( assert np.allclose(refit_flux, final_call["flux"]) assert fit.oot_baseline_parameter_fit_applied is False assert "already flattened" in fit.oot_baseline_parameter_fit_note + assert np.isfinite(fit.oot_baseline_parameter_fit_a0) + assert fit.oot_baseline_parameter_fit_a0_error > 0 + assert np.isfinite(fit.oot_baseline_parameter_fit_a2) + assert fit.oot_baseline_parameter_fit_a2_error > 0 + assert fit.pre_detrending_baseline_source.startswith("out-of-transit airmass/baseline") + assert fit.pre_detrending_baseline_scale_parameter == "a0" + assert fit.pre_detrending_baseline_scale_value == pytest.approx( + fit.oot_baseline_parameter_fit_a0 + ) + assert fit.pre_detrending_baseline_scale_error == pytest.approx( + fit.oot_baseline_parameter_fit_a0_error + ) + assert fit.pre_detrending_baseline_a2_value == pytest.approx( + fit.oot_baseline_parameter_fit_a2 + ) + assert fit.pre_detrending_baseline_a2_error == pytest.approx( + fit.oot_baseline_parameter_fit_a2_error + ) def test_fit_final_lightcurve_uses_oot_baseline_parameter_refit_when_linear_detrend_skips(monkeypatch): diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py index c8cc0bb6..a298aae6 100644 --- a/tests/test_exotic_rprs_retry.py +++ b/tests/test_exotic_rprs_retry.py @@ -1474,6 +1474,12 @@ def fake_run_nested( assert np.allclose(fit_flux, captured["flux"]) assert returned.oot_baseline_parameter_fit_applied is False assert "instead of reusing" in returned.oot_baseline_parameter_fit_note + assert returned.pre_detrending_baseline_source.startswith("selected fast UltraNest fit") + assert returned.pre_detrending_baseline_scale_parameter == "a1" + assert returned.pre_detrending_baseline_scale_value == pytest.approx(1.03) + assert returned.pre_detrending_baseline_scale_error == pytest.approx(0.02) + assert returned.pre_detrending_baseline_a2_value == pytest.approx(0.12) + assert returned.pre_detrending_baseline_a2_error == pytest.approx(0.03) def test_rprs_posterior_retry_walks_bounds_until_retry_cap(monkeypatch): diff --git a/tests/test_output_files.py b/tests/test_output_files.py index e13ba368..ffbe7f97 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -1111,7 +1111,7 @@ def test_transit_outputs_use_rprs_fallback_uncertainty_when_model_error_missing( (0.4, 0.04), ) - assert results["Rp/R*"]["uncertainty"] == "0.005" + assert results["Rp/R*"]["uncertainty"] == "0.0050" def test_final_planetary_params_reports_transit_comparison_catalog_reference(tmp_path): @@ -1214,13 +1214,111 @@ def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclinati assert keys[inclination_index + 1] == "Ratio of Distance to Stellar Radius (a/Rs)" assert keys[inclination_index + 2] == "Impact Parameter (b)" - assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.0 +/- 0.4" + assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.00 +/- 0.40" expected_b, expected_b_error = fit_impact_parameter_value_error(fit) assert expected_b == pytest.approx(12.0 * np.cos(np.deg2rad(88.5))) assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" +def test_final_planetary_params_matches_values_to_two_sigfig_uncertainties(tmp_path): + fit = DummyFit() + fit.errors["a1"] = 0.00023 + fit.errors["a2"] = 0.0031 + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "filter": "V", + "filter_desc": "Johnson V", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.063, 0.083]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Flux normalization (a1)"] == "1.00000 +/- 0.00023" + assert final_params["Airmass coefficient 2 (a2)"] == "0.0000 +/- 0.0031" + assert final_params["Transit Duration (day)"] == "0.073 +/- 0.010" + + +def test_detrended_fixed_baseline_does_not_report_inherited_errors_as_fitted(tmp_path): + fit = DummyFit() + fit.errors["a1"] = 0.00023 + fit.errors["a2"] = 0.0031 + fit.oot_baseline_detrending_applied = True + fit.pre_detrending_baseline_source = "test out-of-transit baseline fit" + fit.pre_detrending_baseline_scale_parameter = "a1" + fit.pre_detrending_baseline_scale_value = 1.004321 + fit.pre_detrending_baseline_scale_error = 0.00023 + fit.pre_detrending_baseline_a2_value = -0.01234 + fit.pre_detrending_baseline_a2_error = 0.0031 + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "filter": "V", + "filter_desc": "Johnson V", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.063, 0.083]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + _, _, results = aavso_dicts( + { + **p_dict, + "pPer": 2.15, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.001, + "aRs": 12.0, + "aRsUnc": 0.4, + "inc": 88.5, + "incUnc": 0.2, + "ecc": 0.0, + }, + fit, + i_dict, + [0.063, 0.083], + (0.1, 0.01), + (0.2, 0.02), + (0.3, 0.03), + (0.4, 0.04), + ) + + assert final_params["Flux normalization (a1)"] == ( + "1.0 (fixed after out-of-transit baseline detrending)" + ) + assert final_params["Airmass coefficient 2 (a2)"] == ( + "0.0 (fixed after out-of-transit baseline detrending)" + ) + assert final_params["Pre-detrending baseline source"] == "test out-of-transit baseline fit" + assert final_params["Pre-detrending airmass coefficient 1 (a1)"] == ( + "1.00432 +/- 0.00023" + ) + assert final_params["Pre-detrending airmass coefficient 2 (a2)"] == ( + "-0.0123 +/- 0.0031" + ) + assert results["Am1"] == {"value": "1.0", "uncertainty": "0"} + assert results["Am2"] == {"value": "0.0", "uncertainty": "0"} + + def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertainties(tmp_path): fit = DummyFit() (tmp_path / "working_artifacts").mkdir() @@ -1242,15 +1340,15 @@ def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertaintie output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] - assert final_params["Mid-Transit Time (Tmid)"].endswith("+/- 0.0001 BJD_TDB") - assert final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] == "0.1234 +/- 0.001" + assert final_params["Mid-Transit Time (Tmid)"].endswith("+/- 0.00010 BJD_TDB") + assert final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] == "0.1234 +/- 0.0010" assert "Transit depth (Rp/Rs)^2" not in final_params assert AREA_DEPTH_LABEL in final_params assert OBSERVABLE_DEPTH_LABEL in final_params assert PRIOR_OBSERVABLE_DEPTH_LABEL in final_params assert OBSERVABLE_DEPTH_DELTA_LABEL in final_params - assert final_params["Orbital Inclination (inc)"] == "88.5 +/- 0.2 " - assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.0 +/- 0.4" + assert final_params["Orbital Inclination (inc)"] == "88.50 +/- 0.20 " + assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.00 +/- 0.40" assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" @@ -1344,7 +1442,7 @@ def test_final_planetary_params_reports_model_and_red_noise_uncertainties(tmp_pa final_params["Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty"] ) assert final_params["Ratio of Planet to Stellar Radius (Rp/R*) model-fit uncertainty"] == ( - "0.1 +/- 0.002" + "0.1000 +/- 0.0020" ) assert "Ratio of Planet to Stellar Radius (Rp/R*) data-fit red-noise uncertainty" in final_params assert "Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty" in final_params diff --git a/tests/test_utils.py b/tests/test_utils.py index 8427b350..4f748032 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -410,6 +410,23 @@ def test_small_numbers(self): assert 0.0002 == result +class TestFormatValueAndUncertainty: + def test_preserves_two_significant_figures_and_matches_value_precision(self): + assert format_value_and_uncertainty(0.073, 0.01) == ("0.073", "0.010") + assert format_value_and_uncertainty(1.0, 0.00023) == ("1.00000", "0.00023") + assert format_value_and_uncertainty(0.0, 0.0031) == ("0.0000", "0.0031") + + def test_formats_uncertainties_above_one_to_two_significant_figures(self): + assert format_value_and_uncertainty(89.3511, 2.16) == ("89.4", "2.2") + assert format_value_and_uncertainty(1234, 100) == ("1230", "1.0e+02") + + def test_recomputes_precision_when_rounding_crosses_a_decade(self): + assert format_value_and_uncertainty(0.0732, 0.00999) == ("0.073", "0.010") + + def test_full_report_text_uses_the_same_precision(self): + assert format_value_with_uncertainty(12.0, 0.4) == "12.00 +/- 0.40" + + class TestGetVal: """tests the get_val() function From fcd35c8c95b0a0227d89338fc06dcc020bf84ed4 Mon Sep 17 00:00:00 2001 From: Michael Fitzgerald Date: Mon, 17 Aug 2026 09:49:03 +1000 Subject: [PATCH 116/116] Making sure raw photometry is captured. --- README.md | 2 +- docs/README.md | 2 +- exotic/exotic.py | 9 +- exotic/output_files.py | 459 +++++++++++++++++++++++----- tests/test_exotic_proper_motion.py | 2 + tests/test_nextastro_variability.py | 2 +- tests/test_output_files.py | 216 ++++++++++++- 7 files changed, 610 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 9b71595a..f0cdd235 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. -Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. Each reduction writes its run log from startup through shutdown to a unique `Diagnostics/EXOTIC_RunLog__pid.log`, so separate or midnight-spanning runs do not overwrite or split one another. +Flux-bearing result files retain both magnitude representations. Final-lightcurve and differential-magnitude CSV rows explicitly include the raw, uncorrected differential magnitude and uncertainty alongside the corrected differential magnitude and uncertainty, plus apparent magnitude and uncertainty where applicable (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row containing both raw and corrected differential values and the applied correction factor. When weighted linear out-of-transit baseline detrending is applied, `#OUT_OF_TRANSIT_BASELINE-XC` records the formula, BJD_TDB reference time, intercept, and slope; the standard `DIFF` and `ERR` rows are restored to their pre-detrending values and `DETREND_2` carries the correction function, making the operation reversible from the AAVSO file. AID rows retain the standard Extended format and store raw `DIFFMAG` and `DIFFERR` values while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. Each reduction writes its run log from startup through shutdown to a unique `Diagnostics/EXOTIC_RunLog__pid.log`, so separate or midnight-spanning runs do not overwrite or split one another. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. diff --git a/docs/README.md b/docs/README.md index 0f444fcc..64bcbeba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -265,7 +265,7 @@ Comparison stars may be supplied in `user_info` using either `"Comparison Star(s Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. -Flux-bearing result files retain both magnitude representations. Final-lightcurve CSV rows include differential magnitude and uncertainty plus apparent magnitude and uncertainty (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row. AID rows retain the standard Extended format and store `DIFFMAG` and `DIFFERR` in the `NOTES` field while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. Each reduction writes its run log from startup through shutdown to a unique `Diagnostics/EXOTIC_RunLog__pid.log`, so separate or midnight-spanning runs do not overwrite or split one another. +Flux-bearing result files retain both magnitude representations. Final-lightcurve and differential-magnitude CSV rows explicitly include the raw, uncorrected differential magnitude and uncertainty alongside the corrected differential magnitude and uncertainty, plus apparent magnitude and uncertainty where applicable (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row containing both raw and corrected differential values and the applied correction factor. When weighted linear out-of-transit baseline detrending is applied, `#OUT_OF_TRANSIT_BASELINE-XC` records the formula, BJD_TDB reference time, intercept, and slope; the standard `DIFF` and `ERR` rows are restored to their pre-detrending values and `DETREND_2` carries the correction function, making the operation reversible from the AAVSO file. AID rows retain the standard Extended format and store raw `DIFFMAG` and `DIFFERR` values while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. Each reduction writes its run log from startup through shutdown to a unique `Diagnostics/EXOTIC_RunLog__pid.log`, so separate or midnight-spanning runs do not overwrite or split one another. For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. diff --git a/exotic/exotic.py b/exotic/exotic.py index 2e821592..71d17444 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -538,6 +538,7 @@ def annotate_out_of_transit_baseline_detrending( note=None, slope=None, intercept=None, + reference_time_bjd_tdb=None, pre_points=0, post_points=0, ): @@ -548,6 +549,7 @@ def annotate_out_of_transit_baseline_detrending( fit.oot_baseline_detrending_note = note fit.oot_baseline_slope = slope fit.oot_baseline_intercept = intercept + fit.oot_baseline_reference_time_bjd_tdb = reference_time_bjd_tdb fit.oot_baseline_pre_points = int(pre_points) if pre_points is not None else 0 fit.oot_baseline_post_points = int(post_points) if post_points is not None else 0 @@ -5703,6 +5705,7 @@ def aligned_selected_array(key, dtype=float): note=detrend_result.get('note'), slope=detrend_result.get('slope'), intercept=detrend_result.get('intercept'), + reference_time_bjd_tdb=detrend_result.get('reference_time_bjd_tdb'), pre_points=detrend_result.get('pre_points', 0), post_points=detrend_result.get('post_points', 0), ) @@ -11808,6 +11811,7 @@ def detrend_flux_on_out_of_transit_baseline( 'baseline': baseline, 'slope': float(slope), 'intercept': float(intercept), + 'reference_time_bjd_tdb': float(mid_transit), 'pre_points': pre_points, 'post_points': post_points, 'ingress_time': ingress_time, @@ -12555,6 +12559,7 @@ def run_oot_baseline_parameter_refit_if_needed(current_fit): note=detrend_result['note'], slope=detrend_result['slope'], intercept=detrend_result['intercept'], + reference_time_bjd_tdb=detrend_result['reference_time_bjd_tdb'], pre_points=detrend_result['pre_points'], post_points=detrend_result['post_points'], ) @@ -28982,8 +28987,8 @@ def save_stellar_variability_magnitude_csv(vsp_params, save, target_name, observ 'Airmass', 'Apparent Magnitude', 'Apparent Magnitude Error', - 'Differential Magnitude', - 'Differential Magnitude Error', + 'Raw Differential Magnitude', + 'Raw Differential Magnitude Error', 'Filter', 'Comparison', ]) diff --git a/exotic/output_files.py b/exotic/output_files.py index f6a0943f..439d5ab4 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -177,11 +177,78 @@ def aavso_airmass_results(fit): def aavso_detrend_model(fit): + oot_baseline_model = out_of_transit_baseline_model(fit) + if oot_baseline_model is not None: + return oot_baseline_model if getattr(fit, 'airmass_fit_skipped', False): return np.ones(len(fit.time), dtype=float) return np.asarray(fit.airmass_model, dtype=float) +def out_of_transit_baseline_model(fit): + """Rebuild the linear baseline divided out before the final transit fit.""" + if not baseline_fixed_after_detrending(fit): + return None + + times = np.asarray(getattr(fit, 'time', []), dtype=float).reshape(-1) + slope = finite_float(getattr(fit, 'oot_baseline_slope', None)) + intercept = finite_float(getattr(fit, 'oot_baseline_intercept', None)) + reference_time = finite_float( + getattr(fit, 'oot_baseline_reference_time_bjd_tdb', None) + ) + if ( + times.size == 0 + or not np.isfinite(slope) + or not np.isfinite(intercept) + or not np.isfinite(reference_time) + ): + return None + + baseline = intercept + slope * (times - reference_time) + if baseline.shape != times.shape or not np.all(np.isfinite(baseline)) or np.any(baseline <= 0): + return None + return baseline + + +def out_of_transit_baseline_detrending_metadata(fit): + """Return the complete reversible linear-baseline contract for AAVSO output.""" + applied = baseline_fixed_after_detrending(fit) + metadata = { + 'applied': applied, + 'note': getattr(fit, 'oot_baseline_detrending_note', None), + } + if not applied: + return metadata + + metadata.update({ + 'model': 'baseline(t) = intercept + slope_per_day * (BJD_TDB - reference_time_bjd_tdb)', + 'forward_correction': 'detrended_flux = raw_flux / baseline(t)', + 'inverse_correction': 'raw_flux = detrended_flux * baseline(t)', + 'reference_time_bjd_tdb': finite_float( + getattr(fit, 'oot_baseline_reference_time_bjd_tdb', None) + ), + 'intercept': finite_float(getattr(fit, 'oot_baseline_intercept', None)), + 'slope_per_day': finite_float(getattr(fit, 'oot_baseline_slope', None)), + 'pre_ingress_point_count': int(getattr(fit, 'oot_baseline_pre_points', 0) or 0), + 'post_egress_point_count': int(getattr(fit, 'oot_baseline_post_points', 0) or 0), + 'serialized_model_available': out_of_transit_baseline_model(fit) is not None, + }) + return metadata + + +def aavso_undetrended_flux_series(fit, detrend_model): + """Return pre-correction flux/error arrays matching the exported correction model.""" + data = np.asarray(getattr(fit, 'data', []), dtype=float).reshape(-1) + data_error = np.asarray(getattr(fit, 'dataerr', []), dtype=float).reshape(-1) + detrend_model = np.asarray(detrend_model, dtype=float).reshape(-1) + if not (data.shape == data_error.shape == detrend_model.shape): + return data, data_error + oot_baseline_model = out_of_transit_baseline_model(fit) + if oot_baseline_model is not None and oot_baseline_model.shape == data.shape: + return data * oot_baseline_model, data_error * oot_baseline_model + return data, data_error + + def baseline_fixed_after_detrending(fit): """Return whether the reported neutral baseline was fixed after detrending.""" @@ -328,6 +395,54 @@ def differential_magnitude_from_vsp_param(vsp_param): return apparent_mag - comparison_mag, differential_err +def differential_magnitude_correction_model(fit, shape): + """Return the relative correction applied to raw target/reference ratios.""" + shape = tuple(shape) + correction_type = 'none' + correction_model = out_of_transit_baseline_model(fit) + if correction_model is not None: + correction_type = 'out_of_transit_linear_baseline' + elif baseline_fixed_after_detrending(fit): + correction_type = 'unavailable' + elif not getattr(fit, 'airmass_fit_skipped', False): + candidate = getattr(fit, 'airmass_model', None) + if candidate is not None: + candidate = np.asarray(candidate, dtype=float).reshape(-1) + if candidate.shape == shape: + correction_model = candidate + correction_type = 'airmass' + + if correction_model is None or np.asarray(correction_model).shape != shape: + correction_model = np.ones(shape, dtype=float) + if correction_type != 'unavailable': + correction_type = 'none' + else: + correction_model = np.asarray(correction_model, dtype=float).reshape(-1) + + valid = np.isfinite(correction_model) & (correction_model > 0) + reference = float(np.nanmedian(correction_model[valid])) if np.any(valid) else 1.0 + if not np.isfinite(reference) or reference <= 0: + reference = 1.0 + relative_model = np.divide( + correction_model, + reference, + out=np.full(shape, np.nan, dtype=float), + where=valid, + ) + correction_applied = bool( + correction_type != 'none' + and np.any(valid) + and not np.allclose(relative_model[valid], 1.0, rtol=0.0, atol=1.0e-12) + ) + return { + 'model': correction_model, + 'relative_model': relative_model, + 'reference': reference, + 'type': correction_type, + 'applied': correction_applied, + } + + def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, apply_airmass_correction=None): """Return target-minus-reference instrumental magnitudes. @@ -349,6 +464,12 @@ def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, if fit_data.size == 0 or fit_times.shape != fit_data.shape: return None + if apply_airmass_correction is None: + apply_airmass_correction = not bool( + getattr(fit, 'stellar_variability_only', False) + ) + apply_airmass_correction = bool(apply_airmass_correction) + target_flux = np.asarray( getattr( fit, @@ -386,15 +507,19 @@ def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, target_flux.shape == fit_data.shape and reference_flux.shape == fit_data.shape ) + correction = differential_magnitude_correction_model(fit, fit_data.shape) + relative_correction_model = correction['relative_model'] if not has_raw_photometry: - target_flux = np.asarray(getattr(fit, 'detrended', fit_data), dtype=float).reshape(-1) - if target_flux.shape != fit_data.shape: - target_flux = fit_data.copy() + # Ordinary fits retain their uncorrected relative flux in ``fit.data``. + # A linear out-of-transit pass instead leaves corrected data on the + # final fit, so restore its raw input with the retained baseline model. + target_flux = fit_data.copy() + target_error = np.asarray(getattr(fit, 'dataerr', []), dtype=float).reshape(-1) + if baseline_fixed_after_detrending(fit) and correction['type'] == 'out_of_transit_linear_baseline': + target_flux = target_flux * relative_correction_model + if target_error.shape == fit_data.shape: + target_error = target_error * relative_correction_model reference_flux = np.ones(fit_data.shape, dtype=float) - target_error = np.asarray( - getattr(fit, 'detrendederr', getattr(fit, 'dataerr', [])), - dtype=float, - ).reshape(-1) reference_error = np.zeros(fit_data.shape, dtype=float) if target_error.shape != fit_data.shape: @@ -402,39 +527,13 @@ def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, if reference_error.shape != fit_data.shape: reference_error = np.full(fit_data.shape, np.nan, dtype=float) - if apply_airmass_correction is None: - apply_airmass_correction = not bool( - getattr(fit, 'stellar_variability_only', False) - ) - # ``fit.detrended`` is already corrected. Only divide an explicitly - # retained raw target/reference ratio by the fitted airmass model. - apply_airmass_correction = bool(apply_airmass_correction and has_raw_photometry) - relative_airmass_model = np.ones(fit_data.shape, dtype=float) - if apply_airmass_correction: - airmass_model = np.asarray( - getattr(fit, 'airmass_model', np.ones(fit_data.shape)), - dtype=float, - ).reshape(-1) - if airmass_model.shape != fit_data.shape: - airmass_model = np.ones(fit_data.shape, dtype=float) - valid_airmass_model = np.isfinite(airmass_model) & (airmass_model > 0) - airmass_reference = ( - float(np.nanmedian(airmass_model[valid_airmass_model])) - if np.any(valid_airmass_model) - else 1.0 - ) - if not np.isfinite(airmass_reference) or airmass_reference <= 0: - airmass_reference = 1.0 - relative_airmass_model = np.divide( - airmass_model, - airmass_reference, - out=np.full(fit_data.shape, np.nan, dtype=float), - where=valid_airmass_model, - ) - with np.errstate(divide='ignore', invalid='ignore'): raw_ratio = np.divide(target_flux, reference_flux) - corrected_ratio = np.divide(raw_ratio, relative_airmass_model) + corrected_ratio = ( + np.divide(raw_ratio, relative_correction_model) + if apply_airmass_correction + else raw_ratio + ) differential_magnitude = -2.5 * np.log10(corrected_ratio) magnitude_factor = 2.5 / np.log(10.0) explicit_error = magnitude_factor * np.sqrt( @@ -483,7 +582,30 @@ def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, 'magnitude': differential_magnitude[keep], 'magnitude_error': differential_error[keep], 'source_mask': keep, - 'airmass_corrected': apply_airmass_correction, + 'airmass_corrected': bool( + apply_airmass_correction + and correction['type'] == 'airmass' + and correction['applied'] + ), + 'correction_applied': bool(apply_airmass_correction and correction['applied']), + 'correction_type': correction['type'] if apply_airmass_correction else 'none', + 'correction_factor': ( + relative_correction_model[keep] + if apply_airmass_correction + else np.ones(np.count_nonzero(keep), dtype=float) + ), + 'raw_measurement_available': bool( + has_raw_photometry + or not ( + correction['type'] == 'unavailable' + or ( + getattr(fit, 'airmass_fit_skipped', False) + and 'input AAVSO file already reports' in str( + getattr(fit, 'airmass_correction_note', '') + ) + ) + ) + ), 'has_raw_photometry': has_raw_photometry, } @@ -498,10 +620,18 @@ def magnitude_series_from_fit(fit, out_of_transit_only=False, result = { 'differential_magnitude': np.full(fit_data.shape, np.nan, dtype=float), 'differential_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'raw_differential_magnitude': np.full(fit_data.shape, np.nan, dtype=float), + 'raw_differential_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'corrected_differential_magnitude': np.full(fit_data.shape, np.nan, dtype=float), + 'corrected_differential_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'differential_magnitude_correction_factor': np.full(fit_data.shape, np.nan, dtype=float), 'apparent_magnitude': np.full(fit_data.shape, np.nan, dtype=float), 'apparent_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), 'band': None, 'airmass_corrected': False, + 'correction_applied': False, + 'correction_type': 'none', + 'raw_measurement_available': False, 'has_raw_photometry': False, 'apparent_calibrated': False, } @@ -519,8 +649,43 @@ def magnitude_series_from_fit(fit, out_of_transit_only=False, result['differential_magnitude'][source_mask] = series['magnitude'] result['differential_magnitude_error'][source_mask] = series['magnitude_error'] result['airmass_corrected'] = bool(series['airmass_corrected']) + result['correction_applied'] = bool(series['correction_applied']) + result['correction_type'] = series['correction_type'] + result['raw_measurement_available'] = bool(series['raw_measurement_available']) result['has_raw_photometry'] = bool(series['has_raw_photometry']) + raw_series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=False, + ) + if raw_series is not None and raw_series.get('raw_measurement_available', False): + raw_mask = np.asarray(raw_series['source_mask'], dtype=bool) + if raw_mask.shape == fit_data.shape: + result['raw_differential_magnitude'][raw_mask] = raw_series['magnitude'] + result['raw_differential_magnitude_error'][raw_mask] = raw_series['magnitude_error'] + + correct_variability = ( + not bool(getattr(fit, 'stellar_variability_only', False)) + if apply_airmass_correction is None + else bool(apply_airmass_correction) + ) + corrected_series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=correct_variability, + ) + if corrected_series is not None: + corrected_mask = np.asarray(corrected_series['source_mask'], dtype=bool) + if corrected_mask.shape == fit_data.shape: + result['corrected_differential_magnitude'][corrected_mask] = corrected_series['magnitude'] + result['corrected_differential_magnitude_error'][corrected_mask] = ( + corrected_series['magnitude_error'] + ) + result['differential_magnitude_correction_factor'][corrected_mask] = ( + corrected_series['correction_factor'] + ) + calibration = apparent_magnitude_calibration_from_vsp_params( getattr(fit, 'stellar_variability_params', None) ) @@ -626,12 +791,27 @@ def write_differential_magnitude_csv(fit, save, target_name, observation_date=No observed_filter=None, out_of_transit_only=False, apply_airmass_correction=None, filename_prefix='DifferentialMagnitude'): - series = differential_magnitude_series_from_fit( + series = magnitude_series_from_fit( fit, out_of_transit_only=out_of_transit_only, apply_airmass_correction=apply_airmass_correction, ) - if series is None: + times = np.asarray( + getattr(fit, 'time', getattr(fit, 'jd_times', [])), + dtype=float, + ).reshape(-1) + airmass = np.asarray( + getattr(fit, 'airmass', np.full(times.shape, np.nan)), + dtype=float, + ).reshape(-1) + if airmass.shape != times.shape: + airmass = np.full(times.shape, np.nan, dtype=float) + raw_magnitude = np.asarray(series['raw_differential_magnitude'], dtype=float) + corrected_magnitude = np.asarray(series['corrected_differential_magnitude'], dtype=float) + if not ( + raw_magnitude.shape == corrected_magnitude.shape == times.shape + and np.any(np.isfinite(raw_magnitude) | np.isfinite(corrected_magnitude)) + ): return None output_dir = Path(save) @@ -652,25 +832,45 @@ def write_differential_magnitude_csv(fit, save, target_name, observation_date=No '# AIRMASS_CORRECTION=' f"{'YES' if series['airmass_corrected'] else 'NO'}\n" ) + handle.write(f"# DIFFERENTIAL_MAGNITUDE_CORRECTION={series['correction_type']}\n") handle.write( - '# BJD_TDB,Airmass,Differential Magnitude,' - 'Differential Magnitude Uncertainty,Filter,Comparison\n' - ) - for time_value, airmass, magnitude, magnitude_error in zip( - series['time'], - series['airmass'], - series['magnitude'], - series['magnitude_error'], - ): - airmass_text = f"{airmass}" if np.isfinite(airmass) else 'na' - error_text = ( - f"{magnitude_error:.{MAGNITUDE_DECIMAL_PLACES}f}" - if np.isfinite(magnitude_error) - else 'na' + '# BJD_TDB,Airmass,Raw Differential Magnitude,' + 'Raw Differential Magnitude Uncertainty,Corrected Differential Magnitude,' + 'Corrected Differential Magnitude Uncertainty,Correction Factor,Filter,Comparison\n' + ) + for index, time_value in enumerate(times): + if not ( + np.isfinite(raw_magnitude[index]) + or np.isfinite(corrected_magnitude[index]) + ): + continue + airmass_text = f"{airmass[index]}" if np.isfinite(airmass[index]) else 'na' + raw_text = format_magnitude( + raw_magnitude[index], default='na', digits=MAGNITUDE_DECIMAL_PLACES + ) + raw_error_text = format_magnitude_error( + series['raw_differential_magnitude_error'][index], + default='na', + digits=MAGNITUDE_DECIMAL_PLACES, + ) + corrected_text = format_magnitude( + corrected_magnitude[index], default='na', digits=MAGNITUDE_DECIMAL_PLACES + ) + corrected_error_text = format_magnitude_error( + series['corrected_differential_magnitude_error'][index], + default='na', + digits=MAGNITUDE_DECIMAL_PLACES, + ) + correction_factor = finite_float( + series['differential_magnitude_correction_factor'][index] + ) + correction_factor_text = ( + f"{correction_factor:.7f}" if np.isfinite(correction_factor) else 'na' ) handle.write( f"{time_value}, {airmass_text}, " - f"{magnitude:.{MAGNITUDE_DECIMAL_PLACES}f}, {error_text}, " + f"{raw_text}, {raw_error_text}, {corrected_text}, {corrected_error_text}, " + f"{correction_factor_text}, " f"{observed_filter or 'na'}, {comparison}\n" ) return output_path @@ -2351,7 +2551,7 @@ def final_lightcurve(self, phase): f.write("# DIFFERENTIAL_MAGNITUDE_AIRMASS_CORRECTED=NO\n") f.write( "# BJD_TDB,Apparent Magnitude,Apparent Magnitude Uncertainty," - "Differential Magnitude,Differential Magnitude Uncertainty,Band,Airmass\n" + "Raw Differential Magnitude,Raw Differential Magnitude Uncertainty,Band,Airmass\n" ) for vsp_p in vsp_params: time_value = finite_float(vsp_p.get('time')) @@ -2383,6 +2583,31 @@ def final_lightcurve(self, phase): magnitude_series = magnitude_series_from_fit(self.fit) band = magnitude_series['band'] or self.i_dict.get('filter') or 'na' + fit_times = np.asarray(getattr(self.fit, 'time', []), dtype=float).reshape(-1) + detrend_model = np.asarray(aavso_detrend_model(self.fit), dtype=float).reshape(-1) + if detrend_model.shape != fit_times.shape: + detrend_model = np.ones(fit_times.shape, dtype=float) + airmass_values = np.asarray( + getattr(self.fit, 'airmass', np.full(fit_times.shape, np.nan)), + dtype=float, + ).reshape(-1) + if airmass_values.shape != fit_times.shape: + airmass_values = np.full(fit_times.shape, np.nan, dtype=float) + corrected_flux_error = np.asarray( + getattr(self.fit, 'detrendederr', []), + dtype=float, + ).reshape(-1) + if corrected_flux_error.shape != fit_times.shape: + fit_data_error = np.asarray(getattr(self.fit, 'dataerr', []), dtype=float).reshape(-1) + if fit_data_error.shape == fit_times.shape: + corrected_flux_error = np.divide( + fit_data_error, + detrend_model, + out=np.full(fit_times.shape, np.nan, dtype=float), + where=np.isfinite(detrend_model) & (detrend_model > 0), + ) + else: + corrected_flux_error = np.full(fit_times.shape, np.nan, dtype=float) with params_file.open('w') as f: f.write(f"# FINAL TIMESERIES OF {self.p_dict['pName']}\n") @@ -2397,26 +2622,47 @@ def final_lightcurve(self, phase): f"{'YES' if magnitude_series['airmass_corrected'] else 'NO'}\n" ) f.write( - "# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass," - "Differential Magnitude,Differential Magnitude Uncertainty," + "# DIFFERENTIAL_MAGNITUDE_CORRECTION=" + f"{magnitude_series['correction_type']}\n" + ) + f.write( + "# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass,Detrend Correction Function," + "Raw Differential Magnitude,Raw Differential Magnitude Uncertainty," + "Corrected Differential Magnitude,Corrected Differential Magnitude Uncertainty," "Apparent Magnitude,Apparent Magnitude Uncertainty,Band\n" ) - for row_index, (bjd, phase, flux, fluxerr, model, am) in enumerate(zip( + for row_index, (bjd, phase, flux, fluxerr, model, am, correction_value) in enumerate(zip( self.fit.time, phase, self.fit.detrended, - self.fit.dataerr / self.fit.airmass_model, + corrected_flux_error, self.fit.transit, - self.fit.airmass_model)): - row = f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}" - differential_mag_text = format_magnitude( - magnitude_series['differential_magnitude'][row_index], + airmass_values, + detrend_model)): + airmass_text = str(am) if np.isfinite(am) else 'na' + correction_text = str(correction_value) if np.isfinite(correction_value) else 'na' + row = ( + f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, " + f"{airmass_text}, {correction_text}" + ) + raw_differential_mag_text = format_magnitude( + magnitude_series['raw_differential_magnitude'][row_index], default="na", digits=MAGNITUDE_DECIMAL_PLACES, ) - differential_error_text = format_magnitude_error( - magnitude_series['differential_magnitude_error'][row_index], + raw_differential_error_text = format_magnitude_error( + magnitude_series['raw_differential_magnitude_error'][row_index], + default="na", + digits=MAGNITUDE_DECIMAL_PLACES, + ) + corrected_differential_mag_text = format_magnitude( + magnitude_series['corrected_differential_magnitude'][row_index], + default="na", + digits=MAGNITUDE_DECIMAL_PLACES, + ) + corrected_differential_error_text = format_magnitude_error( + magnitude_series['corrected_differential_magnitude_error'][row_index], default="na", digits=MAGNITUDE_DECIMAL_PLACES, ) @@ -2429,7 +2675,8 @@ def final_lightcurve(self, phase): default="na", ) row = ( - f"{row}, {differential_mag_text}, {differential_error_text}, " + f"{row}, {raw_differential_mag_text}, {raw_differential_error_text}, " + f"{corrected_differential_mag_text}, {corrected_differential_error_text}, " f"{apparent_mag_text}, {apparent_error_text}, {band}" ) f.write(f"{row}\n") @@ -2724,6 +2971,19 @@ def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coor oot_baseline_note = getattr(self.fit, 'oot_baseline_detrending_note', None) if oot_baseline_note: params_num["Out-of-transit baseline detrending note"] = str(oot_baseline_note) + oot_baseline_metadata = out_of_transit_baseline_detrending_metadata(self.fit) + if oot_baseline_metadata.get('applied'): + params_num["Out-of-transit baseline detrending model"] = str( + oot_baseline_metadata['model'] + ) + for output_label, metadata_key in ( + ("Out-of-transit baseline reference time (BJD_TDB)", 'reference_time_bjd_tdb'), + ("Out-of-transit baseline intercept", 'intercept'), + ("Out-of-transit baseline slope (per day)", 'slope_per_day'), + ): + value = finite_float(oot_baseline_metadata.get(metadata_key)) + if np.isfinite(value): + params_num[output_label] = str(value) sparse_posterior_note = getattr(self.fit, 'sparse_posterior_live_point_extension_note', None) if sparse_posterior_note: params_num["Sparse posterior live-point extension note"] = str(sparse_posterior_note) @@ -2951,6 +3211,10 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, ld0, ld1, ld2, ld3) aavso_airmass_terms = aavso_airmass_results(self.fit) detrend_model = aavso_detrend_model(self.fit) + undetrended_flux, undetrended_flux_error = aavso_undetrended_flux_series( + self.fit, + detrend_model, + ) qc_metadata = build_aavso_qc_metadata(self.fit) fit_quality_metadata = build_fit_quality_metadata(self.fit) rprs_report_error = fit_rprs_report_error(self.fit) @@ -3051,17 +3315,49 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, f.write(format_aavso_json_header("FRAME_FILTERING-XC", frame_filtering_metadata)) f.write(format_aavso_json_header("ASTROMETRY-XC", astrometry_metadata)) f.write(format_aavso_json_header("BAD_PIXEL-XC", bad_pixel_metadata)) + f.write(format_aavso_json_header( + "OUT_OF_TRANSIT_BASELINE-XC", + out_of_transit_baseline_detrending_metadata(self.fit), + )) + f.write(format_aavso_json_header("DETREND_PARAMETERS-XC", { + 'DETREND_1': 'airmass', + 'DETREND_2': ( + 'out_of_transit_linear_baseline_correction_function' + if baseline_fixed_after_detrending(self.fit) + else 'airmass_correction_function' + ), + 'standard_header_preserved': True, + })) f.write(format_aavso_json_header("MAGNITUDE_FIELDS-XC", { 'apparent_magnitude': 'catalogue-calibrated target magnitude', 'apparent_magnitude_error': 'flux and catalogue calibration uncertainty', - 'differential_magnitude': ( + 'raw_differential_magnitude': ( 'target minus selected comparison reference; ' '-2.5 log10(target_flux/reference_flux)' ), - 'differential_magnitude_error': 'flux-only uncertainty', + 'raw_differential_magnitude_error': 'raw target/reference flux-only uncertainty', + 'corrected_differential_magnitude': ( + 'raw differential magnitude after the declared correction factor is removed' + ), + 'corrected_differential_magnitude_error': ( + 'flux-only uncertainty after correction; correction-model uncertainty excluded' + ), + 'differential_magnitude': ( + 'target minus selected comparison reference after the declared correction; ' + 'backward-compatible alias of corrected_differential_magnitude' + ), + 'differential_magnitude_error': ( + 'backward-compatible alias of corrected_differential_magnitude_error' + ), + 'differential_magnitude_correction_factor': ( + 'relative multiplicative flux correction; corrected_flux = raw_flux / factor' + ), 'per_point_header': 'MAGNITUDE-XC', 'band': magnitude_series['band'] or self.i_dict.get('filter'), 'airmass_corrected': magnitude_series['airmass_corrected'], + 'correction_applied': magnitude_series['correction_applied'], + 'correction_type': magnitude_series['correction_type'], + 'raw_measurement_available': magnitude_series['raw_measurement_available'], 'apparent_calibrated': magnitude_series['apparent_calibrated'], 'comparison_reference': ( getattr(self.fit, 'differential_magnitude_reference_label', None) @@ -3072,10 +3368,29 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, for magnitude_index in range(0, len(self.fit.time)): f.write(format_aavso_json_header("MAGNITUDE-XC", { 'date_bjd_tdb': finite_float(self.fit.time[magnitude_index]), + 'raw_differential_magnitude': rounded_magnitude_value( + magnitude_series['raw_differential_magnitude'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'raw_differential_magnitude_error': rounded_magnitude_error( + magnitude_series['raw_differential_magnitude_error'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'corrected_differential_magnitude': rounded_magnitude_value( + magnitude_series['corrected_differential_magnitude'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'corrected_differential_magnitude_error': rounded_magnitude_error( + magnitude_series['corrected_differential_magnitude_error'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), 'differential_magnitude': rounded_magnitude_value( magnitude_series['differential_magnitude'][magnitude_index], digits=MAGNITUDE_DECIMAL_PLACES, ), + 'differential_magnitude_correction_factor': finite_float( + magnitude_series['differential_magnitude_correction_factor'][magnitude_index] + ), 'differential_magnitude_error': rounded_magnitude_error( magnitude_series['differential_magnitude_error'][magnitude_index], digits=MAGNITUDE_DECIMAL_PLACES, @@ -3107,8 +3422,8 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, # f.write(f"{round(self.fit.time[aavsoC], 8)},{round(self.fit.data[aavsoC] / self.fit.parameters['a1'], 7)}," # f"{round(self.fit.dataerr[aavsoC] / self.fit.parameters['a1'], 7)},{round(airmasses[aavsoC], 7)}," # f"{round(self.fit.airmass_model[aavsoC] / self.fit.parameters['a1'], 7)}\n") - f.write(f"{round(self.fit.time[aavsoC], 8)},{round(self.fit.data[aavsoC], 7)}," - f"{round(self.fit.dataerr[aavsoC], 7)},{round(airmasses[aavsoC], 7)}," + f.write(f"{round(self.fit.time[aavsoC], 8)},{round(undetrended_flux[aavsoC], 7)}," + f"{round(undetrended_flux_error[aavsoC], 7)},{round(airmasses[aavsoC], 7)}," f"{round(detrend_model[aavsoC], 7)}\n") copy_aavso_supporting_artifacts( self.dir, diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py index 5d72facb..05e43974 100644 --- a/tests/test_exotic_proper_motion.py +++ b/tests/test_exotic_proper_motion.py @@ -3675,6 +3675,7 @@ def test_detrend_flux_on_out_of_transit_baseline_removes_linear_slope(): assert np.allclose(result["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) assert result["flux"][3] == pytest.approx(0.99, abs=1e-8) assert result["slope"] == pytest.approx(0.02, abs=1e-8) + assert result["reference_time_bjd_tdb"] == pytest.approx(0.0) def test_fit_final_lightcurve_with_oot_baseline_detrending_refits_with_flattened_flux(monkeypatch): @@ -3730,6 +3731,7 @@ def fake_lc_fitter( assert refit_flux[3] == pytest.approx(0.99, abs=1e-8) assert np.allclose(refit_unc[[0, 1, 2, 4, 5, 6]], 0.01 / (1.0 + 0.02 * times[[0, 1, 2, 4, 5, 6]])) assert fit.oot_baseline_detrending_applied is True + assert fit.oot_baseline_reference_time_bjd_tdb == pytest.approx(0.0) assert fit.oot_baseline_pre_points == 3 assert fit.oot_baseline_post_points == 3 diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py index 36bf44d1..52eab16d 100644 --- a/tests/test_nextastro_variability.py +++ b/tests/test_nextastro_variability.py @@ -3506,7 +3506,7 @@ def test_process_fortuitous_variables_write_independent_and_combined_aid_product csv_path = next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')) csv_text = csv_path.read_text(encoding='utf-8') assert 'Apparent Magnitude' in csv_text.splitlines()[0] - assert 'Differential Magnitude' in csv_text.splitlines()[0] + assert 'Raw Differential Magnitude' in csv_text.splitlines()[0] exported_errors = [ float(row.split(',')[3]) for row in csv_text.splitlines()[1:] diff --git a/tests/test_output_files.py b/tests/test_output_files.py index ffbe7f97..18c6ad52 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -11,8 +11,10 @@ PRIOR_OBSERVABLE_DEPTH_LABEL, AIDOutputFiles, OutputFiles, + aavso_detrend_model, aid_comparison_coordinate_headers, aavso_dicts, + aavso_undetrended_flux_series, build_aavso_qc_metadata, fit_empirical_transit_uncertainty, fit_impact_parameter_value_error, @@ -79,8 +81,8 @@ def test_differential_csv_does_not_require_apparent_magnitude_calibration(tmp_pa assert '# AIRMASS_CORRECTION=NO' in output_text assert 'Differential Magnitude' in output_text assert 'Apparent' not in output_text - assert ', 0.7526, 0.0054, V, ' in output_text - assert ', 0.6491, 0.0051, V, ' in output_text + assert ', 0.7526, 0.0054, 0.7526, 0.0054, 1.0000000, V, ' in output_text + assert ', 0.6491, 0.0051, 0.6491, 0.0051, 1.0000000, V, ' in output_text class DummyFit: @@ -300,7 +302,7 @@ def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): assert "# FINAL STELLAR VARIABILITY TIMESERIES OF WASP-194" in output_text assert "Apparent Magnitude,Apparent Magnitude Uncertainty" in output_text - assert "Differential Magnitude,Differential Magnitude Uncertainty" in output_text + assert "Raw Differential Magnitude,Raw Differential Magnitude Uncertainty" in output_text assert "2461229.89899, 13.7378, 0.0042, 1.2378, 0.0021, r, 1.193135" in output_text assert "Flux" not in output_text @@ -325,9 +327,14 @@ def test_final_lightcurve_adds_transit_apparent_magnitude_columns_when_calibrate output_text = next((tmp_path / "working_artifacts").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() - assert "Differential Magnitude,Differential Magnitude Uncertainty" in output_text + assert "Raw Differential Magnitude,Raw Differential Magnitude Uncertainty" in output_text + assert "Corrected Differential Magnitude,Corrected Differential Magnitude Uncertainty" in output_text assert "Apparent Magnitude,Apparent Magnitude Uncertainty,Band" in output_text - assert "2461229.9, 0.1, 1.0, 0.001, 1.0, 1.0, -0.0000, 0.0011, 13.7400" in output_text + assert ( + "2461229.9, 0.1, 1.0, 0.001, 1.0, na, 1.0, " + "-0.0000, 0.0011, -0.0000, 0.0011, 13.7400" + in output_text + ) assert output_text.rstrip().endswith(", r") @@ -400,6 +407,162 @@ def test_magnitude_series_preserves_raw_ratio_for_later_apparent_recalibration() ) +def test_magnitude_series_keeps_raw_and_airmass_corrected_differential_values(): + target_flux = np.array([800.0, 1000.0, 1200.0]) + reference_flux = np.full(3, 1000.0) + target_error = np.full(3, 2.0) + reference_error = np.full(3, 3.0) + airmass_model = np.array([0.8, 1.0, 1.2]) + relative_correction = airmass_model / np.median(airmass_model) + raw_ratio = target_flux / reference_flux + expected_raw = -2.5 * np.log10(raw_ratio) + expected_corrected = -2.5 * np.log10(raw_ratio / relative_correction) + fit = SimpleNamespace( + stellar_variability_only=False, + time=np.array([2461229.9, 2461229.91, 2461229.92]), + data=raw_ratio, + dataerr=np.full(3, 0.001), + detrended=raw_ratio / relative_correction, + detrendederr=np.full(3, 0.001) / relative_correction, + airmass=np.array([1.1, 1.2, 1.3]), + airmass_model=airmass_model, + transit=np.ones(3), + stellar_variability_target_flux=target_flux, + stellar_variability_comp_flux=reference_flux, + stellar_variability_target_flux_error=target_error, + stellar_variability_comp_flux_error=reference_error, + ) + + series = magnitude_series_from_fit(fit) + + np.testing.assert_allclose(series['raw_differential_magnitude'], expected_raw) + np.testing.assert_allclose(series['corrected_differential_magnitude'], expected_corrected) + np.testing.assert_allclose(series['differential_magnitude'], expected_corrected) + np.testing.assert_allclose( + series['raw_differential_magnitude_error'], + series['corrected_differential_magnitude_error'], + ) + np.testing.assert_allclose( + series['differential_magnitude_correction_factor'], + relative_correction, + ) + assert series['correction_type'] == 'airmass' + assert series['correction_applied'] is True + assert series['airmass_corrected'] is True + + +def test_differential_csv_writes_raw_and_corrected_differential_rows(tmp_path): + fit = SimpleNamespace( + stellar_variability_only=False, + time=np.array([2461229.9, 2461229.91]), + data=np.array([0.8, 1.2]), + dataerr=np.full(2, 0.008), + detrended=np.ones(2), + detrendederr=np.full(2, 0.01), + airmass=np.array([1.1, 1.4]), + airmass_model=np.array([0.8, 1.2]), + transit=np.ones(2), + stellar_variability_target_flux=np.array([800.0, 1200.0]), + stellar_variability_comp_flux=np.full(2, 1000.0), + stellar_variability_target_flux_error=np.full(2, 2.0), + stellar_variability_comp_flux_error=np.full(2, 3.0), + ) + + output_path = write_differential_magnitude_csv( + fit, + tmp_path, + 'Transit Target', + observation_date='2026-08-17', + observed_filter='V', + ) + output_lines = output_path.read_text(encoding='utf-8').splitlines() + + assert '# AIRMASS_CORRECTION=YES' in output_lines + assert '# DIFFERENTIAL_MAGNITUDE_CORRECTION=airmass' in output_lines + assert 'Raw Differential Magnitude' in output_lines[2] + assert 'Corrected Differential Magnitude' in output_lines[2] + first_row = [value.strip() for value in output_lines[3].split(',')] + assert first_row[2] == f"{-2.5 * np.log10(0.8):.4f}" + assert first_row[4] == f"{-2.5 * np.log10(0.8 / 0.8):.4f}" + assert first_row[6] == '0.8000000' + + +def test_final_lightcurve_csv_writes_raw_and_corrected_differential_rows(tmp_path): + (tmp_path / 'working_artifacts').mkdir() + fit = SimpleNamespace( + stellar_variability_only=False, + time=np.array([2461229.9, 2461229.91]), + data=np.array([0.8, 1.2]), + dataerr=np.full(2, 0.008), + detrended=np.ones(2), + detrendederr=np.full(2, 0.01), + airmass=np.array([1.1, 1.4]), + airmass_model=np.array([0.8, 1.2]), + transit=np.ones(2), + stellar_variability_target_flux=np.array([800.0, 1200.0]), + stellar_variability_comp_flux=np.full(2, 1000.0), + stellar_variability_target_flux_error=np.full(2, 2.0), + stellar_variability_comp_flux_error=np.full(2, 3.0), + ) + p_dict = {'pName': 'Transit Target b', 'sName': 'Transit Target'} + i_dict = {'save': str(tmp_path), 'date': '2026-08-17', 'filter': 'V'} + + OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([0.1, 0.2])) + output_path = next( + (tmp_path / 'working_artifacts').glob('FinalLightCurve_TransitTargetb_2026-08-17.csv') + ) + output_lines = output_path.read_text(encoding='utf-8').splitlines() + + assert '# DIFFERENTIAL_MAGNITUDE_CORRECTION=airmass' in output_lines + assert 'Raw Differential Magnitude' in output_lines[4] + assert 'Corrected Differential Magnitude' in output_lines[4] + first_row = [value.strip() for value in output_lines[5].split(',')] + assert first_row[5] == '1.1' + assert first_row[6] == '0.8' + assert first_row[7] == f"{-2.5 * np.log10(0.8):.4f}" + assert first_row[9] == f"{-2.5 * np.log10(0.8 / 0.8):.4f}" + + +def test_linear_baseline_metadata_reconstructs_raw_flux_and_differential_magnitude(): + times = np.array([2461229.9, 2461229.91, 2461229.92]) + corrected_flux = np.array([1.0, 0.99, 1.0]) + corrected_error = np.full(3, 0.01) + fit = SimpleNamespace( + stellar_variability_only=False, + time=times, + data=corrected_flux, + dataerr=corrected_error, + detrended=corrected_flux, + detrendederr=corrected_error, + airmass=np.array([1.1, 1.2, 1.3]), + transit=np.ones(3), + oot_baseline_detrending_applied=True, + oot_baseline_reference_time_bjd_tdb=times[1], + oot_baseline_intercept=1.0, + oot_baseline_slope=2.0, + ) + expected_model = 1.0 + 2.0 * (times - times[1]) + + detrend_model = aavso_detrend_model(fit) + raw_flux, raw_error = aavso_undetrended_flux_series(fit, detrend_model) + series = magnitude_series_from_fit(fit) + + np.testing.assert_allclose(detrend_model, expected_model) + np.testing.assert_allclose(raw_flux, corrected_flux * expected_model) + np.testing.assert_allclose(raw_error, corrected_error * expected_model) + np.testing.assert_allclose( + series['raw_differential_magnitude'], + -2.5 * np.log10(corrected_flux * expected_model), + ) + np.testing.assert_allclose( + series['corrected_differential_magnitude'], + -2.5 * np.log10(corrected_flux), + ) + assert series['correction_type'] == 'out_of_transit_linear_baseline' + assert series['correction_applied'] is True + assert series['airmass_corrected'] is False + + def test_calibrated_ensemble_keeps_apparent_magnitudes_independent_of_raw_differential(): target_flux = np.array([1000.0, 1010.0]) raw_reference_flux = np.full(2, 375.0) @@ -490,6 +653,13 @@ def test_observable_depth_is_separate_from_area_depth_for_grazing_geometry(): def test_aavso_output_includes_observatory_location_headers(tmp_path): fit = DummyFit() + fit.oot_baseline_detrending_applied = True + fit.oot_baseline_detrending_note = "Applied weighted linear out-of-transit baseline detrending." + fit.oot_baseline_reference_time_bjd_tdb = fit.time[0] + fit.oot_baseline_intercept = 1.2 + fit.oot_baseline_slope = 0.05 + fit.oot_baseline_pre_points = 4 + fit.oot_baseline_post_points = 5 fit.stellar_variability_target_flux = np.array([500.0]) fit.stellar_variability_comp_flux = np.array([1000.0]) fit.stellar_variability_target_flux_error = np.array([2.0]) @@ -592,8 +762,43 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): assert "#GAIAPMDEC=-9.5" in output_text magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") magnitude_row = aavso_json_header(output_text, "MAGNITUDE-XC") + baseline_metadata = aavso_json_header(output_text, "OUT_OF_TRANSIT_BASELINE-XC") + detrend_metadata = aavso_json_header(output_text, "DETREND_PARAMETERS-XC") + assert baseline_metadata == { + "applied": True, + "forward_correction": "detrended_flux = raw_flux / baseline(t)", + "intercept": 1.2, + "inverse_correction": "raw_flux = detrended_flux * baseline(t)", + "model": ( + "baseline(t) = intercept + slope_per_day * " + "(BJD_TDB - reference_time_bjd_tdb)" + ), + "note": "Applied weighted linear out-of-transit baseline detrending.", + "post_egress_point_count": 5, + "pre_ingress_point_count": 4, + "reference_time_bjd_tdb": fit.time[0], + "serialized_model_available": True, + "slope_per_day": 0.05, + } + assert "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION" in output_text + assert detrend_metadata == { + "DETREND_1": "airmass", + "DETREND_2": "out_of_transit_linear_baseline_correction_function", + "standard_header_preserved": True, + } assert magnitude_fields["apparent_calibrated"] is True assert magnitude_fields["differential_magnitude"].startswith("target minus") + assert magnitude_fields["raw_differential_magnitude"].startswith("target minus") + assert magnitude_fields["differential_magnitude"].endswith( + "corrected_differential_magnitude" + ) + assert magnitude_row["raw_differential_magnitude"] == round(differential_mag, 4) + assert magnitude_row["raw_differential_magnitude_error"] == round(differential_error, 4) + assert magnitude_row["corrected_differential_magnitude"] == round(differential_mag, 4) + assert magnitude_row["corrected_differential_magnitude_error"] == round( + differential_error, + 4, + ) assert magnitude_row["differential_magnitude"] == round(differential_mag, 4) assert magnitude_row["differential_magnitude_error"] == round(differential_error, 4) assert magnitude_row["apparent_magnitude"] == round(12.0 + differential_mag, 4) @@ -601,6 +806,7 @@ def test_aavso_output_includes_observatory_location_headers(tmp_path): np.hypot(0.02, differential_error), 4, ) + assert "2450000.123456,1.2,0.012,1.0,1.2" in output_text def test_aavso_output_omits_obsname_header_when_blank(tmp_path):