From 500097fbfd9234749ff5e926d2a2c3ccf65a996d Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Tue, 14 Apr 2026 13:31:16 +0000 Subject: [PATCH] add cmb observation efficiency --- sotodlib/qa/quality_reports/report_data.py | 10 ++++++++-- sotodlib/site_pipeline/generate_reports.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/sotodlib/qa/quality_reports/report_data.py b/sotodlib/qa/quality_reports/report_data.py index ff5111070..ab5b1b728 100644 --- a/sotodlib/qa/quality_reports/report_data.py +++ b/sotodlib/qa/quality_reports/report_data.py @@ -13,6 +13,7 @@ import sys from influxdb import InfluxDBClient from collections import defaultdict +import traceback from sotodlib.io import hkdb from sotodlib.io.hkdb import HkConfig @@ -176,7 +177,10 @@ def get_apex_data(cfg: ReportDataConfig): return None def date_converter(d): - naive_dt = dt.datetime.fromisoformat(d) + if isinstance(d, bytes): + naive_dt = dt.datetime.fromisoformat(d.decode('utf-8')) + else: + naive_dt = dt.datetime.fromisoformat(d) return naive_dt.replace(tzinfo=dt.timezone.utc) data = np.genfromtxt( @@ -252,7 +256,9 @@ def get_and_merge_apex_pwv(cfg: ReportDataConfig, result=None): try: result_apex = get_apex_data(cfg) except Exception as e: - logger.error(f"get_apex_data failed with {e}") + errmsg = f'{type(e)}: {e}' + tb = ''.join(traceback.format_tb(e.__traceback__)) + logger.error(f"get_apex_data failed with {errmsg}\n{tb}") result_apex = None if result_apex is not None: diff --git a/sotodlib/site_pipeline/generate_reports.py b/sotodlib/site_pipeline/generate_reports.py index 10a981f54..14cce223d 100644 --- a/sotodlib/site_pipeline/generate_reports.py +++ b/sotodlib/site_pipeline/generate_reports.py @@ -379,6 +379,22 @@ def render_report( else: unique_obs = [o for o in data.obs_list if o.obs_type == "obs"] + total_time = (dt.datetime.fromisoformat(stop_time_str) - dt.datetime.fromisoformat(start_time_str)).total_seconds() / 3600 + + if data.pwv is not None: + mask = data.pwv[1] < 3 + timediff = np.diff(data.pwv[0][mask]) + # prevent gaps from adding time unnecessarily + mask = timediff < 120 + good_time = np.sum(timediff[mask]) / 3600 + else: + good_time = 0. + + cmb_good_pwvtime = np.sum(np.array([o.duration for o in unique_obs if o.obs_subtype == "cmb" and o.pwv < 3])) / 3600 + cmb_all_pwvtime = np.sum(np.array([o.duration for o in unique_obs if o.obs_subtype == "cmb" and o.pwv])) / 3600 + idle_good_time = good_time - cmb_good_pwvtime + idle_all_time = total_time - cmb_all_pwvtime + jinja_data = { "data": data, "report_interval": cfg.report_interval.capitalize(), @@ -395,6 +411,10 @@ def render_report( "Number of Cal Observations": len([o for o in unique_obs if o.obs_subtype == "cal"]), "Time Spent on CMB Observations (hrs)": np.round(np.sum(np.array([o.duration for o in unique_obs if o.obs_subtype == "cmb"])) / 3600, 1), "Time Spent on Cal Observations (hrs)": np.round(np.sum(np.array([o.duration for o in unique_obs if o.obs_subtype == "cal"])) / 3600, 1), + "Percentage of Time on CMB Observations (PWV < 3):": np.round(100 * cmb_good_pwvtime / good_time, 2), + "Percentage of Time on CMB Observations:": np.round(100 * cmb_all_pwvtime / total_time, 2), + "Percentage of Time spent not observing CMB (PWV < 3):": np.round(100 * idle_good_time / good_time, 2), + "Percentage of Time spent not observing CMB:": np.round(100 * idle_all_time / total_time, 2), "Average Duration of CMB Observations (hrs)": np.round(np.nanmean(v) / 3600, 2) if (v := [o.duration for o in unique_obs if o.obs_subtype == "cmb"]) else 0, "Average Duration of Cal Observations (hrs)": np.round(np.nanmean(v) / 3600, 2) if (v := [o.duration for o in unique_obs if o.obs_subtype == "cal"]) else 0, "Average Obs PWV (mm)": np.round(np.nanmean([o.pwv for o in unique_obs]), 3),