From 2bf265f04368dd298367ffd8aac62a183f537fe1 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Wed, 1 Apr 2026 18:45:10 -0400 Subject: [PATCH 01/10] feat: barebones epochs system implementation --- sotodlib/utils/epochs.py | 216 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 sotodlib/utils/epochs.py diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py new file mode 100644 index 000000000..78c6b3566 --- /dev/null +++ b/sotodlib/utils/epochs.py @@ -0,0 +1,216 @@ +from copy import deepcopy +from dataclasses import dataclass, field +from itertools import accumulate +from operator import add, mul +from typing import Any, Self + +import yaml +from deepdiff import DeepDiff + + +@dataclass +class Interval: + """ + Smallest unit of the epoch system. + Here we have a time interval containing a set of tags and an arbitrary dict of data. + This can be sliced with standard slicing notation, but the indices are ctime. + + Two `Interval` instances can be combined with the following operators, + all operators are functionally addition but with differing checks: + + * `|`: naive addition, combines `tags` and `data` and simply returns the most inclusive time range + * `+`: same as `|` but will error if time ranges are not overlapping + * `&`: same as `|` but will error if `tags` or `data` are not the same + * `*`: same as `+` but will error if `data` is not the same + * `@`: same as `*` but will error if `tags` is not the same + + In cases where `data` is not the same but the `Invervals` are combined, + the new `data` will keep values from the right hand `Inverval` for overlapping keys. + """ + + name: str + start: float + stop: float + tags: set[str] + data: dict[str, Any] + + def __getitem__(self: Self, val) -> Self: + to_ret = deepcopy(self) + if not isinstance(val, slice): + val = slice(val) + if val.start is not None: + if self.start < val.start: + raise IndexError( + f"Start of slice out of range, interval starts at {self.start} but slice starts at {val.start}!" + ) + to_ret.start = val.start + if val.stop is not None: + if self.stop > val.stop: + raise IndexError( + f"End of slice out of range, interval stops at {self.stop} but slice stops at {val.stop}!" + ) + to_ret.stop = val.stop + return to_ret + + def combine( + self: Self, + tocombine: Self, + in_place: bool = True, + time_check: bool = False, + tag_check: bool = False, + data_check: bool = False, + ) -> Self: + if not isinstance(tocombine, Interval): + raise TypeError( + f"Can only add intervals to other intervals, not {type(tocombine)}!" + ) + combined = self + if in_place: + combined = deepcopy(self) + if time_check and (self.start > tocombine.stop or self.stop < tocombine.start): + raise ValueError( + "Can't combine without overlapping time ranges with time_check=True" + ) + combined.start = min(self.start, tocombine.start) + combined.stop = max(self.stop, tocombine.stop) + + if tag_check and self.tags != tocombine.tags: + raise ValueError("Can't combine without identical tags with tag_check=True") + combined.tags = self.tags.union(tocombine.tags) + + diff = DeepDiff(self.data, self.data) + if data_check and len(diff) != 0: + raise ValueError( + "Can't combine without identical data with data_check=True" + ) + combined.data.update(tocombine.data) + + return combined + + def __or__(self, tocombine) -> Self: + return self.combine(tocombine, False, False, False, False) + + def __ior__(self, tocombine) -> Self: + return self.combine(tocombine, True, False, False, False) + + def __add__(self, tocombine) -> Self: + return self.combine(tocombine, False, True, False, False) + + def __iadd__(self, tocombine) -> Self: + return self.combine(tocombine, True, True, False, False) + + def __and__(self, tocombine) -> Self: + return self.combine(tocombine, False, False, True, True) + + def __iand__(self, tocombine) -> Self: + return self.combine(tocombine, True, False, True, True) + + def __mul__(self, tocombine) -> Self: + return self.combine(tocombine, False, True, False, True) + + def __imul__(self, tocombine) -> Self: + return self.combine(tocombine, True, True, False, True) + + def __matmul__(self, tocombine) -> Self: + return self.combine(tocombine, False, True, True, True) + + def __imatmul__(self, tocombine) -> Self: + return self.combine(tocombine, True, True, True, True) + + +@dataclass +class Epoch: + name: str + covers: tuple[Interval] + strict: bool + _internal: Interval = field(init=False) + + def __post_init__(self): + self._internal = accumulate(self.covers, mul if self.strict else add) + + def __setattr__(self, name, value): + if name == "covers" or name == "strict": + self.__post_init__() + return super().__setattr__(name, value) + + +@dataclass +class Era: + name: str + epochs: tuple[Epoch] + strict: bool + _internal: Interval = field(init=False) + + def __post_init__(self): + self._internal = accumulate( + [e._internal for e in self.epochs], mul if self.strict else add + ) + + def __setattr__(self, name, value): + if name == "covers" or name == "strict": + self.__post_init__() + return super().__setattr__(name, value) + + +@dataclass +class Calendar: + intervals: dict[str, Interval] + eras: dict[str, Era] + data: dict[str, dict[str, Any]] + + @classmethod + def load(cls, fpath: str): + with open(fpath, "r") as f: + cfg = yaml.safe_load(f) + if "intervals" not in cfg: + raise ValueError("At minimum an intervals section must be provided") + + # Collate information in string form + intervals = cfg["intervals"] + eras = {} + data = {} + for key, val in cfg: + if key == "intervals": + continue + if key[0] == "_": + data[key[1:]] = val + eras[key] = val + + # Load intervals + interval_dict = {} + for name, interval in intervals.items(): + interval_data = {} + for key, val in interval.get("data", {}).items(): + val = val.split(".") + if len(val) != 2: + raise ValueError("Data must have format X.Y") + interval_data[key] = data[val[0]][val[1]] + interval_dict[name] = Interval( + name, + interval.get("start", 0), + interval.get("stop", 20000000000), + interval.get("tags", []), + interval_data, + ) + + # Load epochs + for ename, era in eras.items(): + strict = era.get("strict", True) + epochs = [] + for name, ec in era.items(): + covers = [] + for cname in ec.get("covers", []): + iname, slstr = cname.split("[") + slstr = slstr[:-1] + ival = intervals[iname] + slc = slice( + *map( + lambda x: float(x.strip()) if x.strip() else None, + slstr.split(":"), + ) + ) + covers += [ival[slc]] + epochs[name] = Epoch(name, tuple(covers), strict) + eras[ename] = Era(ename, tuple(epochs), strict) + + return Calendar(interval_dict, eras, data) From 139dc0d5b5cf12976129ba7cebeb21f977667230 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Wed, 1 Apr 2026 18:53:38 -0400 Subject: [PATCH 02/10] fix: adjust era strictness --- sotodlib/utils/epochs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index 78c6b3566..94263005a 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -1,7 +1,7 @@ from copy import deepcopy from dataclasses import dataclass, field from itertools import accumulate -from operator import add, mul +from operator import add, mul, or_ from typing import Any, Self import yaml @@ -143,7 +143,7 @@ class Era: def __post_init__(self): self._internal = accumulate( - [e._internal for e in self.epochs], mul if self.strict else add + [e._internal for e in self.epochs], add if self.strict else or_ ) def __setattr__(self, name, value): From 6c471af97368f36ae6d80788cb580cb29944e05f Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Wed, 1 Apr 2026 18:56:45 -0400 Subject: [PATCH 03/10] fix: cast output of accumulate --- sotodlib/utils/epochs.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index 94263005a..6ccef79f9 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from itertools import accumulate from operator import add, mul, or_ -from typing import Any, Self +from typing import Any, Self, cast import yaml from deepdiff import DeepDiff @@ -126,7 +126,9 @@ class Epoch: _internal: Interval = field(init=False) def __post_init__(self): - self._internal = accumulate(self.covers, mul if self.strict else add) + self._internal = cast( + Interval, accumulate(self.covers, mul if self.strict else add) + ) def __setattr__(self, name, value): if name == "covers" or name == "strict": @@ -142,8 +144,9 @@ class Era: _internal: Interval = field(init=False) def __post_init__(self): - self._internal = accumulate( - [e._internal for e in self.epochs], add if self.strict else or_ + self._internal = cast( + Interval, + accumulate([e._internal for e in self.epochs], add if self.strict else or_), ) def __setattr__(self, name, value): From 4fae6dc116b9ea6af5b72822b23e43475e565c52 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Wed, 1 Apr 2026 19:05:02 -0400 Subject: [PATCH 04/10] fix: correct variable name in era post init --- sotodlib/utils/epochs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index 6ccef79f9..d0293b08a 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -150,7 +150,7 @@ def __post_init__(self): ) def __setattr__(self, name, value): - if name == "covers" or name == "strict": + if name == "epochs" or name == "strict": self.__post_init__() return super().__setattr__(name, value) From 852e2143ade3e94074c0867cd5727dbbed8f9636 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Thu, 2 Apr 2026 14:20:12 -0400 Subject: [PATCH 05/10] feat: better strictness system --- sotodlib/utils/epochs.py | 116 +++++++++++++++++++++++++++++---------- 1 file changed, 88 insertions(+), 28 deletions(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index d0293b08a..11c73da2b 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -1,12 +1,16 @@ from copy import deepcopy from dataclasses import dataclass, field from itertools import accumulate -from operator import add, mul, or_ +from operator import add, and_, mul, or_ from typing import Any, Self, cast import yaml from deepdiff import DeepDiff +# TODO: Add check for a specific piece of data -> existance and strictness +# TODO: Add method to search via tags +# TODO: Make Invtervals aware of what Epochs they are members of and make Epochs aware of what Eras they are members of + @dataclass class Interval: @@ -18,11 +22,14 @@ class Interval: Two `Interval` instances can be combined with the following operators, all operators are functionally addition but with differing checks: - * `|`: naive addition, combines `tags` and `data` and simply returns the most inclusive time range - * `+`: same as `|` but will error if time ranges are not overlapping - * `&`: same as `|` but will error if `tags` or `data` are not the same - * `*`: same as `+` but will error if `data` is not the same - * `@`: same as `*` but will error if `tags` is not the same + * `|`: Allow gaps between intervals, but data and tags can differ + * `&`: Intervals must be overlapping, but data and tags can differ + * `*`: Allow gaps between intervals, data must be the same but tags can differ + * `+`: Invervals must be overlapping, data must be the same but tags can differ + * `/`: Allow gaps between intervals, data can differ but tags must be the same + * `-`: Invervals must be overlapping, data can differ but tags must be the same + * `**`: Allow gaps between invervals, data and tags must be the same + * `@`: Allow gaps between intervals, data and tags must be the same In cases where `data` is not the same but the `Invervals` are combined, the new `data` will keep values from the right hand `Inverval` for overlapping keys. @@ -56,9 +63,9 @@ def combine( self: Self, tocombine: Self, in_place: bool = True, - time_check: bool = False, - tag_check: bool = False, - data_check: bool = False, + allow_gap: bool = False, + same_data: bool = False, + same_tag: bool = False, ) -> Self: if not isinstance(tocombine, Interval): raise TypeError( @@ -67,21 +74,23 @@ def combine( combined = self if in_place: combined = deepcopy(self) - if time_check and (self.start > tocombine.stop or self.stop < tocombine.start): + if not allow_gap and ( + self.start > tocombine.stop or self.stop < tocombine.start + ): raise ValueError( - "Can't combine without overlapping time ranges with time_check=True" + "Can't combine without overlapping time ranges with allow_gap=False" ) combined.start = min(self.start, tocombine.start) combined.stop = max(self.stop, tocombine.stop) - if tag_check and self.tags != tocombine.tags: - raise ValueError("Can't combine without identical tags with tag_check=True") + if not same_tag and self.tags != tocombine.tags: + raise ValueError("Can't combine without identical tags with same_tag=False") combined.tags = self.tags.union(tocombine.tags) diff = DeepDiff(self.data, self.data) - if data_check and len(diff) != 0: + if not same_data and len(diff) != 0: raise ValueError( - "Can't combine without identical data with data_check=True" + "Can't combine without identical data with same_data=False" ) combined.data.update(tocombine.data) @@ -93,24 +102,42 @@ def __or__(self, tocombine) -> Self: def __ior__(self, tocombine) -> Self: return self.combine(tocombine, True, False, False, False) - def __add__(self, tocombine) -> Self: + def __and__(self, tocombine) -> Self: return self.combine(tocombine, False, True, False, False) - def __iadd__(self, tocombine) -> Self: + def __iand__(self, tocombine) -> Self: return self.combine(tocombine, True, True, False, False) - def __and__(self, tocombine) -> Self: - return self.combine(tocombine, False, False, True, True) + def __mul__(self, tocombine) -> Self: + return self.combine(tocombine, False, False, True, False) - def __iand__(self, tocombine) -> Self: - return self.combine(tocombine, True, False, True, True) + def __imul__(self, tocombine) -> Self: + return self.combine(tocombine, True, False, True, False) - def __mul__(self, tocombine) -> Self: + def __add__(self, tocombine) -> Self: + return self.combine(tocombine, False, True, True, False) + + def __iadd__(self, tocombine) -> Self: + return self.combine(tocombine, True, True, True, False) + + def __div__(self, tocombine) -> Self: + return self.combine(tocombine, False, False, False, True) + + def __idiv__(self, tocombine) -> Self: + return self.combine(tocombine, True, False, False, True) + + def __sub__(self, tocombine) -> Self: return self.combine(tocombine, False, True, False, True) - def __imul__(self, tocombine) -> Self: + def __isub__(self, tocombine) -> Self: return self.combine(tocombine, True, True, False, True) + def __pow__(self, tocombine) -> Self: + return self.combine(tocombine, False, False, False, True) + + def __ipow__(self, tocombine) -> Self: + return self.combine(tocombine, True, False, False, True) + def __matmul__(self, tocombine) -> Self: return self.combine(tocombine, False, True, True, True) @@ -120,6 +147,21 @@ def __imatmul__(self, tocombine) -> Self: @dataclass class Epoch: + """ + Dataclass for storing a collection of `Invervals`s. + `Inverval`s must be overlapping. + + Attributes + ---------- + name : str + Name for this epoch. + covers: tuple[Interval] + Collections of overlapping `Invervals` that make up an `Epoch`. + strict : bool + If `True` than in addition to being overlapping the `Intervals` must + also contain the same data. + """ + name: str covers: tuple[Interval] strict: bool @@ -127,7 +169,7 @@ class Epoch: def __post_init__(self): self._internal = cast( - Interval, accumulate(self.covers, mul if self.strict else add) + Interval, accumulate(self.covers, mul if self.strict else or_) ) def __setattr__(self, name, value): @@ -138,6 +180,21 @@ def __setattr__(self, name, value): @dataclass class Era: + """ + Dataclass for storing a collection of `Epoch`s. + `Epoch`s must be non-overlapping but can have gaps in time between them. + + Attributes + ---------- + name : str + Name for this era. + epochs : tuple[Epoch] + Collections of non-overlapping `Epochs` that make up an `Era`. + strict : bool + If `True` than in addition to being non-overlapping the `Epochs` must + also contain the same data. + """ + name: str epochs: tuple[Epoch] strict: bool @@ -146,7 +203,9 @@ class Era: def __post_init__(self): self._internal = cast( Interval, - accumulate([e._internal for e in self.epochs], add if self.strict else or_), + accumulate( + [e._internal for e in self.epochs], add if self.strict else and_ + ), ) def __setattr__(self, name, value): @@ -198,7 +257,8 @@ def load(cls, fpath: str): # Load epochs for ename, era in eras.items(): - strict = era.get("strict", True) + strict_era = era.get("strict_era", False) + strict_epoch = era.get("strict_epochs", True) epochs = [] for name, ec in era.items(): covers = [] @@ -213,7 +273,7 @@ def load(cls, fpath: str): ) ) covers += [ival[slc]] - epochs[name] = Epoch(name, tuple(covers), strict) - eras[ename] = Era(ename, tuple(epochs), strict) + epochs[name] = Epoch(name, tuple(covers), strict_epoch) + eras[ename] = Era(ename, tuple(epochs), strict_era) return Calendar(interval_dict, eras, data) From ca89a2c1d7b276036f7230cd7d76b34028afde63 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Fri, 3 Apr 2026 16:28:39 -0400 Subject: [PATCH 06/10] feat: add tools in calendar to modify eras and epochs and check membership of an interval --- sotodlib/utils/epochs.py | 203 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 194 insertions(+), 9 deletions(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index 11c73da2b..f53a2f3ba 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -1,15 +1,15 @@ +import re from copy import deepcopy from dataclasses import dataclass, field from itertools import accumulate from operator import add, and_, mul, or_ -from typing import Any, Self, cast +from typing import Any, Optional, Self, cast import yaml from deepdiff import DeepDiff # TODO: Add check for a specific piece of data -> existance and strictness # TODO: Add method to search via tags -# TODO: Make Invtervals aware of what Epochs they are members of and make Epochs aware of what Eras they are members of @dataclass @@ -41,6 +41,9 @@ class Interval: tags: set[str] data: dict[str, Any] + def __repr__(self) -> str: + return f"{self.name}[{self.start}:{self.stop}]" + def __getitem__(self: Self, val) -> Self: to_ret = deepcopy(self) if not isinstance(val, slice): @@ -59,6 +62,14 @@ def __getitem__(self: Self, val) -> Self: to_ret.stop = val.stop return to_ret + def __eq__(self, other) -> bool: + if not isinstance(other, Interval): + return False + diff = DeepDiff(self.data, other.data) + return bool( + (str(self) == str(other)) * (self.tags == other.tags) * (len(diff) == 0) + ) + def combine( self: Self, tocombine: Self, @@ -87,13 +98,15 @@ def combine( raise ValueError("Can't combine without identical tags with same_tag=False") combined.tags = self.tags.union(tocombine.tags) - diff = DeepDiff(self.data, self.data) + diff = DeepDiff(self.data, tocombine.data) if not same_data and len(diff) != 0: raise ValueError( "Can't combine without identical data with same_data=False" ) combined.data.update(tocombine.data) + combined.name = self.name + "+" + tocombine.name + return combined def __or__(self, tocombine) -> Self: @@ -155,6 +168,8 @@ class Epoch: ---------- name : str Name for this epoch. + era_name : str + Name of the era that this epoch belongs to. covers: tuple[Interval] Collections of overlapping `Invervals` that make up an `Epoch`. strict : bool @@ -163,7 +178,8 @@ class Epoch: """ name: str - covers: tuple[Interval] + era_name: str + covers: tuple[Interval, ...] strict: bool _internal: Interval = field(init=False) @@ -172,11 +188,25 @@ def __post_init__(self): Interval, accumulate(self.covers, mul if self.strict else or_) ) + def __repr__(self) -> str: + return ( + self.name + + "(" + + ",".join([str(ival) for ival in self.covers]) + + ")" + + ("*" * self.strict) + ) + def __setattr__(self, name, value): if name == "covers" or name == "strict": self.__post_init__() return super().__setattr__(name, value) + def __eq__(self, other): + if not isinstance(other, Epoch): + return False + return str(self) == str(other) + @dataclass class Era: @@ -196,7 +226,7 @@ class Era: """ name: str - epochs: tuple[Epoch] + epochs: tuple[Epoch, ...] strict: bool _internal: Interval = field(init=False) @@ -216,12 +246,162 @@ def __setattr__(self, name, value): @dataclass class Calendar: + """ + Class to manage intervals, epochs, and eras in a unified way. + It is reccomended to modify relationships between these using this class, + the `Calender` is not aware of direct modifications. + """ + intervals: dict[str, Interval] + epochs: dict[str, Epoch] eras: dict[str, Era] data: dict[str, dict[str, Any]] + orphan_epochs: list[Epoch] + + def change_strictness( + self, strictness: bool, era: str, epoch: Optional[str] = None + ): + """ + Change the strictness for an `Era` or an `Epoch`. + + Parameters + ---------- + strictness : bool + The new strictness. + era : str + The name of the era to modify. + epoch : Optional, default: None + The name of epoch to modify. + To modify and era set `epoch` to `None`. + """ + if epoch is None: + self.eras[era].strict = strictness + else: + self.epochs[f"{era}.{epoch}"].strict = strictness + + def add_epoch(self, era: str, epoch: Epoch): + """ + Add an `Epoch` to an `Era`. + If the `Epoch` is in the orphans list it will be removed. + + Parameters + ---------- + era : str + The name of the era to modify. + epoch : Epoch + The new epoch to add. + """ + epoch.era_name = era + epoch_list = list(self.eras[era].epochs) + [epoch] + self.eras[era].epochs = tuple(epoch_list) + self.epochs[f"{era}.{epoch.name}"] + self.epoch_orphans = [e for e in self.epoch_orphans if e != epoch] + + def del_epoch(self, era: str, epoch: str): + """ + Delete an `Epoch` from an `Era`. + This will add it to the orphans list + + Parameters + ---------- + era : str + The name of the era to modify. + epoch : str + The name of the epoch to remove. + """ + to_remove = self.epochs[f"{era}.{epoch}"] + self.epoch_orphans += [to_remove] + self.eras[era].epochs = tuple( + e for e in self.eras[era].epochs if e != to_remove + ) + del self.epochs[f"{era}.{epoch}"] + + def add_interval(self, era: str, epoch: str, interval: Interval): + """ + Add an `Inverval` to an `Epoch`. + + Parameters + ---------- + era : str + The name of the era to modify. + epoch : Epoch + The name of epoch to modify, + interval : Interval + The interval to add. + """ + self.epochs[f"{era}.{epoch}"].covers = tuple( + list(self.epochs[f"{era}.{epoch}"].covers) + [interval] + ) + self.intervals[str(interval)] = interval + + def del_interval(self, era: str, epoch: str, interval: str): + """ + Delete an `Inverval` from an `Epoch`. + + Parameters + ---------- + era : str + The name of the era to modify. + epoch : Epoch + The name of epoch to modify, + interval : str + The name of interval (with its slice) to delete. + """ + self.epochs[f"{era}.{epoch}"].covers = tuple( + i + for i in self.epochs[f"{era}.{epoch}"].covers + if i != self.intervals[interval] + ) + + def find_slices(self, interval_name: str) -> list[Interval]: + """ + Find all slices for an interval. + + Parameters + ---------- + interval_name : str + The name of the interval to find slices for. + + Returns + ------- + slices : list[Interval] + List of slices with the correct name. + """ + ikeys = list(self.intervals.keys()) + r = re.compile(f"{interval_name}\\[.*:.*\\]") + ikeys = filter(r.match, ikeys) + slices = [self.intervals[ikey] for ikey in ikeys] + + return slices + + def get_membership(self, interval: str) -> list[tuple[Era, Epoch]]: + """ + Get the `Epochs` than an interval belongs to. + This does not search oprhaned `Epoch`s. + + Parameters + ---------- + interval : str + The name of the interval (with its slice) to search for. + + Returns + ------- + membership : list[tuple[Era, Epoch]] + A list a tuples where each element is an `(Era, Epoch)` pair + where the `Epoch` contains the interval. + """ + ival = self.intervals[interval] + membership = [] + for era in self.eras.values(): + for epoch in era.epochs: + for i in epoch.covers: + if i == ival: + membership += [(era, epoch)] + break + return membership @classmethod - def load(cls, fpath: str): + def load(cls, fpath: str) -> Self: with open(fpath, "r") as f: cfg = yaml.safe_load(f) if "intervals" not in cfg: @@ -247,15 +427,17 @@ def load(cls, fpath: str): if len(val) != 2: raise ValueError("Data must have format X.Y") interval_data[key] = data[val[0]][val[1]] - interval_dict[name] = Interval( + ival = Interval( name, interval.get("start", 0), interval.get("stop", 20000000000), interval.get("tags", []), interval_data, ) + interval_dict[str(ival)] = ival # Load epochs + epoch_dict = {} for ename, era in eras.items(): strict_era = era.get("strict_era", False) strict_epoch = era.get("strict_epochs", True) @@ -266,6 +448,7 @@ def load(cls, fpath: str): iname, slstr = cname.split("[") slstr = slstr[:-1] ival = intervals[iname] + interval_dict[str(ival)] = ival slc = slice( *map( lambda x: float(x.strip()) if x.strip() else None, @@ -273,7 +456,9 @@ def load(cls, fpath: str): ) ) covers += [ival[slc]] - epochs[name] = Epoch(name, tuple(covers), strict_epoch) + epoch = Epoch(name, ename, tuple(covers), strict_epoch) + epochs += [epoch] + epoch_dict[f"{ename}.{name}"] = epoch eras[ename] = Era(ename, tuple(epochs), strict_era) - return Calendar(interval_dict, eras, data) + return cls(interval_dict, epoch_dict, eras, data, []) From 58875d66cd3154202f42201de922825a42c62e10 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Mon, 6 Apr 2026 13:16:43 -0400 Subject: [PATCH 07/10] feat: add functions to check on data presence --- sotodlib/utils/epochs.py | 109 +++++++++++++++++++++++++++++++++++---- 1 file changed, 100 insertions(+), 9 deletions(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index f53a2f3ba..11d9c7a79 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -3,12 +3,11 @@ from dataclasses import dataclass, field from itertools import accumulate from operator import add, and_, mul, or_ -from typing import Any, Optional, Self, cast +from typing import Any, Literal, Optional, Self, cast, overload import yaml from deepdiff import DeepDiff -# TODO: Add check for a specific piece of data -> existance and strictness # TODO: Add method to search via tags @@ -22,12 +21,12 @@ class Interval: Two `Interval` instances can be combined with the following operators, all operators are functionally addition but with differing checks: - * `|`: Allow gaps between intervals, but data and tags can differ - * `&`: Intervals must be overlapping, but data and tags can differ - * `*`: Allow gaps between intervals, data must be the same but tags can differ - * `+`: Invervals must be overlapping, data must be the same but tags can differ - * `/`: Allow gaps between intervals, data can differ but tags must be the same - * `-`: Invervals must be overlapping, data can differ but tags must be the same + * `|`: Intervals must be overlapping, but data and tags can differ + * `&`: Allow gaps between intervals, but data and tags can differ + * `*`: Invervals must be overlapping, data must be the same but tags can differ + * `+`: Allow gaps between intervals, data must be the same but tags can differ + * `/`: Invervals must be overlapping, data can differ but tags must be the same + * `-`: Allow gaps between intervals, data can differ but tags must be the same * `**`: Allow gaps between invervals, data and tags must be the same * `@`: Allow gaps between intervals, data and tags must be the same @@ -207,6 +206,33 @@ def __eq__(self, other): return False return str(self) == str(other) + def check_data(self, field: str, strict: bool = True) -> bool: + """ + Check that all `Interval`s in this `Epoch` contain a certain data field. + Optinally check that they also contain the same value for the field. + + Parameters + ---------- + field : str + The name of the data field to check for. + strict : str + If True then all `Inverval`s within the `Epoch` must share the same + value for the data field. If `False` they simply need to exist. + + """ + if field not in self._internal.data: + return False + if strict: + tester = self if self.strict else deepcopy(self) + try: + tester.strict = True + except ValueError: + return False + return True + return cast( + bool, accumulate([field in ival.data for ival in self.covers], and_) + ) + @dataclass class Era: @@ -243,6 +269,24 @@ def __setattr__(self, name, value): self.__post_init__() return super().__setattr__(name, value) + def check_data(self, field: str, strict: bool = True) -> bool: + """ + Check that all `Epoch`s in this `Era` contain a certain data field. + Optinally check that all of their `Invervals` also contain the same value for the field. + + Parameters + ---------- + field : str + The name of the data field to check for. + strict : str + If True then all `Inverval`s within each `Epoch` must share the same + value for the data field. If `False` they simply need to exist. + + """ + return cast( + bool, accumulate([e.check_data(field, strict) for e in self.epochs], and_) + ) + @dataclass class Calendar: @@ -258,6 +302,52 @@ class Calendar: data: dict[str, dict[str, Any]] orphan_epochs: list[Epoch] + @overload + def find_data_field(self, field: str, search_in: Literal["eras"]) -> list[Era]: + ... + + @overload + def find_data_field(self, field: str, search_in: Literal["epochs"]) -> list[Epoch]: + ... + + @overload + def find_data_field( + self, field: str, search_in: Literal["intervals"] + ) -> list[Interval]: + ... + + def find_data_field( + self, field: str, search_in: Literal["intervals", "epochs", "eras"] + ) -> list: + """ + Return all objects containing a specific data field. + The `search_in` parameter specifies which type of object to search for. + To check that a specific `Era` has a field in all of it's epochs use `Era.check_data`. + + Parameters + ---------- + field : str + The name of the field to check + search_in : Literal["intervals", "eras", "epochs"] + What type of objects to search for. + Note that orphaned epochs will not be searched. + + Returns + ------ + found : list + A list of obects of type `search_in` containing this data field. + """ + if search_in == "intervals": + return [ival for ival in self.intervals.values() if field in ival.data] + elif search_in == "epochs": + return [ + epoch for epoch in self.epochs.values() if field in epoch._internal.data + ] + elif search_in == "eras": + return [era for era in self.eras.values() if field in era._internal.data] + else: + raise ValueError(f"Invalid search_in: {search_in}") + def change_strictness( self, strictness: bool, era: str, epoch: Optional[str] = None ): @@ -272,7 +362,8 @@ def change_strictness( The name of the era to modify. epoch : Optional, default: None The name of epoch to modify. - To modify and era set `epoch` to `None`. + To modify only the era set `epoch` to `None`. + To modify all `Epoch`s in an era set to `all`. """ if epoch is None: self.eras[era].strict = strictness From bc34b53dc713640a488891b9fba8f0a9a3f462fa Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Mon, 6 Apr 2026 14:47:18 -0400 Subject: [PATCH 08/10] feat: add tag search --- sotodlib/utils/epochs.py | 48 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index 11d9c7a79..4ea0590b7 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -8,9 +8,6 @@ import yaml from deepdiff import DeepDiff -# TODO: Add method to search via tags - - @dataclass class Interval: """ @@ -302,6 +299,51 @@ class Calendar: data: dict[str, dict[str, Any]] orphan_epochs: list[Epoch] + @overload + def find_tagged(self, tag: str, search_in: Literal["eras"]) -> list[Era]: + ... + + @overload + def find_tagged(self, tag: str, search_in: Literal["epochs"]) -> list[Epoch]: + ... + + @overload + def find_tagged( + self, tag: str, search_in: Literal["intervals"] + ) -> list[Interval]: + ... + + def find_tagged( + self, tag: str, search_in: Literal["intervals", "epochs", "eras"] + ) -> list: + """ + Return all objects containing a specific tag. + The `search_in` parameter specifies which type of object to search for. + + Parameters + ---------- + tag : str + The tag to search for. + search_in : Literal["intervals", "eras", "epochs"] + What type of objects to search for. + Note that orphaned epochs will not be searched. + + Returns + ------ + found : list + A list of obects of type `search_in` containing this tag. + """ + if search_in == "intervals": + return [ival for ival in self.intervals.values() if field in ival.tags] + elif search_in == "epochs": + return [ + epoch for epoch in self.epochs.values() if field in epoch._internal.tags + ] + elif search_in == "eras": + return [era for era in self.eras.values() if field in era._internal.tags] + else: + raise ValueError(f"Invalid search_in: {search_in}") + @overload def find_data_field(self, field: str, search_in: Literal["eras"]) -> list[Era]: ... From a77a99b725280248767f146ec8bcde3e0a781ed3 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Fri, 10 Apr 2026 15:48:14 -0400 Subject: [PATCH 09/10] fix: switch to reduce --- sotodlib/utils/epochs.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index 4ea0590b7..0d3771f7f 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -1,13 +1,14 @@ import re from copy import deepcopy from dataclasses import dataclass, field -from itertools import accumulate +from functools import reduce from operator import add, and_, mul, or_ from typing import Any, Literal, Optional, Self, cast, overload import yaml from deepdiff import DeepDiff + @dataclass class Interval: """ @@ -181,7 +182,7 @@ class Epoch: def __post_init__(self): self._internal = cast( - Interval, accumulate(self.covers, mul if self.strict else or_) + Interval, reduce(mul if self.strict else or_, self.covers) ) def __repr__(self) -> str: @@ -226,9 +227,7 @@ def check_data(self, field: str, strict: bool = True) -> bool: except ValueError: return False return True - return cast( - bool, accumulate([field in ival.data for ival in self.covers], and_) - ) + return cast(bool, reduce(and_, [field in ival.data for ival in self.covers])) @dataclass @@ -256,9 +255,7 @@ class Era: def __post_init__(self): self._internal = cast( Interval, - accumulate( - [e._internal for e in self.epochs], add if self.strict else and_ - ), + reduce(add if self.strict else and_, [e._internal for e in self.epochs]), ) def __setattr__(self, name, value): @@ -281,7 +278,7 @@ def check_data(self, field: str, strict: bool = True) -> bool: """ return cast( - bool, accumulate([e.check_data(field, strict) for e in self.epochs], and_) + bool, reduce(and_, [e.check_data(field, strict) for e in self.epochs]) ) @@ -308,9 +305,7 @@ def find_tagged(self, tag: str, search_in: Literal["epochs"]) -> list[Epoch]: ... @overload - def find_tagged( - self, tag: str, search_in: Literal["intervals"] - ) -> list[Interval]: + def find_tagged(self, tag: str, search_in: Literal["intervals"]) -> list[Interval]: ... def find_tagged( From 3cd40d84d4a8437ea4ca2c43c0305cc00c1ee1eb Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Tue, 14 Apr 2026 15:04:01 -0400 Subject: [PATCH 10/10] fix: many small bugfixes in loading and initialization --- sotodlib/utils/epochs.py | 47 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/sotodlib/utils/epochs.py b/sotodlib/utils/epochs.py index 0d3771f7f..012ea2005 100644 --- a/sotodlib/utils/epochs.py +++ b/sotodlib/utils/epochs.py @@ -46,13 +46,13 @@ def __getitem__(self: Self, val) -> Self: if not isinstance(val, slice): val = slice(val) if val.start is not None: - if self.start < val.start: + if self.start > val.start: raise IndexError( f"Start of slice out of range, interval starts at {self.start} but slice starts at {val.start}!" ) to_ret.start = val.start if val.stop is not None: - if self.stop > val.stop: + if self.stop < val.stop: raise IndexError( f"End of slice out of range, interval stops at {self.stop} but slice stops at {val.stop}!" ) @@ -91,15 +91,13 @@ def combine( combined.start = min(self.start, tocombine.start) combined.stop = max(self.stop, tocombine.stop) - if not same_tag and self.tags != tocombine.tags: - raise ValueError("Can't combine without identical tags with same_tag=False") + if same_tag and self.tags != tocombine.tags: + raise ValueError("Can't combine without identical tags with same_tag=True") combined.tags = self.tags.union(tocombine.tags) diff = DeepDiff(self.data, tocombine.data) - if not same_data and len(diff) != 0: - raise ValueError( - "Can't combine without identical data with same_data=False" - ) + if same_data and len(diff) != 0: + raise ValueError("Can't combine without identical data with same_data=True") combined.data.update(tocombine.data) combined.name = self.name + "+" + tocombine.name @@ -195,7 +193,7 @@ def __repr__(self) -> str: ) def __setattr__(self, name, value): - if name == "covers" or name == "strict": + if (name == "covers" or name == "strict") and "_internal" in self.__dict__: self.__post_init__() return super().__setattr__(name, value) @@ -207,7 +205,7 @@ def __eq__(self, other): def check_data(self, field: str, strict: bool = True) -> bool: """ Check that all `Interval`s in this `Epoch` contain a certain data field. - Optinally check that they also contain the same value for the field. + Optinally check that they each contain the same value for the field across all intervals. Parameters ---------- @@ -259,7 +257,7 @@ def __post_init__(self): ) def __setattr__(self, name, value): - if name == "epochs" or name == "strict": + if (name == "epochs" or name == "strict") and "_internal" in self.__dict__: self.__post_init__() return super().__setattr__(name, value) @@ -329,13 +327,13 @@ def find_tagged( A list of obects of type `search_in` containing this tag. """ if search_in == "intervals": - return [ival for ival in self.intervals.values() if field in ival.tags] + return [ival for ival in self.intervals.values() if tag in ival.tags] elif search_in == "epochs": return [ - epoch for epoch in self.epochs.values() if field in epoch._internal.tags + epoch for epoch in self.epochs.values() if tag in epoch._internal.tags ] elif search_in == "eras": - return [era for era in self.eras.values() if field in era._internal.tags] + return [era for era in self.eras.values() if tag in era._internal.tags] else: raise ValueError(f"Invalid search_in: {search_in}") @@ -539,11 +537,12 @@ def load(cls, fpath: str) -> Self: intervals = cfg["intervals"] eras = {} data = {} - for key, val in cfg: + for key, val in cfg.items(): if key == "intervals": continue if key[0] == "_": data[key[1:]] = val + continue eras[key] = val # Load intervals @@ -559,10 +558,11 @@ def load(cls, fpath: str) -> Self: name, interval.get("start", 0), interval.get("stop", 20000000000), - interval.get("tags", []), + set(interval.get("tags", [])), interval_data, ) interval_dict[str(ival)] = ival + intervals[name] = ival # Load epochs epoch_dict = {} @@ -571,10 +571,19 @@ def load(cls, fpath: str) -> Self: strict_epoch = era.get("strict_epochs", True) epochs = [] for name, ec in era.items(): + if name in ["strict_era", "strict_epochs"]: + continue covers = [] for cname in ec.get("covers", []): - iname, slstr = cname.split("[") - slstr = slstr[:-1] + csplt = cname.split("[") + if len(csplt) == 1: + iname = cname + slstr = ":" + elif len(csplt) == 2: + iname, slstr = csplt + slstr = slstr[:-1] + else: + raise ValueError(f"Invalid cover string {cname}") ival = intervals[iname] interval_dict[str(ival)] = ival slc = slice( @@ -584,7 +593,7 @@ def load(cls, fpath: str) -> Self: ) ) covers += [ival[slc]] - epoch = Epoch(name, ename, tuple(covers), strict_epoch) + epoch = Epoch(name, ename, tuple(covers), strict=strict_epoch) epochs += [epoch] epoch_dict[f"{ename}.{name}"] = epoch eras[ename] = Era(ename, tuple(epochs), strict_era)