From 6e0ef984f7a6546337504669001bd79d1debf789 Mon Sep 17 00:00:00 2001 From: Sam Park Date: Fri, 27 Mar 2026 12:53:13 -0400 Subject: [PATCH 1/5] Add modular analysis backend config and distribution schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ConfigAnalysis model with backend: "r" | "gtars" (defaults to "r") to support switching analysis engines via config. Add nullable JSONB columns (distributions on BedStats, bedset_stats on BedSets) and BedSetDistributions model for gtars output. Existing code and behavior unchanged — pure additive schema and config additions. Co-Authored-By: Claude Opus 4.6 (1M context) --- bbconf/config_parser/models.py | 13 +++++++++++++ bbconf/db_utils.py | 11 +++++++++++ bbconf/models/bed_models.py | 2 ++ bbconf/models/bedset_models.py | 26 ++++++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/bbconf/config_parser/models.py b/bbconf/config_parser/models.py index 8f1d37ec..bf2f39a4 100644 --- a/bbconf/config_parser/models.py +++ b/bbconf/config_parser/models.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from typing import Literal from pydantic import BaseModel, ConfigDict, computed_field, field_validator from yacman import load_yaml @@ -126,6 +127,17 @@ class ConfigPepHubClient(BaseModel): tag: str | None = DEFAULT_PEPHUB_TAG +class ConfigAnalysis(BaseModel): + """Analysis backend configuration. + + Controls which statistics engine is used for BED file analysis. + """ + + backend: Literal["r", "gtars"] = "r" + + model_config = ConfigDict(extra="forbid") + + class ConfigFile(BaseModel): database: ConfigDB qdrant: ConfigQdrant = None @@ -134,6 +146,7 @@ class ConfigFile(BaseModel): access_methods: AccessMethods = None s3: ConfigS3 = None phc: ConfigPepHubClient = None + analysis: ConfigAnalysis = ConfigAnalysis() model_config = ConfigDict(extra="allow") diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index 050c7e61..8261154b 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -255,6 +255,12 @@ class BedStats(Base): promotercore_percentage: Mapped[Optional[float]] tssdist: Mapped[Optional[float]] + distributions: Mapped[Optional[dict]] = mapped_column( + JSON, + nullable=True, + comment="Full distribution arrays from gtars genomicdist (JSONB)", + ) + bed: Mapped["Bed"] = relationship("Bed", back_populates="stats") @@ -337,6 +343,11 @@ class BedSets(Base): bedset_standard_deviation: Mapped[Optional[dict]] = mapped_column( JSON, comment="Median values of the bedset" ) + bedset_stats: Mapped[Optional[dict]] = mapped_column( + JSON, + nullable=True, + comment="Pre-aggregated distribution statistics from gtars (JSONB)", + ) bedfiles: Mapped[list["BedFileBedSetRelation"]] = relationship( "BedFileBedSetRelation", back_populates="bedset", cascade="all, delete-orphan" diff --git a/bbconf/models/bed_models.py b/bbconf/models/bed_models.py index d16d0c51..fec4a11b 100644 --- a/bbconf/models/bed_models.py +++ b/bbconf/models/bed_models.py @@ -74,6 +74,8 @@ class BedStatsModel(BaseModel): promoterprox_frequency: float | None = None promoterprox_percentage: float | None = None + distributions: dict | None = None + model_config = ConfigDict(extra="ignore", populate_by_name=True) diff --git a/bbconf/models/bedset_models.py b/bbconf/models/bedset_models.py index ca074a99..45dae379 100644 --- a/bbconf/models/bedset_models.py +++ b/bbconf/models/bedset_models.py @@ -1,4 +1,5 @@ import datetime +from typing import Optional from pydantic import BaseModel, ConfigDict, model_validator @@ -7,10 +8,34 @@ class BedSetStats(BaseModel): + """Bedset statistics: mean/sd of scalar columns. + + Populated from bedset_means and bedset_standard_deviation database columns. + """ + mean: BedStatsModel = None sd: BedStatsModel = None +class BedSetDistributions(BaseModel): + """Collection-level aggregated distribution statistics for a bedset. + + Stored in the bedset_stats JSONB database column. Populated when + member bed files have been processed with the gtars analysis backend. + """ + + n_files: int = 0 + composition: Optional[dict] = None + scalar_summaries: Optional[dict] = None + tss_histogram: Optional[dict] = None + widths_histogram: Optional[dict] = None + neighbor_distances: Optional[dict] = None + gc_content: Optional[dict] = None + region_distribution: Optional[dict] = None + partitions: Optional[dict] = None + chromosome_summaries: Optional[dict] = None + + class BedSetPlots(BaseModel): region_commonality: FileModel = None @@ -24,6 +49,7 @@ class BedSetMetadata(BaseModel): submission_date: datetime.datetime = None last_update_date: datetime.datetime = None statistics: BedSetStats | None = None + distributions: BedSetDistributions | None = None plots: BedSetPlots | None = None description: str = None summary: str = None From f0ecd82cb7d10a72b851107f459d061b6e134a5c Mon Sep 17 00:00:00 2001 From: Sam Park Date: Fri, 3 Apr 2026 14:44:24 -0400 Subject: [PATCH 2/5] Fix bedset stats aggregation: skip non-numeric columns The stddev/avg loop over BedStatsModel.model_fields was hitting the new JSON `distributions` column, causing PostgreSQL to fail with "function stddev(json) does not exist". Filter to float fields only. Co-Authored-By: Claude Opus 4.6 (1M context) --- bbconf/modules/bedsets.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bbconf/modules/bedsets.py b/bbconf/modules/bedsets.py index 83566ac3..cad38dce 100644 --- a/bbconf/modules/bedsets.py +++ b/bbconf/modules/bedsets.py @@ -427,7 +427,11 @@ def _calculate_statistics(self, bed_ids: list[str]) -> BedSetStats: """ _LOGGER.info("Calculating bedset statistics") - numeric_columns = BedStatsModel.model_fields + numeric_columns = [ + name + for name, field in BedStatsModel.model_fields.items() + if field.annotation in (float, float | None) + ] bedset_sd = {} bedset_mean = {} From a9c136742918ccd9c3973ad3374238e2493eb908 Mon Sep 17 00:00:00 2001 From: khoroshevskyi Date: Sun, 16 Aug 2026 16:39:46 -0400 Subject: [PATCH 3/5] updated gtars distribution new schema --- bbconf/config_parser/models.py | 21 +++++++-------------- bbconf/db_utils.py | 6 +++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/bbconf/config_parser/models.py b/bbconf/config_parser/models.py index 9b060dbb..695b0d69 100644 --- a/bbconf/config_parser/models.py +++ b/bbconf/config_parser/models.py @@ -73,14 +73,14 @@ class ConfigPath(BaseModel): class AccessMethodsStruct(BaseModel): type: str - description: str = None + description: str | None = None prefix: str class AccessMethods(BaseModel): - http: AccessMethodsStruct = None - s3: AccessMethodsStruct = None - local: AccessMethodsStruct = None + http: AccessMethodsStruct | None = None + s3: AccessMethodsStruct | None = None + local: AccessMethodsStruct | None = None class ConfigS3(BaseModel): @@ -118,13 +118,6 @@ def modify_access(self) -> bool: ) return False - -class ConfigPepHubClient(BaseModel): - namespace: str | None = DEFAULT_PEPHUB_NAMESPACE - name: str | None = DEFAULT_PEPHUB_NAME - tag: str | None = DEFAULT_PEPHUB_TAG - - class ConfigAnalysis(BaseModel): """Analysis backend configuration. @@ -137,11 +130,11 @@ class ConfigAnalysis(BaseModel): class ConfigFile(BaseModel): database: ConfigDB - qdrant: ConfigQdrant = None + qdrant: ConfigQdrant | None = None server: ConfigServer path: ConfigPath - access_methods: AccessMethods = None - s3: ConfigS3 = None + access_methods: AccessMethods | None = None + s3: ConfigS3 | None = None analysis: ConfigAnalysis = ConfigAnalysis() model_config = ConfigDict(extra="allow") diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index 56aa19f6..535fc55f 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -20,7 +20,7 @@ select, text, ) -from sqlalchemy.dialects.postgresql import ARRAY, JSON +from sqlalchemy.dialects.postgresql import ARRAY, JSON, JSONB from sqlalchemy.engine import URL, Engine, create_engine from sqlalchemy.event import listens_for from sqlalchemy.exc import IntegrityError, ProgrammingError @@ -283,7 +283,7 @@ class BedStats(Base): tssdist: Mapped[Optional[float]] distributions: Mapped[Optional[dict]] = mapped_column( - JSON, + JSONB, nullable=True, comment="Full distribution arrays from gtars genomicdist (JSONB)", ) @@ -380,7 +380,7 @@ class BedSets(Base): JSON, comment="Median values of the bedset" ) bedset_stats: Mapped[Optional[dict]] = mapped_column( - JSON, + JSONB, nullable=True, comment="Pre-aggregated distribution statistics from gtars (JSONB)", ) From 96c05bdae6c321d25634500e2b73a2031ca7dfda Mon Sep 17 00:00:00 2001 From: khoroshevskyi Date: Sun, 16 Aug 2026 17:44:50 -0400 Subject: [PATCH 4/5] Fixed config file initialization --- bbconf/config_parser/bedbaseconfig.py | 14 +------------- bbconf/config_parser/models.py | 4 +++- bbconf/config_parser/utils.py | 2 +- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index fea22738..f66880f2 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -129,19 +129,7 @@ def _read_config_file(config_path: str) -> ConfigFile: """ _config = yacman.YAMLConfigManager.from_yaml_file(filepath=config_path).exp - - config_dict = {} - for field_name, annotation in ConfigFile.model_fields.items(): - try: - config_dict[field_name] = annotation.annotation( - **_config.get(field_name) - ) - except TypeError: - # TODO: this should be more specific - config_dict[field_name] = annotation.annotation() - - return ConfigFile(**config_dict) - # return ConfigFile.from_yaml(Path(config_path)) + return ConfigFile(**_config) @property def config(self) -> ConfigFile: diff --git a/bbconf/config_parser/models.py b/bbconf/config_parser/models.py index 695b0d69..69cd5e57 100644 --- a/bbconf/config_parser/models.py +++ b/bbconf/config_parser/models.py @@ -118,6 +118,7 @@ def modify_access(self) -> bool: ) return False + class ConfigAnalysis(BaseModel): """Analysis backend configuration. @@ -128,6 +129,7 @@ class ConfigAnalysis(BaseModel): model_config = ConfigDict(extra="forbid") + class ConfigFile(BaseModel): database: ConfigDB qdrant: ConfigQdrant | None = None @@ -135,7 +137,7 @@ class ConfigFile(BaseModel): path: ConfigPath access_methods: AccessMethods | None = None s3: ConfigS3 | None = None - analysis: ConfigAnalysis = ConfigAnalysis() + analysis: ConfigAnalysis | None = ConfigAnalysis() model_config = ConfigDict(extra="allow") diff --git a/bbconf/config_parser/utils.py b/bbconf/config_parser/utils.py index 46f0a7f1..10376d09 100644 --- a/bbconf/config_parser/utils.py +++ b/bbconf/config_parser/utils.py @@ -27,7 +27,7 @@ def config_analyzer(config_path: str) -> bool: _LOGGER.info(f"Analyzing the configuration file {config_path}...") - _config = yacman.YAMLConfigManager(filepath=config_path).exp + _config = yacman.YAMLConfigManager.from_yaml_file(filepath=config_path).exp config_dict = {} for field_name, annotation in ConfigFile.model_fields.items(): From de184b0aad3bb3f0d21c126f6296727cfbac1256 Mon Sep 17 00:00:00 2001 From: khoroshevskyi Date: Sun, 16 Aug 2026 18:12:58 -0400 Subject: [PATCH 5/5] Added alembic migration for distribution --- ...d_added_genomic_distribution_json_plots.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 bbconf/alembic/versions/845d978eac7d_added_genomic_distribution_json_plots.py diff --git a/bbconf/alembic/versions/845d978eac7d_added_genomic_distribution_json_plots.py b/bbconf/alembic/versions/845d978eac7d_added_genomic_distribution_json_plots.py new file mode 100644 index 00000000..fc486f6f --- /dev/null +++ b/bbconf/alembic/versions/845d978eac7d_added_genomic_distribution_json_plots.py @@ -0,0 +1,60 @@ +"""Added genomic distribution json plots + +Revision ID: 845d978eac7d +Revises: 8b0b706d0827 +Create Date: 2026-08-16 18:02:03.058352 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "845d978eac7d" +down_revision: Union[str, None] = "8b0b706d0827" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + + op.drop_column("bed", "pephub") + op.add_column( + "bed_stats", + sa.Column( + "distributions", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Full distribution arrays from gtars genomicdist (JSONB)", + ), + ) + op.add_column( + "bedsets", + sa.Column( + "bedset_stats", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Pre-aggregated distribution statistics from gtars (JSONB)", + ), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("bedsets", "bedset_stats") + op.drop_column("bed_stats", "distributions") + op.add_column( + "bed", + sa.Column( + "pephub", + sa.BOOLEAN(), + autoincrement=False, + nullable=False, + comment="Whether sample was added to pephub", + ), + )