diff --git a/bbconf/alembic/versions/845d978eac7d_added_genomic_distribution_json_plots.py b/bbconf/alembic/versions/845d978eac7d_added_genomic_distribution_json_plots.py new file mode 100644 index 0000000..fc486f6 --- /dev/null +++ b/bbconf/alembic/versions/845d978eac7d_added_genomic_distribution_json_plots.py @@ -0,0 +1,60 @@ +"""Added genomic distribution json plots + +Revision ID: 845d978eac7d +Revises: 8b0b706d0827 +Create Date: 2026-08-16 18:02:03.058352 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "845d978eac7d" +down_revision: Union[str, None] = "8b0b706d0827" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + + op.drop_column("bed", "pephub") + op.add_column( + "bed_stats", + sa.Column( + "distributions", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Full distribution arrays from gtars genomicdist (JSONB)", + ), + ) + op.add_column( + "bedsets", + sa.Column( + "bedset_stats", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Pre-aggregated distribution statistics from gtars (JSONB)", + ), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("bedsets", "bedset_stats") + op.drop_column("bed_stats", "distributions") + op.add_column( + "bed", + sa.Column( + "pephub", + sa.BOOLEAN(), + autoincrement=False, + nullable=False, + comment="Whether sample was added to pephub", + ), + ) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index fea2273..f66880f 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -129,19 +129,7 @@ def _read_config_file(config_path: str) -> ConfigFile: """ _config = yacman.YAMLConfigManager.from_yaml_file(filepath=config_path).exp - - config_dict = {} - for field_name, annotation in ConfigFile.model_fields.items(): - try: - config_dict[field_name] = annotation.annotation( - **_config.get(field_name) - ) - except TypeError: - # TODO: this should be more specific - config_dict[field_name] = annotation.annotation() - - return ConfigFile(**config_dict) - # return ConfigFile.from_yaml(Path(config_path)) + return ConfigFile(**_config) @property def config(self) -> ConfigFile: diff --git a/bbconf/config_parser/models.py b/bbconf/config_parser/models.py index c2f5ca7..69cd5e5 100644 --- a/bbconf/config_parser/models.py +++ b/bbconf/config_parser/models.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from typing import Literal from pydantic import BaseModel, ConfigDict, computed_field, field_validator from yacman import load_yaml @@ -72,14 +73,14 @@ class ConfigPath(BaseModel): class AccessMethodsStruct(BaseModel): type: str - description: str = None + description: str | None = None prefix: str class AccessMethods(BaseModel): - http: AccessMethodsStruct = None - s3: AccessMethodsStruct = None - local: AccessMethodsStruct = None + http: AccessMethodsStruct | None = None + s3: AccessMethodsStruct | None = None + local: AccessMethodsStruct | None = None class ConfigS3(BaseModel): @@ -118,13 +119,25 @@ def modify_access(self) -> bool: return False +class ConfigAnalysis(BaseModel): + """Analysis backend configuration. + + Controls which statistics engine is used for BED file analysis. + """ + + backend: Literal["r", "gtars"] = "r" + + model_config = ConfigDict(extra="forbid") + + class ConfigFile(BaseModel): database: ConfigDB - qdrant: ConfigQdrant = None + qdrant: ConfigQdrant | None = None server: ConfigServer path: ConfigPath - access_methods: AccessMethods = None - s3: ConfigS3 = None + access_methods: AccessMethods | None = None + s3: ConfigS3 | None = None + analysis: ConfigAnalysis | None = ConfigAnalysis() model_config = ConfigDict(extra="allow") diff --git a/bbconf/config_parser/utils.py b/bbconf/config_parser/utils.py index 46f0a7f..10376d0 100644 --- a/bbconf/config_parser/utils.py +++ b/bbconf/config_parser/utils.py @@ -27,7 +27,7 @@ def config_analyzer(config_path: str) -> bool: _LOGGER.info(f"Analyzing the configuration file {config_path}...") - _config = yacman.YAMLConfigManager(filepath=config_path).exp + _config = yacman.YAMLConfigManager.from_yaml_file(filepath=config_path).exp config_dict = {} for field_name, annotation in ConfigFile.model_fields.items(): diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index e49423c..535fc55 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -20,7 +20,7 @@ select, text, ) -from sqlalchemy.dialects.postgresql import ARRAY, JSON +from sqlalchemy.dialects.postgresql import ARRAY, JSON, JSONB from sqlalchemy.engine import URL, Engine, create_engine from sqlalchemy.event import listens_for from sqlalchemy.exc import IntegrityError, ProgrammingError @@ -282,6 +282,12 @@ class BedStats(Base): promotercore_percentage: Mapped[Optional[float]] tssdist: Mapped[Optional[float]] + distributions: Mapped[Optional[dict]] = mapped_column( + JSONB, + nullable=True, + comment="Full distribution arrays from gtars genomicdist (JSONB)", + ) + bed: Mapped["Bed"] = relationship("Bed", back_populates="stats") __table_args__ = ( @@ -373,6 +379,11 @@ class BedSets(Base): bedset_standard_deviation: Mapped[Optional[dict]] = mapped_column( JSON, comment="Median values of the bedset" ) + bedset_stats: Mapped[Optional[dict]] = mapped_column( + JSONB, + nullable=True, + comment="Pre-aggregated distribution statistics from gtars (JSONB)", + ) bedfile_count: Mapped[int] = mapped_column( default=0, comment="Number of bedfiles in the bedset (denormalized count)" diff --git a/bbconf/models/bed_models.py b/bbconf/models/bed_models.py index 8580b9b..1103594 100644 --- a/bbconf/models/bed_models.py +++ b/bbconf/models/bed_models.py @@ -74,6 +74,8 @@ class BedStatsModel(BaseModel): promoterprox_frequency: float | None = None promoterprox_percentage: float | None = None + distributions: dict | None = None + model_config = ConfigDict(extra="ignore", populate_by_name=True) diff --git a/bbconf/models/bedset_models.py b/bbconf/models/bedset_models.py index 3f3d852..8acbf14 100644 --- a/bbconf/models/bedset_models.py +++ b/bbconf/models/bedset_models.py @@ -1,4 +1,5 @@ import datetime +from typing import Optional from pydantic import BaseModel, ConfigDict, model_validator @@ -7,10 +8,34 @@ class BedSetStats(BaseModel): + """Bedset statistics: mean/sd of scalar columns. + + Populated from bedset_means and bedset_standard_deviation database columns. + """ + mean: BedStatsModel = None sd: BedStatsModel = None +class BedSetDistributions(BaseModel): + """Collection-level aggregated distribution statistics for a bedset. + + Stored in the bedset_stats JSONB database column. Populated when + member bed files have been processed with the gtars analysis backend. + """ + + n_files: int = 0 + composition: Optional[dict] = None + scalar_summaries: Optional[dict] = None + tss_histogram: Optional[dict] = None + widths_histogram: Optional[dict] = None + neighbor_distances: Optional[dict] = None + gc_content: Optional[dict] = None + region_distribution: Optional[dict] = None + partitions: Optional[dict] = None + chromosome_summaries: Optional[dict] = None + + class BedSetPlots(BaseModel): region_commonality: FileModel = None @@ -24,6 +49,7 @@ class BedSetMetadata(BaseModel): submission_date: datetime.datetime = None last_update_date: datetime.datetime = None statistics: BedSetStats | None = None + distributions: BedSetDistributions | None = None plots: BedSetPlots | None = None description: str = None summary: str = None diff --git a/bbconf/modules/bedsets.py b/bbconf/modules/bedsets.py index facfe1e..5b0d38c 100644 --- a/bbconf/modules/bedsets.py +++ b/bbconf/modules/bedsets.py @@ -420,7 +420,11 @@ def _calculate_statistics(self, bed_ids: list[str]) -> BedSetStats: """ _LOGGER.info("Calculating bedset statistics") - numeric_columns = BedStatsModel.model_fields + numeric_columns = [ + name + for name, field in BedStatsModel.model_fields.items() + if field.annotation in (float, float | None) + ] bedset_sd = {} bedset_mean = {}