Efficiency improvements

README.md

@@ -30,3 +30,29 @@ UI development:
 ```terminal
 export VITE_API_HOST=http://localhost:8000
 ```
+
+## Testing
+
+Black-box compliance tests (against any deployment):
+
+```
+pytest tests/api/ --api_root=https://api.bedbase.org
+```
+
+Integration tests (ephemeral Postgres + Qdrant + in-process TestClient, ~30s):
+
+```
+./tests/scripts/test-integration.sh
+```
+
+Manual service control:
+
+```
+./tests/scripts/services.sh start
+RUN_INTEGRATION_TESTS=true pytest tests/integration/ tests/api/
+./tests/scripts/services.sh stop
+```
+
+Integration tests require network access on first boot (bbconf downloads a
+licenses CSV from GitHub and may pull HuggingFace models unless
+`BEDHOST_INIT_ML=false` is set; `test-integration.sh` sets it).

bedhost/_version.py

@@ -1 +1 @@
-__version__ = "0.12.7"
+__version__ = "0.12.8"

bedhost/dependencies.py

@@ -1,25 +1,68 @@
-from cachetools import TTLCache
+from statistics import StatisticsError
 from bbconf.bbagent import BedBaseAgent
-from bbconf.models.base_models import FileStats, UsageModel
+from bbconf.models.base_models import BinValues, FileStats, GEOStatistics, UsageModel
 from bedboss.refgenome_validator.main import ReferenceValidator
 from fastapi import Request
-
-_stats_cache: TTLCache = TTLCache(maxsize=2, ttl=14 * 24 * 60 * 60)
+from . import _LOGGER
 
 
 def get_bbagent(request: Request) -> BedBaseAgent:
     return request.app.state.bbagent
 
 
-def get_usage_data(request: Request) -> UsageModel:
-    return request.app.state.usage_data
+# def get_usage_data(request: Request) -> UsageModel:
+#     return request.app.state.usage_data
 
 
 def get_ref_validator(request: Request) -> ReferenceValidator:
     return request.app.state.ref_validator
 
 
-def fetch_detailed_stats(bbagent: BedBaseAgent, concise: bool = False) -> FileStats:
-    if concise not in _stats_cache:
-        _stats_cache[concise] = bbagent.get_detailed_stats(concise=concise)
-    return _stats_cache[concise]
+def _empty_bin_values() -> BinValues:
+    return BinValues(bins=[], counts=[], mean=0.0, median=0.0)
+
+
+def _empty_file_stats() -> FileStats:
+    """Return a well-formed FileStats with zeroed-out bins.
+
+    Used as a fallback when the underlying database has no bed records, in
+    which case bbconf's ``statistics.mean`` call on an empty list raises
+    ``StatisticsError``.
+    """
+    empty_bins = _empty_bin_values()
+    return FileStats(
+        bed_compliance={},
+        data_format={},
+        file_genome={},
+        file_organism={},
+        file_assay={},
+        cell_line={},
+        geo_status={},
+        bed_comments={},
+        mean_region_width=empty_bins,
+        file_size=empty_bins,
+        number_of_regions=empty_bins,
+        geo=GEOStatistics(
+            number_of_files={},
+            cumulative_number_of_files={},
+            file_sizes=empty_bins,
+        ),
+    )
+
+
+def fetch_detailed_stats(
+    request: Request, bbagent: BedBaseAgent, concise: bool = False
+) -> FileStats:
+    """
+    Fetch detailed file statistics from the BedBaseAgent.
+    """
+
+    if concise not in request.app.state.detailed_stats:
+        _LOGGER.info("Stats are not cached, fetching...")
+        try:
+            request.app.state.detailed_stats[concise] = bbagent.get_detailed_stats(
+                concise=concise
+            )
+        except StatisticsError:
+            return _empty_file_stats()
+    return request.app.state.detailed_stats[concise]

bedhost/helpers.py

@@ -66,10 +66,15 @@ def attach_routers(app):
 
 
 def configure(bbconf_file_path: str) -> BedBaseAgent:
+    # Respect BEDHOST_INIT_ML for CI/smoke deployments that don't need the
+    # ML models (dense/sparse encoders, UMAP, region2vec) loaded. Default
+    # is to initialize them (unchanged behavior for production).
+    init_ml_env = os.environ.get("BEDHOST_INIT_ML", "true").lower()
+    init_ml = init_ml_env not in ("0", "false", "no")
     try:
         # bbconf_file_path = os.environ.get("BEDBASE_CONFIG") or None
-        _LOGGER.info(f"Loading config: '{bbconf_file_path}'")
-        bbc = BedBaseAgent(bbconf_file_path)
+        _LOGGER.info(f"Loading config: '{bbconf_file_path}' (init_ml={init_ml})")
+        bbc = BedBaseAgent(bbconf_file_path, init_ml=init_ml)
     except Exception as e:
         raise BedHostException(f"Bedbase config was not provided or is incorrect: {e}")
     return bbc
@@ -125,17 +130,23 @@ async def wrapper(*args, **kwargs):
                     else:
                         usage_data.files[file_path] = 1
             elif event == "bed_search":
-                query = kwargs.get("query").strip()
-                if query in usage_data.bed_search:
-                    usage_data.bed_search[query] += 1
-                else:
-                    usage_data.bed_search[query] = 1
+                raw_query = kwargs.get("query")
+                # /v1/bedset/list accepts query=None; skip usage tracking in
+                # that case rather than crashing on ``None.strip()``.
+                if raw_query is not None:
+                    query = raw_query.strip()
+                    if query in usage_data.bed_search:
+                        usage_data.bed_search[query] += 1
+                    else:
+                        usage_data.bed_search[query] = 1
             elif event == "bedset_search":
-                query = kwargs.get("query").strip()
-                if query in usage_data.bedset_search:
-                    usage_data.bedset_search[query] += 1
-                else:
-                    usage_data.bedset_search[query] = 1
+                raw_query = kwargs.get("query")
+                if raw_query is not None:
+                    query = raw_query.strip()
+                    if query in usage_data.bedset_search:
+                        usage_data.bedset_search[query] += 1
+                    else:
+                        usage_data.bedset_search[query] = 1
             elif event == "bed_meta":
                 bed_id = kwargs.get("bed_id")
                 if bed_id in usage_data.bed_meta:

bedhost/main.py

@@ -14,9 +14,12 @@
 )
 from bedboss.refgenome_validator.main import ReferenceValidator
 
-from fastapi import FastAPI, Request
+from cachetools import TTLCache
+
+from fastapi import FastAPI, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
 from . import _LOGGER
@@ -84,6 +87,8 @@ async def lifespan(app: FastAPI):
     app.state.bbagent = configure(bbconf_file_path)
     app.state.usage_data = init_model_usage()
 
+    app.state.detailed_stats: TTLCache = TTLCache(maxsize=2, ttl=14 * 24 * 60 * 60)
+
     # Respect BEDHOST_INIT_ML for CI/smoke deployments that don't need the
     # reference genome validator loaded. Default is to initialize it.
     init_ml_env = os.environ.get("BEDHOST_INIT_ML", "true").lower()
@@ -107,6 +112,7 @@ async def lifespan(app: FastAPI):
     app.state.scheduler = scheduler
 
     try:
+        _LOGGER.info("Starting app ...")
         yield
     finally:
         app.state.scheduler.shutdown(wait=False)
@@ -117,6 +123,7 @@ async def lifespan(app: FastAPI):
     description="BED file/sets statistics and image server API",
     version=bedhost_version,
     docs_url="/v1/docs",
+    openapi_url="/v1/openapi.json",
     openapi_tags=tags_metadata,
     lifespan=lifespan,
 )
@@ -137,6 +144,12 @@ async def lifespan(app: FastAPI):
     allow_headers=["*"],
 )
 
+app.mount(
+    "/static",
+    StaticFiles(directory=str(Path(__file__).parent / "static")),
+    name="static",
+)
+
 templates = Jinja2Templates(directory=str(Path(__file__).parent / "templates"))
 templates.env.autoescape = False
 
@@ -160,13 +173,19 @@ async def changelog(request: Request):
 
 
 @app.get("/")
-def lending_page():
+def landing_page():
     return RedirectResponse(url="v1/")
 
 
 def render_markdown(filename: str, request: Request):
-    with open(os.path.join(STATIC_PATH, filename), "r", encoding="utf-8") as input_file:
-        text = input_file.read()
+    path = os.path.join(STATIC_PATH, filename)
+    try:
+        with open(path, "r", encoding="utf-8") as input_file:
+            text = input_file.read()
+    except FileNotFoundError:
+        # e.g. docs/changelog.md was removed from the repo in 2024 but the
+        # route is still registered. Return 404 rather than a 500.
+        raise HTTPException(status_code=404, detail=f"{filename} not found")
     content = markdown.markdown(text)
     return templates.TemplateResponse(request, "page.html", {"content": content})
 

bedhost/routers/base_api.py

@@ -48,14 +48,15 @@ async def get_bedbase_db_stats(
     summary="Get detailed statistics for BEDbase platform, including number of files for each genome",
     response_model=FileStats,
 )
-async def get_detailed_stats(
+def get_detailed_stats(
+    request: Request,
     concise: bool = False,
     bbagent: BedBaseAgent = Depends(get_bbagent),
 ):
     """
     Returns detailed statistics
     """
-    return fetch_detailed_stats(bbagent, concise=concise)
+    return fetch_detailed_stats(request, bbagent=bbagent, concise=concise)
 
 
 @router.get(

bedhost/routers/bed_api.py

@@ -101,7 +101,7 @@ async def list_beds(
     summary="Get metadata for a single BED record",
     response_model=BedMetadataAll,
     response_model_by_alias=False,
-    description=f"Example\n " f"bed_id: {EXAMPLE_BED}",
+    description=f"Example\n bed_id: {EXAMPLE_BED}",
 )
 @count_requests(event="bed_meta")
 async def get_bed_metadata(
@@ -531,6 +531,15 @@ async def text_to_bed_search(
         f"Searching for: '{query}' with limit='{limit}' and offset='{offset}' and genome='{genome}' and assay='{assay}'"
     )
 
+    # Hybrid search depends on the dense encoder, which isn't loaded when
+    # bbconf is initialized with ``init_ml=False`` (CI / BEDHOST_INIT_ML=false).
+    # Return 503 up-front rather than a 500 AttributeError from deep in bbconf.
+    if getattr(bbagent.config, "dense_encoder", None) is None:
+        raise HTTPException(
+            status_code=503,
+            detail="Text search unavailable (ML models disabled)",
+        )
+
     spaceless_query = query.replace(" ", "")
     if len(spaceless_query) == 32 and spaceless_query == query:
         try:

bedhost/templates/page.html

@@ -42,9 +42,7 @@
     <div class="container">
         <footer class="d-flex flex-wrap justify-content-between align-items-center py-1  my-4 border-top">
             <div class="text-end">
-                <img src="https://databio.org/images/logo/University_of_Virginia_Rotunda_logo.svg" alt="databio"  height="45">
-                <img src="https://databio.org/images/logo/University_of_Virginia_logo.svg" alt="databio" height="50">
-                <img src="https://databio.org/images/logo/UVA_CPHG.svg" alt="databio"  height="60" class="ps-3">
+                <img src="/static/UVA_CPHG.png" alt="UVA Center for Public Health Genomics" height="45" class="ps-3">
             </div>
         </footer>
     </div>

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+testpaths = tests
+markers =
+    require_service: marks test as requiring a running bedhost service (deselect with '-m "not require_service"')

requirements/requirements-all.txt

@@ -1,5 +1,5 @@
 # bbconf @ git+https://github.com/databio/bbconf.git@dev#egg=bbconf
-bbconf>=0.14.12
+bbconf>=0.14.13
 fastapi>=0.103.0
 sentence_transformers>=5.0
 logmuse>=0.2.7

tests/api/__init__.py

@@ -0,0 +1 @@
+"""Compliance tests for bedhost API."""

tests/api/conftest.py

@@ -0,0 +1,27 @@
+"""API test fixtures and constants."""
+
+import pytest
+
+
+@pytest.fixture(scope="session")
+def example_bed_id(api_root):
+    """Get an example BED ID from the server."""
+    try:
+        res = api_root.get("/v1/bed/example", timeout=3)
+        if res.status_code == 200:
+            return res.json().get("id")
+    except Exception:
+        pass
+    return None
+
+
+@pytest.fixture(scope="session")
+def example_bedset_id(api_root):
+    """Get an example BEDSET ID from the server."""
+    try:
+        res = api_root.get("/v1/bedset/example", timeout=3)
+        if res.status_code == 200:
+            return res.json().get("id")
+    except Exception:
+        pass
+    return None