Add plugin support

e2e/docker/goosebit/goosebit-external-auth.yaml

@@ -26,7 +26,7 @@ poll_time: 00:00:05
 device_auth:
   enable: true
   mode: external
-  external_url: http://172.20.5.100:8000/api/v1/auth
+  external_url: http://authserver:8000/api/v1/auth
   external_mode: json
   external_json_key: token
 

e2e/docker/goosebit/goosebit.yaml

@@ -74,6 +74,6 @@ storage:
   s3:
     bucket: goosebit
     # region: us-east-1
-    endpoint_url: http://172.20.5.100:9000 # example for self-hosted min.io
+    endpoint_url: http://minio:9000 # example for self-hosted min.io
     access_key_id: minioadmin
     secret_access_key: minioadmin

e2e/docker/swupdate/swupdate.cfg

@@ -6,7 +6,7 @@ globals = {
 
 suricatta = {
     server = "hawkbit";
-    url = "http://172.20.5.102:60053";
+    url = "http://goosebit:60053";
     polldelay = 10;
     tenant = "DEFAULT";
 };

e2e/external_auth/docker-compose.yml

@@ -13,10 +13,6 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
-    networks:
-      goosebit_network:
-        ipv4_address: 172.20.5.100
-
   goosebit:
     container_name: goosebit
     build:
@@ -36,10 +32,6 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
-    networks:
-      goosebit_network:
-        ipv4_address: 172.20.5.102
-
   swupdate:
     container_name: swupdate
     build:
@@ -50,14 +42,3 @@ services:
     depends_on:
       goosebit:
         condition: service_healthy
-    networks:
-      goosebit_network:
-        ipv4_address: 172.20.5.103
-
-networks:
-  goosebit_network:
-    driver: bridge
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.20.5.0/24

e2e/s3/docker-compose.yml

@@ -14,10 +14,6 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
-    networks:
-      goosebit_network:
-        ipv4_address: 172.20.5.100
-
   goosebit:
     container_name: goosebit
     build:
@@ -35,10 +31,6 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
-    networks:
-      goosebit_network:
-        ipv4_address: 172.20.5.102
-
   swupdate:
     container_name: swupdate
     build:
@@ -49,14 +41,3 @@ services:
     depends_on:
       goosebit:
         condition: service_healthy
-    networks:
-      goosebit_network:
-        ipv4_address: 172.20.5.103
-
-networks:
-  goosebit_network:
-    driver: bridge
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.20.5.0/24

goosebit.yaml

@@ -49,6 +49,15 @@ poll_time: 00:01:00
 # Defaults to a randomized value. If this value is not set, user sessions will not persist when app restarts.
 #secret_key: my_very_top_secret_key123
 
+# A list of installed plugins that you want to enable in goosebit
+#plugins:
+#  - goosebit_simple_stats
+
+# A required setting in case you want to test the example plugin
+#simple_stats_show:
+#  - device_count
+#  - software_count
+
 ## Internal settings that usually don't need to be modified
 metrics:
   prometheus:

goosebit/__init__.py

@@ -13,8 +13,9 @@
 from starlette.exceptions import HTTPException as StarletteHTTPException
 from tortoise.exceptions import ValidationError
 
-from goosebit import api, db, ui, updater
+from goosebit import api, db, plugins, ui, updater
 from goosebit.auth import get_user_from_request, login_user, redirect_if_authenticated
+from goosebit.device_manager import DeviceManager
 from goosebit.settings import PWD_CXT, config
 from goosebit.ui.nav import nav
 from goosebit.ui.static import static
@@ -34,7 +35,6 @@ async def lifespan(_: FastAPI):
 
     if db_ready:
         yield
-
     await db.close()
 
 
@@ -63,6 +63,24 @@ async def lifespan(_: FastAPI):
 app.mount("/static", static, name="static")
 Instrumentor.instrument_app(app)
 
+for plugin in plugins.load():
+    if plugin.router is not None:
+        logger.info(f"Adding routing handler for plugin: {plugin.name}")
+        app.include_router(router=plugin.router, prefix=plugin.url_prefix)
+    if plugin.db_model_path is not None:
+        logger.info(f"Adding db handler for plugin: {plugin.name}")
+        db.config.add_models(plugin.db_model_path)
+    if plugin.static_files is not None:
+        logger.info(f"Adding static files handler for plugin: {plugin.name}")
+        app.mount(f"{plugin.url_prefix}/static", plugin.static_files, name=plugin.static_files_name)
+    if plugin.templates is not None:
+        logger.info(f"Adding template handler for plugin: {plugin.name}")
+        templates.add_template_handler(plugin.templates)
+    if plugin.update_source_hook is not None:
+        DeviceManager.add_update_source(plugin.update_source_hook)
+    if plugin.config_data_hook is not None:
+        DeviceManager.add_config_callback(plugin.config_data_hook)
+
 
 # Custom exception handler for Tortoise ValidationError
 @app.exception_handler(ValidationError)
@@ -128,7 +146,7 @@ async def logout(request: Request):
     return resp
 
 
-@app.get("/docs")
+@app.get("/docs", include_in_schema=False)
 async def swagger_docs(request: Request):
     return get_swagger_ui_html(
         title="gooseBit docs",

goosebit/db/config.py

@@ -4,11 +4,18 @@
 from goosebit.db.pg_ssl_context import PostgresSSLContext
 from goosebit.settings import config
 
+
+def add_models(models_path: str):
+    models.append(models_path)
+
+
+models = ["goosebit.db.models", "aerich.models"]
+
 TORTOISE_CONF = {
     "connections": {"default": config.db_uri},
     "apps": {
         "models": {
-            "models": ["goosebit.db.models", "aerich.models"],
+            "models": models,
         },
     },
 }

goosebit/device_manager.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
+import asyncio
 import re
 from enum import StrEnum
-from typing import Optional
+from typing import Any, Awaitable, Callable, Optional
 
 from aiocache import caches
+from fastapi.requests import Request
 
 from goosebit.db.models import (
     Device,
@@ -14,6 +16,7 @@
     UpdateModeEnum,
     UpdateStateEnum,
 )
+from goosebit.schema.updates import UpdateChunk
 
 caches.set_config(
     {
@@ -35,6 +38,9 @@ class HandlingType(StrEnum):
 class DeviceManager:
     _hardware_default = None
 
+    _update_sources: list[Callable[[Request, Device], Awaitable[tuple[HandlingType, UpdateChunk | None]]]] = []
+    _config_callbacks: list[Callable[[Device, dict[str, Any]], Awaitable[None]]] = []
+
     @staticmethod
     async def get_device(dev_id: str) -> Device:
         cache = caches.get("default")
@@ -115,11 +121,23 @@ async def update_feed(device: Device, feed: str):
         await DeviceManager.save_device(device, update_fields=["feed"])
 
     @staticmethod
-    async def update_config_data(device: Device, **kwargs):
+    def add_config_callback(callback: Callable[[Device, dict[str, Any]], Awaitable[None]]):
+        DeviceManager._config_callbacks.append(callback)
+
+    @staticmethod
+    def remove_config_callback(callback: Callable[[Device, dict[str, Any]], Awaitable[None]]):
+        DeviceManager._config_callbacks.remove(callback)
+
+    @staticmethod
+    async def update_config_data(device: Device, **kwargs: dict[str, Any]):
         model = kwargs.get("hw_boardname") or "default"
         revision = kwargs.get("hw_revision") or "default"
         sw_version = kwargs.get("sw_version")
 
+        await asyncio.gather(
+            *[cb(device, **kwargs) for cb in DeviceManager._config_callbacks]  # type: ignore[call-arg]
+        )
+
         hardware = (await Hardware.get_or_create(model=model, revision=revision))[0]
         modified = False
 
@@ -183,6 +201,14 @@ async def _get_software(device: Device) -> Software | None:
         assert device.update_mode == UpdateModeEnum.PINNED
         return None
 
+    @staticmethod
+    def add_update_source(source: Callable[[Request, Device], Awaitable[tuple[HandlingType, UpdateChunk | None]]]):
+        DeviceManager._update_sources.append(source)
+
+    @staticmethod
+    async def get_alt_src_updates(request: Request, device: Device) -> list[tuple[HandlingType, UpdateChunk | None]]:
+        return await asyncio.gather(*[source(request, device) for source in DeviceManager._update_sources])
+
     @staticmethod
     async def get_update(device: Device) -> tuple[HandlingType, Software | None]:
         software = await DeviceManager._get_software(device)

goosebit/plugins/__init__.py

@@ -0,0 +1,32 @@
+import logging
+from importlib.metadata import entry_points
+
+from goosebit.schema.plugins import PluginSchema
+from goosebit.settings import config
+
+logger = logging.getLogger(__name__)
+
+
+def load() -> list:
+    plugin_configs = []
+    logger.info("Checking for plugins to be loaded...")
+    if len(config.plugins) == 0:
+        logger.info("No plugins found.")
+        return []
+    logger.info(f"Found plugins enabled in config: {config.plugins}")
+
+    entries = entry_points(group="goosebit.plugins")
+
+    for plugin in config.plugins:
+        modules = entries.select(name=plugin)
+        for module in modules:  # should be 1 or 0
+            logger.info(f"Loading plugin: {plugin}")
+            loaded_plugin = module.load()
+            if not hasattr(loaded_plugin, "config"):
+                logger.error(f"Failed to load plugin: {plugin}, plugin has not defined config")
+                continue
+            if not isinstance(loaded_plugin.config, PluginSchema):
+                logger.error(f"Failed to load plugin: {plugin}, config is not an instance of PluginSchema")
+                continue
+            plugin_configs.append(loaded_plugin.config)
+    return plugin_configs

goosebit/schema/plugins.py

@@ -0,0 +1,66 @@
+import inspect
+from typing import Any, Awaitable, Callable
+
+from fastapi import APIRouter
+from fastapi.requests import Request
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+from pydantic import BaseModel, Field, computed_field
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+    YamlConfigSettingsSource,
+)
+
+from goosebit.db import Device
+from goosebit.device_manager import HandlingType
+from goosebit.schema.updates import UpdateChunk
+from goosebit.settings import config
+
+
+def get_module_name():
+    module = inspect.getmodule(inspect.stack()[2][0])
+    if module is not None:
+        return module.__name__.split(".")[0]
+    raise TypeError("Could not discover plugin module name")
+
+
+class PluginSchema(BaseModel):
+    class Config:
+        arbitrary_types_allowed = True
+
+    name: str = Field(
+        default_factory=get_module_name
+    )  # get the name of the package this was initialized in (plugin package)
+    router: APIRouter | None = None
+    db_model_path: str | None = None
+    static_files: StaticFiles | None = None
+    templates: Jinja2Templates | None = None
+    update_source_hook: Callable[[Request, Device], Awaitable[tuple[HandlingType, UpdateChunk | None]]] | None = None
+    config_data_hook: Callable[[Device, dict[str, Any]], Awaitable[None]] | None = None
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def url_prefix(self) -> str:
+        return f"/plugins/{self.name}"
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def static_files_name(self) -> str:
+        return f"{self.name}_static"
+
+
+class PluginSettings(BaseSettings):
+    model_config = SettingsConfigDict(extra="ignore")
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return tuple([env_settings, YamlConfigSettingsSource(settings_cls, config.config_file)])

goosebit/schema/updates.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel, Field
+
+
+class UpdateChunkArtifact(BaseModel):
+    filename: str
+    hashes: dict[str, str]
+    size: int
+    links: dict[str, dict[str, str]] = Field(serialization_alias="_links")
+
+
+class UpdateChunk(BaseModel):
+    part: str = "os"
+    version: str = "1"
+    name: str
+    artifacts: list[UpdateChunkArtifact]