Sync Media with MEDIA_PLUGINS setting robustly

changelog.d/1830.changed.md

@@ -0,0 +1,12 @@
+Alter how the `MEDIA_PLUGINS` setting is synced with the Media model. The goal
+is to be able to eventually easily replace the plugin used for a specific media
+type with another. This handles the removal of plugins once installed much
+better.
+
+NOTES:
+
+Added a new management command `sync_media` that should be run after `migrate`
+but before running up the actual server in order to ensure that the `Media`
+table is up to date with installed media plugins even if a plugin has been
+removed since first startup. Destinations using removed plugins will still
+exist, they just won't work.

docker-entrypoint-argus.sh

@@ -5,6 +5,7 @@ git config --global --add safe.directory /argus
 python3 -m pip install -e .
 python3 manage.py collectstatic --noinput
 python3 manage.py migrate --noinput
+python3 manage.py sync_media
 
 # Stop password from being printed in logs
 set +x

docker/cmd-argus.sh

@@ -2,6 +2,7 @@
 
 django-admin collectstatic --noinput
 django-admin migrate --noinput
+django-admin sync_media
 
 # Stop password from being printed in logs
 set +x

src/argus/htmx/destination/forms.py

@@ -13,6 +13,8 @@ def __init__(self, *args, **kwargs):
         # Serializer request the request object
         self.request = kwargs.pop("request", None)
         super().__init__(*args, **kwargs)
+        self._available_media = Media.objects.available()
+        self.fields["media"].queryset = self._available_media
 
     class Meta:
         model = DestinationConfig

src/argus/htmx/destination/views.py

@@ -4,6 +4,7 @@
 from django.utils import timezone
 from django.views.generic import CreateView, DeleteView, ListView, UpdateView
 from django_htmx.http import HttpResponseClientRedirect
+from rest_framework.exceptions import ValidationError
 
 from argus.htmx.modals import DeleteModal
 from argus.notificationprofile.models import DestinationConfig
@@ -58,7 +59,7 @@ def get_prefix(self):
         return self._get_prefix(getattr(self.object, "pk", None))
 
     def get_queryset(self):
-        return super().get_queryset().filter(user=self.request.user).order_by("media", "pk")
+        return super().get_queryset().available().filter(user=self.request.user).order_by("media", "pk")
 
     def get_template_names(self):
         return [f"htmx/destination/destination{self.template_name_suffix}.html"]
@@ -168,6 +169,14 @@ def post(self, request, *args, **kwargs):
         self.object = self.get_object()
         try:
             medium = api_safely_get_medium_object(self.object.media.slug)
+        except ValidationError:
+            # should send 410 probably
+            return HttpResponseRedirect(self.get_success_url())
+
+        if not medium:
+            return HttpResponseRedirect(self.get_success_url())
+
+        try:
             medium.raise_if_not_deletable(self.object)
         except NotificationMedium.NotDeletableError as e:
             update_forms = _get_update_forms(request.user)
@@ -190,7 +199,7 @@ def post(self, request, *args, **kwargs):
 
 
 def _get_update_forms(user) -> list[DestinationFormUpdate]:
-    destinations = user.destinations.all().order_by("media", "pk")
+    destinations = user.destinations.available().order_by("media", "pk")
     return [
         DestinationFormUpdate(instance=destination, prefix=f"destination_{destination.pk}")
         for destination in destinations

src/argus/notificationprofile/admin.py

@@ -65,8 +65,10 @@ class MediaAdmin(admin.ModelAdmin):
     list_display = (
         "slug",
         "name",
+        "installed",
     )
     fields = ("name",)
+    list_filter = ("installed",)
 
 
 class DestinationConfigAdmin(admin.ModelAdmin):

src/argus/notificationprofile/apps.py

@@ -11,16 +11,16 @@ def ready(self):
         from .signals import (
             create_default_timeslot,
             sync_email_destination,
-            sync_media,
             task_background_send_notification,
+            trigger_sync_media,
         )
 
         # uses settings
         from .utils import are_notifications_enabled
 
         post_save.connect(create_default_timeslot, "argus_auth.User")
         post_save.connect(sync_email_destination, "argus_auth.User")
-        post_migrate.connect(sync_media, sender=self)
+        post_migrate.connect(trigger_sync_media, sender=self)
 
         if are_notifications_enabled():
             # sending notifications

src/argus/notificationprofile/management/commands/sync_media.py

@@ -0,0 +1,19 @@
+import sys
+
+from django.core.management.base import BaseCommand
+
+from argus.notificationprofile.media import sync_media
+
+
+class Command(BaseCommand):
+    help = "Sync anything needing syncing that is not covered by migrate"
+
+    def add_arguments(self, parser):
+        parser.add_argument("-l", "--list", action="store_true", help="Show the available syncs that will be run")
+
+    def handle(self, *args, **options):
+        if options.get("list", False):
+            sys.stdout.write("Sync MEDIA_PLUGINS setting to MEDIA table")
+            sys.exit(0)
+
+        sync_media()

src/argus/notificationprofile/media/__init__.py

@@ -30,6 +30,7 @@
 
 
 __all__ = [
+    "sync_media",
     "api_safely_get_medium_object",
     "send_notification",
     "background_send_notification",
@@ -41,18 +42,65 @@
 
 
 # TODO: Raise Incident if media_class not importable?
+
 MEDIA_PLUGINS = getattr(settings, "MEDIA_PLUGINS")
-_media_classes = [import_class_from_dotted_path(media_plugin) for media_plugin in MEDIA_PLUGINS]
-MEDIA_CLASSES_DICT = {media_class.MEDIA_SLUG: media_class for media_class in _media_classes}
+
+
+def _compile_MEDIA_CLASSES_DICT():
+    media_classes_dict = {}
+    for media_plugin in MEDIA_PLUGINS:
+        try:
+            plugin_class = import_class_from_dotted_path(media_plugin)
+        except (ImportError, ModuleNotFoundError, AttributeError) as e:
+            LOG.warning('Failed to load media plugin "%s": %s', media_plugin, str(e))
+        else:
+            media_classes_dict[plugin_class.MEDIA_SLUG] = plugin_class
+    return media_classes_dict
+
+
+MEDIA_CLASSES_DICT = _compile_MEDIA_CLASSES_DICT()
+
+
+def sync_media():
+    """Sync MEDIA_PLUGINS with the Media table
+
+    Check if all media in Media has a respective class"""
+
+    for medium in Media.objects.all():
+        is_installed = medium.slug in MEDIA_CLASSES_DICT.keys()
+        if is_installed != medium.installed:
+            medium.installed = is_installed
+            medium.save(update_fields=["installed"])
+            if not is_installed:
+                LOG.warning("%s plugin is no longer registered in MEDIA_PLUGINS", medium.name)
+            else:
+                LOG.info("%s plugin is again registered in MEDIA_PLUGINS", medium.name)
+
+    # Check if all media plugins are also saved in Media
+    for media_class in MEDIA_CLASSES_DICT.values():
+        if not Media.objects.filter(slug=media_class.MEDIA_SLUG):
+            media = Media(
+                slug=media_class.MEDIA_SLUG,
+                name=media_class.MEDIA_NAME,
+                installed=True,
+            )
+            media.save()
+            LOG.info("%s media plugin is now installed and registered correctly", media.name)
 
 
 def api_safely_get_medium_object(media_slug, version: str = API_STABLE_VERSION):
     try:
-        classobj = MEDIA_CLASSES_DICT[media_slug]
-    except KeyError:
+        medium_obj = Media.objects.get(slug=media_slug)
+    except Media.DoesNotExist:
         raise ValidationError(f'Medium "{media_slug}" is not installed.')
-    obj = classobj(version)
-    return obj
+    if medium_obj.installed:
+        try:
+            classobj = MEDIA_CLASSES_DICT[media_slug]
+        except KeyError:
+            raise ValidationError(f'Python module for "{media_slug}" not found')
+        obj = classobj(version)
+        return obj
+    return None
 
 
 def send_notification(destinations: Iterable[DestinationConfig], *events: Iterable[Event]):

src/argus/notificationprofile/models.py

@@ -179,6 +179,14 @@ def editable_by(self, user: User):
         return False
 
 
+class MediaQuerySet(models.QuerySet):
+    def available(self):
+        return self.filter(installed=True)
+
+    def unavailable(self):
+        return self.filter(installed=False)
+
+
 class Media(models.Model):
     class Meta:
         verbose_name = "Medium"
@@ -189,6 +197,8 @@ class Meta:
     name = models.CharField(max_length=MEDIA_NAME_LENGTH)
     installed = models.BooleanField(default=True)
 
+    objects = MediaQuerySet.as_manager()
+
     def __str__(self) -> str:
         return f"{self.slug}"
 
@@ -198,6 +208,14 @@ def save(self, *args, **kwargs):
         return super(Media, self).save(*args, **kwargs)
 
 
+class DestinationConfigQuerySet(models.QuerySet):
+    def available(self):
+        return self.filter(media__installed=True)
+
+    def unavailable(self):
+        return self.filter(media__installed=False)
+
+
 class DestinationConfig(models.Model):
     class Meta:
         constraints = [
@@ -215,6 +233,8 @@ class Meta:
     settings = models.JSONField()
     managed = models.BooleanField(null=True)
 
+    objects = DestinationConfigQuerySet.as_manager()
+
     def __str__(self):
         if self.label:
             return self.label

src/argus/notificationprofile/signals.py

@@ -4,9 +4,9 @@
 from typing import TYPE_CHECKING
 
 from django.contrib.auth import get_user_model
-from django.db.utils import ProgrammingError
 
 from argus.notificationprofile.media import send_notifications_to_users
+from argus.notificationprofile.media import sync_media
 from argus.notificationprofile.tasks import task_check_for_notifications
 from argus.plannedmaintenance.utils import event_covered_by_planned_maintenance
 
@@ -20,46 +20,16 @@
 User = get_user_model()
 
 __all__ = [
-    "sync_media",
+    "trigger_sync_media",
     "create_default_timeslot",
     "sync_email_destination",
     "task_send_notification",
     "task_background_send_notification",
 ]
 
 
-def sync_media(sender, **kwargs):
-    """Sync MEDIA_PLUGINS with the Media table
-
-    Check if all media in Media has a respective class"""
-
-    from .media import MEDIA_CLASSES_DICT
-
-    apps = kwargs["apps"]
-    try:
-        Media = apps.get_model("argus_notificationprofile", "Media")
-    except ImportError:
-        return
-
-    try:
-        for medium in Media.objects.all():
-            if medium.slug not in MEDIA_CLASSES_DICT.keys():
-                LOG.warning("%s plugin is not registered in MEDIA_PLUGINS", medium.name)
-                # Need to check in case of backwards migrations
-                if getattr(medium, "installed", None):
-                    medium.installed = False
-                    medium.save(update_fields=["installed"])
-    except ProgrammingError:
-        return
-
-    # Check if all media plugins are also saved in Media
-    new_media = [
-        Media(slug=media_class.MEDIA_SLUG, name=media_class.MEDIA_NAME)
-        for media_class in MEDIA_CLASSES_DICT.values()
-        if not Media.objects.filter(slug=media_class.MEDIA_SLUG)
-    ]
-    if new_media:
-        Media.objects.bulk_create(new_media)
+def trigger_sync_media(sender, **kwargs):
+    sync_media()
 
 
 # Create default immediate Timeslot when a user is created

src/argus/notificationprofile/v2/views.py

@@ -133,12 +133,9 @@ def get_context_data(self, **kwargs):
 class MediaViewSet(viewsets.ModelViewSet):
     version = VERSION
     serializer_class = MediaSerializer
-    queryset = Media.objects.none()
+    queryset = Media.objects.available()
     http_method_names = ["get", "head"]
 
-    def get_queryset(self):
-        return Media.objects.all()
-
     @extend_schema(responses={"200": JSONSchemaSerializer})
     @action(methods=["get"], detail=True)
     def json_schema(self, request, pk, *args, **kwargs):
@@ -178,7 +175,7 @@ class DestinationConfigViewSet(rw_viewsets.ModelViewSet):
     http_method_names = ["get", "head", "post", "patch", "delete"]
 
     def get_queryset(self):
-        return self.request.user.destinations.all()
+        return self.request.user.destinations.available()
 
     def perform_create(self, serializer):
         serializer.validated_data["settings"].pop("synced", False)
@@ -188,6 +185,9 @@ def destroy(self, request, *args, **kwargs):
         pk = self.kwargs["pk"]
         destination = get_object_or_404(self.get_queryset(), pk=pk)
 
+        medium = api_safely_get_medium_object(destination.media.slug)
+        if not medium:
+            raise ValidationError(f"Media {destination.media.slug} of destination not installed")
         try:
             medium = api_safely_get_medium_object(destination.media.slug, self.version)
             medium.raise_if_not_deletable(destination)
@@ -197,8 +197,10 @@ def destroy(self, request, *args, **kwargs):
             return super().destroy(destination)
 
     def _is_destination_duplicate(self, destination):
-        other_destinations = DestinationConfig.objects.filter(media=destination.media).filter(
-            ~Q(user_id=destination.user.id)
+        other_destinations = (
+            DestinationConfig.objects.available()
+            .filter(media=destination.media)
+            .filter(~Q(user_id=destination.user.id))
         )
         medium = api_safely_get_medium_object(destination.media_id, self.version)
         destination_in_use = medium.has_duplicate(other_destinations, destination.settings)
@@ -213,6 +215,8 @@ def duplicate(self, request, pk, *args, **kwargs):
             destination = request.user.destinations.get(pk=pk)
         except DestinationConfig.DoesNotExist:
             raise ValidationError(f"Destination with pk={pk} does not exist.")
+        if not destination.media.installed:
+            raise ValidationError(f"Destination with pk={pk} is not supported, plugin not installed correctly.")
         is_duplicate = self._is_destination_duplicate(destination=destination)
         serializer = DuplicateDestinationSerializer({"is_duplicate": is_duplicate})
         return Response(serializer.data)