Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ RUN apt update && apt install --no-install-recommends -y \
libpq-dev
COPY requirements* ./
ARG PIP_REQUIREMENTS=requirements.txt
RUN python -m pip install -U pip \
&& python -m pip install uwsgi
RUN python -m pip install -U pip
# The `--no-deps` has been added because of a pip bug with --require-hashes
# The dependencies have already been resolved during the generation of
# the requirements.txt, so adding `--no-deps` can safely be done.
Expand Down
16 changes: 16 additions & 0 deletions src/dso_api/middleware.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from importlib.metadata import version

from django.conf import settings
from django.http import HttpRequest
from packaging.version import parse
from schematools.contrib.django.models import Profile
Expand All @@ -8,6 +9,21 @@

from dso_api.dbroles import DatabaseRoles

try:
import uwsgi as _uwsgi
except ImportError:
_uwsgi = None


class HarakiriMiddleware:
def __init__(self, get_response):
self._get_response = get_response

def __call__(self, request: HttpRequest):
if _uwsgi is not None:
_uwsgi.set_user_harakiri(settings.UWSGI_HARAKIRI_DEFAULT)
return self._get_response(request)


class AuthMiddleware:
"""
Expand Down
11 changes: 11 additions & 0 deletions src/dso_api/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@
OAUTH_CLIENT_ID = os.getenv("OAUTH_CLIENT_ID", "dso-api-open-api")
OAUTH_CLIENT_ID_ENTRA = os.getenv("OAUTH_CLIENT_ID_ENTRA")

# uWSGI settings
UWSGI_HARAKIRI_DEFAULT = env.int("DSO_UWSGI_HARAKIRI_DEFAULT", default=180)
UWSGI_HARAKIRI_EXEMPT_DATASETS = [
"benkagg",
"brk2",
"gebieden",
"meetbouten",
"monumenten",
"nap",
] # These are the datasets used in the drawing tool on data.amsterdam.nl

# -- Security

Expand Down Expand Up @@ -78,6 +88,7 @@
]

MIDDLEWARE = [
"dso_api.middleware.HarakiriMiddleware",
"django.middleware.gzip.GZipMiddleware",
"django.middleware.security.SecurityMiddleware",
"whitenoise.middleware.WhiteNoiseMiddleware",
Expand Down
1 change: 1 addition & 0 deletions src/requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ requests == 2.34.2
shapely == 2.1.2
urllib3 == 2.8.0
urllib3-mock == 0.3.3
uwsgi == 2.0.31
whitenoise == 6.12.0
zipp == 4.1.2

Expand Down
3 changes: 3 additions & 0 deletions src/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2451,6 +2451,9 @@ urllib3-mock==0.3.3 \
--hash=sha256:702c90042920d771c9902b7b5b542551cc57f259078f4eada47ab4e8cdd11f1a \
--hash=sha256:b210037029ac96beac4f3e7b54f466c394b060525ea5a824803d5f5ed14558f1
# via -r requirements.in
uwsgi==2.0.31 \
--hash=sha256:e8f8b350ccc106ff93a65247b9136f529c14bf96b936ac5b264c6ff9d0c76257
# via -r requirements.in
uwsgi-readiness-check==0.2.0 \
--hash=sha256:b8c4bfba260ec6f1937ca8d8968f2415aba82428637da17653fc04a3378d752e \
--hash=sha256:e7a2b5a94407193bc876efd28157a45833cb69c372f7684004fa729eca2f1ab0
Expand Down
2 changes: 2 additions & 0 deletions src/requirements_dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,8 @@ urllib3==2.8.0
# requests
urllib3-mock==0.3.3
# via -r requirements.in
uwsgi==2.0.31
# via -r requirements.in
uwsgi-readiness-check==0.2.0
# via -r requirements.in
uwsgitop==0.12
Expand Down
13 changes: 13 additions & 0 deletions src/rest_framework_dso/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import sys
from inspect import isgeneratorfunction

from django.conf import settings
from django.http import HttpResponseNotFound, JsonResponse
from django.utils.html import escape
from gisserver.exceptions import ExternalValueError
Expand All @@ -20,6 +21,11 @@
from rest_framework_dso.pagination import DSOPageNumberPagination
from rest_framework_dso.response import StreamingResponse

try:
import uwsgi as _uwsgi
except ImportError:
_uwsgi = None

W3HTMLREF = "https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.5.1"


Expand Down Expand Up @@ -371,6 +377,13 @@ def finalize_response(self, request, response, *args, **kwargs):
pass
response = response.accepted_renderer.finalize_response(response, renderer_context)

exempt_stream = (
response.streaming
and getattr(self, "dataset_id", None) in settings.UWSGI_HARAKIRI_EXEMPT_DATASETS
)
if _uwsgi is not None and exempt_stream:
_uwsgi.set_user_harakiri(0)

return response

def get_view_description(self, **kwargs):
Expand Down
106 changes: 105 additions & 1 deletion src/tests/test_rest_framework_dso/test_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,22 @@
from django.urls import path
from rest_framework import generics, viewsets
from rest_framework.exceptions import ErrorDetail, ValidationError
from rest_framework.renderers import JSONRenderer
from rest_framework.routers import SimpleRouter

from dso_api import middleware
from rest_framework_dso import views
from rest_framework_dso.renderers import HALJSONRenderer
from rest_framework_dso.renderers import CSVRenderer, HALJSONRenderer, RendererMixin
from tests.utils import read_response, read_response_json, read_response_partial

from .models import Location, Movie
from .serializers import LocationSerializer, MovieSerializer


class NonStreamingJSONRenderer(RendererMixin, JSONRenderer):
pass


class MovieDetailAPIView(views.DSOViewMixin, generics.RetrieveAPIView):
serializer_class = MovieSerializer
queryset = Movie.objects.all()
Expand All @@ -24,6 +30,13 @@ class MovieListAPIView(views.DSOViewMixin, generics.ListAPIView):
queryset = Movie.objects.all()


class MovieCSVListAPIView(views.DSOViewMixin, generics.ListAPIView):
serializer_class = MovieSerializer
queryset = Movie.objects.all()
renderer_classes = [CSVRenderer]
dataset_id = "movies"


class MovieViewSet(views.DSOViewMixin, viewsets.ReadOnlyModelViewSet):
serializer_class = MovieSerializer
queryset = Movie.objects.all()
Expand All @@ -40,6 +53,7 @@ class LocationListAPIView(views.DSOViewMixin, generics.ListAPIView):
urlpatterns = [
path("v1/locations", LocationListAPIView.as_view(), name="locations-list"),
path("v1/movies", MovieListAPIView.as_view(), name="movies-list"),
path("v1/movies-csv", MovieCSVListAPIView.as_view(), name="movies-csv-list"),
path("v1/movies/<pk>", MovieDetailAPIView.as_view(), name="movies-detail"),
] + router.get_urls()

Expand All @@ -48,6 +62,38 @@ class LocationListAPIView(views.DSOViewMixin, generics.ListAPIView):
pytestmark = [pytest.mark.urls(__name__)] # enable for all tests in this file


class FakeUwsgi:
def __init__(self):
self.calls = []

def set_user_harakiri(self, seconds):
self.calls.append(seconds)


@pytest.mark.parametrize("raises", [False, True])
def test_harakiri_starts_before_request_handler(settings, monkeypatch, raises):
fake_uwsgi = FakeUwsgi()
monkeypatch.setattr(middleware, "_uwsgi", fake_uwsgi)
settings.UWSGI_HARAKIRI_DEFAULT = 120
request = object()
response = object()

def get_response(received_request):
assert received_request is request
assert fake_uwsgi.calls == [120]
if raises:
raise RuntimeError("handler failed")
return response

handler = middleware.HarakiriMiddleware(get_response)
if raises:
with pytest.raises(RuntimeError, match="handler failed"):
handler(request)
else:
assert handler(request) is response
assert fake_uwsgi.calls == [120]


@pytest.mark.django_db
class TestExpand:
@pytest.mark.parametrize(
Expand Down Expand Up @@ -217,6 +263,64 @@ def test_list_expand_true(self, api_client, movie):
}
assert response["Content-Type"] == "application/hal+json"

@pytest.mark.parametrize("url", ["/v1/movies-csv", "/v1/movies"])
@pytest.mark.parametrize("dataset_id", [None, "movies", "benkagg"])
@pytest.mark.parametrize("exempt_datasets", [[], ["benkagg"], ["benkagg", "movies"]])
def test_streaming_exempts_configured_datasets_from_uwsgi_harakiri(
self, api_client, movie, settings, monkeypatch, url, dataset_id, exempt_datasets
):
fake_uwsgi = FakeUwsgi()
monkeypatch.setattr(views, "_uwsgi", fake_uwsgi)
monkeypatch.setattr(middleware, "_uwsgi", fake_uwsgi)
monkeypatch.setattr(MovieCSVListAPIView, "dataset_id", dataset_id)
monkeypatch.setattr(MovieListAPIView, "dataset_id", dataset_id, raising=False)
settings.UWSGI_HARAKIRI_DEFAULT = 180
settings.UWSGI_HARAKIRI_EXEMPT_DATASETS = exempt_datasets
expected_calls = [180, 0] if dataset_id in exempt_datasets else [180]

response = api_client.get(url)
assert response.streaming
assert fake_uwsgi.calls == expected_calls
read_response(response)

assert response.status_code == 200
assert fake_uwsgi.calls == expected_calls

@pytest.mark.parametrize("timeout", [0, 120])
@pytest.mark.parametrize("invalid_fields", [False, True])
@pytest.mark.parametrize("dataset_id", ["movies", "benkagg"])
def test_non_streaming_sets_uwsgi_harakiri(
self, api_client, movie, settings, monkeypatch, timeout, invalid_fields, dataset_id
):
fake_uwsgi = FakeUwsgi()
monkeypatch.setattr(views, "_uwsgi", fake_uwsgi)
monkeypatch.setattr(middleware, "_uwsgi", fake_uwsgi)
monkeypatch.setattr(MovieListAPIView, "dataset_id", dataset_id, raising=False)
monkeypatch.setattr(MovieListAPIView, "renderer_classes", [NonStreamingJSONRenderer])
settings.UWSGI_HARAKIRI_DEFAULT = timeout
params = {"_fields": "unknown"} if invalid_fields else {}

response = api_client.get(
"/v1/movies",
data=params,
HTTP_ACCEPT="application/json",
)

assert not response.streaming
assert response.status_code == (400 if invalid_fields else 200)
assert fake_uwsgi.calls == [timeout]
read_response(response)

def test_non_streaming_without_uwsgi(self, api_client, movie, monkeypatch):
monkeypatch.setattr(views, "_uwsgi", None)
monkeypatch.setattr(middleware, "_uwsgi", None)

response = api_client.get("/v1/movies", HTTP_ACCEPT="text/html")

assert not response.streaming
assert response.status_code == 200
read_response(response)

@pytest.mark.parametrize(
"params",
[
Expand Down
Loading