Characterization tests

README.rst

@@ -58,3 +58,26 @@ And now you are ready to start the webserver::
 
 Setting the environment variables is needed when developing on a PC.
 Open your browser and point to http://localhost:8888/.
+
+Test
+----
+
+The test suite in ``test/`` contains HTTP-level characterization tests (pytest + tornado's testing tools).
+They run against a faked audio backend, so no JACK, mod-host or MOD hardware is needed.
+
+First complete the Install section above (virtualenv, python requirements and ``make -C utils`` — the tests
+import the webserver, which requires ``utils/libmod_utils.so``).
+
+On Python 3.10 or newer, the pinned tornado 4.3 needs a one-time patch (see the note in ``requirements.txt``)::
+
+    $ sed -i 's/collections.MutableMapping/collections.abc.MutableMapping/' modui-env/lib/python3.*/site-packages/tornado/httputil.py
+
+Install the test requirements and run the suite from the repository root::
+
+    $ source modui-env/bin/activate
+    $ pip3 install -r test-requirements.txt
+    $ pytest
+
+NOTE: ``test/hmi-protocol-integrationtest.py`` is not part of this suite — it is a standalone integration
+test for the HMI serial protocol that requires JACK, mod-host and a serial device, and pytest does not
+collect it.

pytest.ini

@@ -0,0 +1,36 @@
+[pytest]
+testpaths = test
+python_files = test_*.py
+markers =
+    lilv: tests that initialize the global lilv world (opt-in; run with -m lilv).
+        Deliberately excluded from the default run by addopts below -- see
+        docs/characterization-phase-6.md. These tests call modtools.utils.init(),
+        which is process-global and has no de-init in this suite (cleanup()
+        exists but is intentionally not called -- see test/test_effects_lilv.py).
+        A combined run (`pytest -m "(lilv or not lilv) and not lilv_fixture"`
+        -- see the lilv_fixture marker below for why the "and not lilv_fixture"
+        is required) was verified green with no observed leakage into other
+        tests as of phase 6, but -m lilv should still be treated as its own
+        invocation in CI to keep the blast radius of a lilv/JACK-adjacent
+        regression contained to one job.
+    lilv_fixture: the phase-6 stretch-goal tests (test/test_effects_lilv_fixture.py)
+        that scan a hand-written, binaryless .lv2 bundle. Opt-in; run with
+        -m lilv_fixture, and ONLY that -- as its OWN, separate pytest
+        invocation. DO NOT run together with -m lilv: modtools.utils.init()
+        is only safe to call ONCE per process. Calling it a second time (this
+        module's fixture vs. test_effects_lilv.py's) leaves lilv's
+        process-global NamespaceDefinitions singleton bound to a freed world
+        -- confirmed by direct reproduction to silently return empty
+        ports/category data and then SIGSEGV the interpreter on exit. Keeping
+        this marker's tests in a file of their own, with their own single
+        init() call and their own `pytest -m lilv_fixture` invocation, is not
+        a style preference, it is the only way to keep this suite
+        segfault-free.
+        DANGER, easy to get wrong: a bare `-m "lilv or not lilv"` (the
+        phase-6 spec's combined-run check) ALSO collects lilv_fixture tests,
+        because a lilv_fixture-only test is NOT marked lilv and therefore
+        satisfies "not lilv" -- that recreates the exact double-init crash
+        above. When running that combined check, always add
+        `and not lilv_fixture`:
+        `pytest -m "(lilv or not lilv) and not lilv_fixture"`.
+addopts = -m "not lilv and not lilv_fixture"

test-requirements.txt

@@ -0,0 +1,4 @@
+pytest
+
+# Python 3.12 removed ssl.match_hostname, which tornado 4.3 falls back to importing from this package
+backports.ssl_match_hostname; python_version >= "3.12"

test/base.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2012-2023 MOD Audio UG
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""Shared base class for layer-1 characterization tests.
+
+Uses tornado 4.3's tornado.testing.AsyncHTTPTestCase (unittest-based, plain
+tornado-4 idioms -- no async/await) against the real module-level
+mod.webserver.application. conftest.py has already set every MOD_* env var
+before this module (or any test module) imports mod.webserver.
+"""
+
+import json
+
+from tornado.testing import AsyncHTTPTestCase
+
+from conftest import _UserFilesHelper, _USER_FILES_DIR
+
+
+class ModUITestCase(AsyncHTTPTestCase):
+    # Every test_*.py imports this class directly (`from base import
+    # ModUITestCase`) so it can subclass it -- which also puts it in each
+    # test module's globals. pytest's unittest integration collects *any*
+    # TestCase subclass it finds at module scope, regardless of name (the
+    # usual python_classes="Test*" filter does not apply to TestCase
+    # subclasses), so without this it would try to collect ModUITestCase
+    # itself as a test in every module that imports it and fail with
+    # "no attribute 'runTest'" (it defines no test_* methods of its own).
+    # Concrete subclasses below must set __test__ = True to opt back in,
+    # since __test__ is looked up via normal attribute inheritance.
+    __test__ = False
+
+    def runTest(self):
+        # Never actually invoked as a real test (real test_* methods are
+        # collected and run individually by pytest). Exists only because
+        # modern pytest (9.x) instantiates every unittest.TestCase subclass
+        # once with methodName="runTest" during collection, to register
+        # fixture factories (_pytest.unittest.UnitTestCase.newinstance()).
+        # tornado 4.3's own AsyncTestCase.__init__ (tornado/testing.py)
+        # unconditionally does getattr(self, methodName) -- unlike stdlib
+        # unittest.TestCase, it does not special-case a missing "runTest" --
+        # so without this method that probe instantiation raises
+        # AttributeError and collection fails for every test class.
+        pass
+
+    def get_app(self):
+        # Imported lazily (not at module top-level) so that conftest.py's
+        # env-var setup is guaranteed to have already run before mod.webserver
+        # -- and mod.settings, which reads MOD_* env vars at import time --
+        # is ever imported.
+        import mod.webserver
+        return mod.webserver.application
+
+    def seed(self, relpath, content=b"data"):
+        """Write USER_FILES_DIR/<relpath> (creating parent dirs) for a test.
+
+        Thin wrapper around conftest's user_files fixture helper, exposed
+        here because pytest fixture return values can't be injected as
+        parameters into unittest.TestCase test methods (which is what these
+        AsyncHTTPTestCase-derived tests are). The autouse `user_files`
+        fixture in conftest.py still handles wiping USER_FILES_DIR between
+        tests.
+        """
+        return _UserFilesHelper(_USER_FILES_DIR).seed(relpath, content=content)
+
+    def fetch_json(self, path, **kwargs):
+        """GET/POST/etc. `path` and decode the response body as JSON.
+
+        Asserts the response declares a JSON content type, matching the
+        JsonRequestHandler.write() convention used across mod/webserver.py.
+        """
+        response = self.fetch(path, **kwargs)
+        content_type = response.headers.get("Content-Type", "")
+        assert "application/json" in content_type, (
+            "expected JSON content type for %s, got %r (body: %r)"
+            % (path, content_type, response.body)
+        )
+        return response, json.loads(response.body.decode("utf-8"))