feat(lawful-basis): emit lawful basis and external media on every platform

README.md

@@ -383,6 +383,24 @@ All adapters share these common configuration options:
 | `STATE_FILE` | No | `.{adapter}_state.json` | State tracking file |
 | `LOG_LEVEL` | No | `INFO` | Logging level |
 | `ALLOW_UNSIGNED_WEBHOOKS` | No | `false` | See [Webhook authentication](#webhook-authentication) |
+| `MEDIA_BACKEND` | No | `embed` | Where recording audio goes: `embed` (inline base64), `filesystem`, or `s3` |
+| `MEDIA_BASE_URL` | No | - | Public base URL prepended to re-hosted media (`filesystem`/`s3`) |
+| `MEDIA_FILESYSTEM_PATH` | Only if `MEDIA_BACKEND=filesystem` | - | Directory to write published recordings to |
+| `MEDIA_S3_BUCKET` | Only if `MEDIA_BACKEND=s3` | - | S3 (or S3-compatible) bucket for published recordings. Needs the `s3` extra: `pip install ".[s3]"` |
+| `MEDIA_S3_REGION` | No | - | S3 region |
+| `MEDIA_S3_PREFIX` | No | - | Key prefix for published recordings in the bucket |
+| `MEDIA_S3_ENDPOINT_URL` | No | - | Endpoint URL for an S3-compatible store (DigitalOcean Spaces, MinIO, Telnyx Cloud Storage); leave unset for AWS |
+| `LAWFUL_BASIS` | No | - | Why this deployment may hold the recording: `consent`, `contract`, `legal_obligation`, `vital_interests`, `public_task`, or `legitimate_interests`. Unset means no `lawful_basis` attachment is emitted; never defaulted |
+| `LAWFUL_BASIS_PURPOSES` | No | `recording` | Comma-separated purposes granted under the basis |
+| `LAWFUL_BASIS_EXPIRATION` | No | - | ISO 8601 timestamp when the lawful basis expires |
+| `LAWFUL_BASIS_JUSTIFICATION` | No | - | Free-text justification, stored in the attachment's metadata |
+
+The `lawful_basis` attachment (and every other JSON-encoded attachment these
+adapters emit, such as the platform tags attachment) follows
+draft-ietf-vcon-vcon-core-04 §2.3.2: for `encoding: "json"`, `body` is the
+raw JSON value itself (an object or array), not a `json.dumps` string.
+`mediatype: "application/json"` is set alongside it, and `start`/`party`/
+`dialog` are always present, as the Attachment Object requires.
 
 ## Webhook authentication
 

adapters/asterisk/webhook.py

@@ -32,12 +32,29 @@ def create_app(config: AsteriskConfig) -> FastAPI:
     )
 
     # Initialize components
+    publisher = config.build_publisher()
+    if publisher is None:
+        logger.warning(
+            "MEDIA_BACKEND=embed: audio will be inlined as base64, making each "
+            "vCon roughly 1.3x the size of the recording. Set MEDIA_BACKEND=s3 "
+            "for anything beyond a lab."
+        )
+    lawful_basis = config.build_lawful_basis()
+    if not lawful_basis.enabled:
+        logger.warning(
+            "LAWFUL_BASIS is unset, so vCons will carry no record of why this "
+            "deployment may hold the recording. Fine for a lab; not for real "
+            "conversations."
+        )
+
     builder = AsteriskVconBuilder(
         download_recordings=config.download_recordings,
         recording_format=config.recording_format,
         recordings_path=config.recordings_path,
         ari_url=config.asterisk_ari_url,
         ari_auth=config.get_ari_auth(),
+        publisher=publisher,
+        lawful_basis=lawful_basis,
     )
     poster = HttpPoster(config.conserver_url, config.get_headers(), config.ingress_lists)
     tracker = StateTracker(config.state_file)

adapters/bandwidth/webhook.py

@@ -32,10 +32,27 @@ def create_app(config: BandwidthConfig) -> FastAPI:
     )
 
     # Initialize components
+    publisher = config.build_publisher()
+    if publisher is None:
+        logger.warning(
+            "MEDIA_BACKEND=embed: audio will be inlined as base64, making each "
+            "vCon roughly 1.3x the size of the recording. Set MEDIA_BACKEND=s3 "
+            "for anything beyond a lab."
+        )
+    lawful_basis = config.build_lawful_basis()
+    if not lawful_basis.enabled:
+        logger.warning(
+            "LAWFUL_BASIS is unset, so vCons will carry no record of why this "
+            "deployment may hold the recording. Fine for a lab; not for real "
+            "conversations."
+        )
+
     builder = BandwidthVconBuilder(
         download_recordings=config.download_recordings,
         recording_format=config.recording_format,
         api_auth=config.get_api_auth(),
+        publisher=publisher,
+        lawful_basis=lawful_basis,
     )
     poster = HttpPoster(config.conserver_url, config.get_headers(), config.ingress_lists)
     tracker = StateTracker(config.state_file)

adapters/freeswitch/webhook.py

@@ -32,11 +32,28 @@ def create_app(config: FreeSwitchConfig) -> FastAPI:
     )
 
     # Initialize components
+    publisher = config.build_publisher()
+    if publisher is None:
+        logger.warning(
+            "MEDIA_BACKEND=embed: audio will be inlined as base64, making each "
+            "vCon roughly 1.3x the size of the recording. Set MEDIA_BACKEND=s3 "
+            "for anything beyond a lab."
+        )
+    lawful_basis = config.build_lawful_basis()
+    if not lawful_basis.enabled:
+        logger.warning(
+            "LAWFUL_BASIS is unset, so vCons will carry no record of why this "
+            "deployment may hold the recording. Fine for a lab; not for real "
+            "conversations."
+        )
+
     builder = FreeSwitchVconBuilder(
         download_recordings=config.download_recordings,
         recording_format=config.recording_format,
         recordings_path=config.recordings_path,
         recordings_url_base=config.recordings_url_base,
+        publisher=publisher,
+        lawful_basis=lawful_basis,
     )
     poster = HttpPoster(config.conserver_url, config.get_headers(), config.ingress_lists)
     tracker = StateTracker(config.state_file)

adapters/twilio/webhook.py

@@ -31,10 +31,27 @@ def create_app(config: TwilioConfig) -> FastAPI:
     )
 
     # Initialize components
+    publisher = config.build_publisher()
+    if publisher is None:
+        logger.warning(
+            "MEDIA_BACKEND=embed: audio will be inlined as base64, making each "
+            "vCon roughly 1.3x the size of the recording. Set MEDIA_BACKEND=s3 "
+            "for anything beyond a lab."
+        )
+    lawful_basis = config.build_lawful_basis()
+    if not lawful_basis.enabled:
+        logger.warning(
+            "LAWFUL_BASIS is unset, so vCons will carry no record of why this "
+            "deployment may hold the recording. Fine for a lab; not for real "
+            "conversations."
+        )
+
     builder = TwilioVconBuilder(
         download_recordings=config.download_recordings,
         recording_format=config.recording_format,
         twilio_auth=config.get_twilio_auth(),
+        publisher=publisher,
+        lawful_basis=lawful_basis,
     )
     poster = HttpPoster(config.conserver_url, config.get_headers(), config.ingress_lists)
     tracker = StateTracker(config.state_file)

core/base_builder.py

@@ -295,6 +295,26 @@ def build(self, recording_data: BaseRecordingData) -> Vcon | None:
                     vcon.uuid,
                 )
 
+            # The official schema (draft-ietf-vcon-vcon-core-04, Attachment
+            # Object) requires `start`, `party`, and `dialog` on every
+            # attachment, and `mediatype` whenever `body` is present. Neither
+            # vcon-lib's own `add_tag()` (the tags attachment above) nor its
+            # own validator enforces any of that, so every attachment this
+            # builder produces gets backfilled here rather than leaving a
+            # schema-invalid vCon. `created_at` is when this vCon, and every
+            # attachment on it, was produced, which is what "sent/exchanged"
+            # means for a tag or a lawful basis record; every attachment here
+            # is `encoding: "json"`, so `application/json` is always the
+            # right mediatype. Existing values are never overwritten (the
+            # lawful_basis attachment already sets its own party/dialog and
+            # mediatype).
+            for attachment in vcon.vcon_dict.get("attachments", []):
+                attachment.setdefault("start", vcon.created_at)
+                attachment.setdefault("party", 0)
+                attachment.setdefault("dialog", 0)
+                if attachment.get("body") is not None:
+                    attachment.setdefault("mediatype", "application/json")
+
             logger.info(
                 f"Created vCon {vcon.uuid} from recording {recording_data.recording_id} "
                 f"(from: {recording_data.from_number}, to: {recording_data.to_number})"

core/lawful_basis.py

@@ -90,11 +90,22 @@ def purpose_grants(self, granted_at: str) -> list[dict[str, Any]]:
             for purpose in self.purposes
         ]
 
-    def apply(self, vcon: Vcon, party_index: int | None = None) -> bool:
+    def apply(self, vcon: Vcon, party_index: int = 0, dialog_index: int = 0) -> bool:
         """Attach the configured basis. Returns False when none is configured.
 
         Delegates to `vcon-lib`, which owns the shape its own validator and
-        finder expect.
+        finder expect. Under draft-ietf-vcon-vcon-core-04 §2.3.2 (CDDL
+        `body: any`), `body` for `encoding: "json"` is the JSON value itself,
+        not a `json.dumps` string — so vcon-lib 0.9.6's object body is
+        correct as-is and is left untouched. (An earlier revision of this
+        method stringified it against a stale schema fork that typed `body`
+        as a string; that was wrong for -04 and has been reverted.)
+
+        Every adapter here produces exactly one dialog and references party 0
+        as the recording's subject, so `party`/`dialog` default to 0 rather
+        than being left off. `mediatype` is required whenever `body` is
+        present (-04's Attachment Object); vcon-lib does not set it, so it is
+        added here.
         """
         if not self.enabled:
             return False
@@ -115,6 +126,11 @@ def apply(self, vcon: Vcon, party_index: int | None = None) -> bool:
             expiration=self.expiration,
             purpose_grants=self.purpose_grants(granted_at),
             party_index=party_index,
+            dialog_index=dialog_index,
             **extra,
         )
+
+        attachment = vcon.vcon_dict["attachments"][-1]
+        attachment.setdefault("mediatype", "application/json")
+
         return True

tests/adapters/asterisk/test_lawful_basis_and_media.py

@@ -0,0 +1,98 @@
+"""CON-1083: lawful basis and external media, wired the same way as Telnyx.
+
+`core/lawful_basis.py` and `core/media_publisher.py` were already exercised
+through the Telnyx adapter (CON-814/CON-844); this closes the gap for
+Asterisk, whose webhook factory previously built `AsteriskVconBuilder`
+without either `lawful_basis` or `publisher`, so every vCon it produced
+carried no basis and always embedded audio inline.
+"""
+
+import base64
+import hashlib
+import logging
+
+import pytest
+
+from adapters.asterisk.builder import AsteriskRecordingData, AsteriskVconBuilder
+from core.lawful_basis import LawfulBasisConfig
+from core.media_publisher import FilesystemPublisher
+
+AUDIO = b"RIFF" + b"\x00" * 4 + b"WAVEfmt not real audio, just deterministic bytes"
+
+
+def _event(**overrides):
+    base = {
+        "recording_name": "rec-1",
+        "caller_id_num": "+15551234567",
+        "connected_line_num": "+15559876543",
+        "direction": "inbound",
+        "target_uri": "file:/var/spool/asterisk/recording/rec-1.wav",
+    }
+    base.update(overrides)
+    return base
+
+
+@pytest.fixture
+def build(monkeypatch):
+    """Build a vCon with the real builder, audio download stubbed out."""
+    monkeypatch.setattr(AsteriskVconBuilder, "_download_recording", lambda self, rd: AUDIO)
+
+    def _build(**builder_kwargs):
+        builder = AsteriskVconBuilder(recording_format="wav", **builder_kwargs)
+        vcon = builder.build(AsteriskRecordingData(_event()))
+        assert vcon is not None, "builder returned None"
+        return vcon
+
+    return _build
+
+
+def test_lawful_basis_emitted_when_configured(build):
+    vcon = build(lawful_basis=LawfulBasisConfig(lawful_basis="consent", purposes=["recording"]))
+
+    found = vcon.find_lawful_basis_attachments()
+    assert len(found) == 1
+    attachment = found[0]
+
+    assert attachment["purpose"] == "lawful_basis"
+    assert "type" not in attachment
+    assert attachment["party"] == 0
+    assert attachment["dialog"] == 0
+    assert attachment["encoding"] == "json"
+    assert attachment["mediatype"] == "application/json"
+    # draft-ietf-vcon-vcon-core-04 §2.3.2 (CDDL `body: any`): for
+    # `encoding: "json"`, body is the JSON value itself, not a `json.dumps`
+    # string.
+    assert isinstance(attachment["body"], dict), "body must be the JSON object, not a string"
+
+    body = attachment["body"]
+    assert body["lawful_basis"] == "consent"
+
+    assert "lawful_basis" in vcon.to_dict()["extensions"]
+    valid, errors = vcon.is_valid()
+    assert valid, errors
+
+
+def test_lawful_basis_absent_and_warned_when_unset(build, caplog):
+    with caplog.at_level(logging.WARNING, logger="core.base_builder"):
+        vcon = build(lawful_basis=None)
+
+    assert vcon.find_lawful_basis_attachments() == []
+    assert "lawful_basis" not in (vcon.to_dict().get("extensions") or [])
+    assert any("no lawful_basis attachment" in r.getMessage() for r in caplog.records)
+
+
+def test_filesystem_media_backend_produces_url_and_hash_not_inline_body(build, tmp_path):
+    publisher = FilesystemPublisher(destination=tmp_path, base_url="https://media.test/rec")
+    vcon = build(publisher=publisher)
+    dialog = vcon.to_dict()["dialog"][0]
+
+    assert dialog["url"].startswith("https://media.test/rec/")
+    expected_hash = "sha512-" + base64.urlsafe_b64encode(
+        hashlib.sha512(AUDIO).digest()
+    ).decode().rstrip("=")
+    assert dialog["content_hash"] == expected_hash
+    assert "body" not in dialog, "audio must not be inlined when publishing"
+    assert "encoding" not in dialog
+
+    valid, errors = vcon.is_valid()
+    assert valid, errors

tests/adapters/bandwidth/test_lawful_basis_and_media.py

@@ -0,0 +1,96 @@
+"""CON-1083: lawful basis and external media, wired the same way as Telnyx.
+
+Bandwidth's webhook factory previously built `BandwidthVconBuilder` without
+either `lawful_basis` or `publisher`, so every vCon it produced carried no
+basis and always embedded audio inline.
+"""
+
+import base64
+import hashlib
+import logging
+
+import pytest
+
+from adapters.bandwidth.builder import BandwidthRecordingData, BandwidthVconBuilder
+from core.lawful_basis import LawfulBasisConfig
+from core.media_publisher import FilesystemPublisher
+
+AUDIO = b"RIFF" + b"\x00" * 4 + b"WAVEfmt not real audio, just deterministic bytes"
+
+
+def _event(**overrides):
+    base = {
+        "recordingId": "rec-1",
+        "from": "+15551234567",
+        "to": "+15559876543",
+        "direction": "inbound",
+        "mediaUrl": "https://bandwidth.test/media/rec-1.wav",
+    }
+    base.update(overrides)
+    return base
+
+
+@pytest.fixture
+def build(monkeypatch):
+    """Build a vCon with the real builder, audio download stubbed out."""
+    monkeypatch.setattr(BandwidthVconBuilder, "_download_recording", lambda self, rd: AUDIO)
+
+    def _build(**builder_kwargs):
+        builder = BandwidthVconBuilder(recording_format="wav", **builder_kwargs)
+        vcon = builder.build(BandwidthRecordingData(_event()))
+        assert vcon is not None, "builder returned None"
+        return vcon
+
+    return _build
+
+
+def test_lawful_basis_emitted_when_configured(build):
+    vcon = build(lawful_basis=LawfulBasisConfig(lawful_basis="consent", purposes=["recording"]))
+
+    found = vcon.find_lawful_basis_attachments()
+    assert len(found) == 1
+    attachment = found[0]
+
+    assert attachment["purpose"] == "lawful_basis"
+    assert "type" not in attachment
+    assert attachment["party"] == 0
+    assert attachment["dialog"] == 0
+    assert attachment["encoding"] == "json"
+    assert attachment["mediatype"] == "application/json"
+    # draft-ietf-vcon-vcon-core-04 §2.3.2 (CDDL `body: any`): for
+    # `encoding: "json"`, body is the JSON value itself, not a `json.dumps`
+    # string.
+    assert isinstance(attachment["body"], dict), "body must be the JSON object, not a string"
+
+    body = attachment["body"]
+    assert body["lawful_basis"] == "consent"
+
+    assert "lawful_basis" in vcon.to_dict()["extensions"]
+    valid, errors = vcon.is_valid()
+    assert valid, errors
+
+
+def test_lawful_basis_absent_and_warned_when_unset(build, caplog):
+    with caplog.at_level(logging.WARNING, logger="core.base_builder"):
+        vcon = build(lawful_basis=None)
+
+    assert vcon.find_lawful_basis_attachments() == []
+    assert "lawful_basis" not in (vcon.to_dict().get("extensions") or [])
+    assert any("no lawful_basis attachment" in r.getMessage() for r in caplog.records)
+
+
+def test_filesystem_media_backend_produces_url_and_hash_not_inline_body(build, tmp_path):
+    publisher = FilesystemPublisher(destination=tmp_path, base_url="https://media.test/rec")
+    vcon = build(publisher=publisher)
+    dialog = vcon.to_dict()["dialog"][0]
+
+    assert dialog["url"].startswith("https://media.test/rec/")
+    expected_hash = "sha512-" + base64.urlsafe_b64encode(
+        hashlib.sha512(AUDIO).digest()
+    ).decode().rstrip("=")
+    assert dialog["content_hash"] == expected_hash
+    assert "body" not in dialog, "audio must not be inlined when publishing"
+    assert "encoding" not in dialog
+
+    valid, errors = vcon.is_valid()
+    assert valid, errors