From ec1f836a0068bb6e0baf80044a4cbdaa4adf2626 Mon Sep 17 00:00:00 2001 From: sideshowbarker Date: Mon, 27 Jul 2026 11:52:27 +0900 Subject: [PATCH] =?UTF-8?q?LibWeb+LibMedia:=20Reject=20audio=E2=87=94video?= =?UTF-8?q?=20codec=20mismatches=20in=20decodingInfo()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This aligns our mediaCapabilities.decodingInfo() implementation with WPT expectations to reject an audio codec where a video codec is expected, and reject a video codec where an audio codec is expected. The spec (see https://github.com/w3c/media-capabilities/issues/261) doesn’t explicitly state that requirement — but Gecko and WebKit both also reject such codec mismatches (and pass the relevant WPT subtests). This also includes a track_type_from_codec_id() bug fix — to correctly classify Theora as a video codec, rather than incorrectly as audio. Fixes https://github.com/LadybirdBrowser/ladybird/issues/10696 --- Libraries/LibMedia/CodecID.h | 46 +++++++- .../MediaCapabilities.cpp | 105 +++++++++++++----- .../MediaCapabilitiesAPI/MediaCapabilities.h | 5 - Libraries/LibWeb/MimeSniff/MimeType.cpp | 90 ++++++++++++++- Libraries/LibWeb/MimeSniff/MimeType.h | 1 + .../media-capabilities/decodingInfo.any.txt | 35 +++--- .../media-capabilities/decodingInfo.any.js | 11 ++ 7 files changed, 235 insertions(+), 58 deletions(-) diff --git a/Libraries/LibMedia/CodecID.h b/Libraries/LibMedia/CodecID.h index 84e97501a7e1d..da6a9cbb4b05c 100644 --- a/Libraries/LibMedia/CodecID.h +++ b/Libraries/LibMedia/CodecID.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2023, Stephan Vedder + * Copyright (c) 2026-present, the Ladybird developers. * * SPDX-License-Identifier: BSD-2-Clause */ @@ -7,6 +8,7 @@ #pragma once #include +#include #include namespace Media { @@ -56,10 +58,10 @@ inline TrackType track_type_from_codec_id(CodecID codec) case CodecID::H264: case CodecID::H265: case CodecID::AV1: + case CodecID::Theora: return TrackType::Video; case CodecID::MP3: case CodecID::AAC: - case CodecID::Theora: case CodecID::Vorbis: case CodecID::Opus: case CodecID::FLAC: @@ -79,6 +81,48 @@ inline TrackType track_type_from_codec_id(CodecID codec) return TrackType::Unknown; } +// Maps a codec ID string, as used in the codecs parameter of a MIME type, to a CodecID. Returns CodecID::Unknown for +// codec ID strings that aren't recognized. +// https://www.rfc-editor.org/rfc/rfc6381 +inline CodecID codec_id_from_rfc6381_codec_string(StringView codec_string) +{ + // Codec ID strings for codec families such as AVC, HEVC, VP9, AV1 and AAC have period-separated suffixes that + // describe the profile, level, and other parameters of the codec used. + auto is_codec_family = [&](StringView family) { + if (!codec_string.starts_with(family)) + return false; + return codec_string.length() == family.length() || codec_string[family.length()] == '.'; + }; + + if (is_codec_family("avc1"sv) || is_codec_family("avc3"sv)) + return CodecID::H264; + if (is_codec_family("hvc1"sv) || is_codec_family("hev1"sv)) + return CodecID::H265; + if (codec_string == "vp8"sv || is_codec_family("vp08"sv)) + return CodecID::VP8; + if (codec_string == "vp9"sv || is_codec_family("vp09"sv)) + return CodecID::VP9; + if (is_codec_family("av01"sv)) + return CodecID::AV1; + if (codec_string == "theora"sv) + return CodecID::Theora; + if (codec_string == "vorbis"sv) + return CodecID::Vorbis; + if (codec_string == "opus"sv) + return CodecID::Opus; + if (codec_string == "flac"sv) + return CodecID::FLAC; + if (codec_string == "mp3"sv) + return CodecID::MP3; + // MPEG-4 audio object type 0x40 is MPEG-4 AAC and object types 0x66 to 0x68 are MPEG-2 AAC, while object types 0x69 + // and 0x6B are MPEG-1/2 audio, whose ubiquitous layer is MP3. + if (is_codec_family("mp4a.40"sv) || codec_string == "mp4a.66"sv || codec_string == "mp4a.67"sv || codec_string == "mp4a.68"sv) + return CodecID::AAC; + if (codec_string == "mp4a.69"sv || codec_string == "mp4a.6B"sv || codec_string == "mp4a.6b"sv) + return CodecID::MP3; + return CodecID::Unknown; +} + constexpr StringView codec_id_to_string(CodecID codec) { switch (codec) { diff --git a/Libraries/LibWeb/MediaCapabilitiesAPI/MediaCapabilities.cpp b/Libraries/LibWeb/MediaCapabilitiesAPI/MediaCapabilities.cpp index 10be1abe3a1b8..9cdd5dea101f3 100644 --- a/Libraries/LibWeb/MediaCapabilitiesAPI/MediaCapabilities.cpp +++ b/Libraries/LibWeb/MediaCapabilitiesAPI/MediaCapabilities.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2024, Jamie Mansfield * Copyright (c) 2025, Psychpsyo + * Copyright (c) 2026-present, the Ladybird developers. * * SPDX-License-Identifier: BSD-2-Clause */ @@ -8,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -76,26 +78,51 @@ bool is_valid_media_decoding_configuration(MediaDecodingConfiguration const& con return true; } -// https://w3c.github.io/media-capabilities/#valid-audio-mime-type -bool is_valid_audio_mime_type(Utf16View string) +// https://w3c.github.io/media-capabilities/#check-mime-type-validity +static bool check_mime_type_validity(MimeSniff::MimeType const& mime_type, StringView media) { - // A valid audio MIME type is a string that is a valid media MIME type and for which the type per [RFC9110] is - // either audio or application. - auto mime_type = MimeSniff::MimeType::parse(string); - if (!mime_type.has_value()) + // To check MIME type validity given a MIME type record mimeType and a string media, run the following steps: + + // 1. If the type of mimeType per [RFC9110] is neither media nor application, return false. + if (mime_type.type() != media && mime_type.type() != "application"sv) return false; - return mime_type->type() == "audio"sv || mime_type->type() == "application"sv; -} -// https://w3c.github.io/media-capabilities/#valid-video-mime-type -bool is_valid_video_mime_type(Utf16View string) -{ - // A valid video MIME type is a string that is a valid media MIME type and for which the type per [RFC9110] is - // either video or application. - auto mime_type = MimeSniff::MimeType::parse(string); - if (!mime_type.has_value()) + // AD-HOC: The spec doesn't define which combined type and subtype members allow a single media codec, and which + // instead allow multiple media codecs. So this is based on the equivalent lists used by other engines. + bool allows_multiple_media_codecs = mime_type.essence().is_one_of( + "audio/mp4"sv, "audio/ogg"sv, "audio/webm"sv, + "application/mp4"sv, "application/ogg"sv, + "video/mp4"sv, "video/ogg"sv, "video/webm"sv); + + // 2. If the combined type and subtype members of mimeType allow a single media codec and the parameters member of + // mimeType is not empty, return false. + if (!allows_multiple_media_codecs && !mime_type.parameters().is_empty()) return false; - return mime_type->type() == "video"sv || mime_type->type() == "application"sv; + + // 3. If the combined type and subtype members of mimeType allow multiple media codecs, run the following steps: + if (allows_multiple_media_codecs) { + // 1. If the parameters member of mimeType does not contain a single key named "codecs", return false. + auto codecs_iter = mime_type.parameters().find("codecs"sv); + if (mime_type.parameters().size() != 1 || codecs_iter == mime_type.parameters().end()) + return false; + + // 2. If the value of mimeType.parameters["codecs"] does not describe a single media codec, return false. + auto codec_string = codecs_iter->value.bytes_as_string_view().trim_whitespace(); + if (codec_string.is_empty() || codec_string.contains(',')) + return false; + + // AD-HOC: Reject a video config whose contentType gives an audio codec, and reject an audio config whose + // contentType describes a video codec — as WPT expects, even though the spec doesn't require it (yet). + // https://github.com/w3c/media-capabilities/issues/261 + auto track_type = Media::track_type_from_codec_id(Media::codec_id_from_rfc6381_codec_string(codec_string)); + if (media == "audio"sv && track_type == Media::TrackType::Video) + return false; + if (media == "video"sv && track_type == Media::TrackType::Audio) + return false; + } + + // 4. Return true. + return true; } // https://w3c.github.io/media-capabilities/#valid-video-configuration @@ -104,34 +131,52 @@ bool is_valid_video_configuration(VideoConfiguration const& configuration) // To check if a VideoConfiguration configuration is a valid video configuration, the following steps MUST be // run: - // 1. If configuration’s contentType is not a valid video MIME type, return false and abort these steps. - if (!is_valid_video_mime_type(configuration.content_type)) - return false; - - // 2. If framerate is not finite or is not greater than 0, return false and abort these steps. + // 1. If framerate is not finite or is not greater than 0, return false and abort these steps. if (!isfinite(configuration.framerate) || configuration.framerate <= 0) return false; - // 3. If an optional member is specified for a MediaDecodingType or MediaEncodingType to which it’s not + // 2. If an optional member is specified for a MediaDecodingType or MediaEncodingType to which it’s not // applicable, return false and abort these steps. See applicability rules in the member definitions below. // FIXME: Implement this. - // 4. Return true. - return true; + // 3. Let mimeType be the result of running parse a MIME type with configuration’s contentType. + auto mime_type = MimeSniff::MimeType::parse(configuration.content_type); + + // 4. If mimeType is failure, return false. + if (!mime_type.has_value()) + return false; + + // AD-HOC: Also return false if config's contentType is not a valid MIME-type string, such as "video/webm;". The + // spec’s "Parse a MIME type" accepts some strings that don't match the media-type production; but Blink + // instead parses the content type strictly here — and rejects a trailing semicolon. + if (!MimeSniff::is_valid_mime_type_string(configuration.content_type)) + return false; + + // 5. Return the result of running check MIME type validity with mimeType and video. + return check_mime_type_validity(*mime_type, "video"sv); } -// https://w3c.github.io/media-capabilities/#valid-video-configuration +// https://w3c.github.io/media-capabilities/#valid-audio-configuration bool is_valid_audio_configuration(AudioConfiguration const& configuration) { // To check if a AudioConfiguration configuration is a valid audio configuration, the following steps MUST be // run: - // 1. If configuration’s contentType is not a valid audio MIME type, return false and abort these steps. - if (!is_valid_audio_mime_type(configuration.content_type)) + // 1. Let mimeType be the result of running parse a MIME type with configuration’s contentType. + auto mime_type = MimeSniff::MimeType::parse(configuration.content_type); + + // 2. If mimeType is failure, return false. + if (!mime_type.has_value()) return false; - // 2. Return true. - return true; + // AD-HOC: Also return false if config's contentType isn't a valid MIME-type string, such as "audio/mpeg;". The + // spec’s "Parse a MIME type" accepts some strings that don't match the media-type production; but Blink + // instead parses the content type strictly here — and rejects a trailing semicolon. + if (!MimeSniff::is_valid_mime_type_string(configuration.content_type)) + return false; + + // 3. Return the result of running check MIME type validity with mimeType and audio. + return check_mime_type_validity(*mime_type, "audio"sv); } GC_DEFINE_ALLOCATOR(MediaCapabilities); @@ -222,6 +267,8 @@ MediaCapabilitiesDecodingInfo create_a_media_capabilities_decoding_info(MediaDec bool is_able_to_decode_media(MediaDecodingConfiguration const& configuration) { + // FIXME: This only checks the MIME subtype — so a codec unrecognized by LibMedia on an otherwise supported subtype + // is still reported as supported. if (configuration.type != MediaDecodingType::MediaSource) return false; diff --git a/Libraries/LibWeb/MediaCapabilitiesAPI/MediaCapabilities.h b/Libraries/LibWeb/MediaCapabilitiesAPI/MediaCapabilities.h index f5e37fab67c67..e4e8ebf6dd90b 100644 --- a/Libraries/LibWeb/MediaCapabilitiesAPI/MediaCapabilities.h +++ b/Libraries/LibWeb/MediaCapabilitiesAPI/MediaCapabilities.h @@ -63,9 +63,4 @@ MediaCapabilitiesDecodingInfo create_a_media_capabilities_decoding_info(MediaDec bool is_able_to_decode_media(MediaDecodingConfiguration const&); -// https://w3c.github.io/media-capabilities/#valid-audio-mime-type -bool is_valid_audio_mime_type(Utf16View); -// https://w3c.github.io/media-capabilities/#valid-video-mime-type -bool is_valid_video_mime_type(Utf16View); - } diff --git a/Libraries/LibWeb/MimeSniff/MimeType.cpp b/Libraries/LibWeb/MimeSniff/MimeType.cpp index 7f120ea3178de..2a82f91e3a992 100644 --- a/Libraries/LibWeb/MimeSniff/MimeType.cpp +++ b/Libraries/LibWeb/MimeSniff/MimeType.cpp @@ -3,10 +3,12 @@ * Copyright (c) 2022-2023, Linus Groh * Copyright (c) 2022, networkException * Copyright (c) 2024, Jamie Mansfield + * Copyright (c) 2026-present, the Ladybird developers. * * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include @@ -43,18 +45,94 @@ bool is_javascript_mime_type_essence_match(Utf16View string) return false; } -static bool contains_only_http_quoted_string_token_code_points(StringView string) +// https://mimesniff.spec.whatwg.org/#http-quoted-string-token-code-point +// An HTTP quoted-string token code point is U+0009 TAB, a code point in the range U+0020 SPACE to U+007E (~), inclusive, +// or a code point in the range U+0080 through U+00FF (ÿ), inclusive. +static constexpr bool is_http_quoted_string_token_code_point(u32 code_point) { - // https://mimesniff.spec.whatwg.org/#http-quoted-string-token-code-point - // An HTTP quoted-string token code point is U+0009 TAB, a code point in the range U+0020 SPACE to U+007E (~), inclusive, - // or a code point in the range U+0080 through U+00FF (ÿ), inclusive. - for (auto ch : Utf8View(string)) { - if (!(ch == '\t' || (ch >= 0x20 && ch <= 0x7E) || (ch >= 0x80 && ch <= 0xFF))) + return code_point == '\t' || (code_point >= 0x20 && code_point <= 0x7E) || (code_point >= 0x80 && code_point <= 0xFF); +} + +// https://mimesniff.spec.whatwg.org/#valid-mime-type-string +bool is_valid_mime_type_string(Utf16View string) +{ + // A valid MIME type string is a string that matches the media-type token production. In particular, a valid MIME + // type string may include parameters. [HTTP-SEMANTICS] + + // The media-type production per RFC 9110 sections 8.3.1 and 5.6: + // media-type = type "/" subtype parameters + // type = token + // subtype = token + // parameters = *( OWS ";" OWS parameter ) + // parameter = parameter-name "=" parameter-value + // parameter-name = token + // parameter-value = ( token / quoted-string ) + // quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE + // qdtext = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text + // quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + // + // NOTE: RFC 9110 also permits a U+003B (;) that's not followed by a param. But per examples in the MIME Sniffing + // Standard, a string such as "text/html;" isn't a valid MIME type string — so we require a param after each + // U+003B (;) here. + // https://mimesniff.spec.whatwg.org/#example-valid-mime-type-string + Utf16GenericLexer lexer { string }; + + auto consume_http_token = [&lexer]() { + return lexer.consume_while([](char16_t code_point) { return HTTP::is_http_token_code_point(code_point); }); + }; + auto ignore_optional_whitespace = [&lexer]() { + lexer.ignore_while([](char16_t code_point) { return code_point == ' ' || code_point == '\t'; }); + }; + + auto type = consume_http_token(); + if (type.is_empty() || !lexer.consume_specific('/')) + return false; + + auto subtype = consume_http_token(); + if (subtype.is_empty()) + return false; + + while (!lexer.is_eof()) { + ignore_optional_whitespace(); + if (!lexer.consume_specific(';')) + return false; + ignore_optional_whitespace(); + + auto parameter_name = consume_http_token(); + if (parameter_name.is_empty() || !lexer.consume_specific('=')) return false; + + if (lexer.consume_specific('"')) { + while (true) { + if (lexer.is_eof()) + return false; + auto code_point = lexer.consume(); + if (code_point == '"') + break; + if (code_point == '\\') { + if (lexer.is_eof()) + return false; + if (!is_http_quoted_string_token_code_point(lexer.consume())) + return false; + } else if (!is_http_quoted_string_token_code_point(code_point)) { + return false; + } + } + } else { + auto parameter_value = consume_http_token(); + if (parameter_value.is_empty()) + return false; + } } + return true; } +static bool contains_only_http_quoted_string_token_code_points(StringView string) +{ + return all_of(Utf8View(string), is_http_quoted_string_token_code_point); +} + static bool contains_only_http_token_code_points(StringView string) { // https://mimesniff.spec.whatwg.org/#http-token-code-point diff --git a/Libraries/LibWeb/MimeSniff/MimeType.h b/Libraries/LibWeb/MimeSniff/MimeType.h index 31743da9da012..724af04371772 100644 --- a/Libraries/LibWeb/MimeSniff/MimeType.h +++ b/Libraries/LibWeb/MimeSniff/MimeType.h @@ -16,6 +16,7 @@ namespace Web::MimeSniff { bool is_javascript_mime_type_essence_match(Utf16View); +bool is_valid_mime_type_string(Utf16View); // https://mimesniff.spec.whatwg.org/#javascript-mime-type // A JavaScript MIME type is any MIME type whose essence is one of the following: diff --git a/Tests/LibWeb/Text/expected/wpt-import/media-capabilities/decodingInfo.any.txt b/Tests/LibWeb/Text/expected/wpt-import/media-capabilities/decodingInfo.any.txt index 981ad85bf2334..2c515f1576d40 100644 --- a/Tests/LibWeb/Text/expected/wpt-import/media-capabilities/decodingInfo.any.txt +++ b/Tests/LibWeb/Text/expected/wpt-import/media-capabilities/decodingInfo.any.txt @@ -1,9 +1,9 @@ Harness status: OK -Found 33 tests +Found 34 tests -15 Pass -18 Fail +29 Pass +5 Fail Pass Test that decodingInfo rejects if it doesn't get a configuration Pass Test that decodingInfo rejects if the MediaConfiguration isn't valid Pass Test that decodingInfo rejects if the MediaConfiguration does not have a type @@ -12,28 +12,29 @@ Pass Test that decodingInfo rejects if the video configuration has a negative fr Pass Test that decodingInfo rejects if the video configuration has a framerate set to 0 Pass Test that decodingInfo rejects if the video configuration has a framerate set to Infinity Pass Test that decodingInfo rejects if the video configuration contentType doesn't parse -Fail Test that decodingInfo rejects if the video configuration contentType is not a valid MIME type string +Pass Test that decodingInfo rejects if the video configuration contentType is not a valid MIME type string Pass Test that decodingInfo rejects if the video configuration contentType isn't of type video -Fail Test that decodingInfo rejects if the video configuration contentType is of type audio -Fail Test that decodingInfo rejects if the audio configuration contentType is of type video -Fail Test that decodingInfo rejects if the video configuration contentType has more than one parameter -Fail Test that decodingInfo rejects if the video configuration contentType has one parameter that isn't codecs -Fail Test that decodingInfo rejects if the video configuration contentType does not imply a single media codec but has no codecs parameter -Fail Test that decodingInfo rejects if the video configuration contentType has a codecs parameter that indicates multiple video codecs -Fail Test that decodingInfo rejects if the video configuration contentType has a codecs parameter that indicates both an audio and a video codec +Pass Test that decodingInfo rejects if the video configuration contentType is of type audio +Pass Test that decodingInfo rejects if the audio configuration contentType is of type video +Pass Test that decodingInfo rejects if the video configuration contentType has more than one parameter +Pass Test that decodingInfo rejects if the video configuration contentType has one parameter that isn't codecs +Pass Test that decodingInfo rejects if the video configuration contentType does not imply a single media codec but has no codecs parameter +Pass Test that decodingInfo rejects if the video configuration contentType has a codecs parameter that indicates multiple video codecs +Pass Test that decodingInfo rejects if the video configuration contentType has a codecs parameter that indicates both an audio and a video codec Pass Test that decodingInfo rejects if the audio configuration contentType doesn't parse -Fail Test that decodingInfo rejects if the audio configuration contentType is not a valid MIME type string +Pass Test that decodingInfo rejects if the audio configuration contentType is not a valid MIME type string Pass Test that decodingInfo rejects if the audio configuration contentType isn't of type audio -Fail Test that decodingInfo rejects if the audio configuration contentType has more than one parameter -Fail Test that decodingInfo rejects if the audio configuration contentType has one parameter that isn't codecs -Fail Test that decodingInfo rejects if the audio configuration contentType does not imply a single media codec but has no codecs parameter -Fail Test that decodingInfo rejects if the audio configuration contentType has a codecs parameter that indicates multiple audio codecs -Fail Test that decodingInfo rejects if the audio configuration contentType has a codecs parameter that indicates both an audio and a video codec +Pass Test that decodingInfo rejects if the audio configuration contentType has more than one parameter +Pass Test that decodingInfo rejects if the audio configuration contentType has one parameter that isn't codecs +Pass Test that decodingInfo rejects if the audio configuration contentType does not imply a single media codec but has no codecs parameter +Pass Test that decodingInfo rejects if the audio configuration contentType has a codecs parameter that indicates multiple audio codecs +Pass Test that decodingInfo rejects if the audio configuration contentType has a codecs parameter that indicates both an audio and a video codec Fail Test that decodingInfo returns a valid MediaCapabilitiesInfo objects Pass Test that decodingInfo rejects if the MediaConfiguration does not have a valid type Fail Test that decodingInfo with spatialRendering set returns a valid MediaCapabilitiesInfo objects Fail Test that decodingInfo with hdrMetadataType, colorGamut, and transferFunction set returns a valid MediaCapabilitiesInfo objects Fail Test that decodingInfo with mismatched codec color space is unsupported +Fail Test that decodingInfo with type media-source accepts colorGamut and transferFunction Pass Test that decodingInfo rejects if the video configuration has an empty hdrMetadataType Pass Test that decodingInfo rejects if the video configuration has a colorGamut set to true Pass Test that decodingInfo rejects if the video configuration has a transferFunction set to 3 \ No newline at end of file diff --git a/Tests/LibWeb/Text/input/wpt-import/media-capabilities/decodingInfo.any.js b/Tests/LibWeb/Text/input/wpt-import/media-capabilities/decodingInfo.any.js index 510a59d9d781d..852e723224074 100644 --- a/Tests/LibWeb/Text/input/wpt-import/media-capabilities/decodingInfo.any.js +++ b/Tests/LibWeb/Text/input/wpt-import/media-capabilities/decodingInfo.any.js @@ -374,6 +374,17 @@ promise_test(t => { }); }, 'Test that decodingInfo with mismatched codec color space is unsupported'); +promise_test(t => { + return navigator.mediaCapabilities.decodingInfo({ + type: 'media-source', + video: videoConfigurationWithDynamicRange, + }).then(ability => { + assert_equals(typeof ability.supported, "boolean"); + assert_equals(typeof ability.smooth, "boolean"); + assert_equals(typeof ability.powerEfficient, "boolean"); + }); +}, "Test that decodingInfo with type media-source accepts colorGamut and transferFunction"); + promise_test(t => { return promise_rejects_js(t, TypeError, navigator.mediaCapabilities.decodingInfo({ type: 'file',