diff --git a/crates/engine/src/js_dom_bridge_tests/part09.rs b/crates/engine/src/js_dom_bridge_tests/part09.rs index b0fef1604..8add1aaed 100644 --- a/crates/engine/src/js_dom_bridge_tests/part09.rs +++ b/crates/engine/src/js_dom_bridge_tests/part09.rs @@ -6109,3 +6109,93 @@ fn test_media_source_mse_face_t8n() { "append-after-endOfStream：readyState 回转 open + sourceopen 重派（先于 update）+ sb1/sb2 完成事件序" ); } + +/// t8s-3 音频轨 MSE 面：isTypeSupported 音频腿真值（mseAppendAudio 门面在位才 +/// true——能力诚实）+ 组合串逐 leg（bilibili DASH 探针形态翻转）+ audio +/// SourceBuffer 盖章与 appendBuffer 路由（mock 桥记录调用通道）。 +#[test] +fn test_media_source_mse_audio_face_t8s3() { + use zero_script_sandbox::{Sandbox, V8Sandbox}; + let config = zero_script_sandbox::SandboxConfig { + persistent_context: true, + ..Default::default() + }; + let mut sandbox = V8Sandbox::with_config(config).unwrap(); + sandbox.execute(generate_js_dom_shim()).unwrap(); + let mutations: Arc>> = Arc::new(Mutex::new(vec![])); + let dom_html: Arc> = Arc::new(Mutex::new("".to_string())); + let page_url: Arc> = Arc::new(Mutex::new("https://wpt.test/t.html".to_string())); + let canvas_registry: std::sync::Arc> = + std::sync::Arc::new(std::sync::Mutex::new(crate::js_dom_bridge::CanvasRegistry::new())); + register_dom_callbacks(&mut sandbox, &mutations, &dom_html, &page_url, &canvas_registry, None); + + // ① 能力诚实（双向判别）：无 mseAppendAudio（旧宿主形态）→ audio 腿 false、 + // 组合串 false（既有 t8n 钉读数不变）；补挂 mseAppendAudio 后同一查询翻 true。 + sandbox.execute( + r#" + globalThis.__zwVideoBridge = { + mseCreate: function () { return true; }, + mseAppend: function () { return true; }, + mseEnd: function () { return true; }, + mseSetDuration: function () {}, + mseDims: function () { return '320x240'; } + }; + globalThis.__t = [ + String(MediaSource.isTypeSupported('video/mp4; codecs="avc1.42E01E,mp4a.40.2"')), + String(MediaSource.isTypeSupported('audio/mp4; codecs="mp4a.40.2"')), + String(MediaSource.isTypeSupported('audio/mp4; codecs="opus"')), + String(MediaSource.isTypeSupported('video/mp4; codecs="avc1.42E01E,vp9"'))];"#, + ) + .unwrap(); + assert_eq!( + sandbox.execute("globalThis.__t.join(',')").unwrap().value, + "false,false,false,false", + "无 mseAppendAudio：audio 腿 false + 组合串 false（既有能力面不虚报）" + ); + sandbox.execute( + r#" + globalThis.__zwVideoBridge.mseAppendAudio = function (src, b64) { + this.__a = (this.__a || []); this.__a.push('audio:' + atob(b64).length); return true; + }; + globalThis.__zwVideoBridge.mseAppend = function (src, b64) { + this.__a = (this.__a || []); this.__a.push('video:' + atob(b64).length); return true; + }; + globalThis.__t2 = [ + String(MediaSource.isTypeSupported('video/mp4; codecs="avc1.42E01E,mp4a.40.2"')), + String(MediaSource.isTypeSupported('audio/mp4; codecs="mp4a.40.2"')), + String(MediaSource.isTypeSupported('audio/mp4; codecs="opus"')), + String(MediaSource.isTypeSupported('video/mp4; codecs="avc1.42E01E,vp9"')), + String(MediaSource.isTypeSupported('audio/mp4'))];"#, + ) + .unwrap(); + assert_eq!( + sandbox.execute("globalThis.__t2.join(',')").unwrap().value, + "true,true,false,false,true", + "有 mseAppendAudio：bilibili 组合探针翻 true + audio/mp4 aac true + opus/复合 vp9 false + 裸 audio/mp4 true" + ); + + // ② 双 SourceBuffer 轨章 + appendBuffer 路由：video SB 走 mseAppend、audio SB + // 走 mseAppendAudio（registry 双 feed 的 shim 侧契约面）。 + sandbox.execute( + r#" + globalThis.__zwVideoBridge.__a = []; + var ms = new MediaSource(); + globalThis.__msA = ms; + var url = URL.createObjectURL(ms); + ms.addEventListener('sourceopen', function () { + var vsb = ms.addSourceBuffer('video/mp4; codecs="avc1.42E01E"'); + var asb = ms.addSourceBuffer('audio/mp4; codecs="mp4a.40.2"'); + vsb.appendBuffer(new Uint8Array([1, 1, 1, 1])); + asb.appendBuffer(new Uint8Array([2, 2, 2, 2])); + }); + var v = document.createElement('video'); + v.src = url;"#, + ) + .unwrap(); + sandbox.execute("void 0").unwrap(); + assert_eq!( + sandbox.execute("__zwVideoBridge.__a.join(',')").unwrap().value, + "video:4,audio:4", + "appendBuffer 轨路由：video SB→mseAppend、audio SB→mseAppendAudio（各 4B）" + ); +} diff --git a/crates/engine/src/js_dom_shim/part06.js b/crates/engine/src/js_dom_shim/part06.js index daf885b5b..609f9fb3d 100644 --- a/crates/engine/src/js_dom_shim/part06.js +++ b/crates/engine/src/js_dom_shim/part06.js @@ -8862,6 +8862,9 @@ _zwMseEt.call(this); this._zwMs = mediaSource || null; this._zwUpdating = false; + // t8s-3 轨类型章（'video'|'audio'）——addSourceBuffer 按 mimeType 盖章， + // appendBuffer 据此路由视频/音频独立字节流（DASH 双轨分缓冲）。 + this._zwTrackKind = 'video'; }; globalThis.SourceBuffer.prototype = Object.create(_zwMseEt.prototype); Object.defineProperty(globalThis.SourceBuffer.prototype, 'updating', { @@ -8904,7 +8907,15 @@ try { if (msObj._zwUrl && typeof globalThis.__zwVideoBridge === 'object' && typeof globalThis.__zwVideoBridge.mseAppend === 'function') { - ok = globalThis.__zwVideoBridge.mseAppend(msObj._zwUrl, b64) === true; + // t8s-3 轨路由：video SB → 视频流；audio SB → 独立音频流；audio SB + // 遇门面缺席（不可达面——addSourceBuffer 同源守卫已挡）走失败面， + // 绝不把音频字节写入视频流（能力诚实）。 + if (self._zwTrackKind === 'audio') { + ok = typeof globalThis.__zwVideoBridge.mseAppendAudio === 'function' + && globalThis.__zwVideoBridge.mseAppendAudio(msObj._zwUrl, b64) === true; + } else { + ok = globalThis.__zwVideoBridge.mseAppend(msObj._zwUrl, b64) === true; + } } } catch (_eMseAp) {} if (ok) { @@ -8948,23 +8959,41 @@ this._zwAttach = null; }; globalThis.MediaSource.prototype = Object.create(_zwMseEt.prototype); - // spec isTypeSupported 真值表不虚报：唯一真实可解面 = video/mp4 + H.264 - //（avc1/avc3）；audio/*（音频轨 MSE 未实现）与其他容器恒 false；容器无 codecs - // 查询按 Chromium 口径 true（载荷不可解时 appendBuffer error 事件如实报）。 - // decode-h264 feature 关闭（桥缺失）→ 恒 false。 + // spec isTypeSupported 真值表不虚报：video/mp4 + H.264（avc1/avc3）恒真值； + // audio/mp4 + AAC 与组合串音频腿以「音频 append 流在位」（mseAppendAudio + // 门面 = 音频解码链可用，t8s-3）为支持面真值——门面缺席恒 false（能力诚实）。 + // 其他容器恒 false；容器无 codecs 查询按 Chromium 口径 true（载荷不可解时 + // appendBuffer error 事件如实报）。decode-h264 feature 关闭（桥缺失）→ 恒 false。 // https://www.w3.org/TR/media-source/#dom-mediasource-istypesupported globalThis.MediaSource.isTypeSupported = function (type) { if (typeof globalThis.__zwVideoBridge !== 'object' || typeof globalThis.__zwVideoBridge.mseCreate !== 'function') return false; var t = String(type == null ? '' : type).trim().toLowerCase(); - if (t.indexOf('video/mp4') !== 0) return false; + // t8s-3 音频腿：audio/mp4 + AAC（mp4a.40.x/aac）——音频 append 流在位才真 + //（mseAppendAudio 门面 = 解码链可用；组合串逐 leg 全支持才 true—— + // bilibili DASH 探针 'video/mp4; codecs="avc1.42E01E,mp4a.40.2"' 形态）。 + var isAudio = t.indexOf('audio/mp4') === 0; + if (!isAudio && t.indexOf('video/mp4') !== 0) return false; + // 音频 leg 支持面 = 音频 append 流在位（解码链可用，能力诚实）。 + var audioOk = typeof globalThis.__zwVideoBridge.mseAppendAudio === 'function' + && typeof globalThis.__zwVideoBridge.mseAppend === 'function'; + if (isAudio && !audioOk) return false; var m = /codecs\s*=\s*"([^"]*)"/.exec(t) || /codecs\s*=\s*'([^']*)'/.exec(t) || /codecs\s*=\s*([^;]+)/.exec(t); if (!m) return true; var parts = m[1].split(','); for (var i = 0; i < parts.length; i++) { var c = parts[i].trim(); - if (c && !/^avc[13]\./.test(c)) return false; + if (!c) continue; + // AAC 家族：mp4a.40.x（MPEG-4 Audio，AAC-LC 主形态）与 'aac' 别名。 + var isAac = audioOk && /^(mp4a\.40|aac)\b/.test(c); + if (isAudio) { + if (!isAac) return false; + } else if (!/^avc[13]\./.test(c) && !isAac) { + // 组合串逐 leg（bilibili DASH 探针 'avc1…,mp4a.40.2' 形态）——视频腿 + // avc1/avc3 或音频腿 AAC 全支持才 true。 + return false; + } } return true; }; @@ -8980,6 +9009,8 @@ _zwMseThrow('InvalidStateError', 'MediaSource is not open'); } var sb = new globalThis.SourceBuffer(this); + // t8s-3 轨章：audio/mp4 → 音频流（registry 独立 feed）；其余按视频。 + sb._zwTrackKind = t.trim().toLowerCase().indexOf('audio/') === 0 ? 'audio' : 'video'; this.sourceBuffers._zwPush(sb); return sb; }; diff --git a/crates/media/src/mse_feed.rs b/crates/media/src/mse_feed.rs index a1a26adba..a35638c5c 100644 --- a/crates/media/src/mse_feed.rs +++ b/crates/media/src/mse_feed.rs @@ -15,6 +15,7 @@ use std::io::{Read, Seek, SeekFrom}; use std::sync::{Arc, Mutex}; use symphonia::core::codecs::CodecParameters; +use symphonia::core::codecs::audio::{AudioCodecParameters, AudioDecoderOptions}; use symphonia::core::codecs::video::well_known::CODEC_ID_H264; use symphonia::core::formats::probe::Hint; use symphonia::core::formats::{FormatOptions, FormatReader}; @@ -22,6 +23,7 @@ use symphonia::core::io::MediaSourceStream; use symphonia::core::meta::MetadataOptions; use symphonia::core::units::TimeBase; +use crate::audio_decode::{AudioDecodeError, DecodedAudio}; use crate::decode::{ColorSpace, DecodeError, DecodedVideoFrame}; /// append 与解码共享的缓冲（单一写入面：append 互斥；解码只读）。 @@ -448,6 +450,217 @@ impl MseVideoFeed { } } +/// 音轨 probe 选中集：（轨 id、时基、采样率、声道、解码器）。 +type SelectedAudioTrack = ( + u32, + TimeBase, + u32, + u16, + Box, +); + +/// MSE 流式音频轨（t8s-3）：共享缓冲 + 惰性 symphonia reader + 音频解码。 +/// +/// 边缘语义与 [`MseVideoFeed`] 同构（增长重建 / `ended` 真流末 / 读边缘归一）， +/// 差异：轨选择取**音频轨**（symphonia registry 可解的 codec——aac feature 随 +/// workspace features 启用），解码输出 [`DecodedAudio`] 批（f32 交错—— +/// `WebmAudioEntry`/audio clock 消费契约，pts 取包真值非帧计数推导——重建后 +/// 时间轴不回跳）。skip 相按 pts 单调（重建后重放包不对解码器重放）。 +pub struct MseAudioFeed { + handle: MseFeedHandle, + reader: Option>, + decoder: Option>, + track_id: u32, + time_base: TimeBase, + sample_rate: u32, + channels: u16, + at_edge: bool, + /// probe（或重建）时的缓冲长度——symphonia 建 reader 固化 total_len， + /// append 增长对它不可见；读边缘据此判重建。 + len_at_probe: usize, + /// 已交付批的最大 pts（毫秒）——重建跳过相基准。 + last_pts_ms: Option, + skip_until_new: bool, +} + +impl MseAudioFeed { + fn blank(handle: MseFeedHandle) -> Self { + Self { + handle, + reader: None, + decoder: None, + track_id: 0, + time_base: TimeBase::try_new(1, 1_000).expect("1/1000 non-zero"), + sample_rate: 48_000, + channels: 2, + at_edge: false, + len_at_probe: 0, + last_pts_ms: None, + skip_until_new: false, + } + } + + /// 从既有 append 句柄构造（registry MSE 面：audio SourceBuffer 的 append + /// 流挂独立句柄，play 懒建解码器时经此挂接）。 + pub fn from_handle(handle: MseFeedHandle) -> Self { + Self::blank(handle) + } + + fn try_probe(&mut self) -> Result { + if self.reader.is_some() { + return Ok(true); + } + let src = FeedSource { + shared: Arc::clone(&self.handle.shared), + pos: 0, + }; + let mss = MediaSourceStream::new(Box::new(src), Default::default()); + let mut hint = Hint::new(); + hint.with_extension("mp4"); + // 数据不足 / moov 未到齐 → 留待重试（与视频 feed 同：ended 仍失败才硬错）。 + let probed = match symphonia::default::get_probe().probe( + &hint, + mss, + FormatOptions::default(), + MetadataOptions::default(), + ) { + Ok(p) => p, + Err(e) => { + if self.handle.is_ended() { + return Err(AudioDecodeError::Probe(e.to_string())); + } + return Ok(false); + } + }; + // 轨选择：首个 symphonia registry 可解的音频轨（make_audio_decoder 即 + // 支持面真值——aac feature 在位 → mp4a.40.2 可解；不支持 codec 错误上抛）。 + let mut selected: Option = None; + for track in probed.tracks() { + let Some(CodecParameters::Audio(params)) = &track.codec_params else { + continue; + }; + let Some(rate) = params.sample_rate else { + continue; + }; + let audio_params = AudioCodecParameters { + codec: params.codec, + profile: params.profile, + sample_rate: params.sample_rate, + sample_format: params.sample_format, + bits_per_sample: params.bits_per_sample, + bits_per_coded_sample: params.bits_per_coded_sample, + channels: params.channels.clone(), + max_frames_per_packet: params.max_frames_per_packet, + verification_check: params.verification_check, + extra_data: params.extra_data.clone(), + frames_per_block: params.frames_per_block, + }; + let decoder = match symphonia::default::get_codecs() + .make_audio_decoder(&audio_params, &AudioDecoderOptions::default()) + { + Ok(d) => d, + // 该轨 codec 不在注册表 → 尝试下一音轨（多音轨面）；全失败 → + //Ok(None) 等待面不适用（init 段已到齐才进到这里），按不支持硬错。 + Err(_) => continue, + }; + let channels = params.channels.as_ref().map(|c| c.count()).unwrap_or(1) as u16; + let tb = track + .time_base + .unwrap_or_else(|| TimeBase::try_new(1, rate).expect("1/rate non-zero")); + selected = Some((track.id, tb, rate, channels, decoder)); + break; + } + let Some((track_id, tb, rate, channels, decoder)) = selected else { + return Err(AudioDecodeError::NoTrack); + }; + self.track_id = track_id; + self.time_base = tb; + self.sample_rate = rate; + self.channels = channels; + self.decoder = Some(decoder); + self.reader = Some(probed); + self.len_at_probe = self.handle.byte_len(); + Ok(true) + } + + /// 解码下一批（一个 media packet 一批；等待更多数据 / 真流末 → `Ok(None)`， + /// 区分经 [`Self::is_exhausted`]）。 + pub fn next_batch(&mut self) -> Result, AudioDecodeError> { + if !self.try_probe()? { + return Ok(None); + } + loop { + let next = match self.reader.as_mut().expect("probed").next_packet() { + Ok(x) => Ok(x), + Err(symphonia::core::errors::Error::IoError(ref e)) + if e.kind() == std::io::ErrorKind::UnexpectedEof => + { + Ok(None) + } + Err(e) => Err(e), + }; + match next { + Ok(Some(packet)) => { + if packet.track_id != self.track_id { + continue; + } + // 同视频 feed N6：pts 换算 u64 饱和。 + let pts128 = packet.pts.get() as u128 * 1_000 * u128::from(self.time_base.numer.get()) + / u128::from(self.time_base.denom.get()); + let pts_ms = u64::try_from(pts128).unwrap_or(u64::MAX); + // 重建跳过相：pts ≤ 已交付的重放包不重解（时间轴由包 pts + // 真值承载，重复批会使 audio clock 游标回跳）。 + if self.skip_until_new && self.last_pts_ms.is_some_and(|last| pts_ms <= last) { + continue; + } + self.skip_until_new = false; + self.at_edge = false; + let decoded = self + .decoder + .as_mut() + .expect("probed") + .decode(&packet) + .map_err(|e| AudioDecodeError::Decode(e.to_string()))?; + let mut samples = Vec::with_capacity(decoded.samples_interleaved()); + decoded.copy_to_vec_interleaved::(&mut samples); + if samples.is_empty() { + continue; + } + self.last_pts_ms = Some(pts_ms); + return Ok(Some(DecodedAudio { + samples, + sample_rate: self.sample_rate, + channels: self.channels, + pts_ms, + })); + } + Ok(None) => { + self.at_edge = true; + if self.handle.is_ended() { + return Ok(None); + } + if self.handle.byte_len() > self.len_at_probe { + self.reader = None; + self.decoder = None; + self.track_id = 0; + self.skip_until_new = self.last_pts_ms.is_some(); + if self.try_probe()? { + continue; + } + } + return Ok(None); + } + Err(e) => return Err(AudioDecodeError::Decode(e.to_string())), + } + } + } + + /// 真流末（`ended` 且 demux 耗尽）——伴生音频条目停止判定的 MSE 面。 + pub fn is_exhausted(&self) -> bool { + self.at_edge && self.handle.is_ended() + } +} + #[cfg(test)] mod tests { use super::*; @@ -796,6 +1009,392 @@ mod tests { moofmdat } + /// 提取 fixture 音轨素材（t8s-3 钉测）：esds payload + 逐 AAC 包字节。 + /// 提取走 symphonia 自身 demux（与生产同路径——手搓 stsc/stco 样本表遍历 + /// 已证易错，弃用）：probe fixture → 音轨包序列原样收集。 + fn fixture_audio_material() -> (Vec, Vec>) { + use symphonia::core::formats::{FormatOptions, probe::Hint}; + use symphonia::core::io::MediaSourceStream; + use symphonia::core::meta::MetadataOptions; + let data = fixture_bytes(); + let mss = MediaSourceStream::new(Box::new(std::io::Cursor::new(data.clone())), Default::default()); + let mut probed = symphonia::default::get_probe() + .probe(&Hint::new(), mss, FormatOptions::default(), MetadataOptions::default()) + .expect("fixture probe"); + let audio_track_id = probed + .tracks() + .iter() + .find(|t| t.codec_params.as_ref().is_some_and(|c| c.is_audio())) + .map(|t| t.id) + .expect("fixture 音轨"); + // esds：从 stsd 手取（demux 层不透出）——moov → soun trak → stbl/stsd。 + let (moov, _, _) = find_box(&data, b"moov").expect("moov"); + let mut esds: Option> = None; + for (f, s, e) in top_boxes(moov) { + if f != *b"trak" { + continue; + } + let trak = &moov[s..e]; + let (mdia, _, _) = find_box(trak, b"mdia").expect("mdia"); + let (hdlr, _, _) = find_box(mdia, b"hdlr").expect("hdlr"); + if hdlr[8..12] != *b"soun" { + continue; + } + let (minf, _, _) = find_box(mdia, b"minf").expect("minf"); + let (stbl, _, _) = find_box(minf, b"stbl").expect("stbl"); + let (stsd, _, _) = find_box(stbl, b"stsd").expect("stsd"); + let entry_size = u32_at(stsd, 8) as usize; + let entry = &stsd[8..8 + entry_size]; + // mp4a 子 box 从 entry body 28B 固定段后开始（8B 头 + 28B 音频样目）。 + for (cf, cs, ce) in top_boxes(&entry[8 + 28..]) { + if cf == *b"esds" { + esds = Some(entry[8 + 28 + cs..8 + 28 + ce].to_vec()); + } + } + } + let esds = esds.expect("fixture 音轨 esds"); + let mut samples = Vec::new(); + while let Ok(Some(pkt)) = probed.next_packet() { + if pkt.track_id == audio_track_id { + samples.push(pkt.data.to_vec()); + } + } + assert!(!samples.is_empty(), "fixture 音轨样本提取"); + (esds, samples) + } + + fn u32_at(buf: &[u8], off: usize) -> u32 { + u32::from_be_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) + } + + /// 手造最小 fragmented fMP4 **音频-only** 流（DASH 音频轨形态，t8s-3）： + /// ftyp + moov(mvhd/trak(tkhd+mdia(mdhd/hdlr(soun)/minf(smhd/dinf/stbl( + /// stsd(mp4a+esds))/stts/stsc/stsz/stco)))+mvex(trex)) + moof(mfhd/traf( + /// tfhd/tfdt/trun)) + mdat。timescale = 真实采样率（测试先经 + /// `AudioDecoder::open` 取 esds 真值），trex/trun 样本时长 = 1024 + /// （AAC 每帧 1024 样本约定）。 + fn build_fragmented_audio(esds: &[u8], rate: u32, samples: &[Vec]) -> (Vec, Vec) { + let timescale: u32 = rate; + let frame_dur: u32 = 1024; + // mp4a sample entry body：6B reserved + 2B dri + 8B reserved + 2B chan + // + 2B size + 2B pre_defined + 2B reserved + 4B rate(16.16)，随后 esds。 + let mut mp4a_body = Vec::new(); + mp4a_body.extend_from_slice(&[0u8; 6]); + mp4a_body.extend_from_slice(&be32(1)[2..]); + mp4a_body.extend_from_slice(&[0u8; 8]); + mp4a_body.extend_from_slice(&be32(1)[2..]); // channelcount = 1（fixture 单声道） + mp4a_body.extend_from_slice(&be32(16)[2..]); // samplesize + mp4a_body.extend_from_slice(&be32(0)[2..]); + mp4a_body.extend_from_slice(&be32(0)[2..]); + mp4a_body.extend_from_slice(&be32(rate << 16)); // samplerate 16.16 + mp4a_body.extend_from_slice(&box_header(b"esds", esds.len())); + mp4a_body.extend_from_slice(esds); + let mp4a = box_header(b"mp4a", mp4a_body.len()) + .into_iter() + .chain(mp4a_body) + .collect::>(); + let mut stsd_payload = be32(1).to_vec(); + stsd_payload.extend_from_slice(&mp4a); + let stsd = full_box(b"stsd", 0, 0, stsd_payload); + let stts = full_box(b"stts", 0, 0, be32(0).to_vec()); + let stsc = full_box(b"stsc", 0, 0, be32(0).to_vec()); + let mut stsz_payload = be32(0).to_vec(); + stsz_payload.extend_from_slice(&be32(0)); + let stsz = full_box(b"stsz", 0, 0, stsz_payload); + let stco = full_box(b"stco", 0, 0, be32(0).to_vec()); + let stbl_body = [stsd, stts, stsc, stsz, stco].concat(); + let stbl = box_header(b"stbl", stbl_body.len()) + .into_iter() + .chain(stbl_body) + .collect::>(); + let smhd = full_box(b"smhd", 0, 0, be32(0).to_vec()); + let dref = full_box(b"dref", 0, 0, { + let mut p = be32(1).to_vec(); + p.extend_from_slice(&full_box(b"url ", 0, 1, Vec::new())); + p + }); + let dinf = box_header(b"dinf", dref.len()) + .into_iter() + .chain(dref) + .collect::>(); + let minf_body = [smhd, dinf, stbl].concat(); + let minf = box_header(b"minf", minf_body.len()) + .into_iter() + .chain(minf_body) + .collect::>(); + let hdlr = full_box(b"hdlr", 0, 0, { + let mut p = be32(0).to_vec(); + p.extend_from_slice(b"soun"); + p.extend_from_slice(&[0u8; 12]); + p.extend_from_slice(b"SoundHandler\0"); + p + }); + let mdhd = full_box(b"mdhd", 0, 0, { + let mut p = Vec::new(); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(timescale)); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(0x55C00000)); + p + }); + let mdia_body = [mdhd, hdlr, minf].concat(); + let mdia = box_header(b"mdia", mdia_body.len()) + .into_iter() + .chain(mdia_body) + .collect::>(); + let tkhd = full_box(b"tkhd", 0, 7, { + let mut p = Vec::new(); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(2)); // track_id = 2（音轨） + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&[0u8; 8]); + p.extend_from_slice(&be32(0)[2..]); + p.extend_from_slice(&be32(0)[2..]); + p.extend_from_slice(&be32(0x0100)[2..]); // volume = 1.0（音轨） + p.extend_from_slice(&be32(0)[2..]); + p.extend_from_slice(&UNITY_MATRIX.iter().flat_map(|v| v.to_be_bytes()).collect::>()); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(0)); + p + }); + let trak_body = [tkhd, mdia].concat(); + let trak = box_header(b"trak", trak_body.len()) + .into_iter() + .chain(trak_body) + .collect::>(); + let mvhd = full_box(b"mvhd", 0, 0, { + let mut p = Vec::new(); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(timescale)); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(0x00010000)); + p.extend_from_slice(&be32(0x01000000)[..2]); + p.extend_from_slice(&[0u8; 2]); + p.extend_from_slice(&[0u8; 8]); + p.extend_from_slice(&UNITY_MATRIX.iter().flat_map(|v| v.to_be_bytes()).collect::>()); + p.extend_from_slice(&[0u8; 24]); + p.extend_from_slice(&be32(3)); + p + }); + let trex = full_box(b"trex", 0, 0, { + let mut p = be32(2).to_vec(); // track_id = 2 + p.extend_from_slice(&be32(1)); + p.extend_from_slice(&be32(frame_dur)); + p.extend_from_slice(&be32(0)); + p.extend_from_slice(&be32(0x02000000)); // 关键帧旗标 + p + }); + let mvex = box_header(b"mvex", trex.len()) + .into_iter() + .chain(trex) + .collect::>(); + let moov_body = [mvhd, trak, mvex].concat(); + let moov = box_header(b"moov", moov_body.len()) + .into_iter() + .chain(moov_body) + .collect::>(); + let ftyp = box_header(b"ftyp", 12) + .into_iter() + .chain(b"iso5".to_vec()) + .chain(be32(0)) + .chain(b"iso5".to_vec()) + .collect::>(); + let init = [ftyp, moov].concat(); + // media 段：moof + mdat（全部 AAC 帧一个 fragment）。 + let mut trun_payload = be32(samples.len() as u32).to_vec(); + trun_payload.extend_from_slice(&be32(0)); + for _ in samples { + trun_payload.extend_from_slice(&be32(frame_dur)); + trun_payload.extend_from_slice(&be32(0)); + trun_payload.extend_from_slice(&be32(0x02000000)); + } + let trun = full_box(b"trun", 0, 0x000701, trun_payload); + let tfhd = full_box(b"tfhd", 0, 0x020000, be32(2).to_vec()); + let tfdt = full_box(b"tfdt", 1, 0, be32(0).to_vec()); + let mfhd = full_box(b"mfhd", 0, 0, be32(1).to_vec()); + let trun_len = trun.len(); + let traf_body = [tfhd, tfdt, trun].concat(); + let traf = box_header(b"traf", traf_body.len()) + .into_iter() + .chain(traf_body) + .collect::>(); + let moof_body = [mfhd, traf].concat(); + let moof = box_header(b"moof", moof_body.len()) + .into_iter() + .chain(moof_body) + .collect::>(); + let mut mdat_payload = Vec::new(); + for s in samples { + mdat_payload.extend_from_slice(s); + } + let mut moofmdat = moof.clone(); + let mdat_start = moofmdat.len(); + moofmdat.extend_from_slice(&box_header(b"mdat", mdat_payload.len())); + moofmdat.extend_from_slice(&mdat_payload); + let trun_data_start = moof.len() - trun_len + 12 + 4; + moofmdat[trun_data_start..trun_data_start + 4].copy_from_slice(&be32((mdat_start + 8) as u32)); + let mut off = trun_data_start + 4; + for s in samples { + moofmdat[off + 4..off + 8].copy_from_slice(&be32(s.len() as u32)); + off += 12; + } + (init, moofmdat) + } + + /// 钉④（t8s-3 渐进面）：双轨 mp4 渐进流喂——音轨惰性 probe + 分块边缘语义 + /// 与视频 feed 同构；批 pts 单调、采样率/声道一致。 + #[test] + fn mse_audio_feed_progressive_streaming_t8s3() { + let rate = crate::AudioDecoder::open(&fixture_bytes()) + .expect("fixture 音轨可解") + .sample_rate(); + let data = fixture_bytes(); + let cut = data.len() / 2; + let handle = MseFeedHandle::new(); + let mut feed = MseAudioFeed::from_handle(handle.clone()); + handle.append(&data[..cut]); + assert!(feed.next_batch().unwrap().is_none(), "moov 未到齐=等待面"); + assert!(!feed.is_exhausted(), "未 end() 的边缘不是流末"); + handle.append(&data[cut..]); + let mut batches = 0; + let mut last_pts = 0u64; + while let Some(b) = feed.next_batch().unwrap() { + assert_eq!(b.sample_rate, rate); + assert_eq!(b.channels, 1, "fixture 单声道"); + assert!(b.pts_ms >= last_pts, "批 pts 单调"); + assert!(!b.samples.is_empty()); + last_pts = b.pts_ms; + batches += 1; + assert!(batches <= 10_000, "runaway batches"); + } + assert!(batches > 0, "音轨应产出批"); + assert!(!feed.is_exhausted(), "未 end() 前 demux 耗尽仍非真流末"); + handle.end(); + assert!(feed.next_batch().unwrap().is_none()); + assert!(feed.is_exhausted(), "end() 后耗尽即真流末"); + } + + /// 钉⑤（t8s-3 fragmented 面，DASH 音频轨形态）：音频-only fMP4 init+media + /// 段逐段 append——symphonia mvex 流式 demux + AAC 解码，批数与样本表等价。 + #[test] + fn mse_audio_feed_fragmented_fmp4_t8s3() { + let rate = crate::AudioDecoder::open(&fixture_bytes()) + .expect("fixture 音轨可解") + .sample_rate(); + let (esds, samples) = fixture_audio_material(); + let (init, media) = build_fragmented_audio(&esds, rate, &samples); + let handle = MseFeedHandle::new(); + let mut feed = MseAudioFeed::from_handle(handle.clone()); + handle.append(&init); + assert!(feed.next_batch().unwrap().is_none(), "init 段无批（等 media 段）"); + assert!(!feed.is_exhausted()); + handle.append(&media); + let mut batches = 0; + let mut last_pts = 0u64; + while let Some(b) = feed.next_batch().unwrap() { + assert!(b.pts_ms >= last_pts, "批 pts 单调（tfdt/trun 时序）"); + last_pts = b.pts_ms; + batches += 1; + assert!(batches <= 10_000, "runaway batches"); + } + assert_eq!(batches, samples.len(), "逐 AAC 帧一批"); + handle.end(); + assert!(feed.is_exhausted(), "end() 后真流末"); + } + + /// 音频续段构造器（t8s-3 返修 N1）：复刻 build_fragmented_audio 的 moof+mdat + /// 单 fragment 面，tfdt 基准偏移到 `base_dts`（高 pts 续段）、mfhd 序号 `seq`。 + fn build_media_segment_cont_audio(samples: &[Vec], base_dts: u32, seq: u32) -> Vec { + let frame_dur: u32 = 1024; + let mut trun_payload = be32(samples.len() as u32).to_vec(); + trun_payload.extend_from_slice(&be32(0)); + for _ in samples { + trun_payload.extend_from_slice(&be32(frame_dur)); + trun_payload.extend_from_slice(&be32(0)); + trun_payload.extend_from_slice(&be32(0x02000000)); + } + let trun = full_box(b"trun", 0, 0x000701, trun_payload); + let tfhd = full_box(b"tfhd", 0, 0x020000, be32(2).to_vec()); + let tfdt = full_box(b"tfdt", 1, 0, be32(base_dts).to_vec()); + let mfhd = full_box(b"mfhd", 0, 0, be32(seq).to_vec()); + let trun_len = trun.len(); + let traf_body = [tfhd, tfdt, trun].concat(); + let traf = box_header(b"traf", traf_body.len()) + .into_iter() + .chain(traf_body) + .collect::>(); + let moof_body = [mfhd, traf].concat(); + let moof = box_header(b"moof", moof_body.len()) + .into_iter() + .chain(moof_body) + .collect::>(); + let mut mdat_payload = Vec::new(); + for s in samples { + mdat_payload.extend_from_slice(s); + } + let mut moofmdat = moof.clone(); + let mdat_start = moofmdat.len(); + moofmdat.extend_from_slice(&box_header(b"mdat", mdat_payload.len())); + moofmdat.extend_from_slice(&mdat_payload); + let trun_data_start = moof.len() - trun_len + 12 + 4; + moofmdat[trun_data_start..trun_data_start + 4].copy_from_slice(&be32((mdat_start + 8) as u32)); + let mut off = trun_data_start + 4; + for s in samples { + moofmdat[off + 4..off + 8].copy_from_slice(&be32(s.len() as u32)); + off += 12; + } + moofmdat + } + + /// 钉⑥（t8s-3 返修 N1，重建跳相）：读边缘 + 缓冲增长触发重建——skip 相按 + /// last_pts_ms 生效，已交付段不得重放（生产形态：解码泵排到写入边缘 → + /// appendBuffer 续段 → 下次 next_batch 边缘重建续读）。中途（未到边缘） + /// append 走顺序读不触发重建，不在本钉场景。 + #[test] + fn mse_audio_feed_rebuild_skip_phase_t8s3() { + let rate = crate::AudioDecoder::open(&fixture_bytes()) + .expect("fixture 音轨可解") + .sample_rate(); + let (esds, samples) = fixture_audio_material(); + let (init, media) = build_fragmented_audio(&esds, rate, &samples); + let cont = build_media_segment_cont_audio(&samples, (samples.len() as u32) * 1024, 2); + let handle = MseFeedHandle::new(); + let mut feed = MseAudioFeed::from_handle(handle.clone()); + handle.append(&init); + handle.append(&media); + // 排到读边缘：首段 87 批全交付（last_pts_ms = 首段最大 pts）。 + let mut seg1 = 0usize; + let mut last_pts = 0u64; + while let Some(b) = feed.next_batch().unwrap() { + assert!(b.pts_ms >= last_pts, "首段批 pts 单调（亚毫秒间隔允许等值）"); + last_pts = b.pts_ms; + seg1 += 1; + } + assert_eq!(seg1, samples.len(), "首段排空至边缘"); + assert!(!feed.is_exhausted(), "未 end() 的边缘是等待面"); + // 边缘后续喂（缓冲增长）→ 重建 → skip 相丢弃已交付段重放包。 + handle.append(&cont); + let mut seg2 = 0usize; + let mut replays = 0usize; + while let Some(b) = feed.next_batch().unwrap() { + if b.pts_ms <= last_pts { + replays += 1; + } + assert!(b.pts_ms >= last_pts, "续段批 pts 单调（重建后不得回跳/重放）"); + last_pts = b.pts_ms; + seg2 += 1; + assert!(seg2 <= 10_000, "runaway batches"); + } + assert_eq!(replays, 0, "重建后零重放（skip 相生效）"); + assert_eq!(seg2, samples.len(), "续段批数与样本表等价"); + handle.end(); + assert!(feed.is_exhausted(), "end() 后真流末"); + } + /// 钉①（渐进式流喂）：同一 progressive mp4 分两块 append——probe 惰性建 /// reader，帧解码与整文件 open 等价；未 end() 前 None ≠ 真流末。 #[test] diff --git a/crates/webview/src/video_registry.rs b/crates/webview/src/video_registry.rs index a906d78ae..bd57d14a0 100644 --- a/crates/webview/src/video_registry.rs +++ b/crates/webview/src/video_registry.rs @@ -333,6 +333,101 @@ impl WebmAudioEntry { } } +/// MSE 伴生音频条目（t8s-3）——audio SourceBuffer 独立字节流的播放面。 +/// +/// 与 [`WebmAudioEntry`] 同 sink/cursor/增益契约，差异：源是 [`zero_media::MseAudioFeed`] +/// 增长流——`next_batch` 的 None 是「等待更多 append」而非流末，停止判定走 +/// `is_exhausted()`（ended 旗标 + demux 耗尽）；loop 回卷不适用（流式源，FIXME(mse-full-semantics)）。 +#[cfg(feature = "decode-h264")] +struct MseAudioEntry { + feed: zero_media::MseAudioFeed, + sink: NullSink, + sink_started: bool, + playing: bool, + /// 已解码累计游标（毫秒）——audio clock 主时钟的媒体时间真值面。 + cursor_ms: u64, + /// seek 追赶区静默线（毫秒）。 + skip_until_ms: u64, + last_tick_ms: Option, + volume: f32, + muted: bool, +} + +#[cfg(feature = "decode-h264")] +impl MseAudioEntry { + fn new(feed: zero_media::MseAudioFeed) -> Self { + Self { + feed, + sink: NullSink::new(), + sink_started: false, + playing: false, + cursor_ms: 0, + skip_until_ms: 0, + last_tick_ms: None, + volume: 1.0, + muted: false, + } + } + + /// 推进到墙钟 `now_ms`（WebmAudioEntry 同面——batch pts + 采样计数游标）。 + fn advance_to(&mut self, now_ms: u64) -> bool { + if !self.playing { + return false; + } + let last = self.last_tick_ms.unwrap_or(now_ms); + self.last_tick_ms = Some(now_ms); + let target = self.cursor_ms + now_ms.saturating_sub(last); + let gain = if self.muted { 0.0 } else { self.volume }; + let mut wrote = false; + while self.cursor_ms < target { + match self.feed.next_batch() { + Ok(Some(batch)) => { + if !self.sink_started { + let _ = self.sink.start(AudioFormat { + sample_rate: batch.sample_rate, + channels: batch.channels, + }); + self.sink_started = true; + } + // saturating：pts 饱和值（u64::MAX，负 pts/edit-list 形态） + // 不进裸加法（debug overflow panic / release 回绕）。 + self.cursor_ms = batch.pts_ms.saturating_add( + (batch.samples.len() as u64 * 1000) + / (u64::from(batch.sample_rate) * u64::from(batch.channels).max(1)), + ); + if self.cursor_ms <= self.skip_until_ms { + continue; + } + if gain == 1.0 { + let _ = self.sink.write(&batch.samples); + } else { + let gained: Vec = batch.samples.iter().map(|s| s * gain).collect(); + let _ = self.sink.write(&gained); + } + wrote = true; + if self.cursor_ms >= target { + break; + } + } + Ok(None) => { + // MSE 流边缘：ended + 耗尽才停；增长等待面保持播放态 + //（下一 tick 续解码——appendBuffer 与解码泵异步交错）。 + if self.feed.is_exhausted() { + self.playing = false; + } + break; + } + Err(_) => { + // 解码错误停轨（事件面归后续切片——FIXME(mse-full-semantics)）。 + self.playing = false; + break; + } + } + } + wrote + } +} + /// 注册表规范化键（M3 扩批 XXIV）：query/fragment 不敏感——WPT 用例的 cache-buster /// query（`?...Math.random()`）指向同一 fixture 字节；shim 侧 IDL getter 与 runner /// 快照的 URL 编码形态可能不一致（空格/括号编码差异），strip 后以路径为键，两侧 @@ -366,6 +461,13 @@ pub struct VideoPlayerRegistry { /// 声明，语义层 duration 以此优先）。 #[cfg(feature = "decode-h264")] mse_durations: HashMap, + /// MSE 音频 append 句柄（t8s-3）：audio SourceBuffer 的独立字节流（DASH + /// 音轨独立 init/media 段——与视频 feed 分缓冲，同键不同流）。 + #[cfg(feature = "decode-h264")] + mse_audio_feeds: HashMap, + /// MSE 伴生音频条目（t8s-3）：play 懒建——audio clock 主时钟面。 + #[cfg(feature = "decode-h264")] + mse_audio_entries: HashMap, } /// MSE 单 feed 共享缓冲总量封顶（t8n 返修 N2——页面 appendBuffer 字节跨信任 @@ -403,6 +505,8 @@ impl VideoPlayerRegistry { { self.mse_feeds.remove(&key); self.mse_durations.remove(&key); + self.mse_audio_feeds.remove(&key); + self.mse_audio_entries.remove(&key); } } @@ -439,6 +543,8 @@ impl VideoPlayerRegistry { { self.mse_feeds.clear(); self.mse_durations.clear(); + self.mse_audio_feeds.clear(); + self.mse_audio_entries.clear(); } } @@ -448,9 +554,29 @@ impl VideoPlayerRegistry { pub fn mse_create(&mut self, abs_src: &str) { let key = registry_key(abs_src); self.mse_feeds.insert(key, zero_media::MseFeedHandle::new()); + // t8s-3：audio SourceBuffer 注册同步建音频句柄（同 MediaSource 重建语义 + // ——旧流/旧播放面一并复位）。 + self.mse_audio_feeds.insert(key, zero_media::MseFeedHandle::new()); + self.mse_audio_entries.remove(&key); self.mse_durations.remove(&key); } + /// MSE appendBuffer（音频轨，t8s-3）：audio SourceBuffer 字节追入独立缓冲。 + /// 容量封顶/未登记键语义与视频 append 同面。 + #[cfg(feature = "decode-h264")] + pub fn mse_append_audio(&mut self, abs_src: &str, data: &[u8]) -> bool { + match self.mse_audio_feeds.get(®istry_key(abs_src)) { + Some(handle) => { + if handle.byte_len().saturating_add(data.len()) > MSE_FEED_MAX_BYTES { + return false; + } + handle.append(data); + true + } + None => false, + } + } + /// MSE appendBuffer：字节追入共享缓冲（解码侧惰性 probe + 增长重建续读）。 /// 返 false = 未登记键或超容量封顶（shim 走 append error 面——spec /// QuotaExceededError 语义的最小面）。 @@ -470,10 +596,22 @@ impl VideoPlayerRegistry { } } + /// MSE 伴生音频游标读数（毫秒；t8s-3 观测面——未建条目/未登记键 → None）。 + #[cfg(feature = "decode-h264")] + pub fn mse_audio_cursor_ms(&self, abs_src: &str) -> Option { + self.mse_audio_entries.get(®istry_key(abs_src)).map(|e| e.cursor_ms) + } + /// MSE endOfStream：置流末旗标（此后写入边缘 = 真 EOF，player Ended 可达）。 + /// spec endOfStream 是 MediaSource 级 op——视频/音频全 source buffer 的流末 + /// 旗标一并置位（t8s-3 双 feed 面）。 #[cfg(feature = "decode-h264")] pub fn mse_end(&mut self, abs_src: &str) { - if let Some(handle) = self.mse_feeds.get(®istry_key(abs_src)) { + let key = registry_key(abs_src); + if let Some(handle) = self.mse_feeds.get(&key) { + handle.end(); + } + if let Some(handle) = self.mse_audio_feeds.get(&key) { handle.end(); } } @@ -567,7 +705,13 @@ impl VideoPlayerRegistry { entry.volume = volume.clamp(0.0, 1.0); entry.muted = muted; } - // A/V pair 伴生轨同步增益（video 元素的 volume/muted setter 桥推面）。 + // A/V pair 伴生轨同步增益（video 元素的 volume/muted setter 桥推面）； + // MSE 伴生音频同 clamp 契约（t8s-3 返修 N3）。 + #[cfg(feature = "decode-h264")] + if let Some(entry) = self.mse_audio_entries.get_mut(&key) { + entry.volume = volume.clamp(0.0, 1.0); + entry.muted = muted; + } if let Some(entry) = self.av_audio_entries.get_mut(&key) { entry.volume = volume.clamp(0.0, 1.0); entry.muted = muted; @@ -596,6 +740,17 @@ impl VideoPlayerRegistry { { let decoder = VideoTrackDecoder::Mse(Box::new(zero_media::MseVideoFeed::from_handle(handle))); self.players.insert(key, VideoPlayer::new(decoder)); + // t8s-3：audio SourceBuffer 已有字节才建伴生音频条目（纯视频流零 + // 开销——条目缺席走墙钟 tick，t8n 单轨形态零回归）；probe 惰性—— + // 音轨 init 段未到齐时 next_batch 返回等待面，播放态保持。 + if let Some(audio_handle) = self.mse_audio_feeds.get(&key).cloned() + && audio_handle.byte_len() > 0 + { + self.mse_audio_entries.insert( + key, + MseAudioEntry::new(zero_media::MseAudioFeed::from_handle(audio_handle)), + ); + } } if !self.players.contains_key(&key) { // M3 扩批 XVI：解码器构建失败不再消费源字节（此前 sources.remove 前置—— @@ -663,6 +818,19 @@ impl VideoPlayerRegistry { handle, )))); } + // MSE 伴生音频同面回卷（t8s-3 返修 B1）：同句柄重建 feed + 游标 + // 归零——与 av 伴生轨回卷块同契约（audio clock 主时钟读数随 + // reset 归零，复活重播音频腿不静默）。 + #[cfg(feature = "decode-h264")] + if let Some(audio_handle) = self.mse_audio_feeds.get(&key).cloned().filter(|h| h.byte_len() > 0) { + if let Some(entry) = self.mse_audio_entries.get_mut(&key) { + entry.feed = zero_media::MseAudioFeed::from_handle(audio_handle); + entry.sink_started = false; + entry.cursor_ms = 0; + entry.skip_until_ms = 0; + entry.last_tick_ms = None; + } + } // 源字节：单轨源 play 首次即消费（sources.remove），双轨源经 av_sources // 留存（伴生轨 seek 重建共用）——两处任一在即可重建；MSE 键两处皆空 //（不经 settle 登记），天然 no-op。 @@ -698,6 +866,13 @@ impl VideoPlayerRegistry { entry.playing = true; entry.last_tick_ms = Some(now_ms); } + // MSE 伴生音频起播（t8s-3，与视频同锚点时钟；Ended 复活=同句柄重建 feed + // ——reset 面经 players 重建分支，音频条目随 play 重建游标归零）。 + #[cfg(feature = "decode-h264")] + if let Some(entry) = self.mse_audio_entries.get_mut(&key) { + entry.playing = true; + entry.last_tick_ms = Some(now_ms); + } self.players.contains_key(&key) } @@ -711,6 +886,11 @@ impl VideoPlayerRegistry { entry.playing = false; entry.last_tick_ms = None; } + #[cfg(feature = "decode-h264")] + if let Some(entry) = self.mse_audio_entries.get_mut(&key) { + entry.playing = false; + entry.last_tick_ms = None; + } } /// playbackRate 变速（clamp 面 player 内置；未建 player 时登记于建时生效—— @@ -746,6 +926,19 @@ impl VideoPlayerRegistry { } } } + // MSE 伴生音频 seek（t8s-3 返修 N2）：同句柄重建 feed（读游标归流首） + // + 主时钟游标对齐 target + 追赶区静默线消费 skip_until_ms——播放中 + // seek 后下一 tick 主时钟分支以新游标同步视频（不回拉）。 + #[cfg(feature = "decode-h264")] + if let Some(audio_handle) = self.mse_audio_feeds.get(&key).cloned().filter(|h| h.byte_len() > 0) + && let Some(entry) = self.mse_audio_entries.get_mut(&key) + { + entry.feed = zero_media::MseAudioFeed::from_handle(audio_handle); + entry.sink_started = false; + entry.cursor_ms = target_ms; + entry.skip_until_ms = target_ms; + entry.last_tick_ms = None; + } // 伴生音频轨 seek（重建 + 追赶区静默）——master clock 游标对齐 target， // 后续 tick 视频 sync_to_media_time 跟随（不脱轨）。 // codec 泛化重建（A_VORBIS 优先、A_OPUS 次之——与 play 懒建同序）。 @@ -795,6 +988,11 @@ impl VideoPlayerRegistry { /// 由组合时钟驱动；无伴生轨回落视频位置）。 pub fn current_time(&self, abs_src: &str) -> f64 { let key = registry_key(abs_src); + // MSE 伴生音频游标优先（t8s-3 返修 N4——audio clock 主时钟契约与 av 面一致）。 + #[cfg(feature = "decode-h264")] + if let Some(e) = self.mse_audio_entries.get(&key) { + return e.cursor_ms as f64 / 1000.0; + } if let Some(cursor_ms) = self.av_audio_entries.get(&key).map(|e| e.cursor_ms) { return cursor_ms as f64 / 1000.0; } @@ -879,6 +1077,13 @@ impl VideoPlayerRegistry { self.players.values().any(|p| p.is_playing()) || self.audio_entries.values().any(|e| e.playing) || self.av_audio_entries.values().any(|e| e.playing) + || { + #[cfg(feature = "decode-h264")] + if self.mse_audio_entries.values().any(|e| e.playing) { + return true; + } + false + } } /// 音频泵推进（tab_worker 帧泵同节拍调用）：所有播放中的音频条目按实时节奏 @@ -906,6 +1111,18 @@ impl VideoPlayerRegistry { wrote = true; } } + // MSE 伴生音频轨推进（t8s-3）。 + #[cfg(feature = "decode-h264")] + { + let mse_keys: Vec = self.mse_audio_entries.keys().copied().collect(); + for key in mse_keys { + if let Some(entry) = self.mse_audio_entries.get_mut(&key) + && entry.advance_to(now_ms) + { + wrote = true; + } + } + } wrote } @@ -942,6 +1159,20 @@ impl VideoPlayerRegistry { entry.advance_to(now_ms, restart); } } + // MSE 伴生音频轨主时钟先行（t8s-3）。 + #[cfg(feature = "decode-h264")] + let mse_keys: Vec = self + .mse_audio_entries + .keys() + .copied() + .filter(|k| self.players.contains_key(k)) + .collect(); + #[cfg(feature = "decode-h264")] + for key in mse_keys { + if let Some(entry) = self.mse_audio_entries.get_mut(&key) { + entry.advance_to(now_ms); + } + } let mut changed = false; let keys: Vec = self.players.keys().copied().collect(); for key in keys { @@ -949,6 +1180,18 @@ impl VideoPlayerRegistry { continue; }; // A/V pair：视频对齐音频主时钟；纯视频：墙钟 tick。 + #[cfg(feature = "decode-h264")] + if self.mse_audio_entries.get(&key).is_some_and(|e| e.playing) { + let media_ms = self.mse_audio_entries.get(&key).map(|e| e.cursor_ms).unwrap_or(0); + let ticked = player.sync_to_media_time(media_ms as f64); + if let Ok(Some(frame)) = ticked + && let Ok(data) = ImageData::from_rgba(frame.rgba, frame.width, frame.height) + { + image_cache.insert_with_key(ImageKey::new(key), data); + changed = true; + } + continue; + } let ticked = if self.av_audio_entries.contains_key(&key) { let media_ms = self.av_audio_entries.get(&key).map(|e| e.cursor_ms).unwrap_or(0); player.sync_to_media_time(media_ms as f64) @@ -1298,6 +1541,110 @@ mod tests { assert!(!reg.play(MSE_SRC, 0)); } + /// t8s-3 双轨接线钉：audio SourceBuffer 流（独立 feed）→ play 懒建伴生音频 + /// 条目 → 音频泵解码推进（cursor 走 batch pts 真值）+ 视频帧照常产出—— + /// audio clock 主时钟面在 MSE 双轨源上成立。 + #[cfg(feature = "decode-h264")] + #[test] + fn registry_mse_audio_dual_track_t8s3() { + let mut reg = VideoPlayerRegistry::new(); + reg.mse_create(MSE_SRC); + let data = fixture_bytes_named("sample-mp4-h264.mp4"); + assert!(reg.mse_append(MSE_SRC, &data)); + // 音轨流：同 fixture（音轨内嵌——MseAudioFeed probe 取音轨，钉 media 层）。 + assert!(reg.mse_append_audio(MSE_SRC, &data)); + assert!(reg.play(MSE_SRC, 0), "MSE 双轨 play 建 player + 伴生音频条目"); + assert!(reg.is_any_playing(), "播放态（视频 player + 音频条目）"); + // 音频泵推进：cursor 走 batch pts 真值推进（等待面/等待重试不炸）。 + let mut now = 0u64; + let mut wrote = false; + for _ in 0..200 { + now += 100; + wrote |= reg.audio_advance_all(now); + if reg.mse_audio_cursor_ms(MSE_SRC).is_some_and(|c| c > 0) { + break; + } + } + assert!(wrote, "音频泵应产出写入"); + assert!( + reg.mse_audio_cursor_ms(MSE_SRC).is_some_and(|c| c > 0), + "MSE 伴生音频游标推进（batch pts 真值）" + ); + // 视频帧照常产出（音频主时钟分支不破坏视频面）。 + let mut cache = ImageCache::new(16, 64 * 1024 * 1024); + reg.mse_end(MSE_SRC); + let mut now3 = now; + let mut saw_frame = false; + while reg.is_playing(MSE_SRC) { + now3 += 100; + saw_frame |= reg.tick_all(now3, &mut cache); + assert!(now3 < 120_000, "runaway loop"); + } + assert!(saw_frame, "双轨源视频帧泵照常产出"); + // 流末（双 feed 均 end）→ 音频条目解码至耗尽后自停 + is_any_playing 复位 + //（视频 Ended 先到——音频残余批次按泵节奏排空）。 + let mut now4 = now3; + while reg.is_any_playing() { + now4 += 100; + reg.audio_advance_all(now4); + assert!(now4 < 180_000, "runaway loop（音频条目未随流末停）"); + } + assert!(!reg.is_any_playing(), "流末后视频/音频条目全停"); + } + + /// t8s-3 返修 B1 复活面钉：Ended→play 重头播放契约的音频腿——双轨播放至 + /// 流末全停后再次 play，伴生音频条目须随 reset 重建（feed 重置 + 游标归零） + /// 并可继续推进（复活前游标冻结在流末即为缺陷形态）。 + #[cfg(feature = "decode-h264")] + #[test] + fn registry_mse_audio_play_after_ended_rebuilds_entry_t8s3() { + let mut reg = VideoPlayerRegistry::new(); + reg.mse_create(MSE_SRC); + let data = fixture_bytes_named("sample-mp4-h264.mp4"); + assert!(reg.mse_append(MSE_SRC, &data)); + assert!(reg.mse_append_audio(MSE_SRC, &data)); + assert!(reg.play(MSE_SRC, 0)); + let mut now = 0u64; + // 播至流末（视频 Ended + 音频耗尽自停）。 + let mut cache = ImageCache::new(16, 64 * 1024 * 1024); + reg.mse_end(MSE_SRC); + loop { + now += 100; + reg.audio_advance_all(now); + reg.tick_all(now, &mut cache); + if !reg.is_any_playing() { + break; + } + assert!(now < 120_000, "runaway loop（首轮未到流末）"); + } + let cursor_before = reg.mse_audio_cursor_ms(MSE_SRC).unwrap_or(0); + assert!(cursor_before > 0, "首轮音频游标应已推进"); + // 复活：play 重头——音频游标须归零并可继续推进（B1 缺陷形态 = 冻结在 + // cursor_before 且泵零写入）。 + assert!(reg.play(MSE_SRC, now), "Ended 后 play = 重头播放"); + let now2 = now; + assert_eq!( + reg.mse_audio_cursor_ms(MSE_SRC), + Some(0), + "复活后音频游标归零（同句柄 feed 重建）" + ); + let mut wrote = false; + let mut now3 = now2; + for _ in 0..100 { + now3 += 100; + wrote |= reg.audio_advance_all(now3); + if wrote && reg.mse_audio_cursor_ms(MSE_SRC).is_some_and(|c| c > 0) { + break; + } + } + assert!(wrote, "复活后音频泵应恢复写入"); + assert!( + reg.mse_audio_cursor_ms(MSE_SRC) + .is_some_and(|c| c > 0 && c < cursor_before), + "复活后游标重头推进（非冻结）" + ); + } + /// 封顶/未登记拒绝面（t8n 返修 B2）：未登记键 `mse_append` → false /// （registry 侧入口；回调层映射 "0" → shim error + updateend 失败面）。 #[cfg(feature = "decode-h264")] @@ -1373,6 +1720,7 @@ pub fn video_bridge_facade_script() -> String { s.push_str( "globalThis.__zwVideoBridge.mseCreate = function (src) { return __zw_ms_create(src) === '1'; };\ globalThis.__zwVideoBridge.mseAppend = function (src, b64) { return __zw_ms_append(src, b64) === '1'; };\ + globalThis.__zwVideoBridge.mseAppendAudio = function (src, b64) { return __zw_ms_append_audio(src, b64) === '1'; };\ globalThis.__zwVideoBridge.mseEnd = function (src) { return __zw_ms_end(src) === '1'; };\ globalThis.__zwVideoBridge.mseSetDuration = function (src, secs) { __zw_ms_set_duration(src, String(secs)); };\ globalThis.__zwVideoBridge.mseDims = function (src) { return __zw_ms_dims(src); };", @@ -1629,6 +1977,24 @@ pub fn register_video_bridge_callbacks( }), ); let reg_ms = std::sync::Arc::clone(®istry); + sandbox.register_callback( + "__zw_ms_append_audio", + Box::new(move |args| { + let src = args.first().map(String::as_str).unwrap_or(""); + let b64 = args.get(1).map(String::as_str).unwrap_or(""); + match crate::webview::base64_decode(b64) { + Ok(bytes) => { + let ok = reg_ms + .lock() + .unwrap_or_else(|e| e.into_inner()) + .mse_append_audio(src, &bytes); + if ok { "1".into() } else { "0".into() } + } + Err(_) => "0".into(), + } + }), + ); + let reg_ms = std::sync::Arc::clone(®istry); sandbox.register_callback( "__zw_ms_end", Box::new(move |args| {