diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8b0ebb9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.cs] +indent_style = space +indent_size = 4 + +[*.{yml,yaml,json,md,props,csproj}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73eabc9..b9a3eff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,9 +22,12 @@ jobs: - name: Restore dependencies run: dotnet restore - + + - name: Test + run: dotnet test -c Release --no-restore + - name: Publish BydTools.CLI - run: dotnet publish BydTools.CLI/BydTools.CLI.csproj -c Release -r win-x64 -o ./publish + run: dotnet publish BydTools.CLI/BydTools.CLI.csproj -c Release -r win-x64 -o ./publish -p:EnableAotAnalyzer=true -p:EnableTrimAnalyzer=true - name: Clean up unnecessary files run: | diff --git a/BydTools.Wwise/3rdParty/avcodec-vgmstream-59.dll b/BydTools.Audio/3rdParty/avcodec-vgmstream-59.dll similarity index 100% rename from BydTools.Wwise/3rdParty/avcodec-vgmstream-59.dll rename to BydTools.Audio/3rdParty/avcodec-vgmstream-59.dll diff --git a/BydTools.Wwise/3rdParty/avformat-vgmstream-59.dll b/BydTools.Audio/3rdParty/avformat-vgmstream-59.dll similarity index 100% rename from BydTools.Wwise/3rdParty/avformat-vgmstream-59.dll rename to BydTools.Audio/3rdParty/avformat-vgmstream-59.dll diff --git a/BydTools.Wwise/3rdParty/avutil-vgmstream-57.dll b/BydTools.Audio/3rdParty/avutil-vgmstream-57.dll similarity index 100% rename from BydTools.Wwise/3rdParty/avutil-vgmstream-57.dll rename to BydTools.Audio/3rdParty/avutil-vgmstream-57.dll diff --git a/BydTools.Wwise/3rdParty/libvgmstream.dll b/BydTools.Audio/3rdParty/libvgmstream.dll similarity index 100% rename from BydTools.Wwise/3rdParty/libvgmstream.dll rename to BydTools.Audio/3rdParty/libvgmstream.dll diff --git a/BydTools.Wwise/3rdParty/libvorbis.dll b/BydTools.Audio/3rdParty/libvorbis.dll similarity index 100% rename from BydTools.Wwise/3rdParty/libvorbis.dll rename to BydTools.Audio/3rdParty/libvorbis.dll diff --git a/BydTools.Audio/Bnk/BnkParser.cs b/BydTools.Audio/Bnk/BnkParser.cs new file mode 100644 index 0000000..eaa5d16 --- /dev/null +++ b/BydTools.Audio/Bnk/BnkParser.cs @@ -0,0 +1,57 @@ +using System.Buffers.Binary; +using System.Text; + +namespace BydTools.Audio.Bnk; + +public readonly record struct BnkWem(uint Id, int Offset, int Size); + +public static class BnkParser +{ + public static List Parse(ReadOnlySpan data) + { + int dataOffset = 0; + var entries = new List(); + int pos = 0; + while (pos + 8 <= data.Length) + { + string sign = Encoding.ASCII.GetString(data.Slice(pos, 4)); + uint sectionSize = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(pos + 4, 4)); + int sectionStart = pos + 8; + if (sectionSize > data.Length - sectionStart) + throw new InvalidDataException($"BNK section {sign} exceeds the file."); + + if (sign == "DIDX") + { + int count = (int)(sectionSize / 12); + for (int i = 0; i < count; i++) + { + int at = sectionStart + i * 12; + uint id = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(at, 4)); + uint offset = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(at + 4, 4)); + uint size = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(at + 8, 4)); + entries.Add(new BnkWem(id, (int)offset, (int)size)); + } + } + else if (sign == "DATA") + { + dataOffset = sectionStart; + } + + pos = sectionStart + (int)sectionSize; + } + + if (dataOffset == 0 || entries.Count == 0) + return []; + + for (int i = 0; i < entries.Count; i++) + { + var entry = entries[i]; + int absolute = entry.Offset + dataOffset; + if (entry.Size < 0 || absolute < 0 || absolute > data.Length - entry.Size) + throw new InvalidDataException($"BNK WEM {entry.Id} is outside the DATA section."); + entries[i] = entry with { Offset = absolute }; + } + + return entries; + } +} diff --git a/BydTools.Audio/BydTools.Audio.csproj b/BydTools.Audio/BydTools.Audio.csproj new file mode 100644 index 0000000..bcd9cfe --- /dev/null +++ b/BydTools.Audio/BydTools.Audio.csproj @@ -0,0 +1,10 @@ + + + true + + + + + + + diff --git a/BydTools.Audio/Interop/LibVgmstream.cs b/BydTools.Audio/Interop/LibVgmstream.cs new file mode 100644 index 0000000..b38ba54 --- /dev/null +++ b/BydTools.Audio/Interop/LibVgmstream.cs @@ -0,0 +1,90 @@ +using System.Runtime.InteropServices; + +namespace BydTools.Audio.Interop; + +internal static partial class LibVgmstream +{ + private const string DllName = "libvgmstream"; + + [LibraryImport(DllName, EntryPoint = "libvgmstream_get_version")] + public static partial uint GetVersion(); + + [LibraryImport(DllName, EntryPoint = "libvgmstream_init")] + public static partial nint Init(); + + [LibraryImport(DllName, EntryPoint = "libvgmstream_free")] + public static partial void Free(nint lib); + + [LibraryImport(DllName, EntryPoint = "libvgmstream_setup")] + public static partial void Setup(nint lib, ref NativeConfig cfg); + + [LibraryImport(DllName, EntryPoint = "libvgmstream_open_stream")] + public static partial int OpenStream(nint lib, nint libsf, int subsong); + + [LibraryImport(DllName, EntryPoint = "libvgmstream_render")] + public static partial int Render(nint lib); + + public enum SampleFormat : int + { + Pcm16 = 1, + Pcm24 = 2, + Pcm32 = 3, + Float = 4, + } + + /// Matches libvgmstream_config_t from API 1.x. + [StructLayout(LayoutKind.Sequential)] + public struct NativeConfig + { + public byte disable_config_override; + public byte allow_play_forever; + public byte play_forever; + public byte ignore_loop; + public byte force_loop; + public byte really_force_loop; + public byte ignore_fade; + private byte _pad0; + public double loop_count; + public double fade_time; + public double fade_delay; + public int stereo_track; + public int auto_downmix_channels; + public int force_sfmt; + } + + /// Leading fields of libvgmstream_t. Later fields are not read. + [StructLayout(LayoutKind.Sequential)] + public struct NativeHandle + { + public nint Priv; + public nint Format; + public nint Decoder; + } + + /// + /// Leading fields of libvgmstream_format_t. stream_samples is int64 at offset 32. + /// + [StructLayout(LayoutKind.Sequential)] + public struct NativeFormat + { + public int Channels; + public int SampleRate; + public int SampleFormat; + public int SampleSize; + public uint ChannelLayout; + public int SubsongIndex; + public int SubsongCount; + public int InputChannels; + public long StreamSamples; + } + + /// Leading fields of libvgmstream_decoder_t. + [StructLayout(LayoutKind.Sequential)] + public struct NativeDecoder + { + public nint Buffer; + public int BufferSamples; + public int BufferBytes; + public byte Done; + } +} diff --git a/BydTools.Audio/Interop/MemoryStreamfile.cs b/BydTools.Audio/Interop/MemoryStreamfile.cs new file mode 100644 index 0000000..489b404 --- /dev/null +++ b/BydTools.Audio/Interop/MemoryStreamfile.cs @@ -0,0 +1,167 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace BydTools.Audio.Interop; + +/// +/// In-memory libstreamfile_t (vgmstream API 1.x). The name ends in .wem because +/// vgmstream picks the format from the extension. open returns another +/// view of the same buffer, which is what vgmstream does when it reopens a file. +/// +internal sealed unsafe class MemoryStreamfile : IDisposable +{ + private State? _state; + private LibStreamfile* _native; + private int _disposed; + + private MemoryStreamfile(State state, LibStreamfile* native) + { + _state = state; + _native = native; + } + + internal nint Native => (nint)_native; + + public static MemoryStreamfile Create(ReadOnlyMemory data) + { + if (!MemoryMarshal.TryGetArray(data, out ArraySegment segment) || segment.Array is null) + throw new NotSupportedException("WEM data must be backed by a byte array."); + + var state = new State(segment.Array, segment.Offset, segment.Count); + return new MemoryStreamfile(state, Allocate(state)); + } + + internal int ReadAt(long offset, Span destination) + { + var state = _state ?? throw new ObjectDisposedException(nameof(MemoryStreamfile)); + if (offset < 0 || offset >= state.Length || destination.IsEmpty) + return 0; + int count = (int)Math.Min(destination.Length, state.Length - offset); + state.Bytes.AsSpan(state.Offset + (int)offset, count).CopyTo(destination); + return count; + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + return; + _state = null; + if (_native != null) + { + Release(_native); + _native = null; + } + } + + private static LibStreamfile* Allocate(State state) + { + var native = (LibStreamfile*)NativeMemory.AllocZeroed((nuint)sizeof(LibStreamfile)); + native->UserData = GCHandle.ToIntPtr(state.Self); + native->Read = &Read; + native->GetSize = &GetSize; + native->GetName = &GetName; + native->Open = &Open; + native->Close = &Close; + return native; + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] + private static int Read(nint user, byte* dst, long offset, int length) + { + var state = State.From(user); + if (state == null || offset < 0 || length <= 0 || offset >= state.Length) + return 0; + int count = (int)Math.Min(length, state.Length - offset); + fixed (byte* src = state.Bytes) + { + Buffer.MemoryCopy(src + state.Offset + offset, dst, count, count); + } + + return count; + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] + private static long GetSize(nint user) + { + var state = State.From(user); + return state?.Length ?? 0; + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] + private static byte* GetName(nint user) + { + var state = State.From(user); + return state == null ? null : state.Name; + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] + private static LibStreamfile* Open(nint user, byte* filename) + { + var state = State.From(user); + if (state == null) + return null; + var child = new State(state.Bytes, state.Offset, state.Length); + return Allocate(child); + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] + private static void Close(LibStreamfile* streamfile) => Release(streamfile); + + private static void Release(LibStreamfile* streamfile) + { + if (streamfile == null) + return; + var handle = GCHandle.FromIntPtr(streamfile->UserData); + if (handle.Target is State state) + state.Release(); + handle.Free(); + NativeMemory.Free(streamfile); + } + + private sealed class State + { + public State(byte[] bytes, int offset, int length) + { + Bytes = bytes; + Offset = offset; + Length = length; + Self = GCHandle.Alloc(this); + Name = (byte*)NativeMemory.Alloc(10); + "audio.wem\0"u8.CopyTo(new Span(Name, 10)); + } + + public byte[] Bytes { get; } + public int Offset { get; } + public int Length { get; } + public GCHandle Self { get; } + public byte* Name { get; private set; } + + public static State? From(nint user) + { + if (user == 0) + return null; + var handle = GCHandle.FromIntPtr(user); + return handle.Target as State; + } + + public void Release() + { + if (Name != null) + { + NativeMemory.Free(Name); + Name = null; + } + } + } +} + +[StructLayout(LayoutKind.Sequential)] +internal unsafe struct LibStreamfile +{ + public nint UserData; + public delegate* unmanaged[Cdecl] Read; + public delegate* unmanaged[Cdecl] GetSize; + public delegate* unmanaged[Cdecl] GetName; + public delegate* unmanaged[Cdecl] Open; + public delegate* unmanaged[Cdecl] Close; +} diff --git a/BydTools.Audio/Interop/VgmstreamHandle.cs b/BydTools.Audio/Interop/VgmstreamHandle.cs new file mode 100644 index 0000000..89ad626 --- /dev/null +++ b/BydTools.Audio/Interop/VgmstreamHandle.cs @@ -0,0 +1,54 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace BydTools.Audio.Interop; + +internal sealed unsafe class VgmstreamHandle : IDisposable +{ + private nint _handle; + + private VgmstreamHandle(nint handle) => _handle = handle; + + public static VgmstreamHandle Create() + { + nint handle = LibVgmstream.Init(); + return handle == 0 + ? throw new InvalidOperationException("libvgmstream_init returned NULL.") + : new VgmstreamHandle(handle); + } + + public void Setup(LibVgmstream.NativeConfig config) => LibVgmstream.Setup(_handle, ref config); + + public int Open(nint streamfile) => LibVgmstream.OpenStream(_handle, streamfile, 0); + + public int Render() => LibVgmstream.Render(_handle); + + public ref LibVgmstream.NativeFormat Format + { + get + { + var format = ((LibVgmstream.NativeHandle*)_handle)->Format; + if (format == 0) + throw new InvalidOperationException("libvgmstream format is not available."); + return ref Unsafe.AsRef((void*)format); + } + } + + public ref LibVgmstream.NativeDecoder Decoder + { + get + { + var decoder = ((LibVgmstream.NativeHandle*)_handle)->Decoder; + if (decoder == 0) + throw new InvalidOperationException("libvgmstream decoder is not available."); + return ref Unsafe.AsRef((void*)decoder); + } + } + + public void Dispose() + { + nint handle = Interlocked.Exchange(ref _handle, 0); + if (handle != 0) + LibVgmstream.Free(handle); + } +} diff --git a/BydTools.Audio/Naming/AudioNaming.cs b/BydTools.Audio/Naming/AudioNaming.cs new file mode 100644 index 0000000..2beb7b6 --- /dev/null +++ b/BydTools.Audio/Naming/AudioNaming.cs @@ -0,0 +1,109 @@ +using BydTools.Audio.Pck; + +namespace BydTools.Audio.Naming; + +public static class AudioNaming +{ + public static string ResolveWem( + string blockFolder, + string? blockLanguage, + ulong fileId, + PckMapper? mapper, + string extension, + IReadOnlyList? languages, + uint languageId + ) + { + string? mapped = mapper?.GetMappedPath(fileId); + if (mapped != null) + return BuildMapped(blockFolder, blockLanguage, mapped, extension); + + return Unmapped(blockFolder, LanguageFolder(blockLanguage, languages, languageId), $"{fileId}{extension}"); + } + + public static string ResolveBnkWem( + string blockFolder, + string? blockLanguage, + ulong bankFileId, + uint wemId, + PckMapper? mapper, + string extension, + IReadOnlyList? languages, + uint languageId + ) + { + string? mapped = mapper?.GetMappedPath(wemId); + if (mapped != null) + return BuildMapped(blockFolder, blockLanguage, mapped, extension); + + string language = LanguageFolder(blockLanguage, languages, languageId); + return Unmapped(blockFolder, language, $"{bankFileId}_{wemId}{extension}"); + } + + private static string LanguageFolder( + string? blockLanguage, + IReadOnlyList? languages, + uint languageId + ) + { + if (languageId != 0 && languages != null) + { + foreach (var language in languages) + { + if (language.Id == languageId) + return Normalize(language.Name); + } + } + + return Normalize(blockLanguage); + } + + private static string BuildMapped( + string blockFolder, + string? blockLanguage, + string mappedPath, + string extension + ) + { + string withExtension = Path.ChangeExtension(mappedPath, extension); + if (!TryParseMapped(withExtension, out var audioType, out var language, out var realPath)) + { + return Unmapped(blockFolder, Normalize(blockLanguage), Path.GetFileName(withExtension)); + } + + return Path.Combine(blockFolder, Normalize(language), audioType, realPath); + } + + private static bool TryParseMapped( + string mappedPath, + out string audioType, + out string language, + out string realPath + ) + { + string[] parts = mappedPath.Split(['\\', '/'], StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 3) + { + audioType = "unknown"; + language = "Unknown"; + realPath = Path.GetFileName(mappedPath); + return false; + } + + audioType = parts[0]; + language = parts[1]; + realPath = Path.Combine(parts[2..]); + return true; + } + + private static string Unmapped(string blockFolder, string language, string fileName) => + Path.Combine(blockFolder, "unmapped", language, fileName); + + private static string Normalize(string? languageName) + { + if (string.IsNullOrWhiteSpace(languageName)) + return "Unknown"; + string trimmed = languageName.Trim(); + return char.ToUpperInvariant(trimmed[0]) + trimmed[1..].ToLowerInvariant(); + } +} diff --git a/BydTools.Audio/Pck/PckArchive.cs b/BydTools.Audio/Pck/PckArchive.cs new file mode 100644 index 0000000..581f425 --- /dev/null +++ b/BydTools.Audio/Pck/PckArchive.cs @@ -0,0 +1,251 @@ +using System.Buffers.Binary; +using System.Text; + +namespace BydTools.Audio.Pck; + +/// +/// Parsed AKPK archive. is known as soon as +/// returns; entry payloads stay in the original buffer +/// until is called. +/// +public sealed class PckArchive +{ + private const uint AkpkMagic = 0x4B504B41; + private readonly byte[] _decryptedHeader; + + private PckArchive( + bool encrypted, + byte[] decryptedHeader, + IReadOnlyList languages, + IReadOnlyList entries + ) + { + IsVfsEncrypted = encrypted; + _decryptedHeader = decryptedHeader; + Languages = languages; + Entries = entries; + } + + public bool IsVfsEncrypted { get; } + public IReadOnlyList Languages { get; } + public IReadOnlyList Entries { get; } + + public static PckArchive Parse(ReadOnlySpan file) + { + if (file.Length < 8) + throw new InvalidDataException("PCK file is too small."); + + uint magic = BinaryPrimitives.ReadUInt32LittleEndian(file); + uint headerSize = BinaryPrimitives.ReadUInt32LittleEndian(file[4..]); + if (headerSize < 16 || headerSize > file.Length - 8) + throw new InvalidDataException($"PCK header size {headerSize} is not plausible."); + + bool encrypted = magic != AkpkMagic; + var headerContent = file.Slice(8, (int)headerSize); + byte[] decryptedHeader; + if (!encrypted) + { + decryptedHeader = headerContent.ToArray(); + } + else + { + byte[] payload = headerContent[4..].ToArray(); + PckCipher.DecipherInPlace(payload, headerSize, payload.Length, 0); + decryptedHeader = new byte[4 + payload.Length]; + BinaryPrimitives.WriteUInt32LittleEndian(decryptedHeader, 1); + payload.CopyTo(decryptedHeader, 4); + } + + var reader = new SpanReader(decryptedHeader); + uint flag = reader.ReadUInt32(); + if (flag == 0x01000000) + throw new NotSupportedException("Big-endian PCK files are not supported."); + + uint languagesSize = reader.ReadUInt32(); + uint banksSize = reader.ReadUInt32(); + uint soundsSize = reader.ReadUInt32(); + uint overhead = 16; + uint externalsSize = 0; + if (languagesSize + banksSize + soundsSize + overhead < (uint)decryptedHeader.Length) + externalsSize = reader.ReadUInt32(); + + var languages = ParseLanguages(ref reader, languagesSize, decryptedHeader.Length); + var entries = new List(); + ParseSector(ref reader, banksSize, PckSectorType.Bank, entries, decryptedHeader.Length); + ParseSector(ref reader, soundsSize, PckSectorType.Sound, entries, decryptedHeader.Length); + if (externalsSize > 0) + ParseSector(ref reader, externalsSize, PckSectorType.External, entries, decryptedHeader.Length); + + return new PckArchive(encrypted, decryptedHeader, languages, entries); + } + + public void DecipherPayloads(Span file) + { + if (!IsVfsEncrypted) + return; + foreach (var entry in Entries) + { + if (!TrySlice(file, entry, out var slice)) + continue; + PckCipher.DecipherInPlace(slice, (uint)entry.FileId, slice.Length, 0); + } + } + + public byte[] ToPlainPck(ReadOnlySpan file) + { + byte[] copy = file.ToArray(); + if (!IsVfsEncrypted) + return copy; + + BinaryPrimitives.WriteUInt32LittleEndian(copy, AkpkMagic); + if (copy.Length < 8 + _decryptedHeader.Length) + throw new InvalidDataException("PCK buffer is too small for the decrypted header."); + _decryptedHeader.CopyTo(copy, 8); + DecipherPayloads(copy); + return copy; + } + + public static bool TrySlice(Span file, PckFileEntry entry, out Span slice) + { + slice = default; + if (entry.Size > int.MaxValue || entry.Offset < 0 || entry.Offset > file.Length) + return false; + if (entry.Size > file.Length - entry.Offset) + return false; + slice = file.Slice((int)entry.Offset, (int)entry.Size); + return true; + } + + private static List ParseLanguages(ref SpanReader reader, uint sectorSize, int limit) + { + if (sectorSize == 0) + return []; + int sectorStart = reader.Position; + if (sectorSize > limit - sectorStart) + throw new InvalidDataException("PCK language sector exceeds the header."); + uint count = reader.ReadUInt32(); + var languages = new List((int)Math.Min(count, 1024)); + for (uint i = 0; i < count; i++) + { + uint nameOffset = reader.ReadUInt32(); + uint langId = reader.ReadUInt32(); + int saved = reader.Position; + int namePos = sectorStart + (int)nameOffset; + reader.Seek(namePos); + if (reader.Remaining < 2) + throw new InvalidDataException("PCK language name is truncated."); + byte b1 = reader.ReadByte(); + byte b2 = reader.ReadByte(); + reader.Seek(namePos); + int rawLength = b1 == 0 || b2 == 0 ? 0x20 : 0x10; + if (reader.Remaining < rawLength) + throw new InvalidDataException("PCK language name is truncated."); + var raw = reader.ReadBytes(rawLength); + string name = (b1 == 0 || b2 == 0 ? Encoding.Unicode : Encoding.UTF8).GetString(raw); + int nul = name.IndexOf('\0'); + if (nul >= 0) + name = name[..nul]; + languages.Add(new PckLanguage(langId, name)); + reader.Seek(saved); + } + + reader.Seek(sectorStart + (int)sectorSize); + return languages; + } + + private static void ParseSector( + ref SpanReader reader, + uint sectorSize, + PckSectorType sectorType, + List entries, + int limit + ) + { + if (sectorSize == 0) + return; + int sectorStart = reader.Position; + if (sectorSize > limit - sectorStart) + throw new InvalidDataException("PCK sector exceeds the header."); + uint fileCount = reader.ReadUInt32(); + if (fileCount == 0) + { + reader.Seek(sectorStart + (int)sectorSize); + return; + } + + if ((sectorSize - 4) % fileCount != 0) + throw new InvalidDataException("PCK sector size is not a multiple of its entry count."); + uint entrySize = (sectorSize - 4) / fileCount; + bool altMode = entrySize >= 0x18; + + for (uint i = 0; i < fileCount; i++) + { + ulong fileId; + if (altMode && sectorType == PckSectorType.External) + { + uint idLow = reader.ReadUInt32(); + uint idHigh = reader.ReadUInt32(); + fileId = idLow | ((ulong)idHigh << 32); + } + else + { + fileId = reader.ReadUInt32(); + } + + uint blockSize = reader.ReadUInt32(); + uint size = + altMode && sectorType != PckSectorType.External + ? (uint)reader.ReadInt64() + : reader.ReadUInt32(); + uint rawOffset = reader.ReadUInt32(); + uint languageId = reader.ReadUInt32(); + long offset = blockSize != 0 ? (long)rawOffset * blockSize : rawOffset; + entries.Add(new PckFileEntry(fileId, size, offset, languageId, sectorType)); + } + + reader.Seek(sectorStart + (int)sectorSize); + } + + private ref struct SpanReader + { + private readonly ReadOnlySpan _span; + private int _position; + + public SpanReader(ReadOnlySpan span) => _span = span; + + public int Position => _position; + public int Remaining => _span.Length - _position; + + public void Seek(int position) + { + if ((uint)position > (uint)_span.Length) + throw new InvalidDataException($"PCK seek to {position} is outside the header."); + _position = position; + } + + public uint ReadUInt32() + { + var slice = Take(4); + return BinaryPrimitives.ReadUInt32LittleEndian(slice); + } + + public long ReadInt64() + { + var slice = Take(8); + return BinaryPrimitives.ReadInt64LittleEndian(slice); + } + + public byte ReadByte() => Take(1)[0]; + + public ReadOnlySpan ReadBytes(int count) => Take(count); + + private ReadOnlySpan Take(int count) + { + if ((uint)count > (uint)Remaining) + throw new InvalidDataException($"PCK header ended at offset {_position}."); + var slice = _span.Slice(_position, count); + _position += count; + return slice; + } + } +} diff --git a/BydTools.Audio/Pck/PckCipher.cs b/BydTools.Audio/Pck/PckCipher.cs new file mode 100644 index 0000000..a847cc6 --- /dev/null +++ b/BydTools.Audio/Pck/PckCipher.cs @@ -0,0 +1,63 @@ +using System.Buffers.Binary; + +namespace BydTools.Audio.Pck; + +/// Counter XOR used by the VFS layer on top of AKPK. +public static class PckCipher +{ + private const uint ConstM = 0x04E11C23; + private const uint ConstX = 0x9C5A0B29; + + public static void DecipherInPlace(Span data, uint seed, int size, int offsetToFileStart) + { + if (size == 0) + return; + if ((uint)size > (uint)data.Length) + throw new ArgumentOutOfRangeException(nameof(size)); + + static uint GenerateKey(uint counter) + { + uint val = ((counter & 0xFF) ^ ConstX) * ConstM; + val = (val ^ ((counter >> 8) & 0xFF)) * ConstM; + val = (val ^ ((counter >> 16) & 0xFF)) * ConstM; + val = (val ^ ((counter >> 24) & 0xFF)) * ConstM; + return val; + } + + int pos = 0; + uint baseCounter = seed + (uint)(offsetToFileStart >> 2); + int alignedOffset = offsetToFileStart & 0b11; + + if (alignedOffset > 0) + { + uint key = GenerateKey(baseCounter); + Span keyBytes = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(keyBytes, key); + int leading = Math.Min(4 - alignedOffset, size); + for (int i = 0; i < leading; i++) + data[pos++] ^= keyBytes[alignedOffset + i]; + baseCounter++; + } + + int alignedSize = (size - pos) & ~0b11; + int blocks = alignedSize / 4; + for (int block = 0; block < blocks; block++) + { + uint word = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(pos, 4)); + BinaryPrimitives.WriteUInt32LittleEndian( + data.Slice(pos, 4), + word ^ GenerateKey(baseCounter + (uint)block) + ); + pos += 4; + } + + if (pos < size) + { + uint key = GenerateKey(baseCounter + (uint)blocks); + Span keyBytes = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(keyBytes, key); + for (int i = 0; i < size - pos; i++) + data[pos + i] ^= keyBytes[i]; + } + } +} diff --git a/BydTools.Audio/Pck/PckMapper.cs b/BydTools.Audio/Pck/PckMapper.cs new file mode 100644 index 0000000..d1f8fee --- /dev/null +++ b/BydTools.Audio/Pck/PckMapper.cs @@ -0,0 +1,76 @@ +using System.Text; +using System.Text.Json.Nodes; + +namespace BydTools.Audio.Pck; + +/// +/// Resolves Wwise ids to paths using an AudioDialog object. +/// Ids are FNV-1 64-bit hashes of {soundType}/{language}/{path}. +/// +public sealed class PckMapper +{ + private const ulong FnvOffset = 0xcbf29ce484222325; + private const ulong FnvPrime = 0x100000001b3; + private static readonly string[] KnownSoundTypes = ["voice", "music", "sfx"]; + + private readonly Dictionary _idToPath; + + public PckMapper(JsonObject root, string language) + { + _idToPath = Parse(root, language); + } + + public int Count => _idToPath.Count; + + public string? GetMappedPath(ulong fileId) + { + if (_idToPath.TryGetValue(fileId, out string? path)) + return path; + + ulong low = (uint)fileId; + if (low != fileId && _idToPath.TryGetValue(low, out path)) + return path; + return null; + } + + internal static ulong Fnv1_64(ReadOnlySpan data) + { + ulong hash = FnvOffset; + foreach (byte b in data) + { + hash = unchecked(hash * FnvPrime); + hash ^= b; + } + + return hash; + } + + private static Dictionary Parse(JsonObject root, string language) + { + string lang = language.ToLowerInvariant(); + var map = new Dictionary(root.Count * KnownSoundTypes.Length); + foreach (var entry in root) + { + if (entry.Value is not JsonObject row) + continue; + if (!row.TryGetPropertyValue("path", out var pathNode) || pathNode is null) + continue; + string? rawPath = pathNode.GetValue(); + if (string.IsNullOrWhiteSpace(rawPath)) + continue; + + string normalized = rawPath.Replace('\\', '/').Trim(); + foreach (string soundType in KnownSoundTypes) + { + string hashInput = $"{soundType}/{language}/{rawPath}".Replace('\\', '/').ToLowerInvariant(); + ulong hash = Fnv1_64(Encoding.UTF8.GetBytes(hashInput)); + string output = $"{soundType}/{lang}/{normalized}"; + if (output.EndsWith(".wem", StringComparison.OrdinalIgnoreCase)) + output = output[..^4]; + map.TryAdd(hash, output); + } + } + + return map; + } +} diff --git a/BydTools.Audio/Pck/PckModels.cs b/BydTools.Audio/Pck/PckModels.cs new file mode 100644 index 0000000..e13c3ea --- /dev/null +++ b/BydTools.Audio/Pck/PckModels.cs @@ -0,0 +1,18 @@ +namespace BydTools.Audio.Pck; + +public enum PckSectorType +{ + Bank, + Sound, + External, +} + +public sealed record PckLanguage(uint Id, string Name); + +public sealed record PckFileEntry( + ulong FileId, + uint Size, + long Offset, + uint LanguageId, + PckSectorType SectorType +); diff --git a/BydTools.Audio/Wem/IWemDecoder.cs b/BydTools.Audio/Wem/IWemDecoder.cs new file mode 100644 index 0000000..74b63fa --- /dev/null +++ b/BydTools.Audio/Wem/IWemDecoder.cs @@ -0,0 +1,6 @@ +namespace BydTools.Audio.Wem; + +public interface IWemDecoder : IDisposable +{ + void Decode(ReadOnlyMemory wem, Stream wavOutput); +} diff --git a/BydTools.Audio/Wem/WavWriter.cs b/BydTools.Audio/Wem/WavWriter.cs new file mode 100644 index 0000000..c04e129 --- /dev/null +++ b/BydTools.Audio/Wem/WavWriter.cs @@ -0,0 +1,36 @@ +namespace BydTools.Audio.Wem; + +internal static class WavWriter +{ + public static void WriteHeader(Stream stream, int channels, int sampleRate, int bitsPerSample, int dataBytes) + { + int blockAlign = channels * (bitsPerSample / 8); + int avgBytesPerSec = sampleRate * blockAlign; + Span header = stackalloc byte[44]; + "RIFF"u8.CopyTo(header); + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(header[4..], 36 + dataBytes); + "WAVE"u8.CopyTo(header[8..]); + "fmt "u8.CopyTo(header[12..]); + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(header[16..], 16); + System.Buffers.Binary.BinaryPrimitives.WriteInt16LittleEndian(header[20..], 1); + System.Buffers.Binary.BinaryPrimitives.WriteInt16LittleEndian(header[22..], (short)channels); + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(header[24..], sampleRate); + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(header[28..], avgBytesPerSec); + System.Buffers.Binary.BinaryPrimitives.WriteInt16LittleEndian(header[32..], (short)blockAlign); + System.Buffers.Binary.BinaryPrimitives.WriteInt16LittleEndian(header[34..], (short)bitsPerSample); + "data"u8.CopyTo(header[36..]); + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(header[40..], dataBytes); + stream.Write(header); + } + + public static void PatchSizes(Stream stream, int dataBytes) + { + Span size = stackalloc byte[4]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(size, 36 + dataBytes); + stream.Position = 4; + stream.Write(size); + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(size, dataBytes); + stream.Position = 40; + stream.Write(size); + } +} diff --git a/BydTools.Audio/Wem/WemDecoders.cs b/BydTools.Audio/Wem/WemDecoders.cs new file mode 100644 index 0000000..ec90070 --- /dev/null +++ b/BydTools.Audio/Wem/WemDecoders.cs @@ -0,0 +1,199 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using BydTools.Audio.Interop; +using BydTools.Core; + +namespace BydTools.Audio.Wem; + +public static class WemDecoders +{ + public static IWemDecoder? TryCreate(ILogger logger) + { + if (LibVgmstreamDecoder.IsAvailable) + { + logger.Verbose("Engine: libvgmstream (DLL)"); + return new LibVgmstreamDecoder(); + } + + string? cli = VgmstreamCliDecoder.Find(); + if (cli != null) + { + logger.Verbose("Engine: vgmstream-cli ({0})", cli); + return new VgmstreamCliDecoder(cli); + } + + return null; + } +} + +public sealed class LibVgmstreamDecoder : IWemDecoder +{ + private static readonly Lazy Available = new(Probe); + + public static bool IsAvailable => Available.Value; + + public unsafe void Decode(ReadOnlyMemory wem, Stream wavOutput) + { + using var handle = VgmstreamHandle.Create(); + var config = new LibVgmstream.NativeConfig + { + ignore_loop = 1, + force_sfmt = (int)LibVgmstream.SampleFormat.Pcm16, + }; + handle.Setup(config); + + using (var streamfile = MemoryStreamfile.Create(wem)) + { + int opened = handle.Open(streamfile.Native); + if (opened < 0) + throw new InvalidOperationException($"libvgmstream failed to open the stream (error {opened})."); + } + + ref var format = ref handle.Format; + if (format.Channels <= 0 || format.SampleRate <= 0 || format.StreamSamples <= 0) + throw new InvalidOperationException("Invalid stream format."); + + int expected = checked((int)(format.StreamSamples * format.Channels * format.SampleSize)); + WavWriter.WriteHeader(wavOutput, format.Channels, format.SampleRate, format.SampleSize * 8, expected); + + int written = 0; + while (handle.Decoder.Done == 0) + { + int rendered = handle.Render(); + if (rendered < 0) + throw new InvalidOperationException($"libvgmstream render error: {rendered}"); + + ref var decoder = ref handle.Decoder; + if (decoder.BufferBytes <= 0 || decoder.Buffer == 0) + continue; + wavOutput.Write(new ReadOnlySpan((void*)decoder.Buffer, decoder.BufferBytes)); + written += decoder.BufferBytes; + } + + if (written != expected) + WavWriter.PatchSizes(wavOutput, written); + } + + public void Dispose() { } + + private static bool Probe() + { + try + { + uint version = LibVgmstream.GetVersion(); + return version >> 24 == 1; + } + catch (Exception ex) when (ex is DllNotFoundException or BadImageFormatException or EntryPointNotFoundException) + { + return false; + } + } +} + +public sealed class VgmstreamCliDecoder : IWemDecoder +{ + private readonly string _executable; + private readonly string _tempDirectory; + + public VgmstreamCliDecoder(string executable) + { + _executable = executable; + _tempDirectory = Path.Combine(Path.GetTempPath(), "bydtools-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDirectory); + } + + public static string? Find() + { + string name = OperatingSystem.IsWindows() ? "vgmstream-cli.exe" : "vgmstream-cli"; + string? exeDir = Path.GetDirectoryName(Environment.ProcessPath); + if (exeDir != null) + { + string local = Path.Combine(exeDir, name); + if (File.Exists(local)) + return local; + string nested = Path.Combine(exeDir, "vgmstream", name); + if (File.Exists(nested)) + return nested; + } + + string? pathEnv = Environment.GetEnvironmentVariable("PATH"); + if (pathEnv == null) + return null; + foreach (string dir in pathEnv.Split(Path.PathSeparator)) + { + try + { + string candidate = Path.Combine(dir, name); + if (File.Exists(candidate)) + return candidate; + } + catch (ArgumentException) { } + } + + return null; + } + + public void Decode(ReadOnlyMemory wem, Stream wavOutput) + { + string wemPath = Path.Combine(_tempDirectory, Guid.NewGuid().ToString("N") + ".wem"); + string wavPath = Path.ChangeExtension(wemPath, ".wav"); + File.WriteAllBytes(wemPath, wem.Span); + try + { + var start = new ProcessStartInfo + { + FileName = _executable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + start.ArgumentList.Add("-o"); + start.ArgumentList.Add(wavPath); + start.ArgumentList.Add(wemPath); + using var process = Process.Start(start) ?? throw new InvalidOperationException("Failed to start vgmstream-cli."); + Task stdout = process.StandardOutput.ReadToEndAsync(); + Task stderr = process.StandardError.ReadToEndAsync(); + process.WaitForExit(); + string error = stderr.GetAwaiter().GetResult(); + stdout.GetAwaiter().GetResult(); + if (process.ExitCode != 0) + { + string trimmed = error.Trim(); + if (trimmed.Length > 300) + trimmed = trimmed[..300] + "..."; + throw new InvalidOperationException($"vgmstream-cli exited with code {process.ExitCode}: {trimmed}"); + } + + using var wav = File.OpenRead(wavPath); + if (wav.Length == 0) + throw new InvalidOperationException("vgmstream-cli produced no output."); + wav.CopyTo(wavOutput); + } + finally + { + TryDelete(wemPath); + TryDelete(wavPath); + } + } + + public void Dispose() + { + try + { + if (Directory.Exists(_tempDirectory)) + Directory.Delete(_tempDirectory, recursive: true); + } + catch (IOException) { } + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + File.Delete(path); + } + catch (IOException) { } + } +} diff --git a/BydTools.CLI/ArgParser.cs b/BydTools.CLI/ArgParser.cs index 4ac3dd1..abc5cf5 100644 --- a/BydTools.CLI/ArgParser.cs +++ b/BydTools.CLI/ArgParser.cs @@ -1,74 +1,87 @@ namespace BydTools.CLI; +public sealed record CommandOption( + string LongName, + string? ShortName, + bool IsFlag, + bool Required, + string? ValueName, + string Description, + string[] Continuation +); + /// -/// Minimal command-line argument parser with builder pattern. -/// Supports --long and -short aliases for both flags and valued options. -/// Uses dictionary-based lookup for O(1) argument resolution. +/// Small argv parser. Options are declared once and drive both parsing and help. /// -class ArgParser +public sealed class ArgParser { - private readonly record struct OptionDef(string LongName, bool IsFlag); + private readonly List _options = []; + private readonly Dictionary _lookup = new(StringComparer.OrdinalIgnoreCase); - private readonly Dictionary _lookup = []; - private readonly Dictionary _values = []; - private readonly HashSet _flags = []; - private readonly List _errors = []; + public IReadOnlyList Options => _options; + public Dictionary Values { get; private set; } = new(StringComparer.OrdinalIgnoreCase); + public HashSet Flags { get; private set; } = new(StringComparer.OrdinalIgnoreCase); + public List Errors { get; } = []; - public ArgParser AddOption(string longName, string? shortName = null) + public ArgParser Add( + string longName, + string? shortName, + string description, + string? valueName = null, + bool required = false, + params string[] continuation + ) { - var def = new OptionDef(longName, IsFlag: false); - _lookup[$"--{longName}"] = def; + var option = new CommandOption( + longName, + shortName, + IsFlag: valueName == null, + required, + valueName, + description, + continuation + ); + _options.Add(option); + _lookup["--" + longName] = option; if (shortName != null) - _lookup[$"-{shortName}"] = def; + _lookup["-" + shortName] = option; return this; } - public ArgParser AddFlag(string longName, string? shortName = null) - { - var def = new OptionDef(longName, IsFlag: true); - _lookup[$"--{longName}"] = def; - if (shortName != null) - _lookup[$"-{shortName}"] = def; - return this; - } + public bool HasFlag(string longName) => Flags.Contains(longName); + + public string? Get(string longName) => Values.TryGetValue(longName, out var value) ? value : null; - public bool TryParse(string[] args) + public bool Parse(string[] args) { - _values.Clear(); - _flags.Clear(); - _errors.Clear(); + Values = new Dictionary(StringComparer.OrdinalIgnoreCase); + Flags = new HashSet(StringComparer.OrdinalIgnoreCase); + Errors.Clear(); for (int i = 0; i < args.Length; i++) { - var arg = args[i]; - - if (!_lookup.TryGetValue(arg, out var def)) + string arg = args[i]; + if (!_lookup.TryGetValue(arg, out var option)) { - _errors.Add($"Unknown argument: {arg}"); + Errors.Add($"Unknown argument: {arg}"); continue; } - if (def.IsFlag) + if (option.IsFlag) { - _flags.Add(def.LongName); + Flags.Add(option.LongName); + continue; } - else + + if (i + 1 >= args.Length || args[i + 1].StartsWith('-')) { - if (i + 1 >= args.Length) - { - _errors.Add($"--{def.LongName} requires a value."); - continue; - } - _values[def.LongName] = args[++i]; + Errors.Add($"Missing value for {arg}"); + continue; } + + Values[option.LongName] = args[++i]; } - return _errors.Count == 0; + return Errors.Count == 0; } - - public string? GetValue(string longName) => _values.GetValueOrDefault(longName); - - public bool GetFlag(string longName) => _flags.Contains(longName); - - public IReadOnlyList Errors => _errors; } diff --git a/BydTools.CLI/BydTools.CLI.csproj b/BydTools.CLI/BydTools.CLI.csproj index 0633e6f..a5c4b4d 100644 --- a/BydTools.CLI/BydTools.CLI.csproj +++ b/BydTools.CLI/BydTools.CLI.csproj @@ -1,22 +1,14 @@ Exe - net10.0 - enable - enable true true false true + true - - - - - - - + diff --git a/BydTools.CLI/Commands/ICommand.cs b/BydTools.CLI/Commands/ICommand.cs index 9c3b8e2..a762ae9 100644 --- a/BydTools.CLI/Commands/ICommand.cs +++ b/BydTools.CLI/Commands/ICommand.cs @@ -1,12 +1,8 @@ namespace BydTools.CLI.Commands; -/// -/// Represents a CLI subcommand that can be executed with parsed arguments. -/// -interface ICommand +public interface ICommand { string Name { get; } string Description { get; } - void PrintHelp(string exeName); - void Execute(string[] args); + int Execute(string[] args, CancellationToken cancellationToken); } diff --git a/BydTools.CLI/Commands/PckCommand.cs b/BydTools.CLI/Commands/PckCommand.cs index fc3f001..b422d28 100644 --- a/BydTools.CLI/Commands/PckCommand.cs +++ b/BydTools.CLI/Commands/PckCommand.cs @@ -1,389 +1,100 @@ -using System.Collections.Concurrent; -using BydTools.PCK; -using BydTools.Utils; +using BydTools.Audio.Wem; +using BydTools.Extraction; +using BydTools.Extraction.Audio; using BydTools.VFS; -using BydTools.VFS.SparkBuffer; -using BydTools.Wwise; using Spectre.Console; namespace BydTools.CLI.Commands; -sealed class PckCommand : ICommand +public sealed class PckCommand : ICommand { public string Name => "pck"; - public string Description => "Extract audio from VFS"; + public string Description => "Extract audio from a VFS block and convert WEM to WAV"; - // Single source of truth for supported PCK audio block types and their mapping language. - // Adding a new block here enables CLI validation and language resolution at the same time. - private static readonly Dictionary BlockLanguageMap = new() + public int Execute(string[] args, CancellationToken cancellationToken) { - // main audios - { EVFSBlockType.Audio, "Main" }, - // prologue audios (and more?) - { EVFSBlockType.AuditAudio, "Audit" }, - // loading screen audios - { EVFSBlockType.InitAudio, "Initial" }, - // language audios - { EVFSBlockType.AudioChinese, "Chinese" }, - { EVFSBlockType.AudioEnglish, "English" }, - { EVFSBlockType.AudioJapanese, "Japanese" }, - { EVFSBlockType.AudioKorean, "Korean" }, - { EVFSBlockType.HotfixAudio, "Hotfix" }, - }; - - private static readonly string[] AudioBlockTypeNames = - [ - .. BlockLanguageMap.Keys.Select(static t => t.ToString()), - ]; - - public void PrintHelp(string exeName) - { - HelpFormatter.WriteUsage("pck", "--input --output --type "); - - HelpFormatter.WriteSectionHeader("Required"); - HelpFormatter.WriteEntry( - "-i, --input ", - "Game data directory that contains the VFS folder" - ); - HelpFormatter.WriteEntry("-o, --output ", "Output directory"); - HelpFormatter.WriteEntry("-t, --type ", "Audio block type to extract"); - HelpFormatter.WriteBlankLine(); - - HelpFormatter.WriteSectionHeader("Options"); - HelpFormatter.WriteEntry("-m, --mode ", "Extract mode (default: wav)"); - HelpFormatter.WriteEntryContinuation("raw Extract wem without conversion"); - HelpFormatter.WriteEntryContinuation("wav Convert to wav via vgmstream"); - HelpFormatter.WriteEntry("--no-map", "Disable automatic AudioDialog filename mapping"); - HelpFormatter.WriteCommonOptions(); - HelpFormatter.WriteBlankLine(); - - WriteAudioBlockTypes(); - } - - public void Execute(string[] args) - { - var parser = new ArgParser() - .AddFlag("help", "h") - .AddFlag("verbose", "v") - .AddFlag("no-map") - .AddOption("input", "i") - .AddOption("output", "o") - .AddOption("type", "t") - .AddOption("mode", "m") - .AddOption("key"); - - if (!parser.TryParse(args)) - { - foreach (var error in parser.Errors) - Console.Error.WriteLine(error); - PrintHelp(Program.ExecutableName); - return; - } - - if (parser.GetFlag("help")) - { - PrintHelp(Program.ExecutableName); - return; - } - - var gamePath = parser.GetValue("input"); - if (string.IsNullOrWhiteSpace(gamePath)) - { - Console.Error.WriteLine("Error: --input is required."); - PrintHelp(Program.ExecutableName); - return; - } - - var outputDir = parser.GetValue("output"); - if (string.IsNullOrWhiteSpace(outputDir)) - { - Console.Error.WriteLine("Error: --output is required."); - PrintHelp(Program.ExecutableName); - return; - } - - var typeStr = parser.GetValue("type"); - if (string.IsNullOrWhiteSpace(typeStr)) - { - Console.Error.WriteLine("Error: --type is required."); - HelpFormatter.WriteBlankLine(); - WriteAudioBlockTypes(); - return; - } - - if ( - !Enum.TryParse(typeStr, ignoreCase: true, out var blockType) - || !AudioBlockTypeNames.Contains(blockType.ToString()) - ) + var parser = CreateParser(); + if (!parser.Parse(args)) + return VfsCommand.Fail(2, parser.Errors[0]); + if (parser.HasFlag("help")) + { + PrintHelp(parser); + return 0; + } + + string? input = parser.Get("input"); + string? output = parser.Get("output"); + string? typeArg = parser.Get("type"); + if (string.IsNullOrWhiteSpace(input)) + return VfsCommand.Fail(2, "--input is required."); + if (string.IsNullOrWhiteSpace(output)) + return VfsCommand.Fail(2, "--output is required."); + if (string.IsNullOrWhiteSpace(typeArg)) + return VfsCommand.Fail(2, "--type is required."); + + string modeArg = parser.Get("mode") ?? "wav"; + if (modeArg is not ("wav" or "raw")) + return VfsCommand.Fail(2, "--mode must be one of: wav, raw."); + var mode = modeArg == "raw" ? AudioExportMode.Raw : AudioExportMode.Wav; + + byte[] key; + int jobs; + try { - Console.Error.WriteLine($"Error: '{typeStr}' is not a valid audio block type."); - HelpFormatter.WriteBlankLine(); - WriteAudioBlockTypes(); - return; + key = VFSDefine.ResolveKey(parser.Get("platform"), parser.Get("key")); + jobs = VfsCommand.ResolveJobs(parser.Get("jobs")); } - - var mode = parser.GetValue("mode") ?? "wav"; - if (mode is not ("raw" or "wav")) + catch (ArgumentException ex) { - Console.Error.WriteLine("Error: --mode must be one of: raw, wav"); - PrintHelp(Program.ExecutableName); - return; + return VfsCommand.Fail(2, ex.Message); } - var vfsPath = Path.Combine(gamePath, VFSDefine.VFS_DIR); + string gamePath = Path.GetFullPath(input); + if (!Directory.Exists(gamePath)) + return VfsCommand.Fail(1, "Game directory not found: \"{0}\"", gamePath); + string vfsPath = Path.Combine(gamePath, VFSDefine.VfsDirectoryName); if (!Directory.Exists(vfsPath)) - { - Console.Error.WriteLine( - $"Error: VFS directory ({VFSDefine.VFS_DIR}) not found under \"{gamePath}\"." - ); - return; - } - - byte[] chaChaKey = VFSDefine.DefaultChaChaKey; - var keyBase64 = parser.GetValue("key"); - if (!string.IsNullOrWhiteSpace(keyBase64)) - { - try - { - chaChaKey = Convert.FromBase64String(keyBase64); - } - catch (FormatException) - { - Console.Error.WriteLine("Error: --key must be a valid Base64 string."); - return; - } - if (chaChaKey.Length != VFSDefine.KEY_LEN) - { - Console.Error.WriteLine( - $"Error: --key must decode to {VFSDefine.KEY_LEN} bytes (got {chaChaKey.Length})." - ); - return; - } - } + return VfsCommand.Fail(1, "VFS directory not found under \"{0}\".", gamePath); + var logger = new Logger(parser.HasFlag("verbose")); + VfsArchive archive; try { - var logger = new Logger(parser.GetFlag("verbose")); - bool autoMap = !parser.GetFlag("no-map"); - string? language = GetLanguageForBlockType(blockType); - - IWemConverter? wemConverter = null; - if (mode == "wav") - { - wemConverter = ResolveConverter(logger); - if (wemConverter == null) - return; - } - - Console.WriteLine($"Input: {vfsPath}"); - Console.WriteLine($"Output: {outputDir}"); - Console.WriteLine($"Type: {blockType}"); - Console.WriteLine($"Mode: {mode}"); - - PckMapper? mapper = null; - if (autoMap && language != null) - { - mapper = LoadAudioDialogMapper(vfsPath, chaChaKey, language, logger); - } - else if (autoMap && language == null) - { - logger.Info($"Auto-mapping skipped: no language context for {blockType}"); - } - - Directory.CreateDirectory(outputDir); - ExtractAudioBlock( - vfsPath, - chaChaKey, - blockType, - outputDir, - mode, - mapper, - wemConverter, - logger - ); + archive = VfsArchive.Open(vfsPath, key, logger); } catch (Exception ex) { - Console.Error.WriteLine($"Error: {ex.Message}"); - Environment.Exit(1); + return VfsCommand.Fail(1, ex.Message); } - } - - private static void WriteAudioBlockTypes() - { - HelpFormatter.WriteSectionHeader("Audio block types"); - Console.WriteLine($" {string.Join(", ", AudioBlockTypeNames)}"); - } - - private static string? GetLanguageForBlockType(EVFSBlockType type) => - BlockLanguageMap.GetValueOrDefault(type); - private static PckMapper? LoadAudioDialogMapper( - string vfsPath, - byte[] key, - string language, - ILogger logger - ) - { - logger.Info("Loading AudioDialog from Table block..."); - - var tableInfo = VfsReader.ReadBlockInfo(vfsPath, EVFSBlockType.Table, key); - - foreach (var chunk in tableInfo.allChunks) + var block = archive.Find(typeArg); + if (block == null) { - var chunkFile = VfsReader.ResolveChunkPath(vfsPath, EVFSBlockType.Table, chunk); - if (chunkFile == null) - continue; - - using var chunkFs = File.OpenRead(chunkFile); - foreach (var file in chunk.files) - { - if (!file.fileName.EndsWith(".bytes", StringComparison.OrdinalIgnoreCase)) - continue; - - chunkFs.Seek(file.offset, SeekOrigin.Begin); - var data = VfsReader.ReadFileData(chunkFs, file, key); - - using var ms = new MemoryStream(data); - using var br = new BinaryReader(ms); - - string rootName; - try - { - rootName = SparkBufferDumper.GetRootDefinitionName(br); - } - catch - { - continue; - } - - if (rootName != "AudioDialog") - continue; - - logger.Info(" Found AudioDialog, parsing..."); - var json = SparkBufferDumper.Decrypt(data); - using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); - var mapper = new PckMapper(stream, language); - logger.Info($" Mapped {mapper.Count} entries (lang={language})"); - return mapper; - } + return BlockRegistry.TryParse(typeArg, out _) + ? VfsCommand.Fail(1, "Block \"{0}\" was not found in this VFS.", typeArg) + : VfsCommand.Fail(2, "Unknown block type: {0}", typeArg); } - logger.Info(" AudioDialog not found in Table block, skipping auto-map"); - return null; - } - - private static void ExtractAudioBlock( - string vfsPath, - byte[] key, - EVFSBlockType blockType, - string outputDir, - string mode, - PckMapper? mapper, - IWemConverter? wemConverter, - ILogger logger - ) - { - var blockInfo = VfsReader.ReadBlockInfo(vfsPath, blockType, key); - logger.Info($"--- {blockType} ---"); + if (block.Descriptor is { Kind: not ContentKind.Audio }) + logger.Info("Warning: {0} is not an audio block. Only .pck files inside it will be exported.", block.DisplayName); - var pckFiles = new List<(string Name, byte[] Data)>(); - foreach (var chunk in blockInfo.allChunks) + IWemDecoder? decoder = null; + if (mode == AudioExportMode.Wav) { - var chunkFile = VfsReader.ResolveChunkPath(vfsPath, blockType, chunk); - if (chunkFile == null) + decoder = WemDecoders.TryCreate(logger); + if (decoder == null) { - logger.Verbose($" Chunk not found, skipping"); - continue; - } - - using var chunkFs = File.OpenRead(chunkFile); - foreach (var file in chunk.files) - { - if (!file.fileName.EndsWith(".pck", StringComparison.OrdinalIgnoreCase)) - continue; - - chunkFs.Seek(file.offset, SeekOrigin.Begin); - var data = VfsReader.ReadFileData(chunkFs, file, key); - pckFiles.Add((file.fileName, data)); - } - } - - if (pckFiles.Count == 0) - { - logger.Info("No PCK files found in block."); - return; - } - - logger.Info($"Found {pckFiles.Count} PCK file(s)"); - - string ext = mode == "wav" ? ".wav" : ".wem"; - var jobs = new List(); - int mappedCount = 0; - int unmappedCount = 0; - - foreach (var (pckName, pckData) in pckFiles) - { - logger.Info($"Parsing {pckName}..."); - - using var pckStream = new MemoryStream(pckData); - var pckParser = new PckParser(pckStream); - var content = pckParser.Parse(); - - logger.Verbose( - $" {content.Entries.Count} entries, {content.Languages.Count} languages" - ); - - foreach (var entry in content.Entries) - { - byte[] fileData = pckParser.GetFileData(entry); - if (fileData.Length < 4) - continue; - - ReadOnlySpan magic = fileData.AsSpan(0, 4); - - if (magic.SequenceEqual("BKHD"u8)) - { - CollectBnkJobs( - fileData, - entry, - outputDir, - blockType, - mapper, - ext, - content.Languages, - jobs, - ref mappedCount, - ref unmappedCount - ); - } - else if (magic.SequenceEqual("RIFF"u8) || magic.SequenceEqual("RIFX"u8)) - { - string outName = ResolveOutputName( - blockType, - entry.FileId, - mapper, - ext, - content.Languages, - entry.LanguageId, - out bool isMapped - ); - if (isMapped) - mappedCount++; - else - unmappedCount++; - jobs.Add(new AudioJob(fileData, Path.Combine(outputDir, outName))); - } + return VfsCommand.Fail( + 1, + "No WEM decoder found. Place libvgmstream next to the program, or install vgmstream-cli." + ); } } - logger.Info($"Name mapping: mapped={mappedCount}, unmapped={unmappedCount}"); - AnsiConsole.MarkupLine( - $"Collected [blue]{jobs.Count}[/] audio files, processing [blue]{Markup.Escape(mode)}[/]..." - ); - - if (mode == "raw" || wemConverter == null) + string outputPath = Path.GetFullPath(output); + try { + AudioExportResult? result = null; AnsiConsole .Progress() .AutoClear(false) @@ -392,311 +103,77 @@ out bool isMapped new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn(), + new RemainingTimeColumn(), new SpinnerColumn() ) .Start(ctx => { - var task = ctx.AddTask("Extracting", maxValue: jobs.Count); - foreach (var job in jobs) - { - EnsureDirectory(job.OutputPath); - File.WriteAllBytes(job.OutputPath, job.Data); - task.Increment(1); - } + var task = ctx.AddTask("Exporting", maxValue: 0); + result = new AudioExportService(logger).Export( + archive, + block, + outputPath, + mode, + mapNames: !parser.HasFlag("no-map"), + decoder, + jobs, + new SpectreProgress(task), + cancellationToken + ); }); - AnsiConsole.MarkupLine($"[green]Done:[/] {jobs.Count} files extracted."); - return; - } - - int converted = 0; - int failed = 0; - var failMessages = new ConcurrentBag(); - - AnsiConsole - .Progress() - .AutoClear(false) - .HideCompleted(false) - .Columns( - new TaskDescriptionColumn(), - new ProgressBarColumn(), - new PercentageColumn(), - new RemainingTimeColumn(), - new SpinnerColumn() - ) - .Start(ctx => - { - var task = ctx.AddTask("Converting", maxValue: jobs.Count); - Parallel.ForEach( - jobs, - new ParallelOptions - { - MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount), - }, - job => - { - EnsureDirectory(job.OutputPath); - - string tempWem = Path.Combine( - Path.GetTempPath(), - $"byd_{Environment.CurrentManagedThreadId}_{Path.GetFileNameWithoutExtension(job.OutputPath)}.wem" - ); - - try - { - File.WriteAllBytes(tempWem, job.Data); - job.Data = null!; - wemConverter.Convert(tempWem, job.OutputPath); - Interlocked.Increment(ref converted); - } - catch (Exception ex) - { - string fallback = Path.ChangeExtension(job.OutputPath, ".wem"); - EnsureDirectory(fallback); - try - { - if (File.Exists(tempWem)) - File.Copy(tempWem, fallback, true); - } - catch { } - - failMessages.Add(ex.Message); - Interlocked.Increment(ref failed); - } - finally - { - try - { - File.Delete(tempWem); - } - catch { } - } - - task.Increment(1); - } - ); - }); - - foreach (var msg in failMessages) - logger.Verbose($" Failed: {msg}"); - - AnsiConsole.MarkupLine( - $"[green]Done:[/] {converted} converted" - + (failed > 0 ? $", [yellow]{failed} failed[/] (saved as .wem)" : "") - ); - } - - private static void CollectBnkJobs( - byte[] bnkData, - PckFileEntry bankEntry, - string outputDir, - EVFSBlockType blockType, - PckMapper? mapper, - string extension, - List languages, - List jobs, - ref int mappedCount, - ref int unmappedCount - ) - { - var wemEntries = BnkParser.Parse(bnkData); - foreach (var wem in wemEntries) - { - byte[] wemData = new byte[wem.Size]; - Array.Copy(bnkData, wem.Offset, wemData, 0, wem.Size); - - string outName = ResolveBnkWemName( - blockType, - bankEntry.FileId, - wem.Id, - mapper, - extension, - languages, - bankEntry.LanguageId, - out bool isMapped + decoder?.Dispose(); + if (result == null) + return 1; + logger.Info( + "Done. {0} files extracted, {1} converted, {2} failed.", + result.Extracted, + result.Converted, + result.Failed ); - if (isMapped) - mappedCount++; - else - unmappedCount++; - jobs.Add(new AudioJob(wemData, Path.Combine(outputDir, outName))); - } - } - - private sealed class AudioJob(byte[] data, string outputPath) - { - public byte[] Data = data; - public readonly string OutputPath = outputPath; - } - - // ── Output name resolution ────────────────────────────────────── - - private static string ResolveOutputName( - EVFSBlockType blockType, - ulong fileId, - PckMapper? mapper, - string extension, - List? languages, - uint languageId, - out bool isMapped - ) - { - string? mapped = mapper?.GetMappedPath(fileId.ToString()); - if (mapped != null) - { - isMapped = true; - return BuildMappedOutputPath(blockType, mapped, extension); + return result.Failed > 0 && result.Extracted == 0 ? 1 : 0; } - - if (languageId != 0 && languages != null) + catch (Exception ex) when (VfsCommand.IsCancel(ex)) { - var lang = languages.Find(l => l.Id == languageId); - if (lang != null) - { - isMapped = false; - return BuildUnmappedOutputPath( - blockType, - NormalizeLanguageDirectoryName(lang.Name), - $"{fileId}{extension}" - ); - } + decoder?.Dispose(); + return VfsCommand.Fail(1, "Cancelled."); } - - isMapped = false; - return BuildUnmappedOutputPath( - blockType, - NormalizeLanguageDirectoryName(GetLanguageForBlockType(blockType) ?? "Unknown"), - $"{fileId}{extension}" - ); - } - - private static string ResolveBnkWemName( - EVFSBlockType blockType, - ulong bankFileId, - uint wemId, - PckMapper? mapper, - string extension, - List? languages, - uint languageId, - out bool isMapped - ) - { - string? mapped = mapper?.GetMappedPath(wemId.ToString()); - if (mapped != null) - { - isMapped = true; - return BuildMappedOutputPath(blockType, mapped, extension); - } - - isMapped = false; - string language = "Unknown"; - if (languageId != 0 && languages != null) - { - var lang = languages.Find(l => l.Id == languageId); - if (lang != null) - language = NormalizeLanguageDirectoryName(lang.Name); - } - else - { - language = NormalizeLanguageDirectoryName( - GetLanguageForBlockType(blockType) ?? "Unknown" - ); - } - - return BuildUnmappedOutputPath(blockType, language, $"{bankFileId}_{wemId}{extension}"); - } - - private static string BuildMappedOutputPath( - EVFSBlockType blockType, - string mappedPath, - string extension - ) - { - string withExtension = Path.ChangeExtension(mappedPath, extension); - if ( - TryParseMappedPath(withExtension, out var audioType, out var language, out var realPath) - ) - { - return Path.Combine( - blockType.ToString(), - NormalizeLanguageDirectoryName(language), - audioType, - realPath - ); - } - - string fallbackLanguage = NormalizeLanguageDirectoryName( - GetLanguageForBlockType(blockType) ?? "Unknown" - ); - return BuildUnmappedOutputPath( - blockType, - fallbackLanguage, - Path.GetFileName(withExtension) - ); - } - - private static bool TryParseMappedPath( - string mappedPath, - out string audioType, - out string language, - out string realPath - ) - { - string[] parts = mappedPath.Split(['\\', '/'], StringSplitOptions.RemoveEmptyEntries); - if (parts.Length < 3) + catch (Exception ex) { - audioType = "unknown"; - language = "Unknown"; - realPath = Path.GetFileName(mappedPath); - return false; + decoder?.Dispose(); + return VfsCommand.Fail(1, ex.Message); } - - audioType = parts[0]; - language = parts[1]; - realPath = Path.Combine(parts[2..]); - return true; - } - - private static string BuildUnmappedOutputPath( - EVFSBlockType blockType, - string language, - string fileName - ) => Path.Combine(blockType.ToString(), "unmapped", language, fileName); - - private static string NormalizeLanguageDirectoryName(string languageName) - { - if (string.IsNullOrWhiteSpace(languageName)) - return "Unknown"; - - string trimmed = languageName.Trim(); - return char.ToUpperInvariant(trimmed[0]) + trimmed[1..].ToLowerInvariant(); - } - - private static void EnsureDirectory(string filePath) - { - string? dir = Path.GetDirectoryName(filePath); - if (dir != null) - Directory.CreateDirectory(dir); } - private static IWemConverter? ResolveConverter(ILogger logger) + private static ArgParser CreateParser() => + new ArgParser() + .Add("input", "i", "Game data directory that contains the VFS folder", "path", required: true) + .Add("output", "o", "Output directory", "dir", required: true) + .Add("type", "t", "Audio block type", "type", required: true) + .Add("mode", "m", "Output mode", "mode", continuation: "wav (default) or raw.") + .Add("key", null, "ChaCha20 key, Base64 of 32 bytes. Overrides --platform", "base64") + .Add("platform", null, "Key to use when --key is omitted", "name", continuation: "pc (default) or android.") + .Add("jobs", null, "How many WEM files to convert at once", "n", continuation: "Default: processor count.") + .Add("no-map", null, "Skip AudioDialog name mapping") + .Add("verbose", "v", "Print per-file details") + .Add("help", "h", "Show help information"); + + private static void PrintHelp(ArgParser parser) { - if (LibVgmstreamConverter.IsAvailable) - { - logger.Verbose("Engine: libvgmstream (DLL)"); - return new LibVgmstreamConverter(); - } - - if (WemConverter.VgmstreamPath != null) - { - logger.Verbose($"Engine: vgmstream-cli ({WemConverter.VgmstreamPath})"); - return new WemConverter(); - } - - Console.Error.WriteLine( - "Error: vgmstream not found. Place libvgmstream.dll (preferred) " - + "or vgmstream-cli next to the executable, or add to PATH." + HelpFormatter.WriteCommandHelp( + "PCK audio extractor", + "pck --input --output --type ", + parser, + () => + { + HelpFormatter.WriteSection("Audio block types"); + AnsiConsole.MarkupLine( + " [cyan]{0}[/]", + Markup.Escape(string.Join(", ", BlockRegistry.Audio.Select(static d => d.Type.ToString()))) + ); + AnsiConsole.MarkupLine(" [dim]A block discovered in this VFS can also be passed by its groupCfgName.[/]"); + AnsiConsole.WriteLine(); + } ); - return null; } } diff --git a/BydTools.CLI/Commands/VfsCommand.cs b/BydTools.CLI/Commands/VfsCommand.cs index 57b315c..28f44f4 100644 --- a/BydTools.CLI/Commands/VfsCommand.cs +++ b/BydTools.CLI/Commands/VfsCommand.cs @@ -1,198 +1,187 @@ +using BydTools.Extraction; +using BydTools.Extraction.Vfs; using BydTools.VFS; -using BydTools.VFS.PostProcessors; +using Spectre.Console; namespace BydTools.CLI.Commands; -sealed class VfsCommand : ICommand +public sealed class VfsCommand : ICommand { public string Name => "vfs"; - public string Description => "Dump files from VFS"; + public string Description => "Parse VFS block declarations and dump assets"; - public void PrintHelp(string exeName) + public int Execute(string[] args, CancellationToken cancellationToken) { - HelpFormatter.WriteUsage( - "vfs", - "--input --output --blocktype [,type2,...]" - ); + var parser = CreateParser(); + if (!parser.Parse(args)) + return Fail(2, parser.Errors[0]); + if (parser.HasFlag("help")) + { + PrintHelp(parser); + return 0; + } - HelpFormatter.WriteSectionHeader("Required"); - HelpFormatter.WriteEntry( - "-i, --input ", - "Game data directory that contains the VFS folder" - ); - HelpFormatter.WriteEntry("-o, --output ", "Output directory"); - HelpFormatter.WriteEntry( - "-t, --blocktype ", - "Block type to dump (name or numeric value)" - ); - HelpFormatter.WriteEntryContinuation( - "Multiple types can be separated by comma, e.g. Bundle,Lua,Table" - ); - HelpFormatter.WriteBlankLine(); + string? input = parser.Get("input"); + if (string.IsNullOrWhiteSpace(input)) + return Fail(2, "--input is required."); - HelpFormatter.WriteSectionHeader("Options"); - HelpFormatter.WriteEntry("--debug", "Scan subfolders and print block info (no extraction)"); - HelpFormatter.WriteCommonOptions(); - HelpFormatter.WriteBlankLine(); + string gamePath = Path.GetFullPath(input); + if (!Directory.Exists(gamePath)) + return Fail(1, "Game directory not found: \"{0}\"", gamePath); - HelpFormatter.WriteEnumValues("Available block types", VFSDumper.BlockHashMap.Keys); - } + string vfsPath = Path.Combine(gamePath, VFSDefine.VfsDirectoryName); + if (!Directory.Exists(vfsPath)) + return Fail(1, "VFS directory not found under \"{0}\".", gamePath); - public void Execute(string[] args) - { - var parser = new ArgParser() - .AddFlag("help", "h") - .AddFlag("verbose", "v") - .AddFlag("debug") - .AddOption("input", "i") - .AddOption("output", "o") - .AddOption("blocktype", "t") - .AddOption("key"); - - if (!parser.TryParse(args)) + byte[] key; + int jobs; + try { - foreach (var error in parser.Errors) - Console.Error.WriteLine(error); - PrintHelp(Program.ExecutableName); - return; + key = VFSDefine.ResolveKey(parser.Get("platform"), parser.Get("key")); + jobs = ResolveJobs(parser.Get("jobs")); } - - if (parser.GetFlag("help")) + catch (ArgumentException ex) { - PrintHelp(Program.ExecutableName); - return; + return Fail(2, ex.Message); } - var gamePath = parser.GetValue("input"); - if (string.IsNullOrWhiteSpace(gamePath)) + bool debug = parser.HasFlag("debug"); + string? output = parser.Get("output"); + string? blockTypeArg = parser.Get("blocktype"); + if (!debug) { - Console.Error.WriteLine("Error: --input is required."); - PrintHelp(Program.ExecutableName); - return; + if (string.IsNullOrWhiteSpace(output)) + return Fail(2, "--output is required."); + if (string.IsNullOrWhiteSpace(blockTypeArg)) + return Fail(2, "--blocktype is required."); } - var streamingAssetsPath = Path.Combine(gamePath, VFSDefine.VFS_DIR); - if (!Directory.Exists(streamingAssetsPath)) + var logger = new Logger(parser.HasFlag("verbose")); + VfsArchive archive; + try { - Console.Error.WriteLine( - "Error: VFS directory ({1}) not found under \"{0}\".", - gamePath, - VFSDefine.VFS_DIR - ); - return; + archive = VfsArchive.Open(vfsPath, key, logger); } - - byte[]? customKey = null; - var keyBase64 = parser.GetValue("key"); - if (!string.IsNullOrWhiteSpace(keyBase64)) + catch (Exception ex) { - try - { - customKey = Convert.FromBase64String(keyBase64); - } - catch (FormatException) - { - Console.Error.WriteLine("Error: --key must be a valid Base64 string."); - return; - } - - if (customKey.Length != VFSDefine.KEY_LEN) - { - Console.Error.WriteLine( - "Error: --key must decode to {0} bytes (got {1}).", - VFSDefine.KEY_LEN, - customKey.Length - ); - return; - } + return Fail(1, ex.Message); } - var logger = new Logger(parser.GetFlag("verbose")); - var postProcessors = PostProcessorFactory.CreateProcessors(logger); - IVFSDumper dumper = new VFSDumper(logger, postProcessors, customKey); + var extractor = new VfsExtractor(logger); + if (debug) + { + extractor.WriteDebugReport(archive); + return 0; + } - if (parser.GetFlag("debug")) + var blocks = new List(); + foreach (string token in blockTypeArg!.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) { - try - { - dumper.DebugScanBlocks(streamingAssetsPath); - } - catch (Exception ex) + var block = archive.Find(token); + if (block == null) { - Console.Error.WriteLine($"Error: {ex.Message}"); - Environment.Exit(1); + bool known = BlockRegistry.TryParse(token, out _); + return known + ? Fail(1, "Block \"{0}\" was not found in this VFS.", token) + : Fail(2, "Unknown block type: {0}", token); } - return; - } - var outputDir = parser.GetValue("output"); - if (string.IsNullOrWhiteSpace(outputDir)) - { - Console.Error.WriteLine("Error: --output is required."); - PrintHelp(Program.ExecutableName); - return; + blocks.Add(block); } - var blockTypeString = parser.GetValue("blocktype"); - if (string.IsNullOrWhiteSpace(blockTypeString)) + string outputPath = Path.GetFullPath(output!); + try { - Console.Error.WriteLine("Error: --blocktype is required."); - HelpFormatter.WriteBlankLine(); - HelpFormatter.WriteEnumValues("Available block types", VFSDumper.BlockHashMap.Keys); - return; + AnsiConsole + .Progress() + .AutoClear(false) + .HideCompleted(false) + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new RemainingTimeColumn(), + new SpinnerColumn() + ) + .Start(ctx => + { + foreach (var block in blocks) + { + var task = ctx.AddTask(block.DisplayName, maxValue: 0); + extractor.Dump( + block, + outputPath, + jobs, + new SpectreProgress(task), + cancellationToken + ); + } + }); } - - var blockTypes = ParseBlockTypes(blockTypeString); - if (blockTypes == null) - return; - - try + catch (Exception ex) when (IsCancel(ex)) { - Console.WriteLine("Input: {0}", streamingAssetsPath); - Console.WriteLine("Output: {0}", outputDir); - - for (int i = 0; i < blockTypes.Count; i++) - { - if (i > 0) - Console.WriteLine(); - dumper.DumpAssetByType(streamingAssetsPath, blockTypes[i], outputDir); - } + return Fail(1, "Cancelled."); } catch (Exception ex) { - Console.Error.WriteLine($"Error: {ex.Message}"); - Environment.Exit(1); + return Fail(1, ex.Message); } + + return 0; } - private static List? ParseBlockTypes(string raw) + internal static ArgParser CreateParser() => + new ArgParser() + .Add("input", "i", "Game data directory that contains the VFS folder", "path", required: true) + .Add("output", "o", "Output directory for extracted files", "dir", required: true) + .Add( + "blocktype", + "t", + "Block type to extract", + "type", + required: true, + "Separate multiple types with commas." + ) + .Add("key", null, "ChaCha20 key, Base64 of 32 bytes. Overrides --platform", "base64") + .Add("platform", null, "Key to use when --key is omitted", "name", continuation: "pc (default) or android.") + .Add("jobs", null, "How many chunks to read at once", "n", continuation: "Default: processor count.") + .Add("debug", "d", "Scan block declarations without extracting") + .Add("verbose", "v", "Print per-chunk and per-file details") + .Add("help", "h", "Show help information"); + + private static void PrintHelp(ArgParser parser) { - var segments = raw.Split(',', StringSplitOptions.RemoveEmptyEntries); - var result = new List(segments.Length); + HelpFormatter.WriteCommandHelp( + "VFS asset dumper", + "vfs --input --output --blocktype ", + parser, + () => HelpFormatter.WriteEnumValues("Available block types", BlockRegistry.All.Select(static d => d.Type)) + ); + } - foreach (var segment in segments) - { - var trimmed = segment.Trim(); - if (Enum.TryParse(trimmed, ignoreCase: true, out var parsed)) - { - result.Add(parsed); - } - else if ( - byte.TryParse(trimmed, out var btValue) - && Enum.IsDefined(typeof(EVFSBlockType), btValue) - ) - { - result.Add((EVFSBlockType)btValue); - } - else - { - Console.Error.WriteLine("Error: failed to parse blocktype \"{0}\".", trimmed); - HelpFormatter.WriteBlankLine(); - HelpFormatter.WriteEnumValues("Available block types", VFSDumper.BlockHashMap.Keys); - return null; - } - } + internal static int ResolveJobs(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + return Environment.ProcessorCount; + if (!int.TryParse(text, out int jobs) || jobs < 1) + throw new ArgumentException("--jobs must be a positive integer."); + return jobs; + } - return result; + internal static int Fail(int code, string format, params object[] args) + { + Logger.WriteError(args.Length == 0 ? format : string.Format(format, args)); + return code; } + + internal static bool IsCancel(Exception ex) => + ex is OperationCanceledException + || (ex is AggregateException aggregate && aggregate.InnerExceptions.All(IsCancel)); +} + +internal sealed class SpectreProgress(ProgressTask task) : IProgressSink +{ + public void AddTotal(long amount) => task.MaxValue += amount; + + public void Advance(long amount) => task.Increment(amount); } diff --git a/BydTools.CLI/HelpFormatter.cs b/BydTools.CLI/HelpFormatter.cs index 713153b..28a6910 100644 --- a/BydTools.CLI/HelpFormatter.cs +++ b/BydTools.CLI/HelpFormatter.cs @@ -1,78 +1,82 @@ +using BydTools.CLI.Commands; using Spectre.Console; namespace BydTools.CLI; -/// -/// Shared help output formatter for consistent CLI help style. -/// -static class HelpFormatter +public static class HelpFormatter { - private const int FlagColumnWidth = 26; - private const int TypeColumnWidth = 26; - private const int TypeColumnsPerRow = 3; + public const string Executable = "BydTools"; - public static void WriteUsage( - string command, - string requiredArgs, - string? optionalHint = "[options]" - ) + public static void WriteRootHelp(IEnumerable commands) { - AnsiConsole.MarkupLine("[bold yellow]Usage:[/]"); - var parts = new List - { - $"[bold]{Markup.Escape(Program.ExecutableName)}[/]", - $"[green]{Markup.Escape(command)}[/]", - Markup.Escape(requiredArgs), - }; - if (optionalHint != null) - parts.Add($"[grey]{Markup.Escape(optionalHint)}[/]"); - AnsiConsole.MarkupLine($" {string.Join(" ", parts)}"); + var rule = new Rule("[bold]BydTools[/]") { Justification = Justify.Left }; + AnsiConsole.Write(rule); + AnsiConsole.MarkupLine("[dim]Arknights: Endfield data tools[/]"); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Usage:[/]"); + AnsiConsole.MarkupLine($" [cyan]{Executable}[/] [yellow][/] [[options]]"); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Commands:[/]"); + foreach (var command in commands) + AnsiConsole.MarkupLine($" [cyan]{command.Name,-12}[/] {Markup.Escape(command.Description)}"); AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine($"Run [cyan]{Executable} --help[/] for details."); } - public static void WriteSectionHeader(string title) + public static void WriteCommandHelp(string description, string usage, ArgParser parser, Action? extra = null) { - AnsiConsole.MarkupLine($"[bold yellow]{Markup.Escape(title)}:[/]"); - } + var rule = new Rule($"[bold]{Markup.Escape(description)}[/]") { Justification = Justify.Left }; + AnsiConsole.Write(rule); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Usage:[/]"); + AnsiConsole.MarkupLine($" [cyan]{Executable}[/] {Markup.Escape(usage)}"); + AnsiConsole.WriteLine(); - public static void WriteEntry(string flags, string description) - { - AnsiConsole.MarkupLine( - $" [green]{Markup.Escape(flags.PadRight(FlagColumnWidth))}[/]" - + Markup.Escape(description) - ); + var required = parser.Options.Where(static o => o.Required).ToArray(); + var optional = parser.Options.Where(static o => !o.Required).ToArray(); + if (required.Length > 0) + { + AnsiConsole.MarkupLine("[bold]Required:[/]"); + WriteOptions(required); + AnsiConsole.WriteLine(); + } + + if (optional.Length > 0) + { + AnsiConsole.MarkupLine("[bold]Options:[/]"); + WriteOptions(optional); + AnsiConsole.WriteLine(); + } + + extra?.Invoke(); } - public static void WriteEntryContinuation(string text) + public static void WriteSection(string title) { - AnsiConsole.MarkupLine($" {"".PadRight(FlagColumnWidth)}{Markup.Escape(text)}"); + AnsiConsole.MarkupLine($"[bold]{Markup.Escape(title)}[/]"); } - public static void WriteCommonOptions() + public static void WriteEnumValues(string title, IEnumerable values) + where T : struct, Enum { - WriteEntry("-v, --verbose", "Enable verbose output"); - WriteEntry("-h, --help", "Show help information"); + WriteSection(title); + foreach (var value in values) + AnsiConsole.MarkupLine($" [cyan]{value,-24}[/] [dim]({Convert.ToByte(value)})[/]"); + AnsiConsole.WriteLine(); } - public static void WriteBlankLine() => AnsiConsole.WriteLine(); - - /// - /// Formats enum values as a multi-column display: "Name (Value)" padded into columns. - /// - public static void WriteEnumValues(string sectionTitle, IEnumerable values) - where T : Enum + private static void WriteOptions(IEnumerable options) { - WriteSectionHeader(sectionTitle); - - var items = values.Select(v => $"{v} ({Convert.ToByte(v)})").ToList(); - - for (int i = 0; i < items.Count; i += TypeColumnsPerRow) + foreach (var option in options) { - var row = items - .Skip(i) - .Take(TypeColumnsPerRow) - .Select(item => item.PadRight(TypeColumnWidth)); - AnsiConsole.MarkupLine($" [grey]{Markup.Escape(string.Join("", row))}[/]"); + string flags = option.ShortName == null + ? $"--{option.LongName}" + : $"--{option.LongName}, -{option.ShortName}"; + if (option.ValueName != null) + flags += $" <{option.ValueName}>"; + AnsiConsole.MarkupLine($" [cyan]{Markup.Escape(flags),-32}[/] {Markup.Escape(option.Description)}"); + foreach (string line in option.Continuation) + AnsiConsole.MarkupLine($" {"",-32} [dim]{Markup.Escape(line)}[/]"); } } } diff --git a/BydTools.CLI/Logger.cs b/BydTools.CLI/Logger.cs index 9853429..53178a1 100644 --- a/BydTools.CLI/Logger.cs +++ b/BydTools.CLI/Logger.cs @@ -1,58 +1,33 @@ -using BydTools.Utils; -using Spectre.Console; - namespace BydTools.CLI; -/// -/// Console logger with support for verbose mode, powered by Spectre.Console. -/// -public class Logger : ILogger +public sealed class Logger : BydTools.Core.ILogger { - private static readonly IAnsiConsole _stderr = AnsiConsole.Create( - new AnsiConsoleSettings { Out = new AnsiConsoleOutput(Console.Error) } - ); - private readonly bool _verbose; - public Logger(bool verbose = false) - { - _verbose = verbose; - } - - public void Info(string message) => AnsiConsole.MarkupLine(Markup.Escape(message)); + public Logger(bool verbose) => _verbose = verbose; - public void Info(string format, params object[] args) => - AnsiConsole.MarkupLine(Markup.Escape(string.Format(format, args))); + public void Info(string message) => Console.WriteLine(message); - public void InfoNoNewline(string message) => AnsiConsole.Markup(Markup.Escape(message)); + public void Info(string format, params object[] args) => Console.WriteLine(format, args); public void Verbose(string message) { if (_verbose) - AnsiConsole.MarkupLine($"[grey]{Markup.Escape(message)}[/]"); + Console.WriteLine(message); } public void Verbose(string format, params object[] args) { if (_verbose) - AnsiConsole.MarkupLine($"[grey]{Markup.Escape(string.Format(format, args))}[/]"); + Console.WriteLine(format, args); } - public void VerboseNoNewline(string message) - { - if (_verbose) - AnsiConsole.Markup($"[grey]{Markup.Escape(message)}[/]"); - } + public void Error(string message) => WriteError(message); - public void Error(string message) => _stderr.MarkupLine($"[red]{Markup.Escape(message)}[/]"); + public void Error(string format, params object[] args) => WriteError(format, args); - public void Error(string format, params object[] args) => - _stderr.MarkupLine($"[red]{Markup.Escape(string.Format(format, args))}[/]"); + public static void WriteError(string message) => Console.Error.WriteLine(message); - /// - /// Writes a styled error message to stderr with a red "error:" prefix. - /// Use for CLI-level validation/argument errors. - /// - public static void WriteError(string message) => - _stderr.MarkupLine($"[red bold]error:[/] {Markup.Escape(message)}"); + public static void WriteError(string format, params object[] args) => + Console.Error.WriteLine(format, args); } diff --git a/BydTools.CLI/Program.cs b/BydTools.CLI/Program.cs index 46896cb..6d96f19 100644 --- a/BydTools.CLI/Program.cs +++ b/BydTools.CLI/Program.cs @@ -1,56 +1,51 @@ -using System.Reflection; using BydTools.CLI.Commands; namespace BydTools.CLI; -class Program +public static class Program { - private static readonly ICommand[] Commands = [new VfsCommand(), new PckCommand()]; + private static CancellationTokenSource? _cancellation; + private static int _cancelHooked; - internal static string ExecutableName { get; } = - Path.GetFileNameWithoutExtension( - Assembly.GetEntryAssembly()?.GetName().Name - ?? Assembly.GetExecutingAssembly().GetName().Name - ?? "BydTools.CLI" - ); - - static void Main(string[] args) + public static int Main(string[] args) { - if (args.Length == 0) + using var cancellation = new CancellationTokenSource(); + _cancellation = cancellation; + if (Interlocked.Exchange(ref _cancelHooked, 1) == 0) { - PrintHelp(); - return; + Console.CancelKeyPress += (_, eventArgs) => + { + eventArgs.Cancel = true; + _cancellation?.Cancel(); + }; } - var subcommand = args[0].ToLowerInvariant(); - - if (subcommand is "-h" or "--help") + var commands = new ICommand[] { new VfsCommand(), new PckCommand() }; + if (args.Length == 0 || args[0] is "-h" or "--help" or "help") { - PrintHelp(); - return; + HelpFormatter.WriteRootHelp(commands); + return 0; } - var command = Array.Find(Commands, c => c.Name == subcommand); + var command = commands.FirstOrDefault(c => + c.Name.Equals(args[0], StringComparison.OrdinalIgnoreCase) + ); if (command == null) { - Logger.WriteError($"unknown command: {subcommand}"); - PrintHelp(); - return; + Logger.WriteError("Unknown command: {0}", args[0]); + Console.Error.WriteLine(); + HelpFormatter.WriteRootHelp(commands); + return 2; } - command.Execute(args[1..]); - } - - static void PrintHelp() - { - HelpFormatter.WriteUsage("", "[options]", null); - - HelpFormatter.WriteSectionHeader("Commands"); - foreach (var cmd in Commands) - HelpFormatter.WriteEntry(cmd.Name, cmd.Description); - HelpFormatter.WriteBlankLine(); - - HelpFormatter.WriteSectionHeader("Options"); - HelpFormatter.WriteEntry("-h, --help", "Show help information"); + try + { + return command.Execute(args[1..], cancellation.Token); + } + catch (OperationCanceledException) + { + Logger.WriteError("Cancelled."); + return 1; + } } } diff --git a/BydTools.Core/BydTools.Core.csproj b/BydTools.Core/BydTools.Core.csproj new file mode 100644 index 0000000..6b512ec --- /dev/null +++ b/BydTools.Core/BydTools.Core.csproj @@ -0,0 +1 @@ + diff --git a/BydTools.Utils/Crypto/CSChaCha20.cs b/BydTools.Core/Crypto/CSChaCha20.cs similarity index 97% rename from BydTools.Utils/Crypto/CSChaCha20.cs rename to BydTools.Core/Crypto/CSChaCha20.cs index 1b41bdf..870ff69 100644 --- a/BydTools.Utils/Crypto/CSChaCha20.cs +++ b/BydTools.Core/Crypto/CSChaCha20.cs @@ -18,7 +18,7 @@ using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; -namespace BydTools.Utils.Crypto; +namespace BydTools.Core.Crypto; /// /// Chosen SIMD mode diff --git a/BydTools.Utils/Crypto/CSXXTEA.cs b/BydTools.Core/Crypto/CSXXTEA.cs similarity index 96% rename from BydTools.Utils/Crypto/CSXXTEA.cs rename to BydTools.Core/Crypto/CSXXTEA.cs index 88a3114..3635ee9 100644 --- a/BydTools.Utils/Crypto/CSXXTEA.cs +++ b/BydTools.Core/Crypto/CSXXTEA.cs @@ -6,7 +6,7 @@ * Roger M. Needham * * Original Code Author: Ma Bingyao - * Modified for BydTools.Utils + * Modified for BydTools.Core * * The MIT License (MIT) * Copyright (c) 2008-2016 Ma Bingyao @@ -33,7 +33,7 @@ using System; using System.Text; -namespace BydTools.Utils.Crypto; +namespace BydTools.Core.Crypto; /// /// XXTEA (Corrected Block TEA) encryption algorithm implementation diff --git a/BydTools.Utils/ILogger.cs b/BydTools.Core/ILogger.cs similarity index 57% rename from BydTools.Utils/ILogger.cs rename to BydTools.Core/ILogger.cs index d20d37f..7c3ac9c 100644 --- a/BydTools.Utils/ILogger.cs +++ b/BydTools.Core/ILogger.cs @@ -1,16 +1,12 @@ -namespace BydTools.Utils; - -/// -/// Shared logger interface for all BydTools modules. -/// -public interface ILogger -{ - void Info(string message); - void Info(string format, params object[] args); - void InfoNoNewline(string message); - void Verbose(string message); - void Verbose(string format, params object[] args); - void VerboseNoNewline(string message); - void Error(string message); - void Error(string format, params object[] args); -} +namespace BydTools.Core; + +/// Shared logger for library code. The CLI supplies the console implementation. +public interface ILogger +{ + void Info(string message); + void Info(string format, params object[] args); + void Verbose(string message); + void Verbose(string format, params object[] args); + void Error(string message); + void Error(string format, params object[] args); +} diff --git a/BydTools.Core/IO/SpanReader.cs b/BydTools.Core/IO/SpanReader.cs new file mode 100644 index 0000000..b29434f --- /dev/null +++ b/BydTools.Core/IO/SpanReader.cs @@ -0,0 +1,100 @@ +using System.Buffers.Binary; +using System.Text; + +namespace BydTools.Core.IO; + +/// +/// Bounds-checked cursor over a byte span. Seeking past the buffer throws +/// instead of allocating or wrapping. +/// +public ref struct SpanReader +{ + private readonly ReadOnlySpan _span; + private int _position; + + public SpanReader(ReadOnlySpan span) + { + _span = span; + _position = 0; + } + + public int Position => _position; + public int Length => _span.Length; + public int Remaining => _span.Length - _position; + + public void Seek(int position) + { + if ((uint)position > (uint)_span.Length) + { + throw new InvalidDataException( + $"Seek to {position} is outside the buffer ({_span.Length} bytes)." + ); + } + + _position = position; + } + + public void Skip(int count) + { + if (count < 0) + throw new InvalidDataException($"Cannot skip {count} bytes at offset {_position}."); + Seek(_position + count); + } + + /// Advances to the next multiple of (2, 4, or 8). + public void Align(int boundary) + { + int aligned = (_position + (boundary - 1)) & ~(boundary - 1); + Seek(aligned); + } + + public byte ReadByte() => Take(1)[0]; + + public ReadOnlySpan ReadBytes(int count) => Take(count); + + public ushort ReadUInt16() => BinaryPrimitives.ReadUInt16LittleEndian(Take(2)); + + public int ReadInt32() => BinaryPrimitives.ReadInt32LittleEndian(Take(4)); + + public uint ReadUInt32() => BinaryPrimitives.ReadUInt32LittleEndian(Take(4)); + + public uint ReadUInt32BigEndian() => BinaryPrimitives.ReadUInt32BigEndian(Take(4)); + + public long ReadInt64() => BinaryPrimitives.ReadInt64LittleEndian(Take(8)); + + public UInt128 ReadUInt128() => BinaryPrimitives.ReadUInt128LittleEndian(Take(16)); + + public float ReadSingle() => BinaryPrimitives.ReadSingleLittleEndian(Take(4)); + + public double ReadDouble() => BinaryPrimitives.ReadDoubleLittleEndian(Take(8)); + + public bool ReadBoolean() => ReadByte() != 0; + + public string ReadUtf8(int length) => Encoding.UTF8.GetString(Take(length)); + + public string ReadNullTerminatedUtf8() + { + int start = _position; + int relative = _span[start..].IndexOf((byte)0); + if (relative < 0) + throw new InvalidDataException($"Unterminated string at offset {start}."); + + string value = Encoding.UTF8.GetString(_span.Slice(start, relative)); + _position = start + relative + 1; + return value; + } + + private ReadOnlySpan Take(int count) + { + if (count < 0 || (uint)count > (uint)Remaining) + { + throw new InvalidDataException( + $"Need {count} bytes at offset {_position}, buffer length is {_span.Length}." + ); + } + + var slice = _span.Slice(_position, count); + _position += count; + return slice; + } +} diff --git a/BydTools.Core/IO/UInt128Hex.cs b/BydTools.Core/IO/UInt128Hex.cs new file mode 100644 index 0000000..99b5965 --- /dev/null +++ b/BydTools.Core/IO/UInt128Hex.cs @@ -0,0 +1,13 @@ +using System.Buffers.Binary; + +namespace BydTools.Core.IO; + +public static class UInt128Hex +{ + public static string ToHexLittleEndian(this UInt128 value) + { + Span bytes = stackalloc byte[16]; + BinaryPrimitives.WriteUInt128LittleEndian(bytes, value); + return Convert.ToHexString(bytes); + } +} diff --git a/BydTools.Extraction/Audio/AudioExportService.cs b/BydTools.Extraction/Audio/AudioExportService.cs new file mode 100644 index 0000000..b90d843 --- /dev/null +++ b/BydTools.Extraction/Audio/AudioExportService.cs @@ -0,0 +1,350 @@ +using System.Collections.Concurrent; +using BydTools.Audio.Bnk; +using BydTools.Audio.Naming; +using BydTools.Audio.Pck; +using BydTools.Audio.Wem; +using BydTools.Core; +using BydTools.Extraction.Sink; +using BydTools.Formats.SparkBuffer; +using BydTools.VFS; + +namespace BydTools.Extraction.Audio; + +public enum AudioExportMode +{ + Raw, + Wav, +} + +public sealed class AudioExportResult(int Extracted, int Converted, int Failed, IReadOnlyList Failures) +{ + public int Extracted { get; } = Extracted; + public int Converted { get; } = Converted; + public int Failed { get; } = Failed; + public IReadOnlyList Failures { get; } = Failures; +} + +/// +/// Exports one audio block. PCK archives are decrypted and converted one at a time, +/// and WEM jobs point into that archive's buffer instead of copying every file up front. +/// +public sealed class AudioExportService +{ + private readonly ILogger _logger; + + public AudioExportService(ILogger logger) => _logger = logger; + + public AudioExportResult Export( + VfsArchive archive, + VfsBlock block, + string outputDirectory, + AudioExportMode mode, + bool mapNames, + IWemDecoder? decoder, + int jobs, + IProgressSink? progress = null, + CancellationToken cancellationToken = default + ) + { + progress ??= NullProgress.Instance; + string? language = block.Descriptor?.AudioLanguage; + PckMapper? mapper = null; + if (mapNames && language != null) + mapper = LoadMapper(archive, language); + else if (mapNames) + _logger.Info("Auto-mapping skipped: no language context for {0}", block.DisplayName); + + var sink = new DirectoryOutputSink(outputDirectory); + int extracted = 0; + int converted = 0; + int failed = 0; + int mapped = 0; + int unmapped = 0; + var failures = new ConcurrentBag(); + int pckCount = 0; + + foreach (var chunk in block.Info.Chunks) + { + cancellationToken.ThrowIfCancellationRequested(); + string? chunkPath = block.ResolveChunkPath(chunk); + if (chunkPath == null) + { + _logger.Verbose(" Chunk not found, skipping"); + continue; + } + + foreach (var file in chunk.Files) + { + if (!file.Name.EndsWith(".pck", StringComparison.OrdinalIgnoreCase)) + continue; + pckCount++; + byte[] data; + try + { + data = block.ReadFile(chunkPath, file); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + failed++; + failures.Add($"{file.Name}: {ex.Message}"); + continue; + } + + try + { + ExportPck( + data, + file.Name, + block.DisplayName, + language, + mode, + mapper, + decoder, + jobs, + sink, + progress, + cancellationToken, + ref extracted, + ref converted, + ref failed, + ref mapped, + ref unmapped, + failures + ); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + failed++; + failures.Add($"{file.Name}: {ex.Message}"); + } + } + } + + if (pckCount == 0) + _logger.Info("No PCK files found in block."); + else + _logger.Info("Name mapping: mapped={0}, unmapped={1}", mapped, unmapped); + + foreach (string message in failures) + _logger.Verbose(" Failed: {0}", message); + + return new AudioExportResult(extracted, converted, failed, failures.ToArray()); + } + + private void ExportPck( + byte[] data, + string pckName, + string blockFolder, + string? blockLanguage, + AudioExportMode mode, + PckMapper? mapper, + IWemDecoder? decoder, + int jobs, + DirectoryOutputSink sink, + IProgressSink progress, + CancellationToken cancellationToken, + ref int extracted, + ref int converted, + ref int failed, + ref int mapped, + ref int unmapped, + ConcurrentBag failures + ) + { + _logger.Info("Parsing {0}...", pckName); + var pck = PckArchive.Parse(data); + if (pck.IsVfsEncrypted) + pck.DecipherPayloads(data); + _logger.Verbose(" {0} entries, {1} languages", pck.Entries.Count, pck.Languages.Count); + + string extension = mode == AudioExportMode.Wav ? ".wav" : ".wem"; + var jobsToRun = new List(); + foreach (var entry in pck.Entries) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!PckArchive.TrySlice(data, entry, out var slice) || slice.Length < 4) + continue; + + try + { + if (slice[..4].SequenceEqual("BKHD"u8)) + { + var wems = BnkParser.Parse(slice); + foreach (var wem in wems) + { + bool isMapped = mapper?.GetMappedPath(wem.Id) != null; + if (isMapped) + mapped++; + else + unmapped++; + string relative = AudioNaming.ResolveBnkWem( + blockFolder, + blockLanguage, + entry.FileId, + wem.Id, + mapper, + extension, + pck.Languages, + entry.LanguageId + ); + jobsToRun.Add(new AudioJob((int)entry.Offset + wem.Offset, wem.Size, relative)); + } + } + else if (slice[..4].SequenceEqual("RIFF"u8) || slice[..4].SequenceEqual("RIFX"u8)) + { + bool isMapped = mapper?.GetMappedPath(entry.FileId) != null; + if (isMapped) + mapped++; + else + unmapped++; + string relative = AudioNaming.ResolveWem( + blockFolder, + blockLanguage, + entry.FileId, + mapper, + extension, + pck.Languages, + entry.LanguageId + ); + jobsToRun.Add(new AudioJob((int)entry.Offset, (int)entry.Size, relative)); + } + else if (slice[..4].SequenceEqual("PLUG"u8) && mode == AudioExportMode.Raw) + { + string relative = AudioNaming.ResolveWem( + blockFolder, + blockLanguage, + entry.FileId, + mapper, + ".plg", + pck.Languages, + entry.LanguageId + ); + Write(sink, relative, slice); + extracted++; + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + failed++; + failures.Add($"{pckName}#{entry.FileId}: {ex.Message}"); + } + } + + if (jobsToRun.Count == 0) + return; + + progress.AddTotal(jobsToRun.Count); + if (mode == AudioExportMode.Raw || decoder == null) + { + foreach (var job in jobsToRun) + { + cancellationToken.ThrowIfCancellationRequested(); + Write(sink, job.RelativePath, data.AsSpan(job.Offset, job.Length)); + extracted++; + progress.Advance(1); + } + + return; + } + + int localConverted = 0; + int localFailed = 0; + Parallel.ForEach( + jobsToRun, + new ParallelOptions + { + MaxDegreeOfParallelism = Math.Max(1, jobs), + CancellationToken = cancellationToken, + }, + job => + { + try + { + using var output = sink.Create(job.RelativePath); + decoder.Decode(data.AsMemory(job.Offset, job.Length), output); + output.Complete(); + Interlocked.Increment(ref localConverted); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + try + { + string fallback = Path.ChangeExtension(job.RelativePath, ".wem"); + Write(sink, fallback, data.AsSpan(job.Offset, job.Length)); + } + catch (Exception writeEx) when (writeEx is not OperationCanceledException) + { + failures.Add(writeEx.Message); + } + + failures.Add(ex.Message); + Interlocked.Increment(ref localFailed); + } + finally + { + progress.Advance(1); + } + } + ); + converted += localConverted; + failed += localFailed; + extracted += localConverted + localFailed; + } + + private PckMapper? LoadMapper(VfsArchive archive, string language) + { + _logger.Info("Loading AudioDialog from Table block..."); + VfsBlock? table = archive.Find(nameof(EVFSBlockType.Table)); + if (table == null) + { + _logger.Info(" AudioDialog not found in Table block, skipping auto-map"); + return null; + } + + foreach (var chunk in table.Info.Chunks) + { + string? chunkPath = table.ResolveChunkPath(chunk); + if (chunkPath == null) + continue; + foreach (var file in chunk.Files) + { + if (!file.Name.EndsWith(".bytes", StringComparison.OrdinalIgnoreCase)) + continue; + try + { + byte[] data = table.ReadFile(chunkPath, file); + if (SparkBuffer.GetRootName(data) != "AudioDialog") + continue; + _logger.Info(" Found AudioDialog, parsing..."); + var mapper = new PckMapper(SparkBuffer.ReadObject(data), language); + _logger.Info(" Mapped {0} entries (lang={1})", mapper.Count, language); + return mapper; + } + catch (OperationCanceledException) + { + throw; + } + catch + { + continue; + } + } + } + + _logger.Info(" AudioDialog not found in Table block, skipping auto-map"); + return null; + } + + private static void Write(IOutputSink sink, string relativePath, ReadOnlySpan data) + { + using var output = sink.Create(relativePath); + output.Write(data); + output.Complete(); + } + + private readonly record struct AudioJob(int Offset, int Length, string RelativePath); +} diff --git a/BydTools.Extraction/BydTools.Extraction.csproj b/BydTools.Extraction/BydTools.Extraction.csproj new file mode 100644 index 0000000..45eec73 --- /dev/null +++ b/BydTools.Extraction/BydTools.Extraction.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/BydTools.Extraction/IProgressSink.cs b/BydTools.Extraction/IProgressSink.cs new file mode 100644 index 0000000..06932b3 --- /dev/null +++ b/BydTools.Extraction/IProgressSink.cs @@ -0,0 +1,18 @@ +namespace BydTools.Extraction; + +public interface IProgressSink +{ + void AddTotal(long amount); + void Advance(long amount); +} + +public sealed class NullProgress : IProgressSink +{ + public static readonly NullProgress Instance = new(); + + private NullProgress() { } + + public void AddTotal(long amount) { } + + public void Advance(long amount) { } +} diff --git a/BydTools.Extraction/Sink/DirectoryOutputSink.cs b/BydTools.Extraction/Sink/DirectoryOutputSink.cs new file mode 100644 index 0000000..53b3fc9 --- /dev/null +++ b/BydTools.Extraction/Sink/DirectoryOutputSink.cs @@ -0,0 +1,65 @@ +namespace BydTools.Extraction.Sink; + +public interface IOutputSink +{ + OutputFile Create(string relativePath); +} + +/// +/// Writes to path.partial and moves into place only after . +/// Disposing without completing deletes the partial file, including on cancellation. +/// +public sealed class OutputFile : FileStream +{ + private readonly string _partial; + private readonly string _final; + private bool _complete; + private int _disposed; + + internal OutputFile(string partial, string final) + : base(partial, FileMode.Create, FileAccess.Write, FileShare.None) + { + _partial = partial; + _final = final; + } + + public void Complete() => _complete = true; + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + if (!disposing || Interlocked.Exchange(ref _disposed, 1) != 0) + return; + if (_complete) + File.Move(_partial, _final, overwrite: true); + else if (File.Exists(_partial)) + File.Delete(_partial); + } +} + +public sealed class DirectoryOutputSink : IOutputSink +{ + private readonly string _root; + + public DirectoryOutputSink(string root) + { + _root = Path.GetFullPath(root); + Directory.CreateDirectory(_root); + } + + public OutputFile Create(string relativePath) + { + if (string.IsNullOrWhiteSpace(relativePath) || Path.IsPathRooted(relativePath)) + throw new InvalidDataException($"Refusing path \"{relativePath}\"."); + + string full = Path.GetFullPath(Path.Combine(_root, relativePath)); + string prefix = _root.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar; + if (!full.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"Path \"{relativePath}\" escapes the output directory."); + + string? directory = Path.GetDirectoryName(full); + if (!string.IsNullOrEmpty(directory)) + Directory.CreateDirectory(directory); + return new OutputFile(full + ".partial", full); + } +} diff --git a/BydTools.Extraction/Vfs/ContentHandlers.cs b/BydTools.Extraction/Vfs/ContentHandlers.cs new file mode 100644 index 0000000..cce7e20 --- /dev/null +++ b/BydTools.Extraction/Vfs/ContentHandlers.cs @@ -0,0 +1,121 @@ +using BydTools.Audio.Pck; +using BydTools.Extraction.Sink; +using BydTools.Formats.Lua; +using BydTools.Formats.SparkBuffer; +using BydTools.Formats.Usm; + +namespace BydTools.Extraction.Vfs; + +public enum HandleResult +{ + Handled, + Skipped, + Failed, +} + +public interface IContentHandler +{ + bool CanHandle(string relativePath); + HandleResult Handle(string relativePath, ReadOnlyMemory data, IOutputSink sink); +} + +public sealed class SparkBufferHandler : IContentHandler +{ + public bool CanHandle(string relativePath) => + Path.GetExtension(relativePath).Equals(".bytes", StringComparison.OrdinalIgnoreCase); + + public HandleResult Handle(string relativePath, ReadOnlyMemory data, IOutputSink sink) + { + try + { + string json = SparkBuffer.ToJson(data.Span); + using var output = sink.Create(Path.ChangeExtension(relativePath, ".json")); + output.Write(System.Text.Encoding.UTF8.GetBytes(json)); + output.Complete(); + return HandleResult.Handled; + } + catch (Exception) + { + return HandleResult.Failed; + } + } +} + +public sealed class LuaHandler : IContentHandler +{ + public bool CanHandle(string relativePath) => true; + + public HandleResult Handle(string relativePath, ReadOnlyMemory data, IOutputSink sink) + { + try + { + byte[]? decrypted = LuaDecipher.Decrypt(data.Span); + if (decrypted == null || !LuaDecipher.IsValid(decrypted)) + return HandleResult.Failed; + using var output = sink.Create(Path.ChangeExtension(relativePath, ".lua")); + output.Write(decrypted); + output.Complete(); + return HandleResult.Handled; + } + catch (Exception) + { + return HandleResult.Failed; + } + } +} + +public sealed class UsmHandler : IContentHandler +{ + public bool CanHandle(string relativePath) => + relativePath.EndsWith(".usm", StringComparison.OrdinalIgnoreCase); + + public HandleResult Handle(string relativePath, ReadOnlyMemory data, IOutputSink sink) + { + try + { + var streams = CriUsmDemuxer.Demux(data.Span); + if (streams.Length == 0) + return HandleResult.Skipped; + string basePath = Path.ChangeExtension(relativePath, null) ?? relativePath; + var seen = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var stream in streams) + { + seen.TryGetValue(stream.Extension, out int index); + string name = index == 0 ? basePath + stream.Extension : $"{basePath}_{index}{stream.Extension}"; + seen[stream.Extension] = index + 1; + using var output = sink.Create(name); + output.Write(stream.Data); + output.Complete(); + } + + return HandleResult.Handled; + } + catch (Exception) + { + return HandleResult.Failed; + } + } +} + +public sealed class PckDecryptHandler : IContentHandler +{ + public bool CanHandle(string relativePath) => + relativePath.EndsWith(".pck", StringComparison.OrdinalIgnoreCase); + + public HandleResult Handle(string relativePath, ReadOnlyMemory data, IOutputSink sink) + { + try + { + var archive = PckArchive.Parse(data.Span); + byte[] plain = archive.ToPlainPck(data.Span); + using var output = sink.Create(relativePath); + output.Write(plain); + output.Complete(); + return HandleResult.Handled; + } + catch (Exception) + { + return HandleResult.Failed; + } + } +} diff --git a/BydTools.Extraction/Vfs/VfsExtractor.cs b/BydTools.Extraction/Vfs/VfsExtractor.cs new file mode 100644 index 0000000..4cfd911 --- /dev/null +++ b/BydTools.Extraction/Vfs/VfsExtractor.cs @@ -0,0 +1,227 @@ +using BydTools.Core; +using BydTools.Core.IO; +using BydTools.Extraction.Sink; +using BydTools.Formats.Usm; +using BydTools.VFS; + +namespace BydTools.Extraction.Vfs; + +public sealed class VfsExtractResult(int Extracted, int Failed) +{ + public int Extracted { get; } = Extracted; + public int Failed { get; } = Failed; +} + +public sealed class VfsExtractor +{ + private readonly ILogger _logger; + + public VfsExtractor(ILogger logger) => _logger = logger; + + public VfsExtractResult Dump( + VfsBlock block, + string outputDirectory, + int jobs, + IProgressSink? progress = null, + CancellationToken cancellationToken = default + ) + { + progress ??= NullProgress.Instance; + var info = block.Info; + _logger.Info("--- {0} ({1}) ---", block.DisplayName, (byte)info.BlockType); + + int totalFiles = 0; + var typeCounts = new Dictionary(StringComparer.OrdinalIgnoreCase); + long totalBytes = 0; + foreach (var chunk in info.Chunks) + { + totalFiles += chunk.Files.Count; + foreach (var file in chunk.Files) + { + string ext = Path.GetExtension(file.Name).ToLowerInvariant(); + if (string.IsNullOrEmpty(ext)) + ext = "(no extension)"; + typeCounts.TryGetValue(ext, out int count); + typeCounts[ext] = count + 1; + if (file.Length > 0) + totalBytes += file.Length; + } + } + + string details = string.Join(", ", typeCounts.Select(static pair => $"{pair.Value} {pair.Key}")); + _logger.Info("Found {0} files ({1})", totalFiles, details); + WriteBlockInfo(info); + _logger.Info("Extracting..."); + progress.AddTotal(totalBytes); + + var sink = new DirectoryOutputSink(outputDirectory); + IContentHandler? handler = CreateHandler(block.Descriptor?.Kind ?? ContentKind.Generic); + int extracted = 0; + int failed = 0; + var options = new ParallelOptions + { + MaxDegreeOfParallelism = Math.Max(1, jobs), + CancellationToken = cancellationToken, + }; + + Parallel.ForEach( + info.Chunks, + options, + chunk => + { + string? chunkPath = block.ResolveChunkPath(chunk); + if (chunkPath == null) + { + _logger.Verbose(" Chunk file not found: {0}", chunk.FileName); + return; + } + + foreach (var file in chunk.Files) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + byte[] data = block.ReadFile(chunkPath, file); + string relative = file.Name; + if (string.IsNullOrEmpty(relative) && block.Descriptor?.Kind == ContentKind.Video) + { + string usmName = UsmNameReader.TryGetName(data) ?? $"{file.NameHash:X16}.usm"; + relative = Path.Combine("Video", usmName); + _logger.Verbose(" Recovered USM name: {0}", relative); + } + + bool handled = false; + if (handler != null && handler.CanHandle(relative)) + { + HandleResult result = handler.Handle(relative, data, sink); + if (result == HandleResult.Handled) + handled = true; + else if (result == HandleResult.Failed) + Interlocked.Increment(ref failed); + } + + if (!handled) + { + using var output = sink.Create(relative); + output.Write(data); + output.Complete(); + } + + Interlocked.Increment(ref extracted); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Interlocked.Increment(ref failed); + _logger.Verbose(" Failed {0}: {1}", file.Name, ex.Message); + } + finally + { + progress.Advance(Math.Max(0, file.Length)); + } + } + + _logger.Verbose(" Dumped {0} file(s) from chunk {1}", chunk.Files.Count, chunk.FileName); + } + ); + + _logger.Info( + failed == 0 + ? $"Done, {extracted} files extracted." + : $"Done, {extracted} files extracted. {failed} failed." + ); + return new VfsExtractResult(extracted, failed); + } + + public void WriteDebugReport(VfsArchive archive) + { + _logger.Info("Debug: scanning all block declarations under {0}", archive.Root); + _logger.Info(""); + if (archive.Blocks.Count == 0) + { + _logger.Info("No block declarations found."); + return; + } + + foreach (var block in archive.Blocks) + { + var info = block.Info; + string typeLabel = Enum.IsDefined(info.BlockType) + ? $"{info.BlockType} ({(byte)info.BlockType})" + : $"Unknown ({(byte)info.BlockType})"; + _logger.Info( + " [{0}] groupCfgName = {1} | blockType = {2} | chunks = {3} | files = {4}", + info.GroupCfgHashName, + info.GroupCfgName, + typeLabel, + info.Chunks.Count, + info.GroupFileInfoNum + ); + WriteBlockInfo(info); + } + + _logger.Info(""); + _logger.Info("Debug: found {0} block declaration(s).", archive.Blocks.Count); + _logger.Info(""); + _logger.Info("Block type mapping (groupCfgName -> directory hash):"); + foreach (var block in archive.Blocks.OrderBy(static b => b.Info.GroupCfgName, StringComparer.Ordinal)) + _logger.Info(" {0} -> {1}", block.Info.GroupCfgName, block.Info.GroupCfgHashName); + } + + private static IContentHandler? CreateHandler(ContentKind kind) => + kind switch + { + ContentKind.Table => new SparkBufferHandler(), + ContentKind.Lua => new LuaHandler(), + ContentKind.Video => new UsmHandler(), + ContentKind.Audio => new PckDecryptHandler(), + _ => null, + }; + + private void WriteBlockInfo(VfsBlockInfo info) + { + _logger.Verbose("========== BLC INFO =========="); + _logger.Verbose("GroupCfgName : {0}", info.GroupCfgName); + _logger.Verbose("GroupCfgHash : {0}", info.GroupCfgHashName); + _logger.Verbose("Version : {0}", info.Version); + _logger.Verbose("BlockType : {0} ({1})", info.BlockType, (byte)info.BlockType); + _logger.Verbose("FileInfoNum : {0}", info.GroupFileInfoNum); + _logger.Verbose("ChunksLength : {0}", info.GroupChunksLength); + _logger.Verbose("ChunksCount : {0}", info.Chunks.Count); + _logger.Verbose("------------------------------"); + for (int i = 0; i < info.Chunks.Count; i++) + { + var chunk = info.Chunks[i]; + _logger.Verbose("Chunk #{0}:", i); + _logger.Verbose(" md5Name : {0}", chunk.Md5Name.ToHexLittleEndian()); + _logger.Verbose(" contentMD5 : {0}", chunk.ContentMd5.ToHexLittleEndian()); + _logger.Verbose(" length : {0}", chunk.Length); + _logger.Verbose(" blockType : {0} ({1})", chunk.BlockType, (byte)chunk.BlockType); + _logger.Verbose(" fileTag : {0} ({1})", chunk.FileTag, (byte)chunk.FileTag); + _logger.Verbose(" filesCount : {0}", chunk.Files.Count); + for (int j = 0; j < chunk.Files.Count; j++) + { + var file = chunk.Files[j]; + _logger.Verbose(" File #{0}:", j); + _logger.Verbose(" name : {0}", file.Name); + _logger.Verbose(" nameHash : 0x{0:X16}", file.NameHash); + _logger.Verbose(" chunkMD5 : {0}", file.ChunkMd5.ToHexLittleEndian()); + _logger.Verbose(" dataMD5 : {0}", file.DataMd5.ToHexLittleEndian()); + _logger.Verbose(" offset : {0}", file.Offset); + _logger.Verbose(" len : {0}", file.Length); + _logger.Verbose(" blockType : {0} ({1})", file.BlockType, (byte)file.BlockType); + _logger.Verbose(" useEncrypt : {0}", file.Encrypted); + if (file.Encrypted) + _logger.Verbose(" ivSeed : {0}", file.IvSeed); + _logger.Verbose(" fileTag : {0} ({1})", file.FileTag, (byte)file.FileTag); + } + + _logger.Verbose("------------------------------"); + } + + _logger.Verbose("======== END BLC INFO ========"); + } +} diff --git a/BydTools.Formats/BydTools.Formats.csproj b/BydTools.Formats/BydTools.Formats.csproj new file mode 100644 index 0000000..f95b2a3 --- /dev/null +++ b/BydTools.Formats/BydTools.Formats.csproj @@ -0,0 +1,5 @@ + + + + + diff --git a/BydTools.Formats/Lua/LuaDecipher.cs b/BydTools.Formats/Lua/LuaDecipher.cs new file mode 100644 index 0000000..b6af368 --- /dev/null +++ b/BydTools.Formats/Lua/LuaDecipher.cs @@ -0,0 +1,73 @@ +using System.Text; +using System.Text.Unicode; +using BydTools.Core.Crypto; + +namespace BydTools.Formats.Lua; + +/// Decrypts the XXTEA-wrapped Lua shipped inside the Lua VFS block. +public static class LuaDecipher +{ + private static readonly byte[] MasterKey = Encoding.UTF8.GetBytes(DeriveMasterKey()); + + public static byte[]? Decrypt(ReadOnlySpan data) + { + var text = Encoding.UTF8.GetString(data).Trim(); + byte[] cipher; + try + { + cipher = Convert.FromBase64String(text); + } + catch (FormatException) + { + return null; + } + + return XXTEA.Decrypt(cipher, MasterKey); + } + + public static bool IsValid(ReadOnlySpan data) + { + if (data.Length < 4) + return false; + if (data[0] == 0x1B && data[1] == (byte)'L' && data[2] == (byte)'u' && data[3] == (byte)'a') + return true; + if (!IsUtf8(data)) + return false; + + int sample = Math.Min(data.Length, 1000); + string text = Encoding.UTF8.GetString(data[..sample]).TrimStart('\uFEFF', '\r', '\n', ' ', '\t'); + return text.StartsWith("local ", StringComparison.Ordinal) + || text.StartsWith("function ", StringComparison.Ordinal) + || text.StartsWith("return ", StringComparison.Ordinal) + || text.StartsWith("require ", StringComparison.Ordinal) + || text.StartsWith("--", StringComparison.Ordinal) + || text.StartsWith("config ", StringComparison.Ordinal) + || text.Contains("local ", StringComparison.Ordinal) + || text.Contains("function(", StringComparison.Ordinal); + } + + private static bool IsUtf8(ReadOnlySpan data) + { + try + { + Encoding.UTF8.GetString(data); + return Utf8.IsValid(data); + } + catch (ArgumentException) + { + return false; + } + } + + private static string DeriveMasterKey() + { + string[] keys = ["cynb5", "paeky", "xmF5og", "ud35+e", "72iUy", "azWk3", "901lU", "dDfl2"]; + const string initialKey = "Assets/Beyond/InitialAssets/"; + string packed = $"{keys[1]}{keys[5]}{keys[3]}{keys[2]}=="; + byte[] data = Convert.FromBase64String(packed); + byte[] keyBytes = Encoding.UTF8.GetBytes(initialKey); + for (int i = 0; i < data.Length; i++) + data[i] = (byte)(data[i] - keyBytes[i % keyBytes.Length]); + return Encoding.UTF8.GetString(data); + } +} diff --git a/BydTools.Formats/SparkBuffer/SparkBuffer.cs b/BydTools.Formats/SparkBuffer/SparkBuffer.cs new file mode 100644 index 0000000..63eea1d --- /dev/null +++ b/BydTools.Formats/SparkBuffer/SparkBuffer.cs @@ -0,0 +1,454 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using BydTools.Core.IO; + +namespace BydTools.Formats.SparkBuffer; + +public enum SparkType : byte +{ + Bool, + Byte, + Int, + Long, + Float, + Double, + Enum, + String, + Bean, + Array, + Map, +} + +/// +/// Decodes a SparkBuffer blob to JSON. Each call owns its type table, so +/// conversions can run in parallel. +/// +public static class SparkBuffer +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + public static string GetRootName(ReadOnlySpan data) + { + var reader = new SpanReader(data); + reader.ReadInt32(); + int rootOffset = reader.ReadInt32(); + reader.Seek(rootOffset); + reader.ReadByte(); + return reader.ReadNullTerminatedUtf8(); + } + + public static string ToJson(ReadOnlySpan data) => ReadObject(data).ToJsonString(JsonOptions); + + public static JsonObject ReadObject(ReadOnlySpan data) + { + var node = ReadNode(data); + return node as JsonObject + ?? throw new InvalidDataException("SparkBuffer root is not an object."); + } + + private static JsonNode? ReadNode(ReadOnlySpan data) + { + var reader = new SpanReader(data); + var schema = new SparkSchema(); + int typeDefOffset = reader.ReadInt32(); + int rootDefOffset = reader.ReadInt32(); + int dataOffset = reader.ReadInt32(); + + reader.Seek(typeDefOffset); + schema.Read(ref reader); + + reader.Seek(rootDefOffset); + var root = ReadFieldHeader(ref reader); + + reader.Seek(dataOffset); + return root.Type switch + { + SparkType.Bean => ReadBean( + ref reader, + schema, + schema.Bean(root.TypeHash), + pointer: false + ), + SparkType.Map => ReadMap(ref reader, schema, root), + _ => throw new NotSupportedException($"Unsupported root type {root.Type}"), + }; + } + + private static Field ReadFieldHeader(ref SpanReader reader) + { + var field = new Field { Type = (SparkType)reader.ReadByte(), Name = reader.ReadNullTerminatedUtf8() }; + if (field.Type is SparkType.Enum or SparkType.Bean) + { + reader.Align(4); + field.TypeHash = reader.ReadInt32(); + } + + if (field.Type == SparkType.Map) + { + field.Type2 = (SparkType)reader.ReadByte(); + field.Type3 = (SparkType)reader.ReadByte(); + if (field.Type2 is SparkType.Enum or SparkType.Bean) + { + reader.Align(4); + field.TypeHash = reader.ReadInt32(); + } + + if (field.Type3 is SparkType.Enum or SparkType.Bean) + { + reader.Align(4); + field.TypeHash2 = reader.ReadInt32(); + } + } + + return field; + } + + private static JsonObject ReadMap(ref SpanReader reader, SparkSchema schema, Field typeDef) + { + var map = new JsonObject(); + int kvCount = reader.ReadInt32(); + if (kvCount < 0) + throw new InvalidDataException("Negative SparkBuffer map size."); + reader.Skip(checked(kvCount * 8)); + + for (int i = 0; i < kvCount; i++) + { + string key = typeDef.Type2 switch + { + SparkType.String => ReadStringOffset(ref reader), + SparkType.Int => reader.ReadInt32().ToString(), + SparkType.Long => ReadLong(ref reader).ToString(), + _ => throw new NotSupportedException($"Unsupported map key type {typeDef.Type2}"), + }; + + map[key] = typeDef.Type3 switch + { + SparkType.Bean => ReadBean(ref reader, schema, schema.Bean(typeDef.TypeHash2), pointer: true), + SparkType.String => ReadStringOffset(ref reader), + SparkType.Int => reader.ReadInt32(), + SparkType.Float => reader.ReadSingle(), + SparkType.Enum => ReadEnumName(ref reader, schema.Enum(typeDef.TypeHash2)), + SparkType.Bool => ReadBool(ref reader), + _ => throw new NotSupportedException($"Unsupported map value type {typeDef.Type3}"), + }; + } + + return map; + } + + private static JsonObject? ReadBean(ref SpanReader reader, SparkSchema schema, BeanType bean, bool pointer) + { + int? pointerOrigin = null; + if (pointer) + { + int beanOffset = reader.ReadInt32(); + if (beanOffset == -1) + return null; + pointerOrigin = reader.Position; + reader.Seek(beanOffset); + } + + var obj = new JsonObject(); + for (int fieldIndex = 0; fieldIndex < bean.Fields.Length; fieldIndex++) + { + Field beanField = bean.Fields[fieldIndex]; + int? origin = null; + if (beanField.Type == SparkType.Array) + { + int fieldOffset = reader.ReadInt32(); + if (fieldOffset == -1) + { + obj[beanField.Name] = null; + continue; + } + + origin = reader.Position; + reader.Seek(fieldOffset); + } + + switch (beanField.Type) + { + case SparkType.Array: + var array = new JsonArray(); + int itemCount = reader.ReadInt32(); + if (itemCount < 0) + throw new InvalidDataException("Negative SparkBuffer array size."); + for (int n = 0; n < itemCount; n++) + { + array.Add( + beanField.Type2 switch + { + SparkType.String => JsonValue.Create(ReadStringOffset(ref reader)), + SparkType.Bean => ReadBean( + ref reader, + schema, + schema.Bean(beanField.TypeHash), + pointer: true + ), + SparkType.Float => JsonValue.Create(reader.ReadSingle()), + SparkType.Long => JsonValue.Create(ReadLong(ref reader)), + SparkType.Int or SparkType.Enum => JsonValue.Create(reader.ReadInt32()), + SparkType.Bool => JsonValue.Create(reader.ReadBoolean()), + SparkType.Double => JsonValue.Create(ReadDouble(ref reader)), + _ => throw new NotSupportedException( + $"Unsupported array type {beanField.Type2} at offset {reader.Position}" + ), + } + ); + } + + obj[beanField.Name] = array; + break; + case SparkType.Int: + case SparkType.Enum: + obj[beanField.Name] = reader.ReadInt32(); + break; + case SparkType.Long: + obj[beanField.Name] = ReadLong(ref reader); + break; + case SparkType.Float: + obj[beanField.Name] = reader.ReadSingle(); + break; + case SparkType.Double: + obj[beanField.Name] = ReadDouble(ref reader); + break; + case SparkType.String: + obj[beanField.Name] = ReadStringOffset(ref reader); + break; + case SparkType.Bean: + obj[beanField.Name] = ReadBean( + ref reader, + schema, + schema.Bean(beanField.TypeHash), + pointer: true + ); + break; + case SparkType.Bool: + obj[beanField.Name] = reader.ReadBoolean(); + if (fieldIndex + 1 < bean.Fields.Length && bean.Fields[fieldIndex + 1].Type != SparkType.Bool) + reader.Align(4); + break; + case SparkType.Map: + int mapOffset = reader.ReadInt32(); + int mapOrigin = reader.Position; + reader.Seek(mapOffset); + obj[beanField.Name] = ReadMap(ref reader, schema, beanField); + reader.Seek(mapOrigin); + break; + default: + throw new NotSupportedException( + $"Dumping bean field type {beanField.Type} isn't supported at offset {reader.Position}" + ); + } + + if (origin is not null) + reader.Seek(origin.Value); + } + + if (pointerOrigin is not null) + reader.Seek(pointerOrigin.Value); + return obj; + } + + private static string ReadStringOffset(ref SpanReader reader) + { + int offset = reader.ReadInt32(); + if (offset == -1) + return string.Empty; + int saved = reader.Position; + reader.Seek(offset); + string value = reader.ReadNullTerminatedUtf8(); + reader.Seek(saved); + return value; + } + + private static string ReadEnumName(ref SpanReader reader, EnumType type) + { + int value = reader.ReadInt32(); + foreach (var item in type.Items) + { + if (item.Value == value) + return item.Name; + } + + return value.ToString(); + } + + private static bool ReadBool(ref SpanReader reader) + { + bool value = reader.ReadBoolean(); + reader.Align(4); + return value; + } + + private static long ReadLong(ref SpanReader reader) + { + reader.Align(8); + return reader.ReadInt64(); + } + + private static double ReadDouble(ref SpanReader reader) + { + reader.Align(8); + return reader.ReadDouble(); + } + + private struct Field + { + public string Name; + public SparkType Type; + public SparkType Type2; + public SparkType Type3; + public int TypeHash; + public int TypeHash2; + } + + private sealed class BeanType + { + public int TypeHash; + public string Name = ""; + public Field[] Fields = []; + } + + private sealed class EnumType + { + public int TypeHash; + public string Name = ""; + public EnumItem[] Items = []; + } + + private readonly struct EnumItem(string name, int value) + { + public string Name { get; } = name; + public int Value { get; } = value; + } + + private sealed class SparkSchema + { + private readonly Dictionary _beans = []; + private readonly Dictionary _enums = []; + + public BeanType Bean(int hash) => _beans[hash]; + + public EnumType Enum(int hash) => _enums[hash]; + + public void Read(ref SpanReader reader) + { + int count = reader.ReadInt32(); + while (count-- > 0) + { + var sparkType = (SparkType)reader.ReadByte(); + reader.Align(4); + switch (sparkType) + { + case SparkType.Enum: + { + var enumType = ReadEnum(ref reader); + _enums.TryAdd(enumType.TypeHash, enumType); + break; + } + case SparkType.Bean: + { + var beanType = ReadBeanType(ref reader); + _beans.TryAdd(beanType.TypeHash, beanType); + break; + } + default: + throw new InvalidDataException( + $"Invalid spark type {sparkType} in the type definition section." + ); + } + } + } + + private static EnumType ReadEnum(ref SpanReader reader) + { + var enumType = new EnumType { TypeHash = reader.ReadInt32(), Name = reader.ReadNullTerminatedUtf8() }; + reader.Align(4); + int count = reader.ReadInt32(); + if (count < 0) + throw new InvalidDataException("Negative SparkBuffer enum size."); + enumType.Items = new EnumItem[count]; + for (int i = 0; i < count; i++) + { + string name = reader.ReadNullTerminatedUtf8(); + reader.Align(4); + enumType.Items[i] = new EnumItem(name, reader.ReadInt32()); + } + + return enumType; + } + + private static BeanType ReadBeanType(ref SpanReader reader) + { + var bean = new BeanType { TypeHash = reader.ReadInt32(), Name = reader.ReadNullTerminatedUtf8() }; + reader.Align(4); + int fieldCount = reader.ReadInt32(); + if (fieldCount < 0) + throw new InvalidDataException("Negative SparkBuffer field count."); + bean.Fields = new Field[fieldCount]; + for (int i = 0; i < fieldCount; i++) + { + var field = new Field + { + Name = reader.ReadNullTerminatedUtf8(), + Type = (SparkType)reader.ReadByte(), + }; + switch (field.Type) + { + case SparkType.Bool: + case SparkType.Byte: + case SparkType.Int: + case SparkType.Long: + case SparkType.Float: + case SparkType.Double: + case SparkType.String: + break; + case SparkType.Enum: + case SparkType.Bean: + reader.Align(4); + field.TypeHash = reader.ReadInt32(); + break; + case SparkType.Array: + field.Type2 = (SparkType)reader.ReadByte(); + if (field.Type2 is SparkType.Enum or SparkType.Bean) + { + reader.Align(4); + field.TypeHash = reader.ReadInt32(); + } + + break; + case SparkType.Map: + field.Type2 = (SparkType)reader.ReadByte(); + field.Type3 = (SparkType)reader.ReadByte(); + if (field.Type2 is SparkType.Enum or SparkType.Bean) + { + reader.Align(4); + field.TypeHash = reader.ReadInt32(); + } + + if (field.Type3 is SparkType.Enum or SparkType.Bean) + { + reader.Align(4); + field.TypeHash2 = reader.ReadInt32(); + } + + break; + default: + throw new InvalidDataException( + $"Unsupported bean field type {field.Type} at offset {reader.Position}" + ); + } + + bean.Fields[i] = field; + } + + return bean; + } + } +} diff --git a/BydTools.Formats/Usm/CriUsmDemuxer.cs b/BydTools.Formats/Usm/CriUsmDemuxer.cs new file mode 100644 index 0000000..505a4f8 --- /dev/null +++ b/BydTools.Formats/Usm/CriUsmDemuxer.cs @@ -0,0 +1,146 @@ +using System.Buffers.Binary; + +namespace BydTools.Formats.Usm; + +public readonly record struct UsmStream(string Extension, byte[] Data); + +/// +/// Demultiplexes a CRI USM container into elementary video and audio streams. +/// Names are not assigned here; the caller numbers streams of the same extension +/// in the order they first appear, and overwrites on a later run. +/// +public static class CriUsmDemuxer +{ + private static readonly byte[] Crid = "CRID"u8.ToArray(); + private static readonly byte[] Sfv = "@SFV"u8.ToArray(); + private static readonly byte[] Sfa = "@SFA"u8.ToArray(); + private static readonly byte[] Sbt = "@SBT"u8.ToArray(); + private static readonly byte[] Cue = "@CUE"u8.ToArray(); + private static readonly byte[] Alp = "@ALP"u8.ToArray(); + private static readonly byte[] Aixf = "AIXF"u8.ToArray(); + private static readonly byte[] Hca = [0x48, 0x43, 0x41, 0x00]; + + private static readonly byte[] HeaderEnd = + [ + 0x23, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x20, 0x45, 0x4E, 0x44, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, + 0x3D, 0x00, + ]; + + private static readonly byte[] MetadataEnd = + [ + 0x23, 0x4D, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x20, 0x45, 0x4E, 0x44, 0x20, 0x20, + 0x20, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, + 0x3D, 0x00, + ]; + + private static readonly byte[] ContentsEnd = + [ + 0x23, 0x43, 0x4F, 0x4E, 0x54, 0x45, 0x4E, 0x54, 0x53, 0x20, 0x45, 0x4E, 0x44, 0x20, 0x20, + 0x20, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, + 0x3D, 0x00, + ]; + + public static UsmStream[] Demux(ReadOnlySpan usm) + { + int start = usm.IndexOf(Crid); + if (start < 0) + return []; + + var order = new List(); + var streams = new Dictionary(); + int pos = start; + try + { + while (pos + 8 <= usm.Length) + { + var sig = usm.Slice(pos, 4); + if (!IsKnown(sig)) + break; + + uint blockSize = BinaryPrimitives.ReadUInt32BigEndian(usm.Slice(pos + 4, 4)); + if (blockSize > int.MaxValue - 8 || pos > usm.Length - 8 - (int)blockSize) + break; + + bool isAudio = sig.SequenceEqual(Sfa); + bool isVideo = sig.SequenceEqual(Sfv); + if (isAudio || isVideo) + { + byte streamId = isAudio && pos + 0xC < usm.Length ? usm[pos + 0xC] : (byte)0; + uint blockId = BinaryPrimitives.ReadUInt32LittleEndian(sig); + uint key = streamId | blockId; + if (!streams.TryGetValue(key, out var ms)) + { + ms = new MemoryStream(); + streams[key] = ms; + order.Add(key); + } + + int headerSkip = BinaryPrimitives.ReadUInt16BigEndian(usm.Slice(pos + 8, 2)); + int footerSkip = BinaryPrimitives.ReadUInt16BigEndian(usm.Slice(pos + 0xA, 2)); + int cut = (int)blockSize - headerSkip - footerSkip; + int dataStart = pos + 8 + headerSkip; + if (cut > 0 && dataStart >= 0 && dataStart <= usm.Length - cut) + ms.Write(usm.Slice(dataStart, cut)); + } + + pos += 8 + (int)blockSize; + } + + var output = new List(order.Count); + foreach (uint key in order) + { + byte[] elementary = streams[key].ToArray(); + var span = elementary.AsSpan(); + int headerEnd = span.IndexOf(HeaderEnd); + int metadataEnd = span.IndexOf(MetadataEnd); + int dataStart; + if (headerEnd >= 0 || metadataEnd >= 0) + dataStart = Math.Max(headerEnd, metadataEnd) + MetadataEnd.Length; + else + dataStart = 0; + + if ((uint)dataStart > (uint)span.Length) + continue; + + int contentEnd = span[dataStart..].IndexOf(ContentsEnd); + int dataLength = contentEnd >= 0 ? contentEnd : span.Length - dataStart; + if (dataLength <= 0) + continue; + + var payload = span.Slice(dataStart, dataLength).ToArray(); + output.Add(new UsmStream(DetectExtension(key, payload), payload)); + } + + return output.ToArray(); + } + finally + { + foreach (var ms in streams.Values) + ms.Dispose(); + } + } + + private static bool IsKnown(ReadOnlySpan sig) => + sig.SequenceEqual(Crid) + || sig.SequenceEqual(Sfv) + || sig.SequenceEqual(Sfa) + || sig.SequenceEqual(Sbt) + || sig.SequenceEqual(Cue) + || sig.SequenceEqual(Alp); + + private static string DetectExtension(uint streamKey, ReadOnlySpan payload) + { + Span block = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(block, streamKey & 0xFFFFFFF0); + if (!block.SequenceEqual(Sfa)) + return ".m2v"; + if (payload.Length >= 4 && payload[..4].SequenceEqual(Aixf)) + return ".aix"; + if (payload.Length >= 1 && payload[0] == 0x80) + return ".adx"; + if (payload.Length >= 4 && payload[..4].SequenceEqual(Hca)) + return ".hca"; + return ".bin"; + } +} diff --git a/BydTools.Formats/Usm/UsmNameReader.cs b/BydTools.Formats/Usm/UsmNameReader.cs new file mode 100644 index 0000000..747afc7 --- /dev/null +++ b/BydTools.Formats/Usm/UsmNameReader.cs @@ -0,0 +1,51 @@ +using System.Buffers.Binary; + +namespace BydTools.Formats.Usm; + +/// +/// Recovers a file name from a CRI USM header when the VFS entry name is empty. +/// +public static class UsmNameReader +{ + private const int CridSignature = 0x44495243; + private static readonly byte[] DirStreamTag = "CRIUSF_DIR_STREAM"u8.ToArray(); + + public static string? TryGetName(ReadOnlySpan usm) + { + if (usm.Length < 12) + return null; + if (BinaryPrimitives.ReadInt32LittleEndian(usm) != CridSignature) + return null; + + int blockSize = BinaryPrimitives.ReadInt32BigEndian(usm[4..]); + short payloadOffset = BinaryPrimitives.ReadInt16BigEndian(usm[8..]); + short paddingSize = BinaryPrimitives.ReadInt16BigEndian(usm[10..]); + int dataLength = blockSize - paddingSize - payloadOffset; + if (dataLength <= 0 || 12 + dataLength > usm.Length) + return null; + + var buff = usm.Slice(12, dataLength); + int dirStreamIndex = buff.IndexOf(DirStreamTag); + if (dirStreamIndex < 0) + return null; + + int offset = 18; + while (dirStreamIndex + offset < buff.Length) + { + var tail = buff[(dirStreamIndex + offset)..]; + int end = tail.IndexOf((byte)0); + if (end < 0) + end = tail.Length; + string str = System.Text.Encoding.UTF8.GetString(tail[..end]); + if (str.EndsWith(".usm", StringComparison.OrdinalIgnoreCase)) + { + string? root = Path.GetPathRoot(str); + return string.IsNullOrEmpty(root) ? str : str[root.Length..]; + } + + offset += str.Length + 1; + } + + return null; + } +} diff --git a/BydTools.PCK/BydTools.PCK.csproj b/BydTools.PCK/BydTools.PCK.csproj deleted file mode 100644 index 5719d8f..0000000 --- a/BydTools.PCK/BydTools.PCK.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - diff --git a/BydTools.PCK/PckConverter.cs b/BydTools.PCK/PckConverter.cs deleted file mode 100644 index d10faae..0000000 --- a/BydTools.PCK/PckConverter.cs +++ /dev/null @@ -1,282 +0,0 @@ -using BydTools.Utils; -using BydTools.Wwise; - -namespace BydTools.PCK; - -/// -/// Extracts audio from PCK archives and converts WEM to WAV via vgmstream, -/// with optional filename mapping via . -/// -public class PckConverter -{ - private readonly ILogger _logger; - private readonly IWemConverter _wemConverter; - private readonly PckExtractor _extractor; - - public PckConverter(ILogger logger, IWemConverter wemConverter) - : this(logger, wemConverter, new PckExtractor(logger)) { } - - public PckConverter(ILogger logger, IWemConverter wemConverter, PckExtractor extractor) - { - _logger = logger; - _wemConverter = wemConverter; - _extractor = extractor; - } - - /// - /// Extracts and converts audio files from a PCK archive. - /// - /// raw — extract WEM/BNK/PLG without conversion. - /// wav — convert WEM to WAV via vgmstream; failures kept as WEM. - /// - /// - public void ExtractAndConvert( - string pckPath, - string outputDir, - string mode = "wav", - PckMapper? mapper = null - ) - { - if (!File.Exists(pckPath)) - throw new FileNotFoundException("PCK file not found", pckPath); - - Directory.CreateDirectory(outputDir); - - string normalizedMode = mode.ToLowerInvariant(); - if (normalizedMode is not ("raw" or "wav")) - throw new ArgumentException( - $"Invalid mode '{mode}'. Must be 'raw' or 'wav'.", - nameof(mode) - ); - - _logger.Info($"Input: {pckPath}"); - _logger.Info($"Output: {outputDir}"); - _logger.Info($"Mode: {normalizedMode}"); - - if (normalizedMode == "raw") - { - _extractor.ExtractFiles(pckPath, outputDir, mapper); - return; - } - - ConvertToWav(pckPath, outputDir, mapper); - } - - private void ConvertToWav(string pckPath, string outputDir, PckMapper? mapper) - { - string tempDir = Path.Combine(outputDir, ".pck_extract"); - Directory.CreateDirectory(tempDir); - - try - { - using var fileStream = File.OpenRead(pckPath); - var parser = new PckParser(fileStream); - var content = parser.Parse(); - - _logger.Info($"Parsed {content.Entries.Count} entries"); - - var wemJobs = new List(); - var plgJobs = new List(); - - foreach (var entry in content.Entries) - { - byte[] fileData = parser.GetFileData(entry); - if (fileData.Length < 4) - continue; - - ReadOnlySpan magic = fileData.AsSpan(0, 4); - - if (magic.SequenceEqual("BKHD"u8)) - { - CollectBnkWems(fileData, entry, tempDir, mapper, wemJobs); - } - else if (magic.SequenceEqual("RIFF"u8) || magic.SequenceEqual("RIFX"u8)) - { - string key = entry.FileId.ToString(); - string tempPath = Path.Combine(tempDir, $"{key}.wem"); - File.WriteAllBytes(tempPath, fileData); - - WemCodec codec = WemFormatReader.DetectCodec(fileData); - string outName = PckExtractor.ResolveOutputName( - entry.FileId, - mapper, - ".wav", - content.Languages, - entry.LanguageId - ); - wemJobs.Add(new ConvertJob(tempPath, outName, codec)); - } - else if (magic.SequenceEqual("PLUG"u8)) - { - string key = entry.FileId.ToString(); - string tempPath = Path.Combine(tempDir, $"{key}.plg"); - File.WriteAllBytes(tempPath, fileData); - - string outName = PckExtractor.ResolveOutputName( - entry.FileId, - mapper, - ".plg", - content.Languages, - entry.LanguageId - ); - plgJobs.Add(new CopyJob(tempPath, outName)); - } - } - - _logger.Info($"Found {wemJobs.Count} WEM, {plgJobs.Count} PLG files"); - - var codecGroups = wemJobs.GroupBy(j => j.Codec).OrderByDescending(g => g.Count()); - foreach (var g in codecGroups) - _logger.Verbose($" {WemFormatReader.GetCodecName(g.Key)}: {g.Count()}"); - - if (wemJobs.Count > 0) - _logger.Verbose("Converting WEM to WAV..."); - - int converted = 0, - failed = 0; - int done = 0; - int total = wemJobs.Count; - int lastPercent = -1; - - Parallel.ForEach( - wemJobs, - new ParallelOptions - { - MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount), - }, - job => - { - string finalPath = Path.Combine(outputDir, job.OutputName); - EnsureDirectory(finalPath); - - try - { - _wemConverter.Convert(job.TempPath, finalPath); - Interlocked.Increment(ref converted); - } - catch (Exception ex) - { - string wemFallback = Path.ChangeExtension(finalPath, ".wem"); - EnsureDirectory(wemFallback); - try - { - File.Copy(job.TempPath, wemFallback, overwrite: true); - } - catch { } - - _logger.Verbose( - $"[{WemFormatReader.GetCodecName(job.Codec)}] " - + $"{Path.GetFileName(job.TempPath)}: {ex.Message}" - ); - Interlocked.Increment(ref failed); - } - - int current = Interlocked.Increment(ref done); - int percent = current * 100 / total; - if (percent != lastPercent && percent % 10 == 0) - { - lastPercent = percent; - _logger.Verbose($" Progress: {current}/{total} ({percent}%)"); - } - } - ); - - if (plgJobs.Count > 0) - { - _logger.Info("Copying PLG files..."); - foreach (var job in plgJobs) - { - string finalPath = Path.Combine(outputDir, job.OutputName); - EnsureDirectory(finalPath); - File.Copy(job.TempPath, finalPath, overwrite: true); - } - } - - _logger.Info($"Done: {converted} WAV, {failed} failed, {plgJobs.Count} PLG"); - } - finally - { - CleanupTempDir(tempDir); - } - } - - private static void CollectBnkWems( - byte[] bnkData, - PckFileEntry bankEntry, - string tempDir, - PckMapper? mapper, - List wemJobs - ) - { - var wemEntries = BnkParser.Parse(bnkData); - - foreach (var wem in wemEntries) - { - string key = $"{bankEntry.FileId}_{wem.Id}"; - string tempPath = Path.Combine(tempDir, $"{key}.wem"); - - using var fs = File.Create(tempPath); - fs.Write(bnkData, (int)wem.Offset, (int)wem.Size); - - WemCodec codec = WemCodec.Unknown; - if (wem.Size >= 22) - codec = WemFormatReader.DetectCodec(bnkData.AsSpan((int)wem.Offset, (int)wem.Size)); - - string outName = PckExtractor.ResolveBnkWemName( - bankEntry.FileId, - wem.Id, - mapper, - ".wav" - ); - wemJobs.Add(new ConvertJob(tempPath, outName, codec)); - } - } - - private static void CleanupTempDir(string tempDir) - { - if (!Directory.Exists(tempDir)) - return; - - for (int attempt = 0; attempt < 3; attempt++) - { - try - { - Directory.Delete(tempDir, recursive: true); - return; - } - catch (IOException) when (attempt < 2) - { - Thread.Sleep(500); - } - catch - { - break; - } - } - - try - { - foreach (var file in Directory.EnumerateFiles(tempDir)) - { - try - { - File.Delete(file); - } - catch { } - } - Directory.Delete(tempDir, recursive: true); - } - catch { } - } - - private static void EnsureDirectory(string filePath) - { - string? dir = Path.GetDirectoryName(filePath); - if (dir != null) - Directory.CreateDirectory(dir); - } - - private record ConvertJob(string TempPath, string OutputName, WemCodec Codec); - - private record CopyJob(string TempPath, string OutputName); -} diff --git a/BydTools.PCK/PckDefine.cs b/BydTools.PCK/PckDefine.cs deleted file mode 100644 index bbd8908..0000000 --- a/BydTools.PCK/PckDefine.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace BydTools.PCK; - -/// -/// Sector type in a PCK (AKPK) archive header. -/// -public enum PckSectorType -{ - Bank, - Sound, - External, -} - -/// -/// Language entry parsed from a PCK header. -/// -public record PckLanguage(uint Id, string Name); - -/// -/// A file entry parsed from a PCK archive. -/// -/// Wwise file/sound ID (uint32 or uint64 for externals). -/// File data size in bytes. -/// Absolute byte offset within the PCK file. -/// Associated language ID (0 = SFX / language-independent). -/// Which sector this entry belongs to. -public record PckFileEntry( - ulong FileId, - uint Size, - long Offset, - uint LanguageId, - PckSectorType SectorType -); - -/// -/// Complete parsed result of a PCK archive header. -/// -public record PckContent(List Languages, List Entries); diff --git a/BydTools.PCK/PckExtractor.cs b/BydTools.PCK/PckExtractor.cs deleted file mode 100644 index c9b9746..0000000 --- a/BydTools.PCK/PckExtractor.cs +++ /dev/null @@ -1,186 +0,0 @@ -using BydTools.Utils; -using BydTools.Wwise; - -namespace BydTools.PCK; - -/// -/// Extracts audio files from PCK archives, optionally mapping numeric IDs -/// to human-readable paths via . -/// -public class PckExtractor -{ - private readonly ILogger _logger; - - public PckExtractor(ILogger logger) - { - _logger = logger; - } - - /// - /// Extracts all files from a PCK archive to . - /// BNK entries are expanded to their embedded WEM files. - /// - public void ExtractFiles( - string pckPath, - string outputDir, - PckMapper? mapper = null, - bool extractWem = true, - bool extractBnk = true, - bool extractPlg = true - ) - { - if (!File.Exists(pckPath)) - throw new FileNotFoundException("PCK file not found", pckPath); - - Directory.CreateDirectory(outputDir); - - using var fileStream = File.OpenRead(pckPath); - var parser = new PckParser(fileStream); - var content = parser.Parse(); - - _logger.Info( - $"Parsed {content.Entries.Count} entries, {content.Languages.Count} languages" - ); - - int savedCount = 0; - - foreach (var entry in content.Entries) - { - byte[] fileData = parser.GetFileData(entry); - if (fileData.Length < 4) - continue; - - ReadOnlySpan magic = fileData.AsSpan(0, 4); - - if (magic.SequenceEqual("BKHD"u8)) - { - if (!extractBnk) - continue; - - savedCount += ExtractBnkWems(fileData, entry, outputDir, mapper, content.Languages); - } - else if (magic.SequenceEqual("RIFF"u8) || magic.SequenceEqual("RIFX"u8)) - { - if (!extractWem) - continue; - - string name = ResolveOutputName( - entry.FileId, - mapper, - ".wem", - content.Languages, - entry.LanguageId - ); - SaveFile(outputDir, name, fileData); - savedCount++; - } - else if (magic.SequenceEqual("PLUG"u8)) - { - if (!extractPlg) - continue; - - string name = ResolveOutputName( - entry.FileId, - mapper, - ".plg", - content.Languages, - entry.LanguageId - ); - SaveFile(outputDir, name, fileData); - savedCount++; - } - } - - _logger.Info($"Extracted {savedCount} files"); - } - - private int ExtractBnkWems( - byte[] bnkData, - PckFileEntry bankEntry, - string outputDir, - PckMapper? mapper, - List languages - ) - { - var wemEntries = BnkParser.Parse(bnkData); - if (wemEntries.Count == 0) - return 0; - - int count = 0; - foreach (var wem in wemEntries) - { - byte[] wemData = new byte[wem.Size]; - Array.Copy(bnkData, wem.Offset, wemData, 0, wem.Size); - - string name = ResolveBnkWemName(bankEntry.FileId, wem.Id, mapper, ".wem"); - SaveFile(outputDir, name, wemData); - count++; - } - - return count; - } - - internal static string ResolveOutputName( - ulong fileId, - PckMapper? mapper, - string extension, - List? languages = null, - uint languageId = 0 - ) - { - if (TryResolveMapped(fileId.ToString(), mapper, extension, out var resolved)) - return resolved; - - if (languageId != 0 && languages != null) - { - var lang = languages.Find(l => l.Id == languageId); - if (lang != null) - return Path.Combine("unmapped", lang.Name, $"{fileId}{extension}"); - } - - return Path.Combine("unmapped", $"{fileId}{extension}"); - } - - internal static string ResolveBnkWemName( - ulong bankFileId, - uint wemId, - PckMapper? mapper, - string extension - ) - { - if (TryResolveMapped(wemId.ToString(), mapper, extension, out var resolved)) - return resolved; - - return Path.Combine("unmapped", $"{bankFileId}_{wemId}{extension}"); - } - - private static bool TryResolveMapped( - string key, - PckMapper? mapper, - string extension, - out string result - ) - { - if (mapper != null) - { - string? mapped = mapper.GetMappedPath(key); - if (mapped != null) - { - result = Path.ChangeExtension(mapped, extension); - return true; - } - } - - result = ""; - return false; - } - - private static void SaveFile(string baseDir, string relativePath, byte[] data) - { - string fullPath = Path.Combine(baseDir, relativePath); - string? dir = Path.GetDirectoryName(fullPath); - if (dir != null) - Directory.CreateDirectory(dir); - File.WriteAllBytes(fullPath, data); - } -} diff --git a/BydTools.PCK/PckMapper.cs b/BydTools.PCK/PckMapper.cs deleted file mode 100644 index 520aa89..0000000 --- a/BydTools.PCK/PckMapper.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System.Text; -using System.Text.Json; - -namespace BydTools.PCK; - -/// -/// Resolves Wwise file IDs to human-readable paths -/// using an AudioDialog JSON file. -/// -/// Expected format: { "id": { "path": "v1d0/.../file.wem", ... }, ... } -/// -/// -/// PCK file IDs are FNV-1a 64-bit hashes of {soundType}/{language}/{path} (lowercased). -/// This mapper internally tries known prefixes voice, music, and sfx -/// for each AudioDialog row and records all resulting hashes as lookup keys. -/// -/// -public class PckMapper -{ - private const ulong FnvOffset = 0xcbf29ce484222325; - private const ulong FnvPrime = 0x100000001b3; - private static readonly string[] KnownSoundTypes = ["voice", "music", "sfx"]; - - private readonly Dictionary _idToPath; - - public int Count => _idToPath.Count; - - public PckMapper(string jsonPath, string language) - { - if (!File.Exists(jsonPath)) - throw new FileNotFoundException("JSON mapping file not found", jsonPath); - - using var stream = File.OpenRead(jsonPath); - _idToPath = ParseAudioDialog(stream, language); - } - - public PckMapper(Stream stream, string language) - { - _idToPath = ParseAudioDialog(stream, language); - } - - /// - /// Resolves a Wwise file ID to an output path. - /// Returns null if the ID has no mapping. - /// - public string? GetMappedPath(string fileId) - { - if (_idToPath.TryGetValue(fileId, out string? path)) - return path; - - if (ulong.TryParse(fileId, out ulong id64)) - { - uint id32 = (uint)(id64 & 0xFFFFFFFF); - if (id32 != id64 && _idToPath.TryGetValue(id32.ToString(), out path)) - return path; - } - - return null; - } - - internal static ulong Fnv1a64(ReadOnlySpan data) - { - ulong hash = FnvOffset; - foreach (byte b in data) - { - hash = unchecked(hash * FnvPrime); - hash ^= b; - } - return hash; - } - - private static Dictionary ParseAudioDialog(Stream stream, string language) - { - using var doc = JsonDocument.Parse(stream); - var root = doc.RootElement; - - if (root.ValueKind != JsonValueKind.Object) - throw new InvalidDataException("JSON root must be an object (id -> row)"); - - string lang = language.ToLowerInvariant(); - var map = new Dictionary(root.GetPropertyCount() * KnownSoundTypes.Length); - - foreach (var entry in root.EnumerateObject()) - { - if (entry.Value.ValueKind != JsonValueKind.Object) - continue; - - if (!entry.Value.TryGetProperty("path", out var pathElement)) - continue; - - string? rawPath = pathElement.GetString(); - if (string.IsNullOrWhiteSpace(rawPath)) - continue; - - string normalizedPath = rawPath.Replace('/', '\\').Trim(); - foreach (string soundType in KnownSoundTypes) - { - string hashInput = $"{soundType}/{language}/{rawPath}" - .Replace('\\', '/') - .ToLowerInvariant(); - - ulong hash = Fnv1a64(Encoding.UTF8.GetBytes(hashInput)); - string hashKey = hash.ToString(); - - string outputPath = Path.Combine(soundType, lang, normalizedPath); - if (outputPath.EndsWith(".wem", StringComparison.OrdinalIgnoreCase)) - outputPath = outputPath[..^4]; - - map.TryAdd(hashKey, outputPath); - } - } - - return map; - } -} diff --git a/BydTools.PCK/PckParser.cs b/BydTools.PCK/PckParser.cs deleted file mode 100644 index 231ea68..0000000 --- a/BydTools.PCK/PckParser.cs +++ /dev/null @@ -1,331 +0,0 @@ -using System.Buffers.Binary; -using System.Text; - -namespace BydTools.PCK; - -/// -/// PCK (AKPK) archive parser with VFS encryption support. -/// Parses the header into structured sectors (languages, banks, sounds, externals) -/// and provides decrypted file data extraction. -/// -public class PckParser -{ - private const uint AKPK_MAGIC = 0x4B504B41; // "AKPK" little-endian - private const uint CONST_M = 0x04E11C23; - private const uint CONST_X = 0x9C5A0B29; - - private readonly Stream _file; - private bool _isVfsEncrypted; - - public PckParser(Stream fileStream) - { - _file = fileStream ?? throw new ArgumentNullException(nameof(fileStream)); - } - - /// - /// Whether the underlying PCK file uses VFS encryption. - /// Only valid after has been called. - /// - public bool IsVfsEncrypted => _isVfsEncrypted; - - /// - /// XOR-deciphers data in-place using a counter-based key derived from . - /// - public static void DecipherInplace(Span data, uint seed, int size, int offsetToFileStart) - { - if (size == 0) - return; - - static uint GenerateKey(uint counter) - { - uint val = ((counter & 0xFF) ^ CONST_X) * CONST_M & 0xFFFFFFFF; - val = (val ^ ((counter >> 8) & 0xFF)) * CONST_M & 0xFFFFFFFF; - val = (val ^ ((counter >> 16) & 0xFF)) * CONST_M & 0xFFFFFFFF; - val = (val ^ ((counter >> 24) & 0xFF)) * CONST_M & 0xFFFFFFFF; - return val; - } - - int pos = 0; - uint baseCounter = seed + (uint)(offsetToFileStart >> 2); - int alignedOffset = offsetToFileStart & 0b11; - - // Head: handle misaligned beginning - if (alignedOffset > 0) - { - uint key = GenerateKey(baseCounter); - Span keyBytes = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(keyBytes, key); - int bytesLeading = Math.Min(4 - alignedOffset, size); - for (int i = 0; i < bytesLeading; i++) - data[pos++] ^= keyBytes[alignedOffset + i]; - baseCounter++; - } - - // Body: aligned 4-byte blocks - int alignedSize = (size - pos) & ~0b11; - int numBlocks = alignedSize / 4; - Span bodyKeyBytes = stackalloc byte[4]; - for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) - { - uint key = GenerateKey(baseCounter + (uint)blockIdx); - BinaryPrimitives.WriteUInt32LittleEndian(bodyKeyBytes, key); - for (int i = 0; i < 4; i++) - data[pos + i] ^= bodyKeyBytes[i]; - pos += 4; - } - - // Tail: remaining bytes - if (pos < size) - { - uint key = GenerateKey(baseCounter + (uint)numBlocks); - Span keyBytes = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(keyBytes, key); - int bytesRemaining = size - pos; - for (int i = 0; i < bytesRemaining; i++) - data[pos + i] ^= keyBytes[i]; - } - } - - /// - /// Parses the PCK header into structured content (languages + file entries). - /// - public PckContent Parse() - { - byte[] header = ReadDecryptedHeader(); - using var ms = new MemoryStream(header); - using var reader = new BinaryReader(ms); - - uint flag = reader.ReadUInt32(); // endianness: 1 = LE, 0x01000000 = BE - bool bigEndian = flag == 0x01000000; - if (bigEndian) - throw new NotSupportedException("Big-endian PCK files are not supported"); - - uint languagesSectorSize = reader.ReadUInt32(); - uint banksSectorSize = reader.ReadUInt32(); - uint soundsSectorSize = reader.ReadUInt32(); - - // Externals sector exists when the three known sectors + overhead don't fill the header - uint externalsSectorSize = 0; - uint overhead = 4u + 4u + 4u + 4u; // flag + 3 sector-size fields - if ( - languagesSectorSize + banksSectorSize + soundsSectorSize + overhead - < (uint)header.Length - ) - externalsSectorSize = reader.ReadUInt32(); - - var languages = ParseLanguages(reader, languagesSectorSize); - - var entries = new List(); - ParseSector(reader, banksSectorSize, PckSectorType.Bank, entries); - ParseSector(reader, soundsSectorSize, PckSectorType.Sound, entries); - if (externalsSectorSize > 0) - ParseSector(reader, externalsSectorSize, PckSectorType.External, entries); - - return new PckContent(languages, entries); - } - - /// - /// Reads and (if VFS-encrypted) decrypts the raw file data for a given entry. - /// - public byte[] GetFileData(PckFileEntry entry) - { - _file.Seek(entry.Offset, SeekOrigin.Begin); - byte[] data = new byte[entry.Size]; - _file.ReadExactly(data); - - if (_isVfsEncrypted) - DecipherInplace(data, (uint)entry.FileId, data.Length, 0); - - return data; - } - - /// - /// Returns full PCK bytes with VFS-layer encryption removed. - /// If the source is already plain AKPK, returns the original bytes. - /// - public byte[] GetDecryptedPckBytes() - { - _file.Seek(0, SeekOrigin.Begin); - byte[] pckBytes = new byte[_file.Length]; - _file.ReadExactly(pckBytes); - - var content = Parse(); - if (!_isVfsEncrypted) - return pckBytes; - - byte[] decryptedHeader = ReadDecryptedHeader(); - if (pckBytes.Length < 8 + decryptedHeader.Length) - throw new InvalidDataException("PCK buffer is too small for decrypted header"); - - BinaryPrimitives.WriteUInt32LittleEndian(pckBytes.AsSpan(0, 4), AKPK_MAGIC); - decryptedHeader.CopyTo(pckBytes.AsSpan(8, decryptedHeader.Length)); - - foreach (var entry in content.Entries) - { - if (entry.Offset < 0) - continue; - - if (entry.Size > int.MaxValue || entry.Offset > int.MaxValue) - continue; - - int offset = (int)entry.Offset; - int size = (int)entry.Size; - if (size == 0 || offset > pckBytes.Length - size) - continue; - - DecipherInplace(pckBytes.AsSpan(offset, size), (uint)entry.FileId, size, 0); - } - - return pckBytes; - } - - // ── private helpers ────────────────────────────────────────────── - - /// - /// Reads the PCK header from the stream, decrypting it if VFS-encrypted. - /// Returns header content starting from the flag field. - /// - private byte[] ReadDecryptedHeader() - { - _file.Seek(0, SeekOrigin.Begin); - - Span prefix = stackalloc byte[8]; - _file.ReadExactly(prefix); - uint magic = BinaryPrimitives.ReadUInt32LittleEndian(prefix); - uint headerSize = BinaryPrimitives.ReadUInt32LittleEndian(prefix[4..]); - - if (headerSize < 16) - throw new InvalidDataException($"Header size too small: {headerSize}"); - - byte[] headerContent = new byte[headerSize]; - _file.ReadExactly(headerContent); - - if (magic == AKPK_MAGIC) - { - _isVfsEncrypted = false; - return headerContent; - } - - // VFS encrypted: first 4 bytes of headerContent are the (encrypted) flag — skip them. - // Decrypt everything after the flag. - _isVfsEncrypted = true; - byte[] decryptedPayload = new byte[headerSize - 4]; - Array.Copy(headerContent, 4, decryptedPayload, 0, decryptedPayload.Length); - DecipherInplace(decryptedPayload, headerSize, decryptedPayload.Length, 0); - - // Reconstruct: flag(1, LE) + decrypted sector data - byte[] result = new byte[4 + decryptedPayload.Length]; - BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(0, 4), 1); - decryptedPayload.CopyTo(result.AsSpan(4)); - return result; - } - - private static List ParseLanguages(BinaryReader reader, uint sectorSize) - { - if (sectorSize == 0) - return []; - - long sectorStart = reader.BaseStream.Position; - uint count = reader.ReadUInt32(); - var languages = new List((int)count); - - for (int i = 0; i < count; i++) - { - uint nameOffset = reader.ReadUInt32(); - uint langId = reader.ReadUInt32(); - - long savedPos = reader.BaseStream.Position; - reader.BaseStream.Position = sectorStart + nameOffset; - - // Detect encoding: if either of the first 2 bytes is 0x00 → UTF-16LE, else UTF-8 - byte b1 = reader.ReadByte(); - byte b2 = reader.ReadByte(); - reader.BaseStream.Position = sectorStart + nameOffset; - - string name; - if (b1 == 0 || b2 == 0) - { - byte[] raw = reader.ReadBytes(0x20); - name = Encoding.Unicode.GetString(raw); - } - else - { - byte[] raw = reader.ReadBytes(0x10); - name = Encoding.UTF8.GetString(raw); - } - int nullIdx = name.IndexOf('\0'); - if (nullIdx >= 0) - name = name[..nullIdx]; - - languages.Add(new PckLanguage(langId, name)); - reader.BaseStream.Position = savedPos; - } - - reader.BaseStream.Position = sectorStart + sectorSize; - return languages; - } - - /// - /// Parses one file-entry sector (banks, sounds, or externals). - /// Entry size is auto-detected: 20 bytes (normal) or 24 bytes (alt mode). - /// - private static void ParseSector( - BinaryReader reader, - uint sectorSize, - PckSectorType sectorType, - List entries - ) - { - if (sectorSize == 0) - return; - - long sectorStart = reader.BaseStream.Position; - uint fileCount = reader.ReadUInt32(); - if (fileCount == 0) - { - reader.BaseStream.Position = sectorStart + sectorSize; - return; - } - - uint entrySize = (sectorSize - 4) / fileCount; - bool altMode = entrySize >= 0x18; // 24-byte entries - - for (int i = 0; i < fileCount; i++) - { - ulong fileId; - if (altMode && sectorType == PckSectorType.External) - { - // 64-bit file ID: low word first, high word second (LE) - uint idLow = reader.ReadUInt32(); - uint idHigh = reader.ReadUInt32(); - fileId = idLow | ((ulong)idHigh << 32); - } - else - { - fileId = reader.ReadUInt32(); - } - - uint blockSize = reader.ReadUInt32(); - - uint size; - if (altMode && sectorType != PckSectorType.External) - { - // 64-bit size for alt-mode non-external entries - size = (uint)reader.ReadInt64(); - } - else - { - size = reader.ReadUInt32(); - } - - uint rawOffset = reader.ReadUInt32(); - uint languageId = reader.ReadUInt32(); - - long offset = blockSize != 0 ? (long)rawOffset * blockSize : rawOffset; - - entries.Add(new PckFileEntry(fileId, size, offset, languageId, sectorType)); - } - - reader.BaseStream.Position = sectorStart + sectorSize; - } -} diff --git a/BydTools.Tests/BydTools.Tests.csproj b/BydTools.Tests/BydTools.Tests.csproj new file mode 100644 index 0000000..3ab2c19 --- /dev/null +++ b/BydTools.Tests/BydTools.Tests.csproj @@ -0,0 +1,17 @@ + + + true + false + true + + + + + + + + + + + + diff --git a/BydTools.Tests/Fixtures/Samples.cs b/BydTools.Tests/Fixtures/Samples.cs new file mode 100644 index 0000000..a972f8e --- /dev/null +++ b/BydTools.Tests/Fixtures/Samples.cs @@ -0,0 +1,333 @@ +using System.Buffers.Binary; +using System.Globalization; +using System.Text; +using BydTools.Core.Crypto; +using BydTools.Core.IO; +using BydTools.Formats.SparkBuffer; +using BydTools.VFS; + +namespace BydTools.Tests.Fixtures; + +internal static class Samples +{ + public static byte[] Key { get; } = Enumerable.Repeat((byte)0x11, 32).ToArray(); + + public static byte[] MinimalBean() => + Spark( + static (ms, w) => + { + w.Write(1); + w.Write((byte)SparkType.Bean); + Align(ms, 4); + w.Write(0x12345678); + CString(w, "Item"); + Align(ms, 4); + w.Write(1); + CString(w, "value"); + w.Write((byte)SparkType.Int); + }, + static (ms, w) => + { + w.Write((byte)SparkType.Bean); + CString(w, "Root"); + Align(ms, 4); + w.Write(0x12345678); + }, + static w => w.Write(42) + ); + + public static byte[] AudioDialogMap() => + Spark( + static (ms, w) => + { + w.Write(1); + w.Write((byte)SparkType.Bean); + Align(ms, 4); + w.Write(0x1111); + CString(w, "Row"); + Align(ms, 4); + w.Write(1); + CString(w, "path"); + w.Write((byte)SparkType.String); + }, + static (ms, w) => + { + w.Write((byte)SparkType.Map); + CString(w, "AudioDialog"); + w.Write((byte)SparkType.String); + w.Write((byte)SparkType.Bean); + Align(ms, 4); + w.Write(0x1111); + }, + static w => + { + w.Write(1); + w.BaseStream.Position += 8; + long keyPos = w.BaseStream.Position; + w.Write(0); + long valPos = w.BaseStream.Position; + w.Write(0); + long keyStr = w.BaseStream.Position; + CString(w, "100"); + long bean = w.BaseStream.Position; + long pathField = w.BaseStream.Position; + w.Write(0); + long pathStr = w.BaseStream.Position; + CString(w, "v1d0/a.wem"); + w.BaseStream.Position = keyPos; + w.Write((int)keyStr); + w.BaseStream.Position = valPos; + w.Write((int)bean); + w.BaseStream.Position = pathField; + w.Write((int)pathStr); + } + ); + + public static byte[] PlainPck() + { + using var lang = new MemoryStream(); + using (var writer = new BinaryWriter(lang)) + { + writer.Write(1u); + writer.Write(12u); + writer.Write(1u); + writer.Write(Encoding.UTF8.GetBytes("sfx\0")); + } + + byte[] fileData = Encoding.ASCII.GetBytes("RIFF-demo-data"); + using var sounds = new MemoryStream(); + using (var writer = new BinaryWriter(sounds)) + { + writer.Write(1u); + writer.Write(99u); + writer.Write(0u); + writer.Write((uint)fileData.Length); + writer.Write(0u); + writer.Write(1u); + } + + byte[] langBytes = lang.ToArray(); + byte[] soundBytes = sounds.ToArray(); + int headerLength = 16 + langBytes.Length + soundBytes.Length; + BinaryPrimitives.WriteUInt32LittleEndian(soundBytes.AsSpan(16, 4), (uint)(8 + headerLength)); + + using var file = new MemoryStream(); + using var output = new BinaryWriter(file); + output.Write(0x4B504B41u); + output.Write((uint)headerLength); + output.Write(1u); + output.Write((uint)langBytes.Length); + output.Write(0u); + output.Write((uint)soundBytes.Length); + output.Write(langBytes); + output.Write(soundBytes); + output.Write(fileData); + return file.ToArray(); + } + + public const string EncryptedPckBase64 = + "AAAAADgAAAABAAAA8Q3+vBBY1TmbeU/Ds8MmQCnloMlVL3hGtDaKz/eayUzajfzS6NfTT1X5TdnKQyVW/GSf3zjASLfwzu0XY9g7lgt3"; + + public static byte[] Bnk() + { + byte[] wem = Encoding.ASCII.GetBytes("RIFF"); + using var ms = new MemoryStream(); + using var writer = new BinaryWriter(ms); + writer.Write(Encoding.ASCII.GetBytes("BKHD")); + writer.Write(4); + writer.Write(1); + writer.Write(Encoding.ASCII.GetBytes("DIDX")); + writer.Write(12); + writer.Write(7u); + writer.Write(0u); + writer.Write((uint)wem.Length); + writer.Write(Encoding.ASCII.GetBytes("DATA")); + writer.Write(wem.Length); + writer.Write(wem); + return ms.ToArray(); + } + + public static byte[] Usm() + { + using var ms = new MemoryStream(); + Packet(ms, "CRID", "HEAD"u8, 0); + Packet(ms, "@SFV", "VIDEODATA"u8, 0); + Packet(ms, "@SFA", new byte[] { 0x48, 0x43, 0x41, 0x00 }.Concat(Encoding.ASCII.GetBytes("AUDIODATA")).ToArray(), 1); + Packet(ms, "@SFA", new byte[] { 0x48, 0x43, 0x41, 0x00 }.Concat(Encoding.ASCII.GetBytes("MORE")).ToArray(), 2); + return ms.ToArray(); + } + + public static string CreateGame(string root, params VfsFileSpec[] files) + { + var groups = files.GroupBy(static file => file.DirectoryHash); + foreach (var group in groups) + { + var first = group.First(); + string dir = Path.Combine(root, VFSDefine.VfsDirectoryName, first.DirectoryHash); + Directory.CreateDirectory(dir); + byte[] payload = BlockPayload(first, group.ToArray()); + byte[] blc = Encrypt(payload); + File.WriteAllBytes(Path.Combine(dir, first.DirectoryHash + ".blc"), blc); + var chunk = new byte[group.Sum(static file => file.Stored.Length)]; + int offset = 0; + foreach (var file in group) + { + file.Stored.CopyTo(chunk, offset); + offset += file.Stored.Length; + } + + UInt128 md5 = 1; + File.WriteAllBytes(Path.Combine(dir, md5.ToHexLittleEndian() + ".chk"), chunk); + } + + return root; + } + + public static VfsFileSpec Entry( + string directoryHash, + string groupName, + EVFSBlockType type, + string name, + byte[] plain, + bool encrypt = false, + int codeVersion = 3 + ) + { + byte[] stored = plain; + long iv = 0; + if (encrypt) + { + iv = 7; + Span nonce = stackalloc byte[12]; + BinaryPrimitives.WriteInt32LittleEndian(nonce, VFSDefine.ProtocolVersion); + BinaryPrimitives.WriteInt64LittleEndian(nonce[4..], iv); + using var chacha = new CSChaCha20(Key, nonce.ToArray(), 1); + stored = chacha.EncryptBytes(plain); + } + + return new VfsFileSpec(directoryHash, groupName, type, name, plain, stored, encrypt, iv, codeVersion); + } + + private static byte[] BlockPayload(VfsFileSpec header, VfsFileSpec[] files) + { + using var ms = new MemoryStream(); + using var writer = new BinaryWriter(ms); + writer.Write(header.CodeVersion); + if (header.CodeVersion <= 10) + writer.Write(VFSDefine.ProtocolVersion); + byte[] name = Encoding.UTF8.GetBytes(header.GroupName); + writer.Write((ushort)name.Length); + writer.Write(name); + uint hash = uint.Parse(header.DirectoryHash, NumberStyles.HexNumber); + Span hashBytes = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(hashBytes, hash); + writer.Write(hashBytes); + writer.Write(0); + writer.Write(files.Length); + writer.Write(files.Sum(static file => (long)file.Stored.Length)); + writer.Write((byte)header.Type); + writer.Write(1); + Span md5 = stackalloc byte[16]; + md5[0] = 1; + writer.Write(md5); + writer.Write(new byte[16]); + writer.Write(files.Sum(static file => (long)file.Stored.Length)); + writer.Write((byte)header.Type); + writer.Write(files.Length); + long offset = 0; + foreach (var file in files) + { + byte[] fileName = Encoding.UTF8.GetBytes(file.Name); + writer.Write((ushort)fileName.Length); + writer.Write(fileName); + writer.Write(0L); + writer.Write(new byte[16]); + writer.Write(new byte[16]); + writer.Write(offset); + writer.Write((long)file.Stored.Length); + writer.Write((byte)file.Type); + writer.Write((byte)(file.Encrypt ? 1 : 0)); + if (file.Encrypt) + writer.Write(file.IvSeed); + offset += file.Stored.Length; + } + + return ms.ToArray(); + } + + private static byte[] Encrypt(byte[] payload) + { + byte[] nonce = new byte[12]; + BinaryPrimitives.WriteInt32LittleEndian(nonce, VFSDefine.ProtocolVersion); + using var chacha = new CSChaCha20(Key, nonce, 1); + byte[] cipher = chacha.EncryptBytes(payload); + byte[] file = new byte[nonce.Length + cipher.Length]; + nonce.CopyTo(file, 0); + cipher.CopyTo(file, nonce.Length); + return file; + } + + private static void Packet(Stream stream, string signature, ReadOnlySpan payload, byte streamId) + { + const ushort headerSkip = 0x18; + uint blockSize = headerSkip + (uint)payload.Length; + stream.Write(Encoding.ASCII.GetBytes(signature)); + Span size = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(size, blockSize); + stream.Write(size); + Span header = stackalloc byte[headerSkip]; + BinaryPrimitives.WriteUInt16BigEndian(header, headerSkip); + header[4] = streamId; + stream.Write(header); + stream.Write(payload); + } + + private static byte[] Spark( + Action typeDef, + Action rootDef, + Action data + ) + { + using var ms = new MemoryStream(); + using var writer = new BinaryWriter(ms); + writer.Write(0); + writer.Write(0); + writer.Write(0); + long typeDefAt = ms.Position; + typeDef(ms, writer); + long rootAt = ms.Position; + rootDef(ms, writer); + long dataAt = ms.Position; + data(writer); + ms.Position = 0; + writer.Write((int)typeDefAt); + writer.Write((int)rootAt); + writer.Write((int)dataAt); + return ms.ToArray(); + } + + private static void Align(Stream stream, int boundary) + { + long aligned = (stream.Position - 1) + (boundary - ((stream.Position - 1) % boundary)); + while (stream.Position < aligned) + stream.WriteByte(0); + } + + private static void CString(BinaryWriter writer, string text) + { + writer.Write(Encoding.UTF8.GetBytes(text)); + writer.Write((byte)0); + } +} + +internal sealed record VfsFileSpec( + string DirectoryHash, + string GroupName, + EVFSBlockType Type, + string Name, + byte[] Plain, + byte[] Stored, + bool Encrypt, + long IvSeed, + int CodeVersion +); diff --git a/BydTools.Tests/FormatTests.cs b/BydTools.Tests/FormatTests.cs new file mode 100644 index 0000000..1622d9e --- /dev/null +++ b/BydTools.Tests/FormatTests.cs @@ -0,0 +1,186 @@ +using System.Text; +using System.Text.Json.Nodes; +using BydTools.Audio.Bnk; +using BydTools.Audio.Interop; +using BydTools.Audio.Naming; +using BydTools.Audio.Pck; +using BydTools.Formats.Lua; +using BydTools.Formats.SparkBuffer; +using BydTools.Formats.Usm; +using BydTools.Tests.Fixtures; +using BydTools.VFS; +using Xunit; + +namespace BydTools.Tests; + +public class SparkBufferTests +{ + [Fact] + public void Bean_matches_legacy_json() + { + string json = SparkBuffer.ToJson(Samples.MinimalBean()); + Assert.Equal("{\n \"value\": 42\n}", json.Replace("\r\n", "\n")); + Assert.Equal("Root", SparkBuffer.GetRootName(Samples.MinimalBean())); + } + + [Fact] + public void Map_matches_legacy_audio_dialog() + { + byte[] data = Samples.AudioDialogMap(); + Assert.Equal("AudioDialog", SparkBuffer.GetRootName(data)); + string json = SparkBuffer.ToJson(data).Replace("\r\n", "\n"); + Assert.Equal("{\n \"100\": {\n \"path\": \"v1d0/a.wem\"\n }\n}", json); + } + + [Fact] + public void Conversions_do_not_share_type_tables() + { + byte[] bean = Samples.MinimalBean(); + byte[] map = Samples.AudioDialogMap(); + Parallel.For(0, 8, _ => + { + Assert.Equal("Root", SparkBuffer.GetRootName(bean)); + Assert.Contains("\"path\"", SparkBuffer.ToJson(map)); + }); + } +} + +public class PckTests +{ + [Fact] + public void Plain_archive_reads_the_file_entry() + { + var archive = PckArchive.Parse(Samples.PlainPck()); + Assert.False(archive.IsVfsEncrypted); + Assert.Equal("sfx", archive.Languages[0].Name); + Assert.Equal(99ul, archive.Entries[0].FileId); + Assert.Equal(14u, archive.Entries[0].Size); + } + + [Fact] + public void Encrypted_archive_matches_the_legacy_parser() + { + byte[] file = Convert.FromBase64String(Samples.EncryptedPckBase64); + var archive = PckArchive.Parse(file); + Assert.True(archive.IsVfsEncrypted); + archive.DecipherPayloads(file); + Assert.True(PckArchive.TrySlice(file, archive.Entries[0], out var slice)); + Assert.Equal("RIFF-demo-data", Encoding.ASCII.GetString(slice)); + + byte[] plain = archive.ToPlainPck(Convert.FromBase64String(Samples.EncryptedPckBase64)); + Assert.Equal(0x4B504B41u, System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(plain)); + var again = PckArchive.Parse(plain); + Assert.False(again.IsVfsEncrypted); + } +} + +public class PckMapperTests +{ + [Fact] + public void Hashes_match_the_legacy_mapper() + { + var root = (JsonObject)JsonNode.Parse("{\"100\":{\"path\":\"v1d0/a.wem\"}}")!; + var mapper = new PckMapper(root, "Chinese"); + Assert.Equal(3, mapper.Count); + Assert.Equal("voice/chinese/v1d0/a", mapper.GetMappedPath(6563861493395949416)); + Assert.Equal("music/chinese/v1d0/a", mapper.GetMappedPath(3014147426767775513)); + Assert.Equal("sfx/chinese/v1d0/a", mapper.GetMappedPath(9659578313000333649)); + } +} + +public class AudioNamingTests +{ + [Fact] + public void Mapped_and_unmapped_paths_use_the_block_language() + { + var root = (JsonObject)JsonNode.Parse("{\"100\":{\"path\":\"v1d0/a.wem\"}}")!; + var mapper = new PckMapper(root, "Chinese"); + Assert.Equal( + Path.Combine("Audio", "Chinese", "voice", "v1d0", "a.wav"), + AudioNaming.ResolveWem("Audio", "Chinese", 6563861493395949416, mapper, ".wav", null, 0) + ); + Assert.Equal( + Path.Combine("Audio", "unmapped", "Chinese", "99.wav"), + AudioNaming.ResolveWem("Audio", "Chinese", 99, null, ".wav", null, 0) + ); + Assert.Equal( + Path.Combine("Audio", "unmapped", "Chinese", "5_7.wav"), + AudioNaming.ResolveBnkWem("Audio", "Chinese", 5, 7, null, ".wav", null, 0) + ); + } +} + +public class BnkTests +{ + [Fact] + public void Parses_a_single_wem() + { + byte[] bnk = Samples.Bnk(); + var entries = BnkParser.Parse(bnk); + Assert.Equal(7u, entries[0].Id); + Assert.Equal(4, entries[0].Size); + Assert.Equal("RIFF", Encoding.ASCII.GetString(bnk.AsSpan(entries[0].Offset, 4))); + } +} + +public class UsmTests +{ + [Fact] + public void Demux_keeps_video_and_both_audio_streams() + { + var streams = CriUsmDemuxer.Demux(Samples.Usm()); + Assert.Equal(".m2v", streams[0].Extension); + Assert.Equal("VIDEODATA", Encoding.ASCII.GetString(streams[0].Data)); + Assert.Equal(".hca", streams[1].Extension); + Assert.StartsWith("HCA", Encoding.ASCII.GetString(streams[1].Data)); + Assert.Equal(".hca", streams[2].Extension); + } +} + +public class LuaTests +{ + [Fact] + public void Rejects_bytes_that_are_only_ascii_noise() + { + Assert.False(LuaDecipher.IsValid(" "u8)); + Assert.True(LuaDecipher.IsValid("\u001bLua"u8)); + Assert.True(LuaDecipher.IsValid("local x = 1"u8)); + } +} + +public class RegistryTests +{ + [Theory] + [InlineData("Audio", true)] + [InlineData("12", true)] + [InlineData("50", false)] + [InlineData("0", false)] + [InlineData("All", false)] + [InlineData("Raw", false)] + [InlineData("100", false)] + public void Only_catalogued_block_types_parse(string text, bool expected) => + Assert.Equal(expected, BlockRegistry.TryParse(text, out _)); +} + +public class MemoryStreamfileTests +{ + [Fact] + public void Callbacks_read_the_pinned_buffer() + { + byte[] data = [1, 2, 3, 4, 5]; + using var stream = MemoryStreamfile.Create(data); + Assert.Equal(3, stream.ReadAt(1, stackalloc byte[3])); + unsafe + { + var native = (LibStreamfile*)stream.Native; + byte* dst = stackalloc byte[2]; + Assert.Equal(2, native->Read(native->UserData, dst, 0, 2)); + Assert.Equal(1, dst[0]); + Assert.Equal(5, native->GetSize(native->UserData)); + LibStreamfile* child = native->Open(native->UserData, null); + Assert.Equal(1, child->Read(child->UserData, dst, 4, 1)); + Assert.Equal(5, dst[0]); + child->Close(child); + } + } +} diff --git a/BydTools.Tests/PipelineTests.cs b/BydTools.Tests/PipelineTests.cs new file mode 100644 index 0000000..36e117a --- /dev/null +++ b/BydTools.Tests/PipelineTests.cs @@ -0,0 +1,165 @@ +using System.Text; +using BydTools.CLI; +using BydTools.Core; +using BydTools.Extraction.Audio; +using BydTools.Extraction.Vfs; +using BydTools.Tests.Fixtures; +using BydTools.VFS; +using Xunit; + +namespace BydTools.Tests; + +public class PipelineTests +{ + [Fact] + public void Reads_a_block_and_rejects_a_wrong_key() + { + using var root = new TempDir(); + Samples.CreateGame( + root.Path, + Samples.Entry("42A8FCA6", "Table", EVFSBlockType.Table, "hello.txt", "hi"u8.ToArray(), encrypt: true) + ); + var archive = VfsArchive.Open(Vfs(root.Path), Samples.Key, new QuietLogger()); + var block = archive.Find("Table"); + Assert.NotNull(block); + Assert.Equal("Table", block.Info.GroupCfgName); + byte[] data = block.ReadFile(block.ResolveChunkPath(block.Info.Chunks[0])!, block.Info.Chunks[0].Files[0]); + Assert.Equal("hi", Encoding.ASCII.GetString(data)); + + var blc = Directory.GetFiles(Vfs(root.Path), "*.blc", SearchOption.AllDirectories).Single(); + var error = Assert.Throws(() => + BlcReader.Parse(File.ReadAllBytes(blc), Enumerable.Repeat((byte)0x22, 32).ToArray(), "42A8FCA6") + ); + Assert.Contains("key", error.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Code_version_above_ten_does_not_consume_a_second_version() + { + using var root = new TempDir(); + Samples.CreateGame( + root.Path, + Samples.Entry("42A8FCA6", "Table", EVFSBlockType.Table, "a.txt", "z"u8.ToArray(), codeVersion: 11) + ); + var block = VfsArchive.Open(Vfs(root.Path), Samples.Key, new QuietLogger()).Find("Table"); + Assert.NotNull(block); + Assert.Equal(11, block.Info.Version); + Assert.Equal(3, block.Info.CodeVersion); + Assert.Equal("a.txt", block.Info.Chunks[0].Files[0].Name); + } + + [Fact] + public void Dump_converts_spark_and_refuses_paths_that_escape() + { + using var root = new TempDir(); + using var output = new TempDir(); + Samples.CreateGame( + root.Path, + Samples.Entry("42A8FCA6", "Table", EVFSBlockType.Table, "row.bytes", Samples.MinimalBean()), + Samples.Entry("42A8FCA6", "Table", EVFSBlockType.Table, "../outside.txt", "no"u8.ToArray()) + ); + var block = VfsArchive.Open(Vfs(root.Path), Samples.Key, new QuietLogger()).Find("Table")!; + var result = new VfsExtractor(new QuietLogger()).Dump(block, output.Path, jobs: 2); + Assert.Contains("\"value\": 42", File.ReadAllText(Path.Combine(output.Path, "row.json"))); + Assert.False(File.Exists(Path.Combine(output.Path, "row.bytes"))); + Assert.False(File.Exists(Path.Combine(Directory.GetParent(output.Path)!.FullName, "outside.txt"))); + Assert.True(result.Failed >= 1); + } + + [Fact] + public void Usm_names_stay_stable_when_extracted_twice() + { + using var root = new TempDir(); + using var output = new TempDir(); + Samples.CreateGame( + root.Path, + Samples.Entry("55FC21C6", "Video", EVFSBlockType.Video, "clip.usm", Samples.Usm()) + ); + var block = VfsArchive.Open(Vfs(root.Path), Samples.Key, new QuietLogger()).Find("Video")!; + var extractor = new VfsExtractor(new QuietLogger()); + extractor.Dump(block, output.Path, jobs: 1); + extractor.Dump(block, output.Path, jobs: 1); + string[] names = Directory + .GetFiles(output.Path, "*", SearchOption.AllDirectories) + .Select(Path.GetFileName) + .Order(StringComparer.Ordinal) + .ToArray()!; + Assert.Equal(["clip.hca", "clip.m2v", "clip_1.hca"], names); + Assert.Equal("VIDEODATA", File.ReadAllText(Path.Combine(output.Path, "clip.m2v"))); + } + + [Fact] + public void Audio_export_writes_one_pck_without_copying_the_whole_block_first() + { + using var root = new TempDir(); + using var output = new TempDir(); + Samples.CreateGame( + root.Path, + Samples.Entry("24ED34CF", "Audio", EVFSBlockType.Audio, "bank.pck", Samples.PlainPck()) + ); + var archive = VfsArchive.Open(Vfs(root.Path), Samples.Key, new QuietLogger()); + var result = new AudioExportService(new QuietLogger()).Export( + archive, + archive.Find("Audio")!, + output.Path, + AudioExportMode.Raw, + mapNames: true, + decoder: null, + jobs: 1 + ); + Assert.Equal(1, result.Extracted); + Assert.Equal( + "RIFF-demo-data", + File.ReadAllText(Path.Combine(output.Path, "Audio", "unmapped", "Sfx", "99.wem")) + ); + } + + [Fact] + public void Cli_rejects_bad_arguments_and_dumps_a_table() + { + Assert.Equal(0, Program.Main([])); + Assert.Equal(2, Program.Main(["nope"])); + Assert.Equal(2, Program.Main(["vfs"])); + + using var root = new TempDir(); + using var output = new TempDir(); + Samples.CreateGame( + root.Path, + Samples.Entry("42A8FCA6", "Table", EVFSBlockType.Table, "row.bytes", Samples.MinimalBean()) + ); + string key = Convert.ToBase64String(Samples.Key); + Assert.Equal(2, Program.Main(["vfs", "-i", root.Path, "-o", output.Path, "-t", "50", "--key", key])); + Assert.Equal(2, Program.Main(["vfs", "-i", root.Path, "-o", output.Path, "-t", "All", "--key", key])); + Assert.Equal( + 0, + Program.Main(["vfs", "-i", root.Path, "-o", output.Path, "-t", "Table", "--key", key, "--jobs", "1"]) + ); + Assert.True(File.Exists(Path.Combine(output.Path, "row.json"))); + } + + private static string Vfs(string root) => Path.Combine(root, VFSDefine.VfsDirectoryName); + + private sealed class TempDir : IDisposable + { + public TempDir() => Path = Directory.CreateTempSubdirectory("bydtools-").FullName; + + public string Path { get; } + + public void Dispose() => Directory.Delete(Path, recursive: true); + } + + private sealed class QuietLogger : ILogger + { + public void Info(string message) { } + + public void Info(string format, params object[] args) { } + + public void Verbose(string message) { } + + public void Verbose(string format, params object[] args) { } + + public void Error(string message) { } + + public void Error(string format, params object[] args) { } + } +} diff --git a/BydTools.Utils/BydTools.Utils.csproj b/BydTools.Utils/BydTools.Utils.csproj deleted file mode 100644 index 6d665de..0000000 --- a/BydTools.Utils/BydTools.Utils.csproj +++ /dev/null @@ -1,7 +0,0 @@ - - - net10.0 - enable - enable - - diff --git a/BydTools.Utils/Extensions/EndianSpanReader.cs b/BydTools.Utils/Extensions/EndianSpanReader.cs deleted file mode 100644 index 3c9037f..0000000 --- a/BydTools.Utils/Extensions/EndianSpanReader.cs +++ /dev/null @@ -1,93 +0,0 @@ -using System.Buffers.Binary; -using System.Text; - -namespace BydTools.Utils.Extensions; - -/// -/// Extension methods on and for reading -/// primitives and strings with explicit endianness. -/// Ported from BeyondToolsSRC with thread-safe design (no mutable static state). -/// -public static class EndianSpanReader -{ - public static ushort ReadUInt16LE(this ReadOnlySpan data) => - BinaryPrimitives.ReadUInt16LittleEndian(data); - - public static ushort ReadUInt16BE(this ReadOnlySpan data) => - BinaryPrimitives.ReadUInt16BigEndian(data); - - public static short ReadInt16LE(this ReadOnlySpan data) => - BinaryPrimitives.ReadInt16LittleEndian(data); - - public static short ReadInt16BE(this ReadOnlySpan data) => - BinaryPrimitives.ReadInt16BigEndian(data); - - public static uint ReadUInt32LE(this ReadOnlySpan data) => - BinaryPrimitives.ReadUInt32LittleEndian(data); - - public static uint ReadUInt32BE(this ReadOnlySpan data) => - BinaryPrimitives.ReadUInt32BigEndian(data); - - public static int ReadInt32LE(this ReadOnlySpan data) => - BinaryPrimitives.ReadInt32LittleEndian(data); - - public static int ReadInt32BE(this ReadOnlySpan data) => - BinaryPrimitives.ReadInt32BigEndian(data); - - public static long ReadInt64LE(this ReadOnlySpan data) => - BinaryPrimitives.ReadInt64LittleEndian(data); - - public static long ReadInt64BE(this ReadOnlySpan data) => - BinaryPrimitives.ReadInt64BigEndian(data); - - public static UInt128 ReadUInt128LE(this ReadOnlySpan data) => - BinaryPrimitives.ReadUInt128LittleEndian(data); - - public static UInt128 ReadUInt128BE(this ReadOnlySpan data) => - BinaryPrimitives.ReadUInt128BigEndian(data); - - public static float ReadSingleLE(this ReadOnlySpan data) => - BinaryPrimitives.ReadSingleLittleEndian(data); - - public static float ReadSingleBE(this ReadOnlySpan data) => - BinaryPrimitives.ReadSingleBigEndian(data); - - public static string ReadStringToNull(this ReadOnlySpan data, int maxLength = 32767) - { - int limit = Math.Min(data.Length, maxLength); - int count = data[..limit].IndexOf((byte)0); - if (count < 0) - count = limit; - return Encoding.UTF8.GetString(data[..count]); - } - - public static string ReadString(this ReadOnlySpan data, int length) => - Encoding.UTF8.GetString(data[..length]); - - // Span overloads that forward to ReadOnlySpan - public static ushort ReadUInt16LE(this Span data) => - ReadUInt16LE((ReadOnlySpan)data); - - public static ushort ReadUInt16BE(this Span data) => - ReadUInt16BE((ReadOnlySpan)data); - - public static int ReadInt32LE(this Span data) => ReadInt32LE((ReadOnlySpan)data); - - public static int ReadInt32BE(this Span data) => ReadInt32BE((ReadOnlySpan)data); - - public static long ReadInt64LE(this Span data) => ReadInt64LE((ReadOnlySpan)data); - - public static long ReadInt64BE(this Span data) => ReadInt64BE((ReadOnlySpan)data); - - public static UInt128 ReadUInt128LE(this Span data) => - ReadUInt128LE((ReadOnlySpan)data); - - public static UInt128 ReadUInt128BE(this Span data) => - ReadUInt128BE((ReadOnlySpan)data); - - public static string ReadStringToNull(this Span data, int maxLength = 32767) => - ReadStringToNull((ReadOnlySpan)data, maxLength); - - public static string ReadString(this Span data, int length) => - ReadString((ReadOnlySpan)data, length); -} diff --git a/BydTools.Utils/Extensions/StreamExtensions.cs b/BydTools.Utils/Extensions/StreamExtensions.cs deleted file mode 100644 index b760422..0000000 --- a/BydTools.Utils/Extensions/StreamExtensions.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace BydTools.Utils.Extensions -{ - public static class StreamExtensions - { - public static void CopyBytes(this Stream inStream, Stream outStream, long? count = null) - { - if (count == null) - { - inStream.CopyTo(outStream); - return; - } - - long readBytes = 0L; - var buffer = new byte[64 * 1024]; - do - { - var toRead = Math.Min((long)(count - readBytes), buffer.LongLength); - var readNow = inStream.Read(buffer, 0, (int)toRead); - if (readNow == 0) - break; - outStream.Write(buffer, 0, readNow); - readBytes += readNow; - } while (readBytes < count); - } - } -} diff --git a/BydTools.Utils/Extensions/UInt128Extensions.cs b/BydTools.Utils/Extensions/UInt128Extensions.cs deleted file mode 100644 index 0d25d32..0000000 --- a/BydTools.Utils/Extensions/UInt128Extensions.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.Buffers.Binary; - -namespace BydTools.Utils.Extensions; - -/// -/// Extension methods for UInt128 to convert to hex strings. -/// -public static class UInt128Extensions -{ - /// - /// Converts a UInt128 value to a hexadecimal string in big-endian format. - /// This is used for MD5 hash names in BLC/CHK files. - /// - /// The UInt128 value to convert. - /// A 32-character uppercase hexadecimal string. - public static string ToHexStringBigEndian(this UInt128 value) - { - Span bytes = stackalloc byte[16]; - BinaryPrimitives.WriteUInt128BigEndian(bytes, value); - return Convert.ToHexString(bytes); - } - - /// - /// Converts a UInt128 value to a hexadecimal string in little-endian format. - /// - /// The UInt128 value to convert. - /// A 32-character uppercase hexadecimal string. - public static string ToHexStringLittleEndian(this UInt128 value) - { - Span bytes = stackalloc byte[16]; - BinaryPrimitives.WriteUInt128LittleEndian(bytes, value); - return Convert.ToHexString(bytes); - } -} diff --git a/BydTools.Utils/NullLogger.cs b/BydTools.Utils/NullLogger.cs deleted file mode 100644 index 9862ed3..0000000 --- a/BydTools.Utils/NullLogger.cs +++ /dev/null @@ -1,28 +0,0 @@ -namespace BydTools.Utils; - -/// -/// No-op logger implementation that silently discards all messages. -/// Use when logging is not needed but a non-null ILogger is required. -/// -public sealed class NullLogger : ILogger -{ - public static readonly NullLogger Instance = new(); - - private NullLogger() { } - - public void Info(string message) { } - - public void Info(string format, params object[] args) { } - - public void InfoNoNewline(string message) { } - - public void Verbose(string message) { } - - public void Verbose(string format, params object[] args) { } - - public void VerboseNoNewline(string message) { } - - public void Error(string message) { } - - public void Error(string format, params object[] args) { } -} diff --git a/BydTools.Utils/VGMToolbox/ByteConverter.cs b/BydTools.Utils/VGMToolbox/ByteConverter.cs deleted file mode 100644 index 6798d8a..0000000 --- a/BydTools.Utils/VGMToolbox/ByteConverter.cs +++ /dev/null @@ -1,127 +0,0 @@ -using System.Globalization; -using System.Text; - -namespace BydTools.Utils.VGMToolbox; - -/// -/// Provides utilities for byte-level conversions, encoding transformations, -/// and multi-byte integer handling with endianness support. -/// -public static class ByteConverter -{ - // Code page constants - private const int CodePageJapanese = 932; - private const int CodePageCyrillic = 1251; - - #region String Encoding - - /// Decodes bytes using the specified code page. - public static string Decode(byte[] data, int codePage) => - Encoding.GetEncoding(codePage).GetString(data); - - /// Decodes bytes using Japanese (Shift-JIS) encoding. - public static string DecodeJapanese(byte[] data) => Decode(data, CodePageJapanese); - - /// Decodes bytes using Cyrillic (Windows-1251) encoding. - public static string DecodeCyrillic(byte[] data) => Decode(data, CodePageCyrillic); - - /// Decodes bytes using ASCII encoding. - public static string DecodeAscii(byte[] data) => Encoding.ASCII.GetString(data); - - /// Decodes bytes using UTF-16 LE encoding. - public static string DecodeUtf16Le(byte[] data) => Encoding.Unicode.GetString(data); - - #endregion - - #region String Parsing - - /// - /// Parses a numeric string that may be hex (0x prefix) or decimal, - /// and may be negative. Returns the parsed long value. - /// - public static long ParseLong(string value) - { - var isNegative = false; - var span = value.AsSpan(); - - if (span.StartsWith("-")) - { - isNegative = true; - span = span[1..]; - } - - long result; - if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) - { - result = long.Parse(span[2..], NumberStyles.HexNumber); - } - else - { - result = long.Parse(span, NumberStyles.Integer); - } - - return isNegative ? -result : result; - } - - #endregion - - #region Byte Manipulation - - /// Returns the high nibble (upper 4 bits) of a byte. - public static byte GetHighNibble(byte value) => (byte)((value >> 4) & 0x0F); - - /// Returns the low nibble (lower 4 bits) of a byte. - public static byte GetLowNibble(byte value) => (byte)(value & 0x0F); - - /// - /// Converts a hex string to a byte array. - /// Example: "AABBCC" -> [0xAA, 0xBB, 0xCC] - /// - public static byte[] HexStringToBytes(string hex) - { - var result = new byte[hex.Length / 2]; - for (var i = 0; i < result.Length; i++) - { - result[i] = byte.Parse(hex.AsSpan(i * 2, 2), NumberStyles.HexNumber); - } - return result; - } - - #endregion - - #region Multi-byte Integer Conversion - - // Note: BitConverter is used for little-endian, manual reversal for big-endian - - public static int ToInt32(byte[] data, int offset, bool littleEndian) => - littleEndian ? BitConverter.ToInt32(data, offset) : BitConverter.ToInt32(data.AsSpan(offset, 4).ToArray().Reverse().ToArray(), 0); - - public static long ToInt64(byte[] data, int offset, bool littleEndian) => - littleEndian ? BitConverter.ToInt64(data, offset) : BitConverter.ToInt64(data.AsSpan(offset, 8).ToArray().Reverse().ToArray(), 0); - - public static ushort ToUInt16(byte[] data, int offset, bool littleEndian) => - littleEndian ? BitConverter.ToUInt16(data, offset) : BitConverter.ToUInt16(data.AsSpan(offset, 2).ToArray().Reverse().ToArray(), 0); - - public static uint ToUInt32(byte[] data, int offset, bool littleEndian) => - littleEndian ? BitConverter.ToUInt32(data, offset) : BitConverter.ToUInt32(data.AsSpan(offset, 4).ToArray().Reverse().ToArray(), 0); - - public static ulong ToUInt64(byte[] data, int offset, bool littleEndian) => - littleEndian ? BitConverter.ToUInt64(data, offset) : BitConverter.ToUInt64(data.AsSpan(offset, 8).ToArray().Reverse().ToArray(), 0); - - public static byte[] FromInt32(int value, bool littleEndian) => - littleEndian ? BitConverter.GetBytes(value) : BitConverter.GetBytes(value).Reverse().ToArray(); - - public static byte[] FromInt64(long value, bool littleEndian) => - littleEndian ? BitConverter.GetBytes(value) : BitConverter.GetBytes(value).Reverse().ToArray(); - - public static byte[] FromUInt16(ushort value, bool littleEndian) => - littleEndian ? BitConverter.GetBytes(value) : BitConverter.GetBytes(value).Reverse().ToArray(); - - public static byte[] FromUInt32(uint value, bool littleEndian) => - littleEndian ? BitConverter.GetBytes(value) : BitConverter.GetBytes(value).Reverse().ToArray(); - - public static byte[] FromUInt64(ulong value, bool littleEndian) => - littleEndian ? BitConverter.GetBytes(value) : BitConverter.GetBytes(value).Reverse().ToArray(); - - #endregion -} diff --git a/BydTools.Utils/VGMToolbox/CalculatingOffsetDescription.cs b/BydTools.Utils/VGMToolbox/CalculatingOffsetDescription.cs deleted file mode 100644 index c214bde..0000000 --- a/BydTools.Utils/VGMToolbox/CalculatingOffsetDescription.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BydTools.Utils.VGMToolbox; - -public class CalculatingOffsetDescription : OffsetDescription -{ - public const string OFFSET_VARIABLE_STRING = "$V"; - - public string CalculationString { get; set; } = string.Empty; -} diff --git a/BydTools.Utils/VGMToolbox/CalculatingOffsetInfo.cs b/BydTools.Utils/VGMToolbox/CalculatingOffsetInfo.cs deleted file mode 100644 index 0613abf..0000000 --- a/BydTools.Utils/VGMToolbox/CalculatingOffsetInfo.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace BydTools.Utils.VGMToolbox; - -/// -/// Extended offset info that supports variable-based calculations. -/// -public class CalculatingOffsetInfo : OffsetInfo -{ - /// Variable placeholder string used in calculations. - public const string VariablePlaceholder = "$V"; - - /// Calculation expression string. - public string Calculation { get; set; } = string.Empty; -} diff --git a/BydTools.Utils/VGMToolbox/Constants.cs b/BydTools.Utils/VGMToolbox/Constants.cs deleted file mode 100644 index 490e26d..0000000 --- a/BydTools.Utils/VGMToolbox/Constants.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BydTools.Utils.VGMToolbox; - -public static class VGMToolboxConstants -{ - public const int FileReadChunkSize = 71680; - public const string BigEndianByteOrder = "BigEndian"; - public const string LittleEndianByteOrder = "LittleEndian"; -} diff --git a/BydTools.Utils/VGMToolbox/FileUtils.cs b/BydTools.Utils/VGMToolbox/FileUtils.cs deleted file mode 100644 index 907adb2..0000000 --- a/BydTools.Utils/VGMToolbox/FileUtils.cs +++ /dev/null @@ -1,550 +0,0 @@ -using System.Collections; -using System.Diagnostics; -using System.Globalization; -using System.Text; - -namespace BydTools.Utils.VGMToolbox; - -/// -/// Provides file I/O utilities including reading, writing, chunking, -/// and file manipulation operations used by VGMToolbox. -/// -public static class FileUtils -{ - #region Stream Reading - - /// - /// Reads exactly bytes from the stream into the buffer, - /// throwing if EOF is reached prematurely. - /// - public static void ReadExact(Stream stream, byte[] buffer, int count) - { - int offset = 0; - int remaining = count; - - while (remaining > 0) - { - int read = stream.Read(buffer, offset, remaining); - if (read == 0) - { - throw new EndOfStreamException( - $"End of stream reached with {remaining} bytes left to read"); - } - remaining -= read; - offset += read; - } - } - - /// - /// Reads exactly the full length of the buffer from the stream. - /// - public static void ReadExact(Stream stream, byte[] buffer) => - ReadExact(stream, buffer, buffer.Length); - - #endregion - - #region File Statistics - - /// - /// Counts total files in the provided paths (files and directories). - /// - public static int CountFiles(string[] paths, bool includeSubdirectories = true) - { - int total = 0; - var option = includeSubdirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; - - foreach (var path in paths) - { - if (File.Exists(path)) - { - total++; - } - else if (Directory.Exists(path)) - { - total += Directory.GetFiles(path, "*.*", option).Length; - } - } - - return total; - } - - #endregion - - #region File Name Sanitization - - /// - /// Replaces invalid filename characters with underscores. - /// - public static string SanitizeFileName(string fileName) - { - foreach (char c in Path.GetInvalidFileNameChars()) - { - fileName = fileName.Replace(c, '_'); - } - return fileName; - } - - #endregion - - #region Chunk Replacement - - /// - /// Replaces a chunk in the destination file with data from the source file. - /// - public static void ReplaceChunk( - string sourcePath, long sourceOffset, long length, - string destPath, long destOffset) - { - byte[] buffer = new byte[VgmtConstants.FileReadChunkSize]; - - using var source = File.OpenRead(sourcePath); - using var dest = File.Open(destPath, FileMode.Open, FileAccess.Write); - - source.Position = sourceOffset; - dest.Position = destOffset; - - long remaining = length; - while (remaining > 0) - { - int toRead = (int)Math.Min(remaining, buffer.Length); - int read = source.Read(buffer, 0, toRead); - if (read == 0) break; - dest.Write(buffer, 0, read); - remaining -= read; - } - } - - #endregion - - #region Zero Out - - /// - /// Writes zeros to a file chunk at the specified offset. - /// - public static void ZeroOutChunk(string path, long offset, int length) - { - byte[] buffer = new byte[Math.Min(length, VgmtConstants.FileReadChunkSize)]; - int remaining = length; - - using var stream = File.OpenWrite(path); - stream.Position = offset; - - while (remaining > 0) - { - int toWrite = Math.Min(remaining, buffer.Length); - stream.Write(buffer, 0, toWrite); - remaining -= toWrite; - } - } - - #endregion - - #region Interleave Files - - /// - /// Interleaves multiple source files into a single destination file. - /// - public static void InterleaveFiles( - string[] sourceFiles, - uint interleaveSize, - long startOffset, - byte[] padding, - string destFile) - { - using var dest = File.Create(destFile); - var sources = sourceFiles.Select(f => File.OpenRead(f)).ToArray(); - - try - { - long maxLength = sources.Max(s => s.Length); - long currentOffset = startOffset; - - while (currentOffset < maxLength) - { - foreach (var source in sources) - { - if (currentOffset < source.Length) - { - source.Position = currentOffset; - long remainingInSource = source.Length - currentOffset; - long toCopy = Math.Min(remainingInSource, interleaveSize); - - CopyStream(source, dest, toCopy); - - // Pad if necessary - long paddingNeeded = interleaveSize - toCopy; - if (paddingNeeded > 0) - { - WritePadding(dest, padding, (int)paddingNeeded); - } - } - else - { - WritePadding(dest, padding, (int)interleaveSize); - } - } - currentOffset += interleaveSize; - } - } - finally - { - foreach (var source in sources) - { - source.Dispose(); - } - } - } - - private static void CopyStream(Stream source, Stream dest, long count) - { - byte[] buffer = new byte[8192]; - long remaining = count; - - while (remaining > 0) - { - int toRead = (int)Math.Min(remaining, buffer.Length); - int read = source.Read(buffer, 0, toRead); - if (read == 0) break; - dest.Write(buffer, 0, read); - remaining -= read; - } - } - - private static void WritePadding(Stream stream, byte[] padding, int count) - { - int written = 0; - while (written < count) - { - int toWrite = Math.Min(padding.Length, count - written); - stream.Write(padding, 0, toWrite); - written += toWrite; - } - } - - #endregion - - #region File Splitting - - /// - /// Splits a file into chunks of the specified size. - /// - public static string[] SplitFile(string sourceFile, long startOffset, ulong chunkSize, string outputFolder) - { - var files = new List(); - - using var source = File.OpenRead(sourceFile); - long fileLength = source.Length; - long currentOffset = startOffset; - int chunkNumber = 1; - - while (currentOffset < fileLength) - { - string outputFile = Path.Combine(outputFolder, $"{Path.GetFileName(sourceFile)}.{chunkNumber:D3}"); - long remaining = fileLength - currentOffset; - long toExtract = Math.Min((long)chunkSize, remaining); - ParseUtils.ExtractChunkToFile(source, currentOffset, toExtract, outputFile); - files.Add(outputFile); - currentOffset += toExtract; - chunkNumber++; - } - - return files.ToArray(); - } - - #endregion - - #region File Properties - - /// - /// Gets the size of a file, or null if the file doesn't exist or is inaccessible. - /// - public static long? GetFileSize(string path) - { - try - { - var info = new FileInfo(Path.GetFullPath(path)); - return info.Length; - } - catch - { - return null; - } - } - - #endregion - - #region File Creation - - /// - /// Creates a file from a byte array. - /// - public static void CreateFileFromBytes(string path, byte[] data) - { - string? dir = Path.GetDirectoryName(path); - if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - File.WriteAllBytes(path, data); - } - - /// - /// Creates a file from a text string using ASCII encoding. - /// - public static void CreateFileFromText(string path, string text) - { - string? dir = Path.GetDirectoryName(path); - if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - File.WriteAllText(path, text, Encoding.ASCII); - } - - #endregion - - #region Execute External Program - - /// - /// Executes an external program and captures its output. - /// - public static bool ExecuteExternal( - string executablePath, - string arguments, - string workingDirectory, - out string stdout, - out string stderr) - { - using var process = new Process(); - process.StartInfo = new ProcessStartInfo(executablePath, arguments) - { - WorkingDirectory = workingDirectory, - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - stdout = string.Empty; - stderr = string.Empty; - - bool started = process.Start(); - stdout = process.StandardOutput.ReadToEnd(); - stderr = process.StandardError.ReadToEnd(); - process.WaitForExit(); - - return started; - } - - #endregion - - #region Header Operations - - /// - /// Prepends a header to a file. - /// - public static void PrependHeader(byte[] header, string sourceFile, string destFile) - { - byte[] buffer = new byte[8192]; - - using var dest = File.Create(destFile); - dest.Write(header, 0, header.Length); - - using var source = File.OpenRead(sourceFile); - int read; - while ((read = source.Read(buffer, 0, buffer.Length)) > 0) - { - dest.Write(buffer, 0, read); - } - } - - #endregion - - #region Rename by Internal Name - - /// - /// Renames a file based on an internal name stored at a specific offset. - /// - public static void RenameByInternalName( - string path, - long nameOffset, - int maxLength, - byte[]? terminator, - bool preserveExtension) - { - string? directory = Path.GetDirectoryName(path); - if (string.IsNullOrEmpty(directory)) - throw new InvalidOperationException("Cannot determine directory for file"); - - string originalExtension = Path.GetExtension(path); - string internalName; - - using (var fs = File.OpenRead(path)) - { - int nameLength; - if (terminator != null) - { - nameLength = ParseUtils.GetSegmentLength(fs, (int)nameOffset, terminator); - } - else - { - nameLength = maxLength; - } - - if (nameLength < 1) - throw new ArgumentOutOfRangeException(nameof(nameLength), "Name length is less than 1"); - - byte[] nameBytes = ParseUtils.ReadSimpleOffset(fs, nameOffset, nameLength); - - // Replace nulls with spaces and decode as ASCII - for (int i = 0; i < nameBytes.Length; i++) - { - if (nameBytes[i] == 0x00) nameBytes[i] = 0x20; - } - internalName = Encoding.ASCII.GetString(nameBytes).Trim(); - } - - string newPath = Path.Combine(directory, internalName); - if (preserveExtension) - { - newPath = Path.ChangeExtension(newPath, originalExtension); - } - - if (path.Equals(newPath, StringComparison.OrdinalIgnoreCase)) - return; - - // Handle conflicts - if (File.Exists(newPath)) - { - string baseName = Path.GetFileNameWithoutExtension(newPath); - string ext = Path.GetExtension(newPath); - int count = Directory.GetFiles(directory, $"{baseName}*{ext}").Length; - newPath = Path.Combine(directory, $"{baseName}_{count:D4}{ext}"); - } - - File.Move(path, newPath); - } - - #endregion - - #region Remove Chunks - - /// - /// Removes a chunk from a file and returns the path to the new file. - /// - public static string RemoveChunk(string path, long offset, long length) - { - string fullPath = Path.GetFullPath(path); - string? directory = Path.GetDirectoryName(fullPath); - if (directory == null) - throw new InvalidOperationException("Cannot determine directory"); - - string tempPath = Path.ChangeExtension(fullPath, ".cut"); - - using (var source = File.OpenRead(fullPath)) - { - // Copy before the chunk - ParseUtils.ExtractChunkToFile(source, 0, offset, tempPath); - - // Copy after the chunk - using (var dest = File.Open(tempPath, FileMode.Append, FileAccess.Write)) - { - source.Position = offset + length; - byte[] buffer = new byte[1024]; - int read; - while ((read = source.Read(buffer, 0, buffer.Length)) > 0) - { - dest.Write(buffer, 0, read); - } - } - } - - return tempPath; - } - - /// - /// Removes all occurrences of a byte pattern from a file. - /// - public static string RemoveAllChunks(string path, byte[] pattern) - { - using var fs = File.OpenRead(path); - return RemoveAllChunks(fs, pattern); - } - - /// - /// Removes all occurrences of a byte pattern from a file stream. - /// - public static string RemoveAllChunks(FileStream fs, byte[] pattern) - { - long[] offsets = ParseUtils.FindAllOffsets(fs, 0, pattern, false, -1, -1, true); - string destPath = Path.ChangeExtension(fs.Name, ".cut"); - - byte[] buffer = new byte[VgmtConstants.FileReadChunkSize]; - long readOffset = 0; - - using var dest = File.Create(destPath); - - foreach (long offset in offsets) - { - fs.Position = readOffset; - long chunkSize = offset - readOffset; - CopyStreamChunk(fs, dest, chunkSize, buffer); - readOffset = offset + pattern.Length; - } - - // Copy remaining data after last pattern - fs.Position = readOffset; - CopyStreamChunk(fs, dest, fs.Length - readOffset, buffer); - - return destPath; - } - - private static void CopyStreamChunk(Stream source, Stream dest, long count, byte[] buffer) - { - long remaining = count; - while (remaining > 0) - { - int toRead = (int)Math.Min(remaining, buffer.Length); - int read = source.Read(buffer, 0, toRead); - if (read == 0) break; - dest.Write(buffer, 0, read); - remaining -= read; - } - } - - #endregion - - #region External Process Execution - - /// - /// Executes an external program and captures its output. - /// - public static bool Execute( - string executable, - string arguments, - string workingDirectory, - out string stdout, - out string stderr) - { - using var process = new Process(); - process.StartInfo = new ProcessStartInfo(executable, arguments) - { - WorkingDirectory = workingDirectory, - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - bool started = process.Start(); - stdout = process.StandardOutput.ReadToEnd(); - stderr = process.StandardError.ReadToEnd(); - process.WaitForExit(); - - return started; - } - - #endregion -} diff --git a/BydTools.Utils/VGMToolbox/OffsetDescription.cs b/BydTools.Utils/VGMToolbox/OffsetDescription.cs deleted file mode 100644 index e16cf98..0000000 --- a/BydTools.Utils/VGMToolbox/OffsetDescription.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BydTools.Utils.VGMToolbox; - -public class OffsetDescription -{ - public string OffsetValue { get; set; } = string.Empty; - public string OffsetSize { get; set; } = string.Empty; - public string OffsetByteOrder { get; set; } = string.Empty; -} diff --git a/BydTools.Utils/VGMToolbox/OffsetInfo.cs b/BydTools.Utils/VGMToolbox/OffsetInfo.cs deleted file mode 100644 index df3a0d8..0000000 --- a/BydTools.Utils/VGMToolbox/OffsetInfo.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace BydTools.Utils.VGMToolbox; - -/// -/// Describes an offset location, size, and byte order for data extraction. -/// -public class OffsetInfo -{ - /// Offset value as a string (can be hex "0x..." or decimal). - public string OffsetValue { get; set; } = string.Empty; - - /// Size of the data to read (in bytes). - public string Size { get; set; } = string.Empty; - - /// Byte order (BigEndian or LittleEndian). - public string ByteOrder { get; set; } = string.Empty; -} diff --git a/BydTools.Utils/VGMToolbox/ParseUtils.cs b/BydTools.Utils/VGMToolbox/ParseUtils.cs deleted file mode 100644 index aed448c..0000000 --- a/BydTools.Utils/VGMToolbox/ParseUtils.cs +++ /dev/null @@ -1,319 +0,0 @@ -using System.Collections; -using System.Globalization; -using System.Security.Cryptography; -using System.Text; - -namespace BydTools.Utils.VGMToolbox; - -/// -/// Core file parsing utilities for searching, extracting, and analyzing binary data. -/// -public static class ParseUtils -{ - // Log file names - public const string LogFileName = "vgmt_extraction_log.txt"; - public const string BatchFileName = "vgmt_extraction_log.bat"; - public const string VfsCutFolder = "vgmt_vfs_cut"; - - #region Simple Offset Reading - - /// - /// Reads bytes from a byte array at the specified offset. - /// - public static byte[] ReadSimpleOffset(byte[] data, int offset, int length) - { - var result = new byte[length]; - Buffer.BlockCopy(data, offset, result, 0, length); - return result; - } - - /// - /// Reads bytes from a stream at the specified offset without changing stream position. - /// - public static byte[] ReadSimpleOffset(Stream stream, long offset, int length) - { - long originalPosition = stream.Position; - try - { - stream.Position = offset; - var result = new byte[length]; - stream.ReadExactly(result, 0, length); - return result; - } - finally - { - stream.Position = originalPosition; - } - } - - #endregion - - #region Segment Length - - /// - /// Finds the length of a segment by searching for a terminator byte sequence. - /// - public static int GetSegmentLength(byte[] data, int startOffset, byte[] terminator) - { - for (int i = startOffset; i <= data.Length - terminator.Length; i++) - { - if (data.AsSpan(i, terminator.Length).SequenceEqual(terminator)) - { - return i - startOffset; - } - } - return 0; - } - - /// - /// Finds the length of a segment by searching for a terminator in a stream. - /// - public static int GetSegmentLength(Stream stream, int startOffset, byte[] terminator) - { - stream.Position = startOffset; - var checkBytes = new byte[terminator.Length]; - int position = startOffset; - - while (position < stream.Length) - { - stream.Position = position; - stream.Read(checkBytes, 0, 1); - - if (checkBytes[0] == terminator[0]) - { - stream.Position = position; - stream.Read(checkBytes, 0, terminator.Length); - - if (checkBytes.AsSpan().SequenceEqual(terminator)) - { - return position - startOffset; - } - } - position++; - } - - return 0; - } - - #endregion - - #region Find Offsets - - /// - /// Finds the next occurrence of a byte pattern in a stream. - /// - public static long FindNextOffset(Stream stream, long startOffset, byte[] pattern, bool restorePosition = true) - { - long originalPosition = stream.Position; - long position = startOffset; - int patternLength = pattern.Length; - - try - { - while (position < stream.Length - patternLength + 1) - { - stream.Position = position; - var check = new byte[patternLength]; - int read = stream.Read(check, 0, patternLength); - - if (read == patternLength && check.AsSpan().SequenceEqual(pattern)) - { - return position; - } - - position++; - } - - return -1; - } - finally - { - if (restorePosition) - { - stream.Position = originalPosition; - } - } - } - - /// - /// Finds the previous occurrence of a byte pattern in a stream. - /// - public static long FindPreviousOffset(Stream stream, long startOffset, byte[] pattern) - { - long originalPosition = stream.Position; - int patternLength = pattern.Length; - long position = startOffset - patternLength; - - try - { - while (position >= 0) - { - stream.Position = position; - var check = new byte[patternLength]; - int read = stream.Read(check, 0, patternLength); - - if (read == patternLength && check.AsSpan().SequenceEqual(pattern)) - { - if (position == startOffset) - { - position -= patternLength; - continue; - } - return position; - } - - position--; - } - - return -1; - } - finally - { - stream.Position = originalPosition; - } - } - - /// - /// Finds all occurrences of a byte pattern in a stream. - /// - public static long[] FindAllOffsets( - Stream stream, - long startOffset, - byte[] pattern, - bool useModulo, - long moduloDivisor, - long moduloResult, - bool restorePosition) - { - long originalPosition = stream.Position; - var offsets = new List(); - long position = startOffset; - int patternLength = pattern.Length; - - try - { - while (position < stream.Length - patternLength + 1) - { - stream.Position = position; - var check = new byte[patternLength]; - int read = stream.Read(check, 0, patternLength); - - if (read == patternLength && check.AsSpan().SequenceEqual(pattern)) - { - if (!useModulo || position % moduloDivisor == moduloResult) - { - offsets.Add(position); - } - } - - position++; - } - } - finally - { - if (restorePosition) - { - stream.Position = originalPosition; - } - } - - return offsets.ToArray(); - } - - #endregion - - #region Extraction - - /// - /// Extracts a chunk from a stream to a file. - /// - public static void ExtractChunkToFile( - Stream stream, - long offset, - long length, - string outputPath) - { - ExtractChunkToFile(stream, offset, length, outputPath, false, false); - } - - /// - /// Extracts a chunk from a stream to a file with optional logging and batch file generation. - /// - public static void ExtractChunkToFile( - Stream stream, - long offset, - long length, - string outputPath, - bool writeLog, - bool writeBatch) - { - string? outputDir = Path.GetDirectoryName(outputPath); - if (!string.IsNullOrEmpty(outputDir) && !Directory.Exists(outputDir)) - { - Directory.CreateDirectory(outputDir); - } - - // Handle filename conflicts - string finalPath = outputPath; - if (File.Exists(finalPath)) - { - string dir = outputDir ?? string.Empty; - string name = Path.GetFileNameWithoutExtension(finalPath); - string ext = Path.GetExtension(finalPath); - int count = Directory.GetFiles(dir, $"{name}*{ext}", SearchOption.TopDirectoryOnly).Length; - finalPath = Path.Combine(dir, $"{name}_{count:D3}{ext}"); - } - - // Extract data - using var output = File.Create(finalPath); - stream.Position = offset; - - byte[] buffer = new byte[VgmtConstants.FileReadChunkSize]; - long remaining = length; - - while (remaining > 0) - { - int toRead = (int)Math.Min(remaining, buffer.Length); - int read = stream.Read(buffer, 0, toRead); - if (read == 0) break; - output.Write(buffer, 0, read); - remaining -= read; - } - - // Write log if requested - if (writeLog) - { - string logEntry = $"Extracted - Offset: 0x{offset:X8} Length: 0x{length:X8} File: {Path.GetFileName(finalPath)}"; - string logPath = Path.Combine(outputDir ?? string.Empty, LogFileName); - File.AppendAllText(logPath, logEntry + Environment.NewLine); - } - } - - #endregion - - #region Comparison - - /// - /// Compares a segment of a byte array against a pattern. - /// - public static bool MatchesPattern(byte[] data, int offset, byte[] pattern) - { - if (data.Length < offset + pattern.Length) - return false; - - return data.AsSpan(offset, pattern.Length).SequenceEqual(pattern); - } - - /// - /// Compares a segment of a span against a pattern. - /// - public static bool MatchesPattern(ReadOnlySpan data, int offset, ReadOnlySpan pattern) - { - if (data.Length < offset + pattern.Length) - return false; - - return data.Slice(offset, pattern.Length).SequenceEqual(pattern); - } - - #endregion -} diff --git a/BydTools.Utils/VGMToolbox/VgmtConstants.cs b/BydTools.Utils/VGMToolbox/VgmtConstants.cs deleted file mode 100644 index 8057103..0000000 --- a/BydTools.Utils/VGMToolbox/VgmtConstants.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace BydTools.Utils.VGMToolbox; - -/// -/// Constants used by VGMToolbox file parsing utilities. -/// -public static class VgmtConstants -{ - /// Default chunk size for file read operations (71680 bytes). - public const int FileReadChunkSize = 71680; - - /// Big-endian byte order identifier. - public const string BigEndian = "BigEndian"; - - /// Little-endian byte order identifier. - public const string LittleEndian = "LittleEndian"; -} diff --git a/BydTools.VFS/BlcReader.cs b/BydTools.VFS/BlcReader.cs new file mode 100644 index 0000000..d2a42bf --- /dev/null +++ b/BydTools.VFS/BlcReader.cs @@ -0,0 +1,156 @@ +using System.Buffers.Binary; +using BydTools.Core.Crypto; +using BydTools.Core.IO; + +namespace BydTools.VFS; + +/// +/// Decrypts and parses a BLC block declaration. The directory hash is checked +/// before chunk tables are allocated, so a wrong key fails before a garbage +/// count can request a huge array. +/// +public static class BlcReader +{ + private const int MaxNameBytes = 10_000; + private const int MaxChunks = 1_000_000; + private const int MaxFilesPerChunk = 5_000_000; + + public static VfsBlockInfo Read(string blcPath, byte[] key) + { + var file = File.ReadAllBytes(blcPath); + var expectedHash = Path.GetFileNameWithoutExtension(blcPath); + return Parse(file, key, expectedHash); + } + + public static VfsBlockInfo Parse(ReadOnlySpan file, byte[] key, string expectedHash) + { + if (file.Length < VFSDefine.NonceLength) + throw new InvalidDataException("BLC file is smaller than the nonce."); + + int nonceVersion = BinaryPrimitives.ReadInt32LittleEndian(file); + byte[] nonce = file[..VFSDefine.NonceLength].ToArray(); + byte[] cipher = file[VFSDefine.NonceLength..].ToArray(); + + byte[] plain; + using (var chacha = new CSChaCha20(key, nonce, 1)) + plain = chacha.DecryptBytes(cipher); + + if (plain.Length != cipher.Length) + throw new InvalidDataException("ChaCha20 output length does not match the BLC payload."); + + byte[] reconstructed = new byte[file.Length]; + nonce.CopyTo(reconstructed, 0); + plain.CopyTo(reconstructed, VFSDefine.NonceLength); + + try + { + return ParseDecrypted(reconstructed, nonceVersion, expectedHash); + } + catch (InvalidDataException ex) when (!ex.Message.Contains("hash mismatch", StringComparison.Ordinal)) + { + throw new InvalidDataException( + ex.Message + " The ChaCha20 key may be incorrect or the file is corrupt.", + ex + ); + } + } + + private static VfsBlockInfo ParseDecrypted(byte[] file, int nonceVersion, string expectedHash) + { + var reader = new SpanReader(file); + reader.Seek(VFSDefine.NonceLength); + + int codeVersion = reader.ReadInt32(); + int version = codeVersion; + if (codeVersion > 10) + codeVersion = 3; + else + version = reader.ReadInt32(); + + ushort nameLength = reader.ReadUInt16(); + if (nameLength > MaxNameBytes) + throw new InvalidDataException($"BLC name length {nameLength} at offset {reader.Position} is not plausible."); + string groupCfgName = reader.ReadUtf8(nameLength); + + uint hash = reader.ReadUInt32BigEndian(); + reader.Skip(4); + string groupCfgHashName = hash.ToString("X8"); + if (!groupCfgHashName.Equals(expectedHash, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidDataException( + $"BLC hash mismatch: parsed \"{groupCfgHashName}\", expected \"{expectedHash}\". " + + "The ChaCha20 key may be incorrect or outdated." + ); + } + + int groupFileInfoNum = reader.ReadInt32(); + long groupChunksLength = reader.ReadInt64(); + var blockType = (EVFSBlockType)reader.ReadByte(); + int chunkCount = reader.ReadInt32(); + if (chunkCount < 0 || chunkCount > MaxChunks) + throw new InvalidDataException($"BLC chunk count {chunkCount} is not plausible."); + + var chunks = new VfsChunk[chunkCount]; + for (int i = 0; i < chunkCount; i++) + { + var md5Name = reader.ReadUInt128(); + var contentMd5 = reader.ReadUInt128(); + long length = reader.ReadInt64(); + var chunkType = (EVFSBlockType)reader.ReadByte(); + var chunkTag = EVFSFileTag.Base; + if (codeVersion > 3) + chunkTag = (EVFSFileTag)(byte)reader.ReadInt32(); + + int fileCount = reader.ReadInt32(); + if (fileCount < 0 || fileCount > MaxFilesPerChunk) + throw new InvalidDataException($"BLC file count {fileCount} is not plausible."); + + var files = new VfsFile[fileCount]; + for (int j = 0; j < fileCount; j++) + { + ushort fileNameLength = reader.ReadUInt16(); + if (fileNameLength > MaxNameBytes) + throw new InvalidDataException($"File name length {fileNameLength} is not plausible."); + string fileName = reader.ReadUtf8(fileNameLength); + long nameHash = reader.ReadInt64(); + var chunkMd5 = reader.ReadUInt128(); + var dataMd5 = reader.ReadUInt128(); + long offset = reader.ReadInt64(); + long len = reader.ReadInt64(); + var fileType = (EVFSBlockType)reader.ReadByte(); + bool encrypted = reader.ReadByte() != 0; + long ivSeed = encrypted ? reader.ReadInt64() : 0; + var fileTag = EVFSFileTag.Base; + if (codeVersion > 3) + fileTag = (EVFSFileTag)(byte)reader.ReadInt32(); + + files[j] = new VfsFile( + fileName, + nameHash, + chunkMd5, + dataMd5, + offset, + len, + fileType, + encrypted, + ivSeed, + fileTag + ); + } + + chunks[i] = new VfsChunk(md5Name, contentMd5, length, chunkType, chunkTag, files); + } + + return new VfsBlockInfo( + nonceVersion, + codeVersion, + version, + groupCfgName, + groupCfgHashName, + groupFileInfoNum, + groupChunksLength, + blockType, + chunks + ); + } +} diff --git a/BydTools.VFS/BlockRegistry.cs b/BydTools.VFS/BlockRegistry.cs new file mode 100644 index 0000000..4008d09 --- /dev/null +++ b/BydTools.VFS/BlockRegistry.cs @@ -0,0 +1,91 @@ +using System.Collections.Frozen; +using System.Diagnostics.CodeAnalysis; + +namespace BydTools.VFS; + +public enum ContentKind +{ + Generic, + Table, + Lua, + Video, + Audio, +} + +/// +/// One row of the block catalog. Directory hashes are hints: opening a VFS +/// scans real directories, and a mismatch is reported rather than trusted. +/// +public sealed record BlockDescriptor( + EVFSBlockType Type, + string DirHash, + ContentKind Kind, + string? AudioLanguage = null +); + +public static class BlockRegistry +{ + private static readonly BlockDescriptor[] Entries = + [ + new(EVFSBlockType.InitAudio, "07A1BB91", ContentKind.Audio, "Initial"), + new(EVFSBlockType.InitBundle, "0CE8FA57", ContentKind.Generic), + new(EVFSBlockType.BundleManifest, "1CDDBF1F", ContentKind.Generic), + new(EVFSBlockType.InitialExtendData, "3C9D9D2D", ContentKind.Generic), + new(EVFSBlockType.Audio, "24ED34CF", ContentKind.Audio, "Main"), + new(EVFSBlockType.Bundle, "7064D8E2", ContentKind.Generic), + new(EVFSBlockType.DynamicStreaming, "23D53F5D", ContentKind.Generic), + new(EVFSBlockType.Table, "42A8FCA6", ContentKind.Table), + new(EVFSBlockType.Video, "55FC21C6", ContentKind.Video), + new(EVFSBlockType.IV, "A63D7E6A", ContentKind.Generic), + new(EVFSBlockType.Streaming, "C3442D43", ContentKind.Generic), + new(EVFSBlockType.JsonData, "775A31D1", ContentKind.Generic), + new(EVFSBlockType.Lua, "19E3AE45", ContentKind.Lua), + new(EVFSBlockType.IFixPatchOut, "DAFE52C9", ContentKind.Generic), + new(EVFSBlockType.ExtendData, "D6E622F7", ContentKind.Generic), + new(EVFSBlockType.AudioChinese, "E1E7D7CE", ContentKind.Audio, "Chinese"), + new(EVFSBlockType.AudioEnglish, "A31457D0", ContentKind.Audio, "English"), + new(EVFSBlockType.AudioJapanese, "F668D4EE", ContentKind.Audio, "Japanese"), + new(EVFSBlockType.AudioKorean, "E9D31017", ContentKind.Audio, "Korean"), + new(EVFSBlockType.AuditAudio, "1EBAF5C6", ContentKind.Audio, "Audit"), + new(EVFSBlockType.AuditDynamicStreaming, "B9358E30", ContentKind.Generic), + new(EVFSBlockType.AuditIV, "06223FE2", ContentKind.Generic), + new(EVFSBlockType.AuditStreaming, "6432320A", ContentKind.Generic), + new(EVFSBlockType.AuditVideo, "2E6CE44D", ContentKind.Video), + new(EVFSBlockType.HotfixAudio, "F151B649", ContentKind.Audio, "Hotfix"), + ]; + + private static readonly FrozenDictionary ByType = + Entries.ToFrozenDictionary(static d => d.Type); + + public static IReadOnlyList All => Entries; + + public static IEnumerable Audio => + Entries.Where(static d => d.Kind == ContentKind.Audio); + + public static bool TryGet(EVFSBlockType type, [NotNullWhen(true)] out BlockDescriptor? descriptor) => + ByType.TryGetValue(type, out descriptor); + + /// + /// Accepts an enum name or a numeric value, and only when that value is catalogued. + /// and are rejected, + /// as is any number the enum parser would otherwise accept. + /// + public static bool TryParse(string text, [NotNullWhen(true)] out BlockDescriptor? descriptor) + { + if (Enum.TryParse(text, ignoreCase: true, out var named) && ByType.ContainsKey(named)) + { + descriptor = ByType[named]; + return true; + } + + if (byte.TryParse(text, out byte numeric)) + { + var type = (EVFSBlockType)numeric; + if (Enum.IsDefined(type) && ByType.TryGetValue(type, out descriptor)) + return true; + } + + descriptor = null; + return false; + } +} diff --git a/BydTools.VFS/BydTools.VFS.csproj b/BydTools.VFS/BydTools.VFS.csproj index 64b4683..f95b2a3 100644 --- a/BydTools.VFS/BydTools.VFS.csproj +++ b/BydTools.VFS/BydTools.VFS.csproj @@ -1,12 +1,5 @@ - - net10.0 - enable - enable - - - - + diff --git a/BydTools.VFS/CriUsm/BlockSizeInfo.cs b/BydTools.VFS/CriUsm/BlockSizeInfo.cs deleted file mode 100644 index 57c4024..0000000 --- a/BydTools.VFS/CriUsm/BlockSizeInfo.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace BydTools.VFS.CriUsm; - -/// -/// Describes the size characteristics of a block type in an MPEG stream. -/// -public sealed class BlockSizeInfo -{ - /// How the block size is determined. - public PacketSizeType SizeType { get; } - - /// The static size value or size field length. - public int StaticSize { get; } - - /// Creates a new BlockSizeInfo instance. - public BlockSizeInfo(PacketSizeType sizeType, int staticSize) - { - SizeType = sizeType; - StaticSize = staticSize; - } -} diff --git a/BydTools.VFS/CriUsm/CriUsmDemuxer.cs b/BydTools.VFS/CriUsm/CriUsmDemuxer.cs deleted file mode 100644 index 57baea6..0000000 --- a/BydTools.VFS/CriUsm/CriUsmDemuxer.cs +++ /dev/null @@ -1,366 +0,0 @@ -using System.Buffers.Binary; - -namespace BydTools.VFS.CriUsm; - -/// -/// Demultiplexes CRI USM (Sofdec2) containers into separate video and audio streams. -/// Ported from VGMToolbox (CriUsmStream / MpegStream) with in-memory processing. -/// -public static class CriUsmDemuxer -{ - private static readonly byte[] CRID_SIG = "CRID"u8.ToArray(); - private static readonly byte[] SFV_SIG = "@SFV"u8.ToArray(); - private static readonly byte[] SFA_SIG = "@SFA"u8.ToArray(); - private static readonly byte[] SBT_SIG = "@SBT"u8.ToArray(); - private static readonly byte[] CUE_SIG = "@CUE"u8.ToArray(); - private static readonly byte[] ALP_SIG = "@ALP"u8.ToArray(); - - private static readonly byte[] HEADER_END_MARKER = - [ - 0x23, - 0x48, - 0x45, - 0x41, - 0x44, - 0x45, - 0x52, - 0x20, - 0x45, - 0x4E, - 0x44, - 0x20, - 0x20, - 0x20, - 0x20, - 0x20, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x00, - ]; - - private static readonly byte[] METADATA_END_MARKER = - [ - 0x23, - 0x4D, - 0x45, - 0x54, - 0x41, - 0x44, - 0x41, - 0x54, - 0x41, - 0x20, - 0x45, - 0x4E, - 0x44, - 0x20, - 0x20, - 0x20, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x00, - ]; - - private static readonly byte[] CONTENTS_END_MARKER = - [ - 0x23, - 0x43, - 0x4F, - 0x4E, - 0x54, - 0x45, - 0x4E, - 0x54, - 0x53, - 0x20, - 0x45, - 0x4E, - 0x44, - 0x20, - 0x20, - 0x20, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x3D, - 0x00, - ]; - - private static readonly byte[] AIXF_SIG = "AIXF"u8.ToArray(); - private static readonly byte[] HCA_SIG = [0x48, 0x43, 0x41, 0x00]; - - private const int SEARCH_BUFFER_SIZE = 71680; - - /// - /// Demultiplexes USM data into separate video (.m2v) and audio (.adx/.hca/.aix) files. - /// - /// Paths of all output files, or empty array on failure. - public static string[] Demux(byte[] usmData, string outputBasePath) - { - using var stream = new MemoryStream(usmData, writable: false); - return Demux(stream, outputBasePath); - } - - /// - /// Demultiplexes a USM stream into separate video (.m2v) and audio (.adx/.hca/.aix) files. - /// - public static string[] Demux(Stream stream, string outputBasePath) - { - long fileSize = stream.Length; - long currentOffset = FindPattern(stream, 0, CRID_SIG); - if (currentOffset < 0) - return []; - - var streamWriters = new Dictionary(); - Span blockId = stackalloc byte[4]; - Span sizeField = stackalloc byte[4]; - - try - { - while (currentOffset < fileSize) - { - stream.Position = currentOffset; - if (stream.Read(blockId) < 4) - break; - - uint blockIdVal = BitConverter.ToUInt32(blockId); - - if (!IsKnownPacket(blockId)) - break; - - stream.Position = currentOffset + 4; - if (stream.Read(sizeField) < 4) - break; - uint blockSize = BinaryPrimitives.ReadUInt32BigEndian(sizeField); - - bool isAudio = blockId.SequenceEqual(SFA_SIG); - bool isVideo = blockId.SequenceEqual(SFV_SIG); - - if (isAudio || isVideo) - { - byte streamId = isAudio ? ReadByte(stream, currentOffset + 0xC) : (byte)0; - uint streamKey = (uint)(streamId | blockIdVal); - - if (!streamWriters.TryGetValue(streamKey, out var ms)) - { - ms = new MemoryStream(); - streamWriters[streamKey] = ms; - } - - int headerSkip = ReadUInt16BE(stream, currentOffset + 8); - int footerSkip = ReadUInt16BE(stream, currentOffset + 0xA); - int cutSize = (int)(blockSize - headerSkip - footerSkip); - - if (cutSize > 0) - { - long dataStart = currentOffset + 8 + headerSkip; - stream.Position = dataStart; - var payload = GC.AllocateUninitializedArray(cutSize); - stream.ReadExactly(payload); - ms.Write(payload); - } - } - - currentOffset += 8 + blockSize; - } - - return FinalizeStreams(streamWriters, outputBasePath); - } - finally - { - foreach (var ms in streamWriters.Values) - ms.Dispose(); - } - } - - private static string[] FinalizeStreams( - Dictionary streams, - string outputBasePath - ) - { - var outputFiles = new List(streams.Count); - var outputDir = Path.GetDirectoryName(outputBasePath); - if (!string.IsNullOrEmpty(outputDir) && !Directory.Exists(outputDir)) - Directory.CreateDirectory(outputDir); - - foreach (var (streamKey, ms) in streams) - { - ms.Position = 0; - - long headerEndOff = FindPattern(ms, 0, HEADER_END_MARKER); - long metadataEndOff = FindPattern(ms, 0, METADATA_END_MARKER); - - long dataStart; - if (headerEndOff >= 0 || metadataEndOff >= 0) - { - long furthest = Math.Max(headerEndOff, metadataEndOff); - dataStart = furthest + METADATA_END_MARKER.Length; - } - else - { - dataStart = 0; - } - - long contentEndOff = FindPattern(ms, dataStart, CONTENTS_END_MARKER); - long dataLength = - contentEndOff > dataStart ? contentEndOff - dataStart : ms.Length - dataStart; - - if (dataLength <= 0) - continue; - - bool isAudio = IsAudioStreamKey(streamKey); - string ext; - if (isAudio) - { - byte[] sig = new byte[4]; - ms.Position = dataStart; - ms.ReadExactly(sig); - ext = DetectAudioExtension(sig); - } - else - { - ext = ".m2v"; - } - - string outputPath = GetUniqueOutputPath(outputBasePath, ext); - - using var fs = File.Create(outputPath); - ms.Position = dataStart; - var buffer = GC.AllocateUninitializedArray(64 * 1024); - long remaining = dataLength; - while (remaining > 0) - { - int toRead = (int)Math.Min(remaining, buffer.Length); - int read = ms.Read(buffer, 0, toRead); - if (read == 0) - break; - fs.Write(buffer, 0, read); - remaining -= read; - } - - outputFiles.Add(outputPath); - } - - return outputFiles.ToArray(); - } - - private static bool IsKnownPacket(ReadOnlySpan sig) => - sig.SequenceEqual(CRID_SIG) - || sig.SequenceEqual(SFV_SIG) - || sig.SequenceEqual(SFA_SIG) - || sig.SequenceEqual(SBT_SIG) - || sig.SequenceEqual(CUE_SIG) - || sig.SequenceEqual(ALP_SIG); - - private static bool IsAudioStreamKey(uint streamKey) - { - Span blockBytes = stackalloc byte[4]; - BitConverter.TryWriteBytes(blockBytes, streamKey & 0xFFFFFFF0); - return blockBytes.SequenceEqual(SFA_SIG); - } - - private static byte ReadByte(Stream stream, long offset) - { - stream.Position = offset; - int b = stream.ReadByte(); - return b < 0 ? (byte)0 : (byte)b; - } - - private static int ReadUInt16BE(Stream stream, long offset) - { - Span buf = stackalloc byte[2]; - stream.Position = offset; - stream.ReadExactly(buf); - return BinaryPrimitives.ReadUInt16BigEndian(buf); - } - - private static string DetectAudioExtension(ReadOnlySpan signature) - { - if (signature.Length >= 4 && signature[..4].SequenceEqual(AIXF_SIG)) - return ".aix"; - if (signature.Length >= 1 && signature[0] == 0x80) - return ".adx"; - if (signature.Length >= 4 && signature[..4].SequenceEqual(HCA_SIG)) - return ".hca"; - return ".bin"; - } - - private static string GetUniqueOutputPath(string basePath, string ext) - { - string path = basePath + ext; - if (!File.Exists(path)) - return path; - - string dir = Path.GetDirectoryName(basePath) ?? "."; - string name = Path.GetFileNameWithoutExtension(basePath); - int n = 1; - do - { - path = Path.Combine(dir, $"{name}_{n}{ext}"); - n++; - } while (File.Exists(path)); - - return path; - } - - internal static long FindPattern(Stream stream, long startOffset, ReadOnlySpan pattern) - { - var buffer = GC.AllocateUninitializedArray(SEARCH_BUFFER_SIZE); - long absoluteOffset = startOffset; - - while (absoluteOffset < stream.Length) - { - stream.Position = absoluteOffset; - int bytesRead = stream.Read(buffer, 0, buffer.Length); - if (bytesRead < pattern.Length) - break; - - for (int i = 0; i <= bytesRead - pattern.Length; i++) - { - if (buffer.AsSpan(i, pattern.Length).SequenceEqual(pattern)) - return absoluteOffset + i; - } - - absoluteOffset += bytesRead - pattern.Length + 1; - } - - return -1; - } -} diff --git a/BydTools.VFS/CriUsm/MpegStream.cs b/BydTools.VFS/CriUsm/MpegStream.cs deleted file mode 100644 index 237b89e..0000000 --- a/BydTools.VFS/CriUsm/MpegStream.cs +++ /dev/null @@ -1,224 +0,0 @@ -using BydTools.Utils.VGMToolbox; - -namespace BydTools.VFS.CriUsm; - -/// -/// Base class for MPEG Program Stream (PS) demultiplexing. -/// Provides core functionality for parsing packetized elementary streams. -/// -public abstract class MpegStream -{ - // Standard MPEG-2 PS packet signatures - private static readonly byte[] _packStartBytes = [0x00, 0x00, 0x01, 0xBA]; - private static readonly byte[] _packEndBytes = [0x00, 0x00, 0x01, 0xB9]; - - /// Packet block ID definitions and their size characteristics. - protected Dictionary BlockIds { get; } = new(); - - /// Stream ID to file extension mapping. - protected Dictionary StreamExtensions { get; } = new(); - - /// Source file path. - public string FilePath { get; } - - /// Audio file extension. - public string AudioExtension { get; protected set; } = string.Empty; - - /// Video file extension. - public string VideoExtension { get; protected set; } = string.Empty; - - /// Whether the same stream ID is used for multiple audio tracks. - public bool UsesSameIdForMultipleAudioTracks { get; protected set; } - - /// Whether subtitle extraction is supported. - public bool SupportsSubtitleExtraction { get; protected set; } - - /// Whether block sizes are little-endian. - public bool LittleEndianBlockSizes { get; protected set; } - - /// Creates a new MpegStream instance. - protected MpegStream(string path) - { - FilePath = path; - UsesSameIdForMultipleAudioTracks = false; - SupportsSubtitleExtraction = false; - LittleEndianBlockSizes = false; - - // Initialize standard MPEG block IDs - InitializeBlockIds(); - } - - private void InitializeBlockIds() - { - var packEndId = BitConverter.ToUInt32(_packEndBytes, 0); - BlockIds[packEndId] = new BlockSizeInfo(PacketSizeType.Eof, -1); - - var packStartId = BitConverter.ToUInt32(_packStartBytes, 0); - BlockIds[packStartId] = new BlockSizeInfo(PacketSizeType.Static, 0xE); - - // Initialize 0x00-0xAF range - for (byte i = 0; i <= 0xAF; i++) - { - var idBytes = new byte[] { 0x00, 0x00, 0x01, i }; - var id = BitConverter.ToUInt32(idBytes, 0); - BlockIds[id] = new BlockSizeInfo(PacketSizeType.Static, 0xE); - } - } - - /// Returns the packet start signature bytes. - protected virtual byte[] GetPacketStartBytes() => _packStartBytes; - - /// Returns the packet end signature bytes. - protected virtual byte[] GetPacketEndBytes() => _packEndBytes; - - /// Gets the audio packet header size at the current position. - protected abstract int GetAudioHeaderSize(Stream stream, long offset); - - /// Gets the audio packet sub-header size. - protected virtual int GetAudioSubHeaderSize(Stream stream, long offset, byte streamId) => 0; - - /// Gets the video packet header size at the current position. - protected abstract int GetVideoHeaderSize(Stream stream, long offset); - - /// Gets the audio packet footer size. - protected virtual int GetAudioFooterSize(Stream stream, long offset) => 0; - - /// Gets the video packet footer size. - protected virtual int GetVideoFooterSize(Stream stream, long offset) => 0; - - /// Determines if the block ID represents an audio block. - protected virtual bool IsAudioBlock(ReadOnlySpan blockId) => - blockId.Length >= 4 && blockId[3] >= 0xC0 && blockId[3] <= 0xDF; - - /// Determines if the block ID represents a video block. - protected virtual bool IsVideoBlock(ReadOnlySpan blockId) => - blockId.Length >= 4 && blockId[3] >= 0xE0 && blockId[3] <= 0xEF; - - /// Gets the file extension for audio files. - protected virtual string GetAudioExtension(Stream stream, long offset) => AudioExtension; - - /// Gets the file extension for video files. - protected virtual string GetVideoExtension(Stream stream, long offset) => VideoExtension; - - /// Gets the stream ID from the current position. - protected virtual byte GetStreamId(Stream stream, long offset) => 0; - - /// Gets the start offset for parsing. - protected virtual long GetStartOffset(Stream stream, long offset) => 0; - - /// Performs final tasks after demuxing. - protected virtual string[] PostProcess( - Stream source, - Dictionary outputs, - bool addHeader) => Array.Empty(); - - /// - /// Demultiplexes the MPEG stream into separate audio and video files. - /// - public string[] Demux(bool video, bool audio, bool addHeader = false) - { - using var stream = File.OpenRead(FilePath); - return Demux(stream, video, audio, addHeader); - } - - /// - /// Demultiplexes the MPEG stream from a provided stream. - /// - public string[] Demux(Stream stream, bool video, bool audio, bool addHeader = false) - { - var options = new DemuxOptions - { - ExtractVideo = video, - ExtractAudio = audio, - AddHeader = addHeader - }; - - return DemuxInternal(stream, options); - } - - private string[] DemuxInternal(Stream stream, DemuxOptions options) - { - long fileSize = stream.Length; - long currentOffset = GetStartOffset(stream, 0); - - // Find first pack start - currentOffset = ParseUtils.FindNextOffset(stream, currentOffset, GetPacketStartBytes()); - - if (currentOffset == -1) - { - throw new FormatException($"Cannot find pack header in: {Path.GetFileName(FilePath)}"); - } - - var outputs = new Dictionary(); - byte[] blockIdBytes = new byte[4]; - - try - { - while (currentOffset < fileSize) - { - // Read block ID - stream.Position = currentOffset; - stream.ReadExactly(blockIdBytes, 0, 4); - uint blockId = BitConverter.ToUInt32(blockIdBytes, 0); - - if (!BlockIds.TryGetValue(blockId, out var blockInfo)) - { - // Unknown block - skip ahead - currentOffset++; - continue; - } - - // Process based on block type - switch (blockInfo.SizeType) - { - case PacketSizeType.Eof: - // End of file marker - currentOffset = fileSize; - break; - - case PacketSizeType.Static: - currentOffset += blockInfo.StaticSize; - break; - - case PacketSizeType.SizeBytes: - // Read size from following bytes - byte[] sizeBytes = new byte[blockInfo.StaticSize]; - stream.ReadExactly(sizeBytes, 0, blockInfo.StaticSize); - - if (!LittleEndianBlockSizes) - { - Array.Reverse(sizeBytes); - } - - int blockSize = blockInfo.StaticSize switch - { - 4 => BitConverter.ToInt32(sizeBytes, 0), - 2 => BitConverter.ToUInt16(sizeBytes, 0), - 1 => sizeBytes[0], - _ => throw new InvalidDataException($"Unsupported block size field: {blockInfo.StaticSize}") - }; - - currentOffset += 4 + blockInfo.StaticSize + blockSize; - break; - } - } - } - finally - { - foreach (var fs in outputs.Values) - { - fs.Dispose(); - } - } - - return PostProcess(stream, outputs, options.AddHeader); - } - - /// Demultiplexing options. - protected struct DemuxOptions - { - public bool ExtractVideo; - public bool ExtractAudio; - public bool AddHeader; - } -} diff --git a/BydTools.VFS/CriUsm/PacketSizeType.cs b/BydTools.VFS/CriUsm/PacketSizeType.cs deleted file mode 100644 index 8d3eb41..0000000 --- a/BydTools.VFS/CriUsm/PacketSizeType.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace BydTools.VFS.CriUsm; - -/// -/// Defines how packet sizes are determined in MPEG streams. -/// -public enum PacketSizeType -{ - /// Fixed static size. - Static, - - /// Size specified in bytes following the header. - SizeBytes, - - /// End of file marker. - Eof -} diff --git a/BydTools.VFS/CriUsm/UsmNameReader.cs b/BydTools.VFS/CriUsm/UsmNameReader.cs deleted file mode 100644 index 73baaad..0000000 --- a/BydTools.VFS/CriUsm/UsmNameReader.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System.Buffers.Binary; -using BydTools.Utils.Extensions; - -namespace BydTools.VFS.CriUsm; - -/// -/// Extracts embedded file names from CRI USM container headers. -/// When a Video asset in the VFS has an empty fileName, the real name -/// can sometimes be recovered from the CRID / CRIUSF_DIR_STREAM metadata. -/// Ported from BeyondToolsSRC's VFSDump.GetNameFromUsm. -/// -public static class UsmNameReader -{ - private const int CRID_SIGNATURE = 0x44495243; // "CRID" as little-endian int32 - private static readonly byte[] DIR_STREAM_TAG = "CRIUSF_DIR_STREAM"u8.ToArray(); - - /// - /// Attempts to extract a .usm filename from the CRID header block. - /// - /// The filename (without leading root), or null if not found. - public static string? TryGetName(byte[] usmData) - { - if (usmData.Length < 12) - return null; - - var span = usmData.AsSpan(); - - int cridSign = BinaryPrimitives.ReadInt32LittleEndian(span); - if (cridSign != CRID_SIGNATURE) - return null; - - int blockSize = BinaryPrimitives.ReadInt32BigEndian(span[4..]); - short payloadOffset = BinaryPrimitives.ReadInt16BigEndian(span[8..]); - short paddingSize = BinaryPrimitives.ReadInt16BigEndian(span[10..]); - - int dataLen = blockSize - paddingSize - payloadOffset; - if (dataLen <= 0 || 12 + dataLen > usmData.Length) - return null; - - var buff = span.Slice(12, dataLen); - - int dirStreamIndex = buff.IndexOf(DIR_STREAM_TAG); - if (dirStreamIndex < 0) - return null; - - int offset = 18; - while (dirStreamIndex + offset < buff.Length) - { - var str = buff[(dirStreamIndex + offset)..].ReadStringToNull(); - if (str.EndsWith(".usm", StringComparison.OrdinalIgnoreCase)) - { - var root = Path.GetPathRoot(str); - return string.IsNullOrEmpty(root) ? str : str[root.Length..]; - } - offset += str.Length + 1; - } - - return null; - } -} diff --git a/BydTools.VFS/EVFSBlockType.cs b/BydTools.VFS/EVFSBlockType.cs new file mode 100644 index 0000000..98f6b24 --- /dev/null +++ b/BydTools.VFS/EVFSBlockType.cs @@ -0,0 +1,36 @@ +namespace BydTools.VFS; + +/// +/// VFS block types. Member names match groupCfgName in BLC files. +/// Numeric values follow the release client and differ from CBT3. +/// +public enum EVFSBlockType : byte +{ + All = 0, + InitAudio = 1, + InitBundle = 2, + InitialExtendData = 3, + BundleManifest = 4, + IFixPatchOut = 5, + AuditStreaming = 6, + AuditDynamicStreaming = 7, + AuditIV = 8, + AuditAudio = 9, + AuditVideo = 10, + Bundle = 11, + Audio = 12, + Video = 13, + IV = 14, + Streaming = 15, + DynamicStreaming = 16, + Lua = 17, + Table = 18, + JsonData = 19, + ExtendData = 20, + HotfixAudio = 21, + Raw = 100, + AudioChinese = 101, + AudioEnglish = 102, + AudioJapanese = 103, + AudioKorean = 104, +} diff --git a/BydTools.VFS/EVFSFileTag.cs b/BydTools.VFS/EVFSFileTag.cs new file mode 100644 index 0000000..7e96f7c --- /dev/null +++ b/BydTools.VFS/EVFSFileTag.cs @@ -0,0 +1,12 @@ +namespace BydTools.VFS; + +/// +/// File tag introduced with the patch-merge mechanism. Present only when the +/// BLC code version is greater than 3. Names are inferred and not confirmed +/// against the runtime XLua registration. +/// +public enum EVFSFileTag : byte +{ + Base = 0, + Patch = 1, +} diff --git a/BydTools.VFS/IVFSDumper.cs b/BydTools.VFS/IVFSDumper.cs deleted file mode 100644 index b204c33..0000000 --- a/BydTools.VFS/IVFSDumper.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace BydTools.VFS; - -/// -/// Abstraction for VFS block extraction operations. -/// -public interface IVFSDumper -{ - /// - /// Dumps files from a VFS block type to the specified output directory. - /// - void DumpAssetByType(string streamingAssetsPath, EVFSBlockType dumpAssetType, string outputDir); - - /// - /// Scans all subdirectories under the VFS path and prints block declaration info. - /// - void DebugScanBlocks(string streamingAssetsPath); -} diff --git a/BydTools.VFS/Models/EVFSBlockType.cs b/BydTools.VFS/Models/EVFSBlockType.cs deleted file mode 100644 index af83371..0000000 --- a/BydTools.VFS/Models/EVFSBlockType.cs +++ /dev/null @@ -1,171 +0,0 @@ -namespace BydTools.VFS -{ - /// - /// VFS block types as defined in the game's VFS system. - /// Each block type corresponds to a specific category of game assets. - /// The enum member names match the groupCfgName found in BLC files. - /// Values updated for release version (differs from CBT3). - /// - public enum EVFSBlockType : byte - { - /// - /// Used for dumping all block types. - /// - All = 0, - - /// - /// Initial audio assets (AK Package .pck files) - /// Hash: 07A1BB91 - /// - InitAudio = 1, - - /// - /// Initial bundle assets (Asset Bundle .ab files) - /// Hash: 0CE8FA57 - /// - InitBundle = 2, - - /// - /// Initial extended data (.bin files) - /// Hash: 3C9D9D2D - /// - InitialExtendData = 3, - - /// - /// Bundle manifest file (Manifest Map .hgmmap) - /// Hash: 1CDDBF1F - /// - BundleManifest = 4, - - /// - /// IFix patch output (usually empty) - /// Hash: DAFE52C9 - /// - IFixPatchOut = 5, - - /// - /// Audit streaming data (.bytes files) - /// Hash: 6432320A - /// - AuditStreaming = 6, - - /// - /// Audit dynamic streaming data (.bytes files) - /// Hash: B9358E30 - /// - AuditDynamicStreaming = 7, - - /// - /// Audit irradiance volume data (.bytes files) - /// Hash: 06223FE2 - /// - AuditIV = 8, - - /// - /// Audit audio assets (AK Package .pck files) - /// Hash: 1EBAF5C6 - /// - AuditAudio = 9, - - /// - /// Audit video assets (Criware USM .usm files) - /// Hash: 2E6CE44D - /// - AuditVideo = 10, - - /// - /// Main bundle assets (Asset Bundle .ab files) - /// Hash: 7064D8E2 - /// - Bundle = 11, - - /// - /// Main audio assets (AK Package .pck files) - /// Hash: 24ED34CF - /// - Audio = 12, - - /// - /// Video assets (Criware USM .usm files) - /// Hash: 55FC21C6 - /// - Video = 13, - - /// - /// Irradiance Volume data (.bytes files) - /// Hash: A63D7E6A - /// - IV = 14, - - /// - /// Streaming data (.bytes files) - /// Hash: C3442D43 - /// - Streaming = 15, - - /// - /// Dynamic streaming data (.bytes files) - /// Hash: 23D53F5D - /// - DynamicStreaming = 16, - - /// - /// Lua scripts (.lua files, encrypted) - /// Hash: 19E3AE45 - /// - Lua = 17, - - /// - /// Table/data assets (.bytes files, encrypted with SparkBuffer) - /// Hash: 42A8FCA6 - /// - Table = 18, - - /// - /// JSON data (.json files, encrypted) - /// Hash: 775A31D1 - /// - JsonData = 19, - - /// - /// Extended data (.bin files) - /// Hash: D6E622F7 - /// - ExtendData = 20, - - /// - /// Hotfix audio assets - /// Hash: F151B649 - /// - HotfixAudio = 21, - - /// - /// Raw assets - /// - Raw = 100, - - /// - /// Chinese audio assets (AK Package .pck files) - /// Hash: E1E7D7CE - /// - AudioChinese = 101, - - /// - /// English audio assets (AK Package .pck files) - /// Hash: A31457D0 - /// - AudioEnglish = 102, - - /// - /// Japanese audio assets (AK Package .pck files) - /// Hash: F668D4EE - /// - AudioJapanese = 103, - - /// - /// Korean audio assets (AK Package .pck files) - /// Hash: E9D31017 - /// - AudioKorean = 104, - } -} diff --git a/BydTools.VFS/Models/EVFSFileTag.cs b/BydTools.VFS/Models/EVFSFileTag.cs deleted file mode 100644 index a3dee6d..0000000 --- a/BydTools.VFS/Models/EVFSFileTag.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace BydTools.VFS; - -/// -/// File tag for VFS assets, introduced alongside the patch merge mechanism. -/// Binary enum (2 values) registered via manual XLua wrap, not reflection. -/// Actual member names are runtime-obfuscated (StringLiteral_dEDO / StringLiteral_j_OB_); -/// names below are inferred from AddPatchInfo semantics and require runtime -/// confirmation at BeyondVFSEVFSFileTagWrap::__Register. -/// -public enum EVFSFileTag : byte -{ - /// - /// Base package file (inferred). - /// - Base = 0, - - /// - /// Patch / hotfix override file (inferred). - /// - Patch = 1, -} diff --git a/BydTools.VFS/Models/EVFSLoaderPosType.cs b/BydTools.VFS/Models/EVFSLoaderPosType.cs deleted file mode 100644 index 22b0777..0000000 --- a/BydTools.VFS/Models/EVFSLoaderPosType.cs +++ /dev/null @@ -1,43 +0,0 @@ -namespace BydTools.VFS; - -/// -/// Specifies the loader position type for VFS assets. -/// Determines where and how assets are loaded from storage. -/// -public enum EVFSLoaderPosType : byte -{ - /// - /// No specific loader position. - /// - None = 0, - - /// - /// Asset loaded from persistent data path. - /// - PersistAsset = 1, - - /// - /// Asset loaded from streaming assets path. - /// - StreamAsset = 2, - - /// - /// Asset loaded from VFS. - /// - VFS = 10, - - /// - /// Asset loaded from VFS with persistent asset fallback. - /// - VFS_PersistAsset = 11, - - /// - /// Asset loaded from VFS with streaming asset fallback. - /// - VFS_StreamAsset = 12, - - /// - /// Asset loaded from VFS build. - /// - VFS_Build = 13, -} diff --git a/BydTools.VFS/Models/FVFBlockChunkInfo.cs b/BydTools.VFS/Models/FVFBlockChunkInfo.cs deleted file mode 100644 index 3e5824e..0000000 --- a/BydTools.VFS/Models/FVFBlockChunkInfo.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BydTools.VFS; - -/// -/// Represents chunk information within a VFS block. -/// Each FVFBlockChunkInfo corresponds to a single CHK file. -/// -public struct FVFBlockChunkInfo -{ - /// - /// File extension for chunk files. - /// - public const string FILE_EXTENSION = ".chk"; - - /// - /// MD5 hash used as the chunk filename (without extension). - /// When converted to hex string, use big-endian format. - /// - public UInt128 md5Name; - - /// - /// MD5 hash of the chunk content for integrity verification. - /// - public UInt128 contentMD5; - - /// - /// Total length of this chunk in bytes. - /// - public long length; - - /// - /// Block type of this chunk. - /// - public EVFSBlockType blockType; - - /// - /// File tag for this chunk (only present when VFS version > 3). - /// - public EVFSFileTag fileTag; - - /// - /// Array of file information within this chunk. - /// Each file represents a virtual file stored in the CHK. - /// - [JsonIgnore] - public FVFBlockFileInfo[] files; -} diff --git a/BydTools.VFS/Models/FVFBlockFileInfo.cs b/BydTools.VFS/Models/FVFBlockFileInfo.cs deleted file mode 100644 index 6477c14..0000000 --- a/BydTools.VFS/Models/FVFBlockFileInfo.cs +++ /dev/null @@ -1,73 +0,0 @@ -namespace BydTools.VFS; - -/// -/// Represents file information for a virtual file within a VFS chunk. -/// Each FVFBlockFileInfo describes a file's location and encryption status within a CHK file. -/// -public struct FVFBlockFileInfo -{ - /// - /// Virtual file path (e.g., "Data/Bundles/Windows/manifest.hgmmap"). - /// - public string fileName; - - /// - /// Hash of the file name for quick lookup. - /// - public long fileNameHash; - - /// - /// MD5 hash identifying which chunk this file belongs to. - /// Should match the parent chunk's md5Name. - /// - public UInt128 fileChunkMD5Name; - - /// - /// MD5 hash of the file data for integrity verification. - /// - public UInt128 fileDataMD5; - - /// - /// Byte offset of this file within the chunk (CHK file). - /// - public long offset; - - /// - /// Length of the file data in bytes. - /// - public long len; - - /// - /// Block type of this file. - /// - public EVFSBlockType blockType; - - /// - /// Indicates whether this file is encrypted with outer-layer encryption. - /// If true, the file needs to be decrypted using ChaCha20 with ivSeed. - /// Note: This does not indicate whether the file content itself is encrypted - /// (e.g., .ab files may have their own encryption). - /// - public bool bUseEncrypt; - - /// - /// IV seed for ChaCha20 decryption when bUseEncrypt is true. - /// The nonce is constructed as: [version (4 bytes LE)] + [ivSeed (8 bytes LE)] - /// - public long ivSeed; - - /// - /// File tag for this file (only present when VFS version > 3). - /// - public EVFSFileTag fileTag; - - /// - /// Indicates if this is a direct file reference. - /// - public bool bIsDirect; - - /// - /// Loader position type for this file. - /// - public EVFSLoaderPosType loaderPosType; -} diff --git a/BydTools.VFS/Models/VFBlockMainInfo.cs b/BydTools.VFS/Models/VFBlockMainInfo.cs deleted file mode 100644 index 4dda006..0000000 --- a/BydTools.VFS/Models/VFBlockMainInfo.cs +++ /dev/null @@ -1,188 +0,0 @@ -using System.Buffers.Binary; -using System.Runtime.InteropServices; -using System.Text; - -namespace BydTools.VFS; - -/// -/// Represents the main information structure of a VFS block (BLC file). -/// Each VFBlockMainInfo corresponds to a single BLC file. -/// -public class VFBlockMainInfo -{ - /// - /// Parses VFBlockMainInfo from decrypted BLC file bytes. - /// - /// The decrypted byte array containing the BLC data. - /// The starting offset (typically BLOCK_HEAD_LEN after decryption). - public VFBlockMainInfo(byte[] bytes, int offset = 0) - { - // Read version (4 bytes, little-endian) - codeVersion = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); - version = codeVersion; - offset += sizeof(int); - - // Clamp illegal version values (mirrors game logic: version > 10 → forced to 3) - if (codeVersion > 10) - codeVersion = 3; - else - { - version = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(int); - } - - // Read groupCfgName (2 bytes length + UTF-8 string) - ushort groupCfgNameLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(ushort); - groupCfgName = Encoding.UTF8.GetString(bytes.AsSpan(offset, groupCfgNameLength)); - offset += groupCfgNameLength; - - // groupCfgHashName is 8 bytes long but only the first 4 bytes are valid. - groupCfgHashName = BinaryPrimitives - .ReadUInt32BigEndian(bytes.AsSpan(offset)) - .ToString("X8"); - offset += sizeof(long); // Skip full 8 bytes as per the structure - - // Read groupFileInfoNum (4 bytes, little-endian) - groupFileInfoNum = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(int); - - // Read groupChunksLength (8 bytes, little-endian) - groupChunksLength = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(long); - - // Read blockType (1 byte) - blockType = (EVFSBlockType)bytes[offset++]; - - // Read chunk count and allocate array - var chunkCount = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); - allChunks = GC.AllocateUninitializedArray(chunkCount); - offset += sizeof(int); - - // Parse each chunk - foreach (ref var chunk in allChunks.AsSpan()) - { - // Read md5Name (16 bytes as UInt128, little-endian) - chunk.md5Name = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset)); - offset += Marshal.SizeOf(); - - // Read contentMD5 (16 bytes as UInt128, little-endian) - chunk.contentMD5 = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset)); - offset += Marshal.SizeOf(); - - // Read length (8 bytes, little-endian) - chunk.length = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(long); - - // Read blockType (1 byte) - chunk.blockType = (EVFSBlockType)bytes[offset++]; - - // EVFSFileTag (serialized as int32, only low byte is meaningful) - if (codeVersion > 3) - { - chunk.fileTag = (EVFSFileTag)(byte)BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(int); - } - // Read file count and allocate array - var fileCount = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); - chunk.files = GC.AllocateUninitializedArray(fileCount); - offset += sizeof(int); - - // Parse each file in the chunk - foreach (ref var file in chunk.files.AsSpan()) - { - // Read fileName (2 bytes length + UTF-8 string) - ushort fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian( - bytes.AsSpan(offset) - ); - offset += sizeof(ushort); - file.fileName = Encoding.UTF8.GetString(bytes.AsSpan(offset, fileNameLength)); - offset += fileNameLength; - - // Read fileNameHash (8 bytes, little-endian) - file.fileNameHash = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(long); - - // Read fileChunkMD5Name (16 bytes as UInt128, little-endian) - file.fileChunkMD5Name = BinaryPrimitives.ReadUInt128LittleEndian( - bytes.AsSpan(offset) - ); - offset += Marshal.SizeOf(); - - // Read fileDataMD5 (16 bytes as UInt128, little-endian) - file.fileDataMD5 = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset)); - offset += Marshal.SizeOf(); - - // Read offset (8 bytes, little-endian) - file.offset = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(long); - - // Read len (8 bytes, little-endian) - file.len = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(long); - - // Read blockType (1 byte) - file.blockType = (EVFSBlockType)bytes[offset++]; - - // Read bUseEncrypt (1 byte, non-zero is true) - file.bUseEncrypt = bytes[offset++] != 0; - - // If encrypted, read ivSeed (8 bytes, little-endian) - if (file.bUseEncrypt) - { - file.ivSeed = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(long); - } - - // EVFSFileTag (serialized as int32, only low byte is meaningful) - if (codeVersion > 3) - { - file.fileTag = (EVFSFileTag)(byte)BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); - offset += sizeof(int); - } - } - } - } - - /// - /// Blk version. - /// - public int codeVersion; - - /// - /// Some version. - /// - public int version; - - /// - /// The configuration name of this block group (e.g., "BundleManifest", "Audio"). - /// This matches the groupCfgName in the BLC file. - /// - public string groupCfgName; - - /// - /// The hash name of this block group as a hex string (e.g., "1CDDBF1F"). - /// This is used as the directory name in the VFS folder. - /// - public string groupCfgHashName; - - /// - /// The total number of virtual files in this block group. - /// - public int groupFileInfoNum; - - /// - /// The total length of all chunks in this block group in bytes. - /// - public long groupChunksLength; - - /// - /// The block type of this group. - /// - public EVFSBlockType blockType; - - /// - /// Array of chunk information. Each chunk corresponds to a CHK file. - /// - public FVFBlockChunkInfo[] allChunks; -} diff --git a/BydTools.VFS/PostProcessors/IPostProcessor.cs b/BydTools.VFS/PostProcessors/IPostProcessor.cs deleted file mode 100644 index 77f8868..0000000 --- a/BydTools.VFS/PostProcessors/IPostProcessor.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace BydTools.VFS.PostProcessors; - -/// -/// Post-processes extracted VFS file data before writing to disk. -/// Implementations handle format-specific decryption/conversion (e.g. SparkBuffer → JSON, Lua decryption). -/// -public interface IPostProcessor -{ - /// - /// Attempts to post-process the file data. - /// - /// Raw file bytes extracted from a VFS chunk. - /// Target output path (may be changed by the processor). - /// true if the file was handled and written; false to fall back to raw write. - bool TryProcess(byte[] data, string outputPath); -} diff --git a/BydTools.VFS/PostProcessors/LuaPostProcessor.cs b/BydTools.VFS/PostProcessors/LuaPostProcessor.cs deleted file mode 100644 index a4b022d..0000000 --- a/BydTools.VFS/PostProcessors/LuaPostProcessor.cs +++ /dev/null @@ -1,60 +0,0 @@ -using BydTools.Utils; -using BydTools.VFS.xLua; - -namespace BydTools.VFS.PostProcessors; - -/// -/// Decrypts encrypted Lua bytecode files. -/// -public sealed class LuaPostProcessor : IPostProcessor -{ - private readonly ILogger _logger; - private readonly string _masterKey; - - private LuaPostProcessor(ILogger logger, string masterKey) - { - _logger = logger; - _masterKey = masterKey; - } - - /// - /// Creates a if the Lua master key can be derived. - /// Returns null when the key is unavailable. - /// - public static LuaPostProcessor? TryCreate(ILogger logger) - { - var masterKey = LuaDecipher.GetMasterKey(); - if (string.IsNullOrEmpty(masterKey)) - return null; - - logger.Verbose("Lua master key initialized successfully"); - return new LuaPostProcessor(logger, masterKey); - } - - public bool TryProcess(byte[] data, string outputPath) - { - try - { - var base64String = System.Text.Encoding.UTF8.GetString(data).Trim(); - var decryptedLua = LuaDecipher.DecryptLuaWithMasterKey(base64String, _masterKey); - - if (decryptedLua != null && LuaDecipher.IsValidLuaBytecode(decryptedLua)) - { - var luaFilePath = Path.ChangeExtension(outputPath, ".lua"); - File.WriteAllBytes(luaFilePath, decryptedLua); - _logger.Verbose( - $" Decrypted Lua: {Path.GetFileName(outputPath)} -> {Path.GetFileName(luaFilePath)}" - ); - return true; - } - } - catch (Exception ex) - { - _logger.Verbose( - $" Lua decryption failed for {Path.GetFileName(outputPath)}: {ex.Message}" - ); - } - - return false; - } -} diff --git a/BydTools.VFS/PostProcessors/PckPostProcessor.cs b/BydTools.VFS/PostProcessors/PckPostProcessor.cs deleted file mode 100644 index d34a48d..0000000 --- a/BydTools.VFS/PostProcessors/PckPostProcessor.cs +++ /dev/null @@ -1,45 +0,0 @@ -using BydTools.PCK; -using BydTools.Utils; - -namespace BydTools.VFS.PostProcessors; - -/// -/// Decrypts VFS-encrypted .pck payloads to plain AKPK files. -/// -public sealed class PckPostProcessor : IPostProcessor -{ - private readonly ILogger _logger; - - public PckPostProcessor(ILogger logger) - { - _logger = logger; - } - - public bool TryProcess(byte[] data, string outputPath) - { - if (!outputPath.EndsWith(".pck", StringComparison.OrdinalIgnoreCase)) - return false; - - try - { - using var ms = new MemoryStream(data, writable: false); - var parser = new PckParser(ms); - byte[] decrypted = parser.GetDecryptedPckBytes(); - - File.WriteAllBytes(outputPath, decrypted); - if (parser.IsVfsEncrypted) - _logger.Verbose(" Decrypted PCK: {0}", Path.GetFileName(outputPath)); - - return true; - } - catch (Exception ex) - { - _logger.Verbose( - " PCK decryption failed for {0}: {1}", - Path.GetFileName(outputPath), - ex.Message - ); - return false; - } - } -} diff --git a/BydTools.VFS/PostProcessors/PostProcessorFactory.cs b/BydTools.VFS/PostProcessors/PostProcessorFactory.cs deleted file mode 100644 index 83df0b4..0000000 --- a/BydTools.VFS/PostProcessors/PostProcessorFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -using BydTools.Utils; - -namespace BydTools.VFS.PostProcessors; - -/// -/// Creates and registers instances -/// for each supported . -/// -public static class PostProcessorFactory -{ - public static Dictionary CreateProcessors(ILogger logger) - { - var processors = new Dictionary - { - { EVFSBlockType.Table, new SparkBufferPostProcessor(logger) }, - { EVFSBlockType.Video, new VideoPostProcessor(logger) }, - { EVFSBlockType.InitAudio, new PckPostProcessor(logger) }, - { EVFSBlockType.Audio, new PckPostProcessor(logger) }, - { EVFSBlockType.AudioChinese, new PckPostProcessor(logger) }, - { EVFSBlockType.AudioEnglish, new PckPostProcessor(logger) }, - { EVFSBlockType.AudioJapanese, new PckPostProcessor(logger) }, - { EVFSBlockType.AudioKorean, new PckPostProcessor(logger) }, - { EVFSBlockType.AuditAudio, new PckPostProcessor(logger) }, - { EVFSBlockType.HotfixAudio, new PckPostProcessor(logger) }, - }; - - var luaProcessor = LuaPostProcessor.TryCreate(logger); - if (luaProcessor != null) - processors[EVFSBlockType.Lua] = luaProcessor; - - return processors; - } -} diff --git a/BydTools.VFS/PostProcessors/SparkBufferPostProcessor.cs b/BydTools.VFS/PostProcessors/SparkBufferPostProcessor.cs deleted file mode 100644 index 5e6c4ca..0000000 --- a/BydTools.VFS/PostProcessors/SparkBufferPostProcessor.cs +++ /dev/null @@ -1,45 +0,0 @@ -using BydTools.Utils; -using BydTools.VFS.SparkBuffer; - -namespace BydTools.VFS.PostProcessors; - -/// -/// Decrypts SparkBuffer .bytes files to JSON. -/// -public sealed class SparkBufferPostProcessor : IPostProcessor -{ - private readonly ILogger _logger; - - public SparkBufferPostProcessor(ILogger logger) - { - _logger = logger; - } - - public bool TryProcess(byte[] data, string outputPath) - { - if (!Path.GetExtension(outputPath).Equals(".bytes", StringComparison.OrdinalIgnoreCase)) - return false; - - try - { - var decryptedJson = SparkBufferDumper.Decrypt(data); - if (!string.IsNullOrEmpty(decryptedJson)) - { - var jsonFilePath = Path.ChangeExtension(outputPath, ".json"); - File.WriteAllText(jsonFilePath, decryptedJson); - _logger.Verbose( - $" Decrypted SparkBuffer: {Path.GetFileName(outputPath)} -> {Path.GetFileName(jsonFilePath)}" - ); - return true; - } - } - catch (Exception ex) - { - _logger.Verbose( - $" SparkBuffer decryption failed for {Path.GetFileName(outputPath)}: {ex.Message}" - ); - } - - return false; - } -} diff --git a/BydTools.VFS/PostProcessors/VideoPostProcessor.cs b/BydTools.VFS/PostProcessors/VideoPostProcessor.cs deleted file mode 100644 index 0fff09e..0000000 --- a/BydTools.VFS/PostProcessors/VideoPostProcessor.cs +++ /dev/null @@ -1,43 +0,0 @@ -using BydTools.Utils; -using BydTools.VFS.CriUsm; - -namespace BydTools.VFS.PostProcessors; - -/// -/// Post-processor for Video (USM) assets. -/// Demultiplexes CRI USM containers into separate video (.m2v) and audio (.adx/.hca/.aix) streams. -/// -public sealed class VideoPostProcessor : IPostProcessor -{ - private readonly ILogger _logger; - - public VideoPostProcessor(ILogger logger) => _logger = logger; - - public bool TryProcess(byte[] data, string outputPath) - { - if (!outputPath.EndsWith(".usm", StringComparison.OrdinalIgnoreCase)) - return false; - - string basePath = Path.ChangeExtension(outputPath, null); - - try - { - var files = CriUsmDemuxer.Demux(data, basePath); - if (files.Length == 0) - { - _logger.Verbose(" USM demux produced no output, falling back to raw write"); - return false; - } - - foreach (var file in files) - _logger.Verbose(" Demuxed: {0}", Path.GetFileName(file)); - - return true; - } - catch (Exception ex) - { - _logger.Verbose(" USM demux failed: {0}", ex.Message); - return false; - } - } -} diff --git a/BydTools.VFS/SparkBuffer/BeanType.cs b/BydTools.VFS/SparkBuffer/BeanType.cs deleted file mode 100644 index 5823d84..0000000 --- a/BydTools.VFS/SparkBuffer/BeanType.cs +++ /dev/null @@ -1,99 +0,0 @@ -namespace BydTools.VFS.SparkBuffer -{ - public struct BeanType - { - public int typeHash; - public string name; - public Field[] fields; - - public BeanType(BinaryReader reader) - { - typeHash = reader.ReadInt32(); - name = reader.ReadSparkBufferString(); - reader.Align4Bytes(); - var fieldCount = reader.ReadInt32(); - fields = new Field[fieldCount]; - - foreach (ref var field in fields.AsSpan()) - { - field.name = reader.ReadSparkBufferString(); - field.type = reader.ReadSparkType(); - switch (field.type) - { - case SparkType.Bool: - case SparkType.Byte: - case SparkType.Int: - case SparkType.Long: - case SparkType.Float: - case SparkType.Double: - case SparkType.String: - break; - case SparkType.Enum: - case SparkType.Bean: - reader.Align4Bytes(); - field.typeHash = reader.ReadInt32(); - break; - case SparkType.Array: - field.type2 = reader.ReadSparkType(); - - if (field.type2.Value.IsEnumOrBeanType()) - { - reader.Align4Bytes(); - field.typeHash = reader.ReadInt32(); - } - break; - case SparkType.Map: - field.type2 = reader.ReadSparkType(); - field.type3 = reader.ReadSparkType(); - - if (field.type2.Value.IsEnumOrBeanType()) - { - reader.Align4Bytes(); - field.typeHash = reader.ReadInt32(); - } - if (field.type3.Value.IsEnumOrBeanType()) - { - reader.Align4Bytes(); - field.typeHash2 = reader.ReadInt32(); - } - - break; - default: - throw new Exception( - string.Format( - "Unsupported bean field type {0} at position {1}", - field.type, - reader.BaseStream.Position - ) - ); - } - } - } - - public struct Field - { - public string name; - public SparkType type; - - /// - /// or key type - /// - public SparkType? type2; - - /// - /// value type - /// - public SparkType? type3; - - /// - /// , , , or key type hash - /// - public int? typeHash; - - /// - /// value type hash - /// - public int? typeHash2; - } - } -} diff --git a/BydTools.VFS/SparkBuffer/BinaryReaderExtensions.cs b/BydTools.VFS/SparkBuffer/BinaryReaderExtensions.cs deleted file mode 100644 index 06a175c..0000000 --- a/BydTools.VFS/SparkBuffer/BinaryReaderExtensions.cs +++ /dev/null @@ -1,85 +0,0 @@ -using System.Text; - -namespace BydTools.VFS.SparkBuffer -{ - public static class BinaryReaderExtensions - { - public static SparkType ReadSparkType(this BinaryReader reader) => - (SparkType)reader.ReadByte(); - - public static long Seek( - this BinaryReader reader, - long pos, - SeekOrigin seekOrigin = SeekOrigin.Begin - ) => reader.BaseStream.Seek(pos, seekOrigin); - - public static string ReadSparkBufferString(this BinaryReader reader) - { - using MemoryStream buffer = new(); - while (true) - { - byte b = reader.ReadByte(); - if (b == 0) - break; - buffer.WriteByte(b); - } - - return Encoding.UTF8.GetString(buffer.ToArray()); - } - - public static string ReadSparkBufferStringOffset(this BinaryReader reader) - { - var stringOffset = reader.ReadInt32(); - if (stringOffset == -1) - return string.Empty; - - var oldPosition = reader.BaseStream.Position; - - reader.Seek(stringOffset); - var tmp = reader.ReadSparkBufferString(); - - reader.BaseStream.Position = oldPosition; - return tmp; - } - - public static string ReadSparkBufferEnum(this BinaryReader reader, EnumType type) - { - var value = reader.ReadInt32(); - return type - .enums.FirstOrDefault( - x => x.value == value, - new EnumType.EnumItem { name = value.ToString(), value = value } - ) - .name; - } - - public static bool ReadSparkBufferBool(this BinaryReader reader) - { - var value = reader.ReadBoolean(); - reader.Align4Bytes(); - return value; - } - - public static long ReadSparkBufferLong(this BinaryReader reader) - { - reader.Align8Bytes(); - return reader.ReadInt64(); - } - - public static double ReadSparkBufferDouble(this BinaryReader reader) - { - reader.Align8Bytes(); - return reader.ReadDouble(); - } - - public static void Align4Bytes(this BinaryReader reader) => - reader.Seek( - (reader.BaseStream.Position - 1) + (4 - ((reader.BaseStream.Position - 1) % 4)) - ); - - public static void Align8Bytes(this BinaryReader reader) => - reader.Seek( - (reader.BaseStream.Position - 1) + (8 - ((reader.BaseStream.Position - 1) % 8)) - ); - } -} diff --git a/BydTools.VFS/SparkBuffer/EnumType.cs b/BydTools.VFS/SparkBuffer/EnumType.cs deleted file mode 100644 index f3a73d8..0000000 --- a/BydTools.VFS/SparkBuffer/EnumType.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace BydTools.VFS.SparkBuffer -{ - public struct EnumType - { - public int typeHash; - public string name; - public EnumItem[] enums; - - public EnumType(BinaryReader reader) - { - typeHash = reader.ReadInt32(); - name = reader.ReadSparkBufferString(); - reader.Align4Bytes(); - var enumCount = reader.ReadInt32(); - enums = new EnumItem[enumCount]; - - foreach (ref var enumItem in enums.AsSpan()) - { - enumItem.name = reader.ReadSparkBufferString(); - reader.Align4Bytes(); - enumItem.value = reader.ReadInt32(); - } - } - - public struct EnumItem - { - public string name; - public int value; - } - } -} diff --git a/BydTools.VFS/SparkBuffer/SparkBufferDumper.cs b/BydTools.VFS/SparkBuffer/SparkBufferDumper.cs deleted file mode 100644 index 4a4dabd..0000000 --- a/BydTools.VFS/SparkBuffer/SparkBufferDumper.cs +++ /dev/null @@ -1,317 +0,0 @@ -using System.Text.Json.Nodes; - -namespace BydTools.VFS.SparkBuffer -{ - /// - /// Provides functionality to dump SparkBuffer binary files to JSON format. - /// - public class SparkBufferDumper - { - /// - /// Decrypts SparkBuffer data from bytes and returns string representation. - /// The output format depends on the SparkBuffer content structure (JSON format). - /// - /// The SparkBuffer binary data. - /// String representation of the SparkBuffer data. - public static string Decrypt(byte[] data) - { - using var memoryStream = new MemoryStream(data); - using var binaryReader = new BinaryReader(memoryStream); - return Decrypt(binaryReader); - } - - /// - /// Decrypts SparkBuffer data from BinaryReader and returns string representation. - /// The output format depends on the SparkBuffer content structure (JSON format). - /// - /// BinaryReader positioned at the start of the SparkBuffer file. - /// String representation of the SparkBuffer data. - public static string Decrypt(BinaryReader binaryReader) - { - // Clear previous type definitions to avoid conflicts - SparkManager.ClearTypeDefinitions(); - - var typeDefOffset = binaryReader.ReadInt32(); - var rootDefOffset = binaryReader.ReadInt32(); - var dataOffset = binaryReader.ReadInt32(); - - binaryReader.Seek(typeDefOffset); - SparkManager.ReadTypeDefinitions(binaryReader); - - binaryReader.Seek(rootDefOffset); - var rootDef = new BeanType.Field - { - type = binaryReader.ReadSparkType(), - name = binaryReader.ReadSparkBufferString(), - }; - - if (rootDef.type.IsEnumOrBeanType()) - { - binaryReader.Align4Bytes(); - rootDef.typeHash = binaryReader.ReadInt32(); - } - if (rootDef.type == SparkType.Map) - { - rootDef.type2 = binaryReader.ReadSparkType(); - rootDef.type3 = binaryReader.ReadSparkType(); - - if (rootDef.type2.Value.IsEnumOrBeanType()) - { - binaryReader.Align4Bytes(); - rootDef.typeHash = binaryReader.ReadInt32(); - } - if (rootDef.type3.Value.IsEnumOrBeanType()) - { - binaryReader.Align4Bytes(); - rootDef.typeHash2 = binaryReader.ReadInt32(); - } - } - - binaryReader.Seek(dataOffset); - switch (rootDef.type) - { - case SparkType.Bean: - var rootBeanType = SparkManager.BeanTypeFromHash((int)rootDef.typeHash!); - var beanDump = ReadBeanAsJObject(binaryReader, rootBeanType); - return beanDump!.ToJsonString(SparkManager.jsonSerializerOptions); - case SparkType.Map: - var mapDump = ReadMapAsJObject(binaryReader, rootDef); - return mapDump!.ToJsonString(SparkManager.jsonSerializerOptions); - default: - throw new NotSupportedException( - string.Format("Unsupported root type {0}", rootDef.type) - ); - } - } - - /// - /// Reads the root definition name from a SparkBuffer without fully parsing it. - /// - /// BinaryReader positioned at the start of the SparkBuffer file. - /// The name of the root definition. - public static string GetRootDefinitionName(BinaryReader binaryReader) - { - var originalPosition = binaryReader.BaseStream.Position; - - // Read offsets - binaryReader.ReadInt32(); // typeDefOffset - var rootDefOffset = binaryReader.ReadInt32(); - - // Read root definition to get the name - binaryReader.Seek(rootDefOffset); - binaryReader.ReadSparkType(); // type - var name = binaryReader.ReadSparkBufferString(); - - // Reset to original position - binaryReader.BaseStream.Position = originalPosition; - - return name; - } - - private static JsonObject? ReadMapAsJObject( - BinaryReader binaryReader, - BeanType.Field typeDef - ) - { - var mapDump = new JsonObject(); - var kvCount = binaryReader.ReadInt32(); - binaryReader.Seek(kvCount * 8, SeekOrigin.Current); - - for (int i = 0; i < kvCount; i++) - { - var key = typeDef.type2 switch - { - SparkType.String => binaryReader.ReadSparkBufferStringOffset(), - SparkType.Int => binaryReader.ReadInt32().ToString(), - SparkType.Long => binaryReader.ReadSparkBufferLong().ToString(), - _ => throw new NotSupportedException( - string.Format("Unsupported map key type {0}", typeDef.type2) - ), - }; - mapDump[key] = null; - - mapDump[key] = typeDef.type3 switch - { - SparkType.Bean => ReadBeanAsJObject( - binaryReader, - SparkManager.BeanTypeFromHash((int)typeDef.typeHash2!), - true - ), - SparkType.String => binaryReader.ReadSparkBufferStringOffset(), - SparkType.Int => binaryReader.ReadInt32(), - SparkType.Float => binaryReader.ReadSingle(), - SparkType.Enum => binaryReader.ReadSparkBufferEnum( - SparkManager.EnumTypeFromHash(typeDef.typeHash2!.Value) - ), - SparkType.Bool => binaryReader.ReadSparkBufferBool(), - _ => throw new NotSupportedException( - string.Format("Unsupported map value type {0}", typeDef.type3) - ), - }; - } - - return mapDump; - } - - private static JsonObject? ReadBeanAsJObject( - BinaryReader binaryReader, - BeanType beanType, - bool pointer = false - ) - { - long? pointerOrigin = null; - if (pointer) - { - var beanOffset = binaryReader.ReadInt32(); - if (beanOffset == -1) - return null; - - pointerOrigin = binaryReader.BaseStream.Position; - binaryReader.Seek(beanOffset); - } - - var dumpObj = new JsonObject(); - - foreach (var (fieldIndex, beanField) in beanType.fields.Index()) - { - long? origin = null; - if (beanField.type == SparkType.Array) - { - var fieldOffset = binaryReader.ReadInt32(); - if (fieldOffset == -1) - { - dumpObj[beanField.name] = null; - continue; - } - - origin = binaryReader.BaseStream.Position; - binaryReader.Seek(fieldOffset); - } - - switch (beanField.type) - { - case SparkType.Array: - var jArray = new JsonArray(); - - var itemCount = binaryReader.ReadInt32(); - while (itemCount-- > 0) - { - switch (beanField.type2) - { - case SparkType.String: - jArray.Add( - (JsonNode?) - JsonValue.Create( - binaryReader.ReadSparkBufferStringOffset() - ) - ); - break; - case SparkType.Bean: - jArray.Add( - (JsonNode?)ReadBeanAsJObject( - binaryReader, - SparkManager.BeanTypeFromHash((int)beanField.typeHash!), - true - ) - ); - break; - case SparkType.Float: - jArray.Add( - (JsonNode?)JsonValue.Create(binaryReader.ReadSingle()) - ); - break; - case SparkType.Long: - jArray.Add( - (JsonNode?) - JsonValue.Create(binaryReader.ReadSparkBufferLong()) - ); - break; - case SparkType.Int: - case SparkType.Enum: - jArray.Add( - (JsonNode?)JsonValue.Create(binaryReader.ReadInt32()) - ); - break; - case SparkType.Bool: - jArray.Add( - (JsonNode?)JsonValue.Create(binaryReader.ReadBoolean()) - ); - break; - case SparkType.Double: - jArray.Add( - (JsonNode?) - JsonValue.Create(binaryReader.ReadSparkBufferDouble()) - ); - break; - default: - throw new NotSupportedException( - string.Format( - "Unsupported array type {0} on bean array field, position: {1}", - beanField.type2, - binaryReader.BaseStream.Position - ) - ); - } - } - - dumpObj[beanField.name] = jArray; - break; - case SparkType.Int: - case SparkType.Enum: - dumpObj[beanField.name] = binaryReader.ReadInt32(); - break; - case SparkType.Long: - dumpObj[beanField.name] = binaryReader.ReadSparkBufferLong(); - break; - case SparkType.Float: - dumpObj[beanField.name] = binaryReader.ReadSingle(); - break; - case SparkType.Double: - dumpObj[beanField.name] = binaryReader.ReadSparkBufferDouble(); - break; - case SparkType.String: - dumpObj[beanField.name] = binaryReader.ReadSparkBufferStringOffset(); - break; - case SparkType.Bean: - dumpObj[beanField.name] = ReadBeanAsJObject( - binaryReader, - SparkManager.BeanTypeFromHash((int)beanField.typeHash!), - true - ); - break; - case SparkType.Bool: - dumpObj[beanField.name] = binaryReader.ReadBoolean(); - if ( - beanType.fields.Length > fieldIndex + 1 - && beanType.fields[fieldIndex + 1].type != SparkType.Bool - ) - binaryReader.Align4Bytes(); - break; - case SparkType.Map: - var mapOffset = binaryReader.ReadInt32(); - var mapOrigin = binaryReader.BaseStream.Position; - binaryReader.Seek(mapOffset); - dumpObj[beanField.name] = ReadMapAsJObject(binaryReader, beanField); - binaryReader.Seek(mapOrigin); - break; - case SparkType.Byte: - throw new Exception( - string.Format( - "Dumping bean field type {0} isn't supported, position: {1}", - beanField.type, - binaryReader.BaseStream.Position - ) - ); - } - - if (origin is not null) - binaryReader.BaseStream.Position = (long)origin; - } - - if (pointerOrigin is not null) - binaryReader.BaseStream.Position = (long)pointerOrigin; - - return dumpObj; - } - } -} diff --git a/BydTools.VFS/SparkBuffer/SparkManager.cs b/BydTools.VFS/SparkBuffer/SparkManager.cs deleted file mode 100644 index 3826bfb..0000000 --- a/BydTools.VFS/SparkBuffer/SparkManager.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System.Text.Encodings.Web; -using System.Text.Json; - -namespace BydTools.VFS.SparkBuffer -{ - public static class SparkManager - { - public static readonly JsonSerializerOptions jsonSerializerOptions = new() - { - WriteIndented = true, - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; - - private static readonly Dictionary beanTypeMap = []; - private static readonly Dictionary enumTypeMap = []; - - /// - /// Clears all type definitions. Should be called before reading a new SparkBuffer file. - /// - public static void ClearTypeDefinitions() - { - beanTypeMap.Clear(); - enumTypeMap.Clear(); - } - - public static BeanType BeanTypeFromHash(int hash) => beanTypeMap[hash]; - - public static EnumType EnumTypeFromHash(int hash) => enumTypeMap[hash]; - - public static void ReadTypeDefinitions(BinaryReader reader) - { - var typeDefCount = reader.ReadInt32(); - while (typeDefCount-- > 0) - { - var sparkType = reader.ReadSparkType(); - reader.Align4Bytes(); - - switch (sparkType) - { - case SparkType.Enum: - { - var enumType = new EnumType(reader); - enumTypeMap.TryAdd(enumType.typeHash, enumType); - break; - } - case SparkType.Bean: - { - var beanType = new BeanType(reader); - beanTypeMap.TryAdd(beanType.typeHash, beanType); - break; - } - default: -#pragma warning disable CA2208 - throw new ArgumentOutOfRangeException( - nameof(sparkType), - sparkType.ToString(), - "Invalid spark type on type definition section" - ); -#pragma warning restore CA2208 - } - } - } - } -} diff --git a/BydTools.VFS/SparkBuffer/SparkType.cs b/BydTools.VFS/SparkBuffer/SparkType.cs deleted file mode 100644 index 036684f..0000000 --- a/BydTools.VFS/SparkBuffer/SparkType.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace BydTools.VFS.SparkBuffer -{ - public enum SparkType : byte - { - Bool, - Byte, - Int, - Long, - Float, - Double, - Enum, - String, - Bean, - Array, - Map, - } -} diff --git a/BydTools.VFS/SparkBuffer/SparkTypeExtensions.cs b/BydTools.VFS/SparkBuffer/SparkTypeExtensions.cs deleted file mode 100644 index 13c883d..0000000 --- a/BydTools.VFS/SparkBuffer/SparkTypeExtensions.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BydTools.VFS.SparkBuffer -{ - public static class SparkTypeExtensions - { - public static bool IsEnumOrBeanType(this SparkType type) => - type is SparkType.Enum or SparkType.Bean; - } -} diff --git a/BydTools.VFS/VFSDefine.cs b/BydTools.VFS/VFSDefine.cs index 72f0e82..6c91629 100644 --- a/BydTools.VFS/VFSDefine.cs +++ b/BydTools.VFS/VFSDefine.cs @@ -1,50 +1,47 @@ namespace BydTools.VFS; -/// -/// Constants used for VFS (Virtual File System) operations. -/// Based on reverse engineering of Arknights: Endfield CBT3. -/// public static class VFSDefine { - /// - /// ChaCha20 encryption key in Base64 format (PC / default). - /// - public const string CHACHA_KEY_PC_BASE64 = "6VsxesT4KFadI6hr8nHctT6Eb6dckk1nHbqOOPTKUuE="; - - /// - /// ChaCha20 encryption key in Base64 format (Android). - /// - public const string CHACHA_KEY_ANDROID_BASE64 = "eU1cu+MYQiaYdVherRzV86pv/N/lIU/9gIk+5n5Vj4Y="; - - /// - /// Pre-decoded default (PC) ChaCha20 key bytes. - /// - public static readonly byte[] DefaultChaChaKey = Convert.FromBase64String(CHACHA_KEY_PC_BASE64); - - /// - /// VFS directory name within StreamingAssets. - /// - public const string VFS_DIR = "VFS"; - - /// - /// VFS protocol version. - /// - public const int VFS_PROTO_VERSION = 3; - - /// - /// Minimum size of a valid VFB (VFS Block) file header. - /// - public const int VFS_VFB_HEAD_LEN = 16; - - /// - /// Size of the block header used as nonce for ChaCha20 encryption. - /// For BLC: First 12 bytes are used as nonce. - /// For CHK outer-layer encryption: 4 bytes version + 8 bytes ivSeed = 12 bytes nonce. - /// - public const int BLOCK_HEAD_LEN = 12; - - /// - /// ChaCha20 key length in bytes (256 bits). - /// - public const int KEY_LEN = 32; + public const string ChaChaKeyPcBase64 = "6VsxesT4KFadI6hr8nHctT6Eb6dckk1nHbqOOPTKUuE="; + public const string ChaChaKeyAndroidBase64 = "eU1cu+MYQiaYdVherRzV86pv/N/lIU/9gIk+5n5Vj4Y="; + + public static readonly byte[] PcKey = Convert.FromBase64String(ChaChaKeyPcBase64); + public static readonly byte[] AndroidKey = Convert.FromBase64String(ChaChaKeyAndroidBase64); + + public const string VfsDirectoryName = "VFS"; + public const int ProtocolVersion = 3; + public const int NonceLength = 12; + public const int KeyLength = 32; + + public static byte[] ResolveKey(string? platform, string? keyBase64) + { + if (!string.IsNullOrWhiteSpace(keyBase64)) + { + byte[] key; + try + { + key = Convert.FromBase64String(keyBase64); + } + catch (FormatException) + { + throw new ArgumentException("--key must be a valid Base64 string."); + } + + if (key.Length != KeyLength) + { + throw new ArgumentException( + $"--key must decode to {KeyLength} bytes (got {key.Length})." + ); + } + + return key; + } + + return (platform ?? "pc").ToLowerInvariant() switch + { + "pc" => PcKey, + "android" => AndroidKey, + _ => throw new ArgumentException("--platform must be one of: pc, android."), + }; + } } diff --git a/BydTools.VFS/VFSDumper.cs b/BydTools.VFS/VFSDumper.cs deleted file mode 100644 index e1cb55a..0000000 --- a/BydTools.VFS/VFSDumper.cs +++ /dev/null @@ -1,294 +0,0 @@ -using System.Collections.Frozen; -using BydTools.Utils; -using BydTools.Utils.Extensions; -using BydTools.VFS.CriUsm; -using BydTools.VFS.PostProcessors; - -namespace BydTools.VFS; - -/// -/// Extracts files from VFS blocks with pluggable post-processing. -/// -public class VFSDumper : IVFSDumper -{ - private readonly ILogger _logger; - private readonly IReadOnlyDictionary _postProcessors; - private readonly byte[] _chaChaKey; - - public VFSDumper( - ILogger logger, - IReadOnlyDictionary postProcessors, - byte[]? customKey = null - ) - { - _logger = logger; - _postProcessors = postProcessors; - _chaChaKey = customKey ?? VFSDefine.DefaultChaChaKey; - } - - /// - /// Maps EVFSBlockType to groupCfgHashName (directory name). - /// Frozen for thread-safety and optimized read performance. - /// - private static readonly FrozenDictionary blockHashMap = new Dictionary< - EVFSBlockType, - string - > - { - { EVFSBlockType.InitAudio, "07A1BB91" }, - { EVFSBlockType.InitBundle, "0CE8FA57" }, - { EVFSBlockType.BundleManifest, "1CDDBF1F" }, - { EVFSBlockType.InitialExtendData, "3C9D9D2D" }, - { EVFSBlockType.Audio, "24ED34CF" }, - { EVFSBlockType.Bundle, "7064D8E2" }, - { EVFSBlockType.DynamicStreaming, "23D53F5D" }, - { EVFSBlockType.Table, "42A8FCA6" }, - { EVFSBlockType.Video, "55FC21C6" }, - { EVFSBlockType.IV, "A63D7E6A" }, - { EVFSBlockType.Streaming, "C3442D43" }, - { EVFSBlockType.JsonData, "775A31D1" }, - { EVFSBlockType.Lua, "19E3AE45" }, - { EVFSBlockType.IFixPatchOut, "DAFE52C9" }, - { EVFSBlockType.ExtendData, "D6E622F7" }, - { EVFSBlockType.AudioChinese, "E1E7D7CE" }, - { EVFSBlockType.AudioEnglish, "A31457D0" }, - { EVFSBlockType.AudioJapanese, "F668D4EE" }, - { EVFSBlockType.AudioKorean, "E9D31017" }, - { EVFSBlockType.AuditAudio, "1EBAF5C6" }, - { EVFSBlockType.AuditDynamicStreaming, "B9358E30" }, - { EVFSBlockType.AuditIV, "06223FE2" }, - { EVFSBlockType.AuditStreaming, "6432320A" }, - { EVFSBlockType.AuditVideo, "2E6CE44D" }, - { EVFSBlockType.HotfixAudio, "F151B649" }, - }.ToFrozenDictionary(); - - public static IReadOnlyDictionary BlockHashMap => blockHashMap; - - private VFBlockMainInfo ReadBlockInfo(string streamingAssetsPath, EVFSBlockType blockType) => - VfsReader.ReadBlockInfo(streamingAssetsPath, blockType, _chaChaKey); - - private byte[] ReadFileData(FileStream chunkFs, in FVFBlockFileInfo file) => - VfsReader.ReadFileData(chunkFs, file, _chaChaKey); - - public void DumpAssetByType( - string streamingAssetsPath, - EVFSBlockType dumpAssetType, - string outputDir - ) - { - _logger.Info("--- {0} ({1}) ---", dumpAssetType, (byte)dumpAssetType); - - var vfBlockMainInfo = ReadBlockInfo(streamingAssetsPath, dumpAssetType); - var hashName = blockHashMap[dumpAssetType]; - var blockDir = Path.Combine(streamingAssetsPath, hashName); - - var fileTypeCounts = new Dictionary(); - int totalFiles = 0; - foreach (var chunk in vfBlockMainInfo.allChunks) - { - totalFiles += chunk.files.Length; - foreach (var file in chunk.files) - { - var ext = Path.GetExtension(file.fileName).ToLowerInvariant(); - if (string.IsNullOrEmpty(ext)) - ext = "(no extension)"; - fileTypeCounts.TryGetValue(ext, out var count); - fileTypeCounts[ext] = count + 1; - } - } - - var typeDetails = string.Join(", ", fileTypeCounts.Select(kvp => $"{kvp.Value} {kvp.Key}")); - _logger.Info($"Found {totalFiles} files ({typeDetails})"); - - DumpBlockInfo(vfBlockMainInfo); - - var createdDirs = new HashSet(); - _logger.Info("Extracting..."); - - _postProcessors.TryGetValue(dumpAssetType, out var processor); - - int extractedCount = 0; - foreach (var chunk in vfBlockMainInfo.allChunks) - { - var chunkMd5Name = - chunk.md5Name.ToHexStringLittleEndian() + FVFBlockChunkInfo.FILE_EXTENSION; - var chunkFilePath = Path.Combine(blockDir, chunkMd5Name); - - if (!File.Exists(chunkFilePath)) - { - _logger.Verbose($" Chunk file not found: {chunkMd5Name}"); - continue; - } - - using var chunkFs = File.OpenRead(chunkFilePath); - - foreach (var file in chunk.files) - { - chunkFs.Seek(file.offset, SeekOrigin.Begin); - var fileData = ReadFileData(chunkFs, file); - - var fileName = file.fileName; - if (string.IsNullOrEmpty(fileName) && dumpAssetType == EVFSBlockType.Video) - { - var usmName = - UsmNameReader.TryGetName(fileData) ?? $"{file.fileNameHash:X16}.usm"; - fileName = $"Video/{usmName}"; - _logger.Verbose(" Recovered USM name: {0}", fileName); - } - - var filePath = Path.Combine(outputDir, fileName); - var fileDir = Path.GetDirectoryName(filePath); - - if (!string.IsNullOrEmpty(fileDir) && !Directory.Exists(fileDir)) - { - Directory.CreateDirectory(fileDir); - if (createdDirs.Add(fileDir)) - _logger.Info($" Created directory: {fileDir}"); - } - - if (processor != null && processor.TryProcess(fileData, filePath)) - { - extractedCount++; - continue; - } - - File.WriteAllBytes(filePath, fileData); - extractedCount++; - } - - _logger.Verbose( - " Dumped {0} file(s) from chunk {1}", - chunk.files.Length, - chunkMd5Name - ); - } - - _logger.Info($"Done, {extractedCount} files extracted."); - } - - private void DumpBlockInfo(VFBlockMainInfo info) - { - _logger.Verbose("========== BLC INFO =========="); - _logger.Verbose("GroupCfgName : {0}", info.groupCfgName); - _logger.Verbose("GroupCfgHash : {0}", info.groupCfgHashName); - _logger.Verbose("Version : {0}", info.codeVersion); - _logger.Verbose("BlockType : {0} ({1})", info.blockType, (byte)info.blockType); - _logger.Verbose("FileInfoNum : {0}", info.groupFileInfoNum); - _logger.Verbose("ChunksLength : {0}", info.groupChunksLength); - _logger.Verbose("ChunksCount : {0}", info.allChunks.Length); - _logger.Verbose("------------------------------"); - - for (int i = 0; i < info.allChunks.Length; i++) - { - ref readonly var chunk = ref info.allChunks[i]; - _logger.Verbose("Chunk #{0}:", i); - _logger.Verbose(" md5Name : {0}", chunk.md5Name.ToHexStringLittleEndian()); - _logger.Verbose(" contentMD5 : {0}", chunk.contentMD5.ToHexStringLittleEndian()); - _logger.Verbose(" length : {0}", chunk.length); - _logger.Verbose(" blockType : {0} ({1})", chunk.blockType, (byte)chunk.blockType); - _logger.Verbose(" fileTag : {0} ({1})", chunk.fileTag, (byte)chunk.fileTag); - _logger.Verbose(" filesCount : {0}", chunk.files.Length); - - for (int j = 0; j < chunk.files.Length; j++) - { - ref readonly var file = ref chunk.files[j]; - _logger.Verbose(" File #{0}:", j); - _logger.Verbose(" name : {0}", file.fileName); - _logger.Verbose(" nameHash : 0x{0:X16}", file.fileNameHash); - _logger.Verbose( - " chunkMD5 : {0}", - file.fileChunkMD5Name.ToHexStringLittleEndian() - ); - _logger.Verbose( - " dataMD5 : {0}", - file.fileDataMD5.ToHexStringLittleEndian() - ); - _logger.Verbose(" offset : {0}", file.offset); - _logger.Verbose(" len : {0}", file.len); - _logger.Verbose( - " blockType : {0} ({1})", - file.blockType, - (byte)file.blockType - ); - _logger.Verbose(" useEncrypt : {0}", file.bUseEncrypt); - if (file.bUseEncrypt) - _logger.Verbose(" ivSeed : {0}", file.ivSeed); - _logger.Verbose(" fileTag : {0} ({1})", file.fileTag, (byte)file.fileTag); - } - - _logger.Verbose("------------------------------"); - } - - _logger.Verbose("======== END BLC INFO ========"); - } - - public void DebugScanBlocks(string streamingAssetsPath) - { - _logger.Info("Debug: scanning all block declarations under {0}", streamingAssetsPath); - _logger.Info(""); - - var dirs = Directory.GetDirectories(streamingAssetsPath); - if (dirs.Length == 0) - { - _logger.Info("No subdirectories found."); - return; - } - - int found = 0; - var blockTypeMap = new Dictionary(); - - foreach (var dir in dirs.OrderBy(d => d)) - { - var dirName = Path.GetFileName(dir); - var blcPath = Path.Combine(dir, dirName + ".blc"); - - if (!File.Exists(blcPath)) - { - _logger.Verbose(" [{0}] No BLC file found, skipping.", dirName); - continue; - } - - try - { - var info = VfsReader.DecryptBlc(blcPath, _chaChaKey); - - var blockTypeByte = (byte)info.blockType; - var typeLabel = Enum.IsDefined(info.blockType) - ? $"{info.blockType} ({blockTypeByte})" - : $"Unknown ({blockTypeByte})"; - - _logger.Info( - " [{0}] groupCfgName = {1} | blockType = {2} | chunks = {3} | files = {4}", - dirName, - info.groupCfgName, - typeLabel, - info.allChunks.Length, - info.groupFileInfoNum - ); - - blockTypeMap[info.groupCfgName] = dirName; - DumpBlockInfo(info); - found++; - } - catch (Exception ex) - { - _logger.Error(" [{0}] Failed to parse BLC: {1}", dirName, ex.Message); - } - } - - _logger.Info(""); - _logger.Info( - "Debug: found {0} block declaration(s) in {1} subdirectories.", - found, - dirs.Length - ); - - if (blockTypeMap.Count > 0) - { - _logger.Info(""); - _logger.Info("Block type mapping (groupCfgName -> directory hash):"); - foreach (var (name, hash) in blockTypeMap.OrderBy(kv => kv.Key)) - _logger.Info(" {0} -> {1}", name, hash); - } - } -} diff --git a/BydTools.VFS/VfsArchive.cs b/BydTools.VFS/VfsArchive.cs new file mode 100644 index 0000000..d975d3f --- /dev/null +++ b/BydTools.VFS/VfsArchive.cs @@ -0,0 +1,162 @@ +using BydTools.Core; +using BydTools.Core.Crypto; + +namespace BydTools.VFS; + +public sealed class VfsBlock +{ + private readonly byte[] _key; + + internal VfsBlock(string directoryPath, VfsBlockInfo info, BlockDescriptor? descriptor, byte[] key) + { + DirectoryPath = directoryPath; + Info = info; + Descriptor = descriptor; + _key = key; + } + + public string DirectoryPath { get; } + public VfsBlockInfo Info { get; } + public BlockDescriptor? Descriptor { get; } + + public string DisplayName => + Descriptor?.Type.ToString() + ?? (Enum.IsDefined(Info.BlockType) ? Info.BlockType.ToString() : Info.GroupCfgName); + + public string? ResolveChunkPath(VfsChunk chunk) + { + string path = Path.Combine(DirectoryPath, chunk.FileName); + return File.Exists(path) ? path : null; + } + + public byte[] ReadFile(string chunkPath, VfsFile file) + { + if (file.Length < 0 || file.Length > int.MaxValue) + throw new InvalidDataException($"File '{file.Name}' has an invalid length ({file.Length})."); + + int length = (int)file.Length; + byte[] data = new byte[length]; + using var handle = File.OpenHandle( + chunkPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ); + int got = 0; + while (got < length) + { + int n = RandomAccess.Read(handle, data.AsSpan(got), file.Offset + got); + if (n == 0) + { + throw new EndOfStreamException( + $"Chunk ended before '{file.Name}' was fully read ({got}/{length})." + ); + } + + got += n; + } + + if (!file.Encrypted) + return data; + + Span nonce = stackalloc byte[VFSDefine.NonceLength]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(nonce, VFSDefine.ProtocolVersion); + System.Buffers.Binary.BinaryPrimitives.WriteInt64LittleEndian(nonce[4..], file.IvSeed); + using var chacha = new CSChaCha20(_key, nonce, 1); + return chacha.DecryptBytes(data); + } +} + +/// +/// A game VFS directory. Blocks are discovered by decrypting every .blc, +/// so a catalog entry is not required to dump a block the client just added. +/// +public sealed class VfsArchive +{ + private VfsArchive(string root, IReadOnlyList blocks) + { + Root = root; + Blocks = blocks; + } + + public string Root { get; } + public IReadOnlyList Blocks { get; } + + public static VfsArchive Open(string vfsDirectory, byte[] key, ILogger logger) + { + if (!Directory.Exists(vfsDirectory)) + throw new DirectoryNotFoundException($"VFS directory not found: {vfsDirectory}"); + + var blocks = new List(); + Exception? firstError = null; + foreach (var dir in Directory.GetDirectories(vfsDirectory).OrderBy(static d => d, StringComparer.Ordinal)) + { + string dirName = Path.GetFileName(dir); + string blcPath = Path.Combine(dir, dirName + ".blc"); + if (!File.Exists(blcPath)) + { + logger.Verbose(" [{0}] No BLC file found, skipping.", dirName); + continue; + } + + try + { + var info = BlcReader.Read(blcPath, key); + if (info.NonceVersion != VFSDefine.ProtocolVersion) + { + logger.Info( + "Warning: VFS version {0} does not match expected {1}.", + info.NonceVersion, + VFSDefine.ProtocolVersion + ); + } + + BlockRegistry.TryGet(info.BlockType, out var byType); + var byName = BlockRegistry.All.FirstOrDefault(d => + d.Type.ToString().Equals(info.GroupCfgName, StringComparison.OrdinalIgnoreCase) + ); + var descriptor = byName ?? (byType is null ? null : byType); + if ( + descriptor != null + && !descriptor.DirHash.Equals(info.GroupCfgHashName, StringComparison.OrdinalIgnoreCase) + ) + { + logger.Info( + "Warning: {0} directory hash {1} differs from the catalog hash {2}.", + info.GroupCfgName, + info.GroupCfgHashName, + descriptor.DirHash + ); + } + + blocks.Add(new VfsBlock(dir, info, descriptor, key)); + } + catch (Exception ex) + { + firstError ??= ex; + logger.Error(" [{0}] Failed to parse BLC: {1}", dirName, ex.Message); + } + } + + if (blocks.Count == 0 && firstError != null) + throw firstError; + + return new VfsArchive(vfsDirectory, blocks); + } + + public VfsBlock? Find(string nameOrType) + { + if (BlockRegistry.TryParse(nameOrType, out var descriptor)) + { + string typeName = descriptor.Type.ToString(); + return Blocks.FirstOrDefault(b => + b.Info.BlockType == descriptor.Type + || b.Info.GroupCfgName.Equals(typeName, StringComparison.OrdinalIgnoreCase) + ); + } + + return Blocks.FirstOrDefault(b => + b.Info.GroupCfgName.Equals(nameOrType, StringComparison.OrdinalIgnoreCase) + ); + } +} diff --git a/BydTools.VFS/VfsModel.cs b/BydTools.VFS/VfsModel.cs new file mode 100644 index 0000000..9a7fa0d --- /dev/null +++ b/BydTools.VFS/VfsModel.cs @@ -0,0 +1,40 @@ +using BydTools.Core.IO; + +namespace BydTools.VFS; + +public sealed record VfsFile( + string Name, + long NameHash, + UInt128 ChunkMd5, + UInt128 DataMd5, + long Offset, + long Length, + EVFSBlockType BlockType, + bool Encrypted, + long IvSeed, + EVFSFileTag FileTag +); + +public sealed record VfsChunk( + UInt128 Md5Name, + UInt128 ContentMd5, + long Length, + EVFSBlockType BlockType, + EVFSFileTag FileTag, + IReadOnlyList Files +) +{ + public string FileName => Md5Name.ToHexLittleEndian() + ".chk"; +} + +public sealed record VfsBlockInfo( + int NonceVersion, + int CodeVersion, + int Version, + string GroupCfgName, + string GroupCfgHashName, + int GroupFileInfoNum, + long GroupChunksLength, + EVFSBlockType BlockType, + IReadOnlyList Chunks +); diff --git a/BydTools.VFS/VfsReader.cs b/BydTools.VFS/VfsReader.cs deleted file mode 100644 index 9ecfba2..0000000 --- a/BydTools.VFS/VfsReader.cs +++ /dev/null @@ -1,113 +0,0 @@ -using System.Buffers.Binary; -using BydTools.Utils.Crypto; -using BydTools.Utils.Extensions; - -namespace BydTools.VFS; - -/// -/// Shared low-level helpers for reading VFS blocks, chunks, and files. -/// Both VFSDumper and external consumers (e.g. PckCommand) use these -/// instead of duplicating the decrypt-and-parse logic. -/// -public static class VfsReader -{ - /// - /// Decrypts and parses a BLC file for the given block type. - /// Validates the parsed hash against the directory name. - /// - public static VFBlockMainInfo ReadBlockInfo(string vfsPath, EVFSBlockType blockType, byte[] key) - { - if (!VFSDumper.BlockHashMap.TryGetValue(blockType, out var hashName)) - throw new InvalidOperationException($"No hash mapping for block type {blockType}"); - - var blcPath = Path.Combine(vfsPath, hashName, hashName + ".blc"); - if (!File.Exists(blcPath)) - throw new FileNotFoundException($"BLC file not found for {blockType}", blcPath); - - return DecryptBlc(blcPath, key); - } - - /// - /// Decrypts and parses a BLC file by its direct path (for discovery/debug scenarios - /// where the block type is not known in advance). - /// - public static VFBlockMainInfo DecryptBlc(string blcFilePath, byte[] key) - { - var blockFile = File.ReadAllBytes(blcFilePath); - - byte[] nonce = new byte[VFSDefine.BLOCK_HEAD_LEN]; - Buffer.BlockCopy(blockFile, 0, nonce, 0, nonce.Length); - var version = BinaryPrimitives.ReadInt32LittleEndian(nonce); - if (version != VFSDefine.VFS_PROTO_VERSION) - { - Console.WriteLine($"Warning: VFS version {version} does not match expected {VFSDefine.VFS_PROTO_VERSION}."); - } - - using var chacha = new CSChaCha20(key, nonce, 1); - var decrypted = chacha.DecryptBytes(blockFile[VFSDefine.BLOCK_HEAD_LEN..]); - Buffer.BlockCopy(decrypted, 0, blockFile, VFSDefine.BLOCK_HEAD_LEN, decrypted.Length); - - var info = new VFBlockMainInfo(blockFile, VFSDefine.BLOCK_HEAD_LEN); - - var expectedHash = Path.GetFileNameWithoutExtension(blcFilePath); - if (!string.Equals(info.groupCfgHashName, expectedHash, StringComparison.OrdinalIgnoreCase)) - { - throw new InvalidDataException( - $"BLC hash mismatch: parsed \"{info.groupCfgHashName}\", expected \"{expectedHash}\". " - + "The ChaCha20 key may be incorrect or outdated." - ); - } - - return info; - } - - /// - /// Resolves the filesystem path for a chunk file, or null if it doesn't exist. - /// - public static string? ResolveChunkPath( - string vfsPath, - EVFSBlockType blockType, - in FVFBlockChunkInfo chunk - ) - { - if (!VFSDumper.BlockHashMap.TryGetValue(blockType, out var hashName)) - return null; - - var chunkFileName = - chunk.md5Name.ToHexStringLittleEndian() + FVFBlockChunkInfo.FILE_EXTENSION; - var path = Path.Combine(vfsPath, hashName, chunkFileName); - return File.Exists(path) ? path : null; - } - - /// - /// Reads (and optionally decrypts) a single file's data from an open chunk stream. - /// - public static byte[] ReadFileData( - FileStream chunkFs, - in FVFBlockFileInfo file, - byte[] key - ) - { - if (file.bUseEncrypt) - { - byte[] nonce = new byte[VFSDefine.BLOCK_HEAD_LEN]; - Buffer.BlockCopy(BitConverter.GetBytes(VFSDefine.VFS_PROTO_VERSION), 0, nonce, 0, sizeof(int)); - Buffer.BlockCopy( - BitConverter.GetBytes(file.ivSeed), - 0, - nonce, - sizeof(int), - sizeof(long) - ); - - using var chacha = new CSChaCha20(key, nonce, 1); - var encrypted = new byte[file.len]; - chunkFs.ReadExactly(encrypted); - return chacha.DecryptBytes(encrypted); - } - - var data = new byte[file.len]; - chunkFs.ReadExactly(data); - return data; - } -} diff --git a/BydTools.VFS/xLua/LuaDecipher.cs b/BydTools.VFS/xLua/LuaDecipher.cs deleted file mode 100644 index 2ed360b..0000000 --- a/BydTools.VFS/xLua/LuaDecipher.cs +++ /dev/null @@ -1,279 +0,0 @@ -using System; -using System.Linq; -using System.Text; -using BydTools.Utils.Crypto; - -namespace BydTools.VFS.xLua; - -/// -/// Provides functionality to decrypt encrypted Lua bytecode -/// -public static class LuaDecipher -{ - private static readonly string[] Keys = new[] - { - "cynb5", - "paeky", - "xmF5og", - "ud35+e", - "72iUy", - "azWk3", - "901lU", - "dDfl2", - }; - - private const string InitialKey = "Assets/Beyond/InitialAssets/"; - private const string DefaultDecryptionKey = "Assets/Beyond/DynamicAssets/Gameplay/UI/Fonts/"; - - public static byte[]? DecryptLua(string encryptedLuaBase64) - { - try - { - string? masterKey = GetMasterKey(); - if (string.IsNullOrEmpty(masterKey)) - { - return null; - } - - byte[] data = Convert.FromBase64String(encryptedLuaBase64); - return XXTEA.Decrypt(data, Encoding.UTF8.GetBytes(masterKey)); - } - catch (Exception) - { - return null; - } - } - - /// - /// Decrypts Lua bytecode from encrypted byte array - /// - /// Encrypted Lua bytecode - /// Decrypted Lua bytecode as byte array, or null if decryption fails - public static byte[]? DecryptLua(byte[] encryptedData) - { - try - { - string? masterKey = GetMasterKey(); - if (string.IsNullOrEmpty(masterKey)) - { - return null; - } - - return XXTEA.Decrypt(encryptedData, Encoding.UTF8.GetBytes(masterKey)); - } - catch (Exception) - { - return null; - } - } - - /// - /// Decrypts Lua bytecode with a custom key - /// - /// Base64 encoded encrypted Lua bytecode - /// Custom decryption key - /// Decrypted Lua bytecode as byte array, or null if decryption fails - public static byte[]? DecryptLuaWithKey(string encryptedLuaBase64, string customKey) - { - try - { - if (string.IsNullOrEmpty(customKey)) - { - return null; - } - - byte[] data = Convert.FromBase64String(encryptedLuaBase64); - return XXTEA.Decrypt(data, Encoding.UTF8.GetBytes(customKey)); - } - catch (Exception) - { - return null; - } - } - - /// - /// Decrypts Lua bytecode with a custom key - /// - /// Encrypted Lua bytecode - /// Custom decryption key - /// Decrypted Lua bytecode as byte array, or null if decryption fails - public static byte[]? DecryptLuaWithKey(byte[] encryptedData, string customKey) - { - try - { - if (string.IsNullOrEmpty(customKey)) - { - return null; - } - - return XXTEA.Decrypt(encryptedData, Encoding.UTF8.GetBytes(customKey)); - } - catch (Exception) - { - return null; - } - } - - /// - /// Gets the master key used for decryption - /// - /// Master decryption key, or null if key cannot be obtained - public static string? GetMasterKey() - { - if (Keys.Length <= 5) - { - return null; - } - - string encryptedMasterKey = $"{Keys[1]}{Keys[5]}{Keys[3]}{Keys[2]}=="; - - byte[]? masterKeyBytes = DecryptSubtraction(encryptedMasterKey, InitialKey); - if (masterKeyBytes == null || masterKeyBytes.Length == 0) - { - return null; - } - - return Encoding.UTF8.GetString(masterKeyBytes); - } - - /// - /// Decrypts Lua bytecode with a pre-computed master key - /// - /// Base64 encoded encrypted Lua bytecode - /// Pre-computed master key - /// Decrypted Lua bytecode as byte array, or null if decryption fails - public static byte[]? DecryptLuaWithMasterKey(string encryptedLuaBase64, string masterKey) - { - try - { - byte[] data = Convert.FromBase64String(encryptedLuaBase64); - return XXTEA.Decrypt(data, Encoding.UTF8.GetBytes(masterKey)); - } - catch (Exception) - { - return null; - } - } - - /// - /// Decrypts Lua bytecode with a pre-computed master key - /// - /// Encrypted Lua bytecode - /// Pre-computed master key - /// Decrypted Lua bytecode as byte array, or null if decryption fails - public static byte[]? DecryptLuaWithMasterKey(byte[] encryptedData, string masterKey) - { - try - { - return XXTEA.Decrypt(encryptedData, Encoding.UTF8.GetBytes(masterKey)); - } - catch (Exception) - { - return null; - } - } - - /// - /// Decrypts data using subtraction cipher - /// - /// Base64 encoded encrypted text - /// Decryption key (defaults to DefaultDecryptionKey if null or empty) - /// Decrypted data as byte array, or null if decryption fails - private static byte[]? DecryptSubtraction(string encryptedText, string? key = null) - { - try - { - if (string.IsNullOrEmpty(key)) - { - key = DefaultDecryptionKey; - } - - byte[] data = Convert.FromBase64String(encryptedText); - byte[] keyBytes = Encoding.UTF8.GetBytes(key); - int keyLen = keyBytes.Length; - - if (keyLen == 0) - { - return data; - } - - for (int i = 0; i < data.Length; i++) - { - data[i] = (byte)(data[i] - keyBytes[i % keyLen]); - } - - return data; - } - catch (Exception) - { - return null; - } - } - - /// - /// Attempts to decrypt Lua bytecode and returns decryption status - /// - /// Base64 encoded encrypted Lua bytecode - /// Decrypted Lua bytecode (output parameter) - /// True if decryption succeeded, false otherwise - public static bool TryDecryptLua(string encryptedLuaBase64, out byte[]? decryptedData) - { - decryptedData = DecryptLua(encryptedLuaBase64); - return decryptedData != null && decryptedData.Length > 0; - } - - /// - /// Attempts to decrypt Lua bytecode and returns decryption status - /// - /// Encrypted Lua bytecode - /// Decrypted Lua bytecode (output parameter) - /// True if decryption succeeded, false otherwise - public static bool TryDecryptLua(byte[] encryptedData, out byte[]? decryptedData) - { - decryptedData = DecryptLua(encryptedData); - return decryptedData != null && decryptedData.Length > 0; - } - - /// - /// Validates if the decrypted data appears to be valid Lua bytecode or source code - /// - /// Decrypted data to validate - /// True if data appears to be valid Lua bytecode or source code - public static bool IsValidLuaBytecode(byte[]? data) - { - if (data == null || data.Length < 4) - { - return false; - } - - // Check for Lua bytecode signature (0x1B 'L' 'u' 'a') - if (data[0] == 0x1B && data[1] == 0x4C && data[2] == 0x75 && data[3] == 0x61) - { - return true; - } - - // Check if it's valid UTF-8 text (Lua source code) - // Look for common Lua keywords at the start - try - { - var text = Encoding.UTF8.GetString(data.Take(Math.Min(1000, data.Length)).ToArray()); - // Remove BOM and whitespace - text = text.TrimStart('\uFEFF', '\r', '\n', ' ', '\t'); - - // Check for common Lua patterns - return text.StartsWith("local ") - || text.StartsWith("function ") - || text.StartsWith("return ") - || text.StartsWith("require ") - || text.StartsWith("--") - || text.StartsWith("config ") - || text.Contains("local ") - || text.Contains("function(") - || (data.Length > 10 && data.All(b => b < 128 || b >= 0xC0)); // Valid UTF-8 range - } - catch - { - return false; - } - } -} diff --git a/BydTools.Wwise/BnkParser.cs b/BydTools.Wwise/BnkParser.cs deleted file mode 100644 index 93309f0..0000000 --- a/BydTools.Wwise/BnkParser.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace BydTools.Wwise; - -/// -/// Minimal Wwise BNK parser — extracts embedded WEM file entries -/// by reading only the BKHD, DIDX, and DATA sections. -/// -public static class BnkParser -{ - /// - /// A WEM file entry found inside a BNK. - /// is absolute within the source BNK byte array. - /// - public record BnkWemEntry(uint Id, uint Offset, uint Size); - - /// - /// Parses a BNK byte array and returns the embedded WEM entries. - /// Scans sections in order; handles any section arrangement. - /// - public static List Parse(byte[] data) - { - using var reader = new BinaryReader(new MemoryStream(data)); - - long dataOffset = 0; - var entries = new List(); - - while (reader.BaseStream.Position + 8 <= reader.BaseStream.Length) - { - string sign = ReadSignature(reader); - uint sectionSize = reader.ReadUInt32(); - long sectionStart = reader.BaseStream.Position; - - if (sign == "DIDX") - { - int wemCount = (int)(sectionSize / 12); - for (int i = 0; i < wemCount; i++) - { - uint id = reader.ReadUInt32(); - uint offset = reader.ReadUInt32(); - uint size = reader.ReadUInt32(); - entries.Add(new BnkWemEntry(id, offset, size)); - } - } - else if (sign == "DATA") - { - dataOffset = reader.BaseStream.Position; - } - - reader.BaseStream.Position = sectionStart + sectionSize; - } - - if (dataOffset == 0 || entries.Count == 0) - return []; - - for (int i = 0; i < entries.Count; i++) - entries[i] = entries[i] with { Offset = (uint)(entries[i].Offset + dataOffset) }; - - return entries; - } - - /// - /// Parses a BNK file on disk and writes the embedded WEM files - /// to a subdirectory named after the BNK file. - /// - public static void ParseToFiles(string bnkFilePath) - { - byte[] data = File.ReadAllBytes(bnkFilePath); - var entries = Parse(data); - if (entries.Count == 0) - return; - - string outputDir = Path.Combine( - Path.GetDirectoryName(bnkFilePath) ?? ".", - Path.GetFileNameWithoutExtension(bnkFilePath) - ); - Directory.CreateDirectory(outputDir); - - foreach (var entry in entries) - { - string wemPath = Path.Combine(outputDir, $"{entry.Id}.wem"); - using var fs = File.Create(wemPath); - fs.Write(data, (int)entry.Offset, (int)entry.Size); - } - } - - private static string ReadSignature(BinaryReader reader) - { - Span chars = stackalloc char[4]; - for (int i = 0; i < 4; i++) - chars[i] = (char)reader.ReadByte(); - return new string(chars); - } -} diff --git a/BydTools.Wwise/BydTools.Wwise.csproj b/BydTools.Wwise/BydTools.Wwise.csproj deleted file mode 100644 index 5db6a1a..0000000 --- a/BydTools.Wwise/BydTools.Wwise.csproj +++ /dev/null @@ -1,15 +0,0 @@ - - - net10.0 - enable - enable - true - - - - - PreserveNewest - PreserveNewest - - - diff --git a/BydTools.Wwise/IWemConverter.cs b/BydTools.Wwise/IWemConverter.cs deleted file mode 100644 index d465f3a..0000000 --- a/BydTools.Wwise/IWemConverter.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BydTools.Wwise; - -/// -/// Abstraction for converting Wwise WEM audio files to WAV. -/// -public interface IWemConverter -{ - /// - /// Converts a WEM file to WAV. - /// - /// Path to the source WEM file. - /// Path to the output WAV file. - void Convert(string wemPath, string outputPath); -} diff --git a/BydTools.Wwise/Interop/LibVgmstream.cs b/BydTools.Wwise/Interop/LibVgmstream.cs deleted file mode 100644 index 4ac8dbd..0000000 --- a/BydTools.Wwise/Interop/LibVgmstream.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System.Runtime.InteropServices; - -namespace BydTools.Wwise.Interop; - -/// -/// P/Invoke bindings for libvgmstream's public C API. -/// Requires libvgmstream.dll built with -DBUILD_SHARED_LIBS=ON. -/// -internal static partial class LibVgmstream -{ - private const string DllName = "libvgmstream"; - - // ── version ────────────────────────────────────────────────── - - [LibraryImport(DllName, EntryPoint = "libvgmstream_get_version")] - public static partial uint GetVersion(); - - // ── lifecycle ──────────────────────────────────────────────── - - [LibraryImport(DllName, EntryPoint = "libvgmstream_init")] - public static partial nint Init(); - - [LibraryImport(DllName, EntryPoint = "libvgmstream_free")] - public static partial void Free(nint lib); - - // ── config ─────────────────────────────────────────────────── - - [LibraryImport(DllName, EntryPoint = "libvgmstream_setup")] - public static partial void Setup(nint lib, ref NativeConfig cfg); - - // ── stream ─────────────────────────────────────────────────── - - [LibraryImport(DllName, EntryPoint = "libvgmstream_open_stream")] - public static partial int OpenStream(nint lib, nint libsf, int subsong); - - [LibraryImport(DllName, EntryPoint = "libvgmstream_close_stream")] - public static partial void CloseStream(nint lib); - - // ── decode ─────────────────────────────────────────────────── - - [LibraryImport(DllName, EntryPoint = "libvgmstream_render")] - public static partial int Render(nint lib); - - [LibraryImport(DllName, EntryPoint = "libvgmstream_fill")] - public static partial int Fill(nint lib, nint buf, int bufSamples); - - // ── streamfile ─────────────────────────────────────────────── - - [LibraryImport( - DllName, - EntryPoint = "libstreamfile_open_from_stdio", - StringMarshalling = StringMarshalling.Utf8 - )] - public static partial nint OpenStreamfileFromStdio(string filename); - - [LibraryImport(DllName, EntryPoint = "libstreamfile_open_buffered")] - public static partial nint OpenStreamfileBuffered(nint extLibsf); - - // ── log ────────────────────────────────────────────────────── - - [LibraryImport(DllName, EntryPoint = "libvgmstream_set_log")] - public static partial void SetLog(int level, nint callback); - - // ── structs ────────────────────────────────────────────────── - - public enum SampleFormat : int - { - Pcm16 = 1, - Pcm24 = 2, - Pcm32 = 3, - Float = 4, - } - - [StructLayout(LayoutKind.Sequential)] - public struct NativeConfig - { - public byte disable_config_override; - public byte allow_play_forever; - public byte play_forever; - public byte ignore_loop; - public byte force_loop; - public byte really_force_loop; - public byte ignore_fade; - private byte _pad0; - public double loop_count; - public double fade_time; - public double fade_delay; - public int stereo_track; - public int auto_downmix_channels; - public int force_sfmt; - } -} diff --git a/BydTools.Wwise/Interop/VgmstreamHandle.cs b/BydTools.Wwise/Interop/VgmstreamHandle.cs deleted file mode 100644 index 1ae9438..0000000 --- a/BydTools.Wwise/Interop/VgmstreamHandle.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System.Runtime.InteropServices; - -namespace BydTools.Wwise.Interop; - -/// -/// Safe wrapper around a libvgmstream context handle. -/// Reads native struct fields via pointer offsets. -/// -internal sealed class VgmstreamHandle : IDisposable -{ - private nint _handle; - - public nint Ptr => - _handle != 0 ? _handle : throw new ObjectDisposedException(nameof(VgmstreamHandle)); - - public bool IsValid => _handle != 0; - - private VgmstreamHandle(nint handle) => _handle = handle; - - public static VgmstreamHandle Create() - { - nint h = LibVgmstream.Init(); - return h == 0 - ? throw new InvalidOperationException("libvgmstream_init returned NULL") - : new VgmstreamHandle(h); - } - - // ── format access ──────────────────────────────────────────── - - /// Pointer to the libvgmstream_format_t struct. - private nint FormatPtr => Marshal.ReadIntPtr(_handle, nint.Size); // offset 8 on x64 - - /// Pointer to the libvgmstream_decoder_t struct. - private nint DecoderPtr => Marshal.ReadIntPtr(_handle, nint.Size * 2); // offset 16 on x64 - - public int Channels => Marshal.ReadInt32(FormatPtr, 0); - public int SampleRate => Marshal.ReadInt32(FormatPtr, 4); - public LibVgmstream.SampleFormat SampleFmt => - (LibVgmstream.SampleFormat)Marshal.ReadInt32(FormatPtr, 8); - public int SampleSize => Marshal.ReadInt32(FormatPtr, 12); - - /// - /// stream_samples field offset within libvgmstream_format_t. - /// Layout: channels(4) + sample_rate(4) + sample_format(4) + sample_size(4) - /// + channel_layout(4) + subsong_index(4) + subsong_count(4) + input_channels(4) - /// = 32 bytes → stream_samples is at offset 32 (int64). - /// - public long StreamSamples => Marshal.ReadInt64(FormatPtr, 32); - - // ── decoder access ─────────────────────────────────────────── - - public nint DecodedBuf => Marshal.ReadIntPtr(DecoderPtr, 0); - public int DecodedSamples => Marshal.ReadInt32(DecoderPtr, nint.Size); - public int DecodedBytes => Marshal.ReadInt32(DecoderPtr, nint.Size + 4); - public bool Done => Marshal.ReadByte(DecoderPtr, nint.Size + 8) != 0; - - // ── operations ─────────────────────────────────────────────── - - public void Setup(LibVgmstream.NativeConfig cfg) => LibVgmstream.Setup(_handle, ref cfg); - - public int OpenStream(nint streamfile, int subsong = 0) => - LibVgmstream.OpenStream(_handle, streamfile, subsong); - - public void CloseStream() => LibVgmstream.CloseStream(_handle); - - public int Render() => LibVgmstream.Render(_handle); - - // ── dispose ────────────────────────────────────────────────── - - public void Dispose() - { - var h = Interlocked.Exchange(ref _handle, 0); - if (h != 0) - LibVgmstream.Free(h); - } -} diff --git a/BydTools.Wwise/LibVgmstreamConverter.cs b/BydTools.Wwise/LibVgmstreamConverter.cs deleted file mode 100644 index 974b3ae..0000000 --- a/BydTools.Wwise/LibVgmstreamConverter.cs +++ /dev/null @@ -1,135 +0,0 @@ -using System.Runtime.InteropServices; -using BydTools.Wwise.Interop; - -namespace BydTools.Wwise; - -/// -/// Converts WEM files to WAV using libvgmstream.dll (in-process, no external CLI). -/// Thread-safe: each call creates its own vgmstream context. -/// -public sealed class LibVgmstreamConverter : IWemConverter -{ - private static bool? _available; - - public static bool IsAvailable - { - get - { - _available ??= ProbeLibrary(); - return _available.Value; - } - } - - public void Convert(string wemPath, string outputPath) - { - if (!File.Exists(wemPath)) - throw new FileNotFoundException("WEM file not found", wemPath); - - using var handle = VgmstreamHandle.Create(); - SetupHandle(handle); - - nint sf = LibVgmstream.OpenStreamfileFromStdio(wemPath); - if (sf == 0) - throw new InvalidOperationException($"Failed to open streamfile: {wemPath}"); - - try - { - int result = handle.OpenStream(sf); - if (result < 0) - throw new InvalidOperationException( - $"libvgmstream failed to open stream (error {result})" - ); - } - finally - { - CloseStreamfile(sf); - } - - DecodeAndWriteWav(handle, outputPath); - } - - private static void SetupHandle(VgmstreamHandle handle) - { - var cfg = new LibVgmstream.NativeConfig - { - ignore_loop = 1, - force_sfmt = (int)LibVgmstream.SampleFormat.Pcm16, - }; - handle.Setup(cfg); - } - - private static void DecodeAndWriteWav(VgmstreamHandle handle, string outputPath) - { - int channels = handle.Channels; - int sampleRate = handle.SampleRate; - int sampleSize = handle.SampleSize; - long totalSamples = handle.StreamSamples; - - if (channels <= 0 || sampleRate <= 0 || totalSamples <= 0) - throw new InvalidOperationException("Invalid stream format"); - - int expectedBytes = (int)(totalSamples * channels * sampleSize); - - // Write WAV header then stream decoded data directly — no intermediate MemoryStream - using var fs = File.Create(outputPath); - using var writer = new BinaryWriter(fs); - WavWriter.WriteHeader(writer, channels, sampleRate, sampleSize * 8, expectedBytes); - - int writtenBytes = 0; - while (!handle.Done) - { - int renderResult = handle.Render(); - if (renderResult < 0) - throw new InvalidOperationException($"libvgmstream render error: {renderResult}"); - - int bytes = handle.DecodedBytes; - if (bytes <= 0) - continue; - - nint buf = handle.DecodedBuf; - unsafe - { - fs.Write(new ReadOnlySpan((void*)buf, bytes)); - } - writtenBytes += bytes; - } - - if (writtenBytes != expectedBytes) - { - fs.Seek(4, SeekOrigin.Begin); - writer.Write(36 + writtenBytes); - fs.Seek(40, SeekOrigin.Begin); - writer.Write(writtenBytes); - } - } - - private static void CloseStreamfile(nint sf) - { - nint closeFnPtr = Marshal.ReadIntPtr(sf, 40); - if (closeFnPtr != 0) - { - var closeFn = Marshal.GetDelegateForFunctionPointer(closeFnPtr); - closeFn(sf); - } - } - - [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - private delegate void CloseDelegate(nint libsf); - - private static bool ProbeLibrary() - { - try - { - _ = LibVgmstream.GetVersion(); - return true; - } - catch (DllNotFoundException) - { - return false; - } - catch (EntryPointNotFoundException) - { - return false; - } - } -} diff --git a/BydTools.Wwise/WavWriter.cs b/BydTools.Wwise/WavWriter.cs deleted file mode 100644 index 83d3453..0000000 --- a/BydTools.Wwise/WavWriter.cs +++ /dev/null @@ -1,55 +0,0 @@ -namespace BydTools.Wwise; - -/// -/// Minimal WAV file writer for PCM16 interleaved audio. -/// -internal static class WavWriter -{ - public static void WriteHeader( - BinaryWriter w, - int channels, - int sampleRate, - int bitsPerSample, - int dataBytes - ) - { - int blockAlign = channels * (bitsPerSample / 8); - int avgBytesPerSec = sampleRate * blockAlign; - - // RIFF header - w.Write("RIFF"u8); - w.Write(36 + dataBytes); // file size - 8 - w.Write("WAVE"u8); - - // fmt chunk - w.Write("fmt "u8); - w.Write(16); // chunk size - w.Write((short)1); // PCM format - w.Write((short)channels); - w.Write(sampleRate); - w.Write(avgBytesPerSec); - w.Write((short)blockAlign); - w.Write((short)bitsPerSample); - - // data chunk header - w.Write("data"u8); - w.Write(dataBytes); - } - - /// - /// Creates a complete WAV file from raw PCM data. - /// - public static void Write( - string path, - int channels, - int sampleRate, - int bitsPerSample, - byte[] pcmData - ) - { - using var fs = File.Create(path); - using var w = new BinaryWriter(fs); - WriteHeader(w, channels, sampleRate, bitsPerSample, pcmData.Length); - w.Write(pcmData); - } -} diff --git a/BydTools.Wwise/WemConverter.cs b/BydTools.Wwise/WemConverter.cs deleted file mode 100644 index b56e817..0000000 --- a/BydTools.Wwise/WemConverter.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System.Diagnostics; - -namespace BydTools.Wwise; - -/// -/// Converts Wwise WEM audio files to WAV via vgmstream-cli. -/// Supports all codecs that vgmstream can decode: Wwise Vorbis, Opus, PCM, ADPCM, PTADPCM, etc. -/// -public sealed class WemConverter : IWemConverter -{ - private static readonly Lazy _vgmstreamPath = new(FindVgmstream); - - /// - /// Path to the vgmstream-cli executable, or null if not found. - /// - public static string? VgmstreamPath => _vgmstreamPath.Value; - - public void Convert(string wemPath, string outputPath) - { - if (!File.Exists(wemPath)) - throw new FileNotFoundException("WEM file not found", wemPath); - - string vgmstream = - _vgmstreamPath.Value - ?? throw new FileNotFoundException( - "vgmstream-cli not found. Place vgmstream-cli next to the executable or add it to PATH." - ); - - var psi = new ProcessStartInfo - { - FileName = vgmstream, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - psi.ArgumentList.Add("-o"); - psi.ArgumentList.Add(outputPath); - psi.ArgumentList.Add(wemPath); - - using var proc = - Process.Start(psi) - ?? throw new InvalidOperationException("Failed to start vgmstream-cli"); - - proc.StandardOutput.ReadToEnd(); - string stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - - if (proc.ExitCode != 0) - throw new InvalidOperationException( - $"vgmstream-cli exited with code {proc.ExitCode}: {Truncate(stderr)}" - ); - - if (!File.Exists(outputPath) || new FileInfo(outputPath).Length == 0) - throw new InvalidOperationException("vgmstream-cli produced no output"); - } - - private static string Truncate(string msg) - { - string trimmed = msg.Trim(); - return trimmed.Length > 300 ? trimmed[..300] + "..." : trimmed; - } - - private static string? FindVgmstream() - { - string name = OperatingSystem.IsWindows() ? "vgmstream-cli.exe" : "vgmstream-cli"; - - // 1. Next to the current executable - string? exeDir = Path.GetDirectoryName(Environment.ProcessPath); - if (exeDir != null) - { - string local = Path.Combine(exeDir, name); - if (File.Exists(local)) - return local; - - // Also check a vgmstream/ subdirectory - string sub = Path.Combine(exeDir, "vgmstream", name); - if (File.Exists(sub)) - return sub; - } - - // 2. PATH - string? pathEnv = Environment.GetEnvironmentVariable("PATH"); - if (pathEnv != null) - { - foreach (string dir in pathEnv.Split(Path.PathSeparator)) - { - try - { - string candidate = Path.Combine(dir, name); - if (File.Exists(candidate)) - return candidate; - } - catch { } - } - } - - return null; - } -} diff --git a/BydTools.Wwise/WemFormat.cs b/BydTools.Wwise/WemFormat.cs deleted file mode 100644 index 39a9d04..0000000 --- a/BydTools.Wwise/WemFormat.cs +++ /dev/null @@ -1,174 +0,0 @@ -using System.Buffers.Binary; - -namespace BydTools.Wwise; - -/// -/// Wwise codec identifiers found in the WEM RIFF fmt chunk. -/// Values from Audiokinetic Wwise SDK, cross-referenced with vgmstream. -/// -public enum WemCodec : ushort -{ - Unknown = 0, - Pcm = 0x0001, - Adpcm = 0x0002, - PcmFloat = 0x0003, - AdpcmOld = 0x0069, - Wma2 = 0x0161, - WmaPro = 0x0162, - Xma2Chunk = 0x0165, - Xma2Fmt = 0x0166, - Aac = 0xAAC0, - OpusNx = 0x3039, - Opus = 0x3040, - OpusWw = 0x3041, - Dsp = 0xFFF0, - Hevag = 0xFFFB, - Atrac9 = 0xFFFC, - PcmAuthoring = 0xFFFE, - Vorbis = 0xFFFF, - Ptadpcm = 0x8311, -} - -/// -/// Parsed metadata from a WEM file's RIFF header. -/// -public record WemInfo( - WemCodec Codec, - int Channels, - int SampleRate, - int AvgBytesPerSec, - int BlockAlign, - int BitsPerSample -) -{ - public bool IsOpus => Codec is WemCodec.Opus or WemCodec.OpusNx or WemCodec.OpusWw; - public bool IsVorbis => Codec == WemCodec.Vorbis; - public bool IsPcm => Codec is WemCodec.Pcm or WemCodec.PcmAuthoring or WemCodec.PcmFloat; - public bool IsAdpcm => Codec is WemCodec.Adpcm or WemCodec.AdpcmOld or WemCodec.Ptadpcm; -} - -/// -/// Reads minimal metadata from WEM file data without full parsing. -/// -public static class WemFormatReader -{ - /// - /// Detects the codec from raw WEM file bytes. - /// Returns if the format cannot be determined. - /// - public static WemCodec DetectCodec(ReadOnlySpan data) - { - if (data.Length < 22) - return WemCodec.Unknown; - - if (!data[..4].SequenceEqual("RIFF"u8) && !data[..4].SequenceEqual("RIFX"u8)) - return WemCodec.Unknown; - - // Walk RIFF chunks to find "fmt " - int pos = 12; // skip RIFF header (4 magic + 4 size + 4 "WAVE"/"XWMA") - while (pos + 8 <= data.Length) - { - var chunkId = data.Slice(pos, 4); - uint chunkSize = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(pos + 4, 4)); - - if (chunkId.SequenceEqual("fmt "u8)) - { - if (pos + 8 + 2 > data.Length) - return WemCodec.Unknown; - ushort formatTag = BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(pos + 8, 2)); - return Enum.IsDefined(typeof(WemCodec), formatTag) - ? (WemCodec)formatTag - : WemCodec.Unknown; - } - - pos += 8 + (int)chunkSize; - if (pos % 2 != 0) - pos++; // RIFF chunks are word-aligned - } - - return WemCodec.Unknown; - } - - /// - /// Parses full WEM metadata from file bytes. - /// Returns null if the header cannot be parsed. - /// - public static WemInfo? Parse(ReadOnlySpan data) - { - if (data.Length < 28) - return null; - - if (!data[..4].SequenceEqual("RIFF"u8) && !data[..4].SequenceEqual("RIFX"u8)) - return null; - - int pos = 12; - while (pos + 8 <= data.Length) - { - var chunkId = data.Slice(pos, 4); - uint chunkSize = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(pos + 4, 4)); - - if (chunkId.SequenceEqual("fmt "u8)) - { - var fmt = data.Slice( - pos + 8, - (int)Math.Min(chunkSize, (uint)(data.Length - pos - 8)) - ); - if (fmt.Length < 16) - return null; - - ushort formatTag = BinaryPrimitives.ReadUInt16LittleEndian(fmt[..2]); - int channels = BinaryPrimitives.ReadUInt16LittleEndian(fmt[2..4]); - int sampleRate = (int)BinaryPrimitives.ReadUInt32LittleEndian(fmt[4..8]); - int avgBytesPerSec = (int)BinaryPrimitives.ReadUInt32LittleEndian(fmt[8..12]); - int blockAlign = BinaryPrimitives.ReadUInt16LittleEndian(fmt[12..14]); - int bitsPerSample = BinaryPrimitives.ReadUInt16LittleEndian(fmt[14..16]); - - var codec = Enum.IsDefined(typeof(WemCodec), formatTag) - ? (WemCodec)formatTag - : WemCodec.Unknown; - - return new WemInfo( - codec, - channels, - sampleRate, - avgBytesPerSec, - blockAlign, - bitsPerSample - ); - } - - pos += 8 + (int)chunkSize; - if (pos % 2 != 0) - pos++; - } - - return null; - } - - /// - /// Returns a human-readable description for a codec. - /// - public static string GetCodecName(WemCodec codec) => - codec switch - { - WemCodec.Pcm => "PCM", - WemCodec.PcmAuthoring => "PCM (Authoring)", - WemCodec.PcmFloat => "PCM Float", - WemCodec.Adpcm => "IMA ADPCM", - WemCodec.AdpcmOld => "IMA ADPCM (old)", - WemCodec.Ptadpcm => "PTADPCM", - WemCodec.Vorbis => "Wwise Vorbis", - WemCodec.Opus => "Opus", - WemCodec.OpusNx => "Opus NX", - WemCodec.OpusWw => "Wwise Opus", - WemCodec.Aac => "AAC", - WemCodec.Wma2 => "WMAv2", - WemCodec.WmaPro => "WMA Pro", - WemCodec.Xma2Chunk => "XMA2", - WemCodec.Xma2Fmt => "XMA2", - WemCodec.Dsp => "DSP", - WemCodec.Hevag => "HEVAG", - WemCodec.Atrac9 => "ATRAC9", - _ => $"Unknown (0x{(ushort)codec:X4})", - }; -} diff --git a/BydTools.sln b/BydTools.sln index a750c0c..e87ad6c 100644 --- a/BydTools.sln +++ b/BydTools.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31903.59 @@ -7,11 +7,15 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.CLI", "BydTools.CL EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.VFS", "BydTools.VFS\BydTools.VFS.csproj", "{D7B0A018-BA70-447A-A8D6-C89A53735159}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.PCK", "BydTools.PCK\BydTools.PCK.csproj", "{55BB8D83-2535-40AC-AFF0-86D7D1E900B4}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.Core", "BydTools.Core\BydTools.Core.csproj", "{6F107594-B476-4913-91CF-FB5D39AD590D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.Utils", "BydTools.Utils\BydTools.Utils.csproj", "{A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.Formats", "BydTools.Formats\BydTools.Formats.csproj", "{2DFA0D42-2FBD-415B-965F-4FBE51101AB2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.Wwise", "BydTools.Wwise\BydTools.Wwise.csproj", "{B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.Audio", "BydTools.Audio\BydTools.Audio.csproj", "{FBE01096-538E-470F-AE3E-9EEB59754DAD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.Extraction", "BydTools.Extraction\BydTools.Extraction.csproj", "{7045D8FB-C043-4B44-B3CA-960D472D11C2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BydTools.Tests", "BydTools.Tests\BydTools.Tests.csproj", "{1EA61CFA-25D3-4535-9893-38BAE115C113}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -47,42 +51,66 @@ Global {D7B0A018-BA70-447A-A8D6-C89A53735159}.Release|x64.Build.0 = Release|Any CPU {D7B0A018-BA70-447A-A8D6-C89A53735159}.Release|x86.ActiveCfg = Release|Any CPU {D7B0A018-BA70-447A-A8D6-C89A53735159}.Release|x86.Build.0 = Release|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Debug|x64.ActiveCfg = Debug|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Debug|x64.Build.0 = Debug|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Debug|x86.ActiveCfg = Debug|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Debug|x86.Build.0 = Debug|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Release|Any CPU.Build.0 = Release|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Release|x64.ActiveCfg = Release|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Release|x64.Build.0 = Release|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Release|x86.ActiveCfg = Release|Any CPU - {55BB8D83-2535-40AC-AFF0-86D7D1E900B4}.Release|x86.Build.0 = Release|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Debug|x64.ActiveCfg = Debug|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Debug|x64.Build.0 = Debug|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Debug|x86.ActiveCfg = Debug|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Debug|x86.Build.0 = Debug|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Release|Any CPU.Build.0 = Release|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Release|x64.ActiveCfg = Release|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Release|x64.Build.0 = Release|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Release|x86.ActiveCfg = Release|Any CPU - {A9C3D4E5-6F78-4B9C-8D1A-2E3F4A5B6C7D}.Release|x86.Build.0 = Release|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Debug|x64.ActiveCfg = Debug|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Debug|x64.Build.0 = Debug|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Debug|x86.ActiveCfg = Debug|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Debug|x86.Build.0 = Debug|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Release|Any CPU.Build.0 = Release|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Release|x64.ActiveCfg = Release|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Release|x64.Build.0 = Release|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Release|x86.ActiveCfg = Release|Any CPU - {B1C2D3E4-F5A6-47B8-9C0D-1E2F3A4B5C6D}.Release|x86.Build.0 = Release|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Debug|x64.ActiveCfg = Debug|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Debug|x64.Build.0 = Debug|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Debug|x86.ActiveCfg = Debug|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Debug|x86.Build.0 = Debug|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Release|Any CPU.Build.0 = Release|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Release|x64.ActiveCfg = Release|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Release|x64.Build.0 = Release|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Release|x86.ActiveCfg = Release|Any CPU + {6F107594-B476-4913-91CF-FB5D39AD590D}.Release|x86.Build.0 = Release|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Debug|x64.ActiveCfg = Debug|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Debug|x64.Build.0 = Debug|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Debug|x86.ActiveCfg = Debug|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Debug|x86.Build.0 = Debug|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Release|Any CPU.Build.0 = Release|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Release|x64.ActiveCfg = Release|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Release|x64.Build.0 = Release|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Release|x86.ActiveCfg = Release|Any CPU + {2DFA0D42-2FBD-415B-965F-4FBE51101AB2}.Release|x86.Build.0 = Release|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Debug|x64.ActiveCfg = Debug|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Debug|x64.Build.0 = Debug|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Debug|x86.ActiveCfg = Debug|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Debug|x86.Build.0 = Debug|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Release|Any CPU.Build.0 = Release|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Release|x64.ActiveCfg = Release|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Release|x64.Build.0 = Release|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Release|x86.ActiveCfg = Release|Any CPU + {FBE01096-538E-470F-AE3E-9EEB59754DAD}.Release|x86.Build.0 = Release|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Debug|x64.ActiveCfg = Debug|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Debug|x64.Build.0 = Debug|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Debug|x86.ActiveCfg = Debug|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Debug|x86.Build.0 = Debug|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Release|Any CPU.Build.0 = Release|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Release|x64.ActiveCfg = Release|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Release|x64.Build.0 = Release|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Release|x86.ActiveCfg = Release|Any CPU + {7045D8FB-C043-4B44-B3CA-960D472D11C2}.Release|x86.Build.0 = Release|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Debug|x64.ActiveCfg = Debug|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Debug|x64.Build.0 = Debug|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Debug|x86.ActiveCfg = Debug|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Debug|x86.Build.0 = Debug|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Release|Any CPU.Build.0 = Release|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Release|x64.ActiveCfg = Release|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Release|x64.Build.0 = Release|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Release|x86.ActiveCfg = Release|Any CPU + {1EA61CFA-25D3-4535-9893-38BAE115C113}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..549e94a --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + latest + true + true + + diff --git a/NOTICES.md b/NOTICES.md index 628823e..8008582 100644 --- a/NOTICES.md +++ b/NOTICES.md @@ -59,12 +59,29 @@ SOFTWARE. - **Source**: https://github.com/vgmstream/vgmstream - **License**: ISC / MIT (depending on component) - **Copyright**: vgmstream contributors -- **Usage**: `libvgmstream.dll` is used via P/Invoke for WEM → WAV decoding in `BydTools.Wwise`. +- **Usage**: `libvgmstream.dll` is used via P/Invoke for WEM → WAV decoding in `BydTools.Audio`. Falls back to `vgmstream-cli` when the DLL is unavailable. -- **Bundled files** (in `BydTools.Wwise/3rdParty/`): +- **Bundled files** (in `BydTools.Audio/3rdParty/`): - `libvgmstream.dll` — core decoding library - `libvorbis.dll` — Vorbis decoder (Wwise Vorbis) - `avcodec-vgmstream-59.dll` — FFmpeg avcodec (Wwise Opus) - `avformat-vgmstream-59.dll` — FFmpeg avformat - `avutil-vgmstream-57.dll` — FFmpeg avutil - - `swresample-vgmstream-4.dll` — FFmpeg swresample + +--- + +## CSChaCha20 + +- **Source**: https://github.com/KaarloR/CSChaCha20 +- **License**: ISC +- **Copyright**: (c) 2015, 2018 Scott Bennett; (c) 2018-2023 Kaarlo Räihä +- **Usage**: ChaCha20 decryption of BLC payloads and encrypted VFS file entries, in `BydTools.Core`. + +--- + +## XXTEA + +- **Source**: https://github.com/xxtea/xxtea-dotnet +- **License**: MIT +- **Copyright**: (c) 2008-2016 Ma Bingyao. Algorithm by David J. Wheeler and Roger M. Needham. +- **Usage**: Lua script decryption in `BydTools.Formats`. diff --git a/README.md b/README.md index b056eeb..5587a2e 100644 --- a/README.md +++ b/README.md @@ -31,75 +31,79 @@ ## Prerequisites - [.NET 10 SDK](https://dotnet.microsoft.com/download) as runtime -- [ffmpeg](https://ffmpeg.org/) must be in `PATH` +- WAV conversion uses the bundled `libvgmstream` on Windows. Elsewhere, `vgmstream-cli` must be on `PATH`. `raw` mode does not need it. ## Usage > [!NOTE] -> Some features may generate a large number of small files (e.g., `BydTools.VFS` extracting `Bundle` or `BydTools.PCK`), which could affect disk performance or write endurance. +> Dumping `Bundle` or extracting audio can write a large number of small files. -```help +Exit codes: `0` success, `2` bad arguments, `1` a runtime failure. Ctrl+C stops the run and deletes files that were only partly written. + +```text Usage: BydTools [options] Commands: - vfs Dump files from VFS - pck Extract audio from VFS - -Options: - -h, --help Show help information + vfs Dump files from VFS + pck Extract audio from VFS ``` -### BydTools.VFS - -Dump specific types of files from the game's VFS. +### vfs -```help -Usage: - BydTools vfs --input --output --blocktype [,type2,...] [options] +```text +BydTools vfs --input --output --blocktype [,type2,...] Required: - -i, --input Game data directory that contains the VFS folder - -o, --output Output directory - -t, --blocktype Block type to dump (name or numeric value) - Multiple types can be separated by comma, e.g. Bundle,Lua,Table + -i, --input Game data directory that contains the VFS folder + -o, --output Output directory + -t, --blocktype Block type. Commas separate several types. Options: - --debug Scan subfolders and print block info (no extraction) - -v, --verbose Enable verbose output - -h, --help Show help information - -Available block types: - InitAudio, InitBundle, BundleManifest, InitialExtendData, Audio, Bundle, - DynamicStreaming, Table, Video, IV, Streaming, JsonData, Lua, IFixPatchOut, - ExtendData, AudioChinese, AudioEnglish, AudioJapanese, AudioKorean + --key ChaCha20 key (32 bytes). Overrides --platform. + --platform pc (default) or android, used when --key is omitted. + --jobs Chunks read at once. Default: processor count. + -d, --debug Print block declarations without extracting. + -v, --verbose Per-chunk and per-file details. + -h, --help + +Block types: + InitAudio, InitBundle, InitialExtendData, BundleManifest, IFixPatchOut, + AuditStreaming, AuditDynamicStreaming, AuditIV, AuditAudio, AuditVideo, + Bundle, Audio, Video, IV, Streaming, DynamicStreaming, Lua, Table, JsonData, + ExtendData, HotfixAudio, AudioChinese, AudioEnglish, AudioJapanese, AudioKorean ``` -### BydTools.PCK +A directory that is not in this list can still be dumped by its `groupCfgName`. `All`, `Raw`, and numbers that are not a real block type are rejected. `--debug` prints the blocks actually present, including ones the catalog does not know yet. -Extract audio from VFS and convert WEM to WAV. Automatically maps filenames via AudioDialog. +### pck -```help -Usage: - BydTools pck --input --output --type [options] +Extracts `.pck` audio and, in `wav` mode, converts WEM to WAV. Names come from the Table block's AudioDialog when that block is present. + +```text +BydTools pck --input --output --type Required: - -i, --input Game data directory that contains the VFS folder - -o, --output Output directory - -t, --type Audio block type to extract + -i, --input Game data directory that contains the VFS folder + -o, --output Output directory + -t, --type Audio block, or a groupCfgName found in this VFS. Options: - -m, --mode Extract mode (default: wav) - raw Extract wem without conversion - wav Convert to wav via vgmstream - --no-map Disable automatic AudioDialog filename mapping - -v, --verbose Enable verbose output - -h, --help Show help information + -m, --mode wav (default) or raw. + --key ChaCha20 key (32 bytes). Overrides --platform. + --platform pc (default) or android. + --jobs WEM files converted at once. Default: processor count. + --no-map Skip AudioDialog name mapping. + -v, --verbose + -h, --help Audio block types: - InitAudio, Audio, AudioChinese, AudioEnglish, AudioJapanese, AudioKorean + InitAudio, Audio, AudioChinese, AudioEnglish, AudioJapanese, AudioKorean, + AuditAudio, HotfixAudio ``` +Files with no AudioDialog match go under `/unmapped//`. The language is the one stored in the PCK when it is known, otherwise the block's own language (Main, Chinese, Hotfix, and so on). `PLUG` entries are written as `.plg` in `raw` mode and skipped in `wav` mode. Extracting a USM again overwrites the previous video and audio streams instead of creating `_2`, `_3`, and so on. + ## License This project is licensed under [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/). @@ -109,5 +113,7 @@ This project includes code ported from or inspired by the following open-source - [AnimeStudio](https://github.com/Escartem/AnimeStudio) - [AnimeWwise](https://github.com/Escartem/AnimeWwise) - [vgmstream](https://github.com/vgmstream/vgmstream) +- [CSChaCha20](https://github.com/KaarloR/CSChaCha20) +- [XXTEA](https://github.com/xxtea/xxtea-dotnet) See [NOTICES.md](NOTICES.md) for full details. diff --git a/scripts/build-vgmstream-wem-min.ps1 b/scripts/build-vgmstream-wem-min.ps1 index 235869f..5f6a35b 100644 --- a/scripts/build-vgmstream-wem-min.ps1 +++ b/scripts/build-vgmstream-wem-min.ps1 @@ -1,7 +1,7 @@ param( [string]$VsDevCmdPath = "D:\Program Files\Microsoft Visual Studio\18\Enterprise\Common7\Tools\VsDevCmd.bat", [string]$VgmstreamRoot = "F:\Workplace\Endfield\vgmstream", - [string]$Project3rdPartyDir = "$PSScriptRoot\..\BydTools.Wwise\3rdParty", + [string]$Project3rdPartyDir = "$PSScriptRoot\..\BydTools.Audio\3rdParty", [string]$BuildDir = "", [string]$Generator = "Visual Studio 18 2026", [string]$Configuration = "Release",