diff --git a/.gitignore b/.gitignore index b8648c5e1..8f8dabbfa 100644 --- a/.gitignore +++ b/.gitignore @@ -340,3 +340,5 @@ ASALocalRun/ # BeatPulse healthcheck temp database healthchecksdb +# MonoGame build outputs +/GameWorld/ContentProject/Content/bin/ diff --git a/Editors/Reports/DeepSearch/DeepSearchReport.cs b/Editors/Reports/DeepSearch/DeepSearchReport.cs index 82f79b4a8..26edd0f79 100644 --- a/Editors/Reports/DeepSearch/DeepSearchReport.cs +++ b/Editors/Reports/DeepSearch/DeepSearchReport.cs @@ -77,7 +77,7 @@ public List DeepSearch(string searchStr, bool caseSensetive) { var pf = packFile; var ds = pf.DataSource as PackedFileSource; - var bytes = ds.ReadDataForFastSearch(fileStram); + var bytes = ds.ReadData(fileStram); var str = Encoding.ASCII.GetString(bytes); if (str.Contains(searchStr, StringComparison.InvariantCultureIgnoreCase)) diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Fonts/DefaultFont.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Fonts/DefaultFont.xnb deleted file mode 100644 index 3ab760690..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Fonts/DefaultFont.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Bloom.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Bloom.xnb deleted file mode 100644 index 7ed57eefe..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Bloom.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Geometry/BasicShader.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Geometry/BasicShader.xnb deleted file mode 100644 index b1572e49b..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Geometry/BasicShader.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/InstancingShader.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/InstancingShader.xnb deleted file mode 100644 index 12c1d30df..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/InstancingShader.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/LineShader.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/LineShader.xnb deleted file mode 100644 index fc57b5f5f..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/LineShader.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Pbr/MetalRoughness/MetalRoughness_main.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Pbr/MetalRoughness/MetalRoughness_main.xnb deleted file mode 100644 index 0173135ac..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Pbr/MetalRoughness/MetalRoughness_main.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Pbr/SpecGloss/SpecGloss_main.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Pbr/SpecGloss/SpecGloss_main.xnb deleted file mode 100644 index ad07eab48..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/Pbr/SpecGloss/SpecGloss_main.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/TexturePreview.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/TexturePreview.xnb deleted file mode 100644 index 55ac06d61..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Shaders/TexturePreview.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/Brdf_rgba32f_raw.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/Brdf_rgba32f_raw.xnb deleted file mode 100644 index a608d6dd2..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/Brdf_rgba32f_raw.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/DiffuseAmbientLightCubeMap.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/DiffuseAmbientLightCubeMap.xnb deleted file mode 100644 index 72f68bde5..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/DiffuseAmbientLightCubeMap.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/SpecularAmbientLightCubemap.xnb b/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/SpecularAmbientLightCubemap.xnb deleted file mode 100644 index cb4c2fa39..000000000 Binary files a/GameWorld/ContentProject/Content/bin/Windows/Content/Textures/Phazer/SpecularAmbientLightCubemap.xnb and /dev/null differ diff --git a/GameWorld/ContentProject/ContentProject.csproj b/GameWorld/ContentProject/ContentProject.csproj index d654a6e0e..772569eaf 100644 --- a/GameWorld/ContentProject/ContentProject.csproj +++ b/GameWorld/ContentProject/ContentProject.csproj @@ -8,8 +8,11 @@ + + + diff --git a/GameWorld/View3D/Services/ComplexMeshLoader.cs b/GameWorld/View3D/Services/ComplexMeshLoader.cs index deff7c53e..391e4d5d0 100644 --- a/GameWorld/View3D/Services/ComplexMeshLoader.cs +++ b/GameWorld/View3D/Services/ComplexMeshLoader.cs @@ -45,7 +45,7 @@ SceneNode Load(PackFile file, SceneNode parent, AnimationPlayer player, string a _logger.Here().Information($"Attempting to load file {file.Name}"); - switch (file.Extention) + switch (file.Extension) { case ".variantmeshdefinition": LoadVariantMesh(file, ref parent, player, attachmentPointName, onlyLoadRootNode, onlyLoadFirstMesh); diff --git a/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs b/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs index 3f1c0af71..f7e47bc3c 100644 --- a/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs +++ b/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs @@ -89,21 +89,22 @@ void LoadFromPackFileContainer(PackFileContainer packFileContainer) // Handle packfile which are stored in a saved file. // This is done for performance reasons. Opening all the animations files from disk is very slow // creating stream which is reused goes a lot faster! + // https://www.jacksondunstan.com/articles/3568 var groupedAnims = allAnimsInSavedPackedFiles .GroupBy(x => x.DataSource.Parent.FilePath) .ToList(); Parallel.For(0, groupedAnims.Count, index => { - using var handle = File.OpenHandle(groupedAnims[index].Key); - - //https://www.jacksondunstan.com/articles/3568 - var buffer = new byte[100]; - + using var stream = new FileStream(groupedAnims[index].Key, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + foreach (var file in groupedAnims[index]) { - RandomAccess.Read(handle, buffer, file.DataSource.Offset); - FileDiscovered(buffer, packFileContainer, file.FullPath, ref skeletonFileNameList, ref animationList); + var bytes = file.DataSource.ReadData(stream); + if (bytes.Length > 100) + Array.Resize(ref bytes, 100); + + FileDiscovered(bytes, packFileContainer, file.FullPath, ref skeletonFileNameList, ref animationList); } }); @@ -134,7 +135,11 @@ void FileDiscovered(byte[] byteChunk, PackFileContainer container, string fullPa var brokenAnims = new string[] { "rigidmodels\\buildings\\roman_aqueduct_straight\\roman_aqueduct_straight_piece01_destruct01_anim.anim", - "animations\\battle\\raptor02\\subset\\colossal_squig\\deaths\\rp2_colossalsquig_death_01.anim" + "animations\\battle\\raptor02\\subset\\colossal_squig\\deaths\\rp2_colossalsquig_death_01.anim", + "animations\\battle\\humanoid13b\\golgfag\\docking\\hu13b_golgfag_docking_armed_02.anim", + "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_shoot_back_crossbow_01.anim", + "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_reload_crossbow_01.anim", + "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_sp_rider1_shoot_ready_crossbow_01.anim" }; if (brokenAnims.Contains(fullPath)) { @@ -145,6 +150,10 @@ void FileDiscovered(byte[] byteChunk, PackFileContainer container, string fullPa try { + if (byteChunk.Length == 0) + { + throw new Exception("File empty."); + } var animationSkeletonName = AnimationFile.GetAnimationName(byteChunk); lock (_threadLock) diff --git a/Shared/SharedCore/ByteParsing/ByteHelper.cs b/Shared/SharedCore/ByteParsing/ByteHelper.cs index 43a930216..e34629505 100644 --- a/Shared/SharedCore/ByteParsing/ByteHelper.cs +++ b/Shared/SharedCore/ByteParsing/ByteHelper.cs @@ -85,6 +85,5 @@ public static uint GetPropertyTypeSize(T property) return (uint)Marshal.SizeOf(property); } - } } diff --git a/Shared/SharedCore/ErrorHandling/PackFileLog.cs b/Shared/SharedCore/ErrorHandling/PackFileLog.cs new file mode 100644 index 000000000..d813d869a --- /dev/null +++ b/Shared/SharedCore/ErrorHandling/PackFileLog.cs @@ -0,0 +1,140 @@ +using Shared.Core.PackFiles; +using Shared.Core.PackFiles.Models; + +namespace Shared.Core.ErrorHandling +{ + public class CompressionStats + { + public long DiskSize { get; set; } + public long UncompressedSize { get; set; } + + public CompressionStats(long diskSize = 0, long uncompressedSize = 0) + { + DiskSize = diskSize; + UncompressedSize = uncompressedSize; + } + + public void Add(CompressionStats stat) + { + DiskSize += stat.DiskSize; + UncompressedSize += stat.UncompressedSize; + } + } + + public static class PackFileLog + { + private static readonly ILogger s_logger = Logging.CreateStatic(typeof(PackFileLog)); + + public static Dictionary GetCompressionStats(PackFileContainer container) + { + var stats = new Dictionary(); + + foreach (var packFile in container.FileList.Values) + { + if (packFile.DataSource is PackedFileSource source) + { + var format = source.IsCompressed + ? source.CompressionFormat + : CompressionFormat.None; + + if (!stats.TryGetValue(format, out var totals)) + { + totals = new CompressionStats(); + stats[format] = totals; + } + + totals.DiskSize += source.Size; + totals.UncompressedSize += source.IsCompressed ? source.UncompressedSize : 0L; + } + } + + return stats; + } + + public static void LogPackCompression(PackFileContainer container) + { + var stats = GetCompressionStats(container); + var totalFiles = container.FileList.Count; + var packSizeFmt = FormatSize(container.OriginalLoadByteSize); + + var loadingPart = $"Loading {container.Name}.pack ({totalFiles} files, {packSizeFmt})"; + + var fileCounts = new Dictionary(); + foreach (var pf in container.FileList.Values) + { + if (pf.DataSource is PackedFileSource src) + { + var fmt = src.IsCompressed + ? src.CompressionFormat + : CompressionFormat.None; + + if (!fileCounts.TryGetValue(fmt, out var cnt)) + fileCounts[fmt] = 1; + else + fileCounts[fmt] = cnt + 1; + } + } + + var segments = stats + .OrderBy(kvp => kvp.Key) + .Select(kvp => + { + var fmt = kvp.Key; + var count = fileCounts.TryGetValue(fmt, out var c) ? c : 0; + var disk = FormatSize(kvp.Value.DiskSize); + + if (fmt == CompressionFormat.None) + return $"{fmt}: {count} files, {disk} (Disk Size)"; + + var unc = FormatSize(kvp.Value.UncompressedSize); + return $"{fmt}: {count} files, {disk} (Disk Size), {unc} (Uncompressed Size)"; + }) + .ToList(); + + var compressionPart = $"File Compression – {string.Join(" | ", segments)}"; + s_logger.Here().Information($"{loadingPart} | {compressionPart}"); + } + + public static void LogPacksCompression(IDictionary globalStats) + { + var segments = globalStats + .OrderBy(kvp => kvp.Key) + .Select(kvp => + { + var format = kvp.Key; + var diskFormatted = FormatSize(kvp.Value.DiskSize); + + if (format == CompressionFormat.None) + return $"{format}: {diskFormatted} (Disk Size)"; + + var uncompressedFormatted = FormatSize(kvp.Value.UncompressedSize); + return $"{format}: {diskFormatted} (Disk Size), {uncompressedFormatted} (Uncompressed Size)"; + }) + .ToList(); + + var totalDisk = globalStats.Values.Sum(stat => stat.DiskSize); + var totalUncompressed = globalStats.Values.Sum(stat => stat.UncompressedSize); + + var totalDiskFormatted = FormatSize(totalDisk); + var totalUncompressedFormatted = FormatSize(totalUncompressed); + + var totalSegment = $"Total: {totalDiskFormatted} (Disk Size), {totalUncompressedFormatted} (Uncompressed Size)"; + var summary = string.Join(" | ", segments.Append(totalSegment)); + + s_logger.Here().Information($"Size of compressed files in all packs by format - {summary}"); + } + + private static string FormatSize(long bytes) + { + var kb = 1024.0; + var mb = kb * 1024.0; + var gb = mb * 1024.0; + + if (bytes >= gb) + return $"{bytes / gb:F2} GB"; + if (bytes >= mb) + return $"{bytes / mb:F2} MB"; + return $"{bytes / kb:F2} KB"; + } + } +} diff --git a/Shared/SharedCore/PackFiles/IPackFileService.cs b/Shared/SharedCore/PackFiles/IPackFileService.cs index 4ac0b369f..0247656b7 100644 --- a/Shared/SharedCore/PackFiles/IPackFileService.cs +++ b/Shared/SharedCore/PackFiles/IPackFileService.cs @@ -1,4 +1,5 @@ using Shared.Core.PackFiles.Models; +using Shared.Core.Settings; namespace Shared.Core.PackFiles { @@ -22,9 +23,8 @@ public interface IPackFileService void RenameDirectory(PackFileContainer pf, string currentNodeName, string newName); void RenameFile(PackFileContainer pf, PackFile file, string newName); void SaveFile(PackFile file, byte[] data); - void SavePackContainer(PackFileContainer pf, string path, bool createBackup); + void SavePackContainer(PackFileContainer pf, string path, bool createBackup, GameInformation gameInformation); void SetEditablePack(PackFileContainer? pf); void UnloadPackContainer(PackFileContainer pf); } - } diff --git a/Shared/SharedCore/PackFiles/Models/DataSource.cs b/Shared/SharedCore/PackFiles/Models/DataSource.cs index dc4e09108..1b658c96e 100644 --- a/Shared/SharedCore/PackFiles/Models/DataSource.cs +++ b/Shared/SharedCore/PackFiles/Models/DataSource.cs @@ -1,5 +1,5 @@ using Shared.Core.ByteParsing; -using static Shared.Core.PackFiles.PackFileDecrypter; +using Shared.Core.Settings; namespace Shared.Core.PackFiles.Models { @@ -89,16 +89,29 @@ public record PackedFileSource : IDataSource public long Offset { get; private set; } public long Size { get; private set; } public bool IsEncrypted { get; private set; } + public bool IsCompressed { get; set; } + public CompressionFormat CompressionFormat { get; set; } + public uint UncompressedSize { get; set; } public PackedFileSourceParent Parent { get => _parent; } private readonly PackedFileSourceParent _parent; - public PackedFileSource(PackedFileSourceParent parent, long offset, long length, bool isEncrypted) + public PackedFileSource( + PackedFileSourceParent parent, + long offset, + long length, + bool isEncrypted, + bool isCompressed, + CompressionFormat compressionFormat, + uint uncompressedSize) { Offset = offset; _parent = parent; Size = length; IsEncrypted = isEncrypted; + IsCompressed = isCompressed; + CompressionFormat = compressionFormat; + UncompressedSize = uncompressedSize; } public byte[] ReadData() @@ -111,7 +124,9 @@ public byte[] ReadData() } if (IsEncrypted) - data = Decrypt(data); + data = PackFileEncryption.Decrypt(data); + if (IsCompressed) + data = PackFileCompression.Decompress(data); return data; } @@ -120,23 +135,27 @@ public byte[] ReadData(int size) var data = new byte[size]; using (Stream stream = File.Open(_parent.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { - stream.Seek(Offset, SeekOrigin.Begin); - stream.Read(data, 0, size); + stream.Seek(Offset, SeekOrigin.Begin); + stream.Read(data, 0, data.Length); } if (IsEncrypted) - data = Decrypt(data); + data = PackFileEncryption.Decrypt(data); + if (IsCompressed) + data = PackFileCompression.Decompress(data); return data; } - public byte[] ReadDataForFastSearch(Stream knownStream) + public byte[] ReadData(Stream knownStream) { var data = new byte[Size]; knownStream.Seek(Offset, SeekOrigin.Begin); knownStream.Read(data, 0, (int)Size); if (IsEncrypted) - data = Decrypt(data); + data = PackFileEncryption.Decrypt(data); + if (IsCompressed) + data = PackFileCompression.Decompress(data); return data; } @@ -144,6 +163,59 @@ public ByteChunk ReadDataAsChunk() { return new ByteChunk(ReadData()); } + + public void SetCompressionInfo(GameInformation gameInformation, string rootFolder, string extension) + { + // Check if the game supports any compression at all + if (gameInformation.CompressionFormats.All(compressionFormat => compressionFormat == CompressionFormat.None)) + return; + + // We use isTable because non-loc tables don't have an extension + var isTable = rootFolder == "db" || extension == ".loc"; + var hasExtension = !string.IsNullOrEmpty(extension); + + // Don't compress files that aren't tables and don't have extensions + if (!isTable && !hasExtension) + { + CompressionFormat = CompressionFormat.None; + IsCompressed = false; + return; + } + + // Only in WH3 (and newer games?) is the table compression bug fixed + if (isTable && gameInformation.CompressionFormats.Contains(CompressionFormat.Zstd) && gameInformation.Type == GameTypeEnum.Warhammer3) + { + CompressionFormat = CompressionFormat.Zstd; + IsCompressed = true; + return; + } + + // Games that support the other formats won't use Lzma1 as it's legacy so if it's set then it's for a game that only uses it so keep it + if (CompressionFormat == CompressionFormat.Lzma1 && gameInformation.CompressionFormats.Contains(CompressionFormat.Lzma1)) + return; + + // Anything that shouldn't be None or Lz4 is set to Zstd unless the game doesn't support that in which case use None + if (PackFileCompression.NoneFileTypes.Contains(extension)) + { + CompressionFormat = CompressionFormat.None; + IsCompressed = false; + } + else if (PackFileCompression.Lz4FileTypes.Contains(extension) && gameInformation.CompressionFormats.Contains(CompressionFormat.Lz4)) + { + CompressionFormat = CompressionFormat.Lz4; + IsCompressed = true; + } + else if (gameInformation.CompressionFormats.Contains(CompressionFormat.Zstd)) + { + CompressionFormat = CompressionFormat.Zstd; + IsCompressed = true; + } + else + { + CompressionFormat = CompressionFormat.None; + IsCompressed = false; + } + } } public class PackedFileSourceParent diff --git a/Shared/SharedCore/PackFiles/Models/PackFile.cs b/Shared/SharedCore/PackFiles/Models/PackFile.cs index 6619890a5..701a684e1 100644 --- a/Shared/SharedCore/PackFiles/Models/PackFile.cs +++ b/Shared/SharedCore/PackFiles/Models/PackFile.cs @@ -7,7 +7,7 @@ public class PackFile { public IDataSource DataSource { get; set; } public string Name { get; set; } - public string Extention { get => Path.GetExtension(Name); } + public string Extension { get => Path.GetExtension(Name); } public PackFile(string name, IDataSource dataSource) { diff --git a/Shared/SharedCore/PackFiles/Models/PackFileContainer.cs b/Shared/SharedCore/PackFiles/Models/PackFileContainer.cs index 78ea59fa3..83752f7b2 100644 --- a/Shared/SharedCore/PackFiles/Models/PackFileContainer.cs +++ b/Shared/SharedCore/PackFiles/Models/PackFileContainer.cs @@ -1,4 +1,5 @@ -using Shared.Core.Settings; +using System.Text; +using Shared.Core.Settings; namespace Shared.Core.PackFiles.Models { @@ -7,7 +8,7 @@ public class PackFileContainer public string Name { get; set; } public PFHeader Header { get; set; } public bool IsCaPackFile { get; set; } = false; - public string SystemFilePath { get; set; } + public string SystemFilePath { get; set; } public long OriginalLoadByteSize { get; set; } = -1; public Dictionary FileList { get; set; } = []; @@ -24,7 +25,7 @@ public void MergePackFileContainer(PackFileContainer other) return; } - public void SaveToByteArray(BinaryWriter writer) + public void SaveToByteArray(BinaryWriter writer, GameInformation gameInformation) { long fileNamesOffset = 0; var sortedFiles = FileList.OrderBy(x => x.Key, StringComparer.Ordinal).ToList(); @@ -51,24 +52,42 @@ public void SaveToByteArray(BinaryWriter writer) fileNamesOffset2 += fileSize + headerSpesificBytes + strLength; } - Header.FileCount = (uint)FileList.Count(); + Header.FileCount = (uint)FileList.Count; PackFileSerializer.WriteHeader(Header, (uint)fileNamesOffset, writer); // Save all the files foreach (var file in sortedFiles) { - var fileSize = (int)file.Value.DataSource.Size; + var packFile = file.Value; + var packedFileSource = (PackedFileSource)file.Value.DataSource; + var data = packedFileSource.ReadData(); + + var fileExtension = packFile.Extension; + + var segments = file.Key.Split(['\\', '/'], StringSplitOptions.RemoveEmptyEntries); + var rootFolder = segments.First(); + + packedFileSource.SetCompressionInfo(gameInformation, rootFolder, fileExtension); + + var fileSize = data.Length; + if (packedFileSource.IsCompressed) + { + var compressedData = PackFileCompression.Compress(data, packedFileSource.CompressionFormat); + fileSize = compressedData.Length; + } writer.Write(fileSize); + // Timestamp if (Header.HasIndexWithTimeStamp) - writer.Write(0); // timestamp + writer.Write(0); + // Compression if (Header.Version == PackFileVersion.PFH5) - writer.Write((byte)0); // Compression + writer.Write(packedFileSource.IsCompressed); // Filename - foreach (byte c in file.Key) - writer.Write(c); + var fileNameBytes = Encoding.UTF8.GetBytes(file.Key); + writer.Write(fileNameBytes); // Zero terminator writer.Write((byte)0); @@ -82,11 +101,23 @@ public void SaveToByteArray(BinaryWriter writer) // Write the files foreach (var file in sortedFiles) { - var data = file.Value.DataSource.ReadData(); + var packFile = file.Value; + var packedFileSource = (PackedFileSource)packFile.DataSource; + var offset = writer.BaseStream.Position; - var dataLength = data.Length; - var isEncrypted = Header.HasEncryptedData; - file.Value.DataSource = new PackedFileSource(packedFileSourceParent, offset, dataLength, isEncrypted); + var data = packedFileSource.ReadData(); + if (packedFileSource.IsCompressed) + data = PackFileCompression.Compress(data, packedFileSource.CompressionFormat); + + packFile.DataSource = new PackedFileSource( + packedFileSourceParent, + offset, + data.Length, + packedFileSource.IsEncrypted, + packedFileSource.IsCompressed, + packedFileSource.CompressionFormat, + packedFileSource.UncompressedSize); + writer.Write(data); } } diff --git a/Shared/SharedCore/PackFiles/PackFileCompression.cs b/Shared/SharedCore/PackFiles/PackFileCompression.cs new file mode 100644 index 000000000..988378151 --- /dev/null +++ b/Shared/SharedCore/PackFiles/PackFileCompression.cs @@ -0,0 +1,261 @@ +using EasyCompressor; +using K4os.Compression.LZ4.Encoders; +using K4os.Compression.LZ4.Streams; +using ZstdSharp; +using ZstdSharp.Unsafe; + +namespace Shared.Core.PackFiles +{ + public enum CompressionFormat + { + /// Dummy variant to disable compression. + None, + + /// Legacy format. Supported by all PFH5 games (all Post-WH2 games). + /// + /// Specifically, Total War games use the Non-Streamed LZMA1 format with the following custom header: + /// + /// | Bytes | Type | Data | + /// | ----- | ----- | ----------------------------------------------------------------------------------- | + /// | 4 | [u32] | Uncompressed size (as u32, max at 4GB). | + /// | 1 | [u8] | LZMA model properties (lc, lp, pb) in encoded form... I think. Usually it's `0x5D`. | + /// | 4 | [u32] | Dictionary size (as u32)... I think. It's usually `[0x00, 0x00, 0x40, 0x00]`. | + /// + /// For reference, a normal Non-Streamed LZMA1 header (from the original spec) contains: + /// + /// | Bytes | Type | Data | + /// | ----- | ------------- | ----------------------------------------------------------- | + /// | 1 | [u8] | LZMA model properties (lc, lp, pb) in encoded form. | + /// | 4 | [u32] | Dictionary size (32-bit unsigned integer, little-endian). | + /// | 8 | [prim@u64] | Uncompressed size (64-bit unsigned integer, little-endian). | + /// + /// This means one has to move the uncompressed size to the correct place in order for a compressed file to be readable, + /// and one has to remove the uncompressed size and prepend it to the file in order for the game to read the compressed file. + Lzma1, + + /// New format introduced in WH3 6.2. + /// + /// This is a standard Lz4 implementation, with the following tweaks: + /// + /// | Bytes | Type | Data | + /// | ----- | --------- | --------------------------------------------- | + /// | 4 | [u32] | Uncompressed size (as u32, max at 4GB). | + /// | * | &[[`u8`]] | Lz4 data, starting with the Lz4 Magic Number. | + Lz4, + + /// New format introduced in WH3 6.2. + /// + /// This is a standard Zstd implementation, with the following tweaks: + /// + /// | Bytes | Type | Data | + /// | ----- | --------- | ----------------------------------------------- | + /// | 4 | [u32] | Uncompressed size (as u32, max at 4GB). | + /// | * | &[[`u8`]] | Zstd data, starting with the Zstd Magic Number. | + /// + /// By default the Zstd compression is done with the checksum and content size flags enabled. + Zstd + } + + public static class PackFileCompression + { + // LZMA alone doesn't have a defined magic number, but it always starts with one of these, depending on the compression level + private static readonly uint[] s_magicNumbersLzma = [ + 0x0100_005D, + 0x1000_005D, + 0x0800_005D, + 0x2000_005D, + 0x4000_005D, + 0x8000_005D, + 0x0000_005D, + 0x0400_005D, + ]; + private static readonly uint s_magicNumberLz4 = 0x184D_2204; + private static readonly uint s_magicNumberZstd = 0xfd2f_b528; + + public static List NoneFileTypes { get; } = + [ + // Exclusive - In CA packs the files are exclusively in this format + ".bnk", + ".ca_vp8", + ".fxc", + ".hlsl_compiled", + ".log", + ".manifest", + ".wem", + + // Preferred - In CA packs the files are mostly in this format + ".dat", + ".rigid_model_v2", + + // RPFM - How RPFM formats the file + ".rpfm_reserved", + ]; + + public static List Lz4FileTypes { get; } = + [ + // Exclusive - In CA packs the files are exclusively in this format + ".animpack", + ".collision", + ".cs2", + ".exr", + ".mvscene", + ".variantmeshdefinition", + ".wsmodel", + ".xt", + + // Preferred - In CA packs the files are mostly in this format + ".parsed", + ]; + + public static byte[] Decompress(byte[] data) + { + var result = Array.Empty(); + if (data == null || data.Length == 0) + return result; + + using var stream = new MemoryStream(data, false); + using var reader = new BinaryReader(stream); + + // Read the header and get what we need + var uncompressedSize = reader.ReadUInt32(); + var magicNumber = reader.ReadUInt32(); + var compressionFormat = GetCompressionFormat(magicNumber); + stream.Seek(-4, SeekOrigin.Current); + + if (compressionFormat == CompressionFormat.Zstd) + return DecompressZstd(reader, uncompressedSize); + if (compressionFormat == CompressionFormat.Lz4) + return DecompressLz4(reader, uncompressedSize); + else if (compressionFormat == CompressionFormat.Lzma1) + result = DecompressLzma(data, uncompressedSize); + else if (compressionFormat == CompressionFormat.None) + return data; + + if (result.Length != uncompressedSize) + throw new InvalidDataException($"Expected {uncompressedSize:N0} bytes after decompression, but got {result.Length:N0}."); + + return result; + } + + private static byte[] DecompressZstd(BinaryReader reader, uint uncompressedSize) + { + var buffer = new byte[uncompressedSize]; + var output = new MemoryStream(buffer); + using var decompressionStream = new DecompressionStream(reader.BaseStream); + decompressionStream.CopyTo(output); + return output.ToArray(); + } + + private static byte[] DecompressLz4(BinaryReader reader, uint uncompressedSize) + { + var buffer = new byte[uncompressedSize]; + var output = new MemoryStream(buffer); + var decompressor = new LZ4DecoderStream(reader.BaseStream, i => new LZ4ChainDecoder(i.BlockSize, 0)); + decompressor.CopyTo(output); + return output.ToArray(); + } + + private static byte[] DecompressLzma(byte[] data, uint uncompressedSize) + { + var uncompressedSizeFieldSize = sizeof(uint); + var headerDataLength = 5; + var injectedSizeLength = sizeof(ulong); + + // Compute all the offsets + var headerStart = uncompressedSizeFieldSize; + var headerEnd = headerStart + headerDataLength; + var footerStart = headerEnd; + var minTotalSize = footerStart; + + // LZMA1 headers have 13 bytes, but we only have 9 due to using a u32 size + if (data.Length < minTotalSize) + throw new InvalidDataException("File too small to be valid LZMA."); + + // Unlike other formats, in this one we need to inject the uncompressed size in the file header otherwise it won't be a valid lzma file + using var primary = new MemoryStream(data.Length + injectedSizeLength); + primary.Write(data, headerStart, headerDataLength); + primary.Write(BitConverter.GetBytes((ulong)uncompressedSize), 0, injectedSizeLength); + primary.Write(data, footerStart, data.Length - footerStart); + primary.Position = 0; + + try + { + return LZMACompressor.Shared.Decompress(primary.ToArray()); + } + catch + { + // Some files may still fail so fall back to a unknown size (u64::MAX) instead + using var fallback = new MemoryStream(data.Length + injectedSizeLength); + fallback.Write(data, headerStart, headerDataLength); + fallback.Write(BitConverter.GetBytes(ulong.MaxValue), 0, injectedSizeLength); + fallback.Write(data, footerStart, data.Length - footerStart); + fallback.Position = 0; + return LZMACompressor.Shared.Decompress(fallback.ToArray()); + } + } + + public static byte[] Compress(byte[] data, CompressionFormat format) + { + if(format == CompressionFormat.Zstd) + return CompressZstd(data); + else if(format == CompressionFormat.Lz4) + return CompressLz4(data); + else if (format == CompressionFormat.Lzma1) + return CompressLzma1(data); + return data; + } + + private static byte[] CompressZstd(byte[] data) + { + using var stream = new MemoryStream(); + stream.Write(BitConverter.GetBytes((uint)data.Length)); + + using (var compressor = new CompressionStream(stream, 3, leaveOpen: true)) + { + compressor.SetParameter(ZSTD_cParameter.ZSTD_c_contentSizeFlag, 1); + compressor.SetParameter(ZSTD_cParameter.ZSTD_c_checksumFlag, 1); + compressor.SetPledgedSrcSize((ulong)data.Length); + compressor.Write(data, 0, data.Length); + } + + return stream.ToArray(); + } + + private static byte[] CompressLz4(byte[] data) + { + using var stream = new MemoryStream(); + stream.Write(BitConverter.GetBytes((uint)data.Length)); + + using (var encoder = LZ4Stream.Encode(stream, leaveOpen: true)) + encoder.Write(data, 0, data.Length); + + return stream.ToArray(); + } + + private static byte[] CompressLzma1(byte[] data) + { + var compressedData = LZMACompressor.Shared.Compress(data); + if (compressedData.Length < 13) + throw new InvalidDataException("Data cannot be compressed"); + + using var stream = new MemoryStream(); + stream.Write(BitConverter.GetBytes(data.Length), 0, 4); + stream.Write(compressedData, 0, 5); + stream.Write(compressedData, 13, compressedData.Length - 13); + + return stream.ToArray(); + } + + public static CompressionFormat GetCompressionFormat(uint magicNumber) + { + if (magicNumber == s_magicNumberZstd) + return CompressionFormat.Zstd; + else if (magicNumber == s_magicNumberLz4) + return CompressionFormat.Lz4; + else if (s_magicNumbersLzma.Contains(magicNumber)) + return CompressionFormat.Lzma1; + else + return CompressionFormat.None; + } + } +} diff --git a/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs b/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs index 3f9a6ef9e..23b95ddd7 100644 --- a/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs +++ b/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs @@ -1,4 +1,5 @@ -using System.Reflection; +using System.Collections.Concurrent; +using System.Reflection; using System.Text; using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models; @@ -42,7 +43,7 @@ public PackFileContainer LoadSystemFolderAsPackFileContainer(string packFileSyst return container; } - void AddFolderContentToPackFile(PackFileContainer container, string folderPath, string rootPath) + private static void AddFolderContentToPackFile(PackFileContainer container, string folderPath, string rootPath) { var files = Directory.GetFiles(folderPath); foreach (var filePath in files) @@ -73,9 +74,10 @@ void AddFolderContentToPackFile(PackFileContainer container, string folderPath, using var fileStream = File.OpenRead(packFileSystemPath); using var reader = new BinaryReader(fileStream, Encoding.ASCII); - var container = PackFileSerializer.Load(packFileSystemPath, reader, new CustomPackDuplicatePackFileResolver()); + var pack = PackFileSerializer.Load(packFileSystemPath, reader, new CustomPackDuplicatePackFileResolver()); + PackFileLog.LogPackCompression(pack); - return container; + return pack; } catch (Exception e) { @@ -108,26 +110,36 @@ void AddFolderContentToPackFile(PackFileContainer container, string folderPath, } var packList = new List(); - - //foreach(var packFilePath in allCaPackFiles) + var packsCompressionStats = new ConcurrentDictionary(); + Parallel.ForEach(allCaPackFiles, packFilePath => { var path = gameDataFolder + "\\" + packFilePath; if (File.Exists(path)) { - using var fileStram = File.OpenRead(path); - using var reader = new BinaryReader(fileStram, Encoding.ASCII); + using var fileStream = File.OpenRead(path); + using var reader = new BinaryReader(fileStream, Encoding.ASCII); var pack = PackFileSerializer.Load(path, reader, packfileResolver); packList.Add(pack); + + PackFileLog.LogPackCompression(pack); + var packCompressionStats = PackFileLog.GetCompressionStats(pack); + foreach (var kvp in packCompressionStats) + { + if (!packsCompressionStats.TryGetValue(kvp.Key, out var existingStats)) + packsCompressionStats[kvp.Key] = new CompressionStats(kvp.Value.DiskSize, kvp.Value.UncompressedSize); + else + existingStats.Add(kvp.Value); + } } else - { _logger.Here().Warning($"{gameName} pack file '{path}' not found, loading skipped"); - } } ); + PackFileLog.LogPacksCompression(packsCompressionStats); + var caPackFileContainer = new PackFileContainer($"All Game Packs - {gameName}"); caPackFileContainer.IsCaPackFile = true; caPackFileContainer.SystemFilePath = gameDataFolder; @@ -147,6 +159,6 @@ void AddFolderContentToPackFile(PackFileContainer container, string folderPath, _logger.Here().Error($"Trying to get all CA packs in {gameDataFolder}. Error : {e.ToString()}"); return null; } - } // 2000 + } } } diff --git a/Shared/SharedCore/PackFiles/PackFileDecrypter.cs b/Shared/SharedCore/PackFiles/PackFileDecrypter.cs deleted file mode 100644 index 2925c45aa..000000000 --- a/Shared/SharedCore/PackFiles/PackFileDecrypter.cs +++ /dev/null @@ -1,78 +0,0 @@ -using System.Text; - -namespace Shared.Core.PackFiles -{ - public static class PackFileDecrypter - { - private static readonly byte[] s_iNDEX_STRING_KEY = Encoding.ASCII.GetBytes("#:AhppdV-!PEfz&}[]Nv?6w4guU%dF5.fq:n*-qGuhBJJBm&?2tPy!geW/+k#pG?"); - private const uint INDEX_U32_KEY = 0xE10B_73F4; - private const ulong DATA_KEY = 0x8FEB_2A67_40A6_920E; - - public static byte[] Decrypt(byte[] ciphertext) - { - // First, make sure the file ends in a multiple of 8. If not, extend it with zeros. - // We need it because the decoding is done in packs of 8 bytes. - var size = ciphertext.Length; - var padding = 8 - size % 8; - if (padding < 8) - Array.Resize(ref ciphertext, size + padding); - - // Then decrypt the file in packs of 8. It's faster than in packs of 4. - var plaintext = new byte[ciphertext.Length]; - ulong edi = 0; - var chunks = ciphertext.Length / 8; - - using (var memStream = new MemoryStream(ciphertext)) - using (var reader = new BinaryReader(memStream)) - using (var writer = new BinaryWriter(new MemoryStream(plaintext))) - { - for (var i = 0; i < chunks; i++) - { - if (i == chunks - 1) - writer.Write(reader.ReadBytes(8)); // The last chunk is not encrypted. - else - { - var esi = edi; - memStream.Seek((long)esi, SeekOrigin.Begin); - var prod = DATA_KEY * ~edi; - var data = reader.ReadUInt64(); - prod ^= data; - writer.Seek((int)esi, SeekOrigin.Begin); - writer.Write(prod); - } - edi += 8; - } - } - - // Remove the extra bytes we added in the first step. - Array.Resize(ref plaintext, size); - return plaintext; - } - - // This function decrypts the size of a PackedFile. - public static uint DecryptAndReadU32(BinaryReader reader, uint secondKey) - { - var ciphertext = reader.ReadUInt32(); - return ciphertext ^ INDEX_U32_KEY ^ ~secondKey; - } - - // This function decrypts the path of a PackedFile. - public static string DecryptAndReadString(Stream stream, uint secondKey) - { - StringBuilder path = new(); - var index = 0; - while (true) - { - var character = stream.ReadByte(); - if (character == -1) break; - - var decryptedChar = (byte)(character ^ s_iNDEX_STRING_KEY[index % s_iNDEX_STRING_KEY.Length] ^ ~secondKey); - if (decryptedChar == 0) break; - - path.Append((char)decryptedChar); - index++; - } - return path.ToString(); - } - } -} diff --git a/Shared/SharedCore/PackFiles/PackFileEncrypter.cs b/Shared/SharedCore/PackFiles/PackFileEncrypter.cs deleted file mode 100644 index 9fc1eb8cc..000000000 --- a/Shared/SharedCore/PackFiles/PackFileEncrypter.cs +++ /dev/null @@ -1,72 +0,0 @@ -using System.Text; - -namespace Shared.Core.PackFiles -{ - public static class PackFileEncrypter - { - private static readonly byte[] s_iNDEX_STRING_KEY = Encoding.ASCII.GetBytes("#:AhppdV-!PEfz&}[]Nv?6w4guU%dF5.fq:n*-qGuhBJJBm&?2tPy!geW/+k#pG?"); - private const uint INDEX_U32_KEY = 0xE10B_73F4; - private const ulong DATA_KEY = 0x8FEB_2A67_40A6_920E; - - public static byte[] Encrypt(byte[] plaintext) - { - // Ensure the plaintext is a multiple of 8 bytes by padding with zeros if necessary. - var size = plaintext.Length; - var padding = 8 - size % 8; - if (padding < 8) - Array.Resize(ref plaintext, size + padding); - - var ciphertext = new byte[plaintext.Length]; - ulong edi = 0; - var chunks = plaintext.Length / 8; - - using (var memStream = new MemoryStream(plaintext)) - using (var reader = new BinaryReader(memStream)) - using (var writer = new BinaryWriter(new MemoryStream(ciphertext))) - { - for (var i = 0; i < chunks; i++) - { - if (i == chunks - 1) - writer.Write(reader.ReadBytes(8)); // Do not encrypt the last chunk. - else - { - var esi = edi; - memStream.Seek((long)esi, SeekOrigin.Begin); - var data = reader.ReadUInt64(); - var encrypted = data ^ (DATA_KEY * ~edi); - writer.Seek((int)esi, SeekOrigin.Begin); - writer.Write(encrypted); - } - edi += 8; - } - } - - // Remove extra padding for accurate file representation. - Array.Resize(ref ciphertext, size); - return ciphertext; - } - - // This function encrypts a uint32 value (like the PackedFile size). - public static uint EncryptU32(uint plaintext, uint secondKey) - { - return plaintext ^ INDEX_U32_KEY ^ ~secondKey; - } - - // This function encrypts a file path into the pack file. - public static byte[] EncryptString(string path, uint secondKey) - { - var pathBytes = Encoding.ASCII.GetBytes(path); - var encrypted = new byte[pathBytes.Length + 1]; // +1 for null terminator - var index = 0; - - for (var i = 0; i < pathBytes.Length; i++) - { - encrypted[i] = (byte)(pathBytes[i] ^ s_iNDEX_STRING_KEY[index % s_iNDEX_STRING_KEY.Length] ^ ~secondKey); - index++; - } - - encrypted[pathBytes.Length] = 0; // Null terminator - return encrypted; - } - } -} diff --git a/Shared/SharedCore/PackFiles/PackFileEncryption.cs b/Shared/SharedCore/PackFiles/PackFileEncryption.cs new file mode 100644 index 000000000..9e394206a --- /dev/null +++ b/Shared/SharedCore/PackFiles/PackFileEncryption.cs @@ -0,0 +1,158 @@ +using System.Buffers.Binary; +using System.Text; + +namespace Shared.Core.PackFiles +{ + public static class PackFileEncryption + { + private static readonly byte[] s_iNDEX_STRING_KEY = Encoding.ASCII.GetBytes("#:AhppdV-!PEfz&}[]Nv?6w4guU%dF5.fq:n*-qGuhBJJBm&?2tPy!geW/+k#pG?"); + private const uint INDEX_U32_KEY = 0xE10B_73F4; + private const ulong DATA_KEY = 0x8FEB_2A67_40A6_920E; + + public static byte[] Decrypt(byte[] ciphertext) + { + // First, make sure the file ends in a multiple of 8. If not, extend it with zeros. + // We need it because the decoding is done in packs of 8 bytes. + var size = ciphertext.Length; + var padding = 8 - size % 8; + if (padding < 8) + Array.Resize(ref ciphertext, size + padding); + + // Then decrypt the file in packs of 8. It's faster than in packs of 4. + var plaintext = new byte[ciphertext.Length]; + ulong edi = 0; + var chunks = ciphertext.Length / 8; + + using (var memStream = new MemoryStream(ciphertext)) + using (var reader = new BinaryReader(memStream)) + using (var writer = new BinaryWriter(new MemoryStream(plaintext))) + { + for (var i = 0; i < chunks; i++) + { + if (i == chunks - 1) + writer.Write(reader.ReadBytes(8)); // The last chunk is not encrypted. + else + { + var esi = edi; + memStream.Seek((long)esi, SeekOrigin.Begin); + var prod = DATA_KEY * ~edi; + var data = reader.ReadUInt64(); + prod ^= data; + writer.Seek((int)esi, SeekOrigin.Begin); + writer.Write(prod); + } + edi += 8; + } + } + + // Remove the extra bytes we added in the first step. + Array.Resize(ref plaintext, size); + return plaintext; + } + + public static void DecryptInPlace(Span buffer, long entrySize, long entryRelativeOffset = 0) + { + // We need it because the decoding is done in packs of 8 bytes. + if (entrySize <= 8) + return; + + for (var off = 0; off + 8 <= buffer.Length; off += 8) + { + var edi = entryRelativeOffset + off; + if (edi + 8 > entrySize - 8) + break; + + var cipher = BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(off, 8)); + var plain = DATA_KEY * ~((ulong)edi) ^ cipher; + BinaryPrimitives.WriteUInt64LittleEndian(buffer.Slice(off, 8), plain); + } + } + + // This function decrypts the size of a PackedFile. + public static uint DecryptAndReadU32(BinaryReader reader, uint secondKey) + { + var ciphertext = reader.ReadUInt32(); + return ciphertext ^ INDEX_U32_KEY ^ ~secondKey; + } + + // This function decrypts the path of a PackedFile. + public static string DecryptAndReadString(Stream stream, uint secondKey) + { + StringBuilder path = new(); + var index = 0; + while (true) + { + var character = stream.ReadByte(); + if (character == -1) break; + + var decryptedChar = (byte)(character ^ s_iNDEX_STRING_KEY[index % s_iNDEX_STRING_KEY.Length] ^ ~secondKey); + if (decryptedChar == 0) break; + + path.Append((char)decryptedChar); + index++; + } + return path.ToString(); + } + + public static byte[] Encrypt(byte[] plaintext) + { + // Ensure the plaintext is a multiple of 8 bytes by padding with zeros if necessary. + var size = plaintext.Length; + var padding = 8 - size % 8; + if (padding < 8) + Array.Resize(ref plaintext, size + padding); + + var ciphertext = new byte[plaintext.Length]; + ulong edi = 0; + var chunks = plaintext.Length / 8; + + using (var memStream = new MemoryStream(plaintext)) + using (var reader = new BinaryReader(memStream)) + using (var writer = new BinaryWriter(new MemoryStream(ciphertext))) + { + for (var i = 0; i < chunks; i++) + { + if (i == chunks - 1) + writer.Write(reader.ReadBytes(8)); // Do not encrypt the last chunk. + else + { + var esi = edi; + memStream.Seek((long)esi, SeekOrigin.Begin); + var data = reader.ReadUInt64(); + var encrypted = data ^ (DATA_KEY * ~edi); + writer.Seek((int)esi, SeekOrigin.Begin); + writer.Write(encrypted); + } + edi += 8; + } + } + + // Remove extra padding for accurate file representation. + Array.Resize(ref ciphertext, size); + return ciphertext; + } + + // This function encrypts a uint32 value (like the PackedFile size). + public static uint EncryptU32(uint plaintext, uint secondKey) + { + return plaintext ^ INDEX_U32_KEY ^ ~secondKey; + } + + // This function encrypts a file path into the pack file. + public static byte[] EncryptString(string path, uint secondKey) + { + var pathBytes = Encoding.ASCII.GetBytes(path); + var encrypted = new byte[pathBytes.Length + 1]; // +1 for null terminator + var index = 0; + + for (var i = 0; i < pathBytes.Length; i++) + { + encrypted[i] = (byte)(pathBytes[i] ^ s_iNDEX_STRING_KEY[index % s_iNDEX_STRING_KEY.Length] ^ ~secondKey); + index++; + } + + encrypted[pathBytes.Length] = 0; // Null terminator + return encrypted; + } + } +} diff --git a/Shared/SharedCore/PackFiles/PackFileSerializer.cs b/Shared/SharedCore/PackFiles/PackFileSerializer.cs index 3888d4783..f0c45aa98 100644 --- a/Shared/SharedCore/PackFiles/PackFileSerializer.cs +++ b/Shared/SharedCore/PackFiles/PackFileSerializer.cs @@ -2,7 +2,6 @@ using Shared.Core.ErrorHandling; using Shared.Core.PackFiles.Models; using Shared.Core.Settings; -using static Shared.Core.PackFiles.PackFileDecrypter; namespace Shared.Core.PackFiles { @@ -26,7 +25,7 @@ public static class PackFileSerializer { static readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileSerializer)); - public static PackFileContainer Load(string packFileSystemPath, BinaryReader reader, IDuplicatePackFileResolver dubplicatePackFileResolver) + public static PackFileContainer Load(string packFileSystemPath, BinaryReader reader, IDuplicatePackFileResolver duplicatePackFileResolver) { try { @@ -51,7 +50,6 @@ public static PackFileContainer Load(string packFileSystemPath, BinaryReader rea FilePath = packFileSystemPath, }; - //var buffer = reader.ReadBytes((int)output.Header.DataStart - 28); var offset = output.Header.DataStart; @@ -60,43 +58,52 @@ public static PackFileContainer Load(string packFileSystemPath, BinaryReader rea { uint size; if (output.Header.HasEncryptedIndex) - size = DecryptAndReadU32(reader, (uint)(output.Header.FileCount - i - 1)); + size = PackFileEncryption.DecryptAndReadU32(reader, (uint)(output.Header.FileCount - i - 1)); else size = reader.ReadUInt32(); if (output.Header.HasIndexWithTimeStamp) reader.ReadUInt32(); - byte isCompressed = 0; + var isCompressed = false; if (headerVersion == PackFileVersion.PFH5) - isCompressed = reader.ReadByte(); // Is the file actually compressed, or is it just a compressed format? + isCompressed = reader.ReadBoolean(); var fullPackedFileName = IOFunctions.ReadZeroTerminatedAscii(reader, fileNameBuffer).ToLower(); - var packFileName = Path.GetFileName(fullPackedFileName); var isEncrypted = output.Header.HasEncryptedData; - var fileContent = new PackFile(packFileName, new PackedFileSource(packedFileSourceParent, offset, size, isEncrypted)); - if (dubplicatePackFileResolver.CheckForDuplicates) + var compressionFormat = CompressionFormat.None; + uint uncompressedSize = 0; + if (isCompressed) + { + var fileHeader = DetectCompressionInfo(reader, offset, size, isEncrypted); + using var compressionStream = new MemoryStream(fileHeader, false); + using var compressionReader = new BinaryReader(compressionStream); + uncompressedSize = compressionReader.ReadUInt32(); + var magicNumber = compressionReader.ReadUInt32(); + compressionFormat = PackFileCompression.GetCompressionFormat(magicNumber); + } + + var packedFileSource = new PackedFileSource(packedFileSourceParent, offset, size, isEncrypted, isCompressed, compressionFormat, uncompressedSize); + var fileContent = new PackFile(packFileName, packedFileSource); + + if (duplicatePackFileResolver.CheckForDuplicates) { var containsKey = output.FileList.ContainsKey(fullPackedFileName); if (containsKey) { - if (dubplicatePackFileResolver.KeepDuplicateFile(fullPackedFileName)) + if (duplicatePackFileResolver.KeepDuplicateFile(fullPackedFileName)) { _logger.Here().Warning($"Duplicate file found {fullPackedFileName}"); output.FileList.Add(fullPackedFileName + Guid.NewGuid().ToString(), fileContent); } } else - { output.FileList[fullPackedFileName] = fileContent; - } } else - { output.FileList[fullPackedFileName] = fileContent; - } offset += size; } @@ -127,7 +134,7 @@ static PFHeader ReadHeader(BinaryReader reader) if (header.HasEncryptedIndex) { var filesRemaining = header.ReferenceFileCount; - packed_file_index_size = DecryptAndReadU32(reader, filesRemaining); + packed_file_index_size = PackFileEncryption.DecryptAndReadU32(reader, filesRemaining); } // Read the buffer of data stuff @@ -217,5 +224,25 @@ public static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWrit writer.Write((byte)0); } } + + private static byte[] DetectCompressionInfo(BinaryReader reader, long dataOffset, uint entrySize, bool isEncrypted) + { + if (entrySize <= 8 || !isEncrypted && entrySize == 0) + return []; + + var headerLen = 8; + var header = new byte[headerLen]; + + var savedPos = reader.BaseStream.Position; + + reader.BaseStream.Seek(dataOffset, SeekOrigin.Begin); + reader.Read(header, 0, headerLen); + reader.BaseStream.Seek(savedPos, SeekOrigin.Begin); + + if (isEncrypted) + PackFileEncryption.DecryptInPlace(header, entrySize); + + return header; + } } } diff --git a/Shared/SharedCore/PackFiles/PackFileService.cs b/Shared/SharedCore/PackFiles/PackFileService.cs index 667c5c909..880a5135f 100644 --- a/Shared/SharedCore/PackFiles/PackFileService.cs +++ b/Shared/SharedCore/PackFiles/PackFileService.cs @@ -4,12 +4,10 @@ using Shared.Core.Events.Global; using Shared.Core.Misc; using Shared.Core.PackFiles.Models; +using Shared.Core.Settings; namespace Shared.Core.PackFiles { - - - public class PackFileService : IPackFileService { private readonly ILogger _logger = Logging.Create(); @@ -254,7 +252,7 @@ public void SaveFile(PackFile file, byte[] data) _globalEventHub?.PublishGlobalEvent(new PackFileSavedEvent(file)); } - public void SavePackContainer(PackFileContainer pf, string path, bool createBackup) + public void SavePackContainer(PackFileContainer pf, string path, bool createBackup, GameInformation gameInformation) { if (File.Exists(path) && DirectoryHelper.IsFileLocked(path)) { @@ -279,7 +277,7 @@ public void SavePackContainer(PackFileContainer pf, string path, bool createBack using (var memoryStream = new FileStream(path + "_temp", FileMode.OpenOrCreate)) { using var writer = new BinaryWriter(memoryStream); - pf.SaveToByteArray(writer); + pf.SaveToByteArray(writer, gameInformation); } File.Delete(path); @@ -366,5 +364,4 @@ public class SimpleMessageBox : ISimpleMessageBox { public void ShowDialogBox(string message, string title) => MessageBox.Show(message, title); } - } diff --git a/Shared/SharedCore/Services/TouchedFilesRecorder.cs b/Shared/SharedCore/Services/TouchedFilesRecorder.cs index 911d478d4..9a2e5dfda 100644 --- a/Shared/SharedCore/Services/TouchedFilesRecorder.cs +++ b/Shared/SharedCore/Services/TouchedFilesRecorder.cs @@ -1,9 +1,9 @@ -using Serilog; -using Shared.Core.ErrorHandling; +using Shared.Core.ErrorHandling; using Shared.Core.Events; using Shared.Core.Events.Global; using Shared.Core.PackFiles; using Shared.Core.PackFiles.Models; +using Shared.Core.Settings; namespace Shared.Core.Services { @@ -13,12 +13,14 @@ public class TouchedFilesRecorder readonly List<(string FilePath, PackFileContainer Container)> _files = new(); readonly IPackFileService _pfs; private readonly IGlobalEventHub _eventHub; + private readonly ApplicationSettingsService _applicationSettingsService; bool _isStarted = false; - public TouchedFilesRecorder(IPackFileService pfs, IGlobalEventHub eventHub) + public TouchedFilesRecorder(IPackFileService pfs, IGlobalEventHub eventHub, ApplicationSettingsService applicationSettingsService) { _pfs = pfs; _eventHub = eventHub; + _applicationSettingsService = applicationSettingsService; } public void Start() @@ -49,7 +51,8 @@ public void ExtractFilesToPack(string path) foreach (var item in _files) _pfs.CopyFileFromOtherPackFile(item.Container, item.FilePath, newPack); - _pfs.SavePackContainer(newPack, path, false); + var gameInformation = GameInformationDatabase.GetGameById(_applicationSettingsService.CurrentSettings.CurrentGame); + _pfs.SavePackContainer(newPack, path, false, gameInformation); } public void Stop() diff --git a/Shared/SharedCore/Settings/GameInformationBuilder.cs b/Shared/SharedCore/Settings/GameInformationBuilder.cs index 3f09e8485..bdf0c0cb1 100644 --- a/Shared/SharedCore/Settings/GameInformationBuilder.cs +++ b/Shared/SharedCore/Settings/GameInformationBuilder.cs @@ -41,7 +41,7 @@ public static GameInformationBuilder Create(GameTypeEnum type, string displayNam { return new GameInformationBuilder() { - _instance = new GameInformation(type, displayName, Settings.PackFileVersion.PFH5, Settings.GameBnkVersion.Unsupported, Settings.WsModelVersion.Unknown) + _instance = new GameInformation(type, displayName, Settings.PackFileVersion.PFH5, Settings.GameBnkVersion.Unsupported, Settings.WsModelVersion.Unknown, []) }; } diff --git a/Shared/SharedCore/Settings/GameInformationDatabase.cs b/Shared/SharedCore/Settings/GameInformationDatabase.cs index 5141c0b57..5640160ea 100644 --- a/Shared/SharedCore/Settings/GameInformationDatabase.cs +++ b/Shared/SharedCore/Settings/GameInformationDatabase.cs @@ -1,4 +1,6 @@ -namespace Shared.Core.Settings +using Shared.Core.PackFiles; + +namespace Shared.Core.Settings { public enum GameTypeEnum { @@ -46,13 +48,14 @@ public enum WsModelVersion //RmvVersionEnum - public class GameInformation(GameTypeEnum gameType, string displayName, PackFileVersion packFileVersion, GameBnkVersion bankGeneratorVersion, WsModelVersion wsModelVersion) + public class GameInformation(GameTypeEnum gameType, string displayName, PackFileVersion packFileVersion, GameBnkVersion bankGeneratorVersion, WsModelVersion wsModelVersion, List CompressionFormats) { public GameTypeEnum Type { get; } = gameType; public string DisplayName { get; } = displayName; public PackFileVersion PackFileVersion { get; } = packFileVersion; public GameBnkVersion BankGeneratorVersion { get; } = bankGeneratorVersion; public WsModelVersion WsModelVersion { get; } = wsModelVersion; + public List CompressionFormats { get; } = CompressionFormats; } public static class GameInformationDatabase @@ -61,14 +64,14 @@ public static class GameInformationDatabase static GameInformationDatabase() { - var warhammer = new GameInformation(GameTypeEnum.Warhammer, "Warhammer", PackFileVersion.PFH4, GameBnkVersion.Unsupported, WsModelVersion.Unknown ); - var warhammer2 = new GameInformation(GameTypeEnum.Warhammer2, "Warhammer II", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Version1 ); - var warhammer3 = new GameInformation(GameTypeEnum.Warhammer3, "Warhammer III", PackFileVersion.PFH5, GameBnkVersion.Warhammer3, WsModelVersion.Version3); - var troy = new GameInformation(GameTypeEnum.Troy, "Troy", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Unknown); - var threeKingdoms = new GameInformation( GameTypeEnum.ThreeKingdoms, "Three Kingdoms", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Version1 ); - var rome2 = new GameInformation(GameTypeEnum.Rome2, "Rome II", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Unknown); - var attila = new GameInformation(GameTypeEnum.Attila, "Attila", PackFileVersion.PFH5, GameBnkVersion.Attila, WsModelVersion.Unknown); - var pharaoh = new GameInformation(GameTypeEnum.Pharaoh, "Pharaoh", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Unknown); + var warhammer = new GameInformation(GameTypeEnum.Warhammer, "Warhammer", PackFileVersion.PFH4, GameBnkVersion.Unsupported, WsModelVersion.Unknown, [CompressionFormat.None]); + var warhammer2 = new GameInformation(GameTypeEnum.Warhammer2, "Warhammer II", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Version1, [CompressionFormat.Lzma1]); + var warhammer3 = new GameInformation(GameTypeEnum.Warhammer3, "Warhammer III", PackFileVersion.PFH5, GameBnkVersion.Warhammer3, WsModelVersion.Version3, [CompressionFormat.Lzma1, CompressionFormat.Lz4, CompressionFormat.Zstd]); + var troy = new GameInformation(GameTypeEnum.Troy, "Troy", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Unknown, [CompressionFormat.Lzma1]); + var threeKingdoms = new GameInformation( GameTypeEnum.ThreeKingdoms, "Three Kingdoms", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Version1, [CompressionFormat.Lzma1]); + var rome2 = new GameInformation(GameTypeEnum.Rome2, "Rome II", PackFileVersion.PFH4, GameBnkVersion.Unsupported, WsModelVersion.Unknown, [CompressionFormat.None]); + var attila = new GameInformation(GameTypeEnum.Attila, "Attila", PackFileVersion.PFH4, GameBnkVersion.Attila, WsModelVersion.Unknown, [CompressionFormat.None]); + var pharaoh = new GameInformation(GameTypeEnum.Pharaoh, "Pharaoh", PackFileVersion.PFH5, GameBnkVersion.Unsupported, WsModelVersion.Unknown, [CompressionFormat.Lzma1]); Games = [warhammer, warhammer2, warhammer3, troy, threeKingdoms, rome2, attila, pharaoh]; } diff --git a/Shared/SharedCore/Shared.Core.csproj b/Shared/SharedCore/Shared.Core.csproj index 912eb5438..be887969d 100644 --- a/Shared/SharedCore/Shared.Core.csproj +++ b/Shared/SharedCore/Shared.Core.csproj @@ -1,18 +1,21 @@  - - net9.0-windows - enable - enable - + + net9.0-windows + enable + enable + + + + - + @@ -28,7 +31,7 @@ - + @@ -39,5 +42,5 @@ - + diff --git a/Shared/SharedUI/BaseDialogs/PackFileTree/ContextMenu/Commands/SaveAsPackFileContainerCommand.cs b/Shared/SharedUI/BaseDialogs/PackFileTree/ContextMenu/Commands/SaveAsPackFileContainerCommand.cs index 5259c31fb..c52244665 100644 --- a/Shared/SharedUI/BaseDialogs/PackFileTree/ContextMenu/Commands/SaveAsPackFileContainerCommand.cs +++ b/Shared/SharedUI/BaseDialogs/PackFileTree/ContextMenu/Commands/SaveAsPackFileContainerCommand.cs @@ -1,12 +1,12 @@ using System.Windows.Forms; using Shared.Core.PackFiles; +using Shared.Core.Settings; using Shared.Ui.Common; namespace Shared.Ui.BaseDialogs.PackFileTree.ContextMenu.Commands { - public class SaveAsPackFileContainerCommand(IPackFileService packFileService) : IContextMenuCommand + public class SaveAsPackFileContainerCommand(IPackFileService packFileService, ApplicationSettingsService applicationSettingsService) : IContextMenuCommand { - //private readonly ILogger _logger = Logging.Create(); public string GetDisplayName(TreeNode node) => "Save As"; public bool IsEnabled(TreeNode node) => true; @@ -21,7 +21,8 @@ public void Execute(TreeNode _selectedNode) using (new WaitCursor()) { - packFileService.SavePackContainer(_selectedNode.FileOwner, saveFileDialog.FileName, false); + var gameInformation = GameInformationDatabase.GetGameById(applicationSettingsService.CurrentSettings.CurrentGame); + packFileService.SavePackContainer(_selectedNode.FileOwner, saveFileDialog.FileName, false, gameInformation); _selectedNode.UnsavedChanged = false; _selectedNode.ForeachNode((node) => node.UnsavedChanged = false); } diff --git a/Shared/SharedUI/BaseDialogs/PackFileTree/ContextMenu/Commands/SavePackFileContainerCommand.cs b/Shared/SharedUI/BaseDialogs/PackFileTree/ContextMenu/Commands/SavePackFileContainerCommand.cs index 2d3ab4baf..fb5af6391 100644 --- a/Shared/SharedUI/BaseDialogs/PackFileTree/ContextMenu/Commands/SavePackFileContainerCommand.cs +++ b/Shared/SharedUI/BaseDialogs/PackFileTree/ContextMenu/Commands/SavePackFileContainerCommand.cs @@ -4,11 +4,15 @@ using Shared.Core.ErrorHandling; using Shared.Core.PackFiles; using Shared.Core.Services; +using Shared.Core.Settings; using Shared.Ui.Common; namespace Shared.Ui.BaseDialogs.PackFileTree.ContextMenu.Commands { - public class SavePackFileContainerCommand(IPackFileService packFileService, IStandardDialogs standardDialogs) : IContextMenuCommand + public class SavePackFileContainerCommand( + IPackFileService packFileService, + IStandardDialogs standardDialogs, + ApplicationSettingsService applicationSettingsService) : IContextMenuCommand { private readonly ILogger _logger = Logging.Create(); public string GetDisplayName(TreeNode node) => "Save"; @@ -32,7 +36,8 @@ public void Execute(TreeNode _selectedNode) { try { - packFileService.SavePackContainer(_selectedNode.FileOwner, systemPath, false); + var gameInformation = GameInformationDatabase.GetGameById(applicationSettingsService.CurrentSettings.CurrentGame); + packFileService.SavePackContainer(_selectedNode.FileOwner, systemPath, false, gameInformation); } catch (Exception e) { @@ -67,7 +72,8 @@ public void Execute() { try { - packFileService.SavePackContainer(pack, systemPath, false); + var gameInformation = GameInformationDatabase.GetGameById(applicationSettingsService.CurrentSettings.CurrentGame); + packFileService.SavePackContainer(pack, systemPath, false, gameInformation); } catch (Exception e) { diff --git a/Testing/Shared.Core.Test/PackFiles/PackFileCompressionTests.cs b/Testing/Shared.Core.Test/PackFiles/PackFileCompressionTests.cs new file mode 100644 index 000000000..dc941a7aa --- /dev/null +++ b/Testing/Shared.Core.Test/PackFiles/PackFileCompressionTests.cs @@ -0,0 +1,69 @@ +using System.Text; +using Moq; +using Shared.Core.Events; +using Shared.Core.PackFiles; +using Shared.Core.PackFiles.Models; + +namespace Test.Shared.Core.PackFiles +{ + internal class PackFileCompressionTests + { + private IPackFileService _packFileService; + private PackFileContainer _container; + + [SetUp] + public void Setup() + { + var eventHub = new Mock(); + _packFileService = new PackFileService(eventHub.Object); + _container = _packFileService.CreateNewPackFileContainer("EncryptedOutput", PackFileCAType.MOD, true); + + // Use files that aren't tiny so that they can actually be compressed rather than increase in size when being compressed + List files = [ + new("Directory_0", PackFile.CreateFromASCII("file0.txt", new string('A', 1_024))), + new("Directory_0", PackFile.CreateFromASCII("file1.txt", new string('B', 2_048))), + new("Directory_0\\subfolder", PackFile.CreateFromASCII("subfile0.txt", new string('C', 4_096))), + new("Directory_0\\subfolder", PackFile.CreateFromASCII("subfile1.txt", new string('D', 8_192))) + ]; + + _packFileService.AddFilesToPack(_container, files); + } + + [Test] + public void TestCompressAndDecompressPackFile() + { + var compressionFormats = Enum.GetValues(typeof(CompressionFormat)).Cast(); + var originals = _container.FileList + .ToDictionary(file => file.Value.Name, + file => file.Value.DataSource.ReadData()); + + foreach (var fileName in originals.Keys) + { + var data = originals[fileName]; + + foreach (var compressionFormat in compressionFormats) + { + var compressedData = PackFileCompression.Compress(data, compressionFormat); + + if (compressionFormat != CompressionFormat.None) + { + Assert.That(compressedData, Has.Length.LessThan(data.Length), + $"[{compressionFormat}] {fileName} did not reduce in size: {data.Length} --> {compressedData.Length}"); + } + + var decompressed = PackFileCompression.Decompress(compressedData); + Assert.That(decompressed, Has.Length.EqualTo(data.Length), + $"[{compressionFormat}] {fileName} length mismatch"); + + var expected = Encoding.UTF8.GetString(originals[fileName]); + var actual = Encoding.UTF8.GetString(decompressed); + Assert.That(actual, Is.EqualTo(expected), + $"[{compressionFormat}] {fileName} content mismatch after round-trip"); + + // Feed back in for the next iteration + data = decompressed; + } + } + } + } +} diff --git a/Testing/Shared.Core.Test/PackFiles/PackFileEncryptionTests.cs b/Testing/Shared.Core.Test/PackFiles/PackFileEncryptionTests.cs index 8efb6bd7d..6093b5155 100644 --- a/Testing/Shared.Core.Test/PackFiles/PackFileEncryptionTests.cs +++ b/Testing/Shared.Core.Test/PackFiles/PackFileEncryptionTests.cs @@ -36,7 +36,7 @@ public void TestEncryptAndDecryptPackFile() foreach (var file in _container.FileList) { var originalData = file.Value.DataSource.ReadData(); - var encryptedData = PackFileEncrypter.Encrypt(originalData); + var encryptedData = PackFileEncryption.Encrypt(originalData); file.Value.DataSource = new MemorySource(encryptedData); } @@ -47,7 +47,7 @@ public void TestEncryptAndDecryptPackFile() foreach (var file in _container.FileList) { var encryptedData = file.Value.DataSource.ReadData(); - var decryptedContent = PackFileDecrypter.Decrypt(encryptedData); + var decryptedContent = PackFileEncryption.Decrypt(encryptedData); var originalContent = Encoding.UTF8.GetString(decryptedContent); switch (file.Value.Name)