Compression Support

.gitignore

@@ -340,3 +340,5 @@ ASALocalRun/
 # BeatPulse healthcheck temp database
 healthchecksdb
 
+# MonoGame build outputs
+/GameWorld/ContentProject/Content/bin/

Editors/Reports/DeepSearch/DeepSearchReport.cs

@@ -77,7 +77,7 @@ public List<string> DeepSearch(string searchStr, bool caseSensetive)
                               {
                                   var pf = packFile;
                                   var ds = pf.DataSource as PackedFileSource;
-                                  var bytes = ds.ReadDataForFastSearch(fileStram);
+                                  var bytes = ds.ReadData(fileStram);
                                   var str = Encoding.ASCII.GetString(bytes);
 
                                   if (str.Contains(searchStr, StringComparison.InvariantCultureIgnoreCase))

GameWorld/ContentProject/ContentProject.csproj

@@ -8,8 +8,11 @@
     </PropertyGroup>
 
     <ItemGroup>
+      <Compile Remove="Content\bin\**" />
       <Compile Remove="Content\obj\**" />
+      <EmbeddedResource Remove="Content\bin\**" />
       <EmbeddedResource Remove="Content\obj\**" />
+      <None Remove="Content\bin\**" />
       <None Remove="Content\obj\**" />
     </ItemGroup>
 

GameWorld/View3D/Services/ComplexMeshLoader.cs

@@ -45,7 +45,7 @@ SceneNode Load(PackFile file, SceneNode parent, AnimationPlayer player, string a
 
                 _logger.Here().Information($"Attempting to load file {file.Name}");
 
-                switch (file.Extention)
+                switch (file.Extension)
                 {
                     case ".variantmeshdefinition":
                         LoadVariantMesh(file, ref parent, player, attachmentPointName, onlyLoadRootNode, onlyLoadFirstMesh);

GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs

@@ -89,21 +89,22 @@ void LoadFromPackFileContainer(PackFileContainer packFileContainer)
             // Handle packfile which are stored in a saved file.
             // This is done for performance reasons. Opening all the animations files from disk is very slow
             // creating stream which is reused goes a lot faster!
+            // https://www.jacksondunstan.com/articles/3568
             var groupedAnims = allAnimsInSavedPackedFiles
                 .GroupBy(x => x.DataSource.Parent.FilePath)
                 .ToList();
 
             Parallel.For(0, groupedAnims.Count, index =>
             {
-                using var handle = File.OpenHandle(groupedAnims[index].Key);
-
-                //https://www.jacksondunstan.com/articles/3568
-                var buffer = new byte[100];
-
+                using var stream = new FileStream(groupedAnims[index].Key, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+
                 foreach (var file in groupedAnims[index])
                 {
-                    RandomAccess.Read(handle, buffer, file.DataSource.Offset);
-                    FileDiscovered(buffer, packFileContainer, file.FullPath, ref skeletonFileNameList, ref animationList);
+                    var bytes = file.DataSource.ReadData(stream);
+                    if (bytes.Length > 100)
+                        Array.Resize(ref bytes, 100);
+
+                    FileDiscovered(bytes, packFileContainer, file.FullPath, ref skeletonFileNameList, ref animationList);
                 }
             });
 
@@ -134,7 +135,11 @@ void FileDiscovered(byte[] byteChunk, PackFileContainer container, string fullPa
                 var brokenAnims = new string[]
                 {
                      "rigidmodels\\buildings\\roman_aqueduct_straight\\roman_aqueduct_straight_piece01_destruct01_anim.anim",
-                     "animations\\battle\\raptor02\\subset\\colossal_squig\\deaths\\rp2_colossalsquig_death_01.anim"
+                     "animations\\battle\\raptor02\\subset\\colossal_squig\\deaths\\rp2_colossalsquig_death_01.anim",
+                     "animations\\battle\\humanoid13b\\golgfag\\docking\\hu13b_golgfag_docking_armed_02.anim",
+                     "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_shoot_back_crossbow_01.anim",
+                     "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_reload_crossbow_01.anim",
+                     "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_sp_rider1_shoot_ready_crossbow_01.anim"
                 };
                 if (brokenAnims.Contains(fullPath))
                 {
@@ -145,6 +150,10 @@ void FileDiscovered(byte[] byteChunk, PackFileContainer container, string fullPa
 
             try
             {
+                if (byteChunk.Length == 0)
+                {
+                    throw new Exception("File empty.");
+                }
                 var animationSkeletonName = AnimationFile.GetAnimationName(byteChunk);
 
                 lock (_threadLock)

Shared/SharedCore/ByteParsing/ByteHelper.cs

@@ -85,6 +85,5 @@ public static uint GetPropertyTypeSize<T>(T property)
 
             return (uint)Marshal.SizeOf(property);
         }
-
     }
 }

Shared/SharedCore/ErrorHandling/PackFileLog.cs

@@ -0,0 +1,140 @@
+using Shared.Core.PackFiles;
+using Shared.Core.PackFiles.Models;
+
+namespace Shared.Core.ErrorHandling
+{
+    public class CompressionStats
+    {
+        public long DiskSize { get; set; }
+        public long UncompressedSize { get; set; }
+
+        public CompressionStats(long diskSize = 0, long uncompressedSize = 0)
+        {
+            DiskSize = diskSize;
+            UncompressedSize = uncompressedSize;
+        }
+
+        public void Add(CompressionStats stat)
+        {
+            DiskSize += stat.DiskSize;
+            UncompressedSize += stat.UncompressedSize;
+        }
+    }
+
+    public static class PackFileLog
+    {
+        private static readonly ILogger s_logger = Logging.CreateStatic(typeof(PackFileLog));
+
+        public static Dictionary<CompressionFormat, CompressionStats> GetCompressionStats(PackFileContainer container)
+        {
+            var stats = new Dictionary<CompressionFormat, CompressionStats>();
+
+            foreach (var packFile in container.FileList.Values)
+            {
+                if (packFile.DataSource is PackedFileSource source)
+                {
+                    var format = source.IsCompressed
+                        ? source.CompressionFormat
+                        : CompressionFormat.None;
+
+                    if (!stats.TryGetValue(format, out var totals))
+                    {
+                        totals = new CompressionStats();
+                        stats[format] = totals;
+                    }
+
+                    totals.DiskSize += source.Size;
+                    totals.UncompressedSize += source.IsCompressed ? source.UncompressedSize : 0L;
+                }
+            }
+
+            return stats;
+        }
+
+        public static void LogPackCompression(PackFileContainer container)
+        {
+            var stats = GetCompressionStats(container);
+            var totalFiles = container.FileList.Count;
+            var packSizeFmt = FormatSize(container.OriginalLoadByteSize);
+
+            var loadingPart = $"Loading {container.Name}.pack ({totalFiles} files, {packSizeFmt})";
+
+            var fileCounts = new Dictionary<CompressionFormat, int>();
+            foreach (var pf in container.FileList.Values)
+            {
+                if (pf.DataSource is PackedFileSource src)
+                {
+                    var fmt = src.IsCompressed
+                        ? src.CompressionFormat
+                        : CompressionFormat.None;
+
+                    if (!fileCounts.TryGetValue(fmt, out var cnt))
+                        fileCounts[fmt] = 1;
+                    else
+                        fileCounts[fmt] = cnt + 1;
+                }
+            }
+
+            var segments = stats
+                .OrderBy(kvp => kvp.Key)
+                .Select(kvp =>
+                {
+                    var fmt = kvp.Key;
+                    var count = fileCounts.TryGetValue(fmt, out var c) ? c : 0;
+                    var disk = FormatSize(kvp.Value.DiskSize);
+
+                    if (fmt == CompressionFormat.None)
+                        return $"{fmt}: {count} files, {disk} (Disk Size)";
+
+                    var unc = FormatSize(kvp.Value.UncompressedSize);
+                    return $"{fmt}: {count} files, {disk} (Disk Size), {unc} (Uncompressed Size)";
+                })
+                .ToList();
+
+            var compressionPart = $"File Compression – {string.Join(" | ", segments)}";
+            s_logger.Here().Information($"{loadingPart} | {compressionPart}");
+        }
+
+        public static void LogPacksCompression(IDictionary<CompressionFormat, CompressionStats> globalStats)
+        {
+            var segments = globalStats
+                .OrderBy(kvp => kvp.Key)
+                .Select(kvp =>
+                {
+                    var format = kvp.Key;
+                    var diskFormatted = FormatSize(kvp.Value.DiskSize);
+
+                    if (format == CompressionFormat.None)
+                        return $"{format}: {diskFormatted} (Disk Size)";
+
+                    var uncompressedFormatted = FormatSize(kvp.Value.UncompressedSize);
+                    return $"{format}: {diskFormatted} (Disk Size), {uncompressedFormatted} (Uncompressed Size)";
+                })
+                .ToList();
+
+            var totalDisk = globalStats.Values.Sum(stat => stat.DiskSize);
+            var totalUncompressed = globalStats.Values.Sum(stat => stat.UncompressedSize);
+
+            var totalDiskFormatted = FormatSize(totalDisk);
+            var totalUncompressedFormatted = FormatSize(totalUncompressed);
+
+            var totalSegment = $"Total: {totalDiskFormatted} (Disk Size), {totalUncompressedFormatted} (Uncompressed Size)";
+            var summary = string.Join(" | ", segments.Append(totalSegment));
+
+            s_logger.Here().Information($"Size of compressed files in all packs by format - {summary}");
+        }
+
+        private static string FormatSize(long bytes)
+        {
+            var kb = 1024.0;
+            var mb = kb * 1024.0;
+            var gb = mb * 1024.0;
+
+            if (bytes >= gb)
+                return $"{bytes / gb:F2} GB";
+            if (bytes >= mb)
+                return $"{bytes / mb:F2} MB";
+            return $"{bytes / kb:F2} KB";
+        }
+    }
+}

Shared/SharedCore/PackFiles/IPackFileService.cs

@@ -1,4 +1,5 @@
 using Shared.Core.PackFiles.Models;
+using Shared.Core.Settings;
 
 namespace Shared.Core.PackFiles
 {
@@ -22,9 +23,8 @@ public interface IPackFileService
         void RenameDirectory(PackFileContainer pf, string currentNodeName, string newName);
         void RenameFile(PackFileContainer pf, PackFile file, string newName);
         void SaveFile(PackFile file, byte[] data);
-        void SavePackContainer(PackFileContainer pf, string path, bool createBackup);
+        void SavePackContainer(PackFileContainer pf, string path, bool createBackup, GameInformation gameInformation);
         void SetEditablePack(PackFileContainer? pf);
         void UnloadPackContainer(PackFileContainer pf);
     }
-
 }

Shared/SharedCore/PackFiles/Models/DataSource.cs

@@ -1,5 +1,5 @@
 using Shared.Core.ByteParsing;
-using static Shared.Core.PackFiles.PackFileDecrypter;
+using Shared.Core.Settings;
 
 namespace Shared.Core.PackFiles.Models
 {
@@ -89,16 +89,29 @@ public record PackedFileSource : IDataSource
         public long Offset { get; private set; }
         public long Size { get; private set; }
         public bool IsEncrypted { get; private set; }
+        public bool IsCompressed { get; set; }
+        public CompressionFormat CompressionFormat { get; set; }
+        public uint UncompressedSize { get; set; }
         public PackedFileSourceParent Parent { get => _parent; }
 
         private readonly PackedFileSourceParent _parent;
 
-        public PackedFileSource(PackedFileSourceParent parent, long offset, long length, bool isEncrypted)
+        public PackedFileSource(
+            PackedFileSourceParent parent,
+            long offset,
+            long length,
+            bool isEncrypted,
+            bool isCompressed,
+            CompressionFormat compressionFormat,
+            uint uncompressedSize)
         {
             Offset = offset;
             _parent = parent;
             Size = length;
             IsEncrypted = isEncrypted;
+            IsCompressed = isCompressed;
+            CompressionFormat = compressionFormat;
+            UncompressedSize = uncompressedSize;
         }
 
         public byte[] ReadData()
@@ -111,7 +124,9 @@ public byte[] ReadData()
             }
 
             if (IsEncrypted)
-                data = Decrypt(data);
+                data = PackFileEncryption.Decrypt(data);
+            if (IsCompressed)
+                data = PackFileCompression.Decompress(data);
             return data;
         }
 
@@ -120,30 +135,87 @@ public byte[] ReadData(int size)
             var data = new byte[size];
             using (Stream stream = File.Open(_parent.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
             {
-            stream.Seek(Offset, SeekOrigin.Begin);
-                stream.Read(data, 0, size);
+                stream.Seek(Offset, SeekOrigin.Begin);
+                stream.Read(data, 0, data.Length);
             }
 
             if (IsEncrypted)
-                data = Decrypt(data);
+                data = PackFileEncryption.Decrypt(data);
+            if (IsCompressed)
+                data = PackFileCompression.Decompress(data);
             return data;
         }
 
-        public byte[] ReadDataForFastSearch(Stream knownStream)
+        public byte[] ReadData(Stream knownStream)
         {
             var data = new byte[Size];
             knownStream.Seek(Offset, SeekOrigin.Begin);
             knownStream.Read(data, 0, (int)Size);
 
             if (IsEncrypted)
-                data = Decrypt(data);
+                data = PackFileEncryption.Decrypt(data);
+            if (IsCompressed)
+                data = PackFileCompression.Decompress(data);
             return data;
         }
 
         public ByteChunk ReadDataAsChunk()
         {
             return new ByteChunk(ReadData());
         }
+
+        public void SetCompressionInfo(GameInformation gameInformation, string rootFolder, string extension)
+        {
+            // Check if the game supports any compression at all
+            if (gameInformation.CompressionFormats.All(compressionFormat => compressionFormat == CompressionFormat.None))
+                return;
+
+            // We use isTable because non-loc tables don't have an extension
+            var isTable = rootFolder == "db" || extension == ".loc";
+            var hasExtension = !string.IsNullOrEmpty(extension);
+
+            // Don't compress files that aren't tables and don't have extensions
+            if (!isTable && !hasExtension)
+            {
+                CompressionFormat = CompressionFormat.None;
+                IsCompressed = false;
+                return;
+            }
+
+            // Only in WH3 (and newer games?) is the table compression bug fixed
+            if (isTable && gameInformation.CompressionFormats.Contains(CompressionFormat.Zstd) && gameInformation.Type == GameTypeEnum.Warhammer3)
+            {
+                CompressionFormat = CompressionFormat.Zstd;
+                IsCompressed = true;
+                return;
+            }
+
+            // Games that support the other formats won't use Lzma1 as it's legacy so if it's set then it's for a game that only uses it so keep it
+            if (CompressionFormat == CompressionFormat.Lzma1 && gameInformation.CompressionFormats.Contains(CompressionFormat.Lzma1))
+                return;
+
+            // Anything that shouldn't be None or Lz4 is set to Zstd unless the game doesn't support that in which case use None
+            if (PackFileCompression.NoneFileTypes.Contains(extension))
+            {
+                CompressionFormat = CompressionFormat.None;
+                IsCompressed = false;
+            }
+            else if (PackFileCompression.Lz4FileTypes.Contains(extension) && gameInformation.CompressionFormats.Contains(CompressionFormat.Lz4))
+            {
+                CompressionFormat = CompressionFormat.Lz4;
+                IsCompressed = true;
+            }
+            else if (gameInformation.CompressionFormats.Contains(CompressionFormat.Zstd))
+            {
+                CompressionFormat = CompressionFormat.Zstd;
+                IsCompressed = true;
+            }
+            else
+            {
+                CompressionFormat = CompressionFormat.None;
+                IsCompressed = false;
+            }
+        }
     }
 
     public class PackedFileSourceParent