From edb7fa6ae2d319bd0c56880f0776b90ffbebcfa3 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 25 Aug 2025 10:56:48 +0200 Subject: [PATCH 01/23] Fix typo in error from SettingsFieldFinder_FindFieldByAbsoluteName() Range interval is half open so it should be just arrayElemCount. --- src/assets/settings_layout.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assets/settings_layout.cpp b/src/assets/settings_layout.cpp index d1086dae..de92f2d3 100644 --- a/src/assets/settings_layout.cpp +++ b/src/assets/settings_layout.cpp @@ -230,7 +230,7 @@ bool SettingsFieldFinder_FindFieldByAbsoluteName(const SettingsLayoutAsset_s& la if (lookupLayoutIndex < 0 || lookupLayoutIndex >= arrayElemCount) { Error("%s: field \"%s\" is an array with range interval [0,%d), but provided array subscript was %d.\n", - __FUNCTION__, lookupName, arrayElemCount-1, lookupLayoutIndex); + __FUNCTION__, lookupName, arrayElemCount, lookupLayoutIndex); return false; } From 0d950f4272380c41e4a406fea9aff2c8da603821 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 25 Aug 2025 12:07:08 +0200 Subject: [PATCH 02/23] Fix off-by-one in SettingsFieldFinder_FindFieldByAbsoluteOffset() If target falls at the end of the range it must be dropped as well since that falls right outside the indexable items by 1. --- src/assets/settings_layout.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assets/settings_layout.cpp b/src/assets/settings_layout.cpp index de92f2d3..077ceb94 100644 --- a/src/assets/settings_layout.cpp +++ b/src/assets/settings_layout.cpp @@ -70,7 +70,7 @@ bool SettingsFieldFinder_FindFieldByAbsoluteOffset(const SettingsLayoutAsset_s& { const uint32_t totalValueBufSizeAligned = IALIGN(layout.rootLayout.totalValueBufferSize, layout.rootLayout.alignment); - if (targetOffset > result.currentBase + (layout.rootLayout.arrayElemCount * totalValueBufSizeAligned)) + if (targetOffset >= result.currentBase + (layout.rootLayout.arrayElemCount * totalValueBufSizeAligned)) return false; // Beyond this layout. const uint32_t fieldOffset = layout.rootLayout.offsetMap[i]; From b8b68d116182c88f88f5190191705fba829c9afb Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 25 Aug 2025 12:09:43 +0200 Subject: [PATCH 03/23] Fix *bad range* error detection in SettingsFieldFinder_FindFieldByAbsoluteOffset() Fix incorrect *invalid offset (i.e. lands on pad bytes)*. We must apply the current base ontop of the field offset as the offsets in `offsetMap` are relative and `targetOffset` is absolute! This problem only existed for sub-layouts that do not start at offset 0 in the absolute root layout. --- src/assets/settings_layout.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assets/settings_layout.cpp b/src/assets/settings_layout.cpp index 077ceb94..3e9d0709 100644 --- a/src/assets/settings_layout.cpp +++ b/src/assets/settings_layout.cpp @@ -75,7 +75,7 @@ bool SettingsFieldFinder_FindFieldByAbsoluteOffset(const SettingsLayoutAsset_s& const uint32_t fieldOffset = layout.rootLayout.offsetMap[i]; - if (targetOffset < fieldOffset) + if (targetOffset < (result.currentBase + fieldOffset)) return false; // Invalid offset (i.e. we have 2 ints at 4 and 8, but target was 5). const uint32_t originalBase = result.currentBase; From 8a5d359527e370968eadc38146187f30503a9406 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 25 Aug 2025 12:11:53 +0200 Subject: [PATCH 04/23] Improve variable naming for SettingsFieldFinder_FindFieldByAbsoluteOffset() --- src/assets/settings_layout.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/assets/settings_layout.cpp b/src/assets/settings_layout.cpp index 3e9d0709..ae7f6034 100644 --- a/src/assets/settings_layout.cpp +++ b/src/assets/settings_layout.cpp @@ -69,20 +69,19 @@ bool SettingsFieldFinder_FindFieldByAbsoluteOffset(const SettingsLayoutAsset_s& for (size_t i = 0; i < layout.rootLayout.typeMap.size(); i++) { const uint32_t totalValueBufSizeAligned = IALIGN(layout.rootLayout.totalValueBufferSize, layout.rootLayout.alignment); + const uint32_t originalBaseCurrentDepth = result.currentBase; - if (targetOffset >= result.currentBase + (layout.rootLayout.arrayElemCount * totalValueBufSizeAligned)) + if (targetOffset >= originalBaseCurrentDepth + (layout.rootLayout.arrayElemCount * totalValueBufSizeAligned)) return false; // Beyond this layout. const uint32_t fieldOffset = layout.rootLayout.offsetMap[i]; - if (targetOffset < (result.currentBase + fieldOffset)) + if (targetOffset < (originalBaseCurrentDepth + fieldOffset)) return false; // Invalid offset (i.e. we have 2 ints at 4 and 8, but target was 5). - const uint32_t originalBase = result.currentBase; - for (int currArrayIdx = 0; currArrayIdx < layout.rootLayout.arrayElemCount; currArrayIdx++) { - const uint32_t elementBase = result.currentBase + (currArrayIdx * totalValueBufSizeAligned); + const uint32_t elementBase = originalBaseCurrentDepth + (currArrayIdx * totalValueBufSizeAligned); const uint32_t absoluteFieldOffset = elementBase + fieldOffset; const SettingsFieldType_e fieldType = layout.rootLayout.typeMap[i]; @@ -124,7 +123,7 @@ bool SettingsFieldFinder_FindFieldByAbsoluteOffset(const SettingsLayoutAsset_s& return true; } - result.currentBase = originalBase; + result.currentBase = originalBaseCurrentDepth; } } } From 91d3028e57279b4ae7efa5a85811efb560ed4e70 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 25 Aug 2025 13:57:53 +0200 Subject: [PATCH 05/23] Fix crash when item.nameIndex == modNamesCount Off by one, since it indexes into the array it must be tested against modNamesCount-1. --- src/assets/settings.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/assets/settings.cpp b/src/assets/settings.cpp index 86466718..86f5972f 100644 --- a/src/assets/settings.cpp +++ b/src/assets/settings.cpp @@ -273,8 +273,8 @@ static void SettingsAsset_CalculateModValuesBuffers(const rapidjson::Value& modV item.nameIndex = static_cast(nameIndex); - if (item.nameIndex > modNamesCount) - Error("Settings mod value #%zu indexes outside mod names array (%hu > %hu).\n", elemIndex, item.nameIndex, static_cast(modNamesCount)); + if (item.nameIndex >= modNamesCount) + Error("Settings mod value #%zu indexes outside mod names array (%hu > %hu).\n", elemIndex, item.nameIndex, static_cast(modNamesCount-1)); const char* const typeName = JSON_GetValueRequired(elem, "type"); From 67a2aaf606f2a21ae92ccf9e61cfc4812aa19840 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 25 Aug 2025 13:58:36 +0200 Subject: [PATCH 06/23] Fix incorrect error print value Must be numColumns. --- src/assets/datatable.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assets/datatable.cpp b/src/assets/datatable.cpp index 44520d56..e7e33f6f 100644 --- a/src/assets/datatable.cpp +++ b/src/assets/datatable.cpp @@ -29,7 +29,7 @@ static size_t DataTable_SetupRows(const rapidcsv::Document& doc, datatable_t* co // typically happens when there's an empty line in the csv file. if (numTypeNames != dtblHdr->numColumns) - Error("Expected %u columns for type name row, found %u.\n", dtblHdr->numRows, numTypeNames); + Error("Expected %u columns for type name row, found %u.\n", dtblHdr->numColumns, numTypeNames); // Make sure every row (including rows we don't end up storing in the pak), // have the same number of columns as the type row. The column count in the From ea64f27122866b12ac96dbe0791c759c977513b8 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 25 Aug 2025 14:00:04 +0200 Subject: [PATCH 07/23] Fix potential unhandled crash in DataTable_SetupRows() Always use the DataTable_ParseCellFromDocument() helper to retrieve cells, its also faster. --- src/assets/datatable.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/assets/datatable.cpp b/src/assets/datatable.cpp index e7e33f6f..5f0f310d 100644 --- a/src/assets/datatable.cpp +++ b/src/assets/datatable.cpp @@ -20,6 +20,18 @@ static void DataTable_ReportInvalidDataTypeError(const char* const type, const u Error("Invalid data type \"%s\" at cell [%u,%u].\n", type, rowIdx, colIdx); } +template +static T DataTable_ParseCellFromDocument(const rapidcsv::Document& doc, const uint32_t colIdx, const uint32_t rowIdx, const dtblcoltype_t type) +{ + try { + return doc.GetCell(colIdx, rowIdx); + } + catch (const std::exception& ex) { + Error("Exception while parsing %s value from cell [%u,%u]: %s.\n", DataTable_GetStringFromType(type), rowIdx, colIdx, ex.what()); + return T{}; + } +} + template static size_t DataTable_SetupRows(const rapidcsv::Document& doc, datatable_t* const dtblHdr, datatable_asset_t& tmp, std::vector& outTypeRow) { @@ -65,8 +77,8 @@ static size_t DataTable_SetupRows(const rapidcsv::Document& doc, datatable_t* co for (uint32_t j = 0; j < dtblHdr->numRows; ++j) { // this can be std::string since we only deal with the string types here - std::vector row = doc.GetRow(j); - const size_t strLen = row[i].length(); + const std::string cellValue = DataTable_ParseCellFromDocument(doc, i, j, type); + const size_t strLen = cellValue.length(); if (isPrecachedAsset && strLen > 0) tmp.guidRefBufSize += sizeof(PakGuid_t); @@ -116,18 +128,6 @@ static void DataTable_SetupColumns(CPakFileBuilder* const pak, PakPageLump_s& da } } -template -static T DataTable_ParseCellFromDocument(rapidcsv::Document& doc, const uint32_t colIdx, const uint32_t rowIdx, const dtblcoltype_t type) -{ - try { - return doc.GetCell(colIdx, rowIdx); - } - catch (const std::exception& ex) { - Error("Exception while parsing %s value from cell [%u,%u]: %s.\n", DataTable_GetStringFromType(type), rowIdx, colIdx, ex.what()); - return T{}; - } -} - static void DataTable_ReportInvalidValueError(const dtblcoltype_t type, const uint32_t rowIdx, const uint32_t colIdx) { Error("Invalid %s value at cell [%u,%u].\n", DataTable_GetStringFromType(type), rowIdx, colIdx); From 61f2af0d28a1a5360f7eeca467a69090b658307d Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 25 Aug 2025 14:01:21 +0200 Subject: [PATCH 08/23] Improve Vector3D & Vector2D parsing performance Use sscanf over regex since its faster and parsing vectors is too simple to justify the use of regex. --- src/assets/datatable.cpp | 19 +++---------------- src/assets/settings.cpp | 23 ++++++++--------------- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/src/assets/datatable.cpp b/src/assets/datatable.cpp index 5f0f310d..db7ad1fb 100644 --- a/src/assets/datatable.cpp +++ b/src/assets/datatable.cpp @@ -184,25 +184,12 @@ static void DataTable_SetupValues(CPakFileBuilder* const pak, PakAsset_t& asset, } case dtblcoltype_t::Vector: { - std::string val = DataTable_ParseCellFromDocument(doc, colIdx, rowIdx, col.type); - std::smatch sm; + const std::string val = DataTable_ParseCellFromDocument(doc, colIdx, rowIdx, col.type); + Vector3 vec; // get values from format "" - const bool result = std::regex_search(val, sm, std::regex("<(.*),(.*),(.*)>")); - - // 0 - all - // 1 - x - // 2 - y - // 3 - z - if (result && (sm.size() == 4)) - { - const Vector3 vec( - static_cast(atof(sm[1].str().c_str())), - static_cast(atof(sm[2].str().c_str())), - static_cast(atof(sm[3].str().c_str()))); - + if (sscanf_s(val.c_str(), "<%f,%f,%f>", &vec.x, &vec.y, &vec.z) == 3) valbuf.write(vec); - } else DataTable_ReportInvalidValueError(col.type, rowIdx, colIdx); break; diff --git a/src/assets/settings.cpp b/src/assets/settings.cpp index 86f5972f..ceba9740 100644 --- a/src/assets/settings.cpp +++ b/src/assets/settings.cpp @@ -443,23 +443,16 @@ static void SettingsAsset_WriteAssetValue(const char* const string, const size_t static void SettingsAsset_WriteVectorValue(const char* const value, const char* const fieldName, const size_t valueOffset, const bool isVec2, PakPageLump_s& dataLump) { - std::cmatch sm; // get values from format "", or "". - const bool result = std::regex_search(value, sm, std::regex(isVec2 ? "<(.*),(.*)>" : "<(.*),(.*),(.*)>")); - - // 0 - all - // 1 - x - // 2 - y - // 3 - z (if !isVec2) - if (result && (sm.size() == (isVec2 ? 3 : 4))) - { - Vector3* const vec = reinterpret_cast(&dataLump.data[valueOffset]); - vec->x = static_cast(atof(sm[1].str().c_str())); - vec->y = static_cast(atof(sm[2].str().c_str())); + Vector3* const vec = reinterpret_cast(&dataLump.data[valueOffset]); + bool success = false; - if (!isVec2) - vec->z = static_cast(atof(sm[3].str().c_str())); - } + // get values from format "", or "". + if (isVec2) + success = sscanf_s(value, "<%f,%f>", &vec->x, &vec->y) == 2; else + success = sscanf_s(value, "<%f,%f,%f>", &vec->x, &vec->y, &vec->z) == 3; + + if (!success) { const char* const targetType = isVec2 ? "float2" : "float3"; Error("Field \"%s\" has value '%s' that cannot be parsed as %s.\n", fieldName, value, targetType); From 32c82603fd5374252f7ca5d7cf2179e4a46c4a90 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Tue, 26 Aug 2025 00:59:16 +0200 Subject: [PATCH 09/23] Avoid division by zero on invalid UI image atlas Make sure width and height aren't zero since they can be if the texture atlas is corrupt. --- src/assets/ui_image_atlas.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/assets/ui_image_atlas.cpp b/src/assets/ui_image_atlas.cpp index 8a825f8f..22325eb0 100644 --- a/src/assets/ui_image_atlas.cpp +++ b/src/assets/ui_image_atlas.cpp @@ -54,6 +54,9 @@ void Assets::AddUIImageAsset_v10(CPakFileBuilder* const pak, const PakGuid_t ass UIImageAtlasHeader_t* const pHdr = reinterpret_cast(hdrLump.data); const TextureAssetHeader_t* const atlasHdr = reinterpret_cast(atlasAsset->header); + if (atlasHdr->width == 0 || atlasHdr->height == 0) + Error("UI image atlas has invalid dimensions (%hux%hu); width and height cannot be zero!\n", atlasHdr->width, atlasHdr->height); + pHdr->width = atlasHdr->width; pHdr->height = atlasHdr->height; From 98e165e9dfa287a1b7e056178ebc3d092e9ffe9c Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Tue, 26 Aug 2025 01:01:25 +0200 Subject: [PATCH 10/23] Make sure mstudioevent_t::options read is clamped Do not read more than 256 chars, this can happen if this section is corrupt. --- src/assets/animseq.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/assets/animseq.cpp b/src/assets/animseq.cpp index 37295b2f..bd97ebbc 100644 --- a/src/assets/animseq.cpp +++ b/src/assets/animseq.cpp @@ -57,7 +57,10 @@ static void AnimSeq_ParseDependenciesFromData(const uint8_t* const data, std::se const char* end = strchr(start, ' '); if (!end) - end = event->options + strlen(event->options); + { + const size_t maxLen = sizeof(event->options) - (start - event->options); + end = start + strnlen(start, maxLen); + } const size_t nameLen = (end - start); From 28a59a28862904c4fe2fcb88d8a0a654c1ecc79d Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Tue, 26 Aug 2025 01:03:42 +0200 Subject: [PATCH 11/23] Improve performance for ASEQ dependency iteration Use range scan instead of manually advancing the iterator since this turns out to be quite slow actually. Range scan is a lot faster. --- src/assets/animseq.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/assets/animseq.cpp b/src/assets/animseq.cpp index bd97ebbc..b9c96262 100644 --- a/src/assets/animseq.cpp +++ b/src/assets/animseq.cpp @@ -201,15 +201,13 @@ static void AnimSeq_InternalAddAnimSeq(CPakFileBuilder* const pak, const PakGuid hdr->settingsCount = (uint32_t)set.size(); } - for (size_t j = 0; j < set.size(); j++) - { - std::set::iterator it = set.begin(); - std::advance(it, j); + size_t j = 0; - const PakGuid_t guidToCopy = *it; + for (const PakGuid_t& guidToCopy : set) + { reinterpret_cast(&dataLump.data[bufferBase])[j] = guidToCopy; - Pak_RegisterGuidRefAtOffset(guidToCopy, bufferBase + (j * sizeof(PakGuid_t)), dataLump, asset); + ++j; } bufferBase += set.size() * sizeof(PakGuid_t); From 76c23751671e43b4ae31d4413f3ee0fcc20a8465 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Tue, 26 Aug 2025 01:07:15 +0200 Subject: [PATCH 12/23] Report error if we failed to read anim recording pose param values --- src/assets/anim_recording.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/assets/anim_recording.cpp b/src/assets/anim_recording.cpp index 174b51f1..2d2a4011 100644 --- a/src/assets/anim_recording.cpp +++ b/src/assets/anim_recording.cpp @@ -101,7 +101,8 @@ static void AnimRecording_InternalAddAnimRecording(CPakFileBuilder* const pak, c for (int i = 0; i < fileHdr.numElements; i++) { - bio.Read(pHdr->poseParamValues[i]); + if (!bio.Read(pHdr->poseParamValues[i])) + Error("Failed to read animation pose parameter value #%i\n", i); } for (int i = 0; i < fileHdr.numSequences; i++) From ed02498df205cb1bcd39b0ef1fc62bf1cd2c2f87 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Tue, 26 Aug 2025 11:27:38 +0200 Subject: [PATCH 13/23] Major improvements to BinaryIO BinaryIO now returns true if read and write operations were successful, false if they weren't. The size housekeeping logic has also been rewritten. If we seek forward, we will store this amount now and only add it to the total size if we end up writing something at that location. Previously it would always add to the totalSize even if we weren't writing to the seeked pos which is incorrect! --- src/utils/binaryio.cpp | 228 +++++++++++++++++------------------------ src/utils/binaryio.h | 90 +++++++++------- 2 files changed, 148 insertions(+), 170 deletions(-) diff --git a/src/utils/binaryio.cpp b/src/utils/binaryio.cpp index 32c0800a..2b2f5048 100644 --- a/src/utils/binaryio.cpp +++ b/src/utils/binaryio.cpp @@ -3,23 +3,19 @@ #include //----------------------------------------------------------------------------- -// Purpose: CIOStream constructors +// constructors/destructors //----------------------------------------------------------------------------- BinaryIO::BinaryIO() { Reset(); } - -//----------------------------------------------------------------------------- -// Purpose: CIOStream destructor -//----------------------------------------------------------------------------- BinaryIO::~BinaryIO() { Close(); } //----------------------------------------------------------------------------- -// Purpose: get internal stream mode from selected mode +// Get internal stream mode from selected mode //----------------------------------------------------------------------------- static std::ios_base::openmode GetInternalStreamMode(const BinaryIO::Mode_e mode) { @@ -40,10 +36,7 @@ static std::ios_base::openmode GetInternalStreamMode(const BinaryIO::Mode_e mode } //----------------------------------------------------------------------------- -// Purpose: opens the file in specified mode -// Input : *filePath - -// mode - -// Output : true if operation is successful +// Opens the file in specified mode //----------------------------------------------------------------------------- bool BinaryIO::Open(const char* const filePath, const Mode_e mode) { @@ -70,25 +63,26 @@ bool BinaryIO::Open(const char* const filePath, const Mode_e mode) return false; } - m_size = status.st_size; + m_state.totalSize = status.st_size; } return true; } //----------------------------------------------------------------------------- -// Purpose: resets the state +// Resets the internal state //----------------------------------------------------------------------------- void BinaryIO::Reset() { - m_size = 0; - m_skip = 0; + m_state.totalSize = 0; + m_state.skipCount = 0; + m_state.seekGap = 0; m_mode = Mode_e::None; m_flags = 0; } //----------------------------------------------------------------------------- -// Purpose: closes the stream +// Closes the stream //----------------------------------------------------------------------------- void BinaryIO::Close() { @@ -97,7 +91,7 @@ void BinaryIO::Close() } //----------------------------------------------------------------------------- -// Purpose: flushes the ofstream +// Flushes the ofstream //----------------------------------------------------------------------------- void BinaryIO::Flush() { @@ -106,8 +100,7 @@ void BinaryIO::Flush() } //----------------------------------------------------------------------------- -// Purpose: gets the position of the current character in the stream -// Output : std::streampos +// Gets the position of the current character in the stream //----------------------------------------------------------------------------- std::streamoff BinaryIO::TellGet() { @@ -121,9 +114,7 @@ std::streamoff BinaryIO::TellPut() } //----------------------------------------------------------------------------- -// Purpose: sets the position of the current character in the stream -// Input : offset - -// way - +// Sets the position of the current character in the stream //----------------------------------------------------------------------------- void BinaryIO::SeekGet(const std::streamoff offset, const std::ios_base::seekdir way) { @@ -138,7 +129,7 @@ void BinaryIO::SeekPut(const std::streamoff offset, const std::ios_base::seekdir { assert(IsWriteMode()); - CalcSkipDelta(offset, way); + UpdateSeekState(offset, way); m_stream.seekp(offset, way); } void BinaryIO::Seek(const std::streamoff offset, const std::ios_base::seekdir way) @@ -150,8 +141,7 @@ void BinaryIO::Seek(const std::streamoff offset, const std::ios_base::seekdir wa } //----------------------------------------------------------------------------- -// Purpose: returns the data -// Output : std::filebuf* +// Returns the stream buffer //----------------------------------------------------------------------------- const std::filebuf* BinaryIO::GetData() const { @@ -159,12 +149,11 @@ const std::filebuf* BinaryIO::GetData() const } //----------------------------------------------------------------------------- -// Purpose: returns the data size -// Output : std::streampos +// Returns the data size //----------------------------------------------------------------------------- const std::streamoff BinaryIO::GetSize() const { - return m_size; + return m_state.totalSize; } bool BinaryIO::IsReadMode() const @@ -178,8 +167,7 @@ bool BinaryIO::IsWriteMode() const } //----------------------------------------------------------------------------- -// Purpose: checks if we are able to read the file -// Output : true on success, false otherwise +// Checks if we are able to read the file //----------------------------------------------------------------------------- bool BinaryIO::IsReadable() const { @@ -190,8 +178,7 @@ bool BinaryIO::IsReadable() const } //----------------------------------------------------------------------------- -// Purpose: checks if we are able to write to file -// Output : true on success, false otherwise +// Checks if we are able to write to file //----------------------------------------------------------------------------- bool BinaryIO::IsWritable() const { @@ -202,8 +189,7 @@ bool BinaryIO::IsWritable() const } //----------------------------------------------------------------------------- -// Purpose: checks if we hit the end of file -// Output : true on success, false otherwise +// Checks if we hit the end of file //----------------------------------------------------------------------------- bool BinaryIO::IsEof() const { @@ -211,71 +197,60 @@ bool BinaryIO::IsEof() const } //----------------------------------------------------------------------------- -// Purpose: reads a string from the file -// Input : &svOut - -// Output : true on success, false otherwise +// Reads a string from the file //----------------------------------------------------------------------------- bool BinaryIO::ReadString(std::string& out) { if (!IsReadable()) return false; - while (!m_stream.eof()) - { - const char c = Read(); + char c; + while (m_stream.get(c)) + { if (c == '\0') - break; + return true; out += c; } - return true; + return false; // EOF or error, result is truncated. } //----------------------------------------------------------------------------- -// Purpose: reads a string from the file into a fixed size buffer -// Input : *buf - -// len - -// Output : true on success, false otherwise +// Reads a string from the file into a fixed size buffer //----------------------------------------------------------------------------- -bool BinaryIO::ReadString(char* const buf, const size_t len) +bool BinaryIO::ReadString(char* const buf, const size_t len, size_t* const outWriteCount) { + assert(buf && len > 0); + if (!IsReadable()) return false; size_t i = 0; + char c; + bool fullRead = false; - while (i < len && !m_stream.eof()) + while (i < (len - 1) && m_stream.get(c)) { - const char c = Read(); - if (c == '\0') + { + buf[i] = '\0'; + fullRead = true; + break; + } buf[i++] = c; } - return true; -} + if (!fullRead) + buf[i] = '\0'; -//----------------------------------------------------------------------------- -// Purpose: writes a string to the file -// Input : &input - -// Output : true on success, false otherwise -//----------------------------------------------------------------------------- -bool BinaryIO::WriteString(const std::string& input, const bool nullterminate) -{ - if (!IsWritable()) - return false; - - const char* const text = input.c_str(); - const size_t len = input.length() + nullterminate; + if (outWriteCount) + *outWriteCount = i; - m_stream.write(text, len); - CalcAddDelta(len); - - return true; + return fullRead; // if false, string too long and result is truncated. } // limit number of io calls and allocations by just using this static buffer @@ -284,10 +259,9 @@ static constexpr size_t PAD_BUF_SIZE = 4096; const static char s_padBuf[PAD_BUF_SIZE]; //----------------------------------------------------------------------------- -// Purpose: pads the out stream up to count bytes -// Input : count - +// Pads the out stream up to count bytes //----------------------------------------------------------------------------- -void BinaryIO::Pad(const size_t count) +bool BinaryIO::Pad(const size_t count, size_t* const outPadCount) { assert(count > 0); size_t remainder = count; @@ -295,88 +269,78 @@ void BinaryIO::Pad(const size_t count) while (remainder) { const size_t writeCount = (std::min)(remainder, PAD_BUF_SIZE); - Write(s_padBuf, writeCount); + + if (!Write(s_padBuf, writeCount)) + break; remainder -= writeCount; } + + if (outPadCount) + *outPadCount = count - remainder; + + return remainder == 0; } //----------------------------------------------------------------------------- -// Purpose: makes sure that the size gets incremented if we exceeded the end of -// the stream with the delta amount +// If we seek backwards, and then write new data, we should not add +// this to the total output size of the stream as we modify and not +// add. we have to keep by how much we shifted backwards and advanced +// forward until we can start adding again. //----------------------------------------------------------------------------- -void BinaryIO::CalcAddDelta(const size_t count) +void BinaryIO::UpdateSeekState(const std::streamoff offset, const std::ios_base::seekdir way) { - if (m_skip > 0) + std::streamoff targetPos = 0; + + switch (way) // determine the abs target pos of the seek. { - m_skip -= count; + case std::ios_base::beg: + targetPos = offset; + break; + case std::ios_base::cur: + targetPos = (m_state.totalSize - m_state.skipCount) + offset; + break; + case std::ios_base::end: + targetPos = m_state.totalSize + offset; + break; + } - if (m_skip < 0) - { - m_size += -m_skip; // Add the overshoot to the file size. - m_skip = 0; - } + if (targetPos > m_state.totalSize) + { + m_state.seekGap = targetPos - m_state.totalSize; + m_state.skipCount = 0; // cannot have skip and gap. + } + else // seeking within logical bounds. + { + m_state.skipCount = m_state.totalSize - targetPos; + m_state.seekGap = 0; // cannot have skip and gap. } - else - m_size += count; } //----------------------------------------------------------------------------- -// Purpose: if we seek backwards, and then write new data, we should not add -// this to the total output size of the stream as we modify and not -// add. we have to keep by how much we shifted backwards and advanced -// forward until we can start adding again. +// Updates the logical file size after a write of `writeCount` bytes. +// This handles padding from gaps and appending new data. //----------------------------------------------------------------------------- -void BinaryIO::CalcSkipDelta(const std::streamoff offset, const std::ios_base::seekdir way) +void BinaryIO::PostWriteUpdate(const size_t writeCount) { - switch (way) - { - case std::ios_base::beg: + // if we had a gap, append it to the total size as it has been padded out. + if (m_state.seekGap > 0) { - if (offset < 0) - { - assert(false && "Negative offset in std::ios_base::beg is invalid."); - return; - } - - if (offset > m_size) - { - m_size = offset; - m_skip = 0; - } - else - m_skip = m_size - offset; - break; + m_state.totalSize += m_state.seekGap; + m_state.seekGap = 0; } - case std::ios_base::cur: - { - if (offset > 0) - CalcAddDelta(offset); - else - m_skip += -offset; - break; - } - case std::ios_base::end: + + // account for overwriting/appending. + if (m_state.skipCount > 0) { - if (offset >= 0) + if (writeCount >= (size_t)m_state.skipCount) { - m_size += offset; - m_skip = 0; + m_state.totalSize += (writeCount - m_state.skipCount); + m_state.skipCount = 0; } - else - m_skip += -offset; - break; - } - default: - assert(false && "Unsupported seek direction."); - break; + else // Writing within the logical bounds. + m_state.skipCount -= writeCount; } - - // Ensure m_skip is non-negative, this can happen if you call this method - // with cur or end, and a negative value who's absolute value is greater - // than the total stream size. If you hit this, you have a bug somewhere. - assert(m_skip >= 0); - - if (m_skip < 0) - m_skip = 0; + else + m_state.totalSize += writeCount; } diff --git a/src/utils/binaryio.h b/src/utils/binaryio.h index 1c1091df..f9cfd317 100644 --- a/src/utils/binaryio.h +++ b/src/utils/binaryio.h @@ -41,85 +41,99 @@ class BinaryIO bool IsEof() const; //----------------------------------------------------------------------------- - // Purpose: reads any value from the file + // Reads any value from the file with specified size //----------------------------------------------------------------------------- template - inline void Read(T& value) + inline bool Read(T* const value, const size_t size, size_t* const outReadCount = nullptr) { - if (IsReadable()) - m_stream.read(reinterpret_cast(&value), sizeof(value)); + if (!IsReadable()) + return false; + + char* const data = reinterpret_cast(value); + const size_t readCount = m_stream.rdbuf()->sgetn(data, size); + + if (outReadCount) + *outReadCount = readCount; + + return readCount == size; } //----------------------------------------------------------------------------- - // Purpose: reads any value from the file with specified size + // Reads any value from the file //----------------------------------------------------------------------------- template - inline void Read(T* const value, const size_t size) - { - if (IsReadable()) - m_stream.read(reinterpret_cast(value), size); - } - template - inline void Read(T& value, const size_t size) + inline bool Read(T& value, size_t* const outReadCount = nullptr) { - if (IsReadable()) - m_stream.read(reinterpret_cast(&value), size); + return Read(&value, sizeof(T), outReadCount); } //----------------------------------------------------------------------------- - // Purpose: reads any value from the file and returns it + // Reads any value from the file and returns it //----------------------------------------------------------------------------- template - inline T Read() + inline T Read(size_t* const outReadCount = nullptr) { T value{}; - if (!IsReadable()) - return value; + Read(&value, sizeof(T), outReadCount); - m_stream.read(reinterpret_cast(&value), sizeof(value)); return value; } bool ReadString(std::string& svOut); - bool ReadString(char* const pBuf, const size_t nLen); + bool ReadString(char* const pBuf, const size_t nLen, size_t* const outWriteCount = nullptr); //----------------------------------------------------------------------------- - // Purpose: writes any value to the file + // Writes any value to the file with specified size //----------------------------------------------------------------------------- template - inline void Write(const T& value) + inline bool Write(const T* const value, const size_t size, size_t* const outWriteCount = nullptr) { if (!IsWritable()) - return; + return false; + + const char* const data = reinterpret_cast(value); + const size_t writeCount = m_stream.rdbuf()->sputn(data, size); + + PostWriteUpdate(writeCount); - const size_t count = sizeof(value); + if (outWriteCount) + *outWriteCount = writeCount; - m_stream.write(reinterpret_cast(&value), count); - CalcAddDelta(count); + return writeCount == size; } //----------------------------------------------------------------------------- - // Purpose: writes any value to the file with specified size + // Writes any value to the file //----------------------------------------------------------------------------- template - inline void Write(const T* const value, const size_t size) + inline bool Write(const T& value, size_t* const outWriteCount = nullptr) { - if (!IsWritable()) - return; + return Write(&value, sizeof(T), outWriteCount); + } - m_stream.write(reinterpret_cast(value), size); - CalcAddDelta(size); + //----------------------------------------------------------------------------- + // Writes a string to the file + //----------------------------------------------------------------------------- + inline bool WriteString(const std::string& input, const bool nullTerminate, size_t* const outWriteCount = nullptr) + { + return Write(input.c_str(), input.length() + nullTerminate, outWriteCount); } - bool WriteString(const std::string& svInput, const bool nullterminate); - void Pad(const size_t count); + + bool Pad(const size_t count, size_t* const outPadCount = nullptr); protected: - void CalcAddDelta(const size_t count); - void CalcSkipDelta(const std::streamoff offset, const std::ios_base::seekdir way); + void UpdateSeekState(const std::streamoff offset, const std::ios_base::seekdir way); + void PostWriteUpdate(const size_t writeCount); private: + struct CursorState_s + { + std::streamoff totalSize; // Total stream output size. + std::streamoff skipCount; // Amount seeked into logical bounds. + std::streamoff seekGap; // Amount seeked beyond logical bounds. + }; + std::fstream m_stream; // I/O stream. - std::streamoff m_size; // File size. - std::streamoff m_skip; // Amount skipped back. + CursorState_s m_state; // Stream state. std::ios_base::openmode m_flags; // Stream flags. Mode_e m_mode; // Stream mode. }; From 6490f926f6324fb2f0c4a22bf9bd9fd9a8cef320 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Tue, 26 Aug 2025 13:44:49 +0200 Subject: [PATCH 14/23] Use stream itself to calc file size initially The stat func is faster but not secure as it requires another file system opening query. Use the stream itself. This isn't a major issue since the rest of the code will use the fast readily available calculated size. --- src/utils/binaryio.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/utils/binaryio.cpp b/src/utils/binaryio.cpp index 2b2f5048..64af208c 100644 --- a/src/utils/binaryio.cpp +++ b/src/utils/binaryio.cpp @@ -57,13 +57,10 @@ bool BinaryIO::Open(const char* const filePath, const Mode_e mode) if (IsReadMode()) { - struct _stat64 status; - if (_stat64(filePath, &status) != NULL) - { - return false; - } - - m_state.totalSize = status.st_size; + // Calculate the initial size. + m_stream.seekg(0, std::ios::end); + m_state.totalSize = m_stream.tellg(); + m_stream.seekg(0, std::ios::beg); } return true; From 5b3ec3e2ef297fac192f0c02c365c680fd0c2af6 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 6 Oct 2025 13:31:31 +0200 Subject: [PATCH 15/23] Implement error reporting in BinaryIO Throw error if any read or write operation fails, because at that stage the file on disk or data in memory is corrupt! --- src/application/repak.cpp | 1 + src/utils/binaryio.cpp | 72 +++++++++++++++++++++++++++++++++++---- src/utils/binaryio.h | 43 +++++++++-------------- 3 files changed, 83 insertions(+), 33 deletions(-) diff --git a/src/application/repak.cpp b/src/application/repak.cpp index 61db1590..f2854a46 100644 --- a/src/application/repak.cpp +++ b/src/application/repak.cpp @@ -347,6 +347,7 @@ int main(int argc, char** argv) extern bool Console_ColorInit(); Console_ColorInit(); + g_iosmErrorCallback = Error; g_jsonErrorCallback = Error; RePak_HandleCommandLine(argc, argv); diff --git a/src/utils/binaryio.cpp b/src/utils/binaryio.cpp index 64af208c..5deabfea 100644 --- a/src/utils/binaryio.cpp +++ b/src/utils/binaryio.cpp @@ -11,7 +11,7 @@ BinaryIO::BinaryIO() } BinaryIO::~BinaryIO() { - Close(); + Reset(); } //----------------------------------------------------------------------------- @@ -38,17 +38,20 @@ static std::ios_base::openmode GetInternalStreamMode(const BinaryIO::Mode_e mode //----------------------------------------------------------------------------- // Opens the file in specified mode //----------------------------------------------------------------------------- -bool BinaryIO::Open(const char* const filePath, const Mode_e mode) +bool BinaryIO::OpenEx(const char* const filePath, const Mode_e mode, const size_t pathLen) { + assert(filePath && pathLen > 0); + m_flags = GetInternalStreamMode(mode); m_mode = mode; + m_name.assign(filePath, pathLen); if (m_stream.is_open()) { m_stream.close(); } - m_stream.open(filePath, m_flags); + m_stream.open(m_name, m_flags); if (!m_stream.is_open() || !m_stream.good()) { @@ -74,8 +77,8 @@ void BinaryIO::Reset() m_state.totalSize = 0; m_state.skipCount = 0; m_state.seekGap = 0; - m_mode = Mode_e::None; m_flags = 0; + m_mode = Mode_e::None; } //----------------------------------------------------------------------------- @@ -85,6 +88,7 @@ void BinaryIO::Close() { m_stream.close(); Reset(); + m_name.clear(); } //----------------------------------------------------------------------------- @@ -279,6 +283,62 @@ bool BinaryIO::Pad(const size_t count, size_t* const outPadCount) return remainder == 0; } +//----------------------------------------------------------------------------- +// Internal read function. +//----------------------------------------------------------------------------- +bool BinaryIO::DoRead(char* const outBuf, const size_t readCount, size_t* const outReadCount) +{ + if (!IsReadable()) + return false; + + const std::streamoff actualReadCount = m_stream.rdbuf()->sgetn(outBuf, readCount); + + if (outReadCount) + *outReadCount = static_cast(actualReadCount); + + const bool wasSuccesful = actualReadCount == readCount; + + if (!wasSuccesful) + { + if (g_iosmErrorCallback) + { + g_iosmErrorCallback("%s: short read on file \"%s\"; actualReadCount( %zd ) != readCount( %zu ) @ TellGet( %zi )!\n", + __FUNCTION__, m_name.c_str(), actualReadCount, readCount, TellGet()); + } + } + + return wasSuccesful; +} + +//----------------------------------------------------------------------------- +// Internal write function. +//----------------------------------------------------------------------------- +bool BinaryIO::DoWrite(const char* const inBuf, const size_t writeCount, size_t* const outWriteCount) +{ + if (!IsWritable()) + return false; + + const std::streamoff actualWriteCount = m_stream.rdbuf()->sputn(inBuf, writeCount); + + PostWriteUpdate(actualWriteCount); + + if (outWriteCount) + *outWriteCount = actualWriteCount; + + const bool wasSuccesful = actualWriteCount == writeCount; + + if (!wasSuccesful) + { + if (g_iosmErrorCallback) + { + g_iosmErrorCallback("%s: short write on file \"%s\"; actualWriteCount( %zd ) != writeCount( %zu ) @ TellPut( %zi )!\n", + __FUNCTION__, m_name.c_str(), actualWriteCount, writeCount, TellPut()); + } + } + + return wasSuccesful; +} + //----------------------------------------------------------------------------- // If we seek backwards, and then write new data, we should not add // this to the total output size of the stream as we modify and not @@ -318,7 +378,7 @@ void BinaryIO::UpdateSeekState(const std::streamoff offset, const std::ios_base: // Updates the logical file size after a write of `writeCount` bytes. // This handles padding from gaps and appending new data. //----------------------------------------------------------------------------- -void BinaryIO::PostWriteUpdate(const size_t writeCount) +void BinaryIO::PostWriteUpdate(const std::streamoff writeCount) { // if we had a gap, append it to the total size as it has been padded out. if (m_state.seekGap > 0) @@ -330,7 +390,7 @@ void BinaryIO::PostWriteUpdate(const size_t writeCount) // account for overwriting/appending. if (m_state.skipCount > 0) { - if (writeCount >= (size_t)m_state.skipCount) + if (writeCount >= m_state.skipCount) { m_state.totalSize += (writeCount - m_state.skipCount); m_state.skipCount = 0; diff --git a/src/utils/binaryio.h b/src/utils/binaryio.h index f9cfd317..6f166df4 100644 --- a/src/utils/binaryio.h +++ b/src/utils/binaryio.h @@ -1,5 +1,8 @@ #pragma once +typedef void(*BinaryIOLogger_fn)(const char* fmt, ...); +inline BinaryIOLogger_fn g_iosmErrorCallback = nullptr; + class BinaryIO { public: @@ -15,8 +18,8 @@ class BinaryIO BinaryIO(); ~BinaryIO(); - bool Open(const char* const filePath, const Mode_e mode); - inline bool Open(const std::string& filePath, const Mode_e mode) { return Open(filePath.c_str(), mode); }; + inline bool Open(const char* const filePath, const Mode_e mode) { return OpenEx(filePath, mode, strlen(filePath)); } + inline bool Open(const std::string& filePath, const Mode_e mode) { return OpenEx(filePath.c_str(), mode, filePath.length()); }; void Close(); void Reset(); @@ -44,18 +47,9 @@ class BinaryIO // Reads any value from the file with specified size //----------------------------------------------------------------------------- template - inline bool Read(T* const value, const size_t size, size_t* const outReadCount = nullptr) + inline bool Read(T* const outBuf, const size_t readCount, size_t* const outReadCount = nullptr) { - if (!IsReadable()) - return false; - - char* const data = reinterpret_cast(value); - const size_t readCount = m_stream.rdbuf()->sgetn(data, size); - - if (outReadCount) - *outReadCount = readCount; - - return readCount == size; + return DoRead(reinterpret_cast(outBuf), readCount, outReadCount); } //----------------------------------------------------------------------------- @@ -85,20 +79,9 @@ class BinaryIO // Writes any value to the file with specified size //----------------------------------------------------------------------------- template - inline bool Write(const T* const value, const size_t size, size_t* const outWriteCount = nullptr) + inline bool Write(const T* const inBuf, const size_t writeCount, size_t* const outWriteCount = nullptr) { - if (!IsWritable()) - return false; - - const char* const data = reinterpret_cast(value); - const size_t writeCount = m_stream.rdbuf()->sputn(data, size); - - PostWriteUpdate(writeCount); - - if (outWriteCount) - *outWriteCount = writeCount; - - return writeCount == size; + return DoWrite(reinterpret_cast(inBuf), writeCount, outWriteCount); } //----------------------------------------------------------------------------- @@ -121,8 +104,13 @@ class BinaryIO bool Pad(const size_t count, size_t* const outPadCount = nullptr); protected: + bool OpenEx(const char* const filePath, const Mode_e mode, const size_t pathLen); + void UpdateSeekState(const std::streamoff offset, const std::ios_base::seekdir way); - void PostWriteUpdate(const size_t writeCount); + void PostWriteUpdate(const std::streamoff writeCount); + + bool DoRead(char* const outBuf, const size_t readCount, size_t* const outReadCount); + bool DoWrite(const char* const inBuf, const size_t writeCount, size_t* const outWriteCount); private: struct CursorState_s @@ -136,4 +124,5 @@ class BinaryIO CursorState_s m_state; // Stream state. std::ios_base::openmode m_flags; // Stream flags. Mode_e m_mode; // Stream mode. + std::string m_name; // Stream name. }; From 77c6d3e375dcd166e632ab325021fdb114f7b079 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 6 Oct 2025 13:34:46 +0200 Subject: [PATCH 16/23] Use better semantics for IsWritable() and IsReadable() If we read less than requested, we will errors anyways. So just check on good(). --- src/utils/binaryio.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/utils/binaryio.cpp b/src/utils/binaryio.cpp index 5deabfea..891e4647 100644 --- a/src/utils/binaryio.cpp +++ b/src/utils/binaryio.cpp @@ -172,10 +172,7 @@ bool BinaryIO::IsWriteMode() const //----------------------------------------------------------------------------- bool BinaryIO::IsReadable() const { - if (!IsReadMode() || !m_stream || m_stream.eof()) - return false; - - return true; + return IsReadMode() && m_stream.good(); } //----------------------------------------------------------------------------- @@ -183,10 +180,7 @@ bool BinaryIO::IsReadable() const //----------------------------------------------------------------------------- bool BinaryIO::IsWritable() const { - if (!IsWriteMode() || !m_stream) - return false; - - return true; + return IsWriteMode() && m_stream.good(); } //----------------------------------------------------------------------------- From be957a3faaa0e6e94f00e5710a3a39229f8eab8c Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 6 Oct 2025 13:35:16 +0200 Subject: [PATCH 17/23] Initialize s_padBuf Just in case some compiler will complain or push garbage here. --- src/utils/binaryio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/binaryio.cpp b/src/utils/binaryio.cpp index 891e4647..4d6c7bef 100644 --- a/src/utils/binaryio.cpp +++ b/src/utils/binaryio.cpp @@ -251,7 +251,7 @@ bool BinaryIO::ReadString(char* const buf, const size_t len, size_t* const outWr // limit number of io calls and allocations by just using this static buffer // for padding out the stream. static constexpr size_t PAD_BUF_SIZE = 4096; -const static char s_padBuf[PAD_BUF_SIZE]; +const static char s_padBuf[PAD_BUF_SIZE] = {}; //----------------------------------------------------------------------------- // Pads the out stream up to count bytes From b5d86c111063d104bfe07b094718e7a599406814 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Mon, 6 Oct 2025 13:35:44 +0200 Subject: [PATCH 18/23] Check for null on g_jsonErrorCallback --- src/utils/jsonutils.cpp | 16 ++++++++++++---- src/utils/jsonutils.h | 14 ++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/utils/jsonutils.cpp b/src/utils/jsonutils.cpp index 14fac940..516f5998 100644 --- a/src/utils/jsonutils.cpp +++ b/src/utils/jsonutils.cpp @@ -19,7 +19,10 @@ bool JSON_ParseFromFile(const char* const assetPath, const char* const debugName // if there are parsing or validation problems, we will still error as // these are considered unintentional problems. if (mandatory) - g_jsonErrorCallback("%s: couldn't open %s file.\n", __FUNCTION__, debugName); + { + if (g_jsonErrorCallback) + g_jsonErrorCallback("%s: couldn't open %s file.\n", __FUNCTION__, debugName); + } return false; } @@ -28,15 +31,20 @@ bool JSON_ParseFromFile(const char* const assetPath, const char* const debugName if (document.ParseStream(jsonStreamWrapper).HasParseError()) { - g_jsonErrorCallback("%s: %s parse error at position %zu: [%s].\n", __FUNCTION__, debugName, - document.GetErrorOffset(), rapidjson::GetParseError_En(document.GetParseError())); + if (g_jsonErrorCallback) + { + g_jsonErrorCallback("%s: %s parse error at position %zu: [%s].\n", __FUNCTION__, debugName, + document.GetErrorOffset(), rapidjson::GetParseError_En(document.GetParseError())); + } return false; } if (!document.IsObject()) { - g_jsonErrorCallback("%s: %s root was not an object.\n", __FUNCTION__, debugName); + if (g_jsonErrorCallback) + g_jsonErrorCallback("%s: %s root was not an object.\n", __FUNCTION__, debugName); + return false; } diff --git a/src/utils/jsonutils.h b/src/utils/jsonutils.h index a4d00998..b2bd965d 100644 --- a/src/utils/jsonutils.h +++ b/src/utils/jsonutils.h @@ -257,7 +257,9 @@ inline bool JSON_GetRequired(const T& data, typename T::StringRefType member, ty if (JSON_GetIterator(data, member, out)) return true; - g_jsonErrorCallback("%s: unable to find field \"%s\".\n", __FUNCTION__, member.s); + if (g_jsonErrorCallback) + g_jsonErrorCallback("%s: unable to find field \"%s\".\n", __FUNCTION__, member.s); + return false; } @@ -275,8 +277,10 @@ inline bool JSON_GetRequired(const T& data, typename T::StringRefType member, return true; } - g_jsonErrorCallback("%s: field \"%s\" is of type %s, but accessor expected type %s.\n", - __FUNCTION__, member.s, JSON_TypeToString(it->value), JSON_TypeToString(type)); + if (g_jsonErrorCallback) + g_jsonErrorCallback("%s: field \"%s\" is of type %s, but accessor expected type %s.\n", + __FUNCTION__, member.s, JSON_TypeToString(it->value), JSON_TypeToString(type)); + return false; } @@ -462,7 +466,9 @@ inline bool JSON_ParseNumberRequired(const T& data, typename T::StringRefType me return true; } - g_jsonErrorCallback("%s: unable to parse field \"%s\".\n", __FUNCTION__, member.s); + if (g_jsonErrorCallback) + g_jsonErrorCallback("%s: unable to parse field \"%s\".\n", __FUNCTION__, member.s); + return false; } From cc20c48ce2cd9cac20a1826fab895172e20f9192 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Fri, 5 Dec 2025 22:14:18 +0100 Subject: [PATCH 19/23] Implement basic string table For particle effect dictionary. --- src/pch.h | 1 + src/public/symboltable.h | 179 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 src/public/symboltable.h diff --git a/src/pch.h b/src/pch.h index 46b274ad..1546a91d 100644 --- a/src/pch.h +++ b/src/pch.h @@ -10,6 +10,7 @@ #include //#include #include +#include #include #include #include diff --git a/src/public/symboltable.h b/src/public/symboltable.h new file mode 100644 index 00000000..5c813036 --- /dev/null +++ b/src/public/symboltable.h @@ -0,0 +1,179 @@ +#pragma once +#include "utils/logger.h" + +class CStringPool +{ +public: + CStringPool() = default; + + const char* Add(std::string_view str) + { + if (str.empty()) + return ""; + + m_pool.emplace_back(str); + m_totalStringBytes += str.size() +1; // +1 for null terminator. + + return m_pool.back().c_str(); + } + + void AppendRetainedStringSize(const size_t len) + { + m_numUnstaleStrings++; + assert(m_numUnstaleStrings <= m_pool.size()); + m_retainedStringBytes += len +1; + assert(m_retainedStringBytes <= m_totalStringBytes); + } + + inline size_t StringBytesTotal() const noexcept { return m_totalStringBytes; } + inline size_t StringBytesDiscarded() const noexcept { return m_totalStringBytes - m_retainedStringBytes; } + inline size_t StringBytesRetained() const noexcept { return m_retainedStringBytes; } + + inline size_t NumStringsTotal() const noexcept { return m_pool.size(); } + inline size_t NumStringsDiscarded() const noexcept { return m_pool.size() - m_retainedStringBytes; } + inline size_t NumStringsRetained() const noexcept { return m_retainedStringBytes; } + +private: + std::deque m_pool; + size_t m_numUnstaleStrings = 0; + size_t m_totalStringBytes = 0; + size_t m_retainedStringBytes = 0; +}; + +template +class CSymbolTable +{ +public: + using SymbolId_t = uint32_t; + static constexpr SymbolId_t npos = (std::numeric_limits::max)(); + + explicit CSymbolTable(const size_t poolReserve = 1024) + { + m_symbols.reserve(poolReserve); + if constexpr (!CaseInsensitive) + m_lookupSensitive.reserve(poolReserve); + else + m_lookupInsensitive.reserve(poolReserve); + } + + SymbolId_t AddString(const std::string_view& str) + { + auto& lookup = getLookup(); + if (auto it = lookup.find(str); it != lookup.end()) + return it->second; + + if (m_symbols.size() >= static_cast(npos)) + Error("CSymbolTable: SymbolId overflow"); + + const char* stored = m_pool.Add(str); + SymbolId_t id = static_cast(m_symbols.size()); + m_symbols.emplace_back(stored, str.size()); + lookup.emplace(m_symbols.back(), id); + return id; + } + + std::string_view GetString(const SymbolId_t id) const + { + if (id == npos || static_cast(id) >= m_symbols.size()) + Error("CSymbolTable::GetString: invalid ID"); + + return m_symbols[id]; + } + + SymbolId_t Find(const std::string_view& str) const noexcept + { + const auto& lookup = getLookup(); + if (auto it = lookup.find(str); it != lookup.end()) + return it->second; + return npos; + } + + void MarkAsDiscarded(const SymbolId_t id) noexcept + { + if (id == npos || static_cast(id) >= m_symbols.size()) + return; + + if (m_discardSet.find(id) != m_discardSet.end()) + return; + + std::string_view& str = m_symbols[id]; + + if (!str.empty()) + m_pool.AppendRetainedStringSize(str.size()); + + m_discardSet.insert(id); + } + + bool IsDiscarded(const SymbolId_t id) const noexcept + { + return m_discardSet.find(id) != m_discardSet.end(); + } + + inline size_t StringBytesTotal() const noexcept { return m_pool.StringBytesTotal(); } + inline size_t StringBytesDiscarded() const noexcept { return m_pool.StringBytesDiscarded(); } + inline size_t StringBytesRetained() const noexcept { return m_pool.StringBytesRetained(); } + + inline size_t NumStringsTotal() const noexcept { return m_pool.NumStringsTotal(); } + inline size_t NumStringsDiscarded() const noexcept { return m_pool.NumStringsDiscarded(); } + inline size_t NumStringsRetained() const noexcept { return m_pool.NumStringsRetained(); } + +private: + struct CaseInsensitiveHash_s + { + using is_transparent = void; + size_t operator()(std::string_view s) const noexcept + { + uint64_t hash = 1469598103934665603ULL; + for (unsigned char c : s) + { + hash ^= static_cast(std::tolower(c)); + hash *= 1099511628211ULL; + } + return static_cast(hash); + } + }; + + struct CaseInsensitiveEqual_s + { + using is_transparent = void; + bool operator()(std::string_view a, std::string_view b) const noexcept + { + if (a.size() != b.size()) + return false; + + for (size_t i = 0; i < a.size(); ++i) + { + if (std::tolower(static_cast(a[i])) != + std::tolower(static_cast(b[i]))) + return false; + } + + return true; + } + }; + + using LookupSensitive = std::unordered_map; + using LookupInsensitive = std::unordered_map; + + auto& GetLookup() + { + if constexpr (CaseInsensitive) + return m_lookupInsensitive; + else + return m_lookupSensitive; + } + const auto& GetLookup() const + { + if constexpr (CaseInsensitive) + return m_lookupInsensitive; + else + return m_lookupSensitive; + } + + CStringPool m_pool; + std::vector m_symbols; + std::unordered_set m_discardSet; + LookupSensitive m_lookupSensitive; + LookupInsensitive m_lookupInsensitive; +}; From 4450f4ac780c1696a12d1653ecec8187cbaaa192 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Fri, 5 Dec 2025 22:32:54 +0100 Subject: [PATCH 20/23] Use correct sign for DoRead()/DoWrite() validation --- src/utils/binaryio.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/utils/binaryio.cpp b/src/utils/binaryio.cpp index 4d6c7bef..c3338775 100644 --- a/src/utils/binaryio.cpp +++ b/src/utils/binaryio.cpp @@ -285,19 +285,20 @@ bool BinaryIO::DoRead(char* const outBuf, const size_t readCount, size_t* const if (!IsReadable()) return false; + const std::streamoff signedReadCount = static_cast(readCount); const std::streamoff actualReadCount = m_stream.rdbuf()->sgetn(outBuf, readCount); if (outReadCount) *outReadCount = static_cast(actualReadCount); - const bool wasSuccesful = actualReadCount == readCount; + const bool wasSuccesful = actualReadCount == signedReadCount; if (!wasSuccesful) { if (g_iosmErrorCallback) { - g_iosmErrorCallback("%s: short read on file \"%s\"; actualReadCount( %zd ) != readCount( %zu ) @ TellGet( %zi )!\n", - __FUNCTION__, m_name.c_str(), actualReadCount, readCount, TellGet()); + g_iosmErrorCallback("%s: short read on file \"%s\"; actualReadCount( %zd ) != signedReadCount( %zd ) @ TellGet( %zi )!\n", + __FUNCTION__, m_name.c_str(), actualReadCount, signedReadCount, TellGet()); } } @@ -312,21 +313,22 @@ bool BinaryIO::DoWrite(const char* const inBuf, const size_t writeCount, size_t* if (!IsWritable()) return false; - const std::streamoff actualWriteCount = m_stream.rdbuf()->sputn(inBuf, writeCount); + const std::streamoff signedWriteCount = static_cast(writeCount); + const std::streamoff actualWriteCount = m_stream.rdbuf()->sputn(inBuf, signedWriteCount); PostWriteUpdate(actualWriteCount); if (outWriteCount) *outWriteCount = actualWriteCount; - const bool wasSuccesful = actualWriteCount == writeCount; + const bool wasSuccesful = actualWriteCount == signedWriteCount; if (!wasSuccesful) { if (g_iosmErrorCallback) { - g_iosmErrorCallback("%s: short write on file \"%s\"; actualWriteCount( %zd ) != writeCount( %zu ) @ TellPut( %zi )!\n", - __FUNCTION__, m_name.c_str(), actualWriteCount, writeCount, TellPut()); + g_iosmErrorCallback("%s: short write on file \"%s\"; actualWriteCount( %zd ) != signedWriteCount( %zd ) @ TellPut( %zi )!\n", + __FUNCTION__, m_name.c_str(), actualWriteCount, signedWriteCount, TellPut()); } } From 1ac70c9186adc546c936daae36f1895daa310888 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Fri, 5 Dec 2025 22:34:07 +0100 Subject: [PATCH 21/23] Implement tokenizer for BinaryIO --- src/utils/binaryio.cpp | 111 +++++++++++++++++++++++++++++++++++++++++ src/utils/binaryio.h | 10 ++++ 2 files changed, 121 insertions(+) diff --git a/src/utils/binaryio.cpp b/src/utils/binaryio.cpp index c3338775..08fa9858 100644 --- a/src/utils/binaryio.cpp +++ b/src/utils/binaryio.cpp @@ -248,6 +248,117 @@ bool BinaryIO::ReadString(char* const buf, const size_t len, size_t* const outWr return fullRead; // if false, string too long and result is truncated. } +//----------------------------------------------------------------------------- +// Eat first occurring white space from current position +//----------------------------------------------------------------------------- +void BinaryIO::EatWhiteSpace() +{ + char c; + while (Get(c)) + { + if (!isspace((unsigned char)c)) + { + SeekGet(TellGet() - 1); + break; + } + } +} + +//----------------------------------------------------------------------------- +// Parses a token between 2 delimiters into provided scratch buffer +//----------------------------------------------------------------------------- +bool BinaryIO::ParseToken(char* scratch, const char* startDelim, const char* endDelim, size_t maxLen) +{ + if (!endDelim || !*endDelim || maxLen == 0) + return false; + + char emptyBuf = '\0'; + if (!startDelim) + startDelim = &emptyBuf; + + const size_t endLen = strlen(endDelim); + const size_t startPos = TellGet(); + + EatWhiteSpace(); // Leading whitespace. + + for (size_t i = 0; startDelim[i]; ++i) + { + char c = startDelim[i]; + if (!isspace((unsigned char)c)) + { + char got; + if (!Get(got)) + { + SeekGet(startPos); + scratch[0] = '\0'; + return false; + } + if (tolower(got) != tolower(c)) + { + SeekGet(startPos); + scratch[0] = '\0'; + return false; + } + } + else + { + EatWhiteSpace(); + } + } + + EatWhiteSpace(); // After delim. + const size_t tokenStart = TellGet(); + + size_t endMatchPos = 0; + while (true) + { + char c; + if (!Get(c)) + { + SeekGet(startPos); + scratch[0] = '\0'; + return false; + } + + if (c == endDelim[endMatchPos]) + { + ++endMatchPos; + if (endMatchPos == endLen) + break; + } + else + { + endMatchPos = 0; + } + } + + const size_t tokenEnd = TellGet() - endLen; + size_t tokenLen = tokenEnd - tokenStart; + + if (tokenLen >= maxLen) + tokenLen = maxLen - 1; + + // Copy token. + SeekGet(tokenStart); + for (size_t i = 0; i < tokenLen; ++i) + { + if (!Get(scratch[i])) + { + SeekGet(startPos); + scratch[0] = '\0'; + return false; + } + } + + while (tokenLen > 0 && isspace((unsigned char)scratch[tokenLen - 1])) + --tokenLen; + + scratch[tokenLen] = '\0'; + + SeekGet(tokenEnd + endLen); + return true; +} + // limit number of io calls and allocations by just using this static buffer // for padding out the stream. static constexpr size_t PAD_BUF_SIZE = 4096; diff --git a/src/utils/binaryio.h b/src/utils/binaryio.h index 6f166df4..402c24d6 100644 --- a/src/utils/binaryio.h +++ b/src/utils/binaryio.h @@ -75,6 +75,16 @@ class BinaryIO bool ReadString(std::string& svOut); bool ReadString(char* const pBuf, const size_t nLen, size_t* const outWriteCount = nullptr); + template + inline bool Get(T& out) + { + m_stream.get(out); + return m_stream.good(); + } + + void EatWhiteSpace(); + bool ParseToken(char* const scratch, const char* const startDelim, const char* const endDelim, const size_t maxLen); + //----------------------------------------------------------------------------- // Writes any value to the file with specified size //----------------------------------------------------------------------------- From 1dd5bae210b22a254f0d68814aaaf377194abd81 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Fri, 5 Dec 2025 22:32:04 +0100 Subject: [PATCH 22/23] Implement DMX parser Still work in progress but its mostly done (should implement everything particle effects require). --- src/RePak.vcxproj | 7 + src/RePak.vcxproj.filters | 21 ++ src/math/color.h | 29 +- src/math/vector.h | 46 ++++ src/math/vmatrix.h | 6 + src/public/dmarray.h | 49 ++++ src/public/dmattribute.h | 347 ++++++++++++++++++++++++ src/public/dmelement.h | 31 +++ src/public/dmexchange.h | 43 +++ src/utils/DmxTools.cpp | 551 ++++++++++++++++++++++++++++++++++++++ src/utils/DmxTools.h | 6 + src/utils/dmarray.cpp | 3 + 12 files changed, 1136 insertions(+), 3 deletions(-) create mode 100644 src/math/vmatrix.h create mode 100644 src/public/dmarray.h create mode 100644 src/public/dmattribute.h create mode 100644 src/public/dmelement.h create mode 100644 src/public/dmexchange.h create mode 100644 src/utils/DmxTools.cpp create mode 100644 src/utils/DmxTools.h create mode 100644 src/utils/dmarray.cpp diff --git a/src/RePak.vcxproj b/src/RePak.vcxproj index 0d301f90..584a9f7f 100644 --- a/src/RePak.vcxproj +++ b/src/RePak.vcxproj @@ -161,6 +161,8 @@ NotUsing + + @@ -186,9 +188,13 @@ + + + + @@ -285,6 +291,7 @@ + diff --git a/src/RePak.vcxproj.filters b/src/RePak.vcxproj.filters index d068895d..bd5fc94c 100644 --- a/src/RePak.vcxproj.filters +++ b/src/RePak.vcxproj.filters @@ -250,6 +250,12 @@ assets + + utils + + + utils + @@ -609,6 +615,21 @@ public + + utils + + + public + + + public + + + public + + + math + diff --git a/src/math/color.h b/src/math/color.h index b5b28a19..666f7786 100644 --- a/src/math/color.h +++ b/src/math/color.h @@ -1,13 +1,36 @@ #pragma once -struct Color +struct ColorB { - Color() + ColorB() + { + this->Set(0, 0, 0, 0); + }; + + ColorB(uint8_t r, uint8_t g, uint8_t b, uint8_t a) + { + this->Set(r, g, b, a); + } + + void Set(uint8_t newR, uint8_t newG, uint8_t newB, uint8_t newA) + { + this->r = newR; + this->g = newG; + this->b = newB; + this->a = newA; + } + + uint8_t r, g, b, a; +}; + +struct ColorF +{ + ColorF() { this->Set(0.0f, 0.0f, 0.0f, 0.0f); }; - Color(float r, float g, float b, float a) + ColorF(float r, float g, float b, float a) { this->Set(r, g, b, a); } diff --git a/src/math/vector.h b/src/math/vector.h index 15dbb5dc..1b498ab2 100644 --- a/src/math/vector.h +++ b/src/math/vector.h @@ -1,5 +1,28 @@ #pragma once +struct Vector4 +{ + Vector4() + { + this->Set(0.0f, 0.0f, 0.0f, 0.0f); + } + + Vector4(float x, float y, float z, float w) + { + this->Set(x, y, z, w); + } + + void Set(float newX, float newY, float newZ, float newW) + { + this->x = newX; + this->y = newY; + this->z = newZ; + this->w = newW; + } + + float x, y, z, w; +}; + struct Vector3 { Vector3() @@ -64,3 +87,26 @@ struct QAngle float x, y, z; }; + +struct Quaternion +{ + Quaternion() + { + this->Set(0.0f, 0.0f, 0.0f, 0.0f); + } + + Quaternion(float x, float y, float z, float w) + { + this->Set(x, y, z, w); + } + + void Set(float newX, float newY, float newZ, float newW) + { + this->x = newX; + this->y = newY; + this->z = newZ; + this->w = newW; + } + + float x, y, z, w; +}; diff --git a/src/math/vmatrix.h b/src/math/vmatrix.h new file mode 100644 index 00000000..5bf0df1f --- /dev/null +++ b/src/math/vmatrix.h @@ -0,0 +1,6 @@ +#pragma once + +struct VMatrix +{ + float m[4][4]; +}; diff --git a/src/public/dmarray.h b/src/public/dmarray.h new file mode 100644 index 00000000..90348bed --- /dev/null +++ b/src/public/dmarray.h @@ -0,0 +1,49 @@ +#pragma once + +struct DmAttribute_s; + +//template +//struct DmArrayValue_s +//{ +// std::vector data; +//}; +// +//template +//class DmArray +//{ +//public: +// DmArray(DmAttribute_s* attr) +// : m_attr(attr) +// { +// assert(attr != nullptr); +// //assert(attr->type == GetExpectedType()); +// if (!attr->value.ptr) +// { +// // Allocate. +// attr->value.ptr = new DmArrayValue_s(); +// } +// m_vec = static_cast*>(attr->value.ptr); +// } +// +// void AddToTail(const T& val) { m_vec->data.push_back(val); } +// void RemoveAll() { m_vec->data.clear(); } +// void EnsureCapacity(const size_t count) { m_vec->data.reserve(count); } +// size_t Count() const { return m_vec->data.size(); } +// +// T& operator[](size_t i) { return m_vec->data[i]; } +// const T& operator[](size_t i) const { return m_vec->data[i]; } +// +// auto begin() { return m_vec->data.begin(); } +// auto end() { return m_vec->data.end(); } +// +//private: +// DmAttribute_s* m_attr; +// DmArrayValue_s* m_vec; +// +// ///static DmAttributeType_e GetExpectedType(); +//}; + +//template<> inline DmAttributeType_e DmArray::GetExpectedType() { return AT_INT_ARRAY; } +//template<> inline DmAttributeType_e DmArray::GetExpectedType() { return AT_FLOAT_ARRAY; } +//template<> inline DmAttributeType_e DmArray::GetExpectedType() { return AT_STRING_ARRAY; } + diff --git a/src/public/dmattribute.h b/src/public/dmattribute.h new file mode 100644 index 00000000..55b92fa0 --- /dev/null +++ b/src/public/dmattribute.h @@ -0,0 +1,347 @@ +#pragma once +#include "math/vector.h" +#include "math/color.h" +#include "math/vmatrix.h" +#include "dmarray.h" + +// Structs in order for compile-time classification +// to work inside Dma_GetTypeForPod(). +struct DmeHandle_t { int h; }; +struct DmeSymbol_t { int s; }; + +struct DmeTime_s +{ + DmeTime_s() : tms(INT_MIN) {} + explicit DmeTime_s(int tms) : tms(tms) {} + explicit DmeTime_s(float sec) : tms(RoundSecondsToTMS(sec)) {} + explicit DmeTime_s(double sec) : tms(RoundSecondsToTMS(sec)) {} + + static int RoundSecondsToTMS(float sec) { return (int)floor(10000.0f * sec + 0.5f); }; + static int RoundSecondsToTMS(double sec) { return (int)floor(10000.0f * sec + 0.5f); }; + + int tms; +}; + +union DmAttributeValue_u +{ + DmeHandle_t element; + int intVal; + float floatVal; + bool boolVal; + DmeSymbol_t stringSym; + void* ptr; + DmeTime_s timeVal; + ColorB color; + Vector2 vec2; + Vector3 vec3; + Vector4 vec4; + QAngle ang; + Quaternion quat; + VMatrix mat; + + DmAttributeValue_u() { memset(this, 0, sizeof(*this)); } +}; + +enum DmAttributeType_e : int8_t +{ + AT_UNKNOWN = 0, + + AT_FIRST_VALUE_TYPE, + + AT_ELEMENT = AT_FIRST_VALUE_TYPE, + AT_INT, + AT_FLOAT, + AT_BOOL, + AT_STRING, + AT_VOID, + AT_TIME, + AT_COLOR, + AT_VECTOR2, + AT_VECTOR3, + AT_VECTOR4, + AT_QANGLE, + AT_QUATERNION, + AT_VMATRIX, + + AT_FIRST_ARRAY_TYPE, + + AT_ELEMENT_ARRAY = AT_FIRST_ARRAY_TYPE, + AT_INT_ARRAY, + AT_FLOAT_ARRAY, + AT_BOOL_ARRAY, + AT_STRING_ARRAY, + AT_VOID_ARRAY, + AT_TIME_ARRAY, + AT_COLOR_ARRAY, + AT_VECTOR2_ARRAY, + AT_VECTOR3_ARRAY, + AT_VECTOR4_ARRAY, + AT_QANGLE_ARRAY, + AT_QUATERNION_ARRAY, + AT_VMATRIX_ARRAY, + + AT_TYPE_COUNT, + AT_TYPE_INVALID, +}; + +inline const char* Dma_TypeToString(const DmAttributeType_e type) +{ + switch (type) + { + case AT_UNKNOWN: return "unknown"; + case AT_ELEMENT: return "element"; + case AT_INT: return "int"; + case AT_FLOAT: return "float"; + case AT_BOOL: return "bool"; + case AT_STRING: return "string"; + case AT_VOID: return "void"; + case AT_TIME: return "time"; + case AT_COLOR: return "color"; + case AT_VECTOR2: return "vector2"; + case AT_VECTOR3: return "vector3"; + case AT_VECTOR4: return "vector4"; + case AT_QANGLE: return "qangle"; + case AT_QUATERNION: return "quaternion"; + case AT_VMATRIX: return "vmatrix"; + case AT_ELEMENT_ARRAY: return "element array"; + case AT_INT_ARRAY: return "int array"; + case AT_FLOAT_ARRAY: return "float array"; + case AT_BOOL_ARRAY: return "bool array"; + case AT_STRING_ARRAY: return "string array"; + case AT_VOID_ARRAY: return "void array"; + case AT_TIME_ARRAY: return "time array"; + case AT_COLOR_ARRAY: return "color array"; + case AT_VECTOR2_ARRAY: return "vector2 array"; + case AT_VECTOR3_ARRAY: return "vector3 array"; + case AT_VECTOR4_ARRAY: return "vector4 array"; + case AT_QANGLE_ARRAY: return "qangle array"; + case AT_QUATERNION_ARRAY: return "quaternion array"; + case AT_VMATRIX_ARRAY: return "vmatrix array"; + default: return "invalid"; + } +} + +template +constexpr DmAttributeType_e Dma_GetTypeForPod() +{ + if constexpr (std::is_same_v) + return AT_ELEMENT; + else if constexpr (std::is_same_v) + return AT_ELEMENT_ARRAY; + else if constexpr (std::is_same_v) + return AT_INT; + else if constexpr (std::is_same_v) + return AT_INT_ARRAY; + else if constexpr (std::is_same_v) + return AT_FLOAT; + else if constexpr (std::is_same_v) + return AT_FLOAT_ARRAY; + else if constexpr (std::is_same_v) + return AT_BOOL; + else if constexpr (std::is_same_v) + return AT_BOOL_ARRAY; + else if constexpr (std::is_same_v) + return AT_VOID; + else if constexpr (std::is_same_v) + return AT_VOID_ARRAY; + else if constexpr (std::is_same_v) + return AT_STRING; + else if constexpr (std::is_same_v) + return AT_STRING_ARRAY; + else if constexpr (std::is_same_v) + return AT_TIME; + else if constexpr (std::is_same_v) + return AT_TIME_ARRAY; + else if constexpr (std::is_same_v) + return AT_COLOR; + else if constexpr (std::is_same_v) + return AT_COLOR_ARRAY; + else if constexpr (std::is_same_v) + return AT_VECTOR2; + else if constexpr (std::is_same_v) + return AT_VECTOR2_ARRAY; + else if constexpr (std::is_same_v) + return AT_VECTOR3; + else if constexpr (std::is_same_v) + return AT_VECTOR3_ARRAY; + else if constexpr (std::is_same_v) + return AT_VECTOR4; + else if constexpr (std::is_same_v) + return AT_VECTOR4_ARRAY; + else if constexpr (std::is_same_v) + return AT_QANGLE; + else if constexpr (std::is_same_v) + return AT_QANGLE_ARRAY; + else if constexpr (std::is_same_v) + return AT_QUATERNION; + else if constexpr (std::is_same_v) + return AT_QUATERNION_ARRAY; + else if constexpr (std::is_same_v) + return AT_VMATRIX; + else if constexpr (std::is_same_v) + return AT_VMATRIX_ARRAY; + else + static_assert(std::is_same_v, "Cannot classify data type; unsupported."); +} + +struct DmAttribute_s +{ + DmAttributeValue_u value; + DmAttributeType_e type = AT_UNKNOWN; + DmeSymbol_t name{ -1 }; +}; + +template +T Dma_GetValue(const DmAttribute_s& attrib) +{ + constexpr DmAttributeType_e t = Dma_GetTypeForPod(); + +#ifndef NDEBUG + if (attrib.type != t) + Error("%s: requested type does not match stored type", __FUNCTION__); +#endif + if constexpr (t == AT_ELEMENT) + return attrib.value.element; + else if constexpr (t == AT_INT) + return attrib.value.intVal; + else if constexpr (t == AT_FLOAT) + return attrib.value.floatVal; + else if constexpr (t == AT_BOOL) + return attrib.value.boolVal; + else if constexpr (t == AT_STRING) + return attrib.value.stringSym; + else if constexpr (t == AT_VOID) + return attrib.value.ptr; + else if constexpr (t == AT_TIME) + return attrib.value.timeVal; + else if constexpr (t == AT_COLOR) + return attrib.value.color; + else if constexpr (t == AT_VECTOR2) + return attrib.value.vec2; + else if constexpr (t == AT_VECTOR3) + return attrib.value.vec3; + else if constexpr (t == AT_VECTOR4) + return attrib.value.vec4; + else if constexpr (t == AT_QANGLE) + return attrib.value.ang; + else if constexpr (t == AT_QUATERNION) + return attrib.value.quat; + else if constexpr (t == AT_VMATRIX) + return attrib.value.mat; + else + static_assert(std::is_same_v, "Unsupported attribute type"); +} + +template +void Dma_SetValue(DmAttribute_s& attrib, const T& v) +{ + constexpr DmAttributeType_e t = Dma_GetTypeForPod(); + attrib.type = t; + + if constexpr (t == AT_ELEMENT) + attrib.value.element = v; + else if constexpr (t == AT_INT) + attrib.value.intVal = v; + else if constexpr (t == AT_FLOAT) + attrib.value.floatVal = v; + else if constexpr (t == AT_BOOL) + attrib.value.boolVal = v; + else if constexpr (t == AT_STRING) + attrib.value.stringSym = v; + else if constexpr (t == AT_VOID) + attrib.value.ptr = v; + else if constexpr (t == AT_TIME) + attrib.value.timeVal = v; + else if constexpr (t == AT_COLOR) + attrib.value.color = v; + else if constexpr (t == AT_VECTOR2) + attrib.value.vec2 = v; + else if constexpr (t == AT_VECTOR3) + attrib.value.vec3 = v; + else if constexpr (t == AT_VECTOR4) + attrib.value.vec4 = v; + else if constexpr (t == AT_QANGLE) + attrib.value.ang = v; + else if constexpr (t == AT_QUATERNION) + attrib.value.quat = v; + else if constexpr (t == AT_VMATRIX) + attrib.value.mat = v; + else + printf("Unsupported attribute type!\n"); + //static_assert(std::is_same_v, "Unsupported attribute type!"); +} + +inline int Dma_NumComponents(const DmAttributeType_e type) +{ + switch (type) + { + case AT_BOOL: + case AT_INT: + case AT_FLOAT: + case AT_TIME: + return 1; + + case AT_VECTOR2: + return 2; + + case AT_VECTOR3: + case AT_QANGLE: + return 3; + + case AT_COLOR: + case AT_VECTOR4: + case AT_QUATERNION: + return 4; + + case AT_VMATRIX: + return 16; + + case AT_ELEMENT: + case AT_STRING: + case AT_VOID: + default: + return 0; + } +} + +template +struct DmArrayValue_s +{ + std::vector data; +}; + +template +class DmArray +{ +public: + DmArray(DmAttribute_s* attr) + : m_attr(attr) + { + assert(attr != nullptr); + //assert(attr->type == GetExpectedType()); + if (!attr->value.ptr) + { + // Allocate. + attr->value.ptr = new DmArrayValue_s(); + } + attr->type = Dma_GetTypeForPod(); + m_vec = static_cast*>(attr->value.ptr); + } + + void AddToTail(const T& val) { m_vec->data.push_back(val); } + void RemoveAll() { m_vec->data.clear(); } + void EnsureCapacity(const size_t count) { m_vec->data.reserve(count); } + size_t Count() const { return m_vec->data.size(); } + + T& operator[](size_t i) { return m_vec->data[i]; } + const T& operator[](size_t i) const { return m_vec->data[i]; } + + auto begin() { return m_vec->data.begin(); } + auto end() { return m_vec->data.end(); } + +private: + DmAttribute_s* m_attr; + DmArrayValue_s* m_vec; + + ///static DmAttributeType_e GetExpectedType(); +}; diff --git a/src/public/dmelement.h b/src/public/dmelement.h new file mode 100644 index 00000000..1d46eb98 --- /dev/null +++ b/src/public/dmelement.h @@ -0,0 +1,31 @@ +#pragma once +#include "dmattribute.h" + +#define DM_ELEMENT_NULL -1 +#define DM_ELEMENT_EXTERNAL -2 + +struct DmObjectId_s +{ + unsigned char value[16]; +}; + +using DmObjectIdString_s = char[37]; // 37 is max a string representation of the guid will utilize. + +struct DmElement_s +{ + DmObjectId_s id; + DmeSymbol_t name; + DmeSymbol_t type; + std::vector attr; +}; + +inline const DmAttribute_s* Dme_FindAttribute(const DmElement_s& elem, const DmeSymbol_t name) +{ + for (const DmAttribute_s& at : elem.attr) + { + if (at.name.s == name.s) + return &at; + } + + return nullptr; +} diff --git a/src/public/dmexchange.h b/src/public/dmexchange.h new file mode 100644 index 00000000..8cfa49e2 --- /dev/null +++ b/src/public/dmexchange.h @@ -0,0 +1,43 @@ +#pragma once +#include "dmelement.h" +#include "symboltable.h" + +#define DMX_VERSION_STARTING_TOKEN "" +#define DMX_HEADER_COMPONENT_COUNT 4 // + +struct DmxHeader_s +{ + enum + { + MAX_FORMAT_NAME_LENGTH = 64, + MAX_HEADER_LENGTH = 40 + 2 * MAX_FORMAT_NAME_LENGTH, + }; + + DmxHeader_s() + : encodingVersion(-1) + , formatVersion(-1) + { + encodingName[0] = '\0'; + formatName[0] = '\0'; + } + + char encodingName[MAX_FORMAT_NAME_LENGTH]; + int encodingVersion; + char formatName[MAX_FORMAT_NAME_LENGTH]; + int formatVersion; +}; + +enum DmFileId_e +{ + DMFILEID_INVALID = 0xFFFFFFFF +}; + +typedef CSymbolTable DmSymbolTable; +typedef std::vector DmElementList; + +struct DmContext_s +{ + DmSymbolTable symbolTable; + DmElementList elementList; +}; diff --git a/src/utils/DmxTools.cpp b/src/utils/DmxTools.cpp new file mode 100644 index 00000000..c6be4ce9 --- /dev/null +++ b/src/utils/DmxTools.cpp @@ -0,0 +1,551 @@ +#include "pch.h" +#include "DmxTools.h" +#include "public/dmelement.h" +#include "public/dmattribute.h" +#include "public/symboltable.h" + +bool Dmx_ParseHdr(BinaryIO& bio, DmxHeader_s& hdr) +{ + char hdrStr[DmxHeader_s::MAX_HEADER_LENGTH]; + + if (!bio.ParseToken(hdrStr, DMX_VERSION_STARTING_TOKEN, DMX_VERSION_ENDING_TOKEN, sizeof(hdrStr))) + { + Error("%s: failed to tokenize DMX header! (format should start with \"" DMX_VERSION_STARTING_TOKEN "\" and end with \"" DMX_VERSION_ENDING_TOKEN "\")", __FUNCTION__); + return false; + } + + const int scanCount = sscanf_s(hdrStr, "encoding %s %d format %s %d\n", + hdr.encodingName, (uint32_t)sizeof(hdr.encodingName), &hdr.encodingVersion, + hdr.formatName, (uint32_t)sizeof(hdr.formatName), &hdr.formatVersion); + + if (scanCount != DMX_HEADER_COMPONENT_COUNT) + { + Error("%s: failed to parse DMX header! (expected %d components, got %d)", __FUNCTION__, DMX_HEADER_COMPONENT_COUNT, scanCount); + return false; + } + + char c; // Consume the rest of the text so the cursor points past the header. + while (bio.Get(c) && c != '\0'); + + return true; +} + +static bool Dmx_GetStringTable(DmContext_s& ctx, BinaryIO& bio, const int numStrings) +{ + char stack[2048]; + int iter; + + for (iter = 0; iter < numStrings; iter++) + { + size_t strLen; + + if (!bio.ReadString(stack, sizeof(stack), &strLen)) + break; + + ctx.symbolTable.AddString({ stack, strLen }); + } + + if (iter != numStrings) + { + Error("%s: failed to parse DMX string table! (expected %d strings, got %d)", __FUNCTION__, numStrings, iter); + return false; + } + + return true; +} + +//static const char* Dme_GetString(BinaryIO& bio, const DmSymbolTable& table) +//{ +// const int symHnd = bio.Read(); +// return table.GetString(symHnd).data(); +//} + +static bool Dme_ReadElem(BinaryIO& bio, DmElement_s& elem) +{ + elem.name = bio.Read(); + elem.type = bio.Read(); + + return bio.Read(elem.id); +} + +char* Dme_GuidToString(const DmObjectId_s& guid, DmObjectIdString_s out) +{ + static constexpr char hexDigits[] = "0123456789abcdef"; + + const uint8_t* v = guid.value; + int pos = 0; + + for (int i = 0; i < 16; ++i) + { + out[pos++] = hexDigits[v[i] >> 4]; + out[pos++] = hexDigits[v[i] & 0xF]; + + if (i == 3 || i == 5 || i == 7 || i == 9) + out[pos++] = '-'; + } + + out[pos] = '\0'; + return out; +} + +static bool Dmx_DeserializeElements(DmContext_s& ctx, BinaryIO& bio) +{ + const int numElems = bio.Read(); + + if (!numElems) + return true; // valve does this too. + + ctx.elementList.resize(numElems); + + for (int iter = 0; iter < numElems; iter++) + { + if (!Dme_ReadElem(bio, ctx.elementList[iter])) + return false; + } + + //printf("num elems: %d\n", numElems); + return true; +} + +static DmeSymbol_t Dma_ReadSymHnd(BinaryIO& bio) +{ + return bio.Read(); +} + +static DmeHandle_t Dma_ReadElemIdx(BinaryIO& bio) +{ + const DmeHandle_t hnd = bio.Read(); + + if (hnd.h == DM_ELEMENT_EXTERNAL) + Error("%s: external element references are not supported!\n", __FUNCTION__); + + return hnd; +} + +template +static inline T Dma_DoRead(BinaryIO& bio) +{ + return bio.Read(); +} + +template <> +inline DmeTime_s Dma_DoRead(BinaryIO& bio) +{ + return DmeTime_s(Dma_DoRead(bio)); +} + +template +static void Dma_Deserialize(BinaryIO& bio, DmAttribute_s& attrib) +{ + Dma_SetValue(attrib, Dma_DoRead(bio)); +} + +static DmAttributeType_e s_lastType; + +template +static void Dma_ReadArray(BinaryIO& bio, DmAttribute_s& attrib) +{ + DmArray arr(&attrib); + const int numElems = bio.Read(); + + if (!numElems) + return; + + arr.RemoveAll(); + arr.EnsureCapacity(numElems); + + for (int i = 0; i < numElems; i++) + { + const T rd = Dma_DoRead(bio); + arr.AddToTail(rd); + } +} + +static void Dma_DeserializeAny(BinaryIO& bio, DmAttribute_s& attrib, const DmAttributeType_e type) +{ + switch (type) + { + case AT_INT: + Dma_Deserialize(bio, attrib); + break; + case AT_FLOAT: + Dma_Deserialize(bio, attrib); + break; + case AT_BOOL: + Dma_Deserialize(bio, attrib); + break; +// case AT_STRING: + //Dma_Deserialize<>(bio, attrib); // todo +// break; +// case AT_VOID: + //Dma_Deserialize<>(bio, attrib); // todo +// break; + case AT_TIME: + Dma_Deserialize(bio, attrib); + break; + case AT_COLOR: + Dma_Deserialize(bio, attrib); + break; + case AT_VECTOR2: + Dma_Deserialize(bio, attrib); + break; + case AT_VECTOR3: + Dma_Deserialize(bio, attrib); + break; + case AT_VECTOR4: + Dma_Deserialize(bio, attrib); + break; + case AT_QANGLE: + Dma_Deserialize(bio, attrib); + break; + case AT_QUATERNION: + Dma_Deserialize(bio, attrib); + break; + case AT_VMATRIX: + Dma_Deserialize(bio, attrib); + break; + case AT_INT_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_FLOAT_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_BOOL_ARRAY: + Dma_ReadArray(bio, attrib); + break; +// case AT_STRING_ARRAY: + //Unserialize(buf, this->m_pData); +// break; +// case AT_VOID_ARRAY: + //Unserialize(buf, this->m_pData); +// break; + case AT_TIME_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_COLOR_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_VECTOR2_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_VECTOR3_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_VECTOR4_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_QANGLE_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_QUATERNION_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_VMATRIX_ARRAY: + Dma_ReadArray(bio, attrib); + break; + + default: + Error("%s: unhandled attribute type %d @ %zd!\n", __FUNCTION__, type, bio.TellGet()); + } + + s_lastType = type; +} + +static bool Dma_DeserializeAttribute(const DmContext_s& /*ctx*/, BinaryIO& bio, DmAttribute_s& attrib, const DmAttributeType_e type) +{ + switch (type) + { + case AT_ELEMENT: + Dma_SetValue(attrib, Dma_ReadElemIdx(bio)); + break; + case AT_ELEMENT_ARRAY: + Dma_ReadArray(bio, attrib); + break; + case AT_STRING: + Dma_SetValue(attrib, Dma_ReadSymHnd(bio)); + break; + case AT_STRING_ARRAY: + Dma_ReadArray(bio, attrib); + break; + + default: + Dma_DeserializeAny(bio, attrib, type); + } + + return true; +} + +static bool Dmx_DeserializeAttributes(const DmContext_s& ctx, DmElement_s& elem, BinaryIO& bio) +{ + const int numAttribs = bio.Read(); + elem.attr.resize(numAttribs); + + for (int i = 0; i < numAttribs; i++) + { + DmAttribute_s& attrib = elem.attr[i]; + attrib.name = bio.Read(); + + //printf("%s: %s\n",__FUNCTION__, ctx.symbolTable.GetString(attrib.name.s).data()); + const DmAttributeType_e type = bio.Read(); + Dma_DeserializeAttribute(ctx, bio, attrib, type); + } + + //printf("numAttribs: %d\n", numAttribs); + return true; +} + +#define PRINT_ARRAY(fmt, accessor) \ + { \ + auto* arr = static_cast*>(attr.value.ptr); \ + printf("{ "); \ + for (size_t i = 0; i < arr->data.size(); ++i) \ + { \ + printf(fmt, arr->data[i]); \ + if (i + 1 < arr->data.size()) printf(", "); \ + } \ + printf(" }"); \ + } + +static void Dma_Print(const DmContext_s& ctx, const DmAttribute_s& attr) +{ + printf("\t%s(%s) = ", ctx.symbolTable.GetString(attr.name.s).data(), Dma_TypeToString(attr.type)); + + switch (attr.type) + { + case AT_ELEMENT: + printf("#%d (--> %s)", attr.value.element.h, ctx.symbolTable.GetString(ctx.elementList[attr.value.element.h].name.s).data()); + break; + case AT_INT: + printf("%d", attr.value.intVal); + break; + case AT_FLOAT: + printf("%f", attr.value.floatVal); + break; + case AT_BOOL: + printf(attr.value.boolVal ? "true" : "false"); + break; + case AT_STRING: + printf("%s", ctx.symbolTable.GetString(attr.value.stringSym.s).data()); + break; + case AT_VOID: + printf("ptr: %p", attr.value.ptr); + break; + case AT_TIME: + printf("%d tms (%.4f sec)", attr.value.timeVal.tms, attr.value.timeVal.tms / 10000.0f); + break; + case AT_COLOR: + printf("Color(%u, %u, %u, %u)", attr.value.color.r, attr.value.color.g, + attr.value.color.b, attr.value.color.a); + break; + case AT_VECTOR2: + printf("Vector2(%.3f, %.3f)", attr.value.vec2.x, attr.value.vec2.y); + break; + case AT_VECTOR3: + printf("Vector3(%.3f, %.3f, %.3f)", + attr.value.vec3.x, attr.value.vec3.y, attr.value.vec3.z); + break; + case AT_VECTOR4: + printf("Vector4(%.3f, %.3f, %.3f, %.3f)", + attr.value.vec4.x, attr.value.vec4.y, attr.value.vec4.z, attr.value.vec4.w); + break; + case AT_QANGLE: + printf("QAngle(%.3f, %.3f, %.3f)", + attr.value.ang.x, attr.value.ang.y, attr.value.ang.z); + break; + case AT_QUATERNION: + printf("Quaternion(%.3f, %.3f, %.3f, %.3f)", + attr.value.quat.x, attr.value.quat.y, attr.value.quat.z, attr.value.quat.w); + break; + case AT_VMATRIX: + printf("VMatrix[\n"); + for (int i = 0; i < 4; ++i) + { + printf(" %.3f %.3f %.3f %.3f\n", + attr.value.mat.m[i][0], + attr.value.mat.m[i][1], + attr.value.mat.m[i][2], + attr.value.mat.m[i][3]); + } + printf("]"); + break; + case AT_ELEMENT_ARRAY: + if (attr.value.ptr) + { + auto* arr = static_cast*>(attr.value.ptr); + printf("{ "); + for (size_t i = 0; i < arr->data.size(); ++i) + { + const auto& v = arr->data[i]; + printf("#%d (--> %s)", v.h, ctx.symbolTable.GetString(ctx.elementList[v.h].name.s).data()); + if (i + 1 < arr->data.size()) printf(", "); + } + printf(" }"); + } + else printf("{}"); + break; + case AT_INT_ARRAY: + if (attr.value.ptr) { PRINT_ARRAY("%d", int()); } + else printf("{}"); + break; + case AT_FLOAT_ARRAY: + if (attr.value.ptr) { PRINT_ARRAY("%f", float()); } + else printf("{}"); + break; + case AT_BOOL_ARRAY: + if (attr.value.ptr) + { + auto* arr = static_cast*>(attr.value.ptr); + printf("{ "); + for (size_t i = 0; i < arr->data.size(); ++i) + { + printf(arr->data[i] ? "true" : "false"); + if (i + 1 < arr->data.size()) printf(", "); + } + printf(" }"); + } + else printf("{}"); + break; + case AT_VECTOR2_ARRAY: + if (attr.value.ptr) + { + auto* arr = static_cast*>(attr.value.ptr); + printf("{ "); + for (size_t i = 0; i < arr->data.size(); ++i) + { + const auto& v = arr->data[i]; + printf("(%.2f, %.2f)", v.x, v.y); + if (i + 1 < arr->data.size()) printf(", "); + } + printf(" }"); + } + else printf("{}"); + break; + case AT_VECTOR3_ARRAY: + if (attr.value.ptr) + { + auto* arr = static_cast*>(attr.value.ptr); + printf("{ "); + for (size_t i = 0; i < arr->data.size(); ++i) + { + const auto& v = arr->data[i]; + printf("(%.2f, %.2f, %.2f)", v.x, v.y, v.z); + if (i + 1 < arr->data.size()) printf(", "); + } + printf(" }"); + } + else printf("{}"); + break; + case AT_VECTOR4_ARRAY: + if (attr.value.ptr) + { + auto* arr = static_cast*>(attr.value.ptr); + printf("{ "); + for (size_t i = 0; i < arr->data.size(); ++i) + { + const auto& v = arr->data[i]; + printf("(%.2f, %.2f, %.2f, %.2f)", v.x, v.y, v.z, v.w); + if (i + 1 < arr->data.size()) printf(", "); + } + printf(" }"); + } + else printf("{}"); + break; + case AT_QANGLE_ARRAY: + if (attr.value.ptr) + { + auto* arr = static_cast*>(attr.value.ptr); + printf("{ "); + for (size_t i = 0; i < arr->data.size(); ++i) + { + const auto& v = arr->data[i]; + printf("(%.2f, %.2f, %.2f)", v.x, v.y, v.z); + if (i + 1 < arr->data.size()) printf(", "); + } + printf(" }"); + } + else printf("{}"); + break; + case AT_QUATERNION_ARRAY: + if (attr.value.ptr) + { + auto* arr = static_cast*>(attr.value.ptr); + printf("{ "); + for (size_t i = 0; i < arr->data.size(); ++i) + { + const auto& v = arr->data[i]; + printf("(%.2f, %.2f, %.2f, %.2f)", v.x, v.y, v.z, v.w); + if (i + 1 < arr->data.size()) printf(", "); + } + printf(" }"); + } + else printf("{}"); + break; + case AT_STRING_ARRAY: + if (attr.value.ptr) + { + auto* arr = static_cast*>(attr.value.ptr); + printf("{ "); + for (size_t i = 0; i < arr->data.size(); ++i) + { + printf("%s", ctx.symbolTable.GetString(arr->data[i].s).data()); + if (i + 1 < arr->data.size()) printf(", "); + } + printf(" }"); + } + else printf("{}"); + break; + + default: + printf(""); + break; + } + + printf("\n"); +} + +#undef PRINT_ARRAY + +bool Dmx_DeserializeBinary(DmContext_s& ctx, BinaryIO& bio) +{ + const int numStrings = bio.Read(); + + if (!Dmx_GetStringTable(ctx, bio, numStrings)) + return false; + + if (!Dmx_DeserializeElements(ctx, bio)) + return false; + + //size_t j = 0; + //for (auto& t : ctx.elementList) + //{ + // printf("----------------------- dmElem #%zd\n", j++); + // printf("%s(%s)\n", ctx.symbolTable.GetString(t.type.s).data(), ctx.symbolTable.GetString(t.name.s).data()); + + // DmObjectIdString_s idStr; + // Dme_GuidToString(t.id, idStr); + + // printf("{%s}\n", idStr); + //} + + for (size_t i = 0; i < ctx.elementList.size(); i++) + { + if (!Dmx_DeserializeAttributes(ctx, ctx.elementList[i], bio)) + return false; + } + + size_t elemIdx = 0; + for (auto& t : ctx.elementList) + { + printf("%s(%s) #%zu\n", ctx.symbolTable.GetString(t.type.s).data(), ctx.symbolTable.GetString(t.name.s).data(), elemIdx++); + printf("{\n"); + + for (auto& a : t.attr) + { + Dma_Print(ctx, a); + } + + printf("}\n"); + } + + return true; +} diff --git a/src/utils/DmxTools.h b/src/utils/DmxTools.h new file mode 100644 index 00000000..e6af9f79 --- /dev/null +++ b/src/utils/DmxTools.h @@ -0,0 +1,6 @@ +#pragma once +#include "binaryio.h" +#include "public/dmexchange.h" + +extern bool Dmx_ParseHdr(BinaryIO& bio, DmxHeader_s& hdr); +extern bool Dmx_DeserializeBinary(DmContext_s& ctx, BinaryIO& bio); diff --git a/src/utils/dmarray.cpp b/src/utils/dmarray.cpp new file mode 100644 index 00000000..9cc859ab --- /dev/null +++ b/src/utils/dmarray.cpp @@ -0,0 +1,3 @@ +#include "pch.h" +#include "public/dmarray.h" + From 5500a212df93423cf543e5f48966c626a2678ce4 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Fri, 5 Dec 2025 22:54:26 +0100 Subject: [PATCH 23/23] Particle effect types and foundation (WIP) Types for the base of particle effects, still WIP because 180 operators need to be reversed still before it can be properly finished. Currently only the parser works, pak writer hasn't been implemented yet (still work in progress). --- src/RePak.vcxproj | 4 + src/RePak.vcxproj.filters | 12 ++ src/assets/assets.h | 3 + src/assets/particle_effect.cpp | 98 +++++++++++++++ src/assets/particle_operator_params.cpp | 153 +++++++++++++++++++++++ src/assets/particle_operators_params.h | 42 +++++++ src/logic/pakfile.cpp | 1 + src/public/particle_effect.h | 155 ++++++++++++++++++++++++ src/public/rpak.h | 2 + 9 files changed, 470 insertions(+) create mode 100644 src/assets/particle_effect.cpp create mode 100644 src/assets/particle_operator_params.cpp create mode 100644 src/assets/particle_operators_params.h create mode 100644 src/public/particle_effect.h diff --git a/src/RePak.vcxproj b/src/RePak.vcxproj index 584a9f7f..e600bc7f 100644 --- a/src/RePak.vcxproj +++ b/src/RePak.vcxproj @@ -21,6 +21,8 @@ + + @@ -176,6 +178,7 @@ + @@ -200,6 +203,7 @@ + diff --git a/src/RePak.vcxproj.filters b/src/RePak.vcxproj.filters index bd5fc94c..b7b27f67 100644 --- a/src/RePak.vcxproj.filters +++ b/src/RePak.vcxproj.filters @@ -253,9 +253,15 @@ utils + + assets + utils + + assets + @@ -618,6 +624,9 @@ utils + + public + public @@ -630,6 +639,9 @@ math + + public + diff --git a/src/assets/assets.h b/src/assets/assets.h index 093200ca..23d4cfaa 100644 --- a/src/assets/assets.h +++ b/src/assets/assets.h @@ -9,6 +9,7 @@ #define TXLS_VERSION 1 #define UIMG_VERSION 10 #define RLCD_VERSION 0 +#define EFCT_VERSION 2 //#define DTBL_VERSION 1 #define STLT_VERSION 0 #define STGS_VERSION 1 @@ -36,6 +37,8 @@ namespace Assets void AddUIImageAsset_v10(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry); void AddLcdScreenEffect_v0(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry); + void AddParticleEffect_v2(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& /*mapEntry*/); + void AddDataTableAsset(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry); void AddSettingsLayout_v0(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry); void AddSettingsAsset_v1(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry); diff --git a/src/assets/particle_effect.cpp b/src/assets/particle_effect.cpp new file mode 100644 index 00000000..c0e5f7da --- /dev/null +++ b/src/assets/particle_effect.cpp @@ -0,0 +1,98 @@ +#include "pch.h" +#include "assets.h" +#include "particle_operators_params.h" +#include "public/particle_effect.h" +#include "utils/DmxTools.h" + +#define PARTICLE_DEFINITION_ENC "binary" +#define PARTICLE_DEFINITION_ENC_VER 5 +#define PARTICLE_DEFINITION_FMT "pcf" +#define PARTICLE_DEFINITION_FMT_VER 2 + +static void ParticleEffect_ValidateHeader(const DmxHeader_s& hdr) +{ + if (hdr.formatVersion != PARTICLE_DEFINITION_FMT_VER) + Error("Particle effect is version %d, but %d was expected!", hdr.formatVersion, PARTICLE_DEFINITION_FMT_VER); +} + +/*static*/ void ParticleEffect_BakeElement(DmSymbolTable& /*symbolTable*/, DmElement_s& /*elem*/) +{ + //inheritEntityScale = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "Inherit entity scale", true); + //ParticleEffect_BakeParams(); +} + +struct ParticlePageMemory_s +{ + size_t stringPoolStart; + size_t stringDictStart; + size_t elementsStart; +}; + +/*static*/ void ParticleEffect_CreatePageLayoutAndAlloc(const DmContext_s& ctx, ParticlePageMemory_s& outMem, const char* const particleName) +{ + const size_t nameBufLen = strlen(particleName) +1; + + outMem.stringPoolStart = sizeof(EffectAssetData_s) + nameBufLen; + outMem.stringDictStart = IALIGN8(outMem.stringPoolStart + ctx.symbolTable.StringBytesRetained()); + outMem.elementsStart = outMem.stringDictStart + ctx.symbolTable.NumStringsRetained() * sizeof(PagePtr_t); +} + +static void ParticleEffect_ProcessSymbolTable(DmContext_s& ctx) +{ + const DmSymbolTable::SymbolId_t target = ctx.symbolTable.Find("DmeParticleOperator"); + + if (target == DmSymbolTable::npos) + return; // Nothing to process, no string dictionary to build. + + // note(kawe): since baked particle effects must store and expose all fields + // including default fields, we don't need to store the field + // names inside the baked dictionary. Mark everything that we + // plan to discard later so the symbol table is aware and knows + // the total pool size we need for the page buffer. + for (const DmElement_s& elem : ctx.elementList) + { + if (elem.type.s != target) + continue; + + ctx.symbolTable.MarkAsDiscarded(elem.name.s); + + for (const DmAttribute_s& at : elem.attr) + { + if (at.type == AT_STRING) + ctx.symbolTable.MarkAsDiscarded(at.value.stringSym.s); + } + } +} + +static void ParticleEffect_InternalBake(CPakFileBuilder* const pak, /*PakAsset_t& asset,*/ const PakGuid_t /*assetGuid*/, const char* const assetPath) +{ + BinaryIO input; + + if (!input.Open(pak->GetAssetPath() + assetPath, BinaryIO::Mode_e::Read)) + Error("Failed to open particle effect asset \"%s\".\n", assetPath); + + DmxHeader_s header; + + if (!Dmx_ParseHdr(input, header)) + return; + + ParticleEffect_ValidateHeader(header); + DmContext_s ctx; + + if (!Dmx_DeserializeBinary(ctx, input)) + return; + + ParticleEffect_ProcessSymbolTable(ctx); + + std::vector list; + for (auto& elem : ctx.elementList) + { + ParticleDefintionParams_s& params = list.emplace_back(); + ParticleEffect_BakeParams(ctx.symbolTable, elem, params); + } +} + +void Assets::AddParticleEffect_v2(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& /*mapEntry*/) +{ + ParticleEffect_InternalBake(pak, assetGuid, assetPath); +} diff --git a/src/assets/particle_operator_params.cpp b/src/assets/particle_operator_params.cpp new file mode 100644 index 00000000..d9b292c9 --- /dev/null +++ b/src/assets/particle_operator_params.cpp @@ -0,0 +1,153 @@ +#include "pch.h" +#include "particle_operators_params.h" +#include +#include + + +template +T ParticleEffect_FindAndSetField_OrDefault(DmSymbolTable& symbolTable, const DmElement_s& elem, const std::string_view name, const T defaultValue) +{ + const DmSymbolTable::SymbolId_t sym = symbolTable.Find(name); + + if (sym == DmSymbolTable::npos) + return defaultValue; + + const DmAttribute_s* const at = Dme_FindAttribute(elem, DmeSymbol_t(sym)); + + if (!at) + return defaultValue; + + const DmAttributeType_e expectedType = Dma_GetTypeForPod(); + + if (at->type != expectedType) + return defaultValue; + + return Dma_GetValue(*at); +} + +template +T ParticleEffect_FindAndSetField_OrDefault_Clamped(DmSymbolTable& symbolTable, const DmElement_s& elem, const std::string_view name, const T defaultValue, const T minValue, const T maxValue) +{ + const T result = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, name, defaultValue); + + if (result < minValue || result > maxValue) + { + // todo: warning. + return defaultValue; + } + + return result; +} + +/*static*/ void ParticleEffect_BakeParams(DmSymbolTable& symbolTable, const DmElement_s& elem, ParticleDefintionParams_s& params) +{ + params.maxParticles = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "max_particles", 1000); + params.initialParticles = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "initial_particles", 0); + + // todo: materialName strings + + params.boundingBoxMin = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "bounding_box_min", Vector3(-10, -10, -10)); + params.boundingBoxMax = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "bounding_box_max", Vector3(10, 10, 10)); + + params.cullRadius = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "cull_radius", 0.0f); + params.cullControlPoint = ParticleEffect_FindAndSetField_OrDefault_Clamped(symbolTable, elem, "cull_control_point", 0, 0, 63); + + // todo: fallbackReplacementName strings + + params.fallbackThreshold = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "fallback threshold", 1); // todo: is this a float? most likely not, check s0 pak + + params.constantRadius = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "radius", 5.0f); + params.constantColor = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "color", ColorB(255, 255, 255, 255)); + params.constantColorScale = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "color HDR scale", 1.0f); + params.constantRotation = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "rotation", 0.0f); + params.constantRotationSpeed = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "rotation_speed", 0.0f); + params.constantNormal = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "normal", Vector3(0, 0, 1)); + params.constantNormalSpinDegrees = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "normal spin degrees", 0.0f); + // todo: figure out sequence numbers (user data???) + + params.groupId = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "group id", 0); + params.maximumTimeStep = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "maximum time step", 0.1f); + params.maximumSimTime = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "maximum sim tick rate", 0.0f); + params.minimumSimTime = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "minimum sim tick rate", 0.0f); + params.minimumFrames = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "minimum rendered frames", 0); + + params.skipRenderControlPoint = ParticleEffect_FindAndSetField_OrDefault_Clamped(symbolTable, elem, "control point to disable rendering if it is the camera", -1, -1, 63); + params.allowRenderControlPoint = ParticleEffect_FindAndSetField_OrDefault_Clamped(symbolTable, elem, "control point to only enable rendering if it is the camera", -1, -1, 63); + + params.maxDrawDistance = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "maximum draw distance", 100000.0f); + params.minDrawDistance = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "minimum draw distance", 0.0f); + + params.applyDrawDistanceWhenChild = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "apply draw distance when child", false); + params.noDrawTimeToGoToSleep = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "time to sleep when not drawn", 8.0f); + + params.shouldSort = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "Sort particles", true); + params.shouldBatch = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "batch particle systems", false); + + params.viewModelEffect = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "view model effect", false); + params.screenSpaceEffect = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "screen space effect", false); + params.drawWithScreenSpace = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "draw with screen space", false); + params.drawThroughLeafSystem = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "draw through leafsystem", true); + params.checkOwnerDormant = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "check owner dormant", true); + params.maxRecursionDepth = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "maximum portal recursion depth", 8); + params.aggregateRadius = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "aggregation radius", 0.0f); + params.aggregationMinAvailableParticles = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "minimum free particles to aggregate", 0); + params.minimumTimeStep = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "minimum simulation time step", 0.0f); + params.minCpuLevel = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "minimum CPU level", 0); + params.minGpuLevel = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "minimum GPU level", 0); + + params.stopSimulationAfterTime = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "freeze simulation after time", 1000000000.0f); + params.warmUpTime = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "warm-up time", 0.0f); + params.pauseAfterWarmUp = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "warm-up and pause", false); + + params.killIfOverLimit = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "kill if over-limit", true); + params.useHeightInYaw = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "use height in yaw", false); + params.doDrawDuringPhaseShift = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "phase shift - allowed", false); + params.doOnlyDrawDuringPhaseShift = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "phase shift - only", false); + params.inheritsAlphaVisibility = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "inherit alpha visibility", false); + params.inheritsRawVisibility = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "inherit proxy visibility", false); + params.randomSeed = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "random seed override", 0); + + params.renderShadows = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "shadows", false); + params.isScripted = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "scripted", false); + params.reserved = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "reserved", false); + params.sortFromOrigin = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "sort from origin", false); + + params.lowResDrawEnable = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "low res draw enable", false); + params.lowResDist = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "low res draw dist", 800.0f); + + params.inheritEntityScale = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "Inherit entity scale", true); +} + +/*static*/ void ParticleEffect_BakeParams_C_OP_GraphVector_Params(DmSymbolTable& symbolTable, const DmElement_s& elem, C_OP_GraphVector_Params& params) +{ + // TODO: graph array + /* + graph: "0 0 0 0 0.5 1 1 1 1 0 0 0" + */ + params.m_isLooping = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "graph loop", false); + params.m_graphTime = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "graph time", 1.0f); + params.m_graphTimeIsInLifespans = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "graph time is in lifespans", true); + params.m_bUseLocalSpace = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "use local space", false); + params.m_nFieldOutput = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "output field", 0); + params.m_outputScaleType = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "output op", 0); + params.m_LocalCtrlPoint = ParticleEffect_FindAndSetField_OrDefault_Clamped(symbolTable, elem, "local control point", 0, 0, 63); + params.m_inputTimeCtrlPoint = ParticleEffect_FindAndSetField_OrDefault_Clamped(symbolTable, elem, "input time control point", -1, -1, 63); + params.m_vecOutputMin = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "output minimum", Vector3(-1.0f, -1.0f, -1.0f)); + params.m_vecOutputMax = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "output maximum", Vector3(1.0f, 1.0f, 1.0f)); +} + +/*static*/ void ParticleEffect_BakeParams_C_OP_RenderLightSource_Params(DmSymbolTable& symbolTable, const DmElement_s& elem, C_OP_RenderLightSource_Params& params) +{ + params.m_flColorScale = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "color scale", 1.0f); + params.m_bColorScaleByAlpha = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "color scale by alpha", false); + params.m_flRadiusScale = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "radius scale", 1.0f); + params.m_bIsCockpit = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "is cockpit light", false); + params.m_flPriorityMultiplier = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "priority multiplier", 1.0f); + params.m_flFalloffHalfwayFrac = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "falloff halfway fraction", 0.3f); +} + +/*static*/ void ParticleEffect_BakeParams_C_OP_RenderScreenVelocityRotate_Params(DmSymbolTable& symbolTable, const DmElement_s& elem, C_OP_RenderScreenVelocityRotate_Params& params) +{ + params.m_flRotateRateDegrees = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "rotate_rate(dps)", 0.0f); + params.m_flForwardDegrees = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "forward_angle", -90.0f); +} diff --git a/src/assets/particle_operators_params.h b/src/assets/particle_operators_params.h new file mode 100644 index 00000000..f516f860 --- /dev/null +++ b/src/assets/particle_operators_params.h @@ -0,0 +1,42 @@ +#pragma once +#include "math/vector.h" +#include "math/color.h" +#include "utils/DmxTools.h" +#include "public/particle_effect.h" + +struct C_OP_GraphVector_Params +{ + Vector4 m_graph[16]; + bool m_isLooping; + float m_graphTime; + bool m_graphTimeIsInLifespans; + bool m_bUseLocalSpace; + int m_nFieldOutput; + int m_outputScaleType; + int m_LocalCtrlPoint; + int m_inputTimeCtrlPoint; + Vector3 m_vecOutputMin; + Vector3 m_vecOutputMax; +}; + +struct C_OP_RenderPoints_Params +{ +}; + +struct C_OP_RenderLightSource_Params +{ + float m_flColorScale; + bool m_bColorScaleByAlpha; + float m_flRadiusScale; + bool m_bIsCockpit; + float m_flPriorityMultiplier; + float m_flFalloffHalfwayFrac; +}; + +struct C_OP_RenderScreenVelocityRotate_Params +{ + float m_flRotateRateDegrees; + float m_flForwardDegrees; +}; + +extern void ParticleEffect_BakeParams(DmSymbolTable& symbolTable, const DmElement_s& elem, ParticleDefintionParams_s& params); diff --git a/src/logic/pakfile.cpp b/src/logic/pakfile.cpp index 3f84d814..39847fba 100644 --- a/src/logic/pakfile.cpp +++ b/src/logic/pakfile.cpp @@ -22,6 +22,7 @@ static std::unordered_set s_pakAssetHandler {"txan", PakAssetScope_e::kClientOnly, nullptr, Assets::AddTextureAnimAsset_v1}, {"uimg", PakAssetScope_e::kClientOnly, Assets::AddUIImageAsset_v10, Assets::AddUIImageAsset_v10}, {"rlcd", PakAssetScope_e::kClientOnly, Assets::AddLcdScreenEffect_v0, Assets::AddLcdScreenEffect_v0}, + {"efct", PakAssetScope_e::kClientOnly, nullptr, Assets::AddParticleEffect_v2}, {"matl", PakAssetScope_e::kClientOnly, Assets::AddMaterialAsset_v12, Assets::AddMaterialAsset_v15}, {"mt4a", PakAssetScope_e::kClientOnly, nullptr, Assets::AddMaterialForAspectAsset_v3}, {"shdr", PakAssetScope_e::kClientOnly, Assets::AddShaderAsset_v8, Assets::AddShaderAsset_v12}, diff --git a/src/public/particle_effect.h b/src/public/particle_effect.h new file mode 100644 index 00000000..4c618200 --- /dev/null +++ b/src/public/particle_effect.h @@ -0,0 +1,155 @@ +#pragma once +#include "math/vector.h" +#include "math/color.h" +#include "dmelement.h" + +struct UtlStringDisk_s // CUtlString encoded on the disk. +{ + const char* memory; + int64_t allocationCount; + int64_t growSize; + int64_t actualLength; +}; + +struct ParticleDefintionParams_s +{ + int maxParticles; + int initialParticles; + UtlStringDisk_s materialName; + Vector3 boundingBoxMin; + Vector3 boundingBoxMax; + float cullRadius; + int cullControlPoint; + UtlStringDisk_s fallbackReplacementName; + int fallbackThreshold; + float constantRadius; + ColorB constantColor; + float constantColorScale; + float constantRotation; + float constantRotationSpeed; + Vector3 constantNormal; + float constantNormalSpinDegrees; + int constantSequenceNumber; + int constantSequenceNumber1; + int groupId; + float maximumTimeStep; + float maximumSimTime; + float minimumSimTime; + int minimumFrames; + int skipRenderControlPoint; + int allowRenderControlPoint; + float maxDrawDistance; + float minDrawDistance; + bool applyDrawDistanceWhenChild; + char gapBD[3]; // padding. + float noDrawTimeToGoToSleep; + bool shouldSort; + bool shouldBatch; + bool viewModelEffect; + bool screenSpaceEffect; + bool drawWithScreenSpace; + bool drawThroughLeafSystem; + bool checkOwnerDormant; + char gapCB; // padding. + int maxRecursionDepth; + float aggregateRadius; + int aggregationMinAvailableParticles; + float minimumTimeStep; + int minCpuLevel; + int minGpuLevel; + float stopSimulationAfterTime; + float warmUpTime; + bool pauseAfterWarmUp; + bool killIfOverLimit; + bool useHeightInYaw; + bool doDrawDuringPhaseShift; + bool doOnlyDrawDuringPhaseShift; + bool inheritsAlphaVisibility; + bool inheritsRawVisibility; + char gapF3; // padding. + int randomSeed; + bool renderShadows; + bool isScripted; + bool reserved; + bool sortFromOrigin; + bool lowResDrawEnable; + char gapFD[3]; // padding. + float lowResDist; + bool inheritEntityScale; + char gap105[3]; // padding. +}; + +struct ParticleOperatorCommonParams_s +{ + float ppStartFadeInTime; + float opEndFadeInTime; + float opStartFadeOutTime; + float opEndFadeOutTime; + float opFadeOscillatePeriod; + int opTimeOffsetSeed; + float opTimeOffsetMin; + float opTimeOffsetMax; + int opTimeScaleSeed; + float opTimeScaleMin; + float opTimeScaleMax; + int opStrengthScaleSeed; + float opStrengthMinScale; + float opStrengthMaxScale; + int opEndCapState; + bool mute; +}; + +struct ParticleOperatorBaked_s +{ + uint16_t opTypeIndex; // indexes into dictionary, gets the operator type + uint16_t opSpecificParamsDataSize; // struct size of operator (matches with ctor for each operator, so its most likely jsut that) + uint16_t visibilityInputsIndex; // same stuff as source engine but baked (CParticleVisibilityInputs) + uint16_t initializerParamsIndex; // same stuff as source engine but baked + uint16_t opSpecificParamsIndex; // points to struct that is specific to this operator + char pad[10]; // id must be aligned to 16 bytes. + DmObjectId_s id; + ParticleOperatorCommonParams_s common; // similar to source, but also baked +}; + +struct ParticleOperatorList_s +{ + ParticleOperatorBaked_s** operators; + uint64_t count; +}; + +struct ParticleChildParams_s +{ + float delay; + bool endCap; + bool mute; + int elemIdx; +}; + +struct ParticleElementDisk_s +{ + const char* name; + DmObjectId_s id; + ParticleChildParams_s* childRefs; + const char** scriptRefs; + int numChildRefs; + int numScriptRefs; + bool preventNameBasedLookup; + char pad[4]; // params must be aligned to 8 bytes. + ParticleDefintionParams_s params; + ParticleOperatorList_s operators[6]; +}; + +struct EffectAssetData_s +{ + const char* fileName; + ParticleElementDisk_s* elements; + size_t numElements; + const char** stringDict; + size_t numStrings; +}; + +struct EffectAssetHdr_s +{ + EffectAssetData_s* pcfPtr; + size_t pcfCount; +}; diff --git a/src/public/rpak.h b/src/public/rpak.h index d0e27fb9..c7a259be 100644 --- a/src/public/rpak.h +++ b/src/public/rpak.h @@ -31,6 +31,7 @@ #define TYPE_RMDL MAKE_FOURCC('m', 'd', 'l', '_') // mdl_ #define TYPE_UIMG MAKE_FOURCC('u', 'i', 'm', 'g') // uimg #define TYPE_RLCD MAKE_FOURCC('r', 'l', 'c', 'd') // rlcd +#define TYPE_EFCT MAKE_FOURCC('e', 'f', 'c', 't') // efct #define TYPE_PTCH MAKE_FOURCC('P', 't', 'c', 'h') // Ptch #define TYPE_DTBL MAKE_FOURCC('d', 't', 'b', 'l') // dtbl #define TYPE_STLT MAKE_FOURCC('s', 't', 'l', 't') // stlt @@ -53,6 +54,7 @@ enum class AssetType : uint32_t RMDL = TYPE_RMDL, // model UIMG = TYPE_UIMG, // ui image atlas RLCD = TYPE_RLCD, // lcd screen effect + EFCT = TYPE_EFCT, // particle effect PTCH = TYPE_PTCH, // patch DTBL = TYPE_DTBL, // datatable STLT = TYPE_STLT, // settings layout