Particle effects baking support (EFCT)

src/RePak.vcxproj

@@ -21,6 +21,8 @@
     <ClCompile Include="assets\material.cpp" />
     <ClCompile Include="assets\material_for_aspect.cpp" />
     <ClCompile Include="assets\model.cpp" />
+    <ClCompile Include="assets\particle_effect.cpp" />
+    <ClCompile Include="assets\particle_operator_params.cpp" />
     <ClCompile Include="assets\patch.cpp" />
     <ClCompile Include="assets\settings.cpp" />
     <ClCompile Include="assets\settings_layout.cpp" />
@@ -161,6 +163,8 @@
       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">NotUsing</PrecompiledHeader>
     </ClCompile>
     <ClCompile Include="utils\binaryio.cpp" />
+    <ClCompile Include="utils\dmarray.cpp" />
+    <ClCompile Include="utils\DmxTools.cpp" />
     <ClCompile Include="utils\dxutils.cpp" />
     <ClCompile Include="utils\jsonutils.cpp" />
     <ClCompile Include="utils\logger.cpp" />
@@ -174,6 +178,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="assets\assets.h" />
+    <ClInclude Include="assets\particle_operators_params.h" />
     <ClInclude Include="common\const.h" />
     <ClInclude Include="common\decls.h" />
     <ClInclude Include="logic\buildsettings.h" />
@@ -186,14 +191,19 @@
     <ClInclude Include="math\color.h" />
     <ClInclude Include="math\common.h" />
     <ClInclude Include="math\vector.h" />
+    <ClInclude Include="math\vmatrix.h" />
     <ClInclude Include="pch.h" />
     <ClInclude Include="public\animrig.h" />
     <ClInclude Include="public\anim_recording.h" />
+    <ClInclude Include="public\dmattribute.h" />
+    <ClInclude Include="public\dmelement.h" />
+    <ClInclude Include="public\dmexchange.h" />
     <ClInclude Include="public\lcd_screen_effect.h" />
     <ClInclude Include="public\material.h" />
     <ClInclude Include="public\materialflags.h" />
     <ClInclude Include="public\material_for_aspect.h" />
     <ClInclude Include="public\multishader.h" />
+    <ClInclude Include="public\particle_effect.h" />
     <ClInclude Include="public\rpak.h" />
     <ClInclude Include="public\settings.h" />
     <ClInclude Include="public\settings_layout.h" />
@@ -285,6 +295,7 @@
     <ClInclude Include="thirdparty\zstd\zstd.h" />
     <ClInclude Include="thirdparty\zstd\zstd_errors.h" />
     <ClInclude Include="utils\binaryio.h" />
+    <ClInclude Include="utils\DmxTools.h" />
     <ClInclude Include="utils\dxutils.h" />
     <ClInclude Include="utils\jsonutils.h" />
     <ClInclude Include="utils\logger.h" />

src/RePak.vcxproj.filters

@@ -250,6 +250,18 @@
     <ClCompile Include="assets\lcd_screen_effect.cpp">
       <Filter>assets</Filter>
     </ClCompile>
+    <ClCompile Include="utils\DmxTools.cpp">
+      <Filter>utils</Filter>
+    </ClCompile>
+    <ClCompile Include="assets\particle_effect.cpp">
+      <Filter>assets</Filter>
+    </ClCompile>
+    <ClCompile Include="utils\dmarray.cpp">
+      <Filter>utils</Filter>
+    </ClCompile>
+    <ClCompile Include="assets\particle_operator_params.cpp">
+      <Filter>assets</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="assets\assets.h">
@@ -609,6 +621,27 @@
     <ClInclude Include="public\lcd_screen_effect.h">
       <Filter>public</Filter>
     </ClInclude>
+    <ClInclude Include="utils\DmxTools.h">
+      <Filter>utils</Filter>
+    </ClInclude>
+    <ClInclude Include="public\particle_effect.h">
+      <Filter>public</Filter>
+    </ClInclude>
+    <ClInclude Include="public\dmelement.h">
+      <Filter>public</Filter>
+    </ClInclude>
+    <ClInclude Include="public\dmattribute.h">
+      <Filter>public</Filter>
+    </ClInclude>
+    <ClInclude Include="public\dmexchange.h">
+      <Filter>public</Filter>
+    </ClInclude>
+    <ClInclude Include="math\vmatrix.h">
+      <Filter>math</Filter>
+    </ClInclude>
+    <ClInclude Include="assets\particle_operators_params.h">
+      <Filter>public</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="cpp.hint" />

src/application/repak.cpp

@@ -347,6 +347,7 @@ int main(int argc, char** argv)
     extern bool Console_ColorInit();
     Console_ColorInit();
 
+    g_iosmErrorCallback = Error;
     g_jsonErrorCallback = Error;
 
     RePak_HandleCommandLine(argc, argv);

src/assets/anim_recording.cpp

@@ -101,7 +101,8 @@ static void AnimRecording_InternalAddAnimRecording(CPakFileBuilder* const pak, c
 
 	for (int i = 0; i < fileHdr.numElements; i++)
 	{
-		bio.Read(pHdr->poseParamValues[i]);
+		if (!bio.Read(pHdr->poseParamValues[i]))
+			Error("Failed to read animation pose parameter value #%i\n", i);
 	}
 
 	for (int i = 0; i < fileHdr.numSequences; i++)

src/assets/animseq.cpp

@@ -57,7 +57,10 @@ static void AnimSeq_ParseDependenciesFromData(const uint8_t* const data, std::se
 
         const char* end = strchr(start, ' ');
         if (!end)
-            end = event->options + strlen(event->options);
+        {
+            const size_t maxLen = sizeof(event->options) - (start - event->options);
+            end = start + strnlen(start, maxLen);
+        }
 
         const size_t nameLen = (end - start);
 
@@ -198,15 +201,13 @@ static void AnimSeq_InternalAddAnimSeq(CPakFileBuilder* const pak, const PakGuid
             hdr->settingsCount = (uint32_t)set.size();
         }
 
-        for (size_t j = 0; j < set.size(); j++)
-        {
-            std::set<PakGuid_t>::iterator it = set.begin();
-            std::advance(it, j);
+        size_t j = 0;
 
-            const PakGuid_t guidToCopy = *it;
+        for (const PakGuid_t& guidToCopy : set)
+        {
             reinterpret_cast<PakGuid_t*>(&dataLump.data[bufferBase])[j] = guidToCopy;
-
             Pak_RegisterGuidRefAtOffset(guidToCopy, bufferBase + (j * sizeof(PakGuid_t)), dataLump, asset);
+            ++j;
         }
 
         bufferBase += set.size() * sizeof(PakGuid_t);

src/assets/assets.h

@@ -9,6 +9,7 @@
 #define TXLS_VERSION 1
 #define UIMG_VERSION 10
 #define RLCD_VERSION 0
+#define EFCT_VERSION 2
 //#define DTBL_VERSION 1
 #define STLT_VERSION 0
 #define STGS_VERSION 1
@@ -36,6 +37,8 @@ namespace Assets
 	void AddUIImageAsset_v10(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry);
 	void AddLcdScreenEffect_v0(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry);
 
+	void AddParticleEffect_v2(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& /*mapEntry*/);
+
 	void AddDataTableAsset(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry);
 	void AddSettingsLayout_v0(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry);
 	void AddSettingsAsset_v1(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry);

src/assets/datatable.cpp

@@ -20,6 +20,18 @@ static void DataTable_ReportInvalidDataTypeError(const char* const type, const u
     Error("Invalid data type \"%s\" at cell [%u,%u].\n", type, rowIdx, colIdx);
 }
 
+template <typename T>
+static T DataTable_ParseCellFromDocument(const rapidcsv::Document& doc, const uint32_t colIdx, const uint32_t rowIdx, const dtblcoltype_t type)
+{
+    try {
+        return doc.GetCell<T>(colIdx, rowIdx);
+    }
+    catch (const std::exception& ex) {
+        Error("Exception while parsing %s value from cell [%u,%u]: %s.\n", DataTable_GetStringFromType(type), rowIdx, colIdx, ex.what());
+        return T{};
+    }
+}
+
 template <typename datatable_t>
 static size_t DataTable_SetupRows(const rapidcsv::Document& doc, datatable_t* const dtblHdr, datatable_asset_t& tmp, std::vector<std::string>& outTypeRow)
 {
@@ -29,7 +41,7 @@ static size_t DataTable_SetupRows(const rapidcsv::Document& doc, datatable_t* co
 
     // typically happens when there's an empty line in the csv file.
     if (numTypeNames != dtblHdr->numColumns)
-        Error("Expected %u columns for type name row, found %u.\n", dtblHdr->numRows, numTypeNames);
+        Error("Expected %u columns for type name row, found %u.\n", dtblHdr->numColumns, numTypeNames);
 
     // Make sure every row (including rows we don't end up storing in the pak),
     // have the same number of columns as the type row. The column count in the
@@ -65,8 +77,8 @@ static size_t DataTable_SetupRows(const rapidcsv::Document& doc, datatable_t* co
             for (uint32_t j = 0; j < dtblHdr->numRows; ++j)
             {
                 // this can be std::string since we only deal with the string types here
-                std::vector<std::string> row = doc.GetRow<std::string>(j);
-                const size_t strLen = row[i].length();
+                const std::string cellValue = DataTable_ParseCellFromDocument<std::string>(doc, i, j, type);
+                const size_t strLen = cellValue.length();
 
                 if (isPrecachedAsset && strLen > 0)
                     tmp.guidRefBufSize += sizeof(PakGuid_t);
@@ -116,18 +128,6 @@ static void DataTable_SetupColumns(CPakFileBuilder* const pak, PakPageLump_s& da
     }
 }
 
-template <typename T>
-static T DataTable_ParseCellFromDocument(rapidcsv::Document& doc, const uint32_t colIdx, const uint32_t rowIdx, const dtblcoltype_t type)
-{
-    try {
-        return doc.GetCell<T>(colIdx, rowIdx);
-    }
-    catch (const std::exception& ex) {
-        Error("Exception while parsing %s value from cell [%u,%u]: %s.\n", DataTable_GetStringFromType(type), rowIdx, colIdx, ex.what());
-        return T{};
-    }
-}
-
 static void DataTable_ReportInvalidValueError(const dtblcoltype_t type, const uint32_t rowIdx, const uint32_t colIdx)
 {
     Error("Invalid %s value at cell [%u,%u].\n", DataTable_GetStringFromType(type), rowIdx, colIdx);
@@ -184,25 +184,12 @@ static void DataTable_SetupValues(CPakFileBuilder* const pak, PakAsset_t& asset,
             }
             case dtblcoltype_t::Vector:
             {
-                std::string val = DataTable_ParseCellFromDocument<std::string>(doc, colIdx, rowIdx, col.type);
-                std::smatch sm;
+                const std::string val = DataTable_ParseCellFromDocument<std::string>(doc, colIdx, rowIdx, col.type);
+                Vector3 vec;
 
                 // get values from format "<x,y,z>"
-                const bool result = std::regex_search(val, sm, std::regex("<(.*),(.*),(.*)>"));
-
-                // 0 - all
-                // 1 - x
-                // 2 - y
-                // 3 - z
-                if (result && (sm.size() == 4))
-                {
-                    const Vector3 vec(
-                        static_cast<float>(atof(sm[1].str().c_str())),
-                        static_cast<float>(atof(sm[2].str().c_str())),
-                        static_cast<float>(atof(sm[3].str().c_str())));
-
+                if (sscanf_s(val.c_str(), "<%f,%f,%f>", &vec.x, &vec.y, &vec.z) == 3)
                     valbuf.write(vec);
-                }
                 else
                     DataTable_ReportInvalidValueError(col.type, rowIdx, colIdx);
                 break;

src/assets/particle_effect.cpp

@@ -0,0 +1,98 @@
+#include "pch.h"
+#include "assets.h"
+#include "particle_operators_params.h"
+#include "public/particle_effect.h"
+#include "utils/DmxTools.h"
+
+#define PARTICLE_DEFINITION_ENC "binary"
+#define PARTICLE_DEFINITION_ENC_VER 5
+#define PARTICLE_DEFINITION_FMT "pcf"
+#define PARTICLE_DEFINITION_FMT_VER 2
+
+static void ParticleEffect_ValidateHeader(const DmxHeader_s& hdr)
+{
+	if (hdr.formatVersion != PARTICLE_DEFINITION_FMT_VER)
+		Error("Particle effect is version %d, but %d was expected!", hdr.formatVersion, PARTICLE_DEFINITION_FMT_VER);
+}
+
+/*static*/ void ParticleEffect_BakeElement(DmSymbolTable& /*symbolTable*/, DmElement_s& /*elem*/)
+{
+	//inheritEntityScale = ParticleEffect_FindAndSetField_OrDefault(symbolTable, elem, "Inherit entity scale", true);
+	//ParticleEffect_BakeParams();
+}
+
+struct ParticlePageMemory_s
+{
+	size_t stringPoolStart;
+	size_t stringDictStart;
+	size_t elementsStart;
+};
+
+/*static*/ void ParticleEffect_CreatePageLayoutAndAlloc(const DmContext_s& ctx, ParticlePageMemory_s& outMem, const char* const particleName)
+{
+	const size_t nameBufLen = strlen(particleName) +1;
+
+	outMem.stringPoolStart = sizeof(EffectAssetData_s) + nameBufLen;
+	outMem.stringDictStart = IALIGN8(outMem.stringPoolStart + ctx.symbolTable.StringBytesRetained());
+	outMem.elementsStart = outMem.stringDictStart + ctx.symbolTable.NumStringsRetained() * sizeof(PagePtr_t);
+}
+
+static void ParticleEffect_ProcessSymbolTable(DmContext_s& ctx)
+{
+	const DmSymbolTable::SymbolId_t target = ctx.symbolTable.Find("DmeParticleOperator");
+
+	if (target == DmSymbolTable::npos)
+		return; // Nothing to process, no string dictionary to build.
+
+	// note(kawe): since baked particle effects must store and expose all fields
+	//             including default fields, we don't need to store the field
+	//             names inside the baked dictionary. Mark everything that we
+	//             plan to discard later so the symbol table is aware and knows
+	//             the total pool size we need for the page buffer.
+	for (const DmElement_s& elem : ctx.elementList)
+	{
+		if (elem.type.s != target)
+			continue;
+
+		ctx.symbolTable.MarkAsDiscarded(elem.name.s);
+
+		for (const DmAttribute_s& at : elem.attr)
+		{
+			if (at.type == AT_STRING)
+				ctx.symbolTable.MarkAsDiscarded(at.value.stringSym.s);
+		}
+	}
+}
+
+static void ParticleEffect_InternalBake(CPakFileBuilder* const pak, /*PakAsset_t& asset,*/ const PakGuid_t /*assetGuid*/, const char* const assetPath)
+{
+	BinaryIO input;
+
+	if (!input.Open(pak->GetAssetPath() + assetPath, BinaryIO::Mode_e::Read))
+		Error("Failed to open particle effect asset \"%s\".\n", assetPath);
+
+	DmxHeader_s header;
+
+	if (!Dmx_ParseHdr(input, header))
+		return;
+
+	ParticleEffect_ValidateHeader(header);
+	DmContext_s ctx;
+
+	if (!Dmx_DeserializeBinary(ctx, input))
+		return;
+
+	ParticleEffect_ProcessSymbolTable(ctx);
+
+	std::vector<ParticleDefintionParams_s> list;
+	for (auto& elem : ctx.elementList)
+	{
+		ParticleDefintionParams_s& params = list.emplace_back();
+		ParticleEffect_BakeParams(ctx.symbolTable, elem, params);
+	}
+}
+
+void Assets::AddParticleEffect_v2(CPakFileBuilder* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& /*mapEntry*/)
+{
+	ParticleEffect_InternalBake(pak, assetGuid, assetPath);
+}