From fffbc97920caaa0d4c66ad597e4729475cc6faed Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 14 Aug 2024 18:17:14 +0200 Subject: [PATCH 001/102] Throw error message in case the multizone discrete adjoint solver will write restart files during runtime (there is a bug, potentially when clearing the tape). --- Common/src/CConfig.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index d54d1dc3a6c0..8eaccde5472b 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -3412,8 +3412,17 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i /*--- Using default frequency of 250 for all files when steady, and 1 for unsteady. ---*/ for (auto iVolumeFreq = 0; iVolumeFreq < nVolumeOutputFrequencies; iVolumeFreq++){ - VolumeOutputFrequencies[iVolumeFreq] = Time_Domain ? 1 : 250; + if (Multizone_Problem && DiscreteAdjoint) { + VolumeOutputFrequencies[iVolumeFreq] = nOuterIter; + } + else { + VolumeOutputFrequencies[iVolumeFreq] = Time_Domain ? 1 : 250; + } } + } else if (Multizone_Problem && DiscreteAdjoint) { + SU2_MPI::Error(string("OUTPUT_WRT_FREQ cannot be specified for this solver " + "(writing of restart and sensitivity files not possible for multizone discrete adjoint during runtime yet).\n" + "Please remove this option from the config file, output files will be written when solver finalizes.\n"), CURRENT_FUNCTION); } else if (nVolumeOutputFrequencies < nVolumeOutputFiles) { /*--- If there are fewer frequencies than files, repeat the last frequency. * This is useful to define 1 frequency for the restart file and 1 frequency for all the visualization files. ---*/ From ba6ee1c231bed40ed22392942dfb1fd2b1b94f19 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 16 Aug 2024 00:22:57 +0200 Subject: [PATCH 002/102] Softcode indentifier/index. --- Common/include/basic_types/ad_structure.hpp | 26 ++++++++++++++----- Common/include/geometry/dual_grid/CPoint.hpp | 5 ++-- Common/src/geometry/dual_grid/CPoint.cpp | 4 +-- .../drivers/CDiscAdjMultizoneDriver.hpp | 2 +- SU2_CFD/include/variables/CVariable.hpp | 12 ++++----- .../src/drivers/CDiscAdjMultizoneDriver.cpp | 2 +- SU2_CFD/src/solvers/CSolver.cpp | 2 +- SU2_CFD/src/variables/CVariable.cpp | 4 +-- 8 files changed, 34 insertions(+), 23 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index de902141e89a..b8b71f102b88 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -38,6 +38,9 @@ */ namespace AD { #ifndef CODI_REVERSE_TYPE + +using Identifier = int; + /*! * \brief Start the recording of the operations and involved variables. * If called, the computational graph of all operations occuring after the call will be stored, @@ -101,14 +104,20 @@ inline void EndUseAdjoints() {} * \param[in] index - Position in the adjoint vector. * \param[in] val - adjoint value to be set. */ -inline void SetDerivative(int index, const double val) {} +inline void SetDerivative(Identifier index, const double val) {} /*! * \brief Extracts the adjoint value at index * \param[in] index - position in the adjoint vector where the derivative will be extracted. * \return Derivative value. */ -inline double GetDerivative(int index) { return 0.0; } +inline double GetDerivative(Identifier index) { return 0.0; } + +/*! + * \brief Returns the identifier that represents an inactive variable. + * \return Passive index. + */ +inline Identifier GetPassiveIndex() { return 0; } /*! * \brief Clears the currently stored adjoints but keeps the computational graph. @@ -259,7 +268,7 @@ inline void SetExtFuncOut(T&& data, const int size_x, const int size_y) {} * \param[in] data - variable whose gradient information will be extracted. * \param[in] index - where obtained gradient information will be stored. */ -inline void SetIndex(int& index, const su2double& data) {} +inline void SetIndex(Identifier& index, const su2double& data) {} /*! * \brief Pushes back the current tape position to the tape position's vector. @@ -304,6 +313,7 @@ inline void EndNoSharedReading() {} using CheckpointHandler = codi::ExternalFunctionUserData; using Tape = su2double::Tape; +using Identifier = su2double::Identifier; #ifdef HAVE_OPDI using ExtFuncHelper = codi::OpenMPExternalFunctionHelper; @@ -470,14 +480,14 @@ FORCEINLINE void BeginUseAdjoints() { AD::getTape().beginUseAdjointVector(); } FORCEINLINE void EndUseAdjoints() { AD::getTape().endUseAdjointVector(); } -FORCEINLINE void SetIndex(int& index, const su2double& data) { index = data.getIdentifier(); } +FORCEINLINE void SetIndex(Identifier& index, const su2double& data) { index = data.getIdentifier(); } // WARNING: For performance reasons, this method does not perform bounds checking. // When using it, please ensure sufficient adjoint vector size by a call to AD::ResizeAdjoints(). // This method does not perform locking either. // It should be safeguarded by calls to AD::BeginUseAdjoints() and AD::EndUseAdjoints(). -FORCEINLINE void SetDerivative(int index, const double val) { - if (index == 0) // Allow multiple threads to "set the derivative" of passive variables without causing data races. +FORCEINLINE void SetDerivative(Identifier index, const double val) { + if (!AD::getTape().isIdentifierActive(index)) // Allow multiple threads to "set the derivative" of passive variables without causing data races. return; AD::getTape().setGradient(index, val, codi::AdjointsManagement::Manual); @@ -488,10 +498,12 @@ FORCEINLINE void SetDerivative(int index, const double val) { // Otherwise, please ensure sufficient adjoint vector size by a call to AD::ResizeAdjoints(). // This method does not perform locking either. // It should be safeguarded by calls to AD::BeginUseAdjoints() and AD::EndUseAdjoints(). -FORCEINLINE double GetDerivative(int index) { +FORCEINLINE double GetDerivative(Identifier index) { return AD::getTape().getGradient(index, codi::AdjointsManagement::Manual); } +FORCEINLINE Identifier GetPassiveIndex() { return AD::getTape().getPassiveIndex(); } + FORCEINLINE bool IsIdentifierActive(su2double const& value) { return getTape().isIdentifierActive(value.getIdentifier()); } diff --git a/Common/include/geometry/dual_grid/CPoint.hpp b/Common/include/geometry/dual_grid/CPoint.hpp index e497df353079..88d9e57adc40 100644 --- a/Common/include/geometry/dual_grid/CPoint.hpp +++ b/Common/include/geometry/dual_grid/CPoint.hpp @@ -113,9 +113,8 @@ class CPoint { su2activevector MaxLength; /*!< \brief The maximum cell-center to cell-center length. */ su2activevector RoughnessHeight; /*!< \brief Roughness of the nearest wall. */ - su2matrix AD_InputIndex; /*!< \brief Indices of Coord variables in the adjoint vector. */ - su2matrix - AD_OutputIndex; /*!< \brief Indices of Coord variables in the adjoint vector after having been updated. */ + su2matrix AD_InputIndex; /*!< \brief Indices of Coord variables in the adjoint vector. */ + su2matrix AD_OutputIndex; /*!< \brief Indices of Coord variables in the adjoint vector after having been updated. */ /*! * \brief Allocate fields required by the minimal constructor. diff --git a/Common/src/geometry/dual_grid/CPoint.cpp b/Common/src/geometry/dual_grid/CPoint.cpp index 7096dfa6443f..3e1e2a463b68 100644 --- a/Common/src/geometry/dual_grid/CPoint.cpp +++ b/Common/src/geometry/dual_grid/CPoint.cpp @@ -67,8 +67,8 @@ void CPoint::FullAllocation(unsigned short imesh, const CConfig* config) { } if (config->GetDiscrete_Adjoint()) { - AD_InputIndex.resize(npoint, nDim) = 0; - AD_OutputIndex.resize(npoint, nDim) = 0; + AD_InputIndex.resize(npoint, nDim) = AD::GetPassiveIndex(); + AD_OutputIndex.resize(npoint, nDim) = AD::GetPassiveIndex(); } /*--- Multigrid structures. ---*/ diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 7022220fc07c..368001bf0ea0 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -96,7 +96,7 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { bool eval_transfer = false; /*!< \brief Evaluate the transfer section of the tape. */ su2double ObjFunc; /*!< \brief Value of the objective function. */ - int ObjFunc_Index; /*!< \brief Index of the value of the objective function. */ + AD::Identifier ObjFunc_Index; /*!< \brief Index of the value of the objective function. */ CIteration*** direct_iteration; /*!< \brief Array of pointers to the direct iterations. */ COutput** direct_output; /*!< \brief Array of pointers to the direct outputs. */ diff --git a/SU2_CFD/include/variables/CVariable.hpp b/SU2_CFD/include/variables/CVariable.hpp index bc855bc5dca5..3b6e4b9f8d75 100644 --- a/SU2_CFD/include/variables/CVariable.hpp +++ b/SU2_CFD/include/variables/CVariable.hpp @@ -92,8 +92,8 @@ class CVariable { MatrixType Solution_BGS_k; /*!< \brief Old solution container for BGS iterations. */ - su2matrix AD_InputIndex; /*!< \brief Indices of Solution variables in the adjoint vector. */ - su2matrix AD_OutputIndex; /*!< \brief Indices of Solution variables in the adjoint vector after having been updated. */ + su2matrix AD_InputIndex; /*!< \brief Indices of Solution variables in the adjoint vector. */ + su2matrix AD_OutputIndex; /*!< \brief Indices of Solution variables in the adjoint vector after having been updated. */ VectorType SolutionExtra; /*!< \brief Stores adjoint solution for extra solution variables. Currently only streamwise periodic pressure-drop for massflow prescribed flows. */ @@ -118,7 +118,7 @@ class CVariable { assert(false && "A base method of CVariable was used, but it should have been overridden by the derived class."); } - void RegisterContainer(bool input, su2activematrix& variable, su2matrix* ad_index = nullptr) { + void RegisterContainer(bool input, su2activematrix& variable, su2matrix* ad_index = nullptr) { const auto nPoint = variable.rows(); SU2_OMP_FOR_STAT(roundUpDiv(nPoint,omp_get_num_threads())) for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { @@ -133,7 +133,7 @@ class CVariable { END_SU2_OMP_FOR } - void RegisterContainer(bool input, su2activematrix& variable, su2matrix& ad_index) { + void RegisterContainer(bool input, su2activematrix& variable, su2matrix& ad_index) { RegisterContainer(input, variable, &ad_index); } @@ -2180,7 +2180,7 @@ class CVariable { } inline void GetAdjointSolution_time_n(unsigned long iPoint, su2double *adj_sol) const { - int index = 0; + AD::Identifier index = AD::GetPassiveIndex(); for (unsigned long iVar = 0; iVar < Solution_time_n.cols(); iVar++) { AD::SetIndex(index, Solution_time_n(iPoint, iVar)); adj_sol[iVar] = AD::GetDerivative(index); @@ -2188,7 +2188,7 @@ class CVariable { } inline void GetAdjointSolution_time_n1(unsigned long iPoint, su2double *adj_sol) const { - int index = 0; + AD::Identifier index = AD::GetPassiveIndex(); for (unsigned long iVar = 0; iVar < Solution_time_n1.cols(); iVar++) { AD::SetIndex(index, Solution_time_n1(iPoint, iVar)); adj_sol[iVar] = AD::GetDerivative(index); diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 6e5519a63f2e..2dcda92415fc 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -744,7 +744,7 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { if (kind_recording == RECORDING::SOLUTION_VARIABLES) { cout << " Objective function : " << ObjFunc; if (driver_config->GetWrt_AD_Statistics()){ - cout << " (" << ObjFunc_Index << ")\n"; + // cout << " (" << ObjFunc_Index << ")\n"; } cout << endl; } diff --git a/SU2_CFD/src/solvers/CSolver.cpp b/SU2_CFD/src/solvers/CSolver.cpp index 2c6322392c72..6ca51d701fe6 100644 --- a/SU2_CFD/src/solvers/CSolver.cpp +++ b/SU2_CFD/src/solvers/CSolver.cpp @@ -4096,7 +4096,7 @@ void CSolver::SetVertexTractionsAdjoint(CGeometry *geometry, const CConfig *conf unsigned short iMarker, iDim; unsigned long iVertex, iPoint; - int index; + AD::Identifier index; /*--- Loop over all the markers ---*/ for (iMarker = 0; iMarker < config->GetnMarker_All(); iMarker++) { diff --git a/SU2_CFD/src/variables/CVariable.cpp b/SU2_CFD/src/variables/CVariable.cpp index 98ab867b2e92..ece1113bf369 100644 --- a/SU2_CFD/src/variables/CVariable.cpp +++ b/SU2_CFD/src/variables/CVariable.cpp @@ -69,8 +69,8 @@ CVariable::CVariable(unsigned long npoint, unsigned long ndim, unsigned long nva External.resize(nPoint,nVar) = su2double(0.0); if (!adjoint) { - AD_InputIndex.resize(nPoint,nVar) = -1; - AD_OutputIndex.resize(nPoint,nVar) = -1; + AD_InputIndex.resize(nPoint,nVar) = AD::GetPassiveIndex(); + AD_OutputIndex.resize(nPoint,nVar) = AD::GetPassiveIndex(); } } From c54597ba68a39ad59bc59248cc7fdb504a31fa9c Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 16 Aug 2024 00:48:49 +0200 Subject: [PATCH 003/102] Add tag type option to build system. --- meson.build | 2 ++ meson_options.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/meson.build b/meson.build index e6886833f61d..2a1cbdf998c0 100644 --- a/meson.build +++ b/meson.build @@ -95,6 +95,8 @@ if get_option('enable-autodiff') and not omp codi_rev_args += '-DCODI_PRIMAL_REUSE_TAPE' elif get_option('codi-tape') == 'PrimalMultiUse' codi_rev_args += '-DCODI_PRIMAL_MULTIUSE_TAPE' + elif get_option('codi-tape') == 'Tag' + codi_rev_args += '-DCODI_TAG_TAPE' else error('Invalid CoDiPack tape choice @0@'.format(get_option('codi-tape'))) endif diff --git a/meson_options.txt b/meson_options.txt index 55782c62ff80..f062074dacb7 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -23,7 +23,7 @@ option('enable-coolprop', type : 'boolean', value : false, description: 'enable option('enable-mlpcpp', type : 'boolean', value : false, description: 'enable MLPCpp support') option('enable-gprof', type : 'boolean', value : false, description: 'enable profiling through gprof') option('opdi-backend', type : 'combo', choices : ['auto', 'macro', 'ompt'], value : 'auto', description: 'OpDiLib backend choice') -option('codi-tape', type : 'combo', choices : ['JacobianLinear', 'JacobianReuse', 'JacobianMultiUse', 'PrimalLinear', 'PrimalReuse', 'PrimalMultiUse'], value : 'JacobianLinear', description: 'CoDiPack tape choice') +option('codi-tape', type : 'combo', choices : ['JacobianLinear', 'JacobianReuse', 'JacobianMultiUse', 'PrimalLinear', 'PrimalReuse', 'PrimalMultiUse', 'Tag'], value : 'JacobianLinear', description: 'CoDiPack tape choice') option('opdi-shared-read-opt', type : 'boolean', value : true, description : 'OpDiLib shared reading optimization') option('librom_root', type : 'string', value : '', description: 'libROM base directory') option('enable-librom', type : 'boolean', value : false, description: 'enable LLNL libROM support') From a24fc80b741595d9a9f7914bea3b5e7047e53a04 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 26 Aug 2024 00:07:43 +0200 Subject: [PATCH 004/102] Add option to use RealReverseTag as AD type. --- Common/include/code_config.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Common/include/code_config.hpp b/Common/include/code_config.hpp index cbbf2c531535..30fc11de2be0 100644 --- a/Common/include/code_config.hpp +++ b/Common/include/code_config.hpp @@ -110,6 +110,8 @@ using su2double = codi::RealReversePrimal; using su2double = codi::RealReversePrimalIndexGen >; #elif defined(CODI_PRIMAL_MULTIUSE_TAPE) using su2double = codi::RealReversePrimalIndex; +#elif defined(CODI_TAG_TAPE) +using su2double = codi::RealReverseTag; #else #error "Please define a CoDiPack tape." #endif From 263faa01e8e97c1caa8d687fcf1e120ed5cf6342 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 26 Aug 2024 00:28:44 +0200 Subject: [PATCH 005/102] Temporarily adjust/hardcode type size for memory alignment in vectorization.hpp (to cover RealReverseTag). --- Common/include/parallelization/vectorization.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Common/include/parallelization/vectorization.hpp b/Common/include/parallelization/vectorization.hpp index 50316359fc84..3250033730d1 100644 --- a/Common/include/parallelization/vectorization.hpp +++ b/Common/include/parallelization/vectorization.hpp @@ -90,11 +90,11 @@ class Array : public CVecExpr, Scalar_t> { public: using Scalar = Scalar_t; enum : size_t { Size = N }; - enum : size_t { Align = Size * sizeof(Scalar) }; + enum : size_t { Align = Size * 32 }; static constexpr bool StoreAsRef = true; private: - alignas(Size * sizeof(Scalar)) Scalar x_[N]; + alignas(Size * 32) Scalar x_[N]; public: #define ARRAY_BOILERPLATE \ From 11d34b3f2e97041935cebcdc6cb3292ce89627d3 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:26:02 +0200 Subject: [PATCH 006/102] Add config option and indicator for a debug discrete adjoint run. --- Common/include/CConfig.hpp | 9 ++++++++- Common/src/CConfig.cpp | 10 +++++++++- SU2_CFD/src/SU2_CFD.cpp | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index d55416ed681b..052706268751 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -1056,7 +1056,8 @@ class CConfig { long ParMETIS_pointWgt; /*!< \brief Load balancing weight given to points. */ long ParMETIS_edgeWgt; /*!< \brief Load balancing weight given to edges. */ unsigned short DirectDiff; /*!< \brief Direct Differentation mode. */ - bool DiscreteAdjoint; /*!< \brief AD-based discrete adjoint mode. */ + bool DiscreteAdjoint, /*!< \brief AD-based discrete adjoint mode. */ + DiscreteAdjointDebug; /*!< \brief Discrete adjoint debug mode using tags. */ su2double Const_DES; /*!< \brief Detached Eddy Simulation Constant. */ WINDOW_FUNCTION Kind_WindowFct; /*!< \brief Type of window (weight) function for objective functional. */ unsigned short Kind_HybridRANSLES; /*!< \brief Kind of Hybrid RANS/LES. */ @@ -8918,6 +8919,12 @@ class CConfig { */ bool GetDiscrete_Adjoint(void) const { return DiscreteAdjoint; } + /*! + * \brief Get the indicator whether a debug run for the discrete adjoint solver will be started. + * \return the discrete adjoint debug indicator. + */ + bool GetDiscrete_Adjoint_Debug(void) const { return DiscreteAdjointDebug; } + /*! * \brief Get the number of subiterations while a ramp is applied. * \return Number of FSI subiters. diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 8eaccde5472b..9674f3df765e 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1108,11 +1108,19 @@ void CConfig::SetConfig_Options() { addBoolOption("MULTIZONE", Multizone_Problem, NO); /*!\brief PHYSICAL_PROBLEM \n DESCRIPTION: Physical governing equations \n Options: see \link Solver_Map \endlink \n DEFAULT: NONE \ingroup Config*/ addEnumOption("MULTIZONE_SOLVER", Kind_MZSolver, Multizone_Map, ENUM_MULTIZONE::MZ_BLOCK_GAUSS_SEIDEL); -#ifdef CODI_REVERSE_TYPE +#if defined (CODI_REVERSE_TYPE) const bool discAdjDefault = true; +# if defined (CODI_TAG_TAPE) + const bool discAdjDebugDefault = true; +# else + const bool discAdjDebugDefault = false; +# endif #else const bool discAdjDefault = false; + const bool discAdjDebugDefault = false; #endif + // TODO Set the indicator through MATH_PROBLEM + DiscreteAdjointDebug = discAdjDebugDefault; /*!\brief MATH_PROBLEM \n DESCRIPTION: Mathematical problem \n Options: DIRECT, ADJOINT \ingroup Config*/ addMathProblemOption("MATH_PROBLEM", ContinuousAdjoint, false, DiscreteAdjoint, discAdjDefault, Restart_Flow, discAdjDefault); /*!\brief KIND_TURB_MODEL \n DESCRIPTION: Specify turbulence model \n Options: see \link Turb_Model_Map \endlink \n DEFAULT: NONE \ingroup Config*/ diff --git a/SU2_CFD/src/SU2_CFD.cpp b/SU2_CFD/src/SU2_CFD.cpp index 33bb1a93d15e..11aa2669310c 100644 --- a/SU2_CFD/src/SU2_CFD.cpp +++ b/SU2_CFD/src/SU2_CFD.cpp @@ -102,6 +102,7 @@ int main(int argc, char *argv[]) { and perform all the preprocessing. ---*/ const bool disc_adj = config.GetDiscrete_Adjoint(); + const bool disc_adj_debug = config.GetDiscrete_Adjoint_Debug(); const bool multizone = config.GetMultizone_Problem(); const bool harmonic_balance = (config.GetTime_Marching() == TIME_MARCHING::HARMONIC_BALANCE); From 756dc57f04b68b2227d2dc418696b93d54a2d6a0 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:29:45 +0200 Subject: [PATCH 007/102] Add routines for setting tags to AD structure. --- Common/include/basic_types/ad_structure.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index b8b71f102b88..087f51eb66dd 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -270,6 +270,12 @@ inline void SetExtFuncOut(T&& data, const int size_x, const int size_y) {} */ inline void SetIndex(Identifier& index, const su2double& data) {} +/*! + * \brief Sets the tag tape to a specific tag. + * \param[in] tag - the number to which the tag is set. + */ +inline void SetTag(int tag) {} + /*! * \brief Pushes back the current tape position to the tape position's vector. */ @@ -482,6 +488,8 @@ FORCEINLINE void EndUseAdjoints() { AD::getTape().endUseAdjointVector(); } FORCEINLINE void SetIndex(Identifier& index, const su2double& data) { index = data.getIdentifier(); } +FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } + // WARNING: For performance reasons, this method does not perform bounds checking. // When using it, please ensure sufficient adjoint vector size by a call to AD::ResizeAdjoints(). // This method does not perform locking either. From b1e4f6963ab8b263558479aa128c47fee000c2af Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:36:58 +0200 Subject: [PATCH 008/102] Add kinds of recording for tag tapes. --- Common/include/option_structure.hpp | 2 ++ .../include/drivers/CDiscAdjMultizoneDriver.hpp | 2 +- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 17 +++++++++++++++-- .../src/iteration/CDiscAdjFluidIteration.cpp | 5 ++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index ddf570ab6978..5db89f1481bf 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -2493,6 +2493,8 @@ enum class RECORDING { MESH_COORDS, MESH_DEFORM, SOLUTION_AND_MESH, + TAG_INIT_SOLUTION_VARIABLES, + TAG_CHECK_SOLUTION_VARIABLES }; /*! diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 368001bf0ea0..278427cd676b 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -69,7 +69,7 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { }; /*! - * \brief Kinds of recordings (three different ones). + * \brief Kinds of recordings. */ enum class Kind_Tape { FULL_TAPE, /*!< \brief Entire derivative information for a coupled adjoint diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 2dcda92415fc..b77335feeb22 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -597,6 +597,15 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t if(kind_recording != RECORDING::CLEAR_INDICES) { + if (kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES) { + cout << "Set Tag 1" << endl; + AD::SetTag(1); + } + else if (kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { + cout << "Set Tag 2" << endl; + AD::SetTag(2); + } + AD::StartRecording(); AD::Push_TapePosition(); /// START @@ -738,13 +747,17 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { } } + + if (rank == MASTER_NODE) { AD::RegisterOutput(ObjFunc); AD::SetIndex(ObjFunc_Index, ObjFunc); - if (kind_recording == RECORDING::SOLUTION_VARIABLES) { + if (kind_recording == RECORDING::SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { cout << " Objective function : " << ObjFunc; if (driver_config->GetWrt_AD_Statistics()){ - // cout << " (" << ObjFunc_Index << ")\n"; + cout << " (" << ObjFunc_Index << ")\n"; } cout << endl; } diff --git a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp index d25e4a18ecbe..d498866095ab 100644 --- a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp @@ -404,7 +404,10 @@ void CDiscAdjFluidIteration::RegisterInput(CSolver***** solver, CGeometry**** ge SU2_OMP_PARALLEL_(if(solvers0[ADJFLOW_SOL]->GetHasHybridParallel())) { - if (kind_recording == RECORDING::SOLUTION_VARIABLES || kind_recording == RECORDING::SOLUTION_AND_MESH) { + if (kind_recording == RECORDING::SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES || + kind_recording == RECORDING::SOLUTION_AND_MESH) { /*--- Register flow and turbulent variables as input ---*/ if (config[iZone]->GetFluidProblem()) { From cc7b0cf736bf8578b04f1f3ae30dcff1c8dac73b Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:41:30 +0200 Subject: [PATCH 009/102] Comment out checks in AD structure whether an identifier is active. This is a temporary change to avoid false positives when a tag tape is used. They might be unnecessary anyway though. --- Common/include/basic_types/ad_structure.hpp | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 087f51eb66dd..7c43b6d84c38 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -522,7 +522,8 @@ FORCEINLINE void SetPreaccIn() {} template ::value> = 0> FORCEINLINE void SetPreaccIn(const T& data, Ts&&... moreData) { if (!PreaccActive) return; - if (IsIdentifierActive(data)) PreaccHelper.addInput(data); + // if (IsIdentifierActive(data)) + PreaccHelper.addInput(data); SetPreaccIn(moreData...); } @@ -535,9 +536,9 @@ template FORCEINLINE void SetPreaccIn(const T& data, const int size) { if (PreaccActive) { for (int i = 0; i < size; i++) { - if (IsIdentifierActive(data[i])) { + // if (IsIdentifierActive(data[i])) { PreaccHelper.addInput(data[i]); - } + // } } } } @@ -547,9 +548,9 @@ FORCEINLINE void SetPreaccIn(const T& data, const int size_x, const int size_y) if (!PreaccActive) return; for (int i = 0; i < size_x; i++) { for (int j = 0; j < size_y; j++) { - if (IsIdentifierActive(data[i][j])) { + // if (IsIdentifierActive(data[i][j])) { PreaccHelper.addInput(data[i][j]); - } + // } } } } @@ -567,7 +568,8 @@ FORCEINLINE void SetPreaccOut() {} template ::value> = 0> FORCEINLINE void SetPreaccOut(T& data, Ts&&... moreData) { if (!PreaccActive) return; - if (IsIdentifierActive(data)) PreaccHelper.addOutput(data); + // if (IsIdentifierActive(data)) + PreaccHelper.addOutput(data); SetPreaccOut(moreData...); } @@ -575,9 +577,9 @@ template FORCEINLINE void SetPreaccOut(T&& data, const int size) { if (PreaccActive) { for (int i = 0; i < size; i++) { - if (IsIdentifierActive(data[i])) { + // if (IsIdentifierActive(data[i])) { PreaccHelper.addOutput(data[i]); - } + // } } } } @@ -587,9 +589,9 @@ FORCEINLINE void SetPreaccOut(T&& data, const int size_x, const int size_y) { if (!PreaccActive) return; for (int i = 0; i < size_x; i++) { for (int j = 0; j < size_y; j++) { - if (IsIdentifierActive(data[i][j])) { + // if (IsIdentifierActive(data[i][j])) { PreaccHelper.addOutput(data[i][j]); - } + // } } } } From 44cde90c6916b1db92788ebf3aec63b1069ac6f3 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:54:47 +0200 Subject: [PATCH 010/102] Add first WIP version of the tag debug mode to the (multizone) discrete adjoint solver. --- .../drivers/CDiscAdjMultizoneDriver.hpp | 5 +++ SU2_CFD/include/drivers/CDriver.hpp | 5 +++ .../src/drivers/CDiscAdjMultizoneDriver.cpp | 32 +++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 278427cd676b..8b4ef2274808 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -141,6 +141,11 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { */ void StartSolver() override; + /*! + * \brief [Overload] Launch the debug mode for the discrete adjoint multizone solver. + */ + void DebugRun() override; + /*! * \brief Preprocess the multizone iteration */ diff --git a/SU2_CFD/include/drivers/CDriver.hpp b/SU2_CFD/include/drivers/CDriver.hpp index 2f79be415b3d..d93a87b18dbe 100644 --- a/SU2_CFD/include/drivers/CDriver.hpp +++ b/SU2_CFD/include/drivers/CDriver.hpp @@ -425,6 +425,11 @@ class CDriver : public CDriverBase { */ virtual void StartSolver() {} + /*! + * \brief Launch a debug run of the solver. + */ + virtual void DebugRun() {} + /*! * \brief Deallocation routine */ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index b77335feeb22..edc9a88fbc90 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -159,6 +159,16 @@ void CDiscAdjMultizoneDriver::Preprocess(unsigned long TimeIter) { void CDiscAdjMultizoneDriver::StartSolver() { + /*--- Start the debug recording mode for the discrete adjoint solver. ---*/ + + if (driver_config->GetDiscrete_Adjoint_Debug()) { + + Preprocess(0); + + DebugRun(); + return; + } + const bool time_domain = driver_config->GetTime_Domain(); /*--- Main external loop of the solver. Runs for the number of time steps required. ---*/ @@ -217,6 +227,28 @@ void CDiscAdjMultizoneDriver::StartSolver() { } +void CDiscAdjMultizoneDriver::DebugRun() { + + cout <<"\n------------------------------ Start Debug Run -----------------------------" << endl; + + cout <<"\n------------------------------ Check Objective Function Tape ---------------" << endl; + + cout << "Initial recording ..." << endl; + /*--- This recording will assign the initial (same) tag to each registered variable. + * During the recording, each dependent variable will be assigned the same tag. ---*/ + SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + + cout << "Second recording to check first recording ..." << endl; + /*--- This recording repeats the initial recording with a different tag. + * If a variable was used before it became dependent on the inputs, this variable will still carry the tag + * from the initial recording and a mismatch with the "check" recording tag will throw an error. + * In such a case, a possible reason could be that such a variable is set by a post-processing routine while + * for a mathematically correct recording this dependency must be included earlier. ---*/ + SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + + cout <<"\n------------------------------ End Debug Run -----------------------------" << endl; +} + bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { config_container[iZone]->SetInnerIter(iInnerIter); From 0bbd83255c1f17c1c2953f63eaba58fb6ba8502a Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 18:15:39 +0200 Subject: [PATCH 011/102] Revert small change (will be added to CoDi soon). --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index edc9a88fbc90..e9ba6bc53fb3 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -789,7 +789,7 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { cout << " Objective function : " << ObjFunc; if (driver_config->GetWrt_AD_Statistics()){ - cout << " (" << ObjFunc_Index << ")\n"; + // cout << " (" << ObjFunc_Index << ")\n"; } cout << endl; } From 89f6fe971d3d5585e435d9f280049c3acde3faad Mon Sep 17 00:00:00 2001 From: Josh Kelly Date: Thu, 6 Mar 2025 15:21:55 +0000 Subject: [PATCH 012/102] Resolve tagging issue in PassiveAssign --- Common/include/basic_types/datatype_structure.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Common/include/basic_types/datatype_structure.hpp b/Common/include/basic_types/datatype_structure.hpp index 1698cea3f43e..8af2d78a950b 100644 --- a/Common/include/basic_types/datatype_structure.hpp +++ b/Common/include/basic_types/datatype_structure.hpp @@ -89,8 +89,13 @@ void SetDerivative(su2double& data, const passivedouble& val); FORCEINLINE void SetValue(su2double& data, const passivedouble& val) { data.setValue(val); } +FORCEINLINE passivedouble GetValue(const &double data) { return data; } + FORCEINLINE passivedouble GetValue(const su2double& data) { return data.getValue(); } +template +FORCEINLINE passivedouble GetValue(const codi::ExpressionInterface& data) { return data.cast().getValue(); } + FORCEINLINE void SetSecondary(su2double& data, const passivedouble& val) { data.setGradient(val); } FORCEINLINE void SetDerivative(su2double& data, const passivedouble& val) { data.setGradient(val); } From 9b44431c254359d8fa1444fe52c790971979732f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 17 Mar 2025 15:05:21 +0100 Subject: [PATCH 013/102] Quick fix for new GetValue. --- Common/include/basic_types/datatype_structure.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/basic_types/datatype_structure.hpp b/Common/include/basic_types/datatype_structure.hpp index 8af2d78a950b..fd801790c83f 100644 --- a/Common/include/basic_types/datatype_structure.hpp +++ b/Common/include/basic_types/datatype_structure.hpp @@ -89,7 +89,7 @@ void SetDerivative(su2double& data, const passivedouble& val); FORCEINLINE void SetValue(su2double& data, const passivedouble& val) { data.setValue(val); } -FORCEINLINE passivedouble GetValue(const &double data) { return data; } +FORCEINLINE passivedouble GetValue(const double& data) { return data; } FORCEINLINE passivedouble GetValue(const su2double& data) { return data.getValue(); } From a9f851cd36ecbcfc0603183e6d52f73c1a77c95e Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 19 Mar 2025 15:10:03 +0100 Subject: [PATCH 014/102] Align and improve debug mode screen output. --- .../src/drivers/CDiscAdjMultizoneDriver.cpp | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index e9ba6bc53fb3..3192ad3d96d1 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -229,16 +229,12 @@ void CDiscAdjMultizoneDriver::StartSolver() { void CDiscAdjMultizoneDriver::DebugRun() { - cout <<"\n------------------------------ Start Debug Run -----------------------------" << endl; + cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; - cout <<"\n------------------------------ Check Objective Function Tape ---------------" << endl; - - cout << "Initial recording ..." << endl; /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); - cout << "Second recording to check first recording ..." << endl; /*--- This recording repeats the initial recording with a different tag. * If a variable was used before it became dependent on the inputs, this variable will still carry the tag * from the initial recording and a mismatch with the "check" recording tag will throw an error. @@ -246,7 +242,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { * for a mathematically correct recording this dependency must be included earlier. ---*/ SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); - cout <<"\n------------------------------ End Debug Run -----------------------------" << endl; + cout <<"\n----------------------------- End Debug Run ---------------------------" << endl; } bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { @@ -621,6 +617,8 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t case RECORDING::CLEAR_INDICES: cout << "Clearing the computational graph." << endl; break; case RECORDING::MESH_COORDS: cout << "Storing computational graph wrt MESH COORDINATES." << endl; break; case RECORDING::SOLUTION_VARIABLES: cout << "Storing computational graph wrt CONSERVATIVE VARIABLES." << endl; break; + case RECORDING::TAG_INIT_SOLUTION_VARIABLES: cout << "Simulating recording with tag 1." << endl; AD::SetTag(1); break; + case RECORDING::TAG_CHECK_SOLUTION_VARIABLES: cout << "Checking first recording with tag 2." << endl; AD::SetTag(2); break; default: break; } } @@ -629,15 +627,6 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t if(kind_recording != RECORDING::CLEAR_INDICES) { - if (kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES) { - cout << "Set Tag 1" << endl; - AD::SetTag(1); - } - else if (kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { - cout << "Set Tag 2" << endl; - AD::SetTag(2); - } - AD::StartRecording(); AD::Push_TapePosition(); /// START From 409a763b76380fbab496948e0cd83b91c0a69c5f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 19 Mar 2025 15:25:33 +0100 Subject: [PATCH 015/102] Further align debug mode screen output. --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 3192ad3d96d1..ff6297260262 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -242,7 +242,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { * for a mathematically correct recording this dependency must be included earlier. ---*/ SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); - cout <<"\n----------------------------- End Debug Run ---------------------------" << endl; + cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; } bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { From 13bfa9aa8d290e0f753895b8c8118b687537b844 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 20 Mar 2025 15:22:55 +0100 Subject: [PATCH 016/102] Remove identifier check for preaccumulation, as now covered by CoDi. --- Common/include/basic_types/ad_structure.hpp | 26 +++++---------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 7c43b6d84c38..29d5382dddb9 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -512,18 +512,13 @@ FORCEINLINE double GetDerivative(Identifier index) { FORCEINLINE Identifier GetPassiveIndex() { return AD::getTape().getPassiveIndex(); } -FORCEINLINE bool IsIdentifierActive(su2double const& value) { - return getTape().isIdentifierActive(value.getIdentifier()); -} - /*--- Base case for parameter pack expansion. ---*/ FORCEINLINE void SetPreaccIn() {} template ::value> = 0> FORCEINLINE void SetPreaccIn(const T& data, Ts&&... moreData) { if (!PreaccActive) return; - // if (IsIdentifierActive(data)) - PreaccHelper.addInput(data); + PreaccHelper.addInput(data); SetPreaccIn(moreData...); } @@ -536,9 +531,7 @@ template FORCEINLINE void SetPreaccIn(const T& data, const int size) { if (PreaccActive) { for (int i = 0; i < size; i++) { - // if (IsIdentifierActive(data[i])) { - PreaccHelper.addInput(data[i]); - // } + PreaccHelper.addInput(data[i]); } } } @@ -548,9 +541,7 @@ FORCEINLINE void SetPreaccIn(const T& data, const int size_x, const int size_y) if (!PreaccActive) return; for (int i = 0; i < size_x; i++) { for (int j = 0; j < size_y; j++) { - // if (IsIdentifierActive(data[i][j])) { - PreaccHelper.addInput(data[i][j]); - // } + PreaccHelper.addInput(data[i][j]); } } } @@ -568,8 +559,7 @@ FORCEINLINE void SetPreaccOut() {} template ::value> = 0> FORCEINLINE void SetPreaccOut(T& data, Ts&&... moreData) { if (!PreaccActive) return; - // if (IsIdentifierActive(data)) - PreaccHelper.addOutput(data); + PreaccHelper.addOutput(data); SetPreaccOut(moreData...); } @@ -577,9 +567,7 @@ template FORCEINLINE void SetPreaccOut(T&& data, const int size) { if (PreaccActive) { for (int i = 0; i < size; i++) { - // if (IsIdentifierActive(data[i])) { - PreaccHelper.addOutput(data[i]); - // } + PreaccHelper.addOutput(data[i]); } } } @@ -589,9 +577,7 @@ FORCEINLINE void SetPreaccOut(T&& data, const int size_x, const int size_y) { if (!PreaccActive) return; for (int i = 0; i < size_x; i++) { for (int j = 0; j < size_y; j++) { - // if (IsIdentifierActive(data[i][j])) { - PreaccHelper.addOutput(data[i][j]); - // } + PreaccHelper.addOutput(data[i][j]); } } } From 937398387e510d7afc57b40946166f40b56dc89e Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 20 Mar 2025 15:33:02 +0100 Subject: [PATCH 017/102] Remove printing objective function recording index to screen. --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index ff6297260262..2b1828792276 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -776,11 +776,7 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { if (kind_recording == RECORDING::SOLUTION_VARIABLES || kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES || kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { - cout << " Objective function : " << ObjFunc; - if (driver_config->GetWrt_AD_Statistics()){ - // cout << " (" << ObjFunc_Index << ")\n"; - } - cout << endl; + cout << " Objective function : " << ObjFunc << endl; } } } From 76ad14b2f3d0dd6ecb4d4ddc6ba12b4d6e528189 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 3 Apr 2025 14:11:08 +0200 Subject: [PATCH 018/102] Small fix. --- Common/include/basic_types/ad_structure.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 29d5382dddb9..345b4c6e66b7 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -488,8 +488,6 @@ FORCEINLINE void EndUseAdjoints() { AD::getTape().endUseAdjointVector(); } FORCEINLINE void SetIndex(Identifier& index, const su2double& data) { index = data.getIdentifier(); } -FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } - // WARNING: For performance reasons, this method does not perform bounds checking. // When using it, please ensure sufficient adjoint vector size by a call to AD::ResizeAdjoints(). // This method does not perform locking either. @@ -684,6 +682,12 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = true;) } +# ifdef CODI_TAG_TAPE +FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } +# else +FORCEINLINE void SetTag(int tag) { } +# endif + #endif // CODI_REVERSE_TYPE void Initialize(); From 67c97dd171be3ef2702afd67df0ad543221f3a5f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 3 Apr 2025 14:17:05 +0200 Subject: [PATCH 019/102] Simplify config (build type determines the run mode anyway). --- Common/src/CConfig.cpp | 8 +++----- SU2_CFD/src/SU2_CFD.cpp | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 4b3b165c2a09..961c40621d6d 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1111,16 +1111,14 @@ void CConfig::SetConfig_Options() { #if defined (CODI_REVERSE_TYPE) const bool discAdjDefault = true; # if defined (CODI_TAG_TAPE) - const bool discAdjDebugDefault = true; + DiscreteAdjointDebug = true; # else - const bool discAdjDebugDefault = false; + DiscreteAdjointDebug = false; # endif #else const bool discAdjDefault = false; - const bool discAdjDebugDefault = false; + DiscreteAdjointDebug = false; #endif - // TODO Set the indicator through MATH_PROBLEM - DiscreteAdjointDebug = discAdjDebugDefault; /*!\brief MATH_PROBLEM \n DESCRIPTION: Mathematical problem \n Options: DIRECT, ADJOINT \ingroup Config*/ addMathProblemOption("MATH_PROBLEM", ContinuousAdjoint, false, DiscreteAdjoint, discAdjDefault, Restart_Flow, discAdjDefault); /*!\brief KIND_TURB_MODEL \n DESCRIPTION: Specify turbulence model \n Options: see \link Turb_Model_Map \endlink \n DEFAULT: NONE \ingroup Config*/ diff --git a/SU2_CFD/src/SU2_CFD.cpp b/SU2_CFD/src/SU2_CFD.cpp index 4678737f0185..148071f56942 100644 --- a/SU2_CFD/src/SU2_CFD.cpp +++ b/SU2_CFD/src/SU2_CFD.cpp @@ -92,7 +92,6 @@ int main(int argc, char *argv[]) { and perform all the preprocessing. ---*/ const bool disc_adj = config.GetDiscrete_Adjoint(); - const bool disc_adj_debug = config.GetDiscrete_Adjoint_Debug(); const bool multizone = config.GetMultizone_Problem(); const bool harmonic_balance = (config.GetTime_Marching() == TIME_MARCHING::HARMONIC_BALANCE); From 72a6b2a75dd7448bc4009b78030a237ced8aa19c Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 3 Apr 2025 14:40:09 +0200 Subject: [PATCH 020/102] Add debug mode message. --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 2b1828792276..252887538334 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -163,8 +163,9 @@ void CDiscAdjMultizoneDriver::StartSolver() { if (driver_config->GetDiscrete_Adjoint_Debug()) { - Preprocess(0); + cout << "SU2_CFD_AD is compiled for debug mode recording. To resume the discrete adjoint solver, adjust -Dcodi-tape (-Dcodi-tape=JacobianLinear by default) and recompile." << endl; + Preprocess(0); DebugRun(); return; } From bb0f66e8f1057ddf1242b8d0446c9f37ef718f68 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 3 Apr 2025 15:09:57 +0200 Subject: [PATCH 021/102] New messages on master rank only. --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 252887538334..1b2eada10a28 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -163,8 +163,9 @@ void CDiscAdjMultizoneDriver::StartSolver() { if (driver_config->GetDiscrete_Adjoint_Debug()) { - cout << "SU2_CFD_AD is compiled for debug mode recording. To resume the discrete adjoint solver, adjust -Dcodi-tape (-Dcodi-tape=JacobianLinear by default) and recompile." << endl; - + if (rank == MASTER_NODE) { + cout << "\nSU2_CFD_AD is compiled for debug mode recording. To resume the discrete adjoint solver, adjust -Dcodi-tape (-Dcodi-tape=JacobianLinear by default) and recompile." << endl; + } Preprocess(0); DebugRun(); return; @@ -230,7 +231,9 @@ void CDiscAdjMultizoneDriver::StartSolver() { void CDiscAdjMultizoneDriver::DebugRun() { - cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; + if (rank == MASTER_NODE) { + cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; + } /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ @@ -243,7 +246,9 @@ void CDiscAdjMultizoneDriver::DebugRun() { * for a mathematically correct recording this dependency must be included earlier. ---*/ SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); - cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; + if (rank == MASTER_NODE) { + cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; + } } bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { From 99e1551494160642affa83614447fb3508453f2f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 4 Apr 2025 09:26:50 +0200 Subject: [PATCH 022/102] Turn OUTPUT_WRT_FREQ error for disc. adj. multizone into a warning. --- Common/include/CConfig.hpp | 5 +++++ Common/src/CConfig.cpp | 6 +----- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index e36589c8b017..b3848535a20d 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -9605,6 +9605,11 @@ class CConfig { */ unsigned short GetnVolumeOutputFiles() const { return nVolumeOutputFiles; } + /*! + * \brief GetnVolumeOutputFrequencies + */ + unsigned short GetnVolumeOutputFrequencies() const { return nVolumeOutputFrequencies; } + /*! * \brief GetVolumeOutputFrequency * \param[in] iFile: index of file number for which the writing frequency needs to be returned. diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 961c40621d6d..696728ffa573 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -3418,16 +3418,12 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i /*--- Using default frequency of 250 for all files when steady, and 1 for unsteady. ---*/ for (auto iVolumeFreq = 0; iVolumeFreq < nVolumeOutputFrequencies; iVolumeFreq++){ if (Multizone_Problem && DiscreteAdjoint) { - VolumeOutputFrequencies[iVolumeFreq] = nOuterIter; + VolumeOutputFrequencies[iVolumeFreq] = Time_Domain ? 1 : nOuterIter; } else { VolumeOutputFrequencies[iVolumeFreq] = Time_Domain ? 1 : 250; } } - } else if (Multizone_Problem && DiscreteAdjoint) { - SU2_MPI::Error(string("OUTPUT_WRT_FREQ cannot be specified for this solver " - "(writing of restart and sensitivity files not possible for multizone discrete adjoint during runtime yet).\n" - "Please remove this option from the config file, output files will be written when solver finalizes.\n"), CURRENT_FUNCTION); } else if (nVolumeOutputFrequencies < nVolumeOutputFiles) { /*--- If there are fewer frequencies than files, repeat the last frequency. * This is useful to define 1 frequency for the restart file and 1 frequency for all the visualization files. ---*/ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 1b2eada10a28..2695c0782776 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -344,6 +344,21 @@ void CDiscAdjMultizoneDriver::Run() { const bool time_domain = driver_config->GetTime_Domain(); driver_config->Set_StartTime(SU2_MPI::Wtime()); + /*--- Temporary warning because we need to test writing intermediate output to file (requires re-recording). ---*/ + for(iZone = 0; iZone < nZone; iZone++) { + for (auto iVolumeFreq = 0; iVolumeFreq < config_container[iZone]->GetnVolumeOutputFrequencies(); iVolumeFreq++){ + if (config_container[iZone]->GetVolumeOutputFrequency(iVolumeFreq) < nOuterIter) { + if (rank == MASTER_NODE) { + cout << "\nWARNING (iZone = " << iZone + << "): " + "Writing out restart files during solver iterations is not tested for the discrete adjoint multizone solver.\n" + "It is recommended to remove OUTPUT_WRT_FREQ from the config file, output files will be written when solver finalizes." << std::endl; + } + break; + } + } + } + /*--- If the gradient of the objective function is 0 so are the adjoint variables. * Unless in unsteady problems where there are other contributions to the RHS. ---*/ From f12232fadb251f158c6c0fe0825b9e4f90d09202 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 4 Apr 2025 09:49:03 +0200 Subject: [PATCH 023/102] Clang format. --- Common/include/basic_types/ad_structure.hpp | 8 ++++---- Common/include/basic_types/datatype_structure.hpp | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 345b4c6e66b7..90437794050c 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -682,11 +682,11 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = true;) } -# ifdef CODI_TAG_TAPE +#ifdef CODI_TAG_TAPE FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } -# else -FORCEINLINE void SetTag(int tag) { } -# endif +#else +FORCEINLINE void SetTag(int tag) {} +#endif #endif // CODI_REVERSE_TYPE diff --git a/Common/include/basic_types/datatype_structure.hpp b/Common/include/basic_types/datatype_structure.hpp index fd801790c83f..c78580e218ad 100644 --- a/Common/include/basic_types/datatype_structure.hpp +++ b/Common/include/basic_types/datatype_structure.hpp @@ -93,8 +93,10 @@ FORCEINLINE passivedouble GetValue(const double& data) { return data; } FORCEINLINE passivedouble GetValue(const su2double& data) { return data.getValue(); } -template -FORCEINLINE passivedouble GetValue(const codi::ExpressionInterface& data) { return data.cast().getValue(); } +template +FORCEINLINE passivedouble GetValue(const codi::ExpressionInterface& data) { + return data.cast().getValue(); +} FORCEINLINE void SetSecondary(su2double& data, const passivedouble& val) { data.setGradient(val); } From 16e27a2fcccfa5750fc8065e71b8275c9f782fae Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 7 Apr 2025 13:29:33 +0200 Subject: [PATCH 024/102] Clang format. --- Common/include/basic_types/ad_structure.hpp | 4 ++-- Common/include/geometry/dual_grid/CPoint.hpp | 6 ++++-- Common/include/parallelization/vectorization.hpp | 2 +- SU2_CFD/include/variables/CVariable.hpp | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 90437794050c..fb65027ba712 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -493,8 +493,8 @@ FORCEINLINE void SetIndex(Identifier& index, const su2double& data) { index = da // This method does not perform locking either. // It should be safeguarded by calls to AD::BeginUseAdjoints() and AD::EndUseAdjoints(). FORCEINLINE void SetDerivative(Identifier index, const double val) { - if (!AD::getTape().isIdentifierActive(index)) // Allow multiple threads to "set the derivative" of passive variables without causing data races. - return; + // Allow multiple threads to "set the derivative" of passive variables without causing data races. + if (!AD::getTape().isIdentifierActive(index)) return; AD::getTape().setGradient(index, val, codi::AdjointsManagement::Manual); } diff --git a/Common/include/geometry/dual_grid/CPoint.hpp b/Common/include/geometry/dual_grid/CPoint.hpp index 88d9e57adc40..eb6656a8a96f 100644 --- a/Common/include/geometry/dual_grid/CPoint.hpp +++ b/Common/include/geometry/dual_grid/CPoint.hpp @@ -113,8 +113,10 @@ class CPoint { su2activevector MaxLength; /*!< \brief The maximum cell-center to cell-center length. */ su2activevector RoughnessHeight; /*!< \brief Roughness of the nearest wall. */ - su2matrix AD_InputIndex; /*!< \brief Indices of Coord variables in the adjoint vector. */ - su2matrix AD_OutputIndex; /*!< \brief Indices of Coord variables in the adjoint vector after having been updated. */ + su2matrix + AD_InputIndex; /*!< \brief Indices of Coord variables in the adjoint vector before solver iteration. */ + su2matrix + AD_OutputIndex; /*!< \brief Indices of Coord variables in the adjoint vector after solver iteration. */ /*! * \brief Allocate fields required by the minimal constructor. diff --git a/Common/include/parallelization/vectorization.hpp b/Common/include/parallelization/vectorization.hpp index 3250033730d1..aec19541c34b 100644 --- a/Common/include/parallelization/vectorization.hpp +++ b/Common/include/parallelization/vectorization.hpp @@ -94,7 +94,7 @@ class Array : public CVecExpr, Scalar_t> { static constexpr bool StoreAsRef = true; private: - alignas(Size * 32) Scalar x_[N]; + alignas(Size * 32) Scalar x_[N]; public: #define ARRAY_BOILERPLATE \ diff --git a/SU2_CFD/include/variables/CVariable.hpp b/SU2_CFD/include/variables/CVariable.hpp index 7e039142e49f..8b81a7411774 100644 --- a/SU2_CFD/include/variables/CVariable.hpp +++ b/SU2_CFD/include/variables/CVariable.hpp @@ -92,8 +92,8 @@ class CVariable { MatrixType Solution_BGS_k; /*!< \brief Old solution container for BGS iterations. */ - su2matrix AD_InputIndex; /*!< \brief Indices of Solution variables in the adjoint vector. */ - su2matrix AD_OutputIndex; /*!< \brief Indices of Solution variables in the adjoint vector after having been updated. */ + su2matrix AD_InputIndex; /*!< \brief Indices of Solution variables in the adjoint vector before solver iteration. */ + su2matrix AD_OutputIndex; /*!< \brief Indices of Solution variables in the adjoint vector after solver iteration. */ VectorType SolutionExtra; /*!< \brief Stores adjoint solution for extra solution variables. Currently only streamwise periodic pressure-drop for massflow prescribed flows. */ From 0eaad4dbd6a92acd89a51583d72332c61b009900 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 14 Aug 2024 18:17:14 +0200 Subject: [PATCH 025/102] Throw error message in case the multizone discrete adjoint solver will write restart files during runtime (there is a bug, potentially when clearing the tape). --- Common/src/CConfig.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index a18129ed5517..fd571f9b3d25 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -3411,8 +3411,17 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i /*--- Using default frequency of 250 for all files when steady, and 1 for unsteady. ---*/ for (auto iVolumeFreq = 0; iVolumeFreq < nVolumeOutputFrequencies; iVolumeFreq++){ - VolumeOutputFrequencies[iVolumeFreq] = Time_Domain ? 1 : 250; + if (Multizone_Problem && DiscreteAdjoint) { + VolumeOutputFrequencies[iVolumeFreq] = nOuterIter; + } + else { + VolumeOutputFrequencies[iVolumeFreq] = Time_Domain ? 1 : 250; + } } + } else if (Multizone_Problem && DiscreteAdjoint) { + SU2_MPI::Error(string("OUTPUT_WRT_FREQ cannot be specified for this solver " + "(writing of restart and sensitivity files not possible for multizone discrete adjoint during runtime yet).\n" + "Please remove this option from the config file, output files will be written when solver finalizes.\n"), CURRENT_FUNCTION); } else if (nVolumeOutputFrequencies < nVolumeOutputFiles) { /*--- If there are fewer frequencies than files, repeat the last frequency. * This is useful to define 1 frequency for the restart file and 1 frequency for all the visualization files. ---*/ From 6e2f468782aa9995007321528bbbbb2964f31deb Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 16 Aug 2024 00:22:57 +0200 Subject: [PATCH 026/102] Softcode indentifier/index. --- Common/include/basic_types/ad_structure.hpp | 26 ++++++++++++++----- Common/include/geometry/dual_grid/CPoint.hpp | 5 ++-- Common/src/geometry/dual_grid/CPoint.cpp | 4 +-- .../drivers/CDiscAdjMultizoneDriver.hpp | 2 +- SU2_CFD/include/variables/CVariable.hpp | 12 ++++----- .../src/drivers/CDiscAdjMultizoneDriver.cpp | 2 +- SU2_CFD/src/solvers/CSolver.cpp | 2 +- SU2_CFD/src/variables/CVariable.cpp | 4 +-- 8 files changed, 34 insertions(+), 23 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index de902141e89a..b8b71f102b88 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -38,6 +38,9 @@ */ namespace AD { #ifndef CODI_REVERSE_TYPE + +using Identifier = int; + /*! * \brief Start the recording of the operations and involved variables. * If called, the computational graph of all operations occuring after the call will be stored, @@ -101,14 +104,20 @@ inline void EndUseAdjoints() {} * \param[in] index - Position in the adjoint vector. * \param[in] val - adjoint value to be set. */ -inline void SetDerivative(int index, const double val) {} +inline void SetDerivative(Identifier index, const double val) {} /*! * \brief Extracts the adjoint value at index * \param[in] index - position in the adjoint vector where the derivative will be extracted. * \return Derivative value. */ -inline double GetDerivative(int index) { return 0.0; } +inline double GetDerivative(Identifier index) { return 0.0; } + +/*! + * \brief Returns the identifier that represents an inactive variable. + * \return Passive index. + */ +inline Identifier GetPassiveIndex() { return 0; } /*! * \brief Clears the currently stored adjoints but keeps the computational graph. @@ -259,7 +268,7 @@ inline void SetExtFuncOut(T&& data, const int size_x, const int size_y) {} * \param[in] data - variable whose gradient information will be extracted. * \param[in] index - where obtained gradient information will be stored. */ -inline void SetIndex(int& index, const su2double& data) {} +inline void SetIndex(Identifier& index, const su2double& data) {} /*! * \brief Pushes back the current tape position to the tape position's vector. @@ -304,6 +313,7 @@ inline void EndNoSharedReading() {} using CheckpointHandler = codi::ExternalFunctionUserData; using Tape = su2double::Tape; +using Identifier = su2double::Identifier; #ifdef HAVE_OPDI using ExtFuncHelper = codi::OpenMPExternalFunctionHelper; @@ -470,14 +480,14 @@ FORCEINLINE void BeginUseAdjoints() { AD::getTape().beginUseAdjointVector(); } FORCEINLINE void EndUseAdjoints() { AD::getTape().endUseAdjointVector(); } -FORCEINLINE void SetIndex(int& index, const su2double& data) { index = data.getIdentifier(); } +FORCEINLINE void SetIndex(Identifier& index, const su2double& data) { index = data.getIdentifier(); } // WARNING: For performance reasons, this method does not perform bounds checking. // When using it, please ensure sufficient adjoint vector size by a call to AD::ResizeAdjoints(). // This method does not perform locking either. // It should be safeguarded by calls to AD::BeginUseAdjoints() and AD::EndUseAdjoints(). -FORCEINLINE void SetDerivative(int index, const double val) { - if (index == 0) // Allow multiple threads to "set the derivative" of passive variables without causing data races. +FORCEINLINE void SetDerivative(Identifier index, const double val) { + if (!AD::getTape().isIdentifierActive(index)) // Allow multiple threads to "set the derivative" of passive variables without causing data races. return; AD::getTape().setGradient(index, val, codi::AdjointsManagement::Manual); @@ -488,10 +498,12 @@ FORCEINLINE void SetDerivative(int index, const double val) { // Otherwise, please ensure sufficient adjoint vector size by a call to AD::ResizeAdjoints(). // This method does not perform locking either. // It should be safeguarded by calls to AD::BeginUseAdjoints() and AD::EndUseAdjoints(). -FORCEINLINE double GetDerivative(int index) { +FORCEINLINE double GetDerivative(Identifier index) { return AD::getTape().getGradient(index, codi::AdjointsManagement::Manual); } +FORCEINLINE Identifier GetPassiveIndex() { return AD::getTape().getPassiveIndex(); } + FORCEINLINE bool IsIdentifierActive(su2double const& value) { return getTape().isIdentifierActive(value.getIdentifier()); } diff --git a/Common/include/geometry/dual_grid/CPoint.hpp b/Common/include/geometry/dual_grid/CPoint.hpp index e497df353079..88d9e57adc40 100644 --- a/Common/include/geometry/dual_grid/CPoint.hpp +++ b/Common/include/geometry/dual_grid/CPoint.hpp @@ -113,9 +113,8 @@ class CPoint { su2activevector MaxLength; /*!< \brief The maximum cell-center to cell-center length. */ su2activevector RoughnessHeight; /*!< \brief Roughness of the nearest wall. */ - su2matrix AD_InputIndex; /*!< \brief Indices of Coord variables in the adjoint vector. */ - su2matrix - AD_OutputIndex; /*!< \brief Indices of Coord variables in the adjoint vector after having been updated. */ + su2matrix AD_InputIndex; /*!< \brief Indices of Coord variables in the adjoint vector. */ + su2matrix AD_OutputIndex; /*!< \brief Indices of Coord variables in the adjoint vector after having been updated. */ /*! * \brief Allocate fields required by the minimal constructor. diff --git a/Common/src/geometry/dual_grid/CPoint.cpp b/Common/src/geometry/dual_grid/CPoint.cpp index 7096dfa6443f..3e1e2a463b68 100644 --- a/Common/src/geometry/dual_grid/CPoint.cpp +++ b/Common/src/geometry/dual_grid/CPoint.cpp @@ -67,8 +67,8 @@ void CPoint::FullAllocation(unsigned short imesh, const CConfig* config) { } if (config->GetDiscrete_Adjoint()) { - AD_InputIndex.resize(npoint, nDim) = 0; - AD_OutputIndex.resize(npoint, nDim) = 0; + AD_InputIndex.resize(npoint, nDim) = AD::GetPassiveIndex(); + AD_OutputIndex.resize(npoint, nDim) = AD::GetPassiveIndex(); } /*--- Multigrid structures. ---*/ diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 7022220fc07c..368001bf0ea0 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -96,7 +96,7 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { bool eval_transfer = false; /*!< \brief Evaluate the transfer section of the tape. */ su2double ObjFunc; /*!< \brief Value of the objective function. */ - int ObjFunc_Index; /*!< \brief Index of the value of the objective function. */ + AD::Identifier ObjFunc_Index; /*!< \brief Index of the value of the objective function. */ CIteration*** direct_iteration; /*!< \brief Array of pointers to the direct iterations. */ COutput** direct_output; /*!< \brief Array of pointers to the direct outputs. */ diff --git a/SU2_CFD/include/variables/CVariable.hpp b/SU2_CFD/include/variables/CVariable.hpp index 9a7e456860f1..2d93a21c5bd1 100644 --- a/SU2_CFD/include/variables/CVariable.hpp +++ b/SU2_CFD/include/variables/CVariable.hpp @@ -92,8 +92,8 @@ class CVariable { MatrixType Solution_BGS_k; /*!< \brief Old solution container for BGS iterations. */ - su2matrix AD_InputIndex; /*!< \brief Indices of Solution variables in the adjoint vector. */ - su2matrix AD_OutputIndex; /*!< \brief Indices of Solution variables in the adjoint vector after having been updated. */ + su2matrix AD_InputIndex; /*!< \brief Indices of Solution variables in the adjoint vector. */ + su2matrix AD_OutputIndex; /*!< \brief Indices of Solution variables in the adjoint vector after having been updated. */ VectorType SolutionExtra; /*!< \brief Stores adjoint solution for extra solution variables. Currently only streamwise periodic pressure-drop for massflow prescribed flows. */ @@ -118,7 +118,7 @@ class CVariable { assert(false && "A base method of CVariable was used, but it should have been overridden by the derived class."); } - void RegisterContainer(bool input, su2activematrix& variable, su2matrix* ad_index = nullptr) { + void RegisterContainer(bool input, su2activematrix& variable, su2matrix* ad_index = nullptr) { const auto nPoint = variable.rows(); SU2_OMP_FOR_STAT(roundUpDiv(nPoint,omp_get_num_threads())) for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { @@ -133,7 +133,7 @@ class CVariable { END_SU2_OMP_FOR } - void RegisterContainer(bool input, su2activematrix& variable, su2matrix& ad_index) { + void RegisterContainer(bool input, su2activematrix& variable, su2matrix& ad_index) { RegisterContainer(input, variable, &ad_index); } @@ -2187,7 +2187,7 @@ class CVariable { } inline void GetAdjointSolution_time_n(unsigned long iPoint, su2double *adj_sol) const { - int index = 0; + AD::Identifier index = AD::GetPassiveIndex(); for (unsigned long iVar = 0; iVar < Solution_time_n.cols(); iVar++) { AD::SetIndex(index, Solution_time_n(iPoint, iVar)); adj_sol[iVar] = AD::GetDerivative(index); @@ -2195,7 +2195,7 @@ class CVariable { } inline void GetAdjointSolution_time_n1(unsigned long iPoint, su2double *adj_sol) const { - int index = 0; + AD::Identifier index = AD::GetPassiveIndex(); for (unsigned long iVar = 0; iVar < Solution_time_n1.cols(); iVar++) { AD::SetIndex(index, Solution_time_n1(iPoint, iVar)); adj_sol[iVar] = AD::GetDerivative(index); diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 6e5519a63f2e..2dcda92415fc 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -744,7 +744,7 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { if (kind_recording == RECORDING::SOLUTION_VARIABLES) { cout << " Objective function : " << ObjFunc; if (driver_config->GetWrt_AD_Statistics()){ - cout << " (" << ObjFunc_Index << ")\n"; + // cout << " (" << ObjFunc_Index << ")\n"; } cout << endl; } diff --git a/SU2_CFD/src/solvers/CSolver.cpp b/SU2_CFD/src/solvers/CSolver.cpp index 009910e91a2a..df54f2d7746e 100644 --- a/SU2_CFD/src/solvers/CSolver.cpp +++ b/SU2_CFD/src/solvers/CSolver.cpp @@ -4098,7 +4098,7 @@ void CSolver::SetVertexTractionsAdjoint(CGeometry *geometry, const CConfig *conf unsigned short iMarker, iDim; unsigned long iVertex, iPoint; - int index; + AD::Identifier index; /*--- Loop over all the markers ---*/ for (iMarker = 0; iMarker < config->GetnMarker_All(); iMarker++) { diff --git a/SU2_CFD/src/variables/CVariable.cpp b/SU2_CFD/src/variables/CVariable.cpp index 98ab867b2e92..ece1113bf369 100644 --- a/SU2_CFD/src/variables/CVariable.cpp +++ b/SU2_CFD/src/variables/CVariable.cpp @@ -69,8 +69,8 @@ CVariable::CVariable(unsigned long npoint, unsigned long ndim, unsigned long nva External.resize(nPoint,nVar) = su2double(0.0); if (!adjoint) { - AD_InputIndex.resize(nPoint,nVar) = -1; - AD_OutputIndex.resize(nPoint,nVar) = -1; + AD_InputIndex.resize(nPoint,nVar) = AD::GetPassiveIndex(); + AD_OutputIndex.resize(nPoint,nVar) = AD::GetPassiveIndex(); } } From 1ed6c8d9899c6bd6e5f9279a378e5f75898cca85 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 16 Aug 2024 00:48:49 +0200 Subject: [PATCH 027/102] Add tag type option to build system. --- meson.build | 2 ++ meson_options.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/meson.build b/meson.build index e6886833f61d..2a1cbdf998c0 100644 --- a/meson.build +++ b/meson.build @@ -95,6 +95,8 @@ if get_option('enable-autodiff') and not omp codi_rev_args += '-DCODI_PRIMAL_REUSE_TAPE' elif get_option('codi-tape') == 'PrimalMultiUse' codi_rev_args += '-DCODI_PRIMAL_MULTIUSE_TAPE' + elif get_option('codi-tape') == 'Tag' + codi_rev_args += '-DCODI_TAG_TAPE' else error('Invalid CoDiPack tape choice @0@'.format(get_option('codi-tape'))) endif diff --git a/meson_options.txt b/meson_options.txt index 55782c62ff80..f062074dacb7 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -23,7 +23,7 @@ option('enable-coolprop', type : 'boolean', value : false, description: 'enable option('enable-mlpcpp', type : 'boolean', value : false, description: 'enable MLPCpp support') option('enable-gprof', type : 'boolean', value : false, description: 'enable profiling through gprof') option('opdi-backend', type : 'combo', choices : ['auto', 'macro', 'ompt'], value : 'auto', description: 'OpDiLib backend choice') -option('codi-tape', type : 'combo', choices : ['JacobianLinear', 'JacobianReuse', 'JacobianMultiUse', 'PrimalLinear', 'PrimalReuse', 'PrimalMultiUse'], value : 'JacobianLinear', description: 'CoDiPack tape choice') +option('codi-tape', type : 'combo', choices : ['JacobianLinear', 'JacobianReuse', 'JacobianMultiUse', 'PrimalLinear', 'PrimalReuse', 'PrimalMultiUse', 'Tag'], value : 'JacobianLinear', description: 'CoDiPack tape choice') option('opdi-shared-read-opt', type : 'boolean', value : true, description : 'OpDiLib shared reading optimization') option('librom_root', type : 'string', value : '', description: 'libROM base directory') option('enable-librom', type : 'boolean', value : false, description: 'enable LLNL libROM support') From d7e63eb5122dcd741350c7b2d9f97055bd05f578 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 26 Aug 2024 00:07:43 +0200 Subject: [PATCH 028/102] Add option to use RealReverseTag as AD type. --- Common/include/code_config.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Common/include/code_config.hpp b/Common/include/code_config.hpp index cbbf2c531535..30fc11de2be0 100644 --- a/Common/include/code_config.hpp +++ b/Common/include/code_config.hpp @@ -110,6 +110,8 @@ using su2double = codi::RealReversePrimal; using su2double = codi::RealReversePrimalIndexGen >; #elif defined(CODI_PRIMAL_MULTIUSE_TAPE) using su2double = codi::RealReversePrimalIndex; +#elif defined(CODI_TAG_TAPE) +using su2double = codi::RealReverseTag; #else #error "Please define a CoDiPack tape." #endif From 036fe696d31f8f20664b407aad97f292b67d6585 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 26 Aug 2024 00:28:44 +0200 Subject: [PATCH 029/102] Temporarily adjust/hardcode type size for memory alignment in vectorization.hpp (to cover RealReverseTag). --- Common/include/parallelization/vectorization.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Common/include/parallelization/vectorization.hpp b/Common/include/parallelization/vectorization.hpp index 50316359fc84..3250033730d1 100644 --- a/Common/include/parallelization/vectorization.hpp +++ b/Common/include/parallelization/vectorization.hpp @@ -90,11 +90,11 @@ class Array : public CVecExpr, Scalar_t> { public: using Scalar = Scalar_t; enum : size_t { Size = N }; - enum : size_t { Align = Size * sizeof(Scalar) }; + enum : size_t { Align = Size * 32 }; static constexpr bool StoreAsRef = true; private: - alignas(Size * sizeof(Scalar)) Scalar x_[N]; + alignas(Size * 32) Scalar x_[N]; public: #define ARRAY_BOILERPLATE \ From ec79b1e4dc441bbffb1987ed974cb7b23bcd7455 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:26:02 +0200 Subject: [PATCH 030/102] Add config option and indicator for a debug discrete adjoint run. --- Common/include/CConfig.hpp | 9 ++++++++- Common/src/CConfig.cpp | 10 +++++++++- SU2_CFD/src/SU2_CFD.cpp | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index acb669e78193..e36589c8b017 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -1054,7 +1054,8 @@ class CConfig { long ParMETIS_pointWgt; /*!< \brief Load balancing weight given to points. */ long ParMETIS_edgeWgt; /*!< \brief Load balancing weight given to edges. */ unsigned short DirectDiff; /*!< \brief Direct Differentation mode. */ - bool DiscreteAdjoint; /*!< \brief AD-based discrete adjoint mode. */ + bool DiscreteAdjoint, /*!< \brief AD-based discrete adjoint mode. */ + DiscreteAdjointDebug; /*!< \brief Discrete adjoint debug mode using tags. */ su2double Const_DES; /*!< \brief Detached Eddy Simulation Constant. */ WINDOW_FUNCTION Kind_WindowFct; /*!< \brief Type of window (weight) function for objective functional. */ unsigned short Kind_HybridRANSLES; /*!< \brief Kind of Hybrid RANS/LES. */ @@ -8772,6 +8773,12 @@ class CConfig { */ bool GetDiscrete_Adjoint(void) const { return DiscreteAdjoint; } + /*! + * \brief Get the indicator whether a debug run for the discrete adjoint solver will be started. + * \return the discrete adjoint debug indicator. + */ + bool GetDiscrete_Adjoint_Debug(void) const { return DiscreteAdjointDebug; } + /*! * \brief Get the number of subiterations while a ramp is applied. * \return Number of FSI subiters. diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index fd571f9b3d25..4b3b165c2a09 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1108,11 +1108,19 @@ void CConfig::SetConfig_Options() { addBoolOption("MULTIZONE", Multizone_Problem, NO); /*!\brief PHYSICAL_PROBLEM \n DESCRIPTION: Physical governing equations \n Options: see \link Solver_Map \endlink \n DEFAULT: NONE \ingroup Config*/ addEnumOption("MULTIZONE_SOLVER", Kind_MZSolver, Multizone_Map, ENUM_MULTIZONE::MZ_BLOCK_GAUSS_SEIDEL); -#ifdef CODI_REVERSE_TYPE +#if defined (CODI_REVERSE_TYPE) const bool discAdjDefault = true; +# if defined (CODI_TAG_TAPE) + const bool discAdjDebugDefault = true; +# else + const bool discAdjDebugDefault = false; +# endif #else const bool discAdjDefault = false; + const bool discAdjDebugDefault = false; #endif + // TODO Set the indicator through MATH_PROBLEM + DiscreteAdjointDebug = discAdjDebugDefault; /*!\brief MATH_PROBLEM \n DESCRIPTION: Mathematical problem \n Options: DIRECT, ADJOINT \ingroup Config*/ addMathProblemOption("MATH_PROBLEM", ContinuousAdjoint, false, DiscreteAdjoint, discAdjDefault, Restart_Flow, discAdjDefault); /*!\brief KIND_TURB_MODEL \n DESCRIPTION: Specify turbulence model \n Options: see \link Turb_Model_Map \endlink \n DEFAULT: NONE \ingroup Config*/ diff --git a/SU2_CFD/src/SU2_CFD.cpp b/SU2_CFD/src/SU2_CFD.cpp index 148071f56942..4678737f0185 100644 --- a/SU2_CFD/src/SU2_CFD.cpp +++ b/SU2_CFD/src/SU2_CFD.cpp @@ -92,6 +92,7 @@ int main(int argc, char *argv[]) { and perform all the preprocessing. ---*/ const bool disc_adj = config.GetDiscrete_Adjoint(); + const bool disc_adj_debug = config.GetDiscrete_Adjoint_Debug(); const bool multizone = config.GetMultizone_Problem(); const bool harmonic_balance = (config.GetTime_Marching() == TIME_MARCHING::HARMONIC_BALANCE); From b4d797a46d01efa30405a001ddb78b10c1f67c03 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:29:45 +0200 Subject: [PATCH 031/102] Add routines for setting tags to AD structure. --- Common/include/basic_types/ad_structure.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index b8b71f102b88..087f51eb66dd 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -270,6 +270,12 @@ inline void SetExtFuncOut(T&& data, const int size_x, const int size_y) {} */ inline void SetIndex(Identifier& index, const su2double& data) {} +/*! + * \brief Sets the tag tape to a specific tag. + * \param[in] tag - the number to which the tag is set. + */ +inline void SetTag(int tag) {} + /*! * \brief Pushes back the current tape position to the tape position's vector. */ @@ -482,6 +488,8 @@ FORCEINLINE void EndUseAdjoints() { AD::getTape().endUseAdjointVector(); } FORCEINLINE void SetIndex(Identifier& index, const su2double& data) { index = data.getIdentifier(); } +FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } + // WARNING: For performance reasons, this method does not perform bounds checking. // When using it, please ensure sufficient adjoint vector size by a call to AD::ResizeAdjoints(). // This method does not perform locking either. From 642c341133c2bc926638347186ce1953d248d63b Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:36:58 +0200 Subject: [PATCH 032/102] Add kinds of recording for tag tapes. --- Common/include/option_structure.hpp | 2 ++ .../include/drivers/CDiscAdjMultizoneDriver.hpp | 2 +- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 17 +++++++++++++++-- .../src/iteration/CDiscAdjFluidIteration.cpp | 5 ++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 2bc268970883..278fa52b83d2 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -2550,6 +2550,8 @@ enum class RECORDING { MESH_COORDS, MESH_DEFORM, SOLUTION_AND_MESH, + TAG_INIT_SOLUTION_VARIABLES, + TAG_CHECK_SOLUTION_VARIABLES }; /*! diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 368001bf0ea0..278427cd676b 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -69,7 +69,7 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { }; /*! - * \brief Kinds of recordings (three different ones). + * \brief Kinds of recordings. */ enum class Kind_Tape { FULL_TAPE, /*!< \brief Entire derivative information for a coupled adjoint diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 2dcda92415fc..b77335feeb22 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -597,6 +597,15 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t if(kind_recording != RECORDING::CLEAR_INDICES) { + if (kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES) { + cout << "Set Tag 1" << endl; + AD::SetTag(1); + } + else if (kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { + cout << "Set Tag 2" << endl; + AD::SetTag(2); + } + AD::StartRecording(); AD::Push_TapePosition(); /// START @@ -738,13 +747,17 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { } } + + if (rank == MASTER_NODE) { AD::RegisterOutput(ObjFunc); AD::SetIndex(ObjFunc_Index, ObjFunc); - if (kind_recording == RECORDING::SOLUTION_VARIABLES) { + if (kind_recording == RECORDING::SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { cout << " Objective function : " << ObjFunc; if (driver_config->GetWrt_AD_Statistics()){ - // cout << " (" << ObjFunc_Index << ")\n"; + cout << " (" << ObjFunc_Index << ")\n"; } cout << endl; } diff --git a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp index d25e4a18ecbe..d498866095ab 100644 --- a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp @@ -404,7 +404,10 @@ void CDiscAdjFluidIteration::RegisterInput(CSolver***** solver, CGeometry**** ge SU2_OMP_PARALLEL_(if(solvers0[ADJFLOW_SOL]->GetHasHybridParallel())) { - if (kind_recording == RECORDING::SOLUTION_VARIABLES || kind_recording == RECORDING::SOLUTION_AND_MESH) { + if (kind_recording == RECORDING::SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES || + kind_recording == RECORDING::SOLUTION_AND_MESH) { /*--- Register flow and turbulent variables as input ---*/ if (config[iZone]->GetFluidProblem()) { From 5aaaa9bd60945906e25eeada6fd98a6d9f7d34f4 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:41:30 +0200 Subject: [PATCH 033/102] Comment out checks in AD structure whether an identifier is active. This is a temporary change to avoid false positives when a tag tape is used. They might be unnecessary anyway though. --- Common/include/basic_types/ad_structure.hpp | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 087f51eb66dd..7c43b6d84c38 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -522,7 +522,8 @@ FORCEINLINE void SetPreaccIn() {} template ::value> = 0> FORCEINLINE void SetPreaccIn(const T& data, Ts&&... moreData) { if (!PreaccActive) return; - if (IsIdentifierActive(data)) PreaccHelper.addInput(data); + // if (IsIdentifierActive(data)) + PreaccHelper.addInput(data); SetPreaccIn(moreData...); } @@ -535,9 +536,9 @@ template FORCEINLINE void SetPreaccIn(const T& data, const int size) { if (PreaccActive) { for (int i = 0; i < size; i++) { - if (IsIdentifierActive(data[i])) { + // if (IsIdentifierActive(data[i])) { PreaccHelper.addInput(data[i]); - } + // } } } } @@ -547,9 +548,9 @@ FORCEINLINE void SetPreaccIn(const T& data, const int size_x, const int size_y) if (!PreaccActive) return; for (int i = 0; i < size_x; i++) { for (int j = 0; j < size_y; j++) { - if (IsIdentifierActive(data[i][j])) { + // if (IsIdentifierActive(data[i][j])) { PreaccHelper.addInput(data[i][j]); - } + // } } } } @@ -567,7 +568,8 @@ FORCEINLINE void SetPreaccOut() {} template ::value> = 0> FORCEINLINE void SetPreaccOut(T& data, Ts&&... moreData) { if (!PreaccActive) return; - if (IsIdentifierActive(data)) PreaccHelper.addOutput(data); + // if (IsIdentifierActive(data)) + PreaccHelper.addOutput(data); SetPreaccOut(moreData...); } @@ -575,9 +577,9 @@ template FORCEINLINE void SetPreaccOut(T&& data, const int size) { if (PreaccActive) { for (int i = 0; i < size; i++) { - if (IsIdentifierActive(data[i])) { + // if (IsIdentifierActive(data[i])) { PreaccHelper.addOutput(data[i]); - } + // } } } } @@ -587,9 +589,9 @@ FORCEINLINE void SetPreaccOut(T&& data, const int size_x, const int size_y) { if (!PreaccActive) return; for (int i = 0; i < size_x; i++) { for (int j = 0; j < size_y; j++) { - if (IsIdentifierActive(data[i][j])) { + // if (IsIdentifierActive(data[i][j])) { PreaccHelper.addOutput(data[i][j]); - } + // } } } } From 727d303e7bab1496db56c6690322bcbcbc369f78 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 17:54:47 +0200 Subject: [PATCH 034/102] Add first WIP version of the tag debug mode to the (multizone) discrete adjoint solver. --- .../drivers/CDiscAdjMultizoneDriver.hpp | 5 +++ SU2_CFD/include/drivers/CDriver.hpp | 5 +++ .../src/drivers/CDiscAdjMultizoneDriver.cpp | 32 +++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 278427cd676b..8b4ef2274808 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -141,6 +141,11 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { */ void StartSolver() override; + /*! + * \brief [Overload] Launch the debug mode for the discrete adjoint multizone solver. + */ + void DebugRun() override; + /*! * \brief Preprocess the multizone iteration */ diff --git a/SU2_CFD/include/drivers/CDriver.hpp b/SU2_CFD/include/drivers/CDriver.hpp index 2f79be415b3d..d93a87b18dbe 100644 --- a/SU2_CFD/include/drivers/CDriver.hpp +++ b/SU2_CFD/include/drivers/CDriver.hpp @@ -425,6 +425,11 @@ class CDriver : public CDriverBase { */ virtual void StartSolver() {} + /*! + * \brief Launch a debug run of the solver. + */ + virtual void DebugRun() {} + /*! * \brief Deallocation routine */ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index b77335feeb22..edc9a88fbc90 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -159,6 +159,16 @@ void CDiscAdjMultizoneDriver::Preprocess(unsigned long TimeIter) { void CDiscAdjMultizoneDriver::StartSolver() { + /*--- Start the debug recording mode for the discrete adjoint solver. ---*/ + + if (driver_config->GetDiscrete_Adjoint_Debug()) { + + Preprocess(0); + + DebugRun(); + return; + } + const bool time_domain = driver_config->GetTime_Domain(); /*--- Main external loop of the solver. Runs for the number of time steps required. ---*/ @@ -217,6 +227,28 @@ void CDiscAdjMultizoneDriver::StartSolver() { } +void CDiscAdjMultizoneDriver::DebugRun() { + + cout <<"\n------------------------------ Start Debug Run -----------------------------" << endl; + + cout <<"\n------------------------------ Check Objective Function Tape ---------------" << endl; + + cout << "Initial recording ..." << endl; + /*--- This recording will assign the initial (same) tag to each registered variable. + * During the recording, each dependent variable will be assigned the same tag. ---*/ + SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + + cout << "Second recording to check first recording ..." << endl; + /*--- This recording repeats the initial recording with a different tag. + * If a variable was used before it became dependent on the inputs, this variable will still carry the tag + * from the initial recording and a mismatch with the "check" recording tag will throw an error. + * In such a case, a possible reason could be that such a variable is set by a post-processing routine while + * for a mathematically correct recording this dependency must be included earlier. ---*/ + SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + + cout <<"\n------------------------------ End Debug Run -----------------------------" << endl; +} + bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { config_container[iZone]->SetInnerIter(iInnerIter); From cd425715a3de1c904da428e3ddbe750fa605cbc5 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Sep 2024 18:15:39 +0200 Subject: [PATCH 035/102] Revert small change (will be added to CoDi soon). --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index edc9a88fbc90..e9ba6bc53fb3 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -789,7 +789,7 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { cout << " Objective function : " << ObjFunc; if (driver_config->GetWrt_AD_Statistics()){ - cout << " (" << ObjFunc_Index << ")\n"; + // cout << " (" << ObjFunc_Index << ")\n"; } cout << endl; } From b839e3af134231837a374a8167bdd1ac7fc09099 Mon Sep 17 00:00:00 2001 From: Josh Kelly Date: Thu, 6 Mar 2025 15:21:55 +0000 Subject: [PATCH 036/102] Resolve tagging issue in PassiveAssign --- Common/include/basic_types/datatype_structure.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Common/include/basic_types/datatype_structure.hpp b/Common/include/basic_types/datatype_structure.hpp index 1698cea3f43e..8af2d78a950b 100644 --- a/Common/include/basic_types/datatype_structure.hpp +++ b/Common/include/basic_types/datatype_structure.hpp @@ -89,8 +89,13 @@ void SetDerivative(su2double& data, const passivedouble& val); FORCEINLINE void SetValue(su2double& data, const passivedouble& val) { data.setValue(val); } +FORCEINLINE passivedouble GetValue(const &double data) { return data; } + FORCEINLINE passivedouble GetValue(const su2double& data) { return data.getValue(); } +template +FORCEINLINE passivedouble GetValue(const codi::ExpressionInterface& data) { return data.cast().getValue(); } + FORCEINLINE void SetSecondary(su2double& data, const passivedouble& val) { data.setGradient(val); } FORCEINLINE void SetDerivative(su2double& data, const passivedouble& val) { data.setGradient(val); } From cbaafb3ba5914d27380d2579066f6a7b8dca33c1 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 17 Mar 2025 15:05:21 +0100 Subject: [PATCH 037/102] Quick fix for new GetValue. --- Common/include/basic_types/datatype_structure.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/basic_types/datatype_structure.hpp b/Common/include/basic_types/datatype_structure.hpp index 8af2d78a950b..fd801790c83f 100644 --- a/Common/include/basic_types/datatype_structure.hpp +++ b/Common/include/basic_types/datatype_structure.hpp @@ -89,7 +89,7 @@ void SetDerivative(su2double& data, const passivedouble& val); FORCEINLINE void SetValue(su2double& data, const passivedouble& val) { data.setValue(val); } -FORCEINLINE passivedouble GetValue(const &double data) { return data; } +FORCEINLINE passivedouble GetValue(const double& data) { return data; } FORCEINLINE passivedouble GetValue(const su2double& data) { return data.getValue(); } From b1144d4be97086dc2fbc7467f3baca6f1ee7746e Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 19 Mar 2025 15:10:03 +0100 Subject: [PATCH 038/102] Align and improve debug mode screen output. --- .../src/drivers/CDiscAdjMultizoneDriver.cpp | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index e9ba6bc53fb3..3192ad3d96d1 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -229,16 +229,12 @@ void CDiscAdjMultizoneDriver::StartSolver() { void CDiscAdjMultizoneDriver::DebugRun() { - cout <<"\n------------------------------ Start Debug Run -----------------------------" << endl; + cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; - cout <<"\n------------------------------ Check Objective Function Tape ---------------" << endl; - - cout << "Initial recording ..." << endl; /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); - cout << "Second recording to check first recording ..." << endl; /*--- This recording repeats the initial recording with a different tag. * If a variable was used before it became dependent on the inputs, this variable will still carry the tag * from the initial recording and a mismatch with the "check" recording tag will throw an error. @@ -246,7 +242,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { * for a mathematically correct recording this dependency must be included earlier. ---*/ SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); - cout <<"\n------------------------------ End Debug Run -----------------------------" << endl; + cout <<"\n----------------------------- End Debug Run ---------------------------" << endl; } bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { @@ -621,6 +617,8 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t case RECORDING::CLEAR_INDICES: cout << "Clearing the computational graph." << endl; break; case RECORDING::MESH_COORDS: cout << "Storing computational graph wrt MESH COORDINATES." << endl; break; case RECORDING::SOLUTION_VARIABLES: cout << "Storing computational graph wrt CONSERVATIVE VARIABLES." << endl; break; + case RECORDING::TAG_INIT_SOLUTION_VARIABLES: cout << "Simulating recording with tag 1." << endl; AD::SetTag(1); break; + case RECORDING::TAG_CHECK_SOLUTION_VARIABLES: cout << "Checking first recording with tag 2." << endl; AD::SetTag(2); break; default: break; } } @@ -629,15 +627,6 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t if(kind_recording != RECORDING::CLEAR_INDICES) { - if (kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES) { - cout << "Set Tag 1" << endl; - AD::SetTag(1); - } - else if (kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { - cout << "Set Tag 2" << endl; - AD::SetTag(2); - } - AD::StartRecording(); AD::Push_TapePosition(); /// START From 5f15309fc4867c27ef37e1b53de921160cbdae95 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 19 Mar 2025 15:25:33 +0100 Subject: [PATCH 039/102] Further align debug mode screen output. --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 3192ad3d96d1..ff6297260262 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -242,7 +242,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { * for a mathematically correct recording this dependency must be included earlier. ---*/ SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); - cout <<"\n----------------------------- End Debug Run ---------------------------" << endl; + cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; } bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { From f1a54e9e161da8480497841478b4550cbf54ae85 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 20 Mar 2025 15:22:55 +0100 Subject: [PATCH 040/102] Remove identifier check for preaccumulation, as now covered by CoDi. --- Common/include/basic_types/ad_structure.hpp | 26 +++++---------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 7c43b6d84c38..29d5382dddb9 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -512,18 +512,13 @@ FORCEINLINE double GetDerivative(Identifier index) { FORCEINLINE Identifier GetPassiveIndex() { return AD::getTape().getPassiveIndex(); } -FORCEINLINE bool IsIdentifierActive(su2double const& value) { - return getTape().isIdentifierActive(value.getIdentifier()); -} - /*--- Base case for parameter pack expansion. ---*/ FORCEINLINE void SetPreaccIn() {} template ::value> = 0> FORCEINLINE void SetPreaccIn(const T& data, Ts&&... moreData) { if (!PreaccActive) return; - // if (IsIdentifierActive(data)) - PreaccHelper.addInput(data); + PreaccHelper.addInput(data); SetPreaccIn(moreData...); } @@ -536,9 +531,7 @@ template FORCEINLINE void SetPreaccIn(const T& data, const int size) { if (PreaccActive) { for (int i = 0; i < size; i++) { - // if (IsIdentifierActive(data[i])) { - PreaccHelper.addInput(data[i]); - // } + PreaccHelper.addInput(data[i]); } } } @@ -548,9 +541,7 @@ FORCEINLINE void SetPreaccIn(const T& data, const int size_x, const int size_y) if (!PreaccActive) return; for (int i = 0; i < size_x; i++) { for (int j = 0; j < size_y; j++) { - // if (IsIdentifierActive(data[i][j])) { - PreaccHelper.addInput(data[i][j]); - // } + PreaccHelper.addInput(data[i][j]); } } } @@ -568,8 +559,7 @@ FORCEINLINE void SetPreaccOut() {} template ::value> = 0> FORCEINLINE void SetPreaccOut(T& data, Ts&&... moreData) { if (!PreaccActive) return; - // if (IsIdentifierActive(data)) - PreaccHelper.addOutput(data); + PreaccHelper.addOutput(data); SetPreaccOut(moreData...); } @@ -577,9 +567,7 @@ template FORCEINLINE void SetPreaccOut(T&& data, const int size) { if (PreaccActive) { for (int i = 0; i < size; i++) { - // if (IsIdentifierActive(data[i])) { - PreaccHelper.addOutput(data[i]); - // } + PreaccHelper.addOutput(data[i]); } } } @@ -589,9 +577,7 @@ FORCEINLINE void SetPreaccOut(T&& data, const int size_x, const int size_y) { if (!PreaccActive) return; for (int i = 0; i < size_x; i++) { for (int j = 0; j < size_y; j++) { - // if (IsIdentifierActive(data[i][j])) { - PreaccHelper.addOutput(data[i][j]); - // } + PreaccHelper.addOutput(data[i][j]); } } } From 5f8113debb377ab86be638a67f76f50b05b5fc2f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 20 Mar 2025 15:33:02 +0100 Subject: [PATCH 041/102] Remove printing objective function recording index to screen. --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index ff6297260262..2b1828792276 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -776,11 +776,7 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { if (kind_recording == RECORDING::SOLUTION_VARIABLES || kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES || kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { - cout << " Objective function : " << ObjFunc; - if (driver_config->GetWrt_AD_Statistics()){ - // cout << " (" << ObjFunc_Index << ")\n"; - } - cout << endl; + cout << " Objective function : " << ObjFunc << endl; } } } From 92e58ed802f822b32063538b565911996ff17e9e Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 3 Apr 2025 14:11:08 +0200 Subject: [PATCH 042/102] Small fix. --- Common/include/basic_types/ad_structure.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 29d5382dddb9..345b4c6e66b7 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -488,8 +488,6 @@ FORCEINLINE void EndUseAdjoints() { AD::getTape().endUseAdjointVector(); } FORCEINLINE void SetIndex(Identifier& index, const su2double& data) { index = data.getIdentifier(); } -FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } - // WARNING: For performance reasons, this method does not perform bounds checking. // When using it, please ensure sufficient adjoint vector size by a call to AD::ResizeAdjoints(). // This method does not perform locking either. @@ -684,6 +682,12 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = true;) } +# ifdef CODI_TAG_TAPE +FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } +# else +FORCEINLINE void SetTag(int tag) { } +# endif + #endif // CODI_REVERSE_TYPE void Initialize(); From 008b2dc35e58c6143e62e55d00e09131f14f9750 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 3 Apr 2025 14:17:05 +0200 Subject: [PATCH 043/102] Simplify config (build type determines the run mode anyway). --- Common/src/CConfig.cpp | 8 +++----- SU2_CFD/src/SU2_CFD.cpp | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 4b3b165c2a09..961c40621d6d 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1111,16 +1111,14 @@ void CConfig::SetConfig_Options() { #if defined (CODI_REVERSE_TYPE) const bool discAdjDefault = true; # if defined (CODI_TAG_TAPE) - const bool discAdjDebugDefault = true; + DiscreteAdjointDebug = true; # else - const bool discAdjDebugDefault = false; + DiscreteAdjointDebug = false; # endif #else const bool discAdjDefault = false; - const bool discAdjDebugDefault = false; + DiscreteAdjointDebug = false; #endif - // TODO Set the indicator through MATH_PROBLEM - DiscreteAdjointDebug = discAdjDebugDefault; /*!\brief MATH_PROBLEM \n DESCRIPTION: Mathematical problem \n Options: DIRECT, ADJOINT \ingroup Config*/ addMathProblemOption("MATH_PROBLEM", ContinuousAdjoint, false, DiscreteAdjoint, discAdjDefault, Restart_Flow, discAdjDefault); /*!\brief KIND_TURB_MODEL \n DESCRIPTION: Specify turbulence model \n Options: see \link Turb_Model_Map \endlink \n DEFAULT: NONE \ingroup Config*/ diff --git a/SU2_CFD/src/SU2_CFD.cpp b/SU2_CFD/src/SU2_CFD.cpp index 4678737f0185..148071f56942 100644 --- a/SU2_CFD/src/SU2_CFD.cpp +++ b/SU2_CFD/src/SU2_CFD.cpp @@ -92,7 +92,6 @@ int main(int argc, char *argv[]) { and perform all the preprocessing. ---*/ const bool disc_adj = config.GetDiscrete_Adjoint(); - const bool disc_adj_debug = config.GetDiscrete_Adjoint_Debug(); const bool multizone = config.GetMultizone_Problem(); const bool harmonic_balance = (config.GetTime_Marching() == TIME_MARCHING::HARMONIC_BALANCE); From baf99a972b6beb1355a2656d206555ca741369d7 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 3 Apr 2025 14:40:09 +0200 Subject: [PATCH 044/102] Add debug mode message. --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 2b1828792276..252887538334 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -163,8 +163,9 @@ void CDiscAdjMultizoneDriver::StartSolver() { if (driver_config->GetDiscrete_Adjoint_Debug()) { - Preprocess(0); + cout << "SU2_CFD_AD is compiled for debug mode recording. To resume the discrete adjoint solver, adjust -Dcodi-tape (-Dcodi-tape=JacobianLinear by default) and recompile." << endl; + Preprocess(0); DebugRun(); return; } From 3a9825aa3515ad386666583cd62b5bf7f1d8acba Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 3 Apr 2025 15:09:57 +0200 Subject: [PATCH 045/102] New messages on master rank only. --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 252887538334..1b2eada10a28 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -163,8 +163,9 @@ void CDiscAdjMultizoneDriver::StartSolver() { if (driver_config->GetDiscrete_Adjoint_Debug()) { - cout << "SU2_CFD_AD is compiled for debug mode recording. To resume the discrete adjoint solver, adjust -Dcodi-tape (-Dcodi-tape=JacobianLinear by default) and recompile." << endl; - + if (rank == MASTER_NODE) { + cout << "\nSU2_CFD_AD is compiled for debug mode recording. To resume the discrete adjoint solver, adjust -Dcodi-tape (-Dcodi-tape=JacobianLinear by default) and recompile." << endl; + } Preprocess(0); DebugRun(); return; @@ -230,7 +231,9 @@ void CDiscAdjMultizoneDriver::StartSolver() { void CDiscAdjMultizoneDriver::DebugRun() { - cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; + if (rank == MASTER_NODE) { + cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; + } /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ @@ -243,7 +246,9 @@ void CDiscAdjMultizoneDriver::DebugRun() { * for a mathematically correct recording this dependency must be included earlier. ---*/ SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); - cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; + if (rank == MASTER_NODE) { + cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; + } } bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { From ef29b2bc031709e8d6081893a87381f79b218043 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 4 Apr 2025 09:26:50 +0200 Subject: [PATCH 046/102] Turn OUTPUT_WRT_FREQ error for disc. adj. multizone into a warning. --- Common/include/CConfig.hpp | 5 +++++ Common/src/CConfig.cpp | 6 +----- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index e36589c8b017..b3848535a20d 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -9605,6 +9605,11 @@ class CConfig { */ unsigned short GetnVolumeOutputFiles() const { return nVolumeOutputFiles; } + /*! + * \brief GetnVolumeOutputFrequencies + */ + unsigned short GetnVolumeOutputFrequencies() const { return nVolumeOutputFrequencies; } + /*! * \brief GetVolumeOutputFrequency * \param[in] iFile: index of file number for which the writing frequency needs to be returned. diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 961c40621d6d..696728ffa573 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -3418,16 +3418,12 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i /*--- Using default frequency of 250 for all files when steady, and 1 for unsteady. ---*/ for (auto iVolumeFreq = 0; iVolumeFreq < nVolumeOutputFrequencies; iVolumeFreq++){ if (Multizone_Problem && DiscreteAdjoint) { - VolumeOutputFrequencies[iVolumeFreq] = nOuterIter; + VolumeOutputFrequencies[iVolumeFreq] = Time_Domain ? 1 : nOuterIter; } else { VolumeOutputFrequencies[iVolumeFreq] = Time_Domain ? 1 : 250; } } - } else if (Multizone_Problem && DiscreteAdjoint) { - SU2_MPI::Error(string("OUTPUT_WRT_FREQ cannot be specified for this solver " - "(writing of restart and sensitivity files not possible for multizone discrete adjoint during runtime yet).\n" - "Please remove this option from the config file, output files will be written when solver finalizes.\n"), CURRENT_FUNCTION); } else if (nVolumeOutputFrequencies < nVolumeOutputFiles) { /*--- If there are fewer frequencies than files, repeat the last frequency. * This is useful to define 1 frequency for the restart file and 1 frequency for all the visualization files. ---*/ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 1b2eada10a28..2695c0782776 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -344,6 +344,21 @@ void CDiscAdjMultizoneDriver::Run() { const bool time_domain = driver_config->GetTime_Domain(); driver_config->Set_StartTime(SU2_MPI::Wtime()); + /*--- Temporary warning because we need to test writing intermediate output to file (requires re-recording). ---*/ + for(iZone = 0; iZone < nZone; iZone++) { + for (auto iVolumeFreq = 0; iVolumeFreq < config_container[iZone]->GetnVolumeOutputFrequencies(); iVolumeFreq++){ + if (config_container[iZone]->GetVolumeOutputFrequency(iVolumeFreq) < nOuterIter) { + if (rank == MASTER_NODE) { + cout << "\nWARNING (iZone = " << iZone + << "): " + "Writing out restart files during solver iterations is not tested for the discrete adjoint multizone solver.\n" + "It is recommended to remove OUTPUT_WRT_FREQ from the config file, output files will be written when solver finalizes." << std::endl; + } + break; + } + } + } + /*--- If the gradient of the objective function is 0 so are the adjoint variables. * Unless in unsteady problems where there are other contributions to the RHS. ---*/ From 645d144447d5ea04cf05ee670c076d8d3856d488 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 4 Apr 2025 09:49:03 +0200 Subject: [PATCH 047/102] Clang format. --- Common/include/basic_types/ad_structure.hpp | 8 ++++---- Common/include/basic_types/datatype_structure.hpp | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 345b4c6e66b7..90437794050c 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -682,11 +682,11 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = true;) } -# ifdef CODI_TAG_TAPE +#ifdef CODI_TAG_TAPE FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } -# else -FORCEINLINE void SetTag(int tag) { } -# endif +#else +FORCEINLINE void SetTag(int tag) {} +#endif #endif // CODI_REVERSE_TYPE diff --git a/Common/include/basic_types/datatype_structure.hpp b/Common/include/basic_types/datatype_structure.hpp index fd801790c83f..c78580e218ad 100644 --- a/Common/include/basic_types/datatype_structure.hpp +++ b/Common/include/basic_types/datatype_structure.hpp @@ -93,8 +93,10 @@ FORCEINLINE passivedouble GetValue(const double& data) { return data; } FORCEINLINE passivedouble GetValue(const su2double& data) { return data.getValue(); } -template -FORCEINLINE passivedouble GetValue(const codi::ExpressionInterface& data) { return data.cast().getValue(); } +template +FORCEINLINE passivedouble GetValue(const codi::ExpressionInterface& data) { + return data.cast().getValue(); +} FORCEINLINE void SetSecondary(su2double& data, const passivedouble& val) { data.setGradient(val); } From eb5fb72c59b8c4b03656e1aa4f692da42cff8533 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 7 Apr 2025 13:29:33 +0200 Subject: [PATCH 048/102] Clang format. --- Common/include/basic_types/ad_structure.hpp | 4 ++-- Common/include/geometry/dual_grid/CPoint.hpp | 6 ++++-- Common/include/parallelization/vectorization.hpp | 2 +- SU2_CFD/include/variables/CVariable.hpp | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 90437794050c..fb65027ba712 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -493,8 +493,8 @@ FORCEINLINE void SetIndex(Identifier& index, const su2double& data) { index = da // This method does not perform locking either. // It should be safeguarded by calls to AD::BeginUseAdjoints() and AD::EndUseAdjoints(). FORCEINLINE void SetDerivative(Identifier index, const double val) { - if (!AD::getTape().isIdentifierActive(index)) // Allow multiple threads to "set the derivative" of passive variables without causing data races. - return; + // Allow multiple threads to "set the derivative" of passive variables without causing data races. + if (!AD::getTape().isIdentifierActive(index)) return; AD::getTape().setGradient(index, val, codi::AdjointsManagement::Manual); } diff --git a/Common/include/geometry/dual_grid/CPoint.hpp b/Common/include/geometry/dual_grid/CPoint.hpp index 88d9e57adc40..eb6656a8a96f 100644 --- a/Common/include/geometry/dual_grid/CPoint.hpp +++ b/Common/include/geometry/dual_grid/CPoint.hpp @@ -113,8 +113,10 @@ class CPoint { su2activevector MaxLength; /*!< \brief The maximum cell-center to cell-center length. */ su2activevector RoughnessHeight; /*!< \brief Roughness of the nearest wall. */ - su2matrix AD_InputIndex; /*!< \brief Indices of Coord variables in the adjoint vector. */ - su2matrix AD_OutputIndex; /*!< \brief Indices of Coord variables in the adjoint vector after having been updated. */ + su2matrix + AD_InputIndex; /*!< \brief Indices of Coord variables in the adjoint vector before solver iteration. */ + su2matrix + AD_OutputIndex; /*!< \brief Indices of Coord variables in the adjoint vector after solver iteration. */ /*! * \brief Allocate fields required by the minimal constructor. diff --git a/Common/include/parallelization/vectorization.hpp b/Common/include/parallelization/vectorization.hpp index 3250033730d1..aec19541c34b 100644 --- a/Common/include/parallelization/vectorization.hpp +++ b/Common/include/parallelization/vectorization.hpp @@ -94,7 +94,7 @@ class Array : public CVecExpr, Scalar_t> { static constexpr bool StoreAsRef = true; private: - alignas(Size * 32) Scalar x_[N]; + alignas(Size * 32) Scalar x_[N]; public: #define ARRAY_BOILERPLATE \ diff --git a/SU2_CFD/include/variables/CVariable.hpp b/SU2_CFD/include/variables/CVariable.hpp index 2d93a21c5bd1..b124634a284d 100644 --- a/SU2_CFD/include/variables/CVariable.hpp +++ b/SU2_CFD/include/variables/CVariable.hpp @@ -92,8 +92,8 @@ class CVariable { MatrixType Solution_BGS_k; /*!< \brief Old solution container for BGS iterations. */ - su2matrix AD_InputIndex; /*!< \brief Indices of Solution variables in the adjoint vector. */ - su2matrix AD_OutputIndex; /*!< \brief Indices of Solution variables in the adjoint vector after having been updated. */ + su2matrix AD_InputIndex; /*!< \brief Indices of Solution variables in the adjoint vector before solver iteration. */ + su2matrix AD_OutputIndex; /*!< \brief Indices of Solution variables in the adjoint vector after solver iteration. */ VectorType SolutionExtra; /*!< \brief Stores adjoint solution for extra solution variables. Currently only streamwise periodic pressure-drop for massflow prescribed flows. */ From 21290cb8afe2c3e67f6c0b437e0408586a4b8fcf Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 7 May 2025 14:57:45 +0200 Subject: [PATCH 049/102] Resolve merge issue. --- SU2_CFD/include/variables/CVariable.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/include/variables/CVariable.hpp b/SU2_CFD/include/variables/CVariable.hpp index b883d4734af2..3ff773038e11 100644 --- a/SU2_CFD/include/variables/CVariable.hpp +++ b/SU2_CFD/include/variables/CVariable.hpp @@ -137,7 +137,7 @@ class CVariable { RegisterContainer(input, variable, &ad_index); } - void RegisterContainer(bool input, su2activevector& variable, su2vector* ad_index = nullptr) { + void RegisterContainer(bool input, su2activevector& variable, su2vector* ad_index = nullptr) { const auto nPoint = variable.rows(); SU2_OMP_FOR_STAT(roundUpDiv(nPoint,omp_get_num_threads())) for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { From 05bb80e85b2c2e0d28c251927691ccd39891077d Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 8 May 2025 09:10:53 +0200 Subject: [PATCH 050/102] Add option to debug mode to check for mesh coordinates as tape input, too. --- Common/include/CConfig.hpp | 16 +++++++- Common/include/option_structure.hpp | 27 ++++++++++++- Common/src/CConfig.cpp | 6 +++ .../src/drivers/CDiscAdjMultizoneDriver.cpp | 40 +++++++++++++++++-- .../src/iteration/CDiscAdjFluidIteration.cpp | 5 ++- 5 files changed, 86 insertions(+), 8 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 9548ca9d600b..26a3cb122d75 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -596,8 +596,10 @@ class CConfig { //bool ContactResistance = false; /*!< \brief Apply contact resistance for conjugate heat transfer. */ unsigned short nID_DV; /*!< \brief ID for the region of FEM when computed using direct differentiation. */ - bool AD_Mode; /*!< \brief Algorithmic Differentiation support. */ - bool AD_Preaccumulation; /*!< \brief Enable or disable preaccumulation in the AD mode. */ + bool AD_Mode; /*!< \brief Algorithmic Differentiation support. */ + bool AD_Preaccumulation; /*!< \brief Enable or disable preaccumulation in the AD mode. */ + unsigned short AD_CheckTapeType; /*!< \brief Type of tape that is checked in a tape debug run. */ + unsigned short AD_CheckTapeVariables; /*!< \brief Type of variables that are checked in a tape debug run. */ STRUCT_COMPRESS Kind_Material_Compress; /*!< \brief Determines if the material is compressible or incompressible (structural analysis). */ STRUCT_MODEL Kind_Material; /*!< \brief Determines the material model to be used (structural analysis). */ STRUCT_DEFORMATION Kind_Struct_Solver; /*!< \brief Determines the geometric condition (small or large deformations) for structural analysis. */ @@ -9216,6 +9218,16 @@ class CConfig { */ su2double GetConst_DES(void) const { return Const_DES; } + /*! + * \brief Get the type of tape that will be checked in a tape debug run. + */ + bool GetAD_CheckTapeType(void) const { return AD_CheckTapeType; } + + /*! + * \brief Get the type of variables that will be checked for in a tape debug run. + */ + bool GetAD_CheckTapeVariables(void) const { return AD_CheckTapeVariables; } + /*! * \brief Get if AD preaccumulation should be performed. */ diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 0eb63de954ff..be1d493fb7c5 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -2543,6 +2543,29 @@ static const MapType DirectDiff_Var_Map = { MakePair("ELECTRIC_FIELD", D_EFIELD) }; +/*! + * \brief Types of tapes that can be checked in a tape debug run. + */ +enum ENUM_CHECK_TAPE_TYPE { + OBJECTIVE_FUNCTION_TAPE = 0, /*!< \brief Tape that only includes dependencies and objective function calculation. */ + FULL_SOLVER_TAPE = 1 /*!< \brief Tape that includes dependencies and all solvers. */ +}; +static const MapType CheckTapeType_Map = { + MakePair("OBJECTIVE_FUNCTION_TAPE", OBJECTIVE_FUNCTION_TAPE) + MakePair("FULL_SOLVER_TAPE", FULL_SOLVER_TAPE) +}; + +/*! + * \brief Types of variables that can be checked for in a tape debug run. + */ +enum ENUM_CHECK_TAPE_VARIABLES { + SOLVER_VARIABLES = 0, /*!< \brief A (debug) tag will be assigned to solver/conservative variables. */ + MESH_COORDINATES = 1 /*!< \brief A (debug) tag will be assigned to mesh coordinates. */ +}; +static const MapType CheckTapeVariables_Map = { + MakePair("SOLVER_VARIABLES", SOLVER_VARIABLES) + MakePair("MESH_COORDINATES", MESH_COORDINATES) +}; enum class RECORDING { CLEAR_INDICES, @@ -2551,7 +2574,9 @@ enum class RECORDING { MESH_DEFORM, SOLUTION_AND_MESH, TAG_INIT_SOLUTION_VARIABLES, - TAG_CHECK_SOLUTION_VARIABLES + TAG_CHECK_SOLUTION_VARIABLES, + TAG_INIT_MESH_COORDINATES, + TAG_CHECK_MESH_COORDINATES }; /*! diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 96508eb1b70e..1422b66add3e 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2797,6 +2797,12 @@ void CConfig::SetConfig_Options() { /* DESCRIPTION: Preaccumulation in the AD mode. */ addBoolOption("PREACC", AD_Preaccumulation, YES); + /* DESCRIPTION: Specify the tape which is checked in a tape debug run. */ + addEnumOption("CHECK_TAPE_TYPE", AD_CheckTapeType, CheckTapeType_Map, FULL_SOLVER_TAPE); + + /* DESCRIPTION: Specify the tape which is checked in a tape debug run. */ + addEnumOption("CHECK_TAPE_VARIABLES", AD_CheckTapeVariables, CheckTapeVariables_Map, SOLVER_VARIABLES); + /*--- options that are used in the python optimization scripts. These have no effect on the c++ toolsuite ---*/ /*!\par CONFIG_CATEGORY:Python Options\ingroup Config*/ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index ad5691e0d5be..9ca032f409f9 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -237,14 +237,44 @@ void CDiscAdjMultizoneDriver::DebugRun() { /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ - SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION_TAPE) { + if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) { + if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for MESH_COORDINATES." << endl; + SetRecording(RECORDING::TAG_INIT_MESH_COORDINATES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + } + else { + if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLUTION_VARIABLES." << endl; + SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + } + } + else { + if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) { + if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for MESH_COORDINATES." << endl; + SetRecording(RECORDING::TAG_INIT_MESH_COORDINATES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + } + else { + if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for SOLUTION_VARIABLES." << endl; + SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + } + } /*--- This recording repeats the initial recording with a different tag. * If a variable was used before it became dependent on the inputs, this variable will still carry the tag * from the initial recording and a mismatch with the "check" recording tag will throw an error. * In such a case, a possible reason could be that such a variable is set by a post-processing routine while * for a mathematically correct recording this dependency must be included earlier. ---*/ - SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION_TAPE) { + if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) + SetRecording(RECORDING::TAG_CHECK_MESH_COORDINATES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + else + SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + } + else { + if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) + SetRecording(RECORDING::TAG_CHECK_MESH_COORDINATES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + else + SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + } if (rank == MASTER_NODE) { cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; @@ -638,8 +668,10 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t case RECORDING::CLEAR_INDICES: cout << "Clearing the computational graph." << endl; break; case RECORDING::MESH_COORDS: cout << "Storing computational graph wrt MESH COORDINATES." << endl; break; case RECORDING::SOLUTION_VARIABLES: cout << "Storing computational graph wrt CONSERVATIVE VARIABLES." << endl; break; - case RECORDING::TAG_INIT_SOLUTION_VARIABLES: cout << "Simulating recording with tag 1." << endl; AD::SetTag(1); break; - case RECORDING::TAG_CHECK_SOLUTION_VARIABLES: cout << "Checking first recording with tag 2." << endl; AD::SetTag(2); break; + case RECORDING::TAG_INIT_SOLUTION_VARIABLES: cout << "Simulating recording with tag 1 on conservative variables." << endl; AD::SetTag(1); break; + case RECORDING::TAG_CHECK_SOLUTION_VARIABLES: cout << "Checking first recording with tag 2 on conservative variables." << endl; AD::SetTag(2); break; + case RECORDING::TAG_INIT_MESH_COORDINATES: cout << "Simulating recording with tag 1 on mesh coordinates." << endl; AD::SetTag(1); break; + case RECORDING::TAG_CHECK_MESH_COORDINATES: cout << "Checking first recording with tag 2 on mesh coordinates." << endl; AD::SetTag(2); break; default: break; } } diff --git a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp index 0120b8a2b02f..04a2d3541fa3 100644 --- a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp @@ -432,7 +432,10 @@ void CDiscAdjFluidIteration::RegisterInput(CSolver***** solver, CGeometry**** ge } } - if (kind_recording == RECORDING::MESH_COORDS || kind_recording == RECORDING::SOLUTION_AND_MESH) { + if (kind_recording == RECORDING::MESH_COORDS || + kind_recording == RECORDING::SOLUTION_AND_MESH || + kind_recording == RECORDING::TAG_INIT_MESH_COORDINATES || + kind_recording == RECORDING::TAG_CHECK_MESH_COORDINATES) { /*--- Register node coordinates as input ---*/ geometry0->RegisterCoordinates(); From b69abb5e2162067e8c29dc37c06dcdce4120a153 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 8 May 2025 09:12:22 +0200 Subject: [PATCH 051/102] Rename FULL_TAPE to FULL_SOLVER_TAPE. --- SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp | 2 +- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 121bdc45356c..0ff574e725f1 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -72,7 +72,7 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { * \brief Kinds of recordings. */ enum class Kind_Tape { - FULL_TAPE, /*!< \brief Entire derivative information for a coupled adjoint + FULL_SOLVER_TAPE, /*!< \brief Entire derivative information for a coupled adjoint solution update. */ OBJECTIVE_FUNCTION_TAPE, /*!< \brief Record only the dependence of the objective function w.r.t. solver variables (from all zones). */ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 9ca032f409f9..d570fa86aa3c 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -431,8 +431,8 @@ void CDiscAdjMultizoneDriver::Run() { * here. Otherwise, the whole tape of a coupled run will be created. ---*/ if (RecordingState != RECORDING::SOLUTION_VARIABLES) { - SetRecording(RECORDING::CLEAR_INDICES, Kind_Tape::FULL_TAPE, ZONE_0); - SetRecording(RECORDING::SOLUTION_VARIABLES, Kind_Tape::FULL_TAPE, ZONE_0); + SetRecording(RECORDING::CLEAR_INDICES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + SetRecording(RECORDING::SOLUTION_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } /*-- Start loop over zones. ---*/ @@ -578,11 +578,11 @@ void CDiscAdjMultizoneDriver::EvaluateSensitivities(unsigned long Iter, bool for /*--- SetRecording stores the computational graph on one iteration of the direct problem. Calling it with NONE * as argument ensures that all information from a previous recording is removed. ---*/ - SetRecording(RECORDING::CLEAR_INDICES, Kind_Tape::FULL_TAPE, ZONE_0); + SetRecording(RECORDING::CLEAR_INDICES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); /*--- Store the computational graph of one direct iteration with the mesh coordinates as input. ---*/ - SetRecording(RECORDING::MESH_COORDS, Kind_Tape::FULL_TAPE, ZONE_0); + SetRecording(RECORDING::MESH_COORDS, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); /*--- Initialize the adjoint of the output variables of the iteration with the adjoint solution * of the current iteration. The values are passed to the AD tool. ---*/ From 7753a99c71959a4381ab8fb430d9aa8cd6d1fb0f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 8 May 2025 09:13:09 +0200 Subject: [PATCH 052/102] Fix preaccumulation error. --- SU2_CFD/include/solvers/CFVMFlowSolverBase.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl index d3673e1b763a..26c8ddb591a1 100644 --- a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl +++ b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl @@ -671,12 +671,12 @@ void CFVMFlowSolverBase::ComputeVorticityAndStrainMag(const CConfig& confi StrainMag(iPoint) = sqrt(2.0*StrainMag(iPoint)); AD::SetPreaccOut(StrainMag(iPoint)); + AD::EndPreacc(); /*--- Max is not differentiable, so we not register them for preacc. ---*/ strainMax = max(strainMax, StrainMag(iPoint)); omegaMax = max(omegaMax, GeometryToolbox::Norm(3, Vorticity)); - AD::EndPreacc(); } END_SU2_OMP_FOR From f30b478190106ef0731889bfdd458feecc1df851 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 8 May 2025 11:55:22 +0200 Subject: [PATCH 053/102] AD fixes for RegisterVariables (however, it needs more maintenance). --- Common/include/CConfig.hpp | 12 ++++++++++++ Common/include/basic_types/ad_structure.hpp | 8 ++++++++ SU2_CFD/src/solvers/CDiscAdjSolver.cpp | 7 +++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 26a3cb122d75..65c10523d724 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -1899,6 +1899,12 @@ class CConfig { */ su2double GetPressure_FreeStreamND(void) const { return Pressure_FreeStreamND; } + /*! + * \brief Get a reference to the non-dimensionalized freestream pressure (used for AD tracking). + * \return Reference to non-dimensionalized freestream pressure. + */ + su2double& GetPressure_FreeStreamND(void) { return Pressure_FreeStreamND; } + /*! * \brief Get the value of the thermodynamic pressure. * \return Thermodynamic pressure. @@ -1924,6 +1930,12 @@ class CConfig { */ su2double GetTemperature_FreeStreamND(void) const { return Temperature_FreeStreamND; } + /*! + * \brief Get a reference to the non-dimensionalized freestream temperature (used for AD tracking). + * \return Reference to non-dimensionalized freestream temperature. + */ + su2double& GetTemperature_FreeStreamND(void) { return Temperature_FreeStreamND; } + /*! * \brief Get the value of the non-dimensionalized vibrational-electronic freestream temperature. * \return Non-dimensionalized vibrational-electronic freestream temperature. diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 930ebb72432d..71d659b4b5b3 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -276,6 +276,12 @@ inline void SetIndex(Identifier& index, const su2double& data) {} */ inline void SetTag(int tag) {} +/*! + * \brief Sets the tag of a variable to 0. + * \param[in] v - the variable whose tag is cleared. + */ +inline void ClearTagOnVariable(su2double& v) {} + /*! * \brief Pushes back the current tape position to the tape position's vector. */ @@ -684,8 +690,10 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { #ifdef CODI_TAG_TAPE FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } +FORCEINLINE void ClearTagOnVariable(su2double& v) { AD::getTape().clearTagOnVariable(v); } #else FORCEINLINE void SetTag(int tag) {} +FORCEINLINE void ClearTagOnVariable(su2double& v) {} #endif #endif // CODI_REVERSE_TYPE diff --git a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp index 7a7e172a84d1..bbe462c069cb 100644 --- a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp +++ b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp @@ -184,13 +184,16 @@ void CDiscAdjSolver::RegisterVariables(CGeometry *geometry, CConfig *config, boo Alpha = config->GetAoA()*PI_NUMBER/180.0; Beta = config->GetAoS()*PI_NUMBER/180.0; Mach = config->GetMach(); - Pressure = config->GetPressure_FreeStreamND(); - Temperature = config->GetTemperature_FreeStreamND(); + su2double& Pressure = config->GetPressure_FreeStreamND(); + su2double& Temperature = config->GetTemperature_FreeStreamND(); su2double SoundSpeed = 0.0; + AD::StopRecording(); + AD::ClearTagOnVariable(config->GetVelocity_FreeStreamND()[0]); if (nDim == 2) { SoundSpeed = config->GetVelocity_FreeStreamND()[0]*Velocity_Ref/(cos(Alpha)*Mach); } if (nDim == 3) { SoundSpeed = config->GetVelocity_FreeStreamND()[0]*Velocity_Ref/(cos(Alpha)*cos(Beta)*Mach); } + AD::StartRecording(); if (!reset) { AD::RegisterInput(Mach); From 1e57b74436945239606fb81581be7aae3b905556 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 8 May 2025 15:47:50 +0200 Subject: [PATCH 054/102] Small rework of debug mode recording w.r.t. mesh coordinates, improve namings. --- Common/include/option_structure.hpp | 12 +++--- .../src/drivers/CDiscAdjMultizoneDriver.cpp | 38 ++++++++++--------- .../src/iteration/CDiscAdjFluidIteration.cpp | 13 ++++--- 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index be1d493fb7c5..6b76568c43eb 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -2560,11 +2560,11 @@ static const MapType CheckTapeType_Map = { */ enum ENUM_CHECK_TAPE_VARIABLES { SOLVER_VARIABLES = 0, /*!< \brief A (debug) tag will be assigned to solver/conservative variables. */ - MESH_COORDINATES = 1 /*!< \brief A (debug) tag will be assigned to mesh coordinates. */ + MESH_COORDINATES = 1 /*!< \brief A (debug) tag will be assigned to solver/conservative variables and mesh coordinates. */ }; static const MapType CheckTapeVariables_Map = { MakePair("SOLVER_VARIABLES", SOLVER_VARIABLES) - MakePair("MESH_COORDINATES", MESH_COORDINATES) + MakePair("SOLVER_VARIABLES_AND_MESH_COORDINATES", MESH_COORDINATES) }; enum class RECORDING { @@ -2573,10 +2573,10 @@ enum class RECORDING { MESH_COORDS, MESH_DEFORM, SOLUTION_AND_MESH, - TAG_INIT_SOLUTION_VARIABLES, - TAG_CHECK_SOLUTION_VARIABLES, - TAG_INIT_MESH_COORDINATES, - TAG_CHECK_MESH_COORDINATES + TAG_INIT_SOLVER_VARIABLES, + TAG_CHECK_SOLVER_VARIABLES, + TAG_INIT_SOLVER_AND_MESH, + TAG_CHECK_SOLVER_AND_MESH }; /*! diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index d570fa86aa3c..36822f9d68bd 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -239,22 +239,22 @@ void CDiscAdjMultizoneDriver::DebugRun() { * During the recording, each dependent variable will be assigned the same tag. ---*/ if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION_TAPE) { if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) { - if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for MESH_COORDINATES." << endl; - SetRecording(RECORDING::TAG_INIT_MESH_COORDINATES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLVER_VARIABLES_AND_MESH_COORDINATES." << endl; + SetRecording(RECORDING::TAG_INIT_SOLVER_AND_MESH, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); } else { - if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLUTION_VARIABLES." << endl; - SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLVER_VARIABLES." << endl; + SetRecording(RECORDING::TAG_INIT_SOLVER_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); } } else { if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) { - if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for MESH_COORDINATES." << endl; - SetRecording(RECORDING::TAG_INIT_MESH_COORDINATES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for SOLVER_VARIABLES_AND_MESH_COORDINATES." << endl; + SetRecording(RECORDING::TAG_INIT_SOLVER_AND_MESH, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } else { - if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for SOLUTION_VARIABLES." << endl; - SetRecording(RECORDING::TAG_INIT_SOLUTION_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for SOLVER_VARIABLES." << endl; + SetRecording(RECORDING::TAG_INIT_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } } @@ -265,15 +265,15 @@ void CDiscAdjMultizoneDriver::DebugRun() { * for a mathematically correct recording this dependency must be included earlier. ---*/ if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION_TAPE) { if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) - SetRecording(RECORDING::TAG_CHECK_MESH_COORDINATES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + SetRecording(RECORDING::TAG_CHECK_SOLVER_AND_MESH, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); else - SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); + SetRecording(RECORDING::TAG_CHECK_SOLVER_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); } else { if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) - SetRecording(RECORDING::TAG_CHECK_MESH_COORDINATES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + SetRecording(RECORDING::TAG_CHECK_SOLVER_AND_MESH, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); else - SetRecording(RECORDING::TAG_CHECK_SOLUTION_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); + SetRecording(RECORDING::TAG_CHECK_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } if (rank == MASTER_NODE) { @@ -668,10 +668,10 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t case RECORDING::CLEAR_INDICES: cout << "Clearing the computational graph." << endl; break; case RECORDING::MESH_COORDS: cout << "Storing computational graph wrt MESH COORDINATES." << endl; break; case RECORDING::SOLUTION_VARIABLES: cout << "Storing computational graph wrt CONSERVATIVE VARIABLES." << endl; break; - case RECORDING::TAG_INIT_SOLUTION_VARIABLES: cout << "Simulating recording with tag 1 on conservative variables." << endl; AD::SetTag(1); break; - case RECORDING::TAG_CHECK_SOLUTION_VARIABLES: cout << "Checking first recording with tag 2 on conservative variables." << endl; AD::SetTag(2); break; - case RECORDING::TAG_INIT_MESH_COORDINATES: cout << "Simulating recording with tag 1 on mesh coordinates." << endl; AD::SetTag(1); break; - case RECORDING::TAG_CHECK_MESH_COORDINATES: cout << "Checking first recording with tag 2 on mesh coordinates." << endl; AD::SetTag(2); break; + case RECORDING::TAG_INIT_SOLVER_VARIABLES: cout << "Simulating recording with tag 1 on conservative variables." << endl; AD::SetTag(1); break; + case RECORDING::TAG_CHECK_SOLVER_VARIABLES: cout << "Checking first recording with tag 2 on conservative variables." << endl; AD::SetTag(2); break; + case RECORDING::TAG_INIT_SOLVER_AND_MESH: cout << "Simulating recording with tag 1 on mesh coordinates." << endl; AD::SetTag(1); break; + case RECORDING::TAG_CHECK_SOLVER_AND_MESH: cout << "Checking first recording with tag 2 on mesh coordinates." << endl; AD::SetTag(2); break; default: break; } } @@ -827,8 +827,10 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { AD::RegisterOutput(ObjFunc); AD::SetIndex(ObjFunc_Index, ObjFunc); if (kind_recording == RECORDING::SOLUTION_VARIABLES || - kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES || - kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES) { + kind_recording == RECORDING::TAG_INIT_SOLVER_VARIABLES || + kind_recording == RECORDING::TAG_CHECK_SOLVER_VARIABLES || + kind_recording == RECORDING::TAG_INIT_SOLVER_AND_MESH || + kind_recording == RECORDING::TAG_CHECK_SOLVER_AND_MESH) { cout << " Objective function : " << ObjFunc << endl; } } diff --git a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp index 04a2d3541fa3..e0134adc0124 100644 --- a/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CDiscAdjFluidIteration.cpp @@ -405,9 +405,12 @@ void CDiscAdjFluidIteration::RegisterInput(CSolver***** solver, CGeometry**** ge SU2_OMP_PARALLEL_(if(solvers0[ADJFLOW_SOL]->GetHasHybridParallel())) { if (kind_recording == RECORDING::SOLUTION_VARIABLES || - kind_recording == RECORDING::TAG_INIT_SOLUTION_VARIABLES || - kind_recording == RECORDING::TAG_CHECK_SOLUTION_VARIABLES || + kind_recording == RECORDING::TAG_INIT_SOLVER_VARIABLES || + kind_recording == RECORDING::TAG_CHECK_SOLVER_VARIABLES || + kind_recording == RECORDING::TAG_INIT_SOLVER_AND_MESH || + kind_recording == RECORDING::TAG_CHECK_SOLVER_AND_MESH || kind_recording == RECORDING::SOLUTION_AND_MESH) { + /*--- Register flow and turbulent variables as input ---*/ if (config[iZone]->GetFluidProblem()) { @@ -434,10 +437,10 @@ void CDiscAdjFluidIteration::RegisterInput(CSolver***** solver, CGeometry**** ge if (kind_recording == RECORDING::MESH_COORDS || kind_recording == RECORDING::SOLUTION_AND_MESH || - kind_recording == RECORDING::TAG_INIT_MESH_COORDINATES || - kind_recording == RECORDING::TAG_CHECK_MESH_COORDINATES) { - /*--- Register node coordinates as input ---*/ + kind_recording == RECORDING::TAG_INIT_SOLVER_AND_MESH || + kind_recording == RECORDING::TAG_CHECK_SOLVER_AND_MESH) { + /*--- Register node coordinates as input ---*/ geometry0->RegisterCoordinates(); } From fe5b0fb1400921cd96a223327d7966836bf2d2ac Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Mon, 12 May 2025 09:44:23 +0200 Subject: [PATCH 055/102] AD fixes for RegisterVariables, part 2. --- SU2_CFD/src/solvers/CDiscAdjSolver.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp index bbe462c069cb..e5e8a88faa22 100644 --- a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp +++ b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp @@ -178,22 +178,24 @@ void CDiscAdjSolver::RegisterVariables(CGeometry *geometry, CConfig *config, boo /*--- Register farfield values as input ---*/ - if((config->GetKind_Regime() == ENUM_REGIME::COMPRESSIBLE) && (KindDirect_Solver == RUNTIME_FLOW_SYS && !config->GetBoolTurbomachinery())) { + if((config->GetKind_Regime() == ENUM_REGIME::COMPRESSIBLE) && (KindDirect_Solver == RUNTIME_FLOW_SYS && !config->GetBoolTurbomachinery())){ su2double Velocity_Ref = config->GetVelocity_Ref(); Alpha = config->GetAoA()*PI_NUMBER/180.0; Beta = config->GetAoS()*PI_NUMBER/180.0; Mach = config->GetMach(); + /*--- Pressure and Temperature values are accessed directly by the solver, + * so we register the value in the config file. ---*/ su2double& Pressure = config->GetPressure_FreeStreamND(); su2double& Temperature = config->GetTemperature_FreeStreamND(); su2double SoundSpeed = 0.0; - AD::StopRecording(); + // AD::StopRecording(); AD::ClearTagOnVariable(config->GetVelocity_FreeStreamND()[0]); if (nDim == 2) { SoundSpeed = config->GetVelocity_FreeStreamND()[0]*Velocity_Ref/(cos(Alpha)*Mach); } if (nDim == 3) { SoundSpeed = config->GetVelocity_FreeStreamND()[0]*Velocity_Ref/(cos(Alpha)*cos(Beta)*Mach); } - AD::StartRecording(); + // AD::StartRecording(); if (!reset) { AD::RegisterInput(Mach); @@ -214,11 +216,10 @@ void CDiscAdjSolver::RegisterVariables(CGeometry *geometry, CConfig *config, boo config->GetVelocity_FreeStreamND()[2] = sin(Alpha)*cos(Beta)*Mach*SoundSpeed/Velocity_Ref; } - config->SetTemperature_FreeStreamND(Temperature); + // config->SetTemperature_FreeStreamND(Temperature); direct_solver->SetTemperature_Inf(Temperature); - config->SetPressure_FreeStreamND(Pressure); + // config->SetPressure_FreeStreamND(Pressure); direct_solver->SetPressure_Inf(Pressure); - } if ((config->GetKind_Regime() == ENUM_REGIME::COMPRESSIBLE) && (KindDirect_Solver == RUNTIME_FLOW_SYS) && config->GetBoolTurbomachinery()){ @@ -402,6 +403,9 @@ void CDiscAdjSolver::ExtractAdjoint_Variables(CGeometry *geometry, CConfig *conf if ((config->GetKind_Regime() == ENUM_REGIME::COMPRESSIBLE) && (KindDirect_Solver == RUNTIME_FLOW_SYS) && !config->GetBoolTurbomachinery()) { su2double Local_Sens_Press, Local_Sens_Temp, Local_Sens_AoA, Local_Sens_Mach; + su2double& Pressure = config->GetPressure_FreeStreamND(); + su2double& Temperature = config->GetTemperature_FreeStreamND(); + Local_Sens_Mach = SU2_TYPE::GetDerivative(Mach); Local_Sens_AoA = SU2_TYPE::GetDerivative(Alpha); Local_Sens_Temp = SU2_TYPE::GetDerivative(Temperature); From 427e500732630111b7639a1d5043a9baec6b0bb9 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 16 May 2025 10:14:02 +0200 Subject: [PATCH 056/102] Remove unnecessary virtual function, further fixes in RegisterVariables. --- SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp | 2 +- SU2_CFD/include/drivers/CDriver.hpp | 5 ----- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 4 ++-- SU2_CFD/src/solvers/CDiscAdjSolver.cpp | 7 +++---- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 0ff574e725f1..b0c286377cbe 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -144,7 +144,7 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { /*! * \brief [Overload] Launch the debug mode for the discrete adjoint multizone solver. */ - void DebugRun() override; + void DebugRun(); /*! * \brief Preprocess the multizone iteration diff --git a/SU2_CFD/include/drivers/CDriver.hpp b/SU2_CFD/include/drivers/CDriver.hpp index 45c5bd2cdede..93166413b3e5 100644 --- a/SU2_CFD/include/drivers/CDriver.hpp +++ b/SU2_CFD/include/drivers/CDriver.hpp @@ -425,11 +425,6 @@ class CDriver : public CDriverBase { */ virtual void StartSolver() {} - /*! - * \brief Launch a debug run of the solver. - */ - virtual void DebugRun() {} - /*! * \brief Deallocation routine */ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 36822f9d68bd..f140e17d8ed6 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -670,8 +670,8 @@ void CDiscAdjMultizoneDriver::SetRecording(RECORDING kind_recording, Kind_Tape t case RECORDING::SOLUTION_VARIABLES: cout << "Storing computational graph wrt CONSERVATIVE VARIABLES." << endl; break; case RECORDING::TAG_INIT_SOLVER_VARIABLES: cout << "Simulating recording with tag 1 on conservative variables." << endl; AD::SetTag(1); break; case RECORDING::TAG_CHECK_SOLVER_VARIABLES: cout << "Checking first recording with tag 2 on conservative variables." << endl; AD::SetTag(2); break; - case RECORDING::TAG_INIT_SOLVER_AND_MESH: cout << "Simulating recording with tag 1 on mesh coordinates." << endl; AD::SetTag(1); break; - case RECORDING::TAG_CHECK_SOLVER_AND_MESH: cout << "Checking first recording with tag 2 on mesh coordinates." << endl; AD::SetTag(2); break; + case RECORDING::TAG_INIT_SOLVER_AND_MESH: cout << "Simulating recording with tag 1 on conservative variables and mesh coordinates." << endl; AD::SetTag(1); break; + case RECORDING::TAG_CHECK_SOLVER_AND_MESH: cout << "Checking first recording with tag 2 on conservative variables and mesh coordinates." << endl; AD::SetTag(2); break; default: break; } } diff --git a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp index e5e8a88faa22..401f8af4e981 100644 --- a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp +++ b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp @@ -191,11 +191,12 @@ void CDiscAdjSolver::RegisterVariables(CGeometry *geometry, CConfig *config, boo su2double SoundSpeed = 0.0; - // AD::StopRecording(); + AD::StopRecording(); + // Treat Velocity_FreeStreamND config value as non-dependent (in debug mode) AD::ClearTagOnVariable(config->GetVelocity_FreeStreamND()[0]); if (nDim == 2) { SoundSpeed = config->GetVelocity_FreeStreamND()[0]*Velocity_Ref/(cos(Alpha)*Mach); } if (nDim == 3) { SoundSpeed = config->GetVelocity_FreeStreamND()[0]*Velocity_Ref/(cos(Alpha)*cos(Beta)*Mach); } - // AD::StartRecording(); + AD::StartRecording(); if (!reset) { AD::RegisterInput(Mach); @@ -216,9 +217,7 @@ void CDiscAdjSolver::RegisterVariables(CGeometry *geometry, CConfig *config, boo config->GetVelocity_FreeStreamND()[2] = sin(Alpha)*cos(Beta)*Mach*SoundSpeed/Velocity_Ref; } - // config->SetTemperature_FreeStreamND(Temperature); direct_solver->SetTemperature_Inf(Temperature); - // config->SetPressure_FreeStreamND(Pressure); direct_solver->SetPressure_Inf(Pressure); } From 35d187fdd4baf540705dc79e452002fc5ba647e9 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 16 May 2025 10:17:21 +0200 Subject: [PATCH 057/102] Add AD functions to handle and report errors appearing in a tape debug run. --- Common/include/basic_types/ad_structure.hpp | 57 +++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 71d659b4b5b3..85e7b1eaeafe 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -282,6 +282,37 @@ inline void SetTag(int tag) {} */ inline void ClearTagOnVariable(su2double& v) {} +/*! + * \brief Struct to store information about errors during a tag debug run. + */ +struct ErrorReport {}; + +/*! + * \brief Set a reference to the output file of an ErrorReport. + * \param[in] report - the ErrorReport whose output file is set. + * \param[in] output_file - pointer to the output file. + */ +inline void SetDebugReportFile(struct ErrorReport* report, std::ofstream* output_file) {} + +/*! + * \brief Set the ErrorReport to which error information from a tag debug recording is written. + * \param[in] report - the ErrorReport to which error information is written. + */ +inline void SetTagErrorCallback(struct ErrorReport* report) {} + +/*! + * \brief Reset the error counter in an ErrorReport. + * \param[in] report - the ErrorReport whose error counter is resetted. + */ +inline void ResetErrorCounter(struct ErrorReport* report) {} + +/*! + * \brief Get a pointer to the error counter of an ErrorReport. + * \param[in] report - the ErrorReport whose pointer to its error counter is returned. + * \return Pointer to the error counter. + */ +inline unsigned long* GetErrorCount(struct ErrorReport* report) { return NULL; } + /*! * \brief Pushes back the current tape position to the tape position's vector. */ @@ -688,12 +719,38 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = true;) } +struct ErrorReport { + unsigned long ErrorCounter; + std::ofstream* out; +}; + +FORCEINLINE void ResetErrorCounter(struct ErrorReport* report) { report->ErrorCounter = 0; } + +FORCEINLINE void SetDebugReportFile(struct ErrorReport* report, std::ofstream* output_file) { report->out = output_file; } + +FORCEINLINE unsigned long* GetErrorCount(struct ErrorReport* report) { return &(report->ErrorCounter); } + + #ifdef CODI_TAG_TAPE + FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } FORCEINLINE void ClearTagOnVariable(su2double& v) { AD::getTape().clearTagOnVariable(v); } + +static void tagErrorCallback(int const& correctTag, int const& wrongTag, void* userData) { + ErrorReport* report = (ErrorReport*)userData; + + report->ErrorCounter += 1; + *(report->out) << "Use of variable with bad tag '" << wrongTag << "', should be '" << correctTag << "'." << std::endl; +} + +FORCEINLINE void SetTagErrorCallback(struct ErrorReport* report) { + AD::getTape().setTagErrorCallback(tagErrorCallback, report); +} + #else FORCEINLINE void SetTag(int tag) {} FORCEINLINE void ClearTagOnVariable(su2double& v) {} +FORCEINLINE void SetTagErrorCallback(struct ErrorReport* report) {} #endif #endif // CODI_REVERSE_TYPE From 5c36308ec6b04affc11057e555b39dec64c54e18 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 16 May 2025 10:19:32 +0200 Subject: [PATCH 058/102] Add screen and file output to DebugRun in multizone discrete adjoint solver. --- .../drivers/CDiscAdjMultizoneDriver.hpp | 5 +++ .../src/drivers/CDiscAdjMultizoneDriver.cpp | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index b0c286377cbe..653566b64d89 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -146,6 +146,11 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { */ void DebugRun(); + /*! + * \brief [Overload] Launch the debug mode for the discrete adjoint multizone solver. + */ + void DebugRun_ScreenOutput(struct AD::ErrorReport& error_report); + /*! * \brief Preprocess the multizone iteration */ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index f140e17d8ed6..6e16ae160514 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -235,8 +235,17 @@ void CDiscAdjMultizoneDriver::DebugRun() { cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; } + struct AD::ErrorReport error_report; + AD::SetTagErrorCallback(&error_report); + std::ofstream out1("run1_process" + to_string(rank) + ".out"); + std::ofstream out2("run2_process" + to_string(rank) + ".out"); + + AD::ResetErrorCounter(&error_report); + AD::SetDebugReportFile(&error_report, &out1); + /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ + if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION_TAPE) { if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) { if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLVER_VARIABLES_AND_MESH_COORDINATES." << endl; @@ -257,12 +266,17 @@ void CDiscAdjMultizoneDriver::DebugRun() { SetRecording(RECORDING::TAG_INIT_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } } + DebugRun_ScreenOutput(error_report); + + AD::ResetErrorCounter(&error_report); + AD::SetDebugReportFile(&error_report, &out2); /*--- This recording repeats the initial recording with a different tag. * If a variable was used before it became dependent on the inputs, this variable will still carry the tag * from the initial recording and a mismatch with the "check" recording tag will throw an error. * In such a case, a possible reason could be that such a variable is set by a post-processing routine while * for a mathematically correct recording this dependency must be included earlier. ---*/ + if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION_TAPE) { if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) SetRecording(RECORDING::TAG_CHECK_SOLVER_AND_MESH, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); @@ -275,12 +289,32 @@ void CDiscAdjMultizoneDriver::DebugRun() { else SetRecording(RECORDING::TAG_CHECK_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } + DebugRun_ScreenOutput(error_report); if (rank == MASTER_NODE) { cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; } } +void CDiscAdjMultizoneDriver::DebugRun_ScreenOutput(struct AD::ErrorReport& error_report) { + + int total_errors = 0; + int process_error[size]; + + SU2_MPI::Allreduce(AD::GetErrorCount(&error_report), &total_errors, 1, MPI_INT, MPI_SUM, SU2_MPI::GetComm()); + SU2_MPI::Gather(AD::GetErrorCount(&error_report), 1, MPI_INT, &process_error, 1, MPI_INT, 0, SU2_MPI::GetComm()); + + if (rank == MASTER_NODE) { + std::cout << "\nTotal number of detected tape inconsistencies: " << total_errors << std::endl; + if(total_errors > 0 && size > 1) { + std::cout << "\n"; + for (int irank = 0; irank < size; irank++) { + std::cout << "Number of inconsistencies from process " << irank << ": " << process_error[irank] << std::endl; + } + } + } +} + bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { config_container[iZone]->SetInnerIter(iInnerIter); From 9f3072539171e43f9caf5c2e287e1b24d44747db Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 16 May 2025 10:52:40 +0200 Subject: [PATCH 059/102] Clang format. --- Common/include/basic_types/ad_structure.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 85e7b1eaeafe..23262e61ff1c 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -726,11 +726,12 @@ struct ErrorReport { FORCEINLINE void ResetErrorCounter(struct ErrorReport* report) { report->ErrorCounter = 0; } -FORCEINLINE void SetDebugReportFile(struct ErrorReport* report, std::ofstream* output_file) { report->out = output_file; } +FORCEINLINE void SetDebugReportFile(struct ErrorReport* report, std::ofstream* output_file) { + report->out = output_file; +} FORCEINLINE unsigned long* GetErrorCount(struct ErrorReport* report) { return &(report->ErrorCounter); } - #ifdef CODI_TAG_TAPE FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } From 521844a6b17d0db6c2bd0615b6d7a1b08c8b4aa6 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Sat, 17 May 2025 15:04:29 +0200 Subject: [PATCH 060/102] Small fix (vla error). --- Common/include/basic_types/ad_structure.hpp | 3 ++- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 23262e61ff1c..76b15456e35e 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -752,7 +752,8 @@ FORCEINLINE void SetTagErrorCallback(struct ErrorReport* report) { FORCEINLINE void SetTag(int tag) {} FORCEINLINE void ClearTagOnVariable(su2double& v) {} FORCEINLINE void SetTagErrorCallback(struct ErrorReport* report) {} -#endif + +#endif // CODI_TAG_TAPE #endif // CODI_REVERSE_TYPE diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 6e16ae160514..d6733924ded2 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -299,10 +299,10 @@ void CDiscAdjMultizoneDriver::DebugRun() { void CDiscAdjMultizoneDriver::DebugRun_ScreenOutput(struct AD::ErrorReport& error_report) { int total_errors = 0; - int process_error[size]; + std::vector process_error(size); SU2_MPI::Allreduce(AD::GetErrorCount(&error_report), &total_errors, 1, MPI_INT, MPI_SUM, SU2_MPI::GetComm()); - SU2_MPI::Gather(AD::GetErrorCount(&error_report), 1, MPI_INT, &process_error, 1, MPI_INT, 0, SU2_MPI::GetComm()); + SU2_MPI::Gather(AD::GetErrorCount(&error_report), 1, MPI_INT, process_error.data(), 1, MPI_INT, 0, SU2_MPI::GetComm()); if (rank == MASTER_NODE) { std::cout << "\nTotal number of detected tape inconsistencies: " << total_errors << std::endl; From b3db38d96b36c22b956321cf66f29cc7d891ab56 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 29 May 2025 15:28:34 +0200 Subject: [PATCH 061/102] Add tag tape build to regression, upload under folder name ReverseTagNoMPI. --- .github/workflows/regression.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index cef9d68216ed..6558462b45e0 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - config_set: [BaseMPI, ReverseMPI, ForwardMPI, BaseNoMPI, ReverseNoMPI, ForwardNoMPI, BaseOMP, ReverseOMP, ForwardOMP] + config_set: [BaseMPI, ReverseMPI, ForwardMPI, BaseNoMPI, ReverseNoMPI, ForwardNoMPI, ReverseTagNoMPI, BaseOMP, ReverseOMP, ForwardOMP] include: - config_set: BaseMPI flags: '-Denable-pywrapper=true -Denable-coolprop=true -Denable-mpp=true -Dinstall-mpp=true -Denable-mlpcpp=true -Denable-tests=true --warnlevel=2' @@ -37,6 +37,8 @@ jobs: flags: '-Denable-autodiff=true -Denable-normal=false -Dwith-mpi=disabled -Denable-pywrapper=true -Denable-tests=true --warnlevel=3 --werror' - config_set: ForwardNoMPI flags: '-Denable-directdiff=true -Denable-normal=false -Dwith-mpi=disabled -Denable-tests=true --warnlevel=3 --werror' + - config_set: ReverseTagNoMPI + flags: '-Denable-autodiff=true -Denable-normal=false -Dwith-mpi=disabled -Denable-pywrapper=true -Denable-tests=true --warnlevel=3 --werror -Dcodi-tape=Tag' - config_set: BaseOMP flags: '-Dwith-omp=true -Denable-mixedprec=true -Denable-pywrapper=true -Denable-tecio=false --warnlevel=3 --werror' - config_set: ReverseOMP From e146d3cbeca8250eea6fdac5a44a8029ded6ada7 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 30 May 2025 00:32:20 +0200 Subject: [PATCH 062/102] Test for failing serial regression AD (discadj_multiple_ffd_py). --- SU2_CFD/src/solvers/CDiscAdjSolver.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp index 401f8af4e981..fc10a3e2be6b 100644 --- a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp +++ b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp @@ -191,12 +191,10 @@ void CDiscAdjSolver::RegisterVariables(CGeometry *geometry, CConfig *config, boo su2double SoundSpeed = 0.0; - AD::StopRecording(); // Treat Velocity_FreeStreamND config value as non-dependent (in debug mode) AD::ClearTagOnVariable(config->GetVelocity_FreeStreamND()[0]); if (nDim == 2) { SoundSpeed = config->GetVelocity_FreeStreamND()[0]*Velocity_Ref/(cos(Alpha)*Mach); } if (nDim == 3) { SoundSpeed = config->GetVelocity_FreeStreamND()[0]*Velocity_Ref/(cos(Alpha)*cos(Beta)*Mach); } - AD::StartRecording(); if (!reset) { AD::RegisterInput(Mach); From 9282363d4f9a6f9b6b8f698d4b08600df6ba4c8a Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 30 May 2025 13:17:21 +0200 Subject: [PATCH 063/102] Add job to regression (discadj_tape_tests), using standard serial_regression_AD and standard SU2_CFD_AD just to check if it works. --- .github/workflows/regression.yml | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 6558462b45e0..bb4c7f561c40 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -217,6 +217,56 @@ jobs: with: entrypoint: /bin/rm args: -rf install install_bin.tgz src ccache ${{ matrix.config_set }} + discadj_tape_tests: + if: inputs.runner != 'ARM64' + runs-on: ${{ inputs.runner || 'ubuntu-latest' }} + name: Tape Tests + needs: build + strategy: + fail-fast: false + matrix: + testscript: ['serial_regression_AD.py'] + include: + - testscript: 'serial_regression_AD.py' + #tag: TagNoMPI + tag:NoMPI # check + steps: + - name: Pre Cleanup + uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 + with: + entrypoint: /bin/rm + args: -rf install install_bin.tgz src ccache ${{ matrix.config_set }} + - name: Download All artifacts + uses: actions/download-artifact@v4 + - name: Uncompress and Move Binaries + run: | + BIN_FOLDER="$PWD/install/bin" + mkdir -p $BIN_FOLDER + ls -lah $BIN_FOLDER + for type in Base Reverse Forward; do + TYPE_FOLDER="${type}OMP-tsan" + echo "Processing '$TYPE_FOLDER' ..." + if [ -d $TYPE_FOLDER ]; then + pushd $TYPE_FOLDER + ls -lah + tar -zxvf install_bin.tgz + ls -lah install/bin/ + cp -r install/* $BIN_FOLDER/../ + popd; + fi + done + chmod a+x $BIN_FOLDER/* + ls -lahR $BIN_FOLDER + - name: Run Tests in Container + uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 + with: + # -t -c + args: -b ${{github.ref}} -t develop -c develop -s ${{matrix.testscript}} -a "--tagtape" + - name: Cleanup + uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 + with: + entrypoint: /bin/rm + args: -rf install install_bin.tgz src ccache ${{ matrix.config_set }} thread_sanitizer_tests: if: inputs.runner != 'ARM64' runs-on: ${{ inputs.runner || 'ubuntu-latest' }} From 8fe3f902ab46550790e7cce3ab430bccea115d02 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 30 May 2025 14:03:17 +0200 Subject: [PATCH 064/102] Small fix. --- .github/workflows/regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index bb4c7f561c40..8446b10a7f7e 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -228,8 +228,8 @@ jobs: testscript: ['serial_regression_AD.py'] include: - testscript: 'serial_regression_AD.py' - #tag: TagNoMPI - tag:NoMPI # check + #tag: TagNoMP, check: + tag:NoMPI steps: - name: Pre Cleanup uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 From 8f2d45a549b73ab46574df3215d55c7f491e607b Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 30 May 2025 14:06:13 +0200 Subject: [PATCH 065/102] Small fix. --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 8446b10a7f7e..1dd403748aa8 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -229,7 +229,7 @@ jobs: include: - testscript: 'serial_regression_AD.py' #tag: TagNoMP, check: - tag:NoMPI + tag: NoMPI steps: - name: Pre Cleanup uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 From f7254c2d715e75ee21d672e000b71477de0160f2 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 30 May 2025 14:11:12 +0200 Subject: [PATCH 066/102] Small fix. --- .github/workflows/regression.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 1dd403748aa8..5ba3dfaa6b3f 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -226,10 +226,10 @@ jobs: fail-fast: false matrix: testscript: ['serial_regression_AD.py'] - include: - - testscript: 'serial_regression_AD.py' - #tag: TagNoMP, check: - tag: NoMPI + include: + - testscript: 'serial_regression_AD.py' + #tag: TagNoMP, check: + tag: NoMPI steps: - name: Pre Cleanup uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 From 63accc0d5e207a362bb142617fc4d9696e1d70c8 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 30 May 2025 14:45:52 +0200 Subject: [PATCH 067/102] Add --tagtape option to regression script parser. --- TestCases/TestCase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TestCases/TestCase.py b/TestCases/TestCase.py index 03b431ba3a7d..2b452177a75b 100644 --- a/TestCases/TestCase.py +++ b/TestCases/TestCase.py @@ -46,6 +46,7 @@ def parse_args(description: str): parser = argparse.ArgumentParser(description=description) parser.add_argument('--tsan', action='store_true', help='Run thread sanitizer tests. Requires a tsan-enabled SU2 build.') parser.add_argument('--asan', action='store_true', help='Run address sanitizer tests. Requires an asan-enabled SU2 build.') + parser.add_argument('--tagtape', action='store_true', help='Run discrete adjoint tests in tape debug mode. Requires a SU2_CFD_AD build with Tag reverse type.') return parser.parse_args() class TestCase: From 4b3ad4fa4e98165f7906e114c4de687520c73958 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Sat, 31 May 2025 01:03:47 +0200 Subject: [PATCH 068/102] Small fix. --- .github/workflows/regression.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 5ba3dfaa6b3f..6e949919415f 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -228,7 +228,7 @@ jobs: testscript: ['serial_regression_AD.py'] include: - testscript: 'serial_regression_AD.py' - #tag: TagNoMP, check: + #tag: TagNoMPI, check: tag: NoMPI steps: - name: Pre Cleanup @@ -244,7 +244,7 @@ jobs: mkdir -p $BIN_FOLDER ls -lah $BIN_FOLDER for type in Base Reverse Forward; do - TYPE_FOLDER="${type}OMP-tsan" + TYPE_FOLDER="${type}${{matrix.tag}}" echo "Processing '$TYPE_FOLDER' ..." if [ -d $TYPE_FOLDER ]; then pushd $TYPE_FOLDER From 06f6ad6d514ac5bc3f92520cd5dff308f349015e Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 10 Jun 2025 18:15:50 +0200 Subject: [PATCH 069/102] Improve new error report functions in ad_structure.hpp. Suggested by pcarruscag. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- Common/include/basic_types/ad_structure.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 76b15456e35e..e7198641a287 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -724,7 +724,7 @@ struct ErrorReport { std::ofstream* out; }; -FORCEINLINE void ResetErrorCounter(struct ErrorReport* report) { report->ErrorCounter = 0; } +FORCEINLINE void ResetErrorCounter(ErrorReport& report) { report.ErrorCounter = 0; } FORCEINLINE void SetDebugReportFile(struct ErrorReport* report, std::ofstream* output_file) { report->out = output_file; From b99c08599a5dd7855d9cddb9762fb191be26e51c Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 10 Jun 2025 18:16:58 +0200 Subject: [PATCH 070/102] Improve new error report functions in ad_structure.hpp. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- Common/include/basic_types/ad_structure.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index e7198641a287..6c59d8223ed0 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -730,7 +730,7 @@ FORCEINLINE void SetDebugReportFile(struct ErrorReport* report, std::ofstream* o report->out = output_file; } -FORCEINLINE unsigned long* GetErrorCount(struct ErrorReport* report) { return &(report->ErrorCounter); } +FORCEINLINE unsigned long& GetErrorCount(ErrorReport& report) { return report->ErrorCounter; } #ifdef CODI_TAG_TAPE From d94306ddb5d54ede1db821e6fc9a95c5160abccf Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 10 Jun 2025 18:19:12 +0200 Subject: [PATCH 071/102] Add initializations to ErrorReport struct. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- Common/include/basic_types/ad_structure.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 6c59d8223ed0..f06840b8ab84 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -720,8 +720,8 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { } struct ErrorReport { - unsigned long ErrorCounter; - std::ofstream* out; + unsigned long ErrorCounter = 0; + std::ofstream* out = &std::cout; }; FORCEINLINE void ResetErrorCounter(ErrorReport& report) { report.ErrorCounter = 0; } From 9eee3039ba31dec1d5a95cc3bf71eac6850013d0 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 10 Jun 2025 18:21:45 +0200 Subject: [PATCH 072/102] Remove empty line. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index d6733924ded2..c30668597b76 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -855,8 +855,6 @@ void CDiscAdjMultizoneDriver::SetObjFunction(RECORDING kind_recording) { } } - - if (rank == MASTER_NODE) { AD::RegisterOutput(ObjFunc); AD::SetIndex(ObjFunc_Index, ObjFunc); From 8a00108e8c03ba85caf3ddf700474794212cfa1a Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 10 Jun 2025 23:59:44 +0200 Subject: [PATCH 073/102] Small follow-up fixes for error report function changes. --- Common/include/basic_types/ad_structure.hpp | 12 ++++++------ SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index f06840b8ab84..ad417e6c9fd1 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -304,14 +304,14 @@ inline void SetTagErrorCallback(struct ErrorReport* report) {} * \brief Reset the error counter in an ErrorReport. * \param[in] report - the ErrorReport whose error counter is resetted. */ -inline void ResetErrorCounter(struct ErrorReport* report) {} +inline void ResetErrorCounter(struct ErrorReport& report) {} /*! - * \brief Get a pointer to the error counter of an ErrorReport. + * \brief Get the error count of an ErrorReport. * \param[in] report - the ErrorReport whose pointer to its error counter is returned. - * \return Pointer to the error counter. + * \return Value of the error counter. */ -inline unsigned long* GetErrorCount(struct ErrorReport* report) { return NULL; } +inline unsigned long GetErrorCount(ErrorReport& report) { return 0; } /*! * \brief Pushes back the current tape position to the tape position's vector. @@ -721,7 +721,7 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { struct ErrorReport { unsigned long ErrorCounter = 0; - std::ofstream* out = &std::cout; + std::ofstream* out = nullptr; }; FORCEINLINE void ResetErrorCounter(ErrorReport& report) { report.ErrorCounter = 0; } @@ -730,7 +730,7 @@ FORCEINLINE void SetDebugReportFile(struct ErrorReport* report, std::ofstream* o report->out = output_file; } -FORCEINLINE unsigned long& GetErrorCount(ErrorReport& report) { return report->ErrorCounter; } +FORCEINLINE unsigned long GetErrorCount(ErrorReport& report) { return report.ErrorCounter; } #ifdef CODI_TAG_TAPE diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index c30668597b76..923f1dd371a8 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -240,7 +240,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { std::ofstream out1("run1_process" + to_string(rank) + ".out"); std::ofstream out2("run2_process" + to_string(rank) + ".out"); - AD::ResetErrorCounter(&error_report); + AD::ResetErrorCounter(error_report); AD::SetDebugReportFile(&error_report, &out1); /*--- This recording will assign the initial (same) tag to each registered variable. @@ -268,7 +268,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { } DebugRun_ScreenOutput(error_report); - AD::ResetErrorCounter(&error_report); + AD::ResetErrorCounter(error_report); AD::SetDebugReportFile(&error_report, &out2); /*--- This recording repeats the initial recording with a different tag. @@ -298,11 +298,11 @@ void CDiscAdjMultizoneDriver::DebugRun() { void CDiscAdjMultizoneDriver::DebugRun_ScreenOutput(struct AD::ErrorReport& error_report) { + int num_errors = AD::GetErrorCount(error_report); int total_errors = 0; std::vector process_error(size); - - SU2_MPI::Allreduce(AD::GetErrorCount(&error_report), &total_errors, 1, MPI_INT, MPI_SUM, SU2_MPI::GetComm()); - SU2_MPI::Gather(AD::GetErrorCount(&error_report), 1, MPI_INT, process_error.data(), 1, MPI_INT, 0, SU2_MPI::GetComm()); + SU2_MPI::Allreduce(&num_errors, &total_errors, 1, MPI_INT, MPI_SUM, SU2_MPI::GetComm()); + SU2_MPI::Gather(&num_errors, 1, MPI_INT, process_error.data(), 1, MPI_INT, 0, SU2_MPI::GetComm()); if (rank == MASTER_NODE) { std::cout << "\nTotal number of detected tape inconsistencies: " << total_errors << std::endl; From b82fa32744cca4c10a1beb30e9c52eb4abb86084 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 00:09:11 +0200 Subject: [PATCH 074/102] Follow-up fix for error report initialization. --- Common/include/basic_types/ad_structure.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index ad417e6c9fd1..d900b4f3db33 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -721,7 +721,7 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { struct ErrorReport { unsigned long ErrorCounter = 0; - std::ofstream* out = nullptr; + std::ostream* out = &std::cout; }; FORCEINLINE void ResetErrorCounter(ErrorReport& report) { report.ErrorCounter = 0; } From b5680c83332c02cc6a95bd176d019480054ad707 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 00:37:40 +0200 Subject: [PATCH 075/102] Further small fixes for tape debug mode. --- Common/include/CConfig.hpp | 4 ++-- Common/include/option_structure.hpp | 8 ++++---- Common/src/CConfig.cpp | 2 +- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 65c10523d724..dfa2797c799d 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -9233,12 +9233,12 @@ class CConfig { /*! * \brief Get the type of tape that will be checked in a tape debug run. */ - bool GetAD_CheckTapeType(void) const { return AD_CheckTapeType; } + unsigned short GetAD_CheckTapeType(void) const { return AD_CheckTapeType; } /*! * \brief Get the type of variables that will be checked for in a tape debug run. */ - bool GetAD_CheckTapeVariables(void) const { return AD_CheckTapeVariables; } + unsigned short GetAD_CheckTapeVariables(void) const { return AD_CheckTapeVariables; } /*! * \brief Get if AD preaccumulation should be performed. diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 6b76568c43eb..184d86e4d758 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -2547,12 +2547,12 @@ static const MapType DirectDiff_Var_Map = { * \brief Types of tapes that can be checked in a tape debug run. */ enum ENUM_CHECK_TAPE_TYPE { - OBJECTIVE_FUNCTION_TAPE = 0, /*!< \brief Tape that only includes dependencies and objective function calculation. */ - FULL_SOLVER_TAPE = 1 /*!< \brief Tape that includes dependencies and all solvers. */ + OBJECTIVE_FUNCTION = 0, /*!< \brief Tape that only includes dependencies and objective function calculation. */ + FULL_SOLVER = 1 /*!< \brief Tape that includes dependencies and all solvers. */ }; static const MapType CheckTapeType_Map = { - MakePair("OBJECTIVE_FUNCTION_TAPE", OBJECTIVE_FUNCTION_TAPE) - MakePair("FULL_SOLVER_TAPE", FULL_SOLVER_TAPE) + MakePair("OBJECTIVE_FUNCTION_TAPE", OBJECTIVE_FUNCTION) + MakePair("FULL_SOLVER_TAPE", FULL_SOLVER) }; /*! diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 1422b66add3e..5ff13c028e1c 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2798,7 +2798,7 @@ void CConfig::SetConfig_Options() { addBoolOption("PREACC", AD_Preaccumulation, YES); /* DESCRIPTION: Specify the tape which is checked in a tape debug run. */ - addEnumOption("CHECK_TAPE_TYPE", AD_CheckTapeType, CheckTapeType_Map, FULL_SOLVER_TAPE); + addEnumOption("CHECK_TAPE_TYPE", AD_CheckTapeType, CheckTapeType_Map, FULL_SOLVER); /* DESCRIPTION: Specify the tape which is checked in a tape debug run. */ addEnumOption("CHECK_TAPE_VARIABLES", AD_CheckTapeVariables, CheckTapeVariables_Map, SOLVER_VARIABLES); diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 923f1dd371a8..cd20169d5995 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -246,7 +246,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ - if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION_TAPE) { + if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION) { if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) { if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLVER_VARIABLES_AND_MESH_COORDINATES." << endl; SetRecording(RECORDING::TAG_INIT_SOLVER_AND_MESH, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); @@ -277,7 +277,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { * In such a case, a possible reason could be that such a variable is set by a post-processing routine while * for a mathematically correct recording this dependency must be included earlier. ---*/ - if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION_TAPE) { + if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION) { if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) SetRecording(RECORDING::TAG_CHECK_SOLVER_AND_MESH, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); else From 577c54695fad86fbe8b375fad51f2367e25ffb1c Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 01:08:03 +0200 Subject: [PATCH 076/102] Rename new enums for tape debug mode and make them enum classes. --- Common/include/CConfig.hpp | 8 +++---- Common/include/option_structure.hpp | 24 +++++++++---------- Common/src/CConfig.cpp | 4 ++-- .../src/drivers/CDiscAdjMultizoneDriver.cpp | 12 +++++----- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index dfa2797c799d..2e24e4f811af 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -598,8 +598,8 @@ class CConfig { bool AD_Mode; /*!< \brief Algorithmic Differentiation support. */ bool AD_Preaccumulation; /*!< \brief Enable or disable preaccumulation in the AD mode. */ - unsigned short AD_CheckTapeType; /*!< \brief Type of tape that is checked in a tape debug run. */ - unsigned short AD_CheckTapeVariables; /*!< \brief Type of variables that are checked in a tape debug run. */ + CHECK_TAPE_TYPE AD_CheckTapeType; /*!< \brief Type of tape that is checked in a tape debug run. */ + CHECK_TAPE_VARIABLES AD_CheckTapeVariables; /*!< \brief Type of variables that are checked in a tape debug run. */ STRUCT_COMPRESS Kind_Material_Compress; /*!< \brief Determines if the material is compressible or incompressible (structural analysis). */ STRUCT_MODEL Kind_Material; /*!< \brief Determines the material model to be used (structural analysis). */ STRUCT_DEFORMATION Kind_Struct_Solver; /*!< \brief Determines the geometric condition (small or large deformations) for structural analysis. */ @@ -9233,12 +9233,12 @@ class CConfig { /*! * \brief Get the type of tape that will be checked in a tape debug run. */ - unsigned short GetAD_CheckTapeType(void) const { return AD_CheckTapeType; } + CHECK_TAPE_TYPE GetAD_CheckTapeType(void) const { return AD_CheckTapeType; } /*! * \brief Get the type of variables that will be checked for in a tape debug run. */ - unsigned short GetAD_CheckTapeVariables(void) const { return AD_CheckTapeVariables; } + CHECK_TAPE_VARIABLES GetAD_CheckTapeVariables(void) const { return AD_CheckTapeVariables; } /*! * \brief Get if AD preaccumulation should be performed. diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 184d86e4d758..6f118ce29197 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -2546,25 +2546,25 @@ static const MapType DirectDiff_Var_Map = { /*! * \brief Types of tapes that can be checked in a tape debug run. */ -enum ENUM_CHECK_TAPE_TYPE { - OBJECTIVE_FUNCTION = 0, /*!< \brief Tape that only includes dependencies and objective function calculation. */ - FULL_SOLVER = 1 /*!< \brief Tape that includes dependencies and all solvers. */ +enum class CHECK_TAPE_TYPE { + OBJECTIVE_FUNCTION, /*!< \brief Tape that only includes dependencies and objective function calculation. */ + FULL_SOLVER /*!< \brief Tape that includes dependencies and all solvers. */ }; -static const MapType CheckTapeType_Map = { - MakePair("OBJECTIVE_FUNCTION_TAPE", OBJECTIVE_FUNCTION) - MakePair("FULL_SOLVER_TAPE", FULL_SOLVER) +static const MapType CheckTapeType_Map = { + MakePair("OBJECTIVE_FUNCTION_TAPE", CHECK_TAPE_TYPE::OBJECTIVE_FUNCTION) + MakePair("FULL_SOLVER_TAPE", CHECK_TAPE_TYPE::FULL_SOLVER) }; /*! * \brief Types of variables that can be checked for in a tape debug run. */ -enum ENUM_CHECK_TAPE_VARIABLES { - SOLVER_VARIABLES = 0, /*!< \brief A (debug) tag will be assigned to solver/conservative variables. */ - MESH_COORDINATES = 1 /*!< \brief A (debug) tag will be assigned to solver/conservative variables and mesh coordinates. */ +enum class CHECK_TAPE_VARIABLES { + SOLVER_VARIABLES, /*!< \brief A (debug) tag will be assigned to solver/conservative variables. */ + MESH_COORDINATES /*!< \brief A (debug) tag will be assigned to solver/conservative variables and mesh coordinates. */ }; -static const MapType CheckTapeVariables_Map = { - MakePair("SOLVER_VARIABLES", SOLVER_VARIABLES) - MakePair("SOLVER_VARIABLES_AND_MESH_COORDINATES", MESH_COORDINATES) +static const MapType CheckTapeVariables_Map = { + MakePair("SOLVER_VARIABLES", CHECK_TAPE_VARIABLES::SOLVER_VARIABLES) + MakePair("SOLVER_VARIABLES_AND_MESH_COORDINATES", CHECK_TAPE_VARIABLES::MESH_COORDINATES) }; enum class RECORDING { diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 5ff13c028e1c..7d3e2f826e43 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2798,10 +2798,10 @@ void CConfig::SetConfig_Options() { addBoolOption("PREACC", AD_Preaccumulation, YES); /* DESCRIPTION: Specify the tape which is checked in a tape debug run. */ - addEnumOption("CHECK_TAPE_TYPE", AD_CheckTapeType, CheckTapeType_Map, FULL_SOLVER); + addEnumOption("CHECK_TAPE_TYPE", AD_CheckTapeType, CheckTapeType_Map, CHECK_TAPE_TYPE::FULL_SOLVER); /* DESCRIPTION: Specify the tape which is checked in a tape debug run. */ - addEnumOption("CHECK_TAPE_VARIABLES", AD_CheckTapeVariables, CheckTapeVariables_Map, SOLVER_VARIABLES); + addEnumOption("CHECK_TAPE_VARIABLES", AD_CheckTapeVariables, CheckTapeVariables_Map, CHECK_TAPE_VARIABLES::SOLVER_VARIABLES); /*--- options that are used in the python optimization scripts. These have no effect on the c++ toolsuite ---*/ /*!\par CONFIG_CATEGORY:Python Options\ingroup Config*/ diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index cd20169d5995..2215a93c535b 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -246,8 +246,8 @@ void CDiscAdjMultizoneDriver::DebugRun() { /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ - if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION) { - if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) { + if(driver_config->GetAD_CheckTapeType() == CHECK_TAPE_TYPE::OBJECTIVE_FUNCTION) { + if(driver_config->GetAD_CheckTapeVariables() == CHECK_TAPE_VARIABLES::MESH_COORDINATES) { if (rank == MASTER_NODE) cout << "\nChecking OBJECTIVE_FUNCTION_TAPE for SOLVER_VARIABLES_AND_MESH_COORDINATES." << endl; SetRecording(RECORDING::TAG_INIT_SOLVER_AND_MESH, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); } @@ -257,7 +257,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { } } else { - if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) { + if(driver_config->GetAD_CheckTapeVariables() == CHECK_TAPE_VARIABLES::MESH_COORDINATES) { if (rank == MASTER_NODE) cout << "\nChecking FULL_SOLVER_TAPE for SOLVER_VARIABLES_AND_MESH_COORDINATES." << endl; SetRecording(RECORDING::TAG_INIT_SOLVER_AND_MESH, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } @@ -277,14 +277,14 @@ void CDiscAdjMultizoneDriver::DebugRun() { * In such a case, a possible reason could be that such a variable is set by a post-processing routine while * for a mathematically correct recording this dependency must be included earlier. ---*/ - if(driver_config->GetAD_CheckTapeType() == OBJECTIVE_FUNCTION) { - if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) + if(driver_config->GetAD_CheckTapeType() == CHECK_TAPE_TYPE::OBJECTIVE_FUNCTION) { + if(driver_config->GetAD_CheckTapeVariables() == CHECK_TAPE_VARIABLES::MESH_COORDINATES) SetRecording(RECORDING::TAG_CHECK_SOLVER_AND_MESH, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); else SetRecording(RECORDING::TAG_CHECK_SOLVER_VARIABLES, Kind_Tape::OBJECTIVE_FUNCTION_TAPE, ZONE_0); } else { - if(driver_config->GetAD_CheckTapeVariables() == MESH_COORDINATES) + if(driver_config->GetAD_CheckTapeVariables() == CHECK_TAPE_VARIABLES::MESH_COORDINATES) SetRecording(RECORDING::TAG_CHECK_SOLVER_AND_MESH, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); else SetRecording(RECORDING::TAG_CHECK_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); From 009c689157f414d820f8e8d064cae4e7b0b15d25 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 01:23:32 +0200 Subject: [PATCH 077/102] Update for alignment in vectorization.hpp. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- Common/include/parallelization/vectorization.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/parallelization/vectorization.hpp b/Common/include/parallelization/vectorization.hpp index 5a8f8606a3ea..0360b344946d 100644 --- a/Common/include/parallelization/vectorization.hpp +++ b/Common/include/parallelization/vectorization.hpp @@ -90,7 +90,7 @@ class Array : public CVecExpr, Scalar_t> { public: using Scalar = Scalar_t; enum : size_t { Size = N }; - enum : size_t { Align = Size * 32 }; + enum : size_t { Align = Size * alignof(Scalar) }; static constexpr bool StoreAsRef = true; private: From 32bad7a0af89a0940ed0ecc2c661b5685aaa9e5e Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 10:26:12 +0200 Subject: [PATCH 078/102] Remove unused variable. --- SU2_CFD/include/solvers/CDiscAdjSolver.hpp | 2 +- SU2_CFD/src/solvers/CDiscAdjSolver.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/SU2_CFD/include/solvers/CDiscAdjSolver.hpp b/SU2_CFD/include/solvers/CDiscAdjSolver.hpp index 7071c31750f9..d196eb1bff4f 100644 --- a/SU2_CFD/include/solvers/CDiscAdjSolver.hpp +++ b/SU2_CFD/include/solvers/CDiscAdjSolver.hpp @@ -57,7 +57,7 @@ class CDiscAdjSolver final : public CSolver { su2double Total_Sens_BPress; /*!< \brief Total sensitivity to outlet pressure. */ su2double Total_Sens_Density; /*!< \brief Total sensitivity to initial density (incompressible). */ su2double Total_Sens_ModVel; /*!< \brief Total sensitivity to inlet velocity (incompressible). */ - su2double Mach, Alpha, Beta, Pressure, Temperature, BPressure, ModVel; + su2double Mach, Alpha, Beta, Temperature, BPressure, ModVel; su2double TemperatureRad, Total_Sens_Temp_Rad; CDiscAdjVariable* nodes = nullptr; /*!< \brief The highest level in the variable hierarchy this solver can safely use. */ diff --git a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp index fc10a3e2be6b..07caefe91c3c 100644 --- a/SU2_CFD/src/solvers/CDiscAdjSolver.cpp +++ b/SU2_CFD/src/solvers/CDiscAdjSolver.cpp @@ -184,8 +184,8 @@ void CDiscAdjSolver::RegisterVariables(CGeometry *geometry, CConfig *config, boo Alpha = config->GetAoA()*PI_NUMBER/180.0; Beta = config->GetAoS()*PI_NUMBER/180.0; Mach = config->GetMach(); - /*--- Pressure and Temperature values are accessed directly by the solver, - * so we register the value in the config file. ---*/ + /*--- Pressure and Temperature can be registered directly via their config file value + * (no further treatment required here). ---*/ su2double& Pressure = config->GetPressure_FreeStreamND(); su2double& Temperature = config->GetTemperature_FreeStreamND(); From 96be84fec49f9b03e02b62197e1b1da6ea9c1fbe Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 13:33:30 +0200 Subject: [PATCH 079/102] Add config option to determine whether multizone restart and solution filenames are appended the zone number automatically. --- Common/include/CConfig.hpp | 13 ++++++++++--- Common/src/CConfig.cpp | 4 +++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 2e24e4f811af..dcc818ae325f 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -823,6 +823,7 @@ class CConfig { SurfSens_FileName, /*!< \brief Output file for the sensitivity on the surface (discrete adjoint). */ VolSens_FileName, /*!< \brief Output file for the sensitivity in the volume (discrete adjoint). */ ObjFunc_Hess_FileName; /*!< \brief Hessian approximation obtained by the Sobolev smoothing solver. */ + bool Multizone_Adapt_FileName; /*!< \brief Append zone number to solution and restart file names. */ bool Wrt_Performance, /*!< \brief Write the performance summary at the end of a calculation. */ @@ -5398,19 +5399,25 @@ class CConfig { bool GetWrt_Volume_Overwrite(void) const { return Wrt_Volume_Overwrite; } /*! - * \brief Provides the number of varaibles. + * \brief Get whether filenames are appended the zone number automatically (multiphysics solver). + * \return Flag for appending zone numbers to restart and solution filenames. If Flag=true, zone numer is appended. + */ + bool GetMultizone_AdaptFilename(void) const { return Multizone_Adapt_FileName; } + + /*! + * \brief Provides the number of variables. * \return Number of variables. */ unsigned short GetnVar(void); /*! - * \brief Provides the number of varaibles. + * \brief Provides the number of variables. * \return Number of variables. */ unsigned short GetnZone(void) const { return nZone; } /*! - * \brief Provides the number of varaibles. + * \brief Provides the number of variables. * \return Number of variables. */ unsigned short GetiZone(void) const { return iZone; } diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 7d3e2f826e43..509660b2cfa4 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1190,6 +1190,8 @@ void CConfig::SetConfig_Options() { addBoolOption("WRT_VOLUME_OVERWRITE", Wrt_Volume_Overwrite, true); /*!\brief SYSTEM_MEASUREMENTS \n DESCRIPTION: System of measurements \n OPTIONS: see \link Measurements_Map \endlink \n DEFAULT: SI \ingroup Config*/ addEnumOption("SYSTEM_MEASUREMENTS", SystemMeasurements, Measurements_Map, SI); + /*!\brief MULTIZONE_ADAPT_FILENAME \n DESCRIPTION: Append zone numer to restart and solution filenames. \ingroup Config*/ + addBoolOption("MULTIZONE_ADAPT_FILENAME", Multizone_Adapt_FileName, YES); /*!\par CONFIG_CATEGORY: FluidModel \ingroup Config*/ /*!\brief FLUID_MODEL \n DESCRIPTION: Fluid model \n OPTIONS: See \link FluidModel_Map \endlink \n DEFAULT: STANDARD_AIR \ingroup Config*/ @@ -8393,7 +8395,7 @@ string CConfig::GetFilename(string filename, const string& ext, int timeIter) co filename = filename + string(ext); /*--- Append the zone number if multizone problems ---*/ - if (Multizone_Problem) + if (Multizone_Problem && Multizone_Adapt_FileName) filename = GetMultizone_FileName(filename, GetiZone(), ext); /*--- Append the zone number if multiple instance problems ---*/ From 8d5adc1f060d71d5085ce27a988917b5b8c9d041 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 13:35:01 +0200 Subject: [PATCH 080/102] Add test case (discrete adjoint for inviscid NACA0012 via multizone driver). --- .github/workflows/regression.yml | 2 +- TestCases/serial_regression_AD.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 6e949919415f..57c32d277e4f 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -261,7 +261,7 @@ jobs: uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 with: # -t -c - args: -b ${{github.ref}} -t develop -c develop -s ${{matrix.testscript}} -a "--tagtape" + args: -b ${{github.ref}} -t develop -c develop -s ${{matrix.testscript}} -a "--tapetests" - name: Cleanup uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 with: diff --git a/TestCases/serial_regression_AD.py b/TestCases/serial_regression_AD.py index 89f64d54c22b..59bf5b56decb 100644 --- a/TestCases/serial_regression_AD.py +++ b/TestCases/serial_regression_AD.py @@ -53,6 +53,14 @@ def main(): discadj_naca0012.test_vals = [-3.562611, -8.932639, -0.000000, 0.005608] test_list.append(discadj_naca0012) + # Inciscid NACA0012 (via discadj multizone driver) + discadj_naca0012_via_mz = TestCase('discadj_naca0012_via_mz') + discadj_naca0012_via_mz.cfg_dir = "cont_adj_euler/naca0012" + discadj_naca0012_via_mz.cfg_file = "inv_NACA0012_discadj_multizone.cfg" + discadj_naca0012_via_mz.test_iter = 100 + discadj_naca0012_via_mz.test_vals = [-3.563784, -5.975640, -6.326231, -8.929567] + test_list.append(discadj_naca0012_via_mz) + # Inviscid Cylinder 3D (multiple markers) discadj_cylinder3D = TestCase('discadj_cylinder3D') discadj_cylinder3D.cfg_dir = "disc_adj_euler/cylinder3D" From 08e185845d54e0324e3d8a697a62622c2216ee90 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 14:19:49 +0200 Subject: [PATCH 081/102] Add config file belonging to new test case. --- .../inv_NACA0012_discadj_multizone.cfg | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 TestCases/cont_adj_euler/naca0012/inv_NACA0012_discadj_multizone.cfg diff --git a/TestCases/cont_adj_euler/naca0012/inv_NACA0012_discadj_multizone.cfg b/TestCases/cont_adj_euler/naca0012/inv_NACA0012_discadj_multizone.cfg new file mode 100644 index 000000000000..85d1d03c5a24 --- /dev/null +++ b/TestCases/cont_adj_euler/naca0012/inv_NACA0012_discadj_multizone.cfg @@ -0,0 +1,112 @@ +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% % +% SU2 configuration file % +% Case description: Adjoint transonic inviscid flow around a NACA0012 airfoil % +% Author: Thomas D. Economon % +% Institution: Stanford University % +% Date: 2011.11.02 % +% File Version 8.2.0 "Harrier" % +% % +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% ------------- DIRECT, ADJOINT, AND LINEARIZED PROBLEM DEFINITION ------------% +% +MULTIZONE= YES +MULTIZONE_ADAPT_FILENAME= NO +SOLVER= EULER +MATH_PROBLEM= DISCRETE_ADJOINT +RESTART_SOL= NO +READ_BINARY_RESTART= NO + +% ----------- COMPRESSIBLE AND INCOMPRESSIBLE FREE-STREAM DEFINITION ----------% +% +MACH_NUMBER= 0.8 +AOA= 1.25 +FREESTREAM_PRESSURE= 101325.0 +FREESTREAM_TEMPERATURE= 288.15 + +% ---------------------- REFERENCE VALUE DEFINITION ---------------------------% +% +REF_ORIGIN_MOMENT_X = 0.25 +REF_ORIGIN_MOMENT_Y = 0.00 +REF_ORIGIN_MOMENT_Z = 0.00 +REF_LENGTH= 1.0 +REF_AREA= 1.0 + +% ----------------------- BOUNDARY CONDITION DEFINITION -----------------------% +% +MARKER_EULER= ( airfoil ) +MARKER_FAR= ( farfield ) +MARKER_PLOTTING= ( airfoil ) +MARKER_MONITORING= ( airfoil ) + +% ------------- COMMON PARAMETERS TO DEFINE THE NUMERICAL METHOD --------------% +% +NUM_METHOD_GRAD= GREEN_GAUSS +OBJECTIVE_FUNCTION= DRAG +CFL_NUMBER= 5.0 +CFL_ADAPT= NO +CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) +RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) +OUTER_ITER= 150 +INNER_ITER= 1 + +% -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% +% +CONV_NUM_METHOD_FLOW= JST +JST_SENSOR_COEFF= ( 0.5, 0.02 ) +TIME_DISCRE_FLOW= EULER_IMPLICIT + +% ---------------- ADJOINT-FLOW NUMERICAL METHOD DEFINITION -------------------% +% +CONV_NUM_METHOD_ADJFLOW= ROE +MUSCL_ADJFLOW= YES +SLOPE_LIMITER_ADJFLOW= NONE +ADJ_SHARP_LIMITER_COEFF= 3.0 +ADJ_JST_SENSOR_COEFF= ( 0.5, 0.02 ) +CFL_REDUCTION_ADJFLOW= 0.5 +TIME_DISCRE_ADJFLOW= EULER_IMPLICIT + +% ----------------------- DESIGN VARIABLE PARAMETERS --------------------------% +% +DV_KIND= HICKS_HENNE +DV_MARKER= ( airfoil ) +DV_PARAM= ( 1, 0.5 ) +DV_VALUE= 0.01 + +% ------------------------ GRID DEFORMATION PARAMETERS ------------------------% +% +DEFORM_STIFFNESS_TYPE= WALL_DISTANCE + +% --------------------------- CONVERGENCE PARAMETERS --------------------------% +% +CONV_RESIDUAL_MINVAL= -12 +CONV_STARTITER= 10 +CONV_CAUCHY_ELEMS= 100 +CONV_CAUCHY_EPS= 1E-6 + +% ------------------------- INPUT/OUTPUT INFORMATION --------------------------% +% +MESH_FILENAME= mesh_NACA0012_inv.su2 +MESH_FORMAT= SU2 +MESH_OUT_FILENAME= mesh_out.su2 +SOLUTION_FILENAME= solution_flow.dat +SOLUTION_ADJ_FILENAME= solution_adj.dat +TABULAR_FORMAT= TECPLOT +CONV_FILENAME= history +RESTART_FILENAME= restart_flow.dat +RESTART_ADJ_FILENAME= restart_adj.dat +VOLUME_FILENAME= flow +VOLUME_ADJ_FILENAME= adjoint +GRAD_OBJFUNC_FILENAME= of_grad.dat +SURFACE_FILENAME= surface_flow +SURFACE_ADJ_FILENAME= surface_adjoint +OUTPUT_WRT_FREQ= 250 +SCREEN_OUTPUT= (OUTER_ITER, BGS_RES[0]) +OUTPUT_FILES=(RESTART_ASCII) + +% --------------------- OPTIMAL SHAPE DESIGN DEFINITION -----------------------% +% +OPT_OBJECTIVE= DRAG * 0.001 +OPT_CONSTRAINT= ( LIFT > 0.327 ) * 0.001; ( MOMENT_Z > 0.0 ) * 0.001; ( AIRFOIL_THICKNESS > 0.12 ) * 0.001 +DEFINITION_DV= ( 30, 1.0 | airfoil | 0, 0.05 ); ( 30, 1.0 | airfoil | 0, 0.10 ); ( 30, 1.0 | airfoil | 0, 0.15 ); ( 30, 1.0 | airfoil | 0, 0.20 ); ( 30, 1.0 | airfoil | 0, 0.25 ); ( 30, 1.0 | airfoil | 0, 0.30 ); ( 30, 1.0 | airfoil | 0, 0.35 ); ( 30, 1.0 | airfoil | 0, 0.40 ); ( 30, 1.0 | airfoil | 0, 0.45 ); ( 30, 1.0 | airfoil | 0, 0.50 ); ( 30, 1.0 | airfoil | 0, 0.55 ); ( 30, 1.0 | airfoil | 0, 0.60 ); ( 30, 1.0 | airfoil | 0, 0.65 ); ( 30, 1.0 | airfoil | 0, 0.70 ); ( 30, 1.0 | airfoil | 0, 0.75 ); ( 30, 1.0 | airfoil | 0, 0.80 ); ( 30, 1.0 | airfoil | 0, 0.85 ); ( 30, 1.0 | airfoil | 0, 0.90 ); ( 30, 1.0 | airfoil | 0, 0.95 ); ( 30, 1.0 | airfoil | 1, 0.05 ); ( 30, 1.0 | airfoil | 1, 0.10 ); ( 30, 1.0 | airfoil | 1, 0.15 ); ( 30, 1.0 | airfoil | 1, 0.20 ); ( 30, 1.0 | airfoil | 1, 0.25 ); ( 30, 1.0 | airfoil | 1, 0.30 ); ( 30, 1.0 | airfoil | 1, 0.35 ); ( 30, 1.0 | airfoil | 1, 0.40 ); ( 30, 1.0 | airfoil | 1, 0.45 ); ( 30, 1.0 | airfoil | 1, 0.50 ); ( 30, 1.0 | airfoil | 1, 0.55 ); ( 30, 1.0 | airfoil | 1, 0.60 ); ( 30, 1.0 | airfoil | 1, 0.65 ); ( 30, 1.0 | airfoil | 1, 0.70 ); ( 30, 1.0 | airfoil | 1, 0.75 ); ( 30, 1.0 | airfoil | 1, 0.80 ); ( 30, 1.0 | airfoil | 1, 0.85 ); ( 30, 1.0 | airfoil | 1, 0.90 ); ( 30, 1.0 | airfoil | 1, 0.95 ) From d8f3746909d0144cd626a5f28bfde84fa0f3421f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 14:21:27 +0200 Subject: [PATCH 082/102] CodeFactor-encouraged format of new test case. --- TestCases/serial_regression_AD.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/TestCases/serial_regression_AD.py b/TestCases/serial_regression_AD.py index 59bf5b56decb..82a86f20583f 100644 --- a/TestCases/serial_regression_AD.py +++ b/TestCases/serial_regression_AD.py @@ -53,10 +53,10 @@ def main(): discadj_naca0012.test_vals = [-3.562611, -8.932639, -0.000000, 0.005608] test_list.append(discadj_naca0012) - # Inciscid NACA0012 (via discadj multizone driver) - discadj_naca0012_via_mz = TestCase('discadj_naca0012_via_mz') - discadj_naca0012_via_mz.cfg_dir = "cont_adj_euler/naca0012" - discadj_naca0012_via_mz.cfg_file = "inv_NACA0012_discadj_multizone.cfg" + # Inviscid NACA0012 (via discadj multizone driver) + discadj_naca0012_via_mz = TestCase('discadj_naca0012_via_mz') + discadj_naca0012_via_mz.cfg_dir = "cont_adj_euler/naca0012" + discadj_naca0012_via_mz.cfg_file = "inv_NACA0012_discadj_multizone.cfg" discadj_naca0012_via_mz.test_iter = 100 discadj_naca0012_via_mz.test_vals = [-3.563784, -5.975640, -6.326231, -8.929567] test_list.append(discadj_naca0012_via_mz) From 49f802a07e65d49117240671691ee86ffabd2176 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 15:05:35 +0200 Subject: [PATCH 083/102] Small fix. --- TestCases/TestCase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TestCases/TestCase.py b/TestCases/TestCase.py index 2b452177a75b..75174950110f 100644 --- a/TestCases/TestCase.py +++ b/TestCases/TestCase.py @@ -46,7 +46,7 @@ def parse_args(description: str): parser = argparse.ArgumentParser(description=description) parser.add_argument('--tsan', action='store_true', help='Run thread sanitizer tests. Requires a tsan-enabled SU2 build.') parser.add_argument('--asan', action='store_true', help='Run address sanitizer tests. Requires an asan-enabled SU2 build.') - parser.add_argument('--tagtape', action='store_true', help='Run discrete adjoint tests in tape debug mode. Requires a SU2_CFD_AD build with Tag reverse type.') + parser.add_argument('--tapetests', action='store_true', help='Run discrete adjoint tests in tape debug mode. Requires a SU2_CFD_AD build with Tag reverse type.') return parser.parse_args() class TestCase: From 0fb8b29291c7272162c7f492dcbb1ee24ce88e69 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Wed, 11 Jun 2025 23:45:57 +0200 Subject: [PATCH 084/102] Small update for new tag error callback function. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- Common/include/basic_types/ad_structure.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index d900b4f3db33..be1edfbcc4bc 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -738,7 +738,7 @@ FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } FORCEINLINE void ClearTagOnVariable(su2double& v) { AD::getTape().clearTagOnVariable(v); } static void tagErrorCallback(int const& correctTag, int const& wrongTag, void* userData) { - ErrorReport* report = (ErrorReport*)userData; + auto* report = static_cast(userData); report->ErrorCounter += 1; *(report->out) << "Use of variable with bad tag '" << wrongTag << "', should be '" << correctTag << "'." << std::endl; From b1915e72dc953ba68f7f001062d4ed0b31ba10e7 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 17 Jun 2025 10:00:56 +0200 Subject: [PATCH 085/102] Add 'tapetests' argument to (regression) test case functions to inform them whether the test case is run in a tapetest stage or not. --- TestCases/TestCase.py | 33 ++++++++++++++++++++----------- TestCases/serial_regression_AD.py | 18 ++++++++--------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/TestCases/TestCase.py b/TestCases/TestCase.py index 75174950110f..2e66005625ca 100644 --- a/TestCases/TestCase.py +++ b/TestCases/TestCase.py @@ -108,6 +108,9 @@ def __init__(self,tag_in): # multizone problem self.multizone = False + # Indicate tapetest mode + self.enabled_with_tapetests = False + # The test condition. These must be set after initialization self.test_iter = 1 self.ntest_vals = 4 @@ -128,9 +131,10 @@ def __init__(self,tag_in): self.reference_file_aarch64 = "" self.test_file = "of_grad.dat" - def run_test(self, with_tsan=False, with_asan=False): + def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): - if not self.is_enabled(with_tsan, with_asan): + # Check whether this test is valid and can be continued + if not self.is_enabled(with_tsan, with_asan, with_tapetests): return True print('==================== Start Test: %s ===================='%self.tag) @@ -284,9 +288,9 @@ def run_test(self, with_tsan=False, with_asan=False): os.chdir(workdir) return passed - def run_filediff(self, with_tsan=False, with_asan=False): + def run_filediff(self, with_tsan=False, with_asan=False, with_tapetests=False): - if not self.is_enabled(with_tsan, with_asan): + if not self.is_enabled(with_tsan, with_asan, with_tapetests): return True print('==================== Start Test: %s ===================='%self.tag) @@ -481,9 +485,9 @@ def run_filediff(self, with_tsan=False, with_asan=False): os.chdir(workdir) return passed - def run_opt(self, with_tsan=False, with_asan=False): + def run_opt(self, with_tsan=False, with_asan=False, with_tapetests=False): - if not self.is_enabled(with_tsan, with_asan): + if not self.is_enabled(with_tsan, with_asan, with_tapetests): return True print('==================== Start Test: %s ===================='%self.tag) @@ -606,9 +610,9 @@ def run_opt(self, with_tsan=False, with_asan=False): os.chdir(workdir) return passed - def run_geo(self, with_tsan=False, with_asan=False): + def run_geo(self, with_tsan=False, with_asan=False, with_tapetests=False): - if not self.is_enabled(with_tsan, with_asan): + if not self.is_enabled(with_tsan, with_asan, with_tapetests): return True print('==================== Start Test: %s ===================='%self.tag) @@ -746,9 +750,9 @@ def run_geo(self, with_tsan=False, with_asan=False): os.chdir(workdir) return passed - def run_def(self, with_tsan=False, with_asan=False): + def run_def(self, with_tsan=False, with_asan=False, with_tapetests=False): - if not self.is_enabled(with_tsan, with_asan): + if not self.is_enabled(with_tsan, with_asan, with_tapetests): return True print('==================== Start Test: %s ===================='%self.tag) @@ -971,19 +975,24 @@ def disable_restart(self): return - def is_enabled(self, with_tsan=False, with_asan=False): + def is_enabled(self, with_tsan=False, with_asan=False, with_tapetests=False): is_enabled_on_arch = self.cpu_arch in self.enabled_on_cpu_arch if not is_enabled_on_arch: print('Ignoring test "%s" because it is not enabled for the current CPU architecture: %s' % (self.tag, self.cpu_arch)) + # A test case is valid to be continued if neither of the special modes (tsan, asan, tapetests) is active, or if so, the corresponding test case is enabled, too. tsan_compatible = not with_tsan or self.enabled_with_tsan asan_compatible = not with_asan or self.enabled_with_asan + tapetests_compatible = not with_tapetests or self.enabled_with_tapetests if not tsan_compatible: print('Ignoring test "%s" because it is not enabled to run with the thread sanitizer.' % self.tag) - return is_enabled_on_arch and tsan_compatible and asan_compatible + if not tapetests_compatible: + print('Ignoring test "%s" because it is not enabled to run a test of the tape.' % self.tag) + + return is_enabled_on_arch and tsan_compatible and asan_compatible and tapetests_compatible and tapetests_compatible def adjust_test_data(self): diff --git a/TestCases/serial_regression_AD.py b/TestCases/serial_regression_AD.py index 82a86f20583f..7ba3abd62dc1 100644 --- a/TestCases/serial_regression_AD.py +++ b/TestCases/serial_regression_AD.py @@ -234,7 +234,7 @@ def main(): if test.tol == 0.0: test.tol = 0.00001 - pass_list = [ test.run_test(args.tsan, args.asan) for test in test_list ] + pass_list = [ test.run_test(args.tsan, args.asan, args.tapetests) for test in test_list ] ################################### ### Coupled RHT-CFD Adjoint ### @@ -251,7 +251,7 @@ def main(): discadj_rht.reference_file_aarch64 = "of_grad_cd_aarch64.csv.ref" discadj_rht.test_file = "of_grad_cd.csv" discadj_rht.enabled_with_asan = False - pass_list.append(discadj_rht.run_filediff(args.tsan, args.asan)) + pass_list.append(discadj_rht.run_filediff(args.tsan, args.asan, args.tapetests)) test_list.append(discadj_rht) ###################################### @@ -269,7 +269,7 @@ def main(): discadj_euler_py.reference_file_aarch64 = "of_grad_cd_disc_aarch64.dat.ref" discadj_euler_py.test_file = "of_grad_cd.dat" discadj_euler_py.enabled_with_asan = False - pass_list.append(discadj_euler_py.run_filediff(args.tsan, args.asan)) + pass_list.append(discadj_euler_py.run_filediff(args.tsan, args.asan, args.tapetests)) test_list.append(discadj_euler_py) # test discrete_adjoint with multiple ffd boxes @@ -283,7 +283,7 @@ def main(): discadj_multiple_ffd_py.reference_file_aarch64 = "of_grad_cd_aarch64.dat.ref" discadj_multiple_ffd_py.test_file = "of_grad_cd.dat" discadj_multiple_ffd_py.enabled_with_asan = False - pass_list.append(discadj_multiple_ffd_py.run_filediff(args.tsan, args.asan)) + pass_list.append(discadj_multiple_ffd_py.run_filediff(args.tsan, args.asan, args.tapetests)) test_list.append(discadj_multiple_ffd_py) # test direct_differentiation.py @@ -297,7 +297,7 @@ def main(): directdiff_euler_py.reference_file_aarch64 = "of_grad_directdiff_aarch64.dat.ref" directdiff_euler_py.test_file = "DIRECTDIFF/of_grad_directdiff.dat" directdiff_euler_py.enabled_with_asan = False - pass_list.append(directdiff_euler_py.run_filediff(args.tsan, args.asan)) + pass_list.append(directdiff_euler_py.run_filediff(args.tsan, args.asan, args.tapetests)) test_list.append(directdiff_euler_py) # test direct_differentiation.py with multiple ffd boxes @@ -311,7 +311,7 @@ def main(): directdiff_multiple_ffd_py.reference_file_aarch64 = "of_grad_directdiff_aarch64.dat.ref" directdiff_multiple_ffd_py.test_file = "DIRECTDIFF/of_grad_directdiff.dat" directdiff_multiple_ffd_py.enabled_with_asan = False - pass_list.append(directdiff_multiple_ffd_py.run_filediff(args.tsan, args.asan)) + pass_list.append(directdiff_multiple_ffd_py.run_filediff(args.tsan, args.asan, args.tapetests)) test_list.append(directdiff_multiple_ffd_py) # test continuous_adjoint.py, with multiple objectives @@ -338,7 +338,7 @@ def main(): pywrapper_FEA_AD_FlowLoad.new_output = False pywrapper_FEA_AD_FlowLoad.enabled_with_asan = False test_list.append(pywrapper_FEA_AD_FlowLoad) - pass_list.append(pywrapper_FEA_AD_FlowLoad.run_test(args.tsan, args.asan)) + pass_list.append(pywrapper_FEA_AD_FlowLoad.run_test(args.tsan, args.asan, args.tapetests)) # Flow AD Mesh Displacement Sensitivity pywrapper_CFD_AD_MeshDisp = TestCase('pywrapper_CFD_AD_MeshDisp') @@ -352,7 +352,7 @@ def main(): pywrapper_CFD_AD_MeshDisp.new_output = False pywrapper_CFD_AD_MeshDisp.enabled_with_asan = False test_list.append(pywrapper_CFD_AD_MeshDisp) - pass_list.append(pywrapper_CFD_AD_MeshDisp.run_test(args.tsan, args.asan)) + pass_list.append(pywrapper_CFD_AD_MeshDisp.run_test(args.tsan, args.asan, args.tapetests)) ################################### @@ -369,7 +369,7 @@ def main(): grad_smooth_naca0012.reference_file_aarch64 = "of_hess_aarch64.dat.ref" grad_smooth_naca0012.test_file = "of_hess.dat" grad_smooth_naca0012.enabled_with_asan = False - pass_list.append(grad_smooth_naca0012.run_filediff(args.tsan, args.asan)) + pass_list.append(grad_smooth_naca0012.run_filediff(args.tsan, args.asan, args.tapetests)) test_list.append(grad_smooth_naca0012) # Tests summary From fd918eef5e55b02522b1f86baf886c6b91e98419 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 17 Jun 2025 16:06:02 +0200 Subject: [PATCH 086/102] Some renamings for error reports of new tape test mode. --- .../drivers/CDiscAdjMultizoneDriver.hpp | 8 ++++---- .../src/drivers/CDiscAdjMultizoneDriver.cpp | 18 +++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 653566b64d89..e1e7a0a85731 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -142,14 +142,14 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { void StartSolver() override; /*! - * \brief [Overload] Launch the debug mode for the discrete adjoint multizone solver. + * \brief [Overload] Launch the tape test mode for the discrete adjoint multizone solver. */ - void DebugRun(); + void TapeTest(); /*! - * \brief [Overload] Launch the debug mode for the discrete adjoint multizone solver. + * \brief [Overload] Get error numbers after a tape test run of the discrete adjoint multizone solver. */ - void DebugRun_ScreenOutput(struct AD::ErrorReport& error_report); + int TapeTest_GatherErrors(struct AD::ErrorReport& error_report); /*! * \brief Preprocess the multizone iteration diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 2215a93c535b..783b07347a6e 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -167,7 +167,7 @@ void CDiscAdjMultizoneDriver::StartSolver() { cout << "\nSU2_CFD_AD is compiled for debug mode recording. To resume the discrete adjoint solver, adjust -Dcodi-tape (-Dcodi-tape=JacobianLinear by default) and recompile." << endl; } Preprocess(0); - DebugRun(); + TapeTest(); return; } @@ -229,12 +229,13 @@ void CDiscAdjMultizoneDriver::StartSolver() { } -void CDiscAdjMultizoneDriver::DebugRun() { +void CDiscAdjMultizoneDriver::TapeTest() { if (rank == MASTER_NODE) { cout <<"\n---------------------------- Start Debug Run ----------------------------" << endl; } + int total_errors = 0; struct AD::ErrorReport error_report; AD::SetTagErrorCallback(&error_report); std::ofstream out1("run1_process" + to_string(rank) + ".out"); @@ -266,7 +267,7 @@ void CDiscAdjMultizoneDriver::DebugRun() { SetRecording(RECORDING::TAG_INIT_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } } - DebugRun_ScreenOutput(error_report); + total_errors = TapeTest_GatherErrors(error_report); AD::ResetErrorCounter(error_report); AD::SetDebugReportFile(&error_report, &out2); @@ -289,14 +290,16 @@ void CDiscAdjMultizoneDriver::DebugRun() { else SetRecording(RECORDING::TAG_CHECK_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } - DebugRun_ScreenOutput(error_report); + total_errors += TapeTest_GatherErrors(error_report); if (rank == MASTER_NODE) { - cout <<"\n----------------------------- End Debug Run -----------------------------" << endl; + cout << "\n------------------------- Tape Test Run Summary -------------------------" << endl; + cout << "\nTotal number of tape inconsistencies: " << total_errors << endl; + cout << "\n--------------------------- End Tape Test Run ---------------------------" << endl; } } -void CDiscAdjMultizoneDriver::DebugRun_ScreenOutput(struct AD::ErrorReport& error_report) { +int CDiscAdjMultizoneDriver::TapeTest_GatherErrors(struct AD::ErrorReport& error_report) { int num_errors = AD::GetErrorCount(error_report); int total_errors = 0; @@ -305,7 +308,7 @@ void CDiscAdjMultizoneDriver::DebugRun_ScreenOutput(struct AD::ErrorReport& erro SU2_MPI::Gather(&num_errors, 1, MPI_INT, process_error.data(), 1, MPI_INT, 0, SU2_MPI::GetComm()); if (rank == MASTER_NODE) { - std::cout << "\nTotal number of detected tape inconsistencies: " << total_errors << std::endl; + std::cout << "\nNumber of detected tape inconsistencies: " << total_errors << std::endl; if(total_errors > 0 && size > 1) { std::cout << "\n"; for (int irank = 0; irank < size; irank++) { @@ -313,6 +316,7 @@ void CDiscAdjMultizoneDriver::DebugRun_ScreenOutput(struct AD::ErrorReport& erro } } } + return total_errors; } bool CDiscAdjMultizoneDriver::Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode) { From 7c9d4b3524ee4528ce2bdfa541742b3f8ff4f81f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 17 Jun 2025 16:13:35 +0200 Subject: [PATCH 087/102] Add tape test output examination, enable new 'discadj_naca0012_via_mz' test case for tape testing. --- TestCases/TestCase.py | 30 +++++++++++++++++++++++++++--- TestCases/serial_regression_AD.py | 2 ++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/TestCases/TestCase.py b/TestCases/TestCase.py index 2e66005625ca..0f36bef44408 100644 --- a/TestCases/TestCase.py +++ b/TestCases/TestCase.py @@ -115,6 +115,7 @@ def __init__(self,tag_in): self.test_iter = 1 self.ntest_vals = 4 self.test_vals = [] + self.tapetest_vals = [] self.test_vals_aarch64 = [] self.cpu_arch = platform.machine().casefold() self.enabled_on_cpu_arch = ["x86_64","amd64","aarch64","arm64"] @@ -123,6 +124,7 @@ def __init__(self,tag_in): self.command = self.Command() self.timeout = 0 self.tol = 0.0 + self.tapetest_tol = 0 self.tol_file_percent = 0.0 self.comp_threshold = 0.0 @@ -193,7 +195,7 @@ def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): delta_vals = [] sim_vals = [] - if not with_tsan and not with_asan: # sanitizer findings result in non-zero return code, no need to examine the output + if not with_tsan and not with_asan and not with_tapetests: # Sanitizer findings result in non-zero return code, no need to examine the output. Tapetest output is examined separately. # Examine the output f = open(logfilename,'r') output = f.readlines() @@ -245,6 +247,28 @@ def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): #for j in output: # print(j) + if with_tapetests and self.enabled_with_tapetests: # examine the tapetest output + f = open(logfilename,'r') + output = f.readlines() + if not timed_out and len(self.tapetest_vals) != 0: + start_solver = False + for line in output: + if not start_solver: # Don't bother parsing anything before "Total number of tape inconsistencies"; for consistency, we keep the "start_solver" boolean + if line.find('Total number of tape inconsistencies:') > -1: + start_solver=True + raw_data = line.split(':') # Split line into description and the string representing the number of errors + data = raw_data[1].strip() # Clear the string representing the number of errors (for now, expecting a single integer, but zone-wise error numbers are planned) + + if not len(self.tapetest_vals)==len(data): # something went wrong... probably bad input + print("Error in tapetest_vals!") + passed = False + break + for j in range(len(data)): + sim_vals.append( int(data[j]) ) + delta_vals.append( abs(int(data[j])-self.tapetest_vals[j]) ) + if delta_vals[j] > self.tapetest_tol: + exceed_tol = True + passed = False process.communicate() if process.returncode != 0: @@ -267,12 +291,12 @@ def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): if not start_solver: print('ERROR: The code was not able to get to the "Begin solver" section.') - if not with_tsan and not with_asan and iter_missing: + if not with_tsan and not with_asan and not with_tapetests and iter_missing: print('ERROR: The iteration number %d could not be found.'%self.test_iter) print('CPU architecture=%s' % self.cpu_arch) - if len(self.test_vals) != 0: + if len(self.test_vals) != 0 and not with_tapetests: print('test_iter=%d' % self.test_iter) print_vals(self.test_vals, name="test_vals (stored)") diff --git a/TestCases/serial_regression_AD.py b/TestCases/serial_regression_AD.py index 7ba3abd62dc1..687a459cb870 100644 --- a/TestCases/serial_regression_AD.py +++ b/TestCases/serial_regression_AD.py @@ -59,6 +59,8 @@ def main(): discadj_naca0012_via_mz.cfg_file = "inv_NACA0012_discadj_multizone.cfg" discadj_naca0012_via_mz.test_iter = 100 discadj_naca0012_via_mz.test_vals = [-3.563784, -5.975640, -6.326231, -8.929567] + discadj_naca0012_via_mz.enabled_with_tapetests = True + discadj_naca0012_via_mz.tapetest_vals = [0] test_list.append(discadj_naca0012_via_mz) # Inviscid Cylinder 3D (multiple markers) From eb819d2a4d6589ab09b2aa4ba89fe5164ba205e1 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 17 Jun 2025 16:57:08 +0200 Subject: [PATCH 088/102] Small fixes for new tape test output examination. --- TestCases/TestCase.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/TestCases/TestCase.py b/TestCases/TestCase.py index 0f36bef44408..2da91f390462 100644 --- a/TestCases/TestCase.py +++ b/TestCases/TestCase.py @@ -145,6 +145,7 @@ def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): timed_out = False iter_missing = True start_solver = True + tapetest_out = True # if root, add flag to mpirun self.command.allow_mpi_as_root() @@ -197,8 +198,8 @@ def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): if not with_tsan and not with_asan and not with_tapetests: # Sanitizer findings result in non-zero return code, no need to examine the output. Tapetest output is examined separately. # Examine the output - f = open(logfilename,'r') - output = f.readlines() + with open(logfilename,'r') as f: + output = f.readlines() if not timed_out and len(self.test_vals) != 0: start_solver = False for line in output: @@ -248,14 +249,14 @@ def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): # print(j) if with_tapetests and self.enabled_with_tapetests: # examine the tapetest output - f = open(logfilename,'r') - output = f.readlines() + with open(logfilename,'r') as f: + output = f.readlines() if not timed_out and len(self.tapetest_vals) != 0: - start_solver = False + tapetest_out = False for line in output: - if not start_solver: # Don't bother parsing anything before "Total number of tape inconsistencies"; for consistency, we keep the "start_solver" boolean + if not tapetest_out: # Don't bother parsing anything before "Total number of tape inconsistencies" if line.find('Total number of tape inconsistencies:') > -1: - start_solver=True + tapetest_out = True raw_data = line.split(':') # Split line into description and the string representing the number of errors data = raw_data[1].strip() # Clear the string representing the number of errors (for now, expecting a single integer, but zone-wise error numbers are planned) @@ -270,6 +271,10 @@ def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): exceed_tol = True passed = False + if not tapetest_out: + passed = False + + process.communicate() if process.returncode != 0: passed = False @@ -288,9 +293,12 @@ def run_test(self, with_tsan=False, with_asan=False, with_tapetests=False): if exceed_tol: print('ERROR: Difference between computed input and test_vals exceeded tolerance. TOL=%f'%self.tol) - if not start_solver: + if not start_solver and not with_tapetests: print('ERROR: The code was not able to get to the "Begin solver" section.') + if not tapetest_out and with_tapetests: + print('ERROR: The code was not able to get to the "Total number of tape inconsistencies" line.') + if not with_tsan and not with_asan and not with_tapetests and iter_missing: print('ERROR: The iteration number %d could not be found.'%self.test_iter) @@ -725,7 +733,7 @@ def run_geo(self, with_tsan=False, with_asan=False, with_tapetests=False): delta_vals.append( abs(float(data[j])-self.test_vals[j]) ) if delta_vals[j] > self.tol: exceed_tol = True - passed = False + passed = False else: iter_missing = True From 5ca673c473e728d9966c6077fa48cb0502848bcf Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 17 Jun 2025 19:08:15 +0200 Subject: [PATCH 089/102] Provide new tape test case the correct SU2_CFD_AD build (left as sanity check). --- .github/workflows/regression.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 57c32d277e4f..73533f6ce341 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -228,8 +228,7 @@ jobs: testscript: ['serial_regression_AD.py'] include: - testscript: 'serial_regression_AD.py' - #tag: TagNoMPI, check: - tag: NoMPI + tag: TagNoMPI steps: - name: Pre Cleanup uses: docker://ghcr.io/su2code/su2/test-su2:240320-1536 From ff1249aff793f277d8c92b8cd3a33679dbe9ede7 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 17 Jun 2025 22:31:57 +0200 Subject: [PATCH 090/102] Add new config options for tape test mode to config_template.cfg. --- config_template.cfg | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/config_template.cfg b/config_template.cfg index 6b3d5ab396aa..1548e6c57a0e 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -2038,13 +2038,20 @@ TOTAL_DV_PENALTY= 0.0 % % Parameters for the corresponding OF (allowed stress and KS multiplier). STRESS_PENALTY_PARAM= (1.0, 10.0) -% % ---------------- AUTOMATIC DIFFERENTIATION -------------------% % % Preaccumulation in the AD mode. PREACC= YES +% ---------------- AUTOMATIC DIFFERENTIATION (TAPE TEST DEBUG MODE) -------------------% +% +% Specify the kind of tape that is checked (OBJECTIVE_FUNCTION_TAPE, FULL_SOLVER_TAPE) +CHECK_TAPE_TYPE = FULL_SOLVER_TAPE +% +% Specify the variables for which the tape is checked (SOLVER_VARIABLES, SOLVER_VARIABLES_AND_MESH_COORDINATES) +CHECK_TAPE_VARIABLES = SOLVER_VARIABLES + % ---------------- MESH DEFORMATION PARAMETERS (NEW SOLVER) -------------------% % % Use the reformatted pseudo-elastic solver for grid deformation From 62a5f42bc72f15db4430032a2e39eed6836f38dd Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 17 Jun 2025 22:34:41 +0200 Subject: [PATCH 091/102] Add new config option to config_template.cfg for whether solution/restart filenames are extended automatically by the zone number. --- config_template.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config_template.cfg b/config_template.cfg index 1548e6c57a0e..2879a104fb50 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -2380,6 +2380,9 @@ TABULAR_FORMAT= CSV % Set .precision(value) to specified value for SU2_DOT and HISTORY output. Useful for exact gradient validation. OUTPUT_PRECISION= 10 % +% For multizone problems, extend solution and restart filenames automatically by zone number +MULTIZONE_ADAPT_FILENAME= YES +% % Files to output % Possible formats : (TECPLOT_ASCII, TECPLOT, SURFACE_TECPLOT_ASCII, % SURFACE_TECPLOT, CSV, SURFACE_CSV, PARAVIEW_ASCII, PARAVIEW_LEGACY, SURFACE_PARAVIEW_ASCII, From 9fcb160ef1ab74616b79b0ddf61d6b24ed10ea67 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Fri, 20 Jun 2025 13:49:35 +0200 Subject: [PATCH 092/102] Small changes in error report functions for new tape test mode. --- Common/include/basic_types/ad_structure.hpp | 18 +++++++++--------- .../src/drivers/CDiscAdjMultizoneDriver.cpp | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index be1edfbcc4bc..bb28f35dbbb7 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -292,19 +292,19 @@ struct ErrorReport {}; * \param[in] report - the ErrorReport whose output file is set. * \param[in] output_file - pointer to the output file. */ -inline void SetDebugReportFile(struct ErrorReport* report, std::ofstream* output_file) {} +inline void SetDebugReportFile(ErrorReport& report, std::ostream* output_file) {} /*! * \brief Set the ErrorReport to which error information from a tag debug recording is written. * \param[in] report - the ErrorReport to which error information is written. */ -inline void SetTagErrorCallback(struct ErrorReport* report) {} +inline void SetTagErrorCallback(ErrorReport& report) {} /*! * \brief Reset the error counter in an ErrorReport. * \param[in] report - the ErrorReport whose error counter is resetted. */ -inline void ResetErrorCounter(struct ErrorReport& report) {} +inline void ResetErrorCounter(ErrorReport& report) {} /*! * \brief Get the error count of an ErrorReport. @@ -726,11 +726,11 @@ struct ErrorReport { FORCEINLINE void ResetErrorCounter(ErrorReport& report) { report.ErrorCounter = 0; } -FORCEINLINE void SetDebugReportFile(struct ErrorReport* report, std::ofstream* output_file) { - report->out = output_file; +FORCEINLINE void SetDebugReportFile(ErrorReport& report, std::ostream* output_file) { + report.out = output_file; } -FORCEINLINE unsigned long GetErrorCount(ErrorReport& report) { return report.ErrorCounter; } +FORCEINLINE unsigned long GetErrorCount(const ErrorReport& report) { return report.ErrorCounter; } #ifdef CODI_TAG_TAPE @@ -744,14 +744,14 @@ static void tagErrorCallback(int const& correctTag, int const& wrongTag, void* u *(report->out) << "Use of variable with bad tag '" << wrongTag << "', should be '" << correctTag << "'." << std::endl; } -FORCEINLINE void SetTagErrorCallback(struct ErrorReport* report) { - AD::getTape().setTagErrorCallback(tagErrorCallback, report); +FORCEINLINE void SetTagErrorCallback(ErrorReport& report) { + AD::getTape().setTagErrorCallback(tagErrorCallback, &report); } #else FORCEINLINE void SetTag(int tag) {} FORCEINLINE void ClearTagOnVariable(su2double& v) {} -FORCEINLINE void SetTagErrorCallback(struct ErrorReport* report) {} +FORCEINLINE void SetTagErrorCallback(ErrorReport report) {} #endif // CODI_TAG_TAPE diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 783b07347a6e..be55fcae3a3d 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -237,12 +237,12 @@ void CDiscAdjMultizoneDriver::TapeTest() { int total_errors = 0; struct AD::ErrorReport error_report; - AD::SetTagErrorCallback(&error_report); + AD::SetTagErrorCallback(error_report); std::ofstream out1("run1_process" + to_string(rank) + ".out"); std::ofstream out2("run2_process" + to_string(rank) + ".out"); AD::ResetErrorCounter(error_report); - AD::SetDebugReportFile(&error_report, &out1); + AD::SetDebugReportFile(error_report, &out1); /*--- This recording will assign the initial (same) tag to each registered variable. * During the recording, each dependent variable will be assigned the same tag. ---*/ @@ -270,7 +270,7 @@ void CDiscAdjMultizoneDriver::TapeTest() { total_errors = TapeTest_GatherErrors(error_report); AD::ResetErrorCounter(error_report); - AD::SetDebugReportFile(&error_report, &out2); + AD::SetDebugReportFile(error_report, &out2); /*--- This recording repeats the initial recording with a different tag. * If a variable was used before it became dependent on the inputs, this variable will still carry the tag From 9bfbd40481a0657c03aa929f94a916f15502817f Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Sat, 21 Jun 2025 19:30:30 +0200 Subject: [PATCH 093/102] Clang format. --- Common/include/basic_types/ad_structure.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index bb28f35dbbb7..1062e49a39ed 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -726,9 +726,7 @@ struct ErrorReport { FORCEINLINE void ResetErrorCounter(ErrorReport& report) { report.ErrorCounter = 0; } -FORCEINLINE void SetDebugReportFile(ErrorReport& report, std::ostream* output_file) { - report.out = output_file; -} +FORCEINLINE void SetDebugReportFile(ErrorReport& report, std::ostream* output_file) { report.out = output_file; } FORCEINLINE unsigned long GetErrorCount(const ErrorReport& report) { return report.ErrorCounter; } From 3f25e6009994b8de15db14d9204302400080a5b6 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Sat, 21 Jun 2025 19:56:22 +0200 Subject: [PATCH 094/102] Update for alignment in vectorization.hpp. --- Common/include/parallelization/vectorization.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/parallelization/vectorization.hpp b/Common/include/parallelization/vectorization.hpp index 0360b344946d..1e4b8b77f327 100644 --- a/Common/include/parallelization/vectorization.hpp +++ b/Common/include/parallelization/vectorization.hpp @@ -94,7 +94,7 @@ class Array : public CVecExpr, Scalar_t> { static constexpr bool StoreAsRef = true; private: - alignas(Size * 32) Scalar x_[N]; + alignas(Size * alignof(Scalar)) Scalar x_[N]; public: #define ARRAY_BOILERPLATE \ From 91e6172d792c983d7f1dfd000db21c228a577f4e Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Sun, 22 Jun 2025 02:23:37 +0200 Subject: [PATCH 095/102] Fix typo in CConfig.cpp. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- Common/src/CConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 509660b2cfa4..4347ee741129 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1190,7 +1190,7 @@ void CConfig::SetConfig_Options() { addBoolOption("WRT_VOLUME_OVERWRITE", Wrt_Volume_Overwrite, true); /*!\brief SYSTEM_MEASUREMENTS \n DESCRIPTION: System of measurements \n OPTIONS: see \link Measurements_Map \endlink \n DEFAULT: SI \ingroup Config*/ addEnumOption("SYSTEM_MEASUREMENTS", SystemMeasurements, Measurements_Map, SI); - /*!\brief MULTIZONE_ADAPT_FILENAME \n DESCRIPTION: Append zone numer to restart and solution filenames. \ingroup Config*/ + /*!\brief MULTIZONE_ADAPT_FILENAME \n DESCRIPTION: Append zone number to restart and solution filenames. \ingroup Config*/ addBoolOption("MULTIZONE_ADAPT_FILENAME", Multizone_Adapt_FileName, YES); /*!\par CONFIG_CATEGORY: FluidModel \ingroup Config*/ From 4a7c1efd3fe5623e9129ed0ff3a31935ccc1ce57 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Sun, 22 Jun 2025 02:29:57 +0200 Subject: [PATCH 096/102] Small change in output of tape test mode. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index be55fcae3a3d..e820a272fd5f 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -308,7 +308,7 @@ int CDiscAdjMultizoneDriver::TapeTest_GatherErrors(struct AD::ErrorReport& error SU2_MPI::Gather(&num_errors, 1, MPI_INT, process_error.data(), 1, MPI_INT, 0, SU2_MPI::GetComm()); if (rank == MASTER_NODE) { - std::cout << "\nNumber of detected tape inconsistencies: " << total_errors << std::endl; + std::cout << "\nTotal number of detected tape inconsistencies: " << total_errors << std::endl; if(total_errors > 0 && size > 1) { std::cout << "\n"; for (int irank = 0; irank < size; irank++) { From aa5d8f2ea7ba2c0d0edfcd210d5ccd3b5600550a Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Jun 2025 12:23:08 +0200 Subject: [PATCH 097/102] Exclude max variables from recording (follow-up from fixing preaccumulation error). --- Common/include/basic_types/ad_structure.hpp | 15 +++++++++++++++ SU2_CFD/include/solvers/CFVMFlowSolverBase.inl | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 1062e49a39ed..f35ddc6ab21c 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -48,6 +48,17 @@ using Identifier = int; */ inline void StartRecording() {} +/*! + * \brief Pause the recording of the operations and involved variables. + * If called, all operations occuring after the call will not be stored on the computational graph. + */ +inline void PauseRecording() {} + +/*! + * \brief Resume the recording of the operations and variables after the recording had been paused. + */ +inline void ResumeRecording() {} + /*! * \brief Stops the recording of the operations and variables. */ @@ -402,6 +413,10 @@ FORCEINLINE void ResetInput(su2double& data) { data = data.getValue(); } FORCEINLINE void StartRecording() { AD::getTape().setActive(); } +FORCEINLINE void PauseRecording() { AD::getTape().setPassive(); } + +FORCEINLINE void ResumeRecording() { AD::getTape().setActive(); } + FORCEINLINE void StopRecording() { AD::getTape().setPassive(); } FORCEINLINE bool TapeActive() { return AD::getTape().isActive(); } diff --git a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl index 26c8ddb591a1..0bc80882cbd9 100644 --- a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl +++ b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl @@ -673,9 +673,11 @@ void CFVMFlowSolverBase::ComputeVorticityAndStrainMag(const CConfig& confi AD::SetPreaccOut(StrainMag(iPoint)); AD::EndPreacc(); - /*--- Max is not differentiable, so we not register them for preacc. ---*/ + /*--- We don't need to record strainMax and omegaMax. ---*/ + AD::PauseRecording(); strainMax = max(strainMax, StrainMag(iPoint)); omegaMax = max(omegaMax, GeometryToolbox::Norm(3, Vorticity)); + AD::ResumeRecording(); } END_SU2_OMP_FOR From b2987413e50587211bffa9f310ef7e4487c49c24 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Tue, 24 Jun 2025 19:11:27 +0200 Subject: [PATCH 098/102] Undo excluding max computations from preaccumulation region, fix preaccumulation region by setting further inputs and outputs. --- SU2_CFD/include/solvers/CFVMFlowSolverBase.inl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl index 0bc80882cbd9..7484cbdd6658 100644 --- a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl +++ b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl @@ -646,7 +646,10 @@ void CFVMFlowSolverBase::ComputeVorticityAndStrainMag(const CConfig& confi const su2double vy = nodes->GetVelocity(iPoint, 1); const su2double y = geometry->nodes->GetCoord(iPoint, 1); AD::StartPreacc(); + AD::SetPreaccIn(omegaMax); + AD::SetPreaccIn(strainMax); AD::SetPreaccIn(VelocityGradient, nDim, nDim); + AD::SetPreaccIn(Vorticity, 3); AD::SetPreaccIn(vy, y); StrainMag(iPoint) = 0.0; @@ -671,14 +674,13 @@ void CFVMFlowSolverBase::ComputeVorticityAndStrainMag(const CConfig& confi StrainMag(iPoint) = sqrt(2.0*StrainMag(iPoint)); AD::SetPreaccOut(StrainMag(iPoint)); - AD::EndPreacc(); - /*--- We don't need to record strainMax and omegaMax. ---*/ - AD::PauseRecording(); strainMax = max(strainMax, StrainMag(iPoint)); + AD::SetPreaccOut(strainMax); omegaMax = max(omegaMax, GeometryToolbox::Norm(3, Vorticity)); - AD::ResumeRecording(); + AD::SetPreaccOut(omegaMax); + AD::EndPreacc(); } END_SU2_OMP_FOR From 619b6cf4284164b9970efa3d9d8801aabe1ccddb Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 26 Jun 2025 00:30:50 +0200 Subject: [PATCH 099/102] Small update in new tape test mode for MPI gather call. Co-authored-by: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> --- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index e820a272fd5f..065a063c128e 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -305,7 +305,7 @@ int CDiscAdjMultizoneDriver::TapeTest_GatherErrors(struct AD::ErrorReport& error int total_errors = 0; std::vector process_error(size); SU2_MPI::Allreduce(&num_errors, &total_errors, 1, MPI_INT, MPI_SUM, SU2_MPI::GetComm()); - SU2_MPI::Gather(&num_errors, 1, MPI_INT, process_error.data(), 1, MPI_INT, 0, SU2_MPI::GetComm()); + SU2_MPI::Gather(&num_errors, 1, MPI_INT, process_error.data(), 1, MPI_INT, MASTER_NODE, SU2_MPI::GetComm()); if (rank == MASTER_NODE) { std::cout << "\nTotal number of detected tape inconsistencies: " << total_errors << std::endl; From 9b7e676725567cb53222b0dfaf1c97bb75c317b3 Mon Sep 17 00:00:00 2001 From: Ole Burghardt Date: Thu, 26 Jun 2025 00:45:34 +0200 Subject: [PATCH 100/102] Const new TapeTestGatherErrors. --- SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp | 4 ++-- SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index e1e7a0a85731..8eec26d2e82b 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -144,12 +144,12 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { /*! * \brief [Overload] Launch the tape test mode for the discrete adjoint multizone solver. */ - void TapeTest(); + void TapeTest (); /*! * \brief [Overload] Get error numbers after a tape test run of the discrete adjoint multizone solver. */ - int TapeTest_GatherErrors(struct AD::ErrorReport& error_report); + int TapeTestGatherErrors(AD::ErrorReport& error_report) const; /*! * \brief Preprocess the multizone iteration diff --git a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp index 065a063c128e..42d2de2c1c59 100644 --- a/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp +++ b/SU2_CFD/src/drivers/CDiscAdjMultizoneDriver.cpp @@ -236,7 +236,7 @@ void CDiscAdjMultizoneDriver::TapeTest() { } int total_errors = 0; - struct AD::ErrorReport error_report; + AD::ErrorReport error_report; AD::SetTagErrorCallback(error_report); std::ofstream out1("run1_process" + to_string(rank) + ".out"); std::ofstream out2("run2_process" + to_string(rank) + ".out"); @@ -267,7 +267,7 @@ void CDiscAdjMultizoneDriver::TapeTest() { SetRecording(RECORDING::TAG_INIT_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } } - total_errors = TapeTest_GatherErrors(error_report); + total_errors = TapeTestGatherErrors(error_report); AD::ResetErrorCounter(error_report); AD::SetDebugReportFile(error_report, &out2); @@ -290,7 +290,7 @@ void CDiscAdjMultizoneDriver::TapeTest() { else SetRecording(RECORDING::TAG_CHECK_SOLVER_VARIABLES, Kind_Tape::FULL_SOLVER_TAPE, ZONE_0); } - total_errors += TapeTest_GatherErrors(error_report); + total_errors += TapeTestGatherErrors(error_report); if (rank == MASTER_NODE) { cout << "\n------------------------- Tape Test Run Summary -------------------------" << endl; @@ -299,7 +299,7 @@ void CDiscAdjMultizoneDriver::TapeTest() { } } -int CDiscAdjMultizoneDriver::TapeTest_GatherErrors(struct AD::ErrorReport& error_report) { +int CDiscAdjMultizoneDriver::TapeTestGatherErrors(AD::ErrorReport& error_report) const { int num_errors = AD::GetErrorCount(error_report); int total_errors = 0; From 3890998391b536d8b350c1e37a01274438fa872b Mon Sep 17 00:00:00 2001 From: Pedro Gomes <38071223+pcarruscag@users.noreply.github.com> Date: Sat, 28 Jun 2025 08:37:40 -0700 Subject: [PATCH 101/102] Update Common/include/basic_types/ad_structure.hpp --- Common/include/basic_types/ad_structure.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index f35ddc6ab21c..6d1ec65cc28f 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -322,7 +322,7 @@ inline void ResetErrorCounter(ErrorReport& report) {} * \param[in] report - the ErrorReport whose pointer to its error counter is returned. * \return Value of the error counter. */ -inline unsigned long GetErrorCount(ErrorReport& report) { return 0; } +inline unsigned long GetErrorCount(const ErrorReport& report) { return 0; } /*! * \brief Pushes back the current tape position to the tape position's vector. From a42600069bdbb4911d6f2efc7766708cf1ebf759 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sat, 28 Jun 2025 08:49:59 -0700 Subject: [PATCH 102/102] make omega/strain max passive --- Common/include/basic_types/ad_structure.hpp | 2 +- SU2_CFD/include/solvers/CFVMFlowSolverBase.inl | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 6d1ec65cc28f..5a02b23c2c67 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -750,7 +750,7 @@ FORCEINLINE unsigned long GetErrorCount(const ErrorReport& report) { return repo FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } FORCEINLINE void ClearTagOnVariable(su2double& v) { AD::getTape().clearTagOnVariable(v); } -static void tagErrorCallback(int const& correctTag, int const& wrongTag, void* userData) { +static void tagErrorCallback(const int& correctTag, const int& wrongTag, void* userData) { auto* report = static_cast(userData); report->ErrorCounter += 1; diff --git a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl index 7484cbdd6658..9ffb59baf2e8 100644 --- a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl +++ b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl @@ -646,10 +646,7 @@ void CFVMFlowSolverBase::ComputeVorticityAndStrainMag(const CConfig& confi const su2double vy = nodes->GetVelocity(iPoint, 1); const su2double y = geometry->nodes->GetCoord(iPoint, 1); AD::StartPreacc(); - AD::SetPreaccIn(omegaMax); - AD::SetPreaccIn(strainMax); AD::SetPreaccIn(VelocityGradient, nDim, nDim); - AD::SetPreaccIn(Vorticity, 3); AD::SetPreaccIn(vy, y); StrainMag(iPoint) = 0.0; @@ -674,13 +671,11 @@ void CFVMFlowSolverBase::ComputeVorticityAndStrainMag(const CConfig& confi StrainMag(iPoint) = sqrt(2.0*StrainMag(iPoint)); AD::SetPreaccOut(StrainMag(iPoint)); - - strainMax = max(strainMax, StrainMag(iPoint)); - AD::SetPreaccOut(strainMax); - omegaMax = max(omegaMax, GeometryToolbox::Norm(3, Vorticity)); - AD::SetPreaccOut(omegaMax); - AD::EndPreacc(); + + /*--- Max is not differentiable, so we not register them for preacc. ---*/ + strainMax = SU2_TYPE::GetValue(max(strainMax, StrainMag(iPoint))); + omegaMax = SU2_TYPE::GetValue(max(omegaMax, GeometryToolbox::Norm(3, Vorticity))); } END_SU2_OMP_FOR