From 71191b611e0a2f60f100a04d0564673600a18d95 Mon Sep 17 00:00:00 2001 From: sdixon Date: Fri, 17 Jan 2025 14:27:44 +0000 Subject: [PATCH 01/11] UKAEA issue 192: fixing typo in IDL wrapper --- source/wrappers/idl/getstruct.pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/wrappers/idl/getstruct.pro b/source/wrappers/idl/getstruct.pro index 4446137f8..67d4e714b 100755 --- a/source/wrappers/idl/getstruct.pro +++ b/source/wrappers/idl/getstruct.pro @@ -61,7 +61,7 @@ function checknamesarelegal, names, verbose=verbose c = badchars[j] r = strpos(names[i], string(c)) ; Search for illegal characters within the tag name if(r ge 0) then begin - if keword_set(verbose) then print,'WARNING: An Illegal structure name tag has been detected ['+names[i]+'] - Replacing '+string(c)+' with '+sub + if keyword_set(verbose) then print,'WARNING: An Illegal structure name tag has been detected ['+names[i]+'] - Replacing '+string(c)+' with '+sub b = BYTE(names[i]) w = WHERE(b eq c, nw) if(nw gt 0) then begin From fcb1b1aed8eafae86a7f375d772d7b465dabb82c Mon Sep 17 00:00:00 2001 From: sdixon Date: Fri, 16 May 2025 16:01:55 +0100 Subject: [PATCH 02/11] UKAEA issue 192: fixing typo in IDL wrapper --- source/wrappers/idl/getstruct.pro | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/wrappers/idl/getstruct.pro b/source/wrappers/idl/getstruct.pro index 67d4e714b..ac9354379 100755 --- a/source/wrappers/idl/getstruct.pro +++ b/source/wrappers/idl/getstruct.pro @@ -130,7 +130,8 @@ function makeidamstructurearrayatomic, handle, tree, count, rank, shape, debug=d apointer = getidamnodeatomicpointers(handle, tree, debug=debug, verbose=verbose) ; Is this Atomic Component a Pointer ? anamelist2 = anamelist - anamelist2 = checknamesarelegal(anamelist2, verbose=verbose) ; Ensure names are OK: Legal IDL variable names! + ; Ensure names are OK: Legal IDL variable names! + anamelist2 = idl_validname(anamelist2, /CONVERT_ALL) xcount = getidamnodeatomicdatacount(handle, tree, anamelist[0], debug=debug, verbose=verbose) ; Array of non-zero count? if(xcount gt 0) then begin @@ -249,7 +250,8 @@ function makeidamstructureitem, handle, tree, debug=debug, verbose=verbose if(acount gt 0) then begin anamelist2 = anamelist - anamelist2 = checknamesarelegal(anamelist2, verbose=verbose) ; Ensure names are OK: Legal! + ; Ensure names are OK: Legal! + anamelist2 = idl_validname(anamelist2, /CONVERT_ALL) xcount = getidamnodeatomicdatacount(handle, tree, anamelist[0], debug=debug, verbose=verbose); if(xcount gt 0) then begin data = getidamnodeatomicdata(handle, tree, anamelist[0], debug=debug, verbose=verbose) @@ -292,7 +294,8 @@ function makeidamstructureitem, handle, tree, debug=debug, verbose=verbose if(scount gt 0) then begin snamelist2 = snamelist - snamelist2 = checknamesarelegal(snamelist2, verbose=verbose) ; Ensure names are OK: Legal! + ; Ensure names are OK: Legal! + snamelist2 = idl_validname(snamelist2, /CONVERT_ALL) ;childcount = getidamnodechildrencount(handle, tree) ;if(childcount eq 0 and scount eq 1) then begin From 46c99369558edf9207388f4965f67b98fdcde90f Mon Sep 17 00:00:00 2001 From: sdixon Date: Fri, 16 May 2025 16:19:33 +0100 Subject: [PATCH 03/11] UKAEA issue #196: Allowing good data to be returned when the get_bad client flag is set --- source/client/accAPI.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/client/accAPI.cpp b/source/client/accAPI.cpp index 2c2fa449c..bbed39e43 100755 --- a/source/client/accAPI.cpp +++ b/source/client/accAPI.cpp @@ -1143,7 +1143,6 @@ char* getIdamSyntheticData(int handle) int status = getIdamDataStatus(handle); if (handle < 0 || (unsigned int)handle >= data_blocks.size()) return nullptr; if (status == MIN_STATUS && !data_blocks[handle].client_block.get_bad && !get_bad) return nullptr; - if (status != MIN_STATUS && (data_blocks[handle].client_block.get_bad || get_bad)) return nullptr; if (!get_synthetic || data_blocks[handle].error_model == ERROR_MODEL_UNKNOWN) { return data_blocks[handle].data; } @@ -1161,7 +1160,6 @@ char* getIdamData(int handle) int status = getIdamDataStatus(handle); if (handle < 0 || (unsigned int)handle >= data_blocks.size()) return nullptr; if (status == MIN_STATUS && !data_blocks[handle].client_block.get_bad && !get_bad) return nullptr; - if (status != MIN_STATUS && (data_blocks[handle].client_block.get_bad || get_bad)) return nullptr; if (!get_synthetic) { return data_blocks[handle].data; } else { @@ -1498,7 +1496,6 @@ void getIdamDoubleData(int handle, double* fp) int status = getIdamDataStatus(handle); if (handle < 0 || (unsigned int)handle >= data_blocks.size()) return; if (status == MIN_STATUS && !data_blocks[handle].client_block.get_bad && !get_bad) return; - if (status != MIN_STATUS && (data_blocks[handle].client_block.get_bad || get_bad)) return; if (data_blocks[handle].data_type == UDA_TYPE_DOUBLE) { if (!get_synthetic) @@ -1637,7 +1634,6 @@ void getIdamFloatData(int handle, float* fp) return; } if (status == MIN_STATUS && !data_blocks[handle].client_block.get_bad && !get_bad) return; - if (status != MIN_STATUS && (data_blocks[handle].client_block.get_bad || get_bad)) return; if (data_blocks[handle].data_type == UDA_TYPE_FLOAT) { if (!get_synthetic) From 915d70f12c84eea862f35e796566b20faeb22221 Mon Sep 17 00:00:00 2001 From: sdixon Date: Wed, 21 May 2025 17:33:08 +0100 Subject: [PATCH 04/11] replacing all sprintf calls with snprintf --- source/cache/fileCache.cpp | 3 +- source/client/clientAPI.cpp | 6 +-- source/client/connection.cpp | 4 +- source/client/getEnvironment.cpp | 2 +- source/client/makeClientRequestBlock.cpp | 4 +- source/clientserver/expand_path.cpp | 16 +++---- source/clientserver/makeRequestBlock.cpp | 12 ++--- source/clientserver/nameValueSubstitution.cpp | 10 ++-- source/clientserver/protocolXML.cpp | 2 +- source/clientserver/protocolXML2.cpp | 2 +- source/plugins/help/help_plugin.cpp | 44 +++++++++--------- source/plugins/testplugin/testplugin.cpp | 18 ++++---- source/plugins/uda/uda_plugin.cpp | 9 ++-- source/server/fatServer.cpp | 6 +-- source/server/getPluginAddress.cpp | 7 +-- source/server/initPluginList.cpp | 6 +-- source/server/serverPlugin.cpp | 6 +-- source/server/serverStartup.cpp | 2 +- source/server/serverSubsetData.cpp | 2 +- source/server/udaServer.cpp | 32 ++++++------- source/structures/accessors.cpp | 46 +++++++++---------- source/structures/struct.cpp | 2 +- 22 files changed, 122 insertions(+), 119 deletions(-) diff --git a/source/cache/fileCache.cpp b/source/cache/fileCache.cpp index 89b332e9c..e58f2d1a9 100755 --- a/source/cache/fileCache.cpp +++ b/source/cache/fileCache.cpp @@ -490,7 +490,8 @@ int set_entry_state(const CacheEntry& entry, EntryState state) fseek(db, entry.file_position, SEEK_SET); char buffer[3]; - sprintf(buffer, "%d%c", static_cast(state), delimiter[0]); + // note: the EntryState enum currently only has values of {0, 1, 2} so will only print one character + snprintf(buffer, 3, "%d%c", static_cast(state), delimiter[0]); fwrite(buffer, sizeof(char), 2, db); rc = set_db_file_lock_state(db, LockActionType::UNLOCK); // release lock diff --git a/source/client/clientAPI.cpp b/source/client/clientAPI.cpp index 3b16a02cb..4ae826752 100755 --- a/source/client/clientAPI.cpp +++ b/source/client/clientAPI.cpp @@ -59,9 +59,9 @@ int idamClientAPI(const char* file, const char* signal, int pass, int exp_number char data_source[STRING_LENGTH + 1]; if (strlen(file) == 0) { if (pass < 0) - sprintf(data_source, "%d", exp_number); + snprintf(data_source, STRING_LENGTH, "%d", exp_number); else - sprintf(data_source, "%d/%d", exp_number, pass); + snprintf(data_source, STRING_LENGTH, "%d/%d", exp_number, pass); } else { strcpy(data_source, file); } @@ -128,7 +128,7 @@ int idamClientFileAPI(const char* file, const char* signal, const char* format) if (strlen(format) == 0) strcpy(data_source, file); else - sprintf(data_source, "%s::%s", format, file); + snprintf(data_source, STRING_LENGTH, "%s::%s", format, file); if ((err = makeClientRequestBlock(&signal, (const char**)&data_source, 1, &request_block)) != 0) { closeUdaError(); diff --git a/source/client/connection.cpp b/source/client/connection.cpp index 4367ebca2..58f601709 100755 --- a/source/client/connection.cpp +++ b/source/client/connection.cpp @@ -291,7 +291,7 @@ int createConnection() } } udaClientPutHostNameId(hostId); - sprintf(serviceport, "%d", environment->server_port); + snprintf(serviceport, PORT_STRING, "%d", environment->server_port); // Does the host name contain the SSL protocol prefix? If so strip this off @@ -421,7 +421,7 @@ int createConnection() } } udaClientPutHostNameId(hostId); - sprintf(serviceport, "%d", environment->server_port2); + snprintf(serviceport, PORT_STRING, "%d", environment->server_port2); // Does the host name contain the SSL protocol prefix? If so strip this off diff --git a/source/client/getEnvironment.cpp b/source/client/getEnvironment.cpp index 5cf0be6e2..53a9685d6 100755 --- a/source/client/getEnvironment.cpp +++ b/source/client/getEnvironment.cpp @@ -78,7 +78,7 @@ ENVIRONMENT* getIdamClientEnvironment() if (udaEnviron.loglevel <= UDA_LOG_ACCESS) { char cmd[STRING_LENGTH]; - sprintf(cmd, "mkdir -p %s 2>/dev/null", udaEnviron.logdir); + snprintf(cmd, STRING_LENGTH, "mkdir -p %s 2>/dev/null", udaEnviron.logdir); if (system(cmd) != 0) { // TODO: How to log error before log files are open? }; diff --git a/source/client/makeClientRequestBlock.cpp b/source/client/makeClientRequestBlock.cpp index 256efcc47..281619319 100755 --- a/source/client/makeClientRequestBlock.cpp +++ b/source/client/makeClientRequestBlock.cpp @@ -73,7 +73,7 @@ int makeRequestData(const char* data_object, const char* data_source, REQUEST_DA THROW_ERROR(SOURCE_ARG_TOO_LONG, "The Data Source Argument, prefixed with the Device Name, is too long!"); } char* test = (char*)malloc((lstr + 1) * sizeof(char)); - sprintf(test, "%s%s%s", environment->api_device, request->api_delim, request->source); + snprintf(test, (lstr + 1), "%s%s%s", environment->api_device, request->api_delim, request->source); strcpy(request->source, test); free(test); } @@ -85,7 +85,7 @@ int makeRequestData(const char* data_object, const char* data_source, REQUEST_DA THROW_ERROR(SIGNAL_ARG_TOO_LONG, "The Signal/Data Object Argument, prefixed with the Archive Name, is too long!"); } char* test = (char*)malloc((lstr + 1) * sizeof(char)); - sprintf(test, "%s%s%s", environment->api_archive, request->api_delim, request->signal); + snprintf(test, (lstr+1), "%s%s%s", environment->api_archive, request->api_delim, request->signal); strcpy(request->signal, test); free(test); } diff --git a/source/clientserver/expand_path.cpp b/source/clientserver/expand_path.cpp index 5935e5435..8f2fb9756 100755 --- a/source/clientserver/expand_path.cpp +++ b/source/clientserver/expand_path.cpp @@ -419,7 +419,7 @@ int linkReplacement(char* path) // If the user has embedded linux commands within the source string, they will be exposed here // within the client's environment - not the server's. - sprintf(cmd, "ls -l %s 2>&1;", path); + snprintf(cmd, STRING_LENGTH, "ls -l %s 2>&1;", path); errno = 0; if ((ph = popen(cmd, "r")) == nullptr) { @@ -542,7 +542,7 @@ int expandFilePath(char* path, const ENVIRONMENT* environment) */ #ifdef SCRATCHDIR - sprintf(scratch, "/%s/", SCRATCHDIR); + snprintf(scratch, STRING_LENGTH, "/%s/", SCRATCHDIR); lscratch = (int)strlen(scratch); #else strcpy(scratch, "/scratch/"); @@ -558,7 +558,7 @@ int expandFilePath(char* path, const ENVIRONMENT* environment) // Override compiler options if ((env = getenv("UDA_SCRATCHNAME")) != nullptr) { // Check for Environment Variable - sprintf(scratch, "/%s/", env); + snprintf(scratch, STRING_LENGTH, "/%s/", env); lscratch = (int)strlen(scratch); } @@ -647,7 +647,7 @@ int expandFilePath(char* path, const ENVIRONMENT* environment) if ((fp = strrchr(path, '/')) == nullptr) { // Search backwards - extract filename strcpy(work1, path); - sprintf(path, "%s/%s", cwd, work1); // prepend the CWD and return + snprintf(path, STRING_LENGTH, "%s/%s", cwd, work1); // prepend the CWD and return if ((err = linkReplacement(path)) != 0) return err; if ((err = pathReplacement(path, environment)) != 0) return err; @@ -759,7 +759,7 @@ int expandFilePath(char* path, const ENVIRONMENT* environment) //! Prepend the expanded/resolved directory name to the File Name - sprintf(path, "%s/%s", work1, file); // Prepend the path to the filename + snprintf(path, STRING_LENGTH, "%s/%s", work1, file); // Prepend the path to the filename } // End of t1 - t5 tests @@ -793,13 +793,13 @@ int expandFilePath(char* path, const ENVIRONMENT* environment) } if (strlen(netname) > 0 && strlen(host) > 0) { - sprintf(path, "/%s/%s%s", netname, host, work); // prepend /netname/hostname to /scratch/... + snprintf(path, STRING_LENGTH, "/%s/%s%s", netname, host, work); // prepend /netname/hostname to /scratch/... } else { if (strlen(netname) > 0) { - sprintf(path, "/%s%s", netname, work); + snprintf(path, STRING_LENGTH, "/%s%s", netname, work); } else { if (strlen(host) > 0) { - sprintf(path, "/%s%s", host, work); + snprintf(path, STRING_LENGTH, "/%s%s", host, work); } } } diff --git a/source/clientserver/makeRequestBlock.cpp b/source/clientserver/makeRequestBlock.cpp index 70d045e51..73042f442 100644 --- a/source/clientserver/makeRequestBlock.cpp +++ b/source/clientserver/makeRequestBlock.cpp @@ -78,8 +78,8 @@ int makeRequestData(REQUEST_DATA* request, PLUGINLIST pluginList, const ENVIRONM //------------------------------------------------------------------------------ // Check there is something to work with! - sprintf(work, "%s%s", environment->api_archive, environment->api_delim); // default archive - sprintf(work2, "%s%s", environment->api_device, environment->api_delim); // default device + snprintf(work, MAXMETA, "%s%s", environment->api_archive, environment->api_delim); // default archive + snprintf(work2, MAXMETA, "%s%s", environment->api_device, environment->api_delim); // default device LeftTrimString(request->signal); TrimString(request->signal); @@ -348,7 +348,7 @@ int makeRequestData(REQUEST_DATA* request, PLUGINLIST pluginList, const ENVIRONM int id = find_plugin_id_by_format(pluginList.plugin[i].deviceProtocol, &pluginList); if (id >= 0 && pluginList.plugin[id].plugin_class == UDA_PLUGIN_CLASS_SERVER) { - sprintf(work, "%s%s%s", pluginList.plugin[i].deviceProtocol, request->api_delim, + snprintf(work, MAXMETA, "%s%s%s", pluginList.plugin[i].deviceProtocol, request->api_delim, pluginList.plugin[i].deviceHost); UDA_LOG(UDA_LOG_DEBUG, "work#1: %s\n", work); @@ -800,7 +800,7 @@ int source_file_format_test(const char* source, REQUEST_DATA* request, PLUGINLIS FILE* ph = nullptr; int lstr = STRING_LENGTH; char* cmd = (char*)malloc(lstr * sizeof(char)); - sprintf(cmd, "head -c10 %s 2>/dev/null", source); + snprintf(cmd, lstr, "head -c10 %s 2>/dev/null", source); errno = 0; if ((ph = popen(cmd, "r")) == nullptr) { if (errno != 0) addIdamError(SYSTEMERRORTYPE, "sourceFileFormatTest", errno, ""); @@ -829,7 +829,7 @@ int source_file_format_test(const char* source, REQUEST_DATA* request, PLUGINLIS if (STR_EQUALS(cmd, "HDF")) { // Either a netCDF or a HDF5 file: use utility programs to reveal! char* env = getenv("UDA_DUMP_NETCDF"); if (env != nullptr) { - sprintf(cmd, "%s -h %s 2>/dev/null | head -c10 2>/dev/null", env, source); + snprintf(cmd, lstr, "%s -h %s 2>/dev/null | head -c10 2>/dev/null", env, source); errno = 0; if ((ph = popen(cmd, "r")) == nullptr) { if (errno != 0) { @@ -861,7 +861,7 @@ int source_file_format_test(const char* source, REQUEST_DATA* request, PLUGINLIS } else { // an IDA File? char* env = getenv("UDA_DUMP_IDA"); if (env != nullptr) { - sprintf(cmd, "%s -h %s 2>/dev/null 2>/dev/null", env, source); + snprintf(cmd, lstr, "%s -h %s 2>/dev/null 2>/dev/null", env, source); errno = 0; if ((ph = popen(cmd, "r")) == nullptr) { if (errno != 0) { diff --git a/source/clientserver/nameValueSubstitution.cpp b/source/clientserver/nameValueSubstitution.cpp index f2ddfd32b..2edfc5786 100644 --- a/source/clientserver/nameValueSubstitution.cpp +++ b/source/clientserver/nameValueSubstitution.cpp @@ -61,7 +61,7 @@ int name_value_substitution(NAMEVALUELIST* nameValueList, char* tpass) int* placeholderIndex = nullptr; int* tpassIndex = nullptr; int tpassPosition = 0; - unsigned short usedCount = 0; + // unsigned short usedCount = 0; if (nameValueList->pairCount > 0) { // Identify and Count Placeholders do { @@ -134,7 +134,7 @@ int name_value_substitution(NAMEVALUELIST* nameValueList, char* tpass) free(nameValueList->nameValue[placeholderIndex[i]].value); nameValueList->nameValue[placeholderIndex[i]].value = newNameValueList.nameValue[tpassIndex[i]].value; newNameValueList.nameValue[tpassIndex[i]].value = nullptr; - usedCount++; + // usedCount++; UDA_LOG(UDA_LOG_DEBUG, "Placeholder: [%d][%d] %s, Substitution Value [%d] %s\n", i, placeholderIndex[i], nameValueList->nameValue[placeholderIndex[i]].name, @@ -276,11 +276,11 @@ void embedded_value_substitution(NAMEVALUELIST* nameValueList) UDA_LOG(UDA_LOG_DEBUG, "Substituting %s with %s\n", newNameValueList.nameValue[j].value, nameValueList->nameValue[k].value); - char* replace = (char*)malloc( - (strlen(original) + strlen(nameValueList->nameValue[k].value) + 2) * sizeof(char)); + const auto lstr = strlen(original) + strlen(nameValueList->nameValue[k].value) + 2; + char* replace = (char*)malloc(lstr * sizeof(char)); replace[0] = '\0'; p[0] = '\0'; - sprintf(replace, "%s%s%s", original, nameValueList->nameValue[k].value, &p[lstr]); + snprintf(replace, lstr, "%s%s%s", original, nameValueList->nameValue[k].value, &p[lstr]); UDA_LOG(UDA_LOG_DEBUG, "original %s\n", original); UDA_LOG(UDA_LOG_DEBUG, "value %s\n", nameValueList->nameValue[k].value); diff --git a/source/clientserver/protocolXML.cpp b/source/clientserver/protocolXML.cpp index f9b0a448f..5df3ae623 100755 --- a/source/clientserver/protocolXML.cpp +++ b/source/clientserver/protocolXML.cpp @@ -192,7 +192,7 @@ int protocolXML(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALLOC char* env; if ((env = getenv("UDA_WORK_DIR")) != nullptr) { // File to record XDR encoded data - sprintf(tempFile, "%s/idamXDRXXXXXX", env); + snprintf(tempFile, MAXPATH, "%s/idamXDRXXXXXX", env); } // Server calling another server diff --git a/source/clientserver/protocolXML2.cpp b/source/clientserver/protocolXML2.cpp index a6654e1b6..c2f7d501d 100755 --- a/source/clientserver/protocolXML2.cpp +++ b/source/clientserver/protocolXML2.cpp @@ -126,7 +126,7 @@ protocolXML2(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALLOCLIS if ((privateFlags & PRIVATEFLAG_XDRFILE) && protocolVersion >= 5) { // Intermediate XDR File, not stream if ((env = getenv("UDA_WORK_DIR")) != nullptr) { // File to record XDR encoded data - sprintf(tempFile, "%s/idamXDRXXXXXX", env); + snprintf(tempFile, MAXPATH, "%s/idamXDRXXXXXX", env); } } diff --git a/source/plugins/help/help_plugin.cpp b/source/plugins/help/help_plugin.cpp index 72f43e8c8..126e3907c 100755 --- a/source/plugins/help/help_plugin.cpp +++ b/source/plugins/help/help_plugin.cpp @@ -209,7 +209,7 @@ static int do_services(IDAM_PLUGIN_INTERFACE* idam_plugin_interface) count++; } - sprintf(rec, "\nTotal number of registered plugins available: %d\n", count); + snprintf(rec, STRING_LENGTH, "\nTotal number of registered plugins available: %d\n", count); strcat(doc, rec); for (int j = 0; j < 5; j++) { @@ -228,7 +228,7 @@ static int do_services(IDAM_PLUGIN_INTERFACE* idam_plugin_interface) count++; } - sprintf(rec, "\nNumber of plugins for data file formats: %d\n\n", count); + snprintf(rec, STRING_LENGTH, "\nNumber of plugins for data file formats: %d\n\n", count); strcat(doc, rec); for (int i = 0; i < pluginList->count; i++) @@ -237,13 +237,13 @@ static int do_services(IDAM_PLUGIN_INTERFACE* idam_plugin_interface) (pluginList->plugin[i].is_private == UDA_PLUGIN_PUBLIC || (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user)) && pluginList->plugin[i].format[0] != '\0' && pluginList->plugin[i].extension[0] != '\0') { - sprintf(rec, "File format:\t\t%s\n", pluginList->plugin[i].format); + snprintf(rec, STRING_LENGTH, "File format:\t\t%s\n", pluginList->plugin[i].format); strcat(doc, rec); - sprintf(rec, "Filename extension:\t%s\n", pluginList->plugin[i].extension); + snprintf(rec, STRING_LENGTH, "Filename extension:\t%s\n", pluginList->plugin[i].extension); strcat(doc, rec); - sprintf(rec, "Description:\t\t%s\n", pluginList->plugin[i].desc); + snprintf(rec, STRING_LENGTH, "Description:\t\t%s\n", pluginList->plugin[i].desc); strcat(doc, rec); - sprintf(rec, "Example API call:\t%s\n\n", pluginList->plugin[i].example); + snprintf(rec, STRING_LENGTH, "Example API call:\t%s\n\n", pluginList->plugin[i].example); strcat(doc, rec); } break; @@ -257,18 +257,18 @@ static int do_services(IDAM_PLUGIN_INTERFACE* idam_plugin_interface) (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user))) { count++; } - sprintf(rec, "\nNumber of plugins for Libraries: %d\n\n", count); + snprintf(rec, STRING_LENGTH, "\nNumber of plugins for Libraries: %d\n\n", count); strcat(doc, rec); for (int i = 0; i < pluginList->count; i++) if (pluginList->plugin[i].plugin_class == target && pluginList->plugin[i].status == UDA_PLUGIN_OPERATIONAL && (pluginList->plugin[i].is_private == UDA_PLUGIN_PUBLIC || (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user))) { - sprintf(rec, "Library name:\t\t%s\n", pluginList->plugin[i].format); + snprintf(rec, STRING_LENGTH, "Library name:\t\t%s\n", pluginList->plugin[i].format); strcat(doc, rec); - sprintf(rec, "Description:\t\t%s\n", pluginList->plugin[i].desc); + snprintf(rec, STRING_LENGTH, "Description:\t\t%s\n", pluginList->plugin[i].desc); strcat(doc, rec); - sprintf(rec, "Example API call:\t%s\n\n", pluginList->plugin[i].example); + snprintf(rec, STRING_LENGTH, "Example API call:\t%s\n\n", pluginList->plugin[i].example); strcat(doc, rec); } break; @@ -282,18 +282,18 @@ static int do_services(IDAM_PLUGIN_INTERFACE* idam_plugin_interface) (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user))) { count++; } - sprintf(rec, "\nNumber of plugins for Data Servers: %d\n\n", count); + snprintf(rec, STRING_LENGTH, "\nNumber of plugins for Data Servers: %d\n\n", count); strcat(doc, rec); for (int i = 0; i < pluginList->count; i++) if (pluginList->plugin[i].plugin_class == target && pluginList->plugin[i].status == UDA_PLUGIN_OPERATIONAL && (pluginList->plugin[i].is_private == UDA_PLUGIN_PUBLIC || (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user))) { - sprintf(rec, "Server name:\t\t%s\n", pluginList->plugin[i].format); + snprintf(rec, STRING_LENGTH, "Server name:\t\t%s\n", pluginList->plugin[i].format); strcat(doc, rec); - sprintf(rec, "Description:\t\t%s\n", pluginList->plugin[i].desc); + snprintf(rec, STRING_LENGTH, "Description:\t\t%s\n", pluginList->plugin[i].desc); strcat(doc, rec); - sprintf(rec, "Example API call:\t%s\n\n", pluginList->plugin[i].example); + snprintf(rec, STRING_LENGTH, "Example API call:\t%s\n\n", pluginList->plugin[i].example); strcat(doc, rec); } break; @@ -307,18 +307,18 @@ static int do_services(IDAM_PLUGIN_INTERFACE* idam_plugin_interface) (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user))) { count++; } - sprintf(rec, "\nNumber of plugins for External Devices: %d\n\n", count); + snprintf(rec, STRING_LENGTH, "\nNumber of plugins for External Devices: %d\n\n", count); strcat(doc, rec); for (int i = 0; i < pluginList->count; i++) if (pluginList->plugin[i].plugin_class == target && pluginList->plugin[i].status == UDA_PLUGIN_OPERATIONAL && (pluginList->plugin[i].is_private == UDA_PLUGIN_PUBLIC || (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user))) { - sprintf(rec, "External device name:\t%s\n", pluginList->plugin[i].format); + snprintf(rec, STRING_LENGTH, "External device name:\t%s\n", pluginList->plugin[i].format); strcat(doc, rec); - sprintf(rec, "Description:\t\t%s\n", pluginList->plugin[i].desc); + snprintf(rec, STRING_LENGTH, "Description:\t\t%s\n", pluginList->plugin[i].desc); strcat(doc, rec); - sprintf(rec, "Example API call:\t%s\n\n", pluginList->plugin[i].example); + snprintf(rec, STRING_LENGTH, "Example API call:\t%s\n\n", pluginList->plugin[i].example); strcat(doc, rec); } break; @@ -332,18 +332,18 @@ static int do_services(IDAM_PLUGIN_INTERFACE* idam_plugin_interface) (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user))) { count++; } - sprintf(rec, "\nNumber of plugins for Other data services: %d\n\n", count); + snprintf(rec, STRING_LENGTH, "\nNumber of plugins for Other data services: %d\n\n", count); strcat(doc, rec); for (int i = 0; i < pluginList->count; i++) if (pluginList->plugin[i].plugin_class == target && pluginList->plugin[i].status == UDA_PLUGIN_OPERATIONAL && (pluginList->plugin[i].is_private == UDA_PLUGIN_PUBLIC || (pluginList->plugin[i].is_private == UDA_PLUGIN_PRIVATE && !environment->external_user))) { - sprintf(rec, "Data service name:\t%s\n", pluginList->plugin[i].format); + snprintf(rec, STRING_LENGTH, "Data service name:\t%s\n", pluginList->plugin[i].format); strcat(doc, rec); - sprintf(rec, "Description:\t\t%s\n", pluginList->plugin[i].desc); + snprintf(rec, STRING_LENGTH, "Description:\t\t%s\n", pluginList->plugin[i].desc); strcat(doc, rec); - sprintf(rec, "Example API call:\t%s\n\n", pluginList->plugin[i].example); + snprintf(rec, STRING_LENGTH, "Example API call:\t%s\n\n", pluginList->plugin[i].example); strcat(doc, rec); } break; diff --git a/source/plugins/testplugin/testplugin.cpp b/source/plugins/testplugin/testplugin.cpp index 71048b671..ed228f286 100755 --- a/source/plugins/testplugin/testplugin.cpp +++ b/source/plugins/testplugin/testplugin.cpp @@ -3250,12 +3250,12 @@ static int do_test50(IDAM_PLUGIN_INTERFACE* idam_plugin_interface) work[0] = '\0'; strcpy(work, "test50 passed parameters and substitutions\n"); - sprintf(&work[strlen(work)], "Shot number:%d\n", request->exp_number); - sprintf(&work[strlen(work)], "Pass number:%d\n", request->pass); - sprintf(&work[strlen(work)], "substitution parameters:%s\n", request->tpass); - sprintf(&work[strlen(work)], "Number of name-value pairs: %d\n", request->nameValueList.pairCount); + snprintf(&work[strlen(work)], (count - strlen(work)), "Shot number:%d\n", request->exp_number); + snprintf(&work[strlen(work)], (count - strlen(work)), "Pass number:%d\n", request->pass); + snprintf(&work[strlen(work)], (count - strlen(work)), "substitution parameters:%s\n", request->tpass); + snprintf(&work[strlen(work)], (count - strlen(work)), "Number of name-value pairs: %d\n", request->nameValueList.pairCount); for (int i = 0; i < request->nameValueList.pairCount; i++) - sprintf(&work[strlen(work)], "name: %s, value: %s\n", request->nameValueList.nameValue[i].name, + snprintf(&work[strlen(work)], (count - strlen(work)), "name: %s, value: %s\n", request->nameValueList.nameValue[i].name, request->nameValueList.nameValue[i].value); UDA_LOG(UDA_LOG_DEBUG, "test50: %s\n", work); @@ -3899,7 +3899,7 @@ int createUDTSocket(int* usock, int port, int rendezvous) hints.ai_socktype = g_Socket_Type; char service[16]; - sprintf(service, "%d", port); + snprintf(service, 16, "%d", port); if (0 != getaddrinfo(nullptr, service, &hints, &res)) { int err = 9991; @@ -3964,7 +3964,7 @@ int createTCPSocket(SYSSOCKET* ssock, int port, bool rendezvous) hints.ai_socktype = g_Socket_Type; char service[16]; - sprintf(service, "%d", port); + snprintf(service, 16, "%d", port); if (0 != getaddrinfo(nullptr, service, &hints, &res)) { int err = 999; @@ -3999,7 +3999,7 @@ int c_connect(UDTSOCKET* usock, int port) hints.ai_socktype = g_Socket_Type; char buffer[16]; - sprintf(buffer, "%d", port); + snprintf(buffer, 16, "%d", port); if (0 != getaddrinfo(g_Localhost, buffer, &hints, &peer)) { int err = 999; @@ -4023,7 +4023,7 @@ int tcp_connect(SYSSOCKET* ssock, int port) hints.ai_socktype = g_Socket_Type; char buffer[16]; - sprintf(buffer, "%d", port); + snprintf(buffer, 16, "%d", port); if (0 != getaddrinfo(g_Localhost, buffer, &hints, &peer)) { int err = 999; diff --git a/source/plugins/uda/uda_plugin.cpp b/source/plugins/uda/uda_plugin.cpp index 7ced6bd21..c6e45931e 100755 --- a/source/plugins/uda/uda_plugin.cpp +++ b/source/plugins/uda/uda_plugin.cpp @@ -344,11 +344,12 @@ Notes: there are three pathways depending on the request pattern if (pathway == 1) { // Request via the Database - char signal[2 * MAXNAME + 2]; - char source[2 * MAXNAME + 2]; + const auto lstr = 2 * MAXNAME + 2; + char signal[lstr]; + char source[lstr]; - sprintf(signal, "%s::%s", data_source->archive, signal_desc->signal_name); - sprintf(source, "%s::%d", data_source->device_name, data_source->exp_number); + snprintf(signal, lstr, "%s::%s", data_source->archive, signal_desc->signal_name); + snprintf(source, lstr, "%s::%d", data_source->device_name, data_source->exp_number); if (data_source->server[0] != '\0') { char* p = nullptr, * s = nullptr; diff --git a/source/server/fatServer.cpp b/source/server/fatServer.cpp index 37f4ceb7d..016cec2d1 100755 --- a/source/server/fatServer.cpp +++ b/source/server/fatServer.cpp @@ -182,7 +182,7 @@ static int processHierarchicalData(DATA_BLOCK* data_block) char tempFile[MAXPATH]; char* env; if ((env = getenv("UDA_WORK_DIR")) != nullptr) { - sprintf(tempFile, "%s/idamXDRXXXXXX", env); + snprintf(tempFile, MAXPATH, "%s/idamXDRXXXXXX", env); } else { strcpy(tempFile, "/tmp/idamXDRXXXXXX"); } @@ -291,9 +291,9 @@ int handleRequestFat(REQUEST_BLOCK* request_block, REQUEST_BLOCK* request_block0 REQUEST_DATA* request = &request_block->requests[i]; char work[1024]; if (request->api_delim[0] != '\0') { - sprintf(work, "UDA%s", request->api_delim); + snprintf(work, 1024, "UDA%s", request->api_delim); } else { - sprintf(work, "UDA%s", environment.api_delim); + snprintf(work, 1024, "UDA%s", environment.api_delim); } } diff --git a/source/server/getPluginAddress.cpp b/source/server/getPluginAddress.cpp index 6fd360ebe..0a733d2f5 100755 --- a/source/server/getPluginAddress.cpp +++ b/source/server/getPluginAddress.cpp @@ -24,8 +24,9 @@ int getPluginAddress(void** pluginHandle, const char* library, const char* symbo const char* plugin_dir = getenv("UDA_PLUGIN_DIR"); char* full_path; if (plugin_dir != nullptr) { - full_path = (char*)malloc(strlen(plugin_dir) + strlen(library) + 2); - sprintf(full_path, "%s/%s", plugin_dir, library); + const auto lstr = strlen(plugin_dir) + strlen(library) + 2; + full_path = (char*)malloc(lstr); + snprintf(full_path, lstr, "%s/%s", plugin_dir, library); } else { full_path = strdup(library); } @@ -71,4 +72,4 @@ int getPluginAddress(void** pluginHandle, const char* library, const char* symbo } return 0; -} \ No newline at end of file +} diff --git a/source/server/initPluginList.cpp b/source/server/initPluginList.cpp index e5e869d88..f3d10479c 100644 --- a/source/server/initPluginList.cpp +++ b/source/server/initPluginList.cpp @@ -92,11 +92,11 @@ void initPluginList(PLUGINLIST* plugin_list, ENVIRONMENT* environment) if (root == nullptr) { lstr = (int)strlen(filename) + 3; work = (char*)malloc(lstr * sizeof(char)); - sprintf(work, "./%s", filename); // Default ROOT is the server's Working Directory + snprintf(work, lstr, "./%s", filename); // Default ROOT is the server's Working Directory } else { lstr = (int)strlen(filename) + (int)strlen(root) + 2; work = (char*)malloc(lstr * sizeof(char)); - sprintf(work, "%s/%s", root, filename); + snprintf(work, lstr, "%s/%s", root, filename); } } else { lstr = (int)strlen(config) + 1; @@ -352,4 +352,4 @@ void initPluginList(PLUGINLIST* plugin_list, ENVIRONMENT* environment) fclose(conf); } -} \ No newline at end of file +} diff --git a/source/server/serverPlugin.cpp b/source/server/serverPlugin.cpp index f15d689a5..df0c7d4d6 100755 --- a/source/server/serverPlugin.cpp +++ b/source/server/serverPlugin.cpp @@ -165,7 +165,7 @@ int udaServerRedirectStdStreams(int reset) if (env == nullptr) { if ((env = getenv("UDA_WORK_DIR")) != nullptr) { - sprintf(mksdir_template, "%s/idamPLUGINXXXXXX", env); + snprintf(mksdir_template, MAXPATH, "%s/idamPLUGINXXXXXX", env); } else { strcpy(mksdir_template, "/tmp/idamPLUGINXXXXXX"); } @@ -392,7 +392,7 @@ int udaProvenancePlugin(CLIENT_BLOCK* client_block, REQUEST_DATA* original_reque // mimic a client request if (logRecord == nullptr || strlen(logRecord) == 0) { - sprintf(request.signal, "%s::putSignal(uuid='%s',requestedSignal='%s',requestedSource='%s', " + snprintf(request.signal, MAXMETA, "%s::putSignal(uuid='%s',requestedSignal='%s',requestedSource='%s', " "trueSignal='%s', trueSource='%s', trueSourceDOI='%s', execMethod=%d, status=new)", plugin_list->plugin[plugin_id].format, client_block->DOI, original_request->signal, original_request->source, @@ -401,7 +401,7 @@ int udaProvenancePlugin(CLIENT_BLOCK* client_block, REQUEST_DATA* original_reque // need 2> record the server log record - sprintf(request.signal, "%s::putSignal(uuid='%s',logRecord='%s', execMethod=%d, status=update)", + snprintf(request.signal, MAXMETA, "%s::putSignal(uuid='%s',logRecord='%s', execMethod=%d, status=update)", plugin_list->plugin[plugin_id].format, client_block->DOI, logRecord, execMethod); } diff --git a/source/server/serverStartup.cpp b/source/server/serverStartup.cpp index 1a5b92cff..ac49bb380 100755 --- a/source/server/serverStartup.cpp +++ b/source/server/serverStartup.cpp @@ -30,7 +30,7 @@ int startup(void) if (environment->loglevel <= UDA_LOG_ACCESS) { char cmd[STRING_LENGTH]; - sprintf(cmd, "mkdir -p %s 2>/dev/null", environment->logdir); + snprintf(cmd, STRING_LENGTH, "mkdir -p %s 2>/dev/null", environment->logdir); if (system(cmd) != 0) { THROW_ERROR(999, "mkdir command failed"); } diff --git a/source/server/serverSubsetData.cpp b/source/server/serverSubsetData.cpp index 91d90baa8..7be1bdc32 100755 --- a/source/server/serverSubsetData.cpp +++ b/source/server/serverSubsetData.cpp @@ -1078,7 +1078,7 @@ int apply_count(SUBSET subset, DATA_BLOCK* data_block) } data_block->data = (char*)count; data_block->data_units[0] = '\0'; - sprintf(data_block->data_label, "count(dim_id=%d)", dim_id); + snprintf(data_block->data_label, STRING_LENGTH, "count(dim_id=%d)", dim_id); } return 0; diff --git a/source/server/udaServer.cpp b/source/server/udaServer.cpp index a93b47418..f8161da82 100755 --- a/source/server/udaServer.cpp +++ b/source/server/udaServer.cpp @@ -497,9 +497,9 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV // Prepend the client request and test for a redirection request via the proxy's plugin if (request_block->api_delim[0] != '\0') { - sprintf(work, "%s%s", proxyName, request_block->api_delim); + snprintf(work, STRING_LENGTH, "%s%s", proxyName, request_block->api_delim); } else { - sprintf(work, "%s%s", proxyName, environment.api_delim); + snprintf(work, STRING_LENGTH, "%s%s", proxyName, environment.api_delim); } if (strncasecmp(request_block->source, work, strlen(work)) != 0) { @@ -509,9 +509,9 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV if (environment.server_proxy[0] == '\0') { if (request_block->api_delim[0] != '\0') { - sprintf(work, "%s%s%s", proxyName, request_block->api_delim, request_block->source); + snprintf(work, STRING_LENGTH, "%s%s%s", proxyName, request_block->api_delim, request_block->source); } else { - sprintf(work, "%s%s%s", proxyName, environment.api_delim, request_block->source); + snprintf(work, STRING_LENGTH, "%s%s%s", proxyName, environment.api_delim, request_block->source); } strcpy(request_block->source, work); @@ -531,9 +531,9 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV if (environment.server_this[0] != '\0') { if (request_block->api_delim[0] != '\0') { - sprintf(work, "%s%s%s", proxyName, request_block->api_delim, environment.server_this); + snprintf(work, STRING_LENGTH, "%s%s%s", proxyName, request_block->api_delim, environment.server_this); } else { - sprintf(work, "%s%s%s", proxyName, environment.api_delim, environment.server_this); + snprintf(work, STRING_LENGTH, "%s%s%s", proxyName, environment.api_delim, environment.server_this); } if (strstr(request_block->source, work) != nullptr) { @@ -545,14 +545,14 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV if (request_block->source[0] == '/') { if (request_block->api_delim[0] != '\0') - sprintf(work, "%s%s%s%s", proxyName, request_block->api_delim, environment.server_proxy, request_block->source); + snprintf(work, STRING_LENGTH, "%s%s%s%s", proxyName, request_block->api_delim, environment.server_proxy, request_block->source); else - sprintf(work, "%s%s%s%s", proxyName, environment.api_delim, environment.server_proxy, request_block->source); + snprintf(work, STRING_LENGTH, "%s%s%s%s", proxyName, environment.api_delim, environment.server_proxy, request_block->source); } else { if (request_block->api_delim[0] != '\0') - sprintf(work, "%s%s%s/%s", proxyName, request_block->api_delim, environment.server_proxy, request_block->source); + snprintf(work, STRING_LENGTH, "%s%s%s/%s", proxyName, request_block->api_delim, environment.server_proxy, request_block->source); else - sprintf(work, "%s%s%s/%s", proxyName, environment.api_delim, environment.server_proxy, request_block->source); + snprintf(work, STRING_LENGTH, "%s%s%s/%s", proxyName, environment.api_delim, environment.server_proxy, request_block->source); } strcpy(request_block->source, work); } @@ -569,9 +569,9 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV char work[1024]; if (request->api_delim[0] != '\0') { - sprintf(work, "UDA%s", request->api_delim); + snprintf(work, 1024, "UDA%s", request->api_delim); } else { - sprintf(work, "UDA%s", environment.api_delim); + snprintf(work, 1024, "UDA%s", environment.api_delim); } if (environment.server_proxy[0] != '\0' && strncasecmp(request->source, work, strlen(work)) != 0) { @@ -587,9 +587,9 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV // The UDA Plugin strips out the host and port data from the source so the originating server details are never passed. if (request->api_delim[0] != '\0') { - sprintf(work, "UDA%s%s", request->api_delim, environment.server_this); + snprintf(work, 1024, "UDA%s%s", request->api_delim, environment.server_this); } else { - sprintf(work, "UDA%s%s", environment.api_delim, environment.server_this); + snprintf(work, 1024, "UDA%s%s", environment.api_delim, environment.server_this); } if (strstr(request->source, work) != nullptr) { @@ -607,9 +607,9 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV // Prepend the redirection UDA server details if (request->api_delim[0] != '\0') { - sprintf(work, "UDA%s%s/%s", request->api_delim, environment.server_proxy, request->source); + snprintf(work, 1024, "UDA%s%s/%s", request->api_delim, environment.server_proxy, request->source); } else { - sprintf(work, "UDA%s%s/%s", environment.api_delim, environment.server_proxy, request->source); + snprintf(work, 1024, "UDA%s%s/%s", environment.api_delim, environment.server_proxy, request->source); } strcpy(request->source, work); diff --git a/source/structures/accessors.cpp b/source/structures/accessors.cpp index 30a872c0c..e88516946 100755 --- a/source/structures/accessors.cpp +++ b/source/structures/accessors.cpp @@ -827,124 +827,124 @@ void defineField(COMPOUNDFIELD* field, const char* name, const char* desc, int* case SCALARDOUBLE: field->atomictype = UDA_TYPE_DOUBLE; strcpy(field->type, "double"); - sprintf(field->desc, "[double %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[double %s] %s", name, desc); field->size = field->count * sizeof(double); break; case ARRAYDOUBLE: field->atomictype = UDA_TYPE_DOUBLE; strcpy(field->type, "double *"); - sprintf(field->desc, "[double *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[double *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(double*); break; case SCALARFLOAT: field->atomictype = UDA_TYPE_FLOAT; strcpy(field->type, "float"); - sprintf(field->desc, "[float %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[float %s] %s", name, desc); field->size = field->count * sizeof(float); break; case ARRAYFLOAT: field->atomictype = UDA_TYPE_FLOAT; strcpy(field->type, "float *"); - sprintf(field->desc, "[float *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[float *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(float*); break; case SCALARLONG64: field->atomictype = UDA_TYPE_LONG64; strcpy(field->type, "long long"); - sprintf(field->desc, "[long long %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[long long %s] %s", name, desc); field->size = field->count * sizeof(long long); break; case ARRAYLONG64: field->atomictype = UDA_TYPE_LONG64; strcpy(field->type, "long long *"); - sprintf(field->desc, "[long long *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[long long *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(long long*); break; case SCALARULONG64: field->atomictype = UDA_TYPE_UNSIGNED_LONG64; strcpy(field->type, "unsigned long long"); - sprintf(field->desc, "[unsigned long long %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[unsigned long long %s] %s", name, desc); field->size = field->count * sizeof(unsigned long long); break; case ARRAYULONG64: field->atomictype = UDA_TYPE_UNSIGNED_LONG64; strcpy(field->type, "unsigned long long *"); - sprintf(field->desc, "[unsigned long long *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[unsigned long long *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(unsigned long long*); break; case SCALARINT: field->atomictype = UDA_TYPE_INT; strcpy(field->type, "int"); - sprintf(field->desc, "[int %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[int %s] %s", name, desc); field->size = field->count * sizeof(int); break; case ARRAYINT: field->atomictype = UDA_TYPE_INT; strcpy(field->type, "int *"); - sprintf(field->desc, "[int *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[int *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(int*); break; case SCALARUINT: field->atomictype = UDA_TYPE_UNSIGNED_INT; strcpy(field->type, "unsigned int"); - sprintf(field->desc, "[unsigned int %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[unsigned int %s] %s", name, desc); field->size = field->count * sizeof(unsigned int); break; case ARRAYUINT: field->atomictype = UDA_TYPE_UNSIGNED_INT; strcpy(field->type, "unsigned int *"); - sprintf(field->desc, "[unsigned int *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[unsigned int *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(unsigned int*); break; case SCALARSHORT: field->atomictype = UDA_TYPE_SHORT; strcpy(field->type, "short"); - sprintf(field->desc, "[short %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[short %s] %s", name, desc); field->size = field->count * sizeof(short); break; case ARRAYSHORT: field->atomictype = UDA_TYPE_SHORT; strcpy(field->type, "short *"); - sprintf(field->desc, "[short *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[short *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(short*); break; case SCALARUSHORT: field->atomictype = UDA_TYPE_UNSIGNED_SHORT; strcpy(field->type, "unsigned short"); - sprintf(field->desc, "[unsigned short %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[unsigned short %s] %s", name, desc); field->size = field->count * sizeof(unsigned short); break; case ARRAYUSHORT: field->atomictype = UDA_TYPE_UNSIGNED_SHORT; strcpy(field->type, "unsigned short *"); - sprintf(field->desc, "[unsigned short *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[unsigned short *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(unsigned short*); break; case SCALARCHAR: field->atomictype = UDA_TYPE_CHAR; strcpy(field->type, "char"); - sprintf(field->desc, "[char %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[char %s] %s", name, desc); field->size = field->count * sizeof(char); break; case ARRAYCHAR: field->atomictype = UDA_TYPE_CHAR; strcpy(field->type, "char *"); - sprintf(field->desc, "[char *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[char *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(char*); break; case SCALARSTRING: field->atomictype = UDA_TYPE_STRING; strcpy(field->type, "STRING"); - sprintf(field->desc, "[char *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[char *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(char*); field->offset = (int)newoffset((size_t)*offset, "char *"); // must be an explicit char pointer (STRING Convention!) @@ -955,27 +955,27 @@ void defineField(COMPOUNDFIELD* field, const char* name, const char* desc, int* //Bug Fix dgm 07Jul2014: atomictype was missing! field->atomictype = UDA_TYPE_STRING; strcpy(field->type, "STRING *"); - sprintf(field->desc, "[char **%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[char **%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(char**); break; case ARRAYVOID: field->atomictype = UDA_TYPE_VOID; strcpy(field->type, "void *"); - sprintf(field->desc, "[void *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[void *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(void*); break; case SCALARUCHAR: field->atomictype = UDA_TYPE_UNSIGNED_CHAR; strcpy(field->type, "unsigned char"); - sprintf(field->desc, "[char %s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[char %s] %s", name, desc); field->size = field->count * sizeof(unsigned char); break; case ARRAYUCHAR: field->atomictype = UDA_TYPE_UNSIGNED_CHAR; strcpy(field->type, "unsigned char *"); - sprintf(field->desc, "[unsigned char *%s] %s", name, desc); + snprintf(field->desc, MAXELEMENTNAME, "[unsigned char *%s] %s", name, desc); field->pointer = 1; field->size = field->count * sizeof(unsigned char*); break; diff --git a/source/structures/struct.cpp b/source/structures/struct.cpp index 7e83d612a..39c23db83 100755 --- a/source/structures/struct.cpp +++ b/source/structures/struct.cpp @@ -218,7 +218,7 @@ void expandImage(char* buffer, char defnames[MAXELEMENTS][MAXELEMENTNAME], int* } else { for (int j = 0; j < defCount; j++) { if (!strcmp((char*)defnames[j], work)) { - sprintf(work, " = %d]", defvalues[j]); // Array size + snprintf(work, STRING_LENGTH, " = %d]", defvalues[j]); // Array size strncat(expand, &p1[1], p2 - &p1[1]); len = len + (int)(p2 - &p1[1]); expand[len] = '\0'; From 133ca102cfb5678fc387b91b58de7da6363f070b Mon Sep 17 00:00:00 2001 From: sdixon Date: Fri, 6 Jun 2025 17:32:34 +0100 Subject: [PATCH 05/11] updating pyuda subclient registration system --- source/wrappers/python/CMakeLists.txt | 11 +- source/wrappers/python/pyuda/__init__.py | 2 +- source/wrappers/python/pyuda/_client.py | 130 +++++++++++++++--- .../python/tests/breaking_client/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 227 bytes .../python/tests/test_client/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 233 bytes .../test_client.cpython-313-pytest-8.4.0.pyc | Bin 0 -> 1332 bytes .../python/tests/test_client/test_client.py | 20 +++ .../tests/test_subclient_registration.py | 88 ++++++++++++ 10 files changed, 232 insertions(+), 21 deletions(-) create mode 100644 source/wrappers/python/tests/breaking_client/__init__.py create mode 100644 source/wrappers/python/tests/breaking_client/__pycache__/__init__.cpython-313.pyc create mode 100644 source/wrappers/python/tests/test_client/__init__.py create mode 100644 source/wrappers/python/tests/test_client/__pycache__/__init__.cpython-313.pyc create mode 100644 source/wrappers/python/tests/test_client/__pycache__/test_client.cpython-313-pytest-8.4.0.pyc create mode 100644 source/wrappers/python/tests/test_client/test_client.py create mode 100644 source/wrappers/python/tests/test_subclient_registration.py diff --git a/source/wrappers/python/CMakeLists.txt b/source/wrappers/python/CMakeLists.txt index 61ac9da89..0bd80b9f2 100755 --- a/source/wrappers/python/CMakeLists.txt +++ b/source/wrappers/python/CMakeLists.txt @@ -11,10 +11,15 @@ add_custom_command( install( FILES ${CPYTHON_INSTALL_FILE} pyuda/cpyuda/uda.pxd DESTINATION python_installer/pyuda/cpyuda ) -file( GLOB INSTALL_FILES pyuda/*.py ) - +file( GLOB INSTALL_FILES pyuda/*py ) install( FILES ${INSTALL_FILES} DESTINATION python_installer/pyuda ) +# file( GLOB_RECURSE TEST_FILES tests/* ) +install( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests + DESTINATION python_installer + FILES_MATCHING PATTERN "*.py" +) + configure_file( ${CMAKE_CURRENT_LIST_DIR}/pyuda/_version.py.in ${CMAKE_CURRENT_BINARY_DIR}/pyuda/_version.py @ONLY ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/pyuda/_version.py DESTINATION python_installer/pyuda ) @@ -30,4 +35,4 @@ get_filename_component( OPENSSL_LIB_DIR ${OPENSSL_SSL_LIBRARY} DIRECTORY ) get_filename_component( LIBXML_LIB_DIR ${LIBXML2_LIBRARIES} DIRECTORY ) configure_file( ${CMAKE_CURRENT_LIST_DIR}/setup.py.in ${CMAKE_CURRENT_BINARY_DIR}/setup.py @ONLY ) -install( FILES ${CMAKE_CURRENT_BINARY_DIR}/setup.py DESTINATION python_installer ) \ No newline at end of file +install( FILES ${CMAKE_CURRENT_BINARY_DIR}/setup.py DESTINATION python_installer ) diff --git a/source/wrappers/python/pyuda/__init__.py b/source/wrappers/python/pyuda/__init__.py index 3312982b3..615d23ed9 100644 --- a/source/wrappers/python/pyuda/__init__.py +++ b/source/wrappers/python/pyuda/__init__.py @@ -4,7 +4,7 @@ import cpyuda -from ._client import Client +from ._client import Client, SubClientDeprecationWarning from ._signal import Signal from ._video import Video from ._dim import Dim diff --git a/source/wrappers/python/pyuda/_client.py b/source/wrappers/python/pyuda/_client.py index aed473316..47b571d14 100644 --- a/source/wrappers/python/pyuda/_client.py +++ b/source/wrappers/python/pyuda/_client.py @@ -10,15 +10,46 @@ from six import with_metaclass import logging -from collections import namedtuple +# from collections import namedtuple +from collections import defaultdict from collections.abc import Iterable import sys +import os +import importlib +import warnings try: from enum import Enum except ImportError: Enum = object +class SubClientDeprecationWarning(UserWarning): + def __init__(self, message): + super().__init__(SubClientDeprecationWarning, message) + + +def _parse_subclient_register_from_env(): + """ + Parse a list of uda subclient strings from an environment variable + into a map of {module_path: [subclient_classes,]} + + Input string should be formatted as a colon-delimited list, + with each entry containing the module path to the subclient class, + as it would be written to import in python + e.g. UDA_SUBCLIENTS=mast.MastClient:mast.geom.GeometryClient:another_module.AnotherSubClient + + Throws: ValueError if the UDA_SUBCLIENTS string is misformed (e.g. missing required . char) + Throws: KeyError if the environment variable does not exist + + """ + subclients_string = os.environ["UDA_SUBCLIENTS"] + entries = [i for i in subclients_string.split(':') if i != ''] + subclient_register = defaultdict(list) + for entry in entries: + module_path, subclient_class = entry.rsplit('.', maxsplit=1) + subclient_register[module_path].append(subclient_class) + return dict(subclient_register) + class ClientMeta(type): """ Metaclass used to add class level properties @@ -60,29 +91,94 @@ def __init__(self, debug_level=logging.ERROR): self.logger = logging.getLogger(__name__) self._registered_subclients = {} + self.register_all_subclients() + + def _parse_subclient_register_from_yaml(self): + import yaml + subclient_register_path = os.environ["UDA_SUBCLIENT_REGISTER"] + with open(subclient_register_path, 'r') as file: + return yaml.safe_load(file) + + def register_all_subclients(self): + """ + Method attempts to obtain a list of uda subclients from an environment variable + and register the exported methods for use in a standard pyuda client. + + This currently falls back to the (mast-specific) legacy registration routine to + maintain previous behaviour where still required. + + behaviour: + - no environment variable set: fallback to legacy registration routine + - list provided is incorrectly formatted: fallback to legacy registration routine + - list provided is empty: no subclients registered + """ + try: + subclient_register = _parse_subclient_register_from_env() + except (ValueError, KeyError): + self.register_legacy_subclients() + return + + for module_name in subclient_register: + module = importlib.import_module(module_name) + subclient_names = subclient_register[module_name] + + # assume either one or multiple subclients may be specified in each subclient module + if type(subclient_names) is not list: + subclient_names = [subclient_names] + + for subclient_name in subclient_names: + subclient = getattr(module, subclient_name) + self.register_subclient(subclient) + + def register_legacy_subclients(self): + # this warning will annoy all non-mast users until we deprecate + # when do we plan deprecation? + warnings.warn("WARNING: The pyuda client has fallen back to using the legacy " + "subclient registration routine as the UDA_SUBCLIENTS " + "environment variable has not been set, " + "or was incorrectly formatted. \n" + "Note that any errors encountered importing the mast module " + "used in this legacy routine will not be reported, which " + "will frustrate debugging if you require this functionality\n" + "This legacy (mast-specific) routine will be deprecated " + "in a future v3.x release. \n" + "Consider using the UDA_SUBCLIENTS " + "environment variable if your code relies on subclient " + "features.\n" + "You can disable this warning using " + "warnings.simplefilter('ignore', pyuda.SubClientDeprecationWarning)", + SubClientDeprecationWarning) try: from mast.geom import GeomClient from mast import MastClient - self._registered_subclients['geometry'] = GeomClient(self) - self._registered_subclients['geometry_signal_mapping'] = GeomClient(self) - self._registered_subclients['get_images'] = MastClient(self) - self._registered_subclients['get_shot_date_time'] = MastClient(self) - self._registered_subclients['latest_shot'] = MastClient(self) - self._registered_subclients['latest_source_pass'] = MastClient(self) - self._registered_subclients['latest_source_pass_in_range'] = MastClient(self) - self._registered_subclients['listGeomSignals'] = GeomClient(self) - self._registered_subclients['listGeomGroups'] = GeomClient(self) - self._registered_subclients['list'] = MastClient(self) - self._registered_subclients['list_archive_files'] = MastClient(self) - self._registered_subclients['list_archive_directories'] = MastClient(self) - self._registered_subclients['list_file_signals'] = MastClient(self) - self._registered_subclients['list_signals'] = MastClient(self) - self._registered_subclients['put'] = MastClient(self) - self._registered_subclients['list_shots'] = MastClient(self) + mast_client = MastClient(self) + geom_client = GeomClient(self) + self._registered_subclients['geometry'] = geom_client + self._registered_subclients['geometry_signal_mapping'] = geom_client + self._registered_subclients['get_images'] = mast_client + self._registered_subclients['get_shot_date_time'] = mast_client + self._registered_subclients['latest_shot'] = mast_client + self._registered_subclients['latest_source_pass'] = mast_client + self._registered_subclients['latest_source_pass_in_range'] = mast_client + self._registered_subclients['listGeomSignals'] = geom_client + self._registered_subclients['listGeomGroups'] = geom_client + self._registered_subclients['list'] = mast_client + self._registered_subclients['list_archive_files'] = mast_client + self._registered_subclients['list_archive_directories'] = mast_client + self._registered_subclients['list_file_signals'] = mast_client + self._registered_subclients['list_signals'] = mast_client + self._registered_subclients['put'] = mast_client + self._registered_subclients['list_shots'] = mast_client except ImportError: pass + def register_subclient(self, subclient_class): + subclient_class.register(self) + + def register_method(self, method, subclient_instance): + self._registered_subclients[method] = subclient_instance + def get_file(self, source_file, output_file=None): """ Retrieve file using bytes plugin and write to file diff --git a/source/wrappers/python/tests/breaking_client/__init__.py b/source/wrappers/python/tests/breaking_client/__init__.py new file mode 100644 index 000000000..871034edb --- /dev/null +++ b/source/wrappers/python/tests/breaking_client/__init__.py @@ -0,0 +1 @@ +import fake_non_existant_module diff --git a/source/wrappers/python/tests/breaking_client/__pycache__/__init__.cpython-313.pyc b/source/wrappers/python/tests/breaking_client/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d562ee8b284b302746710b64298b233baedec919 GIT binary patch literal 227 zcmey&%ge<81d2k=845u9F^B^LOi;#W9w1{XLoh=yqc=klLpq}-(@T()pC;oiiL}J* z)cCyoy!h0L%;J*7yps6b{FKt1)RhdML27PA>xUMn78UCkr({;-=joTGB!aN6k)D}e zg??rpP)kk@m}jJ4P+5`zGzcn~TBKi+T3n)^RFs;Sotc*&pPZAKnpdJ94^)s@5+AQu iPMj$Tc01_XV85tSxFvu6N0yzLAx;`2J literal 0 HcmV?d00001 diff --git a/source/wrappers/python/tests/test_client/__init__.py b/source/wrappers/python/tests/test_client/__init__.py new file mode 100644 index 000000000..272b3670b --- /dev/null +++ b/source/wrappers/python/tests/test_client/__init__.py @@ -0,0 +1 @@ +from .test_client import TestClient diff --git a/source/wrappers/python/tests/test_client/__pycache__/__init__.cpython-313.pyc b/source/wrappers/python/tests/test_client/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db31649cae1f8b6000918aa7c2ef001d2f52bfd5 GIT binary patch literal 233 zcmey&%ge<81kZjuWvBq@#~=<2FhLog1%QmH48aV+jNS}hj75wJ4Czdo%r6;%!kUb? zxI$8kOPq5uQ}asvG?{L3mjJo($zXmFGf>G&hR;Bf;a0eQXmM&$v3_w%W<`FUerZY~ z2oPhib0GI%#4hT_ZXCm*nk`WZZ$sM literal 0 HcmV?d00001 diff --git a/source/wrappers/python/tests/test_client/__pycache__/test_client.cpython-313-pytest-8.4.0.pyc b/source/wrappers/python/tests/test_client/__pycache__/test_client.cpython-313-pytest-8.4.0.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39e1f93902c4536b88c049a2b38d975b190f3324 GIT binary patch literal 1332 zcma)5&ub)A5U&0?KQb|ivzeHz;qZbePOS~$G8{S35)81kR@#4X%*F7<_Cm(e6_v&@M`s%Cd+46E7Xnp_B z`^P^x0DtgizWfr*b_h%%hS+%o=-9XW&{J*%4}A$>P41hKybyJZxHaj*qgi9@p0xI(rTSFt{-wdcKo2Ulq{f(K9e?m4?0L@RT9@VI~rwh15I zqEoNuJm%O}){~uixef?N<%nHg;qO`=))Ao%w zNRo&$^RgpPzd>oxRR!H_4Rlr>WiTbjN3u zE)%6wVpo$!)`;Tvo47MRGu1gS)BI1)&fHlc3aU-!8T@{0|S}Wad*ZtEojeuu-dN@n8K6Qls~a+MC~MS?%*=LNH|}~ z^vG@$heZ)`kf{o)DA7VJkeoX?Rx#6OaM8N`^%6D-56b2e1Ym3_u%uI<>2&vv|1{5! z&N(@C4-|h>q{EP)S-l~LohxRdc#tT~%KHCH^2OW7GEMX72=kLjAIpf_5mBq}72Uu$ z={~&8H{Zz5wJhYf1I<-k=kMD5xK6(P_7V$}^-m-TR2SkTA5K!n4Iw_C^waWWMTnyW zl}?jPX1Nf^|GrroqOF*%%22^4IJGbP(0WMQr0 zGSoX{zJlKx&5K6!GVnHE`_W&&tV6i=r|SiKGmp?EqpP|Tymb{iLF+Fv_luhNs&>Ux z4<|{gX^>6hz44j2-faq+aZ<5Ev?b~!&uChoVxpy~UAs(W?yFNUe=ho1kX$5gs_zzZ eJ;1w^*RrRd9xgq{aef8(68?s!bN`BsrS~t67dhns literal 0 HcmV?d00001 diff --git a/source/wrappers/python/tests/test_client/test_client.py b/source/wrappers/python/tests/test_client/test_client.py new file mode 100644 index 000000000..8ab6bc0f2 --- /dev/null +++ b/source/wrappers/python/tests/test_client/test_client.py @@ -0,0 +1,20 @@ +class TestClient: + _exported_methods = [ + "speak", + "greet" + ] + + @classmethod + def register(cls, client): + subclient = cls(client) + for method in cls._exported_methods: + client.register_method(method, subclient) + + def __init__(self, client): + self.client = client + + def speak(self): + return "woof" + + def greet(self): + return "hello from the test sub-client" diff --git a/source/wrappers/python/tests/test_subclient_registration.py b/source/wrappers/python/tests/test_subclient_registration.py new file mode 100644 index 000000000..c0611922c --- /dev/null +++ b/source/wrappers/python/tests/test_subclient_registration.py @@ -0,0 +1,88 @@ +import pytest +import pyuda +import os +# import warnings +from pyuda._client import _parse_subclient_register_from_env +import test_client + + +@pytest.mark.parametrize("input_string,expected", + [ + pytest.param("", {}, id="empty_string"), + pytest.param("test.TestClient", + {"test": ["TestClient"]}, + id="single_correct_entry" + ), + pytest.param("mast.MastClient:mast.geom.GeomClient", + {"mast": ["MastClient"], + "mast.geom": ["GeomClient"]}, + id="multiple_correct_entries" + ), + pytest.param("mast.MastClient:mast.MastClient2:geom.GeomClient:geom.GeomClient2", + {"mast": ["MastClient", "MastClient2"], + "geom": ["GeomClient", "GeomClient2"]}, + id="multiple_nested_entries" + ), + ]) +def test_env_parser(input_string, expected): + os.environ["UDA_SUBCLIENTS"] = input_string + assert _parse_subclient_register_from_env() == expected + + +def test_env_parser_throws_when_var_unset(): + del os.environ["UDA_SUBCLIENTS"] + with pytest.raises(KeyError): + _parse_subclient_register_from_env() + + +@pytest.mark.parametrize("input_string,error", + [ + pytest.param("wrong", ValueError, id="no_module_separators"), + pytest.param("wrong.Wrong;not_right.NotRight", + ValueError, + id="wrong_delimiter"), + ]) +def test_env_parser_throws_when_var_misformatted(input_string, error): + os.environ["UDA_SUBCLIENTS"] = input_string + with pytest.raises(error): + _parse_subclient_register_from_env() + + +def test_empty_string_skips_subclient_registration(recwarn): + os.environ["UDA_SUBCLIENTS"] = "" + client = pyuda.Client() + for warning in recwarn: + assert not issubclass(warning.category, pyuda.SubClientDeprecationWarning) + assert client._registered_subclients == {} + + +def test_register_subclient_manually_from_classmethod(): + os.environ["UDA_SUBCLIENTS"] = "" + client = pyuda.Client() + test_client.TestClient.register(client) + assert client.speak() == "woof" + + +def test_register_subclient_manually_from_pyuda_client(): + os.environ["UDA_SUBCLIENTS"] = "" + client = pyuda.Client() + client.register_subclient(test_client.TestClient) + assert client.speak() == "woof" + + +def test_register_sub_client_method_from_env(): + os.environ["UDA_SUBCLIENTS"] = "test_client.TestClient" + client = pyuda.Client() + assert client.speak() == "woof" + + +def test_module_not_found_raised(): + os.environ["UDA_SUBCLIENTS"] = "not_installed.FakeClient" + with pytest.raises(ModuleNotFoundError): + pyuda.Client() + + +def test_module_not_found_for_subclient_dependency(): + os.environ["UDA_SUBCLIENTS"] = "breaking_client.FakeClient" + with pytest.raises(ModuleNotFoundError): + pyuda.Client() From 42577f8f7c9c7f25ab61694a93194a9310b727dc Mon Sep 17 00:00:00 2001 From: sdixon Date: Tue, 10 Jun 2025 12:38:48 +0100 Subject: [PATCH 06/11] updating pyuda subclient registration system --- source/wrappers/python/pyuda/__init__.py | 2 +- source/wrappers/python/pyuda/_client.py | 88 ++++++++++++++----- .../tests/test_subclient_registration.py | 8 +- 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/source/wrappers/python/pyuda/__init__.py b/source/wrappers/python/pyuda/__init__.py index 615d23ed9..6baf9a3ac 100644 --- a/source/wrappers/python/pyuda/__init__.py +++ b/source/wrappers/python/pyuda/__init__.py @@ -4,7 +4,7 @@ import cpyuda -from ._client import Client, SubClientDeprecationWarning +from ._client import Client, UdaSubclientDeprecationWarning from ._signal import Signal from ._video import Video from ._dim import Dim diff --git a/source/wrappers/python/pyuda/_client.py b/source/wrappers/python/pyuda/_client.py index 47b571d14..adab30ee1 100644 --- a/source/wrappers/python/pyuda/_client.py +++ b/source/wrappers/python/pyuda/_client.py @@ -14,7 +14,6 @@ from collections import defaultdict from collections.abc import Iterable import sys -import os import importlib import warnings try: @@ -23,9 +22,17 @@ Enum = object -class SubClientDeprecationWarning(UserWarning): +class UdaSubclientDeprecationWarning(UserWarning): def __init__(self, message): - super().__init__(SubClientDeprecationWarning, message) + super().__init__(UdaSubclientDeprecationWarning, message) + + +class UdaSubclientInterfaceError(cpyuda.UDAException): + pass + + +class UdaSubclientsStringError(cpyuda.UDAException): + pass def _parse_subclient_register_from_env(): @@ -42,7 +49,16 @@ def _parse_subclient_register_from_env(): Throws: KeyError if the environment variable does not exist """ + import re + import os subclients_string = os.environ["UDA_SUBCLIENTS"] + if subclients_string == "": + return {} + + string_validator = re.fullmatch(r'(([a-zA-z]\w*)(\.[a-zA-z]\w*)+:?)+', subclients_string) + if string_validator is None: + raise UdaSubclientsStringError("UDA_SUBCLIENTS string is incorrectly formatted") + entries = [i for i in subclients_string.split(':') if i != ''] subclient_register = defaultdict(list) for entry in entries: @@ -112,43 +128,65 @@ def register_all_subclients(self): - no environment variable set: fallback to legacy registration routine - list provided is incorrectly formatted: fallback to legacy registration routine - list provided is empty: no subclients registered + - subclient does not provide a register method: fallback to legacy routine """ try: subclient_register = _parse_subclient_register_from_env() - except (ValueError, KeyError): + except UdaSubclientsStringError: + warnings.warn("WARNING: cannot parse UDA_SUBCLIENTS string as it is incorrectly formatted. " + "Falling back to legacy subclient registration method. This behaviour will be " + "deprecated along with the fallback method in a later release", + UdaSubclientDeprecationWarning) + self.register_legacy_subclients() + return + except KeyError: + warnings.warn("WARNING: UDA_SUBCLIENTS environment variable not set. Falling back to " + " legacy subclient registration method. This behaviour will be deprecated " + "along with with the fallback method in a later release", + UdaSubclientDeprecationWarning) self.register_legacy_subclients() return - for module_name in subclient_register: - module = importlib.import_module(module_name) - subclient_names = subclient_register[module_name] - - # assume either one or multiple subclients may be specified in each subclient module - if type(subclient_names) is not list: - subclient_names = [subclient_names] - - for subclient_name in subclient_names: - subclient = getattr(module, subclient_name) - self.register_subclient(subclient) + try: + for module_name in subclient_register: + module = importlib.import_module(module_name) + subclient_names = subclient_register[module_name] + + # assume either one or multiple subclients may be specified in each subclient module + if type(subclient_names) is not list: + subclient_names = [subclient_names] + + for subclient_name in subclient_names: + subclient = getattr(module, subclient_name) + self.register_subclient(subclient) + except UdaSubclientInterfaceError: + warnings.warn("WARNING: one of the subclient classes specified did not provide " + "a register method. This may be caused by an old version which does " + "not conform to the new interface. Falling back to legacy subclient " + "registration method. This behaviour will be deprecated along with the " + "fallback method in a later release", UdaSubclientDeprecationWarning) + self.register_legacy_subclients def register_legacy_subclients(self): # this warning will annoy all non-mast users until we deprecate # when do we plan deprecation? warnings.warn("WARNING: The pyuda client has fallen back to using the legacy " - "subclient registration routine as the UDA_SUBCLIENTS " + "subclient registration routine, possibly " + " because the UDA_SUBCLIENTS " "environment variable has not been set, " - "or was incorrectly formatted. \n" + "or was incorrectly formatted. \n\n" "Note that any errors encountered importing the mast module " "used in this legacy routine will not be reported, which " - "will frustrate debugging if you require this functionality\n" + "will frustrate debugging if you require this functionality\n\n" "This legacy (mast-specific) routine will be deprecated " - "in a future v3.x release. \n" + "in a future v3.x release. \n\n" "Consider using the UDA_SUBCLIENTS " "environment variable if your code relies on subclient " - "features.\n" + "features such as list_signals.\n\n" "You can disable this warning using " - "warnings.simplefilter('ignore', pyuda.SubClientDeprecationWarning)", - SubClientDeprecationWarning) + "warnings.simplefilter('ignore', pyuda.UdaSubclientDeprecationWarning) " + "or by setting the UDA_SUBCLIENTS variable with an empty string", + UdaSubclientDeprecationWarning) try: from mast.geom import GeomClient from mast import MastClient @@ -174,9 +212,15 @@ def register_legacy_subclients(self): pass def register_subclient(self, subclient_class): + if not hasattr(subclient_class, "register"): + raise UdaSubclientInterfaceError("The subclient class specified does not provide a \"register\" method") subclient_class.register(self) def register_method(self, method, subclient_instance): + if method in self._registered_subclients: + previous = self._registered_subclients[method].__name__ + warnings.warn(f"The subclient method \"{method.__name__}\" previously registered to {previous} " + f"has been overwritten by {subclient_instance.__name__}") self._registered_subclients[method] = subclient_instance def get_file(self, source_file, output_file=None): diff --git a/source/wrappers/python/tests/test_subclient_registration.py b/source/wrappers/python/tests/test_subclient_registration.py index c0611922c..f118f36ff 100644 --- a/source/wrappers/python/tests/test_subclient_registration.py +++ b/source/wrappers/python/tests/test_subclient_registration.py @@ -2,7 +2,7 @@ import pyuda import os # import warnings -from pyuda._client import _parse_subclient_register_from_env +from pyuda._client import _parse_subclient_register_from_env, UdaSubclientsStringError import test_client @@ -37,9 +37,11 @@ def test_env_parser_throws_when_var_unset(): @pytest.mark.parametrize("input_string,error", [ - pytest.param("wrong", ValueError, id="no_module_separators"), + pytest.param("wrong", + UdaSubclientsStringError, + id="no_module_separators"), pytest.param("wrong.Wrong;not_right.NotRight", - ValueError, + UdaSubclientsStringError, id="wrong_delimiter"), ]) def test_env_parser_throws_when_var_misformatted(input_string, error): From 0b447e53e0169cefbddd630bf95088d88b90c5c7 Mon Sep 17 00:00:00 2001 From: sdixon Date: Wed, 18 Jun 2025 13:47:48 +0100 Subject: [PATCH 07/11] ukaea issue #166: installing pkgconfig files for client-only build --- source/CMakeLists.txt | 6 ++++++ source/server/CMakeLists.txt | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 5be811d40..5cfbab51f 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -86,6 +86,12 @@ foreach( PKGCONFIG_FILE ${PKGCONFIG_FILES} ) ) endforeach() +install( + DIRECTORY + ${CMAKE_BINARY_DIR}/lib/pkgconfig + DESTINATION lib +) + find_program( XINETD_EXE xinetd PATHS /usr/sbin/ /usr/local/sbin/ ) configure_file( diff --git a/source/server/CMakeLists.txt b/source/server/CMakeLists.txt index 1656f1ead..75d062ba4 100755 --- a/source/server/CMakeLists.txt +++ b/source/server/CMakeLists.txt @@ -289,12 +289,6 @@ install( DESTINATION etc ) -install( - DIRECTORY - ${CMAKE_BINARY_DIR}/lib/pkgconfig - DESTINATION lib -) - install( DIRECTORY ${CMAKE_SOURCE_DIR}/source/etc/machine.d From c83e81d3220a9a4694bd6348d6934482561771cd Mon Sep 17 00:00:00 2001 From: sdixon Date: Fri, 28 Nov 2025 13:02:48 +0000 Subject: [PATCH 08/11] updating pkg-config files --- source/CMakeLists.txt | 25 ++++++++++++++++++ source/etc/uda-client.pc.in | 5 ++-- source/wrappers/python/pyuda/_client.py | 3 ++- .../__pycache__/__init__.cpython-313.pyc | Bin 227 -> 0 bytes .../__pycache__/__init__.cpython-313.pyc | Bin 233 -> 0 bytes .../test_client.cpython-313-pytest-8.4.0.pyc | Bin 1332 -> 0 bytes 6 files changed, 30 insertions(+), 3 deletions(-) delete mode 100644 source/wrappers/python/tests/breaking_client/__pycache__/__init__.cpython-313.pyc delete mode 100644 source/wrappers/python/tests/test_client/__pycache__/__init__.cpython-313.pyc delete mode 100644 source/wrappers/python/tests/test_client/__pycache__/test_client.cpython-313-pytest-8.4.0.pyc diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 5cfbab51f..6597a69b9 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -78,6 +78,31 @@ set( PKGCONFIG_FILES uda-plugins.pc ) +get_filename_component( LIBXML_LIB_DIR "${LIBXML2_LIBRARIES}" DIRECTORY ) +set( PKGCONFIG_LIBRARIES "-L${LIBXML_LIB_DIR} -lxml2" ) +set( PKGCONFIG_INCLUDES "-I${LIBXML2_INCLUDE_DIR}" ) +set( PKGCONFIG_REQUIRES fmt ) + +find_package( TIRPC QUIET ) +if( TIRPC_FOUND ) + set( PKGCONFIG_REQUIRES "${PKGCONFIG_REQUIRES} libtirpc" ) +endif() + +if( ENABLE_CAPNP ) + set( PKGCONFIG_REQUIRES "${PKGCONFIG_REQUIRES} capnp" ) + set( PKGCONFIG_LIBRARIES "${PKGCONFIG_LIBRARIES} -luda_serialisation" ) +endif() + +find_package( LibMemcached QUIET ) + +if( LIBMEMCACHED_FOUND AND NOT NO_MEMCACHE ) + set( PKGCONFIG_LIBRARIES "${PKGCONFIG_LIBRARIES} ${LIBMEMCACHED_LIBRARIES}" ) +endif() + +if( TIRPC_FOUND ) + set( PKGCONFIG_REQUIRES "${PKGCONFIG_REQUIRES} libtirpc" ) +endif() + foreach( PKGCONFIG_FILE ${PKGCONFIG_FILES} ) configure_file( "${CMAKE_SOURCE_DIR}/source/etc/${PKGCONFIG_FILE}.in" diff --git a/source/etc/uda-client.pc.in b/source/etc/uda-client.pc.in index 850ca14f4..ebd69e338 100755 --- a/source/etc/uda-client.pc.in +++ b/source/etc/uda-client.pc.in @@ -8,5 +8,6 @@ Name: UDA Description: The Universal Data Access library URL: http://www.iter.org/UDA Version: @PROJECT_VERSION@ -Cflags: -I${includedir} -Libs: -L${libdir} -luda_client @CACHE_LIBRARIES@ +Cflags: -I${includedir} @PKGCONFIG_INCLUDES@ +Libs: -L${libdir} -luda_client @CACHE_LIBRARIES@ @PKGCONFIG_LIBRARIES@ +Requires: @PKGCONFIG_REQUIRES@ diff --git a/source/wrappers/python/pyuda/_client.py b/source/wrappers/python/pyuda/_client.py index adab30ee1..0452e6dfb 100644 --- a/source/wrappers/python/pyuda/_client.py +++ b/source/wrappers/python/pyuda/_client.py @@ -45,7 +45,7 @@ def _parse_subclient_register_from_env(): as it would be written to import in python e.g. UDA_SUBCLIENTS=mast.MastClient:mast.geom.GeometryClient:another_module.AnotherSubClient - Throws: ValueError if the UDA_SUBCLIENTS string is misformed (e.g. missing required . char) + Throws: UdaSubclientsStringError if the UDA_SUBCLIENTS string is misformed (e.g. missing required . char) Throws: KeyError if the environment variable does not exist """ @@ -208,6 +208,7 @@ def register_legacy_subclients(self): self._registered_subclients['list_signals'] = mast_client self._registered_subclients['put'] = mast_client self._registered_subclients['list_shots'] = mast_client + self._registered_subclients['list_sources'] = mast_client except ImportError: pass diff --git a/source/wrappers/python/tests/breaking_client/__pycache__/__init__.cpython-313.pyc b/source/wrappers/python/tests/breaking_client/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index d562ee8b284b302746710b64298b233baedec919..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 227 zcmey&%ge<81d2k=845u9F^B^LOi;#W9w1{XLoh=yqc=klLpq}-(@T()pC;oiiL}J* z)cCyoy!h0L%;J*7yps6b{FKt1)RhdML27PA>xUMn78UCkr({;-=joTGB!aN6k)D}e zg??rpP)kk@m}jJ4P+5`zGzcn~TBKi+T3n)^RFs;Sotc*&pPZAKnpdJ94^)s@5+AQu iPMj$Tc01_XV85tSxFvu6N0yzLAx;`2J diff --git a/source/wrappers/python/tests/test_client/__pycache__/__init__.cpython-313.pyc b/source/wrappers/python/tests/test_client/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index db31649cae1f8b6000918aa7c2ef001d2f52bfd5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 233 zcmey&%ge<81kZjuWvBq@#~=<2FhLog1%QmH48aV+jNS}hj75wJ4Czdo%r6;%!kUb? zxI$8kOPq5uQ}asvG?{L3mjJo($zXmFGf>G&hR;Bf;a0eQXmM&$v3_w%W<`FUerZY~ z2oPhib0GI%#4hT_ZXCm*nk`WZZ$sM diff --git a/source/wrappers/python/tests/test_client/__pycache__/test_client.cpython-313-pytest-8.4.0.pyc b/source/wrappers/python/tests/test_client/__pycache__/test_client.cpython-313-pytest-8.4.0.pyc deleted file mode 100644 index 39e1f93902c4536b88c049a2b38d975b190f3324..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1332 zcma)5&ub)A5U&0?KQb|ivzeHz;qZbePOS~$G8{S35)81kR@#4X%*F7<_Cm(e6_v&@M`s%Cd+46E7Xnp_B z`^P^x0DtgizWfr*b_h%%hS+%o=-9XW&{J*%4}A$>P41hKybyJZxHaj*qgi9@p0xI(rTSFt{-wdcKo2Ulq{f(K9e?m4?0L@RT9@VI~rwh15I zqEoNuJm%O}){~uixef?N<%nHg;qO`=))Ao%w zNRo&$^RgpPzd>oxRR!H_4Rlr>WiTbjN3u zE)%6wVpo$!)`;Tvo47MRGu1gS)BI1)&fHlc3aU-!8T@{0|S}Wad*ZtEojeuu-dN@n8K6Qls~a+MC~MS?%*=LNH|}~ z^vG@$heZ)`kf{o)DA7VJkeoX?Rx#6OaM8N`^%6D-56b2e1Ym3_u%uI<>2&vv|1{5! z&N(@C4-|h>q{EP)S-l~LohxRdc#tT~%KHCH^2OW7GEMX72=kLjAIpf_5mBq}72Uu$ z={~&8H{Zz5wJhYf1I<-k=kMD5xK6(P_7V$}^-m-TR2SkTA5K!n4Iw_C^waWWMTnyW zl}?jPX1Nf^|GrroqOF*%%22^4IJGbP(0WMQr0 zGSoX{zJlKx&5K6!GVnHE`_W&&tV6i=r|SiKGmp?EqpP|Tymb{iLF+Fv_luhNs&>Ux z4<|{gX^>6hz44j2-faq+aZ<5Ev?b~!&uChoVxpy~UAs(W?yFNUe=ho1kX$5gs_zzZ eJ;1w^*RrRd9xgq{aef8(68?s!bN`BsrS~t67dhns From 8bd386eacb1e00882f9b857aee16c2d26b94304c Mon Sep 17 00:00:00 2001 From: sdixon Date: Fri, 28 Nov 2025 14:25:27 +0000 Subject: [PATCH 09/11] adding pyproject.toml --- CMakeLists.txt | 2 +- source/wrappers/python/CMakeLists.txt | 2 ++ source/wrappers/python/pyproject.toml | 35 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 source/wrappers/python/pyproject.toml diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ed5b8545..16ee7b859 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required( VERSION 3.0 ) +cmake_minimum_required( VERSION 3.5 ) ######################################################################################################################## # Macro used to convert Windows path to Unix diff --git a/source/wrappers/python/CMakeLists.txt b/source/wrappers/python/CMakeLists.txt index 0bd80b9f2..058e19dca 100755 --- a/source/wrappers/python/CMakeLists.txt +++ b/source/wrappers/python/CMakeLists.txt @@ -36,3 +36,5 @@ get_filename_component( LIBXML_LIB_DIR ${LIBXML2_LIBRARIES} DIRECTORY ) configure_file( ${CMAKE_CURRENT_LIST_DIR}/setup.py.in ${CMAKE_CURRENT_BINARY_DIR}/setup.py @ONLY ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/setup.py DESTINATION python_installer ) +install( FILES ${CMAKE_CURRENT_LIST_DIR}/pyproject.toml DESTINATION python_installer ) + diff --git a/source/wrappers/python/pyproject.toml b/source/wrappers/python/pyproject.toml new file mode 100644 index 000000000..f14c431a9 --- /dev/null +++ b/source/wrappers/python/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = [ + "setuptools>=42", + "oldest-supported-numpy; python_version <'3.9'", + "numpy>=2.0; python_version >='3.9'", + "Cython>=0.29, <3.1; python_version=='3.8'", + "Cython>=0.29", + "six" +] +build-backend = "setuptools.build_meta" + +[project.urls] +homepage = "https://ukaea.github.io/UDA/" +documentation = "https://ukaea.github.io/UDA/" +source = "https://github.com/ukaea/UDA" +tracker = "https://github.com/ukaea/UDA/issues" + +[project] +name = "pyuda" +dynamic = ["version"] +description = "Universal Data Access (UDA)" +# readme = {file = 'README.md', content-type = "text/markdown"} +license = {text = "Apache-2.0 license"} +dependencies = ["numpy>1.7", "six", "progress"] +requires-python = ">= 3.5" + +classifiers = [ + 'Intended Audience :: Science/Research', + 'Programming Language :: C', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Operating System :: POSIX :: Linux', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', +] From ca0765ab8d0473fbe9979ea0e271538dff505153 Mon Sep 17 00:00:00 2001 From: sdixon Date: Fri, 28 Nov 2025 17:28:11 +0000 Subject: [PATCH 10/11] updating github CI to run for 2.6.X --- .github/workflows/cmake.yml | 240 +++++++++++++++++++++++++++++++++--- .gitlab-ci.yml | 41 ------ 2 files changed, 221 insertions(+), 60 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index ec854254c..a07ed0c09 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -2,39 +2,241 @@ name: Build on: push: - branches: [ "main", "develop", "feature/**", "releases/**" ] + branches: + - 2.6.X + - 'feature/**' + - 'release/**' pull_request: - branches: [ "main" ] + branches: + - 2.6.X -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release +# prevent duplicate CI runs from separate triggers +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true jobs: build: - # The CMake configure and build commands are platform-agnostic and should work equally well on Windows or Mac. - # You can convert this to a matrix build if you need cross-platform coverage. - # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + release: [Release] + ssl: [ON, OFF] + client-only: [ON] + capnp: [ON, OFF] + # exclude: + # - os: windows-latest + # client-only: OFF + # - os: ubuntu-latest + # client-only: ON + # - os: macos-latest + # client-only: ON + + runs-on: ${{ matrix.os }} + + defaults: + run: + shell: bash steps: - uses: actions/checkout@v3 - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y git libboost-dev libssl-dev cmake build-essential pkg-config libxml2-dev + - uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + + - name: Install linux dependencies + if: matrix.os == 'ubuntu-latest' + run: > + sudo apt update && sudo apt install -y + git + libboost-dev + libboost-program-options-dev + libssl-dev + cmake + build-essential + pkg-config + libxml2-dev + ninja-build + python3-dev + python3-pip + python3-venv + + - name: Install Intel compiler + if: matrix.os == 'ubuntu-latest' + run: > + sudo apt install -y wget && + wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && + sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && + echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list && + sudo apt update && + sudo apt install -y intel-basekit && + sudo apt install -y intel-hpckit + + - name: Install macos dependencies + if: matrix.os == 'macos-latest' + run: > + brew update-reset && brew install + git + boost + openssl + cmake + pkg-config + libxml2 + ninja + +# - name: Cache windows packages +# id: cache-vcpkg +# uses: actions/cache@v3 +# env: +# cache-name: cache-vcpkg-packages +# with: +# path: C:/vcpkg +# key: ${{ runner.os }}-build-${{ env.cache-name }} + + # - name: Install windows dependencies + # if: matrix.os == 'windows-latest' + # run: > + # vcpkg install --triplet x64-mingw-static --binarysource="clear;x-gha,readwrite" + # libxml2 + # capnproto + # boost-program-options + # boost-format + # boost-algorithm + # boost-multi-array + # openssl + # dlfcn-win32 + # spdlog + + - name: Configure CMake (linux) + if: matrix.os == 'ubuntu-latest' + run: > + cmake -G Ninja -B build + -DBUILD_SHARED_LIBS=ON + -DCMAKE_BUILD_TYPE=${{ matrix.release }} + -DSSLAUTHENTICATION=${{ matrix.ssl }} + -DCLIENT_ONLY=${{ matrix.client-only }} + -DENABLE_CAPNP=${{ matrix.capnp }} + + - name: Configure CMake (linux Intel) + if: matrix.os == 'ubuntu-latest' + run: > + source /opt/intel/oneapi/setvars.sh && + CXX=icpx CC=icx cmake -G Ninja -B build-intel + -DBUILD_SHARED_LIBS=ON + -DCMAKE_BUILD_TYPE=${{ matrix.release }} + -DSSLAUTHENTICATION=${{ matrix.ssl }} + -DCLIENT_ONLY=${{ matrix.client-only }} + -DENABLE_CAPNP=${{ matrix.capnp }} - - name: Configure CMake - # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + - name: Configure CMake (macos) + if: matrix.os == 'macos-latest' + run: > + cmake -G Ninja -B build + -DBUILD_SHARED_LIBS=ON + -DCMAKE_BUILD_TYPE=${{ matrix.release }} + -DSSLAUTHENTICATION=${{ matrix.ssl }} + -DCLIENT_ONLY=${{ matrix.client-only }} + -DENABLE_CAPNP=${{ matrix.capnp }} + -DOPENSSL_ROOT_DIR="$(brew --prefix openssl@3)" + + # - name: Configure portable XDR + # if: matrix.os == 'windows-latest' + # run: > + # cd extlib + # && cmake -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -B build -G "MinGW Makefiles" . + # + # - name: Build portable XDR + # if: matrix.os == 'windows-latest' + # run: cd extlib && cmake --build build + # + # - name: Install portable XDR + # if: matrix.os == 'windows-latest' + # run: cd extlib && cmake --install build --prefix install + # + # - name: Configure CMake (windows) + # if: matrix.os == 'windows-latest' + # run: > + # XDR_ROOT=extlib/install cmake -G "MinGW Makefiles" -B build + # -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake + # -DBUILD_SHARED_LIBS=ON + # -DCMAKE_BUILD_TYPE=${{ matrix.release }} + # -DSSLAUTHENTICATION=${{ matrix.ssl }} + # -DCLIENT_ONLY=${{ matrix.client-only }} + # -DENABLE_CAPNP=${{ matrix.capnp }} + # -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/uda_install - name: Build - # Build your program with the given configuration - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: cmake --build build --config ${{ matrix.release }} + + - name: Build Intel + if: matrix.os == 'ubuntu-latest' + run: cmake --build build-intel --config ${{ matrix.release }} + + - name: Install + if: matrix.os != 'windows-latest' + run: sudo cmake --install build --config ${{ matrix.release }} + + - name: Install + if: matrix.os == 'windows-latest' + run: cmake --install build --config ${{ matrix.release }} + + - name: Install pyuda + if: matrix.os == 'ubuntu-latest' + run: > + cp -r /usr/local/python_installer ${{github.workspace}}/python_installer && + python3 -m venv ${{github.workspace}}/venv && + source ${{github.workspace}}/venv/bin/activate && + pip3 install --upgrade pip && + pip3 install wheel Cython "numpy<2" six && + pip3 install ${{github.workspace}}/python_installer + + - name: Test pyuda import + if: matrix.os == 'ubuntu-latest' + run: > + source ${{github.workspace}}/venv/bin/activate && + python3 -c 'import pyuda; client=pyuda.Client()' + + # + # systemd not setup on this branch + # + # - name: Run non-SSL system tests + # if: matrix.os == 'ubuntu-latest' && matrix.ssl == 'OFF' && matrix.client_only == 'ON' + # run: > + # sudo cp /usr/local/etc/uda.socket /usr/local/etc/uda@.service /etc/systemd/system && + # sudo systemctl start uda.socket && + # sudo systemctl enable uda.socket && + # sudo chown -R $USER:$USER /usr/local/etc && + # nc -4zv localhost 56565 && + # export UDA_HOST=localhost && + # export UDA_PORT=56565 && + # ./build/test/plugins/plugin_test_testplugin + + # - name: Run SSL system tests + # if: matrix.os == 'ubuntu-latest' && matrix.ssl == 'ON' + # run: > + # sudo cp /usr/local/etc/uda.socket /usr/local/etc/uda@.service /etc/systemd/system && + # sudo chown -R $USER:$USER /usr/local/etc && + # echo "export UDAHOSTNAME=github-ci-ssl" >> /usr/local/etc/udaserver.cfg && + # ./scripts/create_certs.sh && + # mkdir /usr/local/etc/certs && + # cp rootCA.crt server.crt server.key /usr/local/etc/certs && + # sudo systemctl start uda.socket && + # sudo systemctl enable uda.socket && + # nc -4zv localhost 56565 && + # export UDA_HOST=localhost && + # export UDA_PORT=56565 && + # export UDA_CLIENT_SSL_AUTHENTICATE=1 && + # export UDA_CLIENT_CA_SSL_CERT=$PWD/rootCA.crt && + # export UDA_CLIENT_SSL_CERT=$PWD/client.crt && + # export UDA_CLIENT_SSL_KEY=$PWD/client.key && + # ./build/test/plugins/plugin_test_testplugin # - name: Test -# working-directory: ${{github.workspace}}/build +# working-directory: build # # Execute tests defined by the CMake configuration. # # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail -# run: ctest -C ${{env.BUILD_TYPE}} +# run: ctest -C ${{ matrix.release }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 09f4a1e0b..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,41 +0,0 @@ -# use the official gcc image, based on debian -# can use verions as well, like gcc:5.2 -# see https://hub.docker.com/_/gcc/ -image: gcc:7.3 - -build: - stage: build - # instead of calling g++ directly you can also use some build toolkit like make - # install the necessary build tools when needed - before_script: - - apt update && apt -y install cmake python3-dev libssl-dev libboost-dev swig libxml2-dev pkg-config ninja-build libtirpc-dev - script: - - cmake -GNinja -Bbuild -H. -DCMAKE_INSTALL_PREFIX=. -DTARGET_TYPE=OTHER -DNO_MODULES=ON -DFAT_TESTS=ON - - ninja -C build - - ninja -C build install - artifacts: - paths: - - lib/ - - include/ - - bin/ - - etc/ - - python_installer/ - - build/ - # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time - # cache: - # paths: - # - build/ - -# run tests using the binary built before -test: - dependencies: - - build - stage: test - script: - - cd build/test && LD_LIBRARY_PATH=$PWD/../../lib:$LD_LIBRARY_PATH ctest --timeout 60 - artifacts: - reports: - junit: - - "*_out.xml" - - plugins/*_out.xml - - imas/*_out.xml From 07b2b3d47423de5e8ca092bbd4a1aa324dd322a4 Mon Sep 17 00:00:00 2001 From: sdixon Date: Fri, 28 Nov 2025 18:35:04 +0000 Subject: [PATCH 11/11] adding gitlab ci and removing github ci for now --- .github/workflows/cmake.yml | 16 +-- .gitlab-ci.yml | 207 ++++++++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index a07ed0c09..d099e3a4c 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -1,14 +1,18 @@ name: Build +# on compatibility branch we know this won't compile with newest compilers +# probably best to revert to building in a centos7 docker image with gcc 4.8.5 +# disable tests for now on: push: branches: - - 2.6.X - - 'feature/**' - - 'release/**' - pull_request: - branches: - - 2.6.X + - main + # - 2.6.X + # - 'feature/**' + # - 'release/**' + # pull_request: + # branches: + # - 2.6.X # prevent duplicate CI runs from separate triggers concurrency: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..085a6d586 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,207 @@ +workflow: + rules: + - if: '$CI_COMMIT_BRANCH == "2.6.X"' + - if: '$CI_COMMIT_BRANCH =~ /^feature\/.*$/' + - if: '$CI_COMMIT_BRANCH =~ /^release\/.*$/' + - if: '$CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "web"' + +stages: + - build + - test + - package + +default: + tags: + - freia + before_script: + - . /etc/profile + - module use /usr/local/modules/default + - module use /common/software/micromamba/modules + +build: + stage: build + script: + - source scripts/cmake-freia.sh -DCMAKE_INSTALL_PREFIX=install -DCLIENT_ONLY=ON -DNO_JAVA_WRAPPER=ON -DSSLAUTHENTICATION=OFF + - cmake --build build_freia -j --config Release --target install + artifacts: + paths: + - install/ + +build_idl84: + stage: build + script: + - source scripts/cmake-freia-idl84.sh -DCMAKE_INSTALL_PREFIX=install_idl84 -DCLIENT_ONLY=ON -DNO_JAVA_WRAPPER=ON -DSSLAUTHENTICATION=OFF + - cmake --build build_freia -j --config Release --target install + artifacts: + paths: + - install_idl84/ + +.build_pyuda_template: + stage: test + dependencies: + - build + script: + - module load capnproto/0.10.4 fmt/10.0.0 spdlog/1.11.0 + - module swap gcc/11.2.0 + - module use /common/software/micromamba/modules + - module load $PYTHON_MODULE + + - python3 -m venv venv${PYTHON_VERSION} --system-site-packages --symlinks + - source venv${PYTHON_VERSION}/bin/activate + - python3 -m pip install --upgrade pip wheel setuptools + - printf "${PYTHON_REQUIRES}" > requirements.txt + - python3 -m pip install -r requirements.txt + + # - sed -i '/oldest-supported-numpy/,+1d' install/python_installer/pyproject.toml + - COMPILER_DIR=`echo $PATH | awk -F':' '{for (i =1 ; i< NF ; ++i) {print $i}}' | grep gcc` + - export CC=$COMPILER_DIR/gcc + - export CXX=$COMPILER_DIR/g++ + - python3 -m pip install --prefix=install install/python_installer + - mkdir -p dist + - cp -r install/lib/python${PYTHON_VERSION}/site-packages/*uda* dist + + - export PYTHONPATH=dist:$PYTHONPATH + - module use install/modulefiles + - UDA_MODULE=`ls install/modulefiles | grep uda` + - module load $UDA_MODULE + - python -c "import pyuda; client=pyuda.Client();" + + - module load uda-mast + - python -c "import mast" + - python3 -m pip install pytest + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@git.ccfe.ac.uk/MAST-U/mastcodes.git + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_geometry.py + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_images.py + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_meta.py + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_signals.py + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_xpad.py + - python3 mastcodes/uda/python/tests/test_put.py + artifacts: + paths: + - dist + +# python3.5: +# extends: .build_pyuda_template +# variables: +# PYTHON_VERSION: "3.5" +# PYTHON_MODULE: "python/3.5" +# PYTHON_REQUIRES: "Cython==0.29.32\nprogress" +# INSTALL_OPTION: "--prefix=install" + +python3.7: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.7" + PYTHON_MODULE: "python/3.7" + PYTHON_REQUIRES: "progress" + +python3.9: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.9" + PYTHON_MODULE: "python/3.9" + PYTHON_REQUIRES: "progress" + +python3.9_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.9" + PYTHON_MODULE: "python/3.9-minibundle" + PYTHON_REQUIRES: "progress" + +python3.10_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.10" + PYTHON_MODULE: "python/3.10-minibundle" + PYTHON_REQUIRES: "progress" + +python3.11_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.11" + PYTHON_MODULE: "python/3.11-minibundle" + PYTHON_REQUIRES: "progress" + +python3.12_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.12" + PYTHON_MODULE: "python/3.12-minibundle" + PYTHON_REQUIRES: "progress" + +python3.13_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.13" + PYTHON_MODULE: "python/3.13-minibundle" + PYTHON_REQUIRES: "progress" + +.package-template: &package_template + stage: package + dependencies: + - build + - build_idl84 + # - python3.5 + - python3.7 + - python3.9 + - python3.9_minibundle + - python3.10_minibundle + - python3.11_minibundle + - python3.12_minibundle + - python3.13_minibundle + script: + - mkdir -p install/idl/8.3/idl + - mkdir -p install/idl/8.4/idl + + # Move IDL 8.3 files + - mv install/idl/*.pro install/idl/8.3/idl/ + - mv install/dlm install/idl/8.3/ + + # Move IDL 8.4 files + - mv install_idl84/idl/*.pro install/idl/8.4/idl/ + - mv install_idl84/dlm install/idl/8.4/ + + # Create modulefiles + # - UDA_GIT_TAG=$(git describe --tags --abbrev=0) + - MODULEFILE=$(ls install/modulefiles/uda/* | head -n 1) + - cp ${MODULEFILE} ${MODULEFILE}-idl-8.4 + - sed -ri '/([^\$]UDA_IDL_DIR)/ s/$/\/idl\/8.3/' $MODULEFILE + - sed -ri '/([^\$]UDA_IDL_DIR)/ s/$/\/idl\/8.4/' ${MODULEFILE}-idl-8.4 + + # # Create uda-devel modulefile + # - MODULE_TAG=${MODULEFILE##*/} + # - mkdir -p install/modulefiles/uda-devel + # - DEV_MODULEFILE=install/modulefiles/uda-devel/${MODULE_TAG} + # - cp ${MODULEFILE} ${DEV_MODULEFILE} + # - sed -i '/idam/a module load capnproto\nmodule load fmt' ${DEV_MODULEFILE} + + - mkdir -p install/python + - cp -r dist/* install/python + + - TAG=$(git describe --tags || echo "untagged") + - tar -czf uda-$TAG.tar.gz -C install . + # artifacts: + # paths: + # - uda-*.tar.gz + +package:branch: + <<: *package_template + artifacts: + paths: + - uda-*.tar.gz + expire_in: 1 week + only: + - branches + except: + - tags + +package:tag: + <<: *package_template + artifacts: + paths: + - uda-*.tar.gz + expire_in: 2 years + only: + - tags