From 17f09ad526780a374d9cf2396a7dd7f5f269a6b7 Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Wed, 17 Dec 2025 21:22:30 +0000 Subject: [PATCH 1/9] remove redundant data import functionality; make sure to check diff on Save to Server --- pygeoapi_config_dialog.py | 161 +++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 89 deletions(-) diff --git a/pygeoapi_config_dialog.py b/pygeoapi_config_dialog.py index 1a49425..bf7dd9c 100644 --- a/pygeoapi_config_dialog.py +++ b/pygeoapi_config_dialog.py @@ -33,7 +33,7 @@ from .ui_widgets.utils import get_url_status -from .server_config_dialog import Ui_serverDialog +from .server_config_dialog import Ui_serverDialog from .models.top_level.providers.records import ProviderTypes from .ui_widgets.providers.NewProviderWindow import NewProviderWindow @@ -75,10 +75,8 @@ except: pass -headers = { - 'accept': '*/*', - 'Content-Type': 'application/json' -} +headers = {"accept": "*/*", "Content-Type": "application/json"} + def preprocess_for_json(d): """Recursively converts datetime/date objects in a dict to ISO strings.""" @@ -90,27 +88,29 @@ def preprocess_for_json(d): return d.isoformat() return d + class ServerConfigDialog(QDialog, Ui_serverDialog): """ Logic for the Server Configuration Dialog. Inherits from QDialog (functionality) and Ui_serverDialog (layout). """ + def __init__(self, parent=None): super().__init__(parent) - self.setupUi(self) # Builds the UI defined in Designer - + self.setupUi(self) # Builds the UI defined in Designer + # Optional: Set default values based on current config if needed # self.ServerHostlineEdit.setText("localhost") - def get_server_data(self): + def get_server_url(self): """ Retrieve the server configuration data entered by the user. :return: A dictionary with 'host' and 'port' keys. """ host = self.ServerHostlineEdit.text() port = self.ServerSpinBox.value() - protocol = 'http' if self.radioHttp.isChecked() else 'https' - return {'host': host, 'port': port, 'protocol': protocol} + protocol = "http" if self.radioHttp.isChecked() else "https" + return f"{protocol}://{host}:{port}/admin/config" # This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer @@ -118,7 +118,7 @@ def get_server_data(self): os.path.join(os.path.dirname(__file__), "pygeoapi_config_dialog_base.ui") ) - + class PygeoapiConfigDialog(QtWidgets.QDialog, FORM_CLASS): config_data: ConfigData @@ -183,17 +183,25 @@ def on_button_clicked(self, button): # You can also check the standard button type if button == self.buttonBox.button(QDialogButtonBox.Save): + # proceed only if UI data inputs are valid if self._set_validate_ui_data()[0]: if self.serverRadio.isChecked(): + # check #1: show diff with "Procced" and "Cancel" options + if not self._diff_original_and_current_data(): + return + self.server_config(save=True) else: + # check #1: show diff with "Procced" and "Cancel" options + if not self._diff_original_and_current_data(): + return + file_path, _ = QFileDialog.getSaveFileName( self, "Save File", "", "YAML Files (*.yml);;All Files (*)" ) - - # before saving, show diff with "Procced" and "Cancel" options - if file_path and self._diff_original_and_current_data(): + # check #2: valid file path + if file_path: self.save_to_file(file_path) elif button == self.buttonBox.button(QDialogButtonBox.Open): @@ -213,14 +221,13 @@ def server_config(self, save): dialog = ServerConfigDialog(self) - if dialog.exec_(): - data = dialog.get_server_data() - url = f"{data['protocol']}://{data['host']}:{data['port']}/admin/config" - if save == True: + if dialog.exec_(): + url = dialog.get_server_url() + if save: self.push_to_server(url) else: - self.pull_from_server(url) - + self.pull_from_server(url) + def push_to_server(self, url): QMessageBox.information( @@ -240,8 +247,7 @@ def push_to_server(self, url): response = requests.put(url, headers=headers, json=processed_config_dict) response.raise_for_status() - QgsMessageLog.logMessage( - f"Success! Status Code: {response.status_code}") + QgsMessageLog.logMessage(f"Success! Status Code: {response.status_code}") QMessageBox.information( self, @@ -257,9 +263,8 @@ def push_to_server(self, url): f"An error occurred pulling the configuration from the server: {e}", ) - def pull_from_server(self, url): - + QMessageBox.information( self, "Information", @@ -272,8 +277,7 @@ def pull_from_server(self, url): response = requests.get(url, headers=headers) response.raise_for_status() - QgsMessageLog.logMessage( - f"Success! Status Code: {response.status_code}") + QgsMessageLog.logMessage(f"Success! Status Code: {response.status_code}") QMessageBox.information( self, @@ -281,35 +285,10 @@ def pull_from_server(self, url): f"Success! Status Code: {response.status_code}", ) - QgsMessageLog.logMessage( - f"Response: {response.text}") + QgsMessageLog.logMessage(f"Response: {response.text}") - data_dict = response.json() - - self.config_data = ConfigData() - self.config_data.set_data_from_yaml(data_dict) - self.ui_setter.set_ui_from_data() - - # log messages about missing or mistyped values during deserialization - QgsMessageLog.logMessage( - f"Errors during deserialization: {self.config_data.error_message}" - ) - QgsMessageLog.logMessage( - f"Default values used for missing YAML fields: {self.config_data.defaults_message}" - ) - - # summarize all properties missing/overwitten with defaults - # atm, warning with the full list of properties - all_missing_props = self.config_data.all_missing_props - QgsMessageLog.logMessage( - f"All missing or replaced properties: {self.config_data.all_missing_props}" - ) - if len(all_missing_props) > 0: - ReadOnlyTextDialog( - self, - "Warning", - f"All missing or replaced properties (check logs for more details): {self.config_data.all_missing_props}", - ).exec_() + data_dict = response.json() + self.update_config_data_and_ui(data_dict) except requests.exceptions.RequestException as e: QgsMessageLog.logMessage(f"An error occurred: {e}") @@ -320,7 +299,6 @@ def pull_from_server(self, url): f"An error occurred pulling the configuration from the server: {e}", ) - def save_to_file(self, file_path): if file_path: @@ -360,48 +338,53 @@ def open_file(self, file_name): # QApplication.setOverrideCursor(Qt.WaitCursor) with open(file_name, "r", encoding="utf-8") as file: file_content = file.read() + yaml_original_data_dict = yaml.safe_load(file_content) + + self.update_config_data_and_ui(yaml_original_data_dict) - # reset data - self.config_data = ConfigData() + except Exception as e: + QMessageBox.warning(self, "Error", f"Cannot open file:\n{str(e)}") + # finally: + # QApplication.restoreOverrideCursor() - # set data and .all_missing_props: - yaml_original_data = yaml.safe_load(file_content) - self.yaml_original_data = deepcopy(yaml_original_data) + def update_config_data_and_ui(self, data_dict): + """Use the data from local file or local server to reset the ConfigData and UI.""" - self.config_data.set_data_from_yaml(yaml_original_data) + # reset data + self.config_data = ConfigData() - # set UI from data - self.ui_setter.set_ui_from_data() + # set data and .all_missing_props: + self.yaml_original_data = deepcopy(data_dict) + self.config_data.set_data_from_yaml(data_dict) - # log messages about missing or mistyped values during deserialization - # try/except in case of running it from pytests - try: - QgsMessageLog.logMessage( - f"Errors during deserialization: {self.config_data.error_message}" - ) - QgsMessageLog.logMessage( - f"Default values used for missing YAML fields: {self.config_data.defaults_message}" - ) + # set UI from data + self.ui_setter.set_ui_from_data() - # summarize all properties missing/overwitten with defaults - # atm, warning with the full list of properties - QgsMessageLog.logMessage( - f"All missing or replaced properties: {self.config_data.all_missing_props}" - ) + # log messages about missing or mistyped values during deserialization + # try/except in case of running it from pytests + try: + QgsMessageLog.logMessage( + f"Errors during deserialization: {self.config_data.error_message}" + ) + QgsMessageLog.logMessage( + f"Default values used for missing YAML fields: {self.config_data.defaults_message}" + ) - if len(self.config_data.all_missing_props) > 0: - ReadOnlyTextDialog( - self, - "Warning", - f"All missing or replaced properties (check logs for more details): {self.config_data.all_missing_props}", - ).exec_() - except: - pass + # summarize all properties missing/overwitten with defaults + # atm, warning with the full list of properties + all_missing_props = self.config_data.all_missing_props + QgsMessageLog.logMessage( + f"All missing or replaced properties: {all_missing_props}" + ) - except Exception as e: - QMessageBox.warning(self, "Error", f"Cannot open file:\n{str(e)}") - # finally: - # QApplication.restoreOverrideCursor() + if len(all_missing_props) > 0: + ReadOnlyTextDialog( + self, + "Warning", + f"All missing or replaced properties (check logs for more details): {all_missing_props}", + ).exec_() + except: + pass # QgsMessageLog import error in pytests, ignore def _set_validate_ui_data(self) -> tuple[bool, list]: # Set and validate data from UI From 9bd10af9ad6a661a70aa13406aaf1dd4cd9551d3 Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Wed, 17 Dec 2025 21:25:50 +0000 Subject: [PATCH 2/9] remove redundant datetime conversion --- pygeoapi_config_dialog.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/pygeoapi_config_dialog.py b/pygeoapi_config_dialog.py index bf7dd9c..1237522 100644 --- a/pygeoapi_config_dialog.py +++ b/pygeoapi_config_dialog.py @@ -78,17 +78,6 @@ headers = {"accept": "*/*", "Content-Type": "application/json"} -def preprocess_for_json(d): - """Recursively converts datetime/date objects in a dict to ISO strings.""" - if isinstance(d, dict): - return {k: preprocess_for_json(v) for k, v in d.items()} - elif isinstance(d, list): - return [preprocess_for_json(i) for i in d] - elif isinstance(d, (datetime, date)): - return d.isoformat() - return d - - class ServerConfigDialog(QDialog, Ui_serverDialog): """ Logic for the Server Configuration Dialog. @@ -190,7 +179,7 @@ def on_button_clicked(self, button): # check #1: show diff with "Procced" and "Cancel" options if not self._diff_original_and_current_data(): return - + self.server_config(save=True) else: # check #1: show diff with "Procced" and "Cancel" options @@ -236,10 +225,9 @@ def push_to_server(self, url): f"Pushing configuration to: {url}", ) - config_dict = self.config_data.asdict_enum_safe(self.config_data) - - # Pre-process the dictionary to handle datetime objects - processed_config_dict = preprocess_for_json(config_dict) + processed_config_dict = self.config_data.asdict_enum_safe( + self.config_data, datetime_to_str=True + ) # TODO: support authentication through the QT framework try: From 602cf2a7e05919835c4bbb477cfff9cb7f55db42 Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Thu, 18 Dec 2025 14:58:02 +0000 Subject: [PATCH 3/9] centralize stringifying datetime strings for Diff and Save --- pygeoapi_config_dialog.py | 62 +++++++++++++++++++-------------------- tests/test_yaml_save.py | 10 +++++-- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/pygeoapi_config_dialog.py b/pygeoapi_config_dialog.py index 1237522..8e6f8c8 100644 --- a/pygeoapi_config_dialog.py +++ b/pygeoapi_config_dialog.py @@ -149,14 +149,6 @@ class CustomDumper(yaml.SafeDumper): ), ) - def represent_datetime_as_timestamp(dumper, data: datetime): - value = self.config_data.datetime_to_string(data) - - # emit as YAML timestamp → plain scalar, no quotes - return dumper.represent_scalar("tag:yaml.org,2002:timestamp", value) - - self.dumper.add_representer(datetime, represent_datetime_as_timestamp) - # custom assignments self.model = QStringListModel() self.proxy = QSortFilterProxyModel() @@ -177,13 +169,19 @@ def on_button_clicked(self, button): if self.serverRadio.isChecked(): # check #1: show diff with "Procced" and "Cancel" options - if not self._diff_original_and_current_data(): + diff_approved, processed_config_data = ( + self._diff_original_and_current_data() + ) + if not diff_approved: return - self.server_config(save=True) + self.server_config(data_to_push=processed_config_data) else: # check #1: show diff with "Procced" and "Cancel" options - if not self._diff_original_and_current_data(): + diff_approved, processed_config_data = ( + self._diff_original_and_current_data() + ) + if not diff_approved: return file_path, _ = QFileDialog.getSaveFileName( @@ -191,11 +189,11 @@ def on_button_clicked(self, button): ) # check #2: valid file path if file_path: - self.save_to_file(file_path) + self.save_to_file(processed_config_data, file_path) elif button == self.buttonBox.button(QDialogButtonBox.Open): if self.serverRadio.isChecked(): - self.server_config(save=False) + self.server_config(data_to_push=None) else: file_name, _ = QFileDialog.getOpenFileName( self, "Open File", "", "YAML Files (*.yml);;All Files (*)" @@ -206,18 +204,18 @@ def on_button_clicked(self, button): self.reject() return - def server_config(self, save): + def server_config(self, data_to_push: dict | None = None): dialog = ServerConfigDialog(self) if dialog.exec_(): url = dialog.get_server_url() - if save: - self.push_to_server(url) + if data_to_push is not None: + self.push_to_server(url, data_to_push) else: self.pull_from_server(url) - def push_to_server(self, url): + def push_to_server(self, url, data_to_push: dict): QMessageBox.information( self, @@ -225,14 +223,10 @@ def push_to_server(self, url): f"Pushing configuration to: {url}", ) - processed_config_dict = self.config_data.asdict_enum_safe( - self.config_data, datetime_to_str=True - ) - # TODO: support authentication through the QT framework try: # Send the PUT request to Admin API - response = requests.put(url, headers=headers, json=processed_config_dict) + response = requests.put(url, headers=headers, json=data_to_push) response.raise_for_status() QgsMessageLog.logMessage(f"Success! Status Code: {response.status_code}") @@ -248,7 +242,7 @@ def push_to_server(self, url): QMessageBox.critical( self, "Error", - f"An error occurred pulling the configuration from the server: {e}", + f"An error occurred pushing the configuration to the server: {e}", ) def pull_from_server(self, url): @@ -287,14 +281,14 @@ def pull_from_server(self, url): f"An error occurred pulling the configuration from the server: {e}", ) - def save_to_file(self, file_path): + def save_to_file(self, new_config_data: dict, file_path: str): if file_path: QApplication.setOverrideCursor(Qt.WaitCursor) try: with open(file_path, "w", encoding="utf-8") as file: yaml.dump( - self.config_data.asdict_enum_safe(self.config_data), + new_config_data, file, Dumper=self.dumper, default_flow_style=False, @@ -402,14 +396,20 @@ def _set_validate_ui_data(self) -> tuple[bool, list]: QMessageBox.warning(f"Error deserializing: {e}") return - def _diff_original_and_current_data(self) -> tuple[bool, list]: + def _diff_original_and_current_data(self) -> tuple[bool, dict]: """Before saving the file, show the diff and give an option to proceed or cancel.""" + + new_config_data = self.config_data.asdict_enum_safe( + self.config_data, datetime_to_str=True + ) + + # if created from skratch, no original data to compare to if not self.yaml_original_data: - return True + return True, new_config_data diff_data = diff_yaml_dict( self.yaml_original_data, - self.config_data.asdict_enum_safe(self.config_data), + new_config_data, ) if ( @@ -418,7 +418,7 @@ def _diff_original_and_current_data(self) -> tuple[bool, list]: + len(diff_data["changed"]) == 0 ): - return True + return True, new_config_data # add a window with the choice QgsMessageLog.logMessage(f"{diff_data}") @@ -426,9 +426,9 @@ def _diff_original_and_current_data(self) -> tuple[bool, list]: result = dialog.exec_() # returns QDialog.Accepted (1) or QDialog.Rejected (0) if result == QDialog.Accepted: - return True + return True, new_config_data else: - return False + return False, None def open_templates_path_dialog(self): """Defining Server.templates.path path, called from .ui file.""" diff --git a/tests/test_yaml_save.py b/tests/test_yaml_save.py index c5d6f48..d0d8c87 100644 --- a/tests/test_yaml_save.py +++ b/tests/test_yaml_save.py @@ -41,8 +41,11 @@ def test_json_schema_on_open_save(qtbot, sample_yaml: str): dialog.open_file(sample_yaml) # now dialog.config_data has the data stored # Save YAML + processed_config_data = dialog.config_data.asdict_enum_safe( + dialog.config_data, datetime_to_str=True + ) abs_new_yaml_path = sample_yaml.with_name(f"saved_{sample_yaml.name}") - dialog.save_to_file(abs_new_yaml_path) + dialog.save_to_file(processed_config_data, abs_new_yaml_path) result = subprocess.run( [ @@ -103,8 +106,11 @@ def test_open_file_validate_ui_data_save_file(qtbot, sample_yaml: str): yaml1_missing_props = deepcopy(dialog.config_data.all_missing_props) # Save YAML - EVEN THOUGH some mandatory fields might be missing and recorded as empty strings/lists + processed_config_data = dialog.config_data.asdict_enum_safe( + dialog.config_data, datetime_to_str=True + ) abs_new_yaml_path = sample_yaml.with_name(f"saved_updated_{sample_yaml.name}") - dialog.save_to_file(abs_new_yaml_path) + dialog.save_to_file(processed_config_data, abs_new_yaml_path) # open the new file dialog.open_file(abs_new_yaml_path) # now dialog.config_data has the data stored From 33e379decfb7bfb97db5e63e7a28e9df2490a776 Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Fri, 19 Dec 2025 14:19:36 +0000 Subject: [PATCH 4/9] accept +0000 format datetime (comes from requests) --- models/ConfigData.py | 13 +++-------- models/top_level/utils.py | 47 +++++++++++++++++++++++++++++++++++++++ models/utils.py | 20 ++++++++--------- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/models/ConfigData.py b/models/ConfigData.py index ab6ecba..3384596 100644 --- a/models/ConfigData.py +++ b/models/ConfigData.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field, fields, is_dataclass -from datetime import datetime, timezone +from datetime import datetime from enum import Enum from .utils import update_dataclass_from_dict @@ -9,6 +9,7 @@ MetadataConfig, ResourceConfigTemplate, ) +from .top_level.utils import datetime_to_string from .top_level.utils import InlineList from .top_level.providers import ProviderTemplate from .top_level.providers.records import ProviderTypes @@ -141,14 +142,6 @@ def all_missing_props(self): return self._all_missing_props return [] - def datetime_to_string(self, data: datetime): - # normalize to UTC and format with Z - if data.tzinfo is None: - data = data.replace(tzinfo=timezone.utc) - else: - data = data.astimezone(timezone.utc) - return data.strftime("%Y-%m-%dT%H:%M:%SZ") - def asdict_enum_safe(self, obj, datetime_to_str=False): """Overwriting dataclass 'asdict' fuction to replace Enums with strings.""" if is_dataclass(obj): @@ -177,7 +170,7 @@ def asdict_enum_safe(self, obj, datetime_to_str=False): } else: if isinstance(obj, datetime) and datetime_to_str: - return self.datetime_to_string(obj) + return datetime_to_string(obj) else: return obj diff --git a/models/top_level/utils.py b/models/top_level/utils.py index b39e387..972d627 100644 --- a/models/top_level/utils.py +++ b/models/top_level/utils.py @@ -1,5 +1,6 @@ from datetime import datetime, timezone from enum import Enum +import re STRING_SEPARATOR = " | " @@ -64,3 +65,49 @@ def to_iso8601(dt: datetime) -> str: dt = dt.astimezone(timezone.utc) return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def datetime_to_string(data: datetime): + # normalize to UTC and format with Z + if data.tzinfo is None: + data = data.replace(tzinfo=timezone.utc) + else: + data = data.astimezone(timezone.utc) + return data.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def datetime_from_string(value: str) -> datetime | None: + """ + Parse common ISO8601 datetime strings and return a timezone-aware datetime. + Accepts: + - 2025-12-17T12:34:56Z + - 2025-12-17T12:34:56+02:00 + - 2025-12-17T12:34:56+0200 + If no timezone is present, returns a UTC-aware datetime (assumption). + Returns None if parsing fails. + """ + if not isinstance(value, str): + return None + + s = value.strip() + # quick normalization: trailing Z -> +00:00 + if s.endswith("Z"): + s = s[:-1] + "+00:00" + + # normalize +0200 -> +02:00 + s = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s) + + # Try stdlib first (requires offset with colon to return aware dt) + try: + dt = datetime.fromisoformat(s) + except Exception: + dt = None + + if dt is None: + return None + + # If dt is naive, assume UTC + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + + return dt diff --git a/models/utils.py b/models/utils.py index c7eee06..42684be 100644 --- a/models/utils.py +++ b/models/utils.py @@ -4,7 +4,11 @@ from types import UnionType from typing import Any, get_origin, get_args, Union, get_type_hints -from .top_level.utils import InlineList, get_enum_value_from_string +from .top_level.utils import ( + InlineList, + get_enum_value_from_string, + datetime_from_string, +) def update_dataclass_from_dict( @@ -67,12 +71,7 @@ def update_dataclass_from_dict( if (datetime in args or expected_type is datetime) and isinstance( new_value, str ): - try: - new_value = datetime.strptime( - new_value, "%Y-%m-%dT%H:%M:%SZ" - ) - except: - pass + new_value = datetime_from_string(new_value) # Exception: remap str to Enum elif isinstance(expected_type, type) and issubclass( @@ -294,11 +293,10 @@ def _is_instance_of_type(value, expected_type) -> bool: # Exception: try cast str to datetime manually if expected_type is datetime: - try: - datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ") + if datetime_from_string(value) is not None: return True - except: - pass + else: + return False # Fallback for normal types return isinstance(value, expected_type) From e68ee939d3d9385146ce38d8c7d461ca440e3c58 Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Fri, 19 Dec 2025 14:27:44 +0000 Subject: [PATCH 5/9] ignore datetime diffs for different datetime formats --- utils/data_diff.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utils/data_diff.py b/utils/data_diff.py index 81f85ad..ef2d77c 100644 --- a/utils/data_diff.py +++ b/utils/data_diff.py @@ -74,6 +74,15 @@ def diff_obj(obj1: Any, obj2: Any, diff: dict, path: str = "") -> dict: else: if obj1 != obj2: + + # ignore the case where dates came from 'requests' in +00:00 format + if ( + type(obj1) == type(obj2) == str + and obj2.endswith("Z") + and obj1.endswith("+00:00") + ): + return diff + diff["changed"][path] = {"old": obj1, "new": obj2} return diff From 7c7ffc8f864e3052706f87a305b529ba0dd68fa7 Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Fri, 19 Dec 2025 14:45:38 +0000 Subject: [PATCH 6/9] consider case where datetime was parsed from the beginning (was never a string) --- models/ConfigData.py | 2 +- models/top_level/utils.py | 61 --------------------------------------- models/utils.py | 2 +- utils/data_diff.py | 11 +++++++ utils/helper_functions.py | 55 +++++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 63 deletions(-) create mode 100644 utils/helper_functions.py diff --git a/models/ConfigData.py b/models/ConfigData.py index 3384596..dccb56a 100644 --- a/models/ConfigData.py +++ b/models/ConfigData.py @@ -9,7 +9,7 @@ MetadataConfig, ResourceConfigTemplate, ) -from .top_level.utils import datetime_to_string +from ..utils.helper_functions import datetime_to_string from .top_level.utils import InlineList from .top_level.providers import ProviderTemplate from .top_level.providers.records import ProviderTypes diff --git a/models/top_level/utils.py b/models/top_level/utils.py index 972d627..36b8087 100644 --- a/models/top_level/utils.py +++ b/models/top_level/utils.py @@ -1,6 +1,5 @@ from datetime import datetime, timezone from enum import Enum -import re STRING_SEPARATOR = " | " @@ -51,63 +50,3 @@ def bbox_from_list(raw_bbox_list: list): ) return InlineList(list_bbox_val) - - -def to_iso8601(dt: datetime) -> str: - """ - Convert datetime to UTC ISO 8601 string, for both naive and aware datetimes. - """ - if dt.tzinfo is None: - # Treat naive datetime as UTC - dt = dt.replace(tzinfo=timezone.utc) - else: - # Convert to UTC - dt = dt.astimezone(timezone.utc) - - return dt.strftime("%Y-%m-%dT%H:%M:%SZ") - - -def datetime_to_string(data: datetime): - # normalize to UTC and format with Z - if data.tzinfo is None: - data = data.replace(tzinfo=timezone.utc) - else: - data = data.astimezone(timezone.utc) - return data.strftime("%Y-%m-%dT%H:%M:%SZ") - - -def datetime_from_string(value: str) -> datetime | None: - """ - Parse common ISO8601 datetime strings and return a timezone-aware datetime. - Accepts: - - 2025-12-17T12:34:56Z - - 2025-12-17T12:34:56+02:00 - - 2025-12-17T12:34:56+0200 - If no timezone is present, returns a UTC-aware datetime (assumption). - Returns None if parsing fails. - """ - if not isinstance(value, str): - return None - - s = value.strip() - # quick normalization: trailing Z -> +00:00 - if s.endswith("Z"): - s = s[:-1] + "+00:00" - - # normalize +0200 -> +02:00 - s = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s) - - # Try stdlib first (requires offset with colon to return aware dt) - try: - dt = datetime.fromisoformat(s) - except Exception: - dt = None - - if dt is None: - return None - - # If dt is naive, assume UTC - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - - return dt diff --git a/models/utils.py b/models/utils.py index 42684be..f92a5ba 100644 --- a/models/utils.py +++ b/models/utils.py @@ -7,8 +7,8 @@ from .top_level.utils import ( InlineList, get_enum_value_from_string, - datetime_from_string, ) +from ..utils.helper_functions import datetime_from_string def update_dataclass_from_dict( diff --git a/utils/data_diff.py b/utils/data_diff.py index ef2d77c..5aa3a8e 100644 --- a/utils/data_diff.py +++ b/utils/data_diff.py @@ -1,5 +1,8 @@ +from datetime import datetime from typing import Any +from .helper_functions import datetime_to_string + def diff_yaml_dict(obj1: dict, obj2: dict) -> dict: """Returns all added, removed or changed elements between 2 dictionaries.""" @@ -75,6 +78,14 @@ def diff_obj(obj1: Any, obj2: Any, diff: dict, path: str = "") -> dict: else: if obj1 != obj2: + # ignore the case where incoming datetime was never a string + if ( + isinstance(obj1, datetime) + and isinstance(obj2, str) + and datetime_to_string(obj1) == obj2 + ): + return diff + # ignore the case where dates came from 'requests' in +00:00 format if ( type(obj1) == type(obj2) == str diff --git a/utils/helper_functions.py b/utils/helper_functions.py new file mode 100644 index 0000000..a26e21b --- /dev/null +++ b/utils/helper_functions.py @@ -0,0 +1,55 @@ +from datetime import datetime, timezone +import re + + +def datetime_to_string(data: datetime): + # normalize to UTC and format with Z + if data.tzinfo is None: + data = data.replace(tzinfo=timezone.utc) + else: + data = data.astimezone(timezone.utc) + return data.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def datetime_from_string(value: str) -> datetime | None: + """ + Parse common ISO8601 datetime strings and return a timezone-aware datetime. + Accepts: + - 2025-12-17T12:34:56Z + - 2025-12-17T12:34:56+02:00 + - 2025-12-17T12:34:56+0200 + If no timezone is present, returns a UTC-aware datetime (assumption). + Returns None if parsing fails. + """ + + if isinstance(value, datetime): + # If timezone-naive, assume UTC + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value + + if not isinstance(value, str): + return None + + s = value.strip() + # trailing Z -> +00:00 + if s.endswith("Z"): + s = s[:-1] + "+00:00" + + # normalize +0200 -> +02:00 + s = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s) + + # Try stdlib (requires offset with colon to return aware dt) + try: + dt = datetime.fromisoformat(s) + except Exception: + dt = None + + if dt is None: + return None + + # If dt is naive, assume UTC + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + + return dt From 079738910f06d12757da31a942db7bc5be6279a6 Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Fri, 19 Dec 2025 15:26:14 +0000 Subject: [PATCH 7/9] bring back yaml representer removing string quotes from datetime objects --- pygeoapi_config_dialog.py | 28 ++++++++++++++++++++++++---- tests/test_yaml_save.py | 8 ++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/pygeoapi_config_dialog.py b/pygeoapi_config_dialog.py index 8e6f8c8..407c8ea 100644 --- a/pygeoapi_config_dialog.py +++ b/pygeoapi_config_dialog.py @@ -23,12 +23,13 @@ """ from copy import deepcopy -from datetime import date, datetime, timezone +from datetime import datetime import os from wsgiref import headers import requests import yaml +from .utils.helper_functions import datetime_to_string from .utils.data_diff import diff_yaml_dict from .ui_widgets.utils import get_url_status @@ -149,6 +150,15 @@ class CustomDumper(yaml.SafeDumper): ), ) + # make sure datetime items are not saved as strings with quotes + def represent_datetime_as_timestamp(dumper, data: datetime): + value = datetime_to_string(data) + + # emit as YAML timestamp → plain scalar, no quotes + return dumper.represent_scalar("tag:yaml.org,2002:timestamp", value) + + self.dumper.add_representer(datetime, represent_datetime_as_timestamp) + # custom assignments self.model = QStringListModel() self.proxy = QSortFilterProxyModel() @@ -179,7 +189,7 @@ def on_button_clicked(self, button): else: # check #1: show diff with "Procced" and "Cancel" options diff_approved, processed_config_data = ( - self._diff_original_and_current_data() + self._diff_original_and_current_data(get_yaml_output=True) ) if not diff_approved: return @@ -396,7 +406,9 @@ def _set_validate_ui_data(self) -> tuple[bool, list]: QMessageBox.warning(f"Error deserializing: {e}") return - def _diff_original_and_current_data(self) -> tuple[bool, dict]: + def _diff_original_and_current_data( + self, get_yaml_output=False + ) -> tuple[bool, dict]: """Before saving the file, show the diff and give an option to proceed or cancel.""" new_config_data = self.config_data.asdict_enum_safe( @@ -412,6 +424,14 @@ def _diff_original_and_current_data(self) -> tuple[bool, dict]: new_config_data, ) + # if get_yaml_output, preserve datetime objects without string conversion. + # This is needed so the yaml dumper is using representer removing quotes from datetime strings + if get_yaml_output: + new_config_data = self.config_data.asdict_enum_safe( + self.config_data, datetime_to_str=False + ) + + # if no diff detected, directly accept the changes if ( len(diff_data["added"]) + len(diff_data["removed"]) @@ -420,7 +440,7 @@ def _diff_original_and_current_data(self) -> tuple[bool, dict]: ): return True, new_config_data - # add a window with the choice + # if diff detected, show a window with the choice to approve the diff QgsMessageLog.logMessage(f"{diff_data}") dialog = ReadOnlyTextDialog(self, "Warning", diff_data, True) result = dialog.exec_() # returns QDialog.Accepted (1) or QDialog.Rejected (0) diff --git a/tests/test_yaml_save.py b/tests/test_yaml_save.py index d0d8c87..0cae86b 100644 --- a/tests/test_yaml_save.py +++ b/tests/test_yaml_save.py @@ -42,7 +42,7 @@ def test_json_schema_on_open_save(qtbot, sample_yaml: str): # Save YAML processed_config_data = dialog.config_data.asdict_enum_safe( - dialog.config_data, datetime_to_str=True + dialog.config_data, datetime_to_str=False ) abs_new_yaml_path = sample_yaml.with_name(f"saved_{sample_yaml.name}") dialog.save_to_file(processed_config_data, abs_new_yaml_path) @@ -101,13 +101,13 @@ def test_open_file_validate_ui_data_save_file(qtbot, sample_yaml: str): sample_yaml ) # now dialog.config_data has the data stored including .all_missing_props yaml1_data = dialog.config_data.asdict_enum_safe( - deepcopy(dialog.yaml_original_data), True + deepcopy(dialog.yaml_original_data), datetime_to_str=False ) yaml1_missing_props = deepcopy(dialog.config_data.all_missing_props) # Save YAML - EVEN THOUGH some mandatory fields might be missing and recorded as empty strings/lists processed_config_data = dialog.config_data.asdict_enum_safe( - dialog.config_data, datetime_to_str=True + dialog.config_data, datetime_to_str=False ) abs_new_yaml_path = sample_yaml.with_name(f"saved_updated_{sample_yaml.name}") dialog.save_to_file(processed_config_data, abs_new_yaml_path) @@ -115,7 +115,7 @@ def test_open_file_validate_ui_data_save_file(qtbot, sample_yaml: str): # open the new file dialog.open_file(abs_new_yaml_path) # now dialog.config_data has the data stored yaml2_data = dialog.config_data.asdict_enum_safe( - deepcopy(dialog.yaml_original_data), True + deepcopy(dialog.yaml_original_data), datetime_to_str=False ) # get diff between old and new data From a5f584af47a6f9be40a0f86f8aa2e6ae2e937a33 Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Fri, 19 Dec 2025 16:03:48 +0000 Subject: [PATCH 8/9] ensure utf-8 encoding on push_to_server --- pygeoapi_config_dialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeoapi_config_dialog.py b/pygeoapi_config_dialog.py index 407c8ea..6df055e 100644 --- a/pygeoapi_config_dialog.py +++ b/pygeoapi_config_dialog.py @@ -76,7 +76,7 @@ except: pass -headers = {"accept": "*/*", "Content-Type": "application/json"} +headers = {"accept": "*/*", "Content-Type": "application/json; charset=utf-8"} class ServerConfigDialog(QDialog, Ui_serverDialog): From 280df834bf052d825edb9b70f0c6e3bc68f9614d Mon Sep 17 00:00:00 2001 From: Kateryna Konieva Date: Fri, 19 Dec 2025 16:18:05 +0000 Subject: [PATCH 9/9] typo --- utils/data_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/data_diff.py b/utils/data_diff.py index 5aa3a8e..6ce7371 100644 --- a/utils/data_diff.py +++ b/utils/data_diff.py @@ -91,6 +91,7 @@ def diff_obj(obj1: Any, obj2: Any, diff: dict, path: str = "") -> dict: type(obj1) == type(obj2) == str and obj2.endswith("Z") and obj1.endswith("+00:00") + and obj2[:-1] == obj1[:-6] ): return diff