diff --git a/models/ConfigData.py b/models/ConfigData.py index ab6ecba..dccb56a 100644 --- a/models/ConfigData.py +++ b/models/ConfigData.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field, fields, is_dataclass -from datetime import datetime, timezone +from datetime import datetime from enum import Enum from .utils import update_dataclass_from_dict @@ -9,6 +9,7 @@ MetadataConfig, ResourceConfigTemplate, ) +from ..utils.helper_functions import datetime_to_string from .top_level.utils import InlineList from .top_level.providers import ProviderTemplate from .top_level.providers.records import ProviderTypes @@ -141,14 +142,6 @@ def all_missing_props(self): return self._all_missing_props return [] - def datetime_to_string(self, data: datetime): - # normalize to UTC and format with Z - if data.tzinfo is None: - data = data.replace(tzinfo=timezone.utc) - else: - data = data.astimezone(timezone.utc) - return data.strftime("%Y-%m-%dT%H:%M:%SZ") - def asdict_enum_safe(self, obj, datetime_to_str=False): """Overwriting dataclass 'asdict' fuction to replace Enums with strings.""" if is_dataclass(obj): @@ -177,7 +170,7 @@ def asdict_enum_safe(self, obj, datetime_to_str=False): } else: if isinstance(obj, datetime) and datetime_to_str: - return self.datetime_to_string(obj) + return datetime_to_string(obj) else: return obj diff --git a/models/top_level/utils.py b/models/top_level/utils.py index b39e387..36b8087 100644 --- a/models/top_level/utils.py +++ b/models/top_level/utils.py @@ -50,17 +50,3 @@ def bbox_from_list(raw_bbox_list: list): ) return InlineList(list_bbox_val) - - -def to_iso8601(dt: datetime) -> str: - """ - Convert datetime to UTC ISO 8601 string, for both naive and aware datetimes. - """ - if dt.tzinfo is None: - # Treat naive datetime as UTC - dt = dt.replace(tzinfo=timezone.utc) - else: - # Convert to UTC - dt = dt.astimezone(timezone.utc) - - return dt.strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/models/utils.py b/models/utils.py index c7eee06..f92a5ba 100644 --- a/models/utils.py +++ b/models/utils.py @@ -4,7 +4,11 @@ from types import UnionType from typing import Any, get_origin, get_args, Union, get_type_hints -from .top_level.utils import InlineList, get_enum_value_from_string +from .top_level.utils import ( + InlineList, + get_enum_value_from_string, +) +from ..utils.helper_functions import datetime_from_string def update_dataclass_from_dict( @@ -67,12 +71,7 @@ def update_dataclass_from_dict( if (datetime in args or expected_type is datetime) and isinstance( new_value, str ): - try: - new_value = datetime.strptime( - new_value, "%Y-%m-%dT%H:%M:%SZ" - ) - except: - pass + new_value = datetime_from_string(new_value) # Exception: remap str to Enum elif isinstance(expected_type, type) and issubclass( @@ -294,11 +293,10 @@ def _is_instance_of_type(value, expected_type) -> bool: # Exception: try cast str to datetime manually if expected_type is datetime: - try: - datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ") + if datetime_from_string(value) is not None: return True - except: - pass + else: + return False # Fallback for normal types return isinstance(value, expected_type) diff --git a/pygeoapi_config_dialog.py b/pygeoapi_config_dialog.py index 1a49425..6df055e 100644 --- a/pygeoapi_config_dialog.py +++ b/pygeoapi_config_dialog.py @@ -23,17 +23,18 @@ """ from copy import deepcopy -from datetime import date, datetime, timezone +from datetime import datetime import os from wsgiref import headers import requests import yaml +from .utils.helper_functions import datetime_to_string from .utils.data_diff import diff_yaml_dict from .ui_widgets.utils import get_url_status -from .server_config_dialog import Ui_serverDialog +from .server_config_dialog import Ui_serverDialog from .models.top_level.providers.records import ProviderTypes from .ui_widgets.providers.NewProviderWindow import NewProviderWindow @@ -75,42 +76,31 @@ except: pass -headers = { - 'accept': '*/*', - 'Content-Type': 'application/json' -} - -def preprocess_for_json(d): - """Recursively converts datetime/date objects in a dict to ISO strings.""" - if isinstance(d, dict): - return {k: preprocess_for_json(v) for k, v in d.items()} - elif isinstance(d, list): - return [preprocess_for_json(i) for i in d] - elif isinstance(d, (datetime, date)): - return d.isoformat() - return d +headers = {"accept": "*/*", "Content-Type": "application/json; charset=utf-8"} + class ServerConfigDialog(QDialog, Ui_serverDialog): """ Logic for the Server Configuration Dialog. Inherits from QDialog (functionality) and Ui_serverDialog (layout). """ + def __init__(self, parent=None): super().__init__(parent) - self.setupUi(self) # Builds the UI defined in Designer - + self.setupUi(self) # Builds the UI defined in Designer + # Optional: Set default values based on current config if needed # self.ServerHostlineEdit.setText("localhost") - def get_server_data(self): + def get_server_url(self): """ Retrieve the server configuration data entered by the user. :return: A dictionary with 'host' and 'port' keys. """ host = self.ServerHostlineEdit.text() port = self.ServerSpinBox.value() - protocol = 'http' if self.radioHttp.isChecked() else 'https' - return {'host': host, 'port': port, 'protocol': protocol} + protocol = "http" if self.radioHttp.isChecked() else "https" + return f"{protocol}://{host}:{port}/admin/config" # This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer @@ -118,7 +108,7 @@ def get_server_data(self): os.path.join(os.path.dirname(__file__), "pygeoapi_config_dialog_base.ui") ) - + class PygeoapiConfigDialog(QtWidgets.QDialog, FORM_CLASS): config_data: ConfigData @@ -160,8 +150,9 @@ class CustomDumper(yaml.SafeDumper): ), ) + # make sure datetime items are not saved as strings with quotes def represent_datetime_as_timestamp(dumper, data: datetime): - value = self.config_data.datetime_to_string(data) + value = datetime_to_string(data) # emit as YAML timestamp → plain scalar, no quotes return dumper.represent_scalar("tag:yaml.org,2002:timestamp", value) @@ -183,22 +174,36 @@ def on_button_clicked(self, button): # You can also check the standard button type if button == self.buttonBox.button(QDialogButtonBox.Save): + # proceed only if UI data inputs are valid if self._set_validate_ui_data()[0]: if self.serverRadio.isChecked(): - self.server_config(save=True) + # check #1: show diff with "Procced" and "Cancel" options + diff_approved, processed_config_data = ( + self._diff_original_and_current_data() + ) + if not diff_approved: + return + + self.server_config(data_to_push=processed_config_data) else: + # check #1: show diff with "Procced" and "Cancel" options + diff_approved, processed_config_data = ( + self._diff_original_and_current_data(get_yaml_output=True) + ) + if not diff_approved: + return + file_path, _ = QFileDialog.getSaveFileName( self, "Save File", "", "YAML Files (*.yml);;All Files (*)" ) - - # before saving, show diff with "Procced" and "Cancel" options - if file_path and self._diff_original_and_current_data(): - self.save_to_file(file_path) + # check #2: valid file path + if file_path: + self.save_to_file(processed_config_data, file_path) elif button == self.buttonBox.button(QDialogButtonBox.Open): if self.serverRadio.isChecked(): - self.server_config(save=False) + self.server_config(data_to_push=None) else: file_name, _ = QFileDialog.getOpenFileName( self, "Open File", "", "YAML Files (*.yml);;All Files (*)" @@ -209,19 +214,18 @@ def on_button_clicked(self, button): self.reject() return - def server_config(self, save): + def server_config(self, data_to_push: dict | None = None): dialog = ServerConfigDialog(self) - if dialog.exec_(): - data = dialog.get_server_data() - url = f"{data['protocol']}://{data['host']}:{data['port']}/admin/config" - if save == True: - self.push_to_server(url) + if dialog.exec_(): + url = dialog.get_server_url() + if data_to_push is not None: + self.push_to_server(url, data_to_push) else: - self.pull_from_server(url) - - def push_to_server(self, url): + self.pull_from_server(url) + + def push_to_server(self, url, data_to_push: dict): QMessageBox.information( self, @@ -229,19 +233,13 @@ def push_to_server(self, url): f"Pushing configuration to: {url}", ) - config_dict = self.config_data.asdict_enum_safe(self.config_data) - - # Pre-process the dictionary to handle datetime objects - processed_config_dict = preprocess_for_json(config_dict) - # TODO: support authentication through the QT framework try: # Send the PUT request to Admin API - response = requests.put(url, headers=headers, json=processed_config_dict) + response = requests.put(url, headers=headers, json=data_to_push) response.raise_for_status() - QgsMessageLog.logMessage( - f"Success! Status Code: {response.status_code}") + QgsMessageLog.logMessage(f"Success! Status Code: {response.status_code}") QMessageBox.information( self, @@ -254,12 +252,11 @@ def push_to_server(self, url): QMessageBox.critical( self, "Error", - f"An error occurred pulling the configuration from the server: {e}", + f"An error occurred pushing the configuration to the server: {e}", ) - def pull_from_server(self, url): - + QMessageBox.information( self, "Information", @@ -272,8 +269,7 @@ def pull_from_server(self, url): response = requests.get(url, headers=headers) response.raise_for_status() - QgsMessageLog.logMessage( - f"Success! Status Code: {response.status_code}") + QgsMessageLog.logMessage(f"Success! Status Code: {response.status_code}") QMessageBox.information( self, @@ -281,35 +277,10 @@ def pull_from_server(self, url): f"Success! Status Code: {response.status_code}", ) - QgsMessageLog.logMessage( - f"Response: {response.text}") - - data_dict = response.json() - - self.config_data = ConfigData() - self.config_data.set_data_from_yaml(data_dict) - self.ui_setter.set_ui_from_data() - - # log messages about missing or mistyped values during deserialization - QgsMessageLog.logMessage( - f"Errors during deserialization: {self.config_data.error_message}" - ) - QgsMessageLog.logMessage( - f"Default values used for missing YAML fields: {self.config_data.defaults_message}" - ) + QgsMessageLog.logMessage(f"Response: {response.text}") - # summarize all properties missing/overwitten with defaults - # atm, warning with the full list of properties - all_missing_props = self.config_data.all_missing_props - QgsMessageLog.logMessage( - f"All missing or replaced properties: {self.config_data.all_missing_props}" - ) - if len(all_missing_props) > 0: - ReadOnlyTextDialog( - self, - "Warning", - f"All missing or replaced properties (check logs for more details): {self.config_data.all_missing_props}", - ).exec_() + data_dict = response.json() + self.update_config_data_and_ui(data_dict) except requests.exceptions.RequestException as e: QgsMessageLog.logMessage(f"An error occurred: {e}") @@ -320,15 +291,14 @@ def pull_from_server(self, url): f"An error occurred pulling the configuration from the server: {e}", ) - - def save_to_file(self, file_path): + def save_to_file(self, new_config_data: dict, file_path: str): if file_path: QApplication.setOverrideCursor(Qt.WaitCursor) try: with open(file_path, "w", encoding="utf-8") as file: yaml.dump( - self.config_data.asdict_enum_safe(self.config_data), + new_config_data, file, Dumper=self.dumper, default_flow_style=False, @@ -360,48 +330,53 @@ def open_file(self, file_name): # QApplication.setOverrideCursor(Qt.WaitCursor) with open(file_name, "r", encoding="utf-8") as file: file_content = file.read() + yaml_original_data_dict = yaml.safe_load(file_content) + + self.update_config_data_and_ui(yaml_original_data_dict) - # reset data - self.config_data = ConfigData() + except Exception as e: + QMessageBox.warning(self, "Error", f"Cannot open file:\n{str(e)}") + # finally: + # QApplication.restoreOverrideCursor() - # set data and .all_missing_props: - yaml_original_data = yaml.safe_load(file_content) - self.yaml_original_data = deepcopy(yaml_original_data) + def update_config_data_and_ui(self, data_dict): + """Use the data from local file or local server to reset the ConfigData and UI.""" - self.config_data.set_data_from_yaml(yaml_original_data) + # reset data + self.config_data = ConfigData() - # set UI from data - self.ui_setter.set_ui_from_data() + # set data and .all_missing_props: + self.yaml_original_data = deepcopy(data_dict) + self.config_data.set_data_from_yaml(data_dict) - # log messages about missing or mistyped values during deserialization - # try/except in case of running it from pytests - try: - QgsMessageLog.logMessage( - f"Errors during deserialization: {self.config_data.error_message}" - ) - QgsMessageLog.logMessage( - f"Default values used for missing YAML fields: {self.config_data.defaults_message}" - ) + # set UI from data + self.ui_setter.set_ui_from_data() - # summarize all properties missing/overwitten with defaults - # atm, warning with the full list of properties - QgsMessageLog.logMessage( - f"All missing or replaced properties: {self.config_data.all_missing_props}" - ) + # log messages about missing or mistyped values during deserialization + # try/except in case of running it from pytests + try: + QgsMessageLog.logMessage( + f"Errors during deserialization: {self.config_data.error_message}" + ) + QgsMessageLog.logMessage( + f"Default values used for missing YAML fields: {self.config_data.defaults_message}" + ) - if len(self.config_data.all_missing_props) > 0: - ReadOnlyTextDialog( - self, - "Warning", - f"All missing or replaced properties (check logs for more details): {self.config_data.all_missing_props}", - ).exec_() - except: - pass + # summarize all properties missing/overwitten with defaults + # atm, warning with the full list of properties + all_missing_props = self.config_data.all_missing_props + QgsMessageLog.logMessage( + f"All missing or replaced properties: {all_missing_props}" + ) - except Exception as e: - QMessageBox.warning(self, "Error", f"Cannot open file:\n{str(e)}") - # finally: - # QApplication.restoreOverrideCursor() + if len(all_missing_props) > 0: + ReadOnlyTextDialog( + self, + "Warning", + f"All missing or replaced properties (check logs for more details): {all_missing_props}", + ).exec_() + except: + pass # QgsMessageLog import error in pytests, ignore def _set_validate_ui_data(self) -> tuple[bool, list]: # Set and validate data from UI @@ -431,33 +406,49 @@ def _set_validate_ui_data(self) -> tuple[bool, list]: QMessageBox.warning(f"Error deserializing: {e}") return - def _diff_original_and_current_data(self) -> tuple[bool, list]: + def _diff_original_and_current_data( + self, get_yaml_output=False + ) -> tuple[bool, dict]: """Before saving the file, show the diff and give an option to proceed or cancel.""" + + new_config_data = self.config_data.asdict_enum_safe( + self.config_data, datetime_to_str=True + ) + + # if created from skratch, no original data to compare to if not self.yaml_original_data: - return True + return True, new_config_data diff_data = diff_yaml_dict( self.yaml_original_data, - self.config_data.asdict_enum_safe(self.config_data), + new_config_data, ) + # if get_yaml_output, preserve datetime objects without string conversion. + # This is needed so the yaml dumper is using representer removing quotes from datetime strings + if get_yaml_output: + new_config_data = self.config_data.asdict_enum_safe( + self.config_data, datetime_to_str=False + ) + + # if no diff detected, directly accept the changes if ( len(diff_data["added"]) + len(diff_data["removed"]) + len(diff_data["changed"]) == 0 ): - return True + return True, new_config_data - # add a window with the choice + # if diff detected, show a window with the choice to approve the diff QgsMessageLog.logMessage(f"{diff_data}") dialog = ReadOnlyTextDialog(self, "Warning", diff_data, True) result = dialog.exec_() # returns QDialog.Accepted (1) or QDialog.Rejected (0) if result == QDialog.Accepted: - return True + return True, new_config_data else: - return False + return False, None def open_templates_path_dialog(self): """Defining Server.templates.path path, called from .ui file.""" diff --git a/tests/test_yaml_save.py b/tests/test_yaml_save.py index c5d6f48..0cae86b 100644 --- a/tests/test_yaml_save.py +++ b/tests/test_yaml_save.py @@ -41,8 +41,11 @@ def test_json_schema_on_open_save(qtbot, sample_yaml: str): dialog.open_file(sample_yaml) # now dialog.config_data has the data stored # Save YAML + processed_config_data = dialog.config_data.asdict_enum_safe( + dialog.config_data, datetime_to_str=False + ) abs_new_yaml_path = sample_yaml.with_name(f"saved_{sample_yaml.name}") - dialog.save_to_file(abs_new_yaml_path) + dialog.save_to_file(processed_config_data, abs_new_yaml_path) result = subprocess.run( [ @@ -98,18 +101,21 @@ def test_open_file_validate_ui_data_save_file(qtbot, sample_yaml: str): sample_yaml ) # now dialog.config_data has the data stored including .all_missing_props yaml1_data = dialog.config_data.asdict_enum_safe( - deepcopy(dialog.yaml_original_data), True + deepcopy(dialog.yaml_original_data), datetime_to_str=False ) yaml1_missing_props = deepcopy(dialog.config_data.all_missing_props) # Save YAML - EVEN THOUGH some mandatory fields might be missing and recorded as empty strings/lists + processed_config_data = dialog.config_data.asdict_enum_safe( + dialog.config_data, datetime_to_str=False + ) abs_new_yaml_path = sample_yaml.with_name(f"saved_updated_{sample_yaml.name}") - dialog.save_to_file(abs_new_yaml_path) + dialog.save_to_file(processed_config_data, abs_new_yaml_path) # open the new file dialog.open_file(abs_new_yaml_path) # now dialog.config_data has the data stored yaml2_data = dialog.config_data.asdict_enum_safe( - deepcopy(dialog.yaml_original_data), True + deepcopy(dialog.yaml_original_data), datetime_to_str=False ) # get diff between old and new data diff --git a/utils/data_diff.py b/utils/data_diff.py index 81f85ad..6ce7371 100644 --- a/utils/data_diff.py +++ b/utils/data_diff.py @@ -1,5 +1,8 @@ +from datetime import datetime from typing import Any +from .helper_functions import datetime_to_string + def diff_yaml_dict(obj1: dict, obj2: dict) -> dict: """Returns all added, removed or changed elements between 2 dictionaries.""" @@ -74,6 +77,24 @@ def diff_obj(obj1: Any, obj2: Any, diff: dict, path: str = "") -> dict: else: if obj1 != obj2: + + # ignore the case where incoming datetime was never a string + if ( + isinstance(obj1, datetime) + and isinstance(obj2, str) + and datetime_to_string(obj1) == obj2 + ): + return diff + + # ignore the case where dates came from 'requests' in +00:00 format + if ( + type(obj1) == type(obj2) == str + and obj2.endswith("Z") + and obj1.endswith("+00:00") + and obj2[:-1] == obj1[:-6] + ): + return diff + diff["changed"][path] = {"old": obj1, "new": obj2} return diff diff --git a/utils/helper_functions.py b/utils/helper_functions.py new file mode 100644 index 0000000..a26e21b --- /dev/null +++ b/utils/helper_functions.py @@ -0,0 +1,55 @@ +from datetime import datetime, timezone +import re + + +def datetime_to_string(data: datetime): + # normalize to UTC and format with Z + if data.tzinfo is None: + data = data.replace(tzinfo=timezone.utc) + else: + data = data.astimezone(timezone.utc) + return data.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def datetime_from_string(value: str) -> datetime | None: + """ + Parse common ISO8601 datetime strings and return a timezone-aware datetime. + Accepts: + - 2025-12-17T12:34:56Z + - 2025-12-17T12:34:56+02:00 + - 2025-12-17T12:34:56+0200 + If no timezone is present, returns a UTC-aware datetime (assumption). + Returns None if parsing fails. + """ + + if isinstance(value, datetime): + # If timezone-naive, assume UTC + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value + + if not isinstance(value, str): + return None + + s = value.strip() + # trailing Z -> +00:00 + if s.endswith("Z"): + s = s[:-1] + "+00:00" + + # normalize +0200 -> +02:00 + s = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s) + + # Try stdlib (requires offset with colon to return aware dt) + try: + dt = datetime.fromisoformat(s) + except Exception: + dt = None + + if dt is None: + return None + + # If dt is naive, assume UTC + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + + return dt