feat: add options for local page loading and Python code generation in PagePackageController

Author: simontaurus

src/osw/controller/page_package.py

@@ -372,6 +372,17 @@ class CreationConfig(OswBaseModel):
         offline_pages: Optional[Dict[str, WtPage]] = None
         """A dictionary of pages that are already loaded. Pages in this dictionary
         will not be fetched again."""
+        prefer_local_pages: bool = False
+        """Load the pages from the local working directory
+        instead of the server if set to True."""
+        generate_python_code: bool = False
+        """Whether to generate python code for the data models."""
+        python_code_working_dir: Optional[Union[str, Path]] = None
+        """Working directory for python code generation. If set, pydantic v2 data models
+        will be generated in this directory, v1 models in a /v1 subdirectory.
+        """
+        python_code_filename: Optional[str] = "_model_generated.py"
+        """Filename for the generated python code."""
 
         class Config:
             arbitrary_types_allowed = True
@@ -418,6 +429,57 @@ def create(
                 )
             },
         )
+
+        offline_pages = creation_config.offline_pages
+        local_pages = {}
+        if creation_config.prefer_local_pages:
+            # Read the local pages from the package
+            result = wtsite.read_page_package(
+                WtSite.ReadPagePackageParam(
+                    package_name=self.name,
+                    storage_path=Path(creation_config.working_dir),
+                )
+            )
+            local_pages = {p.title: p for p in result.pages}
+            if offline_pages is None:
+                offline_pages = local_pages
+            else:
+                # Merge the local pages with the offline pages
+                offline_pages.update(local_pages)
+
+        if (
+            creation_config.generate_python_code
+            and creation_config.python_code_working_dir is not None
+        ):
+            python_code_path = Path(creation_config.python_code_working_dir)
+            python_code_path /= creation_config.python_code_filename
+            schema_titles = self.page_titles
+            # remove duplicates and entries in ignore_titles
+            schema_titles = list(
+                set(schema_titles)
+                - (
+                    set(creation_config.ignore_titles)
+                    if creation_config.ignore_titles
+                    else set()
+                )
+            )
+            # remove all schemas that do not start with "Category:"
+            schema_titles = [
+                title for title in schema_titles if title.startswith("Category:")
+            ]
+            from osw.core import OSW
+
+            osw_obj = OSW(site=wtsite)
+
+            osw_obj.fetch_schema(
+                fetchSchemaParam=OSW.FetchSchemaParam(
+                    schema_title=schema_titles,
+                    offline_pages=offline_pages,
+                    result_model_path=python_code_path,
+                    mode="replace",
+                )
+            )
+
         # Create a PagePackageConfig instance
         config = package.PagePackageConfig(
             name=self.name,
@@ -432,7 +494,7 @@ def create(
         wtsite.create_page_package(
             WtSite.CreatePagePackageParam(
                 config=config,
-                offline_pages=creation_config.offline_pages,
+                offline_pages=offline_pages,
             )
         )
 

src/osw/wtsite.py

@@ -848,6 +848,9 @@ class ReadPagePackageParam(OswBaseModel):
         """A list of slots that should be read. If None, all slots are read."""
         debug: Optional[bool] = False
         """If True, debug information is printed to the console."""
+        offline: Optional[bool] = True
+        """Skip reading the page content from the webserver
+        before reading the local content, if True."""
 
     class ReadPagePackageResult(OswBaseModel):
         """Return type of read_page_package."""
@@ -957,19 +960,9 @@ def get_slot_content(
                 namespace = page["namespace"].split("_")[-1].capitalize()
                 name = page["name"]
                 # Create the WtPage object
-                page_obj = WtPage(wtSite=self, title=f"{namespace}:{name}")
-                if "main" in selected_slots:
-                    # Main slot is special
-                    slot_content = get_slot_content(
-                        parent_dir=sub_dirs,
-                        url_path=page["urlPath"],
-                        files_in_storage_path=storage_path_content["files"],
-                    )
-                    if slot_content is not None:
-                        page_obj.set_slot_content(
-                            slot_key="main",
-                            content=slot_content,
-                        )
+                page_obj = WtPage(
+                    wtSite=self, title=f"{namespace}:{name}", do_init=not param.offline
+                )
                 if selected_slots is None:
                     _selected_slots = page["slots"]
                 else:
@@ -979,16 +972,29 @@ def get_slot_content(
                         if slot_name in selected_slots
                     }
                 for slot_name, slot_dict in _selected_slots.items():
-                    slot_content = get_slot_content(
-                        parent_dir=sub_dirs,
-                        url_path=slot_dict["urlPath"],
-                        files_in_storage_path=storage_path_content["files"],
-                    )
-                    if slot_content is not None:
-                        page_obj.set_slot_content(
-                            slot_key=slot_name,
-                            content=slot_content,
+                    if slot_name == "main":
+                        # Main slot is special
+                        slot_content = get_slot_content(
+                            parent_dir=sub_dirs,
+                            url_path=page["urlPath"],
+                            files_in_storage_path=storage_path_content["files"],
                         )
+                        if slot_content is not None:
+                            page_obj.set_slot_content(
+                                slot_key="main",
+                                content=slot_content,
+                            )
+                    else:
+                        slot_content = get_slot_content(
+                            parent_dir=sub_dirs,
+                            url_path=slot_dict["urlPath"],
+                            files_in_storage_path=storage_path_content["files"],
+                        )
+                        if slot_content is not None:
+                            page_obj.set_slot_content(
+                                slot_key=slot_name,
+                                content=slot_content,
+                            )
                 pages.append(page_obj)
         return WtSite.ReadPagePackageResult(pages=pages)