sc-103647-add-sitemap-for-docs-pennylane-ai-catalyst

doc/conf.py

@@ -132,6 +132,7 @@ def __getattr__(cls, name):
     "sphinx.ext.viewcode",
     "sphinx_copybutton",
     "sphinxext.opengraph",
+    "sphinx_sitemap",
     "sphinx_automodapi.automodapi",
     "sphinx_automodapi.smart_resolver",
     "sphinx_tabs.tabs",
@@ -158,7 +159,9 @@ def __getattr__(cls, name):
 }
 
 # The base URL with a proper language and version.
-html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "/")
+html_baseurl = os.environ.get(
+    "READTHEDOCS_CANONICAL_URL", "https://docs.pennylane.ai/projects/catalyst/"
+)
 
 autosummary_generate = True
 autosummary_imported_members = False
@@ -299,6 +302,11 @@ def __getattr__(cls, name):
 edit_on_github_project = "PennyLaneAI/catalyst"
 edit_on_github_branch = "main/doc"
 
+# -- Sitemap settings -----------------------------------------------------
+sitemap_url_scheme = "{link}"
+# Exclude C++ API pages from sitemap (runtime internals, not for public indexing)
+sitemap_excludes = ["api/*"]
+
 # ============================================================
 
 # the order in which autodoc lists the documented members
@@ -309,3 +317,21 @@ def __getattr__(cls, name):
 
 # autodoc_default_flags = ['members']
 autosummary_generate = True
+
+
+def add_noindex_to_cpp_pages(
+    app, pagename, templatename, context, doctree
+):  # pylint: disable=unused-argument
+    """
+    Add noindex meta tag to C++ API pages generated by Exhale.
+
+    The C++ runtime API is for reference only and should not be indexed by search engines.
+    """
+    if pagename.startswith("api/"):
+        metatags = context.get("metatags", "")
+        context["metatags"] = metatags + '<meta name="robots" content="noindex, nofollow" />\n'
+
+
+def setup(app):
+    """Sphinx application setup hook."""
+    app.connect("html-page-context", add_noindex_to_cpp_pages)

doc/requirements.txt

@@ -26,6 +26,7 @@ pybind11
 m2r2
 mistune==0.8.4
 sphinxext-opengraph==0.9.0
+sphinx-sitemap
 matplotlib==3.10.0
 lxml_html_clean
 

doc/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+
+Sitemap: https://docs.pennylane.ai/projects/catalyst/sitemap.xml
+