From 6c9bc16653b49ee2a4d3162a51b792000c6184dc Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sat, 20 Dec 2025 16:28:57 -0500 Subject: [PATCH 1/3] eng-1032 Add json-ld experimental export --- apps/roam/src/utils/getExportTypes.ts | 471 ++++++++++++++++++------- apps/roam/src/utils/getPageMetadata.ts | 14 +- apps/website/public/schema/dg_base.ttl | 77 ++++ apps/website/public/schema/dg_core.ttl | 68 ++++ 4 files changed, 500 insertions(+), 130 deletions(-) create mode 100644 apps/website/public/schema/dg_base.ttl create mode 100644 apps/website/public/schema/dg_core.ttl diff --git a/apps/roam/src/utils/getExportTypes.ts b/apps/roam/src/utils/getExportTypes.ts index 6f6e19b99..6a5ade560 100644 --- a/apps/roam/src/utils/getExportTypes.ts +++ b/apps/roam/src/utils/getExportTypes.ts @@ -5,11 +5,14 @@ import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParen import getPageViewType from "roamjs-components/queries/getPageViewType"; import { PullBlock, TreeNode, ViewType } from "roamjs-components/types"; import { Result } from "roamjs-components/types/query-builder"; +import getRoamUrl from "roamjs-components/dom/getRoamUrl"; import XRegExp from "xregexp"; import getDiscourseNodes from "./getDiscourseNodes"; +import type { DiscourseNode } from "./getDiscourseNodes"; import isFlagEnabled from "./isFlagEnabled"; import matchDiscourseNode from "./matchDiscourseNode"; import getDiscourseRelations from "./getDiscourseRelations"; +import type { DiscourseRelation } from "./getDiscourseRelations"; import type { ExportDialogProps } from "~/components/Export"; import getPageMetadata from "./getPageMetadata"; import getDiscourseContextResults from "./getDiscourseContextResults"; @@ -52,7 +55,7 @@ const getContentFromNodes = ({ allNodes, }: { title: string; - allNodes: ReturnType; + allNodes: DiscourseNode[]; }) => { const nodeFormat = allNodes.find((a) => matchDiscourseNode({ title, ...a }), @@ -79,7 +82,7 @@ const getFilename = ({ title?: string; maxFilenameLength: number; simplifiedFilename: boolean; - allNodes: ReturnType; + allNodes: DiscourseNode[]; removeSpecialCharacters: boolean; extension?: string; }) => { @@ -156,7 +159,7 @@ const toMarkdown = ({ embeds: boolean; simplifiedFilename: boolean; maxFilenameLength: number; - allNodes: ReturnType; + allNodes: DiscourseNode[]; removeSpecialCharacters: boolean; linkType: string; flatten?: boolean; @@ -403,138 +406,344 @@ const getExportTypes = ({ }); }; + const pageToMarkdown = async ( + { text, uid, context: _, type, ...rest }: Result, + { + includeDiscourseContext, + appendRefNodeContext, + frontmatter, + optsRefs, + optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }: { + includeDiscourseContext: boolean; + appendRefNodeContext: boolean; + frontmatter: string[]; + optsRefs: boolean; + optsEmbeds: boolean; + simplifiedFilename: boolean; + allNodes: DiscourseNode[]; + maxFilenameLength: number; + removeSpecialCharacters: boolean; + linkType: string; + }, + ): Promise<{ title: string; content: string; uids: Set }> => { + const v = getPageViewType(text) || "bullet"; + const { date, displayName } = getPageMetadata(text); + const treeNode = getFullTreeByParentUid(uid); + + const discourseResults = await handleDiscourseContext({ + includeDiscourseContext, + pageTitle: text, + uid, + appendRefNodeContext, + }); + + const referenceResults = isFlagEnabled("render references") + ? ( + window.roamAlphaAPI.data.fast.q( + `[:find (pull ?pr [:node/title]) (pull ?r [:block/heading [:block/string :as "text"] [:children/view-type :as "viewType"] {:block/children ...}]) :where [?p :node/title "${normalizePageTitle( + text, + )}"] [?r :block/refs ?p] [?r :block/page ?pr]]`, + ) as [PullBlock, PullBlock][] + ).filter( + ([, { [":block/children"]: children }]) => + Array.isArray(children) && children.length, + ) + : []; + + const result: Result = { + ...rest, + date, + text, + uid, + author: displayName, + type, + }; + const yamlLines = handleFrontmatter({ + frontmatter, + rest, + result, + }); + + const content = `${yamlLines}\n\n${treeNode.children + .map((c) => + toMarkdown({ + c, + v, + i: 0, + opts: { + refs: optsRefs, + embeds: optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }, + }), + ) + .join("\n")}\n${ + discourseResults.length + ? `\n###### Discourse Context\n\n${discourseResults + .flatMap((r) => + Object.values(r.results).map( + (t) => + `- **${r.label}::** ${toLink( + getFilename({ + title: t.text, + maxFilenameLength, + simplifiedFilename, + allNodes, + removeSpecialCharacters, + }), + t.uid, + linkType, + )}`, + ), + ) + .join("\n")}\n` + : "" + }${ + referenceResults.length + ? `\n###### References\n\n${referenceResults + .map( + (r_1) => + `${toLink( + getFilename({ + title: r_1[0][":node/title"], + maxFilenameLength, + simplifiedFilename, + allNodes, + removeSpecialCharacters, + }), + r_1[0][":block/uid"] || "", + linkType, + )}\n\n${toMarkdown({ + c: pullBlockToTreeNode(r_1[1], ":bullet"), + opts: { + refs: optsRefs, + embeds: optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }, + })}`, + ) + .join("\n")}\n` + : "" + }`; + const uids = new Set(collectUids(treeNode)); + return { title: text, content, uids }; + }; + + const getJsonLdData = async () => { + const roamUrl = getRoamUrl(); + updateExportProgress({ + progress: 0, + id: exportId, + }); + // skip a beat to let progress render + await new Promise((resolve) => setTimeout(resolve)); + const pageData = await getPageData(); + const numPages = pageData.length + allNodes.length; + let numTreatedPages = 0; + const context = { + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + rdfs: "http://www.w3.org/2000/01/rdf-schema#", + owl: "http://www.w3.org/2002/07/owl#", + dc: "http://purl.org/dc/elements/1.1/", + prov: "http://www.w3.org/ns/prov#", + sioc: "http://rdfs.org/sioc/ns#", + dg: "https://discoursegraphs.com/schema/dg_core#", + subClassOf: "rdfs:subClassOf", + title: "dc:title", + modified: "dc:modified", + created: "dc:date", + creator: "dc:creator", + content: "sioc:content", + source: "dg:source", + destination: "dg:destination", + reltype: "dg:reltype", + pages: `${roamUrl}/page/`, + }; + const settings = { + ...getExportSettings(), + includeDiscourseContext: false, + }; + // TODO : Identify existing CURIES in the node definition + /* eslint-disable @typescript-eslint/naming-convention */ + const nodeSchemaData = await Promise.all( + allNodes.map(async (node: DiscourseNode) => { + const { date, displayName, modified } = getPageMetadata(node.text); + const r = await pageToMarkdown( + { + text: node.text, + uid: node.type, + }, + { ...settings, allNodes }, + ); + numTreatedPages += 1; + updateExportProgress({ + progress: 0.1 + (numTreatedPages / numPages) * 0.75, + id: exportId, + }); + await new Promise((resolve) => setTimeout(resolve)); + return { + "@id": `pages:${node.type}`, + "@type": "dg:NodeSchema", + title: node.text, + content: r.content, + modified: modified?.toJSON(), + created: date.toJSON(), + creator: displayName, + }; + }), + ); + const relSchemaData = allRelations.map((r: DiscourseRelation) => ({ + "@id": `pages:${r.id}`, + "@type": "dg:BinaryRelationSchema", + subClassOf: [ + { + "@type": "owl:Restriction", + "owl:onProperty": "dg:source", + "owl:allValuesFrom": `pages:${r.source}`, + }, + { + "@type": "owl:Restriction", + "owl:onProperty": "dg:destination", + "owl:allValuesFrom": `pages:${r.destination}`, + }, + ], + title: r.label, + })); + const schemaData = [...nodeSchemaData, ...relSchemaData]; + + const schemaUriByName = Object.fromEntries( + schemaData.map((node) => [node.title, node["@id"]]), + ); + + const inverseRelSchemaData = allRelations.map((r: DiscourseRelation) => ({ + "@id": `pages:${r.id}-inverse`, + "@type": "dg:BinaryRelationSchema", + subClassOf: [ + { + "@type": "owl:Restriction", + "owl:onProperty": "dg:source", + "owl:allValuesFrom": `pages:${r.destination}`, + }, + { + "@type": "owl:Restriction", + "owl:onProperty": "dg:destination", + "owl:allValuesFrom": `pages:${r.source}`, + }, + ], + title: r.complement, + "owl:inverse-of": `pages:${r.id}`, + })); + + await Promise.all( + pageData.map(async (page: Result) => { + const r = await pageToMarkdown(page, { + ...settings, + allNodes, + }); + page.content = r.content; + numTreatedPages += 1; + updateExportProgress({ + progress: 0.1 + (numTreatedPages / numPages) * 0.75, + id: exportId, + }); + await new Promise((resolve) => setTimeout(resolve)); + }), + ); + + // skip a beat to let progress render + await new Promise((resolve) => setTimeout(resolve)); + await Promise.all( + pageData.map(async (page: Result) => { + const r = await pageToMarkdown(page, { + ...settings, + allNodes, + }); + page.content = r.content; + numTreatedPages += 1; + updateExportProgress({ + progress: 0.1 + (numTreatedPages / numPages) * 0.75, + id: exportId, + }); + await new Promise((resolve) => setTimeout(resolve)); + }), + ); + + const nodes = pageData.map(({ text, uid, content, type }) => { + const { date, displayName, modified } = getPageMetadata(text); + return { + "@id": `pages:${uid}`, + "@type": schemaUriByName[type], + title: text, + content, + modified: modified?.toJSON(), + created: date.toJSON(), + creator: displayName, + }; + }); + const nodeSet = new Set(pageData.map((n) => n.uid)); + const rels = await getRelationData(); + updateExportProgress({ + progress: 1, + id: exportId, + }); + await new Promise((resolve) => setTimeout(resolve)); + const relations = uniqJsonArray( + rels.filter((r) => nodeSet.has(r.source) && nodeSet.has(r.target)), + ); + const relData = relations.map(({ source, target, label }) => ({ + // no id yet, just a blank node + "@type": "dg:BinaryRelation", + reltype: schemaUriByName[label], + source: `pages:${source}`, + destination: `pages:${target}`, + })); + return { + "@context": context, + "@id": roamUrl, + "prov:generatedAtTime": new Date().toISOString(), + "@graph": [...schemaData, ...inverseRelSchemaData, ...nodes, ...relData], + }; + /* eslint-enable @typescript-eslint/naming-convention */ + }; + return [ { name: "Markdown", callback: async ({ includeDiscourseContext = false }) => { + const settings = { + ...getExportSettings(), + includeDiscourseContext, + }; const { - frontmatter, - optsRefs, - optsEmbeds, simplifiedFilename, maxFilenameLength, removeSpecialCharacters, - linkType, - appendRefNodeContext, - } = getExportSettings(); + } = settings; const allPages = await getPageData(isExportDiscourseGraph); - const gatherings = allPages.map( - ({ text, uid, context: _, type, ...rest }, i, all) => - async () => { - updateExportProgress({ progress: i / all.length, id: exportId }); - // skip a beat to let progress render - await new Promise((resolve) => setTimeout(resolve)); - const v = getPageViewType(text) || "bullet"; - const { date, displayName } = getPageMetadata(text); - const treeNode = getFullTreeByParentUid(uid); - - const discourseResults = await handleDiscourseContext({ - includeDiscourseContext, - pageTitle: text, - uid, - appendRefNodeContext, - }); - - const referenceResults = isFlagEnabled("render references") - ? ( - window.roamAlphaAPI.data.fast.q( - `[:find (pull ?pr [:node/title]) (pull ?r [:block/heading [:block/string :as "text"] [:children/view-type :as "viewType"] {:block/children ...}]) :where [?p :node/title "${normalizePageTitle( - text, - )}"] [?r :block/refs ?p] [?r :block/page ?pr]]`, - ) as [PullBlock, PullBlock][] - ).filter( - ([, { [":block/children"]: children }]) => - Array.isArray(children) && children.length, - ) - : []; - - const result: Result = { - ...rest, - date, - text, - uid, - author: displayName, - type, - }; - const yamlLines = handleFrontmatter({ - frontmatter, - rest, - result, - }); + const gatherings = allPages.map((result, i, all) => async () => { + updateExportProgress({ progress: i / all.length, id: exportId }); + // skip a beat to let progress render + await new Promise((resolve) => setTimeout(resolve)); + return pageToMarkdown(result, { + ...settings, + allNodes, + }); + }); - const content = `${yamlLines}\n\n${treeNode.children - .map((c) => - toMarkdown({ - c, - v, - i: 0, - opts: { - refs: optsRefs, - embeds: optsEmbeds, - simplifiedFilename, - allNodes, - maxFilenameLength, - removeSpecialCharacters, - linkType, - }, - }), - ) - .join("\n")}\n${ - discourseResults.length - ? `\n###### Discourse Context\n\n${discourseResults - .flatMap((r) => - Object.values(r.results).map( - (t) => - `- **${r.label}::** ${toLink( - getFilename({ - title: t.text, - maxFilenameLength, - simplifiedFilename, - allNodes, - removeSpecialCharacters, - }), - t.uid, - linkType, - )}`, - ), - ) - .join("\n")}\n` - : "" - }${ - referenceResults.length - ? `\n###### References\n\n${referenceResults - .map( - (r_1) => - `${toLink( - getFilename({ - title: r_1[0][":node/title"], - maxFilenameLength, - simplifiedFilename, - allNodes, - removeSpecialCharacters, - }), - r_1[0][":block/uid"] || "", - linkType, - )}\n\n${toMarkdown({ - c: pullBlockToTreeNode(r_1[1], ":bullet"), - opts: { - refs: optsRefs, - embeds: optsEmbeds, - simplifiedFilename, - allNodes, - maxFilenameLength, - removeSpecialCharacters, - linkType, - }, - })}`, - ) - .join("\n")}\n` - : "" - }`; - const uids = new Set(collectUids(treeNode)); - return { title: text, content, uids }; - }, - ); const pages = await gatherings.reduce( (p, c) => p.then((arr) => @@ -571,6 +780,18 @@ const getExportTypes = ({ ]; }, }, + { + name: "JSON-LD (Experimental)", + callback: async ({ filename }) => { + const data = await getJsonLdData(); + return [ + { + title: `${filename.replace(/\.json$/, "")}.json`, + content: JSON.stringify(data, undefined, " "), + }, + ]; + }, + }, { name: "Neo4j", callback: async ({ filename }) => { diff --git a/apps/roam/src/utils/getPageMetadata.ts b/apps/roam/src/utils/getPageMetadata.ts index e44416321..f47c767b6 100644 --- a/apps/roam/src/utils/getPageMetadata.ts +++ b/apps/roam/src/utils/getPageMetadata.ts @@ -14,21 +14,25 @@ const getDisplayName = (s: string) => { const getPageMetadata = (title: string, cacheKey?: string) => { const results = window.roamAlphaAPI.q( - `[:find (pull ?p [:create/time :block/uid]) (pull ?cu [:user/uid]) :where [?p :node/title "${normalizePageTitle( + `[:find (pull ?p [:block/uid :create/time [:edit/time :as "modified"]]) (pull ?cu [:user/uid]) :where [?p :node/title "${normalizePageTitle( title, )}"] [?p :create/user ?cu]]`, - ) as [[{ time: number; uid: string }, { uid: string }]]; + ) as [[{ uid: string; time: number; modified: number }, { uid: string }]]; if (results.length) { - const [[{ time: createdTime, uid: id }, { uid }]] = results; + const [[{ uid: id, time: createdTime, modified: modifiedTime }, { uid }]] = + results; const displayName = getDisplayName(uid); const date = new Date(createdTime); - return { displayName, date, id }; + const modified = new Date(modifiedTime); + return { displayName, date, id, modified }; } + const date = new Date(); return { displayName: "Unknown", - date: new Date(), + date, id: "", + modified: date, }; }; diff --git a/apps/website/public/schema/dg_base.ttl b/apps/website/public/schema/dg_base.ttl new file mode 100644 index 000000000..8c9262112 --- /dev/null +++ b/apps/website/public/schema/dg_base.ttl @@ -0,0 +1,77 @@ +@prefix rdf: . +@prefix rdfs: . +@prefix : . +@prefix dc: . +@prefix owl: . +@prefix dgb: . + + + dc:date "2025-11-01" ; + dc:description "Discourse Graphs foundation vocabulary"@en ; + dc:title "Discourse Graphs foundation vocabulary"@en ; + owl:versionInfo "0 (tentative)" ; + a owl:Ontology. + +dgb:Node a owl:Class ; + dc:description "Discourse Graphs nodes"@en . + +dgb:NodeSchema +rdfs:subClassOf owl:Class , dgb:Node ; + dc:description "class of Discourse Graphs node schemata"@en . + +dgb:Role + rdfs:subClassOf owl:ObjectProperty, + [a owl:Restriction; owl:onProperty rdfs:range ; owl:allValuesFrom dgb:RoleSchema ], + [a owl:Restriction; owl:onProperty rdfs:domain ; owl:allValuesFrom dgb:NodeSchema ]; + dc:description "A role within a NodeSchema"@en . + +dgb:BinaryRelationSchema +rdfs:subClassOf dgb:NodeSchema, owl:ObjectProperty ; + dc:description "Discourse Graphs binary relation schema"@en . + +dgb:BinaryRelation +a rdf:Statement, dgb:Node . + +dgb:source +a dgb:Role ; + rdfs:subPropertyOf rdf:subject ; + rdfs:domain dgb:BinaryRelationSchema ; + rdfs:range dgb:Node ; + dc:description "The source of a binary relation"@en . + +dgb:destination +a dgb:Role ; + rdfs:subPropertyOf rdf:object ; + rdfs:domain dgb:BinaryRelationSchema ; + rdfs:range dgb:Node ; + dc:description "The destination of a binary relation"@en . + +dgb:reltype + rdfs:subPropertyOf rdf:predicate ; + rdfs:domain dgb:BinaryRelation ; + rdf:range dgb:BinaryRelationSchema . + + +# examples + +# :x a dgb:NodeSchema . +# :y a dgb:NodeSchema . +# :x0 a :x. +# :y0 a :y. +# :r a dgb:NodeSchema. +# :x_r a dgb:Role ; +# rdfs:domain :r ; +# rdfs:range :x . + +# :r0 a :r; +# :x_r :x0. + +# :br a dgb:BinaryRelationSchema. + +# :br0 +# a dgb:BinaryRelation; +# dgb:reltype :br . +# dgb:source :x0 ; +# dgb:destination :y0 ; + +# # This is "about" :x0 :br :y0; diff --git a/apps/website/public/schema/dg_core.ttl b/apps/website/public/schema/dg_core.ttl new file mode 100644 index 000000000..1b5af6c7a --- /dev/null +++ b/apps/website/public/schema/dg_core.ttl @@ -0,0 +1,68 @@ +@prefix rdf: . +@prefix rdfs: . +@prefix : . +@prefix dc: . +@prefix owl: . +@prefix vs: . +@prefix sioc: . +@prefix prov: . +@prefix dgb: . +@prefix dg: . + +dg:Question a dgb:NodeSchema; + rdfs:label "Question"@en. + +dg:Claim a dgb:NodeSchema; + rdfs:label "Claim"@en. + +dg:Result a dgb:NodeSchema; + rdfs:label "Result"@en. + +dg:Opposes a dgb:BinaryRelationSchema; + rdfs:label "Opposes"@en; + rdfs:range dg:Claim; + rdfs:domain dg:Result. + +# I might actually need to be more verbose than rdfs:range/domain... Still needs research. + +# owl:subClassOf [ +# { +# a owl:Restriction; +# owl:onProperty dgb:source; +# owl:allValuesFrom: dg:Claim +# }, +# { +# a owl:Restriction; +# owl:onProperty dgb:destination; +# owl:allValuesFrom: dg:Result +# }, +# ], + +dg:OpposedBy a dgb:BinaryRelationSchema; + rdfs:label "OpposedBy"@en; + owl:inverseOf dg:Opposes; + rdfs:range dg:Result; + rdfs:domain dg:Claim. + + +dg:Supports a dgb:BinaryRelationSchema; + rdfs:label "Supports"@en; + rdfs:range dg:Claim; + rdfs:domain dg:Result. + +dg:SupportedBy a dgb:BinaryRelationSchema; + rdfs:label "SupportedBy"@en; + owl:inverseOf dg:Supports; + rdfs:range dg:Result; + rdfs:domain dg:Claim. + +dg:Addresses a dgb:BinaryRelationSchema; + rdfs:label "Addresses"@en; + rdfs:range dg:Question; + rdfs:domain dg:Claim. + +dg:AddressedBy a dgb:BinaryRelationSchema; +rdfs:label "AddressedBy"@en; + owl:inverseOf dg:Addresses; + rdfs:range dg:Claim; + rdfs:domain dg:Question. From e96ac8e819846f91c886780a5ef39ee3d05f0ee1 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sat, 20 Dec 2025 18:39:26 -0500 Subject: [PATCH 2/3] refactor --- apps/roam/src/utils/exportUtils.ts | 153 ++++++ apps/roam/src/utils/getExportTypes.ts | 721 ++------------------------ apps/roam/src/utils/jsonld.ts | 185 +++++++ apps/roam/src/utils/pageToMardkown.ts | 350 +++++++++++++ 4 files changed, 739 insertions(+), 670 deletions(-) create mode 100644 apps/roam/src/utils/exportUtils.ts create mode 100644 apps/roam/src/utils/jsonld.ts create mode 100644 apps/roam/src/utils/pageToMardkown.ts diff --git a/apps/roam/src/utils/exportUtils.ts b/apps/roam/src/utils/exportUtils.ts new file mode 100644 index 000000000..d9b2eedad --- /dev/null +++ b/apps/roam/src/utils/exportUtils.ts @@ -0,0 +1,153 @@ +import type { Result } from "roamjs-components/types/query-builder"; +import { PullBlock, TreeNode, ViewType } from "roamjs-components/types"; +import type { DiscourseNode } from "./getDiscourseNodes"; +import matchDiscourseNode from "./matchDiscourseNode"; + +type DiscourseExportResult = Result & { type: string }; + +export const uniqJsonArray = >(arr: T[]) => + Array.from( + new Set( + arr.map((r) => + JSON.stringify( + Object.entries(r).sort(([k], [k2]) => k.localeCompare(k2)), + ), + ), + ), + ).map((entries) => Object.fromEntries(JSON.parse(entries))) as T[]; + +export const getPageData = async ({ + results, + allNodes, + isExportDiscourseGraph, +}: { + results: Result[]; + allNodes: DiscourseNode[]; + isExportDiscourseGraph?: boolean; +}): Promise<(Result & { type: string })[]> => { + const allResults = results || []; + + if (isExportDiscourseGraph) return allResults as DiscourseExportResult[]; + + const matchedTexts = new Set(); + return allNodes.flatMap((n) => + (allResults + ? allResults.flatMap((r) => + Object.keys(r) + .filter((k) => k.endsWith(`-uid`) && k !== "text-uid") + .map((k) => ({ + ...r, + text: r[k.slice(0, -4)].toString(), + uid: r[k] as string, + })) + .concat({ + text: r.text, + uid: r.uid, + }), + ) + : ( + window.roamAlphaAPI.q( + "[:find (pull ?e [:block/uid :node/title]) :where [?e :node/title _]]", + ) as [Record][] + ).map(([{ title, uid }]) => ({ + text: title, + uid, + })) + ) + .filter(({ text }) => { + if (!text) return false; + if (matchedTexts.has(text)) return false; + const isMatch = matchDiscourseNode({ title: text, ...n }); + if (isMatch) matchedTexts.add(text); + + return isMatch; + }) + .map((node) => ({ ...node, type: n.text })), + ); +}; + +const getContentFromNodes = ({ + title, + allNodes, +}: { + title: string; + allNodes: DiscourseNode[]; +}) => { + const nodeFormat = allNodes.find((a) => + matchDiscourseNode({ title, ...a }), + )?.format; + if (!nodeFormat) return title; + const regex = new RegExp( + `^${nodeFormat + .replace(/\[/g, "\\[") + .replace(/]/g, "\\]") + .replace("{content}", "(.*?)") + .replace(/{[^}]+}/g, "(?:.*?)")}$`, + ); + return regex.exec(title)?.[1] || title; +}; + +export const getFilename = ({ + title = "", + maxFilenameLength, + simplifiedFilename, + allNodes, + removeSpecialCharacters, + extension = ".md", +}: { + title?: string; + maxFilenameLength: number; + simplifiedFilename: boolean; + allNodes: DiscourseNode[]; + removeSpecialCharacters: boolean; + extension?: string; +}) => { + const baseName = simplifiedFilename + ? getContentFromNodes({ title, allNodes }) + : title; + const name = `${ + removeSpecialCharacters + ? baseName.replace(/[<>:"/\\|\?*[\]]/g, "") + : baseName + }${extension}`; + + return name.length > maxFilenameLength + ? `${name.substring( + 0, + Math.ceil((maxFilenameLength - 3) / 2), + )}...${name.slice(-Math.floor((maxFilenameLength - 3) / 2))}` + : name; +}; + +export const toLink = (filename: string, uid: string, linkType: string) => { + const extensionRemoved = filename.replace(/\.\w+$/, ""); + if (linkType === "wikilinks") return `[[${extensionRemoved}]]`; + if (linkType === "alias") return `[${filename}](${filename})`; + if (linkType === "roam url") + return `[${extensionRemoved}](https://roamresearch.com/#/app/${window.roamAlphaAPI.graph.name}/page/${uid})`; + return filename; +}; + +export const pullBlockToTreeNode = ( + n: PullBlock, + v: `:${ViewType}`, +): TreeNode => ({ + text: n[":block/string"] || n[":node/title"] || "", + open: typeof n[":block/open"] === "undefined" ? true : n[":block/open"], + order: n[":block/order"] || 0, + uid: n[":block/uid"] || "", + heading: n[":block/heading"] || 0, + viewType: (n[":children/view-type"] || v).slice(1) as ViewType, + editTime: new Date(n[":edit/time"] || 0), + props: { imageResize: {}, iframe: {} }, + textAlign: n[":block/text-align"] || "left", + children: (n[":block/children"] || []) + .sort(({ [":block/order"]: a = 0 }, { [":block/order"]: b = 0 }) => a - b) + .map((r) => pullBlockToTreeNode(r, n[":children/view-type"] || v)), + parents: (n[":block/parents"] || []).map((p) => p[":db/id"] || 0), +}); + +export const collectUids = (t: TreeNode): string[] => [ + t.uid, + ...t.children.flatMap(collectUids), +]; diff --git a/apps/roam/src/utils/getExportTypes.ts b/apps/roam/src/utils/getExportTypes.ts index 6a5ade560..5a47b03f2 100644 --- a/apps/roam/src/utils/getExportTypes.ts +++ b/apps/roam/src/utils/getExportTypes.ts @@ -1,28 +1,26 @@ -import { BLOCK_REF_REGEX } from "roamjs-components/dom/constants"; -import getTextByBlockUid from "roamjs-components/queries/getTextByBlockUid"; import normalizePageTitle from "roamjs-components/queries/normalizePageTitle"; import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; -import getPageViewType from "roamjs-components/queries/getPageViewType"; -import { PullBlock, TreeNode, ViewType } from "roamjs-components/types"; -import { Result } from "roamjs-components/types/query-builder"; -import getRoamUrl from "roamjs-components/dom/getRoamUrl"; -import XRegExp from "xregexp"; +import { PullBlock, TreeNode } from "roamjs-components/types"; +import type { Result } from "roamjs-components/types/query-builder"; import getDiscourseNodes from "./getDiscourseNodes"; -import type { DiscourseNode } from "./getDiscourseNodes"; import isFlagEnabled from "./isFlagEnabled"; -import matchDiscourseNode from "./matchDiscourseNode"; import getDiscourseRelations from "./getDiscourseRelations"; -import type { DiscourseRelation } from "./getDiscourseRelations"; import type { ExportDialogProps } from "~/components/Export"; import getPageMetadata from "./getPageMetadata"; import getDiscourseContextResults from "./getDiscourseContextResults"; import { getRelationDataUtil } from "./getRelationData"; import { ExportTypes } from "./types"; -import { - findReferencedNodeInText, - getReferencedNodeInFormat, -} from "./formatUtils"; import { getExportSettings } from "./getExportSettings"; +import { pageToMarkdown, toMarkdown } from "./pageToMardkown"; +import { getJsonLdData } from "./jsonld"; +import { + uniqJsonArray, + getPageData, + getFilename, + toLink, + pullBlockToTreeNode, + collectUids, +} from "./exportUtils"; export const updateExportProgress = (detail: { progress: number; @@ -34,295 +32,12 @@ export const updateExportProgress = (detail: { }), ); -const pullBlockToTreeNode = (n: PullBlock, v: `:${ViewType}`): TreeNode => ({ - text: n[":block/string"] || n[":node/title"] || "", - open: typeof n[":block/open"] === "undefined" ? true : n[":block/open"], - order: n[":block/order"] || 0, - uid: n[":block/uid"] || "", - heading: n[":block/heading"] || 0, - viewType: (n[":children/view-type"] || v).slice(1) as ViewType, - editTime: new Date(n[":edit/time"] || 0), - props: { imageResize: {}, iframe: {} }, - textAlign: n[":block/text-align"] || "left", - children: (n[":block/children"] || []) - .sort(({ [":block/order"]: a = 0 }, { [":block/order"]: b = 0 }) => a - b) - .map((r) => pullBlockToTreeNode(r, n[":children/view-type"] || v)), - parents: (n[":block/parents"] || []).map((p) => p[":db/id"] || 0), -}); - -const getContentFromNodes = ({ - title, - allNodes, -}: { - title: string; - allNodes: DiscourseNode[]; -}) => { - const nodeFormat = allNodes.find((a) => - matchDiscourseNode({ title, ...a }), - )?.format; - if (!nodeFormat) return title; - const regex = new RegExp( - `^${nodeFormat - .replace(/\[/g, "\\[") - .replace(/]/g, "\\]") - .replace("{content}", "(.*?)") - .replace(/{[^}]+}/g, "(?:.*?)")}$`, - ); - return regex.exec(title)?.[1] || title; -}; - -const getFilename = ({ - title = "", - maxFilenameLength, - simplifiedFilename, - allNodes, - removeSpecialCharacters, - extension = ".md", -}: { - title?: string; - maxFilenameLength: number; - simplifiedFilename: boolean; - allNodes: DiscourseNode[]; - removeSpecialCharacters: boolean; - extension?: string; -}) => { - const baseName = simplifiedFilename - ? getContentFromNodes({ title, allNodes }) - : title; - const name = `${ - removeSpecialCharacters - ? baseName.replace(/[<>:"/\\|\?*[\]]/g, "") - : baseName - }${extension}`; - - return name.length > maxFilenameLength - ? `${name.substring( - 0, - Math.ceil((maxFilenameLength - 3) / 2), - )}...${name.slice(-Math.floor((maxFilenameLength - 3) / 2))}` - : name; -}; - -const uniqJsonArray = >(arr: T[]) => - Array.from( - new Set( - arr.map((r) => - JSON.stringify( - Object.entries(r).sort(([k], [k2]) => k.localeCompare(k2)), - ), - ), - ), - ).map((entries) => Object.fromEntries(JSON.parse(entries))) as T[]; -const viewTypeToPrefix = { - bullet: "- ", - document: "", - numbered: "1. ", -}; - -const collectUids = (t: TreeNode): string[] => [ - t.uid, - ...t.children.flatMap(collectUids), -]; - -const MATCHES_NONE = /$.+^/; - -// Roam embed syntax: {{[[embed]]: ((block-uid)) }} -// Roam embed syntax: {{[[embed-path]]: ((block-uid)) }} -// Also handles multiple parentheses: {{[[embed]]: ((((block-uid)))) }} -const EMBED_REGEX = - /{{\[\[(?:embed|embed-path)\]\]:\s*\(\(+\s*([\w\d-]{9,10})\s*\)\)+\s*}}/; - -// Roam embed-children syntax: {{[[embed-children]]: ((block-uid)) }} -const EMBED_CHILDREN_REGEX = - /{{\[\[embed-children\]\]:\s*\(\(+\s*([\w\d-]{9,10})\s*\)\)+\s*}}/; - -const toLink = (filename: string, uid: string, linkType: string) => { - const extensionRemoved = filename.replace(/\.\w+$/, ""); - if (linkType === "wikilinks") return `[[${extensionRemoved}]]`; - if (linkType === "alias") return `[${filename}](${filename})`; - if (linkType === "roam url") - return `[${extensionRemoved}](https://roamresearch.com/#/app/${window.roamAlphaAPI.graph.name}/page/${uid})`; - return filename; -}; - -const toMarkdown = ({ - c, - i = 0, - v = "bullet", - opts, -}: { - c: TreeNode; - i?: number; - v?: ViewType; - opts: { - refs: boolean; - embeds: boolean; - simplifiedFilename: boolean; - maxFilenameLength: number; - allNodes: DiscourseNode[]; - removeSpecialCharacters: boolean; - linkType: string; - flatten?: boolean; - }; -}): string => { - const { - refs, - embeds, - simplifiedFilename, - maxFilenameLength, - allNodes, - removeSpecialCharacters, - linkType, - flatten = false, - } = opts; - const processedText = c.text - .replace(embeds ? EMBED_REGEX : MATCHES_NONE, (_, blockUid) => { - const reference = getFullTreeByParentUid(blockUid); - return toMarkdown({ c: reference, i, v, opts }); - }) - .replace(embeds ? EMBED_CHILDREN_REGEX : MATCHES_NONE, (_, blockUid) => { - const reference = getFullTreeByParentUid(blockUid); - return reference.children - .map((child) => toMarkdown({ c: child, i, v, opts })) - .join("\n"); - }) - .replace(refs ? BLOCK_REF_REGEX : MATCHES_NONE, (_, blockUid) => { - const reference = getTextByBlockUid(blockUid); - return reference || blockUid; - }) - .replace(/{{\[\[TODO\]\]}}/g, v === "bullet" ? "[ ]" : "- [ ]") - .replace(/{{\[\[DONE\]\]}}/g, v === "bullet" ? "[x]" : "- [x]") - .replace(/\_\_(.+?)\_\_/g, "_$1_") // convert Roam italics __ to markdown italics _ - .replace(/(? { - if (s.name === "match") { - const name = getFilename({ - title: s.value, - allNodes, - maxFilenameLength, - simplifiedFilename, - removeSpecialCharacters, - }); - return toLink(name, c.uid, linkType); - } else if (s.name === "left" || s.name === "right") { - return ""; - } else { - return s.value; - } - }) - .join("") || processedText - : processedText; - const indentation = flatten ? "" : "".padStart(i * 4, " "); - // If this block contains an embed, treat it as document to avoid extra prefixes - const effectiveViewType = - embeds && (EMBED_REGEX.test(c.text) || EMBED_CHILDREN_REGEX.test(c.text)) - ? "document" - : v; - const viewTypePrefix = viewTypeToPrefix[effectiveViewType]; - const headingPrefix = c.heading ? `${"".padStart(c.heading, "#")} ` : ""; - const childrenMarkdown = (c.children || []) - .filter((nested) => !!nested.text || !!nested.children?.length) - .map((nested) => { - const childViewType = v !== "bullet" ? v : nested.viewType || "bullet"; - const childMarkdown = toMarkdown({ - c: nested, - i: i + 1, - v: childViewType, - opts, - }); - return `\n${childMarkdown}`; - }) - .join(""); - const lineBreak = v === "document" ? "\n" : ""; - - return `${indentation}${viewTypePrefix}${headingPrefix}${finalProcessedText}${lineBreak}${childrenMarkdown}`; -}; - -const handleDiscourseContext = async ({ - includeDiscourseContext, - uid, - pageTitle, - appendRefNodeContext, -}: { - includeDiscourseContext: boolean; - uid: string; - pageTitle: string; - appendRefNodeContext: boolean; -}) => { - if (!includeDiscourseContext) return []; - - const discourseResults = await getDiscourseContextResults({ - uid, - }); - if (!appendRefNodeContext) return discourseResults; - - const referencedDiscourseNode = getReferencedNodeInFormat({ uid }); - if (referencedDiscourseNode) { - const referencedResult = findReferencedNodeInText({ - text: pageTitle, - discourseNode: referencedDiscourseNode, - }); - if (!referencedResult) return discourseResults; - const appendedContext = { - label: referencedDiscourseNode.text, - results: { [referencedResult.uid]: referencedResult }, - }; - return [...discourseResults, appendedContext]; - } - - return discourseResults; -}; - -const handleFrontmatter = ({ - frontmatter, - rest, - result, -}: { - frontmatter: string[]; - rest: Record; - result: Result; -}) => { - const yaml = frontmatter.length - ? frontmatter - : [ - "title: {text}", - `url: https://roamresearch.com/#/app/${window.roamAlphaAPI.graph.name}/page/{uid}`, - `author: {author}`, - "date: {date}", - ]; - const resultCols = Object.keys(rest).filter((k) => !k.includes("uid")); - const yamlLines = yaml.concat(resultCols.map((k) => `${k}: {${k}}`)); - const content = yamlLines - .map((s) => - s.replace(/{([^}]+)}/g, (_, capt: string) => { - if (capt === "text") { - // Wrap title in quotes and escape additional quotes - const escapedText = result[capt].toString().replace(/"/g, '\\"'); - return `"${escapedText}"`; - } - return result[capt].toString(); - }), - ) - .join("\n"); - const output = `---\n${content}\n---`; - return output; -}; - type getExportTypesProps = { results?: ExportDialogProps["results"]; exportId: string; isExportDiscourseGraph: boolean; }; -export type DiscourseExportResult = Result & { type: string }; - const getExportTypes = ({ results, exportId, @@ -334,69 +49,29 @@ const getExportTypes = ({ allNodes.map((a) => [a.type, a.text]), ); nodeLabelByType["*"] = "Any"; - const getPageData = async ( - isExportDiscourseGraph?: boolean, - ): Promise<(Result & { type: string })[]> => { - const allResults = results || []; - if (isExportDiscourseGraph) return allResults as DiscourseExportResult[]; - - const matchedTexts = new Set(); - return allNodes.flatMap((n) => - (allResults - ? allResults.flatMap((r) => - Object.keys(r) - .filter((k) => k.endsWith(`-uid`) && k !== "text-uid") - .map((k) => ({ - ...r, - text: r[k.slice(0, -4)].toString(), - uid: r[k] as string, - })) - .concat({ - text: r.text, - uid: r.uid, - }), - ) - : ( - window.roamAlphaAPI.q( - "[:find (pull ?e [:block/uid :node/title]) :where [?e :node/title _]]", - ) as [Record][] - ).map(([{ title, uid }]) => ({ - text: title, - uid, - })) - ) - .filter(({ text }) => { - if (!text) return false; - if (matchedTexts.has(text)) return false; - const isMatch = matchDiscourseNode({ title: text, ...n }); - if (isMatch) matchedTexts.add(text); - - return isMatch; - }) - .map((node) => ({ ...node, type: n.text })), - ); - }; const getRelationData = () => getRelationDataUtil(allRelations, nodeLabelByType); - const getJsonData = async () => { + const getJsonData = async (results: Result[]) => { const grammar = allRelations.map(({ label, destination, source }) => ({ label, destination: nodeLabelByType[destination], source: nodeLabelByType[source], })); - const nodes = (await getPageData()).map(({ text, uid }) => { - const { date, displayName } = getPageMetadata(text); - const { children } = getFullTreeByParentUid(uid); - return { - uid, - title: text, - children, - date: date.toJSON(), - createdBy: displayName, - }; - }); + const nodes = (await getPageData({ results, allNodes })).map( + ({ text, uid }) => { + const { date, displayName } = getPageMetadata(text); + const { children } = getFullTreeByParentUid(uid); + return { + uid, + title: text, + children, + date: date.toJSON(), + createdBy: displayName, + }; + }, + ); const nodeSet = new Set(nodes.map((n) => n.uid)); return getRelationData().then((rels) => { const relations = uniqJsonArray( @@ -406,324 +81,11 @@ const getExportTypes = ({ }); }; - const pageToMarkdown = async ( - { text, uid, context: _, type, ...rest }: Result, - { - includeDiscourseContext, - appendRefNodeContext, - frontmatter, - optsRefs, - optsEmbeds, - simplifiedFilename, - allNodes, - maxFilenameLength, - removeSpecialCharacters, - linkType, - }: { - includeDiscourseContext: boolean; - appendRefNodeContext: boolean; - frontmatter: string[]; - optsRefs: boolean; - optsEmbeds: boolean; - simplifiedFilename: boolean; - allNodes: DiscourseNode[]; - maxFilenameLength: number; - removeSpecialCharacters: boolean; - linkType: string; - }, - ): Promise<{ title: string; content: string; uids: Set }> => { - const v = getPageViewType(text) || "bullet"; - const { date, displayName } = getPageMetadata(text); - const treeNode = getFullTreeByParentUid(uid); - - const discourseResults = await handleDiscourseContext({ - includeDiscourseContext, - pageTitle: text, - uid, - appendRefNodeContext, - }); - - const referenceResults = isFlagEnabled("render references") - ? ( - window.roamAlphaAPI.data.fast.q( - `[:find (pull ?pr [:node/title]) (pull ?r [:block/heading [:block/string :as "text"] [:children/view-type :as "viewType"] {:block/children ...}]) :where [?p :node/title "${normalizePageTitle( - text, - )}"] [?r :block/refs ?p] [?r :block/page ?pr]]`, - ) as [PullBlock, PullBlock][] - ).filter( - ([, { [":block/children"]: children }]) => - Array.isArray(children) && children.length, - ) - : []; - - const result: Result = { - ...rest, - date, - text, - uid, - author: displayName, - type, - }; - const yamlLines = handleFrontmatter({ - frontmatter, - rest, - result, - }); - - const content = `${yamlLines}\n\n${treeNode.children - .map((c) => - toMarkdown({ - c, - v, - i: 0, - opts: { - refs: optsRefs, - embeds: optsEmbeds, - simplifiedFilename, - allNodes, - maxFilenameLength, - removeSpecialCharacters, - linkType, - }, - }), - ) - .join("\n")}\n${ - discourseResults.length - ? `\n###### Discourse Context\n\n${discourseResults - .flatMap((r) => - Object.values(r.results).map( - (t) => - `- **${r.label}::** ${toLink( - getFilename({ - title: t.text, - maxFilenameLength, - simplifiedFilename, - allNodes, - removeSpecialCharacters, - }), - t.uid, - linkType, - )}`, - ), - ) - .join("\n")}\n` - : "" - }${ - referenceResults.length - ? `\n###### References\n\n${referenceResults - .map( - (r_1) => - `${toLink( - getFilename({ - title: r_1[0][":node/title"], - maxFilenameLength, - simplifiedFilename, - allNodes, - removeSpecialCharacters, - }), - r_1[0][":block/uid"] || "", - linkType, - )}\n\n${toMarkdown({ - c: pullBlockToTreeNode(r_1[1], ":bullet"), - opts: { - refs: optsRefs, - embeds: optsEmbeds, - simplifiedFilename, - allNodes, - maxFilenameLength, - removeSpecialCharacters, - linkType, - }, - })}`, - ) - .join("\n")}\n` - : "" - }`; - const uids = new Set(collectUids(treeNode)); - return { title: text, content, uids }; - }; - - const getJsonLdData = async () => { - const roamUrl = getRoamUrl(); - updateExportProgress({ - progress: 0, - id: exportId, - }); - // skip a beat to let progress render - await new Promise((resolve) => setTimeout(resolve)); - const pageData = await getPageData(); - const numPages = pageData.length + allNodes.length; - let numTreatedPages = 0; - const context = { - rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", - rdfs: "http://www.w3.org/2000/01/rdf-schema#", - owl: "http://www.w3.org/2002/07/owl#", - dc: "http://purl.org/dc/elements/1.1/", - prov: "http://www.w3.org/ns/prov#", - sioc: "http://rdfs.org/sioc/ns#", - dg: "https://discoursegraphs.com/schema/dg_core#", - subClassOf: "rdfs:subClassOf", - title: "dc:title", - modified: "dc:modified", - created: "dc:date", - creator: "dc:creator", - content: "sioc:content", - source: "dg:source", - destination: "dg:destination", - reltype: "dg:reltype", - pages: `${roamUrl}/page/`, - }; - const settings = { - ...getExportSettings(), - includeDiscourseContext: false, - }; - // TODO : Identify existing CURIES in the node definition - /* eslint-disable @typescript-eslint/naming-convention */ - const nodeSchemaData = await Promise.all( - allNodes.map(async (node: DiscourseNode) => { - const { date, displayName, modified } = getPageMetadata(node.text); - const r = await pageToMarkdown( - { - text: node.text, - uid: node.type, - }, - { ...settings, allNodes }, - ); - numTreatedPages += 1; - updateExportProgress({ - progress: 0.1 + (numTreatedPages / numPages) * 0.75, - id: exportId, - }); - await new Promise((resolve) => setTimeout(resolve)); - return { - "@id": `pages:${node.type}`, - "@type": "dg:NodeSchema", - title: node.text, - content: r.content, - modified: modified?.toJSON(), - created: date.toJSON(), - creator: displayName, - }; - }), - ); - const relSchemaData = allRelations.map((r: DiscourseRelation) => ({ - "@id": `pages:${r.id}`, - "@type": "dg:BinaryRelationSchema", - subClassOf: [ - { - "@type": "owl:Restriction", - "owl:onProperty": "dg:source", - "owl:allValuesFrom": `pages:${r.source}`, - }, - { - "@type": "owl:Restriction", - "owl:onProperty": "dg:destination", - "owl:allValuesFrom": `pages:${r.destination}`, - }, - ], - title: r.label, - })); - const schemaData = [...nodeSchemaData, ...relSchemaData]; - - const schemaUriByName = Object.fromEntries( - schemaData.map((node) => [node.title, node["@id"]]), - ); - - const inverseRelSchemaData = allRelations.map((r: DiscourseRelation) => ({ - "@id": `pages:${r.id}-inverse`, - "@type": "dg:BinaryRelationSchema", - subClassOf: [ - { - "@type": "owl:Restriction", - "owl:onProperty": "dg:source", - "owl:allValuesFrom": `pages:${r.destination}`, - }, - { - "@type": "owl:Restriction", - "owl:onProperty": "dg:destination", - "owl:allValuesFrom": `pages:${r.source}`, - }, - ], - title: r.complement, - "owl:inverse-of": `pages:${r.id}`, - })); - - await Promise.all( - pageData.map(async (page: Result) => { - const r = await pageToMarkdown(page, { - ...settings, - allNodes, - }); - page.content = r.content; - numTreatedPages += 1; - updateExportProgress({ - progress: 0.1 + (numTreatedPages / numPages) * 0.75, - id: exportId, - }); - await new Promise((resolve) => setTimeout(resolve)); - }), - ); - - // skip a beat to let progress render - await new Promise((resolve) => setTimeout(resolve)); - await Promise.all( - pageData.map(async (page: Result) => { - const r = await pageToMarkdown(page, { - ...settings, - allNodes, - }); - page.content = r.content; - numTreatedPages += 1; - updateExportProgress({ - progress: 0.1 + (numTreatedPages / numPages) * 0.75, - id: exportId, - }); - await new Promise((resolve) => setTimeout(resolve)); - }), - ); - - const nodes = pageData.map(({ text, uid, content, type }) => { - const { date, displayName, modified } = getPageMetadata(text); - return { - "@id": `pages:${uid}`, - "@type": schemaUriByName[type], - title: text, - content, - modified: modified?.toJSON(), - created: date.toJSON(), - creator: displayName, - }; - }); - const nodeSet = new Set(pageData.map((n) => n.uid)); - const rels = await getRelationData(); - updateExportProgress({ - progress: 1, - id: exportId, - }); - await new Promise((resolve) => setTimeout(resolve)); - const relations = uniqJsonArray( - rels.filter((r) => nodeSet.has(r.source) && nodeSet.has(r.target)), - ); - const relData = relations.map(({ source, target, label }) => ({ - // no id yet, just a blank node - "@type": "dg:BinaryRelation", - reltype: schemaUriByName[label], - source: `pages:${source}`, - destination: `pages:${target}`, - })); - return { - "@context": context, - "@id": roamUrl, - "prov:generatedAtTime": new Date().toISOString(), - "@graph": [...schemaData, ...inverseRelSchemaData, ...nodes, ...relData], - }; - /* eslint-enable @typescript-eslint/naming-convention */ - }; - return [ { name: "Markdown", callback: async ({ includeDiscourseContext = false }) => { + if (!results) return []; const settings = { ...getExportSettings(), includeDiscourseContext, @@ -733,7 +95,11 @@ const getExportTypes = ({ maxFilenameLength, removeSpecialCharacters, } = settings; - const allPages = await getPageData(isExportDiscourseGraph); + const allPages = await getPageData({ + results, + allNodes, + isExportDiscourseGraph, + }); const gatherings = allPages.map((result, i, all) => async () => { updateExportProgress({ progress: i / all.length, id: exportId }); // skip a beat to let progress render @@ -771,7 +137,8 @@ const getExportTypes = ({ { name: "JSON", callback: async ({ filename }) => { - const data = await getJsonData(); + if (!results) return []; + const data = await getJsonData(results); return [ { title: `${filename.replace(/\.json$/, "")}.json`, @@ -783,7 +150,15 @@ const getExportTypes = ({ { name: "JSON-LD (Experimental)", callback: async ({ filename }) => { - const data = await getJsonLdData(); + if (!results) return []; + const data = await getJsonLdData({ + results, + allNodes, + allRelations, + nodeLabelByType, + updateExportProgress: (progress: number) => + updateExportProgress({ progress, id: exportId }), + }); return [ { title: `${filename.replace(/\.json$/, "")}.json`, @@ -795,8 +170,9 @@ const getExportTypes = ({ { name: "Neo4j", callback: async ({ filename }) => { + if (!results) return []; const nodeHeader = "uid:ID,label:LABEL,title,author,date\n"; - const nodeData = (await getPageData()) + const nodeData = (await getPageData({ results, allNodes })) .map(({ text, uid, type }) => { const value = text.replace(new RegExp(`^\\[\\[\\w*\\]\\] - `), ""); const { displayName, date } = getPageMetadata(text); @@ -849,6 +225,7 @@ const getExportTypes = ({ { name: "PDF", callback: async ({ includeDiscourseContext = false }) => { + if (!results) return []; const { optsRefs, optsEmbeds, @@ -857,7 +234,11 @@ const getExportTypes = ({ removeSpecialCharacters, linkType, } = getExportSettings(); - const allPages = await getPageData(isExportDiscourseGraph); + const allPages = await getPageData({ + results, + allNodes, + isExportDiscourseGraph, + }); const gatherings = allPages.map(({ text, uid }, i, all) => async () => { updateExportProgress({ progress: i / all.length, id: exportId }); // skip a beat to let progress render diff --git a/apps/roam/src/utils/jsonld.ts b/apps/roam/src/utils/jsonld.ts new file mode 100644 index 000000000..eb5a991ee --- /dev/null +++ b/apps/roam/src/utils/jsonld.ts @@ -0,0 +1,185 @@ +import getRoamUrl from "roamjs-components/dom/getRoamUrl"; +import type { Result } from "roamjs-components/types/query-builder"; +import type { DiscourseRelation } from "./getDiscourseRelations"; +import type { DiscourseNode } from "./getDiscourseNodes"; +import getPageMetadata from "./getPageMetadata"; +import { pageToMarkdown } from "./pageToMardkown"; +import { getRelationDataUtil } from "./getRelationData"; +import { uniqJsonArray, getPageData } from "./exportUtils"; +import { getExportSettings } from "./getExportSettings"; + +export const getJsonLdData = async ({ + results, + allNodes, + allRelations, + nodeLabelByType, + updateExportProgress, +}: { + results: Result[]; + allNodes: DiscourseNode[]; + allRelations: DiscourseRelation[]; + nodeLabelByType: Record; + updateExportProgress: (progress: number) => void; +}) => { + const roamUrl = getRoamUrl(); + const getRelationData = () => + getRelationDataUtil(allRelations, nodeLabelByType); + updateExportProgress(0); + // skip a beat to let progress render + await new Promise((resolve) => setTimeout(resolve)); + const pageData = await getPageData({ results, allNodes }); + const numPages = pageData.length + allNodes.length; + let numTreatedPages = 0; + const context = { + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + rdfs: "http://www.w3.org/2000/01/rdf-schema#", + owl: "http://www.w3.org/2002/07/owl#", + dc: "http://purl.org/dc/elements/1.1/", + prov: "http://www.w3.org/ns/prov#", + sioc: "http://rdfs.org/sioc/ns#", + dg: "https://discoursegraphs.com/schema/dg_core#", + subClassOf: "rdfs:subClassOf", + title: "dc:title", + modified: "dc:modified", + created: "dc:date", + creator: "dc:creator", + content: "sioc:content", + source: "dg:source", + destination: "dg:destination", + reltype: "dg:reltype", + pages: `${roamUrl}/page/`, + }; + const settings = { + ...getExportSettings(), + includeDiscourseContext: false, + }; + // TODO : Identify existing CURIES in the node definition + /* eslint-disable @typescript-eslint/naming-convention */ + const nodeSchemaData = await Promise.all( + allNodes.map(async (node: DiscourseNode) => { + const { date, displayName, modified } = getPageMetadata(node.text); + const r = await pageToMarkdown( + { + text: node.text, + uid: node.type, + }, + { ...settings, allNodes }, + ); + numTreatedPages += 1; + updateExportProgress(0.1 + (numTreatedPages / numPages) * 0.75); + await new Promise((resolve) => setTimeout(resolve)); + return { + "@id": `pages:${node.type}`, + "@type": "dg:NodeSchema", + title: node.text, + content: r.content, + modified: modified?.toJSON(), + created: date.toJSON(), + creator: displayName, + }; + }), + ); + const relSchemaData = allRelations.map((r: DiscourseRelation) => ({ + "@id": `pages:${r.id}`, + "@type": "dg:BinaryRelationSchema", + subClassOf: [ + { + "@type": "owl:Restriction", + "owl:onProperty": "dg:source", + "owl:allValuesFrom": `pages:${r.source}`, + }, + { + "@type": "owl:Restriction", + "owl:onProperty": "dg:destination", + "owl:allValuesFrom": `pages:${r.destination}`, + }, + ], + title: r.label, + })); + const schemaData = [...nodeSchemaData, ...relSchemaData]; + + const schemaUriByName = Object.fromEntries( + schemaData.map((node) => [node.title, node["@id"]]), + ); + + const inverseRelSchemaData = allRelations.map((r: DiscourseRelation) => ({ + "@id": `pages:${r.id}-inverse`, + "@type": "dg:BinaryRelationSchema", + subClassOf: [ + { + "@type": "owl:Restriction", + "owl:onProperty": "dg:source", + "owl:allValuesFrom": `pages:${r.destination}`, + }, + { + "@type": "owl:Restriction", + "owl:onProperty": "dg:destination", + "owl:allValuesFrom": `pages:${r.source}`, + }, + ], + title: r.complement, + "owl:inverse-of": `pages:${r.id}`, + })); + + await Promise.all( + pageData.map(async (page: Result) => { + const r = await pageToMarkdown(page, { + ...settings, + allNodes, + }); + page.content = r.content; + numTreatedPages += 1; + updateExportProgress(0.1 + (numTreatedPages / numPages) * 0.75); + await new Promise((resolve) => setTimeout(resolve)); + }), + ); + + // skip a beat to let progress render + await new Promise((resolve) => setTimeout(resolve)); + await Promise.all( + pageData.map(async (page: Result) => { + const r = await pageToMarkdown(page, { + ...settings, + allNodes, + }); + page.content = r.content; + numTreatedPages += 1; + updateExportProgress(0.1 + (numTreatedPages / numPages) * 0.75); + await new Promise((resolve) => setTimeout(resolve)); + }), + ); + + const nodes = pageData.map(({ text, uid, content, type }) => { + const { date, displayName, modified } = getPageMetadata(text); + return { + "@id": `pages:${uid}`, + "@type": schemaUriByName[type], + title: text, + content, + modified: modified?.toJSON(), + created: date.toJSON(), + creator: displayName, + }; + }); + const nodeSet = new Set(pageData.map((n) => n.uid)); + const rels = await getRelationData(); + updateExportProgress(1); + await new Promise((resolve) => setTimeout(resolve)); + const relations = uniqJsonArray( + rels.filter((r) => nodeSet.has(r.source) && nodeSet.has(r.target)), + ); + const relData = relations.map(({ source, target, label }) => ({ + // no id yet, just a blank node + "@type": "dg:BinaryRelation", + reltype: schemaUriByName[label], + source: `pages:${source}`, + destination: `pages:${target}`, + })); + return { + "@context": context, + "@id": roamUrl, + "prov:generatedAtTime": new Date().toISOString(), + "@graph": [...schemaData, ...inverseRelSchemaData, ...nodes, ...relData], + }; + /* eslint-enable @typescript-eslint/naming-convention */ +}; diff --git a/apps/roam/src/utils/pageToMardkown.ts b/apps/roam/src/utils/pageToMardkown.ts new file mode 100644 index 000000000..1b41f66f6 --- /dev/null +++ b/apps/roam/src/utils/pageToMardkown.ts @@ -0,0 +1,350 @@ +import { BLOCK_REF_REGEX } from "roamjs-components/dom/constants"; +import normalizePageTitle from "roamjs-components/queries/normalizePageTitle"; +import getTextByBlockUid from "roamjs-components/queries/getTextByBlockUid"; +import getPageMetadata from "./getPageMetadata"; +import getPageViewType from "roamjs-components/queries/getPageViewType"; +import type { Result } from "roamjs-components/types/query-builder"; +import type { DiscourseNode } from "./getDiscourseNodes"; +import type { PullBlock, TreeNode, ViewType } from "roamjs-components/types"; +import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; +import getDiscourseContextResults from "./getDiscourseContextResults"; +import isFlagEnabled from "./isFlagEnabled"; +import XRegExp from "xregexp"; +import { + findReferencedNodeInText, + getReferencedNodeInFormat, +} from "./formatUtils"; +import { + getFilename, + toLink, + pullBlockToTreeNode, + collectUids, +} from "./exportUtils"; + +const MATCHES_NONE = /$.+^/; + +// Roam embed syntax: {{[[embed]]: ((block-uid)) }} +// Roam embed syntax: {{[[embed-path]]: ((block-uid)) }} +// Also handles multiple parentheses: {{[[embed]]: ((((block-uid)))) }} +const EMBED_REGEX = + /{{\[\[(?:embed|embed-path)\]\]:\s*\(\(+\s*([\w\d-]{9,10})\s*\)\)+\s*}}/; + +// Roam embed-children syntax: {{[[embed-children]]: ((block-uid)) }} +const EMBED_CHILDREN_REGEX = + /{{\[\[embed-children\]\]:\s*\(\(+\s*([\w\d-]{9,10})\s*\)\)+\s*}}/; + +const viewTypeToPrefix = { + bullet: "- ", + document: "", + numbered: "1. ", +}; + +const handleDiscourseContext = async ({ + includeDiscourseContext, + uid, + pageTitle, + appendRefNodeContext, +}: { + includeDiscourseContext: boolean; + uid: string; + pageTitle: string; + appendRefNodeContext: boolean; +}) => { + if (!includeDiscourseContext) return []; + + const discourseResults = await getDiscourseContextResults({ + uid, + }); + if (!appendRefNodeContext) return discourseResults; + + const referencedDiscourseNode = getReferencedNodeInFormat({ uid }); + if (referencedDiscourseNode) { + const referencedResult = findReferencedNodeInText({ + text: pageTitle, + discourseNode: referencedDiscourseNode, + }); + if (!referencedResult) return discourseResults; + const appendedContext = { + label: referencedDiscourseNode.text, + results: { [referencedResult.uid]: referencedResult }, + }; + return [...discourseResults, appendedContext]; + } + + return discourseResults; +}; + +const handleFrontmatter = ({ + frontmatter, + rest, + result, +}: { + frontmatter: string[]; + rest: Record; + result: Result; +}) => { + const yaml = frontmatter.length + ? frontmatter + : [ + "title: {text}", + `url: https://roamresearch.com/#/app/${window.roamAlphaAPI.graph.name}/page/{uid}`, + `author: {author}`, + "date: {date}", + ]; + const resultCols = Object.keys(rest).filter((k) => !k.includes("uid")); + const yamlLines = yaml.concat(resultCols.map((k) => `${k}: {${k}}`)); + const content = yamlLines + .map((s) => + s.replace(/{([^}]+)}/g, (_, capt: string) => { + if (capt === "text") { + // Wrap title in quotes and escape additional quotes + const escapedText = result[capt].toString().replace(/"/g, '\\"'); + return `"${escapedText}"`; + } + return result[capt].toString(); + }), + ) + .join("\n"); + const output = `---\n${content}\n---`; + return output; +}; + +export const toMarkdown = ({ + c, + i = 0, + v = "bullet", + opts, +}: { + c: TreeNode; + i?: number; + v?: ViewType; + opts: { + refs: boolean; + embeds: boolean; + simplifiedFilename: boolean; + maxFilenameLength: number; + allNodes: DiscourseNode[]; + removeSpecialCharacters: boolean; + linkType: string; + flatten?: boolean; + }; +}): string => { + const { + refs, + embeds, + simplifiedFilename, + maxFilenameLength, + allNodes, + removeSpecialCharacters, + linkType, + flatten = false, + } = opts; + const processedText = c.text + .replace(embeds ? EMBED_REGEX : MATCHES_NONE, (_, blockUid: string) => { + const reference = getFullTreeByParentUid(blockUid); + return toMarkdown({ c: reference, i, v, opts }); + }) + .replace( + embeds ? EMBED_CHILDREN_REGEX : MATCHES_NONE, + (_, blockUid: string) => { + const reference = getFullTreeByParentUid(blockUid); + return reference.children + .map((child) => toMarkdown({ c: child, i, v, opts })) + .join("\n"); + }, + ) + .replace(refs ? BLOCK_REF_REGEX : MATCHES_NONE, (_, blockUid: string) => { + const reference = getTextByBlockUid(blockUid); + return reference || blockUid; + }) + .replace(/{{\[\[TODO\]\]}}/g, v === "bullet" ? "[ ]" : "- [ ]") + .replace(/{{\[\[DONE\]\]}}/g, v === "bullet" ? "[x]" : "- [x]") + .replace(/\_\_(.+?)\_\_/g, "_$1_") // convert Roam italics __ to markdown italics _ + .replace(/(? { + if (s.name === "match") { + const name = getFilename({ + title: s.value, + allNodes, + maxFilenameLength, + simplifiedFilename, + removeSpecialCharacters, + }); + return toLink(name, c.uid, linkType); + } else if (s.name === "left" || s.name === "right") { + return ""; + } else { + return s.value; + } + }) + .join("") || processedText + : processedText; + const indentation = flatten ? "" : "".padStart(i * 4, " "); + // If this block contains an embed, treat it as document to avoid extra prefixes + const effectiveViewType = + embeds && (EMBED_REGEX.test(c.text) || EMBED_CHILDREN_REGEX.test(c.text)) + ? "document" + : v; + const viewTypePrefix = viewTypeToPrefix[effectiveViewType]; + const headingPrefix = c.heading ? `${"".padStart(c.heading, "#")} ` : ""; + const childrenMarkdown = (c.children || []) + .filter((nested) => !!nested.text || !!nested.children?.length) + .map((nested) => { + const childViewType = v !== "bullet" ? v : nested.viewType || "bullet"; + const childMarkdown = toMarkdown({ + c: nested, + i: i + 1, + v: childViewType, + opts, + }); + return `\n${childMarkdown}`; + }) + .join(""); + const lineBreak = v === "document" ? "\n" : ""; + + return `${indentation}${viewTypePrefix}${headingPrefix}${finalProcessedText}${lineBreak}${childrenMarkdown}`; +}; + +export const pageToMarkdown = async ( + { text, uid, context: _, type, ...rest }: Result, + { + includeDiscourseContext, + appendRefNodeContext, + frontmatter, + optsRefs, + optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }: { + includeDiscourseContext: boolean; + appendRefNodeContext: boolean; + frontmatter: string[]; + optsRefs: boolean; + optsEmbeds: boolean; + simplifiedFilename: boolean; + allNodes: DiscourseNode[]; + maxFilenameLength: number; + removeSpecialCharacters: boolean; + linkType: string; + }, +): Promise<{ title: string; content: string; uids: Set }> => { + const v = getPageViewType(text) || "bullet"; + const { date, displayName } = getPageMetadata(text); + const treeNode = getFullTreeByParentUid(uid); + + const discourseResults = await handleDiscourseContext({ + includeDiscourseContext, + pageTitle: text, + uid, + appendRefNodeContext, + }); + + const referenceResults = isFlagEnabled("render references") + ? ( + window.roamAlphaAPI.data.fast.q( + `[:find (pull ?pr [:node/title]) (pull ?r [:block/heading [:block/string :as "text"] [:children/view-type :as "viewType"] {:block/children ...}]) :where [?p :node/title "${normalizePageTitle( + text, + )}"] [?r :block/refs ?p] [?r :block/page ?pr]]`, + ) as [PullBlock, PullBlock][] + ).filter( + ([, { [":block/children"]: children }]) => + Array.isArray(children) && children.length, + ) + : []; + + const result: Result = { + ...rest, + date, + text, + uid, + author: displayName, + type, + }; + const yamlLines = handleFrontmatter({ + frontmatter, + rest, + result, + }); + + const content = `${yamlLines}\n\n${treeNode.children + .map((c) => + toMarkdown({ + c, + v, + i: 0, + opts: { + refs: optsRefs, + embeds: optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }, + }), + ) + .join("\n")}\n${ + discourseResults.length + ? `\n###### Discourse Context\n\n${discourseResults + .flatMap((r) => + Object.values(r.results).map( + (t) => + `- **${r.label}::** ${toLink( + getFilename({ + title: t.text, + maxFilenameLength, + simplifiedFilename, + allNodes, + removeSpecialCharacters, + }), + t.uid, + linkType, + )}`, + ), + ) + .join("\n")}\n` + : "" + }${ + referenceResults.length + ? `\n###### References\n\n${referenceResults + .map( + (r_1) => + `${toLink( + getFilename({ + title: r_1[0][":node/title"], + maxFilenameLength, + simplifiedFilename, + allNodes, + removeSpecialCharacters, + }), + r_1[0][":block/uid"] || "", + linkType, + )}\n\n${toMarkdown({ + c: pullBlockToTreeNode(r_1[1], ":bullet"), + opts: { + refs: optsRefs, + embeds: optsEmbeds, + simplifiedFilename, + allNodes, + maxFilenameLength, + removeSpecialCharacters, + linkType, + }, + })}`, + ) + .join("\n")}\n` + : "" + }`; + const uids = new Set(collectUids(treeNode)); + return { title: text, content, uids }; +}; From 2d884961647ca37889ca0d684da2e029928e43ca Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 22 Dec 2025 08:43:57 -0500 Subject: [PATCH 3/3] More work on ontologies --- apps/roam/src/utils/jsonld.ts | 57 ++++++---------- apps/roam/src/utils/supabaseContext.ts | 6 +- apps/website/public/schema/dg_base.ttl | 71 ++++++++++---------- apps/website/public/schema/dg_core.ttl | 90 ++++++++++++++++---------- 4 files changed, 118 insertions(+), 106 deletions(-) diff --git a/apps/roam/src/utils/jsonld.ts b/apps/roam/src/utils/jsonld.ts index eb5a991ee..8f9a21617 100644 --- a/apps/roam/src/utils/jsonld.ts +++ b/apps/roam/src/utils/jsonld.ts @@ -7,6 +7,7 @@ import { pageToMarkdown } from "./pageToMardkown"; import { getRelationDataUtil } from "./getRelationData"; import { uniqJsonArray, getPageData } from "./exportUtils"; import { getExportSettings } from "./getExportSettings"; +import canonicalRoamUrl from "./canonicalRoamUrl"; export const getJsonLdData = async ({ results, @@ -21,7 +22,7 @@ export const getJsonLdData = async ({ nodeLabelByType: Record; updateExportProgress: (progress: number) => void; }) => { - const roamUrl = getRoamUrl(); + const roamUrl = canonicalRoamUrl(); const getRelationData = () => getRelationDataUtil(allRelations, nodeLabelByType); updateExportProgress(0); @@ -37,16 +38,20 @@ export const getJsonLdData = async ({ dc: "http://purl.org/dc/elements/1.1/", prov: "http://www.w3.org/ns/prov#", sioc: "http://rdfs.org/sioc/ns#", - dg: "https://discoursegraphs.com/schema/dg_core#", + dgb: "https://discoursegraphs.com/schema/dg_base", subClassOf: "rdfs:subClassOf", title: "dc:title", + label: "rdfs:label", modified: "dc:modified", created: "dc:date", creator: "dc:creator", content: "sioc:content", - source: "dg:source", - destination: "dg:destination", - reltype: "dg:reltype", + source: "dgb:source", + destination: "dgb:destination", + predicate: "rdf:predicate", + NodeSchema: "dgb:NodeSchema", + RelationDef: "dgb:RelationDef", + RelationInstance: "dgb:RelationInstance", pages: `${roamUrl}/page/`, }; const settings = { @@ -70,8 +75,8 @@ export const getJsonLdData = async ({ await new Promise((resolve) => setTimeout(resolve)); return { "@id": `pages:${node.type}`, - "@type": "dg:NodeSchema", - title: node.text, + "@type": "dgb:NodeSchema", + "label": node.text, content: r.content, modified: modified?.toJSON(), created: date.toJSON(), @@ -81,20 +86,10 @@ export const getJsonLdData = async ({ ); const relSchemaData = allRelations.map((r: DiscourseRelation) => ({ "@id": `pages:${r.id}`, - "@type": "dg:BinaryRelationSchema", - subClassOf: [ - { - "@type": "owl:Restriction", - "owl:onProperty": "dg:source", - "owl:allValuesFrom": `pages:${r.source}`, - }, - { - "@type": "owl:Restriction", - "owl:onProperty": "dg:destination", - "owl:allValuesFrom": `pages:${r.destination}`, - }, - ], - title: r.label, + "@type": "dgb:RelationDef", + "domain": `pages:${r.source}`; + "range": `pages:${r.destination}`, + "label": r.label, })); const schemaData = [...nodeSchemaData, ...relSchemaData]; @@ -104,19 +99,9 @@ export const getJsonLdData = async ({ const inverseRelSchemaData = allRelations.map((r: DiscourseRelation) => ({ "@id": `pages:${r.id}-inverse`, - "@type": "dg:BinaryRelationSchema", - subClassOf: [ - { - "@type": "owl:Restriction", - "owl:onProperty": "dg:source", - "owl:allValuesFrom": `pages:${r.destination}`, - }, - { - "@type": "owl:Restriction", - "owl:onProperty": "dg:destination", - "owl:allValuesFrom": `pages:${r.source}`, - }, - ], + "@type": "dgb:RelationDef", + "domain": `pages:${r.destination}`; + "range": `pages:${r.source}`, title: r.complement, "owl:inverse-of": `pages:${r.id}`, })); @@ -170,8 +155,8 @@ export const getJsonLdData = async ({ ); const relData = relations.map(({ source, target, label }) => ({ // no id yet, just a blank node - "@type": "dg:BinaryRelation", - reltype: schemaUriByName[label], + "@type": "dgb:RelationInstance", + predicate: schemaUriByName[label], source: `pages:${source}`, destination: `pages:${target}`, })); diff --git a/apps/roam/src/utils/supabaseContext.ts b/apps/roam/src/utils/supabaseContext.ts index 6d09e0366..315ff7746 100644 --- a/apps/roam/src/utils/supabaseContext.ts +++ b/apps/roam/src/utils/supabaseContext.ts @@ -1,8 +1,8 @@ import getCurrentUserEmail from "roamjs-components/queries/getCurrentUserEmail"; import getCurrentUserDisplayName from "roamjs-components/queries/getCurrentUserDisplayName"; import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; -import getRoamUrl from "roamjs-components/dom/getRoamUrl"; +import canonicalRoamUrl from "./canonicalRoamUrl"; import { DISCOURSE_CONFIG_PAGE_TITLE } from "~/utils/renderNodeConfigPage"; import getBlockProps from "~/utils/getBlockProps"; import setBlockProps from "~/utils/setBlockProps"; @@ -27,8 +27,6 @@ export type SupabaseContext = { let _contextCache: SupabaseContext | null = null; -const ROAM_URL_PREFIX = "https://roamresearch.com/#/app/"; - const getOrCreateSpacePassword = () => { const settingsConfigPageUid = getPageUidByPageTitle( DISCOURSE_CONFIG_PAGE_TITLE, @@ -57,7 +55,7 @@ export const getSupabaseContext = async (): Promise => { const personEmail = getCurrentUserEmail(); const personName = getCurrentUserDisplayName(); const spaceName = window.roamAlphaAPI.graph.name; - const url = ROAM_URL_PREFIX + spaceName; + const url = canonicalRoamUrl(spaceName); const platform: Platform = "Roam"; const spaceResult = await fetchOrCreateSpaceDirect({ password: spacePassword, diff --git a/apps/website/public/schema/dg_base.ttl b/apps/website/public/schema/dg_base.ttl index 8c9262112..e19864d09 100644 --- a/apps/website/public/schema/dg_base.ttl +++ b/apps/website/public/schema/dg_base.ttl @@ -6,50 +6,42 @@ @prefix dgb: . - dc:date "2025-11-01" ; - dc:description "Discourse Graphs foundation vocabulary"@en ; - dc:title "Discourse Graphs foundation vocabulary"@en ; + dc:date "2025-12-22" ; + rdfs:comment "DiscourseGraph foundation vocabulary"@en ; + rdfs:label "DiscourseGraph foundation vocabulary"@en ; owl:versionInfo "0 (tentative)" ; a owl:Ontology. -dgb:Node a owl:Class ; - dc:description "Discourse Graphs nodes"@en . +# This is inspired by https://hyperknowledge.org/schemas/hyperknowledge_frames.ttl +# and topic mapping dgb:NodeSchema -rdfs:subClassOf owl:Class , dgb:Node ; - dc:description "class of Discourse Graphs node schemata"@en . +rdfs:subClassOf owl:Class; + rdfs:comment "Subclasses of DiscourseGraph nodes"@en . dgb:Role rdfs:subClassOf owl:ObjectProperty, - [a owl:Restriction; owl:onProperty rdfs:range ; owl:allValuesFrom dgb:RoleSchema ], - [a owl:Restriction; owl:onProperty rdfs:domain ; owl:allValuesFrom dgb:NodeSchema ]; - dc:description "A role within a NodeSchema"@en . + [a owl:Restriction; owl:onProperty rdfs:domain ; owl:allValuesFrom dgb:NodeSchema ], + [a owl:Restriction; owl:onProperty rdfs:range ; owl:allValuesFrom dgb:NodeSchema ]; + rdfs:comment "A role within a node schema"@en . -dgb:BinaryRelationSchema -rdfs:subClassOf dgb:NodeSchema, owl:ObjectProperty ; - dc:description "Discourse Graphs binary relation schema"@en . +dgb:RelationDef rdfs:subClassOf owl:ObjectProperty; + rdfs:comment "DiscourseGraph relations"@en. -dgb:BinaryRelation -a rdf:Statement, dgb:Node . +dgb:RelationInstance rdfs:subClassOf rdf:Statement, dgb:NodeSchema, + [a owl:Restriction; owl:onProperty rdfs:predicate ; owl:allValuesFrom dgb:RelationDef ]. -dgb:source -a dgb:Role ; +dgb:source a dgb:Role ; rdfs:subPropertyOf rdf:subject ; - rdfs:domain dgb:BinaryRelationSchema ; - rdfs:range dgb:Node ; - dc:description "The source of a binary relation"@en . + rdfs:domain dgb:RelationInstance ; + rdfs:range dgb:NodeSchema ; + rdfs:comment "The source of a binary relation"@en . -dgb:destination -a dgb:Role ; +dgb:destination a dgb:Role ; rdfs:subPropertyOf rdf:object ; - rdfs:domain dgb:BinaryRelationSchema ; - rdfs:range dgb:Node ; - dc:description "The destination of a binary relation"@en . - -dgb:reltype - rdfs:subPropertyOf rdf:predicate ; - rdfs:domain dgb:BinaryRelation ; - rdf:range dgb:BinaryRelationSchema . + rdfs:domain dgb:RelationInstance ; + rdfs:range dgb:NodeSchema ; + rdfs:comment "The destination of a binary relation"@en . # examples @@ -66,12 +58,25 @@ dgb:reltype # :r0 a :r; # :x_r :x0. -# :br a dgb:BinaryRelationSchema. +# :br a dgb:RelationDef; +# rdfs:domain :x; +# rdfs:range :y; # :br0 -# a dgb:BinaryRelation; -# dgb:reltype :br . +# a dgb:RelationInstance; +# rdf:predicate :br ; # dgb:source :x0 ; # dgb:destination :y0 ; # # This is "about" :x0 :br :y0; + +# Note: we could also use punning, and define +# :br rdfs:subClassOf dgb:RelationInstance, +# [a owl:Restriction; +# owl:onProperty rdf:predicate ; +# owl:hasValue :br ]. +# Then we can more simply state +# :br0 +# a :br; +# dgb:source :x0 ; +# dgb:destination :y0 ; diff --git a/apps/website/public/schema/dg_core.ttl b/apps/website/public/schema/dg_core.ttl index 1b5af6c7a..7f5affe5b 100644 --- a/apps/website/public/schema/dg_core.ttl +++ b/apps/website/public/schema/dg_core.ttl @@ -10,59 +10,83 @@ @prefix dg: . dg:Question a dgb:NodeSchema; - rdfs:label "Question"@en. + rdfs:label "Question"@en; + rdfs:comment "Scientific unknowns that we want to make known, and are addressable by the systematic application of research methods"@en. dg:Claim a dgb:NodeSchema; - rdfs:label "Claim"@en. + rdfs:label "Claim"@en; + rdfs:comment "Atomic, generalized assertions about the world that (propose to) answer research questions"@en. -dg:Result a dgb:NodeSchema; - rdfs:label "Result"@en. +dg:Evidence a dgb:NodeSchema; + rdfs:label "Evidence"@en; + rdfs:comment "A specific empirical observation from a particular application of a research method"@en. -dg:Opposes a dgb:BinaryRelationSchema; +dg:Source a dgb:NodeSchema; + rdfs:label "Source"@en; + rdfs:comment "Some research source that reports/generates evidence, like an experiment/study, book, conference paper, or journal article"@en. + +dg:opposesCE a dgb:RelationDef; rdfs:label "Opposes"@en; rdfs:range dg:Claim; - rdfs:domain dg:Result. - -# I might actually need to be more verbose than rdfs:range/domain... Still needs research. - -# owl:subClassOf [ -# { -# a owl:Restriction; -# owl:onProperty dgb:source; -# owl:allValuesFrom: dg:Claim -# }, -# { -# a owl:Restriction; -# owl:onProperty dgb:destination; -# owl:allValuesFrom: dg:Result -# }, -# ], - -dg:OpposedBy a dgb:BinaryRelationSchema; - rdfs:label "OpposedBy"@en; + rdfs:domain dg:Evidence. + +dg:opposedByEC a dgb:RelationDef; + rdfs:label "Opposed by"@en; owl:inverseOf dg:Opposes; - rdfs:range dg:Result; + rdfs:range dg:Evidence; rdfs:domain dg:Claim. +dg:supportsCE a dgb:RelationDef; + rdfs:label "Supports"@en; + rdfs:range dg:Claim; + rdfs:domain dg:Evidence. + +dg:supportedByEC a dgb:RelationDef; + rdfs:label "Supported by"@en; + owl:inverseOf dg:Supports; + rdfs:range dg:Evidence; + rdfs:domain dg:Claim. -dg:Supports a dgb:BinaryRelationSchema; +dg:opposesCC a dgb:RelationDef; + rdfs:label "Opposes"@en; + rdfs:range dg:Claim; + rdfs:domain dg:Claim. + +dg:opposedByCC a dgb:RelationDef; + rdfs:label "Opposed by"@en; + owl:inverseOf dg:Opposes; + rdfs:range dg:Claim; + rdfs:domain dg:Claim. + +dg:supportsCC a dgb:RelationDef; rdfs:label "Supports"@en; rdfs:range dg:Claim; - rdfs:domain dg:Result. + rdfs:domain dg:Claim. -dg:SupportedBy a dgb:BinaryRelationSchema; - rdfs:label "SupportedBy"@en; +dg:supportedByCC a dgb:RelationDef; + rdfs:label "Supported by"@en; owl:inverseOf dg:Supports; - rdfs:range dg:Result; + rdfs:range dg:Claim; rdfs:domain dg:Claim. -dg:Addresses a dgb:BinaryRelationSchema; +dg:addresses a dgb:RelationDef; rdfs:label "Addresses"@en; rdfs:range dg:Question; rdfs:domain dg:Claim. -dg:AddressedBy a dgb:BinaryRelationSchema; -rdfs:label "AddressedBy"@en; +dg:addressedBy a dgb:RelationDef; +rdfs:label "Addressed by"@en; owl:inverseOf dg:Addresses; rdfs:range dg:Claim; rdfs:domain dg:Question. + +dg:curatedTo a dgb:RelationDef; + rdfs:label "Curated to"@en; + rdfs:range dg:Source; + rdfs:domain dg:Evidence. + +dg:curatedFrom a dgb:RelationDef; + owl:inverseOf dg:curatedTo; + rdfs:label "Curated from"@en; + rdfs:range dg:Evidence; + rdfs:domain dg:Source.