diff --git a/.gitignore b/.gitignore index dc1b07be8..05a79cfca 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ logs **/*.tgz *.zip *.tgz +.npmrc smoke/package.json smoke/pnpm-lock.yaml diff --git a/README.md b/README.md index 13876523b..6ff5dc25b 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ pnpm run test -t 'test name pattern' path/to/test/file For example, to run a specific test for the CRDT module, in just one project: ```bash -FP_DEBUG=Loader pnpm run test --project file -t 'codec implict iv' crdt +FP_DEBUG='Loader,CRDTClock' pnpm run test --project file -t 'codec implict iv' crdt ``` For testing React components, you can use: diff --git a/cli/package.json b/cli/package.json index 55228dceb..0dc3ca29c 100644 --- a/cli/package.json +++ b/cli/package.json @@ -39,7 +39,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-runtime": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/vendor": "workspace:0.0.0", diff --git a/cloud/3rd-party/package.json b/cloud/3rd-party/package.json index 58520547d..7cb430ac1 100644 --- a/cloud/3rd-party/package.json +++ b/cloud/3rd-party/package.json @@ -39,7 +39,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "react-dom": "^19.1.1", "use-fireproof": "workspace:0.0.0" }, diff --git a/cloud/backend/base/package.json b/cloud/backend/base/package.json index 08addf8b8..a87c426bc 100644 --- a/cloud/backend/base/package.json +++ b/cloud/backend/base/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@cloudflare/workers-types": "^4.20250810.0", "@fireproof/cloud-base": "workspace:0.0.0", "@fireproof/core-base": "workspace:0.0.0", diff --git a/cloud/backend/base/test-helper.ts b/cloud/backend/base/test-helper.ts index 8a37ee8a9..aaaa0fcf2 100644 --- a/cloud/backend/base/test-helper.ts +++ b/cloud/backend/base/test-helper.ts @@ -240,7 +240,7 @@ export async function mockJWK(sthis: SuperThis, claim: Partial = token: keys.strings.privateKey, }); - const id = claim.id ?? sthis.nextId().str; + const id = claim.jti ?? sthis.nextId().str; const claims: ps.TokenForParam = { userId: `hello-${id}`, email: `hello-${id}@test.de`, diff --git a/cloud/backend/cf-d1/package.json b/cloud/backend/cf-d1/package.json index 45ec9c10a..fba6598c2 100644 --- a/cloud/backend/cf-d1/package.json +++ b/cloud/backend/cf-d1/package.json @@ -39,7 +39,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@cloudflare/workers-types": "^4.20250810.0", "@fireproof/cloud-backend-base": "workspace:0.0.0", "@fireproof/cloud-base": "workspace:0.0.0", diff --git a/cloud/backend/node/package.json b/cloud/backend/node/package.json index e31d1ea79..d8e75d7ca 100644 --- a/cloud/backend/node/package.json +++ b/cloud/backend/node/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/cloud-backend-base": "workspace:0.0.0", "@fireproof/cloud-base": "workspace:0.0.0", "@fireproof/core-base": "workspace:0.0.0", diff --git a/cloud/base/package.json b/cloud/base/package.json index 2de5ad4f2..106df6bb4 100644 --- a/cloud/base/package.json +++ b/cloud/base/package.json @@ -38,7 +38,7 @@ "react": ">=18.0.0" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-blockstore": "workspace:0.0.0", "@fireproof/core-runtime": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", diff --git a/cloud/todo-app/package.json b/cloud/todo-app/package.json index 40591938c..b86155aae 100644 --- a/cloud/todo-app/package.json +++ b/cloud/todo-app/package.json @@ -41,7 +41,7 @@ "react": ">=18.0.0" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/vendor": "workspace:0.0.0", "@types/react": "^19.1.8", "react-dom": "^19.1.0", diff --git a/core/base/crdt-clock.ts b/core/base/crdt-clock.ts index 63a438242..3f1604f4e 100644 --- a/core/base/crdt-clock.ts +++ b/core/base/crdt-clock.ts @@ -26,7 +26,7 @@ export class CRDTClockImpl { readonly zoomers = new Map(); readonly watchers = new Map[]) => void>(); - readonly emptyWatchers = new Map(); + readonly noPayloadWatchers = new Map(); readonly blockstore: BaseBlockstore; // ready blockstore @@ -46,7 +46,10 @@ export class CRDTClockImpl { constructor(blockstore: BaseBlockstore) { this.sthis = blockstore.sthis; this.blockstore = blockstore; - this.logger = ensureLogger(blockstore.sthis, "CRDTClock"); + this.logger = ensureLogger(blockstore.sthis, `CRDTClock`) + .With() + .Str("dbName", blockstore.crdtParent?.ledgerParent?.name || "unnamed") + .Logger(); this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger); } @@ -80,7 +83,15 @@ export class CRDTClockImpl { if (!updates.length) { return; } - this.emptyWatchers.forEach((fn) => fn()); + this.logger + .Debug() + .Int("updatesCount", updates.length) + .Int("watchersCount", this.watchers.size) + .Int("noPayloadWatchersCount", this.noPayloadWatchers.size) + .Msg("NOTIFY_WATCHERS: Triggering subscriptions"); + // Always notify both types of watchers - subscription systems need notifications + // regardless of whether there are document updates + this.noPayloadWatchers.forEach((fn) => fn()); this.watchers.forEach((fn) => fn(updates || [])); } @@ -94,9 +105,9 @@ export class CRDTClockImpl { onTock(fn: VoidFn): UnReg { const key = this.sthis.timeOrderedNextId().str; - this.emptyWatchers.set(key, fn); + this.noPayloadWatchers.set(key, fn); return () => { - this.emptyWatchers.delete(key); + this.noPayloadWatchers.delete(key); }; } @@ -114,7 +125,18 @@ export class CRDTClockImpl { // } const noLoader = !localUpdates; - + const needsManualNotification = !localUpdates && (this.watchers.size > 0 || this.noPayloadWatchers.size > 0); + + this.logger + .Debug() + .Bool("localUpdates", localUpdates) + .Int("watchersCount", this.watchers.size) + .Int("noPayloadWatchersCount", this.noPayloadWatchers.size) + .Bool("needsManualNotification", needsManualNotification) + .Int("headLength", newHead.length) + .Int("prevHeadLength", prevHead.length) + .Int("currentHeadLength", this.head.length) + .Msg("INT_APPLY_HEAD: Entry point"); // console.log("int_applyHead", this.applyHeadQueue.size(), this.head, newHead, prevHead, localUpdates); const ogHead = sortClockHead(this.head); newHead = sortClockHead(newHead); @@ -156,6 +178,31 @@ export class CRDTClockImpl { this.transaction = undefined; } this.setHead(advancedHead); + + if (needsManualNotification) { + const changes = await clockChangesSince(this.blockstore, advancedHead, prevHead, {}, this.logger); + const triggerReason = + this.watchers.size > 0 && this.noPayloadWatchers.size > 0 + ? "both" + : this.watchers.size > 0 + ? "watchers" + : "noPayloadWatchers"; + this.logger + .Debug() + .Int("changesCount", changes.result.length) + .Str("triggerReason", triggerReason) + .Int("watchersCount", this.watchers.size) + .Int("noPayloadWatchersCount", this.noPayloadWatchers.size) + .Msg("MANUAL_NOTIFICATION: Checking for changes"); + if (changes.result.length > 0) { + this.logger.Debug().Msg("MANUAL_NOTIFICATION: Calling notifyWatchers with changes"); + this.notifyWatchers(changes.result); + this.noPayloadWatchers.forEach((fn) => fn()); + } else { + this.logger.Debug().Msg("MANUAL_NOTIFICATION: Calling noPayloadWatchers directly"); + this.noPayloadWatchers.forEach((fn) => fn()); + } + } } } diff --git a/core/base/crdt.ts b/core/base/crdt.ts index d4bf7dd80..93ef20e2c 100644 --- a/core/base/crdt.ts +++ b/core/base/crdt.ts @@ -106,6 +106,14 @@ export class CRDTImpl implements CRDT { const crdtMeta = meta as CRDTMeta; if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError(); // console.log("applyMeta-pre", crdtMeta.head, this.clock.head); + this.logger + .Debug() + .Str("newHead", crdtMeta.head.map((h) => h.toString()).join(",")) + .Int("subscribers", this.clock.watchers.size + this.clock.noPayloadWatchers.size) + .Int("headLength", crdtMeta.head.length) + .Int("currentHeadLength", this.clock.head.length) + .Str("dbName", this.opts.name || "unnamed") + .Msg("APPLY_META: Calling applyHead for REMOTE sync"); await this.clock.applyHead(crdtMeta.head, []); // console.log("applyMeta-post", crdtMeta.head, this.clock.head); }, @@ -184,6 +192,15 @@ export class CRDTImpl implements CRDT { }); return { head }; }); + this.logger + .Debug() + .Str("newHead", done.meta.head.map((h) => h.toString()).join(",")) + .Int("subscribers", this.clock.watchers.size + this.clock.noPayloadWatchers.size) + .Int("headLength", done.meta.head.length) + .Int("prevHeadLength", prevHead.length) + .Int("currentHeadLength", this.clock.head.length) + .Str("dbName", this.opts.name || "unnamed") + .Msg("BULK: Calling applyHead for LOCAL write"); await this.clock.applyHead(done.meta.head, prevHead, updates); return done.meta; } diff --git a/core/base/package.json b/core/base/package.json index 675df62ae..78bf9e2aa 100644 --- a/core/base/package.json +++ b/core/base/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-blockstore": "workspace:0.0.0", "@fireproof/core-keybag": "workspace:0.0.0", "@fireproof/core-runtime": "workspace:0.0.0", diff --git a/core/blockstore/package.json b/core/blockstore/package.json index a7a37b8d2..e9d90e442 100644 --- a/core/blockstore/package.json +++ b/core/blockstore/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-gateways-base": "workspace:0.0.0", "@fireproof/core-gateways-cloud": "workspace:0.0.0", "@fireproof/core-gateways-file": "workspace:0.0.0", diff --git a/core/core/package.json b/core/core/package.json index ef3421b69..3d76b8294 100644 --- a/core/core/package.json +++ b/core/core/package.json @@ -39,7 +39,7 @@ "react": ">=18.0.0" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-base": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/vendor": "workspace:0.0.0", diff --git a/core/device-id/certor.ts b/core/device-id/certor.ts new file mode 100644 index 000000000..fa9b51934 --- /dev/null +++ b/core/device-id/certor.ts @@ -0,0 +1,63 @@ +import { toSortedObject } from "@adviser/cement/utils"; +import { Base64EndeCoder } from "@fireproof/core-types-base"; +import { decodeJwt } from "jose"; +import { base58btc } from "multiformats/bases/base58"; +import { sha1 } from "multiformats/hashes/sha1"; +import { sha256 } from "multiformats/hashes/sha2"; +import { CertificatePayload, CertificatePayloadSchema } from "@fireproof/core-types-base/fp-ca-cert-payload.zod.js"; + +export class Certor { + readonly #cert: CertificatePayload; + readonly base64: Base64EndeCoder; + #strCert?: string; + #uint8Cert?: Uint8Array; + + static fromString(base64: Base64EndeCoder, cert: string) { + const certObj = CertificatePayloadSchema.parse(JSON.parse(base64.decode(cert))); + return new Certor(base64, certObj); + } + + static fromJWT(base64: Base64EndeCoder, jwtString: string) { + // const header = decodeProtectedHeader(jwtString); + const payload = decodeJwt(jwtString); + const certObj = CertificatePayloadSchema.parse(payload); + return new Certor(base64, certObj); + } + + constructor(base64: Base64EndeCoder, cert: CertificatePayload) { + this.#cert = cert; + this.base64 = base64; + } + + asCert(): CertificatePayload { + return this.#cert; + } + + parseCertificateSubject(s: string): Record { + const parts: Record = {}; + s.split(",").forEach((part) => { + const [key, value] = part.trim().split("="); + if (key && value) { + parts[key] = value; + } + }); + return parts; + } + + async asSHA1() { + this.#uint8Cert ||= this.base64.decodeUint8(this.asBase64()); + const val = await sha1.digest(this.#uint8Cert); + return base58btc.encode(val.bytes); + } + + async asSHA256() { + this.#uint8Cert ||= this.base64.decodeUint8(this.asBase64()); + const val = await sha256.digest(this.#uint8Cert); + return base58btc.encode(val.bytes); + } + + asBase64() { + this.#strCert ||= this.base64.encode(JSON.stringify(toSortedObject(this.#cert))); + return this.#strCert; + } +} diff --git a/core/device-id/device-id-CA.ts b/core/device-id/device-id-CA.ts new file mode 100644 index 000000000..804659c53 --- /dev/null +++ b/core/device-id/device-id-CA.ts @@ -0,0 +1,206 @@ +import { hashObject } from "@fireproof/core-runtime"; +import { + Base64EndeCoder, + CertificatePayload, + Extensions, + FPDeviceIDPayload, + IssueCertificateResult, + JWKPublic, + Subject, +} from "@fireproof/core-types-base"; +import { SignJWT } from "jose"; +import { DeviceIdKey } from "./device-id-key.js"; +import { DeviceIdValidator } from "./device-id-validator.js"; +import { Certor } from "./certor.js"; +import { Result } from "@adviser/cement"; + +export interface CAActions { + generateSerialNumber(pub: JWKPublic): Promise; +} +interface DeviceIdCAOpts { + readonly base64: Base64EndeCoder; + readonly caKey: DeviceIdKey; + readonly caSubject: Subject; + readonly actions: CAActions; + readonly validityPeriod: number; + readonly caChain: string[]; +} +export interface DeviceIdCAOptsDefaulted { + readonly base64: Base64EndeCoder; + readonly caKey: DeviceIdKey; + readonly caSubject: Subject; + readonly actions: CAActions; + readonly caChain?: string[]; // [] + readonly validityPeriod?: number; // 1 year +} + +function defaultDeviceIdCAOpts(opts: DeviceIdCAOptsDefaulted): DeviceIdCAOpts { + return { + ...opts, + validityPeriod: opts.validityPeriod || 365 * 24 * 60 * 60, // 1 year + caChain: opts.caChain || [], + }; +} + +export class DeviceIdCA { + readonly #opts: DeviceIdCAOpts; + + readonly #caKey: DeviceIdKey; + readonly #caSubject: Subject; + + constructor(opts: DeviceIdCAOptsDefaulted) { + this.#opts = defaultDeviceIdCAOpts(opts); + this.#caKey = opts.caKey; + this.#caSubject = opts.caSubject; + } + + async processCSR(csrJWS: string): Promise> { + const validator = new DeviceIdValidator(); + const validation = await validator.validateCSR(csrJWS); + if (!validation.valid) { + return Result.Err(validation.error); + } + return this.issueCertificate(validation.payload); + } + + async caCertificate(): Promise> { + const rCert = await this.issueCertificate({ + csr: { + subject: this.#caSubject, + publicKey: await this.#caKey.publicKey(), + extensions: {}, + }, + }); + if (rCert.isErr()) { + return Result.Err(rCert); + } + return Result.Ok(Certor.fromJWT(this.#opts.base64, rCert.Ok().certificateJWT).asCert()); + } + + async issueCertificate(devId: FPDeviceIDPayload): Promise> { + const now = Math.floor(Date.now() / 1000); + const serialNumber = await this.#opts.actions.generateSerialNumber(await this.#caKey.publicKey()); + + // Create certificate payload + const certificatePayload: CertificatePayload = { + // Standard JWT claims + iss: this.#caSubject.commonName, // Issuer (CA) + sub: devId.csr.subject.commonName, // Subject + aud: devId.aud || "certificate-users", + iat: now, + nbf: now, // Not before + exp: now + this.#opts.validityPeriod, // 1 year validity + jti: serialNumber, // JWT ID as serial number + + // Certificate-specific claims + certificate: { + version: "3", // X.509 v3 + serialNumber: serialNumber, + + // Subject information + subject: devId.csr.subject, + + // Issuer information + issuer: this.#caSubject, + + // Validity period + validity: { + notBefore: new Date(now * 1000).toISOString(), + notAfter: new Date((now + this.#opts.validityPeriod) * 1000).toISOString(), + }, + + // Public key from CSR + subjectPublicKeyInfo: devId.csr.publicKey, + + // Extensions + // extensions: await this.buildCertificateExtensions(devId.csr.extensions, devId.csr.subject, subjectPubKey), + // Certificate metadata + signatureAlgorithm: "ES256", + keyUsage: ["digitalSignature", "keyEncipherment"], + extendedKeyUsage: ["serverAuth"], + }, + }; + + // Get CA public key for certificate + // const caPublicJWK = await this.#caKey.publicKey(); + const pKey = await this.#caKey.exportPrivateJWK(); + const kid = await this.#caKey.fingerPrint(); + + // Create and sign the certificate JWS + const certificateJWC = await new SignJWT(certificatePayload) + .setProtectedHeader({ + alg: "ES256", + typ: "CERT+JWT", // Custom type for certificate + kid, + x5c: this.#opts.caChain, // CA certificate chain (optional) + // exp: now + this.#opts.validityPeriod, + // crit: ['exp'] // Critical header indicating certificate format + }) + .sign(pKey); + + return Result.Ok({ + certificateJWT: certificateJWC, + certificatePayload: certificatePayload, + format: "JWS", + serialNumber: serialNumber, + issuer: this.#caSubject.commonName, + subject: devId.csr.subject.commonName, + validityPeriod: { + notBefore: new Date(now * 1000), + notAfter: new Date((now + this.#opts.validityPeriod) * 1000), + }, + publicKey: devId.csr.publicKey, + }); + } + + // Build certificate extensions + async buildCertificateExtensions(requestedExtensions: Extensions, subject: Subject, subjectPubKey: JWKPublic) { + const extensions = { + // Basic Constraints + basicConstraints: { + critical: true, + cA: false, // End-entity certificate + pathLenConstraint: null, + }, + + // Key Usage + keyUsage: { + critical: true, + usage: requestedExtensions.keyUsage || ["digitalSignature", "keyEncipherment"], + }, + + // Extended Key Usage + extendedKeyUsage: { + critical: false, + usage: requestedExtensions.extendedKeyUsage || ["serverAuth"], + }, + + // Subject Alternative Name + subjectAltName: { + critical: false, + names: requestedExtensions.subjectAltName || [subject.commonName], + }, + + // Authority Key Identifier (would be CA's key identifier) + authorityKeyIdentifier: { + keyIdentifier: await this.#caKey.fingerPrint(), + }, + + // Subject Key Identifier + subjectKeyIdentifier: { + keyIdentifier: await hashObject(subjectPubKey), + }, + // // CRL Distribution Points + // crlDistributionPoints: { + // distributionPoints: ["https://ca.example.com/crl"] + // }, + // Authority Information Access + // authorityInfoAccess: { + // ocsp: ["https://ocsp.example.com"], + // caIssuers: ["https://ca.example.com/cert"] + // } + }; + + return extensions; + } +} diff --git a/core/device-id/device-id-CSR.ts b/core/device-id/device-id-CSR.ts new file mode 100644 index 000000000..e773ccbb7 --- /dev/null +++ b/core/device-id/device-id-CSR.ts @@ -0,0 +1,58 @@ +import { SignJWT } from "jose"; +import { DeviceIdKey } from "./device-id-key.js"; +import { Subject, Extensions, FPDeviceIDPayload, FPDeviceIDPayloadSchema, SuperThis } from "@fireproof/core-types-base"; +import { exception2Result, Result } from "@adviser/cement"; + +export class DeviceIdCSR { + readonly #key: DeviceIdKey; + readonly #sthis: SuperThis; + constructor(sthis: SuperThis, key: DeviceIdKey) { + this.#key = key; + this.#sthis = sthis; + } + // Create CSR payload + async createCSRPayload(subject: Subject, extensions: Extensions = {}): Promise { + const now = Math.floor(Date.now() / 1000); + return FPDeviceIDPayloadSchema.parse({ + sub: subject.commonName, + iss: "csr-client", + aud: "certificate-authority", + iat: now, + exp: now + 3600, // 1 hour validity + jti: this.#sthis.nextId(16).str, + csr: { + subject: subject, + publicKey: await this.#key.publicKey(), + extensions: { + subjectAltName: extensions.subjectAltName || [], + keyUsage: extensions.keyUsage || ["digitalSignature", "keyEncipherment"], + extendedKeyUsage: extensions.extendedKeyUsage || ["serverAuth"], + }, + }, + }); + } + + // Sign the CSR + async signCSR(payload: FPDeviceIDPayload): Promise> { + return exception2Result(async () => { + const publicJWK = await this.#key.publicKey(); + // Create JWS + const jws = await new SignJWT(payload) + .setProtectedHeader({ + alg: "ES256", + typ: "CSR+JWT", + jwk: publicJWK, // Include public key in header + }) + .setIssuedAt() + .setExpirationTime("1h") + .sign(await this.#key.exportPrivateJWK()); + return jws; + }); + } + + // Complete CSR creation process + async createCSR(subject: Subject, extensions: Partial = {}) { + const payload = await this.createCSRPayload(subject, extensions); + return this.signCSR(payload); + } +} diff --git a/core/device-id/device-id-client.ts b/core/device-id/device-id-client.ts new file mode 100644 index 000000000..317718e5f --- /dev/null +++ b/core/device-id/device-id-client.ts @@ -0,0 +1,77 @@ +// can create a CSR +// can sign Msg + +import { IssueCertificateResult, SuperThis } from "@fireproof/core-types-base"; +import { getKeyBag } from "@fireproof/core-keybag"; +import { ResolveOnce, Result } from "@adviser/cement"; +import { DeviceIdKey } from "./device-id-key.js"; +import { DeviceIdSignMsg } from "./device-id-signed-msg.js"; +import { DeviceIdCSR } from "./device-id-CSR.js"; +import { DeviceIdProtocol } from "./device-id-protocol.js"; + +class MsgSigner { + #x: DeviceIdSignMsg; + + constructor(x: DeviceIdSignMsg) { + this.#x = x; + } + + sign>(payload: T, algorithm?: string): Promise { + return this.#x.sign(payload, algorithm); + } +} + +const onceDeviceId = new ResolveOnce>(); + +export interface DeviceIdTransport { + issueCertificate(csrJWT: string): Promise>; +} + +export class DeviceIdClient { + readonly #sthis: SuperThis; + readonly #transport: DeviceIdProtocol; + + constructor(sthis: SuperThis, transport: DeviceIdProtocol) { + this.#sthis = sthis; + this.#transport = transport; + } + + ensureDeviceId() { + return onceDeviceId.once(async () => { + const kBag = await getKeyBag(this.#sthis); + let deviceIdResult = await kBag.getDeviceId(); + if (deviceIdResult.deviceId.IsNone()) { + const key = await DeviceIdKey.create(); + deviceIdResult = await kBag.setDeviceId(await key.exportPrivateJWK()); + } + const key = await DeviceIdKey.createFromJWK(deviceIdResult.deviceId.unwrap()); + + if (deviceIdResult.cert.IsNone()) { + const csr = new DeviceIdCSR(this.#sthis, key); + const rCsrJWT = await csr.createCSR({ commonName: `fp-dev@${await key.fingerPrint()}` }); + if (rCsrJWT.isErr()) { + return Result.Err(rCsrJWT.Err()); + } + const rCertResult = await this.#transport.issueCertificate(rCsrJWT.Ok()); + if (rCertResult.isErr()) { + return Result.Err(rCertResult.Err()); + } + deviceIdResult = await kBag.setDeviceId(deviceIdResult.deviceId.Unwrap(), rCertResult.Ok()); + } + return Result.Err("No certificate found"); + + // if cert is not there create one or cert is to be renewed + // create csr + // request signing -> get cert + // put into keybag + + // return Result.Ok(new MsgSigner(new DeviceIdSignMsg(this.#sthis.txt.base64, key, cert))); + }); + } + + // sign a message + // @param msg: string // JWT String + sendSigned>(_payload: T, _algorithm?: string): Promise { + throw new Error("Method not implemented."); + } +} diff --git a/core/device-id/device-id-key.ts b/core/device-id/device-id-key.ts new file mode 100644 index 000000000..81f1a109c --- /dev/null +++ b/core/device-id/device-id-key.ts @@ -0,0 +1,56 @@ +import { hashObject } from "@fireproof/core-runtime"; +import { JWKPrivate, JWKPrivateSchema, JWKPublic, JWKPublicSchema } from "@fireproof/core-types-base"; +import { GenerateKeyPairOptions, generateKeyPair, importJWK, exportJWK } from "jose"; + +export class DeviceIdKey { + #privateKey: CryptoKey; + + static async create( + opts: GenerateKeyPairOptions = { + modulusLength: 2048, + extractable: true, + }, + ) { + const pair = await generateKeyPair("ES256", opts); + return new DeviceIdKey(pair.privateKey); + } + + static async createFromJWK( + jwk: JWKPrivate, + opts: GenerateKeyPairOptions = { + extractable: true, + }, + ) { + const pair = await importJWK(jwk, "ES256", opts); + if (pair instanceof Uint8Array) { + throw new Error("Invalid JWK"); + } + return new DeviceIdKey(pair); + } + + private constructor(pair: CryptoKey) { + this.#privateKey = pair; + } + + async fingerPrint() { + return hashObject(await this.exportPrivateJWK()); + } + + async exportPrivateJWK(): Promise { + const jwk = await exportJWK(this.#privateKey); + const { success, data } = JWKPrivateSchema.safeParse(jwk); + if (!success || !data) { + throw new Error("Invalid JWK"); + } + return data; + } + + async publicKey(): Promise { + const privateJWK = await exportJWK(this.#privateKey); + const { success, data } = JWKPublicSchema.safeParse(privateJWK); + if (!success || !data) { + throw new Error("Invalid JWK"); + } + return data; + } +} diff --git a/core/device-id/device-id-protocol.ts b/core/device-id/device-id-protocol.ts new file mode 100644 index 000000000..5ea615cba --- /dev/null +++ b/core/device-id/device-id-protocol.ts @@ -0,0 +1,77 @@ +import { IssueCertificateResult, JWKPrivateSchema, SuperThis } from "@fireproof/core-types-base"; +import { CAActions, DeviceIdCA } from "./device-id-CA.js"; +import { param, Result } from "@adviser/cement"; +import { DeviceIdKey } from "./device-id-key.js"; +import { base58btc } from "multiformats/bases/base58"; +import { DeviceIdVerifyMsg, VerifyWithCertificateResult } from "./device-id-verify-msg.js"; + +async function ensureCA(sthis: SuperThis, actions: CAActions): Promise> { + const rEnv = sthis.env.gets({ + DEVICE_ID_CA_KEY: param.REQUIRED, + DEVICE_ID_CA_COMMON_NAME: param.OPTIONAL, + }); + if (rEnv.isErr()) { + throw rEnv.Err(); + } + const env = rEnv.Ok(); + const { success, data: caKey } = JWKPrivateSchema.safeParse(JSON.parse(sthis.txt.decode(base58btc.decode(env.DEVICE_ID_CA_KEY)))); + if (!success || !caKey) { + throw new Error("Invalid CA key"); + } + + return Result.Ok( + new DeviceIdCA({ + base64: sthis.txt.base64, + caKey: await DeviceIdKey.createFromJWK(caKey), + caSubject: { + commonName: env.DEVICE_ID_CA_COMMON_NAME ?? "Fireproof CA", + }, + actions, + }), + ); +} + +export interface DeviceIdProtocol { + issueCertificate(msg: string): Promise>; + verifyMsg(message: string): Promise; +} + +export interface DeviceIdProtocolSrvOpts { + readonly actions: CAActions; +} + +export class DeviceIdProtocolSrv implements DeviceIdProtocol { + readonly #ca: DeviceIdCA; + readonly #verifyMsg: DeviceIdVerifyMsg; + static async create(sthis: SuperThis, opts: DeviceIdProtocolSrvOpts): Promise> { + const rCa = await ensureCA(sthis, opts.actions); + if (rCa.isErr()) { + return Result.Err(rCa); + } + const rCaCert = await rCa.Ok().caCertificate(); + if (rCaCert.isErr()) { + return Result.Err(rCaCert); + } + const verifyMsg = new DeviceIdVerifyMsg(sthis.txt.base64, [rCaCert.Ok()], { + clockTolerance: 60, + maxAge: 3600, + }); + return Result.Ok(new DeviceIdProtocolSrv(rCa.Ok(), verifyMsg)); + } + + private constructor(ca: DeviceIdCA, verifyMsg: DeviceIdVerifyMsg) { + this.#ca = ca; + this.#verifyMsg = verifyMsg; + } + + // issue a certificate + // @param msg: string // CSR as JWT String + issueCertificate(msg: string): Promise> { + return this.#ca.processCSR(msg); + } + // sign a message + // @param msg: string // JWT String + verifyMsg(message: string): Promise { + return this.#verifyMsg.verifyWithCertificate(message); + } +} diff --git a/core/device-id/device-id-signed-msg.ts b/core/device-id/device-id-signed-msg.ts new file mode 100644 index 000000000..4f20368a5 --- /dev/null +++ b/core/device-id/device-id-signed-msg.ts @@ -0,0 +1,35 @@ +import { Base64EndeCoder, CertificatePayload } from "@fireproof/core-types-base"; +import { SignJWT } from "jose"; +import { Certor } from "./certor.js"; +import { DeviceIdKey } from "./device-id-key.js"; + +export class DeviceIdSignMsg { + readonly #key: DeviceIdKey; + readonly #cert: CertificatePayload; // Cert Signed by DeviceIdCA + readonly base64: Base64EndeCoder; + + constructor(base64: Base64EndeCoder, key: DeviceIdKey, cert: CertificatePayload) { + this.#key = key; + this.#cert = cert; + this.base64 = base64; + } + + async sign>(payload: T, algorithm = "ES256") { + const certor = new Certor(this.base64, this.#cert); + const x5c = [certor.asBase64()]; + const x5t = await certor.asSHA1(); + const x5tS256 = await certor.asSHA256(); + return await new SignJWT(payload) + .setProtectedHeader({ + alg: algorithm, + typ: "JWT", + kid: await this.#key.fingerPrint(), + x5c: x5c, // Certificate chain + x5t: x5t, // SHA-1 thumbprint + "x5t#S256": x5tS256, // SHA-256 thumbprint + }) + .setIssuedAt() + .setExpirationTime("1h") + .sign(await this.#key.exportPrivateJWK()); + } +} diff --git a/core/device-id/device-id-validator.ts b/core/device-id/device-id-validator.ts new file mode 100644 index 000000000..b22b21e25 --- /dev/null +++ b/core/device-id/device-id-validator.ts @@ -0,0 +1,59 @@ +import { FPDeviceIDPayload, JWKPublic, JWKPublicSchema, FPDeviceIDPayloadSchema } from "@fireproof/core-types-base"; +import { jwtVerify, decodeProtectedHeader } from "jose"; + +interface ValidateCSRError { + readonly valid: false; + readonly error: string; +} + +interface ValidateCSRSuccess { + readonly valid: true; + readonly payload: FPDeviceIDPayload; + readonly publicKey: JWKPublic; +} + +type ValidateCSRResult = ValidateCSRError | ValidateCSRSuccess; + +export class DeviceIdValidator { + async validateCSR(csrJWS: string): Promise { + try { + // Parse the JWS header to get the public key + const header = decodeProtectedHeader(csrJWS); + if (!header.jwk) { + throw new Error("No public key in CSR header"); + } + + const { success: successPub, data: publicKey } = JWKPublicSchema.safeParse(header.jwk); + if (!successPub || !publicKey) { + return { + valid: false, + error: "Invalid public key in CSR header", + }; + } + + // Verify the JWS + const { payload: fromPayload } = await jwtVerify(csrJWS, publicKey, { + typ: "CSR+JWT", + }); + + const { success, data: payload } = FPDeviceIDPayloadSchema.safeParse(fromPayload); + if (!success || !payload) { + return { + valid: false, + error: "Invalid CSR payload", + }; + } + + return { + valid: true, + payload: payload, + publicKey, + }; + } catch (error) { + return { + valid: false, + error: (error as Error).message, + }; + } + } +} diff --git a/core/device-id/device-id-verify-msg.ts b/core/device-id/device-id-verify-msg.ts new file mode 100644 index 000000000..59a9cc260 --- /dev/null +++ b/core/device-id/device-id-verify-msg.ts @@ -0,0 +1,327 @@ +import { Base64EndeCoder, CertificatePayload, JWKPublic } from "@fireproof/core-types-base"; +import { jwtVerify, decodeProtectedHeader } from "jose"; +import { Certor } from "./certor.js"; +import { exception2Result, Result } from "@adviser/cement"; + +interface HeaderCertInfo { + readonly certificate: Certor; + readonly certificateChain: Certor[]; + readonly thumbprint?: string; + readonly thumbprintSha256?: string; + readonly keyId?: string; + readonly algorithm?: string; + readonly certificateUrl?: string; + readonly rawHeader: unknown; +} + +interface VerifyWithCertificateSuccess { + readonly valid: true; + readonly payload: unknown; + readonly header: unknown; + readonly certificate: HeaderCertInfo & { + readonly validation: { + readonly valid: true; + readonly subject: Record; + readonly issuer: Record; + readonly serialNumber: string; + readonly notBefore: Date; + readonly notAfter: Date; + readonly publicKey: JWKPublic; + readonly trustedCA?: CertificatePayload; + readonly validityPeriod: { + readonly days: number; + }; + }; + readonly publicKey: JWKPublic; + }; + readonly verificationTimestamp: string; +} + +interface VerifyWithCertificateError { + readonly valid: false; + readonly error: Error; + readonly errorCode: string; + readonly partialResults: { + readonly certificateExtracted: boolean; + readonly jwtSignatureValid: boolean; + readonly certificateInfo?: HeaderCertInfo; + }; + readonly verificationTimestamp: string; +} + +export type VerifyWithCertificateResult = VerifyWithCertificateSuccess | VerifyWithCertificateError; + +interface VerifyWithCertificateOptions { + readonly clockTolerance: number; // Clock skew tolerance in seconds + readonly maxAge?: number; // Maximum JWT age in seconds +} + +export class DeviceIdVerifyMsg { + readonly #base64: Base64EndeCoder; + readonly #trustedCAs: CertificatePayload[]; + readonly #options: VerifyWithCertificateOptions; + + constructor(base64: Base64EndeCoder, trustedCAs: CertificatePayload[], options: VerifyWithCertificateOptions) { + this.#base64 = base64; + this.#trustedCAs = trustedCAs; + this.#options = options; + } + + createVerifyWithCertificateError( + error: Result, + partialResults: Partial = {}, + ): VerifyWithCertificateError { + return { + valid: false, + error: error.Err(), + errorCode: this.getErrorCode(error), + partialResults: { + certificateExtracted: partialResults.certificateExtracted ?? false, + jwtSignatureValid: partialResults.jwtSignatureValid ?? false, + certificateInfo: partialResults.certificateInfo, + }, + verificationTimestamp: new Date().toISOString(), + }; + } + + /** + * Verify JWT and validate certificate + */ + async verifyWithCertificate(jwt: string): Promise { + let certInfo = undefined; + // let publicKey = null; + let jwtPayload = null; + let jwtHeader = null; + + // Step 1: Extract certificate from JWT header + const rCertInfo = this.extractCertificateFromJWT(jwt); + if (rCertInfo.isErr()) { + return this.createVerifyWithCertificateError(rCertInfo); + } + certInfo = rCertInfo.Ok(); + + // Step 2: Validate certificate thumbprint integrity + const rThumbprint = await this.validateCertificateThumbprint(certInfo); + if (rThumbprint.isErr()) { + return this.createVerifyWithCertificateError(rThumbprint, { + certificateExtracted: true, + certificateInfo: certInfo, + }); + } + if (!rThumbprint.Ok()) { + return this.createVerifyWithCertificateError(Result.Err("Certificate thumbprint validation failed"), { + certificateExtracted: true, + certificateInfo: certInfo, + }); + } + + const rVerify = await exception2Result(async () => { + // Step 3: Extract and validate public key from certificate + // console.log("Step 3: Extracting public key from certificate..."); + // publicKey = await extractPublicKeyFromCertificate(certInfo.certificate); + // Step 4: Verify JWT signature with extracted public key + return jwtVerify(jwt, certInfo.certificate.asCert().certificate.subjectPublicKeyInfo, { + clockTolerance: this.#options.clockTolerance, + maxTokenAge: this.#options.maxAge, + }); + }); + if (rVerify.isErr()) { + return this.createVerifyWithCertificateError(rVerify, { + certificateExtracted: true, + certificateInfo: certInfo, + }); + } + const jwtVerification = rVerify.Ok(); + if (!jwtVerification) { + return this.createVerifyWithCertificateError(Result.Err("JWT verification failed"), { + certificateExtracted: true, + certificateInfo: certInfo, + }); + } + + jwtPayload = jwtVerification.payload; + jwtHeader = jwtVerification.protectedHeader; + + // Step 5: Validate certificate properties + const rCertValidation = await this.validateCertificate(certInfo.certificate); + if (rCertValidation.isErr()) { + return this.createVerifyWithCertificateError(rCertValidation, { + certificateExtracted: true, + certificateInfo: certInfo, + jwtSignatureValid: true, + }); + } + + // Step 6: Validate certificate chain if provided + if (certInfo.certificateChain.length > 1) { + return this.createVerifyWithCertificateError(Result.Err("Certificate chain validation not implemented"), { + certificateExtracted: true, + certificateInfo: certInfo, + }); + } + + // Success - return comprehensive result + return { + valid: true, + payload: jwtPayload, + header: jwtHeader, + certificate: { + ...certInfo, + validation: rCertValidation.Ok(), + publicKey: certInfo.certificate.asCert().certificate.subjectPublicKeyInfo, + }, + verificationTimestamp: new Date().toISOString(), + }; + } + + /** + * Extract certificate information from JWT header + */ + extractCertificateFromJWT(jwt: string): Result { + return exception2Result(() => { + // Decode JWT header without verification + const header = decodeProtectedHeader(jwt); + + // Check for certificate in x5c claim + if (!header.x5c || !Array.isArray(header.x5c) || header.x5c.length === 0) { + throw new Error("No certificate chain (x5c) found in JWT header"); + } + + // Convert certificates from base64 to PEM + const certificateChain = header.x5c.map((cert) => Certor.fromString(this.#base64, cert)); + const mainCertificate = certificateChain[0]; + + return { + certificate: mainCertificate, + certificateChain: certificateChain, + thumbprint: header.x5t, + thumbprintSha256: header["x5t#S256"] as string, + keyId: header.kid, + algorithm: header.alg, + certificateUrl: header.x5u, + rawHeader: header, + }; + }); + } + + /** + * Validate certificate thumbprint to ensure integrity + */ + async validateCertificateThumbprint(certInfo: HeaderCertInfo): Promise> { + // Calculate SHA-1 thumbprint + if (certInfo.thumbprint) { + const calculatedThumbprint = await certInfo.certificate.asSHA1(); + // calculateCertThumbprint(certInfo.certificate, "sha1"); + if (certInfo.thumbprint !== calculatedThumbprint) { + return Result.Err(new Error("Certificate SHA-1 thumbprint mismatch - certificate may have been tampered with")); + } + } + + // Calculate SHA-256 thumbprint + if (certInfo.thumbprintSha256) { + const calculatedThumbprintSha256 = await certInfo.certificate.asSHA256(); + if (certInfo.thumbprintSha256 !== calculatedThumbprintSha256) { + return Result.Err(new Error("Certificate SHA-256 thumbprint mismatch - certificate may have been tampered with")); + } + } + return Result.Ok(true); + } + + /** + * Validate certificate properties + */ + async validateCertificate(certor: Certor): Promise> { + const now = new Date(); + return exception2Result(() => { + const cert = certor.asCert(); + // Parse certificate details + const subject = certor.parseCertificateSubject(cert.sub); + const issuer = certor.parseCertificateSubject(cert.iss); + // const isSelfSigned = cert.issuer === cert.subject; + // Basic time validations + const notBefore = new Date(cert.nbf * 1000); + const notAfter = new Date(cert.exp * 1000); + + if (notBefore > now) { + throw new Error(`Certificate is not yet valid (valid from: ${notBefore.toISOString()})`); + } + + if (notAfter < now) { + throw new Error(`Certificate has expired (valid to: ${notAfter.toISOString()})`); + } + + // Self-signed validation + // if (isSelfSigned && !allowSelfSigned) { + // throw new Error("Self-signed certificates are not allowed"); + // } + // Issuer validation + // if (allowedIssuers.length > 0) { + // const issuerMatch = allowedIssuers.some((allowedIssuer) => { + // return cert.issuer.includes(allowedIssuer); + // }); + // if (!issuerMatch) { + // throw new Error(`Certificate issuer not in allowed list: ${cert.issuer}`); + // } + // } + // Key usage validation (simplified) + // if (requiredKeyUsage.length > 0) { + // // In a real implementation, you'd parse the keyUsage extension + // // For now, we assume digital signature is present + // const hasRequiredUsage = requiredKeyUsage.every((usage) => { + // return ["digitalSignature", "keyEncipherment"].includes(usage); + // }); + // if (!hasRequiredUsage) { + // throw new Error("Certificate does not have required key usage"); + // } + // } + // Trust validation for non-self-signed certificates + let trustedCA = null; + trustedCA = this.findTrustedCA(cert, this.#trustedCAs); + if (!trustedCA) { + throw new Error("Certificate not issued by a trusted CA"); + } + + return { + valid: true, + subject: subject, + issuer: issuer, + serialNumber: cert.certificate.serialNumber, + // fingerprint: cert.fingerprint, + // fingerprintSha256: cert.fingerprint256, + notBefore: notBefore, + notAfter: notAfter, + publicKey: cert.certificate.subjectPublicKeyInfo, + // selfSigned: isSelfSigned, + // keyType: cert.publicKey.asymmetricKeyType, + // keySize: cert.publicKey.asymmetricKeySize, + trustedCA: trustedCA, + validityPeriod: { + days: Math.floor((notAfter.getTime() - notBefore.getTime()) / (1000 * 60 * 60 * 24)), + }, + } satisfies VerifyWithCertificateSuccess["certificate"]["validation"]; + }); + } + + findTrustedCA(cert: CertificatePayload, trustedCAs: CertificatePayload[]) { + return trustedCAs.find((trustedCA) => { + try { + return cert.iss === trustedCA.sub; + } catch { + return false; + } + }); + } + + getErrorCode(ierror: unknown) { + const { message: errorMessage } = ierror as Error; + if (errorMessage.includes("thumbprint mismatch")) return "CERT_THUMBPRINT_MISMATCH"; + if (errorMessage.includes("expired")) return "CERT_EXPIRED"; + if (errorMessage.includes("not yet valid")) return "CERT_NOT_YET_VALID"; + if (errorMessage.includes("self-signed")) return "CERT_SELF_SIGNED"; + if (errorMessage.includes("not trusted")) return "CERT_NOT_TRUSTED"; + if (errorMessage.includes("revoked")) return "CERT_REVOKED"; + if (errorMessage.includes("signature verification failed")) return "JWT_SIGNATURE_INVALID"; + if (errorMessage.includes("No certificate")) return "CERT_NOT_FOUND"; + return "VERIFICATION_FAILED"; + } +} diff --git a/core/device-id/index.ts b/core/device-id/index.ts new file mode 100644 index 000000000..17a5dbc83 --- /dev/null +++ b/core/device-id/index.ts @@ -0,0 +1,8 @@ +export * from "./certor.js"; +export * from "./device-id-CA.js"; +export * from "./device-id-CSR.js"; +export * from "./device-id-key.js"; +export * from "./device-id-signed-msg.js"; +export * from "./device-id-verify-msg.js"; +export * from "./device-id-validator.js"; +export * from "./device-id-protocol.js"; diff --git a/core/device-id/package.json b/core/device-id/package.json new file mode 100644 index 000000000..a80209b42 --- /dev/null +++ b/core/device-id/package.json @@ -0,0 +1,51 @@ +{ + "name": "@fireproof/core-device-id", + "version": "0.0.0", + "description": "Live ledger for the web.", + "type": "module", + "main": "./index.js", + "scripts": { + "build": "tsc", + "pack": "core-cli build --doPack", + "publish": "core-cli build" + }, + "keywords": [ + "ledger", + "JSON", + "document", + "IPLD", + "CID", + "IPFS" + ], + "contributors": [ + "J Chris Anderson", + "Alan Shaw", + "Travis Vachon", + "Mikeal Rogers", + "Meno Abels" + ], + "author": "J Chris Anderson", + "license": "AFL-2.0", + "homepage": "https://use-fireproof.com", + "gptdoc": "import { fireproof } from 'use-fireproof'; const db = fireproof('app-db-name'); const ok = await db.put({ anyField: ['any','json'] }); const doc = await db.get(ok.id); await db.del(doc._id); db.subscribe(myRedrawFn); const result = await db.query('anyField', {range : ['a', 'z']}); result.rows.map(({ key }) => key);", + "repository": { + "type": "git", + "url": "git+https://github.com/fireproof-storage/fireproof.git" + }, + "bugs": { + "url": "https://github.com/fireproof-storage/fireproof/issues" + }, + "dependencies": { + "@adviser/cement": "^0.4.26", + "@fireproof/core-keybag": "workspace:0.0.0", + "@fireproof/core-runtime": "workspace:0.0.0", + "@fireproof/core-types-base": "workspace:0.0.0", + "jose": "^6.0.12", + "multiformats": "^13.3.7", + "zod": "^4.0.14" + }, + "devDependencies": { + "@fireproof/core-cli": "workspace:0.0.0", + "@types/node": "^24.1.0" + } +} diff --git a/core/device-id/tsconfig.json b/core/device-id/tsconfig.json new file mode 100644 index 000000000..9f2759456 --- /dev/null +++ b/core/device-id/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + } +} diff --git a/core/device-id/types.ts b/core/device-id/types.ts new file mode 100644 index 000000000..e69de29bb diff --git a/core/gateways/base/meta-key-hack.ts b/core/gateways/base/meta-key-hack.ts index de478fc8c..6ef854770 100644 --- a/core/gateways/base/meta-key-hack.ts +++ b/core/gateways/base/meta-key-hack.ts @@ -163,7 +163,7 @@ async function wrapEncode /* security: we don't want to log the key */ const keyMaterials = await rKex .Ok() - .asV2KeysItem() + .asV2StorageKeyItem() .then((i) => Object.values(i.keys).map((i) => i.key)); return Promise.resolve(Result.Ok(fn(payload, keyMaterials))); diff --git a/core/gateways/base/package.json b/core/gateways/base/package.json index 2356b2cc9..0efed183f 100644 --- a/core/gateways/base/package.json +++ b/core/gateways/base/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-runtime": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/core-types-blockstore": "workspace:0.0.0", diff --git a/core/gateways/cloud/package.json b/core/gateways/cloud/package.json index eda997a1a..dbbc0dc04 100644 --- a/core/gateways/cloud/package.json +++ b/core/gateways/cloud/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-gateways-base": "workspace:0.0.0", "@fireproof/core-protocols-cloud": "workspace:0.0.0", "@fireproof/core-runtime": "workspace:0.0.0", diff --git a/core/gateways/file-deno/package.json b/core/gateways/file-deno/package.json index 2d62ca7bb..3db714f34 100644 --- a/core/gateways/file-deno/package.json +++ b/core/gateways/file-deno/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/vendor": "workspace:0.0.0", "@types/deno": "^2.3.0", diff --git a/core/gateways/file-node/package.json b/core/gateways/file-node/package.json index 44f5ce658..691de6e85 100644 --- a/core/gateways/file-node/package.json +++ b/core/gateways/file-node/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/vendor": "workspace:0.0.0" } diff --git a/core/gateways/file/key-bag-file.ts b/core/gateways/file/key-bag-file.ts index e52aba1ac..20a34155f 100644 --- a/core/gateways/file/key-bag-file.ts +++ b/core/gateways/file/key-bag-file.ts @@ -1,12 +1,5 @@ import { Logger, URI } from "@adviser/cement"; -import { - isNotFoundError, - KeyBagProvider, - V2KeysItem, - SuperThis, - SysFileSystem, - V1StorageKeyItem, -} from "@fireproof/core-types-base"; +import { isNotFoundError, KeyBagProvider, SuperThis, SysFileSystem } from "@fireproof/core-types-base"; import { sysFileSystemFactory } from "./sys-file-system-factory.js"; interface KeyBagCtx { @@ -49,7 +42,7 @@ export class KeyBagProviderFile implements KeyBagProvider { } } - async get(id: string): Promise { + async get(id: string): Promise | undefined> { const ctx = await this._prepare(id); try { const p = await ctx.sysFS.readfile(ctx.fName); @@ -63,8 +56,8 @@ export class KeyBagProviderFile implements KeyBagProvider { } } - async set(item: V2KeysItem): Promise { - const ctx = await this._prepare(item.name); + async set(id: string, item: NonNullable): Promise { + const ctx = await this._prepare(id); const p = this.sthis.txt.encode(JSON.stringify(item, null, 2)); await ctx.sysFS.writefile(ctx.fName, p); } diff --git a/core/gateways/file/package.json b/core/gateways/file/package.json index 572351c1f..a6bc2d8db 100644 --- a/core/gateways/file/package.json +++ b/core/gateways/file/package.json @@ -41,7 +41,7 @@ "@types/node": "^24.3.0" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-gateways-base": "workspace:0.0.0", "@fireproof/core-gateways-file-deno": "workspace:0.0.0", "@fireproof/core-gateways-file-node": "workspace:0.0.0", diff --git a/core/gateways/indexeddb/key-bag-indexeddb.ts b/core/gateways/indexeddb/key-bag-indexeddb.ts index 8711bbf20..0370bcbd6 100644 --- a/core/gateways/indexeddb/key-bag-indexeddb.ts +++ b/core/gateways/indexeddb/key-bag-indexeddb.ts @@ -1,6 +1,6 @@ import { IDBPDatabase, openDB } from "idb"; import { Logger, ResolveOnce, URI } from "@adviser/cement"; -import { KeyBagProvider, V2KeysItem, V1StorageKeyItem, type SuperThis } from "@fireproof/core-types-base"; +import { KeyBagProvider, type SuperThis } from "@fireproof/core-types-base"; import { getPath } from "@fireproof/core-gateways-base"; export class KeyBagProviderIndexedDB implements KeyBagProvider { @@ -39,7 +39,7 @@ export class KeyBagProviderIndexedDB implements KeyBagProvider { await tx.done; } - async get(id: string): Promise { + async get(id: string): Promise | undefined> { const db = await this._prepare(); const tx = db.transaction(["bag"], "readonly"); const keyItem = await tx.objectStore("bag").get(id); @@ -50,10 +50,10 @@ export class KeyBagProviderIndexedDB implements KeyBagProvider { return keyItem; } - async set(item: V2KeysItem): Promise { + async set(id: string, item: NonNullable): Promise { const db = await this._prepare(); const tx = db.transaction(["bag"], "readwrite"); - await tx.objectStore("bag").put(item, item.name); + await tx.objectStore("bag").put(item, id); await tx.done; } } diff --git a/core/gateways/indexeddb/package.json b/core/gateways/indexeddb/package.json index d097a73eb..4a73c4aee 100644 --- a/core/gateways/indexeddb/package.json +++ b/core/gateways/indexeddb/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-gateways-base": "workspace:0.0.0", "@fireproof/core-runtime": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", diff --git a/core/gateways/memory/package.json b/core/gateways/memory/package.json index 14d32e932..12ee2380d 100644 --- a/core/gateways/memory/package.json +++ b/core/gateways/memory/package.json @@ -41,7 +41,7 @@ "@types/node": "^24.3.0" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-gateways-base": "workspace:0.0.0", "@fireproof/core-runtime": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", diff --git a/core/jwks/README.md b/core/jwks/README.md new file mode 100644 index 000000000..76b8c5bfc --- /dev/null +++ b/core/jwks/README.md @@ -0,0 +1,110 @@ +# @fireproof/core-jwks + +JWKS (JSON Web Key Set) validation and fetching for Fireproof. + +## Features + +- ✅ Fetch JWKS from URLs with retry logic and error handling +- ✅ Validate individual keys and entire key sets +- ✅ Support for Clerk authentication URLs +- ✅ Check if keys are current and active +- ✅ Comprehensive error handling and reporting +- ✅ TypeScript support with full type definitions + +## Usage + +```typescript +import { fetchAndValidateJWKS } from "@fireproof/core-jwks"; + +// Fetch and validate Clerk JWKS +const result = await fetchAndValidateJWKS("trusted-glowworm-5", { + allowedKeyTypes: ["RSA"], + allowedUse: ["sig"], + requireKeyId: true, + maxKeys: 5, +}); + +if (result.is_ok()) { + const { jwks, validation } = result.unwrap(); + console.log(`Found ${validation.currentKeysCount} current keys`); +} else { + console.error("JWKS validation failed:", result.unwrap_err().message); +} +``` + +## API + +### Main Functions + +- `fetchJWKS(config, options?)` - Fetch JWKS with retry logic +- `validateJWKS(jwks, options?)` - Validate a JWKS object +- `fetchAndValidateJWKS(config, validationOptions?, fetchOptions?)` - Combined fetch and validate +- `buildJWKSUrl(config)` - Build JWKS URL from configuration + +### Utility Functions + +- `getCurrentKeys(validationResult)` - Get only current/active keys +- `findKeyById(jwks, keyId)` - Find specific key by ID + +### Legacy Compatibility + +- `fetchJwks(url)` - Legacy function (deprecated). **Note: this API throws `JWKSFetchError` on failure**, whereas `fetchJWKS`/`fetchAndValidateJWKS` return a `Result`. Adjust your error handling accordingly. + +```typescript +// Legacy (throws) +try { + const jwks = await fetchJwks("https://example.com/.well-known/jwks.json"); +} catch (error) { + console.error(error.message); +} + +// New (Result) +const res = await fetchJWKS("https://example.com/.well-known/jwks.json"); +if (res.is_err()) { + console.error(res.unwrap_err().message); +} else { + console.log(res.unwrap()); +} +``` + +## Configuration + +Supports multiple input formats: + +- Direct URLs, e.g., `"https://example.com/.well-known/jwks.json"`. +- Clerk tenant shortcuts, e.g., `"trusted-glowworm-5"`. +- Clerk domain hostnames, e.g., `"trusted-glowworm-5.clerk.accounts.dev"`. + +## Error Handling + +The package uses Result types from `@adviser/cement` for comprehensive error handling: + +- `JWKSFetchError` - Network and fetch-related errors +- `JWKSValidationError` - Key validation errors + +## Testing + +Run tests locally: + +```bash +# Basic unit tests +npx vitest run tests/basic.test.ts + +# Integration tests (with live Clerk endpoint) +npx vitest run tests/integration.test.ts + +# All tests +./test-all.sh +``` + +## Structure + +```text +src/ +├── validator.ts # Core JWKS validation logic +├── fetcher.ts # Legacy compatibility layer +└── index.ts # Package exports +tests/ +├── basic.test.ts # Unit tests +└── integration.test.ts # Integration tests with live endpoints +``` diff --git a/core/jwks/package.json b/core/jwks/package.json new file mode 100644 index 000000000..985560f9a --- /dev/null +++ b/core/jwks/package.json @@ -0,0 +1,36 @@ +{ + "name": "@fireproof/core-jwks", + "version": "0.0.0", + "description": "JWKS validation and fetching for Fireproof", + "type": "module", + "main": "./src/index.js", + "types": "./src/index.d.ts", + "scripts": { + "build": "tsc", + "test": "vitest", + "test:run": "vitest run" + }, + "keywords": [ + "jwks", + "jwt", + "validation", + "clerk", + "fireproof" + ], + "author": "Fireproof Team", + "license": "AFL-2.0", + "dependencies": { + "@adviser/cement": "^0.4.25", + "jose": "^6.0.12" + }, + "devDependencies": { + "typescript": "^5.9.2", + "vitest": "^3.2.4" + }, + "exports": { + ".": { + "import": "./src/index.js", + "types": "./src/index.d.ts" + } + } +} diff --git a/core/jwks/src/fetcher.ts b/core/jwks/src/fetcher.ts new file mode 100644 index 000000000..f48218994 --- /dev/null +++ b/core/jwks/src/fetcher.ts @@ -0,0 +1,14 @@ +// Legacy compatibility layer +import { fetchJWKS } from "./validator.js"; + +/** + * Legacy function for backward compatibility + * @deprecated Use fetchJWKS or fetchAndValidateJWKS instead + */ +export async function fetchJwks(url: string) { + const result = await fetchJWKS(url); + if (result.is_err()) { + throw result.unwrap_err(); + } + return result.unwrap(); +} diff --git a/core/jwks/src/index.ts b/core/jwks/src/index.ts new file mode 100644 index 000000000..9e61993b4 --- /dev/null +++ b/core/jwks/src/index.ts @@ -0,0 +1,5 @@ +// Main exports for @fireproof/core-jwks +export * from "./validator.js"; + +// Legacy compatibility - only export the fetchJwks function from fetcher +export { fetchJwks } from "./fetcher.js"; diff --git a/core/jwks/src/validator.ts b/core/jwks/src/validator.ts new file mode 100644 index 000000000..32e784389 --- /dev/null +++ b/core/jwks/src/validator.ts @@ -0,0 +1,278 @@ +import { Result, Option } from "@adviser/cement"; +import { importJWK } from "jose"; + +// Basic JWKS interfaces +export interface JWK { + kty: string; + use?: string; + kid?: string; + alg?: string; + n?: string; // RSA modulus + e?: string; // RSA exponent + crv?: string; // EC curve + x?: string; // EC x coordinate + y?: string; // EC y coordinate + k?: string; // Symmetric key +} + +export interface JWKS { + keys: JWK[]; +} + +export interface JWKSValidationOptions { + allowedKeyTypes?: string[]; + allowedUse?: string[]; + requireKeyId?: boolean; + maxKeys?: number; +} + +export interface KeyValidationResult { + isValid: boolean; + isCurrent: boolean; + keyId?: string; + validationErrors: string[]; + warningMessages: string[]; + originalKey: JWK; +} + +export interface JWKSValidationResult { + isValid: boolean; + validKeysCount: number; + currentKeysCount: number; + totalKeysCount: number; + validationErrors: string[]; + warningMessages: string[]; + keyResults: KeyValidationResult[]; +} + +export class JWKSValidationError extends Error { + constructor( + message: string, + public readonly code: string, + public readonly details?: unknown, + ) { + super(message); + this.name = "JWKSValidationError"; + } +} + +export class JWKSFetchError extends Error { + constructor( + message: string, + public readonly statusCode?: number, + public readonly url?: string, + public readonly originalError?: Error, + ) { + super(message); + this.name = "JWKSFetchError"; + } +} + +// Simple URL builder that works with Clerk +export function buildJWKSUrl(config: string): string { + if (config.startsWith("http://") || config.startsWith("https://")) { + // Validate URL to prevent + return config; + } + + // Handle Clerk-style strings (both "trusted-glowworm-5" and "*.clerk.accounts.dev") + if (config.includes("clerk") || (!config.includes(".") && config.length > 0)) { + const domain = config.includes(".") ? config : `${config}.clerk.accounts.dev`; + return `https://${domain}/.well-known/jwks.json`; + } + + throw new JWKSValidationError("Invalid JWKS configuration", "INVALID_CONFIG", { config }); +} + +// Fetch JWKS with retries +export async function fetchJWKS( + config: string, + options?: { + timeout?: number; + retries?: number; + userAgent?: string; + }, +): Promise> { + try { + const url = buildJWKSUrl(config); + const timeout = options?.timeout ?? 5000; + const retries = options?.retries ?? 3; + const userAgent = options?.userAgent ?? "fireproof-jwks-fetcher/1.0"; + + let lastError: Error | undefined; + + for (let attempt = 0; attempt <= retries; attempt++) { + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + const response = await fetch(url, { + signal: controller.signal, + headers: { + "User-Agent": userAgent, + Accept: "application/json", + "Cache-Control": "no-cache", + }, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + throw new JWKSFetchError(`HTTP ${response.status}: ${response.statusText}`, response.status, url); + } + + const jsonData = await response.json(); + + if (!jsonData?.keys || !Array.isArray(jsonData.keys)) { + throw new JWKSFetchError("Response does not contain a 'keys' array", response.status, url); + } + + return Result.Ok(jsonData as JWKS); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + + // Don't retry on client errors + if (error instanceof JWKSFetchError && error.statusCode && error.statusCode >= 400 && error.statusCode < 500) { + throw error; + } + + // Wait before retry + if (attempt < retries) { + await new Promise((resolve) => setTimeout(resolve, Math.pow(2, attempt) * 1000)); + } + } + } + + throw new JWKSFetchError(`Failed to fetch JWKS after ${retries + 1} attempts`, undefined, url, lastError); + } catch (error) { + if (error instanceof JWKSFetchError) { + return Result.Err(error); + } + return Result.Err( + new JWKSFetchError( + error instanceof Error ? error.message : String(error), + undefined, + undefined, + error instanceof Error ? error : undefined, + ), + ); + } +} + +// Validate individual key +export async function validateJWKSKey(key: JWK, options: JWKSValidationOptions = {}): Promise { + const result: KeyValidationResult = { + isValid: false, + isCurrent: false, + keyId: key.kid, + validationErrors: [], + warningMessages: [], + originalKey: key, + }; + + const allowedKeyTypes = options.allowedKeyTypes ?? ["RSA", "EC"]; + const allowedUse = options.allowedUse ?? ["sig"]; + const requireKeyId = options.requireKeyId ?? true; + + // Basic validations + if (!key.kty) { + result.validationErrors.push("Missing required field 'kty'"); + } else if (!allowedKeyTypes.includes(key.kty)) { + result.validationErrors.push(`Unsupported key type: ${key.kty}`); + } + + if (requireKeyId && !key.kid) { + result.validationErrors.push("Missing required field 'kid'"); + } + + if (key.use && !allowedUse.includes(key.use)) { + result.validationErrors.push(`Unsupported key use: ${key.use}`); + } + + // Key-specific validations + if (key.kty === "RSA" && (!key.n || !key.e)) { + result.validationErrors.push("RSA key missing n or e parameters"); + } + + if (key.kty === "EC" && (!key.crv || !key.x || !key.y)) { + result.validationErrors.push("EC key missing crv, x, or y parameters"); + } + + // Try to import the key + try { + await importJWK(key, key.alg as string | undefined); + result.isCurrent = result.validationErrors.length === 0; + } catch (error) { + result.validationErrors.push(`Key import failed: ${error instanceof Error ? error.message : error}`); + } + + result.isValid = result.validationErrors.length === 0; + return result; +} + +// Validate JWKS +export async function validateJWKS(jwks: JWKS, options: JWKSValidationOptions = {}): Promise { + const result: JWKSValidationResult = { + isValid: false, + validKeysCount: 0, + currentKeysCount: 0, + totalKeysCount: jwks.keys.length, + validationErrors: [], + warningMessages: [], + keyResults: [], + }; + + if (jwks.keys.length === 0) { + result.validationErrors.push("JWKS contains no keys"); + return result; + } + + const maxKeys = options.maxKeys ?? 10; + if (jwks.keys.length > maxKeys) { + result.validationErrors.push(`Too many keys: ${jwks.keys.length} (max: ${maxKeys})`); + return result; + } + + // Validate each key + for (const key of jwks.keys) { + const keyResult = await validateJWKSKey(key, options); + result.keyResults.push(keyResult); + + if (keyResult.isValid) result.validKeysCount++; + if (keyResult.isCurrent) result.currentKeysCount++; + } + + result.isValid = result.validationErrors.length === 0 && result.validKeysCount > 0; + return result; +} + +// Main function +export async function fetchAndValidateJWKS( + config: string, + validationOptions: JWKSValidationOptions = {}, + fetchOptions?: { + timeout?: number; + retries?: number; + userAgent?: string; + }, +): Promise> { + const fetchResult = await fetchJWKS(config, fetchOptions); + if (fetchResult.is_err()) { + return Result.Err(fetchResult.unwrap_err()); + } + + const jwks = fetchResult.unwrap(); + const validation = await validateJWKS(jwks, validationOptions); + + return Result.Ok({ jwks, validation }); +} + +// Utility functions +export function getCurrentKeys(validationResult: JWKSValidationResult): JWK[] { + return validationResult.keyResults.filter((result) => result.isCurrent && result.isValid).map((result) => result.originalKey); +} + +export function findKeyById(jwks: JWKS, keyId: string): Option { + const key = jwks.keys.find((k) => k.kid === keyId); + return key ? Option.Some(key) : Option.None(); +} diff --git a/core/jwks/test-all.sh b/core/jwks/test-all.sh new file mode 100755 index 000000000..f0e9cd0ba --- /dev/null +++ b/core/jwks/test-all.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +printf "🧪 Running JWKS Validator Test Suite" +printf "====================================" + +printf "📋 1. Basic unit tests..." +npx vitest run tests/basic.test.ts --reporter=verbose + +printf "\n🌐 2. Integration tests (with live Clerk endpoint)..." +npx vitest run tests/integration.test.ts --reporter=verbose + +printf "\n📊 3. All tests..." +npx vitest run tests/ --reporter=verbose + +printf "\n✅ Test suite completed!" diff --git a/core/jwks/tests/basic.test.ts b/core/jwks/tests/basic.test.ts new file mode 100644 index 000000000..ebaacc739 --- /dev/null +++ b/core/jwks/tests/basic.test.ts @@ -0,0 +1,19 @@ +import { describe, it, expect } from "vitest"; +import { buildJWKSUrl, JWKSValidationError } from "../src/validator.js"; + +describe("Basic JWKS functionality", () => { + it("should build Clerk URLs correctly", () => { + const result = buildJWKSUrl("trusted-glowworm-5"); + expect(result).toBe("https://trusted-glowworm-5.clerk.accounts.dev/.well-known/jwks.json"); + }); + + it("should handle direct URLs", () => { + const url = "https://example.com/.well-known/jwks.json"; + const result = buildJWKSUrl(url); + expect(result).toBe(url); + }); + + it("should throw on invalid config", () => { + expect(() => buildJWKSUrl("")).toThrow(JWKSValidationError); + }); +}); diff --git a/core/jwks/tests/integration.test.ts b/core/jwks/tests/integration.test.ts new file mode 100644 index 000000000..022c1afd6 --- /dev/null +++ b/core/jwks/tests/integration.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from "vitest"; +import { fetchAndValidateJWKS, buildJWKSUrl } from "../src/validator.js"; + +describe("JWKS Integration Tests", () => { + const runLive = process.env.LIVE_JWKS === "1"; + it.runIf(runLive)("should fetch and validate real Clerk JWKS (integration test)", async () => { + expect(buildJWKSUrl("trusted-glowworm-5")) + .toBe("https://trusted-glowworm-5.clerk.accounts.dev/.well-known/jwks.json"); + }); + + it("should build full Clerk domain URLs", () => { + expect(buildJWKSUrl("trusted-glowworm-5.clerk.accounts.dev")) + .toBe("https://trusted-glowworm-5.clerk.accounts.dev/.well-known/jwks.json"); + }); + + it("should handle direct URLs", () => { + const url = "https://example.com/.well-known/jwks.json"; + expect(buildJWKSUrl(url)).toBe(url); + }); + + // Integration test with real Clerk endpoint (may fail in CI/testing environments) + it("should fetch and validate real Clerk JWKS (integration test)", async () => { + try { + const result = await fetchAndValidateJWKS("trusted-glowworm-5", { + allowedKeyTypes: ["RSA", "EC"], + allowedUse: ["sig"], + requireKeyId: true, + maxKeys: 10 + }, { + timeout: 5000, + retries: 1 + }); + + if (result.is_ok()) { + const { jwks, validation } = result.unwrap(); + + // Basic structure checks + expect(jwks).toHaveProperty("keys"); + expect(Array.isArray(jwks.keys)).toBe(true); + expect(validation.totalKeysCount).toBeGreaterThan(0); + + // Each key should have basic properties + if (jwks.keys.length > 0) { + const firstKey = jwks.keys[0]; + expect(firstKey).toHaveProperty("kty"); + expect(firstKey).toHaveProperty("kid"); + expect(["RSA", "EC", "oct", "OKP"]).toContain(firstKey.kty); + } + + // Validation should work + expect(validation).toHaveProperty("isValid"); + expect(validation).toHaveProperty("currentKeysCount"); + + console.log(`✅ Live test: ${validation.currentKeysCount}/${validation.totalKeysCount} keys are current`); + } else { + // Log error but don't fail test (network issues in CI) + console.warn(`⚠️ Live test failed: ${result.unwrap_err().message}`); + expect(result.unwrap_err().name).toMatch(/JWKSFetchError|JWKSValidationError/); + } + } catch (error) { + console.warn(`⚠️ Live test exception: ${error instanceof Error ? error.message : error}`); + // Don't fail the test - network issues are expected in some environments + } + }, 10000); // 10 second timeout +}); diff --git a/core/jwks/tsconfig.json b/core/jwks/tsconfig.json new file mode 100644 index 000000000..d6850fa87 --- /dev/null +++ b/core/jwks/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "target": "ES2020", + "module": "ES2020", + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "moduleResolution": "node", + "skipLibCheck": true + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "dist", + "tests" + ] +} diff --git a/core/jwks/vitest.config.ts b/core/jwks/vitest.config.ts new file mode 100644 index 000000000..8e730d505 --- /dev/null +++ b/core/jwks/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + }, +}); diff --git a/core/keybag/coerce-keyed-item.ts b/core/keybag/coerce-keyed-item.ts new file mode 100644 index 000000000..af4873f56 --- /dev/null +++ b/core/keybag/coerce-keyed-item.ts @@ -0,0 +1,118 @@ +import { + V1StorageKeyItem, + V2StorageKeyItem, + V2KeysItem, + LegacyKeyedItem, + KeyedItemSchema, + V1StorageKeyItemSchema, + KeyedItem, + KeyBagIf, +} from "@fireproof/core-types-base"; +import { toKeyWithFingerPrint, coerceMaterial } from "./key-with-fingerprint.js"; +import { Logger } from "@adviser/cement"; + +export type ModifiedKeyedItem = KeyedItem & { modified?: boolean }; + +export interface CoerceCtx { + readonly keybag: KeyBagIf; + readonly logger: Logger; +} + +async function toV2KeysItem(ctx: CoerceCtx, ki: Partial): Promise { + if (!ki.name) { + throw ctx.logger.Error().Msg("toV2KeysItem: name is missing").AsError(); + } + if ("key" in ki && ki.key && ki.name) { + // v1 + const fpr = (await toKeyWithFingerPrint(ctx.keybag, coerceMaterial(ctx.keybag, ki.key), true)).Ok().fingerPrint; + return { + modified: true, + id: ki.name, + clazz: "V2StorageKeyItem", + item: { + name: ki.name, + keys: { + [fpr]: { + key: ki.key, + fingerPrint: fpr, + default: true, + }, + }, + }, + }; + } + // fix default + let defKI: V2KeysItem | undefined; + let foundDefKI = false; + let result: V2StorageKeyItem; + if ("keys" in ki && ki.keys) { + result = { + name: ki.name, + keys: ki.keys, + }; + } else { + result = { + name: ki.name, + keys: {}, + }; + } + for (const i of Object.entries(result.keys)) { + if (i[0] !== i[1].fingerPrint) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete result.keys[i[0]]; + result.keys[i[1].fingerPrint] = i[1]; + ctx.logger.Warn().Str("name", ki.name).Msg("fingerPrint mismatch fixed"); + } + if (defKI === undefined) { + defKI = i[1]; + } + if (!foundDefKI && i[1].default) { + defKI = i[1]; + foundDefKI = true; + } else { + (i[1] as { default: boolean }).default = false; + } + } + // if (defKI) { + // result.keys["*"] = defKI; + // } + return { + id: result.name, + clazz: "V2StorageKeyItem", + item: result, + }; +} + +export async function coerceKeyedItem(ctx: CoerceCtx, item: LegacyKeyedItem | undefined): Promise { + if (!item) { + return undefined; + } + if ("clazz" in item) { + const r = KeyedItemSchema.safeParse(item); + return r.success ? item : undefined; + } + // very private only for legacy + function isV1StorageKeyItem(item: LegacyKeyedItem | undefined): item is V1StorageKeyItem { + if (!item) { + return false; + } + const r = V1StorageKeyItemSchema.safeParse(item); + return r.success; + } + function isV2StorageKeysItem(item: LegacyKeyedItem): item is V2StorageKeyItem { + return !!(item as V2StorageKeyItem).keys; + } + + if (isV1StorageKeyItem(item)) { + return toV2KeysItem(ctx, item); + } + + if (isV2StorageKeysItem(item)) { + return { + id: item.name, + clazz: "V2StorageKeyItem", + item, + }; + } + return undefined; +} diff --git a/core/keybag/index.ts b/core/keybag/index.ts index f16098b81..551da79a4 100644 --- a/core/keybag/index.ts +++ b/core/keybag/index.ts @@ -1,2 +1,5 @@ -export * from "./key-bag.js"; +export * from "./key-bag-fingerprint-item.js"; export * from "./key-bag-memory.js"; +export * from "./key-bag-setup.js"; +export * from "./key-bag.js"; +export * from "./key-with-fingerprint.js"; diff --git a/core/keybag/internal-keys-by-fingerprint.ts b/core/keybag/internal-keys-by-fingerprint.ts new file mode 100644 index 000000000..588ee683b --- /dev/null +++ b/core/keybag/internal-keys-by-fingerprint.ts @@ -0,0 +1,364 @@ +import { KeyedResolvOnce, Logger, Result } from "@adviser/cement"; +import { ensureLogger, hashObject } from "@fireproof/core-runtime"; +import { + KeyedV2StorageKeyItem, + KeyedV2StorageKeyItemSchema, + KeysByFingerprint, + KeysItem, + KeyUpsertResult, + KeyWithFingerPrint, + V2StorageKeyItem, +} from "@fireproof/core-types-base"; +import { coerceFingerPrint, coerceMaterial, InternalKeyWithFingerPrint, toKeyWithFingerPrint } from "./key-with-fingerprint.js"; +import { KeyBag } from "./key-bag.js"; +import { base58btc } from "multiformats/bases/base58"; +import { coerceKeyedItem as coerceKeyedItemWithVersionUpdate } from "./coerce-keyed-item.js"; +import z from "zod"; + +// export type InternalKeysItem = Omit & { +// readonly keys: Record; +// readonly id: string; +// }; + +interface InternalKeysByFingerprintFromOpts { + readonly keybag: KeyBag; + readonly name: string; + // readonly keysItem: InternalKeysItem; + readonly modified?: boolean; + readonly opts: { + readonly failIfNotFound?: boolean; + readonly materialStrOrUint8?: string | Uint8Array; + readonly def?: boolean; + }; +} + +type InternalKeyUpsertResult = Omit & { kfp: InternalKeyWithFingerPrint} + +export class InternalKeysByFingerprint implements KeysByFingerprint { + readonly keybag: KeyBag; + readonly name: string; + readonly id: string; + readonly lookUp = new KeyedResolvOnce>(); + // readonly keysItem: InternalKeysItem; + readonly logger: Logger; + + async ensureMaterial( + materialStrOrUint8?: string | Uint8Array, + def?: boolean, + modified?: boolean, + ): Promise> { + if (!modified) { + return Result.Ok(this); + } + if (!materialStrOrUint8) { + return Result.Ok(this); + } + const r = await this.upsert(materialStrOrUint8, def, modified); + if (r.isErr()) { + return Result.Err(r); + } + return Result.Ok(this); + } + + // implicit migration from V1 to V2 + private async toKeysItem(ki: V2StorageKeyItem): Promise { + return Promise.all( + Array.from(Object.values(ki.keys)).map( + async (i) => + new InternalKeyWithFingerPrint({ + fingerPrint: i.fingerPrint, + key: await this.keybag.subtleKey(i.key), + material: { key: base58btc.decode(i.key), keyStr: i.key }, + default: i.default || false, + }), + ), + // [ + // i.fingerPrint, + // await this.keybag.subtleKey(i.key), + // { key: base58btc.decode(i.key), keyStr: i.key }, + // i.default || false, + // ] satisfies [string, CTCryptoKey, KeyMaterial, boolean], + ); + // ).then((i) => i.map((j) => new InternalKeyWithFingerPrint(...j))) + // ).reduce( + // (acc, i) => { + // acc[i.fingerPrint] = i; + // if (i.default) { + // acc["*"] = i; + // } + // return acc; + // }, + // {} as KeysItem + // ); + // return { + // id: this.id, + // name: ki.name, + // keys, + // }; + } + + // is assuming it will not called concurrent or multiple per name + async load(opts: InternalKeysByFingerprintFromOpts["opts"]): Promise> { + const oProvKeysResult = await this.keybag.getRawObj(this.name); + if (oProvKeysResult.IsNone() && opts.failIfNotFound) { + return this.logger.Debug().Msg("failIfNotFound getRawObj").ResultError(); + } + // const provKeysResult = oProvKeysResult + if (oProvKeysResult.IsSome() && !oProvKeysResult.unwrap().success) { + const tsHelp = oProvKeysResult.unwrap(); + if (!tsHelp.success) { + return this.logger + .Error() + .Any({ error: z.formatError(tsHelp.error) }) + .Msg("not LegacyKeyItem") + .ResultError(); + } + } + if (oProvKeysResult.IsSome()) { + const provKeysResult = oProvKeysResult.unwrap(); + const cki = await coerceKeyedItemWithVersionUpdate(this, provKeysResult.data); + if (!cki) { + return this.logger.Error().Any({ item: provKeysResult.data }).Msg("coerce error").ResultError(); + } + const v2StorageResult = KeyedV2StorageKeyItemSchema.safeParse(cki); + if (!v2StorageResult.success) { + return this.logger + .Error() + .Any({ name: this.name, item: provKeysResult.data, error: z.formatError(v2StorageResult.error) }) + .Msg("not V2KeysItems") + .ResultError(); + } + // const keyedItem = { ...v2StorageResult.data, modified: cki.modified }; + + // const v2KeysItem = await this.toV2KeysItem(provKeysItem); + // const keys = Object.values(keyedItem.item.keys).length; + // if (iopts.opts.failIfNotFound && keys === 0) { + // return Result.Err(this.logger.Debug().Str("name", this.name).Msg("no keys getNamedKey").AsError()); + // } + await this.toKeysItem(v2StorageResult.data.item) + .then((items) => + items.map(async (item, idx) => + this.upsert((await item.extract()).key, item.default, cki.modified && idx === items.length - 1), + ), + ) + .then((items) => Promise.all(items)); + } + // this.lookUp.get(i.fingerPrint).once(() => { + // th + // }); + // } + return this.ensureMaterial(opts.materialStrOrUint8 ?? this.keybag.rt.crypto.randomBytes(this.keybag.rt.keyLength), true); + + // if (keys > 0) { + // this.logger + // .Debug() + // .Str("id", id) + // .Str("name", this.name) + // .Any("fprs", Object.keys(keyedItem.item.keys)) + // .Msg("fingerPrint getNamedKey"); + // return InternalKeysByFingerprint.from({ ...this, keysItem: this.keysItem, opts: iopts, modified: keyedItem.modified }); + // } else if (iopts.failIfNotFound) { + // return this.logger.Debug().Str("id", id).Str("name", this.name).Msg("failIfNotFound getNamedKey").ResultError(); + // } + // // lets create a key from the material + // this.keysItem = { name: this.name, keys: {}, id }; + // const rKbfp = await InternalKeysByFingerprint.from({ + // ...this, + // keysItem: this.keysItem, + // opts: { + // materialStrOrUint8: iopts.materialStrOrUint8 ?? this.keybag.rt.crypto.randomBytes(this.keybag.rt.keyLength), + // def: true, + // }, + // modified: true + // }); + // if (rKbfp.isErr()) { + // return rKbfp; + // } + // this.logger + // .Debug() + // .Str("id", id) + // .Str("name", this.name) + // .Any("KeyItems", await rKbfp.Ok().asV2StorageKeyItem()) + // .Msg("createKey getNamedKey-post"); + // return rKbfp; + } + + static async from(kbo: InternalKeysByFingerprintFromOpts): Promise> { + const kbf = new InternalKeysByFingerprint(kbo.keybag, kbo.name); + return kbf.load(kbo.opts); + // retu + // if (rLoad.isErr()) { + // return Result.Err(rLoad); + // } + + // let modified = !!kbo.modified; + // // reverse to keep the first key as default + + // for (const [_, ki] of Object.entries(kbo.keysItem.keys).reverse()) { + // const result = await kbf.upsertNoStore((await ki.asKeysItem()).key, ki.default); + // if (result.isErr()) { + // return Result.Err(result); + // } + // modified ||= result.Ok().modified; + // // if (result.Ok().modified) { + // // throw keyBag.logger.Error().Msg("KeyBag: keysByFingerprint: mismatch unexpected").AsError(); + // // } + // const kur = result.Ok(); + // if (isKeyUpsertResultModified(kur)) { + // if (kur.kfp.fingerPrint !== ki.fingerPrint) { + // return kbo.keybag.logger + // .Error() + // .Any("fprs", { + // fromStorage: ki.fingerPrint, + // calculated: kur.kfp.fingerPrint, + // }) + // .Msg("KeyBag: keysByFingerprint: mismatch") + // .ResultError(); + // } + // } + // } + // let rKur: Result | undefined; + // if (kbo.opts.materialStrOrUint8) { + // // key created if needed + // rKur = await kbf.upsertNoStore(kbo.opts.materialStrOrUint8, kbo.opts.def); + // if (rKur.isErr()) { + // return Result.Err(rKur); + // } + // } + // if (rKur?.Ok().modified || modified) { + // // persit + // await kbo.keybag.setRawObj({ + // id: kbf.name, + // clazz: "V2StorageKeyItem", + // item: await kbf.asV2StorageKeyItem(), + // } satisfies KeyedV2StorageKeyItem); + // } + // return Result.Ok(kbf); + } + + private constructor(keyBag: KeyBag, name: string) { + this.id = keyBag.rt.sthis.nextId().str; + this.logger = ensureLogger(keyBag.rt.sthis, `InternalKeysByFingerprint:${name}:${this.id}`); + this.keybag = keyBag; + this.name = name; + } + + async get(fingerPrint?: string | Uint8Array): Promise { + fingerPrint = coerceFingerPrint(this.keybag, fingerPrint) || "*"; + const ret = this.lookUp.get(fingerPrint).value; + if (!ret) { + this.keybag.logger + .Warn() + .Any({ fprs: this.lookUp.values().map((i) => i.value.Ok().Ok().kfp.fingerPrint), fpr: fingerPrint }) + .Msg("keysByFingerprint:get: not found"); + } + return undefined; + } + async upsert(materialStrOrUint8: string | Uint8Array, def?: boolean, modified?: boolean): Promise> { + const rKur = await this.upsertNoStore(materialStrOrUint8, def); + if (rKur.isErr()) { + return Result.Err(rKur); + } + if (rKur.Ok().modified || modified) { + await this.keybag.setRawObj({ + id: this.name, + clazz: "V2StorageKeyItem", + item: await this.asV2StorageKeyItem(), + } satisfies KeyedV2StorageKeyItem); + } + return rKur; + } + + async upsertNoStore(materialStrOrUint8: string | Uint8Array, def?: boolean): Promise> { + // if (!materialStrOrUint8) { + // return Result.Ok({ + // modified: false, + // }); + // } + const material = coerceMaterial(this.keybag, materialStrOrUint8); + def = !!def; + const rKfp = await toKeyWithFingerPrint(this.keybag, material, !!def); + if (rKfp.isErr()) { + return Result.Err(rKfp); + } + + // critical section + const kfp = rKfp.Ok(); + this.lookUp.unget(kfp.fingerPrint); + return this.lookUp.get(kfp.fingerPrint).once(async () => { + const preHash = await hashObject(await this.asV2StorageKeyItem()); + let val = this.lookUp.get(kfp.fingerPrint).value?.Ok(); + if (val) { + // do not update default if not needed + if (val.kfp.default === def) { + return Result.Ok({ + modified: false, + kfp: val.kfp, + }); + } + } else { + val = { + modified: true, + kfp: new InternalKeyWithFingerPrint({ + default: def, + fingerPrint: kfp.fingerPrint, + key: kfp.key, + material, + }), + }; + } + const keyItems = this.lookUp.values().map((i) => i.value.Ok()); + if (def) { + for (const rI of keyItems) { + const i = rI.Ok().kfp + if (i.default && i.fingerPrint !== kfp.fingerPrint) { + // only update if it's not ourself --> avoid deadlock + this.lookUp.unget(i.fingerPrint); + this.lookUp.get(i.fingerPrint).once(() => Result.Ok({ + modified: true, + kfp: i.setDefault(false), + })); + } + } + } + const postHash = await hashObject(this.asV2StorageKeyItem()); + const ret = Result.Ok({ + modified: preHash !== postHash, + kfp: val.kfp, + }) + if (def || keyItems.length === 0) { + val.kfp.setDefault(true); + this.lookUp.unget("*"); + this.lookUp.get("*").once(() => ret); + } + return ret + }); + } + + async asV2StorageKeyItem(): Promise { + const kis = await Promise.all( + this.lookUp + .values() + .filter((i) => i.key !== "*") + .map((i) => i.value.Ok().Ok().kfp.asKeysItem()), + ); + return { + name: this.name, + keys: kis.reduce( + (acc, i) => { + acc[i.fingerPrint] = i; + return acc; + }, + {} as Record, + ), + }; + } + + // async extract() { + // const ext = new Uint8Array((await this.rt.crypto.exportKey("raw", named.key)) as ArrayBuffer); + // return { + // key: ext, + // keyStr: base58btc.encode(ext), + // }; + // } +} diff --git a/core/keybag/key-bag-fingerprint-item.ts b/core/keybag/key-bag-fingerprint-item.ts new file mode 100644 index 000000000..bac1e6945 --- /dev/null +++ b/core/keybag/key-bag-fingerprint-item.ts @@ -0,0 +1,37 @@ +import { ResolveOnce, ResolveSeq, Result } from "@adviser/cement"; +import { KeysByFingerprint } from "@fireproof/core-types-base"; +import { InternalKeysByFingerprint } from "./internal-keys-by-fingerprint.js"; +import { KeyBag } from "./key-bag.js"; + +interface keyBagFingerprintItemGetOpts { + readonly failIfNotFound: boolean; + readonly materialStrOrUint8?: string | Uint8Array; + readonly def?: boolean; +} + +export class InternalKeyBagFingerprintItem { + readonly name: string; + readonly keybag: KeyBag; + readonly keysByFingerprint = new ResolveOnce>(); + + readonly seq = new ResolveSeq>(); + + constructor(keybag: KeyBag, name: string) { + this.keybag = keybag; + this.name = name; + } + + async getNamedKey(opts: keyBagFingerprintItemGetOpts): Promise> { + return this.seq.add(() => { + return this.keysByFingerprint.once(() => { + return InternalKeysByFingerprint.from({ keybag: this.keybag, opts, name: this.name }); + }).then((r) => { + if (r.isErr()) { + this.keysByFingerprint.reset(); + return r; + } + return r + }); + }); + } +} diff --git a/core/keybag/key-bag-memory.ts b/core/keybag/key-bag-memory.ts index 6f17c5a9f..f45ef3902 100644 --- a/core/keybag/key-bag-memory.ts +++ b/core/keybag/key-bag-memory.ts @@ -1,5 +1,5 @@ import { URI } from "@adviser/cement"; -import { KeyBagProvider, V2KeysItem, V1StorageKeyItem, SuperThis } from "@fireproof/core-types-base"; +import { KeyBagProvider, SuperThis } from "@fireproof/core-types-base"; const memoryKeyBag = new Map(); @@ -35,17 +35,21 @@ export class KeyBagProviderMemory implements KeyBagProvider { return Promise.resolve(); } - async get(id: string): Promise { + async get(id: string): Promise | undefined> { const binKeyItem = memoryKeyBag.get(this.key(id)); if (binKeyItem) { - const ki = JSON.parse(this.sthis.txt.decode(binKeyItem)); - return ki; + try { + const ki = JSON.parse(this.sthis.txt.decode(binKeyItem)); + return ki; + } catch (e) { + /* */ + } } return undefined; } - async set(item: V2KeysItem): Promise { + async set(id: string, item: NonNullable): Promise { const p = this.sthis.txt.encode(JSON.stringify(item, null, 2)); - memoryKeyBag.set(this.key(item.name), p); + memoryKeyBag.set(this.key(id), p); } } diff --git a/core/keybag/key-bag-setup.ts b/core/keybag/key-bag-setup.ts new file mode 100644 index 000000000..4a67d85ce --- /dev/null +++ b/core/keybag/key-bag-setup.ts @@ -0,0 +1,147 @@ +import { URI, runtimeFn, toCryptoRuntime, KeyedResolvOnce } from "@adviser/cement"; +import { KeyBagProviderFile } from "@fireproof/core-gateways-file"; +import { ensureLogger } from "@fireproof/core-runtime"; +import { + KeyedItem, + KeyedDeviceIdKeyBagItem, + KeyedDeviceIdKeyBagItemSchema, + KeyedV2StorageKeyItem, + KeyedV2StorageKeyItemSchema, + SuperThis, + KeyBagProvider, + KeyBagOpts, + KeyBagRuntime, + KeyBagIf, +} from "@fireproof/core-types-base"; +import { KeyBagProviderMemory } from "./key-bag-memory.js"; +import { KeyBag } from "./key-bag.js"; + +export function isDeviceIdKeyBagItem(item: KeyedItem | undefined): item is KeyedDeviceIdKeyBagItem { + if (!item) { + return false; + } + const r = KeyedDeviceIdKeyBagItemSchema.safeParse(item); + return r.success; +} + +export function isV2StorageKeyItem(item: KeyedItem | undefined): item is KeyedV2StorageKeyItem { + if (!item) { + return false; + } + const r = KeyedV2StorageKeyItemSchema.safeParse(item); + return r.success; +} + +export type KeyBackProviderFactory = (url: URI, sthis: SuperThis) => Promise; + +export interface KeyBagProviderFactoryItem { + readonly protocol: string; + // if this is set the default protocol selection is overridden + readonly override?: boolean; + readonly factory: KeyBackProviderFactory; +} + +const keyBagProviderFactories = new Map( + [ + { + protocol: "file:", + factory: async (url: URI, sthis: SuperThis) => { + return new KeyBagProviderFile(url, sthis); + }, + }, + { + protocol: "indexeddb:", + factory: async (url: URI, sthis: SuperThis) => { + const { KeyBagProviderImpl } = await import("@fireproof/core-gateways-indexeddb"); + return new KeyBagProviderImpl(url, sthis); + }, + }, + { + protocol: "memory:", + factory: async (url: URI, sthis: SuperThis) => { + return new KeyBagProviderMemory(url, sthis); + }, + }, + ].map((i) => [i.protocol, i]), +); + +export function registerKeyBagProviderFactory(item: KeyBagProviderFactoryItem) { + const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":"; + keyBagProviderFactories.set(protocol, { + ...item, + protocol, + }); +} + +export function defaultKeyBagUrl(sthis: SuperThis): URI { + let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL"); + let url: URI; + if (runtimeFn().isBrowser) { + url = URI.from(bagFnameOrUrl || "indexeddb://fp-keybag"); + } else { + if (!bagFnameOrUrl) { + const home = sthis.env.get("HOME"); + bagFnameOrUrl = `${home}/.fireproof/keybag`; + url = URI.from(`file://${bagFnameOrUrl}`); + } else { + url = URI.from(bagFnameOrUrl); + } + } + const logger = ensureLogger(sthis, "defaultKeyBagUrl"); + logger.Debug().Url(url).Msg("from env"); + return url; +} + +export function defaultKeyBagOpts(sthis: SuperThis, kbo?: Partial): KeyBagRuntime { + kbo = kbo || {}; + if (kbo.keyRuntime) { + return kbo.keyRuntime; + } + const logger = ensureLogger(sthis, "KeyBag"); + let url: URI; + if (kbo.url) { + url = URI.from(kbo.url); + logger.Debug().Url(url).Msg("from opts"); + } else { + let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL"); + if (runtimeFn().isBrowser) { + url = URI.from(bagFnameOrUrl || "indexeddb://fp-keybag"); + } else { + if (!bagFnameOrUrl) { + const home = sthis.env.get("HOME"); + bagFnameOrUrl = `${home}/.fireproof/keybag`; + url = URI.from(`file://${bagFnameOrUrl}`); + } else { + url = URI.from(bagFnameOrUrl); + } + } + logger.Debug().Url(url).Msg("from env"); + } + const kitem = keyBagProviderFactories.get(url.protocol); + if (!kitem) { + throw logger.Error().Url(url).Msg("unsupported protocol").AsError(); + } + + if (url.hasParam("masterkey")) { + throw logger.Error().Url(url).Msg("masterkey is not supported").AsError(); + } + + return { + url, + crypto: kbo.crypto || toCryptoRuntime({}), + sthis, + logger, + keyLength: kbo.keyLength || 16, + getBagProvider: () => kitem.factory(url, sthis), + id: () => { + return url.toString(); + }, + }; +} + +const _keyBags = new KeyedResolvOnce(); +export async function getKeyBag(sthis: SuperThis, kbo: Partial = {}): Promise { + await sthis.start(); + const rt = defaultKeyBagOpts(sthis, kbo); + return _keyBags.get(rt.id()).once(() => KeyBag.create(rt)); +} diff --git a/core/keybag/key-bag.ts b/core/keybag/key-bag.ts index 267aab270..34cfd09b2 100644 --- a/core/keybag/key-bag.ts +++ b/core/keybag/key-bag.ts @@ -1,464 +1,52 @@ -import { - CTCryptoKey, - KeyedResolvOnce, - Logger, - ResolveOnce, - ResolveSeq, - Result, - runtimeFn, - toCryptoRuntime, - URI, -} from "@adviser/cement"; -import { - isKeyUpsertResultModified, - KeyMaterial, - KeysByFingerprint, - KeyUpsertResult, - KeyWithFingerPrint, -} from "@fireproof/core-types-blockstore"; -import { ensureLogger, hashObject } from "@fireproof/core-runtime"; -import { base58btc } from "multiformats/bases/base58"; +import { Lazy, ResolveOnce, Logger, URI, Result, Option, KeyedResolvOnce, exception2Result } from "@adviser/cement"; +import { hashString, ensureLogger, hashObject } from "@fireproof/core-runtime"; import { KeyBagIf, - KeyBagOpts, - KeyBagProvider, KeyBagRuntime, - V2KeysItem, PARAM, - SuperThis, - V1StorageKeyItem, - V2StorageKeyItem, - KeysItem, + DeviceIdResult, + JWKPrivate, + DeviceIdKeyBagItem, + KeysByFingerprint, + LegacyKeyedItemSchema, + KeyedItem, + KeyedItemSchema, + JWTPayload, + KeyedJwtKeyBagItem, + JWTResult, + KeyedJwtKeyBagItemSchema, + KeyedDeviceIdKeyBagItem, + KeyedDeviceIdKeyBagItemSchema, + JWTPayloadSchema, } from "@fireproof/core-types-base"; -import { KeyBagProviderFile } from "@fireproof/core-gateways-file"; -import { KeyBagProviderMemory } from "./key-bag-memory.js"; - -class keyWithFingerPrint implements KeyWithFingerPrint { - readonly default: boolean; - readonly fingerPrint: string; - readonly key: CTCryptoKey; - #material: KeyMaterial; - - constructor(fpr: string, key: CTCryptoKey, material: KeyMaterial, def: boolean) { - this.fingerPrint = fpr; - this.default = def; - this.key = key; - this.#material = material; - } - - extract(): Promise { - if (this.key.extractable) { - return Promise.resolve(this.#material); - } - throw new Error("Key is not extractable"); - } - - async asV2StorageKeyItem(): Promise { - return { - default: this.default, - fingerPrint: this.fingerPrint, - key: this.#material.keyStr, - }; - } -} -type keysItem = Omit & { - readonly keys: Record; - readonly id: string; -}; - -export function coerceMaterial(kb: KeyBagIf, material: string | Uint8Array): KeyMaterial { - let keyMaterial: Uint8Array; - if (typeof material === "string") { - keyMaterial = base58btc.decode(material); - } else if (material instanceof Uint8Array) { - keyMaterial = material; - } else { - throw kb.logger.Error().Msg("material must be string or Uint8Array").AsError(); - } - return { - key: keyMaterial, - keyStr: base58btc.encode(keyMaterial), - }; -} - -export async function toKeyWithFingerPrint( - keybag: KeyBagIf, - material: KeyMaterial, - def: boolean, -): Promise> { - const key = await keybag.subtleKey(material.key); - const fpr = base58btc.encode(new Uint8Array(await keybag.rt.crypto.digestSHA256(material.key))); - return Result.Ok(new keyWithFingerPrint(fpr, key, material, def)); -} - -export async function toV2StorageKeyItem(keybag: KeyBagIf, material: KeyMaterial, def: boolean): Promise { - const rKfp = await toKeyWithFingerPrint(keybag, material, def); - if (rKfp.isErr()) { - throw rKfp; - } - return { - default: def, - fingerPrint: rKfp.Ok().fingerPrint, - key: material.keyStr, - }; -} - -function coerceFingerPrint(kb: KeyBagIf, fingerPrint?: string | Uint8Array): string | undefined { - if (fingerPrint instanceof Uint8Array) { - fingerPrint = base58btc.encode(fingerPrint); - } - return fingerPrint; -} - -interface KeysByFingerprintFromOpts { - readonly keybag: KeyBag; - readonly prov: KeyBagProvider; - readonly keysItem: keysItem; - readonly modified?: boolean; - readonly opts: { - readonly materialStrOrUint8?: string | Uint8Array; - readonly def?: boolean; - }; -} - -class keysByFingerprint implements KeysByFingerprint { - readonly keybag: KeyBag; - readonly keysItem: keysItem; - readonly prov: KeyBagProvider; - - static async from(kbo: KeysByFingerprintFromOpts): Promise> { - const kbf = new keysByFingerprint(kbo.keybag, kbo.prov, kbo.keysItem); - let modified = !!kbo.modified; - // reverse to keep the first key as default - - for (const [_, ki] of Object.entries(kbo.keysItem.keys).reverse()) { - const result = await kbf.upsertNoStore((await ki.asV2StorageKeyItem()).key, ki.default); - if (result.isErr()) { - throw result; - } - modified = modified || result.Ok().modified; - // if (result.Ok().modified) { - // throw keyBag.logger.Error().Msg("KeyBag: keysByFingerprint: mismatch unexpected").AsError(); - // } - const kur = result.Ok(); - if (isKeyUpsertResultModified(kur)) { - if (kur.kfp.fingerPrint !== ki.fingerPrint) { - throw kbo.keybag.logger - .Error() - .Any("fprs", { - fromStorage: ki.fingerPrint, - calculated: kur.kfp.fingerPrint, - }) - .Msg("KeyBag: keysByFingerprint: mismatch") - .AsError(); - } - } - } - let rKur: Result | undefined; - if (kbo.opts.materialStrOrUint8) { - // key created if needed - rKur = await kbf.upsertNoStore(kbo.opts.materialStrOrUint8, kbo.opts.def); - if (rKur.isErr()) { - throw rKur; - } - } - if (rKur?.Ok().modified || modified) { - // persit - await kbo.prov.set(await kbf.asV2KeysItem()); - } - return Result.Ok(kbf); - } - - private constructor(keyBag: KeyBag, prov: KeyBagProvider, keysItem: keysItem) { - this.prov = prov; - this.keybag = keyBag; - this.keysItem = keysItem; - } - - get id(): string { - return this.keysItem.id; - } - - get name(): string { - return this.keysItem.name; - } - - async get(fingerPrint?: string | Uint8Array): Promise { - fingerPrint = coerceFingerPrint(this.keybag, fingerPrint) || "*"; - const found = this.keysItem.keys[fingerPrint]; - if (found) { - return found; - } - this.keybag.logger - .Warn() - .Any({ fprs: Object.keys(this.keysItem.keys), fpr: fingerPrint, name: this.name, id: this.id }) - .Msg("keysByFingerprint:get: not found"); - return undefined; - } - async upsert(materialStrOrUint8: string | Uint8Array, def?: boolean): Promise> { - const rKur = await this.upsertNoStore(materialStrOrUint8, def); - if (rKur.isErr()) { - return Result.Err(rKur); - } - if (rKur.Ok().modified) { - await this.prov.set(await this.asV2KeysItem()); - } - return rKur; - } - - async upsertNoStore(materialStrOrUint8: string | Uint8Array, def?: boolean): Promise> { - if (!materialStrOrUint8) { - return Result.Ok({ - modified: false, - }); - } - const material = coerceMaterial(this.keybag, materialStrOrUint8); - def = !!def; - const rKfp = await toKeyWithFingerPrint(this.keybag, material, !!def); - if (rKfp.isErr()) { - return Result.Err(rKfp); - } - const preHash = await hashObject(await this.asV2KeysItem()); - const kfp = rKfp.Ok(); - let found = this.keysItem.keys[kfp.fingerPrint]; - if (found) { - if (found.default === def) { - return Result.Ok({ - modified: false, - kfp: found, - }); - } - } else { - found = new keyWithFingerPrint(kfp.fingerPrint, kfp.key, material, def); - } - if (def) { - for (const i of Object.values(this.keysItem.keys)) { - (i as { default: boolean }).default = false; - } - } - if (def || Object.keys(this.keysItem.keys).length === 0) { - (found as { default: boolean }).default = true; - this.keysItem.keys["*"] = found; - } - this.keysItem.keys[kfp.fingerPrint] = found; - - const postHash = await hashObject(this.asV2KeysItem()); - return Result.Ok({ - modified: preHash !== postHash, - kfp: found, - }); - } - - async asV2KeysItem(): Promise { - const my = { ...this.keysItem.keys }; - delete my["*"]; - const kis = await Promise.all(Object.values(my).map((i) => i.asV2StorageKeyItem())); - return { - name: this.name, - keys: kis.reduce( - (acc, i) => { - acc[i.fingerPrint] = i; - return acc; - }, - {} as Record, - ), - }; - } - - // async extract() { - // const ext = new Uint8Array((await this.rt.crypto.exportKey("raw", named.key)) as ArrayBuffer); - // return { - // key: ext, - // keyStr: base58btc.encode(ext), - // }; - // } -} - -interface keyBagFingerprintItemGetOpts { - readonly failIfNotFound: boolean; - readonly materialStrOrUint8?: string | Uint8Array; - readonly def?: boolean; -} - -export interface V2KeysItemUpdated { - readonly modified: boolean; - readonly keysItem: V2KeysItem; -} - -class KeyBagFingerprintItem { - readonly name: string; - readonly keybag: KeyBag; - readonly prov: KeyBagProvider; - readonly logger: Logger; - keysItem?: keysItem; - - readonly #seq: ResolveSeq> = new ResolveSeq>(); - - constructor(keybag: KeyBag, prov: KeyBagProvider, name: string) { - this.keybag = keybag; - this.logger = ensureLogger(keybag.rt.sthis, `KeyBagFingerprintItem:${name}`); - this.name = name; - this.prov = prov; - } - - // implicit migration from V1 to V2 - private async toV2KeysItem(ki: Partial): Promise { - if (!ki.name) { - throw this.logger.Error().Msg("toV2KeysItem: name is missing").AsError(); - } - if ("key" in ki && ki.key && ki.name) { - const fpr = (await toKeyWithFingerPrint(this.keybag, coerceMaterial(this.keybag, ki.key), true)).Ok().fingerPrint; - return { - modified: true, - keysItem: { - name: ki.name, - keys: { - [fpr]: { - key: ki.key, - fingerPrint: fpr, - default: true, - }, - }, - }, - }; - } - // fix default - let defKI: V2StorageKeyItem | undefined; - let foundDefKI = false; - let result: V2KeysItem; - if ("keys" in ki && ki.keys) { - result = { - name: ki.name, - keys: ki.keys, - }; - } else { - result = { - name: ki.name, - keys: {}, - }; - } - for (const i of Object.entries(result.keys)) { - if (i[0] !== i[1].fingerPrint) { - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete result.keys[i[0]]; - result.keys[i[1].fingerPrint] = i[1]; - this.logger.Warn().Str("name", ki.name).Msg("fingerPrint mismatch fixed"); - } - if (defKI === undefined) { - defKI = i[1]; - } - if (!foundDefKI && i[1].default) { - defKI = i[1]; - foundDefKI = true; - } else { - (i[1] as { default: boolean }).default = false; - } - } - // if (defKI) { - // result.keys["*"] = defKI; - // } - return { - modified: false, - keysItem: result, - }; - } +import { base58btc } from "multiformats/bases/base58"; +import { InternalKeyBagFingerprintItem } from "./key-bag-fingerprint-item.js"; +import { decodeJwt, JWK, jwtVerify, JWTVerifyOptions, KeyObject } from "jose"; - private async toKeysItem(ki: V2KeysItem, id: string): Promise { - const keys = ( - await Promise.all( - Array.from(Object.values(ki.keys)).map( - async (i) => - [ - i.fingerPrint, - await this.keybag.subtleKey(i.key), - { key: base58btc.decode(i.key), keyStr: i.key }, - i.default, - ] satisfies [string, CTCryptoKey, KeyMaterial, boolean], - ), - ).then((i) => i.map((j) => new keyWithFingerPrint(...j))) - ).reduce( - (acc, i) => { - acc[i.fingerPrint] = i; - if (i.default) { - acc["*"] = i; - } - return acc; - }, - {} as keysItem["keys"], - ); - return { - id, - name: ki.name, - keys, - }; - } +// this should help to prevent that a the key of the device id is human readable +// thats only a Hausfrauensicherung(german might been offending) +const deviceIdKey = Lazy(() => hashString("FIREProof:deviceId")); - async getNamedKey(opts: keyBagFingerprintItemGetOpts): Promise> { - return this.#seq.add(async () => { - if (this.keysItem) { - // is loaded from provider - return keysByFingerprint.from({ ...this, keysItem: this.keysItem, opts }); - } - const id = this.keybag.rt.sthis.nextId(4).str; //debug - // read from provider and make it a KeysItem (name, keys) +// this is type vise a little weak --- hopefully this will not slash back +type KeyBagItem = InternalKeyBagFingerprintItem | DeviceIdResult | Result; // | DeviceIdItem | JWTItem - let provKeysItem = await this.prov.get(this.name); - if (!provKeysItem) { - provKeysItem = { - name: this.name, - keys: {}, - }; - } - const v2KeysItem = await this.toV2KeysItem(provKeysItem); - const keys = Object.values(v2KeysItem.keysItem.keys).length; - if (opts.failIfNotFound && keys === 0) { - return Result.Err(this.logger.Debug().Str("id", id).Str("name", this.name).Msg("failIfNotFound getNamedKey").AsError()); - } - this.keysItem = await this.toKeysItem(v2KeysItem.keysItem, id); - if (keys > 0) { - this.logger - .Debug() - .Str("id", id) - .Str("name", this.name) - .Any("fprs", Object.keys(v2KeysItem)) - .Msg("fingerPrint getNamedKey"); - return keysByFingerprint.from({ ...this, keysItem: this.keysItem, opts, modified: v2KeysItem.modified }); - } - if (!this.keysItem && opts.failIfNotFound) { - // do not cache - return this.logger.Debug().Str("id", id).Str("name", this.name).Msg("failIfNotFound getNamedKey").ResultError(); - } - this.keysItem = { name: this.name, keys: {}, id }; - const rKbfp = await keysByFingerprint.from({ - ...this, - keysItem: this.keysItem, - opts: { - materialStrOrUint8: opts.materialStrOrUint8 ?? this.keybag.rt.crypto.randomBytes(this.keybag.rt.keyLength), - def: true, - }, - modified: v2KeysItem.modified, - }); - if (rKbfp.isErr()) { - return rKbfp; - } - this.logger - .Debug() - .Str("id", id) - .Str("name", this.name) - .Any("KeyItems", await rKbfp.Ok().asV2KeysItem()) - .Msg("createKey getNamedKey-post"); - return rKbfp; - }); - } -} +const namedKeyItemsPerUrl = new Map>(); export class KeyBag implements KeyBagIf { readonly logger: Logger; readonly rt: KeyBagRuntime; + readonly #namedKeyItems: KeyedResolvOnce; - constructor(rt: KeyBagRuntime) { - this.rt = rt; + static async create(rt: KeyBagRuntime) { + const urlHash = await hashObject(rt.url.toJSON()); + const namedKeyItems = namedKeyItemsPerUrl.get(urlHash) ?? new KeyedResolvOnce(); + return new KeyBag(rt, namedKeyItems); + } + + private constructor(rt: KeyBagRuntime, namedKeyItems: KeyedResolvOnce) { this.logger = ensureLogger(rt.sthis, "KeyBag"); + this.rt = rt; + this.#namedKeyItems = namedKeyItems; } readonly _warnOnce: ResolveOnce = new ResolveOnce(); @@ -509,168 +97,148 @@ export class KeyBag implements KeyBagIf { return Result.Ok(url); } - // flush(): Promise { - // return this._seq.flush(); - // } + async getDeviceId(): Promise { + const id = await deviceIdKey(); + return this.#namedKeyItems.get(id).once(async () => { + const raw = await this.provider().then((p) => p.get(id)); + const r = KeyedDeviceIdKeyBagItemSchema.safeParse(raw); + if (!r.success) { + this.logger + .Error() + .Any({ + item: raw, + errors: r, + }) + .Msg("getDeviceId: unexpected item"); + return { + deviceId: Option.None(), + cert: Option.None(), + }; + } + return { + deviceId: Option.Some(r.data.item.deviceId), + cert: Option.From(r.data.item.cert), + }; + }); + } + async setDeviceId(_deviceId: JWKPrivate, _cert?: DeviceIdKeyBagItem["cert"]): Promise { + const id = await deviceIdKey() + this.#namedKeyItems.unget(id); + return this.#namedKeyItems.get(id).once(async () => { + await this.provider().then((p) => + p.set(id, { + id, + clazz: "DeviceIdKeyBagItem", + item: { + deviceId: _deviceId, + cert: _cert, + }, + } satisfies KeyedDeviceIdKeyBagItem), + ); + const ret = { + deviceId: Option.Some(_deviceId), + cert: Option.From(_cert), + }; + return ret; + }); + } + + async extractClaim(jwtStr: string, key?: CryptoKey | KeyObject | JWK | Uint8Array, opts?: JWTVerifyOptions): Promise { + let claims: JWTPayload | undefined + try { + let unParsed: unknown + if (key) { + unParsed = await jwtVerify(jwtStr, key, opts); + } else { + unParsed = decodeJwt(jwtStr); + } + const r = JWTPayloadSchema.safeParse(unParsed) + if (r.success) { + claims = r.data + } + } catch (e) { + /* */ + } + return claims + } + + async setJwt(name: string, jwtStr: string, key?: CryptoKey | KeyObject | JWK | Uint8Array, opts?: JWTVerifyOptions): Promise> { + // const val = this.#namedKeyItems.get(name).value + const claims = await this.extractClaim(jwtStr, key, opts) + return this.#namedKeyItems.get(name).once(() => { + return exception2Result(() => + this.provider().then((prov) => + prov + .set(name, { + id: name, + clazz: "JwtKeyBagItem", + item: { + jwtStr, + }, + } satisfies KeyedJwtKeyBagItem).then(() => ({ + key: name, + jwt: jwtStr, + claims + } satisfies JWTResult)) + ), + ); + }); + } - // async setNamedKey(name: string, key: string, def?: boolean): Promise> { - // return this._seq.add(() => this._upsertNamedKey(name, key, !!def)); - // } + async getJwt(name: string, key?: CryptoKey | KeyObject | JWK | Uint8Array, opts?: JWTVerifyOptions): Promise> { + if (this.#namedKeyItems.has(name)) { + const ret = await this.#namedKeyItems.get(name).once(() => { + throw new Error("Should never called"); + }); + const p = KeyedJwtKeyBagItemSchema.safeParse(ret); + if (!p.success) { + return Result.Err(p.error); + } + const claims = await this.extractClaim(p.data.item.jwtStr, key, opts) + return Result.Ok({ + key: name, + jwt: p.data.item.jwtStr, + claims: claims + }); + } + return this.logger.Error().Str("name", name).Msg("not found").ResultError(); + } - // async getNamedExtractableKey(name: string, failIfNotFound = false): Promise> { - // const ret = await this.getNamedKey(name, failIfNotFound); - // if (ret.isErr()) { - // return Result.Err(ret) - // } - // const named = ret.Ok(); - // return Result.Ok({ - // ...named, - // extract: async () => { - // const ext = new Uint8Array((await this.rt.crypto.exportKey("raw", named.key)) as ArrayBuffer); - // return { - // key: ext, - // keyStr: base58btc.encode(ext), - // }; - // }, - // }); - // } + async delete(name: string): Promise { + if (this.#namedKeyItems.has(name)) { + await this.provider().then((p) => p.del(name)); + this.#namedKeyItems.unget(name); + return true; + } + return false; + } - private _namedKeyItems = new KeyedResolvOnce(); + readonly provider = Lazy(() => this.rt.getBagProvider()); async getNamedKey( name: string, failIfNotFound = false, materialStrOrUint8?: string | Uint8Array, ): Promise> { - const kItem = await this._namedKeyItems.get(name).once(async () => { - // const id = this.rt.sthis.nextId(4).str; //debug - const prov = await this.rt.getBagProvider(); - return new KeyBagFingerprintItem(this, prov, name); + const kItem = await this.#namedKeyItems.get(name).once(async () => { + return new InternalKeyBagFingerprintItem(this, name); }); return kItem.getNamedKey({ failIfNotFound, materialStrOrUint8 }); } -} - -export type KeyBagFile = Record; - -export function isV1StorageKeyItem(item: V1StorageKeyItem | V2KeysItem): item is V1StorageKeyItem { - return !!(item as V1StorageKeyItem).key; -} - -export function isKeysItem(item: V1StorageKeyItem | V2KeysItem): item is V2KeysItem { - return !!(item as V2KeysItem).keys; -} - -export type KeyBackProviderFactory = (url: URI, sthis: SuperThis) => Promise; - -export interface KeyBagProviderFactoryItem { - readonly protocol: string; - // if this is set the default protocol selection is overridden - readonly override?: boolean; - readonly factory: KeyBackProviderFactory; -} - -const keyBagProviderFactories = new Map( - [ - { - protocol: "file:", - factory: async (url: URI, sthis: SuperThis) => { - return new KeyBagProviderFile(url, sthis); - }, - }, - { - protocol: "indexeddb:", - factory: async (url: URI, sthis: SuperThis) => { - const { KeyBagProviderImpl } = await import("@fireproof/core-gateways-indexeddb"); - return new KeyBagProviderImpl(url, sthis); - }, - }, - { - protocol: "memory:", - factory: async (url: URI, sthis: SuperThis) => { - return new KeyBagProviderMemory(url, sthis); - }, - }, - ].map((i) => [i.protocol, i]), -); -export function registerKeyBagProviderFactory(item: KeyBagProviderFactoryItem) { - const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":"; - keyBagProviderFactories.set(protocol, { - ...item, - protocol, - }); -} - -export function defaultKeyBagUrl(sthis: SuperThis): URI { - let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL"); - let url: URI; - if (runtimeFn().isBrowser) { - url = URI.from(bagFnameOrUrl || "indexeddb://fp-keybag"); - } else { - if (!bagFnameOrUrl) { - const home = sthis.env.get("HOME"); - bagFnameOrUrl = `${home}/.fireproof/keybag`; - url = URI.from(`file://${bagFnameOrUrl}`); - } else { - url = URI.from(bagFnameOrUrl); + async getRawObj(name: string): Promise>> { + const rawObj = await this.provider().then((p) => p.get(name)); + if (!rawObj) { + return Option.None(); } + return Option.Some(LegacyKeyedItemSchema.safeParse(rawObj)); } - const logger = ensureLogger(sthis, "defaultKeyBagUrl"); - logger.Debug().Url(url).Msg("from env"); - return url; -} -export function defaultKeyBagOpts(sthis: SuperThis, kbo?: Partial): KeyBagRuntime { - kbo = kbo || {}; - if (kbo.keyRuntime) { - return kbo.keyRuntime; - } - const logger = ensureLogger(sthis, "KeyBag"); - let url: URI; - if (kbo.url) { - url = URI.from(kbo.url); - logger.Debug().Url(url).Msg("from opts"); - } else { - let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL"); - if (runtimeFn().isBrowser) { - url = URI.from(bagFnameOrUrl || "indexeddb://fp-keybag"); - } else { - if (!bagFnameOrUrl) { - const home = sthis.env.get("HOME"); - bagFnameOrUrl = `${home}/.fireproof/keybag`; - url = URI.from(`file://${bagFnameOrUrl}`); - } else { - url = URI.from(bagFnameOrUrl); - } + async setRawObj(k: KeyedItem): Promise> { + const r = KeyedItemSchema.safeParse(k); + if (!r.success) { + return Result.Err(r.error); } - logger.Debug().Url(url).Msg("from env"); - } - const kitem = keyBagProviderFactories.get(url.protocol); - if (!kitem) { - throw logger.Error().Url(url).Msg("unsupported protocol").AsError(); + return exception2Result(() => this.provider().then((p) => p.set(r.data.id, r.data).then((_) => r.data))); } - - if (url.hasParam("masterkey")) { - throw logger.Error().Url(url).Msg("masterkey is not supported").AsError(); - } - - return { - url, - crypto: kbo.crypto || toCryptoRuntime({}), - sthis, - logger, - keyLength: kbo.keyLength || 16, - getBagProvider: () => kitem.factory(url, sthis), - id: () => { - return url.toString(); - }, - }; -} - -const _keyBags = new KeyedResolvOnce(); -export async function getKeyBag(sthis: SuperThis, kbo: Partial = {}): Promise { - await sthis.start(); - const rt = defaultKeyBagOpts(sthis, kbo); - return _keyBags.get(rt.id()).once(async () => new KeyBag(rt)); } diff --git a/core/keybag/key-with-fingerprint.ts b/core/keybag/key-with-fingerprint.ts new file mode 100644 index 000000000..df7e09c36 --- /dev/null +++ b/core/keybag/key-with-fingerprint.ts @@ -0,0 +1,90 @@ +import { CTCryptoKey, Result } from "@adviser/cement"; +import { KeyBagIf, KeyMaterial, KeysItem, KeyWithFingerPrint } from "@fireproof/core-types-base"; +import { base58btc } from "multiformats/bases/base58"; + +export function coerceMaterial(kb: KeyBagIf, material: string | Uint8Array): KeyMaterial { + let keyMaterial: Uint8Array; + if (typeof material === "string") { + keyMaterial = base58btc.decode(material); + } else if (material instanceof Uint8Array) { + keyMaterial = material; + } else { + throw kb.logger.Error().Msg("material must be string or Uint8Array").AsError(); + } + return { + key: keyMaterial, + keyStr: base58btc.encode(keyMaterial), + }; +} + +export async function toKeyWithFingerPrint( + keybag: KeyBagIf, + material: KeyMaterial, + def: boolean, +): Promise> { + const key = await keybag.subtleKey(material.key); + const fpr = base58btc.encode(new Uint8Array(await keybag.rt.crypto.digestSHA256(material.key))); + return Result.Ok(new InternalKeyWithFingerPrint({ + fingerPrint: fpr, key, material, default: def, + })); +} + +export async function toV2StorageKeyItem(keybag: KeyBagIf, material: KeyMaterial, def: boolean): Promise { + const rKfp = await toKeyWithFingerPrint(keybag, material, def); + if (rKfp.isErr()) { + throw rKfp; + } + return { + default: def, + fingerPrint: rKfp.Ok().fingerPrint, + key: material.keyStr, + }; +} + +export function coerceFingerPrint(kb: KeyBagIf, fingerPrint?: string | Uint8Array): string | undefined { + if (fingerPrint instanceof Uint8Array) { + fingerPrint = base58btc.encode(fingerPrint); + } + return fingerPrint; +} + +export interface InternalKeyWithFingerPrintOpts { + readonly default: boolean; + readonly fingerPrint: string; + readonly key: CTCryptoKey; + readonly material: KeyMaterial; +} + +export class InternalKeyWithFingerPrint implements KeyWithFingerPrint { + readonly default: boolean; + readonly fingerPrint: string; + readonly key: CTCryptoKey; + #material: KeyMaterial; + + constructor(opt: InternalKeyWithFingerPrintOpts) { + this.fingerPrint = opt.fingerPrint; + this.default = opt.default; + this.key = opt.key; + this.#material = opt.material; + } + + setDefault(def: boolean) { + (this as { default: boolean }).default = def; + return this + } + + extract(): Promise { + if (this.key.extractable) { + return Promise.resolve(this.#material); + } + throw new Error("Key is not extractable"); + } + + async asKeysItem(): Promise { + return { + default: this.default, + fingerPrint: this.fingerPrint, + key: this.#material.keyStr, + }; + } +} diff --git a/core/keybag/package.json b/core/keybag/package.json index 774559dc6..0ef473800 100644 --- a/core/keybag/package.json +++ b/core/keybag/package.json @@ -36,13 +36,14 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-gateways-file": "workspace:0.0.0", "@fireproof/core-gateways-indexeddb": "workspace:0.0.0", "@fireproof/core-runtime": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", - "@fireproof/core-types-blockstore": "workspace:0.0.0", "@fireproof/vendor": "workspace:0.0.0", - "multiformats": "^13.4.0" + "jose": "^6.0.12", + "multiformats": "^13.4.0", + "zod": "^4.0.14" } } diff --git a/core/protocols/cloud/package.json b/core/protocols/cloud/package.json index 885d2ee77..b6bf05999 100644 --- a/core/protocols/cloud/package.json +++ b/core/protocols/cloud/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-runtime": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/core-types-protocols-cloud": "workspace:0.0.0", diff --git a/core/protocols/dashboard/package.json b/core/protocols/dashboard/package.json index b9d90a8c3..02b3de3d6 100644 --- a/core/protocols/dashboard/package.json +++ b/core/protocols/dashboard/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-runtime": "workspace:0.0.0", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/core-types-protocols-cloud": "workspace:0.0.0", diff --git a/core/runtime/keyed-crypto.ts b/core/runtime/keyed-crypto.ts index 58052f95f..d47e903f1 100644 --- a/core/runtime/keyed-crypto.ts +++ b/core/runtime/keyed-crypto.ts @@ -1,13 +1,6 @@ import { CryptoRuntime, Logger, URI } from "@adviser/cement"; -import { SuperThis, PARAM, KeyBagIf } from "@fireproof/core-types-base"; -import { - BytesAndKeyWithIv, - CodecOpts, - IvAndKeyAndBytes, - IvKeyIdData, - CryptoAction, - KeysByFingerprint, -} from "@fireproof/core-types-blockstore"; +import { SuperThis, PARAM, KeyBagIf, KeysByFingerprint } from "@fireproof/core-types-base"; +import { BytesAndKeyWithIv, CodecOpts, IvAndKeyAndBytes, IvKeyIdData, CryptoAction } from "@fireproof/core-types-blockstore"; import { ensureLogger, UInt8ArrayEqual } from "./utils.js"; import type { AsyncBlockCodec, ByteView } from "@fireproof/core-types-runtime"; import { base58btc } from "multiformats/bases/base58"; @@ -199,7 +192,7 @@ class noCrypto implements CryptoAction { upsert: () => { throw this.logger.Error().Msg("noCrypto.upsert not implemented").AsError(); }, - asV2KeysItem: () => { + asV2StorageKeyItem: () => { throw this.logger.Error().Msg("noCrypto.asV2KeysItem not implemented").AsError(); }, }; diff --git a/core/runtime/package.json b/core/runtime/package.json index 45a08bb8c..7b137838a 100644 --- a/core/runtime/package.json +++ b/core/runtime/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/core-types-blockstore": "workspace:0.0.0", "@fireproof/core-types-protocols-cloud": "workspace:0.0.0", diff --git a/core/runtime/utils.ts b/core/runtime/utils.ts index 7daee5d03..88f666b73 100644 --- a/core/runtime/utils.ts +++ b/core/runtime/utils.ts @@ -151,6 +151,28 @@ const pathOps = new pathOpsImpl(); const txtOps = ((txtEncoder, txtDecoder) => ({ encode: (input: string) => txtEncoder.encode(input), decode: (input: ToUInt8) => txtDecoder.decode(coerceIntoUint8(input).Ok()), + base64: { + encode: (input: ToUInt8 | string) => { + if (typeof input === "string") { + const data = txtEncoder.encode(input); + return btoa(String.fromCharCode(...data)); + } + let charStr = ""; + for (const i of coerceIntoUint8(input).Ok()) { + charStr += String.fromCharCode(i); + } + return btoa(charStr); + }, + decodeUint8: (input: string) => { + const data = atob(input); + return new Uint8Array(data.split("").map((c) => c.charCodeAt(0))); + }, + decode: (input: string) => { + const data = atob(input); + const uint8 = new Uint8Array(data.split("").map((c) => c.charCodeAt(0))); + return txtDecoder.decode(uint8); + }, + }, // eslint-disable-next-line no-restricted-globals }))(new TextEncoder(), new TextDecoder()); diff --git a/core/tests/blockstore/keyed-crypto-indexeddb-file.test.ts b/core/tests/blockstore/keyed-crypto-indexeddb-file.test.ts index a5769a7c6..6b8024f13 100644 --- a/core/tests/blockstore/keyed-crypto-indexeddb-file.test.ts +++ b/core/tests/blockstore/keyed-crypto-indexeddb-file.test.ts @@ -2,12 +2,12 @@ import { runtimeFn, toCryptoRuntime, URI } from "@adviser/cement"; import { base58btc } from "multiformats/bases/base58"; import { mockLoader, mockSuperThis } from "../helpers.js"; import { ensureSuperThis } from "@fireproof/core-runtime"; -import { V2KeysItem, PARAM } from "@fireproof/core-types-base"; +import { PARAM, KeyWithFingerPrint, V2StorageKeyItem } from "@fireproof/core-types-base"; import { describe, beforeAll, it, expect, beforeEach } from "vitest"; import { coerceMaterial, getKeyBag, toKeyWithFingerPrint } from "@fireproof/core-keybag"; import { KeyBagProviderIndexedDB } from "@fireproof/core-gateways-indexeddb"; import { KeyBagProviderFile } from "@fireproof/core-gateways-file"; -import { KeyWithFingerPrint, Loadable } from "@fireproof/core-types-blockstore"; +import { Loadable } from "@fireproof/core-types-blockstore"; import { createAttachedStores } from "@fireproof/core-blockstore"; describe("KeyBag indexeddb and file", () => { @@ -43,7 +43,7 @@ describe("KeyBag indexeddb and file", () => { const kb = await getKeyBag(sthis, { url: url.toString(), crypto: toCryptoRuntime({ - randomBytes: (size) => new Uint8Array(size).map((_, i) => i), + randomBytes: (size: number) => new Uint8Array(size).map((_, i) => i), }), }); const name = "setkey" + Math.random(); @@ -60,8 +60,8 @@ describe("KeyBag indexeddb and file", () => { expect((await kb.getNamedKey(name2)).Ok()).toEqual(created.Ok()); - let diskBag: V2KeysItem; - let diskBag2: V2KeysItem; + let diskBag: V2StorageKeyItem; + let diskBag2: V2StorageKeyItem; const provider = await kb.rt.getBagProvider(); if (runtimeFn().isBrowser) { const p = provider as KeyBagProviderIndexedDB; @@ -72,12 +72,13 @@ describe("KeyBag indexeddb and file", () => { const { sysFS } = await p._prepare(name); diskBag = await sysFS.readfile((await p._prepare(name)).fName).then((data) => { - return JSON.parse(sthis.txt.decode(data)) as V2KeysItem; + return JSON.parse(sthis.txt.decode(data)) as V2StorageKeyItem; }); diskBag2 = await sysFS.readfile((await p._prepare(name2)).fName).then((data) => { - return JSON.parse(sthis.txt.decode(data)) as V2KeysItem; + return JSON.parse(sthis.txt.decode(data)) as V2StorageKeyItem; }); } + console.log("xxx diskBag", diskBag); expect((await toKeyWithFingerPrint(kb, coerceMaterial(kb, Object.values(diskBag.keys)[0].key), true)).Ok().fingerPrint).toEqual( (await res.Ok().get())?.fingerPrint, ); diff --git a/core/tests/blockstore/keyed-crypto.test.ts b/core/tests/blockstore/keyed-crypto.test.ts index b0e62ea58..82f5901a8 100644 --- a/core/tests/blockstore/keyed-crypto.test.ts +++ b/core/tests/blockstore/keyed-crypto.test.ts @@ -6,11 +6,11 @@ import * as cborg from "cborg"; import type { KeyBagProviderIndexedDB } from "@fireproof/core-gateways-indexeddb"; import { mockLoader, MockSuperThis, mockSuperThis } from "../helpers.js"; import { ensureSuperThis, keyedCryptoFactory, storeType2DataMetaWal } from "@fireproof/core-runtime"; -import { V2KeysItem, PARAM, StoreType } from "@fireproof/core-types-base"; +import { V2KeysItem, PARAM, StoreType, KeyBagIf, KeyWithFingerPrint, V2StorageKeyItem } from "@fireproof/core-types-base"; import { describe, beforeEach, it, expect } from "vitest"; import { coerceMaterial, getKeyBag, KeyBag, toKeyWithFingerPrint } from "@fireproof/core-keybag"; import { KeyBagProviderFile } from "@fireproof/core-gateways-file"; -import { CryptoAction, IvKeyIdData, KeyWithFingerPrint, Loadable } from "@fireproof/core-types-blockstore"; +import { CryptoAction, IvKeyIdData, Loadable, V2SerializedMetaKey } from "@fireproof/core-types-blockstore"; import { createAttachedStores, getDefaultURI } from "@fireproof/core-blockstore"; describe("KeyBag", () => { @@ -97,8 +97,8 @@ describe("KeyBag", () => { expect((await kb.getNamedKey(name2)).Ok()).toEqual(created.Ok()); - let diskBag: V2KeysItem; - let diskBag2: V2KeysItem; + let diskBag: V2StorageKeyItem; + let diskBag2: V2StorageKeyItem; const provider = await kb.rt.getBagProvider(); if (runtimeFn().isBrowser) { const p = provider as KeyBagProviderIndexedDB; @@ -112,10 +112,10 @@ describe("KeyBag", () => { const { sysFS } = await p._prepare(name); diskBag = await sysFS.readfile((await p._prepare(name)).fName).then((data) => { - return JSON.parse(sthis.txt.decode(data)) as V2KeysItem; + return JSON.parse(sthis.txt.decode(data)) as V2StorageKeyItem; }); diskBag2 = await sysFS.readfile((await p._prepare(name2)).fName).then((data) => { - return JSON.parse(sthis.txt.decode(data)) as V2KeysItem; + return JSON.parse(sthis.txt.decode(data)) as V2StorageKeyItem; }); } expect((await toKeyWithFingerPrint(kb, coerceMaterial(kb, Object.values(diskBag.keys)[0].key), true)).Ok().fingerPrint).toEqual( @@ -153,7 +153,7 @@ describe("KeyBag", () => { for (let i = 0; i < 10; ++i) { expect(await kb.getNamedKey(name).then((i) => i.Ok().id)).toEqual(rMyKey.Ok().id); } - expect(Object.keys((await kb.getNamedKey(name).then((i) => i.Ok().asV2KeysItem())).keys).length).toBe(1); + expect(Object.keys((await kb.getNamedKey(name).then((i) => i.Ok().asV2StorageKeyItem())).keys).length).toBe(1); const myKey = (await rMyKey.Ok().get()) as KeyWithFingerPrint; expect(myKey.fingerPrint).toMatch(/^z/); @@ -162,7 +162,7 @@ describe("KeyBag", () => { const myKey1 = (await rMyKey.Ok().get()) as KeyWithFingerPrint; expect(myKey.fingerPrint).toEqual(myKey1.fingerPrint); - expect(Object.keys((await kb.getNamedKey(name).then((i) => i.Ok().asV2KeysItem())).keys).length).toBe(1); + expect(Object.keys((await kb.getNamedKey(name).then((i) => i.Ok().asV2StorageKeyItem())).keys).length).toBe(1); const rMyKey1 = await kb.getNamedKey(name); expect(rMyKey1.Ok()).toEqual(rMyKey.Ok()); @@ -171,12 +171,12 @@ describe("KeyBag", () => { const myKey2 = (await rMyKey1.Ok().get()) as KeyWithFingerPrint; expect(myKey.fingerPrint).toEqual(myKey2.fingerPrint); - expect(Object.keys((await kb.getNamedKey(name).then((i) => i.Ok().asV2KeysItem())).keys).length).toBe(2); + expect(Object.keys((await kb.getNamedKey(name).then((i) => i.Ok().asV2StorageKeyItem())).keys).length).toBe(2); const res = await rMyKey1.Ok().upsert(kb.rt.crypto.randomBytes(kb.rt.keyLength), true); expect(res.isOk()).toBeTruthy(); const myKey3 = (await rMyKey.Ok().get()) as KeyWithFingerPrint; - expect(Object.keys((await kb.getNamedKey(name).then((i) => i.Ok().asV2KeysItem())).keys).length).toBe(3); + expect(Object.keys((await kb.getNamedKey(name).then((i) => i.Ok().asV2StorageKeyItem())).keys).length).toBe(3); expect(myKey.fingerPrint).not.toEqual(myKey3.fingerPrint); }); @@ -214,9 +214,9 @@ describe("KeyBag", () => { expect((await myKey.get())?.fingerPrint).toEqual(fpr); } const provider = await kb.rt.getBagProvider(); - let diskBag: V2KeysItem; + let diskBag: V2StorageKeyItem; if (!("_prepare" in provider)) { - diskBag = (await provider.get(name)) as V2KeysItem; + diskBag = (await provider.get(name)) as V2StorageKeyItem; } else { if (runtimeFn().isBrowser) { const p = provider as KeyBagProviderIndexedDB; @@ -225,7 +225,7 @@ describe("KeyBag", () => { const p = provider as KeyBagProviderFile; const { sysFS } = await p._prepare(name); diskBag = await sysFS.readfile((await p._prepare(name)).fName).then((data) => { - return JSON.parse(sthis.txt.decode(data)) as V2KeysItem; + return JSON.parse(sthis.txt.decode(data)) as V2StorageKeyItem; }); } } @@ -234,7 +234,7 @@ describe("KeyBag", () => { }); describe("KeyedCryptoStore", () => { - let kb: KeyBag; + let kb: KeyBagIf; // let logger: Logger; let baseUrl: URI; const sthis = ensureSuperThis(); @@ -332,7 +332,7 @@ describe("KeyedCryptoStore", () => { }); describe("KeyedCrypto", () => { - let kb: KeyBag; + let kb: KeyBagIf; let kycr: CryptoAction; let keyStr: string; const sthis = ensureSuperThis(); diff --git a/core/tests/fireproof/attachable-subscription.test.ts b/core/tests/fireproof/attachable-subscription.test.ts new file mode 100644 index 000000000..a0e95ff4e --- /dev/null +++ b/core/tests/fireproof/attachable-subscription.test.ts @@ -0,0 +1,512 @@ +import { AppContext, BuildURI, WithoutPromise } from "@adviser/cement"; +import { Attachable, Database, fireproof, GatewayUrlsParam, PARAM, DocBase } from "@fireproof/core"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { ensureSuperThis, sleep } from "@fireproof/core-runtime"; + +const ROWS = 2; + +class AJoinable implements Attachable { + readonly name: string; + readonly db: Database; + + constructor(name: string, db: Database) { + this.name = name; + this.db = db; + } + + async configHash() { + return `joinable-${this.name}`; + } + + prepare(): Promise { + return Promise.resolve({ + car: { + url: BuildURI.from(`memory://car/${this.name}`) + .setParam(PARAM.STORE_KEY, this.db.ledger.opts.storeUrls.data.car.getParam(PARAM.STORE_KEY, "@fireproof:attach@")) + .setParam(PARAM.SELF_REFLECT, "x"), + }, + meta: { + url: BuildURI.from(`memory://meta/${this.name}`) + .setParam(PARAM.STORE_KEY, this.db.ledger.opts.storeUrls.data.meta.getParam(PARAM.STORE_KEY, "@fireproof:attach@")) + .setParam(PARAM.SELF_REFLECT, "x"), + }, + file: { + url: BuildURI.from(`memory://file/${this.name}`) + .setParam(PARAM.STORE_KEY, this.db.ledger.opts.storeUrls.data.file.getParam(PARAM.STORE_KEY, "@fireproof:attach@")) + .setParam(PARAM.SELF_REFLECT, "x"), + }, + }); + } +} + +function aJoinable(name: string, db: Database): Attachable { + return new AJoinable(name, db); +} + +function attachableStoreUrls(name: string, db: Database) { + return { + data: { + car: BuildURI.from(`memory://car/${name}?`) + .setParam(PARAM.STORE_KEY, db.ledger.opts.storeUrls.data.car.getParam(PARAM.STORE_KEY, "")) + .URI(), + meta: BuildURI.from(`memory://meta/${name}`) + .setParam(PARAM.STORE_KEY, db.ledger.opts.storeUrls.data.meta.getParam(PARAM.STORE_KEY, "")) + .URI(), + file: BuildURI.from(`memory://file/${name}`) + .setParam(PARAM.STORE_KEY, db.ledger.opts.storeUrls.data.file.getParam(PARAM.STORE_KEY, "")) + .URI(), + wal: BuildURI.from(`memory://wal/${name}`) + .setParam(PARAM.STORE_KEY, db.ledger.opts.storeUrls.data.wal.getParam(PARAM.STORE_KEY, "")) + .URI(), + }, + }; +} + +async function syncDb(name: string, base: string) { + const db = fireproof(name, { + storeUrls: { + base: BuildURI.from(base).setParam(PARAM.STORE_KEY, "@fireproof:attach@"), + }, + ctx: AppContext.merge({ base }), + }); + await db.ready(); + return db; +} + +async function prepareDb(name: string, base: string) { + { + const db = await syncDb(name, base); + await db.ready(); + const dbId = await db.ledger.crdt.blockstore.loader.attachedStores.local().active.car.id(); + const ret = { db, dbId }; + await writeRow(ret, `initial`); + await db.close(); + } + + const db = await syncDb(name, base); + await db.ready(); + const dbId = await db.ledger.crdt.blockstore.loader.attachedStores.local().active.car.id(); + return { db, dbId }; +} + +async function readDb(name: string, base: string) { + const db = await syncDb(name, base); + const rows = await db.allDocs(); + await db.close(); + return rows.rows.sort((a, b) => a.key.localeCompare(b.key)); +} + +async function writeRow(pdb: WithoutPromise>, style: string) { + const keys: string[] = []; + for (let i = 0; i < ROWS; i++) { + const key = `${pdb.dbId}-${pdb.db.name}-${style}-${i}`; + await pdb.db.put({ + _id: key, + value: key, + type: "test-document", + description: `Test document for ${style}`, + }); + keys.push(key); + } + return keys; +} + +/** + * REMOTE SYNC SUBSCRIPTION BUG REPRODUCTION TESTS + * + * PROBLEM: + * React components using useLiveQuery don't update when remote changes sync via toCloud(). + * Local writes work fine, but remote sync data doesn't trigger React re-renders. + * + * ROOT CAUSE: + * This is NOT a React/use-fireproof bug - it's a core Fireproof subscription system bug. + * The db.subscribe() method only fires for NEW writes, not for EXISTING data that syncs in. + * + * TEST RESULTS: + * ❌ 6 failures: Subscriptions don't fire when existing data syncs via attach() + * ✅ 3 passes: Subscriptions DO fire when new data is written after connection + * + * THE BUG: + * Fireproof treats these differently, but users expect both to trigger subscriptions: + * - ✅ db.put() → subscription fires → React updates (WORKS) + * - ❌ remote data sync → subscription doesn't fire → React doesn't update (BROKEN) + * + * REAL-WORLD IMPACT: + * - User opens React app on phone, writes data, closes app (data syncs to cloud) + * - User opens same React app on laptop + * - App pulls phone data but UI doesn't update (user sees stale data) + * - User must refresh page to see synced data + * + * EXPECTED BEHAVIOR: + * When remote data syncs into local database, subscriptions should fire just like local writes. + * This would make React components update automatically when remote data arrives. + * + * FIX NEEDED: + * Core subscription system needs to treat remote data ingestion the same as local writes. + * Likely fix location: CRDT/ledger layer where remote data is applied to local database. + */ + +describe("Remote Sync Subscription Tests", () => { + const sthis = ensureSuperThis(); + + // Subscription tracking variables + let subscriptionCallbacks: (() => void)[] = []; + const subscriptionCounts = new Map(); + const receivedDocs = new Map(); + // Helper to setup subscription tracking on a database + function setupSubscription(db: Database, dbName: string): Promise { + return new Promise((resolve) => { + subscriptionCounts.set(dbName, 0); + receivedDocs.set(dbName, []); + + const unsubscribe = db.subscribe((docs) => { + const currentCount = subscriptionCounts.get(dbName) || 0; + const currentDocs = receivedDocs.get(dbName) || []; + + subscriptionCounts.set(dbName, currentCount + 1); + receivedDocs.set(dbName, [...currentDocs, ...docs]); + + // Subscription fired successfully - tracked in subscriptionCounts + resolve(); + }, true); + + subscriptionCallbacks.push(unsubscribe); + }); + } + + afterEach(async () => { + // Clean up all subscriptions + subscriptionCallbacks.forEach((unsub) => unsub()); + subscriptionCallbacks = []; + subscriptionCounts.clear(); + receivedDocs.clear(); + }); + + describe("join function", () => { + let db: Database; + let joinableDBs: string[] = []; + + beforeEach(async () => { + const set = sthis.nextId().str; + + db = fireproof(`db-${set}`, { + storeUrls: { + base: `memory://db-${set}`, + }, + }); + + for (let j = 0; j < ROWS; j++) { + await db.put({ _id: `db-${j}`, value: `db-${set}` }); + } + + joinableDBs = await Promise.all( + new Array(1).fill(1).map(async (_, i) => { + const name = `remote-db-${i}-${set}`; + const jdb = fireproof(name, { + storeUrls: attachableStoreUrls(name, db), + }); + for (let j = 0; j < ROWS; j++) { + await jdb.put({ _id: `${i}-${j}`, value: `${i}-${j}` }); + } + expect(await jdb.get(PARAM.GENESIS_CID)).toEqual({ _id: PARAM.GENESIS_CID }); + await jdb.close(); + return name; + }), + ); + + expect(await db.get(PARAM.GENESIS_CID)).toEqual({ _id: PARAM.GENESIS_CID }); + }); + + afterEach(async () => { + await db.close(); + }); + + it("should trigger subscriptions on inbound syncing", async () => { + /* + * WHAT THIS TEST DOES: + * 1. Creates main database with initial data (1 doc) + * 2. Creates remote databases with their own data (1 doc each) + * 3. Sets up subscription on main database + * 4. Attaches remote databases to main database + * 5. Expects subscription to fire when remote data syncs into main database + * + * WHAT SHOULD HAPPEN: + * - Main DB starts with 1 document + * - Remote DBs have 1 document each + * - When attach() completes, main DB should have 2 documents (1 original + 1 from remote) + * - The subscription should fire because the database contents changed (new document arrived) + * - This is equivalent to someone else writing data that syncs into your local database + * + * WHAT ACTUALLY HAPPENS (BUG): + * - ✅ Data syncs correctly (confirmed by debug tests) + * - ❌ Subscription never fires even though database contents changed + * - This means users don't get notified when remote data arrives via toCloud/attach + * + * WHY THIS IS A BUG: + * - From user perspective: remote data arriving should trigger same notifications as local writes + * - React components using useLiveQuery don't update when remote changes sync + * - Breaks the reactive programming model for distributed databases + * + * EXPECTED BEHAVIOR: + * When db.attach() pulls in remote data, it should trigger subscriptions just like db.put() does + */ + + // Setup subscription on main database before attaching remote databases + const subscriptionPromise = setupSubscription(db, "main-db"); + + // Perform the attach operations that should trigger subscriptions + await Promise.all( + joinableDBs.map(async (name) => { + const attached = await db.attach(aJoinable(name, db)); + expect(attached).toBeDefined(); + }), + ); + + // Wait for sync to complete + await sleep(100); + + // Wait for subscription to fire (or timeout) + // 🐛 BUG: This will timeout because subscription never fires for remote data sync + await Promise.race([ + subscriptionPromise, + new Promise((_, reject) => setTimeout(() => reject(new Error("Subscription timeout")), 5000)), + ]); + + // Verify the subscription was triggered + expect(subscriptionCounts.get("main-db")).toBeGreaterThan(0); + expect(subscriptionCounts.get("main-db")).toBeGreaterThanOrEqual(1); // Should fire at least once + + // Verify the data was synced correctly + expect(db.ledger.crdt.blockstore.loader.attachedStores.remotes().length).toBe(joinableDBs.length); + const res = await db.allDocs(); + expect(res.rows.length).toBe(ROWS + ROWS * joinableDBs.length); + + // Verify subscription received the synced documents + const docs = receivedDocs.get("main-db") || []; + expect(docs.length).toBeGreaterThan(0); + // With our fix, subscriptions now properly fire for remote data sync + // The exact number may vary based on sync timing, but we should get all synced documents + expect(docs.length).toBeGreaterThanOrEqual(ROWS * joinableDBs.length); + }); + }); + + describe("sync", () => { + beforeEach(async () => { + // Reset subscription tracking for each sync test + subscriptionCallbacks.forEach((unsub) => unsub()); + subscriptionCallbacks = []; + subscriptionCounts.clear(); + receivedDocs.clear(); + }); + + it("should trigger subscriptions during offline sync reconnection", async () => { + /* + * WHAT THIS TEST SIMULATES: + * This is the classic "offline sync" scenario that users encounter with toCloud(): + * 1. User A writes data and syncs it to cloud storage (outbound database) + * 2. User B is offline, then comes back online and connects to same storage + * 3. User B's database should receive User A's data and notify subscribers + * + * REAL-WORLD SCENARIO: + * - User opens React app on phone, writes some data, closes app (data syncs to cloud) + * - Same user opens React app on laptop later + * - Laptop app should pull phone data and update UI via useLiveQuery + * + * WHAT THIS TEST DOES: + * 1. Creates "outbound" database with data and syncs it to shared namespace + * 2. Creates separate "inbound" database (simulates different device/session) + * 3. Sets up subscription on inbound database + * 4. Connects inbound database to same sync namespace (simulates going online) + * 5. Expects subscription to fire when outbound data syncs into inbound database + * + * WHAT SHOULD HAPPEN: + * - Inbound database starts with 1 document (its own data) + * - When attach() connects to sync namespace, it pulls outbound database's data + * - Inbound database should now have 2 documents (1 original + 1 from outbound) + * - The subscription should fire because database contents changed + * - React app would re-render with the new synced data + * + * WHAT ACTUALLY HAPPENS (BUG): + * - ✅ Data syncs perfectly (confirmed by debug tests) + * - ✅ Database ends up with correct 2 documents + * - ❌ Subscription never fires even though database contents changed + * - ❌ React components using useLiveQuery don't update + * + * WHY THIS IS CRITICAL: + * - This is THE most common sync scenario for distributed apps + * - Users expect React UI to update when remote data syncs in + * - Without this, users have to refresh page or manually re-query + * - Breaks the "live" experience that Fireproof promises + * + * EXPECTED BEHAVIOR: + * Remote sync bringing in existing data should trigger subscriptions just like local writes do + */ + + const id = sthis.nextId().str; + + // Create outbound database and sync data (simulates User A's session) + const poutbound = await prepareDb(`outbound-db-${id}`, "memory://sync-outbound"); + await poutbound.db.attach(aJoinable(`sync-${id}`, poutbound.db)); + await poutbound.db.close(); + const outRows = await readDb(`outbound-db-${id}`, "memory://sync-outbound"); + expect(outRows.length).toBe(ROWS); + + // Create inbound database (simulates User B's session on different device) + const pinbound = await prepareDb(`inbound-db-${id}`, `memory://sync-inbound`); + await pinbound.db.close(); + const inRows = await readDb(`inbound-db-${id}`, "memory://sync-inbound"); + expect(inRows.length).toBe(ROWS); + + // Now test the subscription during sync (User B goes online) + const inbound = await syncDb(`inbound-db-${id}`, `memory://sync-inbound`); + + // Setup subscription BEFORE attaching - this simulates useLiveQuery being active + const subscriptionPromise = setupSubscription(inbound, "inbound-db"); + + // Attach to the same sync namespace - this simulates toCloud() reconnection + // 🐛 BUG: This should trigger subscription but doesn't + await inbound.attach(aJoinable(`sync-${id}`, inbound)); + + // Wait for subscription to fire (or timeout) + // 🐛 BUG: This will timeout because subscription never fires for reconnection sync + await Promise.race([ + subscriptionPromise, + new Promise((_, reject) => setTimeout(() => reject(new Error("Subscription timeout")), 5000)), + ]); + + // Verify the subscription was triggered by remote sync + expect(subscriptionCounts.get("inbound-db")).toBeGreaterThan(0); + expect(subscriptionCounts.get("inbound-db")).toBeGreaterThanOrEqual(1); // Should fire at least once + + // Verify subscription received the synced documents + const docs = receivedDocs.get("inbound-db") || []; + expect(docs.length).toBeGreaterThan(0); + expect(docs.length).toBe(ROWS * 2); // Should receive both inbound and outbound documents + + // Close database after all assertions complete + await inbound.close(); + + // Verify the data was synced correctly + const resultRows = await readDb(`inbound-db-${id}`, "memory://sync-inbound"); + expect(resultRows.length).toBe(ROWS * 2); // inbound + outbound data + }, 100_000); + + it("should trigger subscriptions during online multi-database sync", async () => { + /* + * WHAT THIS TEST DOES (WORKING SCENARIO): + * This test demonstrates the ONE scenario where subscriptions DO work correctly. + * It shows the difference between syncing EXISTING data (broken) vs NEW data (working). + * + * SEQUENCE: + * 1. Creates 3 databases and connects them via attach() (they start empty) + * 2. Sets up subscriptions on all databases + * 3. AFTER connection, writes NEW data to each database + * 4. Expects subscriptions to fire when NEW data syncs between databases + * + * WHY THIS WORKS: + * - Databases start empty, so attach() has no existing data to sync + * - Only NEW writes happen AFTER subscriptions are set up + * - New writes trigger subscriptions locally AND when they sync to remote databases + * - This is "real-time sync" - data written after connection established + * + * CONTRAST WITH FAILING TESTS: + * - Failing tests: Databases have EXISTING data BEFORE attach() + * - Failing tests: Subscription should fire when EXISTING data syncs in + * - Working test: Only NEW data written AFTER attach() syncs + * + * THE PROBLEM: + * Fireproof subscription system distinguishes between: + * ✅ "New writes that sync" (this test - WORKS) + * ❌ "Existing data that syncs" (other tests - BROKEN) + * + * But from user perspective, both should trigger subscriptions because both change database contents! + */ + + const id = sthis.nextId().str; + + // Create multiple databases that will sync together + const dbs = await Promise.all( + Array(3) + .fill(0) + .map(async (_, i) => { + const tdb = await prepareDb(`online-db-${id}-${i}`, `memory://local-${id}-${i}`); + + // Setup subscription on each database + const subscriptionPromise = setupSubscription(tdb.db, `online-db-${i}`); + + // Attach to shared sync namespace (no existing data to sync yet) + await tdb.db.attach(aJoinable(`sync-${id}`, tdb.db)); + + return { ...tdb, subscriptionPromise }; + }), + ); + + // Wait for initial sync to complete (nothing to sync yet) + await sleep(1000); + + // Now write NEW data to databases - this WILL trigger subscriptions ✅ + // This is the key difference: NEW writes vs EXISTING data sync + const keys = []; + for (const [_index, db] of dbs.entries()) { + await sleep(100 * Math.random()); + const dbKeys = await writeRow(db, "add-online"); + keys.push(...dbKeys); + } + + // Wait for sync and subscriptions to propagate + await sleep(1000); + + // Wait for all subscriptions to fire + await Promise.all( + dbs.map(async (db, i) => { + try { + await Promise.race([ + db.subscriptionPromise, + new Promise((_, reject) => setTimeout(() => reject(new Error(`Subscription timeout for db ${i}`)), 5000)), + ]); + } catch (error) { + // Subscription timeout - this is expected if subscriptions don't work for this database + } + }), + ); + + // Verify subscriptions were triggered + let totalSubscriptionFires = 0; + dbs.forEach((_, i) => { + const count = subscriptionCounts.get(`online-db-${i}`) || 0; + totalSubscriptionFires += count; + expect(count).toBeGreaterThan(0); // Each database should have at least one subscription fire + }); + + expect(totalSubscriptionFires).toBeGreaterThan(0); + // With our fix, subscriptions fire more frequently as they should for sync operations + // Each database should fire at least once, but may fire multiple times as sync progresses + expect(totalSubscriptionFires).toBeGreaterThanOrEqual(dbs.length); + + // Verify data was synced correctly across all databases + // Wait for sync completion before checking all keys + await sleep(2000); + + await Promise.all( + dbs.map(async (db) => { + const allDocs = await db.db.allDocs(); + // console.log(allDocs.rows); + expect(allDocs.rows.length).toBe(keys.length * 2); + // for (const key of keys) { + // try { + // const doc = await db.db.get<{ value: string }>(key); + // expect(doc._id).toBe(key); + // expect(doc.value).toBe(key); + // } catch (e) { + // // Document may still be syncing, this is expected in some test runs + // console.log(`Document ${key} not yet synced to database`); + // } + // } + }), + ); + + // Cleanup + await Promise.all(dbs.map((tdb) => tdb.db.close())); + }, 100_000); + }); +}); diff --git a/core/tests/helpers.ts b/core/tests/helpers.ts index c1e371ca1..dd1fe17fb 100644 --- a/core/tests/helpers.ts +++ b/core/tests/helpers.ts @@ -1,10 +1,10 @@ import { BuildURI, MockLogger, runtimeFn, toCryptoRuntime, URI, utils, LogCollector, Logger, AppContext } from "@adviser/cement"; -import { SuperThis, SuperThisOpts, PARAM, Attachable, Attached, CarTransaction, Falsy, DbMeta } from "@fireproof/core"; +import { SuperThis, SuperThisOpts, PARAM, Attachable, Attached, CarTransaction, Falsy, DbMeta, KeyBagIf } from "@fireproof/core"; import { CID } from "multiformats"; import { sha256 } from "multiformats/hashes/sha2"; import * as json from "multiformats/codecs/json"; import { CommitQueue, encodeFile, ensureSuperThis, TaskManager } from "@fireproof/core-runtime"; -import { defaultKeyBagOpts, getKeyBag, KeyBag } from "@fireproof/core-keybag"; +import { defaultKeyBagOpts, getKeyBag } from "@fireproof/core-keybag"; import { AttachedRemotesImpl, toStoreRuntime } from "@fireproof/core-blockstore"; import { Loadable, @@ -136,7 +136,7 @@ class MockLoader implements Loadable { close(): Promise { throw new Error("Method not implemented."); } - keyBag(): Promise { + keyBag(): Promise { return getKeyBag(this.sthis, {}); } // eslint-disable-next-line @typescript-eslint/no-unused-vars diff --git a/core/tests/notes/apply-head-challenge.md b/core/tests/notes/apply-head-challenge.md new file mode 100644 index 000000000..d530f47aa --- /dev/null +++ b/core/tests/notes/apply-head-challenge.md @@ -0,0 +1,417 @@ +# ApplyHead Call Path Challenge - Remote Sync Subscription Bug + +## The Core Problem + +React components using `useLiveQuery` don't update when remote changes sync via `toCloud()`. The subscription system works for local writes but fails for remote sync operations. + +## Key Insight: Two ApplyHead Call Paths + +The `applyHead()` method in `crdt-clock.ts` is called from **two different paths** in `crdt.ts`: + +### Path 1: Local Writes (WORKING ✅) + +``` +User calls db.put() +→ writeQueue.push() +→ crdt.bulk() +→ clock.applyHead(newHead, prevHead, localUpdates=TRUE) +→ notifyWatchers() +→ subscriptions fire +→ React components update +``` + +### Path 2: Remote Sync (BROKEN ❌) + +``` +Remote data arrives +→ applyMeta() +→ clock.applyHead(newHead, prevHead, localUpdates=FALSE) +→ ??? (subscriptions don't fire) +→ React components don't update +``` + +## The Hypothesis + +**The subscription system is only working for the `bulk()` path, not the `applyMeta()` path.** + +This would explain: + +- ✅ Local writes trigger subscriptions (via `bulk()`) +- ❌ Remote sync data doesn't trigger subscriptions (via `applyMeta()`) +- ❌ React components using `useLiveQuery` don't update on remote changes + +## Current Subscription Fix Attempt + +We've added manual subscription triggering in `crdt-clock.ts`: + +```typescript +// In int_applyHead() +const needsManualNotification = !localUpdates && (this.watchers.size > 0 || this.noPayloadWatchers.size > 0); + +if (needsManualNotification) { + const changes = await clockChangesSince(this.blockstore, advancedHead, prevHead, {}, this.logger); + if (changes.result.length > 0) { + this.notifyWatchers(changes.result); + } else { + this.noPayloadWatchers.forEach((fn) => fn()); + } +} +``` + +However, this fix assumes that: + +1. The `applyMeta()` path is reaching `int_applyHead()` +2. The `localUpdates=FALSE` parameter is being set correctly +3. The manual notification logic is executing + +## Investigation Needed + +We need to add logging to trace the execution flow: + +### 1. Log Both Call Sites in `crdt.ts` + +**In `bulk()` method:** + +```typescript +console.log("🔵 BULK: Calling applyHead for LOCAL write", { + localUpdates: true, + newHead: newHead.map((h) => h.toString()), + subscribers: this.clock.watchers.size + this.clock.noPayloadWatchers.size, +}); +await this.clock.applyHead(newHead, prevHead, updates); +``` + +**In `applyMeta()` method:** + +```typescript +console.log("🔴 APPLY_META: Calling applyHead for REMOTE sync", { + localUpdates: false, + newHead: newHead.map((h) => h.toString()), + subscribers: this.clock.watchers.size + this.clock.noPayloadWatchers.size, +}); +await this.clock.applyHead(newHead, prevHead, false); +``` + +### 2. Log Entry Point in `crdt-clock.ts` + +**In `int_applyHead()` method:** + +```typescript +console.log("⚡ INT_APPLY_HEAD: Entry point", { + localUpdates, + watchersCount: this.watchers.size, + noPayloadWatchersCount: this.noPayloadWatchers.size, + needsManualNotification: !localUpdates && (this.watchers.size > 0 || this.noPayloadWatchers.size > 0), +}); +``` + +### 3. Log Notification Calls + +**In `notifyWatchers()` method:** + +```typescript +console.log("🔔 NOTIFY_WATCHERS: Triggering subscriptions", { + updatesCount: updates.length, + watchersCount: this.watchers.size, + noPayloadWatchersCount: this.noPayloadWatchers.size, + filteredUpdates: updates.map((u) => ({ id: u.id, value: u.value })), +}); +``` + +**In manual notification path:** + +```typescript +if (needsManualNotification) { + console.log("🛠️ MANUAL_NOTIFICATION: Checking for changes", { changes: changes.result.length }); + if (changes.result.length > 0) { + console.log("🛠️ MANUAL_NOTIFICATION: Calling notifyWatchers with changes"); + this.notifyWatchers(changes.result); + } else { + console.log("🛠️ MANUAL_NOTIFICATION: Calling noPayloadWatchers directly"); + this.noPayloadWatchers.forEach((fn) => fn()); + } +} +``` + +## Expected Log Output Analysis + +### For Local Writes (Working Case) + +``` +🔵 BULK: Calling applyHead for LOCAL write { localUpdates: true, newHead: [...], subscribers: 1 } +⚡ INT_APPLY_HEAD: Entry point { localUpdates: true, watchersCount: 1, noPayloadWatchersCount: 0, needsManualNotification: false } +🔔 NOTIFY_WATCHERS: Triggering subscriptions { updatesCount: 1, watchersCount: 1, noPayloadWatchersCount: 0, ... } +``` + +### For Remote Sync (Broken Case - What We Should See) + +``` +🔴 APPLY_META: Calling applyHead for REMOTE sync { localUpdates: false, newHead: [...], subscribers: 1 } +⚡ INT_APPLY_HEAD: Entry point { localUpdates: false, watchersCount: 1, noPayloadWatchersCount: 0, needsManualNotification: true } +🛠️ MANUAL_NOTIFICATION: Checking for changes { changes: 1 } +🛠️ MANUAL_NOTIFICATION: Calling notifyWatchers with changes +🔔 NOTIFY_WATCHERS: Triggering subscriptions { updatesCount: 1, watchersCount: 1, noPayloadWatchersCount: 0, ... } +``` + +### For Remote Sync (If Broken - What We Might Actually See) + +``` +🔴 APPLY_META: Calling applyHead for REMOTE sync { localUpdates: false, newHead: [...], subscribers: 1 } +⚡ INT_APPLY_HEAD: Entry point { localUpdates: false, watchersCount: 1, noPayloadWatchersCount: 0, needsManualNotification: true } +🛠️ MANUAL_NOTIFICATION: Checking for changes { changes: 0 } +🛠️ MANUAL_NOTIFICATION: Calling noPayloadWatchers directly +// No NOTIFY_WATCHERS log = bug found! +``` + +**OR even worse:** + +``` +🔴 APPLY_META: Calling applyHead for REMOTE sync { localUpdates: false, newHead: [...], subscribers: 0 } +⚡ INT_APPLY_HEAD: Entry point { localUpdates: false, watchersCount: 0, noPayloadWatchersCount: 0, needsManualNotification: false } +// No manual notification = subscriptions not set up yet when applyMeta is called! +``` + +## Potential Root Causes to Investigate + +### 1. Timing Issue + +- `applyMeta()` might be called before subscriptions are set up +- Remote sync happens during database initialization +- Subscribers not registered yet when remote data arrives + +### 2. Code Path Not Executing + +- `applyMeta()` path might not reach `int_applyHead()` at all +- Different parameter passing between bulk and applyMeta +- Early returns preventing execution + +### 3. Manual Notification Logic Bug + +- Our fix logic might have conditions that don't match real scenarios +- `clockChangesSince()` might return different results for remote sync +- EmptyWatchers vs watchers distinction not working as expected + +### 4. Subscription Setup Mismatch + +- `use-fireproof` might be using different subscription patterns +- React hooks setup timing vs remote sync timing +- Database ready state vs subscription ready state + +## Test Strategy + +1. **Run the failing subscription test with logging** +2. **Compare logs between local write test (working) and remote sync test (broken)** +3. **Identify exactly where the execution path diverges** +4. **Fix the root cause based on evidence** + +## Success Criteria + +The fix is successful when: + +1. ✅ Both local writes AND remote sync operations produce similar log patterns +2. ✅ `🔔 NOTIFY_WATCHERS` logs appear for both paths +3. ✅ Subscription tests pass for both `updates: true` and `updates: false` modes +4. ✅ React components using `useLiveQuery` update on remote sync + +## File Locations for Investigation + +- **Call sites**: `/Users/jchris/code/fp/fireproof/core/base/crdt.ts` +- **Clock implementation**: `/Users/jchris/code/fp/fireproof/core/base/crdt-clock.ts` +- **Test file**: `/Users/jchris/code/fp/fireproof/core/tests/fireproof/attachable-subscription.test.ts` +- **use-fireproof**: `/Users/jchris/code/fp/fireproof/core/tests/node_modules/use-fireproof/react/use-live-query.ts` + +--- + +## Running Targeted Tests + +### Existing Tests to Run + +**1. Run the comprehensive subscription tests:** + +```bash +pnpm test fireproof/attachable-subscription.test.ts --reporter=verbose +``` + +**2. Run a specific failing test with logs:** + +```bash +pnpm test fireproof/attachable-subscription.test.ts -t "should trigger subscriptions on inbound syncing" --reporter=verbose +``` + +**3. Run database tests that exercise both paths:** + +```bash +pnpm test fireproof/database.test.ts -t "basic Ledger with subscription" --reporter=verbose +``` + +### New Simple Tests to Write + +Create these minimal tests in `/Users/jchris/code/fp/fireproof/core/tests/fireproof/apply-head-logging.test.ts`: + +#### Test 1: Local Write Path Logging + +```typescript +import { fireproof } from "@fireproof/core"; +import { describe, expect, it } from "vitest"; + +describe("ApplyHead Path Logging", () => { + it("should log BULK path for local writes", async () => { + const db = fireproof("test-bulk-path"); + + // Setup subscription to ensure watchers exist + let notified = false; + const unsubscribe = db.subscribe(() => { + notified = true; + }, true); + + // Perform local write - should trigger BULK path + console.log("🧪 TEST: Starting local write"); + await db.put({ _id: "test-local", value: "local-data" }); + + // Wait for async operations + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(notified).toBe(true); + unsubscribe(); + await db.close(); + + console.log("🧪 TEST: Local write completed"); + }); +}); +``` + +#### Test 2: Remote Sync Path Logging + +```typescript +it("should log APPLY_META path for remote sync", async () => { + const set = "test-remote-path"; + + // Create source database with data + const sourceDb = fireproof(`source-${set}`, { + storeUrls: { base: `memory://source-${set}` }, + }); + await sourceDb.put({ _id: "test-remote", value: "remote-data" }); + + // Create target database + const targetDb = fireproof(`target-${set}`, { + storeUrls: { base: `memory://target-${set}` }, + }); + + // Setup subscription to ensure watchers exist + let notified = false; + const unsubscribe = targetDb.subscribe(() => { + notified = true; + }, true); + + console.log("🧪 TEST: Starting remote sync"); + + // Trigger remote sync - should trigger APPLY_META path + // (This needs to be implemented based on the actual sync mechanism) + // await targetDb.attach(someAttachable); + + // Wait for async operations + await new Promise((resolve) => setTimeout(resolve, 500)); + + console.log("🧪 TEST: Remote sync completed, notified:", notified); + + unsubscribe(); + await sourceDb.close(); + await targetDb.close(); +}); +``` + +#### Test 3: Side-by-Side Comparison + +```typescript +it("should show log differences between local and remote paths", async () => { + console.log("\n=== COMPARISON TEST START ==="); + + const db = fireproof("test-comparison"); + + let localNotified = false; + let remoteNotified = false; + + const unsubscribe = db.subscribe(() => { + console.log("📬 SUBSCRIPTION: Notification received"); + localNotified = true; // We'll use this for both for now + }, true); + + // Phase 1: Local write + console.log("\n--- PHASE 1: LOCAL WRITE ---"); + await db.put({ _id: "local-test", value: "local" }); + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Phase 2: Simulate remote sync scenario + console.log("\n--- PHASE 2: REMOTE SYNC SIMULATION ---"); + // TODO: Implement actual remote sync trigger + // For now, just show the logging setup is working + console.log("🔄 REMOTE: Would trigger applyMeta path here"); + + console.log("\n--- RESULTS ---"); + console.log("Local write notified:", localNotified); + console.log("Remote sync notified:", remoteNotified); + + unsubscribe(); + await db.close(); + + console.log("=== COMPARISON TEST END ===\n"); +}); +``` + +### Running the New Tests + +**Run the new logging tests:** + +```bash +pnpm test fireproof/apply-head-logging.test.ts --reporter=verbose +``` + +**Run with debug output:** + +```bash +FP_DEBUG=1 pnpm test fireproof/apply-head-logging.test.ts --reporter=verbose +``` + +### Expected Log Analysis + +When running these tests, look for: + +**✅ Successful Local Write Logs:** + +``` +🧪 TEST: Starting local write +🔵 BULK: Calling applyHead for LOCAL write { localUpdates: true, ... } +⚡ INT_APPLY_HEAD: Entry point { localUpdates: true, needsManualNotification: false } +🔔 NOTIFY_WATCHERS: Triggering subscriptions { updatesCount: 1, ... } +📬 SUBSCRIPTION: Notification received +🧪 TEST: Local write completed +``` + +**❌ Missing Remote Sync Logs:** + +``` +🧪 TEST: Starting remote sync +🔄 REMOTE: Would trigger applyMeta path here +// MISSING: 🔴 APPLY_META logs +// MISSING: ⚡ INT_APPLY_HEAD logs +// MISSING: 🔔 NOTIFY_WATCHERS logs +🧪 TEST: Remote sync completed, notified: false +``` + +This pattern will immediately reveal whether the `applyMeta()` path is: + +1. **Not being called at all** (no 🔴 logs) +2. **Not reaching int_applyHead** (🔴 logs but no ⚡ logs) +3. **Not triggering notifications** (🔴 and ⚡ logs but no 🔔 logs) + +### Iterative Testing Strategy + +1. **Start with existing tests** - Add logging and run attachable-subscription.test.ts +2. **Create minimal reproduction** - Use the simple tests above +3. **Identify the break point** - Follow the missing logs +4. **Fix incrementally** - Address each missing log in sequence +5. **Validate with original tests** - Ensure comprehensive tests pass + +--- + +_This investigation will definitively identify whether the `applyMeta()` → `applyHead()` → `notifyWatchers()` chain is broken and exactly where the execution path diverges from the working `bulk()` case._ diff --git a/core/tests/package.json b/core/tests/package.json index c47aa23b7..1a32837e8 100644 --- a/core/tests/package.json +++ b/core/tests/package.json @@ -40,10 +40,11 @@ "react": ">=18.0.0" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core": "workspace:0.0.0", "@fireproof/core-base": "workspace:0.0.0", "@fireproof/core-blockstore": "workspace:0.0.0", + "@fireproof/core-device-id": "workspace:0.0.0", "@fireproof/core-gateways-base": "workspace:0.0.0", "@fireproof/core-gateways-file": "workspace:0.0.0", "@fireproof/core-gateways-indexeddb": "workspace:0.0.0", @@ -62,6 +63,7 @@ "@types/node": "^24.3.0", "cborg": "^4.2.12", "charwise": "^3.0.1", + "jose": "^6.0.12", "use-fireproof": "workspace:0.0.0", "uuidv7": "^1.0.2" }, diff --git a/core/tests/runtime/device-id.test.ts b/core/tests/runtime/device-id.test.ts new file mode 100644 index 000000000..d9a833b0f --- /dev/null +++ b/core/tests/runtime/device-id.test.ts @@ -0,0 +1,456 @@ +import { describe, it, expect, beforeEach, assert } from "vitest"; +import { decodeProtectedHeader, importJWK, jwtVerify } from "jose"; +import { ensureSuperThis } from "@fireproof/core-runtime"; +import { + DeviceIdCA, + DeviceIdCSR, + DeviceIdKey, + DeviceIdSignMsg, + DeviceIdValidator, + DeviceIdVerifyMsg, +} from "@fireproof/core-device-id"; +import { CertificatePayload, CertificatePayloadSchema, Extensions, JWKPrivate, Subject } from "@fireproof/core-types-base"; + +const sthis = ensureSuperThis(); + +describe("DeviceIdKey", () => { + it("should export private key as JWK", async () => { + const key = await DeviceIdKey.create(); + const jwk = await key.exportPrivateJWK(); + + expect(jwk).toBeDefined(); + expect(jwk.kty).toBe("EC"); + if (jwk.kty !== "EC") { + assert.fail("Invalid JWK"); + } + expect(jwk.d).toBeDefined(); // Private key component + + const imported = await DeviceIdKey.createFromJWK(jwk as JWKPrivate); + const jwk2 = await imported.exportPrivateJWK(); + expect(jwk2).toEqual(jwk); + + expect(await key.publicKey()).toEqual(await imported.publicKey()); + + expect(await key.publicKey()).toEqual({ + crv: "P-256", + kty: "EC", + x: expect.any(String), + y: expect.any(String), + }); + }); +}); + +describe("DeviceIdCSR and DeviceIdValidator integration", () => { + it("should create and validate a CSR successfully", async () => { + // Create a key and CSR + const key = await DeviceIdKey.create(); + const csr = new DeviceIdCSR(sthis, key); + + const subject: Subject = { + commonName: "test.example.com", + organization: "Test Corp", + locality: "San Francisco", + stateOrProvinceName: "California", + countryName: "US", + }; + + const extensions: Extensions = { + subjectAltName: ["test.example.com", "www.test.example.com"], + keyUsage: ["digitalSignature", "keyEncipherment"], + extendedKeyUsage: ["serverAuth", "clientAuth"], + }; + + // Create the CSR + const rCsrJWS = await csr.createCSR(subject, extensions); + if (rCsrJWS.isErr()) { + assert.fail(rCsrJWS.Err().message); + } + const csrJWS = rCsrJWS.Ok(); + expect(csrJWS).toBeDefined(); + expect(typeof csrJWS).toBe("string"); + + // Validate the CSR + const validator = new DeviceIdValidator(); + const validation = await validator.validateCSR(csrJWS); + + expect(validation.valid).toBe(true); + if (!validation.valid) { + throw new Error(`Validation failed: ${validation.error}`); + } + expect(validation.payload).toBeDefined(); + expect(validation.publicKey).toBeDefined(); + + if (!validation.payload) { + throw new Error("No payload"); + } + // Verify payload structure + const payload = validation.payload; + expect(payload.sub).toBe(subject.commonName); + expect(payload.iss).toBe("csr-client"); + expect(payload.aud).toBe("certificate-authority"); + expect(payload.csr.subject).toEqual(subject); + expect(payload.csr.extensions.subjectAltName).toEqual(extensions.subjectAltName); + expect(payload.csr.extensions.keyUsage).toEqual(extensions.keyUsage); + expect(payload.csr.extensions.extendedKeyUsage).toEqual(extensions.extendedKeyUsage); + }); + + it("should fail validation for tampered CSR", async () => { + const key = await DeviceIdKey.create(); + const csr = new DeviceIdCSR(sthis, key); + + const subject = { commonName: "test.example.com" }; + const rCsrJWS = await csr.createCSR(subject); + if (rCsrJWS.isErr()) { + assert.fail(rCsrJWS.Err().message); + } + const csrJWS = rCsrJWS.Ok(); + + // Tamper with the CSR + const tamperedCSR = csrJWS.slice(0, -10) + "tampered123"; + + const validator = new DeviceIdValidator(); + const validation = await validator.validateCSR(tamperedCSR); + + expect(validation.valid).toBe(false); + if (validation.valid) { + throw new Error("Validation should have failed"); + } + expect(validation.error).toBeDefined(); + }); + + it("should fail validation for CSR without public key in header", async () => { + const validator = new DeviceIdValidator(); + const invalidCSR = "eyJhbGciOiJFUzI1NiIsInR5cCI6IkNTUitKV1QifQ.eyJzdWIiOiJ0ZXN0In0.invalid"; + + const validation = await validator.validateCSR(invalidCSR); + + expect(validation.valid).toBe(false); + if (validation.valid) { + throw new Error("Validation should have failed"); + } + expect(validation.error).toContain("No public key in CSR header"); + }); +}); + +describe("DeviceIdCA certificate generation and validation", () => { + it("should generate and validate a certificate from CSR", async () => { + // Create CA key and subject + const caKey = await DeviceIdKey.create(); + const caSubject = { + commonName: "Test CA", + organization: "Test CA Corp", + locality: "San Francisco", + stateOrProvinceName: "California", + countryName: "US", + }; + + // Mock CA actions + const mockActions = { + generateSerialNumber: async () => crypto.randomUUID(), + }; + + // Create CA + const ca = new DeviceIdCA({ + base64: sthis.txt.base64, + caKey, + caSubject, + actions: mockActions, + }); + + // Create device key and CSR + const deviceKey = await DeviceIdKey.create(); + const csr = new DeviceIdCSR(sthis, deviceKey); + + const subject = { + commonName: "device.example.com", + organization: "Device Corp", + locality: "New York", + stateOrProvinceName: "New York", + countryName: "US", + }; + + const extensions: Extensions = { + subjectAltName: ["device.example.com", "alt.device.example.com"], + keyUsage: ["digitalSignature", "keyEncipherment"], + extendedKeyUsage: ["serverAuth"], + }; + + // Create CSR + const rCsrJWS = await csr.createCSR(subject, extensions); + if (rCsrJWS.isErr()) { + assert.fail(rCsrJWS.Err().message); + } + const csrJWS = rCsrJWS.Ok(); + + // Process CSR and generate certificate + const rCertificate = await ca.processCSR(csrJWS); + if (rCertificate.isErr()) { + assert.fail(rCertificate.Err().message); + } + const certificate = rCertificate.Ok(); + // Verify certificate structure + expect(certificate.certificateJWT).toBeDefined(); + expect(certificate.format).toBe("JWS"); + expect(certificate.serialNumber).toBeDefined(); + expect(certificate.issuer).toBe(caSubject.commonName); + expect(certificate.subject).toBe(subject.commonName); + expect(certificate.validityPeriod.notBefore).toBeInstanceOf(Date); + expect(certificate.validityPeriod.notAfter).toBeInstanceOf(Date); + expect(certificate.publicKey).toBeDefined(); + + // Verify certificate JWS signature with CA public key + const caPublicKey = await caKey.publicKey(); + const caKeyForVerification = await importJWK(caPublicKey, "ES256"); + + const { payload: certPayload, protectedHeader } = await jwtVerify(certificate.certificateJWT, caKeyForVerification, { + typ: "CERT+JWT", + }); + + // Verify certificate payload + expect(certPayload.iss).toBe(caSubject.commonName); + expect(certPayload.sub).toBe(subject.commonName); + expect(certPayload.jti).toBe(certificate.serialNumber); + expect(certPayload.certificate).toBeDefined(); + + // Verify certificate extensions + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const cert = certPayload.certificate as any; + expect(cert.subject).toEqual(subject); + expect(cert.issuer).toEqual(caSubject); + expect(cert.subjectPublicKeyInfo).toEqual(certificate.publicKey); + // expect(cert.extensions.subjectAltName.names).toEqual(extensions.subjectAltName); + // expect(cert.extensions.keyUsage.usage).toEqual(extensions.keyUsage); + // expect(cert.extensions.extendedKeyUsage.usage).toEqual(extensions.extendedKeyUsage); + + // Verify protected header + expect(protectedHeader.alg).toBe("ES256"); + expect(protectedHeader.typ).toBe("CERT+JWT"); + expect(protectedHeader.kid).toBe(await caKey.fingerPrint()); + }); + + it("should reject invalid CSR", async () => { + const caKey = await DeviceIdKey.create(); + const caSubject = { commonName: "Test CA" }; + const mockActions = { + generateSerialNumber: async () => crypto.randomUUID(), + }; + + const ca = new DeviceIdCA({ + base64: sthis.txt.base64, + caKey, + caSubject, + actions: mockActions, + }); + + const invalidCSR = "invalid.csr.string"; + + await expect(ca.processCSR(invalidCSR)).rejects.toThrow("CSR validation failed"); + }); +}); + +describe("DeviceIdSignMsg", () => { + let deviceKey: DeviceIdKey; + let caKey: DeviceIdKey; + let certificate: CertificatePayload; + const { base64 } = ensureSuperThis().txt; + + let ca: DeviceIdCA; + // Create CA + const caSubject = { + commonName: "Test CA", + organization: "Test CA Corp", + }; + + const mockActions = { + generateSerialNumber: async () => crypto.randomUUID(), + }; + beforeEach(async () => { + // Setup base64 encoder + + // Create CA and device keys + caKey = await DeviceIdKey.create(); + deviceKey = await DeviceIdKey.create(); + + ca = new DeviceIdCA({ + base64: sthis.txt.base64, + caKey, + caSubject, + actions: mockActions, + }); + + // Create CSR and get certificate + const csr = new DeviceIdCSR(sthis, deviceKey); + const subject = { + commonName: "device.example.com", + organization: "Device Corp", + }; + + const rCsrJWS = await csr.createCSR(subject); + if (rCsrJWS.isErr()) { + assert.fail(rCsrJWS.Err().message); + } + const csrJWS = rCsrJWS.Ok(); + const rCertResult = await ca.processCSR(csrJWS); + + // Extract certificate payload from JWS + const caPublicKey = await caKey.publicKey(); + const caKeyForVerification = await importJWK(caPublicKey, "ES256"); + const { payload } = await jwtVerify(rCertResult.Ok().certificateJWT, caKeyForVerification, { typ: "CERT+JWT" }); + certificate = CertificatePayloadSchema.parse(payload); + }); + + it("should sign a payload and include certificate information", async () => { + const signMsg = new DeviceIdSignMsg(base64, deviceKey, certificate); + const payload = { message: "test payload", timestamp: Date.now() }; + + const jwt = await signMsg.sign(payload); + expect(jwt).toBeDefined(); + expect(typeof jwt).toBe("string"); + + // Decode header to verify certificate information + const header = decodeProtectedHeader(jwt); + expect(header.alg).toBe("ES256"); + expect(header.typ).toBe("JWT"); + expect(header.kid).toBe(await deviceKey.fingerPrint()); + expect(header.x5c).toBeDefined(); + expect(Array.isArray(header.x5c)).toBe(true); + expect(header.x5c?.length).toBe(1); + expect(header.x5t).toBeDefined(); + expect(header["x5t#S256"]).toBeDefined(); + }); + + it("should verify signed JWT with device public key", async () => { + const signMsg = new DeviceIdSignMsg(base64, deviceKey, certificate); + const payload = { message: "verification test", id: 123 }; + + const jwt = await signMsg.sign(payload); + + // Verify with device public key + const devicePublicKey = await deviceKey.publicKey(); + const deviceKeyForVerification = await importJWK(devicePublicKey, "ES256"); + + const { payload: verifiedPayload } = await jwtVerify(jwt, deviceKeyForVerification); + expect(verifiedPayload.message).toBe(payload.message); + expect(verifiedPayload.id).toBe(payload.id); + expect(verifiedPayload.iat).toBeDefined(); + expect(verifiedPayload.exp).toBeDefined(); + }); + + it("should include valid certificate thumbprints", async () => { + const signMsg = new DeviceIdSignMsg(base64, deviceKey, certificate); + const payload = { test: "thumbprint validation" }; + + const jwt = await signMsg.sign(payload); + const header = decodeProtectedHeader(jwt); + + // Verify thumbprints are base58btc encoded strings + expect(typeof header.x5t).toBe("string"); + expect(header.x5t?.length).toBeGreaterThan(0); + expect(typeof header["x5t#S256"]).toBe("string"); + expect((header["x5t#S256"] as string).length).toBeGreaterThan(0); + }); + + it("should fail verification with wrong key", async () => { + const signMsg = new DeviceIdSignMsg(base64, deviceKey, certificate); + const payload = { message: "wrong key test" }; + + const jwt = await signMsg.sign(payload); + + // Try to verify with different key + const wrongKey = await DeviceIdKey.create(); + const wrongPublicKey = await wrongKey.publicKey(); + const wrongKeyForVerification = await importJWK(wrongPublicKey, "ES256"); + + await expect(jwtVerify(jwt, wrongKeyForVerification)).rejects.toThrow(); + }); + it("should verify JWT with valid certificate", async () => { + const signMsg = new DeviceIdSignMsg(base64, deviceKey, certificate); + const payload = { message: "verification test", id: 123 }; + const jwt = await signMsg.sign(payload); + expect(jwt).toBeDefined(); + expect(typeof jwt).toBe("string"); + + const caCert = await ca.caCertificate(); + + const deviceVerifyMsg = new DeviceIdVerifyMsg(base64, [caCert.Ok()], { + clockTolerance: 60, + maxAge: 3600, + }); + + const ret = await deviceVerifyMsg.verifyWithCertificate(jwt); + expect(ret.valid).toBe(true); + }); + + it.skip("change the caKey", async () => { + const signMsg = new DeviceIdSignMsg(base64, deviceKey, certificate); + const payload = { message: "verification test", id: 123 }; + const jwt = await signMsg.sign(payload); + expect(jwt).toBeDefined(); + expect(typeof jwt).toBe("string"); + + const newCaKey = await DeviceIdKey.create(); + const newCa = new DeviceIdCA({ + base64: sthis.txt.base64, + caKey: newCaKey, + caSubject, + actions: mockActions, + }); + + const deviceVerifyMsg = new DeviceIdVerifyMsg(base64, [(await newCa.caCertificate()).Ok()], { + clockTolerance: 60, + maxAge: 3600, + }); + + const ret = await deviceVerifyMsg.verifyWithCertificate(jwt); + expect(ret.valid).toBe(false); + }); + + it("use a new deviceId ", async () => { + const newDeviceKey = await DeviceIdKey.create(); + const signMsg = new DeviceIdSignMsg(base64, newDeviceKey, certificate); + const payload = { message: "verification test", id: 123 }; + const jwt = await signMsg.sign(payload); + expect(jwt).toBeDefined(); + expect(typeof jwt).toBe("string"); + + const newCaKey = await DeviceIdKey.create(); + const newCa = new DeviceIdCA({ + base64: sthis.txt.base64, + caKey: newCaKey, + caSubject, + actions: mockActions, + }); + + const deviceVerifyMsg = new DeviceIdVerifyMsg(base64, [(await newCa.caCertificate()).Ok()], { + clockTolerance: 60, + maxAge: 3600, + }); + + const ret = await deviceVerifyMsg.verifyWithCertificate(jwt); + expect(ret.valid).toBe(false); + }); + + it("use a forged caCert", async () => { + const signMsg = new DeviceIdSignMsg(base64, deviceKey, { ...certificate, nbf: certificate.nbf + 1 }); + const payload = { message: "verification test", id: 123 }; + const jwt = await signMsg.sign(payload); + expect(jwt).toBeDefined(); + expect(typeof jwt).toBe("string"); + + const newCaKey = await DeviceIdKey.create(); + const newCa = new DeviceIdCA({ + base64: sthis.txt.base64, + caKey: newCaKey, + caSubject, + actions: mockActions, + }); + + const deviceVerifyMsg = new DeviceIdVerifyMsg(base64, [(await newCa.caCertificate()).Ok()], { + clockTolerance: 60, + maxAge: 3600, + }); + + const ret = await deviceVerifyMsg.verifyWithCertificate(jwt); + expect(ret.valid).toBe(false); + }); +}); diff --git a/core/tests/runtime/key-bag.test.ts b/core/tests/runtime/key-bag.test.ts index 2e329ffc1..00821b8ad 100644 --- a/core/tests/runtime/key-bag.test.ts +++ b/core/tests/runtime/key-bag.test.ts @@ -1,21 +1,31 @@ -import { Result, URI } from "@adviser/cement"; -import { getKeyBag, KeyBagProviderMemory } from "@fireproof/core-keybag"; +import { BuildURI, Result, URI } from "@adviser/cement"; +import { DeviceIdKey } from "@fireproof/core-device-id"; +import { getKeyBag, KeyBag, KeyBagProviderMemory } from "@fireproof/core-keybag"; import { ensureSuperThis } from "@fireproof/core-runtime"; -import { KeyBagIf, V2KeysItem, V2StorageKeyItem } from "@fireproof/core-types-base"; -import { isKeyUpsertResultModified, KeysByFingerprint } from "@fireproof/core-types-blockstore"; +import { + IssueCertificateResult, + JWKPrivate, + JWTResult, + KeyBagIf, + KeyedV2StorageKeyItem, + KeysByFingerprint, + KeyWithFingerPrint, +} from "@fireproof/core-types-base"; +import { isKeyUpsertResultModified } from "@fireproof/core-types-blockstore"; +import { UnsecuredJWT } from "jose"; import { base58btc } from "multiformats/bases/base58"; -import { assert, describe, expect, it } from "vitest"; - -// const v2Keybag = { -// name: "@test-v1-keys-wal@", -// keys: { -// z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ: { -// default: true, -// fingerPrint: "z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ", -// key: "zL89nmBmogeRptW9b7e9j7L", -// }, -// }, -// }; +import { assert, beforeEach, describe, expect, it } from "vitest"; + +const v2Keybag = { + name: "@test-v1-keys-wal@", + keys: { + z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ: { + default: true, + fingerPrint: "z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ", + key: "zL89nmBmogeRptW9b7e9j7L", + }, + }, +}; const v1Keybag = { name: "@test-v1-keys-wal@", @@ -37,12 +47,8 @@ async function keyExtracted( } async function calculateFingerprint(rKbf: Result, kb: KeyBagIf): Promise { - const item = await rKbf.Ok().get(); - const v2Item = await ( - item as unknown as { - asV2StorageKeyItem: () => Promise; - } - ).asV2StorageKeyItem(); + const item = (await rKbf.Ok().get()) as KeyWithFingerPrint; + const v2Item = await item.asKeysItem(); const keyBytes = base58btc.decode(v2Item.key); const hash = await kb.rt.crypto.digestSHA256(keyBytes); return base58btc.encode(new Uint8Array(hash)); @@ -54,7 +60,7 @@ describe("KeyBag", () => { await sthis.start(); const kp = new KeyBagProviderMemory(URI.from("memory://./dist/tests/"), sthis); - kp.set(v1Keybag as unknown as V2KeysItem); + await kp.set(v1Keybag.name, v1Keybag); const kb = await getKeyBag(sthis, { url: "memory://./dist/tests/?extractKey=_deprecated_internal_api", @@ -64,7 +70,59 @@ describe("KeyBag", () => { const fpr = await calculateFingerprint(rKbf, kb); - expect(await rKbf.Ok().asV2KeysItem()).toEqual({ + expect(await rKbf.Ok().asV2StorageKeyItem()).toEqual({ + keys: { + z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ: { + default: true, + fingerPrint: fpr, + key: "zL89nmBmogeRptW9b7e9j7L", + }, + }, + name: "@test-v1-keys-wal@", + }); + + const kb2 = await getKeyBag(sthis, { + url: "memory://./dist/tests/?extractKey=_deprecated_internal_api", + }); + for (const rkbf of [rKbf, await kb2.getNamedKey("@test-v1-keys-wal@")]) { + expect(await keyExtracted(rkbf)).toEqual({ + fingerPrint: "z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ", + key: "zL89nmBmogeRptW9b7e9j7L", + }); + expect(await keyExtracted(rkbf, "z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ")).toEqual({ + fingerPrint: "z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ", + key: "zL89nmBmogeRptW9b7e9j7L", + }); + expect(await keyExtracted(rkbf, "kaputt-x")).toBeUndefined(); + } + expect(await kp.get(v1Keybag.name)).toEqual({ + keys: { + z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ: { + default: true, + fingerPrint: "z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ", + key: "zL89nmBmogeRptW9b7e9j7L", + }, + }, + name: "@test-v1-keys-wal@", + }); + }); + + it("v2 migration", async () => { + const sthis = ensureSuperThis(); + await sthis.start(); + + const kp = new KeyBagProviderMemory(URI.from("memory://./dist/tests/"), sthis); + kp.set(v2Keybag.name, v2Keybag); + + const kb = await getKeyBag(sthis, { + url: "memory://./dist/tests/?extractKey=_deprecated_internal_api", + }); + const rKbf = await kb.getNamedKey("@test-v1-keys-wal@"); + expect(rKbf.isOk()).toBeTruthy(); + + const fpr = await calculateFingerprint(rKbf, kb); + + expect(await rKbf.Ok().asV2StorageKeyItem()).toEqual({ keys: { z7oNYUrGpALe6U5ePvhdD3ufHdLerw4wPWHJERE3383zJ: { default: true, @@ -106,7 +164,7 @@ describe("KeyBag", () => { await sthis.start(); const kp = new KeyBagProviderMemory(URI.from("memory://./dist/tests/"), sthis); - kp.set(v1Keybag as unknown as V2KeysItem); + kp.set(v1Keybag.name, v1Keybag); const kb = await getKeyBag(sthis, { url: "memory://./dist/tests/?extractKey=_deprecated_internal_api", @@ -114,7 +172,7 @@ describe("KeyBag", () => { const key1Material = kb.rt.crypto.randomBytes(kb.rt.keyLength); const rKbf = await kb.getNamedKey("kaputt", false, key1Material); expect(rKbf.isOk()).toBeTruthy(); - const one = await rKbf.Ok().asV2KeysItem(); + const one = await rKbf.Ok().asV2StorageKeyItem(); expect(Object.keys(one.keys).length).toBe(1); const key1Fpr = (await rKbf.Ok().get())?.fingerPrint; expect(await keyExtracted(rKbf)).toEqual({ @@ -130,18 +188,18 @@ describe("KeyBag", () => { return; } - const keys2 = await rKbf.Ok().asV2KeysItem(); + const keys2 = await rKbf.Ok().asV2StorageKeyItem(); expect(Object.keys(keys2.keys).length).toBe(2); expect(await kp.get("kaputt")).toEqual(keys2); const rKbf2 = await kb.getNamedKey("kaputt"); - expect(Object.keys(await rKbf2.Ok().asV2KeysItem()).length).toBe(2); + expect(Object.keys((await rKbf2.Ok().asV2StorageKeyItem()).keys).length).toBe(2); expect((await rKbf2.Ok().get())?.fingerPrint).toEqual(key1Fpr); - const asKeysItem = await rKbf.Ok().asV2KeysItem(); + const asKeysItem = (await rKbf.Ok().asV2StorageKeyItem()).keys; expect(asKeysItem.name).toEqual("kaputt"); expect(Array.from(Object.values(asKeysItem.keys))).toEqual([ { @@ -168,11 +226,11 @@ describe("KeyBag", () => { Object.keys( await rKbf2 .Ok() - .asV2KeysItem() + .asV2StorageKeyItem() .then((i) => i.keys), ).length, ).toBe(3); - const v2Key3 = await (key3.kfp as unknown as { asV2StorageKeyItem: () => Promise }).asV2StorageKeyItem(); + const v2Key3 = await key3.kfp.asKeysItem(); expect(await kp.get("kaputt")).toEqual({ keys: { @@ -188,7 +246,7 @@ describe("KeyBag", () => { const sthis = ensureSuperThis(); await sthis.start(); - const kb = await getKeyBag(sthis); + const kb = (await getKeyBag(sthis)) as KeyBag; const keyName = "simple" + sthis.nextId().str; @@ -196,7 +254,7 @@ describe("KeyBag", () => { const kfp1 = await rKbf1 .Ok() .get() - .then((i) => (i as unknown as { asV2StorageKeyItem: () => Promise }).asV2StorageKeyItem()); + .then((i) => i?.asKeysItem()); expect(kfp1?.fingerPrint).toBeTypeOf("string"); const rKbf2 = await kb.getNamedKey(keyName); @@ -220,7 +278,7 @@ describe("KeyBag", () => { await sthis.start(); const kp = new KeyBagProviderMemory(URI.from("memory://./dist/tests/"), sthis); - kp.set(v1Keybag as unknown as V2KeysItem); + await kp.set(v1Keybag.name, v1Keybag); const kb = await getKeyBag(sthis, { url: "memory://./dist/tests/?extractKey=_deprecated_internal_api", @@ -233,11 +291,148 @@ describe("KeyBag", () => { expect(await kb.getNamedKey(name).then((i) => i.Ok().id)).toEqual(rMyKey.Ok().id); } }); + + describe("test device id", async () => { + const sthis = ensureSuperThis(); + let kb: KeyBagIf; + let key: JWKPrivate; + const fakeCert: IssueCertificateResult = { + certificateJWT: "JWT", + certificatePayload: { + iss: "", + sub: "", + aud: "", + iat: 0, + nbf: 0, + exp: 0, + jti: "", + certificate: { + version: "3", + serialNumber: "4711", + subject: { + commonName: "Subject", + }, + issuer: { + commonName: "Issuer", + }, + validity: { + notBefore: new Date().toISOString(), + notAfter: new Date().toISOString(), + }, + subjectPublicKeyInfo: { + kty: "EC", + crv: "P-256", + x: "x", + y: "y", + }, + signatureAlgorithm: "ES256", + keyUsage: [], + extendedKeyUsage: [], + }, + }, + format: "JWS", + serialNumber: "", + issuer: "", + subject: "", + validityPeriod: { + notBefore: new Date(), + notAfter: new Date(), + }, + publicKey: { + kty: "EC", + crv: "P-256", + x: "x", + y: "y", + }, + }; + beforeEach(async () => { + await sthis.start(); + const id = sthis.nextId().str; + const url = BuildURI.from(`memory://./dist/tests/${id}`).setParam("extractKey", "_deprecated_internal_api").URI(); + kb = await getKeyBag(sthis, { url }); + key = await DeviceIdKey.create().then((i) => i.exportPrivateJWK()); + }); + it("return none if not set", async () => { + const devId = await kb.getDeviceId(); + expect(devId.deviceId.IsNone()).toBeTruthy(); + expect(devId.cert.IsNone()).toBeTruthy(); + }); + it("set and get device id", async () => { + const rSet = await kb.setDeviceId(key); + expect(rSet.deviceId.IsSome()).toBeTruthy(); + expect(rSet.deviceId.Unwrap()).toEqual(key); + expect(rSet.cert.IsNone()).toBeTruthy(); + + const rGet = await kb.getDeviceId(); + expect(rGet.deviceId.IsSome()).toBeTruthy(); + expect(rGet.deviceId.Unwrap()).toEqual(key); + expect(rGet.cert.IsNone()).toBeTruthy(); + }); + + it("set and get device id with cert", async () => { + const rSet = await kb.setDeviceId(key, fakeCert); + expect(rSet.deviceId.Unwrap()).toEqual(fakeCert); + expect(rSet.deviceId.Unwrap()).toEqual(key); + expect(rSet.cert.IsNone()).toBeTruthy(); + + const rGet = await kb.getDeviceId(); + expect(rSet.deviceId.Unwrap()).toEqual(fakeCert); + expect(rGet.deviceId.Unwrap()).toEqual(key); + expect(rGet.cert.IsNone()).toBeTruthy(); + }); + + it("set and get device id stepped cert", async () => { + await kb.setDeviceId(key); + const rSet = await kb.setDeviceId(key, fakeCert); + expect(rSet.deviceId.Unwrap()).toEqual(fakeCert); + expect(rSet.deviceId.Unwrap()).toEqual(key); + expect(rSet.cert.IsNone()).toBeTruthy(); + + const rGet = await kb.getDeviceId(); + expect(rSet.deviceId.Unwrap()).toEqual(fakeCert); + expect(rGet.deviceId.Unwrap()).toEqual(key); + expect(rGet.cert.IsNone()).toBeTruthy(); + }); + }); + describe("jwt", () => { + const sthis = ensureSuperThis(); + + let kb: KeyBagIf; + beforeEach(async () => { + await sthis.start(); + kb = await getKeyBag(sthis, { + url: "memory://./dist/murks/?extractKey=_deprecated_internal_api", + }); + }); + + it("set and get jwt", async () => { + const rNotForundGet = await kb.getJwt("test"); + expect(rNotForundGet.isOk()).toBeFalsy(); + + const jwt = new UnsecuredJWT({ hello: "world" }) + .setIssuedAt() + .setIssuer("fpcloud") + .setAudience("fpcloud-app") + .setExpirationTime("24h") + .setSubject("Test") + .encode(); + + const rSet = await kb.setJwt("test", jwt); + expect(rSet.isOk()).toBeTruthy(); + + const rGet = await kb.getJwt("test"); + expect(rGet.isOk()).toBeTruthy(); + expect(rGet.Ok()).toEqual({ + key: "test", + jwt, + } satisfies JWTResult); + }); + }); }); -function resetDefault(keys: Record) { +function resetDefault(keys: KeyedV2StorageKeyItem["item"]["keys"]) { return Array.from(Object.values(keys)).reduce( (acc, i) => ({ ...acc, [i.fingerPrint]: { ...i, default: false } }), - {} as Record, + {} as KeyedV2StorageKeyItem["item"]["keys"], ); } diff --git a/core/tests/runtime/meta-key-hack.test.ts b/core/tests/runtime/meta-key-hack.test.ts index 5c7cd0437..09cc9a6c1 100644 --- a/core/tests/runtime/meta-key-hack.test.ts +++ b/core/tests/runtime/meta-key-hack.test.ts @@ -51,7 +51,7 @@ describe("MetaKeyHack", () => { await db.put({ val: "test" }); const dataStore = loader.attachedStores.local().active.car; - const kb = new KeyBag(db.ledger.opts.keyBag); + const kb = await KeyBag.create(db.ledger.opts.keyBag); const rDataStoreKeyItem = await kb.getNamedKey(dataStore.url().getParam(PARAM.STORE_KEY) ?? ""); await rDataStoreKeyItem.Ok().upsert("zBUFMmu5c3VdCa4r2DZTzhR", false); @@ -63,7 +63,7 @@ describe("MetaKeyHack", () => { const rGet = await memGw.get(rUrl.Ok(), sthis); const metas = JSON.parse(ctx.loader.sthis.txt.decode(rGet.Ok())) as V2SerializedMetaKey; const keyMaterials = metas.keys; - const dataStoreKeyMaterial = await rDataStoreKeyItem.Ok().asV2KeysItem(); + const dataStoreKeyMaterial = await rDataStoreKeyItem.Ok().asV2StorageKeyItem(); expect(keyMaterials.length).toBeGreaterThan(0); expect(dataStoreKeyMaterial).toEqual({ keys: { diff --git a/core/tests/runtime/text-en-decoder.test.ts b/core/tests/runtime/text-en-decoder.test.ts new file mode 100644 index 000000000..786ad1bc5 --- /dev/null +++ b/core/tests/runtime/text-en-decoder.test.ts @@ -0,0 +1,32 @@ +import { ensureSuperThis } from "@fireproof/core-runtime"; +import { describe, expect, it } from "vitest"; + +describe("text encoder", function () { + const sthis = ensureSuperThis(); + it("should encode and decode", function () { + const input = "hello world"; + const encoded = sthis.txt.encode(input); + const decoded = sthis.txt.decode(encoded); + expect(decoded).toEqual(input); + }); + it("base64", function () { + const input = "hello world"; + const encoded = sthis.txt.base64.encode(input); + const decoded = sthis.txt.base64.decode(encoded); + expect(decoded).toEqual(input); + expect(encoded).toEqual("aGVsbG8gd29ybGQ="); + }); + + it("base64 binary", function () { + const input = new Uint8Array( + new Array(0x10000) + .fill(0) + .map((_, i) => [i % 256, i >> 8]) + .flat(), + ); + const encoded = sthis.txt.base64.encode(input); + const decoded = sthis.txt.base64.decodeUint8(encoded); + expect(decoded).toEqual(input); + expect(input.length).toEqual(decoded.length); + }); +}); diff --git a/core/types/base/device-id-keybag-item.zod.ts b/core/types/base/device-id-keybag-item.zod.ts new file mode 100644 index 000000000..3f2f75403 --- /dev/null +++ b/core/types/base/device-id-keybag-item.zod.ts @@ -0,0 +1,17 @@ +import z from "zod"; +import { JWKPrivateSchema } from "./jwk-private.zod.js"; +import { CertificatePayloadSchema } from "./fp-ca-cert-payload.zod.js"; + +export const DeviceIdKeyBagItemSchema = z + .object({ + deviceId: JWKPrivateSchema, + cert: z + .object({ + certificateJWT: z.string(), + certificatePayload: CertificatePayloadSchema, + }) + .optional(), + }) + .readonly(); + +export type DeviceIdKeyBagItem = z.infer; diff --git a/core/types/base/device-id.ts b/core/types/base/device-id.ts new file mode 100644 index 000000000..159125462 --- /dev/null +++ b/core/types/base/device-id.ts @@ -0,0 +1,16 @@ +import { CertificatePayload } from "./fp-ca-cert-payload.zod.js"; +import { JWKPublic } from "./jwk-public.zod.js"; + +export interface IssueCertificateResult { + readonly certificateJWT: string; // JWT String + readonly certificatePayload: CertificatePayload; + readonly format: "JWS"; + readonly serialNumber: string; + readonly issuer: string; + readonly subject: string; + readonly validityPeriod: { + readonly notBefore: Date; + readonly notAfter: Date; + }; + readonly publicKey: JWKPublic; +} diff --git a/core/types/base/fp-ca-cert-payload.zod.ts b/core/types/base/fp-ca-cert-payload.zod.ts new file mode 100644 index 000000000..527301029 --- /dev/null +++ b/core/types/base/fp-ca-cert-payload.zod.ts @@ -0,0 +1,65 @@ +import { z } from "zod"; + +import { ExtensionsSchema, SubjectSchema } from "./fp-device-id-payload.zod.js"; +import { JWKPublicSchema } from "./jwk-public.zod.js"; +// Certificate Payload Schema +const CertificateSchema = z.object({ + version: z.literal("3"), // X.509 v3 + serialNumber: z.string(), + subject: SubjectSchema, + issuer: SubjectSchema, + validity: z.object({ + notBefore: z.string().datetime(), + notAfter: z.string().datetime(), + }), + subjectPublicKeyInfo: JWKPublicSchema, + signatureAlgorithm: z.literal("ES256"), + keyUsage: z.array( + z.enum([ + "digitalSignature", + "nonRepudiation", + "keyEncipherment", + "dataEncipherment", + "keyAgreement", + "keyCertSign", + "cRLSign", + "encipherOnly", + "decipherOnly", + ]), + ), + extendedKeyUsage: z.array( + z.enum([ + "serverAuth", + "clientAuth", + "codeSigning", + "emailProtection", + "timeStamping", + "OCSPSigning", + "ipsecIKE", + "msCodeInd", + "msCodeCom", + "msCTLSign", + "msEFS", + ]), + ), + extensions: ExtensionsSchema.optional(), +}); + +export const CertificatePayloadSchema = z + .object({ + // Standard JWT claims + iss: z.string(), // Issuer (CA) + sub: z.string(), // Subject + aud: z.string().or(z.array(z.string())), + iat: z.number().int(), + nbf: z.number().int(), // Not before + exp: z.number().int(), // Expiration + jti: z.string(), // JWT ID as serial number + + // Certificate-specific claims + certificate: CertificateSchema, + }) + .readonly(); + +export type Certificate = z.infer; +export type CertificatePayload = z.infer; diff --git a/core/types/base/fp-device-id-payload.zod.ts b/core/types/base/fp-device-id-payload.zod.ts new file mode 100644 index 000000000..614612d33 --- /dev/null +++ b/core/types/base/fp-device-id-payload.zod.ts @@ -0,0 +1,105 @@ +import { z } from "zod"; +import { JWKPublicSchema } from "./jwk-public.zod.js"; +import { JWTPayloadSchema } from "./jwt-payload.zod.js"; + +// Subject Schema +export const SubjectSchema = z.object({ + commonName: z.string(), //.optional(), + countryName: z.string().length(2).optional(), // ISO 3166-1 alpha-2 + stateOrProvinceName: z.string().optional(), + locality: z.string().optional(), + organization: z.string().optional(), + organizationalUnitName: z.string().optional(), + emailAddress: z.string().email().optional(), + serialNumber: z.string().optional(), + streetAddress: z.string().optional(), + postalCode: z.string().optional(), + businessCategory: z.string().optional(), + jurisdictionCountryName: z.string().length(2).optional(), + jurisdictionStateOrProvinceName: z.string().optional(), + jurisdictionLocalityName: z.string().optional(), +}); + +export type Subject = z.infer; + +// Extensions Schema +export const ExtensionsSchema = z.object({ + subjectAltName: z.array(z.string()).optional(), + keyUsage: z + .array( + z.enum([ + "digitalSignature", + "nonRepudiation", + "keyEncipherment", + "dataEncipherment", + "keyAgreement", + "keyCertSign", + "cRLSign", + "encipherOnly", + "decipherOnly", + ]), + ) + .optional(), + extendedKeyUsage: z + .array( + z.enum([ + "serverAuth", + "clientAuth", + "codeSigning", + "emailProtection", + "timeStamping", + "OCSPSigning", + "ipsecIKE", + "msCodeInd", + "msCodeCom", + "msCTLSign", + "msEFS", + ]), + ) + .optional(), + basicConstraints: z + .object({ + cA: z.boolean().optional(), + pathLenConstraint: z.number().int().min(0).optional(), + }) + .optional(), + authorityKeyIdentifier: z.string().optional(), + subjectKeyIdentifier: z.string().optional(), + certificatePolicies: z + .array( + z.object({ + policyIdentifier: z.string(), + policyQualifiers: z.array(z.string()).optional(), + }), + ) + .optional(), + crlDistributionPoints: z.array(z.string().url()).optional(), + authorityInfoAccess: z + .object({ + ocsp: z.array(z.string().url()).optional(), + caIssuers: z.array(z.string().url()).optional(), + }) + .optional(), + nameConstraints: z + .object({ + permitted: z.array(z.string()).optional(), + excluded: z.array(z.string()).optional(), + }) + .optional(), +}); + +export type Extensions = z.infer; + +// Main FPDeviceIDPayload Schema +export const FPDeviceIDPayloadSchema = JWTPayloadSchema.extend({ + csr: z + .object({ + subject: SubjectSchema, + publicKey: JWKPublicSchema, + extensions: ExtensionsSchema, + }) + .readonly(), +}).readonly(); + +// Type inference +export type FPDeviceIDPayload = z.infer; diff --git a/core/types/base/index.ts b/core/types/base/index.ts index 667182a82..77c0c92a8 100644 --- a/core/types/base/index.ts +++ b/core/types/base/index.ts @@ -1,3 +1,13 @@ export * from "./indexer.js"; export * from "./key-bag-if.js"; export * from "./types.js"; + +export * from "./jwt-payload.zod.js"; +export * from "./fp-device-id-payload.zod.js"; +export * from "./fp-ca-cert-payload.zod.js"; +export * from "./jwk-public.zod.js"; +export * from "./jwk-private.zod.js"; +export * from "./device-id.js"; +export * from "./device-id-keybag-item.zod.js"; + +export * from "./keybag-storage.zod.js"; diff --git a/core/types/base/jwk-private.zod.ts b/core/types/base/jwk-private.zod.ts new file mode 100644 index 000000000..79e3c0de8 --- /dev/null +++ b/core/types/base/jwk-private.zod.ts @@ -0,0 +1,56 @@ +import { z } from "zod"; + +// JWK Schema + +export const JWKPrivateSchema = z + .object({ + kty: z.enum(["RSA", "EC", "oct", "OKP"]), + use: z.enum(["sig", "enc"]).optional(), + key_ops: z + .array(z.enum(["sign", "verify", "encrypt", "decrypt", "wrapKey", "unwrapKey", "deriveKey", "deriveBits"])) + .optional(), + alg: z.string().optional(), + kid: z.string().optional(), + x5u: z.string().url().optional(), + x5c: z.array(z.string()).optional(), + x5t: z.string().optional(), + "x5t#S256": z.string().optional(), + }) + .and( + z.discriminatedUnion("kty", [ + // RSA Key + z.object({ + kty: z.literal("RSA"), + n: z.string(), // modulus + e: z.string(), // exponent + d: z.string(), // private exponent + p: z.string(), // first prime factor + q: z.string(), // second prime factor + dp: z.string(), // first factor CRT exponent + dq: z.string(), // second factor CRT exponent + qi: z.string(), // first CRT coefficient + }), + // Elliptic Curve Key + z.object({ + kty: z.literal("EC"), + crv: z.enum(["P-256", "P-384", "P-521", "secp256k1"]), + x: z.string(), // x coordinate + y: z.string(), // y coordinate + d: z.string(), // private key + }), + // Octet sequence (symmetric key) + z.object({ + kty: z.literal("oct"), + k: z.string(), // key value + }), + // Octet string key pairs (Ed25519, Ed448, X25519, X448) + z.object({ + kty: z.literal("OKP"), + crv: z.enum(["Ed25519", "Ed448", "X25519", "X448"]), + x: z.string(), // public key + d: z.string(), // private key + }), + ]), + ); + +export type JWKPrivate = z.infer; diff --git a/core/types/base/jwk-public.zod.ts b/core/types/base/jwk-public.zod.ts new file mode 100644 index 000000000..06dcf84be --- /dev/null +++ b/core/types/base/jwk-public.zod.ts @@ -0,0 +1,56 @@ +import { z } from "zod"; + +// JWK Schema + +export const JWKPublicSchema = z + .object({ + kty: z.enum(["RSA", "EC", "oct", "OKP"]), + use: z.enum(["sig", "enc"]).optional(), + key_ops: z + .array(z.enum(["sign", "verify", "encrypt", "decrypt", "wrapKey", "unwrapKey", "deriveKey", "deriveBits"])) + .optional(), + alg: z.string().optional(), + kid: z.string().optional(), + x5u: z.string().url().optional(), + x5c: z.array(z.string()).optional(), + x5t: z.string().optional(), + "x5t#S256": z.string().optional(), + }) + .and( + z.discriminatedUnion("kty", [ + // RSA Key + z.object({ + kty: z.literal("RSA"), + n: z.string(), // modulus + e: z.string(), // exponent + // d: z.string().optional(), // private exponent + // p: z.string().optional(), // first prime factor + // q: z.string().optional(), // second prime factor + // dp: z.string().optional(), // first factor CRT exponent + // dq: z.string().optional(), // second factor CRT exponent + // qi: z.string().optional(), // first CRT coefficient + }), + // Elliptic Curve Key + z.object({ + kty: z.literal("EC"), + crv: z.enum(["P-256", "P-384", "P-521", "secp256k1"]), + x: z.string(), // x coordinate + y: z.string(), // y coordinate + // d: z.string().optional(), // private key + }), + // Octet sequence (symmetric key) + z.object({ + kty: z.literal("oct"), + k: z.string(), // key value + }), + // Octet string key pairs (Ed25519, Ed448, X25519, X448) + z.object({ + kty: z.literal("OKP"), + crv: z.enum(["Ed25519", "Ed448", "X25519", "X448"]), + x: z.string(), // public key + // d: z.string().optional(), // private key + }), + ]), + ); + +export type JWKPublic = z.infer; diff --git a/core/types/base/jwt-payload.zod.ts b/core/types/base/jwt-payload.zod.ts new file mode 100644 index 000000000..3182c4773 --- /dev/null +++ b/core/types/base/jwt-payload.zod.ts @@ -0,0 +1,14 @@ +import z from "zod"; + +// JWT Payload Schema (standard claims) +export const JWTPayloadSchema = z.object({ + iss: z.string().optional(), // issuer + sub: z.string().optional(), // subject + aud: z.union([z.string(), z.array(z.string())]).optional(), // audience + exp: z.number().int().optional(), // expiration time + nbf: z.number().int().optional(), // not before + iat: z.number().int().optional(), // issued at + jti: z.string().optional(), // JWT ID +}); + +export type JWTPayload = z.infer; diff --git a/core/types/base/key-bag-if.ts b/core/types/base/key-bag-if.ts index 99af79f4f..4985c368f 100644 --- a/core/types/base/key-bag-if.ts +++ b/core/types/base/key-bag-if.ts @@ -1,6 +1,20 @@ -import { Logger, URI, Result } from "@adviser/cement"; -import { KeysByFingerprint } from "@fireproof/core-types-blockstore"; -import { KeyBagRuntime } from "./types.js"; +import { Logger, URI, Result, Option } from "@adviser/cement"; +import { KeyBagRuntime, KeysByFingerprint } from "./types.js"; +import { JWKPrivate } from "./jwk-private.zod.js"; +import { DeviceIdKeyBagItem } from "./device-id-keybag-item.zod.js"; +import { JWTPayload } from "./jwt-payload.zod.js"; +import type { JWK, JWTVerifyOptions, KeyObject } from "jose"; + +export interface DeviceIdResult { + readonly deviceId: Option; + readonly cert: Option; +} + +export interface JWTResult { + readonly key: string; + readonly jwt: string; + readonly claims?: JWTPayload; +} export interface KeyBagIf { readonly logger: Logger; @@ -12,4 +26,12 @@ export interface KeyBagIf { // flush(): Promise; getNamedKey(name: string, failIfNotFound?: boolean, material?: string | Uint8Array): Promise>; + + setJwt(name: string, jwtStr: string, key?: CryptoKey | KeyObject | JWK | Uint8Array, opts?: JWTVerifyOptions): Promise>; + getJwt(name: string, key?: CryptoKey | KeyObject | JWK | Uint8Array, opts?: JWTVerifyOptions): Promise>; + + delete(name: string): Promise; + + getDeviceId(): Promise; + setDeviceId(deviceId: JWKPrivate, rIssueCert?: DeviceIdKeyBagItem["cert"]): Promise; } diff --git a/core/types/base/keybag-storage.zod.ts b/core/types/base/keybag-storage.zod.ts new file mode 100644 index 000000000..9e2fdc9c6 --- /dev/null +++ b/core/types/base/keybag-storage.zod.ts @@ -0,0 +1,67 @@ +import { z } from "zod"; +import { DeviceIdKeyBagItemSchema } from "./device-id-keybag-item.zod.js"; + +export const V1StorageKeyItemSchema = z + .object({ + name: z.string(), + key: z.string(), + }) + .readonly(); + +export const V2KeysItemSchema = z + .object({ + key: z.string(), // material + fingerPrint: z.string(), + default: z.boolean().optional(), + }) + .readonly(); + +export type KeysItem = z.infer; + +export const V2StorageKeyItemSchema = z + .object({ + name: z.string(), + keys: z.record(z.string(), V2KeysItemSchema), + }) + .readonly(); + +export type V1StorageKeyItem = z.infer; +export type V2StorageKeyItem = z.infer; +export type V2KeysItem = z.infer; + +export const KeyedV2StorageKeyItemSchema = z + .object({ + id: z.string(), + clazz: z.literal("V2StorageKeyItem"), + item: V2StorageKeyItemSchema, + }) + .readonly(); + +export const KeyedDeviceIdKeyBagItemSchema = z + .object({ + id: z.string(), + clazz: z.literal("DeviceIdKeyBagItem"), + item: DeviceIdKeyBagItemSchema, + }) + .readonly(); + +export const KeyedJwtKeyBagItemSchema = z + .object({ + id: z.string(), + clazz: z.literal("JwtKeyBagItem"), + item: z.object({ + jwtStr: z.string(), + }), + }) + .readonly(); + +export const KeyedItemSchema = KeyedV2StorageKeyItemSchema.or(KeyedDeviceIdKeyBagItemSchema).or(KeyedJwtKeyBagItemSchema); +export type KeyedItem = z.infer; + +export type KeyedV2StorageKeyItem = z.infer; +export type KeyedDeviceIdKeyBagItem = z.infer; +export type KeyedJwtKeyBagItem = z.infer; + +export const LegacyKeyedItemSchema = KeyedItemSchema.or(V1StorageKeyItemSchema).or(V2StorageKeyItemSchema).readonly(); + +export type LegacyKeyedItem = z.infer; diff --git a/core/types/base/package.json b/core/types/base/package.json index 46d26fe28..8d139ed75 100644 --- a/core/types/base/package.json +++ b/core/types/base/package.json @@ -36,11 +36,13 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-types-blockstore": "workspace:0.0.0", "@fireproof/vendor": "workspace:0.0.0", "@web3-storage/pail": "^0.6.2", + "jose": "^6.0.12", "multiformats": "^13.4.0", - "prolly-trees": "^1.0.4" + "prolly-trees": "^1.0.4", + "zod": "^4.0.14" } } diff --git a/core/types/base/types.ts b/core/types/base/types.ts index ca120741b..4ae0613c8 100644 --- a/core/types/base/types.ts +++ b/core/types/base/types.ts @@ -25,6 +25,7 @@ import type { import type { IndexIf } from "./indexer.js"; import { SerdeGatewayInterceptor } from "@fireproof/core-types-blockstore"; +import { KeysItem, V2StorageKeyItem } from "./keybag-storage.zod.js"; export class NotFoundError extends Error { readonly code = "ENOENT"; @@ -120,10 +121,21 @@ export interface PathOps { export type ToUInt8 = Uint8Array | Result; export type PromiseToUInt8 = ToUInt8 | Promise | Promise>; +export interface Base64EndeCoder { + encode(input: string | ToUInt8): string; + decodeUint8(input: string): Uint8Array; + decode(input: string): string; +} export interface TextEndeCoder { encode(input: string): Uint8Array; decode(input: ToUInt8): string; + readonly base64: Base64EndeCoder; +} + +export interface TextEndeCodable { + txt: TextEndeCoder; } + export interface SuperThisOpts { // readonly crypto?: CryptoRuntime; readonly logger: Logger; @@ -399,6 +411,15 @@ export type VoidFn = () => void; export type UnReg = () => void; export interface CRDTClock { readonly head: ClockHead; + /** + * Active update listeners registered via `onTick`. Keyed by unique IDs generated by `SuperThis.timeOrderedNextId()`. + */ + readonly watchers: Map[]) => void>; + /** + * Listeners that can be notified without loading update payloads (registered via `onTock`). + */ + readonly noPayloadWatchers: Map; + onTock(fn: VoidFn): UnReg; onTick(fn: (updates: DocUpdate[]) => void): UnReg; applyHead(newHead: ClockHead, prevHead: ClockHead, updates?: DocUpdate[]): Promise; @@ -724,23 +745,6 @@ export interface Ledger extends HasCRDT { // compact(): Promise; } -export interface V1StorageKeyItem { - readonly name: string; - readonly key: string; -} - -export interface V2StorageKeyItem { - readonly key: string; // material - readonly fingerPrint: string; - readonly default: boolean; -} - -// Serialized Version -export interface V2KeysItem { - readonly name: string; - readonly keys: Record; -} - export interface KeyMaterial { readonly key: Uint8Array; readonly keyStr: string; @@ -751,7 +755,7 @@ export interface KeyWithFingerPrint { readonly fingerPrint: string; readonly key: CTCryptoKey; extract(): Promise; - asV2StorageKeyItem(): Promise; + asKeysItem(): Promise; } export interface KeyUpsertResultModified { @@ -765,6 +769,7 @@ export function isKeyUpsertResultModified(r: KeyUpsertResult): r is KeyUpsertRes export interface KeyUpsertResultNotModified { readonly modified: false; + readonly kfp: KeyWithFingerPrint } export type KeyUpsertResult = KeyUpsertResultModified | KeyUpsertResultNotModified; @@ -774,17 +779,12 @@ export interface KeysByFingerprint { readonly name: string; get(fingerPrint?: string | Uint8Array): Promise; upsert(key: string | Uint8Array, def?: boolean): Promise>; - asV2KeysItem(): Promise; -} - -export interface KeysItem { - readonly name: string; - readonly keys: Record; + asV2StorageKeyItem(): Promise; } export interface KeyBagProvider { - get(id: string): Promise; - set(item: V2KeysItem): Promise; + get(id: string): Promise | undefined>; + set(id: string, item: NonNullable): Promise; del(id: string): Promise; } diff --git a/core/types/blockstore/package.json b/core/types/blockstore/package.json index f460d9808..a4093b9cf 100644 --- a/core/types/blockstore/package.json +++ b/core/types/blockstore/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/core-types-runtime": "workspace:0.0.0", "@fireproof/vendor": "workspace:0.0.0", diff --git a/core/types/blockstore/types.ts b/core/types/blockstore/types.ts index 0daeda776..52a712361 100644 --- a/core/types/blockstore/types.ts +++ b/core/types/blockstore/types.ts @@ -12,9 +12,10 @@ import { SuperThis, TraceFn, KeyBagRuntime, - V2KeysItem, KeyBagIf, CompactStrategy, + KeysByFingerprint, + KeyWithFingerPrint, } from "@fireproof/core-types-base"; import { CoerceURI, CryptoRuntime, CTCryptoKey, Future, Logger, Result, URI, AppContext } from "@adviser/cement"; import { EventBlock } from "@web3-storage/pail/clock"; @@ -179,18 +180,6 @@ export interface CodecOpts { readonly noIVVerify: boolean; } -export interface KeyMaterial { - readonly key: Uint8Array; - readonly keyStr: string; -} - -export interface KeyWithFingerPrint { - readonly default: boolean; - readonly fingerPrint: string; - readonly key: CTCryptoKey; - extract(): Promise; -} - export interface KeyUpsertResultModified { readonly modified: true; readonly kfp: KeyWithFingerPrint; @@ -206,13 +195,13 @@ export interface KeyUpsertResultNotModified { export type KeyUpsertResult = KeyUpsertResultModified | KeyUpsertResultNotModified; -export interface KeysByFingerprint { - readonly id: string; - readonly name: string; - get(fingerPrint?: string | Uint8Array): Promise; - upsert(key: string | Uint8Array, def?: boolean): Promise>; - asV2KeysItem(): Promise; -} +// export interface KeysByFingerprint { +// readonly id: string; +// readonly name: string; +// get(fingerPrint?: string | Uint8Array): Promise; +// upsert(key: string | Uint8Array, def?: boolean): Promise>; +// asKeyedV2StorageKeyItem(): Promise; +// } export interface CryptoAction { readonly ivLength: number; // in bytes only 12 and 16 are allowed diff --git a/core/types/protocols/cloud/gateway-control.ts b/core/types/protocols/cloud/gateway-control.ts index ae3593624..dca9524fa 100644 --- a/core/types/protocols/cloud/gateway-control.ts +++ b/core/types/protocols/cloud/gateway-control.ts @@ -1,6 +1,6 @@ import { Logger, CoerceURI, URI, AppContext } from "@adviser/cement"; import { Attachable, SuperThis } from "@fireproof/core-types-base"; -import { FPCloudClaim } from "./msg-types.js"; +import { FPCloudClaim } from "./msg-types.zod.js"; export interface ToCloudAttachable extends Attachable { token?: string; diff --git a/core/types/protocols/cloud/index.ts b/core/types/protocols/cloud/index.ts index bd08b9b81..d680d8fdb 100644 --- a/core/types/protocols/cloud/index.ts +++ b/core/types/protocols/cloud/index.ts @@ -2,4 +2,5 @@ export * from "./msg-types-data.js"; export * from "./msg-types-meta.js"; export * from "./msg-types-wal.js"; export * from "./msg-types.js"; +export * from "./msg-types.zod.js"; export * from "./gateway-control.js"; diff --git a/core/types/protocols/cloud/msg-types.ts b/core/types/protocols/cloud/msg-types.ts index 0885f18fd..23828bc65 100644 --- a/core/types/protocols/cloud/msg-types.ts +++ b/core/types/protocols/cloud/msg-types.ts @@ -1,7 +1,7 @@ import { Future, Logger, Result } from "@adviser/cement"; import { SuperThis } from "@fireproof/core-types-base"; import { CalculatePreSignedUrl } from "./msg-types-data.js"; -import type { JWTPayload } from "jose"; +import { FPCloudClaim, ReadWrite, Role, TenantLedger } from "./msg-types.zod.js"; // import { PreSignedMsg } from "./pre-signed-url.js"; export const VERSION = "FP-MSG-1.0"; @@ -13,7 +13,7 @@ export interface BaseTokenParam { readonly validFor: number; } -export type ReadWrite = "read" | "write"; +// export type ReadWrite = "read" | "write"; export function toReadWrite(i?: string): ReadWrite { if (!i) { @@ -27,7 +27,7 @@ export function toReadWrite(i?: string): ReadWrite { } } -export type Role = "admin" | "owner" | "member"; +// export type Role = "admin" | "owner" | "member"; export function toRole(i?: string): Role { if (!i) { @@ -43,30 +43,8 @@ export function toRole(i?: string): Role { } } -interface TenantClaim { - readonly id: string; - readonly role: Role; -} - -interface LedgerClaim { - readonly id: string; - readonly role: Role; - readonly right: ReadWrite; -} - // export type RoleClaim = TenantClaim | LedgerClaim; -export interface FPCloudClaim extends JWTPayload { - readonly userId: string; - readonly email: string; - readonly nickname?: string; - readonly provider?: "github" | "google"; - readonly created: Date; - readonly tenants: TenantClaim[]; - readonly ledgers: LedgerClaim[]; - readonly selected: TenantLedger; -} - // export interface FPWaitTokenResult { // readonly type: "FPWaitTokenResult"; // readonly token: string; @@ -169,11 +147,6 @@ export interface FPCloudAuthType extends AuthType { export type AuthFactory = (tp?: Partial) => Promise>; -export interface TenantLedger { - readonly tenant: string; - readonly ledger: string; -} - export function keyTenantLedger(t: TenantLedger): string { return `${t.tenant}:${t.ledger}`; } diff --git a/core/types/protocols/cloud/msg-types.zod.ts b/core/types/protocols/cloud/msg-types.zod.ts new file mode 100644 index 000000000..1bfdcc9c9 --- /dev/null +++ b/core/types/protocols/cloud/msg-types.zod.ts @@ -0,0 +1,56 @@ +import { z } from "zod"; +import { JWTPayloadSchema } from "@fireproof/core-types-base"; + +// Role and ReadWrite enums +export const RoleSchema = z.enum(["admin", "owner", "member"]); +export const ReadWriteSchema = z.enum(["read", "write"]); + +// Related interface schemas +export const TenantClaimSchema = z.object({ + id: z.string(), + role: RoleSchema, +}); + +export const LedgerClaimSchema = z.object({ + id: z.string(), + role: RoleSchema, + right: ReadWriteSchema, +}); + +export const TenantLedgerSchema = z.object({ + tenant: z.string(), + ledger: z.string(), +}); + +// Main FPCloudClaim schema +export const FPCloudClaimSchema = JWTPayloadSchema.extend({ + userId: z.string(), + email: z.email(), + nickname: z.string().optional(), + provider: z.enum(["github", "google"]).optional(), + created: z.date(), + tenants: z.array(TenantClaimSchema), + ledgers: z.array(LedgerClaimSchema), + selected: TenantLedgerSchema, +}); + +// Type inference from schemas +export type Role = z.infer; +export type ReadWrite = z.infer; +export type TenantClaim = z.infer; +export type LedgerClaim = z.infer; +export type TenantLedger = z.infer; +export type FPCloudClaim = z.infer; + +// For parsing JWT payload with date transformation +export const FPCloudClaimParseSchema = JWTPayloadSchema.extend({ + userId: z.string(), + email: z.email(), + nickname: z.string().optional(), + provider: z.enum(["github", "google"]).optional(), + // Transform string to Date if needed (common in JWT parsing) + created: z.union([z.date(), z.string().transform((str) => new Date(str)), z.number().transform((num) => new Date(num))]), + tenants: z.array(TenantClaimSchema), + ledgers: z.array(LedgerClaimSchema), + selected: TenantLedgerSchema, +}); diff --git a/core/types/protocols/cloud/package.json b/core/types/protocols/cloud/package.json index 7bd09638c..2e1c073d5 100644 --- a/core/types/protocols/cloud/package.json +++ b/core/types/protocols/cloud/package.json @@ -36,12 +36,13 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-types-base": "workspace:0.0.0", "@fireproof/core-types-blockstore": "workspace:0.0.0", "@fireproof/vendor": "workspace:0.0.0", "jose": "^6.0.12", - "multiformats": "^13.4.0" + "multiformats": "^13.4.0", + "zod": "^4.0.14" }, "devDependencies": { "@fireproof/core-cli": "workspace:0.0.0" diff --git a/core/types/runtime/package.json b/core/types/runtime/package.json index 1573f7870..97ad152cb 100644 --- a/core/types/runtime/package.json +++ b/core/types/runtime/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/fireproof-storage/fireproof/issues" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/vendor": "workspace:0.0.0", "multiformats": "^13.4.0" } diff --git a/dashboard/package.json b/dashboard/package.json index c11250d16..fc5c76c69 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -22,7 +22,7 @@ "publish": "echo skip" }, "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@clerk/backend": "^2.7.1", "@clerk/clerk-js": "^5.77.0", "@clerk/clerk-react": "^5.40.0", @@ -47,7 +47,8 @@ "react-router-dom": "^7.7.1", "react-simple-code-editor": "^0.14.1", "use-editable": "^2.3.3", - "use-fireproof": "workspace:0.0.0" + "use-fireproof": "workspace:0.0.0", + "zod": "^4.0.14" }, "devDependencies": { "@clerk/clerk-react": "^5.40.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4accb65b8..45b54074e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,8 +71,8 @@ importers: cli: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-runtime': specifier: workspace:0.0.0 version: link:../core/runtime @@ -126,8 +126,8 @@ importers: cloud/3rd-party: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) react-dom: specifier: ^19.1.1 version: 19.1.1(react@19.1.1) @@ -151,8 +151,8 @@ importers: cloud/backend/base: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@cloudflare/workers-types': specifier: ^4.20250810.0 version: 4.20250819.0 @@ -215,8 +215,8 @@ importers: cloud/backend/cf-d1: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@cloudflare/workers-types': specifier: ^4.20250810.0 version: 4.20250819.0 @@ -276,8 +276,8 @@ importers: cloud/backend/node: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/cloud-backend-base': specifier: workspace:0.0.0 version: link:../base @@ -337,8 +337,8 @@ importers: cloud/base: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-blockstore': specifier: workspace:0.0.0 version: link:../../core/blockstore @@ -377,8 +377,8 @@ importers: cloud/todo-app: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/vendor': specifier: workspace:0.0.0 version: link:../../vendor @@ -405,8 +405,8 @@ importers: core/base: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-blockstore': specifier: workspace:0.0.0 version: link:../blockstore @@ -451,8 +451,8 @@ importers: core/blockstore: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-gateways-base': specifier: workspace:0.0.0 version: link:../gateways/base @@ -511,8 +511,8 @@ importers: core/core: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-base': specifier: workspace:0.0.0 version: link:../base @@ -529,11 +529,42 @@ importers: specifier: '>=18.0.0' version: 19.1.1 + core/device-id: + dependencies: + '@adviser/cement': + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) + '@fireproof/core-keybag': + specifier: workspace:0.0.0 + version: link:../keybag + '@fireproof/core-runtime': + specifier: workspace:0.0.0 + version: link:../runtime + '@fireproof/core-types-base': + specifier: workspace:0.0.0 + version: link:../types/base + jose: + specifier: ^6.0.12 + version: 6.0.12 + multiformats: + specifier: ^13.3.7 + version: 13.4.0 + zod: + specifier: ^4.0.14 + version: 4.0.14 + devDependencies: + '@fireproof/core-cli': + specifier: workspace:0.0.0 + version: link:../../cli + '@types/node': + specifier: ^24.1.0 + version: 24.3.0 + core/gateways/base: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-runtime': specifier: workspace:0.0.0 version: link:../../runtime @@ -556,8 +587,8 @@ importers: core/gateways/cloud: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-gateways-base': specifier: workspace:0.0.0 version: link:../base @@ -586,8 +617,8 @@ importers: core/gateways/file: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-gateways-base': specifier: workspace:0.0.0 version: link:../base @@ -623,8 +654,8 @@ importers: core/gateways/file-deno: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-types-base': specifier: workspace:0.0.0 version: link:../../types/base @@ -641,8 +672,8 @@ importers: core/gateways/file-node: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-types-base': specifier: workspace:0.0.0 version: link:../../types/base @@ -653,8 +684,8 @@ importers: core/gateways/indexeddb: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-gateways-base': specifier: workspace:0.0.0 version: link:../base @@ -677,8 +708,8 @@ importers: core/gateways/memory: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-gateways-base': specifier: workspace:0.0.0 version: link:../base @@ -705,11 +736,27 @@ importers: specifier: ^24.3.0 version: 24.3.0 - core/keybag: + core/jwks: dependencies: '@adviser/cement': specifier: ^0.4.25 version: 0.4.25(typescript@5.9.2) + jose: + specifier: ^6.0.12 + version: 6.0.12 + devDependencies: + typescript: + specifier: ^5.9.2 + version: 5.9.2 + vitest: + specifier: ^3.2.4 + version: 3.2.4(@types/node@24.3.0)(@vitest/browser@3.2.4)(jiti@1.21.7)(tsx@4.20.4)(yaml@2.8.1) + + core/keybag: + dependencies: + '@adviser/cement': + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-gateways-file': specifier: workspace:0.0.0 version: link:../gateways/file @@ -722,21 +769,24 @@ importers: '@fireproof/core-types-base': specifier: workspace:0.0.0 version: link:../types/base - '@fireproof/core-types-blockstore': - specifier: workspace:0.0.0 - version: link:../types/blockstore '@fireproof/vendor': specifier: workspace:0.0.0 version: link:../../vendor + jose: + specifier: ^6.0.12 + version: 6.0.12 multiformats: specifier: ^13.4.0 version: 13.4.0 + zod: + specifier: ^4.0.14 + version: 4.0.14 core/protocols/cloud: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-runtime': specifier: workspace:0.0.0 version: link:../../runtime @@ -759,8 +809,8 @@ importers: core/protocols/dashboard: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-runtime': specifier: workspace:0.0.0 version: link:../../runtime @@ -777,8 +827,8 @@ importers: core/runtime: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-types-base': specifier: workspace:0.0.0 version: link:../types/base @@ -811,8 +861,8 @@ importers: core/tests: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core': specifier: workspace:0.0.0 version: link:../core @@ -822,6 +872,9 @@ importers: '@fireproof/core-blockstore': specifier: workspace:0.0.0 version: link:../blockstore + '@fireproof/core-device-id': + specifier: workspace:0.0.0 + version: link:../device-id '@fireproof/core-gateways-base': specifier: workspace:0.0.0 version: link:../gateways/base @@ -876,6 +929,9 @@ importers: charwise: specifier: ^3.0.1 version: 3.0.1 + jose: + specifier: ^6.0.12 + version: 6.0.12 react: specifier: '>=18.0.0' version: 19.1.1 @@ -908,8 +964,8 @@ importers: core/types/base: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-types-blockstore': specifier: workspace:0.0.0 version: link:../blockstore @@ -919,18 +975,24 @@ importers: '@web3-storage/pail': specifier: ^0.6.2 version: 0.6.2 + jose: + specifier: ^6.0.12 + version: 6.0.12 multiformats: specifier: ^13.4.0 version: 13.4.0 prolly-trees: specifier: ^1.0.4 version: 1.0.4 + zod: + specifier: ^4.0.14 + version: 4.0.14 core/types/blockstore: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-types-base': specifier: workspace:0.0.0 version: link:../base @@ -954,8 +1016,8 @@ importers: core/types/protocols/cloud: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-types-base': specifier: workspace:0.0.0 version: link:../../base @@ -971,6 +1033,9 @@ importers: multiformats: specifier: ^13.4.0 version: 13.4.0 + zod: + specifier: ^4.0.14 + version: 4.0.14 devDependencies: '@fireproof/core-cli': specifier: workspace:0.0.0 @@ -979,8 +1044,8 @@ importers: core/types/runtime: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/vendor': specifier: workspace:0.0.0 version: link:../../../vendor @@ -991,8 +1056,8 @@ importers: dashboard: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@clerk/backend': specifier: ^2.7.1 version: 2.9.2(react-dom@19.1.1(react@19.1.1))(react@19.1.1) @@ -1068,6 +1133,9 @@ importers: use-fireproof: specifier: workspace:0.0.0 version: link:../use-fireproof + zod: + specifier: ^4.0.14 + version: 4.0.14 devDependencies: '@cloudflare/vite-plugin': specifier: ^1.10.1 @@ -1157,8 +1225,8 @@ importers: use-fireproof: dependencies: '@adviser/cement': - specifier: ^0.4.25 - version: 0.4.25(typescript@5.9.2) + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) '@fireproof/core-base': specifier: workspace:0.0.0 version: link:../core/base @@ -1223,6 +1291,9 @@ importers: vendor: dependencies: + '@adviser/cement': + specifier: ^0.4.26 + version: 0.4.26(typescript@5.9.2) yocto-queue: specifier: ^1.2.1 version: 1.2.1 @@ -1251,8 +1322,8 @@ importers: packages: - '@adviser/cement@0.4.25': - resolution: {integrity: sha512-1M0spovP5ztpfrNkrydIeYpnH7RKR7HP8172LU1nDpYsKzXjWKP7AAaVs+qEtgw91+Gphb43q91bKZ9mRgX8cA==} + '@adviser/cement@0.4.26': + resolution: {integrity: sha512-zq4XGHheoLtU8pThMpNwfm2MUWQqv7NTEtRn5doy/R3OVt2f8iYjvJZ7jp8YKQFy3ul8kL8l8DoDJNUy8FqLHw==} engines: {node: '>=20.19.0'} hasBin: true @@ -5637,6 +5708,9 @@ packages: zod@3.22.3: resolution: {integrity: sha512-EjIevzuJRiRPbVH4mGc8nApb/lVLKVpmUhAaR5R5doKGfAnGJ6Gr3CViAVjP+4FWSxCsybeWQdcgCtbX+7oZug==} + zod@4.0.14: + resolution: {integrity: sha512-nGFJTnJN6cM2v9kXL+SOBq3AtjQby3Mv5ySGFof5UGRHrRioSJ5iG680cYNjE/yWk671nROcpPj4hAS8nyLhSw==} + zx@8.8.0: resolution: {integrity: sha512-v0VZXgSHusDvTtZROno3Ws8xkE1uNSSwH/yF8Fm+ZwBrYhr+bRNNpsnTJ32eR/t6umc7lAz5WqdP800ugW9zFA==} engines: {node: '>= 12.17.0'} @@ -5644,7 +5718,7 @@ packages: snapshots: - '@adviser/cement@0.4.25(typescript@5.9.2)': + '@adviser/cement@0.4.26(typescript@5.9.2)': dependencies: ts-essentials: 10.1.1(typescript@5.9.2) yaml: 2.8.1 @@ -10092,4 +10166,6 @@ snapshots: zod@3.22.3: {} + zod@4.0.14: {} + zx@8.8.0: {} diff --git a/use-fireproof/package.json b/use-fireproof/package.json index cde05436c..776253656 100644 --- a/use-fireproof/package.json +++ b/use-fireproof/package.json @@ -22,7 +22,7 @@ "license": "AFL-2.0", "gptdoc": "Fireproof/React/Usage: import { useFireproof } from 'use-fireproof'; function WordCounterApp() { const { useLiveQuery, useDocument } = useFireproof('my-word-app'); const { doc: wordInput, merge: updateWordInput, save: saveWordInput, reset: clearWordInput } = useDocument({ word: '', timestamp: Date.now() }); const recentWords = useLiveQuery('timestamp', { descending: true, limit: 10 }); const { doc: { totalSubmitted }, merge: updateTotalSubmitted, save: saveTotalSubmitted } = useDocument({ _id: 'word-counter', totalSubmitted: 0 }); const handleWordSubmission = (e) => { e.preventDefault(); updateTotalSubmitted({ totalSubmitted: totalSubmitted + 1 }); saveTotalSubmitted(); saveWordInput(); clearWordInput();}; return (<>

{totalSubmitted} words submitted

updateWordInput({ word: e.target.value })} placeholder='Enter a word' />
    {recentWords.docs.map(entry => (
  • {entry.word}
    • ))}
) } export default WordCounterApp;", "dependencies": { - "@adviser/cement": "^0.4.25", + "@adviser/cement": "^0.4.26", "@fireproof/core-base": "workspace:0.0.0", "@fireproof/core-gateways-cloud": "workspace:0.0.0", "@fireproof/core-keybag": "workspace:0.0.0", diff --git a/use-fireproof/react/types.ts b/use-fireproof/react/types.ts index cdff65150..5ecd36a51 100644 --- a/use-fireproof/react/types.ts +++ b/use-fireproof/react/types.ts @@ -13,10 +13,10 @@ import type { DocWithId, IndexKeyType, FPIndexRow, - KeyBagProvider, MapFn, QueryOpts, SuperThis, + KeyBagIf, } from "@fireproof/core-types-base"; import { ToCloudAttachable, TokenAndClaims } from "@fireproof/core-types-protocols-cloud"; @@ -125,7 +125,7 @@ export interface WebToCloudCtx { readonly dashboardURI: string; // https://dev.connect.fireproof.direct/fp/cloud/api/token readonly tokenApiURI: string; // https://dev.connect.fireproof.direct/api // stores connection and token - keyBag?: KeyBagProvider; + keyBag?: KeyBagIf; // readonly uiURI: string; // default "https://dev.connect.fireproof.direct/api" // url param name for token readonly tokenParam: string; diff --git a/use-fireproof/react/use-attach.ts b/use-fireproof/react/use-attach.ts index 0dfa08d70..5fe9160b0 100644 --- a/use-fireproof/react/use-attach.ts +++ b/use-fireproof/react/use-attach.ts @@ -4,8 +4,8 @@ import { useEffect, useState } from "react"; import { AttachState as AttachHook, UseFPConfig, WebCtxHook, WebToCloudCtx } from "./types.js"; import { AppContext, BuildURI, exception2Result, KeyedResolvOnce, ResolveOnce } from "@adviser/cement"; import { decodeJwt } from "jose/jwt/decode"; -import { SuperThis, Database, KeyBagProvider } from "@fireproof/core-types-base"; -import { ensureSuperThis, hashString } from "@fireproof/core-runtime"; +import { SuperThis, Database, KeyBagIf } from "@fireproof/core-types-base"; +import { ensureSuperThis } from "@fireproof/core-runtime"; import { FPCloudClaim, ToCloudAttachable, @@ -13,7 +13,7 @@ import { TokenAndClaims, TokenStrategie, } from "@fireproof/core-types-protocols-cloud"; -import { isKeysItem, isV1StorageKeyItem } from "@fireproof/core-keybag"; +import { getKeyBag } from "@fireproof/core-keybag"; export const WebCtx = "webCtx"; @@ -27,7 +27,7 @@ class WebCtxImpl implements WebToCloudCtx { // readonly uiURI: string; readonly tokenParam: string; // if not provided set in ready - keyBag?: KeyBagProvider; + keyBag?: KeyBagIf; readonly sthis: SuperThis; dbId!: string; @@ -59,7 +59,7 @@ class WebCtxImpl implements WebToCloudCtx { async ready(db: Database): Promise { this.dbId = await db.ledger.refId(); - this.keyBag = this.keyBag ?? (await db.ledger.opts.keyBag.getBagProvider()); + this.keyBag = this.keyBag ?? (await getKeyBag(this.sthis)); } async onAction(token?: TokenAndClaims) { @@ -89,18 +89,12 @@ class WebCtxImpl implements WebToCloudCtx { return this.opts.token(); } const tc = await this._tokenAndClaims.once(async () => { - const ret = await this.keyBag?.get(`${this.dbId}/urlToken`); - if (!ret) { - return undefined; - } - let token: string; - if (isV1StorageKeyItem(ret)) { - token = ret.key; - } else if (isKeysItem(ret)) { - token = ret.keys[this.tokenParam].key; - } else { - return undefined; + const ret = await this.keyBag?.getJwt(`${this.dbId}/urlToken`); + if (!ret || ret.Err()) { + return; } + const key = ret.Ok(); + const token = key.jwt; const claims = decodeJwt(token) as FPCloudClaim; return { token, @@ -118,7 +112,7 @@ class WebCtxImpl implements WebToCloudCtx { return this.opts.resetToken(); } this._tokenAndClaims.reset(); - await this.keyBag?.del(`${this.dbId}/urlToken`); + await this.keyBag?.delete(`${this.dbId}/urlToken`); this.onAction(); } @@ -129,18 +123,8 @@ class WebCtxImpl implements WebToCloudCtx { const oldToken = await this.token(); if (oldToken?.token !== token.token) { this._tokenAndClaims.reset(); - // set this._tokenAndClaims.once(() => token); - await this.keyBag?.set({ - name: `${this.dbId}/urlToken`, - keys: { - [this.tokenParam]: { - key: token.token, - fingerPrint: await hashString(token.token), - default: false, - }, - }, - }); + await this.keyBag?.setJwt(`${this.dbId}/urlToken`, token.token); this.onAction(token); } } diff --git a/vendor/package.json b/vendor/package.json index 5ea205401..8aa5577d9 100644 --- a/vendor/package.json +++ b/vendor/package.json @@ -29,6 +29,7 @@ "zx": "^8.8.0" }, "dependencies": { + "@adviser/cement": "^0.4.26", "yocto-queue": "^1.2.1" } }