diff --git a/src/connection-from-store.ts b/src/connection-from-store.ts index 89a9d50e..bdf6bba7 100644 --- a/src/connection-from-store.ts +++ b/src/connection-from-store.ts @@ -1,5 +1,5 @@ -import { BuildURI, runtimeFn } from "@adviser/cement"; -import { bs, Database, SuperThis } from "@fireproof/core"; +import { BuildURI, CoerceURI, runtimeFn, URI } from "@adviser/cement"; +import { bs, Database, ensureLogger, SuperThis } from "@fireproof/core"; // export interface StoreOptions { // readonly data: bs.DataStore; @@ -7,71 +7,63 @@ import { bs, Database, SuperThis } from "@fireproof/core"; // readonly wal: bs.WALState; // } -// export class ConnectionFromStore extends bs.ConnectionBase { -// stores?: { -// readonly data: bs.DataStore; -// readonly meta: bs.MetaStore; -// } = undefined; +export class ConnectionFromStore extends bs.ConnectionBase { + stores?: { + readonly data: bs.DataStore; + readonly meta: bs.MetaStore; + } = undefined; -// // readonly urlData: URI; -// // readonly urlMeta: URI; + // readonly urlData: URI; + // readonly urlMeta: URI; -// readonly sthis: SuperThis; -// constructor(sthis: SuperThis, url: URI) { -// const logger = ensureLogger(sthis, "ConnectionFromStore", { -// url: () => url.toString(), -// this: 1, -// log: 1, -// }); -// super(url, logger); -// this.sthis = sthis; -// // this.urlData = url; -// // this.urlMeta = url; -// } -// async onConnect(): Promise { -// this.logger.Debug().Msg("onConnect-start"); -// // const stores = { -// // base: this.url, -// // // data: this.urlData, -// // // meta: this.urlMeta, -// // }; -// const rName = this.url.getParamResult("name"); -// if (rName.isErr()) { -// throw this.logger.Error().Err(rName).Msg("missing Parameter").AsError(); -// } -// const storeRuntime = bs.toStoreRuntime(this.sthis); -// const loader: bs.StoreFactoryItem = { -// url: this.url, -// loader: { -// ebOpts: { -// logger: this.logger, -// storeUrls: { -// data: this.url, -// meta: this.url, -// file: this.url, -// wal: this.url, -// }, -// // store: { stores }, -// storeRuntime, -// } as bs.Loadable["ebOpts"], -// sthis: this.sthis, -// } as bs.Loadable, -// }; + readonly sthis: SuperThis; + constructor(sthis: SuperThis, url: URI) { + const logger = ensureLogger(sthis, "ConnectionFromStore", { + url: () => url.toString(), + this: 1, + log: 1, + }); + super(url, logger); + this.sthis = sthis; + // this.urlData = url; + // this.urlMeta = url; + } + async onConnect(): Promise { + this.logger.Debug().Msg("onConnect-start"); + const stores = { + base: this.url, + // data: this.urlData, + // meta: this.urlMeta, + }; + const rName = this.url.getParamResult("name"); + if (rName.isErr()) { + throw this.logger.Error().Err(rName).Msg("missing Parameter").AsError(); + } + const storeRuntime = bs.toStoreRuntime({ stores }, this.sthis); + const loader = { + name: rName.Ok(), + ebOpts: { + logger: this.logger, + store: { stores }, + storeRuntime, + }, + sthis: this.sthis, + } as bs.Loadable; -// this.stores = { -// data: await storeRuntime.makeDataStore(loader), -// meta: await storeRuntime.makeMetaStore(loader), -// }; -// // await this.stores.data.start(); -// // await this.stores.meta.start(); -// this.logger.Debug().Msg("onConnect-done"); -// return; -// } -// } + this.stores = { + data: await storeRuntime.makeDataStore(loader), + meta: await storeRuntime.makeMetaStore(loader), + }; + // await this.stores.data.start(); + // await this.stores.meta.start(); + this.logger.Debug().Msg("onConnect-done"); + return; + } +} -// export function connectionFactory(sthis: SuperThis, iurl: CoerceURI): bs.ConnectionBase { -// return new ConnectionFromStore(sthis, URI.from(iurl)); -// } +export function connectionFactory(sthis: SuperThis, iurl: CoerceURI): bs.ConnectionBase { + return new ConnectionFromStore(sthis, URI.from(iurl)); +} export function makeKeyBagUrlExtractable(sthis: SuperThis) { let base = sthis.env.get("FP_KEYBAG_URL"); diff --git a/src/drive/README.md b/src/drive/README.md new file mode 100644 index 00000000..0b0c9bd4 --- /dev/null +++ b/src/drive/README.md @@ -0,0 +1,44 @@ + +# `@fireproof/drive` + +[Fireproof](https://use-fireproof.com) is an embedded JavaScript document database that runs in the browser (or anywhere with JavaScript) and **[connects to any cloud](https://www.npmjs.com/package/@fireproof/connect)**. + +This module, `@fireproof/drive`, allows you to connect your Fireproof database to google drive via pre defined google drive api endpoints, enabling you to sync your data across multiple users in real-time. + +## Get started + +We assume you already have an app that uses Fireproof in the browser, and you want to setup collaboration among multiple users via the cloud. To write your first Fireproof app, see the [Fireproof quickstart](https://use-fireproof.com/docs/react-tutorial), otherwise read on. + +### 1. Install + +In your existing Fireproof app install the connector: + +```sh +npm install @fireproof/drive +``` + +### 2. Connect + +You're all done on the server, and ready to develop locally and then deploy with no further changes. Now you just need to register the google drive gateway in your client code. Fireproof already deployed the google drive api endpoints, so you don't need anything except fresh access token to sync data with your drive + +```js +// you already have this in your app +import { useFireproof } from "use-fireproof"; +// add this line +import { registerGDriveStoreProtocol } from "@fireproof/drive"; +``` + +You should call registerGDriveStoreProtocol('gdrive:', access_token) before calling useFireproof() hook + +```js +registerGDriveStoreProtocol('gdrive:', access_token) +const { database } = useFireproof("my-app-database-name", { + storeUrls: { + base: "gdrive://", + }, + }); +``` + +### 3. Collaborate + +Now you can use Fireproof as you normally would, and it will sync in realtime with other users. Any existing apps you have that use the [live query](https://use-fireproof.com/docs/react-hooks/use-live-query) or [subscription](https://use-fireproof.com/docs/database-api/database#subscribe) APIs will automatically render multi-user updates. diff --git a/src/drive/drive-gateway.test.ts b/src/drive/drive-gateway.test.ts index ff4e4cb5..26524e82 100644 --- a/src/drive/drive-gateway.test.ts +++ b/src/drive/drive-gateway.test.ts @@ -1,20 +1,16 @@ import { fireproof } from "@fireproof/core"; -import { registerGDriveStoreProtocol } from "./drive-gateway.js"; +import {registerGDriveStoreProtocol} from "./drive-gateway.ts" import { describe, it } from "vitest"; import { smokeDB } from "../../tests/helper.js"; -describe("store-register", () => { - it("should store and retrieve data", async () => { - const unreg = registerGDriveStoreProtocol("gdrive:"); - const db = fireproof("my-database", { - store: { - stores: { - base: "gdrive://www.googleapis.com/?auth=testtoken", - }, +describe("store-register", { timeout: 100000 }, () => { + it("should store and retrieve data", { timeout: 100000 }, async () => { + registerGDriveStoreProtocol("gdrive:", "ya29.a0AeXRPp7h4fANs0x2Y9nL3TCvL96bAnUJmwQ0oOlXPcKSmnajd_h8X2yxA8vdUo62CiKySJzrhYHMhwQVqvLnNVrHaSgR23PuF6rZXLXMApAu6rfRWtVFFKS8pYjEm36VW5csE656Z5bHXzWLXitqz9we-7zskpMNbak_NWOMaCgYKAXoSAQ8SFQHGX2MiXQzoB1I3l33KyL1DJICHNw0175") + const db = fireproof("diy-0.0", { + storeUrls: { + base: "gdrive://home-improvement", }, }); await smokeDB(db); - await db.destroy(); - unreg(); }); }); diff --git a/src/drive/drive-gateway.ts b/src/drive/drive-gateway.ts index 075346e5..4e0984f6 100644 --- a/src/drive/drive-gateway.ts +++ b/src/drive/drive-gateway.ts @@ -2,6 +2,7 @@ import { BuildURI, KeyedResolvOnce, Logger, Result, URI } from "@adviser/cement" import { bs, getStore, SuperThis, ensureSuperLog, NotFoundError, isNotFoundError } from "@fireproof/core"; interface GDriveGatewayParams { + readonly auth: string; readonly driveURL: string; } @@ -65,115 +66,109 @@ export class GDriveGateway implements bs.Gateway { } async put(url: URI, body: Uint8Array): Promise { - const { pathPart } = getStore(url, this.sthis, (...args) => args.join("/")); - const rParams = url.getParamsResult("auth", "name"); + const rParams = url.getParamsResult("name", "store"); if (rParams.isErr()) { return this.logger.Error().Url(url).Err(rParams).Msg("Put Error").ResultError(); } - const { auth } = rParams.Ok(); + const { store } = rParams.Ok(); let { name } = rParams.Ok(); const index = url.getParam("index"); - if (!index) { + if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth); - if (fileId.Err()) { + const fileId = await this.#search(name, this.#toStore(store)); + if (fileId.isErr()) { if (isNotFoundError(fileId.Err())) { - const done = await this.#insert(name, body, auth, pathPart); + const fileData = new Blob([body], { type: "application/octet-stream" }); + + const done = await this.#insert(name, body, this.#toStore(store)); if (done.isErr()) { return done; } return Result.Ok(undefined); } } - const fileMetadata = await this.#get(fileId.Ok(), pathPart, auth); - const fileData = new Blob([body], { type: "application/json" }); - const done = await this.#update(fileId.Ok(), fileMetadata, fileData, auth, pathPart); + const fileData = new Blob([body], { type: "application/octet-stream" }); + + const done = await this.#update(fileId.Ok(), fileData, this.#toStore(store)); if (done.isErr()) { return done; } return Result.Ok(undefined); } - async #delete(fileId: string, auth: string): Promise> { + async #delete(fileId: string): Promise> { const url = this.params.driveURL; const headers = { - Authorization: `Bearer ${auth}`, + Authorization: `Bearer ${this.params.auth}`, }; try { - const response = await fetch(url + fileId, { + const response = await fetch(BuildURI.from(url).appendRelative('drive').appendRelative('v3').appendRelative('files').appendRelative(fileId).toString(), { method: "DELETE", - headers: headers, + headers }); return await response.json(); } catch (err) { - return this.logger.Error().Url(url).Any("init", auth).Err(err).Msg("Could not delete").ResultError(); + return this.logger.Error().Url(url).Any("init", this.params.auth).Err(err).Msg("Could not delete").ResultError(); } } - async #get(fileId: string, type: "data" | "wal" | "meta", auth: string): Promise> { + async #get(fileId: string): Promise> { let response; let headers; const url = BuildURI.from(this.params.driveURL); headers = { - Authorization: `Bearer ${auth}`, + Authorization: `Bearer ${this.params.auth}`, "Content-Type": "application/json", }; - if (type == "meta") { - response = await fetch(url + fileId, { - method: "GET", - headers, - }); - return Result.Ok(new Uint8Array(await response.arrayBuffer())); - } else { - headers = { - Authorization: `Bearer ${auth}`, - }; - response = await fetch(url.appendRelative(fileId).setParam("alt", "media").toString(), { - method: "GET", - headers, - }); - return Result.Ok(new Uint8Array(await response.arrayBuffer())); - } + + headers = { + Authorization: `Bearer ${this.params.auth}`, + }; + response = await fetch(url.appendRelative('drive').appendRelative('v3').appendRelative('files').appendRelative(fileId).setParam("alt", "media").toString(), { + method: "GET", + headers, + }); + return Result.Ok(new Uint8Array(await response.arrayBuffer())); + } async get(url: URI): Promise { - const { pathPart } = getStore(url, this.sthis, (...args) => args.join("/")); - const rParams = url.getParamsResult("auth", "name"); + const rParams = url.getParamsResult("name", "store"); if (rParams.isErr()) { return Result.Err(rParams.Err()); } let { name } = rParams.Ok(); - const { auth } = rParams.Ok(); + const { store } = rParams.Ok(); const index = url.getParam("index"); if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth); - if (fileId.Err()) { + const fileId = await this.#search(name, this.#toStore(store)); + if (fileId.isErr()) { return Result.Err(fileId.Err()); } - const response = await this.#get(fileId.Ok(), pathPart, auth); + const response = await this.#get(fileId.Ok()); return response; } async delete(url: URI): Promise { - const rParams = url.getParamsResult("auth", "name"); + const rParams = url.getParamsResult("name", "store"); if (rParams.isErr()) { return Result.Err(rParams.Err()); } - const { auth } = rParams.Ok(); + const { store } = rParams.Ok(); let { name } = rParams.Ok(); const index = url.getParam("index"); if (index) { name += `-${index}`; } - const fileId = await this.#search(name, auth); + const fileId = await this.#search(name, this.#toStore(store)); if (fileId.isErr()) { return fileId; } - return await this.#delete(fileId.Ok(), auth); + return await this.#delete(fileId.Ok()); } async subscribe(url: URI, callback: (msg: Uint8Array) => void): Promise { @@ -212,43 +207,29 @@ export class GDriveGateway implements bs.Gateway { async #update( fileId: string, - fileMetadata: object, fileData: Blob, - auth: string, store: "data" | "wal" | "meta" ): Promise> { const url = BuildURI.from(this.params.driveURL); - const boundary = "-------" + this.sthis.nextId(16).str; + const headers = { - Authorization: `Bearer ${auth}`, - "Content-Type": `multipart/related; boundary="${boundary}"`, + Authorization: `Bearer ${this.params.auth}`, + "Content-Type": `fireproof/${store}`, }; const response = await fetch( - url.appendRelative(fileId).setParam("uploadType", "multipart").setParam("fields", "id").toString(), + url.appendRelative('upload').appendRelative('drive').appendRelative('v3').appendRelative('files').appendRelative(fileId).setParam("uploadType", "media").toString(), { method: "PATCH", headers, - body: new ReadableStream({ - start: (controller) => { - controller.enqueue(this.sthis.txt.encode(`--${boundary}\r\n`)); - controller.enqueue(this.sthis.txt.encode("Content-Type: application/json\r\n\r\n")); - controller.enqueue(this.sthis.txt.encode(JSON.stringify(fileMetadata) + "\r\n")); - controller.enqueue(this.sthis.txt.encode(`--${boundary}\r\n`)); - controller.enqueue(this.sthis.txt.encode(`Content-Type: fireproof/${store}\r\n\r\n`)); - controller.enqueue(fileData.arrayBuffer()); - controller.enqueue(this.sthis.txt.encode("\r\n")); - controller.enqueue(this.sthis.txt.encode(`--${boundary}--\r\n`)); - controller.close(); - }, - }), + body: fileData } ); if (!response.ok) { - return this.logger.Error().Any({ auth, store }).Msg("Insert Error").ResultError(); + return this.logger.Error().Any({ store }).Msg("Insert Error").ResultError(); } return Result.Ok(fileId); } - async #insert(fileName: string, content: Uint8Array, auth: string, store: string): Promise> { + async #insert(fileName: string, content: Uint8Array, store: "data" | "wal" | "meta"): Promise> { const url = BuildURI.from(this.params.driveURL); const mime = `fireproof/${store}`; const file = new Blob([content], { type: mime }); @@ -261,26 +242,26 @@ export class GDriveGateway implements bs.Gateway { form.append("file", file); try { const response = await fetch( - url.setParam("uploadType", "multipart").setParam("supportsAllDrives", "true").toString(), + url.appendRelative('upload/drive/v3/files').setParam("uploadType", "multipart").setParam("supportsAllDrives", "true").toString(), { method: "POST", - headers: { Authorization: "Bearer " + auth }, + headers: { Authorization: "Bearer " + this.params.auth }, body: form, } ); const jsonRes = (await response.json()) as { id: string }; return Result.Ok(jsonRes.id); } catch (err) { - return this.logger.Error().Any({ auth, store }).Err(err).Msg("Insert Error").ResultError(); + return this.logger.Error().Any({ store }).Err(err).Msg("Insert Error").ResultError(); } } - async #search(fileName: string, auth: string): Promise> { + async #search(fileName: string, store: "data" | "wal" | "meta"): Promise> { try { const response = await fetch( - BuildURI.from("https://www.googleapis.com/drive/v3/files").setParam("q=name", `"${fileName}"`).toString(), + BuildURI.from("https://www.googleapis.com/drive/v3/files").setParam("q=mimeType", `"fireproof/${store}" and name="${fileName}"`).toString(), { headers: { - Authorization: "Bearer " + auth, + Authorization: "Bearer " + this.params.auth, }, } ); @@ -293,9 +274,13 @@ export class GDriveGateway implements bs.Gateway { } return Result.Err(new NotFoundError("File not found")); } catch (err) { - return this.logger.Error().Any({ auth, fileName }).Err(err).Msg("Fetch Error").ResultError(); + return this.logger.Error().Any({ fileName }).Err(err).Msg("Fetch Error").ResultError(); } } + #toStore(s: string): "data" | "wal" | "meta" { + if (["data", "wal", "meta"].includes(s)) { return s as "data"|"wal"|"meta" } + throw new Error(`unknown Store:${s}`) + } } // function generateRandom21DigitNumber() { // let num = Math.floor(Math.random() * 9) + 1; // First digit can't be 0 @@ -305,20 +290,24 @@ export class GDriveGateway implements bs.Gateway { // return num.toString(); // } + + const onceregisterGDriveStoreProtocol = new KeyedResolvOnce<() => void>(); -export function registerGDriveStoreProtocol(protocol = "gdrive:") { +export function registerGDriveStoreProtocol(protocol = "gdrive:", auth: string) { return onceregisterGDriveStoreProtocol.get(protocol).once(() => { URI.protocolHasHostpart(protocol); return bs.registerStoreProtocol({ - protocol, + protocol: protocol, + isDefault: false, defaultURI: (): URI => { return URI.from("gdrive://"); }, gateway: async (sthis): Promise => { return new GDriveGateway(sthis, { - driveURL: "https://www.googleapis.com/upload/drive/v3/files/", + auth: auth, + driveURL: "https://www.googleapis.com/", }); - }, + } }); }); } diff --git a/tests/helper.ts b/tests/helper.ts index c8100280..e31cbf12 100644 --- a/tests/helper.ts +++ b/tests/helper.ts @@ -1,4 +1,5 @@ import { Database } from "@fireproof/core"; +import {expect} from "vitest" export async function smokeDB(db: Database) { const ran = db.sthis.nextId().str;