diff --git a/packages/cli/src/deploy/beta.ts b/packages/cli/src/deploy/beta.ts deleted file mode 100644 index 83308b5fe..000000000 --- a/packages/cli/src/deploy/beta.ts +++ /dev/null @@ -1,46 +0,0 @@ -// beta v2 version of CLI deploy - -import Project from '@openfn/project'; -import { DeployConfig, deployProject } from '@openfn/deploy'; -import type { Logger } from '../util/logger'; -import { Opts } from '../options'; -import { loadAppAuthConfig } from '../projects/util'; - -export type DeployOptionsBeta = Required< - Pick< - Opts, - | 'beta' - | 'command' - | 'log' - | 'logJson' - | 'apiKey' - | 'endpoint' - | 'path' - | 'workspace' - > ->; - -export async function handler(options: DeployOptionsBeta, logger: Logger) { - const config = loadAppAuthConfig(options, logger); - - // TMP use options.path to set the directory for now - // We'll need to manage this a bit better - const project = await Project.from('fs', { root: options.workspace || '.' }); - // TODO: work out if there's any diff - - // generate state for the provisioner - const state = project.serialize('state', { format: 'json' }); - - logger.debug('Converted local project to app state:'); - logger.debug(JSON.stringify(state, null, 2)); - - // TODO not totally sold on endpoint handling right now - config.endpoint ??= project.openfn?.endpoint!; - - logger.info('Sending project to app...'); - - // TODO do I really want to use this deploy function? Is it suitable? - await deployProject(config as DeployConfig, state); - - logger.success('Updated project at', config.endpoint); -} diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index f5fe682f7..f10d19a6a 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -7,7 +7,7 @@ import { } from '@openfn/deploy'; import type { Logger } from '../util/logger'; import { DeployOptions } from './command'; -import * as beta from './beta'; +import * as beta from '../projects/beta'; export type DeployFn = typeof deploy; diff --git a/packages/cli/src/projects/deploy.ts b/packages/cli/src/projects/deploy.ts new file mode 100644 index 000000000..c14f74edb --- /dev/null +++ b/packages/cli/src/projects/deploy.ts @@ -0,0 +1,152 @@ +// beta v2 version of CLI deploy + +/** + * New plan for great glory + * + * - from('fs') does NOT take project file into account + * - deploy must first fetch (and ensure no conflcits) + * - deploy must then load the project from disk + * - deploy must then merge into that project + * - then call provisioner + * - finally write to disk + * + * + * PLUS: diff summary (changed workflows and steps) + * PLUS: confirm + * PLUS: dry run + * + * + * + * One possible probllem for deploy + * + * The idea is we fetch the latest server version, + * write that to disk, merge our changes, and push + * + * But what if our project file is ahead of the server? A fetch + * will conflict and we don't want to throw. + * + * The project may be ahead because: a), we checked out another branch + * and stashed changes, b) we can ran some kind of reconcilation/merge, + * c) we did a manual export (take my fs and write it to the project) + * + * So basically when fetching, we need to check for divergence in history. + * When fetching, for each workflow, we need to decide whether to keep or reject the + * server version based on the history. + * + * + * + * This is super complex and we're getting into merge territory + * First priority is: if there's a problem (that's a super difficult thing!) warn the user + * Second priority is: help the user resolve it + * + * + * The local project files are giving me a headache. But we should be strict and say: + * the project is ALWAYS a representation of the remote. It is invalid for that project + * to represent the local system + * + * So this idea that I can "save" the local to the project file is wrong + * The idea thatwhen I checkout, I "stash" to a project file is wrong + * + * I should be able to export a project to any arbitrary file, yes + * And when checking out and there are conflicts, I should be able to create a duplicate + * file to save my changes without git. + * I think that means checkout errors (it detects changes will be lost), but you have the option to + * stash a temporary local project to be checkedout later + * + * + * This clarify and strictness will I think really help + * + * So: the local project is NEVER ahead of the server + * (but what if the user edited it and it is? I think the system igores it and that's just a force push) + */ + +import Project from '@openfn/project'; +import { DeployConfig, deployProject } from '@openfn/deploy'; +import type { Logger } from '../util/logger'; +import { Opts } from '../options'; +import { loadAppAuthConfig } from './util'; + +export type DeployOptionsBeta = Required< + Pick< + Opts, + | 'beta' + | 'command' + | 'log' + | 'logJson' + | 'apiKey' + | 'endpoint' + | 'path' + | 'workspace' + > +>; + +export async function handler(options: DeployOptionsBeta, logger: Logger) { + const config = loadAppAuthConfig(options, logger); + + // First step, fetch the latest version and write + // this may throw! + try { + await fetch(options, logger); + } catch (e) { + // If fetch failed because of compatiblity, what do we do? + // + // Basically we failed to write to the local project file + // If -f is true, do we: + // a) force-fetch the latest project + // b) or force merge into the old project, and then force push? + // + // The file system may be in a real mess if fs, project and app are all diverged! + // So I think we: + // Log an error: the server has diverged from your local copy + // Run fetch to resolve the conflict (it'll throw too!) + // Pass -f to ignore your local project and pull the latest app changes + // (changes shouldn't be lost here, because its the file system that's kind) + // + // Actually, the FS is king here. + // + // What if: + // Locally I've changed workflow A + // Remove has changed workflow B + // I basically want to keep my workflow A changes and keep the workflow B changes + // But if we force, we'll force our local workflow into the project, overriding it + // Gods its complicated + // What I think you actually want to do is: + // force pull the remote version + // merge only your changed workflows onto the remote + // but merge doesn't work like that + // So either I need merge to be able to merge the fs with a project (sort of like an expand-and-merge) + // Or deploy should accept a list of workflows (only apply these workflows) + // The problem with the deploy is that the local fs will still be out of date + // + // What about openfn project reconcile + // This will fetch the remote project + // check it out into your fs + // any changed workflows you'll be promoted to: + // - keep local + // - keep app + // - keep both + // if keep both, two folders will be created. The user must manually merge + // this leaves you with a file system that can either be merged, deployed or exported + } + + // TMP use options.path to set the directory for now + // We'll need to manage this a bit better + const project = await Project.from('fs', { root: options.workspace || '.' }); + // TODO: work out if there's any diff + + // generate state for the provisioner + const state = project.serialize('state', { format: 'json' }); + + logger.debug('Converted local project to app state:'); + logger.debug(JSON.stringify(state, null, 2)); + + // TODO not totally sold on endpoint handling right now + config.endpoint ??= project.openfn?.endpoint!; + + logger.info('Sending project to app...'); + + // TODO do I really want to use this deploy function? Is it suitable? + await deployProject(config as DeployConfig, state); + + logger.success('Updated project at', config.endpoint); +} diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index fa7fe7811..8c86683c6 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -87,6 +87,8 @@ const fromProject = async ( logger: Logger ): Promise => { logger.debug('Loading Repo from ', path.resolve(rootDir)); + + // TODO make sure this does not break after changes const project = await Project.from('fs', { root: rootDir }); logger.debug('Loading workflow ', workflowName); const workflow = project.getWorkflow(workflowName); diff --git a/packages/cli/test/projects/deploy.test.ts b/packages/cli/test/projects/deploy.test.ts new file mode 100644 index 000000000..8a3d1fbc7 --- /dev/null +++ b/packages/cli/test/projects/deploy.test.ts @@ -0,0 +1,17 @@ +import test from 'ava'; + +// what will deploy tests look like? + +// deploy a project for the first time (this doesn't work though?) + +// deploy a change to a project + +// deploy a change to a project but fetch latest first + +// throw when trying to deploy to a diverged remote project + +// force deploy an incompatible project + +// don't post the final version if dry-run is set + +// TODO diff + confirm diff --git a/packages/project/README.md b/packages/project/README.md index 954a016e2..6d0d6d33e 100644 --- a/packages/project/README.md +++ b/packages/project/README.md @@ -8,6 +8,18 @@ A single Project can be Checked Out to disk at a time, meaning its source workfl A Workspace is a set of related Projects , including a Project and its associated Sandboxes, or a Project deployed to apps in multiple web domains +## Structure and Artifects + +openfn.yaml + +project file + +sort of a mix of project.yaml, state.json and config.json + +This is strictly a representation of a server-side project, it's like the last-sync-state. CLI-only or offline projects do not have one. + +It's also a portable representation of the project + ### Serializing and Parsing The main idea of Projects is that a Project represents a set of OpenFn workflows defined in any format and present a standard JS-friendly interface to manipulate and reason about them. diff --git a/packages/project/src/parse/from-fs.ts b/packages/project/src/parse/from-fs.ts index 1c0e805fc..612c2b838 100644 --- a/packages/project/src/parse/from-fs.ts +++ b/packages/project/src/parse/from-fs.ts @@ -4,53 +4,34 @@ import { glob } from 'glob'; import * as l from '@openfn/lexicon'; import { Project } from '../Project'; -import getIdentifier from '../util/get-identifier'; import { yamlToJson } from '../util/yaml'; import { buildConfig, loadWorkspaceFile, findWorkspaceFile, } from '../util/config'; -import fromProject from './from-project'; import { omit } from 'lodash-es'; +import { Logger } from '@openfn/logger'; export type FromFsConfig = { root: string; + logger?: Logger; }; // Parse a single project from a root folder +// Note that this does NOT attempt to load UUIDS from the project file +// It just builds the project on disk +// I suppose we could take an option? export const parseProject = async (options: FromFsConfig) => { - const { root } = options; + const { root, logger } = options; const { type, content } = findWorkspaceFile(root); const context = loadWorkspaceFile(content, type as any); const config = buildConfig(context.workspace); - // Now we need to look for the corresponding state file - // Need to load UUIDs and other app settings from this - // If we load it as a Project, uuid tracking is way easier - let state: Project | null = null; - const identifier = getIdentifier({ - endpoint: context.project?.endpoint, - env: context.project?.env, - }); - try { - const format = config.formats?.project ?? config.formats?.project ?? 'yaml'; - const statePath = path.join( - root, - config.dirs?.projects ?? '.projects', - `${identifier}.${format}` - ); - const stateFile = await fs.readFile(statePath, 'utf8'); - - state = fromProject(stateFile, config); - } catch (e) { - console.warn(`Failed to find state file for ${identifier}`); - // console.warn(e); - } - const proj: any = { - name: state?.name, + id: context.project?.id, + name: context.project?.name, openfn: omit(context.project, ['id']), config: config, workflows: [], @@ -74,30 +55,21 @@ export const parseProject = async (options: FromFsConfig) => { const wf = fileType === 'yaml' ? yamlToJson(candidate) : JSON.parse(candidate); if (wf.id && Array.isArray(wf.steps)) { - // load settings from the state file - const wfState = state?.getWorkflow(wf.id); - - wf.openfn = Object.assign({}, wfState?.openfn, { - uuid: wfState?.openfn?.uuid ?? null, - }); - - //console.log('Loading workflow at ', filePath); // TODO logger.debug + //logger?.log('Loading workflow at ', filePath); // TODO logger.debug for (const step of wf.steps) { // This is the saved, remote view of the step // TODO if the id has changed, how do we track? - const stateStep = wfState?.get(step.id); if (step.expression && step.expression.endsWith('.js')) { const dir = path.dirname(filePath); const exprPath = path.join(dir, step.expression); try { - console.debug(`Loaded expression from ${exprPath}`); + logger?.debug(`Loaded expression from ${exprPath}`); step.expression = await fs.readFile(exprPath, 'utf-8'); } catch (e) { - console.error(`Error loading expression from ${exprPath}`); + logger?.error(`Error loading expression from ${exprPath}`); // throw? } } - step.openfn = Object.assign({}, stateStep?.openfn); // Now track UUIDs for edges against state for (const target in step.next || {}) { @@ -105,15 +77,13 @@ export const parseProject = async (options: FromFsConfig) => { const bool = step.next[target]; step.next[target] = { condition: bool }; } - const uuid = state?.getUUID(wf.id, step.id, target) ?? null; - step.next[target].openfn = { uuid }; } } proj.workflows.push(wf); } } catch (e) { - console.log(e); + logger?.log(e); // not valid json // should probably maybe a big deal about this huh? continue; diff --git a/packages/project/test/parse/from-fs.test.ts b/packages/project/test/parse/from-fs.test.ts index 5dff39c5e..1a3d1e103 100644 --- a/packages/project/test/parse/from-fs.test.ts +++ b/packages/project/test/parse/from-fs.test.ts @@ -1,221 +1,280 @@ import test from 'ava'; import mock from 'mock-fs'; import { parseProject } from '../../src/parse/from-fs'; +import { jsonToYaml } from '../../src/util/yaml'; +import { buildConfig } from '../../src/util/config'; -const s = JSON.stringify; - -// mock several projects and use them through the tests -// TODO: the state files here are all in v1 format - need to add tests with v2 -// Probably need to rethink all these tests tbh -mock({ - '/p1/openfn.json': s({ - // this must be the whole deploy name right? - // else how do we know? - workflowRoot: 'workflows', - formats: { - openfn: 'json', - project: 'json', - workflow: 'json', - }, - project: { - id: 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00', - env: 'staging', - endpoint: 'https://app.openfn.org', - name: 'My Project', - description: '...', - // Note that we exclude app options here - // That stuff is all in the project.yaml, not useful here - }, - }), - '/p1/workflows/my-workflow': {}, - '/p1/workflows/my-workflow/my-workflow.json': s({ - id: 'my-workflow', - name: 'My Workflow', - steps: [ - { - id: 'a', - expression: 'job.js', - next: { - b: true, - }, - }, - { - id: 'b', - expression: './job.js', - next: { - c: false, - }, - }, - ], // TODO handle expressions too! - // TODO maybe test the options key though - }), - '/p1/workflows/my-workflow/job.js': `fn(s => s)`, - // keep a state file (just the stuff we need for uuids) - '/p1/.projects/staging@app.openfn.org.json': s({ - workflows: [ - { - id: '', - name: 'My Workflow', - jobs: [ - { - id: '', - name: 'a', - project_credential_id: 'p', - }, - { - id: '', - name: 'b', - }, - ], - triggers: [], - edges: [ - { - id: '', - source_job_id: '', - target_job_id: '', - }, - ], - }, - ], - }), - - // junk to throw the tests - '/p1/random.json': s({ - // not a workflow file! this should be ignored - }), - '/p1/workflows/my-workflow/random.json': s({ - // not a workflow file! this should be ignored - }), - - // p2 is all yaml based - '/p2/openfn.yaml': ` - workflowRoot: wfs - formats: - openfn: yaml - project: yaml - workflow: yaml - project: - env: main - id: "123" - endpoint: app.openfn.org`, - '/p2/wfs/my-workflow/my-workflow.yaml': ` - id: my-workflow - name: My Workflow - steps: - - id: job - adaptor: "@openfn/language-common@latest" - expression: ./job.js - `, - '/p2/wfs/my-workflow/job.js': `fn(s => s)`, - // TODO state here - quite a good test - - // p3 uses custom yaml - '/p3/openfn.yaml': ` -workspace: - x: 1 - y: 2 -project: -`, - '/p3/wfs/my-workflow/my-workflow.yaml': ` - id: my-workflow - name: My Workflow - steps: - - id: job - adaptor: "@openfn/language-common@latest" - expression: ./job.js - `, - '/p3/wfs/my-workflow/job.js': `fn(s => s)`, +test.afterEach(() => { + files = {}; + mock.restore(); }); -test('should load workspace config from json', async (t) => { - const project = await parseProject({ root: '/p1' }); +let files: Record = {}; + +function mockFile(path: string, content: string | object) { + if (path.endsWith('.yaml')) { + content = jsonToYaml(content); + } else if (path.endsWith('.json')) { + content = JSON.stringify(content); + } + + files[path] = content; + mock(files); +} + +test.serial('should load workspace config from json', async (t) => { + mockFile( + '/ws/openfn.json', + buildConfig({ + formats: { + openfn: 'json', + project: 'json', + workflow: 'json', + }, + // @ts-ignore ensure we include custom properties + x: 1, + }) + ); + + const project = await parseProject({ root: '/ws' }); t.deepEqual(project.config, { - workflowRoot: 'workflows', + x: 1, dirs: { projects: '.projects', workflows: 'workflows' }, formats: { openfn: 'json', project: 'json', workflow: 'json' }, }); }); -test('should load custom config props and include default', async (t) => { - const project = await parseProject({ root: '/p3' }); +test.serial('should load workspace config from yaml', async (t) => { + mockFile( + '/ws/openfn.yaml', + buildConfig({ + formats: { + openfn: 'yaml', + project: 'yaml', + workflow: 'yaml', + }, + // @ts-ignore ensure we include custom properties + x: 1, + }) + ); + + const project = await parseProject({ root: '/ws' }); t.deepEqual(project.config, { x: 1, - y: 2, dirs: { projects: '.projects', workflows: 'workflows' }, formats: { openfn: 'yaml', project: 'yaml', workflow: 'yaml' }, }); }); -test('should load the workspace config from json', async (t) => { - const project = await parseProject({ root: '/p1' }); +test.serial('should load single workflow', async (t) => { + mockFile('/ws/openfn.yaml', buildConfig()); - t.deepEqual(project.openfn, { - name: 'My Project', - env: 'staging', - endpoint: 'https://app.openfn.org', - description: '...', + mockFile('/ws/workflows/my-workflow/my-workflow.yaml', { + id: 'my-workflow', + name: 'My Workflow', + steps: [ + { + id: 'a', + expression: 'job.js', + }, + ], }); -}); -test('should load a workflow from the file system', async (t) => { - const project = await parseProject({ root: '/p1' }); + mockFile('/ws/workflows/my-workflow/job.js', `fn(s => s)`); + + const project = await parseProject({ root: '/ws' }); t.is(project.workflows.length, 1); - const [wf] = project.workflows; + const wf = project.getWorkflow('my-workflow'); + t.truthy(wf); t.is(wf.id, 'my-workflow'); - t.is(wf.openfn.uuid, ''); - t.is(wf.steps[0].expression, 'fn(s => s)'); + t.is(wf.name, 'My Workflow'); }); -test('should load a workflow from the file system and expand shorthand links', async (t) => { - const project = await parseProject({ root: '/p1' }); +test.serial('should load single workflow from json', async (t) => { + mockFile( + '/ws/openfn.yaml', + buildConfig({ + formats: { + workflow: 'json', + }, + }) + ); + + mockFile('/ws/workflows/my-workflow/my-workflow.json', { + id: 'my-workflow', + name: 'My Workflow', + steps: [ + { + id: 'a', + expression: 'job.js', + }, + ], + }); + + mockFile('/ws/workflows/my-workflow/job.js', `fn(s => s)`); + + const project = await parseProject({ root: '/ws' }); t.is(project.workflows.length, 1); - const [wf] = project.workflows; - t.is(typeof wf.steps[1].next.c, 'object'); + const wf = project.getWorkflow('my-workflow'); + t.truthy(wf); + t.is(wf.id, 'my-workflow'); + t.is(wf.name, 'My Workflow'); }); -test('should track the UUID of a step', async (t) => { - const project = await parseProject({ root: '/p1' }); +test.serial('should load single workflow from custom path', async (t) => { + mockFile( + '/ws/openfn.yaml', + buildConfig({ + dirs: { + workflows: 'custom-wfs', + projects: '.projects', + }, + }) + ); + + mockFile('/ws/custom-wfs/my-workflow/my-workflow.yaml', { + id: 'my-workflow', + name: 'My Workflow', + steps: [ + { + id: 'a', + expression: 'job.js', + }, + ], + }); + + mockFile('/ws/custom-wfs/my-workflow/job.js', `fn(s => s)`); + + const project = await parseProject({ root: '/ws' }); - const [wf] = project.workflows; + t.is(project.workflows.length, 1); - t.truthy(wf.steps[0].openfn); - t.is(wf.steps[0].openfn.uuid, ''); + const wf = project.getWorkflow('my-workflow'); + t.truthy(wf); + t.is(wf.id, 'my-workflow'); + t.is(wf.name, 'My Workflow'); }); -// TODO also test this on different openfn objects -test('should track openfn props from state file on a step', async (t) => { - const project = await parseProject({ root: '/p1' }); +test.serial('should include multiple workflows', async (t) => { + mockFile('/ws/openfn.yaml', buildConfig()); + + mockFile('/ws/workflows/workflow-1/workflow-1.yaml', { + id: 'workflow-1', + name: 'Workflow 1', + steps: [ + { + id: 'a', + expression: 'job.js', + }, + ], + }); + + mockFile('/ws/workflows/workflow-1/job.js', `fn(s => s)`); - const [wf] = project.workflows; + mockFile('/ws/workflows/workflow-2/workflow-2.yaml', { + id: 'workflow-2', + name: 'Workflow 2', + steps: [ + { + id: 'b', + expression: 'job.js', + }, + ], + }); - t.truthy(wf.steps[0].openfn); - t.is(wf.steps[0].openfn.project_credential_id, 'p'); -}); + mockFile('/ws/workflows/workflow-2/job.js', `fn(s => ({ data: [] }))`); -test('should track the UUID of an edge', async (t) => { - const project = await parseProject({ root: '/p1' }); + const project = await parseProject({ root: '/ws' }); - const [wf] = project.workflows; + t.is(project.workflows.length, 2); - t.truthy(wf.steps[0].next?.b.openfn); - t.is(wf.steps[0].next?.b.openfn.uuid, ''); + const wf1 = project.getWorkflow('workflow-1'); + t.truthy(wf1); + t.is(wf1.id, 'workflow-1'); + t.is(wf1.name, 'Workflow 1'); + + const wf2 = project.getWorkflow('workflow-2'); + t.truthy(wf2); + t.is(wf2.id, 'workflow-2'); + t.is(wf2.name, 'Workflow 2'); }); -test.todo('should track the UUID of a trigger'); -// maybe track other things that aren't in workflow.yaml? +test.serial('should load a workflow expression', async (t) => { + mockFile('/ws/openfn.yaml', buildConfig()); + + mockFile('/ws/workflows/my-workflow/my-workflow.yaml', { + id: 'my-workflow', + name: 'My Workflow', + steps: [ + { + id: 'a', + expression: 'job.js', + }, + ], + }); -test('should load a project from yaml', async (t) => { - const project = await parseProject({ root: '/p2' }); + mockFile('/ws/workflows/my-workflow/job.js', `fn(s => s)`); + const project = await parseProject({ root: '/ws' }); t.is(project.workflows.length, 1); - const [wf] = project.workflows; - t.is(wf.id, 'my-workflow'); + const wf = project.getWorkflow('my-workflow'); + + t.truthy(wf); + t.is(wf.steps[0].expression, 'fn(s => s)'); }); + +test.serial( + 'should return empty workflows array when no workflows found', + async (t) => { + mockFile('/ws/openfn.yaml', buildConfig()); + + const project = await parseProject({ root: '/ws' }); + + t.is(project.workflows.length, 0); + } +); + +test.serial( + 'should load a workflow from the file system and expand shorthand links', + async (t) => { + mockFile('/ws/openfn.yaml', buildConfig()); + + mockFile('/ws/workflows/my-workflow/my-workflow.yaml', { + id: 'my-workflow', + name: 'My Workflow', + steps: [ + { + id: 'a', + expression: 'job.js', + next: { + b: true, + }, + }, + { + id: 'b', + expression: './job.js', + next: { + c: false, + }, + }, + ], + }); + + mockFile('/ws/workflows/my-workflow/job.js', `fn(s => s)`); + + const project = await parseProject({ root: '/ws' }); + + t.is(project.workflows.length, 1); + const [wf] = project.workflows; + + t.is(typeof wf.steps[1].next.c, 'object'); + } +);