New deploy command

packages/cli/src/deploy/handler.ts

@@ -7,7 +7,7 @@ import {
 } from '@openfn/deploy';
 import type { Logger } from '../util/logger';
 import { DeployOptions } from './command';
-import * as beta from './beta';
+import * as beta from '../projects/beta';
 
 export type DeployFn = typeof deploy;
 

packages/cli/src/projects/deploy.ts

@@ -0,0 +1,152 @@
+// beta v2 version of CLI deploy
+
+/**
+ * New plan for great glory
+ *
+ * - from('fs') does NOT take project file into account
+ * - deploy must first fetch (and ensure no conflcits)
+ * - deploy must then load the project from disk
+ * - deploy must then merge into that project
+ * - then call provisioner
+ * - finally write to disk
+ *
+ *
+ * PLUS: diff summary (changed workflows and steps)
+ * PLUS: confirm
+ * PLUS: dry run
+ *
+ *
+ *
+ * One possible probllem for deploy
+ *
+ * The idea is we fetch the latest server version,
+ * write that to disk, merge our changes, and push
+ *
+ * But what if our project file is ahead of the server? A fetch
+ * will conflict and we don't want to throw.
+ *
+ * The project may be ahead because: a), we checked out another branch
+ * and stashed changes, b) we can ran some kind of reconcilation/merge,
+ * c) we did a manual export (take my fs and write it to the project)
+ *
+ * So basically when fetching, we need to check for divergence in history.
+ * When fetching, for each workflow, we need to decide whether to keep or reject the
+ * server version based on the history.
+ *
+ *
+ *
+ * This is super complex and we're getting into merge territory
+ * First priority is: if there's a problem (that's a super difficult thing!) warn the user
+ * Second priority is: help the user resolve it
+ *
+ *
+ * The local project files are giving me a headache. But we should be strict and say:
+ * the project is ALWAYS a representation of the remote. It is invalid for that project
+ * to represent the local system
+ *
+ * So this idea that I can "save" the local to the project file is wrong
+ * The idea thatwhen I checkout, I "stash" to a project file is wrong
+ *
+ * I should be able to export a project to any arbitrary file, yes
+ * And when checking out and there are conflicts, I should be able to create a duplicate
+ * file to save my changes without git.
+ * I think that means checkout errors (it detects changes will be lost), but you have the option to
+ * stash a temporary local project to be checkedout later
+ *
+ *
+ * This clarify and strictness will I think really help
+ *
+ * So: the local project is NEVER ahead of the server
+ * (but what if the user edited it and it is? I think the system igores it and that's just a force push)
+ */
+
+import Project from '@openfn/project';
+import { DeployConfig, deployProject } from '@openfn/deploy';
+import type { Logger } from '../util/logger';
+import { Opts } from '../options';
+import { loadAppAuthConfig } from './util';
+
+export type DeployOptionsBeta = Required<
+  Pick<
+    Opts,
+    | 'beta'
+    | 'command'
+    | 'log'
+    | 'logJson'
+    | 'apiKey'
+    | 'endpoint'
+    | 'path'
+    | 'workspace'
+  >
+>;
+
+export async function handler(options: DeployOptionsBeta, logger: Logger) {
+  const config = loadAppAuthConfig(options, logger);
+
+  // First step, fetch the latest version and write
+  // this may throw!
+  try {
+    await fetch(options, logger);
+  } catch (e) {
+    // If fetch failed because of compatiblity, what do we do?
+    //
+    // Basically we failed to write to the local project file
+    // If -f is true, do we:
+    // a) force-fetch the latest project
+    // b) or force merge into the old project, and then force push?
+    //
+    // The file system may be in a real mess if fs, project and app are all diverged!
+    // So I think we:
+    // Log an error: the server has diverged from your local copy
+    // Run fetch to resolve the conflict (it'll throw too!)
+    // Pass -f to ignore your local project and pull the latest app changes
+    // (changes shouldn't be lost here, because its the file system that's kind)
+    //
+    // Actually, the FS is king here.
+    //
+    // What if:
+    // Locally I've changed workflow A
+    // Remove has changed workflow B
+    // I basically want to keep my workflow A changes and keep the workflow B changes
+    // But if we force, we'll force our local workflow into the project, overriding it
+    // Gods its complicated
+    // What I think you actually want to do is:
+    // force pull the remote version
+    // merge only your changed workflows onto the remote
+    // but merge doesn't work like that
+    // So either I need merge to be able to merge the fs with a project (sort of like an expand-and-merge)
+    // Or deploy should accept a list of workflows (only apply these workflows)
+    // The problem with the deploy is that the local fs will still be out of date
+    //
+    // What about openfn project reconcile
+    // This will fetch the remote project
+    // check it out into your fs
+    // any changed workflows you'll be promoted to:
+    // - keep local
+    // - keep app
+    // - keep both
+    // if keep both, two folders will be created. The user must manually merge
+    // this leaves you with a file system that can either be merged, deployed or exported
+  }
+
+  // TMP use options.path to set the directory for now
+  // We'll need to manage this a bit better
+  const project = await Project.from('fs', { root: options.workspace || '.' });
+  // TODO: work out if there's any diff
+
+  // generate state for the provisioner
+  const state = project.serialize('state', { format: 'json' });
+
+  logger.debug('Converted local project to app state:');
+  logger.debug(JSON.stringify(state, null, 2));
+
+  // TODO not totally sold on endpoint handling right now
+  config.endpoint ??= project.openfn?.endpoint!;
+
+  logger.info('Sending project to app...');
+
+  // TODO do I really want to use this deploy function? Is it suitable?
+  await deployProject(config as DeployConfig, state);
+
+  logger.success('Updated project at', config.endpoint);
+}

packages/cli/src/util/load-plan.ts

@@ -87,6 +87,8 @@ const fromProject = async (
   logger: Logger
 ): Promise<any> => {
   logger.debug('Loading Repo from ', path.resolve(rootDir));
+
+  // TODO make sure this does not break after changes
   const project = await Project.from('fs', { root: rootDir });
   logger.debug('Loading workflow ', workflowName);
   const workflow = project.getWorkflow(workflowName);

packages/cli/test/projects/deploy.test.ts

@@ -0,0 +1,17 @@
+import test from 'ava';
+
+// what will deploy tests look like?
+
+// deploy a project for the first time (this doesn't work though?)
+
+// deploy a change to a project
+
+// deploy a change to a project but fetch latest first
+
+// throw when trying to deploy to a diverged remote project
+
+// force deploy an incompatible project
+
+// don't post the final version if dry-run is set
+
+// TODO diff + confirm

packages/project/README.md

@@ -8,6 +8,18 @@ A single Project can be Checked Out to disk at a time, meaning its source workfl
 
 A Workspace is a set of related Projects , including a Project and its associated Sandboxes, or a Project deployed to apps in multiple web domains
 
+## Structure and Artifects
+
+openfn.yaml
+
+project file
+
+sort of a mix of project.yaml, state.json and config.json
+
+This is strictly a representation of a server-side project, it's like the last-sync-state. CLI-only or offline projects do not have one.
+
+It's also a portable representation of the project
+
 ### Serializing and Parsing
 
 The main idea of Projects is that a Project represents a set of OpenFn workflows defined in any format and present a standard JS-friendly interface to manipulate and reason about them.

packages/project/src/parse/from-fs.ts

@@ -4,53 +4,34 @@ import { glob } from 'glob';
 import * as l from '@openfn/lexicon';
 
 import { Project } from '../Project';
-import getIdentifier from '../util/get-identifier';
 import { yamlToJson } from '../util/yaml';
 import {
   buildConfig,
   loadWorkspaceFile,
   findWorkspaceFile,
 } from '../util/config';
-import fromProject from './from-project';
 import { omit } from 'lodash-es';
+import { Logger } from '@openfn/logger';
 
 export type FromFsConfig = {
   root: string;
+  logger?: Logger;
 };
 
 // Parse a single project from a root folder
+// Note that this does NOT attempt to load UUIDS from the project file
+// It just builds the project on disk
+// I suppose we could take an option?
 export const parseProject = async (options: FromFsConfig) => {
-  const { root } = options;
+  const { root, logger } = options;
 
   const { type, content } = findWorkspaceFile(root);
   const context = loadWorkspaceFile(content, type as any);
   const config = buildConfig(context.workspace);
 
-  // Now we need to look for the corresponding state file
-  // Need to load UUIDs and other app settings from this
-  // If we load it as a Project, uuid tracking is way easier
-  let state: Project | null = null;
-  const identifier = getIdentifier({
-    endpoint: context.project?.endpoint,
-    env: context.project?.env,
-  });
-  try {
-    const format = config.formats?.project ?? config.formats?.project ?? 'yaml';
-    const statePath = path.join(
-      root,
-      config.dirs?.projects ?? '.projects',
-      `${identifier}.${format}`
-    );
-    const stateFile = await fs.readFile(statePath, 'utf8');
-
-    state = fromProject(stateFile, config);
-  } catch (e) {
-    console.warn(`Failed to find state file for ${identifier}`);
-    // console.warn(e);
-  }
-
   const proj: any = {
-    name: state?.name,
+    id: context.project?.id,
+    name: context.project?.name,
     openfn: omit(context.project, ['id']),
     config: config,
     workflows: [],
@@ -74,46 +55,35 @@ export const parseProject = async (options: FromFsConfig) => {
       const wf =
         fileType === 'yaml' ? yamlToJson(candidate) : JSON.parse(candidate);
       if (wf.id && Array.isArray(wf.steps)) {
-        // load settings from the state file
-        const wfState = state?.getWorkflow(wf.id);
-
-        wf.openfn = Object.assign({}, wfState?.openfn, {
-          uuid: wfState?.openfn?.uuid ?? null,
-        });
-
-        //console.log('Loading workflow at ', filePath); // TODO logger.debug
+        //logger?.log('Loading workflow at ', filePath); // TODO logger.debug
         for (const step of wf.steps) {
           // This is the saved, remote view of the step
           // TODO if the id has changed, how do we track?
-          const stateStep = wfState?.get(step.id);
           if (step.expression && step.expression.endsWith('.js')) {
             const dir = path.dirname(filePath);
             const exprPath = path.join(dir, step.expression);
             try {
-              console.debug(`Loaded expression from ${exprPath}`);
+              logger?.debug(`Loaded expression from ${exprPath}`);
               step.expression = await fs.readFile(exprPath, 'utf-8');
             } catch (e) {
-              console.error(`Error loading expression from ${exprPath}`);
+              logger?.error(`Error loading expression from ${exprPath}`);
               // throw?
             }
           }
-          step.openfn = Object.assign({}, stateStep?.openfn);
 
           // Now track UUIDs for edges against state
           for (const target in step.next || {}) {
             if (typeof step.next[target] === 'boolean') {
               const bool = step.next[target];
               step.next[target] = { condition: bool };
             }
-            const uuid = state?.getUUID(wf.id, step.id, target) ?? null;
-            step.next[target].openfn = { uuid };
           }
         }
 
         proj.workflows.push(wf);
       }
     } catch (e) {
-      console.log(e);
+      logger?.log(e);
       // not valid json
       // should probably maybe a big deal about this huh?
       continue;