CLI: fix credentials on deploy --beta

packages/cli/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @openfn/cli
 
+## 1.20.2
+
+### Patch Changes
+
+- 6766d96: Fixed an issue where credentials can get dropped in `deploy --beta`
+- Updated dependencies [6766d96]
+  - @openfn/project@0.9.2
+
 ## 1.20.1
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.20.1",
+  "version": "1.20.2",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/deploy/beta.ts

@@ -4,34 +4,28 @@ import Project from '@openfn/project';
 import { DeployConfig, deployProject } from '@openfn/deploy';
 import type { Logger } from '../util/logger';
 import { Opts } from '../options';
+import { loadAppAuthConfig } from '../projects/util';
 
 export type DeployOptionsBeta = Required<
   Pick<
     Opts,
-    'beta' | 'command' | 'log' | 'logJson' | 'apiKey' | 'endpoint' | 'path'
+    | 'beta'
+    | 'command'
+    | 'log'
+    | 'logJson'
+    | 'apiKey'
+    | 'endpoint'
+    | 'path'
+    | 'workspace'
   >
 >;
 
 export async function handler(options: DeployOptionsBeta, logger: Logger) {
-  const { OPENFN_API_KEY } = process.env;
-
-  const { endpoint } = options;
-
-  const config: Partial<DeployConfig> = {
-    apiKey: options.apiKey,
-  };
-
-  if (!options.apiKey && OPENFN_API_KEY) {
-    logger.info('Using OPENFN_API_KEY environment variable');
-    config.apiKey = OPENFN_API_KEY;
-  }
+  const config = loadAppAuthConfig(options, logger);
 
   // TMP use options.path to set the directory for now
   // We'll need to manage this a bit better
-  const project = await Project.from('fs', { root: options.path || '.' });
-  // Why is there an id on openfn here?
-  console.log({ openfn: project.openfn });
-
+  const project = await Project.from('fs', { root: options.workspace || '.' });
   // TODO: work out if there's any diff
 
   // generate state for the provisioner
@@ -41,7 +35,7 @@ export async function handler(options: DeployOptionsBeta, logger: Logger) {
   logger.debug(JSON.stringify(state, null, 2));
 
   // TODO not totally sold on endpoint handling right now
-  config.endpoint = endpoint || project.openfn?.endpoint;
+  config.endpoint ??= project.openfn?.endpoint!;
 
   logger.info('Sending project to app...');
 

packages/cli/src/deploy/command.ts

@@ -1,5 +1,5 @@
 import yargs from 'yargs';
-import { build, ensure } from '../util/command-builders';
+import { build, ensure, override } from '../util/command-builders';
 import { Opts } from '../options';
 import * as o from '../options';
 
@@ -26,6 +26,8 @@ const options = [
   o.logJson,
   o.projectPath,
   o.statePath,
+
+  override(o.workspace, { hidden: true }),
 ];
 
 const deployCommand: yargs.CommandModule<DeployOptions> = {

packages/project/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/project
 
+## 0.9.2
+
+### Patch Changes
+
+- 6766d96: Fixed an issue where credentials can get dropped in `deploy --beta`
+
 ## 0.9.1
 
 ### Patch Changes

packages/project/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "scripts": {
     "test": "pnpm ava",

packages/project/src/Workflow.ts

@@ -90,7 +90,7 @@ class Workflow {
     return this;
   }
 
-  // Get properties on any step or edge by id
+  // Get properties on any step or edge by id or uuid
   get(id: string): WithMeta<l.Step | l.Trigger | l.StepEdge> {
     const item = this.index.edges[id] || this.index.steps[id];
     if (!item) {

packages/project/src/parse/from-fs.ts

@@ -75,14 +75,17 @@ export const parseProject = async (options: FromFsConfig) => {
         fileType === 'yaml' ? yamlToJson(candidate) : JSON.parse(candidate);
       if (wf.id && Array.isArray(wf.steps)) {
         // load settings from the state file
-        const wfState: any = (state && state.getWorkflow(wf.id)) ?? {};
-        wf.openfn = {
-          uuid: wfState.openfn?.uuid ?? null,
-          // TODO do we need to transfer more stuff? Options maybe?
-        };
+        const wfState = state?.getWorkflow(wf.id);
+
+        wf.openfn = Object.assign({}, wfState?.openfn, {
+          uuid: wfState?.openfn?.uuid ?? null,
+        });
 
         //console.log('Loading workflow at ', filePath); // TODO logger.debug
         for (const step of wf.steps) {
+          // This is the saved, remote view of the step
+          // TODO if the id has changed, how do we track?
+          const stateStep = wfState?.get(step.id);
           if (step.expression && step.expression.endsWith('.js')) {
             const dir = path.dirname(filePath);
             const exprPath = path.join(dir, step.expression);
@@ -94,12 +97,7 @@ export const parseProject = async (options: FromFsConfig) => {
               // throw?
             }
           }
-          // check the state file for a matching uuid
-          // TODO this isn't quite right - what if there are other openfn keys to write?
-          // We need to return not just the UUID, but all the openfn keys
-          // TODO do we need to slugify the id here? Not really tbh?
-          const uuid = state?.getUUID(wf.id, step.id) ?? null;
-          step.openfn = { uuid };
+          step.openfn = Object.assign({}, stateStep?.openfn);
 
           // Now track UUIDs for edges against state
           for (const target in step.next || {}) {

packages/project/test/fixtures/sample-v1-project.ts

@@ -1,4 +1,5 @@
 import type { Provisioner } from '@openfn/lexicon/lightning';
+import { cloneDeep } from 'lodash-es';
 
 const state: Provisioner.Project = {
   id: 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00',
@@ -55,3 +56,11 @@ const state: Provisioner.Project = {
 };
 
 export default state;
+
+const withCreds = cloneDeep(state);
+Object.assign(withCreds.workflows[0].jobs[0], {
+  project_credential_id: 'p',
+  keychain_credential_id: 'k',
+});
+
+export { withCreds };

packages/project/test/fixtures/sample-v2-project.ts

@@ -16,7 +16,7 @@ export const json: SerializedProject = {
         {
           name: 'b',
           id: 'b',
-          openfn: { uuid: 3 },
+          openfn: { uuid: 3, project_credential_id: 'x' },
           expression: 'fn()',
           adaptor: 'common',
         },
@@ -50,6 +50,7 @@ workflows:
         id: b
         openfn:
           uuid: 3
+          project_credential_id: x
         expression: fn()
         adaptor: common
       - id: trigger

packages/project/test/parse/from-app-state.test.ts

@@ -2,7 +2,7 @@ import test from 'ava';
 import fromAppState, { mapWorkflow } from '../../src/parse/from-app-state';
 import { clone, cloneDeep } from 'lodash-es';
 
-import state from '../fixtures/sample-v1-project';
+import state, { withCreds } from '../fixtures/sample-v1-project';
 
 // I don't think this file really represents anything
 // loosely maps to the old config file
@@ -177,6 +177,27 @@ test('mapWorkflow: map a simple job', (t) => {
   });
 });
 
+// todo surprised this works
+test('mapWorkflow: map a job with project and keychain credentials', (t) => {
+  const wf = withCreds.workflows[0];
+  wf.jobs.map(console.log);
+  const mapped = mapWorkflow(wf);
+
+  const [_trigger, job] = mapped.steps;
+  t.deepEqual(job, {
+    id: 'transform-data',
+    name: 'Transform data',
+    adaptor: '@openfn/language-common@latest',
+    expression:
+      '// Check out the Job Writing Guide for help getting started:\n// https://docs.openfn.org/documentation/jobs/job-writing-guide\n',
+    openfn: {
+      uuid: '66add020-e6eb-4eec-836b-20008afca816',
+      project_credential_id: 'p',
+      keychain_credential_id: 'k',
+    },
+  });
+});
+
 test('should create a Project from prov state yaml', (t) => {
   const yaml = `id: e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00
 name: aaa

packages/project/test/parse/from-fs.test.ts

@@ -5,6 +5,8 @@ import { parseProject } from '../../src/parse/from-fs';
 const s = JSON.stringify;
 
 // mock several projects and use them through the tests
+// TODO: the state files here are all in v1 format - need to add tests with v2
+// Probably need to rethink all these tests tbh
 mock({
   '/p1/openfn.json': s({
     // this must be the whole deploy name right?
@@ -25,14 +27,6 @@ mock({
       // That stuff is all in the project.yaml, not useful here
     },
   }),
-  // // hmm I don't think we need to model this
-  // // in the PROJECT. It should all be implicit.
-  // '/p1/projects/staging@app.openfn.org.json': s({
-  //   openfn: {
-  //     projectId: 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00',
-  //     endpoint: 'http://localhost:4000',
-  //   },
-  // }),
   '/p1/workflows/my-workflow': {},
   '/p1/workflows/my-workflow/my-workflow.json': s({
     id: 'my-workflow',
@@ -64,9 +58,9 @@ mock({
         name: 'My Workflow',
         jobs: [
           {
-            // pretend this is a uuid
             id: '<uuid-1>',
             name: 'a',
+            project_credential_id: 'p',
           },
           {
             id: '<uuid-2>',
@@ -194,6 +188,16 @@ test('should track the UUID of a step', async (t) => {
   t.is(wf.steps[0].openfn.uuid, '<uuid-1>');
 });
 
+// TODO also test this on different openfn objects
+test('should track openfn props from state file on a step', async (t) => {
+  const project = await parseProject({ root: '/p1' });
+
+  const [wf] = project.workflows;
+
+  t.truthy(wf.steps[0].openfn);
+  t.is(wf.steps[0].openfn.project_credential_id, 'p');
+});
+
 test('should track the UUID of an edge', async (t) => {
   const project = await parseProject({ root: '/p1' });
 

packages/project/test/parse/from-project.test.ts

@@ -93,6 +93,7 @@ test('import from a v2 project as JSON', async (t) => {
         adaptor: 'common',
         openfn: {
           uuid: 3,
+          project_credential_id: 'x',
         },
       },
       {
@@ -138,6 +139,7 @@ test('import from a v2 project as YAML', async (t) => {
         adaptor: 'common',
         openfn: {
           uuid: 3,
+          project_credential_id: 'x',
         },
       },
       {

packages/project/test/serialize/to-app-state.test.ts

@@ -33,6 +33,8 @@ const state: Provisioner.Project = {
           name: 'Transform data',
           body: '// Check out the Job Writing Guide for help getting started:\n// https://docs.openfn.org/documentation/jobs/job-writing-guide\n',
           adaptor: '@openfn/language-common@latest',
+
+          // TODO make sure these get serialized back
           project_credential_id: null,
           keychain_credential_id: null,
         },
@@ -223,6 +225,39 @@ test('should write openfn keys to objects', (t) => {
   t.is(state.workflows[0].edges[0].x, 1);
 });
 
+test('should handle credentials', (t) => {
+  const data = {
+    id: 'my-project',
+    workflows: [
+      {
+        id: 'wf',
+        steps: [
+          {
+            id: 'trigger',
+            type: 'webhook',
+            next: {
+              step: {},
+            },
+          },
+          {
+            id: 'step',
+            expression: '.',
+            openfn: {
+              keychain_credential_id: 'k',
+              project_credential_id: 'p',
+            },
+          },
+        ],
+      },
+    ],
+  };
+
+  const state = toAppState(new Project(data), { format: 'json' });
+  const [job] = state.workflows[0].jobs;
+  t.is(job.keychain_credential_id, 'k');
+  t.is(job.project_credential_id, 'p');
+});
+
 test.todo('handle edge labels');
 
 test('serialize steps and trigger in alphabetical order', (t) => {

packages/project/test/serialize/to-project.test.ts

@@ -18,7 +18,7 @@ const createProject = () => {
     },
     workflows: [
       generateWorkflow(
-        'trigger(type=webhook)-b(expression="fn()",adaptor=common)',
+        'trigger(type=webhook)-b(expression="fn()",adaptor=common,project_credential_id=x)',
         {
           uuidSeed: 1,
           openfnUuid: true,
@@ -61,7 +61,7 @@ test('should exclude null values in yaml', (t) => {
 
   // force some null values into the workflow structure
   proj.workflows[0].openfn.concurrency = null;
-  proj.workflows[0].steps[1].openfn.project_credential_id = null;
+  proj.workflows[0].steps[1].openfn.keychain_credential_id = null;
 
   const yaml = proj.serialize('project', { format: 'yaml' });
   t.deepEqual(yaml, v2.yaml);