Support channels in v2 sync (#1566)

* project: read and write channels via channels.yaml

* project: fix sandbox merge dropping channels and keep remote channel ids on merge

* cli: deploy channel changes and deletions, and clear stale channels.yaml on checkout

* project: track removed channels on merge and send them as deletes

Move channel delete detection out of the CLI deploy handler and into the project's merge step, so any caller of toAppState sends removed channels to Lightning as deletes.

* project: store channels under a channels key in resources.yaml instead of channels.yaml

* project: key resources.yaml channels by id with a required name

* project: match channels by their resources.yaml id so renames keep the channel

* Resource Diffs (#1572)

* restructure project-diff

* diff channels in project

* log resource diff in CLI

* changeset

* one more changeset for the road

* format

* simplify

---------

Co-authored-by: Joe Clark <joe@openfn.org>

Author: midigofrank

.changeset/green-olives-cross.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': patch
+---
+
+Show resource diffs on deploy

.changeset/project-diff-resources.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Project diff now returns { workflows, resources } and includes channel changes

.changeset/sandbox-merge-channels.md

@@ -0,0 +1,7 @@
+---
+'@openfn/project': minor
+'@openfn/cli': minor
+'@openfn/lexicon': patch
+---
+
+Manage project channels locally through a new resources.yaml file

packages/cli/src/projects/checkout.ts

@@ -131,6 +131,11 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => {
       logger?.warn('WARNING! No content for file', f);
     }
   }
+  // Remove any resources.yaml left over from the previous project, so its
+  // channels don't get deployed to this one
+  if (!files['resources.yaml']) {
+    await rimraf(path.join(workspacePath, 'resources.yaml'));
+  }
   if (options.createCredentials) {
     createProjectCredentials(workspacePath, switchProject, logger);
   }

packages/cli/src/projects/deploy.ts

@@ -1,9 +1,11 @@
 import yargs from 'yargs';
 import Project, {
   MergeProjectOptions,
+  toResourceChannels,
   versionsEqual,
   Workspace,
 } from '@openfn/project';
+import { isEqual } from 'lodash-es';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
 
@@ -171,6 +173,15 @@ export const deletedCollections = (
     .map((c) => ({ id: c.uuid as string, name: c.name, delete: true }));
 };
 
+// A missing resources.yaml (local.channels undefined) means channels aren't
+// managed locally, so there's nothing to change
+export const channelsChanged = (local: Project, remote: Project) =>
+  !!local.channels &&
+  !isEqual(
+    toResourceChannels(local.channels, local.credentials),
+    toResourceChannels(remote.channels, remote.credentials)
+  );
+
 export type SyncResult = {
   merged: Project;
   remoteProject: Project;
@@ -244,10 +255,11 @@ const syncProjects = async (
 
   // TODO: what if remote diff and the version checked disagree for some reason?
   const workflowDiffs = mergeCandidates.length
-    ? remoteProject.diff(localProject, mergeCandidates)
+    ? remoteProject.diff(localProject, mergeCandidates).workflows
     : [];
   const didCollectionsChange = collectionsChanged(localProject, remoteProject);
-  if (!workflowDiffs.length && !didCollectionsChange) {
+  const didChannelsChange = channelsChanged(localProject, remoteProject);
+  if (!workflowDiffs.length && !didCollectionsChange && !didChannelsChange) {
     logger.success('Nothing to deploy');
     return null;
   }

packages/cli/src/projects/diff.ts

@@ -1,5 +1,6 @@
 import c from 'chalk';
 import Project, { generateStepDiff, generateEdgeDiff } from '@openfn/project';
+import type { ChannelDiff } from '@openfn/project';
 import type { StepChange, EdgeChange } from '@openfn/project';
 import type { Logger } from '../util/logger';
 
@@ -60,6 +61,26 @@ const printStepDiff = (steps: StepChange[], logger: Logger) => {
   }
 };
 
+const printChannelDiff = (channels: ChannelDiff[], logger: Logger) => {
+  logger.always('Channels:');
+  for (const { name, type, changes } of channels) {
+    if (type === 'added') {
+      logger.always(c.green(`  ${name}: added`));
+    } else if (type === 'removed') {
+      logger.always(c.red(`  ${name}: removed`));
+    } else {
+      logger.always(c.yellow(`  ${name}: changed`));
+      for (const key in changes) {
+        const { from, to } = changes[key];
+        logger.always(
+          c.yellow(`    - ${key}: "${from ?? ''}" -> "${to ?? ''}"`)
+        );
+      }
+    }
+  }
+  logger.break();
+};
+
 // TODO need to include collection diffs
 // https://github.com/OpenFn/kit/issues/1524
 export const printRichDiff = (
@@ -68,8 +89,11 @@ export const printRichDiff = (
   locallyChangedWorkflows: string[],
   logger: Logger
 ) => {
-  const diffs = remote.diff(local, locallyChangedWorkflows);
-  if (diffs.length === 0) {
+  const { workflows: diffs, resources } = remote.diff(
+    local,
+    locallyChangedWorkflows
+  );
+  if (diffs.length === 0 && resources.channels.length === 0) {
     logger.info('No workflow changes detected');
     return diffs;
   }
@@ -112,5 +136,9 @@ export const printRichDiff = (
     logger.break();
   }
 
+  if (resources.channels?.length > 0) {
+    printChannelDiff(resources.channels, logger);
+  }
+
   return diffs;
 };

packages/cli/test/projects/deploy.test.ts

@@ -17,6 +17,7 @@ import {
   hasRemoteDiverged,
   collectionsChanged,
   deletedCollections,
+  channelsChanged,
 } from '../../src/projects/deploy';
 import { printRichDiff } from '../../src/projects/diff';
 import {
@@ -1117,3 +1118,96 @@ test('deletedCollections: nothing to delete when every remote name survives', (t
 
   t.deepEqual(deletedCollections(merged, remote), []);
 });
+
+const channel = {
+  name: 'my-channel',
+  destination_url: 'https://example.com',
+  enabled: true,
+};
+
+test('channelsChanged: false when there is no local resources.yaml', (t) => {
+  const local = { channels: undefined } as unknown as Project;
+  const remote = {
+    channels: [{ ...channel, id: 'chan-1' }],
+  } as unknown as Project;
+
+  t.false(channelsChanged(local, remote));
+});
+
+test('channelsChanged: true when a channel is edited locally', (t) => {
+  const local = {
+    channels: [{ ...channel, enabled: false }],
+  } as unknown as Project;
+  const remote = {
+    channels: [{ ...channel, id: 'chan-1' }],
+  } as unknown as Project;
+
+  t.true(channelsChanged(local, remote));
+});
+
+test('channelsChanged: false when a credential name matches the remote uuid', (t) => {
+  const local = {
+    channels: [{ ...channel, destination_credential_id: 'me@openfn.org|cred' }],
+  } as unknown as Project;
+  const remote = {
+    credentials: [{ uuid: 'cred-uuid', name: 'cred', owner: 'me@openfn.org' }],
+    channels: [
+      { ...channel, id: 'chan-1', destination_credential_id: 'cred-uuid' },
+    ],
+  } as unknown as Project;
+
+  t.false(channelsChanged(local, remote));
+});
+
+const diffChannel = (name: string, extra = {}) => ({
+  name,
+  destination_url: `https://example.com/${name}`,
+  enabled: true,
+  ...extra,
+});
+
+test('printRichDiff: should report a channel-only change', (t) => {
+  const local = new Project({
+    name: 'local',
+    workflows: [],
+    channels: [diffChannel('one')],
+  });
+  const remote = new Project({ name: 'remote', workflows: [], channels: [] });
+
+  printRichDiff(local, remote, [], logger);
+
+  t.truthy(logger._find('always', /following changes to the remote project/));
+  t.truthy(logger._find('always', /Channels:/));
+  t.truthy(logger._find('always', /one: added/));
+});
+
+test('printRichDiff: should report removed channels', (t) => {
+  const local = new Project({ name: 'local', workflows: [], channels: [] });
+  const remote = new Project({
+    name: 'remote',
+    workflows: [],
+    channels: [diffChannel('one')],
+  });
+
+  printRichDiff(local, remote, [], logger);
+
+  t.truthy(logger._find('always', /one: removed/));
+});
+
+test('printRichDiff: should list changed channel fields', (t) => {
+  const local = new Project({
+    name: 'local',
+    workflows: [],
+    channels: [diffChannel('one', { enabled: false })],
+  });
+  const remote = new Project({
+    name: 'remote',
+    workflows: [],
+    channels: [diffChannel('one')],
+  });
+
+  printRichDiff(local, remote, [], logger);
+
+  t.truthy(logger._find('always', /one: changed/));
+  t.truthy(logger._find('always', /enabled: "true" -> "false"/));
+});

packages/lexicon/core.d.ts

@@ -4,6 +4,7 @@ import type { RawSourceMap } from 'source-map';
 import {
   Credential,
   Collection,
+  Channel,
   Job,
   Trigger,
   ProjectSpec,
@@ -47,6 +48,10 @@ export interface ProjectState extends WithState<ProjectSpec, ProjectMeta> {
   // tracks each collection's uuid once it's been created on the server
   collections?: Array<CollectionState>;
 
+  // override channels - locally authored channels (channels.yaml) have no
+  // id until they're deployed
+  channels?: Array<ChannelState>;
+
   /** Stuff only used by the CLI for this project */
   cli?: LocalMeta;
 }
@@ -131,6 +136,12 @@ export interface CollectionState extends Collection {
   uuid?: UUID;
 }
 
+// Like step.configuration, destination_credential_id may hold a credential
+// name (owner|name) rather than a uuid until it's serialized for Lightning
+// `key` is the local id from resources.yaml, used to match channels on merge.
+// It's never sent to Lightning
+export type ChannelState = Omit<Channel, 'id'> & { id?: string; key?: string };
+
 type FileFormats = 'yaml' | 'json';
 
 // This is the old workspace config file, up to 0.6

packages/project/src/Project.ts

@@ -77,7 +77,10 @@ export class Project {
 
   collections?: Array<CollectionState>;
 
-  channels?: l.Channel[];
+  channels?: l.ChannelState[];
+
+  // channels removed locally - to-app-state sends these as delete: true
+  removedChannels: l.ChannelState[] = [];
 
   credentials: Credential[];
 
@@ -279,7 +282,7 @@ export class Project {
     return Object.values(creds);
   }
 
-  // Compare this project with another and return a list of workflow changes
+  // Compare this project with another and return the changes
   diff(project: Project, workflows: string[] = []) {
     return projectDiff(this, project, workflows);
   }

packages/project/src/index.ts

@@ -9,7 +9,12 @@ export { Workspace, yamlToJson, jsonToYaml };
 export { generateWorkflow, generateProject } from './gen/generator';
 
 export { diff } from './util/project-diff';
-export type { WorkflowDiff, DiffType } from './util/project-diff';
+export type {
+  WorkflowDiff,
+  ChannelDiff,
+  ProjectDiff,
+  DiffType,
+} from './util/project-diff';
 
 export { generateStepDiff, generateEdgeDiff } from './util/workflow-diff';
 export type {
@@ -29,3 +34,5 @@ export { default as detectVersion } from './util/detect-version';
 export type { MergeProjectOptions } from './merge/merge-project';
 
 export { mergeCollections } from './merge/merge-project';
+
+export { toResourceChannels } from './util/resources';