Release CLI 1.29.0

integration-tests/cli/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @openfn/integration-tests-cli
 
+## 1.0.16
+
+### Patch Changes
+
+- Updated dependencies [cbe2800]
+  - @openfn/project@0.14.2
+
 ## 1.0.15
 
 ### Patch Changes

integration-tests/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-cli",
   "private": true,
-  "version": "1.0.15",
+  "version": "1.0.16",
   "description": "CLI integration tests",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",

packages/cli/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @openfn/cli
 
+## 1.29.0
+
+### Minor Changes
+
+- 2d570c6: Support github sync with new `project` commands
+
+  GitHub Sync is currently designed with a pair of actions which trigger the legacy `openfn deploy` and `openfn pull` commands.
+
+  This update adds support for the `openfn.yaml` file to the commands: so if you run `openfn project pull` in a github synced repo, the repo goes into v2 mode. The legacy deploy and pull commands will "redirect" to v2. You should leave the legacy `config.json` file, but state.json and spec.yaml files can be removed.
+
+  Initaliasing GitHub sync from the app will continue to use the legacy file structure for the initial commit. If you want to switch to v2, either create an empty openfn.yaml and trigger a sync, or locally run `openfn project pull` and commit changes.
+
+  Be aware that v2 sync only supports a single `openfn.yaml` and `workflows` folder at a time - so a sync which pulls from multiple connected apps will not work well. It should however be safe to push changes to multiple apps.
+
+### Patch Changes
+
+- cbe2800: Improve --help docs and error messages
+- Updated dependencies [cbe2800]
+  - @openfn/project@0.14.2
+
 ## 1.28.2
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.28.2",
+  "version": "1.29.0",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/deploy/command.ts

@@ -1,12 +1,13 @@
 import yargs from 'yargs';
 import { build, ensure, override } from '../util/command-builders';
 import { Opts } from '../options';
+import { Opts as POpts } from '../projects/options';
 import * as o from '../options';
 import * as o2 from '../projects/options';
 
 export type DeployOptions = Required<
   Pick<
-    Opts,
+    Opts & POpts,
     | 'beta'
     | 'command'
     | 'configPath'
@@ -15,6 +16,7 @@ export type DeployOptions = Required<
     | 'logJson'
     | 'projectPath'
     | 'statePath'
+    | 'workspace'
   >
 >;
 

packages/cli/src/deploy/handler.ts

@@ -8,6 +8,8 @@ import {
 import type { Logger } from '../util/logger';
 import { DeployOptions } from './command';
 import * as beta from '../projects/deploy';
+import path from 'node:path';
+import { fileExists } from '../util/file-exists';
 
 export type DeployFn = typeof deploy;
 
@@ -32,6 +34,17 @@ async function deployHandler(
   try {
     const config = mergeOverrides(await getConfig(options.configPath), options);
 
+    const v2ConfigPath = path.join(
+      options.workspace || process.cwd(),
+      'openfn.yaml'
+    );
+    if (await fileExists(v2ConfigPath)) {
+      logger.always(
+        'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy)'
+      );
+      return beta.handler({ ...options, force: true }, logger);
+    }
+
     logger.debug('Deploying with config', JSON.stringify(config, null, 2));
 
     if (options.confirm === false) {
@@ -76,13 +89,18 @@ function mergeOverrides(
   config: DeployConfig,
   options: DeployOptions
 ): DeployConfig {
+  const workspace = options.workspace || process.cwd();
+  const resolveRelative = (p: string) =>
+    path.isAbsolute(p) ? p : path.join(workspace, p);
+  const specPath = pickFirst(options.projectPath, config.specPath);
+  const statePath = pickFirst(options.statePath, config.statePath);
   return {
     ...config,
     apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey),
     endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint),
-    statePath: pickFirst(options.statePath, config.statePath),
-    specPath: pickFirst(options.projectPath, config.specPath),
-    configPath: options.configPath,
+    statePath: resolveRelative(statePath),
+    specPath: resolveRelative(specPath),
+    configPath: resolveRelative(options.configPath),
     requireConfirmation: pickFirst(options.confirm, config.requireConfirmation),
   };
 }

packages/cli/src/execute/command.ts

@@ -84,33 +84,35 @@ const options = [
 const executeCommand: yargs.CommandModule<ExecuteOptions> = {
   command: 'execute <path> [workflow]',
   describe: `Run an openfn expression or workflow. Get more help by running openfn <command> help.
-  \nExecute will run a expression/workflow at the path and write the output state to disk (to ./state.json unless otherwise specified)
-  \nRemember to include the adaptor name with -a. Auto install adaptors with the -i flag.`,
+  \nExecute will run a expression/workflow at the path and write the output state to disk (to ./state.json unless otherwise specified)`,
   aliases: ['$0'],
   handler: ensure('execute', options),
   builder: (yargs) =>
     build(options, yargs)
       // TODO this is now path or workflow name
       .positional('path', {
-        describe:
-          'The path to load the job or workflow from (a .js or .json file or a dir containing a job.js file)',
+        describe: 'The path or name of the workflow or expression to run',
         demandOption: true,
       })
+      .example(
+        'openfn my-workflow',
+        'Execute a workflow called my-workflow. Requires an openfn.yaml file (created by openfn project pull)'
+      )
       .example(
         'openfn foo/job.js',
-        'Execute foo/job.js with no adaptor and write the final state to foo/job.json'
+        'Execute foo/job.js (with no adaptor) and write the final state to foo/job.json'
       )
       .example(
         'openfn workflow.json',
         'Execute the workflow contained in workflow.json'
       )
       .example(
         'openfn job.js -a common --log info',
-        'Execute job.js with common adaptor and info-level logging'
+        'Execute job.js with common adaptor (which will be automatically installed) and info-level logging'
       )
       .example(
-        'openfn compile job.js -a http',
-        'Compile the expression at job.js with the http adaptor and print the code to stdout'
+        'openfn job.js -ma common',
+        'Execute job.js with the build of the common adaptor found in your adaptors monorepo'
       ),
 };
 

packages/cli/src/projects/pull.ts

@@ -23,6 +23,7 @@ export type PullOptions = Pick<
   | 'project'
   | 'confirm'
   | 'snapshots'
+  | 'force'
 >;
 
 const options = [
@@ -61,6 +62,7 @@ export const command: yargs.CommandModule<PullOptions> = {
 };
 
 export async function handler(options: PullOptions, logger: Logger) {
+  options.workspace ??= process.cwd();
   ensureProjectId(options, logger);
 
   await fetch(options, logger);

packages/cli/src/pull/command.ts

@@ -1,12 +1,13 @@
 import yargs from 'yargs';
 import { build, ensure, override } from '../util/command-builders';
 import { Opts } from '../options';
+import { Opts as POpts } from '../projects/options';
 import * as o from '../options';
 import * as po from '../projects/options';
 
 export type PullOptions = Required<
   Pick<
-    Opts,
+    Opts & POpts,
     | 'beta'
     | 'command'
     | 'log'
@@ -17,6 +18,7 @@ export type PullOptions = Required<
     | 'projectId'
     | 'confirm'
     | 'snapshots'
+    | 'workspace'
   >
 >;
 

packages/cli/src/pull/handler.ts

@@ -11,6 +11,7 @@ import {
 import type { Logger } from '../util/logger';
 import { PullOptions } from '../pull/command';
 import beta from '../projects/pull';
+import { fileExists } from '../util/file-exists';
 
 async function pullHandler(options: PullOptions, logger: Logger) {
   if (options.beta) {
@@ -21,6 +22,20 @@ async function pullHandler(options: PullOptions, logger: Logger) {
   try {
     const config = mergeOverrides(await getConfig(options.configPath), options);
 
+    const v2ConfigPath = path.join(
+      options.workspace || process.cwd(),
+      'openfn.yaml'
+    );
+    if (await fileExists(v2ConfigPath)) {
+      logger.always(
+        'Detected openfn.yaml file - switching to v2 pull (openfn project pull)'
+      );
+      return beta(
+        { ...options, project: options.projectId, force: true },
+        logger
+      );
+    }
+
     if (process.env['OPENFN_API_KEY']) {
       logger.info('Using OPENFN_API_KEY environment variable');
       config.apiKey = process.env['OPENFN_API_KEY'];
@@ -127,11 +142,20 @@ function mergeOverrides(
   config: DeployConfig,
   options: PullOptions
 ): DeployConfig {
+  const workspace = options.workspace || process.cwd();
   return {
     ...config,
     apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey),
     endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint),
-    configPath: options.configPath,
+    configPath: path.isAbsolute(options.configPath)
+      ? options.configPath
+      : path.join(workspace, options.configPath),
+    specPath: path.isAbsolute(config.specPath)
+      ? config.specPath
+      : path.join(workspace, config.specPath),
+    statePath: path.isAbsolute(config.statePath)
+      ? config.statePath
+      : path.join(workspace, config.statePath),
     requireConfirmation: pickFirst(options.confirm, config.requireConfirmation),
   };
 }

packages/cli/src/util/file-exists.ts

@@ -0,0 +1,10 @@
+import fs from 'fs/promises';
+
+export async function fileExists(filePath: string) {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.isFile();
+  } catch (error) {
+    return false;
+  }
+}

packages/cli/test/pull/handler.test.ts

@@ -0,0 +1,38 @@
+import test from 'ava';
+import mockfs from 'mock-fs';
+import { createMockLogger } from '@openfn/logger';
+import pullHandler from '../../src/pull/handler';
+import { PullOptions } from '../../src/pull/command';
+
+test.beforeEach(() => {
+  mockfs.restore();
+});
+
+test.afterEach(() => {
+  mockfs.restore();
+});
+
+const options: PullOptions = {
+  beta: false,
+  command: 'pull',
+  projectPath: './project.yaml',
+  configPath: './config.json',
+  projectId: 'abc-123',
+  confirm: false,
+  snapshots: [],
+};
+
+test.serial(
+  'redirects to beta handler when openfn.yaml exists in cwd',
+  async (t) => {
+    const logger = createMockLogger('', { level: 'debug' });
+    mockfs({
+      ['./config.json']: `{"apiKey": "123"}`,
+      ['./openfn.yaml']: '',
+    });
+
+    await t.throwsAsync(() => pullHandler(options, logger));
+
+    t.truthy(logger._find('always', /Detected openfn.yaml file/i));
+  }
+);

packages/cli/test/util/file-exists.test.ts

@@ -0,0 +1,22 @@
+import test from 'ava';
+import mockfs from 'mock-fs';
+import { fileExists } from '../../src/util/file-exists';
+
+test.afterEach(() => {
+  mockfs.restore();
+});
+
+test('returns true for an existing file', async (t) => {
+  mockfs({ './test.txt': 'content' });
+  t.true(await fileExists('./test.txt'));
+});
+
+test('returns false for a non-existent path', async (t) => {
+  mockfs({});
+  t.false(await fileExists('./nonexistent.txt'));
+});
+
+test('returns false for a directory', async (t) => {
+  mockfs({ './mydir': {} });
+  t.false(await fileExists('./mydir'));
+});

packages/project/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/project
 
+## 0.14.2
+
+### Patch Changes
+
+- cbe2800: Suppress error message when generating app state without a credentials.yaml
+
 ## 0.14.1
 
 ### Patch Changes

packages/project/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.14.1",
+  "version": "0.14.2",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "scripts": {
     "test": "pnpm ava",

packages/project/src/Workspace.ts

@@ -43,7 +43,7 @@ export class Workspace {
     } catch (e) {
       if (validate) {
         this.logger.warn(
-          `Could not find openfn.yaml at ${workspacePath}. Using default values.`
+          `Could not find openfn.yaml at ${workspacePath}. Using default configuration.`
         );
       }
     }

packages/project/src/serialize/to-app-state.ts

@@ -141,11 +141,12 @@ export const mapWorkflow = (
           });
           if (mappedCredential) {
             projectCredentialId = mappedCredential.uuid;
-          } else {
-            console.warn(`WARING! Failed to map credential ${projectCredentialId} - Lightning may throw an error.
-
-Ensure the credential exists in project.yaml and try again (maybe ensure the credential is attached to the project in the app and run project fetch)`);
           }
+          //            else {
+          //             console.warn(`WARING! Failed to map credential ${projectCredentialId} - Lightning may throw an error.
+
+          // Ensure the credential exists in project.yaml and try again (maybe ensure the credential is attached to the project in the app and run project fetch)`);
+          //           }
           otherOpenFnProps.project_credential_id = projectCredentialId;
         }
       }