From 2d570c6351f073232b5cace15fae5885dbbb3f19 Mon Sep 17 00:00:00 2001 From: "Farhan Y." Date: Thu, 26 Feb 2026 15:19:05 +0000 Subject: [PATCH 1/3] feat: redirect pull & deploy command to projects (#1263) * feat: redirect to v2 when openfn.yaml exists * tests: file-exists * feat: update * feat: pass project id * tests: add test for redirect * feat: add workspace to old pull * feat: support workspace in deploy * fix: path to spec & state file with workspace * chore: fix formatting * chore: handle comments * tweaks * changeset --------- Co-authored-by: Joe Clark --- .changeset/vast-weeks-tap.md | 13 ++++++++ packages/cli/src/deploy/command.ts | 4 ++- packages/cli/src/deploy/handler.ts | 24 ++++++++++++-- packages/cli/src/projects/pull.ts | 2 ++ packages/cli/src/pull/command.ts | 4 ++- packages/cli/src/pull/handler.ts | 26 ++++++++++++++- packages/cli/src/util/file-exists.ts | 10 ++++++ packages/cli/test/pull/handler.test.ts | 38 ++++++++++++++++++++++ packages/cli/test/util/file-exists.test.ts | 22 +++++++++++++ 9 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 .changeset/vast-weeks-tap.md create mode 100644 packages/cli/src/util/file-exists.ts create mode 100644 packages/cli/test/pull/handler.test.ts create mode 100644 packages/cli/test/util/file-exists.test.ts diff --git a/.changeset/vast-weeks-tap.md b/.changeset/vast-weeks-tap.md new file mode 100644 index 000000000..953c4560a --- /dev/null +++ b/.changeset/vast-weeks-tap.md @@ -0,0 +1,13 @@ +--- +'@openfn/cli': minor +--- + +Support github sync with new `project` commands + +GitHub Sync is currently designed with a pair of actions which trigger the legacy `openfn deploy` and `openfn pull` commands. + +This update adds support for the `openfn.yaml` file to the commands: so if you run `openfn project pull` in a github synced repo, the repo goes into v2 mode. The legacy deploy and pull commands will "redirect" to v2. You should leave the legacy `config.json` file, but state.json and spec.yaml files can be removed. + +Initaliasing GitHub sync from the app will continue to use the legacy file structure for the initial commit. If you want to switch to v2, either create an empty openfn.yaml and trigger a sync, or locally run `openfn project pull` and commit changes. + +Be aware that v2 sync only supports a single `openfn.yaml` and `workflows` folder at a time - so a sync which pulls from multiple connected apps will not work well. It should however be safe to push changes to multiple apps. diff --git a/packages/cli/src/deploy/command.ts b/packages/cli/src/deploy/command.ts index 79ed15cc4..6b144f8ee 100644 --- a/packages/cli/src/deploy/command.ts +++ b/packages/cli/src/deploy/command.ts @@ -1,12 +1,13 @@ import yargs from 'yargs'; import { build, ensure, override } from '../util/command-builders'; import { Opts } from '../options'; +import { Opts as POpts } from '../projects/options'; import * as o from '../options'; import * as o2 from '../projects/options'; export type DeployOptions = Required< Pick< - Opts, + Opts & POpts, | 'beta' | 'command' | 'configPath' @@ -15,6 +16,7 @@ export type DeployOptions = Required< | 'logJson' | 'projectPath' | 'statePath' + | 'workspace' > >; diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index c61b70ba9..4e834a0fb 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -8,6 +8,8 @@ import { import type { Logger } from '../util/logger'; import { DeployOptions } from './command'; import * as beta from '../projects/deploy'; +import path from 'node:path'; +import { fileExists } from '../util/file-exists'; export type DeployFn = typeof deploy; @@ -32,6 +34,17 @@ async function deployHandler( try { const config = mergeOverrides(await getConfig(options.configPath), options); + const v2ConfigPath = path.join( + options.workspace || process.cwd(), + 'openfn.yaml' + ); + if (await fileExists(v2ConfigPath)) { + logger.always( + 'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy)' + ); + return beta.handler({ ...options, force: true }, logger); + } + logger.debug('Deploying with config', JSON.stringify(config, null, 2)); if (options.confirm === false) { @@ -76,13 +89,18 @@ function mergeOverrides( config: DeployConfig, options: DeployOptions ): DeployConfig { + const workspace = options.workspace || process.cwd(); + const resolveRelative = (p: string) => + path.isAbsolute(p) ? p : path.join(workspace, p); + const specPath = pickFirst(options.projectPath, config.specPath); + const statePath = pickFirst(options.statePath, config.statePath); return { ...config, apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey), endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint), - statePath: pickFirst(options.statePath, config.statePath), - specPath: pickFirst(options.projectPath, config.specPath), - configPath: options.configPath, + statePath: resolveRelative(statePath), + specPath: resolveRelative(specPath), + configPath: resolveRelative(options.configPath), requireConfirmation: pickFirst(options.confirm, config.requireConfirmation), }; } diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts index 6a3492561..1df4af557 100644 --- a/packages/cli/src/projects/pull.ts +++ b/packages/cli/src/projects/pull.ts @@ -23,6 +23,7 @@ export type PullOptions = Pick< | 'project' | 'confirm' | 'snapshots' + | 'force' >; const options = [ @@ -61,6 +62,7 @@ export const command: yargs.CommandModule = { }; export async function handler(options: PullOptions, logger: Logger) { + options.workspace ??= process.cwd(); ensureProjectId(options, logger); await fetch(options, logger); diff --git a/packages/cli/src/pull/command.ts b/packages/cli/src/pull/command.ts index ba6dd6620..314164623 100644 --- a/packages/cli/src/pull/command.ts +++ b/packages/cli/src/pull/command.ts @@ -1,12 +1,13 @@ import yargs from 'yargs'; import { build, ensure, override } from '../util/command-builders'; import { Opts } from '../options'; +import { Opts as POpts } from '../projects/options'; import * as o from '../options'; import * as po from '../projects/options'; export type PullOptions = Required< Pick< - Opts, + Opts & POpts, | 'beta' | 'command' | 'log' @@ -17,6 +18,7 @@ export type PullOptions = Required< | 'projectId' | 'confirm' | 'snapshots' + | 'workspace' > >; diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts index fe96e7188..2981ce5ed 100644 --- a/packages/cli/src/pull/handler.ts +++ b/packages/cli/src/pull/handler.ts @@ -11,6 +11,7 @@ import { import type { Logger } from '../util/logger'; import { PullOptions } from '../pull/command'; import beta from '../projects/pull'; +import { fileExists } from '../util/file-exists'; async function pullHandler(options: PullOptions, logger: Logger) { if (options.beta) { @@ -21,6 +22,20 @@ async function pullHandler(options: PullOptions, logger: Logger) { try { const config = mergeOverrides(await getConfig(options.configPath), options); + const v2ConfigPath = path.join( + options.workspace || process.cwd(), + 'openfn.yaml' + ); + if (await fileExists(v2ConfigPath)) { + logger.always( + 'Detected openfn.yaml file - switching to v2 pull (openfn project pull)' + ); + return beta( + { ...options, project: options.projectId, force: true }, + logger + ); + } + if (process.env['OPENFN_API_KEY']) { logger.info('Using OPENFN_API_KEY environment variable'); config.apiKey = process.env['OPENFN_API_KEY']; @@ -127,11 +142,20 @@ function mergeOverrides( config: DeployConfig, options: PullOptions ): DeployConfig { + const workspace = options.workspace || process.cwd(); return { ...config, apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey), endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint), - configPath: options.configPath, + configPath: path.isAbsolute(options.configPath) + ? options.configPath + : path.join(workspace, options.configPath), + specPath: path.isAbsolute(config.specPath) + ? config.specPath + : path.join(workspace, config.specPath), + statePath: path.isAbsolute(config.statePath) + ? config.statePath + : path.join(workspace, config.statePath), requireConfirmation: pickFirst(options.confirm, config.requireConfirmation), }; } diff --git a/packages/cli/src/util/file-exists.ts b/packages/cli/src/util/file-exists.ts new file mode 100644 index 000000000..97c6e61d9 --- /dev/null +++ b/packages/cli/src/util/file-exists.ts @@ -0,0 +1,10 @@ +import fs from 'fs/promises'; + +export async function fileExists(filePath: string) { + try { + const stats = await fs.stat(filePath); + return stats.isFile(); + } catch (error) { + return false; + } +} diff --git a/packages/cli/test/pull/handler.test.ts b/packages/cli/test/pull/handler.test.ts new file mode 100644 index 000000000..3ef838dff --- /dev/null +++ b/packages/cli/test/pull/handler.test.ts @@ -0,0 +1,38 @@ +import test from 'ava'; +import mockfs from 'mock-fs'; +import { createMockLogger } from '@openfn/logger'; +import pullHandler from '../../src/pull/handler'; +import { PullOptions } from '../../src/pull/command'; + +test.beforeEach(() => { + mockfs.restore(); +}); + +test.afterEach(() => { + mockfs.restore(); +}); + +const options: PullOptions = { + beta: false, + command: 'pull', + projectPath: './project.yaml', + configPath: './config.json', + projectId: 'abc-123', + confirm: false, + snapshots: [], +}; + +test.serial( + 'redirects to beta handler when openfn.yaml exists in cwd', + async (t) => { + const logger = createMockLogger('', { level: 'debug' }); + mockfs({ + ['./config.json']: `{"apiKey": "123"}`, + ['./openfn.yaml']: '', + }); + + await t.throwsAsync(() => pullHandler(options, logger)); + + t.truthy(logger._find('always', /Detected openfn.yaml file/i)); + } +); diff --git a/packages/cli/test/util/file-exists.test.ts b/packages/cli/test/util/file-exists.test.ts new file mode 100644 index 000000000..a6c8ce7b4 --- /dev/null +++ b/packages/cli/test/util/file-exists.test.ts @@ -0,0 +1,22 @@ +import test from 'ava'; +import mockfs from 'mock-fs'; +import { fileExists } from '../../src/util/file-exists'; + +test.afterEach(() => { + mockfs.restore(); +}); + +test('returns true for an existing file', async (t) => { + mockfs({ './test.txt': 'content' }); + t.true(await fileExists('./test.txt')); +}); + +test('returns false for a non-existent path', async (t) => { + mockfs({}); + t.false(await fileExists('./nonexistent.txt')); +}); + +test('returns false for a directory', async (t) => { + mockfs({ './mydir': {} }); + t.false(await fileExists('./mydir')); +}); From cbe2800f6f42de1adb0c38c3d545c1f88a731e08 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 26 Feb 2026 16:19:45 +0100 Subject: [PATCH 2/3] CLI: some small UX improvements (#1271) * project: relax error warning when finding unmapped credentials * update cli docs * tweak messaging * changeset --- .changeset/five-lamps-sin.md | 5 +++++ .changeset/slow-worms-try.md | 5 +++++ packages/cli/src/execute/command.ts | 18 ++++++++++-------- packages/project/src/Workspace.ts | 2 +- packages/project/src/serialize/to-app-state.ts | 9 +++++---- 5 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 .changeset/five-lamps-sin.md create mode 100644 .changeset/slow-worms-try.md diff --git a/.changeset/five-lamps-sin.md b/.changeset/five-lamps-sin.md new file mode 100644 index 000000000..4012288e8 --- /dev/null +++ b/.changeset/five-lamps-sin.md @@ -0,0 +1,5 @@ +--- +'@openfn/project': patch +--- + +Suppress error message when generating app state without a credentials.yaml diff --git a/.changeset/slow-worms-try.md b/.changeset/slow-worms-try.md new file mode 100644 index 000000000..5d8f783d4 --- /dev/null +++ b/.changeset/slow-worms-try.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': patch +--- + +Improve --help docs and error messages diff --git a/packages/cli/src/execute/command.ts b/packages/cli/src/execute/command.ts index 77e859203..808750ed1 100644 --- a/packages/cli/src/execute/command.ts +++ b/packages/cli/src/execute/command.ts @@ -84,21 +84,23 @@ const options = [ const executeCommand: yargs.CommandModule = { command: 'execute [workflow]', describe: `Run an openfn expression or workflow. Get more help by running openfn help. - \nExecute will run a expression/workflow at the path and write the output state to disk (to ./state.json unless otherwise specified) - \nRemember to include the adaptor name with -a. Auto install adaptors with the -i flag.`, + \nExecute will run a expression/workflow at the path and write the output state to disk (to ./state.json unless otherwise specified)`, aliases: ['$0'], handler: ensure('execute', options), builder: (yargs) => build(options, yargs) // TODO this is now path or workflow name .positional('path', { - describe: - 'The path to load the job or workflow from (a .js or .json file or a dir containing a job.js file)', + describe: 'The path or name of the workflow or expression to run', demandOption: true, }) + .example( + 'openfn my-workflow', + 'Execute a workflow called my-workflow. Requires an openfn.yaml file (created by openfn project pull)' + ) .example( 'openfn foo/job.js', - 'Execute foo/job.js with no adaptor and write the final state to foo/job.json' + 'Execute foo/job.js (with no adaptor) and write the final state to foo/job.json' ) .example( 'openfn workflow.json', @@ -106,11 +108,11 @@ const executeCommand: yargs.CommandModule = { ) .example( 'openfn job.js -a common --log info', - 'Execute job.js with common adaptor and info-level logging' + 'Execute job.js with common adaptor (which will be automatically installed) and info-level logging' ) .example( - 'openfn compile job.js -a http', - 'Compile the expression at job.js with the http adaptor and print the code to stdout' + 'openfn job.js -ma common', + 'Execute job.js with the build of the common adaptor found in your adaptors monorepo' ), }; diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index 93d44b3f8..618bdbcf2 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -43,7 +43,7 @@ export class Workspace { } catch (e) { if (validate) { this.logger.warn( - `Could not find openfn.yaml at ${workspacePath}. Using default values.` + `Could not find openfn.yaml at ${workspacePath}. Using default configuration.` ); } } diff --git a/packages/project/src/serialize/to-app-state.ts b/packages/project/src/serialize/to-app-state.ts index 737e16fed..2b7602344 100644 --- a/packages/project/src/serialize/to-app-state.ts +++ b/packages/project/src/serialize/to-app-state.ts @@ -141,11 +141,12 @@ export const mapWorkflow = ( }); if (mappedCredential) { projectCredentialId = mappedCredential.uuid; - } else { - console.warn(`WARING! Failed to map credential ${projectCredentialId} - Lightning may throw an error. - -Ensure the credential exists in project.yaml and try again (maybe ensure the credential is attached to the project in the app and run project fetch)`); } + // else { + // console.warn(`WARING! Failed to map credential ${projectCredentialId} - Lightning may throw an error. + + // Ensure the credential exists in project.yaml and try again (maybe ensure the credential is attached to the project in the app and run project fetch)`); + // } otherOpenFnProps.project_credential_id = projectCredentialId; } } From 5f47e81aa33b6e396fdc215403b228cbbc346938 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 26 Feb 2026 15:20:22 +0000 Subject: [PATCH 3/3] version: cli@1.29.0 --- .changeset/five-lamps-sin.md | 5 ----- .changeset/slow-worms-try.md | 5 ----- .changeset/vast-weeks-tap.md | 13 ------------- integration-tests/cli/CHANGELOG.md | 7 +++++++ integration-tests/cli/package.json | 2 +- packages/cli/CHANGELOG.md | 20 ++++++++++++++++++++ packages/cli/package.json | 2 +- packages/project/CHANGELOG.md | 6 ++++++ packages/project/package.json | 2 +- 9 files changed, 36 insertions(+), 26 deletions(-) delete mode 100644 .changeset/five-lamps-sin.md delete mode 100644 .changeset/slow-worms-try.md delete mode 100644 .changeset/vast-weeks-tap.md diff --git a/.changeset/five-lamps-sin.md b/.changeset/five-lamps-sin.md deleted file mode 100644 index 4012288e8..000000000 --- a/.changeset/five-lamps-sin.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/project': patch ---- - -Suppress error message when generating app state without a credentials.yaml diff --git a/.changeset/slow-worms-try.md b/.changeset/slow-worms-try.md deleted file mode 100644 index 5d8f783d4..000000000 --- a/.changeset/slow-worms-try.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/cli': patch ---- - -Improve --help docs and error messages diff --git a/.changeset/vast-weeks-tap.md b/.changeset/vast-weeks-tap.md deleted file mode 100644 index 953c4560a..000000000 --- a/.changeset/vast-weeks-tap.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -'@openfn/cli': minor ---- - -Support github sync with new `project` commands - -GitHub Sync is currently designed with a pair of actions which trigger the legacy `openfn deploy` and `openfn pull` commands. - -This update adds support for the `openfn.yaml` file to the commands: so if you run `openfn project pull` in a github synced repo, the repo goes into v2 mode. The legacy deploy and pull commands will "redirect" to v2. You should leave the legacy `config.json` file, but state.json and spec.yaml files can be removed. - -Initaliasing GitHub sync from the app will continue to use the legacy file structure for the initial commit. If you want to switch to v2, either create an empty openfn.yaml and trigger a sync, or locally run `openfn project pull` and commit changes. - -Be aware that v2 sync only supports a single `openfn.yaml` and `workflows` folder at a time - so a sync which pulls from multiple connected apps will not work well. It should however be safe to push changes to multiple apps. diff --git a/integration-tests/cli/CHANGELOG.md b/integration-tests/cli/CHANGELOG.md index f22a475d8..5513eaa95 100644 --- a/integration-tests/cli/CHANGELOG.md +++ b/integration-tests/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/integration-tests-cli +## 1.0.16 + +### Patch Changes + +- Updated dependencies [cbe2800] + - @openfn/project@0.14.2 + ## 1.0.15 ### Patch Changes diff --git a/integration-tests/cli/package.json b/integration-tests/cli/package.json index 6ad5957c3..1569cfc3e 100644 --- a/integration-tests/cli/package.json +++ b/integration-tests/cli/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-cli", "private": true, - "version": "1.0.15", + "version": "1.0.16", "description": "CLI integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 351b9dab5..43e120e9a 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,25 @@ # @openfn/cli +## 1.29.0 + +### Minor Changes + +- 2d570c6: Support github sync with new `project` commands + + GitHub Sync is currently designed with a pair of actions which trigger the legacy `openfn deploy` and `openfn pull` commands. + + This update adds support for the `openfn.yaml` file to the commands: so if you run `openfn project pull` in a github synced repo, the repo goes into v2 mode. The legacy deploy and pull commands will "redirect" to v2. You should leave the legacy `config.json` file, but state.json and spec.yaml files can be removed. + + Initaliasing GitHub sync from the app will continue to use the legacy file structure for the initial commit. If you want to switch to v2, either create an empty openfn.yaml and trigger a sync, or locally run `openfn project pull` and commit changes. + + Be aware that v2 sync only supports a single `openfn.yaml` and `workflows` folder at a time - so a sync which pulls from multiple connected apps will not work well. It should however be safe to push changes to multiple apps. + +### Patch Changes + +- cbe2800: Improve --help docs and error messages +- Updated dependencies [cbe2800] + - @openfn/project@0.14.2 + ## 1.28.2 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 3519b9015..21e1e3527 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.28.2", + "version": "1.29.0", "description": "CLI devtools for the OpenFn toolchain", "engines": { "node": ">=18", diff --git a/packages/project/CHANGELOG.md b/packages/project/CHANGELOG.md index 60ddd603a..c8d16aa47 100644 --- a/packages/project/CHANGELOG.md +++ b/packages/project/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/project +## 0.14.2 + +### Patch Changes + +- cbe2800: Suppress error message when generating app state without a credentials.yaml + ## 0.14.1 ### Patch Changes diff --git a/packages/project/package.json b/packages/project/package.json index 473fbffb3..d1f2f5641 100644 --- a/packages/project/package.json +++ b/packages/project/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/project", - "version": "0.14.1", + "version": "0.14.2", "description": "Read, serialize, replicate and sync OpenFn projects", "scripts": { "test": "pnpm ava",