diff --git a/integration-tests/cli/CHANGELOG.md b/integration-tests/cli/CHANGELOG.md index f22a475d8..5513eaa95 100644 --- a/integration-tests/cli/CHANGELOG.md +++ b/integration-tests/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/integration-tests-cli +## 1.0.16 + +### Patch Changes + +- Updated dependencies [cbe2800] + - @openfn/project@0.14.2 + ## 1.0.15 ### Patch Changes diff --git a/integration-tests/cli/package.json b/integration-tests/cli/package.json index 6ad5957c3..1569cfc3e 100644 --- a/integration-tests/cli/package.json +++ b/integration-tests/cli/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-cli", "private": true, - "version": "1.0.15", + "version": "1.0.16", "description": "CLI integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 351b9dab5..43e120e9a 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,25 @@ # @openfn/cli +## 1.29.0 + +### Minor Changes + +- 2d570c6: Support github sync with new `project` commands + + GitHub Sync is currently designed with a pair of actions which trigger the legacy `openfn deploy` and `openfn pull` commands. + + This update adds support for the `openfn.yaml` file to the commands: so if you run `openfn project pull` in a github synced repo, the repo goes into v2 mode. The legacy deploy and pull commands will "redirect" to v2. You should leave the legacy `config.json` file, but state.json and spec.yaml files can be removed. + + Initaliasing GitHub sync from the app will continue to use the legacy file structure for the initial commit. If you want to switch to v2, either create an empty openfn.yaml and trigger a sync, or locally run `openfn project pull` and commit changes. + + Be aware that v2 sync only supports a single `openfn.yaml` and `workflows` folder at a time - so a sync which pulls from multiple connected apps will not work well. It should however be safe to push changes to multiple apps. + +### Patch Changes + +- cbe2800: Improve --help docs and error messages +- Updated dependencies [cbe2800] + - @openfn/project@0.14.2 + ## 1.28.2 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 3519b9015..21e1e3527 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.28.2", + "version": "1.29.0", "description": "CLI devtools for the OpenFn toolchain", "engines": { "node": ">=18", diff --git a/packages/cli/src/deploy/command.ts b/packages/cli/src/deploy/command.ts index 79ed15cc4..6b144f8ee 100644 --- a/packages/cli/src/deploy/command.ts +++ b/packages/cli/src/deploy/command.ts @@ -1,12 +1,13 @@ import yargs from 'yargs'; import { build, ensure, override } from '../util/command-builders'; import { Opts } from '../options'; +import { Opts as POpts } from '../projects/options'; import * as o from '../options'; import * as o2 from '../projects/options'; export type DeployOptions = Required< Pick< - Opts, + Opts & POpts, | 'beta' | 'command' | 'configPath' @@ -15,6 +16,7 @@ export type DeployOptions = Required< | 'logJson' | 'projectPath' | 'statePath' + | 'workspace' > >; diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index c61b70ba9..4e834a0fb 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -8,6 +8,8 @@ import { import type { Logger } from '../util/logger'; import { DeployOptions } from './command'; import * as beta from '../projects/deploy'; +import path from 'node:path'; +import { fileExists } from '../util/file-exists'; export type DeployFn = typeof deploy; @@ -32,6 +34,17 @@ async function deployHandler( try { const config = mergeOverrides(await getConfig(options.configPath), options); + const v2ConfigPath = path.join( + options.workspace || process.cwd(), + 'openfn.yaml' + ); + if (await fileExists(v2ConfigPath)) { + logger.always( + 'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy)' + ); + return beta.handler({ ...options, force: true }, logger); + } + logger.debug('Deploying with config', JSON.stringify(config, null, 2)); if (options.confirm === false) { @@ -76,13 +89,18 @@ function mergeOverrides( config: DeployConfig, options: DeployOptions ): DeployConfig { + const workspace = options.workspace || process.cwd(); + const resolveRelative = (p: string) => + path.isAbsolute(p) ? p : path.join(workspace, p); + const specPath = pickFirst(options.projectPath, config.specPath); + const statePath = pickFirst(options.statePath, config.statePath); return { ...config, apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey), endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint), - statePath: pickFirst(options.statePath, config.statePath), - specPath: pickFirst(options.projectPath, config.specPath), - configPath: options.configPath, + statePath: resolveRelative(statePath), + specPath: resolveRelative(specPath), + configPath: resolveRelative(options.configPath), requireConfirmation: pickFirst(options.confirm, config.requireConfirmation), }; } diff --git a/packages/cli/src/execute/command.ts b/packages/cli/src/execute/command.ts index 77e859203..808750ed1 100644 --- a/packages/cli/src/execute/command.ts +++ b/packages/cli/src/execute/command.ts @@ -84,21 +84,23 @@ const options = [ const executeCommand: yargs.CommandModule = { command: 'execute [workflow]', describe: `Run an openfn expression or workflow. Get more help by running openfn help. - \nExecute will run a expression/workflow at the path and write the output state to disk (to ./state.json unless otherwise specified) - \nRemember to include the adaptor name with -a. Auto install adaptors with the -i flag.`, + \nExecute will run a expression/workflow at the path and write the output state to disk (to ./state.json unless otherwise specified)`, aliases: ['$0'], handler: ensure('execute', options), builder: (yargs) => build(options, yargs) // TODO this is now path or workflow name .positional('path', { - describe: - 'The path to load the job or workflow from (a .js or .json file or a dir containing a job.js file)', + describe: 'The path or name of the workflow or expression to run', demandOption: true, }) + .example( + 'openfn my-workflow', + 'Execute a workflow called my-workflow. Requires an openfn.yaml file (created by openfn project pull)' + ) .example( 'openfn foo/job.js', - 'Execute foo/job.js with no adaptor and write the final state to foo/job.json' + 'Execute foo/job.js (with no adaptor) and write the final state to foo/job.json' ) .example( 'openfn workflow.json', @@ -106,11 +108,11 @@ const executeCommand: yargs.CommandModule = { ) .example( 'openfn job.js -a common --log info', - 'Execute job.js with common adaptor and info-level logging' + 'Execute job.js with common adaptor (which will be automatically installed) and info-level logging' ) .example( - 'openfn compile job.js -a http', - 'Compile the expression at job.js with the http adaptor and print the code to stdout' + 'openfn job.js -ma common', + 'Execute job.js with the build of the common adaptor found in your adaptors monorepo' ), }; diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts index 6a3492561..1df4af557 100644 --- a/packages/cli/src/projects/pull.ts +++ b/packages/cli/src/projects/pull.ts @@ -23,6 +23,7 @@ export type PullOptions = Pick< | 'project' | 'confirm' | 'snapshots' + | 'force' >; const options = [ @@ -61,6 +62,7 @@ export const command: yargs.CommandModule = { }; export async function handler(options: PullOptions, logger: Logger) { + options.workspace ??= process.cwd(); ensureProjectId(options, logger); await fetch(options, logger); diff --git a/packages/cli/src/pull/command.ts b/packages/cli/src/pull/command.ts index ba6dd6620..314164623 100644 --- a/packages/cli/src/pull/command.ts +++ b/packages/cli/src/pull/command.ts @@ -1,12 +1,13 @@ import yargs from 'yargs'; import { build, ensure, override } from '../util/command-builders'; import { Opts } from '../options'; +import { Opts as POpts } from '../projects/options'; import * as o from '../options'; import * as po from '../projects/options'; export type PullOptions = Required< Pick< - Opts, + Opts & POpts, | 'beta' | 'command' | 'log' @@ -17,6 +18,7 @@ export type PullOptions = Required< | 'projectId' | 'confirm' | 'snapshots' + | 'workspace' > >; diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts index fe96e7188..2981ce5ed 100644 --- a/packages/cli/src/pull/handler.ts +++ b/packages/cli/src/pull/handler.ts @@ -11,6 +11,7 @@ import { import type { Logger } from '../util/logger'; import { PullOptions } from '../pull/command'; import beta from '../projects/pull'; +import { fileExists } from '../util/file-exists'; async function pullHandler(options: PullOptions, logger: Logger) { if (options.beta) { @@ -21,6 +22,20 @@ async function pullHandler(options: PullOptions, logger: Logger) { try { const config = mergeOverrides(await getConfig(options.configPath), options); + const v2ConfigPath = path.join( + options.workspace || process.cwd(), + 'openfn.yaml' + ); + if (await fileExists(v2ConfigPath)) { + logger.always( + 'Detected openfn.yaml file - switching to v2 pull (openfn project pull)' + ); + return beta( + { ...options, project: options.projectId, force: true }, + logger + ); + } + if (process.env['OPENFN_API_KEY']) { logger.info('Using OPENFN_API_KEY environment variable'); config.apiKey = process.env['OPENFN_API_KEY']; @@ -127,11 +142,20 @@ function mergeOverrides( config: DeployConfig, options: PullOptions ): DeployConfig { + const workspace = options.workspace || process.cwd(); return { ...config, apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey), endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint), - configPath: options.configPath, + configPath: path.isAbsolute(options.configPath) + ? options.configPath + : path.join(workspace, options.configPath), + specPath: path.isAbsolute(config.specPath) + ? config.specPath + : path.join(workspace, config.specPath), + statePath: path.isAbsolute(config.statePath) + ? config.statePath + : path.join(workspace, config.statePath), requireConfirmation: pickFirst(options.confirm, config.requireConfirmation), }; } diff --git a/packages/cli/src/util/file-exists.ts b/packages/cli/src/util/file-exists.ts new file mode 100644 index 000000000..97c6e61d9 --- /dev/null +++ b/packages/cli/src/util/file-exists.ts @@ -0,0 +1,10 @@ +import fs from 'fs/promises'; + +export async function fileExists(filePath: string) { + try { + const stats = await fs.stat(filePath); + return stats.isFile(); + } catch (error) { + return false; + } +} diff --git a/packages/cli/test/pull/handler.test.ts b/packages/cli/test/pull/handler.test.ts new file mode 100644 index 000000000..3ef838dff --- /dev/null +++ b/packages/cli/test/pull/handler.test.ts @@ -0,0 +1,38 @@ +import test from 'ava'; +import mockfs from 'mock-fs'; +import { createMockLogger } from '@openfn/logger'; +import pullHandler from '../../src/pull/handler'; +import { PullOptions } from '../../src/pull/command'; + +test.beforeEach(() => { + mockfs.restore(); +}); + +test.afterEach(() => { + mockfs.restore(); +}); + +const options: PullOptions = { + beta: false, + command: 'pull', + projectPath: './project.yaml', + configPath: './config.json', + projectId: 'abc-123', + confirm: false, + snapshots: [], +}; + +test.serial( + 'redirects to beta handler when openfn.yaml exists in cwd', + async (t) => { + const logger = createMockLogger('', { level: 'debug' }); + mockfs({ + ['./config.json']: `{"apiKey": "123"}`, + ['./openfn.yaml']: '', + }); + + await t.throwsAsync(() => pullHandler(options, logger)); + + t.truthy(logger._find('always', /Detected openfn.yaml file/i)); + } +); diff --git a/packages/cli/test/util/file-exists.test.ts b/packages/cli/test/util/file-exists.test.ts new file mode 100644 index 000000000..a6c8ce7b4 --- /dev/null +++ b/packages/cli/test/util/file-exists.test.ts @@ -0,0 +1,22 @@ +import test from 'ava'; +import mockfs from 'mock-fs'; +import { fileExists } from '../../src/util/file-exists'; + +test.afterEach(() => { + mockfs.restore(); +}); + +test('returns true for an existing file', async (t) => { + mockfs({ './test.txt': 'content' }); + t.true(await fileExists('./test.txt')); +}); + +test('returns false for a non-existent path', async (t) => { + mockfs({}); + t.false(await fileExists('./nonexistent.txt')); +}); + +test('returns false for a directory', async (t) => { + mockfs({ './mydir': {} }); + t.false(await fileExists('./mydir')); +}); diff --git a/packages/project/CHANGELOG.md b/packages/project/CHANGELOG.md index 60ddd603a..c8d16aa47 100644 --- a/packages/project/CHANGELOG.md +++ b/packages/project/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/project +## 0.14.2 + +### Patch Changes + +- cbe2800: Suppress error message when generating app state without a credentials.yaml + ## 0.14.1 ### Patch Changes diff --git a/packages/project/package.json b/packages/project/package.json index 473fbffb3..d1f2f5641 100644 --- a/packages/project/package.json +++ b/packages/project/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/project", - "version": "0.14.1", + "version": "0.14.2", "description": "Read, serialize, replicate and sync OpenFn projects", "scripts": { "test": "pnpm ava", diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index 93d44b3f8..618bdbcf2 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -43,7 +43,7 @@ export class Workspace { } catch (e) { if (validate) { this.logger.warn( - `Could not find openfn.yaml at ${workspacePath}. Using default values.` + `Could not find openfn.yaml at ${workspacePath}. Using default configuration.` ); } } diff --git a/packages/project/src/serialize/to-app-state.ts b/packages/project/src/serialize/to-app-state.ts index 737e16fed..2b7602344 100644 --- a/packages/project/src/serialize/to-app-state.ts +++ b/packages/project/src/serialize/to-app-state.ts @@ -141,11 +141,12 @@ export const mapWorkflow = ( }); if (mappedCredential) { projectCredentialId = mappedCredential.uuid; - } else { - console.warn(`WARING! Failed to map credential ${projectCredentialId} - Lightning may throw an error. - -Ensure the credential exists in project.yaml and try again (maybe ensure the credential is attached to the project in the app and run project fetch)`); } + // else { + // console.warn(`WARING! Failed to map credential ${projectCredentialId} - Lightning may throw an error. + + // Ensure the credential exists in project.yaml and try again (maybe ensure the credential is attached to the project in the app and run project fetch)`); + // } otherOpenFnProps.project_credential_id = projectCredentialId; } }