diff --git a/README.md b/README.md index 8dacd7c..37e001b 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ $ npx @nitpicker/cli crawl https://example.com $ npx @nitpicker/cli crawl https://example.com --interval 5000 $ npx @nitpicker/cli crawl https://example.com --parallels 50 $ npx @nitpicker/cli crawl https://example.com --no-image +$ npx @nitpicker/cli crawl https://example.com --single $ npx @nitpicker/cli crawl https://example.com --no-fetch-external $ npx @nitpicker/cli crawl https://example.com --no-recursive $ npx @nitpicker/cli crawl https://example.com --scope "www.example.com, www3.example.com, https://blog.example.com/blog" diff --git a/packages/@nitpicker/cli/src/commands/crawl.spec.ts b/packages/@nitpicker/cli/src/commands/crawl.spec.ts new file mode 100644 index 0000000..cd0f8eb --- /dev/null +++ b/packages/@nitpicker/cli/src/commands/crawl.spec.ts @@ -0,0 +1,172 @@ +import type { CrawlerOrchestrator as OrchestratorType } from '@nitpicker/crawler'; + +import { afterEach, beforeEach, describe, it, expect, vi } from 'vitest'; + +const mockCrawling = vi.fn(); + +vi.mock('@nitpicker/crawler', () => ({ + CrawlerOrchestrator: { + crawling: mockCrawling, + }, +})); + +vi.mock('../crawl/event-assignments.js', () => ({ + eventAssignments: vi.fn().mockResolvedValue(), +})); + +vi.mock('../crawl/debug.js', () => ({ + log: vi.fn(), + verbosely: vi.fn(), +})); + +type CrawlFlags = Parameters[1]; + +/** + * Minimal flags matching the shape produced by the CLI parser. + * @param overrides - Flag values to override defaults. + */ +function createFlags(overrides: Partial = {}): CrawlFlags { + return { + resume: undefined, + interval: undefined, + image: true, + fetchExternal: true, + parallels: undefined, + recursive: true, + scope: undefined, + exclude: undefined, + excludeKeyword: undefined, + excludeUrl: undefined, + disableQueries: undefined, + imageFileSizeThreshold: undefined, + single: undefined, + maxExcludedDepth: undefined, + retry: 3, + list: undefined, + listFile: undefined, + userAgent: undefined, + ignoreRobots: undefined, + output: undefined, + verbose: undefined, + silent: undefined, + diff: undefined, + ...overrides, + } as CrawlFlags; +} + +/** Sets up the fake orchestrator that mockCrawling returns. */ +function setupFakeOrchestrator() { + const fakeOrchestrator = { + write: vi.fn().mockResolvedValue(), + garbageCollect: vi.fn(), + archive: { filePath: '/tmp/test.nitpicker' }, + } as unknown as OrchestratorType; + + mockCrawling.mockImplementation((_urls, _opts, cb) => { + cb?.(fakeOrchestrator, { baseUrl: 'https://example.com' }); + return Promise.resolve(fakeOrchestrator); + }); +} + +describe('startCrawl', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.resetModules(); + setupFakeOrchestrator(); + }); + + it('--single フラグが true の場合、recursive: false で CrawlerOrchestrator.crawling を呼び出す', async () => { + const { startCrawl } = await import('./crawl.js'); + await startCrawl(['https://example.com'], createFlags({ single: true })); + + expect(mockCrawling).toHaveBeenCalledWith( + ['https://example.com'], + expect.objectContaining({ recursive: false }), + expect.any(Function), + ); + }); + + it('--single フラグが未指定の場合、recursive はフラグの値がそのまま渡される', async () => { + const { startCrawl } = await import('./crawl.js'); + await startCrawl(['https://example.com'], createFlags({ recursive: true })); + + expect(mockCrawling).toHaveBeenCalledWith( + ['https://example.com'], + expect.objectContaining({ recursive: true }), + expect.any(Function), + ); + }); + + it('--single と --recursive が同時指定された場合、--single が優先され recursive: false になる', async () => { + const { startCrawl } = await import('./crawl.js'); + await startCrawl( + ['https://example.com'], + createFlags({ single: true, recursive: true }), + ); + + expect(mockCrawling).toHaveBeenCalledWith( + ['https://example.com'], + expect.objectContaining({ recursive: false }), + expect.any(Function), + ); + }); + + it('--list モードでも recursive: false になる', async () => { + const { startCrawl } = await import('./crawl.js'); + await startCrawl( + ['https://example.com'], + createFlags({ list: ['https://example.com/a'] }), + ); + + expect(mockCrawling).toHaveBeenCalledWith( + ['https://example.com'], + expect.objectContaining({ recursive: false, list: true }), + expect.any(Function), + ); + }); +}); + +describe('crawl', () => { + let consoleWarnSpy: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + vi.resetModules(); + setupFakeOrchestrator(); + consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('--single と --list を同時指定した場合、警告を出力する', async () => { + const { crawl } = await import('./crawl.js'); + await crawl([], createFlags({ single: true, list: ['https://example.com/a'] })); + + expect(consoleWarnSpy).toHaveBeenCalledWith( + 'Warning: --single is ignored when --list or --list-file is specified.', + ); + }); + + it('--single と --list-file を同時指定した場合、警告を出力する', async () => { + const { crawl } = await import('./crawl.js'); + + vi.mock('@d-zero/readtext/list', () => ({ + readList: vi.fn().mockResolvedValue(['https://example.com/a']), + })); + + await crawl([], createFlags({ single: true, listFile: '/tmp/list.txt' })); + + expect(consoleWarnSpy).toHaveBeenCalledWith( + 'Warning: --single is ignored when --list or --list-file is specified.', + ); + }); + + it('--single のみの場合、警告を出力しない', async () => { + const { crawl } = await import('./crawl.js'); + await crawl(['https://example.com'], createFlags({ single: true })); + + expect(consoleWarnSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/@nitpicker/cli/src/commands/crawl.ts b/packages/@nitpicker/cli/src/commands/crawl.ts index 3748a96..85b876a 100644 --- a/packages/@nitpicker/cli/src/commands/crawl.ts +++ b/packages/@nitpicker/cli/src/commands/crawl.ts @@ -184,7 +184,8 @@ export async function startCrawl(siteUrl: string[], flags: CrawlFlags): Promise< ...mapFlagsToCrawlConfig(flags), filePath: flags.output, list: isList, - recursive: isList ? false : flags.recursive, + // --single(単一ページモード)および --list モードでは再帰クロールを無効化 + recursive: isList || flags.single ? false : flags.recursive, }, (orchestrator, config) => { run( @@ -290,6 +291,11 @@ export async function crawl(args: string[], flags: CrawlFlags) { return; } + if (flags.single && (flags.list?.length || flags.listFile)) { + // eslint-disable-next-line no-console + console.warn('Warning: --single is ignored when --list or --list-file is specified.'); + } + if (flags.listFile) { const list = await readList(path.resolve(process.cwd(), flags.listFile)); flags.list = list; diff --git a/packages/@nitpicker/cli/src/commands/pipeline.spec.ts b/packages/@nitpicker/cli/src/commands/pipeline.spec.ts index 45759ce..064aabe 100644 --- a/packages/@nitpicker/cli/src/commands/pipeline.spec.ts +++ b/packages/@nitpicker/cli/src/commands/pipeline.spec.ts @@ -237,6 +237,21 @@ describe('pipeline command', () => { expect(reportFn).toHaveBeenCalledWith([archivePath], expect.any(Object)); }); + it('passes --single flag to startCrawl', async () => { + vi.mocked(startCrawlFn).mockResolvedValue('/tmp/site.nitpicker'); + vi.mocked(analyzeFn).mockResolvedValue(); + + await pipeline(['https://example.com'], { + ...defaultFlags, + single: true, + }); + + expect(startCrawlFn).toHaveBeenCalledWith( + ['https://example.com'], + expect.objectContaining({ single: true }), + ); + }); + it('propagates error when startCrawl rejects', async () => { const crawlError = new Error('Crawl failed'); vi.mocked(startCrawlFn).mockRejectedValue(crawlError);