diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4015d00..37a9af0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -25,10 +25,10 @@ packages/ ↑ └── crawler ── @nitpicker/cli ← @d-zero/roar(外部) ↑ ↑ ↑ ↑ - │ core │ report-google-sheets - │ ↑ │ - │ analyze-* プラグイン - └── @d-zero/dealer(外部) + │ core │ report-google-sheets ← @d-zero/google-sheets(外部) + │ ↑ │ ↑ + │ analyze-* プラグイン │ + └── @d-zero/dealer(外部)──┘ ``` > **Note**: CLI は analyze プラグインに直接依存する(`npx` 実行時のモジュール解決のため)。新規 analyze プラグイン追加時は `@nitpicker/cli/package.json` の `dependencies` にも追加すること。 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a6ff4d..de6ebef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ fix(core): prevent analyze results from being silently empty docs: update README with new CLI options ``` -commitlint がプリコミットフックで検証します。 +`.commitlintrc` で設定されていますが、現在 Git フックには組み込まれていません。CI やレビュー時にコミットメッセージを確認してください。 ## コーディング規約 diff --git a/packages/@nitpicker/cli/src/commands/crawl.spec.ts b/packages/@nitpicker/cli/src/commands/crawl.spec.ts index cd0f8eb..f1fe2b4 100644 --- a/packages/@nitpicker/cli/src/commands/crawl.spec.ts +++ b/packages/@nitpicker/cli/src/commands/crawl.spec.ts @@ -1,24 +1,46 @@ import type { CrawlerOrchestrator as OrchestratorType } from '@nitpicker/crawler'; +import path from 'node:path'; + import { afterEach, beforeEach, describe, it, expect, vi } from 'vitest'; const mockCrawling = vi.fn(); +const mockResume = vi.fn(); vi.mock('@nitpicker/crawler', () => ({ CrawlerOrchestrator: { crawling: mockCrawling, + resume: mockResume, }, })); +const mockEventAssignments = vi.fn().mockResolvedValue(); + vi.mock('../crawl/event-assignments.js', () => ({ - eventAssignments: vi.fn().mockResolvedValue(), + eventAssignments: mockEventAssignments, })); +const mockVerbosely = vi.fn(); +const mockLog = vi.fn(); + vi.mock('../crawl/debug.js', () => ({ - log: vi.fn(), - verbosely: vi.fn(), + log: mockLog, + verbosely: mockVerbosely, +})); + +const mockDiff = vi.fn().mockResolvedValue(); + +vi.mock('../crawl/diff.js', () => ({ + diff: mockDiff, +})); + +const mockReadList = vi.fn().mockResolvedValue(['https://example.com/from-file']); + +vi.mock('@d-zero/readtext/list', () => ({ + readList: mockReadList, })); +// eslint-disable-next-line @typescript-eslint/consistent-type-imports type CrawlFlags = Parameters[1]; /** @@ -66,12 +88,18 @@ function setupFakeOrchestrator() { cb?.(fakeOrchestrator, { baseUrl: 'https://example.com' }); return Promise.resolve(fakeOrchestrator); }); + + mockResume.mockImplementation((_path, _opts, cb) => { + cb?.(fakeOrchestrator, { baseUrl: 'https://example.com' }); + return Promise.resolve(fakeOrchestrator); + }); + + return fakeOrchestrator; } describe('startCrawl', () => { beforeEach(() => { vi.clearAllMocks(); - vi.resetModules(); setupFakeOrchestrator(); }); @@ -124,6 +152,34 @@ describe('startCrawl', () => { expect.any(Function), ); }); + + it('--output フラグを filePath として渡す', async () => { + const { startCrawl } = await import('./crawl.js'); + await startCrawl(['https://example.com'], createFlags({ output: '/custom/output' })); + + expect(mockCrawling).toHaveBeenCalledWith( + ['https://example.com'], + expect.objectContaining({ filePath: '/custom/output' }), + expect.any(Function), + ); + }); + + it('アーカイブファイルパスを返す', async () => { + const { startCrawl } = await import('./crawl.js'); + const result = await startCrawl(['https://example.com'], createFlags()); + + expect(result).toBe('/tmp/test.nitpicker'); + }); + + it('イベントエラー発生時に CrawlAggregateError をスローする', async () => { + mockEventAssignments.mockRejectedValueOnce(new Error('scrape failed')); + + const { startCrawl, CrawlAggregateError } = await import('./crawl.js'); + + await expect(startCrawl(['https://example.com'], createFlags())).rejects.toThrow( + CrawlAggregateError, + ); + }); }); describe('crawl', () => { @@ -131,7 +187,6 @@ describe('crawl', () => { beforeEach(() => { vi.clearAllMocks(); - vi.resetModules(); setupFakeOrchestrator(); consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); }); @@ -151,11 +206,6 @@ describe('crawl', () => { it('--single と --list-file を同時指定した場合、警告を出力する', async () => { const { crawl } = await import('./crawl.js'); - - vi.mock('@d-zero/readtext/list', () => ({ - readList: vi.fn().mockResolvedValue(['https://example.com/a']), - })); - await crawl([], createFlags({ single: true, listFile: '/tmp/list.txt' })); expect(consoleWarnSpy).toHaveBeenCalledWith( @@ -169,4 +219,138 @@ describe('crawl', () => { expect(consoleWarnSpy).not.toHaveBeenCalled(); }); + + it('--diff モードで引数が2つの場合、diff() を呼び出す', async () => { + const { crawl } = await import('./crawl.js'); + await crawl(['a.nitpicker', 'b.nitpicker'], createFlags({ diff: true })); + + expect(mockDiff).toHaveBeenCalledWith('a.nitpicker', 'b.nitpicker'); + expect(mockCrawling).not.toHaveBeenCalled(); + }); + + it('--diff モードで引数が不足している場合、エラーを投げる', async () => { + const { crawl } = await import('./crawl.js'); + + await expect(crawl([], createFlags({ diff: true }))).rejects.toThrow( + 'Please provide two file paths to compare', + ); + }); + + it('--diff モードで引数が1つの場合、エラーを投げる', async () => { + const { crawl } = await import('./crawl.js'); + + await expect(crawl(['a.nitpicker'], createFlags({ diff: true }))).rejects.toThrow( + 'Please provide two file paths to compare', + ); + }); + + it('--resume に絶対パスを指定した場合、そのまま渡す', async () => { + const { crawl } = await import('./crawl.js'); + await crawl([], createFlags({ resume: '/absolute/stub' })); + + expect(mockResume).toHaveBeenCalledWith( + '/absolute/stub', + expect.any(Object), + expect.any(Function), + ); + expect(mockCrawling).not.toHaveBeenCalled(); + }); + + it('--resume に相対パスを指定した場合、resolve して渡す', async () => { + const { crawl } = await import('./crawl.js'); + await crawl([], createFlags({ resume: 'relative/stub' })); + + expect(mockResume).toHaveBeenCalledWith( + path.resolve(process.cwd(), 'relative/stub'), + expect.any(Object), + expect.any(Function), + ); + }); + + it('--resume と --output を同時指定した場合、エラーを投げる', async () => { + const { crawl } = await import('./crawl.js'); + + await expect( + crawl([], createFlags({ resume: '/tmp/stub', output: '/tmp/out' })), + ).rejects.toThrow( + '--output flag is not supported with --resume. The archive path is determined by the stub file.', + ); + }); + + it('--verbose フラグで verbosely() を呼び出す', async () => { + const { crawl } = await import('./crawl.js'); + await crawl(['https://example.com'], createFlags({ verbose: true })); + + expect(mockVerbosely).toHaveBeenCalled(); + }); + + it('--verbose が未指定の場合、verbosely() を呼び出さない', async () => { + const { crawl } = await import('./crawl.js'); + await crawl(['https://example.com'], createFlags()); + + expect(mockVerbosely).not.toHaveBeenCalled(); + }); + + it('--verbose と --silent を同時指定した場合、verbosely() を呼び出さない', async () => { + const { crawl } = await import('./crawl.js'); + await crawl(['https://example.com'], createFlags({ verbose: true, silent: true })); + + expect(mockVerbosely).not.toHaveBeenCalled(); + }); + + it('--list-file フラグでファイルからURLリストを読み込んで startCrawl を呼び出す', async () => { + const { crawl } = await import('./crawl.js'); + await crawl([], createFlags({ listFile: '/tmp/urls.txt' })); + + expect(mockReadList).toHaveBeenCalledWith( + path.resolve(process.cwd(), '/tmp/urls.txt'), + ); + expect(mockCrawling).toHaveBeenCalledWith( + ['https://example.com/from-file'], + expect.objectContaining({ list: true }), + expect.any(Function), + ); + }); + + it('--list と args を両方指定した場合、マージして startCrawl を呼び出す', async () => { + const { crawl } = await import('./crawl.js'); + await crawl( + ['https://example.com/arg'], + createFlags({ list: ['https://example.com/list'] }), + ); + + expect(mockCrawling).toHaveBeenCalledWith( + ['https://example.com/list', 'https://example.com/arg'], + expect.any(Object), + expect.any(Function), + ); + }); + + it('単一 URL 引数で startCrawl を呼び出す', async () => { + const { crawl } = await import('./crawl.js'); + await crawl(['https://example.com'], createFlags()); + + expect(mockCrawling).toHaveBeenCalledWith( + ['https://example.com'], + expect.any(Object), + expect.any(Function), + ); + }); + + it('引数なし・フラグなしの場合、何も呼び出さずに正常終了する', async () => { + const { crawl } = await import('./crawl.js'); + await crawl([], createFlags()); + + expect(mockCrawling).not.toHaveBeenCalled(); + expect(mockResume).not.toHaveBeenCalled(); + expect(mockDiff).not.toHaveBeenCalled(); + }); + + it('常に log() でフラグをログ出力する', async () => { + const { crawl } = await import('./crawl.js'); + const flags = createFlags(); + await crawl([], flags); + + expect(mockLog).toHaveBeenCalledWith('Options: %O', flags); + }); }); diff --git a/packages/@nitpicker/cli/src/crawl/debug.spec.ts b/packages/@nitpicker/cli/src/crawl/debug.spec.ts new file mode 100644 index 0000000..4af5c0f --- /dev/null +++ b/packages/@nitpicker/cli/src/crawl/debug.spec.ts @@ -0,0 +1,38 @@ +import debug from 'debug'; +import { describe, it, expect, vi, afterEach } from 'vitest'; + +import { log, verbosely } from './debug.js'; + +describe('log', () => { + it('Nitpicker:CLI 名前空間の debug インスタンスである', () => { + expect(log.namespace).toBe('Nitpicker:CLI'); + }); +}); + +describe('verbosely', () => { + afterEach(() => { + debug.disable(); + }); + + it('Nitpicker 名前空間が無効な場合、debug.enable を呼び出す', () => { + debug.disable(); + const enableSpy = vi.spyOn(debug, 'enable'); + + verbosely(); + + expect(enableSpy).toHaveBeenCalledWith( + 'Nitpicker*,-Nitpicker:Crawler:Deal,-Nitpicker:Scraper:DOM:Details:*,-Nitpicker:Scraper:Resource:*', + ); + enableSpy.mockRestore(); + }); + + it('Nitpicker 名前空間がすでに有効な場合、再度 enable を呼び出さない', () => { + debug.enable('Nitpicker*'); + const enableSpy = vi.spyOn(debug, 'enable'); + + verbosely(); + + expect(enableSpy).not.toHaveBeenCalled(); + enableSpy.mockRestore(); + }); +}); diff --git a/packages/@nitpicker/cli/src/crawl/diff.spec.ts b/packages/@nitpicker/cli/src/crawl/diff.spec.ts new file mode 100644 index 0000000..9120cef --- /dev/null +++ b/packages/@nitpicker/cli/src/crawl/diff.spec.ts @@ -0,0 +1,205 @@ +import { Archive } from '@nitpicker/crawler'; +import { afterEach, beforeEach, describe, it, expect, vi } from 'vitest'; + +const mockGetPagesA = vi.fn(); +const mockCloseA = vi.fn().mockResolvedValue(); +const mockGetPagesB = vi.fn(); +const mockCloseB = vi.fn().mockResolvedValue(); + +vi.mock('@nitpicker/crawler', () => ({ + Archive: { + open: vi.fn(), + }, +})); + +vi.mock('@d-zero/shared/sort-url', () => ({ + sortUrl: vi.fn((urls: string[]) => + [...urls] + .toSorted((a, b) => a.localeCompare(b)) + .map((url) => ({ withoutHashAndAuth: url })), + ), +})); + +const mockWriteFile = vi.fn().mockResolvedValue(); + +vi.mock('node:fs/promises', () => ({ + default: { + writeFile: (...args: unknown[]) => mockWriteFile(...args), + }, +})); + +import { diff } from './diff.js'; + +/** + * Creates a mock Page object. + * @param url - The page URL + * @param options - Page property overrides + * @param options.isPage - Whether the page is an HTML page + * @param options.isExternal - Whether the page is external + * @param options.status - HTTP status code + */ +function createMockPage( + url: string, + options: { + isPage?: boolean; + isExternal?: boolean; + status?: number | null; + } = {}, +) { + return { + url: { withoutHashAndAuth: url }, + isPage: () => options.isPage ?? true, + isExternal: options.isExternal ?? false, + status: 'status' in options ? options.status : 200, + }; +} + +describe('diff', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(Archive.open).mockImplementation(({ filePath }) => { + if (typeof filePath === 'string' && filePath.includes('b')) { + return Promise.resolve({ + getPages: mockGetPagesB, + close: mockCloseB, + }) as never; + } + return Promise.resolve({ + getPages: mockGetPagesA, + close: mockCloseA, + }) as never; + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('アクティブな内部ページの URL を a.txt と b.txt に書き出す', async () => { + mockGetPagesA.mockResolvedValueOnce([createMockPage('https://example.com/')]); + mockGetPagesB.mockResolvedValueOnce([createMockPage('https://example.com/about')]); + + await diff('archive-a.nitpicker', 'archive-b.nitpicker'); + + expect(mockWriteFile).toHaveBeenCalledWith('a.txt', 'https://example.com/', 'utf8'); + expect(mockWriteFile).toHaveBeenCalledWith( + 'b.txt', + 'https://example.com/about', + 'utf8', + ); + }); + + it('外部ページをフィルタリングする', async () => { + mockGetPagesA.mockResolvedValueOnce([ + createMockPage('https://example.com/', { isExternal: false }), + createMockPage('https://external.com/', { isExternal: true }), + ]); + mockGetPagesB.mockResolvedValueOnce([]); + + await diff('a.nitpicker', 'b.nitpicker'); + + const aContent = mockWriteFile.mock.calls.find( + (c: unknown[]) => c[0] === 'a.txt', + )?.[1] as string; + expect(aContent).toBe('https://example.com/'); + expect(aContent).not.toContain('external.com'); + }); + + it('isPage() が false のページをフィルタリングする', async () => { + mockGetPagesA.mockResolvedValueOnce([ + createMockPage('https://example.com/', { isPage: true }), + createMockPage('https://example.com/image.png', { isPage: false }), + ]); + mockGetPagesB.mockResolvedValueOnce([]); + + await diff('a.nitpicker', 'b.nitpicker'); + + const aContent = mockWriteFile.mock.calls.find( + (c: unknown[]) => c[0] === 'a.txt', + )?.[1] as string; + expect(aContent).toBe('https://example.com/'); + }); + + it('ステータス 400 以上のページをフィルタリングする', async () => { + mockGetPagesA.mockResolvedValueOnce([ + createMockPage('https://example.com/', { status: 200 }), + createMockPage('https://example.com/404', { status: 404 }), + createMockPage('https://example.com/500', { status: 500 }), + ]); + mockGetPagesB.mockResolvedValueOnce([]); + + await diff('a.nitpicker', 'b.nitpicker'); + + const aContent = mockWriteFile.mock.calls.find( + (c: unknown[]) => c[0] === 'a.txt', + )?.[1] as string; + expect(aContent).toBe('https://example.com/'); + }); + + it('ステータスが null のページをフィルタリングする', async () => { + mockGetPagesA.mockResolvedValueOnce([ + createMockPage('https://example.com/', { status: 200 }), + createMockPage('https://example.com/null', { status: null }), + ]); + mockGetPagesB.mockResolvedValueOnce([]); + + await diff('a.nitpicker', 'b.nitpicker'); + + const aContent = mockWriteFile.mock.calls.find( + (c: unknown[]) => c[0] === 'a.txt', + )?.[1] as string; + expect(aContent).toBe('https://example.com/'); + }); + + it('3xx ステータスのページを含める', async () => { + mockGetPagesA.mockResolvedValueOnce([ + createMockPage('https://example.com/redirect', { status: 301 }), + ]); + mockGetPagesB.mockResolvedValueOnce([]); + + await diff('a.nitpicker', 'b.nitpicker'); + + const aContent = mockWriteFile.mock.calls.find( + (c: unknown[]) => c[0] === 'a.txt', + )?.[1] as string; + expect(aContent).toBe('https://example.com/redirect'); + }); + + it('複数ページをソートして改行区切りで書き出す', async () => { + mockGetPagesA.mockResolvedValueOnce([ + createMockPage('https://example.com/c'), + createMockPage('https://example.com/a'), + createMockPage('https://example.com/b'), + ]); + mockGetPagesB.mockResolvedValueOnce([]); + + await diff('a.nitpicker', 'b.nitpicker'); + + const aContent = mockWriteFile.mock.calls.find( + (c: unknown[]) => c[0] === 'a.txt', + )?.[1] as string; + expect(aContent).toBe( + 'https://example.com/a\nhttps://example.com/b\nhttps://example.com/c', + ); + }); + + it('archiveA と archiveB の両方を close する', async () => { + mockGetPagesA.mockResolvedValueOnce([]); + mockGetPagesB.mockResolvedValueOnce([]); + + await diff('a.nitpicker', 'b.nitpicker'); + + expect(mockCloseA).toHaveBeenCalledTimes(1); + expect(mockCloseB).toHaveBeenCalledTimes(1); + }); + + it('Archive.open にファイルパスを渡す', async () => { + mockGetPagesA.mockResolvedValueOnce([]); + mockGetPagesB.mockResolvedValueOnce([]); + + await diff('first.nitpicker', 'second-b.nitpicker'); + + expect(Archive.open).toHaveBeenCalledWith({ filePath: 'first.nitpicker' }); + expect(Archive.open).toHaveBeenCalledWith({ filePath: 'second-b.nitpicker' }); + }); +}); diff --git a/packages/@nitpicker/cli/src/crawl/event-assignments.spec.ts b/packages/@nitpicker/cli/src/crawl/event-assignments.spec.ts new file mode 100644 index 0000000..ca00522 --- /dev/null +++ b/packages/@nitpicker/cli/src/crawl/event-assignments.spec.ts @@ -0,0 +1,98 @@ +import type { CrawlerOrchestrator } from '@nitpicker/crawler'; + +import { afterEach, beforeEach, describe, it, expect, vi } from 'vitest'; + +import { eventAssignments } from './event-assignments.js'; + +type EventHandler = (...args: unknown[]) => void; + +/** + * Mock orchestrator with controllable event emission. + */ +interface MockOrchestrator extends CrawlerOrchestrator { + /** + * Emits a registered event synchronously. + * @param event - Event name + * @param args - Event arguments + */ + emit(event: string, ...args: unknown[]): void; +} + +/** + * Creates a mock CrawlerOrchestrator with controllable event emission. + */ +function createMockOrchestrator(): MockOrchestrator { + const handlers: Record = {}; + return { + on: vi.fn((event: string, handler: EventHandler) => { + handlers[event] ??= []; + handlers[event].push(handler); + }), + emit(event: string, ...args: unknown[]): void { + for (const handler of handlers[event] ?? []) { + handler(...args); + } + }, + } as unknown as MockOrchestrator; +} + +describe('eventAssignments', () => { + let stderrSpy: ReturnType; + + beforeEach(() => { + stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('logType が silent の場合、即座に resolve する', async () => { + const orchestrator = createMockOrchestrator(); + await eventAssignments(orchestrator, ['header'], 'silent'); + + expect(orchestrator.on).not.toHaveBeenCalled(); + expect(stderrSpy).not.toHaveBeenCalled(); + }); + + it('初期ログを stderr に出力する', () => { + const orchestrator = createMockOrchestrator(); + void eventAssignments(orchestrator, ['🐳 header', ' key: value'], 'normal'); + + expect(stderrSpy).toHaveBeenCalledTimes(1); + const output = stderrSpy.mock.calls[0]![0] as string; + expect(output).toContain('header'); + expect(output).toContain('key: value'); + }); + + it('error イベントで reject する', async () => { + const orchestrator = createMockOrchestrator(); + const promise = eventAssignments(orchestrator, ['header'], 'normal'); + + const error = new Error('crawl error'); + orchestrator.emit('error', error); + + await expect(promise).rejects.toBe(error); + }); + + it('writeFileStart イベントでファイルパスを stderr に出力する', async () => { + const orchestrator = createMockOrchestrator(); + const promise = eventAssignments(orchestrator, ['header'], 'normal'); + + orchestrator.emit('writeFileStart', { filePath: '/tmp/out.nitpicker' }); + orchestrator.emit('writeFileEnd', { filePath: '/tmp/out.nitpicker' }); + await promise; + + const calls = stderrSpy.mock.calls.map((c) => c[0] as string); + expect(calls.some((c) => c.includes('/tmp/out.nitpicker'))).toBe(true); + }); + + it('writeFileEnd イベントで resolve する', async () => { + const orchestrator = createMockOrchestrator(); + const promise = eventAssignments(orchestrator, ['header'], 'normal'); + + orchestrator.emit('writeFileEnd', { filePath: '/tmp/out.nitpicker' }); + + await expect(promise).resolves.toBeUndefined(); + }); +});