Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ $ npx @nitpicker/cli crawl https://example.com
$ npx @nitpicker/cli crawl https://example.com --interval 5000
$ npx @nitpicker/cli crawl https://example.com --parallels 50
$ npx @nitpicker/cli crawl https://example.com --no-image
$ npx @nitpicker/cli crawl https://example.com --single
$ npx @nitpicker/cli crawl https://example.com --no-fetch-external
$ npx @nitpicker/cli crawl https://example.com --no-recursive
$ npx @nitpicker/cli crawl https://example.com --scope "www.example.com, www3.example.com, https://blog.example.com/blog"
Expand Down
172 changes: 172 additions & 0 deletions packages/@nitpicker/cli/src/commands/crawl.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
import type { CrawlerOrchestrator as OrchestratorType } from '@nitpicker/crawler';

import { afterEach, beforeEach, describe, it, expect, vi } from 'vitest';

const mockCrawling = vi.fn();

vi.mock('@nitpicker/crawler', () => ({
CrawlerOrchestrator: {
crawling: mockCrawling,
},
}));

vi.mock('../crawl/event-assignments.js', () => ({
eventAssignments: vi.fn().mockResolvedValue(),
}));

vi.mock('../crawl/debug.js', () => ({
log: vi.fn(),
verbosely: vi.fn(),
}));

type CrawlFlags = Parameters<typeof import('./crawl.js').startCrawl>[1];

Check warning on line 22 in packages/@nitpicker/cli/src/commands/crawl.spec.ts

View workflow job for this annotation

GitHub Actions / lint

`import()` type annotations are forbidden

/**
* Minimal flags matching the shape produced by the CLI parser.
* @param overrides - Flag values to override defaults.
*/
function createFlags(overrides: Partial<CrawlFlags> = {}): CrawlFlags {
return {
resume: undefined,
interval: undefined,
image: true,
fetchExternal: true,
parallels: undefined,
recursive: true,
scope: undefined,
exclude: undefined,
excludeKeyword: undefined,
excludeUrl: undefined,
disableQueries: undefined,
imageFileSizeThreshold: undefined,
single: undefined,
maxExcludedDepth: undefined,
retry: 3,
list: undefined,
listFile: undefined,
userAgent: undefined,
ignoreRobots: undefined,
output: undefined,
verbose: undefined,
silent: undefined,
diff: undefined,
...overrides,
} as CrawlFlags;
}

/** Sets up the fake orchestrator that mockCrawling returns. */
function setupFakeOrchestrator() {
const fakeOrchestrator = {
write: vi.fn().mockResolvedValue(),
garbageCollect: vi.fn(),
archive: { filePath: '/tmp/test.nitpicker' },
} as unknown as OrchestratorType;

mockCrawling.mockImplementation((_urls, _opts, cb) => {
cb?.(fakeOrchestrator, { baseUrl: 'https://example.com' });
return Promise.resolve(fakeOrchestrator);
});
}

describe('startCrawl', () => {
beforeEach(() => {
vi.clearAllMocks();
vi.resetModules();
setupFakeOrchestrator();
});

it('--single フラグが true の場合、recursive: false で CrawlerOrchestrator.crawling を呼び出す', async () => {
const { startCrawl } = await import('./crawl.js');
await startCrawl(['https://example.com'], createFlags({ single: true }));

expect(mockCrawling).toHaveBeenCalledWith(
['https://example.com'],
expect.objectContaining({ recursive: false }),
expect.any(Function),
);
});

it('--single フラグが未指定の場合、recursive はフラグの値がそのまま渡される', async () => {
const { startCrawl } = await import('./crawl.js');
await startCrawl(['https://example.com'], createFlags({ recursive: true }));

expect(mockCrawling).toHaveBeenCalledWith(
['https://example.com'],
expect.objectContaining({ recursive: true }),
expect.any(Function),
);
});

it('--single と --recursive が同時指定された場合、--single が優先され recursive: false になる', async () => {
const { startCrawl } = await import('./crawl.js');
await startCrawl(
['https://example.com'],
createFlags({ single: true, recursive: true }),
);

expect(mockCrawling).toHaveBeenCalledWith(
['https://example.com'],
expect.objectContaining({ recursive: false }),
expect.any(Function),
);
});

it('--list モードでも recursive: false になる', async () => {
const { startCrawl } = await import('./crawl.js');
await startCrawl(
['https://example.com'],
createFlags({ list: ['https://example.com/a'] }),
);

expect(mockCrawling).toHaveBeenCalledWith(
['https://example.com'],
expect.objectContaining({ recursive: false, list: true }),
expect.any(Function),
);
});
});

describe('crawl', () => {
let consoleWarnSpy: ReturnType<typeof vi.spyOn>;

beforeEach(() => {
vi.clearAllMocks();
vi.resetModules();
setupFakeOrchestrator();
consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
});

afterEach(() => {
vi.restoreAllMocks();
});

it('--single と --list を同時指定した場合、警告を出力する', async () => {
const { crawl } = await import('./crawl.js');
await crawl([], createFlags({ single: true, list: ['https://example.com/a'] }));

expect(consoleWarnSpy).toHaveBeenCalledWith(
'Warning: --single is ignored when --list or --list-file is specified.',
);
});

it('--single と --list-file を同時指定した場合、警告を出力する', async () => {
const { crawl } = await import('./crawl.js');

vi.mock('@d-zero/readtext/list', () => ({
readList: vi.fn().mockResolvedValue(['https://example.com/a']),
}));

await crawl([], createFlags({ single: true, listFile: '/tmp/list.txt' }));

expect(consoleWarnSpy).toHaveBeenCalledWith(
'Warning: --single is ignored when --list or --list-file is specified.',
);
});

it('--single のみの場合、警告を出力しない', async () => {
const { crawl } = await import('./crawl.js');
await crawl(['https://example.com'], createFlags({ single: true }));

expect(consoleWarnSpy).not.toHaveBeenCalled();
});
});
8 changes: 7 additions & 1 deletion packages/@nitpicker/cli/src/commands/crawl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ export async function startCrawl(siteUrl: string[], flags: CrawlFlags): Promise<
...mapFlagsToCrawlConfig(flags),
filePath: flags.output,
list: isList,
recursive: isList ? false : flags.recursive,
// --single(単一ページモード)および --list モードでは再帰クロールを無効化
recursive: isList || flags.single ? false : flags.recursive,
},
(orchestrator, config) => {
run(
Expand Down Expand Up @@ -290,6 +291,11 @@ export async function crawl(args: string[], flags: CrawlFlags) {
return;
}

if (flags.single && (flags.list?.length || flags.listFile)) {
// eslint-disable-next-line no-console
console.warn('Warning: --single is ignored when --list or --list-file is specified.');
}

if (flags.listFile) {
const list = await readList(path.resolve(process.cwd(), flags.listFile));
flags.list = list;
Expand Down
15 changes: 15 additions & 0 deletions packages/@nitpicker/cli/src/commands/pipeline.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,21 @@ describe('pipeline command', () => {
expect(reportFn).toHaveBeenCalledWith([archivePath], expect.any(Object));
});

it('passes --single flag to startCrawl', async () => {
vi.mocked(startCrawlFn).mockResolvedValue('/tmp/site.nitpicker');
vi.mocked(analyzeFn).mockResolvedValue();

await pipeline(['https://example.com'], {
...defaultFlags,
single: true,
});

expect(startCrawlFn).toHaveBeenCalledWith(
['https://example.com'],
expect.objectContaining({ single: true }),
);
});

it('propagates error when startCrawl rejects', async () => {
const crawlError = new Error('Crawl failed');
vi.mocked(startCrawlFn).mockRejectedValue(crawlError);
Expand Down