Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions src/__tests__/utils/help.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import {describe, it, expect} from 'vitest';
import {Command} from 'commander';
import {format_examples, add_examples} from '../../utils/help';
import type {Example} from '../../utils/help';
import {scraper_command} from '../../commands/scraper';
import {discover_command} from '../../commands/discover';
import {search_command} from '../../commands/search';
import {pipelines_command} from '../../commands/dataset';
import {scrape_command} from '../../commands/scrape';

describe('utils/help.format_examples', ()=>{
it('returns empty string for no examples', ()=>{
expect(format_examples([])).toBe('');
});

it('renders a single example with comment + dollar-prefixed command', ()=>{
const exs: Example[] = [{description: 'Do a thing', command: 'cmd x'}];
const out = format_examples(exs);
expect(out).toContain('\nExamples:\n');
expect(out).toContain(' # Do a thing');
expect(out).toContain(' $ cmd x');
});

it('separates multiple examples with a blank line', ()=>{
const exs: Example[] = [
{description: 'A', command: 'cmd a'},
{description: 'B', command: 'cmd b'},
];
const out = format_examples(exs);
expect(out).toMatch(/cmd a\n\n # B/);
});
});

describe('utils/help.add_examples attaches to a Commander command', ()=>{
it('appears in the rendered --help output', ()=>{
const cmd = new Command('demo')
.description('A demo command')
.argument('<thing>', 'The thing');
add_examples(cmd, [
{description: 'Demo this', command: 'demo widget'},
]);
let captured = '';
cmd.configureOutput({writeOut: (s)=>{ captured += s; }});
cmd.outputHelp();
expect(captured).toContain('Examples:');
expect(captured).toContain('# Demo this');
expect(captured).toContain('$ demo widget');
});
});

// Helper: render the FULL --help output, including addHelpText('after', ...)
// content (which helpInformation() alone does not include).
const render_help = (cmd: Command): string=>{
let captured = '';
cmd.configureOutput({
writeOut: (s)=>{ captured += s; },
writeErr: (s)=>{ captured += s; },
});
cmd.outputHelp();
return captured;
};

// Regression: every Scraper-Studio-adjacent command must ship an Examples
// section. This is the contract this PR establishes; if a future command
// or refactor drops the Examples block, this test fires.
describe('every customer-facing command has Examples in --help', ()=>{
const scraper_create =
scraper_command.commands.find(c=>c.name() == 'create')!;
const scraper_run =
scraper_command.commands.find(c=>c.name() == 'run')!;

const cases: [string, Command][] = [
['scraper create', scraper_create],
['scraper run', scraper_run],
['discover', discover_command],
['search', search_command],
['pipelines', pipelines_command],
['scrape', scrape_command],
];

for (const [name, cmd] of cases)
{
it(`${name} --help includes an Examples section`, ()=>{
const help = render_help(cmd);
const examples_idx = help.indexOf('\nExamples:\n');
expect(examples_idx, `${name} is missing the Examples section`)
.toBeGreaterThan(-1);
const examples_block = help.slice(examples_idx);
// At least one example (`$ brightdata ...`) must be present
// inside the Examples block.
expect(examples_block,
`${name} Examples section has no $ brightdata command`)
.toMatch(/\$\s+brightdata\s/);
// House style C5: examples use real, scrape-friendly domains,
// never the fake example.com / example.test placeholders.
// (The `--deliver-webhook` default in option text is N10, out
// of scope for this PR, so we scope this check to the Examples
// block only, not the whole --help output.)
expect(examples_block,
`${name} examples leak the fake example.com domain`)
.not.toMatch(/https?:\/\/(www\.)?example\.com/);
});
}
});
19 changes: 19 additions & 0 deletions src/commands/dataset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {get, post} from '../utils/client';
import {print, dim, fail} from '../utils/output';
import {start as start_spinner} from '../utils/spinner';
import {parse_timeout, poll_until} from '../utils/polling';
import {add_examples} from '../utils/help';
import type {
Webdata_format,
Webdata_opts,
Expand Down Expand Up @@ -305,4 +306,22 @@ const pipelines_command = new Command('pipelines')
.option('-k, --api-key <key>', 'Override API key')
.action(handle_pipelines);

add_examples(pipelines_command, [
{
description: 'List every available pipeline type',
command: 'brightdata pipelines list',
},
{
description: 'Scrape a single Amazon product page',
command: 'brightdata pipelines amazon_product '
+'https://www.amazon.com/dp/B08N5WRWNW --pretty',
},
{
description: 'Pull a LinkedIn person profile by URL, save as CSV',
command: 'brightdata pipelines linkedin_person_profile '
+'https://www.linkedin.com/in/satyanadella --format csv '
+'-o profile.csv',
},
]);

export {pipelines_command, handle_pipelines};
19 changes: 19 additions & 0 deletions src/commands/discover.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {ensure_authenticated} from '../utils/auth';
import {start as start_spinner} from '../utils/spinner';
import {parse_timeout, poll_until} from '../utils/polling';
import {print, print_table, dim, fail, is_tty} from '../utils/output';
import {add_examples} from '../utils/help';
import type {
Discover_request,
Discover_trigger_response,
Expand Down Expand Up @@ -184,5 +185,23 @@ const discover_command = new Command('discover')
.option('-k, --api-key <key>', 'Override API key')
.action(handle_discover);

add_examples(discover_command, [
{
description: 'Discover and rank URLs relevant to a query',
command: 'brightdata discover "open-source AI agent frameworks"',
},
{
description: 'Add an AI intent filter and cap results',
command: 'brightdata discover "vector databases" '
+'--intent "production-ready, self-hostable" --num-results 10',
},
{
description: 'Localize to a country and include page content as '
+'markdown',
command: 'brightdata discover "best coffee in Berlin" '
+'--country DE --include-content',
},
]);

export {discover_command, handle_discover, build_request, extract_status,
format_markdown, print_discover_table};
19 changes: 19 additions & 0 deletions src/commands/scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {ensure_authenticated} from '../utils/auth';
import {resolve} from '../utils/config';
import {start as start_spinner} from '../utils/spinner';
import {print, success, fail, info} from '../utils/output';
import {add_examples} from '../utils/help';
import type {
Scrape_format,
Scrape_request,
Expand Down Expand Up @@ -106,4 +107,22 @@ const scrape_command = new Command('scrape')
.option('-k, --api-key <key>', 'Override API key')
.action(handle_scrape);

add_examples(scrape_command, [
{
description: 'Scrape a public page and get markdown (default '
+'format)',
command: 'brightdata scrape https://news.ycombinator.com',
},
{
description: 'Return JSON with response metadata, save to a file',
command: 'brightdata scrape https://news.ycombinator.com '
+'--format json --pretty -o hn.json',
},
{
description: 'Geo-target Germany with a mobile user agent',
command: 'brightdata scrape https://www.google.com/search?q=heise '
+'--country de --mobile',
},
]);

export {scrape_command, handle_scrape};
45 changes: 45 additions & 0 deletions src/commands/scraper.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {Command} from 'commander';
import {add_examples} from '../utils/help';
import {post, get} from '../utils/client';
import {load as load_config} from '../utils/config';
import {ensure_authenticated} from '../utils/auth';
Expand Down Expand Up @@ -572,6 +573,50 @@ const run_subcommand = new Command('run')
.option('-k, --api-key <key>', 'Override API key')
.action(handle_run_scraper);

add_examples(create_subcommand, [
{
description: 'Build a scraper for a public page (AI generation '
+'takes 5 to 10 minutes)',
command: 'brightdata scraper create https://news.ycombinator.com '
+'"Extract the top 30 stories: title, url, points, author, '
+'comment count."',
},
{
description: 'Name the scraper and save the full AI output for '
+'inspection',
command: 'brightdata scraper create https://www.ycombinator.com/'
+'companies?batch=W26 "For each company card, extract name, '
+'vertical, tagline, link" --name yc-w26 --pretty -o create.json',
},
{
description: 'Custom delivery webhook (default is a stub, set '
+'this when wiring to your own backend)',
command: 'brightdata scraper create https://news.ycombinator.com '
+'"Extract top stories" --deliver-webhook '
+'https://your-app.test/scraper-callback',
},
]);

add_examples(run_subcommand, [
{
description: 'Run a scraper against a single URL (async, polls '
+'until done)',
command: 'brightdata scraper run c_mp3tuab31lswoxvpws '
+'https://news.ycombinator.com --pretty',
},
{
description: 'Sync mode for small fast pages (server-side 25 to '
+'50 second cap)',
command: 'brightdata scraper run c_mp3tuab31lswoxvpws '
+'https://news.ycombinator.com --sync',
},
{
description: 'Save output as CSV (extension chooses format)',
command: 'brightdata scraper run c_mp3tuab31lswoxvpws '
+'https://news.ycombinator.com -o stories.csv',
},
]);

const scraper_command = new Command('scraper')
.description('Build and manage Bright Data scrapers')
.addCommand(create_subcommand)
Expand Down
18 changes: 18 additions & 0 deletions src/commands/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {ensure_authenticated} from '../utils/auth';
import {resolve} from '../utils/config';
import {start as start_spinner} from '../utils/spinner';
import {print, print_table, fail, dim} from '../utils/output';
import {add_examples} from '../utils/help';
import type {
Search_engine,
Search_type,
Expand Down Expand Up @@ -280,4 +281,21 @@ const search_command = new Command('search')
.option('-k, --api-key <key>', 'Override API key')
.action(handle_search);

add_examples(search_command, [
{
description: 'Web search via Google (default engine)',
command: 'brightdata search "web scraping best practices 2026"',
},
{
description: 'News search localized to a country',
command: 'brightdata search "AI agent funding" '
+'--type news --country us --pretty',
},
{
description: 'Bing image search, mobile device profile',
command: 'brightdata search "san francisco skyline" '
+'--engine bing --type images --device mobile',
},
]);

export {search_command, handle_search};
28 changes: 28 additions & 0 deletions src/utils/help.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import {Command} from 'commander';

type Example = {
description: string; // one-line "what this does"
command: string; // the actual shell command, real domains, no example.com
};

const format_examples = (examples: Example[]): string=>{
if (!examples.length)
return '';
const blocks = examples.map(ex=>
` # ${ex.description}\n $ ${ex.command}`).join('\n\n');
return '\nExamples:\n'+blocks+'\n';
};

// Attach an "Examples:" section to a Commander command, rendered after the
// auto-generated Usage/Options block when the user runs `<cmd> --help`.
//
// House style:
// - Use real, public, scrape-friendly domains (Hacker News, Y Combinator,
// real SERP queries). Never example.com or fake IDs.
// - One line of comment per example, then the literal command on the next.
// - Order: simplest first, then a useful variant or two. Cap at 3-4.
const add_examples = (cmd: Command, examples: Example[]): Command=>
cmd.addHelpText('after', format_examples(examples));

export {add_examples, format_examples};
export type {Example};