diff --git a/src/commands/crawl.js b/src/commands/crawl.js index 0752755..9dfabc4 100644 --- a/src/commands/crawl.js +++ b/src/commands/crawl.js @@ -98,6 +98,11 @@ command.builder = { type: 'boolean', default: false, }, + 'user-agent': { + describe: 'The user-agent to send with the request', + type: 'string', + default: 'Quant (+http://api.quantcdn.io)', + }, }; /** @@ -151,6 +156,9 @@ command.handler = async function(argv) { crawl.maxConcurrency = argv.concurrency; crawl.respectRobotsTxt = argv.robots; crawl.acceptCookies = argv.cookies; + crawl.customHeaders = { + 'User-Agent': argv['user-agent'], + }; const quant = client(config);