Merge pull request #261 from quantcdn/feat/specify-ua-crawl

Feat: User-agent support for crawl.

src/commands/crawl.js

@@ -98,6 +98,11 @@ command.builder = {
     type: 'boolean',
     default: false,
   },
+  'user-agent': {
+    describe: 'The user-agent to send with the request',
+    type: 'string',
+    default: 'Quant (+http://api.quantcdn.io)',
+  },
 };
 
 /**
@@ -151,6 +156,9 @@ command.handler = async function(argv) {
   crawl.maxConcurrency = argv.concurrency;
   crawl.respectRobotsTxt = argv.robots;
   crawl.acceptCookies = argv.cookies;
+  crawl.customHeaders = {
+    'User-Agent': argv['user-agent'],
+  };
 
   const quant = client(config);