WebSocket is not open when 'maxrequestreached' event is reached

[LeMoussel]

• Version: [email protected]
• Platform / OS version: Windows 10
• Node.js version: v10.15.0
• Link: http://f4b1.com/ (Rem: same error for other sites)

f4b1.com

Login - f4b1.com
Contact - f4b1.com
Les différents paramètres APN opérateur : Free Mobile, Orange, SFR, Bouygues et MVNO
Maximum request reached!
Crawl Terminated

(node:14984) UnhandledPromiseRejectionWarning: Error: WebSocket is not open: readyState 3 (CLOSED)
at WebSocket.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\ws\lib\websocket.js:314:19)
at Connection.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Connection.js:84:21)
at Browser._createPageInContext (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:168:47)
at BrowserContext.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:260:26)
at Browser.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:160:33)
at HCCrawler._newCrawler (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:554:38)
at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:336:32)
at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:358:19)
(node:14984) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 3)
(node:14984) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
(node:14984) UnhandledPromiseRejectionWarning: Error: WebSocket is not open: readyState 3 (CLOSED)
at WebSocket.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\ws\lib\websocket.js:314:19)
at Connection.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Connection.js:84:21)
at Browser._createPageInContext (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:168:47)
at BrowserContext.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:260:26)
at Browser.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:160:33)
at HCCrawler._newCrawler (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:554:38)
at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:336:32)
at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:358:19)
(node:14984) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 7)
(node:14984) UnhandledPromiseRejectionWarning: Error: WebSocket is not open: readyState 3 (CLOSED)
at WebSocket.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\ws\lib\websocket.js:314:19)
at Connection.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Connection.js:84:21)
at Browser._createPageInContext (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:168:47)
at BrowserContext.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:260:26)
at Browser.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:160:33)
at HCCrawler._newCrawler (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:554:38)
at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:336:32)
at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:358:19)
at process._tickCallback (internal/process/next_tick.js:68:7)
(node:14984) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 9)

Example code:

const HCCrawler = require('headless-chrome-crawler');
const findChrome = require('chrome-finder'); // https://github.com/gwuhaolin/chrome-finder

(async () => {
  const crawler = await HCCrawler.launch({
    maxRequest: 5,
    maxDepth: 2,
    maxConcurrency: 4,
    obeyRobotsTxt: true,
    executablePath: findChrome(),  // use local Chrome browser
    headless: false,

    evaluatePage: (() => ({
      title: $('title').text(),
    })),

    onSuccess: (result => {
      console.log(result.result.title);
    }),
  });

  crawler.on('maxrequestreached', () => {
    console.log('Maximum request reached!');
  });

  await crawler.queue('http://f4b1.com/');

  await crawler.onIdle();
  await crawler.close();
})().then(() => { console.log('Crawl Terminated'); });

[popstas]

@LeMoussel, try my quick fix - popstas@43a7c14