From b77b00ed49fe59c662f984ac016a1a5b3d79c805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Gonz=C3=A1lez?= Date: Wed, 12 Jun 2024 16:18:17 -0600 Subject: [PATCH] try another lambda chromium --- projects/scrape-txs/package-lock.json | 153 +++++++++++++++++++++++++- projects/scrape-txs/package.json | 1 + projects/scrape-txs/src/lib/run.ts | 8 +- projects/scrape-txs/tsconfig.json | 4 +- 4 files changed, 159 insertions(+), 7 deletions(-) diff --git a/projects/scrape-txs/package-lock.json b/projects/scrape-txs/package-lock.json index 0ef8442..82ef115 100644 --- a/projects/scrape-txs/package-lock.json +++ b/projects/scrape-txs/package-lock.json @@ -7,6 +7,7 @@ "name": "scrape-txs", "dependencies": { "@aws-sdk/client-s3": "^3.596.0", + "@sparticuz/chromium": "^123.0.1", "commander": "^12.0.0", "dayjs": "~1.11.10", "decimal.js": "^10.4.3", @@ -1508,6 +1509,18 @@ "node": ">=16.0.0" } }, + "node_modules/@sparticuz/chromium": { + "version": "123.0.1", + "resolved": "https://registry.npmjs.org/@sparticuz/chromium/-/chromium-123.0.1.tgz", + "integrity": "sha512-RPrA99xrddbXXZjhUH1WZkTCK3QDQ77t/0y+vULtiW1uAlWLBxorxGnNTuzBDk6eNolxOVFrXZ2aoAN+/eB5TA==", + "dependencies": { + "follow-redirects": "^1.15.6", + "tar-fs": "^3.0.5" + }, + "engines": { + "node": ">= 16" + } + }, "node_modules/@tsconfig/node20": { "version": "20.1.2", "resolved": "https://registry.npmjs.org/@tsconfig/node20/-/node20-20.1.2.tgz", @@ -1598,12 +1611,58 @@ "node": ">=8" } }, + "node_modules/b4a": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", + "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==" + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/bare-events": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", + "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", + "optional": true + }, + "node_modules/bare-fs": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.1.tgz", + "integrity": "sha512-W/Hfxc/6VehXlsgFtbB5B4xFcsCl+pAh30cYhoFyXErf6oGrwjh8SwiPAdHgpmWonKuYpZgGywN0SXt7dgsADA==", + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^2.0.0", + "bare-stream": "^2.0.0" + } + }, + "node_modules/bare-os": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.3.0.tgz", + "integrity": "sha512-oPb8oMM1xZbhRQBngTgpcQ5gXw6kjOaRsSWsIeNyRxGed2w/ARyP7ScBYpWR1qfX2E5rS3gBw6OWcSQo+s+kUg==", + "optional": true + }, + "node_modules/bare-path": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", + "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", + "optional": true, + "dependencies": { + "bare-os": "^2.1.0" + } + }, + "node_modules/bare-stream": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.1.2.tgz", + "integrity": "sha512-az/7TFOh4Gk9Tqs1/xMFq5FuFoeZ9hZ3orsM2x69u8NXVUDXZnpdhG8mZY/Pv6DF954MGn+iIt4rFrG34eQsvg==", + "optional": true, + "dependencies": { + "streamx": "^2.18.0" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -1781,6 +1840,14 @@ "url": "https://dotenvx.com" } }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -1790,6 +1857,11 @@ "node": ">=0.8.0" } }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", @@ -1848,6 +1920,25 @@ "node": ">=8" } }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -2489,7 +2580,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } @@ -2791,6 +2881,15 @@ "integrity": "sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==", "dev": true }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/queue-lit": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/queue-lit/-/queue-lit-1.5.2.tgz", @@ -2820,6 +2919,11 @@ } ] }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -2937,6 +3041,19 @@ "node": ">= 10.x" } }, + "node_modules/streamx": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.18.0.tgz", + "integrity": "sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ==", + "dependencies": { + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/strnum": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", @@ -2966,6 +3083,37 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tar-fs": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", + "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.0.tgz", + "integrity": "sha512-TmLJNj6UgX8xcUZo4UDStGQtDiTzF7BzWlzn9g7UWrjkpHr5uJTK1ld16wZ3LXb2vb6jH8qU89dW5whuMdXYdw==", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -3048,8 +3196,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/xtend": { "version": "4.0.2", diff --git a/projects/scrape-txs/package.json b/projects/scrape-txs/package.json index 6afd0a7..b969ec7 100644 --- a/projects/scrape-txs/package.json +++ b/projects/scrape-txs/package.json @@ -19,6 +19,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.596.0", + "@sparticuz/chromium": "^123.0.1", "commander": "^12.0.0", "dayjs": "~1.11.10", "decimal.js": "^10.4.3", diff --git a/projects/scrape-txs/src/lib/run.ts b/projects/scrape-txs/src/lib/run.ts index ea693d4..566cbb2 100644 --- a/projects/scrape-txs/src/lib/run.ts +++ b/projects/scrape-txs/src/lib/run.ts @@ -1,8 +1,8 @@ import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import type dayjs from 'dayjs'; import fs from 'node:fs/promises'; -import { chromium, errors as playwrightErrors, type Browser } from 'playwright'; -import { launchChromium } from 'playwright-aws-lambda'; +import { chromium, type Browser } from 'playwright'; +import lambdaChromium from '@sparticuz/chromium'; import { z } from 'zod'; import { bacGtScrape } from './bac-gt/scrape'; import { bancoIndustrialScrape } from './banco-industrial/scrape'; @@ -20,7 +20,9 @@ export async function run(months: dayjs.Dayjs[]) { z.enum(['bancoIndustrialGt', 'bacGt'] as const).parse(process.env.BANK_KEY); const browser = isLambda() - ? ((await launchChromium({ + ? ((await chromium.launch({ + args: lambdaChromium.args, + executablePath: await lambdaChromium.executablePath(), headless: true, })) as Browser) : await chromium.launch({ diff --git a/projects/scrape-txs/tsconfig.json b/projects/scrape-txs/tsconfig.json index 9c3c6c9..3649483 100644 --- a/projects/scrape-txs/tsconfig.json +++ b/projects/scrape-txs/tsconfig.json @@ -18,7 +18,9 @@ // Best practices "strict": true, "skipLibCheck": true, - "noFallthroughCasesInSwitch": true + "noFallthroughCasesInSwitch": true, + + "allowSyntheticDefaultImports": true // Some stricter flags // "noUnusedLocals": true,