From f3bc8b18f70488922434a4ad7967c214670ed40d Mon Sep 17 00:00:00 2001 From: "Andrew D.Gramigna" Date: Sat, 14 Oct 2023 22:25:12 -0400 Subject: [PATCH] add chunk size secret back in --- job_board_scraper/run_job_scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_board_scraper/run_job_scraper.py b/job_board_scraper/run_job_scraper.py index 1501d4c..e431334 100644 --- a/job_board_scraper/run_job_scraper.py +++ b/job_board_scraper/run_job_scraper.py @@ -29,7 +29,7 @@ def run_spider(single_url_chunk, chunk_number): process.start() if __name__ == '__main__': - chunk_size = 16 + chunk_size = int(os.environ.get("CHUNK_SIZE")) connection = psycopg2.connect(host=os.environ.get("PG_HOST"), user=os.environ.get("PG_USER"), password=os.environ.get("PG_PASSWORD"), dbname=os.environ.get("PG_DATABASE")) cursor = connection.cursor()