add chunk size secret back in

adgramigna · Oct 15, 2023 · f3bc8b1 · f3bc8b1
1 parent fea4347
commit f3bc8b1
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/job_board_scraper/run_job_scraper.py b/job_board_scraper/run_job_scraper.py
@@ -29,7 +29,7 @@ def run_spider(single_url_chunk, chunk_number):
     process.start()
 
 if __name__ == '__main__':
-    chunk_size = 16
+    chunk_size = int(os.environ.get("CHUNK_SIZE"))
 
     connection = psycopg2.connect(host=os.environ.get("PG_HOST"), user=os.environ.get("PG_USER"), password=os.environ.get("PG_PASSWORD"), dbname=os.environ.get("PG_DATABASE"))
     cursor = connection.cursor()