Accessing meta sometimes causes memory leaks, apparently

kelvinn · Oct 28, 2023 · 00e7d5e · 00e7d5e
1 parent 1eca46f
commit 00e7d5e
Show file tree

Hide file tree

Showing 5 changed files with 5 additions and 5 deletions.
diff --git a/price_monitor/spiders/backcountry.py b/price_monitor/spiders/backcountry.py
@@ -32,7 +32,7 @@ class BackcountrySpider(CrawlSpider):
     ]
 
     def parse_detail_page(self, response):
-        item = response.meta.get('item', {})
+        item = {} # response.meta.get('item', {})
         item['url'] = response.url
         brand = response.css(BRAND_SELECTOR).extract_first("").strip()
         item['title'] = f"""{brand} {response.css(TITLE_SELECTOR).extract_first("").strip()}"""

diff --git a/price_monitor/spiders/montbell.py b/price_monitor/spiders/montbell.py
@@ -26,7 +26,7 @@ class MontbellSpider(CrawlSpider):
     ]
 
     def parse_detail_page(self, response):
-        item = response.meta.get('item', {})
+        item = {} # response.meta.get('item', {})
         item['url'] = response.url
         item['title'] = response.css(TITLE_SELECTOR).extract_first("").strip()
         item['price'] = self.get_price(response)

diff --git a/price_monitor/spiders/patagonia.py b/price_monitor/spiders/patagonia.py
@@ -38,7 +38,7 @@ def parse_detail_page(self, response):
         @scrapes url title price
         """
         self.logger.info('Parse Detail Page function called on %s', response.url)
-        item = response.meta.get('item', {})
+        item = {} # response.meta.get('item', {})
         item['url'] = response.url
         item['title'] = response.css(TITLE_SELECTOR).extract_first("").strip()
         item['price'] = self.get_price(response)

diff --git a/price_monitor/spiders/rei.py b/price_monitor/spiders/rei.py
@@ -35,7 +35,7 @@ def parse_detail_page(self, response):
         try:
             product = json.loads(response.xpath('//script[@type="application/ld+json"]//text()').extract_first())
 
-            item = response.meta.get('item', {})
+            item = {} # response.meta.get('item', {})
             item['url'] = response.url
             item['title'] = product.get('name')
             item['price'] = self.get_price(product) or 0

diff --git a/price_monitor/spiders/trekkinn.py b/price_monitor/spiders/trekkinn.py
@@ -27,7 +27,7 @@ class TrekkinnSpider(CrawlSpider):
     ]
 
     def parse_detail_page(self, response):
-        item = response.meta.get('item', {})
+        item = {} # response.meta.get('item', {})
         item['url'] = response.url
         item['title'] = response.css(TITLE_SELECTOR).extract_first("").strip()
         item['price'] = self.get_price(response)