diff --git a/scrapy_crawler/DBWatchDog/DailyScheduler/spiders/PriceUpdateSpider.py b/scrapy_crawler/DBWatchDog/DailyScheduler/spiders/PriceUpdateSpider.py index cbf8931..1fea611 100644 --- a/scrapy_crawler/DBWatchDog/DailyScheduler/spiders/PriceUpdateSpider.py +++ b/scrapy_crawler/DBWatchDog/DailyScheduler/spiders/PriceUpdateSpider.py @@ -9,7 +9,10 @@ from scrapy_crawler.common.db import Deal, get_engine from scrapy_crawler.common.db.models import Trade from scrapy_crawler.common.slack.SlackBots import ExceptionSlackBot -from scrapy_crawler.common.utils.helpers import init_cloudwatch_logger +from scrapy_crawler.common.utils.helpers import ( + get_local_timestring, + init_cloudwatch_logger, +) class PriceUpdateSpider(scrapy.Spider): @@ -44,6 +47,7 @@ def get_soldout_items(self) -> List[Deal]: self.session.query(Deal) .filter(Deal.sold == true()) .filter(Deal.deleted_at == null()) + .filter(Deal.last_crawled >= get_local_timestring(days=-1)) ) return item.all() diff --git a/scrapy_crawler/common/utils/helpers.py b/scrapy_crawler/common/utils/helpers.py index e56e3a0..43427b3 100644 --- a/scrapy_crawler/common/utils/helpers.py +++ b/scrapy_crawler/common/utils/helpers.py @@ -25,8 +25,8 @@ from scrapy_crawler.DBWatchDog.items import IpadItem, IphoneItem, MacbookItem -def get_local_timestring() -> str: - return (datetime.now()).strftime("%Y-%m-%d %H:%M:%S") +def get_local_timestring(days: int = 0) -> str: + return (datetime.now() + timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S") def get_timestamp() -> int: