Merge branch 'develop' into feature/cached_headers

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.8.0
+    rev: 24.10.0
     hooks:
       - id: black
   - repo: https://github.com/codespell-project/codespell

CHANGELOG.md

@@ -4,6 +4,12 @@ If there is no "Upgrading" header for that version, no post-upgrade actions need
 
 
 ## Upcoming
+### Improvements
+- Logins that fail due to 'Too Many Requests' or 'Internal Server Error' errors will now be retried once
+([#311](https://github.com/jdholtz/auto-southwest-check-in/pull/311) by [@dmytrokoren](https://github.com/dmytorkoren))
+
+
+## 8.1 (2024-11-03)
 ### New Features
 - Fare drops can now be checked for all flights on the same day or all nonstop flights on the same day
 ([#303](https://github.com/jdholtz/auto-southwest-check-in/pulls/303))
@@ -15,6 +21,15 @@ If there is no "Upgrading" header for that version, no post-upgrade actions need
 ### Improvements
 - Potentially speed up the check-in process
     - Check-ins now start exactly 24 hours before a flight (instead of 24 hours and 5 seconds)
+- Improve detection evasion in the Docker image by using a virtual display
+([#307](https://github.com/jdholtz/auto-southwest-check-in/pull/307) by [@dmytrokoren](https://github.com/dmytrokoren))
+
+### Bug Fixes
+- Fix headers not being retrieved, causing a webdriver timeout
+([#314](https://github.com/jdholtz/auto-southwest-check-in/pulls/314) by [@sephamorr](https://github.com/sephamorr))
+
+### Upgrading
+- Upgrade the dependencies to the latest versions by running `pip install -r requirements.txt`
 
 
 ## 8.0 (2024-08-17)

Dockerfile

@@ -6,12 +6,13 @@ WORKDIR /app
 # this Docker image already downloads a compatible chromedriver
 ENV AUTO_SOUTHWEST_CHECK_IN_DOCKER=1
 
-RUN apk add --update --no-cache chromium chromium-chromedriver
+RUN apk add --update --no-cache chromium chromium-chromedriver xvfb xauth
 
 RUN adduser -D auto-southwest-check-in -h /app
+RUN chown -R auto-southwest-check-in:auto-southwest-check-in /app
 USER auto-southwest-check-in
 
-COPY requirements.txt requirements.txt
+COPY requirements.txt ./
 RUN pip3 install --upgrade pip && pip3 install --no-cache-dir -r requirements.txt
 
 COPY . .

lib/checkin_scheduler.py

@@ -146,18 +146,20 @@ def _remove_old_flights(self, flights: List[Flight]) -> None:
 
         # Copy the list because it can potentially change inside the loop
         for flight in self.flights[:]:
-            if flight not in flights:
-                flight_idx = self.flights.index(flight)
-                flight_time = flight.get_display_time(twenty_four_hr_time)
-                print(
-                    f"Flight from {flight.departure_airport} to {flight.destination_airport} on "
-                    f"{flight_time} is no longer scheduled. Stopping its check-in\n"
-                )  # Don't log as it has sensitive information
-
-                self.checkin_handlers[flight_idx].stop_check_in()
-
-                self.checkin_handlers.pop(flight_idx)
-                self.flights.pop(flight_idx)
+            if flight in flights:
+                continue
+
+            flight_idx = self.flights.index(flight)
+            flight_time = flight.get_display_time(twenty_four_hr_time)
+            print(
+                f"Flight from {flight.departure_airport} to {flight.destination_airport} on "
+                f"{flight_time} is no longer scheduled. Stopping its check-in\n"
+            )  # Don't log as it has sensitive information
+
+            self.checkin_handlers[flight_idx].stop_check_in()
+
+            self.checkin_handlers.pop(flight_idx)
+            self.flights.pop(flight_idx)
 
         logger.debug(
             "Successfully removed old flights. %d flights are now scheduled", len(self.flights)

lib/reservation_monitor.py

@@ -20,6 +20,9 @@
 from .webdriver import WebDriver
 
 TOO_MANY_REQUESTS_CODE = 429
+INTERNAL_SERVER_ERROR_CODE = 500
+
+RETRY_WAIT_SECONDS = 20
 
 logger = get_logger(__name__)
 
@@ -194,42 +197,62 @@ def _check(self) -> bool:
         # this scope
         return False
 
-    def _get_reservations(self) -> Tuple[List[Dict[str, Any]], bool]:
+    def _get_reservations(self, max_retries: int = 1) -> Tuple[List[Dict[str, Any]], bool]:
         """
-        Returns a list of reservations and a boolean indicating if reservation
-        scheduling should be skipped.
+        Attempts to retrieve a list of reservations and returns a tuple containing the list
+        of reservations and a boolean indicating whether reservation scheduling should be skipped.
 
-        Reservation scheduling will be skipped if a Too Many Requests error or timeout occurs
-        because new headers might not be valid and a list of reservations could not be retrieved.
+        The method will retry fetching reservations once in case of a timeout
+        or a Too Many Requests error. If the retry fails, reservation scheduling will be
+        skipped until the next scheduled attempt.
         """
-        logger.debug("Retrieving reservations for account")
-        webdriver = WebDriver(self.checkin_scheduler)
+        logger.debug("Retrieving reservations for account (max retries: %d)", max_retries)
 
-        try:
-            reservations = webdriver.get_reservations(self)
-        except DriverTimeoutError:
-            logger.debug(
-                "Timeout while retrieving reservations during login. Skipping reservation retrieval"
-            )
-            self.notification_handler.timeout_during_retrieval("account")
-            return [], True
-        except LoginError as err:
-            if err.status_code == TOO_MANY_REQUESTS_CODE:
-                # Don't exit when a Too Many Requests error happens. Instead, just skip the
-                # retrieval until the next time.
+        for attempt in range(max_retries + 1):
+            webdriver = WebDriver(self.checkin_scheduler)
+
+            try:
+                reservations = webdriver.get_reservations(self)
                 logger.debug(
-                    "Encountered a Too Many Requests error while logging in. Skipping reservation "
-                    "retrieval"
+                    "Successfully retrieved %d reservations after %d attempts",
+                    len(reservations),
+                    attempt + 1,
                 )
-                self.notification_handler.too_many_requests_during_login()
-                return [], True
-
-            logger.debug("Error logging in. %s. Exiting", err)
-            self.notification_handler.failed_login(err)
-            sys.exit(1)
-
-        logger.debug("Successfully retrieved %d reservations", len(reservations))
-        return reservations, False
+                return reservations, False
+
+            except DriverTimeoutError:
+                if attempt < max_retries:
+                    logger.debug("Timeout while retrieving reservations during login. Retrying")
+                    logger.debug("Waiting for %d seconds before retrying", RETRY_WAIT_SECONDS)
+                    time.sleep(RETRY_WAIT_SECONDS)
+                else:
+                    logger.debug(
+                        "Timeout persisted after %d retries. Skipping reservation retrieval",
+                        max_retries,
+                    )
+                    self.notification_handler.timeout_during_retrieval("account")
+
+            except LoginError as err:
+                if err.status_code in [TOO_MANY_REQUESTS_CODE, INTERNAL_SERVER_ERROR_CODE]:
+                    if attempt < max_retries:
+                        logger.debug(
+                            "Encountered an error (status: %d) while logging in. Retrying",
+                            err.status_code,
+                        )
+                        logger.debug("Waiting for %d seconds before retrying", RETRY_WAIT_SECONDS)
+                        time.sleep(RETRY_WAIT_SECONDS)
+                    else:
+                        logger.debug(
+                            "Error (status: %d) persists. Skipping reservation retrieval",
+                            err.status_code,
+                        )
+                        self.notification_handler.too_many_requests_during_login()
+                else:
+                    logger.debug("Error logging in. %s. Exiting", err)
+                    self.notification_handler.failed_login(err)
+                    sys.exit(1)
+
+        return [], True
 
     def _stop_monitoring(self) -> None:
         print(f"\nStopping monitoring for account with username {self.username}")

lib/webdriver.py

@@ -8,6 +8,7 @@
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
+from sbvirtualdisplay import Display
 from seleniumbase import Driver
 from seleniumbase.fixtures import page_actions as seleniumbase_actions
 
@@ -21,7 +22,7 @@
 BASE_URL = "https://mobile.southwest.com"
 LOGIN_URL = BASE_URL + "/api/security/v4/security/token"
 TRIPS_URL = BASE_URL + "/api/mobile-misc/v1/mobile-misc/page/upcoming-trips"
-HEADERS_URL = BASE_URL + "/api/chase/v2/chase/offers"
+HEADERS_URL = BASE_URL + "/api/mobile-air-booking/v1/mobile-air-booking/feature/shopping-details"
 
 # Southwest's code when logging in with the incorrect information
 INVALID_CREDENTIALS_CODE = 400518024
@@ -53,6 +54,7 @@ def __init__(self, checkin_scheduler: CheckInScheduler) -> None:
         self.checkin_scheduler = checkin_scheduler
         self.headers_set = False
         self.debug_screenshots = self._should_take_screenshots()
+        self.display = None
         self.headers_listener_enabled = True
         self.cached_headers_path = os.path.join(os.getcwd(), "cached_headers.json")
         self.cache_duration = timedelta(minutes=30)  # Duration to keep headers cached
@@ -93,7 +95,7 @@ def set_headers(self) -> None:
         self._wait_for_attribute("headers_set")
         self._take_debug_screenshot(driver, "post_headers.png")
 
-        driver.quit()
+        self._quit_driver(driver)
 
     def get_reservations(self, account_monitor: AccountMonitor) -> List[JSON]:
         """
@@ -126,26 +128,31 @@ def get_reservations(self, account_monitor: AccountMonitor) -> List[JSON]:
         # instead of requesting again later
         reservations = self._fetch_reservations(driver)
 
-        driver.quit()
+        self._quit_driver(driver)
         return reservations
 
     def _get_driver(self) -> Driver:
         logger.debug("Starting webdriver for current session")
         browser_path = self.checkin_scheduler.reservation_monitor.config.browser_path
 
+        # This environment variable is set in the Docker image
+        is_docker = os.environ.get("AUTO_SOUTHWEST_CHECK_IN_DOCKER") == "1"
+
         driver_version = "mlatest"
-        if os.environ.get("AUTO_SOUTHWEST_CHECK_IN_DOCKER") == "1":
-            # This environment variable is set in the Docker image. Makes sure a new driver
-            # is not downloaded as the Docker image already has the correct driver
+        if is_docker:
+            self._start_display()
+            # Make sure a new driver is not downloaded as the Docker image
+            # already has the correct driver
             driver_version = "keep"
 
         driver = Driver(
             binary_location=browser_path,
             driver_version=driver_version,
-            headless=True,
+            headed=is_docker,
+            headless=not is_docker,
             uc_cdp_events=True,
             undetectable=True,
-            is_mobile=True,
+            incognito=True,
         )
         logger.debug("Using browser version: %s", driver.caps["browserVersion"])
 
@@ -235,7 +242,7 @@ def _wait_for_login(self, driver: Driver, account_monitor: AccountMonitor) -> No
         # Handle login errors
         if self.login_status_code != 200:
             self._cached_headers_manager(login_failed=True)
-            driver.quit()
+            self._quit_driver(driver)
             error = self._handle_login_error(login_response)
             raise error
 
@@ -341,3 +348,24 @@ def _set_account_name(self, account_monitor: AccountMonitor, response: JSON) ->
             f"Successfully logged in to {account_monitor.first_name} "
             f"{account_monitor.last_name}'s account\n"
         )  # Don't log as it contains sensitive information
+
+    def _quit_driver(self, driver: Driver) -> None:
+        driver.quit()
+        self._stop_display()
+
+    def _start_display(self) -> None:
+        try:
+            self.display = Display(size=(1440, 1880), backend="xvfb")
+            self.display.start()
+
+            if self.display.is_alive():
+                logger.debug("Started virtual display successfully")
+            else:
+                logger.debug("Started virtual display but is not active")
+        except Exception as e:
+            logger.debug("Failed to start display: %s", e)
+
+    def _stop_display(self) -> None:
+        if self.display is not None:
+            self.display.stop()
+            logger.debug("Stopped virtual display successfully")

requirements.txt

@@ -1,5 +1,5 @@
 apprise==1.9.0
 ntplib==0.4.0
 pytz==2024.1  # Remove when this script only supports Python 3.9+
-requests==2.31.0
-seleniumbase==4.31.2
+requests==2.32.3
+seleniumbase==4.32.6

southwest.py

@@ -4,7 +4,7 @@
 import sys
 from typing import List
 
-__version__ = "v8.0"
+__version__ = "v8.1"
 
 __doc__ = """
 Schedule a check-in:

tests/unit/test_reservation_monitor.py

@@ -254,6 +254,7 @@ def test_check_skips_scheduling_if_an_error_occurs(self, mocker: MockerFixture)
     def test_get_reservations_skips_retrieval_on_driver_timeout(
         self, mocker: MockerFixture
     ) -> None:
+        mocker.patch("time.sleep")
         mocker.patch.object(WebDriver, "get_reservations", side_effect=DriverTimeoutError)
         mock_timeout_notif = mocker.patch.object(NotificationHandler, "timeout_during_retrieval")
 
@@ -266,6 +267,7 @@ def test_get_reservations_skips_retrieval_on_driver_timeout(
     def test_get_reservations_skips_retrieval_on_too_many_requests_error(
         self, mocker: MockerFixture
     ) -> None:
+        mocker.patch("time.sleep")
         mocker.patch.object(
             WebDriver, "get_reservations", side_effect=LoginError("", TOO_MANY_REQUESTS_CODE)
         )

tests/unit/test_webdriver.py

@@ -84,15 +84,22 @@ def test_get_driver_returns_a_webdriver_with_one_request(self, mock_chrome: mock
 
         assert mock_chrome.call_args.kwargs.get("driver_version") == "mlatest"
 
-    def test_get_driver_keeps_driver_version_in_docker(self, mock_chrome: mock.Mock) -> None:
+    def test_get_driver_keeps_is_correctly_configured_in_docker(
+        self, mocker: MockerFixture, mock_chrome: mock.Mock
+    ) -> None:
+        """The driver version should be kept and the virtual display should be started"""
+
         # This env variable will be set in the Docker image
         os.environ["AUTO_SOUTHWEST_CHECK_IN_DOCKER"] = "1"
 
+        mock_start_display = mocker.patch.object(self.driver, "_start_display")
+
         driver = self.driver._get_driver()
         driver.add_cdp_listener.assert_called_once()
         driver.open.assert_called_once()
 
         assert mock_chrome.call_args.kwargs.get("driver_version") == "keep"
+        mock_start_display.assert_called_once()
 
     def test_headers_listener_sets_headers_when_correct_url(self, mocker: MockerFixture) -> None:
         mocker.patch.object(self.driver, "_get_needed_headers", return_value={"test": "headers"})
@@ -272,3 +279,43 @@ def test_set_account_name_sets_the_correct_values_for_the_name(
 
         assert mock_account_monitor.first_name == "John"
         assert mock_account_monitor.last_name == "Doe"
+
+    def test_quit_driver_cleans_up_webdriver(
+        self, mocker: MockerFixture, mock_chrome: mock.Mock
+    ) -> None:
+        mock_stop_display = mocker.patch.object(self.driver, "_stop_display")
+        self.driver._quit_driver(mock_chrome)
+
+        mock_chrome.quit.assert_called_once()
+        mock_stop_display.assert_called_once()
+
+    # Make sure start_display handles the virtual display both being alive and not
+    @pytest.mark.parametrize("is_alive", [True, False])
+    def test_start_display_starts_virtual_display(
+        self, mocker: MockerFixture, is_alive: bool
+    ) -> None:
+        mock_display = mocker.patch("lib.webdriver.Display")
+        mock_display.return_value.is_alive.return_value = is_alive
+
+        self.driver._start_display()
+        mock_display.assert_called_once()
+
+    def test_start_display_ignores_error_when_display_fails_to_start(
+        self, mocker: MockerFixture
+    ) -> None:
+        mocker.patch("lib.webdriver.Display", side_effect=Exception)
+        self.driver._start_display()
+
+    def test_stop_display_stops_virtual_display(self, mocker: MockerFixture) -> None:
+        mock_display = mocker.patch("lib.webdriver.Display")
+        self.driver.display = mock_display
+
+        self.driver._stop_display()
+        mock_display.stop.assert_called_once()
+
+    def test_stop_display_ignores_if_display_is_not_set(self, mocker: MockerFixture) -> None:
+        mock_display = mocker.patch("lib.webdriver.Display")
+        self.driver.display = None
+
+        self.driver._stop_display()
+        mock_display.stop.assert_not_called()