Skip to content
This repository has been archived by the owner on Jan 25, 2024. It is now read-only.

Commit

Permalink
♻️ chore(deps): Update deps, but also improve scrap class (#129)
Browse files Browse the repository at this point in the history
- Quit driver after remove driver file
- Add execution rights on temporary driver file

  • Updating exceptiongroup (1.1.1 -> 1.1.3)
  • Updating trio (0.22.0 -> 0.22.2)
  • Updating caio (0.9.12 -> 0.9.13)
  • Updating certifi (2023.5.7 -> 2023.7.22)
  • Updating charset-normalizer (3.1.0 -> 3.2.0)
  • Updating typing-extensions (4.6.3 -> 4.7.1)
  • Updating urllib3 (1.26.16 -> 2.0.4)
  • Updating aiofile (3.8.7 -> 3.8.8)
  • Updating click (8.1.3 -> 8.1.7)
  • Updating pathspec (0.11.1 -> 0.11.2)
  • Updating platformdirs (3.6.0 -> 3.10.0)
  • Updating pluggy (1.1.0 -> 1.3.0)
  • Updating pydantic (1.10.9 -> 1.10.12)
  • Updating selenium (4.9.0 -> 4.11.2)
  • Updating black (23.3.0 -> 23.7.0)
  • Updating pillow (9.5.0 -> 10.0.0)
  • Updating undetected-chromedriver (3.5.0 -> 3.5.3)
  • Loading branch information
vvatelot authored Sep 1, 2023
1 parent 2a611c0 commit 8e8bc01
Show file tree
Hide file tree
Showing 3 changed files with 271 additions and 299 deletions.
11 changes: 6 additions & 5 deletions ecoindex_scraper/scrap.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import asyncio
from datetime import datetime
from genericpath import exists
from os import remove
from os import chmod, remove
from shutil import copyfile
from time import sleep
from typing import Dict, Tuple
Expand All @@ -10,6 +9,7 @@
import undetected_chromedriver as uc
from ecoindex.ecoindex import get_ecoindex
from ecoindex.models import PageMetrics, PageType, Result, ScreenShot, WindowSize
from genericpath import exists
from pydantic.networks import HttpUrl
from selenium.common.exceptions import JavascriptException, NoSuchElementException

Expand Down Expand Up @@ -53,16 +53,17 @@ def __init__(
if driver_executable_path:
self.driver_executable_path = f"/tmp/chromedriver_{uuid4()}"
copyfile(driver_executable_path, self.driver_executable_path)
chmod(self.driver_executable_path, 0o755)
else:
self.driver_executable_path = None

def __del__(self):
if hasattr(self, "driver"):
self.driver.quit()

if self.driver_executable_path and exists(self.driver_executable_path):
remove(self.driver_executable_path)

if hasattr(self, "driver"):
self.driver.quit()

def _handle_network_response_received(self, eventdata):
if eventdata["params"]["response"]["url"].startswith("http"):
self.all_requests[eventdata["params"]["requestId"]] = {
Expand Down
Loading

0 comments on commit 8e8bc01

Please sign in to comment.