Skip to content

Commit

Permalink
Handle 404
Browse files Browse the repository at this point in the history
  • Loading branch information
KarimPwnz committed Jun 29, 2023
1 parent 38dbf75 commit 2e39edc
Showing 1 changed file with 8 additions and 0 deletions.
8 changes: 8 additions & 0 deletions scripts/github_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
from utils import create_client, setup_logging


class NotFoundException(Exception):
pass

class GitHubDownloader:
def __init__(self, client, output_path):
self._client = client
Expand All @@ -18,6 +21,8 @@ def _url_to_path(url):

async def download(self, url):
async with self._client.get(url) as resp:
if resp.status == 404:
raise NotFoundException
return await resp.text()

async def save(self, url, content):
Expand All @@ -29,6 +34,9 @@ async def download_and_save(self, url):
logging.info("Downloading %s", url)
try:
content = await self.download(url)
except NotFoundException:
logging.warning("File not found: %s", url)
return
except Exception as e:
logging.exception("Exception occurred while downloading: %s", str(e))
return
Expand Down

0 comments on commit 2e39edc

Please sign in to comment.