Skip to content

Commit

Permalink
ruff
Browse files Browse the repository at this point in the history
  • Loading branch information
jxnl committed Oct 5, 2024
1 parent e2a144f commit 5e6ba05
Showing 1 changed file with 9 additions and 8 deletions.
17 changes: 9 additions & 8 deletions make_sitemap.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import os
import asyncio
import yaml
from typing import Generator, Tuple, Dict, Optional, List
from typing import Optional
from collections.abc import Generator
from openai import AsyncOpenAI
import typer
from rich.console import Console
Expand All @@ -15,7 +16,7 @@

def traverse_docs(
root_dir: str = "docs",
) -> Generator[Tuple[str, str, str], None, None]:
) -> Generator[tuple[str, str, str], None, None]:
"""
Recursively traverse the docs folder and yield the path, content, and content hash of each file.
Expand All @@ -31,7 +32,7 @@ def traverse_docs(
file_path = os.path.join(root, file)
relative_path = os.path.relpath(file_path, root_dir)

with open(file_path, "r", encoding="utf-8") as f:
with open(file_path, encoding="utf-8") as f:
content = f.read()

content_hash = hashlib.md5(content.encode()).hexdigest()
Expand Down Expand Up @@ -101,16 +102,16 @@ async def generate_sitemap(
client = AsyncOpenAI(api_key=api_key)

# Load existing sitemap if it exists
existing_sitemap: Dict[str, Dict[str, str]] = {}
existing_sitemap: dict[str, dict[str, str]] = {}
if os.path.exists(output_file):
with open(output_file, "r", encoding="utf-8") as sitemap_file:
with open(output_file, encoding="utf-8") as sitemap_file:
existing_sitemap = yaml.safe_load(sitemap_file) or {}

sitemap_data: Dict[str, Dict[str, str]] = {}
sitemap_data: dict[str, dict[str, str]] = {}

async def process_file(
path: str, content: str, content_hash: str
) -> Tuple[str, Dict[str, str]]:
) -> tuple[str, dict[str, str]]:
if (
path in existing_sitemap
and existing_sitemap[path].get("hash") == content_hash
Expand All @@ -125,7 +126,7 @@ async def process_file(
)
return path, {"summary": "Failed to generate summary", "hash": content_hash}

files_to_process: List[Tuple[str, str, str]] = list(traverse_docs(root_dir))
files_to_process: list[tuple[str, str, str]] = list(traverse_docs(root_dir))
total_files = len(files_to_process)

with Progress() as progress:
Expand Down

0 comments on commit 5e6ba05

Please sign in to comment.