JFetcher v3.4.0

功能变动：

- **使用 Gotify 发送工作流运行失败通知**
- **使用 PostgreSQL 存储日志**
- **不再支持 Python 3.8，并适配最新版 sshared**
- 项目管理工具切换到 uv
- 使用 uv 进行镜像构建
- 调整部署文件，增加 PostgreSQL 和 Gotify 对应的 Docker 网络
- 调整部署配置
- 优化 requirements.txt 文件生成
- 更新依赖库

错误修复：

- 修复简书积分兑换平台单侧无挂单时采集报错的问题
- 修复配置文件映射异常的问题

.dockerignore

@@ -1,8 +1,7 @@
 **/.git/
 **/.github/
+**/.venv/
 **/.ruff_cache/
 **/__pycache__/
-**/pyproject.toml
-**/poetry.lock
-**/requirements-dev.txt
-**/config.yaml
+**/config.example.toml
+**/config.toml

.github/workflows/static-check.yaml

@@ -7,16 +7,14 @@ jobs:
   static-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - name: Install Poetry
-        run: pipx install poetry
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.x"
-          cache: "poetry"
-      - name: Install Dependencies
-        run: poetry install --all-extras
-      - name: Lint With Ruff
-        run: poetry run ruff check --output-format=github .
-      - name: Type checking with Pyright
-        run: poetry run pyright --warnings .
+    - uses: actions/checkout@v4
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+    - name: Install Python
+      run: uv python install
+    - name: Install Dependencies
+      run: uv sync --all-extras
+    - name: Lint With Ruff
+      run: uv run ruff check --output-format=github .
+    - name: Type checking with Pyright
+      run: uv run pyright --warnings .

Dockerfile

@@ -4,12 +4,13 @@ ENV TZ Asia/Shanghai
 
 WORKDIR /app
 
-COPY requirements.txt .
-RUN pip install uv --no-cache-dir --disable-pip-version-check && \
-    uv pip install --system --no-cache -r requirements.txt && \
-    cd /usr/local/bin && \
-    rm uv uvx
+COPY pyproject.toml uv.lock .
+
+RUN --mount=from=ghcr.io/astral-sh/uv:0.4.0,source=/uv,target=/bin/uv \
+    uv sync --frozen --no-dev --no-cache
 
 COPY . .
 
+ENV PATH="/app/.venv/bin:$PATH"
+
 CMD ["python", "main.py"]

config.example.toml

@@ -1,26 +1,21 @@
-# MongoDB 数据库
-[mongodb]
-    # 地址
+[mongo]
     host = "localhost"
-    # 端口
     port = 27017
-    # 数据库名
     database = "jfetcher"
 
-# 日志
+[postgres]
+    host = "localhost"
+    port = 5432
+    user = "postgres"
+    password = "postgres"
+    database = "jfetcher"
+
 [logging]
-    # 是否保存日志
-    enable_save = true
-    # 日志展示等级
     display_level = "DEBUG"
-    # 日志保存等级
     save_level = "DEBUG"
 
-# 飞书消息推送
-[feishu_notification]
-    # 是否启用消息推送
+[notify]
     enabled = true
-    # 消息推送 Webhook URL
-    webhook_url = ""
-    # 任务运行失败消息卡片 ID
-    failure_card_id = ""
+    host = "localhost"
+    port = 8701
+    token = ""

docker-compose.yml

@@ -1,26 +1,32 @@
 networks:
+  gotify:
+    external: true
   mongodb:
     external: true
+  postgres:
+    external: true
   prefect:
     external: true
 
 services:
   main:
-    image: jfetcher:3.3.0
+    image: jfetcher:3.4.0
     container_name: jfetcher
     build: .
     volumes:
-      - ./config.yaml:/app/config.yaml:ro
+      - ./config.toml:/app/config.toml:ro
     networks:
+      - gotify
       - mongodb
+      - postgres
       - prefect
     environment:
       - PYTHONUNBUFFERED=1
       - PREFECT_API_URL=http://prefect:4200/api
     deploy:
       resources:
         limits:
-          memory: 512M
+          memory: 768M
       restart_policy:
         condition: on-failure
         delay: 5s

jobs/__init__.py

@@ -1,5 +1,6 @@
+from collections.abc import Coroutine
 from importlib import import_module
-from typing import Any, Coroutine, Set, Tuple
+from typing import Any
 
 from prefect.deployments.runner import RunnerDeployment
 
@@ -21,7 +22,7 @@ def import_deployment(path: str) -> DeploymentType:
     return getattr(module, deployment_obj_name)
 
 
-DEPLOYMENT_PATHS: Set[str] = {
+DEPLOYMENT_PATHS: set[str] = {
     "jobs.jianshu.article_earning_ranking:deployment",
     "jobs.jianshu.assets_ranking:deployment",
     "jobs.jianshu.daily_update_ranking:deployment",
@@ -31,6 +32,6 @@ def import_deployment(path: str) -> DeploymentType:
 }
 
 
-DEPLOYMENTS: Tuple[DeploymentType, ...] = tuple(
+DEPLOYMENTS: tuple[DeploymentType, ...] = tuple(
     import_deployment(x) for x in DEPLOYMENT_PATHS
 )

jobs/jianshu/article_earning_ranking.py

@@ -1,5 +1,5 @@
 from datetime import datetime, timedelta
-from typing import List, Optional, Tuple
+from typing import Optional
 
 from jkit.config import CONFIG
 from jkit.exceptions import ResourceUnavailableError
@@ -30,7 +30,7 @@
 @retry(attempts=5, delay=10)
 async def get_author_slug_and_info(
     item: RecordField,
-) -> Tuple[Optional[str], Optional[UserInfo]]:
+) -> tuple[Optional[str], Optional[UserInfo]]:
     flow_run_name = get_flow_run_name()
 
     if not item.slug:
@@ -98,7 +98,7 @@ async def main() -> None:
 
     date = get_today_as_datetime() - timedelta(days=1)
 
-    data: List[ArticleEarningRankingRecordDocument] = []
+    data: list[ArticleEarningRankingRecordDocument] = []
     async for item in ArticleEarningRanking(date.date()):
         processed_item = await process_item(item, date=date)
         data.append(processed_item)

jobs/jianshu/assets_ranking.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import List, Optional, Tuple
+from typing import Optional
 
 from jkit.config import CONFIG
 from jkit.exceptions import ResourceUnavailableError
@@ -29,7 +29,7 @@
 @retry(attempts=5, delay=10)
 async def get_fp_ftn_amount(
     item: AssetsRankingRecord, /
-) -> Tuple[Optional[float], Optional[float]]:
+) -> tuple[Optional[float], Optional[float]]:
     flow_run_name = get_flow_run_name()
 
     if not item.user_info.slug:
@@ -94,7 +94,7 @@ async def main() -> None:
 
     date = get_today_as_datetime()
 
-    data: List[AssetsRankingRecordDocument] = []
+    data: list[AssetsRankingRecordDocument] = []
     async for item in AssetsRanking():
         processed_item = await process_item(item, date=date)
         data.append(processed_item)

jobs/jianshu/daily_update_ranking.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import List
 
 from jkit.ranking.daily_update import (
     DailyUpdateRanking,
@@ -46,7 +45,7 @@ async def main() -> None:
 
     date = get_today_as_datetime()
 
-    data: List[DailyUpdateRankingRecordDocument] = []
+    data: list[DailyUpdateRankingRecordDocument] = []
     async for item in DailyUpdateRanking():
         processed_item = await process_item(item, date=date)
         data.append(processed_item)

jobs/jianshu/lottery.py

@@ -1,5 +1,3 @@
-from typing import List
-
 from jkit.lottery import Lottery, LotteryWinRecord
 from prefect import flow
 
@@ -44,7 +42,7 @@ async def main() -> None:
     if stop_id == 0:
         logger.warn("数据库中没有记录", flow_run_name=flow_run_name)
 
-    data: List[LotteryWinRecordDocument] = []
+    data: list[LotteryWinRecordDocument] = []
     async for item in Lottery().iter_win_records():
         if item.id == stop_id:
             break

jobs/jianshu/lp_recommend.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import List, Optional
+from typing import Optional
 
 from jkit.collection import Collection, CollectionArticleInfo
 from prefect import flow
@@ -72,7 +72,7 @@ async def main() -> None:
 
     date = get_today_as_datetime()
 
-    data: List[LPRecommendedArticleRecordDocument] = []
+    data: list[LPRecommendedArticleRecordDocument] = []
     itered_items_count = 0
     async for item in LP_RECOMMENDED_COLLECTION.iter_articles():
         processed_item = await process_item(item, date=date)

jobs/jpep/ftn_trade.py

@@ -1,5 +1,5 @@
 from datetime import datetime, timedelta
-from typing import List, Literal
+from typing import Literal
 
 from jkit.jpep.ftn_macket import FTNMacket, FTNMacketOrderRecord
 from prefect import flow
@@ -83,12 +83,15 @@ async def main(type: Literal["buy", "sell"]) -> None:  # noqa: A002
 
     fetch_time = get_fetch_time()
 
-    data: List[FTNTradeOrderDocument] = []
+    data: list[FTNTradeOrderDocument] = []
     async for item in FTNMacket().iter_orders(type=type):
         processed_item = await process_item(item, time=fetch_time, type=type)
         data.append(processed_item)
 
-    await FTNTradeOrderDocument.insert_many(data)
+    if data:
+        await FTNTradeOrderDocument.insert_many(data)
+    else:
+        logger.warn("没有可采集的挂单信息，跳过数据写入")
 
     log_flow_run_success(logger, data_count=len(data))
 

models/__init__.py

@@ -1,5 +1,3 @@
-from typing import Tuple, Type
-
 from sshared.mongo import Document
 
 from .jianshu.article_earning_ranking_record import ArticleEarningRankingRecordDocument
@@ -12,7 +10,7 @@
 from .jpep.ftn_trade_order import FTNTradeOrderDocument
 from .jpep.user import UserDocument as JPEPUserDocument
 
-MODELS: Tuple[Type[Document], ...] = (
+MODELS: tuple[type[Document], ...] = (
     ArticleEarningRankingRecordDocument,
     AssetsRankingRecordDocument,
     DailyUpdateRankingRecordDocument,

models/jianshu/user.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from enum import Enum
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
 from jkit.msgspec_constraints import PositiveInt, UserName, UserSlug, UserUploadedUrl
 from sshared.mongo import Document, Index
@@ -19,7 +19,7 @@ class UserDocument(Document, frozen=True):
     updated_at: datetime
     id: Optional[PositiveInt]
     name: Optional[UserName]
-    history_names: List[UserName]
+    history_names: list[UserName]
     avatar_url: Optional[UserUploadedUrl]
 
     class Meta:  # type: ignore
@@ -66,7 +66,7 @@ async def insert_or_update_one(
         if updated_at < db_data.updated_at:
             return
 
-        data_to_set: Dict[str, Any] = {
+        data_to_set: dict[str, Any] = {
             # 刷新数据更新时间
             "updatedAt": updated_at
         }

models/jpep/user.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Any, Dict, Optional
+from typing import Any, Optional
 
 from jkit.msgspec_constraints import NonNegativeInt, PositiveInt
 from sshared.mongo import Document, Field, Index
@@ -61,7 +61,7 @@ async def insert_or_update_one(
         if updated_at < db_data.updated_at:
             return
 
-        update_data: Dict[str, Any] = {
+        update_data: dict[str, Any] = {
             "$set": {
                 # 即使没有要更新的数据，也要刷新更新时间
                 "updatedAt": updated_at,