Skip to content

Commit

Permalink
JFetcher v3.3.0
Browse files Browse the repository at this point in the history 
功能变动：

- 恢复采集简书积分兑换平台交易数据
- 使用 UV 构建 Docker 镜像
- 升级到 sspeedup v0.9.0
  • Loading branch information
@FHU-yezi
FHU-yezi committed Jul 19, 2024
2 parents 378cc13 + 1c1fbe6 commit 691fc75
Show file tree
Hide file tree
Showing 14 changed files with 292 additions and 290 deletions.
9 changes: 4 additions & 5 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@ ENV TZ Asia/Shanghai
WORKDIR /app

COPY requirements.txt .
RUN pip install \
-r requirements.txt \
--no-cache-dir \
--no-compile \
--disable-pip-version-check
RUN pip install uv --no-cache-dir --disable-pip-version-check && \
uv pip install --system --no-cache -r requirements.txt && \
cd /usr/local/bin && \
rm uv uvx

COPY . .

Expand Down
1 change: 0 additions & 1 deletion constants.py
View file

This file was deleted.

2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ networks:

services:
main:
image: jfetcher:3.2.0
image: jfetcher:3.3.0
container_name: jfetcher
build: .
volumes:
Expand Down
2 changes: 2 additions & 0 deletions jobs/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ def import_deployment(path: str) -> DeploymentType:
"jobs.jianshu.assets_ranking:deployment",
"jobs.jianshu.daily_update_ranking:deployment",
"jobs.jianshu.lp_recommend:deployment",
"jobs.jpep.ftn_trade:buy_deployment",
"jobs.jpep.ftn_trade:sell_deployment",
}


Expand Down
2 changes: 1 addition & 1 deletion jobs/jianshu/article_earning_ranking.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ async def process_item(
to_author=item.fp_to_author_anount,
to_voter=item.fp_to_voter_amount,
),
).validate()
)


@flow(
Expand Down
2 changes: 1 addition & 1 deletion jobs/jianshu/assets_ranking.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ async def process_item(
assets=item.assets_amount,
),
user_slug=item.user_info.slug,
).validate()
)


@flow(
Expand Down
2 changes: 1 addition & 1 deletion jobs/jianshu/daily_update_ranking.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ async def process_item(
ranking=item.ranking,
days=item.days,
user_slug=item.user_info.slug,
).validate()
)


@flow(
Expand Down
2 changes: 1 addition & 1 deletion jobs/jianshu/lottery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ async def process_item(item: LotteryWinRecord) -> LotteryWinRecordDocument:
time=item.time,
award_name=item.award_name,
user_slug=item.user_info.slug,
).validate()
)


@flow(
Expand Down
2 changes: 1 addition & 1 deletion jobs/jianshu/lp_recommend.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ async def process_item(
can_comment=item.can_comment,
description=item.description,
author_slug=item.author_info.slug,
).validate()
)


@flow(
Expand Down
2 changes: 1 addition & 1 deletion jobs/jpep/ftn_trade.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ async def process_item(
minimum_trade=item.minimum_trade_amount,
),
publisher_id=item.publisher_info.id,
).validate()
)


@flow(
Expand Down
497 changes: 250 additions & 247 deletions poetry.lock
View file

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ package-mode = false
python = "^3.8"
prefect = "^2.19.0"
jkit = "^3.0.0a16"
sshared = { version = "^0.8.0", extras = ["mongo"] }
tomli = "^2.0.1"
sshared = { version = "^0.9.0", extras = ["config", "mongo"] }

[tool.poetry.group.dev.dependencies]
ruff = "^0.5.0"
Expand Down
30 changes: 15 additions & 15 deletions requirements-dev.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,23 @@ async-timeout==4.0.3 ; python_version >= "3.8" and python_version < "3.12.0"
asyncpg==0.29.0 ; python_version >= "3.8" and python_version < "4.0"
attrs==23.2.0 ; python_version >= "3.8" and python_version < "4.0"
backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9"
cachetools==5.3.3 ; python_version >= "3.8" and python_version < "4.0"
certifi==2024.6.2 ; python_version >= "3.8" and python_version < "4.0"
cachetools==5.4.0 ; python_version >= "3.8" and python_version < "4.0"
certifi==2024.7.4 ; python_version >= "3.8" and python_version < "4.0"
cffi==1.16.0 ; python_version >= "3.8" and python_version < "4.0" and platform_python_implementation != "PyPy"
charset-normalizer==3.3.2 ; python_version >= "3.8" and python_version < "4.0"
click==8.1.7 ; python_version >= "3.8" and python_version < "4.0"
cloudpickle==3.0.0 ; python_version >= "3.8" and python_version < "4.0"
colorama==0.4.6 ; python_version >= "3.8" and python_version < "4.0"
coolname==2.2.0 ; python_version >= "3.8" and python_version < "4.0"
croniter==2.0.5 ; python_version >= "3.8" and python_version < "4.0"
croniter==2.0.7 ; python_version >= "3.8" and python_version < "4.0"
cryptography==42.0.8 ; python_version >= "3.8" and python_version < "4.0"
dateparser==1.2.0 ; python_version >= "3.8" and python_version < "4.0"
dnspython==2.6.1 ; python_version >= "3.8" and python_version < "4.0"
docker==7.1.0 ; python_version >= "3.8" and python_version < "4.0"
email-validator==2.2.0 ; python_version >= "3.8" and python_version < "4.0"
exceptiongroup==1.2.1 ; python_version >= "3.8" and python_version < "3.11"
exceptiongroup==1.2.2 ; python_version >= "3.8" and python_version < "3.11"
fsspec==2024.6.1 ; python_version >= "3.8" and python_version < "4.0"
google-auth==2.31.0 ; python_version >= "3.8" and python_version < "4.0"
google-auth==2.32.0 ; python_version >= "3.8" and python_version < "4.0"
graphviz==0.20.3 ; python_version >= "3.8" and python_version < "4.0"
greenlet==3.0.3 ; python_version >= "3.8" and python_version < "4.0"
griffe==0.47.0 ; python_version >= "3.8" and python_version < "4.0"
Expand All @@ -34,7 +34,7 @@ h2==4.1.0 ; python_version >= "3.8" and python_version < "4.0"
hpack==4.0.0 ; python_version >= "3.8" and python_version < "4.0"
httpcore==1.0.5 ; python_version >= "3.8" and python_version < "4.0"
httpx[http2]==0.27.0 ; python_version >= "3.8" and python_version < "4.0"
humanize==4.9.0 ; python_version >= "3.8" and python_version < "4.0"
humanize==4.10.0 ; python_version >= "3.8" and python_version < "4.0"
hyperframe==6.0.1 ; python_version >= "3.8" and python_version < "4.0"
idna==3.7 ; python_version >= "3.8" and python_version < "4.0"
importlib-metadata==8.0.0 ; python_version >= "3.8" and python_version < "3.10"
Expand All @@ -46,14 +46,14 @@ jkit==3.0.0a16 ; python_version >= "3.8" and python_version < "4.0"
jsonpatch==1.33 ; python_version >= "3.8" and python_version < "4.0"
jsonpointer==3.0.0 ; python_version >= "3.8" and python_version < "4.0"
jsonschema-specifications==2023.12.1 ; python_version >= "3.8" and python_version < "4.0"
jsonschema==4.22.0 ; python_version >= "3.8" and python_version < "4.0"
jsonschema==4.23.0 ; python_version >= "3.8" and python_version < "4.0"
kubernetes==29.0.0 ; python_version >= "3.8" and python_version < "4.0"
mako==1.3.5 ; python_version >= "3.8" and python_version < "4.0"
markdown-it-py==3.0.0 ; python_version >= "3.8" and python_version < "4.0"
markdown==3.6 ; python_version >= "3.8" and python_version < "4.0"
markupsafe==2.1.5 ; python_version >= "3.8" and python_version < "4.0"
mdurl==0.1.2 ; python_version >= "3.8" and python_version < "4.0"
motor==3.5.0 ; python_version >= "3.8" and python_version < "4.0"
motor==3.5.1 ; python_version >= "3.8" and python_version < "4.0"
msgspec==0.18.6 ; python_version >= "3.8" and python_version < "4.0"
nodeenv==1.9.1 ; python_version >= "3.8" and python_version < "4.0"
oauthlib==3.2.2 ; python_version >= "3.8" and python_version < "4.0"
Expand All @@ -63,15 +63,15 @@ pathspec==0.12.1 ; python_version >= "3.8" and python_version < "4.0"
pendulum==2.1.2 ; python_version >= "3.8" and python_version < "3.12"
pendulum==3.0.0 ; python_version >= "3.12" and python_version < "4.0"
pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9"
prefect==2.19.7 ; python_version >= "3.8" and python_version < "4.0"
prefect==2.19.8 ; python_version >= "3.8" and python_version < "4.0"
pyasn1-modules==0.4.0 ; python_version >= "3.8" and python_version < "4.0"
pyasn1==0.6.0 ; python_version >= "3.8" and python_version < "4.0"
pycparser==2.22 ; python_version >= "3.8" and python_version < "4.0" and platform_python_implementation != "PyPy"
pydantic-core==2.20.0 ; python_version >= "3.8" and python_version < "4.0"
pydantic[email]==2.8.0 ; python_version >= "3.8" and python_version < "4.0"
pydantic-core==2.20.1 ; python_version >= "3.8" and python_version < "4.0"
pydantic[email]==2.8.2 ; python_version >= "3.8" and python_version < "4.0"
pygments==2.18.0 ; python_version >= "3.8" and python_version < "4.0"
pymongo==4.8.0 ; python_version >= "3.8" and python_version < "4.0"
pyright==1.1.370 ; python_version >= "3.8" and python_version < "4.0"
pyright==1.1.372 ; python_version >= "3.8" and python_version < "4.0"
python-dateutil==2.9.0.post0 ; python_version >= "3.8" and python_version < "4.0"
python-multipart==0.0.9 ; python_version >= "3.8" and python_version < "4.0"
python-slugify==8.0.4 ; python_version >= "3.8" and python_version < "4.0"
Expand All @@ -86,17 +86,17 @@ requests-oauthlib==2.0.0 ; python_version >= "3.8" and python_version < "4.0"
requests==2.32.3 ; python_version >= "3.8" and python_version < "4.0"
rfc3339-validator==0.1.4 ; python_version >= "3.8" and python_version < "4.0"
rich==13.7.1 ; python_version >= "3.8" and python_version < "4.0"
rpds-py==0.18.1 ; python_version >= "3.8" and python_version < "4.0"
rpds-py==0.19.0 ; python_version >= "3.8" and python_version < "4.0"
rsa==4.9 ; python_version >= "3.8" and python_version < "4"
ruamel-yaml-clib==0.2.8 ; platform_python_implementation == "CPython" and python_version < "3.13" and python_version >= "3.8"
ruamel-yaml==0.18.6 ; python_version >= "3.8" and python_version < "4.0"
ruff==0.5.0 ; python_version >= "3.8" and python_version < "4.0"
ruff==0.5.3 ; python_version >= "3.8" and python_version < "4.0"
shellingham==1.5.4 ; python_version >= "3.8" and python_version < "4.0"
six==1.16.0 ; python_version >= "3.8" and python_version < "4.0"
sniffio==1.3.1 ; python_version >= "3.8" and python_version < "4.0"
sqlalchemy==2.0.31 ; python_version >= "3.8" and python_version < "4.0"
sqlalchemy[asyncio]==2.0.31 ; python_version >= "3.8" and python_version < "4.0"
sshared[mongo]==0.8.0 ; python_version >= "3.8" and python_version < "4.0"
sshared[config,mongo]==0.9.0 ; python_version >= "3.8" and python_version < "4.0"
text-unidecode==1.3 ; python_version >= "3.8" and python_version < "4.0"
toml==0.10.2 ; python_version >= "3.8" and python_version < "4.0"
tomli==2.0.1 ; python_version >= "3.8" and python_version < "4.0"
Expand Down
26 changes: 13 additions & 13 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,23 @@ async-timeout==4.0.3 ; python_version >= "3.8" and python_version < "3.12.0"
asyncpg==0.29.0 ; python_version >= "3.8" and python_version < "4.0"
attrs==23.2.0 ; python_version >= "3.8" and python_version < "4.0"
backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9"
cachetools==5.3.3 ; python_version >= "3.8" and python_version < "4.0"
certifi==2024.6.2 ; python_version >= "3.8" and python_version < "4.0"
cachetools==5.4.0 ; python_version >= "3.8" and python_version < "4.0"
certifi==2024.7.4 ; python_version >= "3.8" and python_version < "4.0"
cffi==1.16.0 ; python_version >= "3.8" and python_version < "4.0" and platform_python_implementation != "PyPy"
charset-normalizer==3.3.2 ; python_version >= "3.8" and python_version < "4.0"
click==8.1.7 ; python_version >= "3.8" and python_version < "4.0"
cloudpickle==3.0.0 ; python_version >= "3.8" and python_version < "4.0"
colorama==0.4.6 ; python_version >= "3.8" and python_version < "4.0"
coolname==2.2.0 ; python_version >= "3.8" and python_version < "4.0"
croniter==2.0.5 ; python_version >= "3.8" and python_version < "4.0"
croniter==2.0.7 ; python_version >= "3.8" and python_version < "4.0"
cryptography==42.0.8 ; python_version >= "3.8" and python_version < "4.0"
dateparser==1.2.0 ; python_version >= "3.8" and python_version < "4.0"
dnspython==2.6.1 ; python_version >= "3.8" and python_version < "4.0"
docker==7.1.0 ; python_version >= "3.8" and python_version < "4.0"
email-validator==2.2.0 ; python_version >= "3.8" and python_version < "4.0"
exceptiongroup==1.2.1 ; python_version >= "3.8" and python_version < "3.11"
exceptiongroup==1.2.2 ; python_version >= "3.8" and python_version < "3.11"
fsspec==2024.6.1 ; python_version >= "3.8" and python_version < "4.0"
google-auth==2.31.0 ; python_version >= "3.8" and python_version < "4.0"
google-auth==2.32.0 ; python_version >= "3.8" and python_version < "4.0"
graphviz==0.20.3 ; python_version >= "3.8" and python_version < "4.0"
greenlet==3.0.3 ; python_version >= "3.8" and python_version < "4.0"
griffe==0.47.0 ; python_version >= "3.8" and python_version < "4.0"
Expand All @@ -34,7 +34,7 @@ h2==4.1.0 ; python_version >= "3.8" and python_version < "4.0"
hpack==4.0.0 ; python_version >= "3.8" and python_version < "4.0"
httpcore==1.0.5 ; python_version >= "3.8" and python_version < "4.0"
httpx[http2]==0.27.0 ; python_version >= "3.8" and python_version < "4.0"
humanize==4.9.0 ; python_version >= "3.8" and python_version < "4.0"
humanize==4.10.0 ; python_version >= "3.8" and python_version < "4.0"
hyperframe==6.0.1 ; python_version >= "3.8" and python_version < "4.0"
idna==3.7 ; python_version >= "3.8" and python_version < "4.0"
importlib-metadata==8.0.0 ; python_version >= "3.8" and python_version < "3.10"
Expand All @@ -46,14 +46,14 @@ jkit==3.0.0a16 ; python_version >= "3.8" and python_version < "4.0"
jsonpatch==1.33 ; python_version >= "3.8" and python_version < "4.0"
jsonpointer==3.0.0 ; python_version >= "3.8" and python_version < "4.0"
jsonschema-specifications==2023.12.1 ; python_version >= "3.8" and python_version < "4.0"
jsonschema==4.22.0 ; python_version >= "3.8" and python_version < "4.0"
jsonschema==4.23.0 ; python_version >= "3.8" and python_version < "4.0"
kubernetes==29.0.0 ; python_version >= "3.8" and python_version < "4.0"
mako==1.3.5 ; python_version >= "3.8" and python_version < "4.0"
markdown-it-py==3.0.0 ; python_version >= "3.8" and python_version < "4.0"
markdown==3.6 ; python_version >= "3.8" and python_version < "4.0"
markupsafe==2.1.5 ; python_version >= "3.8" and python_version < "4.0"
mdurl==0.1.2 ; python_version >= "3.8" and python_version < "4.0"
motor==3.5.0 ; python_version >= "3.8" and python_version < "4.0"
motor==3.5.1 ; python_version >= "3.8" and python_version < "4.0"
msgspec==0.18.6 ; python_version >= "3.8" and python_version < "4.0"
oauthlib==3.2.2 ; python_version >= "3.8" and python_version < "4.0"
orjson==3.10.6 ; python_version >= "3.8" and python_version < "4.0"
Expand All @@ -62,12 +62,12 @@ pathspec==0.12.1 ; python_version >= "3.8" and python_version < "4.0"
pendulum==2.1.2 ; python_version >= "3.8" and python_version < "3.12"
pendulum==3.0.0 ; python_version >= "3.12" and python_version < "4.0"
pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9"
prefect==2.19.7 ; python_version >= "3.8" and python_version < "4.0"
prefect==2.19.8 ; python_version >= "3.8" and python_version < "4.0"
pyasn1-modules==0.4.0 ; python_version >= "3.8" and python_version < "4.0"
pyasn1==0.6.0 ; python_version >= "3.8" and python_version < "4.0"
pycparser==2.22 ; python_version >= "3.8" and python_version < "4.0" and platform_python_implementation != "PyPy"
pydantic-core==2.20.0 ; python_version >= "3.8" and python_version < "4.0"
pydantic[email]==2.8.0 ; python_version >= "3.8" and python_version < "4.0"
pydantic-core==2.20.1 ; python_version >= "3.8" and python_version < "4.0"
pydantic[email]==2.8.2 ; python_version >= "3.8" and python_version < "4.0"
pygments==2.18.0 ; python_version >= "3.8" and python_version < "4.0"
pymongo==4.8.0 ; python_version >= "3.8" and python_version < "4.0"
python-dateutil==2.9.0.post0 ; python_version >= "3.8" and python_version < "4.0"
Expand All @@ -84,7 +84,7 @@ requests-oauthlib==2.0.0 ; python_version >= "3.8" and python_version < "4.0"
requests==2.32.3 ; python_version >= "3.8" and python_version < "4.0"
rfc3339-validator==0.1.4 ; python_version >= "3.8" and python_version < "4.0"
rich==13.7.1 ; python_version >= "3.8" and python_version < "4.0"
rpds-py==0.18.1 ; python_version >= "3.8" and python_version < "4.0"
rpds-py==0.19.0 ; python_version >= "3.8" and python_version < "4.0"
rsa==4.9 ; python_version >= "3.8" and python_version < "4"
ruamel-yaml-clib==0.2.8 ; platform_python_implementation == "CPython" and python_version < "3.13" and python_version >= "3.8"
ruamel-yaml==0.18.6 ; python_version >= "3.8" and python_version < "4.0"
Expand All @@ -93,7 +93,7 @@ six==1.16.0 ; python_version >= "3.8" and python_version < "4.0"
sniffio==1.3.1 ; python_version >= "3.8" and python_version < "4.0"
sqlalchemy==2.0.31 ; python_version >= "3.8" and python_version < "4.0"
sqlalchemy[asyncio]==2.0.31 ; python_version >= "3.8" and python_version < "4.0"
sshared[mongo]==0.8.0 ; python_version >= "3.8" and python_version < "4.0"
sshared[config,mongo]==0.9.0 ; python_version >= "3.8" and python_version < "4.0"
text-unidecode==1.3 ; python_version >= "3.8" and python_version < "4.0"
toml==0.10.2 ; python_version >= "3.8" and python_version < "4.0"
tomli==2.0.1 ; python_version >= "3.8" and python_version < "4.0"
Expand Down

0 comments on commit 691fc75

Please sign in to comment.