refactor: 重构数据库初始化与迁移逻辑

FHU-yezi · Nov 11, 2024 · cbb06bf · cbb06bf
1 parent 3b8da07
commit cbb06bf
Show file tree

Hide file tree

Showing 22 changed files with 150 additions and 202 deletions.
diff --git a/README.md b/README.md
@@ -8,19 +8,21 @@
 
 ## 数据库准备
 
-创建用户：
+进入 `sql` 目录：
 
-```sql
-CREATE ROLE jfetcher LOGIN PASSWORD 'jfetcher';
+```shell
+cd sql
 ```
 
-创建数据库：
+如果您需要修改数据库用户名和密码，请修改 `sql` 目录下的 `0.sql` 和每个子目录下的 `0.sql` 文件。
 
-```sql
-CREATE DATABASE jianshu WITH OWNER = jfetcher;
-CREATE DATABASE jpep WITH OWNER = jfetcher;
-CREATE DATABASE logs;
-```
+您需要一个具有创建用户和数据库权限的用户（一般是超级用户）来完成数据库准备。
+
+每个目录中的 SQL 脚本均应按照编号顺序执行。
+
+首先，执行 `sql` 目录下的脚本。
+
+依次切换到与 `sql` 下的子目录（数据库目录）名称相同的数据库中，先执行每个表目录中的 SQL 脚本，再执行数据库目录下的 SQL 脚本。
 
 ## 配置
 
@@ -37,7 +39,7 @@ cp config.example.toml config.toml
 - logging.host 填写 `postgres`
 - notify.host 填写 `gotify`
 
-同时，您需要填写正确的 `postgres.user` 和 `postgres.password`。
+同时，您需要填写正确的 `{db_name}_postgres.user` 和 `{db_name}_postgres.password`。
 
 ## 使用 Docker 部署
 

diff --git a/models/__init__.py b/models/__init__.py
@@ -1,25 +1,6 @@
-from models.jianshu.article_earning_ranking_record import (
-    ArticleEarningRankingRecord,
-)
-from models.jianshu.daily_update_ranking_record import DailyUpdateRankingRecord
-from models.jianshu.user import User as JianshuUser
-from models.jianshu.user_assets_ranking_record import UserAssetsRankingRecord
-from models.jpep.credit_record import CreditRecord
-from models.jpep.ftn_macket_record import FTNMacketRecord
-from models.jpep.ftn_order import FTNOrder
-from models.jpep.user import User as JPEPUser
 from utils.db import jianshu_pool, jpep_pool
 
 
 async def init_db() -> None:
     await jianshu_pool.prepare()
     await jpep_pool.prepare()
-
-    await ArticleEarningRankingRecord.init()
-    await DailyUpdateRankingRecord.init()
-    await JianshuUser.init()
-    await UserAssetsRankingRecord.init()
-    await CreditRecord.init()
-    await FTNMacketRecord.init()
-    await FTNOrder.init()
-    await JPEPUser.init()
diff --git a/models/jianshu/article_earning_ranking_record.py b/models/jianshu/article_earning_ranking_record.py
@@ -17,24 +17,6 @@ class ArticleEarningRankingRecord(Table, frozen=True):
     author_earning: PositiveFloat
     voter_earning: PositiveFloat
 
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jianshu_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS article_earning_ranking_records (
-                    date DATE NOT NULL,
-                    ranking SMALLINT NOT NULL,
-                    slug VARCHAR(12),
-                    title TEXT,
-                    author_slug VARCHAR(12),
-                    author_earning NUMERIC NOT NULL,
-                    voter_earning NUMERIC NOT NULL,
-                    CONSTRAINT pk_article_earning_ranking_records_date_ranking PRIMARY KEY (date, ranking)
-                );
-                """  # noqa: E501
-            )
-
     @classmethod
     async def insert_many(cls, data: list["ArticleEarningRankingRecord"]) -> None:
         for item in data:

diff --git a/models/jianshu/daily_update_ranking_record.py b/models/jianshu/daily_update_ranking_record.py
@@ -12,21 +12,6 @@ class DailyUpdateRankingRecord(Table, frozen=True):
     slug: NonEmptyStr
     days: PositiveInt
 
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jianshu_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS daily_update_ranking_records (
-                    date DATE NOT NULL,
-                    ranking SMALLINT NOT NULL,
-                    slug VARCHAR(12),
-                    days SMALLINT,
-                    CONSTRAINT pk_daily_update_ranking_records_date_slug PRIMARY KEY (date, slug)
-                );
-                """  # noqa: E501
-            )
-
     @classmethod
     async def insert_many(cls, data: list["DailyUpdateRankingRecord"]) -> None:
         for item in data:

diff --git a/models/jianshu/user.py b/models/jianshu/user.py
@@ -2,7 +2,7 @@
 from enum import Enum
 from typing import Optional
 
-from sshared.postgres import Table, create_enum
+from sshared.postgres import Table
 from sshared.strict_struct import NonEmptyStr, PositiveInt
 
 from utils.db import jianshu_pool
@@ -22,30 +22,6 @@ class User(Table, frozen=True):
     history_names: list[NonEmptyStr]
     avatar_url: Optional[NonEmptyStr]
 
-    @classmethod
-    async def _create_enum(cls) -> None:
-        async with jianshu_pool.get_conn() as conn:
-            await create_enum(
-                conn=conn, name="enum_users_status", enum_class=StatusEnum
-            )
-
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jianshu_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS users (
-                    slug VARCHAR(12) CONSTRAINT pk_users_slug PRIMARY KEY,
-                    status enum_users_status NOT NULL,
-                    update_time TIMESTAMP NOT NULL,
-                    id INTEGER,
-                    name VARCHAR(15),
-                    history_names VARCHAR(15)[] NOT NULL,
-                    avatar_url TEXT
-                );
-                """
-            )
-
     async def create(self) -> None:
         self.validate()
 
@@ -118,7 +94,7 @@ async def upsert(
             async with conn.transaction():
                 # 更新更新时间
                 await conn.execute(
-                    "UPDATE users SET update_time = %s WHERE slug = %s",
+                    "UPDATE users SET update_time = %s WHERE slug = %s;",
                     (datetime.now(), slug),
                 )
 
@@ -129,21 +105,21 @@ async def upsert(
                 # 如果没有存储 ID，进行添加
                 if not user.id and id:
                     await conn.execute(
-                        "UPDATE users SET id = %s WHERE slug = %s",
+                        "UPDATE users SET id = %s WHERE slug = %s;",
                         (id, slug),
                     )
 
                 # 如果没有存储昵称，进行添加
                 if not user.name and name:
                     await conn.execute(
-                        "UPDATE users SET name = %s WHERE slug = %s",
+                        "UPDATE users SET name = %s WHERE slug = %s;",
                         (name, slug),
                     )
 
                 # 更新昵称
                 if user.name and name and user.name != name:
                     await conn.execute(
-                        "UPDATE users SET name = %s WHERE slug = %s",
+                        "UPDATE users SET name = %s WHERE slug = %s;",
                         (name, slug),
                     )
                     await conn.execute(
@@ -155,13 +131,13 @@ async def upsert(
                 # 如果没有存储头像链接，进行添加
                 if not user.avatar_url and avatar_url:
                     await conn.execute(
-                        "UPDATE users SET avatar_url = %s WHERE slug = %s",
+                        "UPDATE users SET avatar_url = %s WHERE slug = %s;",
                         (avatar_url, slug),
                     )
 
                 # 更新头像链接
                 if user.avatar_url and avatar_url and user.avatar_url != avatar_url:
                     await conn.execute(
-                        "UPDATE users SET avatar_url = %s WHERE slug = %s",
+                        "UPDATE users SET avatar_url = %s WHERE slug = %s;",
                         (avatar_url, slug),
                     )
diff --git a/models/jianshu/user_assets_ranking_record.py b/models/jianshu/user_assets_ranking_record.py
@@ -16,23 +16,6 @@ class UserAssetsRankingRecord(Table, frozen=True):
     ftn: Optional[NonNegativeFloat]
     assets: Optional[NonNegativeFloat]
 
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jianshu_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS user_assets_ranking_records (
-                    date DATE NOT NULL,
-                    ranking SMALLINT NOT NULL,
-                    slug VARCHAR(12),
-                    fp NUMERIC,
-                    ftn NUMERIC,
-                    assets NUMERIC,
-                    CONSTRAINT pk_user_assets_ranking_records_date_ranking PRIMARY KEY (date, ranking)
-                );
-                """  # noqa: E501
-            )
-
     @classmethod
     async def insert_many(cls, data: list["UserAssetsRankingRecord"]) -> None:
         for item in data:

diff --git a/models/jpep/credit_record.py b/models/jpep/credit_record.py
@@ -15,20 +15,6 @@ class CreditRecord(Table, frozen=True):
     user_id: PositiveInt
     credit: NonNegativeInt
 
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jpep_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS credit_records (
-                    time TIMESTAMP NOT NULL,
-                    user_id INTEGER NOT NULL,
-                    credit INTEGER NOT NULL,
-                    CONSTRAINT pk_credit_records_time_user_id PRIMARY KEY (time, user_id)
-                );
-                """  # noqa: E501
-            )
-
     async def create(self) -> None:
         self.validate()
 

diff --git a/models/jpep/ftn_macket_record.py b/models/jpep/ftn_macket_record.py
@@ -16,40 +16,6 @@ class FTNMacketRecord(Table, frozen=True):
     remaining_amount: int
     minimum_trade_amount: PositiveInt
 
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jpep_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS ftn_macket_records (
-                    fetch_time TIMESTAMP NOT NULL,
-                    id INTEGER NOT NULL,
-                    price NUMERIC NOT NULL,
-                    traded_count SMALLINT NOT NULL,
-                    total_amount INTEGER NOT NULL,
-                    traded_amount INTEGER NOT NULL,
-                    remaining_amount INTEGER NOT NULL,
-                    minimum_trade_amount INTEGER NOT NULL,
-                    CONSTRAINT pk_ftn_macket_records_fetch_time_id PRIMARY KEY (fetch_time, id)
-                ) PARTITION BY RANGE (fetch_time);
-                """  # noqa: E501
-            )
-            await conn.execute(
-                "CREATE TABLE IF NOT EXISTS ftn_macket_records_2023 PARTITION "
-                "OF ftn_macket_records FOR VALUES FROM ('2023-01-01 00:00:00') "
-                "TO ('2023-12-31 23:59:59');"
-            )
-            await conn.execute(
-                "CREATE TABLE IF NOT EXISTS ftn_macket_records_2024 PARTITION "
-                "OF ftn_macket_records FOR VALUES FROM ('2024-01-01 00:00:00') "
-                "TO ('2024-12-31 23:59:59');"
-            )
-            await conn.execute(
-                "CREATE TABLE IF NOT EXISTS ftn_macket_records_2025 PARTITION "
-                "OF ftn_macket_records FOR VALUES FROM ('2025-01-01 00:00:00') "
-                "TO ('2025-12-31 23:59:59');"
-            )
-
     @classmethod
     async def insert_many(cls, data: list["FTNMacketRecord"]) -> None:
         for item in data:

diff --git a/models/jpep/ftn_order.py b/models/jpep/ftn_order.py
@@ -2,7 +2,7 @@
 from enum import Enum
 from typing import Optional
 
-from sshared.postgres import Table, create_enum
+from sshared.postgres import Table
 from sshared.strict_struct import PositiveInt
 
 from utils.db import jpep_pool
@@ -20,28 +20,6 @@ class FTNOrder(Table, frozen=True):
     publish_time: datetime
     last_seen_time: Optional[datetime]
 
-    @classmethod
-    async def _create_enum(cls) -> None:
-        async with jpep_pool.get_conn() as conn:
-            await create_enum(
-                conn=conn, name="enum_ftn_orders_type", enum_class=TypeEnum
-            )
-
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jpep_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS ftn_orders (
-                    id INTEGER CONSTRAINT pk_ftn_orders_id PRIMARY KEY,
-                    type enum_ftn_orders_type NOT NULL,
-                    publisher_id INTEGER NOT NULL,
-                    publish_time TIMESTAMP NOT NULL,
-                    last_seen_time TIMESTAMP
-                );
-                """
-            )
-
     async def create(self) -> None:
         async with jpep_pool.get_conn() as conn:
             await conn.execute(

diff --git a/models/jpep/user.py b/models/jpep/user.py
@@ -17,21 +17,6 @@ class User(Table, frozen=True):
     hashed_name: NonEmptyStr
     avatar_url: Optional[NonEmptyStr]
 
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jpep_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS users (
-                    id INTEGER NOT NULL CONSTRAINT pk_users_id PRIMARY KEY,
-                    update_time TIMESTAMP NOT NULL,
-                    name TEXT NOT NULL,
-                    hashed_name VARCHAR(9) NOT NULL,
-                    avatar_url TEXT
-                );
-                """
-            )
-
     async def create(self) -> None:
         self.validate()
 
@@ -53,7 +38,7 @@ async def get_by_id(cls, id: int) -> Optional["User"]:  # noqa: A002
         async with jpep_pool.get_conn() as conn:
             cursor = await conn.execute(
                 "SELECT update_time, name, hashed_name, avatar_url "
-                "FROM users WHERE id = %s",
+                "FROM users WHERE id = %s;",
                 (id,),
             )
 
@@ -98,28 +83,28 @@ async def upsert(
             async with conn.transaction():
                 # 更新更新时间
                 await conn.execute(
-                    "UPDATE users SET update_time = %s WHERE id = %s",
+                    "UPDATE users SET update_time = %s WHERE id = %s;",
                     (datetime.now(), id),
                 )
 
                 # 更新昵称和哈希后昵称
                 if user.name and name and user.name != name:
                     # 哈希后昵称一定会跟随昵称变化，一同更新
                     await conn.execute(
-                        "UPDATE users SET name = %s, hashed_name = %s WHERE id = %s",
+                        "UPDATE users SET name = %s, hashed_name = %s WHERE id = %s;",
                         (name, hashed_name, id),
                     )
 
                 # 如果没有存储头像链接，进行添加
                 if not user.avatar_url and avatar_url:
                     await conn.execute(
-                        "UPDATE users SET avatar_url = %s WHERE id = %s",
+                        "UPDATE users SET avatar_url = %s WHERE id = %s;",
                         (avatar_url, id),
                     )
 
                 # 更新头像链接
                 if user.avatar_url and avatar_url and user.avatar_url != avatar_url:
                     await conn.execute(
-                        "UPDATE users SET avatar_url = %s WHERE id = %s",
+                        "UPDATE users SET avatar_url = %s WHERE id = %s;",
                         (avatar_url, id),
                     )