diff --git a/README.md b/README.md index 3b37c10..88806d5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Gorm Sharding plugin using SQL parser and replace for splits large tables into s Gorm Sharding 是一个业务污染小，高性能的数据库分表方案。通过 SQL 解析和替换，实现分表逻辑，让查询正确的根据规则执行到分表里面。 -[![](https://mermaid.ink/img/eyJjb2RlIjoiZ3JhcGggVERcbmZpcnN0KFwiU1FMIFF1ZXJ5PGJyPjxicj5cbnNlbGVjdCAqIGZyb20gb3JkZXJzPGJyPlxud2hlcmUgdXNlcl9pZCA9ID8gYW5kIHN0YXR1cyA9ID88YnI-XG5saW1pdCAxMCBvcmRlciBieSBpZCBkZXNjPGJyPlxuYXJncyA9IFsxMDAsIDFdXCIpXG5cbmZpcnN0LS0tPnwgR29ybSBRdWVyeSB8ZGJbXCJHb3JtIERCXCJdXG5cbnN1YmdyYXBoIFwiR29ybVwiXG4gIGRiLS0-Z29ybV9xdWVyeVxuICBnb3JtX3F1ZXJ5W1wiY29ublBvb2wuUXVlcnlDb250ZXh0KHNxbCwgYXJncyk8YnI-XCJdXG5lbmRcblxuc3ViZ3JhcGggXCJkYXRhYmFzZS9zcWwgLSBDb25uXCJcbiAgRXhlY0NvbnRleHRbL1wiRXhlY0NvbnRleHRcIi9dXG4gIFF1ZXJ5Q29udGV4dFsvXCJRdWVyeUNvbnRleHRcIi9dXG4gIFF1ZXJ5Um93Q29udGV4dFsvXCJRdWVyeVJvd0NvbnRleHRcIi9dXG4gIGdvcm1fcXVlcnktLT5Db25uXG4gIENvbm4oW1wiQ29ublwiXSlcbiAgQ29ubi0tPkV4ZWNDb250ZXh0XG4gIENvbm4tLT5RdWVyeUNvbnRleHRcbiAgQ29ubi0tPlF1ZXJ5Um93Q29udGV4dFxuZW5kXG5cbnN1YmdyYXBoIHNoYXJkaW5nIFtcIk15Q29ublBvb2xcIl1cbiAgUXVlcnlDb250ZXh0LS0-cm91dGVyLS0-Zm9ybWF0X3NxbC0tPnBhcnNlLS0-Y2hlY2tfdGFibGVcbiAgcm91dGVyW1tcInJvdXRlcihzcWwsIGFyZ3MpPGJyPjxicj5cIl1dXG4gIGZvcm1hdF9zcWw-XCJGb3JtYXQgc3FsLCBhcmdzIGZvciBnZXQgZnVsbCBTUUw8YnI-PGJyPlxuICAgIHNxbCA9IHNlbGVjdCAqIGZyb20gb3JkZXJzPGJyPlxuICAgIHdoZXJlIHVzZXJfaWQgPSAxMDAgYW5kIHN0YXR1cyA9IDE8YnI-XG4gICAgbGltaXQgMTAgb3JkZXIgYnkgaWQgZGVzY1wiXVxuXG4gIGNoZWNrX3RhYmxle1wiQ2hlY2sgc2hhcmRpbmcgcnVsZXM8YnI-YnkgdGFibGUgbmFtZVwifVxuICBjaGVja190YWJsZS0tPnwgRXhpc3QgfHByb2Nlc3NfYXN0XG4gIGNoZWNrX3RhYmxlXzF7e1wiUmV0dXJuIFJhdyBTUUxcIn19XG4gIG5vdF9tYXRjaF9lcnJvclsvXCJSZXR1cm4gRXJyb3I8YnI-U1FMIHF1ZXJ5IG11c3QgaGFzIHNoYXJkaW5nIGtleVwiXFxdXG5cbiAgcGFyc2VbW1wiUGFyc2VyIFNRTCB0byBnZXQgQVNUPGJyPlxuICA8YnI-XG4gIGFzdCA9IHNxbHBhcnNlci5QYXJzZShzcWwpXCJdXVxuXG4gIGNoZWNrX3RhYmxlLS4tPnwgTm90IGV4aXN0IHxjaGVja190YWJsZV8xXG4gIHByb2Nlc3NfYXN0KChcIlNoYXJkaW5nIHJ1bGVzXCIpKVxuICBnZXRfbmV3X3RhYmxlX25hbWVbW1wiVXNlIHZhbHVlIGluIFdoZXJlVmFsdWUgKDEwMCkgZm9yIGdldCBzaGFyZGluZyB0YWJsZSBpbmRleDxicj5vcmRlcnMgKyAoMTAwICUgMTYpPGJyPlNoYXJkaW5nIFRhYmxlID0gb3JkZXJzXzRcIl1dXG4gIG5ld19zcWx7e1wic2VsZWN0ICogZnJvbSBvcmRlcnNfNDxicj53aGVyZSB1c2VyX2lkID0gMTAwIGFuZCBzdGF0dXMgPSAxPGJyPmxpbWl0IDEwIG9yZGVyIGJ5IGlkIGRlc2NcIn19XG5cbiAgcHJvY2Vzc19hc3QtLi0-fCBOb3QgbWF0Y2ggU2hhcmRpbmdLZXkgfG5vdF9tYXRjaF9lcnJvclxuICBwcm9jZXNzX2FzdC0tPnwgTWF0Y2ggU2hhcmRpbmdLZXkgfG1hdGNoX3NoYXJkaW5nX2tleS0tPnwgR2V0IHRhYmxlIG5hbWUgfGdldF9uZXdfdGFibGVfbmFtZS0tPnwgUmVwbGFjZSBUYWJsZU5hbWUgdG8gZ2V0IG5ldyBTUUwgfG5ld19zcWxcbmVuZFxuXG5cbnN1YmdyYXBoIGRhdGFiYXNlIFtEYXRhYmFzZV1cbiAgcG9zdHNfb3RoZXJbKFwib3JkZXJzXzAsIG9yZGVyc18xIC4uLiBvcmRlcnNfM1wiKV1cbiAgcG9zdHNfNFsob3JkZXJzXzQpXVxuICBwb3N0c19sYXN0WyhcIm9yZGVyc181IC4uLiBvcmRlcnNfMTVcIildXG4gIG90aGVyX3RhYmxlc1soT3RoZXIgbm9uLXNoYXJkaW5nIHRhYmxlczxicj51c2Vycywgc3RvY2tzLCB0b3BpY3MgLi4uKV1cblxuICBuZXdfc3FsLS0-fCBTaGFyZGluZyBRdWVyeSB8IG9yZGVyc180XG4gIGNoZWNrX3RhYmxlXzEtLi0-fCBOb25lIHNoYXJkaW5nIFF1ZXJ5IHxvdGhlcl90YWJsZXNcbmVuZFxuXG5wb3N0c180LS0-cmVzdWx0XG5vdGhlcl90YWJsZXMtLi0-cmVzdWx0XG5yZXN1bHRbL1F1ZXJ5IHJlc3VsdHNcXF0iLCJtZXJtYWlkIjp7InRoZW1lIjoiZGVmYXVsdCJ9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlLCJhdXRvU3luYyI6dHJ1ZSwidXBkYXRlRGlhZ3JhbSI6ZmFsc2V9)](https://mermaid.live/edit#eyJjb2RlIjoiZ3JhcGggVERcbmZpcnN0KFwiU1FMIFF1ZXJ5PGJyPjxicj5cbnNlbGVjdCAqIGZyb20gb3JkZXJzPGJyPlxud2hlcmUgdXNlcl9pZCA9ID8gYW5kIHN0YXR1cyA9ID88YnI-XG5saW1pdCAxMCBvcmRlciBieSBpZCBkZXNjPGJyPlxuYXJncyA9IFsxMDAsIDFdXCIpXG5cbmZpcnN0LS0tPnwgR29ybSBRdWVyeSB8ZGJbXCJHb3JtIERCXCJdXG5cbnN1YmdyYXBoIFwiR29ybVwiXG4gIGRiLS0-Z29ybV9xdWVyeVxuICBnb3JtX3F1ZXJ5W1wiY29ublBvb2wuUXVlcnlDb250ZXh0KHNxbCwgYXJncyk8YnI-XCJdXG5lbmRcblxuc3ViZ3JhcGggXCJkYXRhYmFzZS9zcWwgLSBDb25uXCJcbiAgRXhlY0NvbnRleHRbL1wiRXhlY0NvbnRleHRcIi9dXG4gIFF1ZXJ5Q29udGV4dFsvXCJRdWVyeUNvbnRleHRcIi9dXG4gIFF1ZXJ5Um93Q29udGV4dFsvXCJRdWVyeVJvd0NvbnRleHRcIi9dXG4gIGdvcm1fcXVlcnktLT5Db25uXG4gIENvbm4oW1wiQ29ublwiXSlcbiAgQ29ubi0tPkV4ZWNDb250ZXh0XG4gIENvbm4tLT5RdWVyeUNvbnRleHRcbiAgQ29ubi0tPlF1ZXJ5Um93Q29udGV4dFxuZW5kXG5cbnN1YmdyYXBoIHNoYXJkaW5nIFtcIk15Q29ublBvb2xcIl1cbiAgUXVlcnlDb250ZXh0LS0-cm91dGVyLS0-Zm9ybWF0X3NxbC0tPnBhcnNlLS0-Y2hlY2tfdGFibGVcbiAgcm91dGVyW1tcInJvdXRlcihzcWwsIGFyZ3MpPGJyPjxicj5cIl1dXG4gIGZvcm1hdF9zcWw-XCJGb3JtYXQgc3FsLCBhcmdzIGZvciBnZXQgZnVsbCBTUUw8YnI-PGJyPlxuICAgIHNxbCA9IHNlbGVjdCAqIGZyb20gb3JkZXJzPGJyPlxuICAgIHdoZXJlIHVzZXJfaWQgPSAxMDAgYW5kIHN0YXR1cyA9IDE8YnI-XG4gICAgbGltaXQgMTAgb3JkZXIgYnkgaWQgZGVzY1wiXVxuXG4gIGNoZWNrX3RhYmxle1wiQ2hlY2sgc2hhcmRpbmcgcnVsZXM8YnI-YnkgdGFibGUgbmFtZVwifVxuICBjaGVja190YWJsZS0tPnwgRXhpc3QgfHByb2Nlc3NfYXN0XG4gIGNoZWNrX3RhYmxlXzF7e1wiUmV0dXJuIFJhdyBTUUxcIn19XG4gIG5vdF9tYXRjaF9lcnJvclsvXCJSZXR1cm4gRXJyb3I8YnI-U1FMIHF1ZXJ5IG11c3QgaGFzIHNoYXJkaW5nIGtleVwiXFxdXG5cbiAgcGFyc2VbW1wiUGFyc2VyIFNRTCB0byBnZXQgQVNUPGJyPlxuICA8YnI-XG4gIGFzdCA9IHNxbHBhcnNlci5QYXJzZShzcWwpXCJdXVxuXG4gIGNoZWNrX3RhYmxlLS4tPnwgTm90IGV4aXN0IHxjaGVja190YWJsZV8xXG4gIHByb2Nlc3NfYXN0KChcIlNoYXJkaW5nIHJ1bGVzXCIpKVxuICBnZXRfbmV3X3RhYmxlX25hbWVbW1wiVXNlIHZhbHVlIGluIFdoZXJlVmFsdWUgKDEwMCkgZm9yIGdldCBzaGFyZGluZyB0YWJsZSBpbmRleDxicj5vcmRlcnMgKyAoMTAwICUgMTYpPGJyPlNoYXJkaW5nIFRhYmxlID0gb3JkZXJzXzRcIl1dXG4gIG5ld19zcWx7e1wic2VsZWN0ICogZnJvbSBvcmRlcnNfNDxicj53aGVyZSB1c2VyX2lkID0gMTAwIGFuZCBzdGF0dXMgPSAxPGJyPmxpbWl0IDEwIG9yZGVyIGJ5IGlkIGRlc2NcIn19XG5cbiAgcHJvY2Vzc19hc3QtLi0-fCBOb3QgbWF0Y2ggU2hhcmRpbmdLZXkgfG5vdF9tYXRjaF9lcnJvclxuICBwcm9jZXNzX2FzdC0tPnwgTWF0Y2ggU2hhcmRpbmdLZXkgfG1hdGNoX3NoYXJkaW5nX2tleS0tPnwgR2V0IHRhYmxlIG5hbWUgfGdldF9uZXdfdGFibGVfbmFtZS0tPnwgUmVwbGFjZSBUYWJsZU5hbWUgdG8gZ2V0IG5ldyBTUUwgfG5ld19zcWxcbmVuZFxuXG5cbnN1YmdyYXBoIGRhdGFiYXNlIFtEYXRhYmFzZV1cbiAgcG9zdHNfb3RoZXJbKFwib3JkZXJzXzAsIG9yZGVyc18xIC4uLiBvcmRlcnNfM1wiKV1cbiAgcG9zdHNfNFsob3JkZXJzXzQpXVxuICBwb3N0c19sYXN0WyhcIm9yZGVyc181IC4uLiBvcmRlcnNfMTVcIildXG4gIG90aGVyX3RhYmxlc1soT3RoZXIgbm9uLXNoYXJkaW5nIHRhYmxlczxicj51c2Vycywgc3RvY2tzLCB0b3BpY3MgLi4uKV1cblxuICBuZXdfc3FsLS0-fCBTaGFyZGluZyBRdWVyeSB8IG9yZGVyc180XG4gIGNoZWNrX3RhYmxlXzEtLi0-fCBOb25lIHNoYXJkaW5nIFF1ZXJ5IHxvdGhlcl90YWJsZXNcbmVuZFxuXG5wb3N0c180LS0-cmVzdWx0XG5vdGhlcl90YWJsZXMtLi0-cmVzdWx0XG5yZXN1bHRbL1F1ZXJ5IHJlc3VsdHNcXF0iLCJtZXJtYWlkIjoie1xuICBcInRoZW1lXCI6IFwiZGVmYXVsdFwiXG59IiwidXBkYXRlRWRpdG9yIjpmYWxzZSwiYXV0b1N5bmMiOnRydWUsInVwZGF0ZURpYWdyYW0iOmZhbHNlfQ) +![Example](./docs/query.svg) ## Features diff --git a/docs/query.md b/docs/query.md new file mode 100644 index 0000000..d463e69 --- /dev/null +++ b/docs/query.md @@ -0,0 +1,69 @@ +Use https://mermaid.live for generate SVG. + +```mermaid +graph TD +first("SQL Query

+select * from orders
+where user_id = ? and status = ?
+limit 10 order by id desc
+args = [100, 1]") + +first--->| Gorm Query |db["Gorm DB"] + +subgraph "Gorm" + db-->gorm_query + gorm_query["connPool.QueryContext(sql, args)
"] +end + +subgraph "database/sql - Conn" + ExecContext[/"ExecContext"/] + QueryContext[/"QueryContext"/] + QueryRowContext[/"QueryRowContext"/] + gorm_query-->Conn + Conn(["Conn"]) + Conn-->ExecContext + Conn-->QueryContext + Conn-->QueryRowContext +end + +subgraph sharding ["MyConnPool"] + QueryContext-->router-->format_sql-->parse-->check_table + router[["router(sql, args)

"]] + format_sql>"Format sql, args for get full SQL

+ sql = select * from orders
+ where user_id = 100 and status = 1
+ limit 10 order by id desc"] + + check_table{"Check sharding rules
by table name"} + check_table-->| Exist |process_ast + check_table_1{{"Return Raw SQL"}} + not_match_error[/"Return Error
SQL query must has sharding key"\] + + parse[["Parser SQL to get AST
+
+ ast = sqlparser.Parse(sql)"]] + + check_table-.->| Not exist |check_table_1 + process_ast(("Sharding rules")) + get_new_table_name[["Use value in WhereValue (100) for get sharding table index
orders + (100 % 16)
Sharding Table = orders_4"]] + new_sql{{"select * from orders_4
where user_id = 100 and status = 1
limit 10 order by id desc"}} + + process_ast-.->| Not match ShardingKey |not_match_error + process_ast-->| Match ShardingKey |match_sharding_key-->| Get table name |get_new_table_name-->| Replace TableName to get new SQL |new_sql +end + + +subgraph database [Database] + posts_other[("orders_0, orders_1 ... orders_3")] + posts_4[(orders_4)] + posts_last[("orders_5 ... orders_15")] + other_tables[(Other non-sharding tables
users, stocks, topics ...)] + + new_sql-->| Sharding Query | orders_4 + check_table_1-.->| None sharding Query |other_tables +end + +posts_4-->result +other_tables-.->result +result[/Query results\] +``` diff --git a/docs/query.svg b/docs/query.svg new file mode 100644 index 0000000..cb056db --- /dev/null +++ b/docs/query.svg @@ -0,0 +1,10 @@ +
Database
MyConnPool
database/sql - Conn
Gorm
Gorm Query
Exist
Not exist
Not match ShardingKey
Match ShardingKey
Get table name
Replace TableName to get new SQL
Sharding Query
None sharding Query
orders_0, orders_1 ... orders_3
orders_4
orders_5 ... orders_15
Other non-sharding tables
users, stocks, topics ...
orders_4
Check sharding rules
by table name
Parser SQL to get AST
+
+ ast = sqlparser.Parse(sql)
Format sql, args for get full SQL

+ sql = select * from orders
+ where user_id = 100 and status = 1
+ limit 10 order by id desc
router(sql, args)

Sharding rules
Return Raw SQL
Return Error
SQL query must has sharding key
Use value in WhereValue (100) for get sharding table index
orders + (100 % 16)
Sharding Table = orders_4
select * from orders_4
where user_id = 100 and status = 1
limit 10 order by id desc
match_sharding_key
ExecContext
QueryContext
QueryRowContext
Conn
connPool.QueryContext(sql, args)
Gorm DB
SQL Query

+select * from orders
+where user_id = ? and status = ?
+limit 10 order by id desc
+args = [100, 1]
Query results
 \ No newline at end of file diff --git a/images/example-scenario.drawio b/images/example-scenario.drawio deleted file mode 100644 index b113d78..0000000 --- a/images/example-scenario.drawio +++ /dev/null @@ -1 +0,0 @@ -7Vpdb5swFP01eZnUyAYCyWOXdp32IbXLw7qnyg0OsBlMjfO1Xz8b7AAhiUgbIGpGpBJfXxvjc869dtyeOQ5XdwzF/nfqYtIzgLvqmTc9w4CWYYubtKwziwOVwWOBq5xywyT4i5URKOs8cHFScuSUEh7EZeOURhGe8pINMUaXZbcZJeWnxsjDFcNkikjV+jNwuZ9Zh4aT2z/jwPP1k6E9ympCpJ3VmyQ+cumyYDJve+aYUcqzb+FqjImcPD0vWbtPe2o3A2M44nUauI8/XujT08sX7thfYz55AIuHK93NApG5emM1Wr7WU4BdMSOqSBn3qUcjRG5z60dG55GL5XOAKOU+3yiNhREK42/M+VrBi+acCpPPQ6JqZzTiqhJaqjymhLJ0BOYQyI+wZ2OTA9o7CcqU0Dmb4kNvrsiEmIf5AT9rA5XgOKYh5mwt2jFMEA8W5XEgRTZv46eaXjOG1gWHmAYRTwo930uDcFC6GdiKNEo1cLiF7XH+4ks2Al0qvEpuSvlyDHesV1ClQXLgVcAfJQf7EAxU+Zds1AfQVOWblSJpWlgXCveYBWJGMFO2hDP6BxeeA9KrwvYOKGm+kZIFWrwC9R0BwyZc4VKig/0yp7riKkkRvBYOcBiv8so0dutJzo3mLL2KJtuT9wQTGecN8EGGc0ZDcaPMxUzICSx9zLC4i6Qh/qZdAihAUyMUL5wNMuuqQt8cWcm7pR9wPIlRitlSpLctWgaEFOhhAzSCToU4EY1wlbPq5fQQFphxvDrMnyreqoFVlr5SPljm2crR0cEvZCrtd3qCmJeaUsya+h10qd9d6BypX7usXy2mUi9HaVrr10ahFFn0nMRZZNCdPjPtCoG65ITJ1V2l0R6pNx8BZgCZANWMAJtscvoIYIFqCICthgA4utQQYNUMAU6XIcDqOIXfUSaz9sRHzA0irw1twmcEsdFBdjaNsjg3Yi2I0xq2Kc7BydFXCzMidreSwsmcyL1N86DiAbRAda3ePKhbu63N7qsYcI02MbXfjqm5C9MsMT9l6fY0XcKt7L0nW9dL6u8xQuxK360GCJ2sLy97OzWzNwS7EW0nfTvNih2WxJ6vvJuR/7ks3juT/+Dc5D9sll6j0aixXHLxZHLOjUz6GOfkq80ZSni7q00AhqiT1eYAnNtyE+7/ESmJUVQLVWsXqv1+vwBl1td70qdV46caq1Uk9+8G/yN5cAt4bkjqg/Z2F+n5mVvpxK136LTtjA97jWbW9aKYn/Nnx6/5f0uYt/8A \ No newline at end of file diff --git a/images/example-scenario.svg b/images/example-scenario.svg deleted file mode 100644 index 83ebe74..0000000 --- a/images/example-scenario.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
select * from orders where uid = 100
select * from orders where uid = 100
orders 
100000000 rows 
orders...
Gorm Sharding
Gorm Sharding
slow results
slow results
orders_000 100000 rows 
orders_000 10...
orders_100
100000 rows 
orders_100...
orders_999 100000 rows 
orders_999 10...
fast results
fast results
...
...
...
...Text is not SVG - cannot display \ No newline at end of file