diff --git a/.gitignore b/.gitignore index c40a91c..1bdef09 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,4 @@ ./**/config/pro.json yarn*.log node_modules -doc logs \ No newline at end of file diff --git a/README.md b/README.md index 5e01313..04a11ab 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,6 @@ If you want to deploy your own elara service for node service. **Redis ^5** and ``` nodejs ^14 -ts-node yarn ``` diff --git a/doc/elara-deploy.pdf b/doc/elara-deploy.pdf new file mode 100644 index 0000000..4e09b7d Binary files /dev/null and b/doc/elara-deploy.pdf differ diff --git a/doc/elara-design.md b/doc/elara-design.md new file mode 100644 index 0000000..91d793f --- /dev/null +++ b/doc/elara-design.md @@ -0,0 +1,438 @@ +#### Elara v2.0 + +Elara v2.0在原先的业务逻辑上用typescript进行重构升级。重构目标： + +``` +1. 提高项目的可维护性 +2. 提高项目的稳定性 +3. 提高项目可用性，水平扩展 +4. 解决Elara1.0目前已知的问题 +5. 优化资源管理及服务能力 +``` + +## 项目架构 + +按服务模块分包，每个包相当于一个微服务。 + +##### 包结构 + +```js +pakages +|___job // 定时任务,收集elara项目统计数据 +|___api // elara官网api服务 +|___suducer // wsrpc的缓存服务，针对最新区块数据 +|___lib // 公共库 +|___script // 相应脚本 +|___wsrpc // elara核心服务，rpc请求，ws订阅管理 + +``` + +##### 服务结构 + +```js +wsrpc +|___logs // 输出日志 +|___config // 配置 + |___default.json // common config + |___dev.json // NODE_ENV === dev, development + |___pro.json // NODE_ENV === pro， production + |___test.json // NODE_ENV === test，所有环境配置应与pro相同 +|___src + |___dao // 持久层数据对象 + |___suber // suber组件 + |___cacher // 缓存，来自suducer + |___kver // kv + |___noder // 节点直连 + |___recorder // archive + |___index.ts // 导出，初始化，组件入口 + |___suber.test.ts // 组件测试 + |___puber + |___matcher + |___global.ts // global变量定义 + |___interface.ts // 接口定义 + |___ ... +|___dist // 编译后js目标文件 +|___tests // 集成测试 +|___app.ts // 入口文件 +|___package.json +|___tsconfig.json +|___ .env // 环境变量配置 +|___pm2.json +|___.gitignore +``` + +#### 命名规范 + +- 公共导出类型以 `CapTypeT`格式，以`T`结尾 +- 全局共享变量，模块共享变量均以命名空间 `G`导出 +- 导出方式以文件夹，而不是文件，即文件夹下`index.ts`统一导出 +- 能使用`const`的尽量不用`let`，禁止`var` + + + +## Elara服务 + +Elara v2.0服务分为两个逻辑部分，Elara官网服务以及polkadot生态服务。官网服务包括`api`和`job`两个模块。**api**负责Elara官网的登录，项目创建，以及dashboard等服务。**job**通过redis stream的消息队列，接收wsrpc模块的原始请求统计数据，将其格式化存入redis。polkadot生态服务包括`wsrpc`和`suducer`，**wsrpc**是polkadot生态节点的代理服务，为减轻节点的服务压力以及提高节点的服务能力，由Elara做统一订阅和资源分发。**suducer**为wsrpc服务做单独的缓存订阅，将节点最新区块数据以及高频率请求做周期缓存。 + +#### 依赖 + +`postgresql`： 用于存储Elara项目数据以及用户数据 + +`redis ^5.0`： 用于Elara缓存服务，包括用户和项目的状态，统计数据，节点数据和业务缓存数据等 + +`nodejs ^14`: 用于运行Elara服务 + +#### 配置 + +##### 节点配置 + +在redis的db3中，需要为支持的链和节点配置启动信息。同类型节点多实例需要配置不同的ID。 + +```ts +Z_chain_list // 支持的链列表，zset类型 score: chain_name + 0: polkadot + 1: kusama +``` + +每个配置节点有自己的类型和id, + +```ts +type = 'node' | 'kv' | 'memory' +``` + +节点配置需要在*有序集合*· `Z_chain_[name]_ids`中注册才生效，其中nodeId需要和节点配置suffix中的数值对应。 + +```ts +// polkadot有效的两个节点配置，0和1 +Z_Chain_polkadot_ids // score: nodeId + 0: 0 // nodeId 0 和 H_Chain_polkadot_0 后缀0对应 + 1: 1 +``` + +比如polkadot节点0的同步节点和kv节点1 + +```ts +H_Chain_polkadot_0 // polkadot链节点0配置，后缀0和ids列表中nodeId对应 +{ + name: polkadot + baseUrl: 127.0.0.1 + wsPort: 9944 + rpcPort: 9933 + type: node + nodeId: 0 + poolSize: 50 // suber初始化连接池大小 +} + +H_Chain_polkadot_1 // polkadot链kv节点1配置 +{ + name: polkadot + baseUrl: 127.0.0.1 + wsPort: 9002 + type: kv + nodeId: 1 + poolSize: 50 +} +``` + +##### 环境配置 + +每个服务需要指定运行环境变量`NODE_ENV`,来读取`config/`路径下相应的配置文件(`pro.json|dev.json`)。 + +```ts +NODE_ENV = pro | dev +``` + +job等服务由额外的环境变量，参照`.env_example`进行设置。 + +**NOTE**：job和api中的AUTH变量，是base64的编码，需要对应。 + +### wsrpc + +wsrpc由三部分构成，`matcher、suber`和`puber`。matcher负责suber和puber的映射管理；puber负责Apps的连接请求以及请求分发；suber负责向各节点建立资源连接，为puber提供后端数据能力。 + +当前wsrpc采用1-1的订阅模型，后续可以升级为N-1以提高Elara服务能力和client端响应能力。 + +#### suber + +负责初始化节点连接资源,处理绑定到suber对象上的message回调。suber对象根据资源类型分为以下几种 + +```ts +cacher: 缓存资源，有suducer服务负责更新 +noder: 节点直连资源，在其他策略失效时，转发到该资源对象上 +kver: elara-kv，节点订阅托管资源 +recorder: 节点历史数据，由elara-archive提供(暂未接通) +``` + +每个suber对象会维护一个puber列表(集合)，并且有全局的suber对象和订阅话题列表来管理对象的资源。 + +```ts +interface Suber { + id: IDT, + chain: string, + nodeId: number, + url: string, + ws: WebSocket, + type: NodeType, + stat: SuberStat, + pubers?: Set, // {pubId} +} +class Suber { + + private static g: ChainSuber = {} + + private static topic: Record = {} + + // ---snip---- +} +``` + +##### 服务策略 + +优先对应的资源对象提供服务，当非noder资源失效时，将请求转发到noder上。 + +**NOTE**：kver资源失效，只能是在kver服务层失败的情况下才能感知 + +###### cacher缓存有效性 + +wsrpc周期性对cacher数据进行监测，有效监测周期为5秒，当累计未更新3次后会将cacher有效状态置为false。 + +##### message处理 + +suber的message分为以下几类 + +``` +1. node response // 同步节点数据，根据实际消息类型进行解析 +2. kv response // kv节点数据，需要解构 +3. ping // 节点健康检查心跳包 +``` + +kv节点数据解构后成为标准同步节点数据包。 + +###### 非订阅消息 + +对于非订阅消息，收到直接转发给puber，并将request cache上下文清除。 + +###### 订阅消息 + +- 首次订阅成功 + + 更新request cache，将subscription ID记录，用以接收后续的订阅回复。 + + **NOTE**：当订阅回复先于订阅成功消息包时，本地缓存1分钟，在收到订阅成功后，将缓存的订阅回复一并转发，并清除订阅回复缓存。 + +- 订阅回复 + + 转发puber，无论消息是error还是result，只做转发 + +- 取消订阅 + + 取消成功，清除相应订阅cache的上下文，转发取消订阅成功回复，当puber关闭导致的取消订阅无需转发。 + + 取消失败，转发取消订阅结果 + +#### puber + +接收来自client端的连接和请求，通过matcher申请suber资源。根据请求的资源类型，调用suber相应的资源对象进行请求。 + +```ts +interface Puber { + id: IDT, + pid: IDT, + chain: string, + nodeId: number, + ws: WebSocket, + topics: Set, // {subscribeId} + subId: IDT, // suber id + kvSubId?: IDT, // kv + memSubId?: IDT // memory node +} + +class Puber { + private static g: Record = {} + + private static reqs: Record> = {} + // ----snip----- +} +``` + +puber对象为了提高资源清理时的效率，维护了自己的请求缓存和订阅列表。每一个puber和链、项目以及节点实例绑定。 + +#### matcher + +负责puber和suber的绑定，以及请求的缓存管理。当puber和suber绑定后，完整的链路也就建立。matcher负责puber和suber的订阅管理，以及资源清除。在suber或者puber断开连接或者服务不可用时，管理相应的资源释放与重新分配。 + +###### suber异常 + +清除suber对象的资源，并通知绑定的puber关闭。 + +**NOTE：** polkadotjs不管代理节点的连通性，Elara只有将Apps的连接通道关闭转为不可用状态，才能触发apps的重连动作。apps会有重试机制，两次之后会清除之前的订阅列表，导致后续的订阅无法正常接收，哪怕Elara恢复，apps也无法恢复，必须刷新页面重新建立连接。 + +###### puber异常 + +``` +1. 释放相应的连接，并取消已订阅的topic +2. 清除request上下文 +3. 清除puber资源 +``` + +###### 节点异常 + +当前节点异常立刻清除已分配的所有资源，重新尝试建立新连接。 + +TODO：连接保持，如何让apps能够恢复？ + +### suducer + +suducer启动时会初始化一次性资源并注册周期task，当订阅节点的runtimeversion更新时，会更新一次性资源。 + +##### 资源列表 + +###### 订阅 + +``` +state_subscribeRuntimeVersion +``` + +###### 一次性资源 + +``` +"rpc_methods", +"system_name", +"system_version", +"system_chain", +"system_chainType", +"system_properties", +"state_getMetadata", +"state_getRuntimeVersion" +``` + +`chain_getBlockHash[0]`也在一次性缓存策略当中 + +###### 周期更新 + +更新周期5秒 + +``` +"system_syncState", +"system_health", +"chain_getHeader", +"chain_getBlock", +"chain_getBlockHash", +"chain_getFinalizedHead" +``` + +### api + +Elara官网后端服务 + +##### api文档 + +详细api查看知识库文档`elara-ts_api_doc.md` + +### job + +负责处理wsrpc发送的请求统计数据，对不同维度的统计信息进行归档，以供api服务使用。 + +```ts +1. Elara项目汇总统计 +2. 每条链汇总统计 +3. Elara项目汇总天统计 +4. 项目天统计 +5. 项目小时统计 + +// job/src/service.ts handleStat +KEY.hTotal(), +KEY.hChainTotal(req.chain), +KEY.hDaily(today), +KEY.hProDaily(chain, pid, today), +KEY.hProHourly(chain, pid, currentHourStamp()) +``` + +同时处理每个项目相应的带宽、国家统计数据。对于Elara项目进行资源限额检查，超过账户限额会更新用户和项目的可用状态，并在UTC+8时间0点重置状态。 + +#### 编译执行 + +```js +yarn install +// ts-node 执行 +yarn service_name start // service_name = wsrpc | job | suducer | api + +// node 执行 +yarn build +yarn service_name start:node +``` + +#### RPC列表 + +``` +// 同块周期 6s +chain_getHeader() +chain_getBlock() +chain_getBlockHash() +chain_getFinalizedHead() +system_syncState() +system_health() +state_getRuntimeVersion(blockhash) +-- author_pendingExtrinsics() + +// 周期要求低 10min +---system_peers() +state_getMetadata([blockHash]) --》 runtimeversion + + + +// 常量，服务节点启动时更新 or 1day， 节点变更 +rpc_methods() +system_name() +system_version() 节点重连更新 +system_chain() +system_chainType() +system_properties() + + +// 链历史数据 history kv 数据同步问题 +chain_getBlock(hash) '0x8caa40fc2c7e7cb9613e5b25ce24c93d14b8625368ea49d9a812ad29eacde879' +chain_getBlockHash(number) 块高度，超出返回错误 +chain_getHeader(hash) // 块hash +state_getStorage(key, blockhash) +state_queryStorageAt(key, blockhash) +--| get_transaction + +// 订阅 + author_submitAndWatchExtrinsic 节点直推 + author_unwatchExtrinsic + + chain_subscribeAllHeads + chain_unsubscribeAllHeads + chain_subscribeNewHeads + chain_unsubscribeNewHeads + chain_subscribeFinalizedHeads + chain_unsubscribeFinalizedHeads + + state_subscribeRuntimeVersion + state_unsubscribeRuntimeVersion + state_subscribeStorage // 区块更新就有pub，逐渐增大 + state_unsubscribeStorage + +// 禁止 + // 泄露节点信息 + system_nodeROles() + system_localListenAddresses() + system_localPeerId() + + system_addLogFilter + system_resetLogFilter + system_addReservedPeer + system_removeReservedPeer + + // 改变链上数据 + author_insertKey() + author_rotateKey() + author_removeExtrinsic + + // + offchain_localStorageSet + +// 其他，直接走rpc，需要列表筛选判定有效 +``` + diff --git a/doc/elara-redis_table_design.md b/doc/elara-redis_table_design.md new file mode 100644 index 0000000..0336a16 --- /dev/null +++ b/doc/elara-redis_table_design.md @@ -0,0 +1,120 @@ +### 分库 + +按业务模块进行分库，提高查询和数据管理；应对业务增长； + +| 模块 | index | 描述 | +| ------------- | ----- | ---------------------------------------------- | +| Account | 0 | 账户相关信息，黑白名单，`vip` | +| Project | 1 | project相关数据，增删改查 | +| Stat统计数据 | 2 | 统计相关，project请求统计，`elara`服务数据统计 | +| Chain配置 | 3 | 链相关数据，链名，`url`，端口，网络，等 | +| Cache缓存数据 | 4 | 周期数据，优化数据等 | + +### 字段设计原则 + +- 表明相关业务 +- 表明数据类型 +- 易于筛选分类查询 + +#### 服务原则 + +分为`Elara`和用户，`Elara`项目方的数据，以及基于`UID`的用户数据 + +#### 数据类型前缀 + +遵从`redis`数据操作命令格式 + +| 类型 | 前缀 | +| ------ | ---- | +| List | L_ | +| Hash | H_ | +| Set | S_ | +| `ZSet` | Z_ | +| String | none | + +#### 统一字段 + +module 表明当前分库的业务模块。 + +可选值： ['Account', 'Project', 'Stat', 'Chain', 'Cache'] + +### `Elara` `Redis`数据设计 + +##### 表名 + +`Type_Module_Item_[Chain]` + +#### stat + +##### 需求 + +1. 项目的统计信息 +2. `elara`总体项目统计信息 + + + +| | | +| ---- | ---- | +| | | +| | | +| | | +| | | +| | | +| | | +| | | +| | | +| | | + + + +#### project + +1. ##### 需求 + + - 列表是否要有序 + - 统计 + - 筛选 + + | 查询 | 命令 | + | ---------------------------- | ------------------------------------------------------- | + | 所有项目数 | get Project_Num | + | 某条链项目数 | keys Z_Project_list\_*_[CHAIN] ; zcard | + | 用户的项目数 | keys Z_Project_list\_[UID]_*; zcard | + | 用户某条链的项目数 | zcard Z_Project_list\_[UID]_[CHAIN] | + | 某条链的项目列表 | keys H_Project\_[CHAIN]_* ; hgetall | + | 用户的项目列表 | keys Z_Project\_list_[UID]*; hgetall | + | 用户某条链的项目列表 | zrange Z_Project_list\_[UID]_[CHIAN] 0 -1; hgetall | + | 列表按创建时间排序(链维度) | zrangebyscore | + | / 列表按更新时间排序(链维度) | 暂不支持 | + | - 某段时间内的项目数 | zcount Z_Project_list\_[UID]_[CHIAN] starttime endtime | + | - 某段时间内的项目列表 | zrange Z_Project_list\_[UID]_[CHIAN] starttime endtime | + + + +2. ##### 数据结构 + + `UID | chain | PID | createtime | lasttime |` + + `UID` 用户筛选 + `chain` 链筛选 + `createtime / lasttime` 创建和更新时间排序 + +3. ##### 实现 + +```js +// Chain名小写，防止不匹配 +module 'Project' +Project_Num 0 // 所有项目数量 + +H_Project_[CHAIN]_[PID] // project详情记录，原先project_info_[PID] +Z_Project_list_[UID]_[CHAIN] // project列表，按UID-CHAIN分类，原先projects_[UID] + +// 有序集合 +- 以craetetime为score +- key值为pid +``` + + + +​ + diff --git a/doc/elara-ts_api_doc.md b/doc/elara-ts_api_doc.md new file mode 100644 index 0000000..6bda396 --- /dev/null +++ b/doc/elara-ts_api_doc.md @@ -0,0 +1,745 @@ + +# @elara/api v0.1.0 + +elara api service + +# Table of contents + +- [auth](#auth) + - [github](#github) + - [login](#login) + - [logout](#logout) +- [chain](#chain) + - [list](#list) +- [NonAuth](#NonAuth) + - [dayilyStatis](#dayilyStatis) + - [lastDaysOfAll](#lastDaysOfAll) + - [totalStatis](#totalStatis) +- [project](#project) + - [countOfChain](#countOfChain) + - [countOfUser](#countOfUser) + - [create](#create) + - [delete](#delete) + - [list](#list) + - [projectByChainPid](#projectByChainPid) + - [projectById](#projectById) + - [updateLimit](#updateLimit) + - [updateName](#updateName) +- [stat](#stat) + - [countryRequestMap](#countryRequestMap) + - [dayilyOfProject](#dayilyOfProject) + - [lastDaysOfProject](#lastDaysOfProject) + - [lastHoursOfProject](#lastHoursOfProject) + - [latestError](#latestError) + - [requestRank](#requestRank) +- [user](#user) + - [detail](#detail) + - [userDailyStatistic](#userDailyStatistic) + +___ + + +# auth + +## github +[Back to top](#top) + +

clientID，clientSecret，access GitHub

+ +``` +GET /auth/github +``` + +## login +[Back to top](#top) + +``` +GET /auth/login +``` + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| User | `Object` |

user obeject with limit resource and project count

| + +### Success response example + +#### Success response example - `Success` + +```json +{ + code: 0, + msg: 'ok', + data: { + user: { + id: 1, + name: 'Bruce', + githubId: 'TestUID', + limit: { + projectNum: 10, // max prject count + bwDayLimit, + reqDayLimit, + ... + } + }, + projectNum: 7 + } +} +``` + +## logout +[Back to top](#top) + +``` +GET /auth/logout +``` + +# chain + +## list +[Back to top](#top) + +``` +POST /chain/list +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| userId | `Integer` | | + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Record | `Object` |

chain list map by network,Record<String, ChainInfo[]>

| + +### Success response example + +#### Success response example - `Success:` + +```json +{ + code: 0, + msg: 'ok', + data: { + 'live': [{ + id: 1, + name: 'polkadot', + team: 'parity', + network: 'Polkadot', + status: 'active', 'active' | 'inactive' | 'empty' + count: 1 // project count + + }, {}], + 'test': [] + } +} +``` + +# NonAuth + +## dayilyStatis +[Back to top](#top) + +

today statistic

+ +``` +GET /public/daily +``` + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Stat | `StatT` |

totay statistic record

| +| Stat.reqCnt | `Integer` |

total request count

| +| Stat.wsConn | `Integer` |

ws connection count

| +| Stat.subCnt | `Integer` |

ws subscribe count

| +| Stat.subResCnt | `Integer` |

ws response count in subscription

| +| Stat.bw | `Integer` |

total bandwidth

| +| Stat.delay | `Integer` |

average delay ms

| +| Stat.inReqCnt | `Integer` |

invalid request count

| +| Stat.timeoutDelay | `Integer` |

average timeout ms

| +| Stat.timeoutCnt | `Integer` |

timeout count

| +| Stat.ctMap | `Integer` |

request country map {'US': 3, 'CZ': 100 , 'unknow': 1}

| + +## lastDaysOfAll +[Back to top](#top) + +

last days statistic record of elara

+ +``` +POST /public/days +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| days | `Integer` |

how many days to view

_Size range: >=1, <=30_
| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Stat | `Object` |

stat day duration info

| +| Stat.timeline | `String[]` | | +| Stat.stats | `StatInfoT[]` | | +| Stat.stats.request | `Integer` |

request count

| +| Stat.stats.bandwidth | `Integer` |

bandwidth in bytes

| + +## totalStatis +[Back to top](#top) + +

total statistic

+ +``` +GET /public/stat +``` + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Stat | `StatInfoT` |

total statistic record with request & bandwidth

| +| Stat.request | `Integer` |

total request count

| +| Stat.bandwidth | `Integer` |

total request bandwidth in byte

| + +# project + +## countOfChain +[Back to top](#top) + +``` +POST /project/count/chain +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| chain | `String` |

chain name

| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| none | `Number` |

count of chain

| + +## countOfUser +[Back to top](#top) + +``` +POST /project/count/user +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| userId | `Number` |

user id

| +| byChain | `Boolean` |

by chain or not

| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| none | `Number` |

count of user

| +| Counts | `Object[]` |

count list of user & chain

| +| Counts.chain | `String` |

chain

| +| Counts.count | `String` |

count of chain

| + +### Success response example + +#### Success response example - `SuccessByChain:` + +```json +{ + "code": 0, + "msg": "ok", + "data": [ + { + "chain": "Kusuma", + "count": "2" + }, + { + "chain": "jupiter", + "count": "2" + } + ] +} +``` + +#### Success response example - `Success:` + +```json +{ + code: 0, + msg: 'ok', + data: 4 +} +``` + +## create +[Back to top](#top) + +``` +POST /project/create +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| userId | `Number` |

user id

| +| chain | `String` |

chain

| +| team | `String` |

team name

| +| name | `String` |

project name to create [0-9a-zA-Z]{4,32}

| +| reqSecLimit | `Number` | **optional**

request second limit

| +| reqDayLimit | `Number` | **optional**

request day limit

| +| bwDayLimit | `Number` | **optional**

bandwidth day limit

| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| none | `ProAttr` |

project created

| + +## delete +[Back to top](#top) + +

logic delete

+ +``` +POST /project/delete +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| id | `Number` |

project id

| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| none | `Boolean` |

delte result, success or not

| + +## list +[Back to top](#top) + +

get project list according to [userId, chain], if both, list of userId & chain

+ +``` +POST /project/list +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| userId | `Number` | **optional**

integer userId, list of userId

| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| project | `ProAttr[]` |

list of project

| +| project.id | `Number` |

project id, integer

| +| project.pid | `String` |

project pid, 16 bytes hex string

| +| project.name | `String` |

project name

| +| project.status | `String` |

project status ['active', 'stop', 'suspend']

| +| project.chain | `String` |

chain name

| +| project.team | `String` |

team name

| +| project.secret | `String` |

project secret, reserved field

| +| project.userId | `Number` |

association user id

| +| project.reqSecLimit | `Number` |

request count of second limit

| +| project.reqDayLimit | `Number` |

request count of day limit

| +| project.bwDayLimit | `Number` |

bandwidth of day limit

| + +### Success response example + +#### Success response example - `Success:` + +```json +{ + code: 0, + msg: 'ok', + data: ProAttr[] +} +``` + +### Error response example + +#### Error response example - `Error:` + +```json +{ + code: 400, // non 0 code + msg: error message, + data: {} +} +``` + +## projectByChainPid +[Back to top](#top) + +``` +POST /project/detail/chainpid +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| chain | `String` | | +| pid | `String` | | + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| project | `ProAttr` | | + +## projectById +[Back to top](#top) + +``` +POST /project/detail/id +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| id | `Number` | | + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| project | `ProAttr` | | + +## updateLimit +[Back to top](#top) + +

udpate project resource limit, reqSecLimit/reqDayLimit/bwDayLimit

+ +``` +POST /project/update/limit/ +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| id | `Number` |

project id

| +| reqSecLimit | `Number` | **optional**

request second limit

| +| reqDayLimit | `Number` | **optional**

request day limit

| +| bwDayLimit | `Number` | **optional**

bandwidth day limit

| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| none | `null` | | + +## updateName +[Back to top](#top) + +``` +POST /project/update/name +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| userId | `Number` | | +| chain | `String` |

which chain belongs to

| +| id | `Number` |

project id

| +| name | `String` |

new project name

| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| none | `String` |

new name

| + +# stat + +## countryRequestMap +[Back to top](#top) + +``` +POST /stat/project/country +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| size | `Integer` |

size of page

_Size range: >=1_
| +| page | `Integer` |

page offset

| +| chain | `String` | | +| pid | `String` | | + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Object | `Object` |

page object

| +| Object.total | `Integer` |

total records

| +| Object.size | `Integer` |

page size

| +| Object.page | `Integer` |

page offset

| +| Object.pages | `Integer` |

total pages

| +| Object.list | `Object[]` |

record list

| +| Object.list.country | `String` | | +| Object.list.percentage | `String` |

request count ratio

| + +## dayilyOfProject +[Back to top](#top) + +

today statistic record of project

+ +``` +POST /stat/project/daily +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| chain | `String` |

chain name

| +| pid | `String` |

project pid

| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Stat | `StatT` |

statistic record

| + +## lastDaysOfProject +[Back to top](#top) + +

last days statistic record of project

+ +``` +POST /stat/project/days +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| chain | `String` |

chain name

| +| pid | `String` |

project pid

| +| days | `Integer` |

how many days to view

_Size range: >=1, <=30_
| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Stat | `Object` |

stat day duration info

| +| Stat.timeline | `String[]` | | +| Stat.stats | `StatInfoT[]` | | +| Stat.stats.request | `Integer` |

request count

| +| Stat.stats.bandwidth | `Integer` |

bandwidth in bytes

| + +## lastHoursOfProject +[Back to top](#top) + +

last hours statistic record of project

+ +``` +POST /stat/project/hours +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| pid | `String` |

project pid

| +| hours | `Integer` |

how many hours to view

_Size range: >=1, <=24_
| + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Stat | `Object` |

stat hour duration info

| +| Stat.timeline | `String[]` | | +| Stat.stats | `StatInfoT[]` | | +| Stat.stats.request | `Integer` |

request count

| +| Stat.stats.bandwidth | `Integer` |

bandwidth in bytes

| + +## latestError +[Back to top](#top) + +

latest error record of all

+ +``` +POST /stat/latest/error +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| size | `Integer` |

size of page

_Size range: >=1_
| +| page | `Integer` |

page offset

| +| chain | `String` | | +| pid | `String` | | + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Object | `Object` |

page object

| +| Object.total | `Integer` |

total records

| +| Object.size | `Integer` |

page size

| +| Object.page | `Integer` |

page offset

| +| Object.pages | `Integer` |

total pages

| +| Object.list | `Object[]` |

record list

| +| Object.list.proto | `String` |

http | ws

| +| Object.list.mehtod | `String` | | +| Object.list.delay | `String` | | +| Object.list.code | `String` | | +| Object.list.time | `String` |

timestamp string YYYY-MM-DD HH:mm

| + +## requestRank +[Back to top](#top) + +

today statistic record of project

+ +``` +POST /stat/project/rank +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| chain | `String` | | +| pid | `String` | | + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Rank | `Object` |

resource rank info

| + +### Success response example + +#### Success response example - `Success:` + +```json +{ + code: 0, + msg: 'ok', + data: { + bandwidth: { + total: 1000 // bandwidth bytes, + list: [{ method: 'systen_health', value: 100 }] + }, + request: { + total: 1024 // request count, + list: [{ method: 'systen_health', value: 10 }] + } + } +} +``` + +# user + +## detail +[Back to top](#top) + +

user detail info

+ +``` +GET /user/detail +``` + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| User | `UserAttr` | | +| User.id | `Integer` |

user id

| +| User.name | `String` |

user ame

| +| User.status | `String` | _Size range: 'active','suspend','barred'_
| +| User.level | `String` | _Size range: 'normal', 'bronzer','silver','gold'_
| +| User.loginType | `String` |

now is github

| +| User.githubId | `String` |

origin uid

| +| User.phone | `String` | **optional** | +| User.mail | `String` | **optional** | + +## userDailyStatistic +[Back to top](#top) + +

user detail info

+ +``` +POST /user/detail/statistic +``` + +### Parameters - `Parameter` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| userId | `Integer` | | + +### Success response + +#### Success response - `Success 200` + +| Name | Type | Description | +|----------|------------|---------------------------------------| +| Stat | `StatT` |

user statistic of all project

| + diff --git a/package.json b/package.json index 3e1f374..eec795e 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,9 @@ "homepage": "https://github.com/patractlabs/elara-ts#readme", "devDependencies": { "@types/jest": "^26.0.23", + "@types/node": "^16.4.7", "jest": "^26.6.3", + "ts-node": "^10", "lerna": "^4.0.0", "minimist": "^1.2.5", "ts-jest": "^26.5.6", diff --git a/packages/README.md b/packages/README.md deleted file mode 100644 index 23cb39f..0000000 --- a/packages/README.md +++ /dev/null @@ -1,54 +0,0 @@ -## `elara-ts`项目设计 - -按服务模块分包，每个包相当于一个微服务。包内项目推荐按照组件原则构建，而不是`MVC`。 - -#### 组织结构 - -##### 项目结构 - -```js -pakages -|___chain // 平行链服务，动态管理链 -|___gateway // api网关 -|___job // 定时任务等 -|___stat // elara数据统计，项目统计 -|___lib // 公共库 -|___ws // elara核心服务，rpc请求，ws订阅管理 -|___account // 账户管理，认证 - -``` - -##### 包结构 - -```js -ws -|___logs // 输出日志 -|___config // 配置 - |___default.json // common config - |___dev.json // NODE_ENV === dev, development - |___pro.json // NODE_ENV === pro， production - |___test.json // NODE_ENV === test，所有环境配置应与pro相同 -|___src - |___lib // 内部公共库 - |___esuber // esuber组件 - |___global.ts // global变量定义 - |___interface.ts // 数据结构 - |___index.ts // 导出，初始化，组件入口 - |___suber.test.ts // 组件测试 - |___epuber - |___matcher -|___test // 集成测试 -|___package.json -|___tsconfig.json -|___pm2.json -|___.gitignore -``` - -#### 命名规范 - -- 公共导出类型以 `CapTypeT`格式，以`T`结尾 -- 全局共享变量，模块共享变量均以命名空间 `G`导出 -- 导出方式以文件夹，而不是文件，即文件夹下`index.ts`统一导出 -- 编码优先函数组件式而不是类 -- 能使用`const`的尽量不用`let`，禁止`var` - diff --git a/packages/scripts/redis-init.ts b/packages/scripts/redis-init.ts index 4462574..7acf6d8 100644 --- a/packages/scripts/redis-init.ts +++ b/packages/scripts/redis-init.ts @@ -1,8 +1,24 @@ -import { Redis, DBT } from '@elara/lib' -import { ChainConfig, ChainType, KEYS, Network } from '@elara/lib' +import { Redis, DBT, KEYS } from '@elara/lib' const cKEY = KEYS.Chain +enum NodeType { + Node = 'node', + Kv = 'kv', + Mem = 'memory' +} + +interface ChainConfig { + name: string, + nodeId: number, // default 0, elara node instance id + type: NodeType, + baseUrl: string, // host + rpcPort: number, // default 9933 + wsPort: number, // default 9944 + poolSize: number, + [key: string]: any // for redis +} + // default localhost:6379, configure your own connection // NOTE: don't change DBT.Chain type const cRd = new Redis(DBT.Chain).getClient() @@ -10,15 +26,12 @@ const cRd = new Redis(DBT.Chain).getClient() const newChain = async (chain: string) => { const polkadot: ChainConfig = { name: chain, + type: NodeType.Node, baseUrl: '127.0.0.1', // node url wsPort: 19944, // websocket port rpcPort: 19933, // http port - network: Network.Live, // optional - chainType: ChainType.Relay, // optional - serverId: 0, // node instance id, when multi node deploy - kvEnable: false, // if true, Elara-kv need - kvPort: 9002, - kvBaseUrl: '127.0.0.1' + nodeId: 0, // node instance id, when multi node deploy + poolSize: 20 } await cRd.hmset(cKEY.hChain(chain, 0), polkadot) let cnt = await cRd.incr(cKEY.chainNum()) diff --git a/yarn.lock b/yarn.lock index 103f77a..bd64ff1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -303,6 +303,18 @@ exec-sh "^0.3.2" minimist "^1.2.0" +"@cspotcode/source-map-consumer@0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b" + integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg== + +"@cspotcode/source-map-support@0.7.0": + version "0.7.0" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.7.0.tgz#4789840aa859e46d2f3173727ab707c66bf344f5" + integrity sha512-X4xqRHqN8ACt2aHVe51OxeA2HjbcL4MqFqXkrmQszJ1NOUuUu5u6Vqx/0lZSVNku7velL5FC/s5uEAj1lsBMhA== + dependencies: + "@cspotcode/source-map-consumer" "0.8.0" + "@dabh/diagnostics@^2.0.2": version "2.0.2" resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.2.tgz#290d08f7b381b8f94607dc8f471a12c675f9db31" @@ -1379,6 +1391,26 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== +"@tsconfig/node10@^1.0.7": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.8.tgz#c1e4e80d6f964fbecb3359c43bd48b40f7cadad9" + integrity sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg== + +"@tsconfig/node12@^1.0.7": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.9.tgz#62c1f6dee2ebd9aead80dc3afa56810e58e1a04c" + integrity sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw== + +"@tsconfig/node14@^1.0.0": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2" + integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg== + +"@tsconfig/node16@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e" + integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA== + "@types/accepts@*": version "1.3.5" resolved "https://registry.yarnpkg.com/@types/accepts/-/accepts-1.3.5.tgz#c34bec115cfc746e04fe5a059df4ce7e7b391575" @@ -1750,6 +1782,11 @@ acorn-walk@^7.1.1: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== +acorn-walk@^8.1.1: + version "8.2.0" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" + integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== + acorn@^7.1.1: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" @@ -1760,6 +1797,11 @@ acorn@^8.2.4: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.4.1.tgz#56c36251fc7cabc7096adc18f05afe814321a28c" integrity sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA== +acorn@^8.4.1: + version "8.7.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.0.tgz#90951fde0f8f09df93549481e5fc141445b791cf" + integrity sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ== + add-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa" @@ -1874,6 +1916,11 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" +arg@^4.1.0: + version "4.1.3" + resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" + integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== + argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -2695,6 +2742,11 @@ crc@^3.4.4: dependencies: buffer "^5.1.0" +create-require@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" + integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== + cron-parser@^3.1.0: version "3.5.0" resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-3.5.0.tgz#b1a9da9514c0310aa7ef99c2f3f1d0f8c235257c" @@ -2941,6 +2993,11 @@ diff-sequences@^26.6.2: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.6.2.tgz#48ba99157de1923412eed41db6b6d4aa9ca7c0b1" integrity sha512-Mv/TDa3nZ9sbc5soK+OoA74BsS3mL37yixCvUAQkiuA4Wz6YtwP/K47n2rv2ovzHZvoiQeA5FTQOschKkEwB0Q== +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -5192,7 +5249,7 @@ make-dir@^3.0.0: dependencies: semver "^6.0.0" -make-error@1.x: +make-error@1.x, make-error@^1.1.1: version "1.3.6" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== @@ -7576,6 +7633,24 @@ ts-jest@^26.5.6: semver "7.x" yargs-parser "20.x" +ts-node@^10: + version "10.4.0" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.4.0.tgz#680f88945885f4e6cf450e7f0d6223dd404895f7" + integrity sha512-g0FlPvvCXSIO1JDF6S232P5jPYqBkRL9qly81ZgAOSU7rwI0stphCgd2kLiCrU9DjQCrJMWEqcNSjQL02s6d8A== + dependencies: + "@cspotcode/source-map-support" "0.7.0" + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.2" + acorn "^8.4.1" + acorn-walk "^8.1.1" + arg "^4.1.0" + create-require "^1.1.0" + diff "^4.0.1" + make-error "^1.1.1" + yn "3.1.1" + tslib@^1.9.0: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" @@ -8170,3 +8245,8 @@ ylru@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/ylru/-/ylru-1.2.1.tgz#f576b63341547989c1de7ba288760923b27fe84f" integrity sha512-faQrqNMzcPCHGVC2aaOINk13K+aaBDUPjGWl0teOXywElLjyVAB6Oe2jj62jHYtwsU49jXhScYbvPENK+6zAvQ== + +yn@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" + integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==