From 97bd4014d611d62f024245b2e350b4155d112f76 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:20:32 +0530 Subject: [PATCH] Support nested entity sorting in plural GQL queries (#456) * Move method to apply block filter to base database * Handle nested entity sorting * Handle nested entity sorting for distinct on type queries * Handle nested entity sorting for all query types --- packages/util/src/database.ts | 124 ++++++++++++++++++++++++++-- packages/util/src/graph/database.ts | 86 +++++++------------ 2 files changed, 146 insertions(+), 64 deletions(-) diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index f5f06028d..a4b0e402e 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -268,7 +268,7 @@ export class Database { queryBuilder = this.buildQuery(repo, queryBuilder, where); if (queryOptions.orderBy) { - queryBuilder = this.orderQuery(repo, queryBuilder, queryOptions); + queryBuilder = await this.orderQuery(repo, queryBuilder, queryOptions); } queryBuilder.addOrderBy('event.id', 'ASC'); @@ -1050,9 +1050,9 @@ export class Database { assert(columnMetadata); if (relation.isArray) { - relationSubQuery = relationSubQuery.where(`${relationTableName}.id = ANY(${alias}.${columnMetadata.databaseName})`); + relationSubQuery = relationSubQuery.where(`${relationTableName}.id = ANY("${alias}".${columnMetadata.databaseName})`); } else { - relationSubQuery = relationSubQuery.where(`${relationTableName}.id = ${alias}.${columnMetadata.databaseName}`); + relationSubQuery = relationSubQuery.where(`${relationTableName}.id = "${alias}".${columnMetadata.databaseName}`); } } @@ -1071,29 +1071,139 @@ export class Database { whereBuilder.andWhere(`EXISTS (${relationSubQuery.getQuery()})`, relationSubQuery.getParameters()); } - orderQuery ( + async orderQuery ( repo: Repository, selectQueryBuilder: SelectQueryBuilder, orderOptions: { orderBy?: string, orderDirection?: string }, + relations: Readonly<{ [key: string]: any }> = {}, + block: Readonly = {}, columnPrefix = '', alias?: string - ): SelectQueryBuilder { + ): Promise> { if (!alias) { alias = selectQueryBuilder.alias; } - const { orderBy, orderDirection } = orderOptions; - assert(orderBy); + const { orderBy: orderByWithSuffix, orderDirection } = orderOptions; + assert(orderByWithSuffix); + + // Nested sort key of form relationField__relationColumn + const [orderBy, suffix] = orderByWithSuffix.split('__'); const columnMetadata = repo.metadata.findColumnWithPropertyName(orderBy); assert(columnMetadata); + // Handle nested entity sort + const relation = relations[orderBy]; + if (suffix && relation) { + return this.orderQueryNested( + repo, + selectQueryBuilder, + { relationField: orderBy, orderBy: suffix, orderDirection }, + relation, + block, + columnPrefix, + alias + ); + } + return selectQueryBuilder.addOrderBy( `"${alias}"."${columnPrefix}${columnMetadata.databaseName}"`, orderDirection === 'desc' ? 'DESC' : 'ASC' ); } + async orderQueryNested ( + repo: Repository, + selectQueryBuilder: SelectQueryBuilder, + orderOptions: { relationField: string, orderBy: string, orderDirection?: string }, + relation: Readonly = {}, + block: Readonly = {}, + columnPrefix = '', + alias: string + ): Promise> { + const { relationField, orderBy, orderDirection } = orderOptions; + + const columnMetadata = repo.metadata.findColumnWithPropertyName(relationField); + assert(columnMetadata); + + const relationRepo = this.conn.getRepository(relation.entity); + const relationTableName = relationRepo.metadata.tableName; + + const relationColumnMetaData = relationRepo.metadata.findColumnWithPropertyName(orderBy); + assert(relationColumnMetaData); + + const queryRunner = repo.queryRunner; + assert(queryRunner); + + // Perform a groupBy(id) and max(block number) to get the latest version of related entities + let subQuery = relationRepo.createQueryBuilder('subTable', queryRunner) + .select('subTable.id', 'id') + .addSelect('MAX(subTable.block_number)', 'block_number') + .where('subTable.is_pruned = :isPruned', { isPruned: false }) + .groupBy('subTable.id'); + + subQuery = await this.applyBlockHeightFilter(queryRunner, subQuery, block, 'subTable'); + + // Self join to select required columns + const latestRelatedEntitiesAlias = `latest${relationField}Entities`; + const relationSubQuery: SelectQueryBuilder = relationRepo.createQueryBuilder(relationTableName, queryRunner) + .select(`${relationTableName}.id`, 'id') + .addSelect(`${relationTableName}.${relationColumnMetaData.databaseName}`, `${relationColumnMetaData.databaseName}`) + .innerJoin( + `(${subQuery.getQuery()})`, + latestRelatedEntitiesAlias, + `${relationTableName}.id = "${latestRelatedEntitiesAlias}"."id" AND ${relationTableName}.block_number = "${latestRelatedEntitiesAlias}"."block_number"` + ) + .setParameters(subQuery.getParameters()); + + // Join with related table to get the required field to sort on + const relatedEntitiesAlias = `related${relationField}`; + selectQueryBuilder = selectQueryBuilder + .innerJoin( + `(${relationSubQuery.getQuery()})`, + relatedEntitiesAlias, + `"${alias}"."${columnPrefix}${columnMetadata.databaseName}" = "${relatedEntitiesAlias}".id` + ) + .setParameters(relationSubQuery.getParameters()); + + // Apply sort + return selectQueryBuilder + .addSelect(`"${relatedEntitiesAlias}"."${relationColumnMetaData.databaseName}"`) + .addOrderBy( + `"${relatedEntitiesAlias}"."${relationColumnMetaData.databaseName}"`, + orderDirection === 'desc' ? 'DESC' : 'ASC' + ); + } + + async applyBlockHeightFilter ( + queryRunner: QueryRunner, + queryBuilder: SelectQueryBuilder, + block: CanonicalBlockHeight, + alias: string + ): Promise> { + // Block hash takes precedence over number if provided + if (block.hash) { + if (!block.canonicalBlockHashes) { + const { canonicalBlockNumber, blockHashes } = await this.getFrothyRegion(queryRunner, block.hash); + + // Update the block field to avoid firing the same query further + block.number = canonicalBlockNumber; + block.canonicalBlockHashes = blockHashes; + } + + queryBuilder = queryBuilder + .andWhere(new Brackets(qb => { + qb.where(`${alias}.block_hash IN (:...blockHashes)`, { blockHashes: block.canonicalBlockHashes }) + .orWhere(`${alias}.block_number <= :canonicalBlockNumber`, { canonicalBlockNumber: block.number }); + })); + } else if (block.number) { + queryBuilder = queryBuilder.andWhere(`${alias}.block_number <= :blockNumber`, { blockNumber: block.number }); + } + + return queryBuilder; + } + async _fetchBlockCount (): Promise { const res = await this._conn.getRepository('block_progress') .count(); diff --git a/packages/util/src/graph/database.ts b/packages/util/src/graph/database.ts index ed8d20719..b2eca9deb 100644 --- a/packages/util/src/graph/database.ts +++ b/packages/util/src/graph/database.ts @@ -4,7 +4,6 @@ import assert from 'assert'; import { - Brackets, Connection, FindOneOptions, In, @@ -22,7 +21,7 @@ import { SelectionNode } from 'graphql'; import _ from 'lodash'; import debug from 'debug'; -import { BlockHeight, Database as BaseDatabase, QueryOptions, Where, CanonicalBlockHeight } from '../database'; +import { Database as BaseDatabase, QueryOptions, Where, CanonicalBlockHeight } from '../database'; import { BlockProgressInterface } from '../types'; import { cachePrunedEntitiesCount, eventProcessingLoadEntityCacheHitCount, eventProcessingLoadEntityCount, eventProcessingLoadEntityDBQueryDuration } from '../metrics'; import { ServerConfig } from '../config'; @@ -220,7 +219,7 @@ export class GraphDatabase { entityType: (new () => Entity), id: string, relationsMap: Map, - block: BlockHeight = {}, + block: CanonicalBlockHeight = {}, selections: ReadonlyArray = [] ): Promise { let { hash: blockHash, number: blockNumber } = block; @@ -260,7 +259,7 @@ export class GraphDatabase { async loadEntityRelations ( queryRunner: QueryRunner, - block: BlockHeight, + block: CanonicalBlockHeight, relationsMap: Map, entityType: new () => Entity, entityData: any, selections: ReadonlyArray = [] @@ -365,6 +364,7 @@ export class GraphDatabase { queryRunner, entityType, latestEntityType, + relationsMap, block, where, queryOptions @@ -442,7 +442,7 @@ export class GraphDatabase { delete where[FILTER_CHANGE_BLOCK]; } - subQuery = await this._applyBlockHeightFilter(queryRunner, subQuery, block, 'subTable'); + subQuery = await this._baseDatabase.applyBlockHeightFilter(queryRunner, subQuery, block, 'subTable'); let selectQueryBuilder = repo.createQueryBuilder(tableName) .innerJoin( @@ -455,10 +455,10 @@ export class GraphDatabase { selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where, relationsMap.get(entityType), block); if (queryOptions.orderBy) { - selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions); + selectQueryBuilder = await this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, relationsMap.get(entityType), block); } - selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }); + selectQueryBuilder = await this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }); if (queryOptions.skip) { selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip); @@ -499,21 +499,20 @@ export class GraphDatabase { delete where[FILTER_CHANGE_BLOCK]; } - subQuery = await this._applyBlockHeightFilter(queryRunner, subQuery, block, 'subTable'); + subQuery = await this._baseDatabase.applyBlockHeightFilter(queryRunner, subQuery, block, 'subTable'); subQuery = this._baseDatabase.buildQuery(repo, subQuery, where, relationsMap.get(entityType), block); let selectQueryBuilder = queryRunner.manager.createQueryBuilder() - .from( - `(${subQuery.getQuery()})`, - 'latestEntities' - ) + .select('"latestEntities".*') + .from(`(${subQuery.getQuery()})`, 'latestEntities') .setParameters(subQuery.getParameters()) as SelectQueryBuilder; if (queryOptions.orderBy) { - selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, 'subTable_'); + selectQueryBuilder = await this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, relationsMap.get(entityType), block, 'subTable_'); if (queryOptions.orderBy !== 'id') { - selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, 'subTable_'); + // Order by id if ordered by some non-id column (for rows having same value for the column ordered on) + selectQueryBuilder = await this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, {}, {}, 'subTable_'); } } @@ -535,7 +534,7 @@ export class GraphDatabase { queryRunner: QueryRunner, entityType: new () => Entity, relationsMap: Map, - block: BlockHeight, + block: CanonicalBlockHeight, where: Where = {} ): Promise { const repo = queryRunner.manager.getRepository(entityType); @@ -551,7 +550,7 @@ export class GraphDatabase { delete where[FILTER_CHANGE_BLOCK]; } - selectQueryBuilder = await this._applyBlockHeightFilter(queryRunner, selectQueryBuilder, block, tableName); + selectQueryBuilder = await this._baseDatabase.applyBlockHeightFilter(queryRunner, selectQueryBuilder, block, tableName); selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where, relationsMap.get(entityType), block); @@ -564,7 +563,7 @@ export class GraphDatabase { queryRunner: QueryRunner, entityType: new () => Entity, relationsMap: Map, - block: BlockHeight, + block: CanonicalBlockHeight, where: Where = {}, queryOptions: QueryOptions = {} ): Promise { @@ -579,15 +578,15 @@ export class GraphDatabase { delete where[FILTER_CHANGE_BLOCK]; } - selectQueryBuilder = await this._applyBlockHeightFilter(queryRunner, selectQueryBuilder, block, tableName); + selectQueryBuilder = await this._baseDatabase.applyBlockHeightFilter(queryRunner, selectQueryBuilder, block, tableName); selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where, relationsMap.get(entityType), block); if (queryOptions.orderBy) { - selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions); + selectQueryBuilder = await this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, relationsMap.get(entityType), block); } - selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }); + selectQueryBuilder = await this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }); if (queryOptions.skip) { selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip); @@ -646,10 +645,10 @@ export class GraphDatabase { selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where, relationsMap.get(entityType), {}, 'latest'); if (queryOptions.orderBy) { - selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, '', 'latest'); + selectQueryBuilder = await this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, relationsMap.get(entityType), {}, '', 'latest'); } - selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, '', 'latest'); + selectQueryBuilder = await this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, {}, {}, '', 'latest'); if (queryOptions.skip) { selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip); @@ -666,7 +665,8 @@ export class GraphDatabase { queryRunner: QueryRunner, entityType: new () => Entity, latestEntity: new () => any, - block: BlockHeight, + relationsMap: Map, + block: CanonicalBlockHeight, where: Where = {}, queryOptions: QueryOptions = {} ): Promise { @@ -684,7 +684,7 @@ export class GraphDatabase { delete where[FILTER_CHANGE_BLOCK]; } - subQuery = await this._applyBlockHeightFilter(queryRunner, subQuery, block, 'subTable'); + subQuery = await this._baseDatabase.applyBlockHeightFilter(queryRunner, subQuery, block, 'subTable'); let selectQueryBuilder = latestEntityRepo.createQueryBuilder('latest') .select('*') @@ -698,13 +698,13 @@ export class GraphDatabase { 'result' ) as SelectQueryBuilder; - selectQueryBuilder = this._baseDatabase.buildQuery(latestEntityRepo, selectQueryBuilder, where, {}, {}, 'latest'); + selectQueryBuilder = this._baseDatabase.buildQuery(latestEntityRepo, selectQueryBuilder, where, relationsMap.get(entityType), block, 'latest'); if (queryOptions.orderBy) { - selectQueryBuilder = this._baseDatabase.orderQuery(latestEntityRepo, selectQueryBuilder, queryOptions, '', 'latest'); + selectQueryBuilder = await this._baseDatabase.orderQuery(latestEntityRepo, selectQueryBuilder, queryOptions, relationsMap.get(entityType), block, '', 'latest'); } - selectQueryBuilder = this._baseDatabase.orderQuery(latestEntityRepo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, '', 'latest'); + selectQueryBuilder = await this._baseDatabase.orderQuery(latestEntityRepo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, {}, {}, '', 'latest'); if (queryOptions.skip) { selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip); @@ -722,7 +722,7 @@ export class GraphDatabase { async loadEntitiesRelations ( queryRunner: QueryRunner, - block: BlockHeight, + block: CanonicalBlockHeight, relationsMap: Map, entity: new () => Entity, entities: Entity[], @@ -752,7 +752,7 @@ export class GraphDatabase { async loadRelation ( queryRunner: QueryRunner, - block: BlockHeight, + block: CanonicalBlockHeight, relationsMap: Map, relations: { [key: string]: any }, entities: Entity[], @@ -1229,34 +1229,6 @@ export class GraphDatabase { ); } - async _applyBlockHeightFilter ( - queryRunner: QueryRunner, - queryBuilder: SelectQueryBuilder, - block: CanonicalBlockHeight, - alias: string - ): Promise> { - // Block hash takes precedence over number if provided - if (block.hash) { - if (!block.canonicalBlockHashes) { - const { canonicalBlockNumber, blockHashes } = await this._baseDatabase.getFrothyRegion(queryRunner, block.hash); - - // Update the block field to avoid firing the same query further - block.number = canonicalBlockNumber; - block.canonicalBlockHashes = blockHashes; - } - - queryBuilder = queryBuilder - .andWhere(new Brackets(qb => { - qb.where(`${alias}.block_hash IN (:...blockHashes)`, { blockHashes: block.canonicalBlockHashes }) - .orWhere(`${alias}.block_number <= :canonicalBlockNumber`, { canonicalBlockNumber: block.number }); - })); - } else if (block.number) { - queryBuilder = queryBuilder.andWhere(`${alias}.block_number <= :blockNumber`, { blockNumber: block.number }); - } - - return queryBuilder; - } - _measureCachedPrunedEntities (): void { const totalEntities = Array.from(this.cachedEntities.latestPrunedEntities.values()) .reduce((acc, idEntitiesMap) => acc + idEntitiesMap.size, 0);