Skip to content

Commit

Permalink
Relation selection and filtering (#453)
Browse files Browse the repository at this point in the history 
* feat: relation filtering feature + relation selection fix

affects: @joystream/hydra-cli, @joystream/hydra-indexer-gateway

* feat: relation filtering feature + relation selection fix II

affects: @joystream/hydra-cli

* feat: relation filtering feature + relation selection fix III + dependency update

affects: @joystream/hydra-cli, @joystream/hydra-indexer-gateway, @joystream/hydra-processor,
@joystream/hydra-typegen, sample
  • Loading branch information
@ondratra
ondratra authored Oct 30, 2021
1 parent fd03b32 commit 59cb3bc
Show file tree
Hide file tree
Showing 31 changed files with 3,470 additions and 11,800 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
"prettier": "2.0.2"
},
"resolutions": {
"typeorm": "0.2.34",
"@polkadot/types": "4.16.2"
},
"lint-staged": {
Expand Down
3 changes: 1 addition & 2 deletions packages/hydra-cli/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,7 @@
"lodash": "^4.17.20",
"pg": "^8.3.2",
"pg-listen": "^1.7.0",
"@joystream/warthog": "^2.40.0",
"typeorm": "^0.2.31"
"@joystream/warthog": "^2.40.0"
},
"devDependencies": {
"@oclif/dev-cli": "^1",
Expand Down
9 changes: 9 additions & 0 deletions packages/hydra-cli/src/codegen/WarthogWrapper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,9 @@ export default class WarthogWrapper {
scripts: Record<string, string>
dependencies: Record<string, string>
devDependencies: Record<string, string>

// temporary fix for specific typeorm version needed (see https://github.com/Joystream/hydra/pull/453 for more info)
resolutions: Record<string, string>
}

// Ensure version is greater than '0.0.0'
Expand All @@ -207,6 +210,12 @@ export default class WarthogWrapper {
...extraDependencies,
}

// temporary fix for specific typeorm version needed (see https://github.com/Joystream/hydra/pull/453 for more info)
pkgFile.resolutions = {
...(pkgFile.resolutions || {}),
typeorm: '0.2.34',
}

debug(`Writing package.json: ${JSON.stringify(pkgFile, null, 2)}`)

fs.writeFileSync('package.json', JSON.stringify(pkgFile, null, 2))
Expand Down
148 changes: 5 additions & 143 deletions packages/hydra-cli/src/templates/entities/service.ts.mst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,150 +62,12 @@ export class {{className}}Service extends WarthogBaseService<{{className}}> {
limit?: number,
offset?: number,
fields?: string[]): Promise<{{className}}[]> {

const where = <{{className}}WhereInput>(_where || {})

{{#fieldResolvers}}
// remove relation filters to enable warthog query builders
{{#relationType.isTO}}
const { {{fieldName}} } = where
delete where.{{fieldName}};
{{/relationType.isTO}}

{{#relationType.isTM}}
const { {{fieldName}}_some, {{fieldName}}_none, {{fieldName}}_every } = where

if ((+!!{{fieldName}}_some) + (+!!{{fieldName}}_none) + (+!!{{fieldName}}_every) > 1) {
throw new Error(`A query can have at most one of none, some, every clauses on a relation field`)
}

delete where.{{fieldName}}_some;
delete where.{{fieldName}}_none;
delete where.{{fieldName}}_every;
{{/relationType.isTM}}
{{/fieldResolvers}}

let mainQuery = this.buildFindQueryWithParams(
<any>where,
orderBy,
undefined,
fields,
'main'
).take(undefined); // remove LIMIT

let parameters = mainQuery.getParameters();

{{#crossFilters}}

{{#fieldResolvers}}
{{#relationType.isTO}}

if ({{fieldName}}) {
// OTO or MTO
const {{fieldName}}Query = this.{{fieldName}}Service.buildFindQueryWithParams(
<any>{{fieldName}},
undefined,
undefined,
['id'],
'{{fieldName}}'
).take(undefined); // remove the default LIMIT


mainQuery = mainQuery
.andWhere(`"{{typeormAliasName}}"."{{fieldNameColumnName}}_id" IN (${ {{fieldName}}Query.getQuery() })`);

parameters = { ...parameters, ...{{fieldName}}Query.getParameters() };

}
{{/relationType.isTO}}

{{#relationType.isTM}}

const {{fieldName}}Filter = {{fieldName}}_some || {{fieldName}}_none || {{fieldName}}_every

if ({{fieldName}}Filter) {

const {{fieldName}}Query = this.{{fieldName}}Service.buildFindQueryWithParams(<any>{{fieldName}}Filter,
undefined,
undefined,
['id'],
'{{fieldName}}'
).take(undefined); //remove the default LIMIT

parameters = { ...parameters, ...{{fieldName}}Query.getParameters() }

const subQueryFiltered = this
.getQueryBuilder()
.select([])
.leftJoin(
'{{typeormAliasName}}.{{fieldName}}',
'{{fieldName}}_filtered',
`{{fieldName}}_filtered.id IN (${ {{fieldName}}Query.getQuery() })`
)
.groupBy('{{typeormAliasName}}_id')
.addSelect('count({{fieldName}}_filtered.id)', 'cnt_filtered')
.addSelect('{{typeormAliasName}}.id', '{{typeormAliasName}}_id');

const subQueryTotal = this
.getQueryBuilder()
.select([])
.leftJoin('{{typeormAliasName}}.{{fieldName}}', '{{fieldName}}_total')
.groupBy('{{typeormAliasName}}_id')
.addSelect('count({{fieldName}}_total.id)', 'cnt_total')
.addSelect('{{typeormAliasName}}.id', '{{typeormAliasName}}_id');

const subQuery = `
SELECT
f.{{typeormAliasName}}_id {{typeormAliasName}}_id, f.cnt_filtered cnt_filtered, t.cnt_total cnt_total
FROM
(${subQueryTotal.getQuery()}) t, (${subQueryFiltered.getQuery()}) f
WHERE
t.{{typeormAliasName}}_id = f.{{typeormAliasName}}_id`;


if ({{fieldName}}_none) {
mainQuery = mainQuery.andWhere(`{{typeormAliasName}}.id IN
(SELECT
{{fieldName}}_subq.{{typeormAliasName}}_id
FROM
(${subQuery}) {{fieldName}}_subq
WHERE
{{fieldName}}_subq.cnt_filtered = 0
)`)
}

if ({{fieldName}}_some) {
mainQuery = mainQuery.andWhere(`{{typeormAliasName}}.id IN
(SELECT
{{fieldName}}_subq.{{typeormAliasName}}_id
FROM
(${subQuery}) {{fieldName}}_subq
WHERE
{{fieldName}}_subq.cnt_filtered > 0
)`)
}

if ({{fieldName}}_every) {
mainQuery = mainQuery.andWhere(`{{typeormAliasName}}.id IN
(SELECT
{{fieldName}}_subq.{{typeormAliasName}}_id
FROM
(${subQuery}) {{fieldName}}_subq
WHERE
{{fieldName}}_subq.cnt_filtered > 0
AND {{fieldName}}_subq.cnt_filtered = {{fieldName}}_subq.cnt_total
)`)
}
}
{{/relationType.isTM}}

{{/fieldResolvers}}

{{/crossFilters}}

mainQuery = mainQuery.setParameters(parameters);
const limitOffset = {
limit: limit || 50,
offset
};

return mainQuery.take(limit || 50).skip(offset || 0).getMany();
return this.buildFindQuery(_where, orderBy, limitOffset, fields).getMany();
}


Expand Down
136 changes: 10 additions & 126 deletions packages/hydra-cli/src/templates/graphql-server/src/WarthogBaseService.ts.mst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,146 +16,30 @@ export class WarthogBaseService<E extends BaseModel> extends BaseService<E> {
orderBy?: string | string[],
pageOptions?: LimitOffset,
fields?: string[],
paramKeyPrefix: string = 'param',
aliases: (field: string) => string | undefined = () => undefined
): SelectQueryBuilder<E> {
const DEFAULT_LIMIT = 50;
let qb = this.manager.createQueryBuilder<E>(this.entityClass, this.klass);
if (!pageOptions) {
pageOptions = {
limit: DEFAULT_LIMIT,
};
}

qb = qb.take(pageOptions.limit || DEFAULT_LIMIT);

if (pageOptions.offset) {
qb = qb.skip(pageOptions.offset);
}
let qb = this.buildFindQuery(where, undefined, pageOptions);

if (fields) {
// We always need to select ID or dataloaders will not function properly
// We always need to select ID or dataloaders will not function properly.
if (fields.indexOf('id') === -1) {
fields.push('id');
}

// Querybuilder requires you to prefix all fields with the table alias. It also requires you to
// specify the field name using it's TypeORM attribute name, not the camel-cased DB column name
qb = qb.select(`${this.klass}.id`, aliases('id'));
fields.forEach(
(field) => field !== 'id' && qb.addSelect(`${this.klass}.${field}`, aliases(field))
);
}

qb = addOrderBy(orderBy, qb, (attr) => this.attrToDBColumn(attr));

// Soft-deletes are filtered out by default, setting `deletedAt_all` is the only way to turn this off
const hasDeletedAts = Object.keys(where).find((key) => key.indexOf('deletedAt_') === 0);
// If no deletedAt filters specified, hide them by default
if (!hasDeletedAts) {
// eslint-disable-next-line @typescript-eslint/camelcase
where.deletedAt_eq = null; // Filter out soft-deleted items
} else if (typeof where.deletedAt_all !== 'undefined') {
// Delete this param so that it doesn't try to filter on the magic `all` param
// Put this here so that we delete it even if `deletedAt_all: false` specified
delete where.deletedAt_all;
} else {
// If we get here, the user has added a different deletedAt filter, like deletedAt_gt: <date>
// do nothing because the specific deleted at filters will be added by processWhereOptions
}

// Keep track of a counter so that TypeORM doesn't reuse our variables that get passed into the query if they
// happen to reference the same column
const paramKeyCounter = { counter: 0 };
const processWheres = (
qb: SelectQueryBuilder<E>,
where: WhereFilterAttributes
): SelectQueryBuilder<E> => {
// where is of shape { userName_contains: 'a' }
Object.keys(where).forEach((k: string) => {
const paramKey = `${paramKeyPrefix}${paramKeyCounter.counter}`;
// increment counter each time we add a new where clause so that TypeORM doesn't reuse our input variables
paramKeyCounter.counter = paramKeyCounter.counter + 1;
const key = k as keyof W; // userName_contains
const parts = key.toString().split('_'); // ['userName', 'contains']
const attr = parts[0]; // userName
const operator = parts.length > 1 ? parts[1] : 'eq'; // contains

return addQueryBuilderWhereItem(
qb,
paramKey,
this.attrToDBColumn(attr),
operator,
where[key]
);
});
return qb;
};

// WhereExpression comes in the following shape:
// {
// AND?: WhereInput[];
// OR?: WhereInput[];
// [key: string]: string | number | null;
// }
const processWhereInput = (
qb: SelectQueryBuilder<E>,
where: WhereExpression
): SelectQueryBuilder<E> => {
const { AND, OR, ...rest } = where;

if (AND && AND.length) {
const ands = AND.filter((value) => JSON.stringify(value) !== '{}');
if (ands.length) {
qb.andWhere(
new Brackets((qb2) => {
ands.forEach((where: WhereExpression) => {
if (Object.keys(where).length === 0) {
return; // disregard empty where objects
}
qb2.andWhere(
new Brackets((qb3) => {
processWhereInput(qb3 as SelectQueryBuilder<any>, where);
return qb3;
})
);
});
})
);
}
}

if (OR && OR.length) {
const ors = OR.filter((value) => JSON.stringify(value) !== '{}');
if (ors.length) {
qb.andWhere(
new Brackets((qb2) => {
ors.forEach((where: WhereExpression) => {
if (Object.keys(where).length === 0) {
return; // disregard empty where objects
}

qb2.orWhere(
new Brackets((qb3) => {
processWhereInput(qb3 as SelectQueryBuilder<any>, where);
return qb3;
})
);
});
})
);
fields.forEach((field) => {
if (field === 'id' || !this.columnMap[field]) {
return;
}
}

if (rest) {
processWheres(qb, rest);
}
return qb;
};

if (Object.keys(where).length) {
processWhereInput(qb, where);
qb = qb.addSelect(`${this.klass}.${field}`, aliases(field));
});
}

qb = addOrderBy(orderBy, qb, (attr) => this.attrToDBColumn(attr));

return qb;
}
}
Expand Down
5 changes: 2 additions & 3 deletions packages/hydra-cli/src/templates/interfaces/service.ts.mst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export class {{className}}Service {
@Inject('{{className}}Service') public readonly {{camelName}}Service: {{className}}Service,
{{/subclasses}}
) {
{{#subclasses}}
{{#subclasses}}
this.typeToService['{{className}}'] = {{camelName}}Service;
{{/subclasses}}
}
Expand Down Expand Up @@ -74,7 +74,6 @@ export class {{className}}Service {
undefined,
undefined,
commonFields,
camelCase(t),
(field) => snakeCase(field)
)
.addSelect(`'${t}'`, 'type')
Expand Down Expand Up @@ -112,7 +111,7 @@ export class {{className}}Service {
limit,
undefined,
fields.filter((f) => service.columnMap[f] !== undefined)
);
).then(tmpResults => tmpResults.map(item => (item.type = t, item)));
});

const result = (await Promise.all<Event[]>(entityPromises)).reduce(
Expand Down
2 changes: 1 addition & 1 deletion packages/hydra-cli/src/templates/scaffold/mappings/package.json.mst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"dependencies": {
"@joystream/hydra-common": "{{{hydraCommonVersion}}}",
"@polkadot/types": "~2.10.2-7",
"warthog": "{{{hydraWarthogVersion}}}"
"@joystream/warthog": "{{{hydraWarthogVersion}}}"
},
"devDependencies": {
"copyfiles": "^2.4.1",
Expand Down
Loading

0 comments on commit 59cb3bc

Please sign in to comment.