From c8c5d3dc8fc68dc5128199f5e2ab418ca5c70725 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 15 Jan 2025 15:12:41 +0200 Subject: [PATCH 01/11] code polish (spaces + correct test name) --- .../postgres/pre-agg-allow-non-strict.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts index 7779b7c2ca309..dee82a3d3b875 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts @@ -205,7 +205,7 @@ describe( const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(getCube(true, false, false)); - + it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); const [request] = getQueries(compiler, joinGraph, cubeEvaluator); @@ -266,7 +266,7 @@ describe( const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(getCube(false, true, false)); - + it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); const [request] = getQueries(compiler, joinGraph, cubeEvaluator); @@ -294,7 +294,7 @@ describe( expect(query.indexOf('cube__hourly_data')).toEqual(-1); }); - it('hour query with the `week` granularity match `HourlyData`', async () => { + it('hour query with the `week` granularity match `DailyData`', async () => { await compiler.compile(); const [,,, request] = getQueries(compiler, joinGraph, cubeEvaluator); const [query] = request.buildSqlAndParams(); @@ -327,7 +327,7 @@ describe( const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(getCube(false, false, true)); - + it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); const [request] = getQueries(compiler, joinGraph, cubeEvaluator); @@ -388,7 +388,7 @@ describe( const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(getCube(true, true, false)); - + it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); const [request] = getQueries(compiler, joinGraph, cubeEvaluator); @@ -449,7 +449,7 @@ describe( const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(getCube(true, false, true)); - + it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); const [request] = getQueries(compiler, joinGraph, cubeEvaluator); @@ -510,7 +510,7 @@ describe( const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(getCube(false, true, true)); - + it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); const [request] = getQueries(compiler, joinGraph, cubeEvaluator); @@ -571,7 +571,7 @@ describe( const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(getCube(true, true, true)); - + it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); const [request] = getQueries(compiler, joinGraph, cubeEvaluator); From 90246516fed22eff49c3e9755d68082fd7978889 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 15 Jan 2025 18:42:24 +0200 Subject: [PATCH 02/11] fix(schema-compiler): Fix queries with time dimensions without granularities don't hit pre-aggregations with allow_non_strict_date_range_match=true --- .../src/adapter/BaseTimeDimension.ts | 6 ++-- .../src/adapter/PreAggregations.js | 36 ++++++++++++++----- .../postgres/pre-aggregations.test.ts | 28 +++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts index c5305250c0fa7..12dee4693e5f6 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts @@ -39,7 +39,7 @@ export class BaseTimeDimension extends BaseFilter { } // TODO: find and fix all hidden references to granularity to rely on granularityObj instead? - public get granularity(): string | undefined { + public get granularity(): string | null | undefined { return this.granularityObj?.granularity; } @@ -217,7 +217,7 @@ export class BaseTimeDimension extends BaseFilter { return this.query.dateTimeCast(this.query.paramAllocator.allocateParam(this.dateRange ? this.dateToFormatted() : BUILD_RANGE_END_LOCAL)); } - public dateRangeGranularity() { + public dateRangeGranularity(): string | null { if (!this.dateRange) { return null; } @@ -262,7 +262,7 @@ export class BaseTimeDimension extends BaseFilter { } public resolvedGranularity() { - return this.granularityObj?.resolvedGranularity(); + return this.granularityObj ? this.granularityObj.resolvedGranularity() : this.dateRangeGranularity(); } public isPredefinedGranularity(): boolean { diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index efc642af9d727..96e80c7beea5e 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -439,18 +439,34 @@ export class PreAggregations { static sortTimeDimensionsWithRollupGranularity(timeDimensions) { return timeDimensions && R.sortBy( R.prop(0), - timeDimensions.map(d => (d.isPredefinedGranularity() ? - [d.expressionPath(), d.rollupGranularity(), null] : - // For custom granularities we need to add its name to the list (for exact matches) - [d.expressionPath(), d.rollupGranularity(), d.granularity] - )) + timeDimensions.map(d => { + const res = [d.expressionPath(), d.rollupGranularity()]; + if (d.isPredefinedGranularity()) { + res.push(null); + } else if (d.granularity && d.granularity !== res[1]) { + // For custom granularities we need to add its name to the list (for exact matches) + res.push(d.granularity); + } + return res; + }) ) || []; } static timeDimensionsAsIs(timeDimensions) { return timeDimensions && R.sortBy( R.prop(0), - timeDimensions.map(d => [d.expressionPath(), d.granularity]), + timeDimensions.map(d => { + const res = [d.expressionPath()]; + const resolvedGranularity = d.resolvedGranularity(); + if (d.granularity && d.granularity !== resolvedGranularity) { + // For custom granularities we need to add its name to the list (for exact matches) + res.push(...[resolvedGranularity, d.granularity]); + } else { + // For query timeDimension without granularity we pass the resolved from dataRange as fallback + res.push(resolvedGranularity); + } + return res; + }), ) || []; } @@ -628,13 +644,15 @@ export class PreAggregations { * @returns {Array>} */ const expandTimeDimension = (timeDimension) => { - const [dimension, granularity, customGranularity] = timeDimension; + const [dimension, granularity, customOrResolvedGranularity] = timeDimension; const res = expandGranularity(granularity) .map((newGranularity) => [dimension, newGranularity]); - if (customGranularity) { + if (customOrResolvedGranularity) { // For custom granularities we add it upfront to the list (for exact matches) - res.unshift([dimension, customGranularity]); + // For queries with timeDimension but without granularity specified we use resolved + // granularity from date range + res.unshift([dimension, customOrResolvedGranularity]); } return res; diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index eb24efc8ff4b3..67334caa87a1f 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -229,6 +229,7 @@ describe('PreAggregations', () => { dimensions: [sourceAndId, source], timeDimension: createdAt, granularity: 'hour', + allowNonStrictDateRangeMatch: true }, visitorsMultiplied: { measures: [count], @@ -546,6 +547,33 @@ describe('PreAggregations', () => { }); })); + it('simple pre-aggregation (with no granularity in query)', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.count' + ], + timeDimensions: [{ + dimension: 'visitors.createdAt', + dateRange: ['2017-01-01 00:00:00.000', '2017-01-29 22:59:59.999'] + }], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(queryAndParams[0]).toMatch(/visitors_source_and_id_rollup/); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + visitors__count: '5' + }] + ); + }); + })); + it('leaf measure pre-aggregation', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ From fbf4ccf8e13566e8e3bd00ab8f04fea3154b2f1e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Jan 2025 17:15:28 +0200 Subject: [PATCH 03/11] export some functions from backend-shared/time --- packages/cubejs-backend-shared/src/time.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cubejs-backend-shared/src/time.ts b/packages/cubejs-backend-shared/src/time.ts index 2d795cf074bb8..e828fd978fc73 100644 --- a/packages/cubejs-backend-shared/src/time.ts +++ b/packages/cubejs-backend-shared/src/time.ts @@ -76,7 +76,7 @@ export function subtractInterval(date: moment.Moment, interval: ParsedInterval): /** * Returns the closest date prior to date parameter aligned with the origin point */ -function alignToOrigin(startDate: moment.Moment, interval: ParsedInterval, origin: moment.Moment): moment.Moment { +export const alignToOrigin = (startDate: moment.Moment, interval: ParsedInterval, origin: moment.Moment): moment.Moment => { let alignedDate = startDate.clone(); let intervalOp; let isIntervalNegative = false; @@ -111,9 +111,9 @@ function alignToOrigin(startDate: moment.Moment, interval: ParsedInterval, origi } return alignedDate; -} +}; -function parsedSqlIntervalToDuration(parsedInterval: ParsedInterval): moment.Duration { +export const parsedSqlIntervalToDuration = (parsedInterval: ParsedInterval): moment.Duration => { const duration = moment.duration(); Object.entries(parsedInterval).forEach(([key, value]) => { @@ -121,7 +121,7 @@ function parsedSqlIntervalToDuration(parsedInterval: ParsedInterval): moment.Dur }); return duration; -} +}; function checkSeriesForDateRange(intervalStr: string, [startStr, endStr]: QueryDateRange): void { const intervalParsed = parseSqlInterval(intervalStr); From bc6d208de39418ba1317a450bc249bab0825954f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Jan 2025 17:16:16 +0200 Subject: [PATCH 04/11] implement granularity.isAlignedWithDateRange(...) --- .../src/adapter/Granularity.ts | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index ff595a5261b3a..8bc243142ac6f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -1,8 +1,12 @@ import moment from 'moment-timezone'; import { addInterval, - isPredefinedGranularity, parseSqlInterval, - QueryDateRange, timeSeries, + alignToOrigin, + isPredefinedGranularity, + parsedSqlIntervalToDuration, + parseSqlInterval, + QueryDateRange, + timeSeries, timeSeriesFromCustomInterval, TimeSeriesOptions } from '@cubejs-backend/shared'; @@ -159,6 +163,25 @@ export class Granularity { } } + public isAlignedWithDateRange([startStr, endStr]: QueryDateRange): boolean { + const intervalParsed = parseSqlInterval(this.granularityInterval); + const grIntervalDuration = parsedSqlIntervalToDuration(intervalParsed); + const msFrom = moment.tz(startStr, this.queryTimezone); + const msTo = moment.tz(endStr, this.queryTimezone).add(1, 'ms'); + const dateRangeDuration = moment.duration(msTo.diff(msFrom)); + + if (dateRangeDuration.asMilliseconds() % grIntervalDuration.asMilliseconds() !== 0) { + return false; + } + + const closestDate = alignToOrigin(msFrom, intervalParsed, this.origin); + if (!msFrom.isSame(closestDate)) { + return false; + } + + return true; + } + public isNaturalAligned(): boolean { const intParsed = this.granularityInterval.split(' '); From e2576d69b96afb94a96169de55fd20498ddddc71 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Jan 2025 21:16:39 +0200 Subject: [PATCH 05/11] remove unused/unneeded --- .../src/adapter/BaseTimeDimension.ts | 4 ++-- .../src/adapter/Granularity.ts | 8 -------- .../src/adapter/PreAggregations.js | 20 ------------------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts index 12dee4693e5f6..f1fe2bd9210f4 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts @@ -229,9 +229,9 @@ export class BaseTimeDimension extends BaseFilter { ); } - protected rollupGranularityValue: any | null = null; + protected rollupGranularityValue: string | null = null; - public rollupGranularity() { + public rollupGranularity(): string | null { if (!this.rollupGranularityValue) { this.rollupGranularityValue = this.query.cacheValue( diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index 8bc243142ac6f..e450b2a5cdb94 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -109,14 +109,6 @@ export class Granularity { return timeSeriesFromCustomInterval(this.granularityInterval, dateRange, moment(this.originLocalFormatted()), options); } - public resolvedGranularity(): string { - if (this.predefinedGranularity) { - return this.granularity; - } - - return this.granularityFromInterval(); - } - /** * Returns the smallest granularity for the granularityInterval */ diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index 96e80c7beea5e..a21e0828c35f2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -501,26 +501,6 @@ export class PreAggregations { ); } - canUsePreAggregationAndCheckIfRefValid(query) { - const transformedQuery = PreAggregations.transformQueryToCanUseForm(query); - return (refs) => PreAggregations.canUsePreAggregationForTransformedQueryFn( - transformedQuery, refs - ); - } - - checkAutoRollupPreAggregationValid(refs) { - try { - this.autoRollupPreAggregationQuery(null, refs); // TODO null - return true; - } catch (e) { - if (e instanceof UserError || e.toString().indexOf('ReferenceError') !== -1) { - return false; - } else { - throw e; - } - } - } - /** * Returns function to determine whether pre-aggregation can be used or not * for specified query, or its value for `refs` if specified. From 4382d9ca519ced50ac4caccf8412c11c92ab5ae9 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Jan 2025 22:21:51 +0200 Subject: [PATCH 06/11] fix/implement pre-agg matching --- .../src/adapter/BaseTimeDimension.ts | 13 ++-- .../src/adapter/Granularity.ts | 8 +++ .../src/adapter/PreAggregations.js | 70 ++++++++----------- 3 files changed, 44 insertions(+), 47 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts index f1fe2bd9210f4..fffe78d49a003 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts @@ -241,7 +241,14 @@ export class BaseTimeDimension extends BaseFilter { return this.dateRangeGranularity(); } - return this.query.minGranularity(this.granularityObj.minGranularity(), this.dateRangeGranularity()); + // If we have granularity and date range, we need to check + // that the interval and the granularity offset are stacked/fits with date range + if (this.granularityObj.isPredefined() || + !this.granularityObj.isAlignedWithDateRange([this.dateFromFormatted(), this.dateToFormatted()])) { + return this.query.minGranularity(this.granularityObj.minGranularity(), this.dateRangeGranularity()); + } + + return this.granularityObj.granularity; } ); } @@ -265,10 +272,6 @@ export class BaseTimeDimension extends BaseFilter { return this.granularityObj ? this.granularityObj.resolvedGranularity() : this.dateRangeGranularity(); } - public isPredefinedGranularity(): boolean { - return this.granularityObj?.isPredefined() || false; - } - public wildcardRange() { return [FROM_PARTITION_RANGE, TO_PARTITION_RANGE]; } diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index e450b2a5cdb94..8bc243142ac6f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -109,6 +109,14 @@ export class Granularity { return timeSeriesFromCustomInterval(this.granularityInterval, dateRange, moment(this.originLocalFormatted()), options); } + public resolvedGranularity(): string { + if (this.predefinedGranularity) { + return this.granularity; + } + + return this.granularityFromInterval(); + } + /** * Returns the smallest granularity for the granularityInterval */ diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index a21e0828c35f2..64dd556e3ee2c 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -439,34 +439,14 @@ export class PreAggregations { static sortTimeDimensionsWithRollupGranularity(timeDimensions) { return timeDimensions && R.sortBy( R.prop(0), - timeDimensions.map(d => { - const res = [d.expressionPath(), d.rollupGranularity()]; - if (d.isPredefinedGranularity()) { - res.push(null); - } else if (d.granularity && d.granularity !== res[1]) { - // For custom granularities we need to add its name to the list (for exact matches) - res.push(d.granularity); - } - return res; - }) + timeDimensions.map(d => [d.expressionPath(), d.rollupGranularity(), d.granularityObj?.minGranularity()]) ) || []; } static timeDimensionsAsIs(timeDimensions) { return timeDimensions && R.sortBy( R.prop(0), - timeDimensions.map(d => { - const res = [d.expressionPath()]; - const resolvedGranularity = d.resolvedGranularity(); - if (d.granularity && d.granularity !== resolvedGranularity) { - // For custom granularities we need to add its name to the list (for exact matches) - res.push(...[resolvedGranularity, d.granularity]); - } else { - // For query timeDimension without granularity we pass the resolved from dataRange as fallback - res.push(resolvedGranularity); - } - return res; - }), + timeDimensions.map(d => [d.expressionPath(), d.resolvedGranularity(), d.granularityObj?.minGranularity()]), ) || []; } @@ -557,8 +537,8 @@ export class PreAggregations { const qryTimeDimensions = references.allowNonStrictDateRangeMatch ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions.map(t => t.slice(0, 2)); - // slice above is used to exclude possible custom granularity returned from sortTimeDimensionsWithRollupGranularity() - + // slices above/below are used to exclude granularity on 3rd position returned from sortTimeDimensionsWithRollupGranularity() + const qryDimensions = transformedQuery.timeDimensions.map(t => t.slice(0, 2)); const backAliasMeasures = backAlias(references.measures); const backAliasSortedDimensions = backAlias(references.sortedDimensions || references.dimensions); const backAliasDimensions = backAlias(references.dimensions); @@ -570,7 +550,7 @@ export class PreAggregations { R.equals(qryTimeDimensions, refTimeDimensions) ) && ( transformedQuery.isAdditive || - R.equals(transformedQuery.timeDimensions, refTimeDimensions) + R.equals(qryDimensions, refTimeDimensions) ) && ( filterDimensionsSingleValueEqual && references.dimensions.length === filterDimensionsSingleValueEqual.size && @@ -585,14 +565,29 @@ export class PreAggregations { }; /** - * Wrap granularity string into an array. - * @param {string} granularity + * Expand granularity into array of granularity hierarchy. + * resolvedGranularity might be a custom granularity name, in this case + * we insert it into all expanded hierarchies using its minimal granularity passed + * as minGranularity param + * @param {string} resolvedGranularity + * @param {string} minGranularity * @returns {Array} */ - const expandGranularity = (granularity) => ( - transformedQuery.granularityHierarchies[granularity] || - [granularity] - ); + const expandGranularity = (resolvedGranularity, minGranularity) => { + if (!resolvedGranularity) { + return []; + } + + let predefinedHierarchy = transformedQuery.granularityHierarchies[resolvedGranularity]; + + if (predefinedHierarchy) { + return predefinedHierarchy; + } + + predefinedHierarchy = transformedQuery.granularityHierarchies[minGranularity]; + // Custom granularity should be the first in list for exact match hit + return [resolvedGranularity, ...predefinedHierarchy]; + }; /** * Determine whether time dimensions match to the window granularity or not. @@ -624,18 +619,9 @@ export class PreAggregations { * @returns {Array>} */ const expandTimeDimension = (timeDimension) => { - const [dimension, granularity, customOrResolvedGranularity] = timeDimension; - const res = expandGranularity(granularity) + const [dimension, resolvedGranularity, minGranularity] = timeDimension; + return expandGranularity(resolvedGranularity, minGranularity) .map((newGranularity) => [dimension, newGranularity]); - - if (customOrResolvedGranularity) { - // For custom granularities we add it upfront to the list (for exact matches) - // For queries with timeDimension but without granularity specified we use resolved - // granularity from date range - res.unshift([dimension, customOrResolvedGranularity]); - } - - return res; }; /** From aabc5ab2ebce712a2df18f2f6fa199d276004984 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Jan 2025 22:22:05 +0200 Subject: [PATCH 07/11] =?UTF-8?q?fix=20test=20for=C2=A0pre-agg=20with=20al?= =?UTF-8?q?lowNonStrictDateRangeMatch:=20true?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../postgres/pre-aggregations.test.ts | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 67334caa87a1f..91c5a0ebb2fc0 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -547,14 +547,15 @@ describe('PreAggregations', () => { }); })); - it('simple pre-aggregation (with no granularity in query)', () => compiler.compile().then(() => { + it('simple pre-aggregation (allowNonStrictDateRangeMatch: true)', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.count' ], timeDimensions: [{ dimension: 'visitors.createdAt', - dateRange: ['2017-01-01 00:00:00.000', '2017-01-29 22:59:59.999'] + dateRange: ['2017-01-01 00:10:00.000', '2017-01-29 22:59:59.999'], + granularity: 'hour', }], timezone: 'America/Los_Angeles', preAggregationsSchema: '' @@ -567,9 +568,24 @@ describe('PreAggregations', () => { return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { expect(res).toEqual( - [{ - visitors__count: '5' - }] + [ + { + visitors__count: '1', + visitors__created_at_hour: '2017-01-02T16:00:00.000Z', + }, + { + visitors__count: '1', + visitors__created_at_hour: '2017-01-04T16:00:00.000Z', + }, + { + visitors__count: '1', + visitors__created_at_hour: '2017-01-05T16:00:00.000Z', + }, + { + visitors__count: '2', + visitors__created_at_hour: '2017-01-06T16:00:00.000Z', + }, + ] ); }); })); From 7d49c3be7b1682be4c3e884bfa180f5f627f831a Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Jan 2025 23:47:32 +0200 Subject: [PATCH 08/11] =?UTF-8?q?test=20for=C2=A0custom=20granularity=20ma?= =?UTF-8?q?tch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../postgres/pre-aggregations.test.ts | 56 ++++++++++++++++++- .../test/unit/pre-agg-time-dim-match.test.ts | 48 +++++++++++++--- 2 files changed, 94 insertions(+), 10 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 91c5a0ebb2fc0..b3d191d8f5066 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -94,7 +94,13 @@ describe('PreAggregations', () => { }, createdAt: { type: 'time', - sql: 'created_at' + sql: 'created_at', + granularities: { + hourTenMinOffset: { + interval: '1 hour', + offset: '10 minutes' + } + } }, signedUpAt: { type: 'time', @@ -224,6 +230,11 @@ describe('PreAggregations', () => { granularity: 'hour', partitionGranularity: 'month' }, + countCustomGranularity: { + measures: [count], + timeDimension: createdAt, + granularity: 'hourTenMinOffset' + }, sourceAndIdRollup: { measures: [count], dimensions: [sourceAndId, source], @@ -590,6 +601,49 @@ describe('PreAggregations', () => { }); })); + it('simple pre-aggregation with custom granularity (exact match)', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.count' + ], + timeDimensions: [{ + dimension: 'visitors.createdAt', + dateRange: ['2017-01-01 00:10:00.000', '2017-01-29 22:09:59.999'], + granularity: 'hourTenMinOffset', + }], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(queryAndParams[0]).toMatch(/visitors_count_custom_granularity/); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [ + { + visitors__count: '1', + visitors__created_at_hourTenMinOffset: '2017-01-02T15:10:00.000Z', + }, + { + visitors__count: '1', + visitors__created_at_hourTenMinOffset: '2017-01-04T15:10:00.000Z', + }, + { + visitors__count: '1', + visitors__created_at_hourTenMinOffset: '2017-01-05T15:10:00.000Z', + }, + { + visitors__count: '2', + visitors__created_at_hourTenMinOffset: '2017-01-06T15:10:00.000Z', + }, + ] + ); + }); + })); + it('leaf measure pre-aggregation', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index 83120f1564950..326f1f725134a 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -16,7 +16,8 @@ describe('Pre Aggregation by filter match tests', () => { }, one_week_by_sunday: { interval: '1 week', - offset: '-1 day' + // offset: '-1 day' // offsets might lead to flaky tests through years + origin: '2025-01-05 00:00:00' }, two_weeks_by_1st_feb_00am: { interval: '2 weeks', @@ -38,6 +39,8 @@ describe('Pre Aggregation by filter match tests', () => { preAggTimeGranularity: string, queryAggTimeGranularity: string, queryTimeZone: string = 'America/Los_Angeles', + dateRange: [ string, string ] = ['2017-01-01', '2017-03-31'], + allowNonStrictDateRangeMatch: boolean = false ) { const aaa: any = { type: 'rollup', @@ -46,8 +49,7 @@ describe('Pre Aggregation by filter match tests', () => { timeDimension: 'cube.created', granularity: preAggTimeGranularity, partitionGranularity: 'year', - // Enabling only for custom granularities - allowNonStrictDateRangeMatch: !/^(minute|hour|day|week|month|quarter|year)$/.test(preAggTimeGranularity) + allowNonStrictDateRangeMatch }; const cube: any = { @@ -65,7 +67,7 @@ describe('Pre Aggregation by filter match tests', () => { // aaa.sortedDimensions = aaa.dimensions; // aaa.sortedDimensions.sort(); - aaa.sortedTimeDimensions = [[aaa.timeDimension, aaa.granularity]]; + aaa.sortedTimeDimensions = [[aaa.timeDimension, aaa.granularity, 'day']]; return compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -73,7 +75,7 @@ describe('Pre Aggregation by filter match tests', () => { timeDimensions: [{ dimension: 'cube.created', granularity: queryAggTimeGranularity, - dateRange: { from: '2017-01-01', to: '2017-03-31' } + dateRange, }], timezone: queryTimeZone, }); @@ -92,15 +94,43 @@ describe('Pre Aggregation by filter match tests', () => { )); it('1 count measure, one_week_by_sunday, one_week_by_sunday', () => testPreAggregationMatch( - true, ['count'], 'one_week_by_sunday', 'one_week_by_sunday' + true, + ['count'], + 'one_week_by_sunday', + 'one_week_by_sunday', + 'UTC', + ['2024-02-11', '2024-03-02'] )); - it('1 count measure, two_weeks_by_1st_feb_00am, two_weeks_by_1st_feb_00am', () => testPreAggregationMatch( - true, ['count'], 'two_weeks_by_1st_feb_00am', 'two_weeks_by_1st_feb_00am' + it('1 count measure, one_week_by_sunday, one_week_by_sunday (dst)', () => testPreAggregationMatch( + true, + ['count'], + 'one_week_by_sunday', + 'one_week_by_sunday', + 'UTC', + ['2024-02-25', '2024-03-30'], // DST Switch happens here, but still must work! + )); + + it('1 count measure, two_weeks_by_1st_feb_00am, two_weeks_by_1st_feb_00am (match)', () => testPreAggregationMatch( + true, + ['count'], + 'two_weeks_by_1st_feb_00am', + 'two_weeks_by_1st_feb_00am', + 'UTC', + ['2024-01-18', '2024-02-28'] + )); + + it('1 count measure, two_weeks_by_1st_feb_00am, two_weeks_by_1st_feb_00am (miss)', () => testPreAggregationMatch( + false, + ['count'], + 'two_weeks_by_1st_feb_00am', + 'two_weeks_by_1st_feb_00am', + 'UTC', + ['2024-01-18', '2024-02-07'], // Interval not aligned )); it('1 count measure, day, one_week_by_sunday', () => testPreAggregationMatch( - true, ['count'], 'day', 'one_week_by_sunday' + true, ['count'], 'day', 'one_week_by_sunday', 'UTC' )); it('1 count measure, day, two_weeks_by_1st_feb_00am', () => testPreAggregationMatch( From 1581867c9d7588b87719898abb8d73f68f094052 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 29 Jan 2025 17:38:04 +0200 Subject: [PATCH 09/11] =?UTF-8?q?fix=20isAlignedWithDateRange=20for=C2=A0D?= =?UTF-8?q?STs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cubejs-schema-compiler/src/adapter/Granularity.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index 8bc243142ac6f..d635bd3c6be40 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -168,9 +168,13 @@ export class Granularity { const grIntervalDuration = parsedSqlIntervalToDuration(intervalParsed); const msFrom = moment.tz(startStr, this.queryTimezone); const msTo = moment.tz(endStr, this.queryTimezone).add(1, 'ms'); - const dateRangeDuration = moment.duration(msTo.diff(msFrom)); - if (dateRangeDuration.asMilliseconds() % grIntervalDuration.asMilliseconds() !== 0) { + // We can't simply compare interval milliseconds because of DSTs. + const testDate = msFrom.clone(); + while (testDate.isBefore(msTo)) { + testDate.add(grIntervalDuration); + } + if (!testDate.isSame(msTo)) { return false; } From 52e36a5fcc2d071595e2d3fbcf6d5099910db318 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 29 Jan 2025 17:38:12 +0200 Subject: [PATCH 10/11] fix test --- .../test/unit/pre-agg-time-dim-match.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index 326f1f725134a..32f644fdd6a34 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -107,7 +107,7 @@ describe('Pre Aggregation by filter match tests', () => { ['count'], 'one_week_by_sunday', 'one_week_by_sunday', - 'UTC', + 'America/Los_Angeles', ['2024-02-25', '2024-03-30'], // DST Switch happens here, but still must work! )); From 73895567de67542caf6bd376010d84ef7df5618c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 31 Jan 2025 13:20:08 +0200 Subject: [PATCH 11/11] =?UTF-8?q?cached=20granularityHierarchies()=20for?= =?UTF-8?q?=C2=A0query=20timezone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/adapter/BaseQuery.js | 46 ++++++++++++++- .../src/adapter/BaseTimeDimension.ts | 4 ++ .../src/adapter/PreAggregations.js | 56 +++++++------------ .../src/compiler/CubeEvaluator.ts | 13 ++++- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index f7154b203e0e6..92e2e14e6acec 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -26,6 +26,7 @@ import { BaseTimeDimension } from './BaseTimeDimension'; import { ParamAllocator } from './ParamAllocator'; import { PreAggregations } from './PreAggregations'; import { SqlParser } from '../parser/SqlParser'; +import { Granularity } from './Granularity'; const DEFAULT_PREAGGREGATIONS_SCHEMA = 'stb_pre_aggregations'; @@ -1379,7 +1380,47 @@ export class BaseQuery { } granularityHierarchies() { - return R.fromPairs(Object.keys(standardGranularitiesParents).map(k => [k, this.granularityParentHierarchy(k)])); + return this.cacheValue( + // If time dimension custom granularity in data model is defined without + // timezone information they are treated in query timezone. + // Because of that it's not possible to correctly precalculate + // granularities hierarchies on startup as they are specific for each timezone. + ['granularityHierarchies', this.timezone], + () => R.reduce( + (hierarchies, cube) => R.reduce( + (acc, [tdName, td]) => { + const dimensionKey = `${cube}.${tdName}`; + + // constructing standard granularities for time dimension + const standardEntries = R.fromPairs( + R.keys(standardGranularitiesParents).map(gr => [ + `${dimensionKey}.${gr}`, + standardGranularitiesParents[gr], + ]), + ); + + // If we have custom granularities in time dimension + const customEntries = td.granularities + ? R.fromPairs( + R.keys(td.granularities).map(granularityName => { + const grObj = new Granularity(this, { dimension: dimensionKey, granularity: granularityName }); + return [ + `${dimensionKey}.${granularityName}`, + [granularityName, ...standardGranularitiesParents[grObj.minGranularity()]], + ]; + }), + ) + : {}; + + return { ...acc, ...standardEntries, ...customEntries }; + }, + hierarchies, + R.toPairs(this.cubeEvaluator.timeDimensionsForCube(cube)), + ), + {}, + R.keys(this.cubeEvaluator.evaluatedCubes), + ), + ); } granularityParentHierarchy(granularity) { @@ -1646,7 +1687,8 @@ export class BaseQuery { /** * - * @param {{sql: string, on: {cubeName: string, expression: Function}, joinType: 'LEFT' | 'INNER', alias: string}} customJoin + * @param {{sql: string, on: {cubeName: string, expression: Function}, joinType: 'LEFT' | 'INNER', alias: string}} + * customJoin * @returns {JoinItem} */ customSubQueryJoin(customJoin) { diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts index fffe78d49a003..7f8869bc4968a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts @@ -241,6 +241,10 @@ export class BaseTimeDimension extends BaseFilter { return this.dateRangeGranularity(); } + if (!this.dateRange) { + return this.granularityObj.minGranularity(); + } + // If we have granularity and date range, we need to check // that the interval and the granularity offset are stacked/fits with date range if (this.granularityObj.isPredefined() || diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index 64dd556e3ee2c..cd2b1c2015f22 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -439,14 +439,14 @@ export class PreAggregations { static sortTimeDimensionsWithRollupGranularity(timeDimensions) { return timeDimensions && R.sortBy( R.prop(0), - timeDimensions.map(d => [d.expressionPath(), d.rollupGranularity(), d.granularityObj?.minGranularity()]) + timeDimensions.map(d => [d.expressionPath(), d.rollupGranularity()]) ) || []; } static timeDimensionsAsIs(timeDimensions) { return timeDimensions && R.sortBy( R.prop(0), - timeDimensions.map(d => [d.expressionPath(), d.resolvedGranularity(), d.granularityObj?.minGranularity()]), + timeDimensions.map(d => [d.expressionPath(), d.resolvedGranularity()]), ) || []; } @@ -536,9 +536,7 @@ export class PreAggregations { backAlias(references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions)); const qryTimeDimensions = references.allowNonStrictDateRangeMatch ? transformedQuery.timeDimensions - : transformedQuery.sortedTimeDimensions.map(t => t.slice(0, 2)); - // slices above/below are used to exclude granularity on 3rd position returned from sortTimeDimensionsWithRollupGranularity() - const qryDimensions = transformedQuery.timeDimensions.map(t => t.slice(0, 2)); + : transformedQuery.sortedTimeDimensions; const backAliasMeasures = backAlias(references.measures); const backAliasSortedDimensions = backAlias(references.sortedDimensions || references.dimensions); const backAliasDimensions = backAlias(references.dimensions); @@ -550,7 +548,7 @@ export class PreAggregations { R.equals(qryTimeDimensions, refTimeDimensions) ) && ( transformedQuery.isAdditive || - R.equals(qryDimensions, refTimeDimensions) + R.equals(transformedQuery.timeDimensions, refTimeDimensions) ) && ( filterDimensionsSingleValueEqual && references.dimensions.length === filterDimensionsSingleValueEqual.size && @@ -566,28 +564,14 @@ export class PreAggregations { /** * Expand granularity into array of granularity hierarchy. - * resolvedGranularity might be a custom granularity name, in this case - * we insert it into all expanded hierarchies using its minimal granularity passed - * as minGranularity param - * @param {string} resolvedGranularity - * @param {string} minGranularity + * @param {string} dimension Dimension in the form of `cube.timeDimension` + * @param {string} granularity Granularity * @returns {Array} */ - const expandGranularity = (resolvedGranularity, minGranularity) => { - if (!resolvedGranularity) { - return []; - } - - let predefinedHierarchy = transformedQuery.granularityHierarchies[resolvedGranularity]; - - if (predefinedHierarchy) { - return predefinedHierarchy; - } - - predefinedHierarchy = transformedQuery.granularityHierarchies[minGranularity]; - // Custom granularity should be the first in list for exact match hit - return [resolvedGranularity, ...predefinedHierarchy]; - }; + const expandGranularity = (dimension, granularity) => ( + transformedQuery.granularityHierarchies[`${dimension}.${granularity}`] || + [granularity] + ); /** * Determine whether time dimensions match to the window granularity or not. @@ -602,15 +586,15 @@ export class PreAggregations { references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions); - return expandGranularity(transformedQuery.windowGranularity) - .map( - windowGranularity => R.all( - td => td[1] === windowGranularity, - sortedTimeDimensions, + return sortedTimeDimensions + .map(td => expandGranularity(td[0], transformedQuery.windowGranularity)) + .some( + expandedGranularities => expandedGranularities.some( + windowGranularity => sortedTimeDimensions.every( + td => td[1] === windowGranularity + ) ) - ) - .filter(x => !!x) - .length > 0; + ); }; /** @@ -619,8 +603,8 @@ export class PreAggregations { * @returns {Array>} */ const expandTimeDimension = (timeDimension) => { - const [dimension, resolvedGranularity, minGranularity] = timeDimension; - return expandGranularity(resolvedGranularity, minGranularity) + const [dimension, resolvedGranularity] = timeDimension; + return expandGranularity(dimension, resolvedGranularity) .map((newGranularity) => [dimension, newGranularity]); }; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index b5bf5bbb29535..56dbd50f12d1c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -448,8 +448,8 @@ export class CubeEvaluator extends CubeSymbols { public timeDimensionPathsForCube(cube: any) { return R.compose( - R.map(nameToDefinition => `${cube}.${nameToDefinition[0]}`), - R.toPairs, + R.map(dimName => `${cube}.${dimName}`), + R.keys, // @ts-ignore R.filter((d: any) => d.type === 'time') // @ts-ignore @@ -460,12 +460,19 @@ export class CubeEvaluator extends CubeSymbols { return this.cubeFromPath(cube).measures || {}; } + public timeDimensionsForCube(cube) { + return R.filter( + (d: any) => d.type === 'time', + this.cubeFromPath(cube).dimensions || {} + ); + } + public preAggregationsForCube(path: string) { return this.cubeFromPath(path).preAggregations || {}; } /** - * Returns pre-aggregations filtered by the spcified selector. + * Returns pre-aggregations filtered by the specified selector. * @param {{ * scheduled: boolean, * dataSource: Array,