From 849cd73c3b6e63309393b1e16263600fe7455818 Mon Sep 17 00:00:00 2001 From: Matt <1009003+tantaman@users.noreply.github.com> Date: Sat, 9 Mar 2024 07:07:33 -0500 Subject: [PATCH] [wip] organize and document things for code review --- src/zql/README.md | 101 +++++++++++++++++++++ src/zql/ast-to-ivm/pipelineBuilder.test.ts | 2 +- src/zql/ast-to-ivm/pipelineBuilder.ts | 2 +- src/zql/{query => ast}/ZqlAst.ts | 0 src/zql/context/context.ts | 43 +++++++++ src/zql/index.ts | 2 + src/zql/query/EntityQuery.test.ts | 2 +- src/zql/query/EntityQuery.ts | 4 +- src/zql/query/IEntityQuery.ts | 2 +- src/zql/query/Statement.test.ts | 2 +- src/zql/query/Statement.ts | 4 +- src/zql/query/context/contextProvider.ts | 43 --------- 12 files changed, 155 insertions(+), 52 deletions(-) create mode 100644 src/zql/README.md rename src/zql/{query => ast}/ZqlAst.ts (100%) create mode 100644 src/zql/context/context.ts create mode 100644 src/zql/index.ts delete mode 100644 src/zql/query/context/contextProvider.ts diff --git a/src/zql/README.md b/src/zql/README.md new file mode 100644 index 0000000..9f4bd78 --- /dev/null +++ b/src/zql/README.md @@ -0,0 +1,101 @@ +# ZQL + +[EntityQuery](./query/EntityQuery.ts) is the main entrypoint for everything query related: + +- building +- preparing +- running +- and materializing queries + +# Creating an EntityQuery + +First, build your schema for rails as you normally would: + +```ts +const issueSchema = z.object({ + id: z.string(), + title: z.string(), + created: z.date(), + status: z.enum(['active', 'inactive']), +}); +type Issue = z.infer; +``` + +Then you can create a well-typed query builder + +```ts +const query = new EntityQuery(context, 'issue'); +``` + +- The first param to `EntityQuery` is the integration point between the query builder and Replicache. It provides the query builder with a way to gain access to the current Replicache instance and collections. See [`makeTestContext`](./context/context.ts) for an example. +- The second param to `EntityQuery` is the same `prefix` parameter that is passed to rails `generate`. It is used to identify the collection being queried. + +> Note: this'll eventually be folded into a method returned by [`GenerateResult`](../generate.ts) so users are not exposed to either parameter on `EntityQuery`. + +# EntityQuery + +[EntityQuery.ts](./query/EntityQuery.ts) + +`EntityQuery` holds all the various query methods and is responsible for building the [`AST`](./ast/ZqlAst.ts) to represent the query. Any time a method is invoked on `EntityQuery`, a new `EntityQuery` instance is returned containing the new `AST` for that query. + +Example: + +```ts +const derivedQuery = query + .where(...) + .join(...) + .select('id', 'title', 'joined.thing', ...) + .asc(...); +``` + +Under the hood, `where`, `join`, `select`, etc. are all making a copy of and updating the internal `AST`. + +Key points: + +1. `EntityQuery` is immutable. Each method invoked on it returns a new query. This prevents queries that have been passed around from being modified out from under their users. This also makes it easy to fork queries that start from a common base. +2. `EntityQuery` is a 100% type safe interface to the user. Layers below `EntityQuery` which are internal to the framework do need to ditch type safety in a number of places but, since the interface is typed, we know the types coming in are correct. +3. The order in which methods are invoked on `EntityQuery` that return `this` does not and will not ever matter. All permutations will return the same AST and result in the same query. + +Once a user has built a query they can turn it into a prepared statement. + +# Prepared Statements + +[`Statement.ts`](./query/Statement.ts) + +A prepared statement is used to: + +1. Manage the lifetime of a query +2. Materialize a query +3. In the future, change bindings of the query +4. De-duplicate queries + +```ts +const stmt = derivedQuery.prepare(); +``` + +## Manage Lifetime + +Statements need to be destroyed when they're no longer used. This'll clean up any data flow pipelines and views uniquely used by the statement or decrement the refcount for those that are shared. + +```ts +stmt.destroy(); +``` + +## Materialization + +Materialization is essentially the process of running the query. This'll be covered after describing how prepared statements are built. + +# AST -> Prepared Statement + +When the user calls `query.prepare()` the `AST` held by the query is converted into a differential dataflow pipeline. +This pipeline is held by the prepared statement so it does not need to be re-built on future uses of the statement. + +The Pipeline Builder is responsible for this conversion of the AST. + +# Pipeline Builder + +[`pipelineBuilder.ts`](./ast-to-ivm/pipelineBuilder.ts) + +The Pipeline Builder is "medium simple" right now. It isn't as dead simple as it could be given the current set of available operations but neither is it as complex as it could be given where we want to go (grouping, joins, having, sub-queries, etc.). + +It exists in a happy place to make our current task easy-ish while being able diff --git a/src/zql/ast-to-ivm/pipelineBuilder.test.ts b/src/zql/ast-to-ivm/pipelineBuilder.test.ts index 7031701..82255c2 100644 --- a/src/zql/ast-to-ivm/pipelineBuilder.test.ts +++ b/src/zql/ast-to-ivm/pipelineBuilder.test.ts @@ -3,7 +3,7 @@ import {Materialite} from '../ivm/Materialite.js'; import {z} from 'zod'; import {EntityQuery} from '../query/EntityQuery.js'; import {buildPipeline} from './pipelineBuilder.js'; -import {makeTestContext} from '../query/context/contextProvider.js'; +import {makeTestContext} from '../context/context.js'; const e1 = z.object({ id: z.string(), diff --git a/src/zql/ast-to-ivm/pipelineBuilder.ts b/src/zql/ast-to-ivm/pipelineBuilder.ts index f44c40e..f24fab2 100644 --- a/src/zql/ast-to-ivm/pipelineBuilder.ts +++ b/src/zql/ast-to-ivm/pipelineBuilder.ts @@ -1,6 +1,6 @@ import {nullthrows} from '../error/InvariantViolation.js'; import {DifferenceStream} from '../ivm/graph/DifferenceStream.js'; -import {AST, Condition, ConditionList, Operator} from '../query/ZqlAst.js'; +import {AST, Condition, ConditionList, Operator} from '../ast/ZqlAst.js'; export const orderingProp = Symbol(); diff --git a/src/zql/query/ZqlAst.ts b/src/zql/ast/ZqlAst.ts similarity index 100% rename from src/zql/query/ZqlAst.ts rename to src/zql/ast/ZqlAst.ts diff --git a/src/zql/context/context.ts b/src/zql/context/context.ts new file mode 100644 index 0000000..bf79692 --- /dev/null +++ b/src/zql/context/context.ts @@ -0,0 +1,43 @@ +import {Materialite} from '../ivm/Materialite.js'; +import {ISource} from '../ivm/source/ISource.js'; +import {Entity} from '../../generate.js'; + +export type Context = { + materialite: Materialite; + getSource: ( + name: string, + ordering?: [string[], 'asc' | 'desc'], + ) => ISource; + destroy: () => void; +}; + +export function makeTestContext(): Context { + const materialite = new Materialite(); + const sources = new Map>(); + const getSource = (name: string) => { + if (!sources.has(name)) { + sources.set(name, materialite.newStatelessSource()); + } + return sources.get(name)! as ISource; + }; + return {materialite, getSource, destroy() {}}; +} + +// const replicacheContexts = new Map(); +// export function getReplicacheContext(tx: ReadTransaction): Context { +// let existing = replicacheContexts.get(tx.clientID); +// if (!existing) { +// existing = { +// materialite: new Materialite(), +// getSource: (name, _ordering?: [string[], 'asc' | 'desc']) => { +// throw new Error(`Source not found: ${name}`); +// }, +// destroy() { +// replicacheContexts.delete(tx.clientID); +// }, +// }; +// replicacheContexts.set(tx.clientID, existing); +// } + +// return existing; +// } diff --git a/src/zql/index.ts b/src/zql/index.ts new file mode 100644 index 0000000..b2a4243 --- /dev/null +++ b/src/zql/index.ts @@ -0,0 +1,2 @@ +export {EntityQuery} from './query/EntityQuery.js'; +export {Context} from './context/context.js'; diff --git a/src/zql/query/EntityQuery.test.ts b/src/zql/query/EntityQuery.test.ts index 0171fb5..f0417f1 100644 --- a/src/zql/query/EntityQuery.test.ts +++ b/src/zql/query/EntityQuery.test.ts @@ -2,7 +2,7 @@ import {expect, expectTypeOf, test} from 'vitest'; import {z} from 'zod'; import {EntityQuery} from './EntityQuery.js'; import {Misuse} from '../error/Misuse.js'; -import {makeTestContext} from './context/contextProvider.js'; +import {makeTestContext} from '../context/context.js'; const context = makeTestContext(); test('query types', () => { diff --git a/src/zql/query/EntityQuery.ts b/src/zql/query/EntityQuery.ts index ea6f1ce..dab494e 100644 --- a/src/zql/query/EntityQuery.ts +++ b/src/zql/query/EntityQuery.ts @@ -3,8 +3,8 @@ import {Misuse} from '../error/Misuse.js'; import {EntitySchema} from '../schema/EntitySchema.js'; import {IEntityQuery, Selectable, SelectedFields} from './IEntityQuery.js'; import {IStatement, Statement} from './Statement.js'; -import {AST, Operator, Primitive} from './ZqlAst.js'; -import {Context} from './context/contextProvider.js'; +import {AST, Operator, Primitive} from '../ast/ZqlAst.js'; +import {Context} from '../context/context.js'; let aliasCount = 0; diff --git a/src/zql/query/IEntityQuery.ts b/src/zql/query/IEntityQuery.ts index f76137a..f025817 100644 --- a/src/zql/query/IEntityQuery.ts +++ b/src/zql/query/IEntityQuery.ts @@ -2,7 +2,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import {EntitySchema} from '../schema/EntitySchema.js'; import {IStatement} from './Statement.js'; -import {AST, Operator} from './ZqlAst.js'; +import {AST, Operator} from '../ast/ZqlAst.js'; export type SelectedFields[]> = Pick< T, diff --git a/src/zql/query/Statement.test.ts b/src/zql/query/Statement.test.ts index 94f9e5c..363790e 100644 --- a/src/zql/query/Statement.test.ts +++ b/src/zql/query/Statement.test.ts @@ -1,5 +1,5 @@ import {expect, test} from 'vitest'; -import {makeTestContext} from './context/contextProvider.js'; +import {makeTestContext} from '../context/context.js'; import {z} from 'zod'; import {EntityQuery} from './EntityQuery.js'; diff --git a/src/zql/query/Statement.ts b/src/zql/query/Statement.ts index 914a4f1..56eb629 100644 --- a/src/zql/query/Statement.ts +++ b/src/zql/query/Statement.ts @@ -3,10 +3,10 @@ import {IView} from '../ivm/view/IView.js'; import {PersistentTreeView} from '../ivm/view/TreeView.js'; import {EntitySchema} from '../schema/EntitySchema.js'; import {MakeHumanReadable, IEntityQuery} from './IEntityQuery.js'; -import {Context} from './context/contextProvider.js'; +import {Context} from '../context/context.js'; import {DifferenceStream} from '../ivm/graph/DifferenceStream.js'; import {ValueView} from '../ivm/view/PrimitiveView.js'; -import {Primitive} from './ZqlAst.js'; +import {Primitive} from '../ast/ZqlAst.js'; import {Entity} from '../../generate.js'; export interface IStatement { diff --git a/src/zql/query/context/contextProvider.ts b/src/zql/query/context/contextProvider.ts deleted file mode 100644 index 4bc556c..0000000 --- a/src/zql/query/context/contextProvider.ts +++ /dev/null @@ -1,43 +0,0 @@ -import {Materialite} from '../../ivm/Materialite.js'; -import {ISource} from '../../ivm/source/ISource.js'; -import {Entity, ReadTransaction} from '../../../generate.js'; - -export type Context = { - materialite: Materialite; - getSource: ( - name: string, - ordering?: [string[], 'asc' | 'desc'], - ) => ISource; - destroy: () => void; -}; - -export function makeTestContext(): Context { - const materialite = new Materialite(); - const sources = new Map>(); - const getSource = (name: string) => { - if (!sources.has(name)) { - sources.set(name, materialite.newStatelessSource()); - } - return sources.get(name)! as ISource; - }; - return {materialite, getSource, destroy() {}}; -} - -const replicacheContexts = new Map(); -export function getReplicacheContext(tx: ReadTransaction): Context { - let existing = replicacheContexts.get(tx.clientID); - if (!existing) { - existing = { - materialite: new Materialite(), - getSource: (name, _ordering?: [string[], 'asc' | 'desc']) => { - throw new Error(`Source not found: ${name}`); - }, - destroy() { - replicacheContexts.delete(tx.clientID); - }, - }; - replicacheContexts.set(tx.clientID, existing); - } - - return existing; -}