[wip] organize and document things for code review

src/zql/README.md

@@ -0,0 +1,101 @@
+# ZQL
+
+[EntityQuery](./query/EntityQuery.ts) is the main entrypoint for everything query related:
+
+- building
+- preparing
+- running
+- and materializing queries
+
+# Creating an EntityQuery
+
+First, build your schema for rails as you normally would:
+
+```ts
+const issueSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  created: z.date(),
+  status: z.enum(['active', 'inactive']),
+});
+type Issue = z.infer<typeof issueSchema>;
+```
+
+Then you can create a well-typed query builder
+
+```ts
+const query = new EntityQuery<Issue>(context, 'issue');
+```
+
+- The first param to `EntityQuery` is the integration point between the query builder and Replicache. It provides the query builder with a way to gain access to the current Replicache instance and collections. See [`makeTestContext`](./context/context.ts) for an example.
+- The second param to `EntityQuery` is the same `prefix` parameter that is passed to rails `generate`. It is used to identify the collection being queried.
+
+> Note: this'll eventually be folded into a method returned by [`GenerateResult`](../generate.ts) so users are not exposed to either parameter on `EntityQuery`.
+
+# EntityQuery
+
+[EntityQuery.ts](./query/EntityQuery.ts)
+
+`EntityQuery` holds all the various query methods and is responsible for building the [`AST`](./ast/ZqlAst.ts) to represent the query. Any time a method is invoked on `EntityQuery`, a new `EntityQuery` instance is returned containing the new `AST` for that query.
+
+Example:
+
+```ts
+const derivedQuery = query
+  .where(...)
+  .join(...)
+  .select('id', 'title', 'joined.thing', ...)
+  .asc(...);
+```
+
+Under the hood, `where`, `join`, `select`, etc. are all making a copy of and updating the internal `AST`.
+
+Key points:
+
+1. `EntityQuery` is immutable. Each method invoked on it returns a new query. This prevents queries that have been passed around from being modified out from under their users. This also makes it easy to fork queries that start from a common base.
+2. `EntityQuery` is a 100% type safe interface to the user. Layers below `EntityQuery` which are internal to the framework do need to ditch type safety in a number of places but, since the interface is typed, we know the types coming in are correct.
+3. The order in which methods are invoked on `EntityQuery` that return `this` does not and will not ever matter. All permutations will return the same AST and result in the same query.
+
+Once a user has built a query they can turn it into a prepared statement.
+
+# Prepared Statements
+
+[`Statement.ts`](./query/Statement.ts)
+
+A prepared statement is used to:
+
+1. Manage the lifetime of a query
+2. Materialize a query
+3. In the future, change bindings of the query
+4. De-duplicate queries
+
+```ts
+const stmt = derivedQuery.prepare();
+```
+
+## Manage Lifetime
+
+Statements need to be destroyed when they're no longer used. This'll clean up any data flow pipelines and views uniquely used by the statement or decrement the refcount for those that are shared.
+
+```ts
+stmt.destroy();
+```
+
+## Materialization
+
+Materialization is essentially the process of running the query. This'll be covered after describing how prepared statements are built.
+
+# AST -> Prepared Statement
+
+When the user calls `query.prepare()` the `AST` held by the query is converted into a differential dataflow pipeline.
+This pipeline is held by the prepared statement so it does not need to be re-built on future uses of the statement.
+
+The Pipeline Builder is responsible for this conversion of the AST.
+
+# Pipeline Builder
+
+[`pipelineBuilder.ts`](./ast-to-ivm/pipelineBuilder.ts)
+
+The Pipeline Builder is "medium simple" right now. It isn't as dead simple as it could be given the current set of available operations but neither is it as complex as it could be given where we want to go (grouping, joins, having, sub-queries, etc.).
+
+It exists in a happy place to make our current task easy-ish while being able

src/zql/ast-to-ivm/pipelineBuilder.test.ts

@@ -3,7 +3,7 @@ import {Materialite} from '../ivm/Materialite.js';
 import {z} from 'zod';
 import {EntityQuery} from '../query/EntityQuery.js';
 import {buildPipeline} from './pipelineBuilder.js';
-import {makeTestContext} from '../query/context/contextProvider.js';
+import {makeTestContext} from '../context/context.js';
 
 const e1 = z.object({
   id: z.string(),

src/zql/ast-to-ivm/pipelineBuilder.ts

@@ -1,6 +1,6 @@
 import {nullthrows} from '../error/InvariantViolation.js';
 import {DifferenceStream} from '../ivm/graph/DifferenceStream.js';
-import {AST, Condition, ConditionList, Operator} from '../query/ZqlAst.js';
+import {AST, Condition, ConditionList, Operator} from '../ast/ZqlAst.js';
 
 export const orderingProp = Symbol();
 

src/zql/context/context.ts

@@ -0,0 +1,43 @@
+import {Materialite} from '../ivm/Materialite.js';
+import {ISource} from '../ivm/source/ISource.js';
+import {Entity} from '../../generate.js';
+
+export type Context = {
+  materialite: Materialite;
+  getSource: <T extends Entity>(
+    name: string,
+    ordering?: [string[], 'asc' | 'desc'],
+  ) => ISource<T>;
+  destroy: () => void;
+};
+
+export function makeTestContext(): Context {
+  const materialite = new Materialite();
+  const sources = new Map<string, ISource<unknown>>();
+  const getSource = <T extends Entity>(name: string) => {
+    if (!sources.has(name)) {
+      sources.set(name, materialite.newStatelessSource<T>());
+    }
+    return sources.get(name)! as ISource<T>;
+  };
+  return {materialite, getSource, destroy() {}};
+}
+
+// const replicacheContexts = new Map<string, Context>();
+// export function getReplicacheContext(tx: ReadTransaction): Context {
+//   let existing = replicacheContexts.get(tx.clientID);
+//   if (!existing) {
+//     existing = {
+//       materialite: new Materialite(),
+//       getSource: (name, _ordering?: [string[], 'asc' | 'desc']) => {
+//         throw new Error(`Source not found: ${name}`);
+//       },
+//       destroy() {
+//         replicacheContexts.delete(tx.clientID);
+//       },
+//     };
+//     replicacheContexts.set(tx.clientID, existing);
+//   }
+
+//   return existing;
+// }

src/zql/index.ts

@@ -0,0 +1,2 @@
+export {EntityQuery} from './query/EntityQuery.js';
+export {Context} from './context/context.js';

src/zql/query/EntityQuery.test.ts

@@ -2,7 +2,7 @@ import {expect, expectTypeOf, test} from 'vitest';
 import {z} from 'zod';
 import {EntityQuery} from './EntityQuery.js';
 import {Misuse} from '../error/Misuse.js';
-import {makeTestContext} from './context/contextProvider.js';
+import {makeTestContext} from '../context/context.js';
 
 const context = makeTestContext();
 test('query types', () => {

src/zql/query/EntityQuery.ts

@@ -3,8 +3,8 @@ import {Misuse} from '../error/Misuse.js';
 import {EntitySchema} from '../schema/EntitySchema.js';
 import {IEntityQuery, Selectable, SelectedFields} from './IEntityQuery.js';
 import {IStatement, Statement} from './Statement.js';
-import {AST, Operator, Primitive} from './ZqlAst.js';
-import {Context} from './context/contextProvider.js';
+import {AST, Operator, Primitive} from '../ast/ZqlAst.js';
+import {Context} from '../context/context.js';
 
 let aliasCount = 0;
 

src/zql/query/IEntityQuery.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import {EntitySchema} from '../schema/EntitySchema.js';
 import {IStatement} from './Statement.js';
-import {AST, Operator} from './ZqlAst.js';
+import {AST, Operator} from '../ast/ZqlAst.js';
 
 export type SelectedFields<T, Fields extends Selectable<any>[]> = Pick<
   T,

src/zql/query/Statement.test.ts

@@ -1,5 +1,5 @@
 import {expect, test} from 'vitest';
-import {makeTestContext} from './context/contextProvider.js';
+import {makeTestContext} from '../context/context.js';
 import {z} from 'zod';
 import {EntityQuery} from './EntityQuery.js';
 

src/zql/query/Statement.ts

@@ -3,10 +3,10 @@ import {IView} from '../ivm/view/IView.js';
 import {PersistentTreeView} from '../ivm/view/TreeView.js';
 import {EntitySchema} from '../schema/EntitySchema.js';
 import {MakeHumanReadable, IEntityQuery} from './IEntityQuery.js';
-import {Context} from './context/contextProvider.js';
+import {Context} from '../context/context.js';
 import {DifferenceStream} from '../ivm/graph/DifferenceStream.js';
 import {ValueView} from '../ivm/view/PrimitiveView.js';
-import {Primitive} from './ZqlAst.js';
+import {Primitive} from '../ast/ZqlAst.js';
 import {Entity} from '../../generate.js';
 
 export interface IStatement<TReturn> {