break into packages

README.md

@@ -8,33 +8,34 @@ cdb is a database built for learning about the inner workings of databases. cdb
 
 ### SELECT
 ```mermaid
-flowchart LR
-begin[ ]
+graph LR
+begin(( ))
 explain([EXPLAIN])
 select([SELECT])
+all[*]
 from([FROM])
 
 begin --> explain
 begin --> select
 explain --> select
-select --> *
-* --> from
+select --> all
+all --> from
 from --> table
 ```
 
 ### CREATE
 ```mermaid
-flowchart LR
-begin[ ]
+graph LR
+begin(( ))
 explain([EXPLAIN])
 create([CREATE])
 table([TABLE])
 colType([INTEGER or TEXT])
-lparen("(")
-colSep(",")
-rparen(")")
-tableIdent("Table Identifier")
-colIdent("Column Identifier")
+lparen["("]
+colSep[","]
+rparen[")"]
+tableIdent["Table Identifier"]
+colIdent["Column Identifier"]
 
 begin --> explain
 begin --> create
@@ -52,20 +53,20 @@ colSep --> colIdent
 
 ### INSERT
 ```mermaid
-flowchart LR
-begin[ ]
+graph LR
+begin(( ))
 explain([EXPLAIN])
 insert([INSERT])
 into([INTO])
-tableIdent("Table Identifier")
-lparen("(")
-rparen(")")
-colSep(",")
+tableIdent["Table Identifier"]
+lparen["("]
+rparen[")"]
+colSep[","]
 values([VALUES])
-lparen2("(")
-rparen2("(")
-colSep2(",")
-literal("literal")
+lparen2["("]
+rparen2["("]
+colSep2[","]
+literal["literal"]
 
 begin --> explain
 begin --> insert
@@ -85,3 +86,73 @@ colSep2 --> literal
 literal --> rparen2
 colSep2 --> rparen2
 ```
+
+## Architecture
+```mermaid
+---
+title: Packages
+---
+graph LR
+    REPL --> DB
+    DB --> Compiler
+    subgraph Compiler
+    Lexer --> Parser --> AST
+    end
+    AST --> Planner
+    Planner --> VM
+    Planner --> Catalog
+    VM --> KV
+    subgraph KV
+    Cursor
+    Catalog
+    Encoder
+    end
+    KV --> Pager
+    subgraph Pager
+    Storage
+    end
+```
+### REPL (Read Eval Print Loop)
+The REPL works with the DB (Database) layer and is responsible for two things.
+Passing down the SQL strings that are read by the REPL to the DB. Printing out
+execution results that are returned from the DB layer. The REPL can be thought
+of as an adapter layer.
+
+### DB (Database)
+The DB (Database) layer is an interface that is called by adapters like the
+REPL. In theory, the DB layer could be called directly by a consumer of the
+package or by something like a TCP connection adapter.
+
+### Compiler
+The Compiler is responsible for converting a raw SQL string to a AST (Abstract
+syntax tree). In doing this, the compiler performs two major steps known as
+lexing and parsing.
+
+### Planner
+The Planner is what is known as a query planner. The planner takes the AST
+generated by the compiler and performs steps to generate an optimal "byte code"
+routine consisting of commands defined in the VM. This routine can be examined
+by prefixing any SQL statement with the `EXPLAIN` keyword.
+
+### VM (Virtual Machine)
+The VM defines a set of commands that can be executed or explained. Each command
+performs basic calls into the KV layer that make up a query execution. This
+mechanism makes queries predictable and consistent.
+
+### KV (Key Value)
+The KV layer implements a data structure known as a
+[B+ tree](https://en.wikipedia.org/wiki/B%2B_tree) this tree enables the
+database to perform fast lookups. At this layer data is encoded into byte slices
+enabled by the `Encoder` in this layer. The layer implements a cursor
+abstraction which enables queries to scan and seek the B trees associated with a
+table or index. Additionally this layer maintains the `Catalog`, an in memory
+representation of the database schema.
+
+### Pager
+The Pager sits on top of a contiguous block of bytes defined in the `Storage`
+interface. This block is typically a single file enabling the database to
+persist data, but it can be an in memory representation. The pager abstracts
+this block into pages which represent nodes in the KV layer's B tree. The pager
+is capable of caching the pages. The pager implements a read write mutex for
+concurrency control. The pager implements atomic writes to its storage through
+an algorithm known as the journal.

db/db.go

@@ -0,0 +1,64 @@
+// db serves as an interface for the database where raw SQL goes in and
+// convenient data structures come out. db is intended to be consumed by things
+// like a repl (read eval print loop), a program, or a transport protocol such
+// as http.
+package db
+
+import (
+	"fmt"
+
+	"github.com/chirst/cdb/compiler"
+	"github.com/chirst/cdb/kv"
+	"github.com/chirst/cdb/planner"
+	"github.com/chirst/cdb/vm"
+)
+
+type executor interface {
+	Execute(*vm.ExecutionPlan) *vm.ExecuteResult
+}
+
+type dbCatalog interface {
+	GetColumns(tableOrIndexName string) ([]string, error)
+	GetRootPageNumber(tableOrIndexName string) (int, error)
+}
+
+type DB struct {
+	vm      executor
+	catalog dbCatalog
+}
+
+func New(useMemory bool) (*DB, error) {
+	kv, err := kv.New(useMemory)
+	if err != nil {
+		return nil, err
+	}
+	return &DB{
+		vm:      vm.New(kv),
+		catalog: kv.GetCatalog(),
+	}, nil
+}
+
+func (db *DB) Execute(sql string) vm.ExecuteResult {
+	tokens := compiler.NewLexer(sql).Lex()
+	statement, err := compiler.NewParser(tokens).Parse()
+	if err != nil {
+		return vm.ExecuteResult{Err: err}
+	}
+	executionPlan, err := db.getExecutionPlanFor(statement)
+	if err != nil {
+		return vm.ExecuteResult{Err: err}
+	}
+	return *db.vm.Execute(executionPlan)
+}
+
+func (db *DB) getExecutionPlanFor(statement compiler.Stmt) (*vm.ExecutionPlan, error) {
+	switch s := statement.(type) {
+	case *compiler.SelectStmt:
+		return planner.NewSelect(db.catalog).GetPlan(s)
+	case *compiler.CreateStmt:
+		return planner.NewCreate(db.catalog).GetPlan(s)
+	case *compiler.InsertStmt:
+		return planner.NewInsert(db.catalog).GetPlan(s)
+	}
+	return nil, fmt.Errorf("statement not supported")
+}

db_test.go → db/db_test.go

@@ -1,23 +1,23 @@
-package main
+package db
 
 import (
 	"testing"
 )
 
 func TestExecute(t *testing.T) {
-	db, err := newDb(true)
+	db, err := New(true)
 	if err != nil {
 		t.Fatal(err)
 	}
 	createSql := "CREATE TABLE person (id INTEGER, first_name TEXT, last_name TEXT, age INTEGER)"
-	createRes := db.execute(createSql)
-	if createRes.err != nil {
-		t.Fatal(err.Error())
+	createRes := db.Execute(createSql)
+	if createRes.Err != nil {
+		t.Fatal(createRes.Err.Error())
 	}
 	selectSchemaSql := "SELECT * FROM cdb_schema"
-	schemaRes := db.execute(selectSchemaSql)
-	if schemaRes.err != nil {
-		t.Fatal(schemaRes.err.Error())
+	schemaRes := db.Execute(selectSchemaSql)
+	if schemaRes.Err != nil {
+		t.Fatal(schemaRes.Err.Error())
 	}
 	schemaSelectExpectations := []string{
 		"1",
@@ -28,19 +28,19 @@ func TestExecute(t *testing.T) {
 		"{\"columns\":[{\"name\":\"id\",\"type\":\"INTEGER\"},{\"name\":\"first_name\",\"type\":\"TEXT\"},{\"name\":\"last_name\",\"type\":\"TEXT\"},{\"name\":\"age\",\"type\":\"INTEGER\"}]}",
 	}
 	for i, s := range schemaSelectExpectations {
-		if c := *schemaRes.resultRows[1][i]; c != s {
+		if c := *schemaRes.ResultRows[1][i]; c != s {
 			t.Fatalf("expected %s got %s", s, c)
 		}
 	}
 	insertSql := "INSERT INTO person (first_name, last_name, age) VALUES ('John', 'Smith', 50)"
-	insertRes := db.execute(insertSql)
-	if insertRes.err != nil {
-		t.Fatal(err.Error())
+	insertRes := db.Execute(insertSql)
+	if insertRes.Err != nil {
+		t.Fatal(insertRes.Err.Error())
 	}
 	selectPersonSql := "SELECT * FROM person"
-	selectPersonRes := db.execute(selectPersonSql)
-	if selectPersonRes.err != nil {
-		t.Fatal(err.Error())
+	selectPersonRes := db.Execute(selectPersonSql)
+	if selectPersonRes.Err != nil {
+		t.Fatal(selectPersonRes.Err.Error())
 	}
 	selectPersonExpectations := []string{
 		"1",
@@ -49,7 +49,7 @@ func TestExecute(t *testing.T) {
 		"50",
 	}
 	for i, s := range selectPersonExpectations {
-		if c := *selectPersonRes.resultRows[1][i]; c != s {
+		if c := *selectPersonRes.ResultRows[1][i]; c != s {
 			t.Fatalf("expected %s got %s", s, c)
 		}
 	}