Add DB Versioning

cmd/serve_migrate.go

@@ -1,16 +1,12 @@
 package cmd
 
 import (
-	"database/sql"
-	"fmt"
-	"os"
 	"path/filepath"
 
-	"github.com/charmbracelet/log"
-
 	"github.com/charmbracelet/charm/server"
 	"github.com/charmbracelet/charm/server/db/sqlite"
-	"github.com/charmbracelet/charm/server/db/sqlite/migration"
+	"github.com/charmbracelet/charm/server/storage"
+	"github.com/charmbracelet/log"
 	"github.com/spf13/cobra"
 
 	_ "modernc.org/sqlite" // sqlite driver
@@ -26,30 +22,11 @@ var ServeMigrationCmd = &cobra.Command{
 	Args:    cobra.NoArgs,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		cfg := server.DefaultConfig()
-		dp := filepath.Join(cfg.DataDir, "db", sqlite.DbName)
-		_, err := os.Stat(dp)
-		if err != nil {
-			return fmt.Errorf("database does not exist: %s", err)
-		}
-		db := sqlite.NewDB(dp)
-		for _, m := range []migration.Migration{
-			migration.Migration0001,
-		} {
-			log.Print("Running migration", "id", fmt.Sprintf("%04d", m.ID), "name", m.Name)
-			err = db.WrapTransaction(func(tx *sql.Tx) error {
-				_, err := tx.Exec(m.SQL)
-				if err != nil {
-					return err
-				}
-				return nil
-			})
-			if err != nil {
-				break
-			}
-		}
+		dp := filepath.Join(cfg.DataDir, "db")
+		err := storage.EnsureDir(dp, 0o700)
 		if err != nil {
-			return err
+			log.Fatal("could not init sqlite path", "err", err)
 		}
-		return nil
+		return sqlite.NewDB(filepath.Join(dp, sqlite.DbName)).Migrate()
 	},
 }

server/db/sqlite/migration/0001_create_tables.go

@@ -0,0 +1,80 @@
+package migration
+
+// Migration0001 is the initial migration.
+var Migration0001 = Migration{
+	Version: 1,
+	Name:    "create tables",
+	SQL: `
+CREATE TABLE IF NOT EXISTS charm_user(
+  id INTEGER NOT NULL PRIMARY KEY,
+  charm_id uuid UNIQUE NOT NULL,
+  name varchar(50) UNIQUE,
+  email varchar(254),
+  bio varchar(1000),
+    created_at timestamp default current_timestamp
+);
+
+CREATE TABLE IF NOT EXISTS public_key(
+  id INTEGER NOT NULL PRIMARY KEY,
+  user_id integer NOT NULL,
+  public_key varchar(2048) NOT NULL,
+    created_at timestamp default current_timestamp,
+  UNIQUE (user_id, public_key),
+  CONSTRAINT user_id_fk
+  FOREIGN KEY (user_id)
+  REFERENCES charm_user (id)
+  ON DELETE CASCADE
+  ON UPDATE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS encrypt_key(
+  id INTEGER NOT NULL PRIMARY KEY,
+  public_key_id integer NOT NULL,
+  global_id uuid NOT NULL,
+    created_at timestamp default current_timestamp,
+  encrypted_key varchar(2048) NOT NULL,
+  CONSTRAINT public_key_id_fk
+  FOREIGN KEY (public_key_id)
+  REFERENCES public_key (id)
+  ON DELETE CASCADE
+  ON UPDATE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS named_seq(
+  id INTEGER NOT NULL PRIMARY KEY,
+  user_id integer NOT NULL,
+  seq integer NOT NULL DEFAULT 0,
+  name varchar(1024) NOT NULL,
+  UNIQUE (user_id, name),
+  CONSTRAINT user_id_fk
+  FOREIGN KEY (user_id)
+  REFERENCES charm_user (id)
+  ON DELETE CASCADE
+  ON UPDATE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS news(
+  id INTEGER NOT NULL PRIMARY KEY,
+  subject text,
+  body text,
+  created_at timestamp default current_timestamp
+);
+
+CREATE TABLE IF NOT EXISTS news_tag(
+  id INTEGER NOT NULL PRIMARY KEY,
+  tag varchar(250),
+  news_id integer NOT NULL,
+  CONSTRAINT news_id_fk
+  FOREIGN KEY (news_id)
+  REFERENCES news (id)
+  ON DELETE CASCADE
+  ON UPDATE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS token(
+  id INTEGER NOT NULL PRIMARY KEY,
+  pin text UNIQUE NOT NULL,
+  created_at timestamp default current_timestamp
+);
+`,
+}

server/db/sqlite/migration/0001_foreign_keys.go → server/db/sqlite/migration/0002_foreign_keys.go

@@ -1,9 +1,9 @@
 package migration
 
-// Migration0001 is the initial migration.
-var Migration0001 = Migration{
-	ID:   1,
-	Name: "foreign keys",
+// Migration0002 is the initial inclusion of foreign keys.
+var Migration0002 = Migration{
+	Version: 2,
+	Name:    "foreign keys",
 	SQL: `
 PRAGMA foreign_keys=off;
 

server/db/sqlite/migration/migration.go

@@ -1,8 +1,75 @@
 package migration
 
+import (
+	"fmt"
+	"time"
+
+	"github.com/charmbracelet/log"
+)
+
+// Migrations is a list of all migrations.
+// The migrations must be in sequence starting from 1.
+var Migrations = []Migration{
+	Migration0001,
+	Migration0002,
+}
+
 // Migration is a db migration script.
 type Migration struct {
-	ID   int
-	Name string
-	SQL  string
+	Version int
+	Name    string
+	SQL     string
+}
+
+type Version struct {
+	Version     int
+	Name        *string
+	CompletedAt *time.Time
+	ErrorAt     *time.Time
+	Comment     *string
+	CreatedAt   *time.Time
+	UpdatedAt   *time.Time
+}
+
+func (v Version) String() string {
+	return fmt.Sprintf(
+		"Version: %d, Name: %s, CompletedAt: %s, ErrorAt: %s, Comment: %s, CreatedAt: %s, UpdatedAt: %s",
+		v.Version,
+		safeString(v.Name),
+		safeTime(v.CompletedAt),
+		safeTime(v.ErrorAt),
+		safeString(v.Comment),
+		safeTime(v.CreatedAt),
+		safeTime(v.UpdatedAt),
+	)
+}
+func safeString(s *string) string {
+	if s != nil {
+		return *s
+	}
+	return "nil"
+}
+func safeTime(t *time.Time) string {
+	if t != nil {
+		return t.Format(time.RFC3339)
+	}
+	return "nil"
+}
+
+// Validate validates the migration sequence.
+// It returns an error if the sequence is not valid.
+// Each migration must have a unique version number and
+// the version numbers must be in sequence starting from 1.
+func Validate() error {
+	log.Info("validating migrations")
+	// later, this could be changed to ensure all versions are sequential starting from the first item in the array
+	// this would remove the requirement to have all versions starting from 1.
+	// this would allow to 'prune' or 'compact' previous versions in some way while continuing the general version scheme.
+	for i, m := range Migrations {
+		if i+1 != m.Version {
+			log.Error("migration is not in sequence", "expected", i+1, "actual", m.Version, "migration", m)
+			return fmt.Errorf("migration %d is not in sequence, expected %d, name %s", m.Version, i+1, m.Name)
+		}
+	}
+	return nil
 }

server/db/sqlite/sql.go

@@ -1,87 +1,35 @@
 package sqlite
 
 const (
-	sqlCreateUserTable = `CREATE TABLE IF NOT EXISTS charm_user(
-                        id INTEGER NOT NULL PRIMARY KEY,
-                        charm_id uuid UNIQUE NOT NULL,
-                        name varchar(50) UNIQUE,
-                        email varchar(254),
-                        bio varchar(1000),
-                        created_at timestamp default current_timestamp
-                        )`
-
-	sqlCreatePublicKeyTable = `CREATE TABLE IF NOT EXISTS public_key(
-                             id INTEGER NOT NULL PRIMARY KEY,
-                             user_id integer NOT NULL,
-                             public_key varchar(2048) NOT NULL,
-                             created_at timestamp default current_timestamp,
-                             UNIQUE (user_id, public_key),
-                             CONSTRAINT user_id_fk
-                                FOREIGN KEY (user_id)
-                                REFERENCES charm_user (id)
-                                ON DELETE CASCADE
-                                ON UPDATE CASCADE
-                            )`
-
-	sqlCreateEncryptKeyTable = `CREATE TABLE IF NOT EXISTS encrypt_key(
-                              id INTEGER NOT NULL PRIMARY KEY,
-                              public_key_id integer NOT NULL,
-                              global_id uuid NOT NULL,
-                              created_at timestamp default current_timestamp,
-                              encrypted_key varchar(2048) NOT NULL,
-                              CONSTRAINT public_key_id_fk
-                                FOREIGN KEY (public_key_id)
-                                REFERENCES public_key (id)
-                                ON DELETE CASCADE
-                                ON UPDATE CASCADE
-                            )`
-
-	sqlCreateNamedSeqTable = `CREATE TABLE IF NOT EXISTS named_seq(
-                            id INTEGER NOT NULL PRIMARY KEY,
-                            user_id integer NOT NULL,
-                            seq integer NOT NULL DEFAULT 0,
-                            name varchar(1024) NOT NULL,
-                            UNIQUE (user_id, name),
-                            CONSTRAINT user_id_fk
-                                FOREIGN KEY (user_id)
-                                REFERENCES charm_user (id)
-                                ON DELETE CASCADE
-                                ON UPDATE CASCADE
-                            )`
-
-	sqlCreateNewsTable = `CREATE TABLE IF NOT EXISTS news(
-                        id INTEGER NOT NULL PRIMARY KEY,
-                        subject text,
-                        body text,
-                        created_at timestamp default current_timestamp
-                        )`
-
-	sqlCreateNewsTagTable = `CREATE TABLE IF NOT EXISTS news_tag(
-                           id INTEGER NOT NULL PRIMARY KEY,
-                           tag varchar(250),
-                           news_id integer NOT NULL,
-                           CONSTRAINT news_id_fk
-                                FOREIGN KEY (news_id)
-                                REFERENCES news (id)
-                                ON DELETE CASCADE
-                                ON UPDATE CASCADE
-                           )`
-
-	sqlCreateTokenTable = `CREATE TABLE IF NOT EXISTS token(
-                           id INTEGER NOT NULL PRIMARY KEY,
-                           pin text UNIQUE NOT NULL,
-                           created_at timestamp default current_timestamp
-                           )`
-
-	sqlSelectUserWithName         = `SELECT id, charm_id, name, email, bio, created_at FROM charm_user WHERE name like ?`
-	sqlSelectUserWithCharmID      = `SELECT id, charm_id, name, email, bio, created_at FROM charm_user WHERE charm_id = ?`
-	sqlSelectUserWithID           = `SELECT id, charm_id, name, email, bio, created_at FROM charm_user WHERE id = ?`
-	sqlSelectUserPublicKeys       = `SELECT id, public_key, created_at FROM public_key WHERE user_id = ?`
-	sqlSelectNumberUserPublicKeys = `SELECT count(*) FROM public_key WHERE user_id = ?`
-	sqlSelectPublicKey            = `SELECT id, user_id, public_key FROM public_key WHERE public_key = ?`
-	sqlSelectEncryptKey           = `SELECT global_id, encrypted_key, created_at FROM encrypt_key WHERE public_key_id = ? AND global_id = ?`
-	sqlSelectEncryptKeys          = `SELECT global_id, encrypted_key, created_at FROM encrypt_key WHERE public_key_id = ? ORDER BY created_at ASC`
-	sqlSelectNamedSeq             = `SELECT seq FROM named_seq WHERE user_id = ? AND name = ?`
+	sqlCreateVersionTable = `
+		CREATE TABLE IF NOT EXISTS version (
+			id INTEGER PRIMARY KEY,
+			version INTEGER NOT NULL,
+			name TEXT NOT NULL,
+			completed_at DATETIME,
+			error_at DATETIME,
+			comment TEXT,
+			created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+			updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+			UNIQUE(version)
+		);`
+	sqlDropVersionTable             = `DROP TABLE IF EXISTS version;`
+	sqlSelectVersionTableCount      = `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='version';`
+	sqlSelectVersionCount           = `SELECT count(*) FROM version;`
+	sqlSelectLatestVersion          = `SELECT version, name, completed_at, error_at, comment, created_at, updated_at FROM version ORDER BY version DESC LIMIT 1;`
+	sqlSelectIncompleteVersionCount = `SELECT count(*) FROM version WHERE completed_at IS NULL;`
+	sqlInsertVersion                = `INSERT INTO version (version, name, comment) VALUES (?, ?, ?);`
+	sqlUpdateCompleteVersion        = `UPDATE version SET completed_at = CURRENT_TIMESTAMP WHERE version = ?;`
+	sqlUpdateErrorVersion           = `UPDATE version SET error_at = CURRENT_TIMESTAMP, comment = ? WHERE version = ?;`
+	sqlSelectUserWithName           = `SELECT id, charm_id, name, email, bio, created_at FROM charm_user WHERE name like ?`
+	sqlSelectUserWithCharmID        = `SELECT id, charm_id, name, email, bio, created_at FROM charm_user WHERE charm_id = ?`
+	sqlSelectUserWithID             = `SELECT id, charm_id, name, email, bio, created_at FROM charm_user WHERE id = ?`
+	sqlSelectUserPublicKeys         = `SELECT id, public_key, created_at FROM public_key WHERE user_id = ?`
+	sqlSelectNumberUserPublicKeys   = `SELECT count(*) FROM public_key WHERE user_id = ?`
+	sqlSelectPublicKey              = `SELECT id, user_id, public_key FROM public_key WHERE public_key = ?`
+	sqlSelectEncryptKey             = `SELECT global_id, encrypted_key, created_at FROM encrypt_key WHERE public_key_id = ? AND global_id = ?`
+	sqlSelectEncryptKeys            = `SELECT global_id, encrypted_key, created_at FROM encrypt_key WHERE public_key_id = ? ORDER BY created_at ASC`
+	sqlSelectNamedSeq               = `SELECT seq FROM named_seq WHERE user_id = ? AND name = ?`
 
 	sqlInsertUser = `INSERT INTO charm_user (charm_id) VALUES (?)`
 
@@ -116,9 +64,10 @@ const (
 	sqlCountUserNames = `SELECT COUNT(*) FROM charm_user WHERE name <> ''`
 
 	sqlSelectNews     = `SELECT id, subject, body, created_at FROM news WHERE id = ?`
-	sqlSelectNewsList = `SELECT n.id, n.subject, n.created_at FROM news AS n
-	                     INNER JOIN news_tag AS t ON t.news_id = n.id
-	                     WHERE t.tag = ?
-	                     ORDER BY n.created_at desc
-	                     LIMIT 50 OFFSET ?`
+	sqlSelectNewsList = `
+		SELECT n.id, n.subject, n.created_at FROM news AS n
+		INNER JOIN news_tag AS t ON t.news_id = n.id
+		WHERE t.tag = ?
+		ORDER BY n.created_at desc
+		LIMIT 50 OFFSET ?`
 )