From 2da240f595b6bf9af34f9cab175e14bdbc42b498 Mon Sep 17 00:00:00 2001 From: Pedro Franceschi Date: Thu, 28 Apr 2016 01:37:02 -0300 Subject: [PATCH] Implement composite type support. --- action/alterattribute.go | 63 +++++++++++ action/createattribute.go | 40 +++++++ action/createtype.go | 35 ++++-- action/dropattribute.go | 33 ++++++ data/sql/setup.sql | 41 ++++++- data/sql/source_trigger.sql | 4 +- database/attribute.go | 88 +++++++++++++++ database/attribute_test.go | 219 ++++++++++++++++++++++++++++++++++++ database/enum.go | 7 +- database/enum_test.go | 15 +++ database/schema.go | 19 +++- database/schema_test.go | 2 + database/table_test.go | 2 + database/type.go | 7 ++ database/type_test.go | 38 ++++++- 15 files changed, 588 insertions(+), 25 deletions(-) create mode 100644 action/alterattribute.go create mode 100644 action/createattribute.go create mode 100644 action/dropattribute.go create mode 100644 database/attribute.go create mode 100644 database/attribute_test.go diff --git a/action/alterattribute.go b/action/alterattribute.go new file mode 100644 index 0000000..2a2d7af --- /dev/null +++ b/action/alterattribute.go @@ -0,0 +1,63 @@ +package action + +import ( + "encoding/gob" + "fmt" +) + +type AlterAttribute struct { + SchemaName string + TypeName string + Column Column + NewColumn Column +} + +// Register type for gob +func init() { + gob.Register(&AlterAttribute{}) +} + +func (a *AlterAttribute) Execute(c *Context) error { + if a.Column.Name != a.NewColumn.Name { + _, err := c.Tx.Exec( + fmt.Sprintf( + "ALTER TYPE \"%s\".\"%s\" RENAME ATTRIBUTE \"%s\" TO \"%s\";", + a.SchemaName, + a.TypeName, + a.Column.Name, + a.NewColumn.Name, + ), + ) + + if err != nil { + return err + } + } + + if a.Column.Type != a.NewColumn.Type { + _, err := c.Tx.Exec( + fmt.Sprintf( + "ALTER TYPE \"%s\".\"%s\" ALTER ATTRIBUTE \"%s\" TYPE %s\"%s\";", + a.SchemaName, + a.TypeName, + a.NewColumn.Name, + a.NewColumn.GetTypeSchemaStr(a.SchemaName), + a.NewColumn.Type, + ), + ) + + if err != nil { + return err + } + } + + return nil +} + +func (a *AlterAttribute) Filter(targetExpression string) bool { + return true +} + +func (a *AlterAttribute) NeedsSeparatedBatch() bool { + return false +} diff --git a/action/createattribute.go b/action/createattribute.go new file mode 100644 index 0000000..6ec4c25 --- /dev/null +++ b/action/createattribute.go @@ -0,0 +1,40 @@ +package action + +import ( + "encoding/gob" + "fmt" +) + +type CreateAttribute struct { + SchemaName string + TypeName string + Column Column +} + +// Register type for gob +func init() { + gob.Register(&CreateAttribute{}) +} + +func (a *CreateAttribute) Execute(c *Context) error { + _, err := c.Tx.Exec( + fmt.Sprintf( + "ALTER TYPE \"%s\".\"%s\" ADD ATTRIBUTE \"%s\" %s\"%s\";", + a.SchemaName, + a.TypeName, + a.Column.Name, + a.Column.GetTypeSchemaStr(a.SchemaName), + a.Column.Type, + ), + ) + + return err +} + +func (a *CreateAttribute) Filter(targetExpression string) bool { + return true +} + +func (a *CreateAttribute) NeedsSeparatedBatch() bool { + return false +} diff --git a/action/createtype.go b/action/createtype.go index 7581514..9089f3e 100644 --- a/action/createtype.go +++ b/action/createtype.go @@ -8,6 +8,7 @@ import ( type CreateType struct { SchemaName string TypeName string + TypeType string } // Register type for gob @@ -16,15 +17,31 @@ func init() { } func (a *CreateType) Execute(c *Context) error { - _, err := c.Tx.Exec( - fmt.Sprintf( - "CREATE TYPE \"%s\".\"%s\" AS ENUM ();", - a.SchemaName, - a.TypeName, - ), - ) - - return err + if a.TypeType == "c" { + // Composite type + _, err := c.Tx.Exec( + fmt.Sprintf( + "CREATE TYPE \"%s\".\"%s\" AS ();", + a.SchemaName, + a.TypeName, + ), + ) + + return err + } else if a.TypeType == "e" { + // Enum + _, err := c.Tx.Exec( + fmt.Sprintf( + "CREATE TYPE \"%s\".\"%s\" AS ENUM ();", + a.SchemaName, + a.TypeName, + ), + ) + + return err + } + + return nil } func (a *CreateType) Filter(targetExpression string) bool { diff --git a/action/dropattribute.go b/action/dropattribute.go new file mode 100644 index 0000000..df32167 --- /dev/null +++ b/action/dropattribute.go @@ -0,0 +1,33 @@ +package action + +import ( + "encoding/gob" + "fmt" +) + +type DropAttribute struct { + SchemaName string + TypeName string + Column Column +} + +// Register type for gob +func init() { + gob.Register(&DropAttribute{}) +} + +func (a *DropAttribute) Execute(c *Context) error { + _, err := c.Tx.Exec( + fmt.Sprintf("ALTER TYPE \"%s\".\"%s\" DROP ATTRIBUTE \"%s\";", a.SchemaName, a.TypeName, a.Column.Name), + ) + + return err +} + +func (a *DropAttribute) Filter(targetExpression string) bool { + return true +} + +func (a *DropAttribute) NeedsSeparatedBatch() bool { + return false +} diff --git a/data/sql/setup.sql b/data/sql/setup.sql index b6f1ca5..31190ab 100644 --- a/data/sql/setup.sql +++ b/data/sql/setup.sql @@ -76,7 +76,7 @@ CREATE TABLE IF NOT EXISTS teleport.batch_events ( -- Returns current schema of all tables in all schemas as a JSON -- JSON array containing each column's definition. -CREATE OR REPLACE FUNCTION get_schema() RETURNS text AS $$ +CREATE OR REPLACE FUNCTION teleport_get_schema() RETURNS text AS $$ BEGIN RETURN ( SELECT json_agg(row_to_json(data)) FROM ( @@ -132,7 +132,7 @@ BEGIN -- Ordinary columns are numbered from 1 up. -- System columns have negative numbers. attr.attr_num > 0 - ) AS attributes, + ) AS columns, ( -- The view pg_indexes provides access to useful information about each index -- in the database. @@ -181,6 +181,7 @@ BEGIN SELECT pgtype.oid AS oid, pgtype.typnamespace AS namespace_oid, + pgtype.typtype AS type_type, pgtype.typname AS type_name, ( -- The catalog pg_attribute stores information about table columns. There will be @@ -197,8 +198,38 @@ BEGIN ) enum WHERE enum.type_oid = pgtype.oid - ) AS enums + ) AS enums, + ( + -- The catalog pg_attribute stores information about table columns. There will be + -- exactly one pg_attribute row for every column in every table in the database. + -- (There will also be attribute entries for indexes, and indeed all objects that + -- have pg_class entries.) + SELECT array_to_json(array_agg(row_to_json(attr))) + FROM ( + SELECT + a.attrelid AS class_oid, + a.attname AS attr_name, + a.attnum AS attr_num, + t.typname AS type_name, + ( + SELECT n.nspname + FROM pg_namespace n + WHERE n.oid = t.typnamespace + ) AS type_schema, + t.oid AS type_oid + FROM pg_attribute a + INNER JOIN pg_type t + ON a.atttypid = t.oid + ) attr + WHERE + attr.class_oid = pgtype.typrelid AND + -- Ordinary columns are numbered from 1 up. + -- System columns have negative numbers. + attr.attr_num > 0 + ) AS attributes FROM pg_type pgtype + LEFT JOIN pg_class class + ON class.oid = pgtype.typrelid -- typtype is: -- b for a base type -- c for a composite type (e.g., a table's row type) @@ -206,7 +237,9 @@ BEGIN -- e for an enum type -- p for a pseudo-type -- r for a range type - WHERE typtype = 'e' + WHERE + (pgtype.typtype = 'e') OR + (pgtype.typtype = 'c' AND class.relkind = 'c') ) pgtype WHERE pgtype.namespace_oid = namespace.oid ) AS types, diff --git a/data/sql/source_trigger.sql b/data/sql/source_trigger.sql index 6a0a4b2..1d1b668 100644 --- a/data/sql/source_trigger.sql +++ b/data/sql/source_trigger.sql @@ -28,7 +28,7 @@ BEGIN WITH all_json_key_value AS ( SELECT 'pre' AS key, data::json AS value FROM teleport.event WHERE id = event_row.id UNION ALL - SELECT 'post' AS key, get_schema()::json AS value + SELECT 'post' AS key, teleport_get_schema()::json AS value ) UPDATE teleport.event SET status = 'waiting_batch', @@ -40,7 +40,7 @@ BEGIN INSERT INTO teleport.event (data, kind, trigger_tag, trigger_event, transaction_id, status) VALUES ( - get_schema()::text, + teleport_get_schema()::text, 'ddl', 'ddl_command_start', '', diff --git a/database/attribute.go b/database/attribute.go new file mode 100644 index 0000000..aad7e6c --- /dev/null +++ b/database/attribute.go @@ -0,0 +1,88 @@ +package database + +import ( + "github.com/pagarme/teleport/action" + "github.com/pagarme/teleport/batcher/ddldiff" +) + +// Define a class column +type Attribute struct { + Name string `json:"attr_name"` + Num int `json:"attr_num"` + TypeName string `json:"type_name"` + TypeSchema string `json:"type_schema"` + TypeOid string `json:"type_oid"` + Type *Type +} + +func (a *Attribute) IsNativeType() bool { + return a.TypeSchema == "pg_catalog" +} + +// Implements Diffable +func (post *Attribute) Diff(other ddldiff.Diffable, context ddldiff.Context) []action.Action { + actions := make([]action.Action, 0) + + if other == nil { + actions = append(actions, &action.CreateAttribute{ + context.Schema, + post.Type.Name, + action.Column{ + post.Name, + post.TypeName, + post.IsNativeType(), + }, + }) + } else { + pre := other.(*Attribute) + + if pre.Name != post.Name || pre.TypeOid != post.TypeOid { + actions = append(actions, &action.AlterAttribute{ + context.Schema, + post.Type.Name, + action.Column{ + pre.Name, + pre.TypeName, + pre.IsNativeType(), + }, + action.Column{ + post.Name, + post.TypeName, + post.IsNativeType(), + }, + }) + } + } + + return actions +} + +func (a *Attribute) Children() []ddldiff.Diffable { + return []ddldiff.Diffable{} +} + +func (a *Attribute) Drop(context ddldiff.Context) []action.Action { + return []action.Action{ + &action.DropAttribute{ + context.Schema, + a.Type.Name, + action.Column{ + a.Name, + a.TypeName, + a.IsNativeType(), + }, + }, + } +} + +func (a *Attribute) IsEqual(other ddldiff.Diffable) bool { + if other == nil { + return false + } + + if otherAttr, ok := other.(*Attribute); ok { + return (a.Num == otherAttr.Num) + } + + return false +} diff --git a/database/attribute_test.go b/database/attribute_test.go new file mode 100644 index 0000000..a607357 --- /dev/null +++ b/database/attribute_test.go @@ -0,0 +1,219 @@ +package database + +import ( + "github.com/pagarme/teleport/action" + "github.com/pagarme/teleport/batcher/ddldiff" + "reflect" + "testing" +) + +func init() { + schema = &Schema{ + "123", + "test_schema", + []*Table{}, + nil, + nil, + nil, + } + + typ = &Type{ + "789", + "test_type", + "c", + []*Enum{}, + []*Attribute{}, + schema, + } + + defaultContext = ddldiff.Context{ + Schema: "default_context", + } +} + +func TestAttributeDiff(t *testing.T) { + var tests = []struct { + pre *Attribute + post *Attribute + output []action.Action + }{ + { + // Diff a column creation + nil, + &Attribute{ + "test_col", + 1, + "text", + "pg_catalog", + "0", + typ, + }, + []action.Action{ + &action.CreateAttribute{ + "default_context", + "test_type", + action.Column{ + "test_col", + "text", + true, + }, + }, + }, + }, + { + // Diff a column update + &Attribute{ + "test_col", + 1, + "text", + "pg_catalog", + "0", + typ, + }, + &Attribute{ + "test_col_2", + 1, + "json", + "pg_catalog", + "0", + typ, + }, + []action.Action{ + &action.AlterAttribute{ + "default_context", + "test_type", + action.Column{ + "test_col", + "text", + true, + }, + action.Column{ + "test_col_2", + "json", + true, + }, + }, + }, + }, + } + + for _, test := range tests { + // Avoid passing a interface with nil pointer + // to Diff and breaking comparisons with nil. + var preObj ddldiff.Diffable + if test.pre == nil { + preObj = nil + } else { + preObj = test.pre + } + + actions := test.post.Diff(preObj, defaultContext) + + if !reflect.DeepEqual(actions, test.output) { + t.Errorf( + "diff %#v with %#v => %v, want %d", + test.pre, + test.post, + actions, + test.output, + ) + } + } +} + +func TestAttributeChildren(t *testing.T) { + attr := &Attribute{ + "test_col_2", + 1, + "json", + "pg_catalog", + "0", + typ, + } + + children := attr.Children() + + if len(children) != 0 { + t.Errorf("attr children => %d, want %d", len(children), 0) + } +} + +func TestAttributeDrop(t *testing.T) { + attr := &Attribute{ + "test_col_2", + 1, + "json", + "pg_catalog", + "0", + typ, + } + + actions := attr.Drop(defaultContext) + + if len(actions) != 1 { + t.Errorf("actions => %d, want %d", len(actions), 1) + } + + dropAction, ok := actions[0].(*action.DropAttribute) + + if !ok { + t.Errorf("action is not DropAttribute") + } + + if dropAction.SchemaName != defaultContext.Schema { + t.Errorf("drop action schema name => %s, want %s", dropAction.SchemaName, defaultContext.Schema) + } + + if dropAction.TypeName != typ.Name { + t.Errorf("drop action table name => %s, want %s", dropAction.TypeName, typ.Name) + } + + if dropAction.Column.Name != attr.Name { + t.Errorf("drop action column name => %s, want %s", dropAction.Column.Name, attr.Name) + } + + if dropAction.Column.Type != attr.TypeName { + t.Errorf("drop action column name => %s, want %s", dropAction.Column.Type, attr.TypeName) + } +} + +func TestAttributeIsEqual(t *testing.T) { + pre := &Attribute{ + "test_col", + 1, + "text", + "pg_catalog", + "0", + typ, + } + + post := &Attribute{ + "test_col_2", + 1, + "int4", + "pg_catalog", + "0", + typ, + } + + if !post.IsEqual(pre) { + t.Errorf("expect classes to be equal") + } + + post.Name = pre.Name + post.Num = 2 + + if post.IsEqual(pre) { + t.Errorf("expect classes not to be equal") + } + + preOtherType := &Enum{ + "123", + "test_enum1_renamed", + typ, + } + + if post.IsEqual(preOtherType) { + t.Errorf("expect two different types not to be equal") + } +} diff --git a/database/enum.go b/database/enum.go index 4097aab..2e57b55 100644 --- a/database/enum.go +++ b/database/enum.go @@ -38,6 +38,9 @@ func (e *Enum) IsEqual(other ddldiff.Diffable) bool { return false } - otherEnum := other.(*Enum) - return (e.Oid == otherEnum.Oid) + if otherEnum, ok := other.(*Enum); ok { + return (e.Oid == otherEnum.Oid) + } + + return false } diff --git a/database/enum_test.go b/database/enum_test.go index 0378b70..b2ec958 100644 --- a/database/enum_test.go +++ b/database/enum_test.go @@ -22,7 +22,9 @@ func init() { typ = &Type{ "789", "test_type", + "c", []*Enum{}, + []*Attribute{}, schema, } @@ -144,4 +146,17 @@ func TestEnumIsEqual(t *testing.T) { if post.IsEqual(pre) { t.Errorf("expect enums not to be equal") } + + preOtherType := &Attribute{ + "test_col_2", + 1, + "int4", + "pg_catalog", + "0", + nil, + } + + if post.IsEqual(preOtherType) { + t.Errorf("expect two different types not to be equal") + } } diff --git a/database/schema.go b/database/schema.go index 4ade48d..49b231d 100644 --- a/database/schema.go +++ b/database/schema.go @@ -47,13 +47,16 @@ func (s *Schema) fillParentReferences() { for _, enum := range typ.Enums { enum.Type = typ } + for _, attr := range typ.Attributes { + attr.Type = typ + } } } // Fetches the schema from the database and update Schema func (db *Database) RefreshSchema() error { // Get schema from query - rows, err := db.runQuery("SELECT get_schema();") + rows, err := db.runQuery("SELECT teleport_get_schema();") if err != nil { return err @@ -111,8 +114,18 @@ func (post *Schema) Diff(other ddldiff.Diffable, context ddldiff.Context) []acti func (s *Schema) Children() []ddldiff.Diffable { children := make([]ddldiff.Diffable, 0) - for i, _ := range s.Types { - children = append(children, s.Types[i]) + // Add enums first + for i, typ := range s.Types { + if typ.Type == "e" { + children = append(children, s.Types[i]) + } + } + + // ... then composite types... + for i, typ := range s.Types { + if typ.Type == "c" { + children = append(children, s.Types[i]) + } } for i, _ := range s.Functions { diff --git a/database/schema_test.go b/database/schema_test.go index fcf080b..a45fdb9 100644 --- a/database/schema_test.go +++ b/database/schema_test.go @@ -152,7 +152,9 @@ func TestSchemaChildren(t *testing.T) { &Type{ "789", "test_type", + "c", []*Enum{}, + []*Attribute{}, nil, }, } diff --git a/database/table_test.go b/database/table_test.go index 12b656e..77a30cf 100644 --- a/database/table_test.go +++ b/database/table_test.go @@ -356,7 +356,9 @@ func TestTableIsEqual(t *testing.T) { preOtherType := &Type{ "789", "test_type", + "c", []*Enum{}, + []*Attribute{}, nil, } diff --git a/database/type.go b/database/type.go index d7677cb..72a4653 100644 --- a/database/type.go +++ b/database/type.go @@ -8,7 +8,9 @@ import ( type Type struct { Oid string `json:"oid"` Name string `json:"type_name"` + Type string `json:"type_type"` Enums []*Enum `json:"enums"` + Attributes []*Attribute `json:"attributes"` Schema *Schema } @@ -19,6 +21,7 @@ func (post *Type) Diff(other ddldiff.Diffable, context ddldiff.Context) []action actions = append(actions, &action.CreateType{ context.Schema, post.Name, + post.Type, }) } else { pre := other.(*Type) @@ -42,6 +45,10 @@ func (t *Type) Children() []ddldiff.Diffable { children = append(children, enum) } + for _, attr := range t.Attributes { + children = append(children, attr) + } + return children } diff --git a/database/type_test.go b/database/type_test.go index aac92e4..ac27678 100644 --- a/database/type_test.go +++ b/database/type_test.go @@ -34,6 +34,7 @@ func TestTypeDiff(t *testing.T) { &Type{ "789", "test_type", + "e", []*Enum{ &Enum{ "123", @@ -46,12 +47,14 @@ func TestTypeDiff(t *testing.T) { nil, }, }, + []*Attribute{}, schema, }, []action.Action{ &action.CreateType{ "default_context", "test_type", + "e", }, }, }, @@ -60,6 +63,7 @@ func TestTypeDiff(t *testing.T) { &Type{ "789", "test_type", + "e", []*Enum{ &Enum{ "123", @@ -72,11 +76,13 @@ func TestTypeDiff(t *testing.T) { nil, }, }, + []*Attribute{}, schema, }, &Type{ "789", "test_type_renamed", + "e", []*Enum{ &Enum{ "123", @@ -89,6 +95,7 @@ func TestTypeDiff(t *testing.T) { nil, }, }, + []*Attribute{}, schema, }, []action.Action{ @@ -134,23 +141,38 @@ func TestTypeChildren(t *testing.T) { }, } + attrs := []*Attribute{ + &Attribute{ + "test_col", + 1, + "text", + "pg_catalog", + "0", + nil, + }, + } + typ := &Type{ "789", "test_type", + "c", enums, + attrs, nil, } children := typ.Children() - if len(children) != 1 { + if len(children) != 2 { t.Errorf("children => %d, want %d", len(children), 1) } - for i, child := range children { - if child != enums[i] { - t.Errorf("child %i => %v, want %v", i, child, enums[i]) - } + if children[0] != enums[0] { + t.Errorf("child 0 => %v, want %v", children[0], enums[0]) + } + + if children[1] != attrs[0] { + t.Errorf("child 1 => %v, want %v", children[1], attrs[0]) } } @@ -158,6 +180,7 @@ func TestTypeDrop(t *testing.T) { typ := &Type{ "789", "test_type", + "c", []*Enum{ &Enum{ "123", @@ -165,6 +188,7 @@ func TestTypeDrop(t *testing.T) { nil, }, }, + []*Attribute{}, schema, } @@ -193,6 +217,7 @@ func TestTypeIsEqual(t *testing.T) { pre := &Type{ "789", "test_type", + "c", []*Enum{ &Enum{ "123", @@ -205,12 +230,14 @@ func TestTypeIsEqual(t *testing.T) { nil, }, }, + []*Attribute{}, schema, } post := &Type{ "789", "test_type_renamed", + "c", []*Enum{ &Enum{ "123", @@ -223,6 +250,7 @@ func TestTypeIsEqual(t *testing.T) { nil, }, }, + []*Attribute{}, schema, }