diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 2f24dd89a..5acaca567 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -39,6 +39,9 @@ jobs: run: ./run-clang-format/run-clang-format.py -r --clang-format-executable clang-format-11 modules sources stress test third_party - name: Test + env: + CC: ${{ matrix.compiler }} + CC_FOR_BUILD: ${{ matrix.compiler }} run: make run_test - name: Prepare for release diff --git a/Makefile b/Makefile index 6f4c19772..1ce2e43cf 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ run_test_prep: build_asan copy_asan_bin build_dbg copy_dbg_bin build_release cop run_test: # change dir, test would not work with absolute path ./cleanup-docker.sh - docker-compose -f ./docker-compose-test.yml up --exit-code-from odyssey odyssey openldapr + docker-compose -f ./docker-compose-test.yml up --exit-code-from odyssey submit-cov: mkdir cov-build && cd cov-build diff --git a/docker-compose-test.yml b/docker-compose-test.yml index be2c5e9e1..86b88ca1a 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -23,16 +23,6 @@ services: networks: od_net: ipv4_address: 192.168.233.16 - - gorm-test: - build: - context: https://github.com/pg-sharding/gorm-spqr.git#main - environment: - DB_HOST: "localhost" - DB_PORT: "6432" - DB_USER: "spqr-console" - DB_NAME: "spqr-console" - network_mode: "host" networks: od_net: diff --git a/docker/Dockerfile b/docker/Dockerfile index 498c60b64..a14a6ff86 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -56,8 +56,6 @@ RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ COPY ./docker/pg/pg_hba-test.conf /etc/postgresql/14/main/pg_hba.conf -COPY ./docker-compose-test.yml /docker-compose-test.yml - RUN mkdir test_dir COPY . /test_dir RUN cd /test_dir && make run_test_prep && cp /test_dir/docker/bin/* /usr/bin/ @@ -84,8 +82,6 @@ COPY ./docker/gorm /gorm COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh -RUN touch ./logs.txt - RUN chmod a+x /usr/local/bin/entrypoint.sh ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e4ad9ee2b..551efe8cd 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -6,6 +6,11 @@ set -ex setup +# gorm +ody-start +/gorm/test.sh +ody-stop + # # odyssey rule-address test # /rule-address/test.sh # if [ $? -eq 1 ] @@ -66,15 +71,6 @@ setup # exit 1 # fi -# gorm -ody-start -/gorm/test.sh -if [ $? -eq 1 ] -then - exit 1 -fi -ody-stop - # /usr/bin/odyssey-asan /etc/odyssey/odyssey.conf # ody-stop diff --git a/docker/gorm/gorm-spqr/Dockerfile b/docker/gorm/gorm-spqr/Dockerfile new file mode 100644 index 000000000..75e76cd2b --- /dev/null +++ b/docker/gorm/gorm-spqr/Dockerfile @@ -0,0 +1,8 @@ +FROM golang:1.21 + +WORKDIR /root +COPY . /root + +RUN go get ./... + +ENTRYPOINT go test github.com/pg-sharding/gorm-spqr/tests diff --git a/docker/gorm/gorm-spqr/README.md b/docker/gorm/gorm-spqr/README.md new file mode 100644 index 000000000..256757258 --- /dev/null +++ b/docker/gorm/gorm-spqr/README.md @@ -0,0 +1,2 @@ +# gorm-spqr +Tool for testing compatibility of SPQR and gorm diff --git a/docker/gorm/gorm-spqr/controllers/person.go b/docker/gorm/gorm-spqr/controllers/person.go new file mode 100644 index 000000000..85989cd3f --- /dev/null +++ b/docker/gorm/gorm-spqr/controllers/person.go @@ -0,0 +1,55 @@ +package controllers + +import "github.com/pg-sharding/gorm-spqr/models" + +func GetAllPersons() []models.Person { + var res []models.Person + models.DB.Find(&res) + + return res +} + +func GetPerson(id uint32) (*models.Person, error) { + var person models.Person + if err := models.DB.Where("id = ?", id).First(&person).Error; err != nil { + return nil, err + } + return &person, nil +} + +func WritePerson(person *models.Person) error { + tx := models.DB.Create(person) + return tx.Error +} + +func UpdatePerson(person *models.Person) error { + var current models.Person + if err := models.DB.Where("id = ?", person.ID).First(¤t).Error; err != nil { + return err + } + models.DB.Save(&person) + return nil +} + +func DeletePerson(id uint32) error { + var person models.Person + if err := models.DB.Where("id = ?", id).First(&person).Error; err != nil { + return err + } + + models.DB.Delete(&person) + return nil +} + +func WritePeople(people []*models.Person) error { + tx := models.DB.Create(people) + return tx.Error +} + +func GetPeople(from uint32, to uint32) ([]*models.Person, error) { + people := make([]*models.Person, 0) + if err := models.DB.Where("id >= ? AND id <= ?", from, to).Find(&people).Error; err != nil { + return nil, err + } + return people, nil +} diff --git a/docker/gorm/gorm-spqr/go.mod b/docker/gorm/gorm-spqr/go.mod new file mode 100644 index 000000000..15f8ee592 --- /dev/null +++ b/docker/gorm/gorm-spqr/go.mod @@ -0,0 +1,20 @@ +module github.com/pg-sharding/gorm-spqr + +go 1.21 + +require ( + github.com/jackc/pgx/v5 v5.4.3 + gorm.io/driver/postgres v1.5.4 + gorm.io/gorm v1.25.5 + gotest.tools/v3 v3.5.1 +) + +require ( + github.com/google/go-cmp v0.5.9 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/text v0.13.0 // indirect +) diff --git a/docker/gorm/gorm-spqr/go.sum b/docker/gorm/gorm-spqr/go.sum new file mode 100644 index 000000000..fdbd13655 --- /dev/null +++ b/docker/gorm/gorm-spqr/go.sum @@ -0,0 +1,36 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/docker/gorm/gorm-spqr/models/person.go b/docker/gorm/gorm-spqr/models/person.go new file mode 100644 index 000000000..232a6f437 --- /dev/null +++ b/docker/gorm/gorm-spqr/models/person.go @@ -0,0 +1,8 @@ +package models + +type Person struct { + ID uint32 `json:"id" gorm:"primary_key"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + Email string `json:"email"` +} diff --git a/docker/gorm/gorm-spqr/models/setup.go b/docker/gorm/gorm-spqr/models/setup.go new file mode 100644 index 000000000..6587dce65 --- /dev/null +++ b/docker/gorm/gorm-spqr/models/setup.go @@ -0,0 +1,77 @@ +package models + +import ( + "context" + "fmt" + "github.com/jackc/pgx/v5" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "os" +) + +var DB *gorm.DB + +func SetupSharding() { + dsn := fmt.Sprintf( + "host=%s user=%s dbname=%s port=%s %s", + os.Getenv("DB_HOST"), + "spqr-console", + "spqr-console", + os.Getenv("DB_PORT"), + os.Getenv("EXTRA_PARAMS"), + ) + conn, err := pgx.Connect(context.Background(), dsn) + if err != nil { + panic(fmt.Errorf("failed to connect to database: %s", err)) + } + defer func() { + _ = conn.Close(context.Background()) + }() + + _, err = conn.Exec(context.Background(), "CREATE DISTRIBUTION ds1 COLUMN TYPES integer;") + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "could not setup sharding: %s\n", err) + } + _, err = conn.Exec(context.Background(), "CREATE SHARDING RULE r1 COLUMNS id FOR DISTRIBUTION ds1;") + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "could not setup sharding: %s\n", err) + } + _, err = conn.Exec(context.Background(), "CREATE KEY RANGE krid1 FROM 1 TO 100 ROUTE TO sh1 FOR DISTRIBUTION ds1;") + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "could not setup sharding: %s\n", err) + } + _, err = conn.Exec(context.Background(), "CREATE KEY RANGE krid2 FROM 100 TO 200 ROUTE TO sh2 FOR DISTRIBUTION ds1;") + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "could not setup sharding: %s\n", err) + } + _, err = conn.Exec(context.Background(), "ALTER DISTRIBUTION ds1 ATTACH RELATION people DISTRIBUTION KEY id;") + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "could not setup sharding: %s\n", err) + } +} + +func ConnectDatabase() { + dsn := fmt.Sprintf( + "host=%s user=%s password=%s dbname=%s port=%s TimeZone=UTC %s", + os.Getenv("DB_HOST"), + os.Getenv("DB_USER"), + os.Getenv("DB_PASSWORD"), + os.Getenv("DB_NAME"), + os.Getenv("DB_PORT"), + os.Getenv("EXTRA_PARAMS"), + ) + database, err := gorm.Open(postgres.New(postgres.Config{ + DSN: dsn, + PreferSimpleProtocol: true, + }), &gorm.Config{}) + + if err != nil { + panic("Failed to connect to database") + } + + if err = database.AutoMigrate(&Person{}); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "could not migrate: %s", err) + } + + DB = database +} diff --git a/docker/gorm/gorm-spqr/tests/person_test.go b/docker/gorm/gorm-spqr/tests/person_test.go new file mode 100644 index 000000000..ccc7e297f --- /dev/null +++ b/docker/gorm/gorm-spqr/tests/person_test.go @@ -0,0 +1,157 @@ +package tests + +import ( + "fmt" + "os" + "testing" + + "github.com/pg-sharding/gorm-spqr/controllers" + "github.com/pg-sharding/gorm-spqr/models" + "github.com/pg-sharding/gorm-spqr/utils" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +func setup() { + models.ConnectDatabase() +} + +func tearDown(t *testing.T) { + if err := utils.DeleteAll(); err != nil { + t.Errorf("Error running tearDown: %s", err) + } +} + +func TestMain(m *testing.M) { + models.SetupSharding() + code := m.Run() + _ = utils.DeleteAll() + os.Exit(code) +} + +func TestCreatePerson(t *testing.T) { + setup() + + var person = models.Person{ + ID: 1, + FirstName: "John", + LastName: "Smith", + } + + err := controllers.WritePerson(&person) + assert.Check(t, err, "could not write: %s", err) + + var allPersons = controllers.GetAllPersons() + assert.Check(t, is.Len(allPersons, 1), "Expected to have 1 person in db, got %d", len(allPersons)) + + tearDown(t) +} + +func TestReadPerson(t *testing.T) { + setup() + + var person = models.Person{ + ID: 1, + FirstName: "John", + LastName: "Smith", + } + + err := controllers.WritePerson(&person) + assert.Check(t, err, "could not write: %s", err) + + var personDb *models.Person + personDb, err = controllers.GetPerson(person.ID) + assert.Check(t, err, "error getting person: %s", err) + assert.DeepEqual(t, person, *personDb) + + tearDown(t) +} + +func TestUpdatePerson(t *testing.T) { + setup() + + var person = models.Person{ + ID: 1, + FirstName: "John", + LastName: "Smith", + } + + err := controllers.WritePerson(&person) + assert.Check(t, err, "could not write: %s", err) + + person.Email = "foo@bar" + + err = controllers.UpdatePerson(&person) + assert.Check(t, err, "error updating: %s", err) + + var personDb *models.Person + personDb, err = controllers.GetPerson(person.ID) + assert.Check(t, err, "error getting person: %s", err) + assert.DeepEqual(t, person, *personDb) + + tearDown(t) +} + +func TestDeletePerson(t *testing.T) { + setup() + + var person = models.Person{ + ID: 1, + FirstName: "John", + LastName: "Smith", + } + + err := controllers.WritePerson(&person) + assert.Check(t, err, "could not write: %s", err) + + var allPersons = controllers.GetAllPersons() + assert.Check(t, is.Len(allPersons, 1), "Expected to have 1 person in db, got %d", len(allPersons)) + + err = controllers.DeletePerson(person.ID) + assert.Check(t, err, "error deleting person: %s", err) + + allPersons = controllers.GetAllPersons() + assert.Check(t, is.Len(allPersons, 0), "Expected to have no people in db, got %d", len(allPersons)) + + tearDown(t) +} + +func TestMultipleWrite(t *testing.T) { + setup() + + var firstShard = make([]*models.Person, 0) + for i := 1; i < 100; i++ { + person := &models.Person{ + ID: uint32(i), + FirstName: fmt.Sprintf("Name_%d", i), + } + firstShard = append(firstShard, person) + } + + err := controllers.WritePeople(firstShard) + assert.Check(t, err, "could not write: %s", err) + + var secondShard = make([]*models.Person, 0) + for i := 100; i < 200; i++ { + person := &models.Person{ + ID: uint32(i), + FirstName: fmt.Sprintf("Name_%d", i), + } + secondShard = append(secondShard, person) + } + + err = controllers.WritePeople(secondShard) + assert.Check(t, err, "could not write: %s", err) + + firstShardPeople, err := controllers.GetPeople(uint32(1), uint32(99)) + assert.Check(t, err, "could not get people: %s", err) + + assert.Check(t, is.Equal(len(firstShardPeople), 99), "expected to have %d records on 1st shard, got %d", 99, len(firstShardPeople)) + + secondShardPeople, err := controllers.GetPeople(uint32(100), uint32(199)) + assert.Check(t, err, "could not get people: %s", err) + + assert.Check(t, is.Equal(len(secondShardPeople), 100), "expected to have %d records on 1st shard, got %d", 100, len(secondShardPeople)) + + tearDown(t) +} diff --git a/docker/gorm/gorm-spqr/utils/utils.go b/docker/gorm/gorm-spqr/utils/utils.go new file mode 100644 index 000000000..f2a782f78 --- /dev/null +++ b/docker/gorm/gorm-spqr/utils/utils.go @@ -0,0 +1,7 @@ +package utils + +import "github.com/pg-sharding/gorm-spqr/models" + +func DeleteAll() error { + return models.DB.Migrator().DropTable(&models.Person{}) +} diff --git a/docker/gorm/test.sh b/docker/gorm/test.sh index 08c43520e..d4f131400 100755 --- a/docker/gorm/test.sh +++ b/docker/gorm/test.sh @@ -1,4 +1,2 @@ -# docker run -e DB_HOST='odyssey' -e DB_PORT=6432 -e DB_USER='spqr-console' -e DB_NAME='spqr-console' --network=odyssey_od_net gorm-tests -sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose -sudo chmod +x /usr/local/bin/docker-compose -COMPOSE_PROJECT_NAME=odyssey docker-compose -f /docker-compose-test.yml up gorm-test \ No newline at end of file +docker build -t gorm-tests /gorm/gorm-spqr +docker run -e DB_HOST='odyssey' -e DB_PORT=6432 -e DB_USER='spqr-console' -e DB_NAME='spqr-console' --network=odyssey_od_net gorm-tests \ No newline at end of file