diff --git a/Makefile b/Makefile index a8ac961f02..eacc4c69ae 100644 --- a/Makefile +++ b/Makefile @@ -135,6 +135,7 @@ go-tidy: make -C flyteplugins go-tidy make -C flytestdlib go-tidy make -C flytecopilot go-tidy + make -C flytectl go-tidy .PHONY: lint-helm-charts lint-helm-charts: diff --git a/flyteadmin/go.mod b/flyteadmin/go.mod index 09f43329f7..82e2189f34 100644 --- a/flyteadmin/go.mod +++ b/flyteadmin/go.mod @@ -48,7 +48,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - github.com/wI2L/jsondiff v0.5.0 + github.com/wI2L/jsondiff v0.6.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.24.0 golang.org/x/net v0.27.0 @@ -167,6 +167,7 @@ require ( github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/shamaton/msgpack/v2 v2.2.2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -176,10 +177,11 @@ require ( github.com/spf13/viper v1.11.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect diff --git a/flyteadmin/go.sum b/flyteadmin/go.sum index f872388bff..afd775c3ba 100644 --- a/flyteadmin/go.sum +++ b/flyteadmin/go.sum @@ -1159,6 +1159,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/santhosh-tekuri/jsonschema/v2 v2.1.0/go.mod h1:yzJzKUGV4RbWqWIBBP4wSOBqavX5saE02yirLS0OTyg= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -1271,8 +1272,8 @@ github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH github.com/tidwall/gjson v1.6.8/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/gjson v1.7.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -1280,8 +1281,9 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= @@ -1302,8 +1304,8 @@ github.com/unrolled/secure v0.0.0-20181005190816-ff9db2ff917f/go.mod h1:mnPT77IA github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= -github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= @@ -1319,6 +1321,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.elastic.co/apm v1.8.0/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= go.elastic.co/apm/module/apmhttp v1.8.0/go.mod h1:9LPFlEON51/lRbnWDfqAWErihIiAFDUMfMV27YjoWQ8= go.elastic.co/apm/module/apmot v1.8.0/go.mod h1:Q5Xzabte8G/fkvDjr1jlDuOSUt9hkVWNZEHh6ZNaTjI= diff --git a/flytectl/go.mod b/flytectl/go.mod index b657a02d4d..a8b0fe8bb2 100644 --- a/flytectl/go.mod +++ b/flytectl/go.mod @@ -141,6 +141,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/shamaton/msgpack/v2 v2.2.2 // indirect github.com/spf13/afero v1.9.2 // indirect github.com/spf13/cast v1.4.1 // indirect @@ -148,6 +149,12 @@ require ( github.com/spf13/viper v1.11.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/wI2L/jsondiff v0.6.0 // indirect + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/flytectl/go.sum b/flytectl/go.sum index f57ca65c0a..9f81c0ec9d 100644 --- a/flytectl/go.sum +++ b/flytectl/go.sum @@ -418,6 +418,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/shamaton/msgpack/v2 v2.2.2 h1:GOIg0c9LV04VwzOOqZSrmsv/JzjNOOMxnS/HvOHGdgs= github.com/shamaton/msgpack/v2 v2.2.2/go.mod h1:6khjYnkx73f7VQU7wjcFS9DFjs+59naVWJv1TB7qdOI= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -457,6 +459,18 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -466,6 +480,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zalando/go-keyring v0.1.1 h1:w2V9lcx/Uj4l+dzAf1m9s+DJ1O8ROkEHnynonHjTcYE= github.com/zalando/go-keyring v0.1.1/go.mod h1:OIC+OZ28XbmwFxU/Rp9V7eKzZjamBJwRzC8UFJH9+L8= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/flyteidl/go.mod b/flyteidl/go.mod index 0da94cea32..6653ce5a87 100644 --- a/flyteidl/go.mod +++ b/flyteidl/go.mod @@ -5,6 +5,7 @@ go 1.22 require ( github.com/flyteorg/flyte/flytestdlib v0.0.0-00010101000000-000000000000 github.com/go-test/deep v1.0.7 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/protobuf v1.5.3 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 @@ -55,7 +56,6 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect diff --git a/flytepropeller/go.mod b/flytepropeller/go.mod index c819278d90..c129312e44 100644 --- a/flytepropeller/go.mod +++ b/flytepropeller/go.mod @@ -22,11 +22,14 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 + github.com/santhosh-tekuri/jsonschema v1.2.4 github.com/shamaton/msgpack/v2 v2.2.2 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 + github.com/wI2L/jsondiff v0.6.0 + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 @@ -124,6 +127,10 @@ require ( github.com/spf13/viper v1.11.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect diff --git a/flytepropeller/go.sum b/flytepropeller/go.sum index 37a8766913..63c498dc77 100644 --- a/flytepropeller/go.sum +++ b/flytepropeller/go.sum @@ -373,6 +373,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/shamaton/msgpack/v2 v2.2.2 h1:GOIg0c9LV04VwzOOqZSrmsv/JzjNOOMxnS/HvOHGdgs= github.com/shamaton/msgpack/v2 v2.2.2/go.mod h1:6khjYnkx73f7VQU7wjcFS9DFjs+59naVWJv1TB7qdOI= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -410,11 +412,25 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/flytepropeller/pkg/compiler/validators/typing.go b/flytepropeller/pkg/compiler/validators/typing.go index 66847410f7..5f8812a602 100644 --- a/flytepropeller/pkg/compiler/validators/typing.go +++ b/flytepropeller/pkg/compiler/validators/typing.go @@ -1,11 +1,18 @@ package validators import ( + "bytes" + "context" + "encoding/json" "strings" structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/santhosh-tekuri/jsonschema" + "github.com/wI2L/jsondiff" + jscmp "gitlab.com/yvesf/json-schema-compare" flyte "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" + "github.com/flyteorg/flyte/flytestdlib/logger" ) type typeChecker interface { @@ -16,6 +23,67 @@ type trivialChecker struct { literalType *flyte.LiteralType } +func isSuperTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { + // Check if the source schema is a supertype of the target schema, beyond simple inheritance. + // For custom types, we expect the JSON schemas in the metadata to come from the same JSON schema package, + // specifically draft 2020-12 from Mashumaro. + + srcSchemaBytes, _ := json.Marshal(sourceMetaData.GetFields()) + tgtSchemaBytes, _ := json.Marshal(targetMetaData.GetFields()) + + compiler := jsonschema.NewCompiler() + + err := compiler.AddResource("src", bytes.NewReader(srcSchemaBytes)) + if err != nil { + return false + } + err = compiler.AddResource("tgt", bytes.NewReader(tgtSchemaBytes)) + if err != nil { + return false + } + + srcSchema, _ := compiler.Compile("src") + tgtSchema, _ := compiler.Compile("tgt") + + // Compare the two schemas + errs := jscmp.Compare(tgtSchema, srcSchema) + + // Ignore the "not implemented" errors from json-schema-compare (additionalProperties, additionalItems, etc.) + // While handling nested structures, we might have multiple "not implemented" errors for a single field as well. + // If all the errors are "not implemented", we can consider the source schema as a supertype of the target schema. + for _, err := range errs { + if !strings.Contains(err.Error(), "not implemented") { + return false + } + } + + return true +} + +func isSameTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { + srcSchemaBytes, err := json.Marshal(sourceMetaData.GetFields()) + if err != nil { + logger.Infof(context.Background(), "Failed to marshal source metadata: %v", err) + return false + } + + tgtSchemaBytes, err := json.Marshal(targetMetaData.GetFields()) + if err != nil { + logger.Infof(context.Background(), "Failed to marshal target metadata: %v", err) + return false + } + + // Use jsondiff to compare the two schemas + patch, err := jsondiff.CompareJSON(srcSchemaBytes, tgtSchemaBytes) + if err != nil { + logger.Infof(context.Background(), "Failed to compare JSON schemas: %v", err) + return false + } + + // If the length of the patch is zero, the two JSON structures are identical + return len(patch) == 0 +} + // CastsFrom is a trivial type checker merely checks if types match exactly. func (t trivialChecker) CastsFrom(upstreamType *flyte.LiteralType) bool { // If upstream is an enum, it can be consumed as a string downstream @@ -35,6 +103,23 @@ func (t trivialChecker) CastsFrom(upstreamType *flyte.LiteralType) bool { return false } + // Related Issue: https://github.com/flyteorg/flyte/issues/5489 + // RFC: https://github.com/flyteorg/flyte/blob/master/rfc/system/5741-binary-idl-with-message-pack.md#flytepropeller + if upstreamType.GetSimple() == flyte.SimpleType_STRUCT && t.literalType.GetSimple() == flyte.SimpleType_STRUCT { + // Json Schema is stored in Metadata + upstreamMetaData := upstreamType.GetMetadata() + downstreamMetaData := t.literalType.GetMetadata() + + // There's bug in flytekit's dataclass Transformer to generate JSON Scheam before, + // in some case, we the JSON Schema will be nil, so we can only pass it to support + // backward compatible. (reference task should be supported.) + if upstreamMetaData == nil || downstreamMetaData == nil { + return true + } + + return isSameTypeInJSON(upstreamMetaData, downstreamMetaData) || isSuperTypeInJSON(upstreamMetaData, downstreamMetaData) + } + // Ignore metadata when comparing types. upstreamTypeCopy := *upstreamType downstreamTypeCopy := *t.literalType diff --git a/flytepropeller/pkg/compiler/validators/typing_test.go b/flytepropeller/pkg/compiler/validators/typing_test.go index f2e407b986..2f5bc5531d 100644 --- a/flytepropeller/pkg/compiler/validators/typing_test.go +++ b/flytepropeller/pkg/compiler/validators/typing_test.go @@ -5,6 +5,7 @@ import ( structpb "github.com/golang/protobuf/ptypes/struct" "github.com/stretchr/testify/assert" + structpb2 "google.golang.org/protobuf/types/known/structpb" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) @@ -557,6 +558,837 @@ func TestMapCasting(t *testing.T) { assert.True(t, castable, "castable from Struct to struct") }) + t.Run("SameDataclassOneLevel(draft 2020-12)", func(t *testing.T) { + /* + @dataclass + class A: + a: int + */ + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + ) + assert.True(t, castable, "same dataclass castable with one level properties") + }) + + t.Run("SameDataclassTwoLevel(draft 2020-12)", func(t *testing.T) { + /* + @dataclass + class A: + a: int + + @dataclass + class B: + b: A + */ + + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "b"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "B"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "b": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "required": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": { + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "b"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "B"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "b": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "required": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": { + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + ) + assert.True(t, castable, "same dataclass castable with two level properties") + }) + + t.Run("DiffDataclassTwoLevel(draft 2020-12)", func(t *testing.T) { + /* + @dataclass + class A: + a: int + + @dataclass + class B: + b: A + + @dataclass + class C: + c: str + + @dataclass + class D: + d: C + + Compare B and D + */ + + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "b"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "B"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "b": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "required": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": { + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "d"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "D"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "d": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "required": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "c"}}}, + }, + }, + }, + "title": { + Kind: &structpb.Value_StringValue{StringValue: "C"}, + }, + "properties": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "c": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "string"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + ) + assert.False(t, castable, "different dataclass with two level properties not castable") + }) + + t.Run("SameBaseModelOneLevel(draft 2020-12)", func(t *testing.T) { + /* + class A(BaseModel): + a: int + */ + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + }, + }, + }, + ) + assert.True(t, castable, "same basemodel castable with one level properties") + }) + + t.Run("BigToSmallAndChildToParent(dataclass draft 2020-12)", func(t *testing.T) { + /* + @dataclass + class A: + a: int + + @dataclass + class B(A): + b: Optional[str] = None + */ + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{ + {Kind: &structpb.Value_StringValue{StringValue: "a"}}, + {Kind: &structpb.Value_StringValue{StringValue: "b"}}, + }, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "B"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + "b": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "default": { + Kind: &structpb.Value_NullValue{}, + }, + "anyOf": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{ + { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "string"}, + }, + }, + }, + }, + }, + { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "null"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + ) + assert.True(t, castable, "dataclass castable from child to parent (big to small)") + }) + + t.Run("SmallToBigAndParentToChild(dataclass draft 2020-12)", func(t *testing.T) { + /* + @dataclass + class A: + a: int + + @dataclass + class B(A): + b: Optional[str] = None + */ + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{ + {Kind: &structpb.Value_StringValue{StringValue: "a"}}, + {Kind: &structpb.Value_StringValue{StringValue: "b"}}, + }, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "B"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + "b": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "default": { + Kind: &structpb.Value_NullValue{}, + }, + "anyOf": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{ + { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "string"}, + }, + }, + }, + }, + }, + { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "null"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + ) + assert.False(t, castable, "dataclass not castable from parent to child (small to big)") + }) + t.Run("MismatchedMapNestLevels_Scalar", func(t *testing.T) { castable := AreTypesCastable( &core.LiteralType{ diff --git a/go.mod b/go.mod index 65cbc7be79..48cf680956 100644 --- a/go.mod +++ b/go.mod @@ -164,6 +164,7 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/robfig/cron/v3 v3.0.0 // indirect github.com/samber/lo v1.47.0 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.10.0+incompatible // indirect github.com/shamaton/msgpack/v2 v2.2.2 // indirect @@ -175,11 +176,12 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/testify v1.9.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/wI2L/jsondiff v0.5.0 // indirect + github.com/wI2L/jsondiff v0.6.0 // indirect + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/go.sum b/go.sum index d0d2474eec..d61dd5d2d9 100644 --- a/go.sum +++ b/go.sum @@ -1198,6 +1198,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/santhosh-tekuri/jsonschema/v2 v2.1.0/go.mod h1:yzJzKUGV4RbWqWIBBP4wSOBqavX5saE02yirLS0OTyg= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -1310,8 +1311,8 @@ github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH github.com/tidwall/gjson v1.6.8/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/gjson v1.7.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -1319,8 +1320,9 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= @@ -1341,8 +1343,8 @@ github.com/unrolled/secure v0.0.0-20181005190816-ff9db2ff917f/go.mod h1:mnPT77IA github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= -github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= @@ -1358,6 +1360,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.elastic.co/apm v1.8.0/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= go.elastic.co/apm/module/apmhttp v1.8.0/go.mod h1:9LPFlEON51/lRbnWDfqAWErihIiAFDUMfMV27YjoWQ8= go.elastic.co/apm/module/apmot v1.8.0/go.mod h1:Q5Xzabte8G/fkvDjr1jlDuOSUt9hkVWNZEHh6ZNaTjI=