diff --git a/docs/src/config/README.md b/docs/src/config/README.md index 2ec9db7..341305b 100755 --- a/docs/src/config/README.md +++ b/docs/src/config/README.md @@ -28,6 +28,7 @@ checks: The following check types are available: - [file](#file) + - [filediff](#filediff) - [yaml](#yaml) - [yamllint](#yamllint) - [json](#json) @@ -64,6 +65,42 @@ file: disallowed-pattern: '^(adminer|phpmyadmin|bigdump)?\.php$' ``` +### filediff +Checks for content changes in a file. + +| Field | Default | Required | Description | +|----------------|:-------:|:--------:|-------------------------------------------------------------------------------------------------------| +| target-file | - | Yes | The file to check for content changes | +| source-file | - | Yes | The file with the original content used for checking. Source file can be either remote or local file. | +| source-context | - | No | The key-value mapping to compile the source file if it is a Jinja2 template | +| context-lines | 0 | No | Specify the number context lines around the line changes in the diff | +| ignore-missing | false | No | Specify whether a missing target file is a fail | + +#### Example +```yaml +filediff: + - name: "Gitlab CI file - PaaS" + source: https://github.com/govCMS/scaffold/raw/master/.gitlab-ci.paas.yml + target: .gitlab-ci.paas.yml + ignore-missing: true + context-lines: 0 + severity: low + source-context: + GOVCMS_VERSION: 9 +``` +Result: +``` +# Breaches were detected + + ### Gitlab CI file - PaaS + -- [Target file .gitlab-ci.paas.yml is different from Source file https://github.com/govCMS/scaffold/raw/master/.gitlab-ci.paas.yml] diff: +--- https://github.com/govCMS/scaffold/raw/master/.gitlab-ci.paas.yml ++++ .gitlab-ci.paas.yml +@@ -9 +9 @@ +- ref: "8.x-master" ++ ref: "9.x-master" +``` + ### yaml Checks yaml files for the presence or absence of required/disallowed values. diff --git a/go.mod b/go.mod index 6e8585f..ff1c2ce 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,11 @@ require ( github.com/hasura/go-graphql-client v0.9.2 github.com/jmespath/go-jmespath v0.4.0 github.com/minio/selfupdate v0.4.0 - github.com/sirupsen/logrus v1.9.0 + github.com/nikolalohinski/gonja/v2 v2.1.5 + github.com/pmezard/go-difflib v1.0.0 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.4 github.com/vmware-labs/yaml-jsonpath v0.3.2 golang.org/x/oauth2 v0.6.0 gopkg.in/yaml.v3 v3.0.1 @@ -25,13 +27,17 @@ require ( github.com/antchfx/xpath v1.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.16.3 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/temoto/robotstxt v1.1.2 // indirect golang.org/x/crypto v0.17.0 // indirect diff --git a/go.sum b/go.sum index 3228d56..3cb329d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= @@ -13,6 +15,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 h1:aRd8M7HJVZOqn/vhOzrGcQH0lNAMkqMn+pXUYkatmcA= github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -20,6 +24,8 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= @@ -28,6 +34,8 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= @@ -52,9 +60,11 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= @@ -73,8 +83,9 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -91,24 +102,33 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/minio/selfupdate v0.4.0 h1:A7t07pN4Ch1tBTIRStW0KhUVyykz+2muCqFsITQeEW8= github.com/minio/selfupdate v0.4.0/go.mod h1:mcDkzMgq8PRcpCRJo/NlPY7U45O5dfYl2Y0Rg7IustY= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nikolalohinski/gonja/v2 v2.1.5 h1:oD8R+GpKMw6Xex9hmWvCQiWlvHfnbmSmu3F5nZ5eRI4= +github.com/nikolalohinski/gonja/v2 v2.1.5/go.mod h1:l9DuWJvT/BddBr2SsmEimESD6msSqRw7u5HzI2Um+sc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -116,8 +136,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= @@ -166,6 +187,8 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= diff --git a/pkg/checks/file/file.go b/pkg/checks/file/file.go new file mode 100644 index 0000000..4b8b773 --- /dev/null +++ b/pkg/checks/file/file.go @@ -0,0 +1,14 @@ +package file + +import "github.com/salsadigitalauorg/shipshape/pkg/config" + +//go:generate go run ../../../cmd/gen.go registry --checkpackage=file + +func RegisterChecks() { + config.ChecksRegistry[File] = func() config.Check { return &FileCheck{} } + config.ChecksRegistry[FileDiff] = func() config.Check { return &FileDiffCheck{} } +} + +func init() { + RegisterChecks() +} diff --git a/pkg/checks/file/filecheck.go b/pkg/checks/file/filecheck.go index 768666e..d616f63 100644 --- a/pkg/checks/file/filecheck.go +++ b/pkg/checks/file/filecheck.go @@ -8,8 +8,6 @@ import ( "github.com/salsadigitalauorg/shipshape/pkg/utils" ) -//go:generate go run ../../../cmd/gen.go registry --checkpackage=file - // FileCheck is a simple File absence check which can be for a single // file or a pattern. type FileCheck struct { @@ -22,14 +20,6 @@ type FileCheck struct { const File config.CheckType = "file" -func RegisterChecks() { - config.ChecksRegistry[File] = func() config.Check { return &FileCheck{} } -} - -func init() { - RegisterChecks() -} - // Merge implementation for file check. func (c *FileCheck) Merge(mergeCheck config.Check) error { fileMergeCheck := mergeCheck.(*FileCheck) diff --git a/pkg/checks/file/filediffcheck.go b/pkg/checks/file/filediffcheck.go new file mode 100644 index 0000000..488d5c0 --- /dev/null +++ b/pkg/checks/file/filediffcheck.go @@ -0,0 +1,148 @@ +package file + +import ( + "errors" + "fmt" + "github.com/nikolalohinski/gonja/v2" + "github.com/nikolalohinski/gonja/v2/exec" + "github.com/pmezard/go-difflib/difflib" + "github.com/salsadigitalauorg/shipshape/pkg/config" + "github.com/salsadigitalauorg/shipshape/pkg/result" + "github.com/salsadigitalauorg/shipshape/pkg/utils" + "io/fs" + "os" + "path/filepath" +) + +type FileDiffCheck struct { + config.CheckBase `yaml:",inline"` + // TargetFile will be compared with SourceFile. + TargetFile string `yaml:"target"` + // SourceFile can be a local file or a remote URI. + SourceFile string `yaml:"source"` + // SourceContext list of key-values to compile the source file as a Jinja template. + SourceContext map[string]any `yaml:"source-context"` + // ContextLines number of context lines around the line changes. + ContextLines int `yaml:"context-lines"` + // IgnoreMissing allows non-existent files to not be counted as a Fail. + // Using a pointer here so that we can differentiate between + // false (default value) and an empty value. + IgnoreMissing *bool `yaml:"ignore-missing"` +} + +const FileDiff config.CheckType = "filediff" + +// RequiresData implementation for FileDiffCheck. +func (c *FileDiffCheck) RequiresData() bool { return true } + +// FetchData implementation for FileDiffCheck +func (c *FileDiffCheck) FetchData() { + if len(c.SourceFile) == 0 { + c.AddBreach(&result.ValueBreach{Value: "no source file provided"}) + return + } + + if len(c.TargetFile) == 0 { + c.AddBreach(&result.ValueBreach{Value: "no target file provided"}) + return + } + + c.DataMap = map[string][]byte{} + var err error + // Fetch the target file. + c.DataMap["target"], err = os.ReadFile(filepath.Join(config.ProjectDir, c.TargetFile)) + if err != nil { + // No failure if missing file and ignoring missing. + var pathError *fs.PathError + if errors.As(err, &pathError) && c.IgnoreMissing != nil && *c.IgnoreMissing { + c.AddPass(fmt.Sprintf("Target file %s does not exist", c.TargetFile)) + c.Result.Status = result.Pass + return + } else { + c.AddBreach(&result.ValueBreach{ + ValueLabel: "error reading target file: " + c.TargetFile, + Value: err.Error()}) + return + } + } + + // Fetch the source file. + if utils.StringIsUrl(c.SourceFile) { + c.DataMap["source"], err = utils.FetchContentFromUrl(c.SourceFile) + } else { + c.DataMap["source"], err = os.ReadFile(filepath.Join(config.ProjectDir, c.SourceFile)) + } + + if err != nil { + c.AddBreach(&result.ValueBreach{ + ValueLabel: "error fetching source file: " + c.SourceFile, + Value: err.Error()}) + return + } + + // Parse the source file as a Jinja template. + if c.SourceContext != nil && len(c.SourceContext) > 0 { + jinjaTemplate, jinjaErr := gonja.FromBytes(c.DataMap["source"]) + if jinjaErr != nil { + c.AddBreach(&result.ValueBreach{ + ValueLabel: "error parsing source file: " + c.SourceFile, + Value: jinjaErr.Error()}) + return + } + + jinjaContext := exec.NewContext(c.SourceContext) + c.DataMap["source"], jinjaErr = jinjaTemplate.ExecuteToBytes(jinjaContext) + if jinjaErr != nil { + c.AddBreach(&result.ValueBreach{ + ValueLabel: "error compiling source file with source context: " + c.SourceFile, + Value: jinjaErr.Error()}) + return + } + } + + return +} + +// Merge implementation for FileDiffCheck check. +func (c *FileDiffCheck) Merge(mergeCheck config.Check) error { + yCheck := mergeCheck.(*FileDiffCheck) + if err := c.CheckBase.Merge(&yCheck.CheckBase); err != nil { + return err + } + + if yCheck.ContextLines != 0 && yCheck.ContextLines != c.ContextLines { + c.ContextLines = yCheck.ContextLines + } + + if yCheck.SourceContext != nil && len(yCheck.SourceContext) != 0 { + c.SourceContext = yCheck.SourceContext + } + + utils.MergeString(&c.SourceFile, yCheck.SourceFile) + utils.MergeString(&c.TargetFile, yCheck.TargetFile) + utils.MergeBoolPtrs(c.IgnoreMissing, yCheck.IgnoreMissing) + return nil +} + +// UnmarshalDataMap implementation for FileDiffCheck check. +func (c *FileDiffCheck) UnmarshalDataMap() {} + +// RunCheck implementation for FileDiffCheck check. +func (c *FileDiffCheck) RunCheck() { + unifiedDiff := difflib.UnifiedDiff{ + A: difflib.SplitLines(string(c.DataMap["source"])), + B: difflib.SplitLines(string(c.DataMap["target"])), + FromFile: c.SourceFile, + ToFile: c.TargetFile, + Context: c.ContextLines, + } + diff, _ := difflib.GetUnifiedDiffString(unifiedDiff) + if len(diff) == 0 { + c.AddPass(fmt.Sprintf("Target file %s is identical to Source file %s", c.TargetFile, c.SourceFile)) + c.Result.Status = result.Pass + } else { + c.AddBreach(&result.ValueBreach{ + ValueLabel: fmt.Sprintf("Target file %s is different from Source file %s", c.TargetFile, c.SourceFile), + Value: fmt.Sprintf("diff: \n%s", diff)}) + } +} diff --git a/pkg/checks/file/filediffcheck_test.go b/pkg/checks/file/filediffcheck_test.go new file mode 100644 index 0000000..0703e0f --- /dev/null +++ b/pkg/checks/file/filediffcheck_test.go @@ -0,0 +1,291 @@ +package file_test + +import ( + "github.com/salsadigitalauorg/shipshape/pkg/checks/file" + "github.com/salsadigitalauorg/shipshape/pkg/config" + "github.com/salsadigitalauorg/shipshape/pkg/result" + "github.com/stretchr/testify/assert" + "testing" +) + +var cTrue = true + +func TestFileDiffCheck_Merge(t *testing.T) { + assertions := assert.New(t) + + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck1"}, + SourceFile: "source-initial", + TargetFile: "target-initial", + } + err := c.Merge(&file.FileDiffCheck{ + SourceFile: "source-final", + TargetFile: "target-final", + }) + assertions.Nil(err) + assertions.EqualValues(file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck1"}, + SourceFile: "source-final", + TargetFile: "target-final", + }, c) + + c = file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck2"}, + SourceFile: "source-initial", + TargetFile: "target-initial", + ContextLines: 0, + } + err = c.Merge(&file.FileDiffCheck{ + ContextLines: 1, + }) + assertions.Nil(err) + assertions.EqualValues(file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck2"}, + SourceFile: "source-initial", + TargetFile: "target-initial", + ContextLines: 1, + }, c) + + c = file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck3"}, + SourceFile: "source-initial", + TargetFile: "target-initial", + SourceContext: map[string]any{"key1": "value1"}, + } + err = c.Merge(&file.FileDiffCheck{ + SourceContext: map[string]any{"key1": "value2"}, + }) + assertions.Nil(err) + assertions.EqualValues(file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck3"}, + SourceFile: "source-initial", + TargetFile: "target-initial", + SourceContext: map[string]any{"key1": "value2"}, + }, c) +} + +func TestFileDiffCheck_FetchData(t *testing.T) { + assertions := assert.New(t) + + config.ProjectDir = "testdata/filediff/" + + t.Run("failOnNoSource", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck"}, + TargetFile: "file1.txt", + } + c.Init(file.FileDiff) + c.FetchData() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Fail, c.Result.Status) + assertions.Equal(0, len(c.Result.Passes)) + assertions.EqualValues( + []result.Breach{&result.ValueBreach{ + CheckType: "filediff", + CheckName: "filediffcheck", + BreachType: result.BreachTypeValue, + Severity: "normal", + Value: "no source file provided", + }}, + c.Result.Breaches, + ) + }) + + t.Run("failOnNoTarget", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck"}, + SourceFile: "file1.txt", + } + c.Init(file.FileDiff) + c.FetchData() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Fail, c.Result.Status) + assertions.Equal(0, len(c.Result.Passes)) + assertions.EqualValues( + []result.Breach{&result.ValueBreach{ + CheckType: "filediff", + CheckName: "filediffcheck", + BreachType: result.BreachTypeValue, + Severity: "normal", + Value: "no target file provided", + }}, + c.Result.Breaches, + ) + }) + + t.Run("failOnSourceNotExist", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck1"}, + SourceFile: "file0.txt", + TargetFile: "file1.txt", + } + c.Init(file.FileDiff) + c.FetchData() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Fail, c.Result.Status) + assertions.Equal(0, len(c.Result.Passes)) + assertions.EqualValues( + []result.Breach{&result.ValueBreach{ + CheckType: "filediff", + CheckName: "filediffcheck1", + BreachType: result.BreachTypeValue, + Severity: "normal", + ValueLabel: "error fetching source file: file0.txt", + Value: "open testdata/filediff/file0.txt: no such file or directory", + }}, + c.Result.Breaches, + ) + }) + + t.Run("failOnTargetNotExist", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck1"}, + SourceFile: "file1.txt", + TargetFile: "file0.txt", + } + c.Init(file.FileDiff) + c.FetchData() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Fail, c.Result.Status) + assertions.Equal(0, len(c.Result.Passes)) + assertions.EqualValues( + []result.Breach{&result.ValueBreach{ + CheckType: "filediff", + CheckName: "filediffcheck1", + BreachType: result.BreachTypeValue, + Severity: "normal", + ValueLabel: "error reading target file: file0.txt", + Value: "open testdata/filediff/file0.txt: no such file or directory", + }}, + c.Result.Breaches, + ) + }) + + t.Run("passOnIgnoreMissingTarget", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck1"}, + SourceFile: "file1.txt", + TargetFile: "file0.txt", + IgnoreMissing: &cTrue, + } + c.Init(file.FileDiff) + c.FetchData() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Pass, c.Result.Status) + assertions.Equal(0, len(c.Result.Breaches)) + assertions.EqualValues([]string{"Target file file0.txt does not exist"}, c.Result.Passes) + }) + + t.Run("failOnMalformedJinjaSource", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck1"}, + SourceFile: "file3.txt", + TargetFile: "file2.txt", + SourceContext: map[string]any{"VERSION": 1}, + } + c.Init(file.FileDiff) + c.FetchData() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Fail, c.Result.Status) + assertions.Equal(0, len(c.Result.Passes)) + assertions.EqualValues( + []result.Breach{&result.ValueBreach{ + CheckType: "filediff", + CheckName: "filediffcheck1", + BreachType: result.BreachTypeValue, + Severity: "normal", + ValueLabel: "error parsing source file: file3.txt", + Value: "failed to parse template 'This is file #{{ VERSION }.\n': '}}' expected here (Line: 0 Col: 0, near \"Unexpected delimiter \"}\"\")", + }}, + c.Result.Breaches, + ) + }) +} + +func TestFileDiffCheck_RunCheck(t *testing.T) { + assertions := assert.New(t) + + config.ProjectDir = "testdata/filediff/" + + t.Run("passOnIdenticalFiles", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck"}, + SourceFile: "file1.txt", + TargetFile: "file1.txt", + } + c.Init(file.FileDiff) + c.FetchData() + c.RunCheck() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Pass, c.Result.Status) + assertions.Equal(0, len(c.Result.Breaches)) + assertions.EqualValues([]string{"Target file file1.txt is identical to Source file file1.txt"}, c.Result.Passes) + }) + + t.Run("failOnDifferentFiles", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck"}, + SourceFile: "file1.txt", + TargetFile: "file2.txt", + } + c.Init(file.FileDiff) + c.FetchData() + c.RunCheck() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Fail, c.Result.Status) + assertions.Equal(0, len(c.Result.Passes)) + assertions.EqualValues( + []result.Breach{&result.ValueBreach{ + CheckType: "filediff", + CheckName: "filediffcheck", + BreachType: result.BreachTypeValue, + Severity: "normal", + ValueLabel: "Target file file2.txt is different from Source file file1.txt", + Value: "diff: \n--- file1.txt\n+++ file2.txt\n@@ -1 +1 @@\n-This is file #1.\n+This is file #2.\n", + }}, + c.Result.Breaches, + ) + }) + + t.Run("passOnIdenticalJinjaFiles", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck"}, + SourceFile: "file4.txt", + TargetFile: "file1.txt", + SourceContext: map[string]any{"VERSION": 1}, + } + c.Init(file.FileDiff) + c.FetchData() + c.RunCheck() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Pass, c.Result.Status) + assertions.Equal(0, len(c.Result.Breaches)) + assertions.EqualValues([]string{"Target file file1.txt is identical to Source file file4.txt"}, c.Result.Passes) + }) + + t.Run("failOnDifferentJinjaFiles", func(t *testing.T) { + c := file.FileDiffCheck{ + CheckBase: config.CheckBase{Name: "filediffcheck"}, + SourceFile: "file4.txt", + TargetFile: "file2.txt", + SourceContext: map[string]any{"VERSION": 1}, + } + c.Init(file.FileDiff) + c.FetchData() + c.RunCheck() + c.Result.DetermineResultStatus(false) + assertions.Equal(result.Fail, c.Result.Status) + assertions.Equal(0, len(c.Result.Passes)) + assertions.EqualValues( + []result.Breach{&result.ValueBreach{ + CheckType: "filediff", + CheckName: "filediffcheck", + BreachType: result.BreachTypeValue, + Severity: "normal", + ValueLabel: "Target file file2.txt is different from Source file file4.txt", + Value: "diff: \n--- file4.txt\n+++ file2.txt\n@@ -1 +1 @@\n-This is file #1.\n+This is file #2.\n", + }}, + c.Result.Breaches, + ) + }) +} diff --git a/pkg/checks/file/testdata/filediff/file1.txt b/pkg/checks/file/testdata/filediff/file1.txt new file mode 100644 index 0000000..16cd50e --- /dev/null +++ b/pkg/checks/file/testdata/filediff/file1.txt @@ -0,0 +1 @@ +This is file #1. diff --git a/pkg/checks/file/testdata/filediff/file2.txt b/pkg/checks/file/testdata/filediff/file2.txt new file mode 100644 index 0000000..8f243e7 --- /dev/null +++ b/pkg/checks/file/testdata/filediff/file2.txt @@ -0,0 +1 @@ +This is file #2. diff --git a/pkg/checks/file/testdata/filediff/file3.txt b/pkg/checks/file/testdata/filediff/file3.txt new file mode 100644 index 0000000..3c50037 --- /dev/null +++ b/pkg/checks/file/testdata/filediff/file3.txt @@ -0,0 +1 @@ +This is file #{{ VERSION }. diff --git a/pkg/checks/file/testdata/filediff/file4.txt b/pkg/checks/file/testdata/filediff/file4.txt new file mode 100644 index 0000000..9739c6a --- /dev/null +++ b/pkg/checks/file/testdata/filediff/file4.txt @@ -0,0 +1 @@ +This is file #{{ VERSION }}.