From 82e581196697bf0158f9ba34a926aca951ff73d8 Mon Sep 17 00:00:00 2001 From: schnee Date: Sat, 7 Dec 2024 17:47:52 +0800 Subject: [PATCH 01/10] feat: devsandbox add diffs api --- cnb-builder-shim/Makefile | 1 + cnb-builder-shim/go.mod | 10 +- cnb-builder-shim/go.sum | 32 +--- .../internal/devsandbox/config/loader.go | 5 +- .../internal/devsandbox/config/types.go | 2 +- .../devsandbox/filediffer/filediffer.go | 159 ++++++++++++++++ .../filediffer/filediffer_suite_test.go | 13 ++ .../devsandbox/filediffer/filediffer_test.go | 169 ++++++++++++++++++ .../internal/devsandbox/filediffer/types.go | 41 +++++ .../internal/devsandbox/webserver/server.go | 73 +++++++- 10 files changed, 457 insertions(+), 48 deletions(-) create mode 100644 cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go create mode 100644 cnb-builder-shim/internal/devsandbox/filediffer/filediffer_suite_test.go create mode 100644 cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go create mode 100644 cnb-builder-shim/internal/devsandbox/filediffer/types.go diff --git a/cnb-builder-shim/Makefile b/cnb-builder-shim/Makefile index 27b1c83243..dfe370b62a 100644 --- a/cnb-builder-shim/Makefile +++ b/cnb-builder-shim/Makefile @@ -45,6 +45,7 @@ dev-launcher: heroku-builder-bionic: ## Build builder image which is based on `cloudnative-buildpacks/builders/heroku-builder` docker buildx bake heroku-builder-bionic -f docker-build/heroku-builder/docker-bake.hcl + .PHONY: heroku-builder-jammy heroku-builder-jammy: ## Build builder image which is based on `cloudnative-buildpacks/builders/heroku-builder` **experimental** IMAGE_NAME=mirrors.tencent.com/bkpaas/bk-builder-heroku-jammy docker buildx bake heroku-builder-jammy -f docker-build/heroku-builder/docker-bake.hcl diff --git a/cnb-builder-shim/go.mod b/cnb-builder-shim/go.mod index 27b58b6013..88080c220a 100644 --- a/cnb-builder-shim/go.mod +++ b/cnb-builder-shim/go.mod @@ -6,6 +6,8 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/buildpacks/lifecycle v0.18.3 github.com/caarlos0/env/v10 v10.0.0 + github.com/docker/docker v27.3.1+incompatible + github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.9.1 github.com/go-logr/logr v1.4.2 github.com/google/go-containerregistry v0.16.1 @@ -16,25 +18,24 @@ require ( github.com/pelletier/go-toml/v2 v2.2.1 github.com/pkg/errors v0.9.1 github.com/shabbywu/logfmtr v0.2.3 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/docker/cli v24.0.6+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v27.3.1+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gin-contrib/cors v1.7.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -63,8 +64,6 @@ require ( github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/stretchr/testify v1.9.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/ulikunitz/xz v0.5.11 // indirect @@ -74,7 +73,6 @@ require ( go.opentelemetry.io/otel/trace v1.31.0 // indirect golang.org/x/arch v0.7.0 // indirect golang.org/x/crypto v0.28.0 // indirect - golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect diff --git a/cnb-builder-shim/go.sum b/cnb-builder-shim/go.sum index 52b2a5098d..ac9383cf09 100644 --- a/cnb-builder-shim/go.sum +++ b/cnb-builder-shim/go.sum @@ -1,3 +1,5 @@ +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= @@ -11,18 +13,12 @@ github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/buildpacks/lifecycle v0.18.3 h1:vNrQmzOpgdvyu+hYeQJesdpAGcyyYFR+R4zHWQhBOVo= github.com/buildpacks/lifecycle v0.18.3/go.mod h1:lLVVv+RTs2NA5T1Dh/cuvjW4LcUhYsBuY8+mxO3V3HI= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -56,8 +52,6 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw= @@ -78,8 +72,6 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -118,8 +110,6 @@ github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2g github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -130,8 +120,6 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -162,8 +150,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -198,15 +184,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -226,23 +208,17 @@ go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+q go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -250,8 +226,6 @@ golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/cnb-builder-shim/internal/devsandbox/config/loader.go b/cnb-builder-shim/internal/devsandbox/config/loader.go index ce4318209e..6d344040e4 100644 --- a/cnb-builder-shim/internal/devsandbox/config/loader.go +++ b/cnb-builder-shim/internal/devsandbox/config/loader.go @@ -48,14 +48,13 @@ func loadCORSConfigFromEnv() (CORSConfig, error) { AllowCredentials: allowCredentials, }, nil } + func loadServiceConfigFromEnv() (ServiceConfig, error) { corsConfig, err := loadCORSConfigFromEnv() if err != nil { return ServiceConfig{}, err } - return ServiceConfig{ - CORS: corsConfig, - }, nil + return ServiceConfig{CORS: corsConfig}, nil } func loadConfigFromEnv() (*Config, error) { diff --git a/cnb-builder-shim/internal/devsandbox/config/types.go b/cnb-builder-shim/internal/devsandbox/config/types.go index 7f4077fcd6..6d35dcc319 100644 --- a/cnb-builder-shim/internal/devsandbox/config/types.go +++ b/cnb-builder-shim/internal/devsandbox/config/types.go @@ -25,7 +25,7 @@ type SourceCodeConfig struct { type CORSConfig struct { // 允许的来源 AllowOrigins []string - // 允许的HTTP方法 + // 允许的 HTTP 方法 AllowMethods []string // 允许的请求头 AllowHeaders []string diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go new file mode 100644 index 0000000000..f4ef0ef500 --- /dev/null +++ b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go @@ -0,0 +1,159 @@ +/* + * TencentBlueKing is pleased to support the open source community by making + * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. + * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * We undertake not to change the open source license (MIT license) applicable + * to the current version of the project delivered to anyone in the future. + */ +package filediffer + +import ( + "bytes" + "io" + "os" + "os/exec" + "path" + "strings" + + "github.com/pkg/errors" +) + +// FileDiffer 文件变更对比器 +type FileDiffer struct { + srcPath string +} + +// New ... +func New() *FileDiffer { + return &FileDiffer{} +} + +// Prepare 准备步骤 +func (d *FileDiffer) Prepare(srcPath string) error { + d.srcPath = srcPath + + _, err := os.Stat(path.Join(d.srcPath, ".git")) + // 如果对应目录下存在 .git 目录，跳过 + if err == nil { + return nil + } + + // 没有 .git 目录（使用子目录部署的情况），执行以下命令以初始化 + // 1. git init + // 2. git add . + // 3. git commit --author="bkpaas " -m "init" + if os.IsNotExist(err) { + commands := [][]string{ + {"init"}, + {"add", "."}, + {"commit", "--author=bkpaas ", "-m", "init"}, + } + for _, cmd := range commands { + if _, err = d.runGitCommand(cmd...); err != nil { + return err + } + } + return nil + } + + // 其他错误，直接返回 + return err +} + +// Diff 对比输出文件变更信息 +func (d *FileDiffer) Diff() ([]File, error) { + if _, err := d.runGitCommand("add", "."); err != nil { + return nil, err + } + output, err := d.runGitCommand("diff", "--cached", "--name-status") + if err != nil { + return nil, err + } + + lines := strings.Split(output, "\n") + files := []File{} + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + return nil, errors.Errorf("invalid line: %s", line) + } + var action FileAction + switch fields[0] { + case "A": + action = FileActionAdded + case "M": + action = FileActionModified + case "D": + action = FileActionDeleted + default: + return nil, errors.Errorf("unknown action: %s", fields[0]) + } + // 强制忽略部分变更文件 + if d.mustIgnoreFile(fields[1]) { + continue + } + + var content string + // 如果是删除操作，不加载文件 + if action != FileActionDeleted { + if content, err = d.loadFileContent(fields[1]); err != nil { + return nil, err + } + } + files = append(files, File{Action: action, Path: fields[1], Content: content}) + } + return files, nil +} + +// 执行 Git 命令 +func (d *FileDiffer) runGitCommand(args ...string) (string, error) { + cmd := exec.Command("git", args...) + cmd.Dir = d.srcPath + + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return "", errors.Errorf("git command failed: %s, output: %s", err, out.String()) + } + + return out.String(), nil +} + +// 加载指定文件内容 +func (d *FileDiffer) loadFileContent(filepath string) (string, error) { + file, err := os.Open(path.Join(d.srcPath, filepath)) + if err != nil { + return "", err + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + return "", err + } + return string(content), nil +} + +// 判断变更的文件是否需要被忽略 +func (d *FileDiffer) mustIgnoreFile(filepath string) bool { + for _, prefix := range forceIgnoreFilePathPrefixes { + if strings.HasPrefix(filepath, prefix) { + return true + } + } + return false +} diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_suite_test.go b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_suite_test.go new file mode 100644 index 0000000000..206e8ca5ca --- /dev/null +++ b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_suite_test.go @@ -0,0 +1,13 @@ +package filediffer_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestFileDiffer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "File Differ Suite") +} diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go new file mode 100644 index 0000000000..3dfcd2de06 --- /dev/null +++ b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go @@ -0,0 +1,169 @@ +/* + * TencentBlueKing is pleased to support the open source community by making + * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. + * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * We undertake not to change the open source license (MIT license) applicable + * to the current version of the project delivered to anyone in the future. + */ + +package filediffer + +import ( + "os" + "os/exec" + "path" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Test Differ", func() { + var tmpDir string + + var initFile = func(dir, filename, content string) error { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + + file, err := os.Create(path.Join(dir, filename)) + if err != nil { + return err + } + defer file.Close() + + // 把文件名写入文件作为内容 + _, err = file.WriteString(content) + return err + } + + var editFile = func(dir, filename, content string) error { + file, err := os.OpenFile(path.Join(dir, filename), os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer file.Close() + + // 追加内容到文件 + _, err = file.WriteString(content) + return err + } + + var runGitCommand = func(dir string, args ...string) error { + cmd := exec.Command("git", args...) + cmd.Dir = dir + return cmd.Run() + } + + // 初始化临时目录 & 文件 + BeforeEach(func() { + tmpDir, _ = os.MkdirTemp("", "filediffer") + for _, filename := range []string{"example.txt", "example.py", "example.go"} { + Expect(initFile(tmpDir, filename, path.Ext(filename))).To(BeNil()) + } + }) + // 清理临时目录 + AfterEach(func() { + Expect(os.RemoveAll(tmpDir)).To(BeNil()) + }) + + Context("Test FileDiffer", func() { + It("with .git", func() { + Expect(runGitCommand(tmpDir, "init")).To(BeNil()) + Expect(runGitCommand(tmpDir, "add", ".")).To(BeNil()) + Expect(runGitCommand(tmpDir, "commit", "--author=bkpaas ", "-m", "init")).To(BeNil()) + + differ := New() + Expect(differ.Prepare(tmpDir)).To(BeNil()) + + files, err := differ.Diff() + Expect(err).To(BeNil()) + Expect(files).To(HaveLen(0)) + + _ = initFile(tmpDir, "example.html", ".html") + _ = initFile(tmpDir, "example.js", ".js") + + files, err = differ.Diff() + Expect(err).To(BeNil()) + Expect(files).To(Equal([]File{ + {Action: FileActionAdded, Path: "example.html", Content: ".html"}, + {Action: FileActionAdded, Path: "example.js", Content: ".js"}, + })) + + _ = os.Remove(path.Join(tmpDir, "example.html")) + _ = os.Remove(path.Join(tmpDir, "example.py")) + files, err = differ.Diff() + Expect(err).To(BeNil()) + Expect(files).To(Equal([]File{ + {Action: FileActionAdded, Path: "example.js", Content: ".js"}, + {Action: FileActionDeleted, Path: "example.py", Content: ""}, + })) + + _ = editFile(tmpDir, "example.go", "gogo") + files, err = differ.Diff() + Expect(err).To(BeNil()) + Expect(files).To(Equal([]File{ + {Action: FileActionModified, Path: "example.go", Content: "gogo"}, + {Action: FileActionAdded, Path: "example.js", Content: ".js"}, + {Action: FileActionDeleted, Path: "example.py", Content: ""}, + })) + }) + + It("without .git", func() { + differ := New() + Expect(differ.Prepare(tmpDir)).To(BeNil()) + + files, err := differ.Diff() + Expect(err).To(BeNil()) + Expect(files).To(HaveLen(0)) + + _ = initFile(path.Join(tmpDir, "webfe/templates"), "example.html", ".html") + _ = initFile(path.Join(tmpDir, "webfe/static"), "example.js", ".js") + _ = initFile(path.Join(tmpDir, "webfe/static"), "example.css", ".css") + _ = os.Remove(path.Join(tmpDir, "webfe/templates/example.html")) + _ = os.Remove(path.Join(tmpDir, "example.py")) + _ = editFile(tmpDir, "example.go", "gogo") + _ = editFile(path.Join(tmpDir, "webfe/static"), "example.js", "js-js") + _ = editFile(tmpDir, "example.txt", "txt no.1") + + files, err = differ.Diff() + Expect(err).To(BeNil()) + Expect(files).To(HaveLen(5)) + Expect(files).To(Equal([]File{ + {Action: FileActionModified, Path: "example.go", Content: "gogo"}, + {Action: FileActionDeleted, Path: "example.py", Content: ""}, + {Action: FileActionModified, Path: "example.txt", Content: "txt no.1"}, + {Action: FileActionAdded, Path: "webfe/static/example.css", Content: ".css"}, + {Action: FileActionAdded, Path: "webfe/static/example.js", Content: "js-js"}, + })) + }) + + It("with ignore", func() { + differ := New() + Expect(differ.Prepare(tmpDir)).To(BeNil()) + + _ = initFile(path.Join(tmpDir, "webfe/templates"), "example.html", ".html") + _ = initFile(path.Join(tmpDir, "v3logs"), "celery.log", "celery is running...") + _ = initFile(path.Join(tmpDir, "v3logs"), "gunicorn.log", "gunicorn is running...") + _ = editFile(tmpDir, "example.txt", "txt no.1") + _ = os.Remove(path.Join(tmpDir, "example.go")) + + files, err := differ.Diff() + Expect(err).To(BeNil()) + Expect(files).To(Equal([]File{ + {Action: FileActionDeleted, Path: "example.go", Content: ""}, + {Action: FileActionModified, Path: "example.txt", Content: "txt no.1"}, + {Action: FileActionAdded, Path: "webfe/templates/example.html", Content: ".html"}, + })) + }) + }) +}) diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/types.go b/cnb-builder-shim/internal/devsandbox/filediffer/types.go new file mode 100644 index 0000000000..66090d5f25 --- /dev/null +++ b/cnb-builder-shim/internal/devsandbox/filediffer/types.go @@ -0,0 +1,41 @@ +/* + * TencentBlueKing is pleased to support the open source community by making + * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. + * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * We undertake not to change the open source license (MIT license) applicable + * to the current version of the project delivered to anyone in the future. + */ + +package filediffer + +// 强制忽略的文件路径前缀 +var forceIgnoreFilePathPrefixes = []string{"v3logs"} + +// FileAction 文件操作类型 +type FileAction string + +const ( + // FileActionAdded 文件增加 + FileActionAdded FileAction = "added" + // FileActionModified 文件变更 + FileActionModified FileAction = "modified" + // FileActionDeleted 文件删除 + FileActionDeleted FileAction = "deleted" +) + +// File 文件详情 +type File struct { + Action FileAction `json:"action"` + Path string `json:"path"` + Content string `json:"content"` +} diff --git a/cnb-builder-shim/internal/devsandbox/webserver/server.go b/cnb-builder-shim/internal/devsandbox/webserver/server.go index cce8589e9f..eed9d260e8 100644 --- a/cnb-builder-shim/internal/devsandbox/webserver/server.go +++ b/cnb-builder-shim/internal/devsandbox/webserver/server.go @@ -35,6 +35,7 @@ import ( "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/filediffer" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/webserver/service" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/appdesc" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils" @@ -93,6 +94,7 @@ func New(lg *logr.Logger) (*WebServer, error) { r.GET("/app_logs", AppLogHandler()) r.GET("/processes/status", ProcessStatusHandler()) r.GET("/processes/list", ProcessListHandler()) + r.GET("/diffs", DiffsHandler()) return s, nil } @@ -139,41 +141,56 @@ func DeployHandler(s *WebServer, svc service.DeployServiceHandler) gin.HandlerFu // 创建临时文件夹 tmpDir, err := os.MkdirTemp("", "source-*") if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"message": fmt.Sprintf("create tmp dir err: %s", err.Error())}) + c.JSON( + http.StatusInternalServerError, + gin.H{"message": fmt.Sprintf("create tmp dir err: %s", err.Error())}, + ) return } defer os.RemoveAll(tmpDir) file, err := c.FormFile("file") if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"message": fmt.Sprintf("get form err: %s", err.Error())}) + c.JSON( + http.StatusInternalServerError, + gin.H{"message": fmt.Sprintf("get form err: %s", err.Error())}, + ) return } fileName := filepath.Base(file.Filename) dst := path.Join(s.env.UploadDir, fileName) if len(dst) > 0 && dst[len(dst)-1] == '.' { - c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("invalid file name: %s", file.Filename)}) + c.JSON( + http.StatusBadRequest, + gin.H{"message": fmt.Sprintf("invalid file name: %s", file.Filename)}, + ) return } if err = c.SaveUploadedFile(file, dst); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"message": fmt.Sprintf("upload file err: %s", err.Error())}) + c.JSON( + http.StatusInternalServerError, + gin.H{"message": fmt.Sprintf("upload file err: %s", err.Error())}, + ) return } // 解压文件到临时目录 if err = utils.Unzip(dst, tmpDir); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"message": fmt.Sprintf("unzip file err: %s", err.Error())}) + c.JSON( + http.StatusInternalServerError, + gin.H{"message": fmt.Sprintf("unzip file err: %s", err.Error())}, + ) return } srcFilePath = path.Join(tmpDir, strings.TrimSuffix(fileName, filepath.Ext(fileName))) case config.BK_REPO: srcFilePath = config.G.SourceCode.Workspace case config.GIT: - c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("unsupported source fetch method: %s", config.G.SourceCode.FetchMethod)}) - return + fallthrough default: - c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("unsupported source fetch method: %s", config.G.SourceCode.FetchMethod)}) + errMsg := fmt.Sprintf("unsupported source fetch method: %s", config.G.SourceCode.FetchMethod) + c.JSON(http.StatusBadRequest, gin.H{"message": errMsg}) return } @@ -184,7 +201,11 @@ func DeployHandler(s *WebServer, svc service.DeployServiceHandler) gin.HandlerFu } select { - case s.ch <- devsandbox.AppReloadEvent{ID: status.DeployID, Rebuild: status.StepOpts.Rebuild, Relaunch: status.StepOpts.Relaunch}: + case s.ch <- devsandbox.AppReloadEvent{ + ID: status.DeployID, + Rebuild: status.StepOpts.Rebuild, + Relaunch: status.StepOpts.Relaunch, + }: c.JSON(http.StatusOK, gin.H{"deployID": status.DeployID}) default: c.JSON( @@ -262,6 +283,40 @@ func ProcessListHandler() gin.HandlerFunc { } } +// DiffsHandler 提供文件变更信息 +func DiffsHandler() gin.HandlerFunc { + return func(c *gin.Context) { + // 由于目前 HTTP 附带文件的源码初始化逻辑不同，暂时不支持 TODO 后续重构时需要统一 + if config.G.SourceCode.FetchMethod != config.BK_REPO { + c.JSON( + http.StatusBadRequest, + gin.H{"message": fmt.Sprintf("unsupported fetch method: %s", config.G.SourceCode.FetchMethod)}, + ) + return + } + + differ := filediffer.New() + // 初始化 + if err := differ.Prepare(config.G.SourceCode.Workspace); err != nil { + c.JSON( + http.StatusInternalServerError, + gin.H{"message": fmt.Sprintf("file differ prepare failed: %s", err)}, + ) + return + } + // 获取文件变更信息 + files, err := differ.Diff() + if err != nil { + c.JSON( + http.StatusInternalServerError, + gin.H{"message": fmt.Sprintf("failed to diff files: %s", err)}, + ) + return + } + c.JSON(http.StatusOK, gin.H{"data": files}) + } +} + // HealthzHandler ... func HealthzHandler() gin.HandlerFunc { return func(c *gin.Context) { From 59b9e98a31ae2250344c726f2d0592dc19f77bd5 Mon Sep 17 00:00:00 2001 From: schnee Date: Sun, 8 Dec 2024 19:37:23 +0800 Subject: [PATCH 02/10] fix: FileDiffer add WithContent option --- cnb-builder-shim/cmd/dev-entrypoint/init.go | 6 ++++ .../devsandbox/filediffer/filediffer.go | 32 +++++++++++++------ .../devsandbox/filediffer/filediffer_test.go | 12 ++++--- .../internal/devsandbox/webserver/server.go | 2 +- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/cnb-builder-shim/cmd/dev-entrypoint/init.go b/cnb-builder-shim/cmd/dev-entrypoint/init.go index 5990e87d6e..e67613d7e8 100644 --- a/cnb-builder-shim/cmd/dev-entrypoint/init.go +++ b/cnb-builder-shim/cmd/dev-entrypoint/init.go @@ -29,6 +29,7 @@ import ( "github.com/pkg/errors" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/filediffer" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/fetcher/http" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils" ) @@ -168,6 +169,11 @@ func initializeSourceCode() error { case config.GIT: return fmt.Errorf("TODO: clone git from revision") } + + // 初始化文件对比器 + if err = filediffer.New().Prepare(workspace); err != nil { + return errors.Wrap(err, "file differ preparing") + } return nil } diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go index f4ef0ef500..3ff7b333d3 100644 --- a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go +++ b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go @@ -30,12 +30,17 @@ import ( // FileDiffer 文件变更对比器 type FileDiffer struct { - srcPath string + srcPath string + withContent bool } // New ... -func New() *FileDiffer { - return &FileDiffer{} +func New(opts ...Option) *FileDiffer { + differ := &FileDiffer{} + for _, opt := range opts { + opt(differ) + } + return differ } // Prepare 准备步骤 @@ -48,15 +53,14 @@ func (d *FileDiffer) Prepare(srcPath string) error { return nil } - // 没有 .git 目录（使用子目录部署的情况），执行以下命令以初始化 - // 1. git init - // 2. git add . - // 3. git commit --author="bkpaas " -m "init" + // 没有 .git 目录（使用子目录部署的情况），执行若干命令以初始化 if os.IsNotExist(err) { commands := [][]string{ {"init"}, {"add", "."}, - {"commit", "--author=bkpaas ", "-m", "init"}, + {"config", "user.name", "bkpaas"}, + {"config", "user.email", "bkpaas@example.com"}, + {"commit", "-m", "init"}, } for _, cmd := range commands { if _, err = d.runGitCommand(cmd...); err != nil { @@ -108,7 +112,7 @@ func (d *FileDiffer) Diff() ([]File, error) { var content string // 如果是删除操作，不加载文件 - if action != FileActionDeleted { + if d.withContent && action != FileActionDeleted { if content, err = d.loadFileContent(fields[1]); err != nil { return nil, err } @@ -157,3 +161,13 @@ func (d *FileDiffer) mustIgnoreFile(filepath string) bool { } return false } + +// Option Differ 选项 +type Option func(*FileDiffer) + +// WithContent Diff 时是否加载文件内容 +func WithContent() Option { + return func(d *FileDiffer) { + d.withContent = true + } +} diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go index 3dfcd2de06..e887f2652f 100644 --- a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go +++ b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go @@ -66,7 +66,7 @@ var _ = Describe("Test Differ", func() { // 初始化临时目录 & 文件 BeforeEach(func() { - tmpDir, _ = os.MkdirTemp("", "filediffer") + tmpDir, _ = os.MkdirTemp("", "file-differ") for _, filename := range []string{"example.txt", "example.py", "example.go"} { Expect(initFile(tmpDir, filename, path.Ext(filename))).To(BeNil()) } @@ -80,9 +80,11 @@ var _ = Describe("Test Differ", func() { It("with .git", func() { Expect(runGitCommand(tmpDir, "init")).To(BeNil()) Expect(runGitCommand(tmpDir, "add", ".")).To(BeNil()) - Expect(runGitCommand(tmpDir, "commit", "--author=bkpaas ", "-m", "init")).To(BeNil()) + Expect(runGitCommand(tmpDir, "config", "user.name", "bkpaas")).To(BeNil()) + Expect(runGitCommand(tmpDir, "config", "user.email", "bkpaas@example.com")).To(BeNil()) + Expect(runGitCommand(tmpDir, "commit", "-m", "init")).To(BeNil()) - differ := New() + differ := New(WithContent()) Expect(differ.Prepare(tmpDir)).To(BeNil()) files, err := differ.Diff() @@ -119,7 +121,7 @@ var _ = Describe("Test Differ", func() { }) It("without .git", func() { - differ := New() + differ := New(WithContent()) Expect(differ.Prepare(tmpDir)).To(BeNil()) files, err := differ.Diff() @@ -148,7 +150,7 @@ var _ = Describe("Test Differ", func() { }) It("with ignore", func() { - differ := New() + differ := New(WithContent()) Expect(differ.Prepare(tmpDir)).To(BeNil()) _ = initFile(path.Join(tmpDir, "webfe/templates"), "example.html", ".html") diff --git a/cnb-builder-shim/internal/devsandbox/webserver/server.go b/cnb-builder-shim/internal/devsandbox/webserver/server.go index eed9d260e8..6370db442e 100644 --- a/cnb-builder-shim/internal/devsandbox/webserver/server.go +++ b/cnb-builder-shim/internal/devsandbox/webserver/server.go @@ -295,7 +295,7 @@ func DiffsHandler() gin.HandlerFunc { return } - differ := filediffer.New() + differ := filediffer.New(filediffer.WithContent()) // 初始化 if err := differ.Prepare(config.G.SourceCode.Workspace); err != nil { c.JSON( From bb76dca9bac78bd0e408d228a75eb484e6502e4a Mon Sep 17 00:00:00 2001 From: schnee Date: Mon, 9 Dec 2024 15:01:39 +0800 Subject: [PATCH 03/10] feat: diffs api support query param content --- cnb-builder-shim/README.md | 12 +++++++----- .../devsandbox/filediffer/filediffer_test.go | 18 ++++++++++++++++++ .../internal/devsandbox/webserver/server.go | 10 ++++++++-- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/cnb-builder-shim/README.md b/cnb-builder-shim/README.md index 3c40ac0425..67bdb7912f 100644 --- a/cnb-builder-shim/README.md +++ b/cnb-builder-shim/README.md @@ -61,7 +61,6 @@ tar -czvf /tmp/source.tgz -C examples/python-flask . 首先，通过环境变量设置一些关键参数： - ```bash # BUILDER_SHIM_IMAGE：构建工具镜像地址 export BUILDER_SHIM_IMAGE='mirrors.tencent.com/bkpaas/bk-builder-heroku-bionic:latest' @@ -82,7 +81,7 @@ docker run --rm \ -e USE_DOCKER_DAEMON=true \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ --mount type=bind,source=/tmp/source.tgz,target=/tmp/source.tgz \ - $BUILDER_SHIM_IMAGE + $BUILDER_SHIM_IMAGE ``` 要点说明： @@ -102,7 +101,7 @@ docker run --rm \ make heroku-dev ``` -其将使用默认的“云原生 builder 镜像”，名称为 `mirrors.tencent.com/bkpaas/builder-heroku-bionic:latest`，你也可以传递环境变量修改该默认名： +其将使用默认的 “云原生 builder 镜像”，名称为 `mirrors.tencent.com/bkpaas/builder-heroku-bionic:latest`，你也可以传递环境变量修改该默认名： ```shell BUILDER_IMAGE_NAME="my-builder-heroku-bionic" BUILDER_IMAGE_TAG="my-tag" DEV_IMAGE_NAME="bk-dev-heroku-bionic" DEV_IMAGE_TAG="latest" make heroku-dev @@ -112,6 +111,7 @@ BUILDER_IMAGE_NAME="my-builder-heroku-bionic" BUILDER_IMAGE_TAG="my-tag" DEV_IMA - `DEV_IMAGE_TAG`: 目标开发镜像 tag ### 2. 启动镜像 + 首先，通过环境变量设置一些关键参数。参数设置可选，仅当容器中默认的地址无效时，进行设置： ```shell @@ -133,7 +133,8 @@ docker run -d --net=host \ bk-dev-heroku-bionic:latest ``` -### 3. 通过 http 请求完成源码的 hot-reload +### 3. 通过 http 请求完成源码的 hot-reload + #### 3.1 请求源码部署 请求命令 @@ -149,7 +150,7 @@ curl --location 'http://{devsandbox_ip}:8000/deploys' \ ```json { - "deployID": "aaf79f28271e47bebf8448b63bddd04f" + "deployID": "aaf79f28271e47bebf8448b63bddd04f" } ``` @@ -162,6 +163,7 @@ curl --location 'http://{devsandbox_ip}:8000/deploys' \ # deployID 是上一步请求部署时返回的 deployID curl --location 'http://{devsandbox_ip}:8000/deploys/{deployID}/results?log=true' --header 'Authorization: Bearer xxx' ``` + 结果示例 ```json diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go index e887f2652f..78d78a63ee 100644 --- a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go +++ b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go @@ -167,5 +167,23 @@ var _ = Describe("Test Differ", func() { {Action: FileActionAdded, Path: "webfe/templates/example.html", Content: ".html"}, })) }) + + It("without content", func() { + differ := New() + Expect(differ.Prepare(tmpDir)).To(BeNil()) + + _ = initFile(path.Join(tmpDir, "webfe/static"), "example.css", "css") + _ = initFile(path.Join(tmpDir, "v3logs"), "celery.log", "celery is running...") + _ = editFile(tmpDir, "example.py", "python") + _ = os.Remove(path.Join(tmpDir, "example.go")) + + files, err := differ.Diff() + Expect(err).To(BeNil()) + Expect(files).To(Equal([]File{ + {Action: FileActionDeleted, Path: "example.go", Content: ""}, + {Action: FileActionModified, Path: "example.py", Content: ""}, + {Action: FileActionAdded, Path: "webfe/static/example.css", Content: ""}, + })) + }) }) }) diff --git a/cnb-builder-shim/internal/devsandbox/webserver/server.go b/cnb-builder-shim/internal/devsandbox/webserver/server.go index 6370db442e..39f43a7466 100644 --- a/cnb-builder-shim/internal/devsandbox/webserver/server.go +++ b/cnb-builder-shim/internal/devsandbox/webserver/server.go @@ -286,7 +286,8 @@ func ProcessListHandler() gin.HandlerFunc { // DiffsHandler 提供文件变更信息 func DiffsHandler() gin.HandlerFunc { return func(c *gin.Context) { - // 由于目前 HTTP 附带文件的源码初始化逻辑不同，暂时不支持 TODO 后续重构时需要统一 + // 由于目前 HTTP 附带文件的源码初始化逻辑不同，暂时不支持 + // TODO 后续重构时需要统一 if config.G.SourceCode.FetchMethod != config.BK_REPO { c.JSON( http.StatusBadRequest, @@ -295,8 +296,13 @@ func DiffsHandler() gin.HandlerFunc { return } - differ := filediffer.New(filediffer.WithContent()) // 初始化 + opts := []filediffer.Option{} + if c.Query("content") == "true" { + opts = append(opts, filediffer.WithContent()) + } + differ := filediffer.New(opts...) + if err := differ.Prepare(config.G.SourceCode.Workspace); err != nil { c.JSON( http.StatusInternalServerError, From 92a78e42f2aa01385c6be08d50ec91cde70ff180 Mon Sep 17 00:00:00 2001 From: schnee Date: Wed, 11 Dec 2024 12:09:44 +0800 Subject: [PATCH 04/10] refactor: rename FileDiffer to VersionController --- cnb-builder-shim/cmd/dev-entrypoint/init.go | 8 +- cnb-builder-shim/cmd/dev-entrypoint/main.go | 2 +- .../filediffer/filediffer_suite_test.go | 13 --- .../internal/devsandbox/vcs/types.go | 94 +++++++++++++++ .../internal/devsandbox/vcs/types_test.go | 83 ++++++++++++++ .../{filediffer/filediffer.go => vcs/vcs.go} | 108 ++++++++++-------- .../types.go => vcs/vcs_suite_test.go} | 26 ++--- .../filediffer_test.go => vcs/vcs_test.go} | 69 ++++++----- .../internal/devsandbox/webserver/server.go | 24 ++-- 9 files changed, 310 insertions(+), 117 deletions(-) delete mode 100644 cnb-builder-shim/internal/devsandbox/filediffer/filediffer_suite_test.go create mode 100644 cnb-builder-shim/internal/devsandbox/vcs/types.go create mode 100644 cnb-builder-shim/internal/devsandbox/vcs/types_test.go rename cnb-builder-shim/internal/devsandbox/{filediffer/filediffer.go => vcs/vcs.go} (53%) rename cnb-builder-shim/internal/devsandbox/{filediffer/types.go => vcs/vcs_suite_test.go} (62%) rename cnb-builder-shim/internal/devsandbox/{filediffer/filediffer_test.go => vcs/vcs_test.go} (78%) diff --git a/cnb-builder-shim/cmd/dev-entrypoint/init.go b/cnb-builder-shim/cmd/dev-entrypoint/init.go index e67613d7e8..bd8c3a3618 100644 --- a/cnb-builder-shim/cmd/dev-entrypoint/init.go +++ b/cnb-builder-shim/cmd/dev-entrypoint/init.go @@ -20,6 +20,7 @@ package main import ( "fmt" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/vcs" "os" "path/filepath" "strings" @@ -29,7 +30,6 @@ import ( "github.com/pkg/errors" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config" - "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/filediffer" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/fetcher/http" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils" ) @@ -171,8 +171,8 @@ func initializeSourceCode() error { } // 初始化文件对比器 - if err = filediffer.New().Prepare(workspace); err != nil { - return errors.Wrap(err, "file differ preparing") + if err = vcs.New().Prepare(workspace); err != nil { + return errors.Wrap(err, "version controller preparing") } return nil } @@ -183,7 +183,7 @@ func ensureWorkspace(workspace string) (err error) { if _, err = os.Stat(workspace); os.IsNotExist(err) { // 文件夹不存在，创建文件夹 logger.Info("create workspace directory") - if err := os.MkdirAll(workspace, 0750); err != nil { + if err = os.MkdirAll(workspace, 0750); err != nil { return errors.Wrap(err, "create workspace directory") } return nil diff --git a/cnb-builder-shim/cmd/dev-entrypoint/main.go b/cnb-builder-shim/cmd/dev-entrypoint/main.go index 81e5750226..208f48e9cf 100644 --- a/cnb-builder-shim/cmd/dev-entrypoint/main.go +++ b/cnb-builder-shim/cmd/dev-entrypoint/main.go @@ -19,11 +19,11 @@ package main import ( + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/webserver" "os" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config" - "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/webserver" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/logging" ) diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_suite_test.go b/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_suite_test.go deleted file mode 100644 index 206e8ca5ca..0000000000 --- a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package filediffer_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestFileDiffer(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "File Differ Suite") -} diff --git a/cnb-builder-shim/internal/devsandbox/vcs/types.go b/cnb-builder-shim/internal/devsandbox/vcs/types.go new file mode 100644 index 0000000000..b4b0127c3c --- /dev/null +++ b/cnb-builder-shim/internal/devsandbox/vcs/types.go @@ -0,0 +1,94 @@ +/* + * TencentBlueKing is pleased to support the open source community by making + * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. + * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * We undertake not to change the open source license (MIT license) applicable + * to the current version of the project delivered to anyone in the future. + */ + +// Package vcs 版本控制系统（VersionControlSystem） +package vcs + +import ( + "slices" + "strings" +) + +// 强制忽略的文件路径前缀 +var forceIgnoreFilePathPrefixes = []string{"v3logs"} + +// FileAction 文件操作类型 +type FileAction string + +const ( + // FileActionAdded 文件增加 + FileActionAdded FileAction = "added" + // FileActionModified 文件变更 + FileActionModified FileAction = "modified" + // FileActionDeleted 文件删除 + FileActionDeleted FileAction = "deleted" +) + +// File 文件详情 +type File struct { + Action FileAction `json:"action"` + Path string `json:"path"` + Content string `json:"content"` +} + +// Files 文件列表 +type Files []File + +// DirTree 目录树 +type DirTree struct { + Name string `json:"name"` + Dirs []*DirTree `json:"dirs"` + Files Files `json:"files"` +} + +// AsTree 转换成目录树形式 +func (files Files) AsTree() DirTree { + // 按照文件路径排序 + slices.SortFunc(files, func(a, b File) int { + return strings.Compare(a.Path, b.Path) + }) + + root := DirTree{Name: "/"} + var cur *DirTree + for _, f := range files { + parts := strings.Split(f.Path, "/") + cur = &root + // 循环创建目录树 + for _, part := range parts[:len(parts)-1] { + if part == "" { + continue + } + exists := false + for _, dir := range cur.Dirs { + if dir.Name == part { + cur = dir + exists = true + break + } + } + if !exists { + cur.Dirs = append(cur.Dirs, &DirTree{Name: part}) + cur = cur.Dirs[len(cur.Dirs)-1] + } + } + // 目录树格式下，路径即为文件名 + f.Path = parts[len(parts)-1] + cur.Files = append(cur.Files, f) + } + return root +} diff --git a/cnb-builder-shim/internal/devsandbox/vcs/types_test.go b/cnb-builder-shim/internal/devsandbox/vcs/types_test.go new file mode 100644 index 0000000000..2701a1d8f4 --- /dev/null +++ b/cnb-builder-shim/internal/devsandbox/vcs/types_test.go @@ -0,0 +1,83 @@ +/* + * TencentBlueKing is pleased to support the open source community by making + * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. + * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * We undertake not to change the open source license (MIT license) applicable + * to the current version of the project delivered to anyone in the future. + */ + +package vcs + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Test Types", func() { + + Context("Test Files", func() { + It("AsTree", func() { + files := Files{ + {Action: FileActionAdded, Path: "README.md"}, + {Action: FileActionAdded, Path: "webfe/static/example.css"}, + {Action: FileActionAdded, Path: "webfe/static/example.js"}, + {Action: FileActionDeleted, Path: "api/main.py"}, + {Action: FileActionDeleted, Path: "webfe/templates/example.html"}, + {Action: FileActionModified, Path: "backend/main.go"}, + {Action: FileActionModified, Path: "backend/types.go"}, + {Action: FileActionModified, Path: "docs/example.txt"}, + } + + excepted := DirTree{ + Name: "/", + Dirs: []*DirTree{ + { + Name: "api", Files: Files{ + {Action: FileActionDeleted, Path: "main.py"}, + }, + }, + { + Name: "backend", Files: Files{ + {Action: FileActionModified, Path: "main.go"}, + {Action: FileActionModified, Path: "types.go"}, + }, + }, + { + Name: "docs", Files: Files{ + {Action: FileActionModified, Path: "example.txt"}, + }, + }, + { + Name: "webfe", Dirs: []*DirTree{ + { + Name: "static", Files: Files{ + {Action: FileActionAdded, Path: "example.css"}, + {Action: FileActionAdded, Path: "example.js"}, + }, + }, + { + Name: "templates", Files: Files{ + {Action: FileActionDeleted, Path: "example.html"}, + }, + }, + }, + }, + }, + Files: Files{ + {Action: FileActionAdded, Path: "README.md"}, + }, + } + Expect(files.AsTree()).To(Equal(excepted)) + }) + }) +}) diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go b/cnb-builder-shim/internal/devsandbox/vcs/vcs.go similarity index 53% rename from cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go rename to cnb-builder-shim/internal/devsandbox/vcs/vcs.go index 3ff7b333d3..416284ff11 100644 --- a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/vcs.go @@ -15,7 +15,7 @@ * We undertake not to change the open source license (MIT license) applicable * to the current version of the project delivered to anyone in the future. */ -package filediffer +package vcs import ( "bytes" @@ -24,30 +24,31 @@ import ( "os/exec" "path" "strings" + "unicode" "github.com/pkg/errors" ) -// FileDiffer 文件变更对比器 -type FileDiffer struct { +// VersionController 版本控制器（基于 Git） +type VersionController struct { srcPath string withContent bool } // New ... -func New(opts ...Option) *FileDiffer { - differ := &FileDiffer{} +func New(opts ...Option) *VersionController { + v := &VersionController{} for _, opt := range opts { - opt(differ) + opt(v) } - return differ + return v } // Prepare 准备步骤 -func (d *FileDiffer) Prepare(srcPath string) error { - d.srcPath = srcPath +func (v *VersionController) Prepare(srcPath string) error { + v.srcPath = srcPath - _, err := os.Stat(path.Join(d.srcPath, ".git")) + _, err := os.Stat(path.Join(v.srcPath, ".git")) // 如果对应目录下存在 .git 目录，跳过 if err == nil { return nil @@ -63,7 +64,7 @@ func (d *FileDiffer) Prepare(srcPath string) error { {"commit", "-m", "init"}, } for _, cmd := range commands { - if _, err = d.runGitCommand(cmd...); err != nil { + if _, err = v.runGitCommand(cmd...); err != nil { return err } } @@ -75,57 +76,48 @@ func (d *FileDiffer) Prepare(srcPath string) error { } // Diff 对比输出文件变更信息 -func (d *FileDiffer) Diff() ([]File, error) { - if _, err := d.runGitCommand("add", "."); err != nil { +func (v *VersionController) Diff() (Files, error) { + // 将所有文件添加到暂存区 + if _, err := v.runGitCommand("add", "."); err != nil { return nil, err } - output, err := d.runGitCommand("diff", "--cached", "--name-status") + // 设置不要转义特殊字符 + if _, err := v.runGitCommand("config", "core.quotepath", "false"); err != nil { + return nil, err + } + // 执行 diff 命令输出变更文件目录 + output, err := v.runGitCommand("diff", "--cached", "--name-status", "--no-renames") if err != nil { return nil, err } lines := strings.Split(output, "\n") - files := []File{} + files := Files{} for _, line := range lines { - if line == "" { + action, filePath, pErr := v.parseDiffLine(line) + if pErr != nil { continue } - fields := strings.Fields(line) - if len(fields) != 2 { - return nil, errors.Errorf("invalid line: %s", line) - } - var action FileAction - switch fields[0] { - case "A": - action = FileActionAdded - case "M": - action = FileActionModified - case "D": - action = FileActionDeleted - default: - return nil, errors.Errorf("unknown action: %s", fields[0]) - } // 强制忽略部分变更文件 - if d.mustIgnoreFile(fields[1]) { + if v.shouldIgnoreFile(filePath) { continue } - var content string // 如果是删除操作，不加载文件 - if d.withContent && action != FileActionDeleted { - if content, err = d.loadFileContent(fields[1]); err != nil { + if v.withContent && action != FileActionDeleted { + if content, err = v.loadFileContent(filePath); err != nil { return nil, err } } - files = append(files, File{Action: action, Path: fields[1], Content: content}) + files = append(files, File{Action: action, Path: filePath, Content: content}) } return files, nil } // 执行 Git 命令 -func (d *FileDiffer) runGitCommand(args ...string) (string, error) { +func (v *VersionController) runGitCommand(args ...string) (string, error) { cmd := exec.Command("git", args...) - cmd.Dir = d.srcPath + cmd.Dir = v.srcPath var out bytes.Buffer cmd.Stdout = &out @@ -138,8 +130,8 @@ func (d *FileDiffer) runGitCommand(args ...string) (string, error) { } // 加载指定文件内容 -func (d *FileDiffer) loadFileContent(filepath string) (string, error) { - file, err := os.Open(path.Join(d.srcPath, filepath)) +func (v *VersionController) loadFileContent(filepath string) (string, error) { + file, err := os.Open(path.Join(v.srcPath, filepath)) if err != nil { return "", err } @@ -152,8 +144,32 @@ func (d *FileDiffer) loadFileContent(filepath string) (string, error) { return string(content), nil } +// 解析 git diff 输出的文件变更信息 +func (v *VersionController) parseDiffLine(line string) (FileAction, string, error) { + // diff 输出格式形如： + // A backend/example.go + // M webfe/example.js + // D api/example.py + index := strings.IndexFunc(line, unicode.IsSpace) + if index == -1 { + return "", "", errors.Errorf("invalid diff line: `%s`", line) + } + + rawAction, filepath := line[:index], strings.TrimSpace(line[index+1:]) + switch rawAction { + case "A": + return FileActionAdded, filepath, nil + case "M": + return FileActionModified, filepath, nil + case "D": + return FileActionDeleted, filepath, nil + default: + return "", "", errors.Errorf("unknown action: %s", rawAction) + } +} + // 判断变更的文件是否需要被忽略 -func (d *FileDiffer) mustIgnoreFile(filepath string) bool { +func (v *VersionController) shouldIgnoreFile(filepath string) bool { for _, prefix := range forceIgnoreFilePathPrefixes { if strings.HasPrefix(filepath, prefix) { return true @@ -162,12 +178,12 @@ func (d *FileDiffer) mustIgnoreFile(filepath string) bool { return false } -// Option Differ 选项 -type Option func(*FileDiffer) +// Option VersionController 选项 +type Option func(*VersionController) -// WithContent Diff 时是否加载文件内容 +// WithContent Diff 时加载文件内容 func WithContent() Option { - return func(d *FileDiffer) { - d.withContent = true + return func(v *VersionController) { + v.withContent = true } } diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/types.go b/cnb-builder-shim/internal/devsandbox/vcs/vcs_suite_test.go similarity index 62% rename from cnb-builder-shim/internal/devsandbox/filediffer/types.go rename to cnb-builder-shim/internal/devsandbox/vcs/vcs_suite_test.go index 66090d5f25..a6b45a5a90 100644 --- a/cnb-builder-shim/internal/devsandbox/filediffer/types.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/vcs_suite_test.go @@ -16,26 +16,16 @@ * to the current version of the project delivered to anyone in the future. */ -package filediffer +package vcs_test -// 强制忽略的文件路径前缀 -var forceIgnoreFilePathPrefixes = []string{"v3logs"} +import ( + "testing" -// FileAction 文件操作类型 -type FileAction string - -const ( - // FileActionAdded 文件增加 - FileActionAdded FileAction = "added" - // FileActionModified 文件变更 - FileActionModified FileAction = "modified" - // FileActionDeleted 文件删除 - FileActionDeleted FileAction = "deleted" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -// File 文件详情 -type File struct { - Action FileAction `json:"action"` - Path string `json:"path"` - Content string `json:"content"` +func TestVCS(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VCS Suite") } diff --git a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go b/cnb-builder-shim/internal/devsandbox/vcs/vcs_test.go similarity index 78% rename from cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go rename to cnb-builder-shim/internal/devsandbox/vcs/vcs_test.go index 78d78a63ee..b2187fd74b 100644 --- a/cnb-builder-shim/internal/devsandbox/filediffer/filediffer_test.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/vcs_test.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package filediffer +package vcs import ( "os" @@ -27,7 +27,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Test Differ", func() { +var _ = Describe("Test VersionController", func() { var tmpDir string var initFile = func(dir, filename, content string) error { @@ -66,7 +66,7 @@ var _ = Describe("Test Differ", func() { // 初始化临时目录 & 文件 BeforeEach(func() { - tmpDir, _ = os.MkdirTemp("", "file-differ") + tmpDir, _ = os.MkdirTemp("", "vcs") for _, filename := range []string{"example.txt", "example.py", "example.go"} { Expect(initFile(tmpDir, filename, path.Ext(filename))).To(BeNil()) } @@ -76,7 +76,7 @@ var _ = Describe("Test Differ", func() { Expect(os.RemoveAll(tmpDir)).To(BeNil()) }) - Context("Test FileDiffer", func() { + Context("Test VersionController", func() { It("with .git", func() { Expect(runGitCommand(tmpDir, "init")).To(BeNil()) Expect(runGitCommand(tmpDir, "add", ".")).To(BeNil()) @@ -84,36 +84,36 @@ var _ = Describe("Test Differ", func() { Expect(runGitCommand(tmpDir, "config", "user.email", "bkpaas@example.com")).To(BeNil()) Expect(runGitCommand(tmpDir, "commit", "-m", "init")).To(BeNil()) - differ := New(WithContent()) - Expect(differ.Prepare(tmpDir)).To(BeNil()) + verCtrl := New(WithContent()) + Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) - files, err := differ.Diff() + files, err := verCtrl.Diff() Expect(err).To(BeNil()) Expect(files).To(HaveLen(0)) _ = initFile(tmpDir, "example.html", ".html") _ = initFile(tmpDir, "example.js", ".js") - files, err = differ.Diff() + files, err = verCtrl.Diff() Expect(err).To(BeNil()) - Expect(files).To(Equal([]File{ + Expect(files).To(Equal(Files{ {Action: FileActionAdded, Path: "example.html", Content: ".html"}, {Action: FileActionAdded, Path: "example.js", Content: ".js"}, })) _ = os.Remove(path.Join(tmpDir, "example.html")) _ = os.Remove(path.Join(tmpDir, "example.py")) - files, err = differ.Diff() + files, err = verCtrl.Diff() Expect(err).To(BeNil()) - Expect(files).To(Equal([]File{ + Expect(files).To(Equal(Files{ {Action: FileActionAdded, Path: "example.js", Content: ".js"}, {Action: FileActionDeleted, Path: "example.py", Content: ""}, })) _ = editFile(tmpDir, "example.go", "gogo") - files, err = differ.Diff() + files, err = verCtrl.Diff() Expect(err).To(BeNil()) - Expect(files).To(Equal([]File{ + Expect(files).To(Equal(Files{ {Action: FileActionModified, Path: "example.go", Content: "gogo"}, {Action: FileActionAdded, Path: "example.js", Content: ".js"}, {Action: FileActionDeleted, Path: "example.py", Content: ""}, @@ -121,10 +121,10 @@ var _ = Describe("Test Differ", func() { }) It("without .git", func() { - differ := New(WithContent()) - Expect(differ.Prepare(tmpDir)).To(BeNil()) + verCtrl := New(WithContent()) + Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) - files, err := differ.Diff() + files, err := verCtrl.Diff() Expect(err).To(BeNil()) Expect(files).To(HaveLen(0)) @@ -137,10 +137,10 @@ var _ = Describe("Test Differ", func() { _ = editFile(path.Join(tmpDir, "webfe/static"), "example.js", "js-js") _ = editFile(tmpDir, "example.txt", "txt no.1") - files, err = differ.Diff() + files, err = verCtrl.Diff() Expect(err).To(BeNil()) Expect(files).To(HaveLen(5)) - Expect(files).To(Equal([]File{ + Expect(files).To(Equal(Files{ {Action: FileActionModified, Path: "example.go", Content: "gogo"}, {Action: FileActionDeleted, Path: "example.py", Content: ""}, {Action: FileActionModified, Path: "example.txt", Content: "txt no.1"}, @@ -150,8 +150,8 @@ var _ = Describe("Test Differ", func() { }) It("with ignore", func() { - differ := New(WithContent()) - Expect(differ.Prepare(tmpDir)).To(BeNil()) + verCtrl := New(WithContent()) + Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) _ = initFile(path.Join(tmpDir, "webfe/templates"), "example.html", ".html") _ = initFile(path.Join(tmpDir, "v3logs"), "celery.log", "celery is running...") @@ -159,9 +159,9 @@ var _ = Describe("Test Differ", func() { _ = editFile(tmpDir, "example.txt", "txt no.1") _ = os.Remove(path.Join(tmpDir, "example.go")) - files, err := differ.Diff() + files, err := verCtrl.Diff() Expect(err).To(BeNil()) - Expect(files).To(Equal([]File{ + Expect(files).To(Equal(Files{ {Action: FileActionDeleted, Path: "example.go", Content: ""}, {Action: FileActionModified, Path: "example.txt", Content: "txt no.1"}, {Action: FileActionAdded, Path: "webfe/templates/example.html", Content: ".html"}, @@ -169,21 +169,38 @@ var _ = Describe("Test Differ", func() { }) It("without content", func() { - differ := New() - Expect(differ.Prepare(tmpDir)).To(BeNil()) + verCtrl := New() + Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) _ = initFile(path.Join(tmpDir, "webfe/static"), "example.css", "css") _ = initFile(path.Join(tmpDir, "v3logs"), "celery.log", "celery is running...") _ = editFile(tmpDir, "example.py", "python") _ = os.Remove(path.Join(tmpDir, "example.go")) - files, err := differ.Diff() + files, err := verCtrl.Diff() Expect(err).To(BeNil()) - Expect(files).To(Equal([]File{ + Expect(files).To(Equal(Files{ {Action: FileActionDeleted, Path: "example.go", Content: ""}, {Action: FileActionModified, Path: "example.py", Content: ""}, {Action: FileActionAdded, Path: "webfe/static/example.css", Content: ""}, })) }) + + It("with special chars", func() { + verCtrl := New(WithContent()) + Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) + + _ = initFile(tmpDir, "example space.css", "css 代码") + _ = initFile(tmpDir, "example——中文.js", "js 代码") + _ = initFile(tmpDir, "example.tab.html", "html 代码") + + files, err := verCtrl.Diff() + Expect(err).To(BeNil()) + Expect(files).To(Equal(Files{ + {Action: FileActionAdded, Path: "example space.css", Content: "css 代码"}, + {Action: FileActionAdded, Path: "example.tab.html", Content: "html 代码"}, + {Action: FileActionAdded, Path: "example——中文.js", Content: "js 代码"}, + })) + }) }) }) diff --git a/cnb-builder-shim/internal/devsandbox/webserver/server.go b/cnb-builder-shim/internal/devsandbox/webserver/server.go index 39f43a7466..8e2dd7b792 100644 --- a/cnb-builder-shim/internal/devsandbox/webserver/server.go +++ b/cnb-builder-shim/internal/devsandbox/webserver/server.go @@ -20,6 +20,8 @@ package webserver import ( "fmt" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/vcs" "net/http" "os" "path" @@ -33,9 +35,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-logr/logr" - "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config" - "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/filediffer" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/webserver/service" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/appdesc" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils" @@ -297,21 +297,21 @@ func DiffsHandler() gin.HandlerFunc { } // 初始化 - opts := []filediffer.Option{} + opts := []vcs.Option{} if c.Query("content") == "true" { - opts = append(opts, filediffer.WithContent()) + opts = append(opts, vcs.WithContent()) } - differ := filediffer.New(opts...) + verCtrl := vcs.New(opts...) - if err := differ.Prepare(config.G.SourceCode.Workspace); err != nil { + if err := verCtrl.Prepare(config.G.SourceCode.Workspace); err != nil { c.JSON( http.StatusInternalServerError, - gin.H{"message": fmt.Sprintf("file differ prepare failed: %s", err)}, + gin.H{"message": fmt.Sprintf("vcs prepare failed: %s", err)}, ) return } // 获取文件变更信息 - files, err := differ.Diff() + files, err := verCtrl.Diff() if err != nil { c.JSON( http.StatusInternalServerError, @@ -319,7 +319,13 @@ func DiffsHandler() gin.HandlerFunc { ) return } - c.JSON(http.StatusOK, gin.H{"data": files}) + // 如果指定 tree 为 true，则返回目录树格式 + if c.Query("tree") == "true" { + c.JSON(http.StatusOK, gin.H{"total": len(files), "tree": files.AsTree()}) + return + } + // 默认返回变更文件列表 + c.JSON(http.StatusOK, gin.H{"total": len(files), "files": files}) } } From 3cb94753db4115dcd1baab9106f66815adca5fc8 Mon Sep 17 00:00:00 2001 From: schnee Date: Wed, 11 Dec 2024 18:20:18 +0800 Subject: [PATCH 05/10] feat: support compress diff tree --- cnb-builder-shim/cmd/dev-entrypoint/init.go | 4 +- cnb-builder-shim/cmd/dev-entrypoint/main.go | 2 +- .../internal/devsandbox/vcs/types.go | 39 ++++++++--- .../internal/devsandbox/vcs/types_test.go | 66 ++++++++++++++++++- .../internal/devsandbox/vcs/vcs.go | 6 ++ .../internal/devsandbox/webserver/server.go | 35 +++++++++- 6 files changed, 137 insertions(+), 15 deletions(-) diff --git a/cnb-builder-shim/cmd/dev-entrypoint/init.go b/cnb-builder-shim/cmd/dev-entrypoint/init.go index bd8c3a3618..97ab78bee0 100644 --- a/cnb-builder-shim/cmd/dev-entrypoint/init.go +++ b/cnb-builder-shim/cmd/dev-entrypoint/init.go @@ -20,7 +20,6 @@ package main import ( "fmt" - "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/vcs" "os" "path/filepath" "strings" @@ -30,6 +29,7 @@ import ( "github.com/pkg/errors" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/vcs" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/fetcher/http" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils" ) @@ -170,7 +170,7 @@ func initializeSourceCode() error { return fmt.Errorf("TODO: clone git from revision") } - // 初始化文件对比器 + // 初始化版本控制器 if err = vcs.New().Prepare(workspace); err != nil { return errors.Wrap(err, "version controller preparing") } diff --git a/cnb-builder-shim/cmd/dev-entrypoint/main.go b/cnb-builder-shim/cmd/dev-entrypoint/main.go index 208f48e9cf..81e5750226 100644 --- a/cnb-builder-shim/cmd/dev-entrypoint/main.go +++ b/cnb-builder-shim/cmd/dev-entrypoint/main.go @@ -19,11 +19,11 @@ package main import ( - "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/webserver" "os" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/webserver" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/logging" ) diff --git a/cnb-builder-shim/internal/devsandbox/vcs/types.go b/cnb-builder-shim/internal/devsandbox/vcs/types.go index b4b0127c3c..3c005083a1 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/types.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/types.go @@ -20,6 +20,7 @@ package vcs import ( + "path" "slices" "strings" ) @@ -49,15 +50,8 @@ type File struct { // Files 文件列表 type Files []File -// DirTree 目录树 -type DirTree struct { - Name string `json:"name"` - Dirs []*DirTree `json:"dirs"` - Files Files `json:"files"` -} - // AsTree 转换成目录树形式 -func (files Files) AsTree() DirTree { +func (files Files) AsTree() *DirTree { // 按照文件路径排序 slices.SortFunc(files, func(a, b File) int { return strings.Compare(a.Path, b.Path) @@ -90,5 +84,32 @@ func (files Files) AsTree() DirTree { f.Path = parts[len(parts)-1] cur.Files = append(cur.Files, f) } - return root + + // 压缩目录树，避免过多的嵌套层级 + return root.Compress() +} + +// DirTree 目录树 +type DirTree struct { + Name string `json:"name"` + Dirs []*DirTree `json:"dirs"` + Files Files `json:"files"` +} + +// Compress 压缩 +func (t *DirTree) Compress() *DirTree { + // 根目录不需要压缩，只有子目录需要 + for idx, dt := range t.Dirs { + t.Dirs[idx] = dt.compress(dt.Name) + } + return t +} + +func (t *DirTree) compress(prefix string) *DirTree { + if len(t.Files) == 0 && len(t.Dirs) == 1 { + subDir := t.Dirs[0] + subDir.Name = path.Join(prefix, subDir.Name) + return subDir.compress(subDir.Name) + } + return t } diff --git a/cnb-builder-shim/internal/devsandbox/vcs/types_test.go b/cnb-builder-shim/internal/devsandbox/vcs/types_test.go index 2701a1d8f4..9733d4053e 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/types_test.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/types_test.go @@ -38,7 +38,7 @@ var _ = Describe("Test Types", func() { {Action: FileActionModified, Path: "docs/example.txt"}, } - excepted := DirTree{ + excepted := &DirTree{ Name: "/", Dirs: []*DirTree{ { @@ -79,5 +79,69 @@ var _ = Describe("Test Types", func() { } Expect(files.AsTree()).To(Equal(excepted)) }) + + It("AsTree with compress", func() { + files := Files{ + {Action: FileActionAdded, Path: "webfe/static/example.css"}, + {Action: FileActionAdded, Path: "webfe/static/example.js"}, + {Action: FileActionDeleted, Path: "api/main.py"}, + {Action: FileActionDeleted, Path: "api/utils/stringx.py"}, + {Action: FileActionModified, Path: "backend/cmd/main.go"}, + {Action: FileActionModified, Path: "backend/pkg/types.go"}, + {Action: FileActionModified, Path: "docs/common/example.txt"}, + {Action: FileActionDeleted, Path: "mako/templates/mako/about_us.mako"}, + } + + excepted := &DirTree{ + Name: "/", + Dirs: []*DirTree{ + { + Name: "api", + Dirs: []*DirTree{ + { + Name: "utils", Files: Files{ + {Action: FileActionDeleted, Path: "stringx.py"}, + }, + }, + }, + Files: Files{ + {Action: FileActionDeleted, Path: "main.py"}, + }, + }, + { + Name: "backend", Dirs: []*DirTree{ + { + Name: "cmd", Files: Files{ + {Action: FileActionModified, Path: "main.go"}, + }, + }, + { + Name: "pkg", Files: Files{ + {Action: FileActionModified, Path: "types.go"}, + }, + }, + }, + }, + { + Name: "docs/common", Files: Files{ + {Action: FileActionModified, Path: "example.txt"}, + }, + }, + { + Name: "mako/templates/mako", + Files: Files{ + {Action: FileActionDeleted, Path: "about_us.mako"}, + }, + }, + { + Name: "webfe/static", Files: Files{ + {Action: FileActionAdded, Path: "example.css"}, + {Action: FileActionAdded, Path: "example.js"}, + }, + }, + }, + } + Expect(files.AsTree()).To(Equal(excepted)) + }) }) }) diff --git a/cnb-builder-shim/internal/devsandbox/vcs/vcs.go b/cnb-builder-shim/internal/devsandbox/vcs/vcs.go index 416284ff11..3e4a108397 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/vcs.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/vcs.go @@ -114,6 +114,12 @@ func (v *VersionController) Diff() (Files, error) { return files, nil } +// Commit 提交变更 +func (v *VersionController) Commit(message string) error { + _, err := v.runGitCommand("commit", "-m", message) + return err +} + // 执行 Git 命令 func (v *VersionController) runGitCommand(args ...string) (string, error) { cmd := exec.Command("git", args...) diff --git a/cnb-builder-shim/internal/devsandbox/webserver/server.go b/cnb-builder-shim/internal/devsandbox/webserver/server.go index 8e2dd7b792..498fe14e90 100644 --- a/cnb-builder-shim/internal/devsandbox/webserver/server.go +++ b/cnb-builder-shim/internal/devsandbox/webserver/server.go @@ -20,8 +20,6 @@ package webserver import ( "fmt" - "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox" - "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/vcs" "net/http" "os" "path" @@ -35,7 +33,9 @@ import ( "github.com/gin-gonic/gin" "github.com/go-logr/logr" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config" + "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/vcs" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/webserver/service" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/appdesc" "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils" @@ -95,6 +95,7 @@ func New(lg *logr.Logger) (*WebServer, error) { r.GET("/processes/status", ProcessStatusHandler()) r.GET("/processes/list", ProcessListHandler()) r.GET("/diffs", DiffsHandler()) + r.GET("/commit", CommitHandler()) return s, nil } @@ -329,6 +330,36 @@ func DiffsHandler() gin.HandlerFunc { } } +// CommitHandler 提交文件变更 +func CommitHandler() gin.HandlerFunc { + return func(c *gin.Context) { + // 由于目前 HTTP 附带文件的源码初始化逻辑不同，暂时不支持 + // TODO 后续重构时需要统一 + if config.G.SourceCode.FetchMethod != config.BK_REPO { + c.JSON( + http.StatusBadRequest, + gin.H{"message": fmt.Sprintf("unsupported fetch method: %s", config.G.SourceCode.FetchMethod)}, + ) + return + } + + commitMsg := c.Query("message") + if commitMsg == "" { + c.JSON(http.StatusBadRequest, gin.H{"message": "commit message is empty"}) + return + } + // 提交变更 + if err := vcs.New().Commit(commitMsg); err != nil { + c.JSON( + http.StatusInternalServerError, + gin.H{"message": fmt.Sprintf("failed to commit files: %s", err)}, + ) + return + } + c.JSON(http.StatusOK, gin.H{"message": "ok"}) + } +} + // HealthzHandler ... func HealthzHandler() gin.HandlerFunc { return func(c *gin.Context) { From 477f6366b18e4ce5c184bc6c55d54057a74185a4 Mon Sep 17 00:00:00 2001 From: schnee Date: Thu, 12 Dec 2024 15:14:24 +0800 Subject: [PATCH 06/10] fix: code diff & commit api urls add codes/ prefix --- .../internal/devsandbox/webserver/server.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cnb-builder-shim/internal/devsandbox/webserver/server.go b/cnb-builder-shim/internal/devsandbox/webserver/server.go index 498fe14e90..f3cf1adbf4 100644 --- a/cnb-builder-shim/internal/devsandbox/webserver/server.go +++ b/cnb-builder-shim/internal/devsandbox/webserver/server.go @@ -94,8 +94,8 @@ func New(lg *logr.Logger) (*WebServer, error) { r.GET("/app_logs", AppLogHandler()) r.GET("/processes/status", ProcessStatusHandler()) r.GET("/processes/list", ProcessListHandler()) - r.GET("/diffs", DiffsHandler()) - r.GET("/commit", CommitHandler()) + r.GET("/codes/diffs", CodeDiffsHandler()) + r.GET("/codes/commit", CodeCommitHandler()) return s, nil } @@ -284,8 +284,8 @@ func ProcessListHandler() gin.HandlerFunc { } } -// DiffsHandler 提供文件变更信息 -func DiffsHandler() gin.HandlerFunc { +// CodeDiffsHandler 提供文件变更信息 +func CodeDiffsHandler() gin.HandlerFunc { return func(c *gin.Context) { // 由于目前 HTTP 附带文件的源码初始化逻辑不同，暂时不支持 // TODO 后续重构时需要统一 @@ -330,8 +330,8 @@ func DiffsHandler() gin.HandlerFunc { } } -// CommitHandler 提交文件变更 -func CommitHandler() gin.HandlerFunc { +// CodeCommitHandler 提交文件变更 +func CodeCommitHandler() gin.HandlerFunc { return func(c *gin.Context) { // 由于目前 HTTP 附带文件的源码初始化逻辑不同，暂时不支持 // TODO 后续重构时需要统一 From ed741f2c47f63613688440fb112327847507f2fd Mon Sep 17 00:00:00 2001 From: schnee Date: Fri, 13 Dec 2024 10:46:43 +0800 Subject: [PATCH 07/10] fix: set srcPath when create VersionController --- cnb-builder-shim/cmd/dev-entrypoint/init.go | 2 +- .../internal/devsandbox/vcs/vcs.go | 17 +++++++++++----- .../internal/devsandbox/vcs/vcs_test.go | 20 +++++++++---------- .../internal/devsandbox/webserver/server.go | 11 ++-------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cnb-builder-shim/cmd/dev-entrypoint/init.go b/cnb-builder-shim/cmd/dev-entrypoint/init.go index 97ab78bee0..7e400609b3 100644 --- a/cnb-builder-shim/cmd/dev-entrypoint/init.go +++ b/cnb-builder-shim/cmd/dev-entrypoint/init.go @@ -171,7 +171,7 @@ func initializeSourceCode() error { } // 初始化版本控制器 - if err = vcs.New().Prepare(workspace); err != nil { + if err = vcs.New(workspace).Prepare(); err != nil { return errors.Wrap(err, "version controller preparing") } return nil diff --git a/cnb-builder-shim/internal/devsandbox/vcs/vcs.go b/cnb-builder-shim/internal/devsandbox/vcs/vcs.go index 3e4a108397..576393c0b1 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/vcs.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/vcs.go @@ -36,8 +36,10 @@ type VersionController struct { } // New ... -func New(opts ...Option) *VersionController { - v := &VersionController{} +func New(srcPath string, opts ...Option) *VersionController { + v := &VersionController{ + srcPath: srcPath, + } for _, opt := range opts { opt(v) } @@ -45,9 +47,7 @@ func New(opts ...Option) *VersionController { } // Prepare 准备步骤 -func (v *VersionController) Prepare(srcPath string) error { - v.srcPath = srcPath - +func (v *VersionController) Prepare() error { _, err := os.Stat(path.Join(v.srcPath, ".git")) // 如果对应目录下存在 .git 目录，跳过 if err == nil { @@ -77,6 +77,9 @@ func (v *VersionController) Prepare(srcPath string) error { // Diff 对比输出文件变更信息 func (v *VersionController) Diff() (Files, error) { + if err := v.Prepare(); err != nil { + return nil, err + } // 将所有文件添加到暂存区 if _, err := v.runGitCommand("add", "."); err != nil { return nil, err @@ -116,6 +119,10 @@ func (v *VersionController) Diff() (Files, error) { // Commit 提交变更 func (v *VersionController) Commit(message string) error { + if err := v.Prepare(); err != nil { + return err + } + _, err := v.runGitCommand("commit", "-m", message) return err } diff --git a/cnb-builder-shim/internal/devsandbox/vcs/vcs_test.go b/cnb-builder-shim/internal/devsandbox/vcs/vcs_test.go index b2187fd74b..ca81450187 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/vcs_test.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/vcs_test.go @@ -84,8 +84,8 @@ var _ = Describe("Test VersionController", func() { Expect(runGitCommand(tmpDir, "config", "user.email", "bkpaas@example.com")).To(BeNil()) Expect(runGitCommand(tmpDir, "commit", "-m", "init")).To(BeNil()) - verCtrl := New(WithContent()) - Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) + verCtrl := New(tmpDir, WithContent()) + Expect(verCtrl.Prepare()).To(BeNil()) files, err := verCtrl.Diff() Expect(err).To(BeNil()) @@ -121,8 +121,8 @@ var _ = Describe("Test VersionController", func() { }) It("without .git", func() { - verCtrl := New(WithContent()) - Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) + verCtrl := New(tmpDir, WithContent()) + Expect(verCtrl.Prepare()).To(BeNil()) files, err := verCtrl.Diff() Expect(err).To(BeNil()) @@ -150,8 +150,8 @@ var _ = Describe("Test VersionController", func() { }) It("with ignore", func() { - verCtrl := New(WithContent()) - Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) + verCtrl := New(tmpDir, WithContent()) + Expect(verCtrl.Prepare()).To(BeNil()) _ = initFile(path.Join(tmpDir, "webfe/templates"), "example.html", ".html") _ = initFile(path.Join(tmpDir, "v3logs"), "celery.log", "celery is running...") @@ -169,8 +169,8 @@ var _ = Describe("Test VersionController", func() { }) It("without content", func() { - verCtrl := New() - Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) + verCtrl := New(tmpDir) + Expect(verCtrl.Prepare()).To(BeNil()) _ = initFile(path.Join(tmpDir, "webfe/static"), "example.css", "css") _ = initFile(path.Join(tmpDir, "v3logs"), "celery.log", "celery is running...") @@ -187,8 +187,8 @@ var _ = Describe("Test VersionController", func() { }) It("with special chars", func() { - verCtrl := New(WithContent()) - Expect(verCtrl.Prepare(tmpDir)).To(BeNil()) + verCtrl := New(tmpDir, WithContent()) + Expect(verCtrl.Prepare()).To(BeNil()) _ = initFile(tmpDir, "example space.css", "css 代码") _ = initFile(tmpDir, "example——中文.js", "js 代码") diff --git a/cnb-builder-shim/internal/devsandbox/webserver/server.go b/cnb-builder-shim/internal/devsandbox/webserver/server.go index f3cf1adbf4..0d09b15c98 100644 --- a/cnb-builder-shim/internal/devsandbox/webserver/server.go +++ b/cnb-builder-shim/internal/devsandbox/webserver/server.go @@ -302,15 +302,8 @@ func CodeDiffsHandler() gin.HandlerFunc { if c.Query("content") == "true" { opts = append(opts, vcs.WithContent()) } - verCtrl := vcs.New(opts...) + verCtrl := vcs.New(config.G.SourceCode.Workspace, opts...) - if err := verCtrl.Prepare(config.G.SourceCode.Workspace); err != nil { - c.JSON( - http.StatusInternalServerError, - gin.H{"message": fmt.Sprintf("vcs prepare failed: %s", err)}, - ) - return - } // 获取文件变更信息 files, err := verCtrl.Diff() if err != nil { @@ -349,7 +342,7 @@ func CodeCommitHandler() gin.HandlerFunc { return } // 提交变更 - if err := vcs.New().Commit(commitMsg); err != nil { + if err := vcs.New(config.G.SourceCode.Workspace).Commit(commitMsg); err != nil { c.JSON( http.StatusInternalServerError, gin.H{"message": fmt.Sprintf("failed to commit files: %s", err)}, From c63d82d51df56a2536e675d87d4c3c8c56cf924d Mon Sep 17 00:00:00 2001 From: schnee Date: Fri, 13 Dec 2024 11:03:54 +0800 Subject: [PATCH 08/10] fix: VersionController.Commit also set user.name & user.email --- cnb-builder-shim/internal/devsandbox/vcs/vcs.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cnb-builder-shim/internal/devsandbox/vcs/vcs.go b/cnb-builder-shim/internal/devsandbox/vcs/vcs.go index 576393c0b1..172a841bef 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/vcs.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/vcs.go @@ -123,8 +123,18 @@ func (v *VersionController) Commit(message string) error { return err } - _, err := v.runGitCommand("commit", "-m", message) - return err + commands := [][]string{ + {"add", "."}, + {"config", "user.name", "bkpaas"}, + {"config", "user.email", "bkpaas@example.com"}, + {"commit", "-m", message}, + } + for _, cmd := range commands { + if _, err := v.runGitCommand(cmd...); err != nil { + return err + } + } + return nil } // 执行 Git 命令 From d25cbeccbe9445158041bf1943c4a4bc00873213 Mon Sep 17 00:00:00 2001 From: schnee Date: Fri, 13 Dec 2024 14:47:00 +0800 Subject: [PATCH 09/10] fix: remove / in root diff tree name --- cnb-builder-shim/internal/devsandbox/vcs/types.go | 2 +- cnb-builder-shim/internal/devsandbox/vcs/types_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cnb-builder-shim/internal/devsandbox/vcs/types.go b/cnb-builder-shim/internal/devsandbox/vcs/types.go index 3c005083a1..0200cb2b41 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/types.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/types.go @@ -57,7 +57,7 @@ func (files Files) AsTree() *DirTree { return strings.Compare(a.Path, b.Path) }) - root := DirTree{Name: "/"} + root := DirTree{Name: ""} var cur *DirTree for _, f := range files { parts := strings.Split(f.Path, "/") diff --git a/cnb-builder-shim/internal/devsandbox/vcs/types_test.go b/cnb-builder-shim/internal/devsandbox/vcs/types_test.go index 9733d4053e..1598805bf3 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/types_test.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/types_test.go @@ -39,7 +39,7 @@ var _ = Describe("Test Types", func() { } excepted := &DirTree{ - Name: "/", + Name: "", Dirs: []*DirTree{ { Name: "api", Files: Files{ @@ -93,7 +93,7 @@ var _ = Describe("Test Types", func() { } excepted := &DirTree{ - Name: "/", + Name: "", Dirs: []*DirTree{ { Name: "api", From fa2ef8c00e402cf8eae3e1c12fb7773d119b0e1c Mon Sep 17 00:00:00 2001 From: schnee Date: Sat, 14 Dec 2024 16:07:37 +0800 Subject: [PATCH 10/10] pref: better dir_tree compress --- .../internal/devsandbox/vcs/types.go | 8 +++-- .../internal/devsandbox/vcs/types_test.go | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/cnb-builder-shim/internal/devsandbox/vcs/types.go b/cnb-builder-shim/internal/devsandbox/vcs/types.go index 0200cb2b41..62eb5786c1 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/types.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/types.go @@ -99,8 +99,8 @@ type DirTree struct { // Compress 压缩 func (t *DirTree) Compress() *DirTree { // 根目录不需要压缩，只有子目录需要 - for idx, dt := range t.Dirs { - t.Dirs[idx] = dt.compress(dt.Name) + for idx, d := range t.Dirs { + t.Dirs[idx] = d.compress(d.Name) } return t } @@ -111,5 +111,9 @@ func (t *DirTree) compress(prefix string) *DirTree { subDir.Name = path.Join(prefix, subDir.Name) return subDir.compress(subDir.Name) } + + for idx, d := range t.Dirs { + t.Dirs[idx] = d.compress(d.Name) + } return t } diff --git a/cnb-builder-shim/internal/devsandbox/vcs/types_test.go b/cnb-builder-shim/internal/devsandbox/vcs/types_test.go index 1598805bf3..d69b730409 100644 --- a/cnb-builder-shim/internal/devsandbox/vcs/types_test.go +++ b/cnb-builder-shim/internal/devsandbox/vcs/types_test.go @@ -90,6 +90,10 @@ var _ = Describe("Test Types", func() { {Action: FileActionModified, Path: "backend/pkg/types.go"}, {Action: FileActionModified, Path: "docs/common/example.txt"}, {Action: FileActionDeleted, Path: "mako/templates/mako/about_us.mako"}, + {Action: FileActionDeleted, Path: "example/templates/example/home/templates/home/home.html"}, + {Action: FileActionDeleted, Path: "example/templates/example/home/app.js"}, + {Action: FileActionDeleted, Path: "example/templates/example/home.html"}, + {Action: FileActionDeleted, Path: "example/apps.py"}, } excepted := &DirTree{ @@ -127,6 +131,36 @@ var _ = Describe("Test Types", func() { {Action: FileActionModified, Path: "example.txt"}, }, }, + { + Name: "example", + Dirs: []*DirTree{ + { + Name: "templates/example", + Dirs: []*DirTree{ + { + Name: "home", + Dirs: []*DirTree{ + { + Name: "templates/home", + Files: Files{ + {Action: FileActionDeleted, Path: "home.html"}, + }, + }, + }, + Files: Files{ + {Action: FileActionDeleted, Path: "app.js"}, + }, + }, + }, + Files: Files{ + {Action: FileActionDeleted, Path: "home.html"}, + }, + }, + }, + Files: Files{ + {Action: FileActionDeleted, Path: "apps.py"}, + }, + }, { Name: "mako/templates/mako", Files: Files{