diff --git a/.github/.golangci.yml b/.github/.golangci.yml index 606b2a5d3..8b6770d24 100644 --- a/.github/.golangci.yml +++ b/.github/.golangci.yml @@ -223,6 +223,8 @@ issues: - gochecknoglobals - gosec - mnd + - funlen # it's not common to fragment test functions + - dupl # tests are repetitive by nature # https://github.com/go-critic/go-critic/issues/926 - linters: - gocritic diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e4105038c..16faa5073 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,7 +29,7 @@ updates: golang-dependencies: patterns: - "*" - + - package-ecosystem: docker directories: - "/app" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 63db47e7d..b395b7ddd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: - id: check-symlinks - id: end-of-file-fixer - id: check-yaml - exclude: ^(project-operator|user-tools-operator|hack/scripts/helmfile/helmfile.yaml)/ + exclude: ^(project-operator|user-tools-operator|hack/scripts/helmfile/helmfile.yaml) - repo: https://github.com/zricethezav/gitleaks rev: v8.21.2 @@ -19,7 +19,7 @@ repos: rev: v0.5.1 hooks: - id: golangci-lint - args: ['--config', '.github/.golangci.yml', 'app/api/...', 'repo-cloner/...'] + args: ['--config', '.github/.golangci.yml', "--build-tags=integration,unit", 'app/api/...', 'repo-cloner/...'] - repo: https://github.com/gruntwork-io/pre-commit rev: v0.1.24 diff --git a/app/Dockerfile b/app/Dockerfile index f65930279..8a137f198 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -31,6 +31,12 @@ RUN go build \ -trimpath \ -o kdl-server http/main.go +RUN go build \ + -ldflags="-s -w \ + -extldflags '-static'" \ + -trimpath \ + -o sync-minio-data scripts/sync_minio_data.go + ### # build app-ui ### @@ -80,7 +86,7 @@ RUN apk update && \ -u ${UID} \ -G ${USER} ${USER} && \ # setup application directories - mkdir -p /app /var/log/app && \ + mkdir -p /app/scripts /var/log/app && \ chown -R ${UID}:${GID} /app /var/log/app && \ # setup SSH with secure permissions mkdir -p ${HOME}/.ssh && \ @@ -91,6 +97,7 @@ WORKDIR /app # copy from app-api and app-ui stages COPY --from=api-builder --chown=${USER}:${USER} /app/kdl-server ./ +COPY --from=api-builder --chown=${USER}:${USER} /app/sync-minio-data ./scripts/ COPY --from=ui-builder --chown=${USER}:${USER} /app/build/ /public RUN setcap 'cap_net_bind_service=+ep' /app/kdl-server diff --git a/app/api/README.md b/app/api/README.md index 05ac9b028..3f6ee36ea 100644 --- a/app/api/README.md +++ b/app/api/README.md @@ -104,8 +104,49 @@ In order to develop in a local environment there are several things to consider: 5. Run `go run http/main.go` (or launch it from your preferred IDE) 6. You can now access the graphQL playground at +### Additional scripts + +Asides from starting KDL server, there are some additional go scripts that can +be used from inside the kdl-server pod: + +#### Sync MinIO data + +This script syncs data from the KDL server database to the MinIO instance. + +Logic inside the script is: + +* find all users, and for each user: + * create a user in MinIO + * update MinIO Credentials for the user in database + +* find all projects, and for each project: + * create a policy in MinIO for the project + * create a user for the project and assign to the project's policy + * update MinIO Credentials for the project in database + * for each project's member: + * join the user to the project in MinIO + +To run the script execute: + +```bash +/app/scripts/sync-minio-data +``` + ## Development +### GraphQL + +The GraphQL schema is defined in the `../graphql/schema.graphqls` file. The +schema is used to generate the Go code for the GraphQL server. To generate the +code execute: + +```console +go generate ./... +``` + +Configuration for the code generation is defined in the `gqlgen.yml` It is well +documented and can be customized to fit the needs of the project. + ### Running tests To create new tests install [GoMock](https://github.com/golang/mock). Mocks used on tests are generated with **mockgen**, when you need a new mock, add the following: diff --git a/app/api/entity/capabilities.go b/app/api/entity/capabilities.go index c767b6cb1..47d8c4597 100644 --- a/app/api/entity/capabilities.go +++ b/app/api/entity/capabilities.go @@ -157,7 +157,7 @@ func checkTolerationSeconds(toleration map[string]interface{}) error { if ok { switch seconds.(type) { - case int, int32, int64: + case int, int32, int64, float64: default: return wrapErrWithValue(ErrCapabilitiesInvalidSeconds, fmt.Sprintf("%v", seconds)) } diff --git a/app/api/entity/minio.go b/app/api/entity/minio.go new file mode 100644 index 000000000..3208d09f7 --- /dev/null +++ b/app/api/entity/minio.go @@ -0,0 +1,6 @@ +package entity + +type MinioAccessKey struct { + AccessKey string + SecretKey string +} diff --git a/app/api/entity/project.go b/app/api/entity/project.go index 1b41a0281..591fa4997 100644 --- a/app/api/entity/project.go +++ b/app/api/entity/project.go @@ -16,6 +16,7 @@ type Project struct { Error *string Repository Repository Members []Member + MinioAccessKey MinioAccessKey } // NewProject is a constructor function. diff --git a/app/api/entity/user.go b/app/api/entity/user.go index ea1ded40e..8ee8409b2 100644 --- a/app/api/entity/user.go +++ b/app/api/entity/user.go @@ -25,16 +25,17 @@ type APIToken struct { // User entity definition. type User struct { - ID string - Email string - Username string - Sub string - Deleted bool - CreationDate time.Time - AccessLevel AccessLevel - SSHKey SSHKey - LastActivity *time.Time - APITokens []APIToken + ID string + Email string + Username string + Sub string + Deleted bool + CreationDate time.Time + AccessLevel AccessLevel + SSHKey SSHKey + LastActivity *time.Time + APITokens []APIToken + MinioAccessKey MinioAccessKey } func (u User) UsernameSlug() string { diff --git a/app/api/go.mod b/app/api/go.mod index 86711f3f6..796908c1f 100644 --- a/app/api/go.mod +++ b/app/api/go.mod @@ -11,10 +11,12 @@ require ( github.com/golang/mock v1.6.0 github.com/gosimple/slug v1.15.0 github.com/kelseyhightower/envconfig v1.4.0 + github.com/minio/madmin-go/v2 v2.2.1 github.com/minio/minio-go/v7 v7.0.83 github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.34.0 github.com/testcontainers/testcontainers-go/modules/k3s v0.34.0 + github.com/testcontainers/testcontainers-go/modules/minio v0.34.0 github.com/vektah/gqlparser/v2 v2.5.21 go.mongodb.org/mongo-driver v1.17.2 go.uber.org/zap v1.27.0 @@ -27,6 +29,13 @@ require ( k8s.io/client-go v0.32.0 ) +require ( + github.com/philhofer/fwd v1.1.2 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/secure-io/sio-go v0.3.1 // indirect + github.com/tinylib/msgp v1.1.8 // indirect +) + require ( dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect @@ -71,7 +80,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/lufia/plan9stats v0.0.0-20230110061619-bbe2e5e100de // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/minio/md5-simd v1.1.2 // indirect @@ -90,7 +99,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/segmentio/ksuid v1.0.4 // indirect diff --git a/app/api/go.sum b/app/api/go.sum index e060d3e32..c9c7e6c52 100644 --- a/app/api/go.sum +++ b/app/api/go.sum @@ -143,12 +143,15 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20230110061619-bbe2e5e100de h1:V53FWzU6KAZVi1tPp5UIsMoUWJ2/PNwYIDXnu7QuBCE= +github.com/lufia/plan9stats v0.0.0-20230110061619-bbe2e5e100de/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/minio/madmin-go/v2 v2.2.1 h1:zyRcXBm013VF6+7wefOpJEU6N2f7/7uFKsOrcn44DpM= +github.com/minio/madmin-go/v2 v2.2.1/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.83 h1:W4Kokksvlz3OKf3OqIlzDNKd4MERlC2oN8YptwJ0+GA= @@ -184,14 +187,19 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -200,6 +208,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc= +github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= @@ -235,6 +245,10 @@ github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybu github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= github.com/testcontainers/testcontainers-go/modules/k3s v0.34.0 h1:uY4iM8S8BK4Ha5vvDj/DBnmzXlEqYGGQedKxZYby20E= github.com/testcontainers/testcontainers-go/modules/k3s v0.34.0/go.mod h1:uOBwCScZA5fGwkpAnkwcQWhlwRXMCX+5/mmQ9o0enCY= +github.com/testcontainers/testcontainers-go/modules/minio v0.34.0 h1:OpUqT7VV/d+wriDMHcCZCUfOoFE6wiHnGVzJOXqq8lU= +github.com/testcontainers/testcontainers-go/modules/minio v0.34.0/go.mod h1:0iaOtVNCzu04KcXHgmdNE7aelKaMUwC9x1M0oe6h1sw= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -285,6 +299,7 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= @@ -293,6 +308,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -301,6 +317,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= @@ -310,11 +327,13 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -326,6 +345,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -333,6 +353,7 @@ golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -340,6 +361,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= @@ -350,6 +372,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/app/api/http/main.go b/app/api/http/main.go index b1ae4f6ba..d7a6ff78e 100644 --- a/app/api/http/main.go +++ b/app/api/http/main.go @@ -21,9 +21,11 @@ import ( "github.com/konstellation-io/kdl-server/app/api/infrastructure/graph" "github.com/konstellation-io/kdl-server/app/api/infrastructure/graph/generated" "github.com/konstellation-io/kdl-server/app/api/infrastructure/k8s" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioservice" "github.com/konstellation-io/kdl-server/app/api/infrastructure/mongodb" "github.com/konstellation-io/kdl-server/app/api/pkg/clock" + "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" "github.com/konstellation-io/kdl-server/app/api/pkg/mongodbutils" "github.com/konstellation-io/kdl-server/app/api/pkg/sshhelper" "github.com/konstellation-io/kdl-server/app/api/usecase/capabilities" @@ -34,9 +36,10 @@ import ( ) type deps struct { - minioService minioservice.MinioService - k8sClient k8s.ClientInterface - mongodbClient *mongodbutils.MongoDB + minioService minioservice.MinioService + minioAdminService minioadminservice.MinioAdminInterface + k8sClient k8s.ClientInterface + mongodbClient *mongodbutils.MongoDB } func loadDependencies(logger logr.Logger, cfg config.Config) deps { @@ -49,7 +52,15 @@ func loadDependencies(logger logr.Logger, cfg config.Config) deps { os.Exit(1) } - // Setting k8s client + // Setting minio admin service + minioAdminService, err := minioadminservice.NewMinioAdminService( + logger, cfg.Minio.Endpoint, cfg.Minio.AccessKey, cfg.Minio.SecretKey, + ) + if err != nil { + logger.Error(err, "Error connecting to Minio for administration") + os.Exit(1) + } + k8sClient, err := k8s.NewK8sClient(logger, cfg) if err != nil { logger.Error(err, "Error creating k8s client") @@ -64,9 +75,10 @@ func loadDependencies(logger logr.Logger, cfg config.Config) deps { } return deps{ - minioService: minioService, - k8sClient: k8sClient, - mongodbClient: mongodbClient, + minioService: minioService, + minioAdminService: minioAdminService, + k8sClient: k8sClient, + mongodbClient: mongodbClient, } } @@ -100,17 +112,24 @@ func loadInteractors( logger logr.Logger, cfg config.Config, k8sClient k8s.ClientInterface, - minioService minioservice.MinioService, + dependencies deps, repos dbRepos, ) useCaseInteractors { realClock := clock.NewRealClock() sshHelper := sshhelper.NewGenerator(logger) + randomGenerator := kdlutil.NewRandomGenerator() capabilitiesInteractor := capabilities.NewInteractor(logger, cfg, repos.capabilitiesRepo, k8sClient) - projectInteractor := project.NewInteractor(logger, k8sClient, minioService, realClock, repos.projectRepo, repos.userActivityRepo) + projectInteractor := project.NewInteractor( + logger, k8sClient, dependencies.minioService, dependencies.minioAdminService, realClock, repos.projectRepo, + repos.userActivityRepo, randomGenerator, + ) runtimeInteractor := runtime.NewInteractor(logger, k8sClient, repos.runtimeRepo) - userInteractor := user.NewInteractor(logger, cfg, repos.userRepo, repos.userActivityRepo, repos.runtimeRepo, repos.capabilitiesRepo, - sshHelper, realClock, k8sClient) + + userInteractor := user.NewInteractor( + logger, cfg, repos.userRepo, repos.userActivityRepo, repos.runtimeRepo, repos.capabilitiesRepo, + sshHelper, realClock, k8sClient, dependencies.minioAdminService, randomGenerator, + ) configmapInteractor := configmap.NewInteractor(logger, cfg, k8sClient, projectInteractor, userInteractor) return useCaseInteractors{ @@ -139,7 +158,7 @@ func main() { defer dependencies.mongodbClient.Disconnect() repos := loadRepos(logger, cfg.MongoDB.DBName, dependencies.mongodbClient) - interactors := loadInteractors(logger, cfg, dependencies.k8sClient, dependencies.minioService, repos) + interactors := loadInteractors(logger, cfg, dependencies.k8sClient, dependencies, repos) // Execute actions before starting the HTTP server actionsBeforeStartingHTTPServer(logger, repos, interactors) diff --git a/app/api/http/middleware/auth_test.go b/app/api/http/middleware/auth_test.go index b6dcc112a..760debf75 100644 --- a/app/api/http/middleware/auth_test.go +++ b/app/api/http/middleware/auth_test.go @@ -15,6 +15,7 @@ import ( "github.com/konstellation-io/kdl-server/app/api/entity" "github.com/konstellation-io/kdl-server/app/api/http/middleware" "github.com/konstellation-io/kdl-server/app/api/infrastructure/config" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" "github.com/konstellation-io/kdl-server/app/api/usecase/capabilities" "github.com/konstellation-io/kdl-server/app/api/usecase/runtime" @@ -23,6 +24,7 @@ import ( "github.com/konstellation-io/kdl-server/app/api/infrastructure/k8s" "github.com/konstellation-io/kdl-server/app/api/pkg/clock" + "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" "github.com/konstellation-io/kdl-server/app/api/pkg/sshhelper" "github.com/konstellation-io/kdl-server/app/api/usecase/project" "github.com/konstellation-io/kdl-server/app/api/usecase/user" @@ -31,15 +33,17 @@ import ( var errUnexpected = errors.New("some error") type userMocks struct { - logger logr.Logger - cfg config.Config - repo *user.MockRepository - runtimeRepo *runtime.MockRepository - capabilitiesRepo *capabilities.MockRepository - sshGenerator *sshhelper.MockSSHKeyGenerator - clock *clock.MockClock - k8sClient *k8s.MockClientInterface - userActivityRepo *project.MockUserActivityRepo + logger logr.Logger + cfg config.Config + repo *user.MockRepository + userActivityRepo *project.MockUserActivityRepo + runtimeRepo *runtime.MockRepository + capabilitiesRepo *capabilities.MockRepository + sshGenerator *sshhelper.MockSSHKeyGenerator + clock *clock.MockClock + k8sClient *k8s.MockClientInterface + minioAdminService *minioadminservice.MockMinioAdminInterface + randomGenerator *kdlutil.MockRandomGenerator } type AuthMiddlewareTestSuite struct { @@ -64,6 +68,8 @@ func (ts *AuthMiddlewareTestSuite) SetupSuite() { ts.mocks.sshGenerator = sshhelper.NewMockSSHKeyGenerator(ts.ctrl) ts.mocks.k8sClient = k8s.NewMockClientInterface(ts.ctrl) ts.mocks.userActivityRepo = project.NewMockUserActivityRepo(ts.ctrl) + ts.mocks.minioAdminService = minioadminservice.NewMockMinioAdminInterface(ts.ctrl) + ts.mocks.randomGenerator = kdlutil.NewMockRandomGenerator(ts.ctrl) zapLog, err := zap.NewDevelopment() ts.Require().NoError(err) @@ -72,8 +78,9 @@ func (ts *AuthMiddlewareTestSuite) SetupSuite() { ts.mocks.cfg = config.Config{} - ts.interactor = user.NewInteractor(ts.mocks.logger, ts.mocks.cfg, ts.mocks.repo, ts.mocks.userActivityRepo, - ts.mocks.runtimeRepo, ts.mocks.capabilitiesRepo, ts.mocks.sshGenerator, ts.mocks.clock, ts.mocks.k8sClient) + ts.interactor = user.NewInteractor(ts.mocks.logger, ts.mocks.cfg, ts.mocks.repo, ts.mocks.userActivityRepo, ts.mocks.runtimeRepo, + ts.mocks.capabilitiesRepo, ts.mocks.sshGenerator, ts.mocks.clock, ts.mocks.k8sClient, ts.mocks.minioAdminService, + ts.mocks.randomGenerator) } func (ts *AuthMiddlewareTestSuite) TestAuthMiddlewareNoEmailHeader() { @@ -113,13 +120,15 @@ func (ts *AuthMiddlewareTestSuite) TestAuthMiddlewareNoUserHeader() { func (ts *AuthMiddlewareTestSuite) TestMiddlewareAuthUsernameNotFound() { // Arrange const ( - id = "user.1234" - email = "user@email.com" - username = "user" - sub = "d5d70477-5192-4182-b80e-5d34550eb4fe" - accessLevel = entity.AccessLevelViewer - publicSSHKey = "test-ssh-key-public" - privateSSHKey = "test-ssh-key-private" + id = "user.1234" + email = "user@email.com" + username = "user" + sub = "d5d70477-5192-4182-b80e-5d34550eb4fe" + accessLevel = entity.AccessLevelViewer + publicSSHKey = "test-ssh-key-public" + privateSSHKey = "test-ssh-key-private" + minioAccessKey = "user-user" // derived from username + minioSecretKey = "secret123" ) ctx := context.Background() @@ -157,6 +166,8 @@ func (ts *AuthMiddlewareTestSuite) TestMiddlewareAuthUsernameNotFound() { ts.mocks.sshGenerator.EXPECT().NewKeys().Return(sshKey, nil) ts.mocks.repo.EXPECT().Create(ctx, u).Return(id, nil) ts.mocks.repo.EXPECT().Get(ctx, id).Return(expectedUser, nil) + ts.mocks.randomGenerator.EXPECT().GenerateRandomString(40).Return(minioSecretKey, nil) + ts.mocks.minioAdminService.EXPECT().CreateUser(ctx, u.UsernameSlug(), minioSecretKey).Return(minioAccessKey, nil) ts.mocks.k8sClient.EXPECT().CreateUserSSHKeySecret(ctx, u, publicSSHKey, privateSSHKey) ts.mocks.k8sClient.EXPECT().CreateUserServiceAccount(ctx, expectedUser.UsernameSlug()) ts.mocks.clock.EXPECT().Now().Return(now) diff --git a/app/api/infrastructure/graph/generated/generated.go b/app/api/infrastructure/graph/generated/generated.go index 7698398e6..cbc7a566d 100644 --- a/app/api/infrastructure/graph/generated/generated.go +++ b/app/api/infrastructure/graph/generated/generated.go @@ -71,6 +71,11 @@ type ComplexityRoot struct { User func(childComplexity int) int } + MinioAccessKey struct { + AccessKey func(childComplexity int) int + SecretKey func(childComplexity int) int + } + Mutation struct { AddAPIToken func(childComplexity int, input *model.APITokenInput) int AddMembers func(childComplexity int, input model.AddMembersInput) int @@ -94,6 +99,7 @@ type ComplexityRoot struct { ID func(childComplexity int) int LastActivationDate func(childComplexity int) int Members func(childComplexity int) int + MinioAccessKey func(childComplexity int) int Name func(childComplexity int) int NeedAccess func(childComplexity int) int Repository func(childComplexity int) int @@ -302,6 +308,20 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Member.User(childComplexity), true + case "MinioAccessKey.accessKey": + if e.complexity.MinioAccessKey.AccessKey == nil { + break + } + + return e.complexity.MinioAccessKey.AccessKey(childComplexity), true + + case "MinioAccessKey.secretKey": + if e.complexity.MinioAccessKey.SecretKey == nil { + break + } + + return e.complexity.MinioAccessKey.SecretKey(childComplexity), true + case "Mutation.addApiToken": if e.complexity.Mutation.AddAPIToken == nil { break @@ -485,6 +505,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Project.Members(childComplexity), true + case "Project.minioAccessKey": + if e.complexity.Project.MinioAccessKey == nil { + break + } + + return e.complexity.Project.MinioAccessKey(childComplexity), true + case "Project.name": if e.complexity.Project.Name == nil { break @@ -979,6 +1006,7 @@ type Project { toolUrls: ToolUrls! needAccess: Boolean! archived: Boolean! + minioAccessKey: MinioAccessKey! } type Repository { @@ -992,6 +1020,11 @@ type Member { addedDate: String! } +type MinioAccessKey { + accessKey: String! + secretKey: String! +} + type ToolUrls { knowledgeGalaxy: String! filebrowser: String! @@ -2010,6 +2043,94 @@ func (ec *executionContext) fieldContext_Member_addedDate(_ context.Context, fie return fc, nil } +func (ec *executionContext) _MinioAccessKey_accessKey(ctx context.Context, field graphql.CollectedField, obj *entity.MinioAccessKey) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_MinioAccessKey_accessKey(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.AccessKey, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_MinioAccessKey_accessKey(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "MinioAccessKey", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _MinioAccessKey_secretKey(ctx context.Context, field graphql.CollectedField, obj *entity.MinioAccessKey) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_MinioAccessKey_secretKey(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.SecretKey, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_MinioAccessKey_secretKey(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "MinioAccessKey", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _Mutation_regenerateSSHKey(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Mutation_regenerateSSHKey(ctx, field) if err != nil { @@ -2287,6 +2408,8 @@ func (ec *executionContext) fieldContext_Mutation_addMembers(ctx context.Context return ec.fieldContext_Project_needAccess(ctx, field) case "archived": return ec.fieldContext_Project_archived(ctx, field) + case "minioAccessKey": + return ec.fieldContext_Project_minioAccessKey(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Project", field.Name) }, @@ -2368,6 +2491,8 @@ func (ec *executionContext) fieldContext_Mutation_createProject(ctx context.Cont return ec.fieldContext_Project_needAccess(ctx, field) case "archived": return ec.fieldContext_Project_archived(ctx, field) + case "minioAccessKey": + return ec.fieldContext_Project_minioAccessKey(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Project", field.Name) }, @@ -2446,6 +2571,8 @@ func (ec *executionContext) fieldContext_Mutation_deleteProject(ctx context.Cont return ec.fieldContext_Project_needAccess(ctx, field) case "archived": return ec.fieldContext_Project_archived(ctx, field) + case "minioAccessKey": + return ec.fieldContext_Project_minioAccessKey(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Project", field.Name) }, @@ -2527,6 +2654,8 @@ func (ec *executionContext) fieldContext_Mutation_removeMembers(ctx context.Cont return ec.fieldContext_Project_needAccess(ctx, field) case "archived": return ec.fieldContext_Project_archived(ctx, field) + case "minioAccessKey": + return ec.fieldContext_Project_minioAccessKey(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Project", field.Name) }, @@ -2608,6 +2737,8 @@ func (ec *executionContext) fieldContext_Mutation_updateMembers(ctx context.Cont return ec.fieldContext_Project_needAccess(ctx, field) case "archived": return ec.fieldContext_Project_archived(ctx, field) + case "minioAccessKey": + return ec.fieldContext_Project_minioAccessKey(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Project", field.Name) }, @@ -2689,6 +2820,8 @@ func (ec *executionContext) fieldContext_Mutation_updateProject(ctx context.Cont return ec.fieldContext_Project_needAccess(ctx, field) case "archived": return ec.fieldContext_Project_archived(ctx, field) + case "minioAccessKey": + return ec.fieldContext_Project_minioAccessKey(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Project", field.Name) }, @@ -3382,6 +3515,56 @@ func (ec *executionContext) fieldContext_Project_archived(_ context.Context, fie return fc, nil } +func (ec *executionContext) _Project_minioAccessKey(ctx context.Context, field graphql.CollectedField, obj *entity.Project) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Project_minioAccessKey(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { + ctx = rctx // use context from middleware stack in children + return obj.MinioAccessKey, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(entity.MinioAccessKey) + fc.Result = res + return ec.marshalNMinioAccessKey2githubᚗcomᚋkonstellationᚑioᚋkdlᚑserverᚋappᚋapiᚋentityᚐMinioAccessKey(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Project_minioAccessKey(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Project", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "accessKey": + return ec.fieldContext_MinioAccessKey_accessKey(ctx, field) + case "secretKey": + return ec.fieldContext_MinioAccessKey_secretKey(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type MinioAccessKey", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _QualityProjectDesc_quality(ctx context.Context, field graphql.CollectedField, obj *model.QualityProjectDesc) (ret graphql.Marshaler) { fc, err := ec.fieldContext_QualityProjectDesc_quality(ctx, field) if err != nil { @@ -3649,6 +3832,8 @@ func (ec *executionContext) fieldContext_Query_project(ctx context.Context, fiel return ec.fieldContext_Project_needAccess(ctx, field) case "archived": return ec.fieldContext_Project_archived(ctx, field) + case "minioAccessKey": + return ec.fieldContext_Project_minioAccessKey(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Project", field.Name) }, @@ -3730,6 +3915,8 @@ func (ec *executionContext) fieldContext_Query_projects(_ context.Context, field return ec.fieldContext_Project_needAccess(ctx, field) case "archived": return ec.fieldContext_Project_archived(ctx, field) + case "minioAccessKey": + return ec.fieldContext_Project_minioAccessKey(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Project", field.Name) }, @@ -7667,6 +7854,50 @@ func (ec *executionContext) _Member(ctx context.Context, sel ast.SelectionSet, o return out } +var minioAccessKeyImplementors = []string{"MinioAccessKey"} + +func (ec *executionContext) _MinioAccessKey(ctx context.Context, sel ast.SelectionSet, obj *entity.MinioAccessKey) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, minioAccessKeyImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("MinioAccessKey") + case "accessKey": + out.Values[i] = ec._MinioAccessKey_accessKey(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "secretKey": + out.Values[i] = ec._MinioAccessKey_secretKey(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var mutationImplementors = []string{"Mutation"} func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet) graphql.Marshaler { @@ -7938,6 +8169,11 @@ func (ec *executionContext) _Project(ctx context.Context, sel ast.SelectionSet, if out.Values[i] == graphql.Null { atomic.AddUint32(&out.Invalids, 1) } + case "minioAccessKey": + out.Values[i] = ec._Project_minioAccessKey(ctx, field, obj) + if out.Values[i] == graphql.Null { + atomic.AddUint32(&out.Invalids, 1) + } default: panic("unknown field " + strconv.Quote(field.Name)) } @@ -9290,6 +9526,10 @@ func (ec *executionContext) marshalNMember2ᚕgithubᚗcomᚋkonstellationᚑio return ret } +func (ec *executionContext) marshalNMinioAccessKey2githubᚗcomᚋkonstellationᚑioᚋkdlᚑserverᚋappᚋapiᚋentityᚐMinioAccessKey(ctx context.Context, sel ast.SelectionSet, v entity.MinioAccessKey) graphql.Marshaler { + return ec._MinioAccessKey(ctx, sel, &v) +} + func (ec *executionContext) marshalNProject2githubᚗcomᚋkonstellationᚑioᚋkdlᚑserverᚋappᚋapiᚋentityᚐProject(ctx context.Context, sel ast.SelectionSet, v entity.Project) graphql.Marshaler { return ec._Project(ctx, sel, &v) } diff --git a/app/api/infrastructure/k8s/error.go b/app/api/infrastructure/k8s/error.go index 400ce3a34..6e54e292c 100644 --- a/app/api/infrastructure/k8s/error.go +++ b/app/api/infrastructure/k8s/error.go @@ -4,9 +4,13 @@ import "errors" var errCRDNoMetadata = errors.New("CRD does not have a 'metadata' field") var errCRDNoSpec = errors.New("CRD does not have a 'spec' field") -var errCDRNoSpecUsername = errors.New("CRD does not have a 'spec.username' field") var errCRDNoSpecMlflow = errors.New("CRD does not have a 'spec.mlflow' field") var errCRDNoSpecMlflowEnv = errors.New("CRD does not have a 'spec.mlflow.env' field") +var errCRDNoSpecFilebrowser = errors.New("CRD does not have a 'spec.filebrowser' field") +var errCRDNoSpecFilebrowserEnv = errors.New("CRD does not have a 'spec.filebrowser.env' field") var errCRDNoSpecVscodeRuntime = errors.New("CRD does not have a 'spec.vscodeRuntime' field") +var errCRDNoSpecVscodeRuntimeEnv = errors.New("CRD does not have a 'spec.vscodeRuntime.env' field") var errCRDNoSpecVscodeRuntimeImage = errors.New("CRD does not have a 'spec.vscodeRuntime.image' field") var errCRDNoSpecPodLabels = errors.New("CRD does not have a 'spec.podLabels' field") +var errCRDCantEncodeInputData = errors.New("can't encode input data for CR") +var errCRDCantDecodeInputData = errors.New("can't decode input data from CR") diff --git a/app/api/infrastructure/k8s/interface.go b/app/api/infrastructure/k8s/interface.go index 31b4f4382..cc02b7c4e 100644 --- a/app/api/infrastructure/k8s/interface.go +++ b/app/api/infrastructure/k8s/interface.go @@ -18,15 +18,15 @@ type ClientInterface interface { CreateSecret(ctx context.Context, name string, values, labels map[string]string) error UpdateSecret(ctx context.Context, name string, values, labels map[string]string) error GetSecret(ctx context.Context, name string) (map[string][]byte, error) - CreateKDLUserToolsCR(ctx context.Context, username string, data UserToolsData) error + CreateKDLUserToolsCR(ctx context.Context, data UserToolsData) error DeleteUserToolsCR(ctx context.Context, username string) error - UpdateKDLUserToolsCR(ctx context.Context, resourceName string, data UserToolsData, crd *map[string]interface{}) error + UpdateKDLUserToolsCR(ctx context.Context, resourceName string, crd *map[string]interface{}) error ListKDLUserToolsCR(ctx context.Context) ([]unstructured.Unstructured, error) GetKDLUserToolsCR(ctx context.Context, username string) (*unstructured.Unstructured, error) IsUserToolPODRunning(ctx context.Context, username string) (bool, error) GetRuntimeIDFromUserTools(ctx context.Context, username string) (string, error) GetCapabilitiesIDFromUserTools(ctx context.Context, username string) (string, error) - CreateKDLProjectCR(ctx context.Context, projectID string) error + CreateKDLProjectCR(ctx context.Context, data ProjectData) error DeleteKDLProjectCR(ctx context.Context, projectID string) error UpdateKDLProjectsCR(ctx context.Context, projectID string, crd *map[string]interface{}) error ListKDLProjectsNameCR(ctx context.Context) ([]string, error) diff --git a/app/api/infrastructure/k8s/kdlproject.go b/app/api/infrastructure/k8s/kdlproject.go index 74c412e64..64fda9c88 100644 --- a/app/api/infrastructure/k8s/kdlproject.go +++ b/app/api/infrastructure/k8s/kdlproject.go @@ -2,18 +2,61 @@ package k8s import ( "context" + "encoding/json" + "github.com/konstellation-io/kdl-server/app/api/entity" "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) +type ProjectData struct { + ProjectID string + MinioAccessKey entity.MinioAccessKey +} + +func (k *Client) ProjectDataToMap(data ProjectData) (map[string]string, error) { + minioAccessKeyJSON, err := json.Marshal(data.MinioAccessKey) + if err != nil { + return nil, err + } + + return map[string]string{ + "projectId": data.ProjectID, + "minioAccessKey": string(minioAccessKeyJSON), + }, nil +} + +func (k *Client) MapToProjectData(data map[string]interface{}) (ProjectData, error) { + minioAccessKeyJSON, ok := data["minioAccessKey"].(string) + if !ok { + return ProjectData{}, errCRDCantDecodeInputData + } + + var minioAccessKey entity.MinioAccessKey + + err := json.Unmarshal([]byte(minioAccessKeyJSON), &minioAccessKey) + if err != nil { + return ProjectData{}, err + } + + projectID, ok := data["projectId"].(string) + if !ok { + return ProjectData{}, errCRDCantDecodeInputData + } + + return ProjectData{ + ProjectID: projectID, + MinioAccessKey: minioAccessKey, + }, nil +} + func (k *Client) GetConfigMapTemplateNameKDLProject() string { return k.cfg.ReleaseName + "-server-project-template" } -func (k *Client) updateKDLProjectTemplate(projectID string, crd *map[string]interface{}) (*map[string]interface{}, error) { +func (k *Client) updateKDLProjectTemplate(data ProjectData, crd *map[string]interface{}) (*map[string]interface{}, error) { crdToUpdate := *crd // update the spec.projectId in the CRD object @@ -22,9 +65,15 @@ func (k *Client) updateKDLProjectTemplate(projectID string, crd *map[string]inte return nil, errCRDNoSpec } - spec["projectId"] = projectID + inputData, err := k.ProjectDataToMap(data) + if err != nil { + return nil, errCRDCantEncodeInputData + } + + spec["inputData"] = inputData + spec["projectId"] = data.ProjectID - // update spec.mlflow.env.ARTIFACTS_BUCKET in the CRD object + // update spec.mlflow.env in the CRD object mlflow, ok := spec["mlflow"].(map[string]interface{}) if !ok { @@ -36,8 +85,9 @@ func (k *Client) updateKDLProjectTemplate(projectID string, crd *map[string]inte return nil, errCRDNoSpecMlflowEnv } - mlflowEnv["ARTIFACTS_BUCKET"] = projectID - // FUTURE: update AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY with minIO values for the project + mlflowEnv["ARTIFACTS_BUCKET"] = data.ProjectID + mlflowEnv["AWS_ACCESS_KEY_ID"] = data.MinioAccessKey.AccessKey + mlflowEnv["AWS_SECRET_ACCESS_KEY"] = data.MinioAccessKey.SecretKey // update metadata.name and metadata.namespace in the CRD object metadata, ok := crdToUpdate["metadata"].(map[string]interface{}) @@ -46,13 +96,28 @@ func (k *Client) updateKDLProjectTemplate(projectID string, crd *map[string]inte return nil, errCRDNoMetadata } - metadata["name"] = projectID + metadata["name"] = data.ProjectID metadata["namespace"] = k.cfg.Kubernetes.Namespace + // update spec.filebrowser.env in the CRD object + filebrowser, ok := spec["filebrowser"].(map[string]interface{}) + + if !ok { + return nil, errCRDNoSpecFilebrowser + } + + filebrowserEnv, ok := filebrowser["env"].(map[string]interface{}) + if !ok { + return nil, errCRDNoSpecFilebrowserEnv + } + + filebrowserEnv["AWS_S3_ACCESS_KEY_ID"] = data.MinioAccessKey.AccessKey + filebrowserEnv["AWS_S3_SECRET_ACCESS_KEY"] = data.MinioAccessKey.SecretKey + return &crdToUpdate, nil } -func (k *Client) CreateKDLProjectCR(ctx context.Context, projectID string) error { +func (k *Client) CreateKDLProjectCR(ctx context.Context, data ProjectData) error { // get the CRD template from the ConfigMap configMap, err := k.GetConfigMap(ctx, k.GetConfigMapTemplateNameKDLProject()) @@ -67,7 +132,7 @@ func (k *Client) CreateKDLProjectCR(ctx context.Context, projectID string) error } // update the CRD object with correct values - crdUpdated, err := k.updateKDLProjectTemplate(projectID, &crd) + crdUpdated, err := k.updateKDLProjectTemplate(data, &crd) if err != nil { return err } @@ -84,7 +149,7 @@ func (k *Client) CreateKDLProjectCR(ctx context.Context, projectID string) error return err } - k.logger.Info("KDL project created correctly in k8s", "projectName", projectID) + k.logger.Info("KDL project created correctly in k8s", "projectName", data.ProjectID) return nil } @@ -126,28 +191,45 @@ func (k *Client) GetKDLProjectCR(ctx context.Context, name string) (*unstructure } func (k *Client) UpdateKDLProjectsCR(ctx context.Context, projectID string, crd *map[string]interface{}) error { - // update the CRD object with correct values - crdUpdated, err := k.updateKDLProjectTemplate(projectID, crd) + k.logger.Info("Updating kdl project", "projectName", projectID) + + // Get existing CR + existingKDLProject, err := k.GetKDLProjectCR(ctx, projectID) if err != nil { return err } - // to update current CRD object, we need to get the existing CRD object and update the spec field - specValue, ok := (*crdUpdated)["spec"] + // Recover the input data from the existing CR + existingSpec, _, err := unstructured.NestedMap(existingKDLProject.Object, "spec") + if err != nil { + return errCRDCantDecodeInputData + } + + inputDataMap, ok := existingSpec["inputData"].(map[string]interface{}) if !ok { - return errCRDNoSpec + return errCRDCantDecodeInputData } - existingKDLProject, err := k.GetKDLProjectCR(ctx, projectID) + inputData, err := k.MapToProjectData(inputDataMap) + if err != nil { + return errCRDCantDecodeInputData + } + + // Re-apply the input data with the updated template + crdUpdated, err := k.updateKDLProjectTemplate(inputData, crd) if err != nil { return err } + // to update current CRD object, we need to get the existing CRD object and update the spec field + specValue, ok := (*crdUpdated)["spec"] + if !ok { + return errCRDNoSpec + } + existingKDLProject.Object["spec"] = specValue // CRD object is now updated and ready to be created - k.logger.Info("Updating kdl project", "projectName", projectID) - _, err = k.kdlProjectRes.Namespace(k.cfg.Kubernetes.Namespace).Update(ctx, existingKDLProject, metav1.UpdateOptions{}) if err != nil { k.logger.Error(err, "Error updating KDL Project CR in k8s", "projectName", projectID) diff --git a/app/api/infrastructure/k8s/kdlproject_test.go b/app/api/infrastructure/k8s/kdlproject_test.go index 1cf0632d9..01d5be126 100644 --- a/app/api/infrastructure/k8s/kdlproject_test.go +++ b/app/api/infrastructure/k8s/kdlproject_test.go @@ -5,6 +5,8 @@ package k8s_test import ( "context" + "github.com/konstellation-io/kdl-server/app/api/entity" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/k8s" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -12,8 +14,18 @@ import ( const ( projectID = "test-project-id" configMapKdlProjectName = "kdl-server-project-template" + projectMinioAccessKey = "project-test-project-id" + projectMinioSecretKey = "testproject123" ) +var projectData = k8s.ProjectData{ + ProjectID: projectID, + MinioAccessKey: entity.MinioAccessKey{ + AccessKey: projectMinioAccessKey, + SecretKey: projectMinioSecretKey, + }, +} + func (s *testSuite) createKDLProjectConfigMapTemplate() { yamlContent := ` apiVersion: kdl.konstellation.io/v1 @@ -29,6 +41,8 @@ spec: mlflow: env: ARTIFACTS_BUCKET: my-demo-bucket + filebrowser: + env: {} ` _, err := s.Clientset.CoreV1().ConfigMaps(namespace).Create( context.Background(), &v1.ConfigMap{ @@ -47,16 +61,40 @@ spec: func (s *testSuite) TestCreateKDLProjectCR_and_DeleteKDLProjectCR() { s.createKDLProjectConfigMapTemplate() - err := s.Client.CreateKDLProjectCR(context.Background(), projectID) + err := s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().NoError(err) + // Retrieve the Custom Resource + resource, err := s.kdlProjectRes.Namespace(namespace).Get(context.Background(), projectID, metav1.GetOptions{}) + s.Require().NoError(err) + + // Check its data + spec, _ := resource.Object["spec"].(map[string]interface{}) + mlflow, _ := spec["mlflow"].(map[string]interface{}) + mlflowEnv, _ := mlflow["env"].(map[string]interface{}) + filebrowser, _ := spec["filebrowser"].(map[string]interface{}) + filebrowserEnv, _ := filebrowser["env"].(map[string]interface{}) + + s.Require().Equal(projectID, spec["projectId"]) + s.Require().Equal(projectMinioAccessKey, mlflowEnv["AWS_ACCESS_KEY_ID"]) + s.Require().Equal(projectMinioSecretKey, mlflowEnv["AWS_SECRET_ACCESS_KEY"]) + s.Require().Equal(projectMinioAccessKey, filebrowserEnv["AWS_S3_ACCESS_KEY_ID"]) + s.Require().Equal(projectMinioSecretKey, filebrowserEnv["AWS_S3_SECRET_ACCESS_KEY"]) + + // Check the input data itself is stored as well + minioAccessKeyJSON := "{\"AccessKey\":\"project-test-project-id\",\"SecretKey\":\"testproject123\"}" + inputData, _ := spec["inputData"].(map[string]interface{}) + inputProjectID, _ := inputData["projectId"].(string) + s.Require().Equal(projectID, inputProjectID) + s.Require().Equal(minioAccessKeyJSON, inputData["minioAccessKey"]) + // Delete the CR err = s.Client.DeleteKDLProjectCR(context.Background(), projectID) s.Require().NoError(err) } func (s *testSuite) TestCreateKDLProjectCR_NoConfigMap() { - err := s.Client.CreateKDLProjectCR(context.Background(), projectID) + err := s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().Error(err) } @@ -84,7 +122,7 @@ metadata: ) s.Require().NoError(err) - err = s.Client.CreateKDLProjectCR(context.Background(), projectID) + err = s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().Error(err) } @@ -111,7 +149,7 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLProjectCR(context.Background(), projectID) + err = s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().Error(err) } @@ -141,7 +179,7 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLProjectCR(context.Background(), projectID) + err = s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().Error(err) } @@ -174,14 +212,14 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLProjectCR(context.Background(), projectID) + err = s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().Error(err) } func (s *testSuite) TestListKDLProjectsNameCR() { // Arrange by creating the CR s.createKDLProjectConfigMapTemplate() - err := s.Client.CreateKDLProjectCR(context.Background(), projectID) + err := s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().NoError(err) // Assert the CR exists @@ -205,7 +243,7 @@ func (s *testSuite) TestListKDLProjectsNameCR_EmptyList() { func (s *testSuite) TestGetKDLProjectCR() { // Arrange by creating the CR s.createKDLProjectConfigMapTemplate() - err := s.Client.CreateKDLProjectCR(context.Background(), projectID) + err := s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().NoError(err) // Assert the CR exists @@ -228,18 +266,28 @@ func (s *testSuite) TestGetKDLProjectCR_Empty() { func (s *testSuite) TestUpdateKDLProjectsCR() { // Arrange by creating the CR s.createKDLProjectConfigMapTemplate() - err := s.Client.CreateKDLProjectCR(context.Background(), projectID) + err := s.Client.CreateKDLProjectCR(context.Background(), projectData) s.Require().NoError(err) // Update the CR crd := map[string]interface{}{ "spec": map[string]interface{}{ + "inputData": map[string]interface{}{ + "projectId": "my-demo-projectId", + "minioAccessKey": map[string]interface{}{ + "AccessKey": "my-demo-accessKey", + "SecretKey": "my-demo-secretKey", + }, + }, "projectId": "my-demo-projectId", "mlflow": map[string]interface{}{ "env": map[string]interface{}{ "ARTIFACTS_BUCKET": "my-demo-bucket", }, }, + "filebrowser": map[string]interface{}{ + "env": map[string]interface{}{}, + }, }, "metadata": map[string]interface{}{ "name": "my-demo-name", diff --git a/app/api/infrastructure/k8s/kdlusertools.go b/app/api/infrastructure/k8s/kdlusertools.go index 2e136f690..3f1c63137 100644 --- a/app/api/infrastructure/k8s/kdlusertools.go +++ b/app/api/infrastructure/k8s/kdlusertools.go @@ -2,6 +2,7 @@ package k8s import ( "context" + "encoding/json" "fmt" "github.com/gosimple/slug" @@ -15,10 +16,94 @@ import ( ) type UserToolsData struct { - RuntimeID string - RuntimeImage string - RuntimeTag string - Capabilities entity.Capabilities + Username string + SlugUsername string + RuntimeID string + RuntimeImage string + RuntimeTag string + Capabilities entity.Capabilities + MinioAccessKey entity.MinioAccessKey +} + +func (k *Client) UserToolsDataToMap(data UserToolsData) (map[string]string, error) { + minioAccessKeyJSON, err := json.Marshal(data.MinioAccessKey) + if err != nil { + return nil, err + } + + capabilitiesJSON, err := json.Marshal(data.Capabilities) + if err != nil { + return nil, err + } + + return map[string]string{ + "username": data.Username, + "slugUsername": data.SlugUsername, + "runtimeID": data.RuntimeID, + "runtimeImage": data.RuntimeImage, + "runtimeTag": data.RuntimeTag, + "capabilities": string(capabilitiesJSON), + "minioAccessKey": string(minioAccessKeyJSON), + }, err +} + +func (k *Client) MapToUserToolsData(data map[string]interface{}) (UserToolsData, error) { + var capabilities entity.Capabilities + + capStr, ok := data["capabilities"].(string) + if !ok { + return UserToolsData{}, errCRDCantDecodeInputData + } + + if err := json.Unmarshal([]byte(capStr), &capabilities); err != nil { + return UserToolsData{}, errCRDCantDecodeInputData + } + + var minioAccessKey entity.MinioAccessKey + + minioStr, ok := data["minioAccessKey"].(string) + if !ok { + return UserToolsData{}, errCRDCantDecodeInputData + } + + if err := json.Unmarshal([]byte(minioStr), &minioAccessKey); err != nil { + return UserToolsData{}, errCRDCantDecodeInputData + } + + username, ok := data["username"].(string) + if !ok { + return UserToolsData{}, errCRDCantDecodeInputData + } + + slugUsername, ok := data["slugUsername"].(string) + if !ok { + return UserToolsData{}, errCRDCantDecodeInputData + } + + runtimeID, ok := data["runtimeID"].(string) + if !ok { + return UserToolsData{}, errCRDCantDecodeInputData + } + + runtimeImage, ok := data["runtimeImage"].(string) + if !ok { + return UserToolsData{}, errCRDCantDecodeInputData + } + + runtimeTag, ok := data["runtimeTag"].(string) + if !ok { + return UserToolsData{}, errCRDCantDecodeInputData + } + + return UserToolsData{ + Username: username, + SlugUsername: slugUsername, + RuntimeID: runtimeID, + RuntimeImage: runtimeImage, + RuntimeTag: runtimeTag, + Capabilities: capabilities, + MinioAccessKey: minioAccessKey, + }, nil } // DeleteUserToolsCR removes a given user tools custom resource from Kubernetes. @@ -55,7 +140,7 @@ func (k *Client) DeleteUserToolsCR(ctx context.Context, username string) error { func (k *Client) updateUserToolsTemplate( crd *map[string]interface{}, - slugUsername, resName, username string, + resName string, data UserToolsData, ) (*map[string]interface{}, error) { crdToUpdate := *crd @@ -75,8 +160,15 @@ func (k *Client) updateUserToolsTemplate( return nil, errCRDNoSpec } - spec["username"] = username - spec["usernameSlug"] = slugUsername + inputData, err := k.UserToolsDataToMap(data) + if err != nil { + return nil, errCRDCantEncodeInputData + } + + spec["inputData"] = inputData + + spec["username"] = data.Username + spec["usernameSlug"] = data.SlugUsername // update spec.vscodeRuntime.image.repository and spec.vscodeRuntime.image.tag in the CRD object vscodeRuntime, ok := spec["vscodeRuntime"].(map[string]interface{}) @@ -91,7 +183,14 @@ func (k *Client) updateUserToolsTemplate( vscodeRuntimeImage["repository"] = data.RuntimeImage vscodeRuntimeImage["tag"] = data.RuntimeTag - // FUTURE: update spec.vscodeRuntime.env.MINIO_ACCESS_KEY and spec.vscodeRuntime.env.MINIO_SECRET_KEY with minIO values for the user + + env, ok := vscodeRuntime["env"].(map[string]interface{}) + if !ok { + return nil, errCRDNoSpecVscodeRuntimeEnv + } + + env["AWS_ACCESS_KEY"] = data.MinioAccessKey.AccessKey + env["AWS_SECRET_KEY"] = data.MinioAccessKey.SecretKey if data.Capabilities.ID != "" { if err := data.Capabilities.Validate(); err != nil { @@ -138,11 +237,9 @@ func (k *Client) GetConfigMapTemplateNameKDLUserTools() string { // CreateKDLUserToolsCR creates the user tools Custom Resource in Kubernetes. func (k *Client) CreateKDLUserToolsCR( ctx context.Context, - username string, data UserToolsData, ) error { - slugUsername := k.getSlugUsername(username) - resName := fmt.Sprintf("usertools-%s", slugUsername) + resName := fmt.Sprintf("usertools-%s", data.SlugUsername) configMap, err := k.GetConfigMap(ctx, k.GetConfigMapTemplateNameKDLUserTools()) if err != nil { @@ -156,7 +253,7 @@ func (k *Client) CreateKDLUserToolsCR( } // update the CRD object with correct values - crdUpdated, err := k.updateUserToolsTemplate(&crd, slugUsername, resName, username, data) + crdUpdated, err := k.updateUserToolsTemplate(&crd, resName, data) if err != nil { return err } @@ -165,7 +262,7 @@ func (k *Client) CreateKDLUserToolsCR( Object: *crdUpdated, } - _, err = k.kdlUserToolsRes.Namespace(k.cfg.Kubernetes.Namespace).Create(ctx, definition, metav1.CreateOptions{}) + _, err = k.kdlUserToolsRes.Namespace(k.cfg.Kubernetes.Namespace).Create(ctx, definition, metav1.CreateOptions{FieldValidation: "Strict"}) if err != nil { k.logger.Error(err, "Error creating user tools") return err @@ -276,26 +373,34 @@ func (k *Client) GetKDLUserToolsCR(ctx context.Context, resourceName string) (*u return object, nil } -func (k *Client) UpdateKDLUserToolsCR(ctx context.Context, resourceName string, data UserToolsData, crd *map[string]interface{}) error { +func (k *Client) UpdateKDLUserToolsCR(ctx context.Context, resourceName string, crd *map[string]interface{}) error { + // CRD object is now updated and ready to be created + k.logger.Info("Updating KDL User Tools CR in k8s", "resourceName", resourceName) + + // Get existing CR existingKDLUserTool, err := k.GetKDLUserToolsCR(ctx, resourceName) if err != nil { return err } - spec, ok := existingKDLUserTool.Object["spec"].(map[string]interface{}) - if !ok { - return errCRDNoSpec + // Recover the input data from the existing CR + existingSpec, _, err := unstructured.NestedMap(existingKDLUserTool.Object, "spec") + if err != nil { + return errCRDCantDecodeInputData } - username, ok := spec["username"].(string) + inputDataMap, ok := existingSpec["inputData"].(map[string]interface{}) if !ok { - return errCDRNoSpecUsername + return errCRDCantDecodeInputData } - slugUsername := k.getSlugUsername(username) + inputData, err := k.MapToUserToolsData(inputDataMap) + if err != nil { + return errCRDCantDecodeInputData + } - // update the CRD object with correct values - crdUpdated, err := k.updateUserToolsTemplate(crd, slugUsername, resourceName, username, data) + // Re-apply the input data with the updated template + crdUpdated, err := k.updateUserToolsTemplate(crd, resourceName, inputData) if err != nil { return err } @@ -308,15 +413,12 @@ func (k *Client) UpdateKDLUserToolsCR(ctx context.Context, resourceName string, existingKDLUserTool.Object["spec"] = specValue - // CRD object is now updated and ready to be created - k.logger.Info("Updating KDL User Tools CR in k8s", "username", username) - _, err = k.kdlUserToolsRes.Namespace(k.cfg.Kubernetes.Namespace).Update(ctx, existingKDLUserTool, metav1.UpdateOptions{}) if err != nil { return err } - k.logger.Info("Updated KDL User Tools CR in k8s", "username", username) + k.logger.Info("Updated KDL User Tools CR in k8s", "resourceName", resourceName) return nil } diff --git a/app/api/infrastructure/k8s/kdlusertools_test.go b/app/api/infrastructure/k8s/kdlusertools_test.go index 4de28c877..20d2bdfbf 100644 --- a/app/api/infrastructure/k8s/kdlusertools_test.go +++ b/app/api/infrastructure/k8s/kdlusertools_test.go @@ -4,6 +4,7 @@ package k8s_test import ( "context" + "encoding/json" "time" "github.com/konstellation-io/kdl-server/app/api/entity" @@ -14,19 +15,26 @@ import ( const ( username = "test.username" + slugUsername = "test-username" resName = "usertools-test-username" + minioAccessKey = "user-test-username" + minioSecretKey = "username123" runtimeID = "test-runtime-id" runtimeImage = "test-runtime-image" runtimeTag = "test-runtime-tag" configMapKdlUserToolTemplateName = "kdl-server-user-tools-template" ) -var data = k8s.UserToolsData{ +var userToolsData = k8s.UserToolsData{ + Username: username, + SlugUsername: slugUsername, RuntimeID: runtimeID, RuntimeImage: runtimeImage, RuntimeTag: runtimeTag, } var dataWithCapabilities = k8s.UserToolsData{ + Username: username, + SlugUsername: slugUsername, RuntimeID: runtimeID, RuntimeImage: runtimeImage, RuntimeTag: runtimeTag, @@ -46,6 +54,10 @@ var dataWithCapabilities = k8s.UserToolsData{ }, Affinities: map[string]interface{}{"key": "value"}, }, + MinioAccessKey: entity.MinioAccessKey{ + AccessKey: minioAccessKey, + SecretKey: minioSecretKey, + }, } func (s *testSuite) createKDLUserToolsConfigMapTemplate() { @@ -64,12 +76,13 @@ spec: image: repository: my-demo-repository tag: my-demo-tag + env: {} nodeSelector: {} tolerations: [] affinity: {} podLabels: runtimeId: my-demo-runtime-id - capabilitiesId: my-demo-capabilities-id + capabilityId: my-demo-capabilities-id ` _, err := s.Clientset.CoreV1().ConfigMaps(namespace).Create( context.Background(), &v1.ConfigMap{ @@ -94,9 +107,59 @@ func (s *testSuite) TestCreateKDLUserToolsCR_and_DeleteUserToolsCR() { cancelCreateUserToolCR() }() - err := s.Client.CreateKDLUserToolsCR(ctx, username, dataWithCapabilities) + err := s.Client.CreateKDLUserToolsCR(ctx, dataWithCapabilities) + s.Require().NoError(err) + + // Retrieve the Custom Resource + resource, err := s.kdlUserToolsRes.Namespace(namespace).Get(context.Background(), resName, metav1.GetOptions{}) + s.Require().NoError(err) + + // Check its data + spec, _ := resource.Object["spec"].(map[string]interface{}) + vscodeRuntime, _ := spec["vscodeRuntime"].(map[string]interface{}) + image, _ := vscodeRuntime["image"].(map[string]interface{}) + env, _ := vscodeRuntime["env"].(map[string]interface{}) + podLabels, _ := spec["podLabels"].(map[string]interface{}) + affinity, _ := spec["affinity"].(map[string]interface{}) + tolerations, _ := spec["tolerations"].([]interface{}) + toleration, _ := tolerations[0].(map[string]interface{}) + + s.Require().Equal(runtimeImage, image["repository"]) + s.Require().Equal(runtimeTag, image["tag"]) + s.Require().Equal(runtimeID, podLabels["runtimeId"]) + s.Require().Equal("test-capability-id", podLabels["capabilityId"]) + s.Require().Equal(minioAccessKey, env["AWS_ACCESS_KEY"]) + s.Require().Equal(minioSecretKey, env["AWS_SECRET_KEY"]) + s.Require().Equal("value", affinity["key"]) + s.Require().Equal("Equal", toleration["operator"]) + s.Require().Equal("value1", toleration["value"]) + s.Require().Equal("NoExecute", toleration["effect"]) + s.Require().Equal(int64(100), toleration["tolerationSeconds"]) + + // Check the input data itself is stored as well + inputData, _ := spec["inputData"].(map[string]interface{}) + inputUserName, _ := inputData["username"].(string) + inputSlugUserName, _ := inputData["slugUsername"].(string) + inputRuntimeID, _ := inputData["runtimeID"].(string) + inputRuntimeImage, _ := inputData["runtimeImage"].(string) + inputRuntimeTag, _ := inputData["runtimeTag"].(string) + + s.Require().Equal(username, inputUserName) + s.Require().Equal(slugUsername, inputSlugUserName) + s.Require().Equal(runtimeID, inputRuntimeID) + s.Require().Equal(runtimeImage, inputRuntimeImage) + s.Require().Equal(runtimeTag, inputRuntimeTag) + + var decodedMinioAccessKey entity.MinioAccessKey + + inputMinioAccessKey, _ := inputData["minioAccessKey"].(string) + + err = json.Unmarshal([]byte(inputMinioAccessKey), &decodedMinioAccessKey) s.Require().NoError(err) + s.Require().Equal(minioAccessKey, decodedMinioAccessKey.AccessKey) + s.Require().Equal(minioSecretKey, decodedMinioAccessKey.SecretKey) + // Delete the CR // create go routine to cancel the context in 5 seconds. Risk of flaky test ctx, cancelDeleteUserToolsCR := context.WithCancel(context.Background()) @@ -110,7 +173,7 @@ func (s *testSuite) TestCreateKDLUserToolsCR_and_DeleteUserToolsCR() { } func (s *testSuite) TestCreateKDLUserToolsCR_NoConfigMap() { - err := s.Client.CreateKDLUserToolsCR(context.Background(), username, data) + err := s.Client.CreateKDLUserToolsCR(context.Background(), userToolsData) s.Require().Error(err) } @@ -125,7 +188,7 @@ func (s *testSuite) TestCreateKDLUserToolsCR_ConfigMapWithoutTemplate() { ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, data) + err = s.Client.CreateKDLUserToolsCR(context.Background(), userToolsData) s.Require().Error(err) } @@ -147,7 +210,7 @@ kind: KDLUserTools ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, data) + err = s.Client.CreateKDLUserToolsCR(context.Background(), userToolsData) s.Require().Error(err) } @@ -172,7 +235,7 @@ metadata: ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, data) + err = s.Client.CreateKDLUserToolsCR(context.Background(), userToolsData) s.Require().Error(err) } @@ -199,7 +262,7 @@ metadata: ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, data) + err = s.Client.CreateKDLUserToolsCR(context.Background(), userToolsData) s.Require().Error(err) } @@ -229,7 +292,7 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, data) + err = s.Client.CreateKDLUserToolsCR(context.Background(), userToolsData) s.Require().Error(err) } @@ -260,7 +323,7 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, data) + err = s.Client.CreateKDLUserToolsCR(context.Background(), userToolsData) s.Require().Error(err) } @@ -280,6 +343,7 @@ spec: image: repository: my-demo-repository tag: my-demo-tag + env: {} ` _, err := s.Clientset.CoreV1().ConfigMaps(namespace).Create( context.Background(), &v1.ConfigMap{ @@ -294,7 +358,7 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, dataWithCapabilities) + err = s.Client.CreateKDLUserToolsCR(context.Background(), dataWithCapabilities) s.Require().Error(err) } @@ -314,6 +378,7 @@ spec: image: repository: my-demo-repository tag: my-demo-tag + env: {} nodeSelector: {} ` _, err := s.Clientset.CoreV1().ConfigMaps(namespace).Create( @@ -329,7 +394,7 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, dataWithCapabilities) + err = s.Client.CreateKDLUserToolsCR(context.Background(), dataWithCapabilities) s.Require().Error(err) } @@ -349,6 +414,7 @@ spec: image: repository: my-demo-repository tag: my-demo-tag + env: {} nodeSelector: {} tolerations: [] ` @@ -365,7 +431,7 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, dataWithCapabilities) + err = s.Client.CreateKDLUserToolsCR(context.Background(), dataWithCapabilities) s.Require().Error(err) } @@ -385,6 +451,7 @@ spec: image: repository: my-demo-repository tag: my-demo-tag + env: {} nodeSelector: {} tolerations: [] affinity: {} @@ -402,7 +469,7 @@ spec: ) s.Require().NoError(err) - err = s.Client.CreateKDLUserToolsCR(context.Background(), username, data) + err = s.Client.CreateKDLUserToolsCR(context.Background(), userToolsData) s.Require().Error(err) } @@ -415,7 +482,7 @@ func (s *testSuite) TestListKDLUserToolsCR() { cancelCreateUserToolCR() }() - err := s.Client.CreateKDLUserToolsCR(ctx, username, dataWithCapabilities) + err := s.Client.CreateKDLUserToolsCR(ctx, dataWithCapabilities) s.Require().NoError(err) // List the CR @@ -451,7 +518,7 @@ func (s *testSuite) TestGetKDLUserToolsCR() { cancelCreateUserToolCR() }() - err := s.Client.CreateKDLUserToolsCR(ctx, username, dataWithCapabilities) + err := s.Client.CreateKDLUserToolsCR(ctx, dataWithCapabilities) s.Require().NoError(err) // Get the CR @@ -487,7 +554,7 @@ func (s *testSuite) TestUpdateKDLUserToolsCR() { cancelCreateUserToolCR() }() - err := s.Client.CreateKDLUserToolsCR(ctx, username, dataWithCapabilities) + err := s.Client.CreateKDLUserToolsCR(ctx, dataWithCapabilities) s.Require().NoError(err) // Update the CR @@ -500,6 +567,7 @@ func (s *testSuite) TestUpdateKDLUserToolsCR() { "repository": "new-repo", "tag": "new-tag", }, + "env": map[string]interface{}{}, }, "podLabels": map[string]interface{}{ "runtimeId": "new-runtime-id", @@ -510,9 +578,35 @@ func (s *testSuite) TestUpdateKDLUserToolsCR() { "name": "new-res-name", }, } - err = s.Client.UpdateKDLUserToolsCR(context.Background(), resName, data, &crd) + err = s.Client.UpdateKDLUserToolsCR(context.Background(), resName, &crd) + s.Require().NoError(err) + + // Retrieve the Custom Resource + resource, err := s.kdlUserToolsRes.Namespace(namespace).Get(context.Background(), resName, metav1.GetOptions{}) s.Require().NoError(err) + // Check the data is still there, in spite of the template update + spec, _ := resource.Object["spec"].(map[string]interface{}) + vscodeRuntime, _ := spec["vscodeRuntime"].(map[string]interface{}) + image, _ := vscodeRuntime["image"].(map[string]interface{}) + env, _ := vscodeRuntime["env"].(map[string]interface{}) + podLabels, _ := spec["podLabels"].(map[string]interface{}) + affinity, _ := spec["affinity"].(map[string]interface{}) + tolerations, _ := spec["tolerations"].([]interface{}) + toleration, _ := tolerations[0].(map[string]interface{}) + + s.Require().Equal(runtimeImage, image["repository"]) + s.Require().Equal(runtimeTag, image["tag"]) + s.Require().Equal(runtimeID, podLabels["runtimeId"]) + s.Require().Equal("test-capability-id", podLabels["capabilityId"]) + s.Require().Equal(minioAccessKey, env["AWS_ACCESS_KEY"]) + s.Require().Equal(minioSecretKey, env["AWS_SECRET_KEY"]) + s.Require().Equal("value", affinity["key"]) + s.Require().Equal("Equal", toleration["operator"]) + s.Require().Equal("value1", toleration["value"]) + s.Require().Equal("NoExecute", toleration["effect"]) + s.Require().Equal(int64(100), toleration["tolerationSeconds"]) + // Delete the CR // create go routine to cancel the context in 5 seconds. Risk of flaky test ctx, cancelDeleteUserToolsCR := context.WithCancel(context.Background()) diff --git a/app/api/infrastructure/k8s/mocks_interface.go b/app/api/infrastructure/k8s/mocks_interface.go index 980014e4f..5f614e892 100644 --- a/app/api/infrastructure/k8s/mocks_interface.go +++ b/app/api/infrastructure/k8s/mocks_interface.go @@ -68,31 +68,31 @@ func (mr *MockClientInterfaceMockRecorder) CreateConfigMapWatcher(ctx interface{ } // CreateKDLProjectCR mocks base method. -func (m *MockClientInterface) CreateKDLProjectCR(ctx context.Context, projectID string) error { +func (m *MockClientInterface) CreateKDLProjectCR(ctx context.Context, data ProjectData) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateKDLProjectCR", ctx, projectID) + ret := m.ctrl.Call(m, "CreateKDLProjectCR", ctx, data) ret0, _ := ret[0].(error) return ret0 } // CreateKDLProjectCR indicates an expected call of CreateKDLProjectCR. -func (mr *MockClientInterfaceMockRecorder) CreateKDLProjectCR(ctx, projectID interface{}) *gomock.Call { +func (mr *MockClientInterfaceMockRecorder) CreateKDLProjectCR(ctx, data interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateKDLProjectCR", reflect.TypeOf((*MockClientInterface)(nil).CreateKDLProjectCR), ctx, projectID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateKDLProjectCR", reflect.TypeOf((*MockClientInterface)(nil).CreateKDLProjectCR), ctx, data) } // CreateKDLUserToolsCR mocks base method. -func (m *MockClientInterface) CreateKDLUserToolsCR(ctx context.Context, username string, data UserToolsData) error { +func (m *MockClientInterface) CreateKDLUserToolsCR(ctx context.Context, data UserToolsData) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateKDLUserToolsCR", ctx, username, data) + ret := m.ctrl.Call(m, "CreateKDLUserToolsCR", ctx, data) ret0, _ := ret[0].(error) return ret0 } // CreateKDLUserToolsCR indicates an expected call of CreateKDLUserToolsCR. -func (mr *MockClientInterfaceMockRecorder) CreateKDLUserToolsCR(ctx, username, data interface{}) *gomock.Call { +func (mr *MockClientInterfaceMockRecorder) CreateKDLUserToolsCR(ctx, data interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateKDLUserToolsCR", reflect.TypeOf((*MockClientInterface)(nil).CreateKDLUserToolsCR), ctx, username, data) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateKDLUserToolsCR", reflect.TypeOf((*MockClientInterface)(nil).CreateKDLUserToolsCR), ctx, data) } // CreateSecret mocks base method. @@ -418,17 +418,17 @@ func (mr *MockClientInterfaceMockRecorder) UpdateKDLProjectsCR(ctx, projectID, c } // UpdateKDLUserToolsCR mocks base method. -func (m *MockClientInterface) UpdateKDLUserToolsCR(ctx context.Context, resourceName string, data UserToolsData, crd *map[string]interface{}) error { +func (m *MockClientInterface) UpdateKDLUserToolsCR(ctx context.Context, resourceName string, crd *map[string]interface{}) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateKDLUserToolsCR", ctx, resourceName, data, crd) + ret := m.ctrl.Call(m, "UpdateKDLUserToolsCR", ctx, resourceName, crd) ret0, _ := ret[0].(error) return ret0 } // UpdateKDLUserToolsCR indicates an expected call of UpdateKDLUserToolsCR. -func (mr *MockClientInterfaceMockRecorder) UpdateKDLUserToolsCR(ctx, resourceName, data, crd interface{}) *gomock.Call { +func (mr *MockClientInterfaceMockRecorder) UpdateKDLUserToolsCR(ctx, resourceName, crd interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateKDLUserToolsCR", reflect.TypeOf((*MockClientInterface)(nil).UpdateKDLUserToolsCR), ctx, resourceName, data, crd) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateKDLUserToolsCR", reflect.TypeOf((*MockClientInterface)(nil).UpdateKDLUserToolsCR), ctx, resourceName, crd) } // UpdateSecret mocks base method. diff --git a/app/api/infrastructure/k8s/suite_test.go b/app/api/infrastructure/k8s/suite_test.go index 9ab2004d8..358650d2e 100644 --- a/app/api/infrastructure/k8s/suite_test.go +++ b/app/api/infrastructure/k8s/suite_test.go @@ -41,48 +41,99 @@ const ( type testSuite struct { suite.Suite - Container *k3s.K3sContainer - Client *k8s.Client - Clientset *kubernetes.Clientset + Container *k3s.K3sContainer + Client *k8s.Client + Clientset *kubernetes.Clientset + kdlUserToolsRes dynamic.NamespaceableResourceInterface + kdlProjectRes dynamic.NamespaceableResourceInterface } func TestSuite(t *testing.T) { suite.Run(t, new(testSuite)) } -func (s *testSuite) defineCRD(restcfg *rest.Config) { - // Create a clientset for CRD operations - apiExtensionsClient, err := apiextensionsclient.NewForConfig(restcfg) - s.Require().NoError(err) - - // Define the CRD for KDLUserTools - crdKdlUserTools := &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kdlusertools.kdl.konstellation.io", // Format: plural.group +var preserve = true +var crdKdlUserTools = &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kdlusertools.kdl.konstellation.io", // Format: plural.group + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "kdl.konstellation.io", + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "kdlusertools", + Singular: "kdlusertool", + Kind: "KDLUserTools", + ListKind: "KDLUserToolsList", }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "kdl.konstellation.io", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "kdlusertools", - Singular: "kdlusertool", - Kind: "KDLUserTools", - ListKind: "KDLUserToolsList", - }, - Scope: apiextensionsv1.NamespaceScoped, - Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ - { - Name: "v1", - Served: true, - Storage: true, - Schema: &apiextensionsv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextensionsv1.JSONSchemaProps{ - "spec": { - Type: "object", - Properties: map[string]apiextensionsv1.JSONSchemaProps{ - "username": {Type: "string"}, - "usernameSlug": {Type: "string"}, + Scope: apiextensionsv1.NamespaceScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "spec": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "inputData": { + Type: "object", + AdditionalProperties: &apiextensionsv1.JSONSchemaPropsOrBool{ + Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "username": {Type: "string"}, + "usernameSlug": {Type: "string"}, + "vscodeRuntime": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "image": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "repository": {Type: "string"}, + "tag": {Type: "string"}, + }, + }, + "env": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{}, + XPreserveUnknownFields: &preserve, + }, + }, + }, + "affinity": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{}, + XPreserveUnknownFields: &preserve, + }, + "nodeSelector": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{}, + XPreserveUnknownFields: &preserve, + }, + "tolerations": { + Type: "array", + Items: &apiextensionsv1.JSONSchemaPropsOrArray{ + Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "effect": {Type: "string"}, + "key": {Type: "string"}, + "operator": {Type: "string"}, + "tolerationSeconds": {Type: "integer"}, + "value": {Type: "string"}, + }, + }, + }, + }, + "podLabels": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{}, + XPreserveUnknownFields: &preserve, }, }, }, @@ -91,52 +142,91 @@ func (s *testSuite) defineCRD(restcfg *rest.Config) { }, }, }, - } + }, +} + +func (s *testSuite) defineCRDKDLUserTools(restcfg *rest.Config) { + // Create a clientset for CRD operations + apiExtensionsClient, err := apiextensionsclient.NewForConfig(restcfg) + s.Require().NoError(err) // Create the CRD KDLUserTools in the cluster _, err = apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crdKdlUserTools, metav1.CreateOptions{}) s.Require().NoError(err) +} - // Define the CRD for KDLProject - crdKdlProject := &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kdlprojects.kdl.konstellation.io", // Format: plural.group +var crdKdlProject = &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kdlprojects.kdl.konstellation.io", // Format: plural.group + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "kdl.konstellation.io", + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "kdlprojects", + Singular: "kdlproject", + Kind: "KDLProject", + ListKind: "KDLProjectList", }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "kdl.konstellation.io", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "kdlprojects", - Singular: "kdlproject", - Kind: "KDLProject", - ListKind: "KDLProjectList", - }, - Scope: apiextensionsv1.NamespaceScoped, - Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ - { - Name: "v1", - Served: true, - Storage: true, - Schema: &apiextensionsv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextensionsv1.JSONSchemaProps{ - "spec": { - Type: "object", - Properties: map[string]apiextensionsv1.JSONSchemaProps{ - "projectId": { - Type: "string", + Scope: apiextensionsv1.NamespaceScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "spec": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "inputData": { + Type: "object", + AdditionalProperties: &apiextensionsv1.JSONSchemaPropsOrBool{ + Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "projectId": { + Type: "string", + }, + "mlflow": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "env": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{}, + XPreserveUnknownFields: &preserve, + }, + }, + }, + "filebrowser": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "env": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{}, + XPreserveUnknownFields: &preserve, + }, }, }, - Required: []string{"projectId"}, }, + Required: []string{"projectId"}, }, - Required: []string{"spec"}, }, + Required: []string{"spec"}, }, }, }, }, - } + }, +} + +func (s *testSuite) defineCRDKDLProject(restcfg *rest.Config) { + // Create a clientset for CRD operations + apiExtensionsClient, err := apiextensionsclient.NewForConfig(restcfg) + s.Require().NoError(err) // Create the CRD KDLProject in the cluster _, err = apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crdKdlProject, metav1.CreateOptions{}) @@ -187,27 +277,28 @@ func (s *testSuite) SetupSuite() { dynamicClient, err := dynamic.NewForConfig(restcfg) s.Require().NoError(err) - kdlUserToolsRes := dynamicClient.Resource(schema.GroupVersionResource{ + s.kdlUserToolsRes = dynamicClient.Resource(schema.GroupVersionResource{ Group: kdlUserToolsGroup, Version: kdlUserToolsVersion, Resource: kdlUserToolsResource, }) - kdlProjectRes := dynamicClient.Resource(schema.GroupVersionResource{ + s.kdlProjectRes = dynamicClient.Resource(schema.GroupVersionResource{ Group: kdlProjectGroup, Version: kdlProjectVersion, Resource: kdlProjectResource, }) - s.defineCRD(restcfg) + s.defineCRDKDLProject(restcfg) + s.defineCRDKDLUserTools(restcfg) // Create the client s.Client = k8s.New( logger, cfg, s.Clientset, - kdlUserToolsRes, - kdlProjectRes, + s.kdlUserToolsRes, + s.kdlProjectRes, ) } diff --git a/app/api/infrastructure/minioadminservice/error.go b/app/api/infrastructure/minioadminservice/error.go new file mode 100644 index 000000000..a50cbe23c --- /dev/null +++ b/app/api/infrastructure/minioadminservice/error.go @@ -0,0 +1,5 @@ +package minioadminservice + +import "errors" + +var errEmptySlug = errors.New("user slug can not be empty") diff --git a/app/api/infrastructure/minioadminservice/interface.go b/app/api/infrastructure/minioadminservice/interface.go new file mode 100644 index 000000000..c13ab61a1 --- /dev/null +++ b/app/api/infrastructure/minioadminservice/interface.go @@ -0,0 +1,17 @@ +package minioadminservice + +import "context" + +//go:generate mockgen -source=${GOFILE} -destination=mocks_${GOFILE} -package=${GOPACKAGE} + +// MinioAdminInterface defines all Minio admin operations. +type MinioAdminInterface interface { + CreateUser(ctx context.Context, userSlug, secretKey string) (string, error) + DeleteUser(ctx context.Context, userSlug string) error + CreateProjectPolicy(ctx context.Context, projectName string) error + DeleteProjectPolicy(ctx context.Context, projectName string) error + CreateProjectUser(ctx context.Context, projectName, secretKey string) (string, error) + DeleteProjectUser(ctx context.Context, projectName string) error + JoinProject(ctx context.Context, userSlug, projectName string) error + LeaveProject(ctx context.Context, userSlug, projectName string) error +} diff --git a/app/api/infrastructure/minioadminservice/mocks_interface.go b/app/api/infrastructure/minioadminservice/mocks_interface.go new file mode 100644 index 000000000..b5dab7542 --- /dev/null +++ b/app/api/infrastructure/minioadminservice/mocks_interface.go @@ -0,0 +1,149 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: interface.go + +// Package minioadminservice is a generated GoMock package. +package minioadminservice + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockMinioAdminInterface is a mock of MinioAdminInterface interface. +type MockMinioAdminInterface struct { + ctrl *gomock.Controller + recorder *MockMinioAdminInterfaceMockRecorder +} + +// MockMinioAdminInterfaceMockRecorder is the mock recorder for MockMinioAdminInterface. +type MockMinioAdminInterfaceMockRecorder struct { + mock *MockMinioAdminInterface +} + +// NewMockMinioAdminInterface creates a new mock instance. +func NewMockMinioAdminInterface(ctrl *gomock.Controller) *MockMinioAdminInterface { + mock := &MockMinioAdminInterface{ctrl: ctrl} + mock.recorder = &MockMinioAdminInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMinioAdminInterface) EXPECT() *MockMinioAdminInterfaceMockRecorder { + return m.recorder +} + +// CreateProjectPolicy mocks base method. +func (m *MockMinioAdminInterface) CreateProjectPolicy(ctx context.Context, projectName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateProjectPolicy", ctx, projectName) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateProjectPolicy indicates an expected call of CreateProjectPolicy. +func (mr *MockMinioAdminInterfaceMockRecorder) CreateProjectPolicy(ctx, projectName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateProjectPolicy", reflect.TypeOf((*MockMinioAdminInterface)(nil).CreateProjectPolicy), ctx, projectName) +} + +// CreateProjectUser mocks base method. +func (m *MockMinioAdminInterface) CreateProjectUser(ctx context.Context, projectName, secretKey string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateProjectUser", ctx, projectName, secretKey) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateProjectUser indicates an expected call of CreateProjectUser. +func (mr *MockMinioAdminInterfaceMockRecorder) CreateProjectUser(ctx, projectName, secretKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateProjectUser", reflect.TypeOf((*MockMinioAdminInterface)(nil).CreateProjectUser), ctx, projectName, secretKey) +} + +// CreateUser mocks base method. +func (m *MockMinioAdminInterface) CreateUser(ctx context.Context, userSlug, secretKey string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateUser", ctx, userSlug, secretKey) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateUser indicates an expected call of CreateUser. +func (mr *MockMinioAdminInterfaceMockRecorder) CreateUser(ctx, userSlug, secretKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateUser", reflect.TypeOf((*MockMinioAdminInterface)(nil).CreateUser), ctx, userSlug, secretKey) +} + +// DeleteProjectPolicy mocks base method. +func (m *MockMinioAdminInterface) DeleteProjectPolicy(ctx context.Context, projectName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteProjectPolicy", ctx, projectName) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteProjectPolicy indicates an expected call of DeleteProjectPolicy. +func (mr *MockMinioAdminInterfaceMockRecorder) DeleteProjectPolicy(ctx, projectName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteProjectPolicy", reflect.TypeOf((*MockMinioAdminInterface)(nil).DeleteProjectPolicy), ctx, projectName) +} + +// DeleteProjectUser mocks base method. +func (m *MockMinioAdminInterface) DeleteProjectUser(ctx context.Context, projectName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteProjectUser", ctx, projectName) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteProjectUser indicates an expected call of DeleteProjectUser. +func (mr *MockMinioAdminInterfaceMockRecorder) DeleteProjectUser(ctx, projectName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteProjectUser", reflect.TypeOf((*MockMinioAdminInterface)(nil).DeleteProjectUser), ctx, projectName) +} + +// DeleteUser mocks base method. +func (m *MockMinioAdminInterface) DeleteUser(ctx context.Context, userSlug string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteUser", ctx, userSlug) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteUser indicates an expected call of DeleteUser. +func (mr *MockMinioAdminInterfaceMockRecorder) DeleteUser(ctx, userSlug interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteUser", reflect.TypeOf((*MockMinioAdminInterface)(nil).DeleteUser), ctx, userSlug) +} + +// JoinProject mocks base method. +func (m *MockMinioAdminInterface) JoinProject(ctx context.Context, userSlug, projectName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "JoinProject", ctx, userSlug, projectName) + ret0, _ := ret[0].(error) + return ret0 +} + +// JoinProject indicates an expected call of JoinProject. +func (mr *MockMinioAdminInterfaceMockRecorder) JoinProject(ctx, userSlug, projectName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "JoinProject", reflect.TypeOf((*MockMinioAdminInterface)(nil).JoinProject), ctx, userSlug, projectName) +} + +// LeaveProject mocks base method. +func (m *MockMinioAdminInterface) LeaveProject(ctx context.Context, userSlug, projectName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LeaveProject", ctx, userSlug, projectName) + ret0, _ := ret[0].(error) + return ret0 +} + +// LeaveProject indicates an expected call of LeaveProject. +func (mr *MockMinioAdminInterfaceMockRecorder) LeaveProject(ctx, userSlug, projectName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LeaveProject", reflect.TypeOf((*MockMinioAdminInterface)(nil).LeaveProject), ctx, userSlug, projectName) +} diff --git a/app/api/infrastructure/minioadminservice/service.go b/app/api/infrastructure/minioadminservice/service.go new file mode 100644 index 000000000..cf3dd5c81 --- /dev/null +++ b/app/api/infrastructure/minioadminservice/service.go @@ -0,0 +1,184 @@ +package minioadminservice + +import ( + "bytes" + "context" + "errors" + "fmt" + "text/template" + + "github.com/go-logr/logr" + "github.com/minio/madmin-go/v2" + + _ "embed" +) + +//go:embed templates/policy.go.tmpl +var policyTemplate string + +type MinioAdminService struct { + logger logr.Logger + client *madmin.AdminClient +} + +// Assure implementation adheres to interface. +var _ MinioAdminInterface = (*MinioAdminService)(nil) + +// Constructor for MinioAdminService. +func NewMinioAdminService(logger logr.Logger, endpoint, accessKey, secretKey string) (*MinioAdminService, error) { + client, err := madmin.New(endpoint, accessKey, secretKey, false) + if err != nil { + return nil, err + } + + return &MinioAdminService{logger: logger, client: client}, nil +} + +func (m *MinioAdminService) getUserAccessKey(userSlug string) string { + return fmt.Sprintf("user-%s", userSlug) +} + +func (m *MinioAdminService) getProjectAccessKey(projectName string) string { + return fmt.Sprintf("project-%s", projectName) +} + +func (m *MinioAdminService) CreateUser(ctx context.Context, userSlug, secretKey string) (string, error) { + if userSlug == "" { + return "", errEmptySlug + } + + accessKey := m.getUserAccessKey(userSlug) + + m.logger.Info("Creating user", "userSlug", userSlug, "accessKey", accessKey) + + err := m.client.AddUser(ctx, accessKey, secretKey) + if err != nil { + return "", fmt.Errorf("failed to create user %s, access key %s: %w", userSlug, accessKey, err) + } + + return accessKey, nil +} + +func (m *MinioAdminService) CreateProjectUser(ctx context.Context, projectName, secretKey string) (string, error) { + accessKey := m.getProjectAccessKey(projectName) + + m.logger.Info("Creating project user", "projectName", projectName) + + err := m.client.AddUser(ctx, accessKey, secretKey) + if err != nil { + return "", fmt.Errorf("failed to create project user %s, access key %s: %w", projectName, accessKey, err) + } + + m.logger.Info("Associating user to policy", "accessKey", accessKey, "policyName", projectName) + + err = m.client.SetPolicy(ctx, projectName, accessKey, false) + if err != nil { + return "", fmt.Errorf("failed associate project user %s: %w", projectName, err) + } + + return accessKey, nil +} + +func (m *MinioAdminService) removeUser(ctx context.Context, accessKey string) error { + err := m.client.RemoveUser(ctx, accessKey) + if err != nil { + var target madmin.ErrorResponse + + if errors.As(err, &target) && target.Code == "XMinioAdminNoSuchUser" { + return nil // ignore error for idempotence + } + } + + return err +} + +func (m *MinioAdminService) DeleteUser(ctx context.Context, userSlug string) error { + accessKey := m.getUserAccessKey(userSlug) + + m.logger.Info("Deleting user", "userSlug", userSlug, "accessKey", accessKey) + + return m.removeUser(ctx, accessKey) +} + +func (m *MinioAdminService) DeleteProjectUser(ctx context.Context, projectName string) error { + accessKey := m.getProjectAccessKey(projectName) + + m.logger.Info("Deleting project user", "projectName", projectName, "accessKey", accessKey) + + return m.removeUser(ctx, accessKey) +} + +func (m *MinioAdminService) CreateProjectPolicy(ctx context.Context, projectName string) error { + tmpl, err := template.New("policy").Parse(policyTemplate) + if err != nil { + return fmt.Errorf("failed to parse policy template: %w", err) + } + + var policyBuffer bytes.Buffer + err = tmpl.Execute(&policyBuffer, struct{ BucketName string }{BucketName: projectName}) + + if err != nil { + return fmt.Errorf("failed to apply policy template: %w", err) + } + + m.logger.Info("Updating policy", "projectName", projectName) + + err = m.client.AddCannedPolicy(ctx, projectName, policyBuffer.Bytes()) + if err != nil { + return fmt.Errorf("failed to add policy %s: %w", projectName, err) + } + + return err +} + +func (m *MinioAdminService) DeleteProjectPolicy(ctx context.Context, projectName string) error { + m.logger.Info("Deleting policy", "policyName", projectName) + + err := m.client.RemoveCannedPolicy(ctx, projectName) + if err != nil { + return fmt.Errorf("failed to remove policy %s: %w", projectName, err) + } + + return err +} + +func (m *MinioAdminService) updateProjectMembership(ctx context.Context, userSlug, projectName string, remove bool) error { + accessKey := m.getUserAccessKey(userSlug) + + return m.client.UpdateGroupMembers(ctx, madmin.GroupAddRemove{ + Group: projectName, + IsRemove: remove, + Members: []string{accessKey}, + }) +} + +func (m *MinioAdminService) JoinProject(ctx context.Context, userSlug, projectName string) error { + m.logger.Info("Adding user to project group", "userSlug", userSlug, "projectName", projectName) + + err := m.updateProjectMembership(ctx, userSlug, projectName, false) + if err != nil { + return fmt.Errorf("failed to add user %s to group %s: %w", userSlug, projectName, err) + } + + /* Adding group to policy. This has to be done after adding a user to group, + because the first user creates the group itself. */ + m.logger.Info("Associating homonimous group to policy", "policyName", projectName) + + err = m.client.SetPolicy(ctx, projectName, projectName, true) + if err != nil { + return fmt.Errorf("failed associate group to policy %s: %w", projectName, err) + } + + return err +} + +func (m *MinioAdminService) LeaveProject(ctx context.Context, userSlug, projectName string) error { + m.logger.Info("Removing user from project group", "userSlug", userSlug, "projectName", projectName) + + err := m.updateProjectMembership(ctx, userSlug, projectName, true) + if err != nil { + return fmt.Errorf("failed to remove user %s from group %s: %w", userSlug, projectName, err) + } + + return err +} diff --git a/app/api/infrastructure/minioadminservice/service_test.go b/app/api/infrastructure/minioadminservice/service_test.go new file mode 100644 index 000000000..da751394b --- /dev/null +++ b/app/api/infrastructure/minioadminservice/service_test.go @@ -0,0 +1,306 @@ +//go:build integration + +package minioadminservice_test + +import ( + "bytes" + "context" + "testing" + + "bou.ke/monkey" + "go.uber.org/zap" + + "github.com/go-logr/zapr" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" + "github.com/minio/madmin-go/v2" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/suite" + "github.com/testcontainers/testcontainers-go" + tcminio "github.com/testcontainers/testcontainers-go/modules/minio" +) + +const ( + username = "admin" + password = "admin123" +) + +type TestSuite struct { + suite.Suite + container *tcminio.MinioContainer + adminClient *madmin.AdminClient + client *minio.Client + service *minioadminservice.MinioAdminService +} + +func TestMinioAdminServiceTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (s *TestSuite) SetupSuite() { + ctx := context.Background() + + zapLog, err := zap.NewDevelopment() + s.Require().NoError(err) + + logger := zapr.NewLogger(zapLog) + + s.container, err = tcminio.Run(ctx, "minio/minio:RELEASE.2024-01-16T16-07-38Z", + tcminio.WithUsername(username), tcminio.WithPassword(password)) + s.Require().NoError(err) + + endpoint, err := s.container.ConnectionString(ctx) + s.Require().NoError(err) + + s.service, err = minioadminservice.NewMinioAdminService(logger, endpoint, username, password) + s.Require().NoError(err) + + s.adminClient, err = madmin.New(endpoint, username, password, false) + s.Require().NoError(err) + + s.client, err = minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(username, password, ""), + Secure: false, + }) + s.Require().NoError(err) +} + +func (s *TestSuite) TearDownSuite() { + monkey.UnpatchAll() + s.Require().NoError(testcontainers.TerminateContainer(s.container)) +} + +func (s *TestSuite) TearDownTest() { + ctx := context.Background() + + for _, group := range []string{"project1", "project2"} { + _ = s.adminClient.UpdateGroupMembers(ctx, madmin.GroupAddRemove{ + Group: group, + IsRemove: true, + Members: []string{"user-foo"}, + }) + + // a call to this method on an empty group removes it + _ = s.adminClient.UpdateGroupMembers(ctx, madmin.GroupAddRemove{ + Group: group, + IsRemove: true, + Members: []string{}, + }) + } + + users, err := s.adminClient.ListUsers(ctx) + s.Require().NoError(err) + + for username := range users { + err = s.adminClient.RemoveUser(ctx, username) + s.Require().NoError(err) + } + + err = s.adminClient.RemoveCannedPolicy(ctx, "project1") + s.Require().NoError(err) + + err = s.adminClient.RemoveCannedPolicy(ctx, "project2") + s.Require().NoError(err) + + err = s.client.RemoveObject(ctx, "project1", "hello.txt", minio.RemoveObjectOptions{}) + s.Require().NoError(err) + err = s.client.RemoveBucket(ctx, "project1") + s.Require().NoError(err) +} + +func (s *TestSuite) SetupTest() { + ctx := context.Background() + + err := s.client.MakeBucket(ctx, "project1", minio.MakeBucketOptions{}) + s.Require().NoError(err) + + _, err = s.client.PutObject(ctx, "project1", "hello.txt", + bytes.NewReader([]byte("hello world")), -1, minio.PutObjectOptions{}) + s.Require().NoError(err) +} + +func (s *TestSuite) TestCreateUser() { + ctx := context.Background() + + const ( + // # gitleaks ignore + username string = "foo" + password string = "-i2YaLei0ohwayaes_hz" // gitleaks:allow + ) + + accessKey, err := s.service.CreateUser(ctx, username, password) + s.Require().NoError(err) + s.Require().Equal("user-foo", accessKey) + + users, err := s.adminClient.ListUsers(ctx) + s.Require().NoError(err) + s.Len(users, 1) + _, ok := users[accessKey] + s.Require().True(ok) +} + +func (s *TestSuite) TestDeleteUser() { + ctx := context.Background() + + accessKey, err := s.service.CreateUser(ctx, "foo", "foo12345678") + s.Require().NoError(err) + s.Require().Equal("user-foo", accessKey) + + err = s.service.DeleteUser(ctx, "foo") + s.Require().NoError(err) + + users, err := s.adminClient.ListUsers(ctx) + s.Require().NoError(err) + s.Empty(users) +} + +func (s *TestSuite) TestCreateProjectUser() { + ctx := context.Background() + + const ( + // # gitleaks ignore + projectName string = "trinity" + password string = "-i2YaLei0ohwayaes_hz" // gitleaks:allow + ) + + err := s.service.CreateProjectPolicy(ctx, projectName) + s.Require().NoError(err) + + accessKey, err := s.service.CreateProjectUser(ctx, projectName, password) + s.Require().NoError(err) + s.Require().Equal("project-trinity", accessKey) + + users, err := s.adminClient.ListUsers(ctx) + s.Require().NoError(err) + s.Len(users, 1) + _, ok := users[accessKey] + s.Require().True(ok) +} + +func (s *TestSuite) TestDeleteProjectUser() { + ctx := context.Background() + + const ( + // # gitleaks ignore + projectName string = "trinity" + password string = "-i2YaLei0ohwayaes_hz" // gitleaks:allow + ) + + err := s.service.CreateProjectPolicy(ctx, projectName) + s.Require().NoError(err) + + accessKey, err := s.service.CreateProjectUser(ctx, projectName, password) + s.Require().NoError(err) + s.Require().Equal("project-trinity", accessKey) + + err = s.service.DeleteProjectUser(ctx, projectName) + s.Require().NoError(err) + + users, err := s.adminClient.ListUsers(ctx) + s.Require().NoError(err) + s.Empty(users) +} + +func (s *TestSuite) TestDeleteUserIdempotence() { + ctx := context.Background() + + err := s.service.DeleteUser(ctx, "nonexistent") + s.Require().NoError(err) +} + +func (s *TestSuite) TestAssignProject() { + ctx := context.Background() + + err := s.service.CreateProjectPolicy(ctx, "project1") + s.Require().NoError(err) + + accessKey, err := s.service.CreateUser(ctx, "foo", "foo12345678") + s.Require().NoError(err) + s.Require().Equal("user-foo", accessKey) + + err = s.service.JoinProject(ctx, "foo", "project1") + s.Require().NoError(err) + + // User login + endpoint, err := s.container.ConnectionString(ctx) + s.Require().NoError(err) + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4("user-foo", "foo12345678", ""), + Secure: false, + }) + s.Require().NoError(err) + + // List buckets + buckets, err := client.ListBuckets(ctx) + s.Require().NoError(err) + + s.Len(buckets, 1) + s.Equal("project1", buckets[0].Name) + + // List objects + objectCh := client.ListObjects(ctx, "project1", minio.ListObjectsOptions{}) + objects := make([]minio.ObjectInfo, 0) + + for object := range objectCh { + s.Require().NoError(object.Err) + objects = append(objects, object) + } + + s.Require().NoError(err) + s.Len(objects, 1) + s.Equal("hello.txt", objects[0].Key) +} + +func (s *TestSuite) TestBucketNotAllowed() { + ctx := context.Background() + + err := s.service.CreateProjectPolicy(ctx, "project2") + s.Require().NoError(err) + + accessKey, err := s.service.CreateUser(ctx, "foo", "foo12345678") + s.Require().NoError(err) + s.Require().Equal("user-foo", accessKey) + + err = s.service.JoinProject(ctx, "foo", "project2") + s.Require().NoError(err) + + // User login + endpoint, err := s.container.ConnectionString(ctx) + s.Require().NoError(err) + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4("user-foo", "foo12345678", ""), + Secure: false, + }) + s.Require().NoError(err) + + // List objects shall fail + objectCh := client.ListObjects(ctx, "project1", minio.ListObjectsOptions{}) + object := <-objectCh + s.Require().Error(object.Err) +} + +func (s *TestSuite) TestDeletePolicy() { + ctx := context.Background() + + err := s.service.CreateProjectPolicy(ctx, "project1") + s.Require().NoError(err) + + err = s.service.DeleteProjectPolicy(ctx, "project1") + s.Require().NoError(err) + + policies, err := s.adminClient.ListCannedPolicies(ctx) + s.Require().NoError(err) + + for policy := range policies { + s.Require().NotEqual("project1", policy) + } +} + +func (s *TestSuite) TestDeletePolicyIdempotence() { + ctx := context.Background() + + err := s.service.DeleteProjectPolicy(ctx, "nonexistent") + s.Require().NoError(err) +} diff --git a/app/api/infrastructure/minioadminservice/templates/policy.go.tmpl b/app/api/infrastructure/minioadminservice/templates/policy.go.tmpl new file mode 100644 index 000000000..2263b22ce --- /dev/null +++ b/app/api/infrastructure/minioadminservice/templates/policy.go.tmpl @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::{{ .BucketName }}", + "arn:aws:s3:::{{ .BucketName }}/*" + ] + } + ] +} diff --git a/app/api/infrastructure/mongodb/mongo_test.go b/app/api/infrastructure/mongodb/mongo_test.go index ac165fefd..7086cf85b 100644 --- a/app/api/infrastructure/mongodb/mongo_test.go +++ b/app/api/infrastructure/mongodb/mongo_test.go @@ -23,14 +23,10 @@ import ( ) const ( - dbName = "kdl" - capabilitiesCollName = "capabilities" - projectCollName = "projects" - runtimesCollName = "runtimes" - userActivityCollName = "userActivity" - userCollName = "users" - rootUsername = "root" - rootPassword = "root" + dbName = "kdl" + userCollName = "users" + rootUsername = "root" + rootPassword = "root" ) var ( @@ -77,6 +73,10 @@ var projectExamples = map[string]entity.Project{ AddedDate: testTimeExample, }, }, + MinioAccessKey: entity.MinioAccessKey{ + AccessKey: "project-project1", + SecretKey: "accessKey1", + }, }, "project2": { ID: "2", @@ -99,6 +99,10 @@ var projectExamples = map[string]entity.Project{ AddedDate: testTimeExample, }, }, + MinioAccessKey: entity.MinioAccessKey{ + AccessKey: "project-project2", + SecretKey: "accessKey2", + }, }, } @@ -136,6 +140,10 @@ var userExamples = map[string]entity.User{ Private: "privateKey1", CreationDate: testTimeExample, }, + MinioAccessKey: entity.MinioAccessKey{ + AccessKey: "accessKey1", + SecretKey: "secretKey1", + }, }, "user2": { ID: primitive.NewObjectID().Hex(), diff --git a/app/api/infrastructure/mongodb/project.go b/app/api/infrastructure/mongodb/project.go index 23e18015f..64c68e2da 100644 --- a/app/api/infrastructure/mongodb/project.go +++ b/app/api/infrastructure/mongodb/project.go @@ -5,12 +5,12 @@ import ( "errors" "time" + "github.com/go-logr/logr" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/go-logr/logr" "github.com/konstellation-io/kdl-server/app/api/entity" "github.com/konstellation-io/kdl-server/app/api/pkg/mongodbutils" "github.com/konstellation-io/kdl-server/app/api/usecase/project" @@ -34,6 +34,8 @@ type projectDTO struct { RepoName string `bson:"repo_name"` URL string `bson:"url"` Members []memberDTO `bson:"members"` + MinioAccessKey string `bson:"minio_access_key"` + MinioSecretKey string `bson:"minio_secret_key"` } type ProjectRepo struct { @@ -175,6 +177,13 @@ func (m *ProjectRepo) UpdateArchived(ctx context.Context, projectID string, arch return m.updateProjectFields(ctx, projectID, bson.M{"archived": archived}) } +func (m *ProjectRepo) UpdateMinioAccess(ctx context.Context, projectID, accessKey, secretKey string) error { + return m.updateProjectFields(ctx, projectID, bson.M{ + "minio_access_key": accessKey, + "minio_secret_key": secretKey, + }) +} + func (m *ProjectRepo) DeleteOne(ctx context.Context, projectID string) error { filter := bson.M{ "_id": projectID, @@ -245,6 +254,8 @@ func (m *ProjectRepo) entityToDTO(p entity.Project) (projectDTO, error) { RepoName: p.Repository.RepoName, URL: p.Repository.URL, Archived: p.Archived, + MinioAccessKey: p.MinioAccessKey.AccessKey, + MinioSecretKey: p.MinioAccessKey.SecretKey, } memberDTOS, err := m.membersToDTOs(p.Members) @@ -288,6 +299,10 @@ func (m *ProjectRepo) dtoToEntity(dto projectDTO) entity.Project { RepoName: dto.RepoName, }, Archived: dto.Archived, + MinioAccessKey: entity.MinioAccessKey{ + AccessKey: dto.MinioAccessKey, + SecretKey: dto.MinioSecretKey, + }, } p.Members = make([]entity.Member, len(dto.Members)) diff --git a/app/api/infrastructure/mongodb/user.go b/app/api/infrastructure/mongodb/user.go index 6b8a88a42..e3c6d94a4 100644 --- a/app/api/infrastructure/mongodb/user.go +++ b/app/api/infrastructure/mongodb/user.go @@ -5,12 +5,12 @@ import ( "errors" "time" + "github.com/go-logr/logr" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/go-logr/logr" "github.com/konstellation-io/kdl-server/app/api/entity" "github.com/konstellation-io/kdl-server/app/api/pkg/mongodbutils" "github.com/konstellation-io/kdl-server/app/api/usecase/user" @@ -33,6 +33,8 @@ type userDTO struct { PublicSSHKey string `bson:"public_ssh_key"` PrivateSSHKey string `bson:"private_ssh_key"` SSHKeyCreationDate time.Time `bson:"ssh_key_creation_date"` + MinioAccessKey string `bson:"minio_access_key"` + MinioSecretKey string `bson:"minio_secret_key"` } type UserRepo struct { @@ -198,6 +200,13 @@ func (m *UserRepo) UpdateEmail(ctx context.Context, username, email string) erro return m.updateUserFields(ctx, username, bson.M{"email": email}) } +func (m *UserRepo) UpdateMinioAccess(ctx context.Context, username, accessKey, secretKey string) error { + return m.updateUserFields(ctx, username, bson.M{ + "minio_access_key": accessKey, + "minio_secret_key": secretKey, + }) +} + func (m *UserRepo) UpdateSub(ctx context.Context, username, sub string) error { return m.updateUserFields(ctx, username, bson.M{"sub": sub}) } @@ -290,6 +299,8 @@ func (m *UserRepo) entityToDTO(u entity.User) (userDTO, error) { CreationDate: u.CreationDate, LastActivity: *u.LastActivity, Deleted: u.Deleted, + MinioAccessKey: u.MinioAccessKey.AccessKey, + MinioSecretKey: u.MinioAccessKey.SecretKey, } if u.ID != "" { @@ -319,6 +330,10 @@ func (m *UserRepo) dtoToEntity(dto userDTO) entity.User { CreationDate: dto.SSHKeyCreationDate, }, Deleted: dto.Deleted, + MinioAccessKey: entity.MinioAccessKey{ + AccessKey: dto.MinioAccessKey, + SecretKey: dto.MinioSecretKey, + }, } } diff --git a/app/api/pkg/kdlutil/mocks_random.go b/app/api/pkg/kdlutil/mocks_random.go new file mode 100644 index 000000000..dbbf4c194 --- /dev/null +++ b/app/api/pkg/kdlutil/mocks_random.go @@ -0,0 +1,49 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: random.go + +// Package kdlutil is a generated GoMock package. +package kdlutil + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockRandomGenerator is a mock of RandomGenerator interface. +type MockRandomGenerator struct { + ctrl *gomock.Controller + recorder *MockRandomGeneratorMockRecorder +} + +// MockRandomGeneratorMockRecorder is the mock recorder for MockRandomGenerator. +type MockRandomGeneratorMockRecorder struct { + mock *MockRandomGenerator +} + +// NewMockRandomGenerator creates a new mock instance. +func NewMockRandomGenerator(ctrl *gomock.Controller) *MockRandomGenerator { + mock := &MockRandomGenerator{ctrl: ctrl} + mock.recorder = &MockRandomGeneratorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRandomGenerator) EXPECT() *MockRandomGeneratorMockRecorder { + return m.recorder +} + +// GenerateRandomString mocks base method. +func (m *MockRandomGenerator) GenerateRandomString(n int) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateRandomString", n) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateRandomString indicates an expected call of GenerateRandomString. +func (mr *MockRandomGeneratorMockRecorder) GenerateRandomString(n interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateRandomString", reflect.TypeOf((*MockRandomGenerator)(nil).GenerateRandomString), n) +} diff --git a/app/api/pkg/kdlutil/random.go b/app/api/pkg/kdlutil/random.go index f112dba64..c97ce0737 100644 --- a/app/api/pkg/kdlutil/random.go +++ b/app/api/pkg/kdlutil/random.go @@ -5,11 +5,22 @@ import ( "math/big" ) +//go:generate mockgen -source=${GOFILE} -destination=mocks_${GOFILE} -package=${GOPACKAGE} +type RandomGenerator interface { + GenerateRandomString(n int) (string, error) +} + +type RandomGeneratorImplementation struct{} + +func NewRandomGenerator() *RandomGeneratorImplementation { + return &RandomGeneratorImplementation{} +} + // GenerateRandomString returns a securely generated random string. // It will return an error if the system's secure random // number generator fails to function correctly, in which // case the caller should not continue. -func GenerateRandomString(n int) (string, error) { +func (r *RandomGeneratorImplementation) GenerateRandomString(n int) (string, error) { const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-" ret := make([]byte, n) diff --git a/app/api/scripts/minio/sync.go b/app/api/scripts/minio/sync.go new file mode 100644 index 000000000..21b9c6cc4 --- /dev/null +++ b/app/api/scripts/minio/sync.go @@ -0,0 +1,115 @@ +package minio + +import ( + "context" + + "github.com/go-logr/logr" + + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" + "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" + "github.com/konstellation-io/kdl-server/app/api/usecase/project" + "github.com/konstellation-io/kdl-server/app/api/usecase/user" +) + +func SyncUsers( + logger logr.Logger, + userRepo user.Repository, + minioAdminService minioadminservice.MinioAdminInterface, + randomGenerator kdlutil.RandomGenerator, +) error { + ctx := context.Background() + // find all users in the database and create MinIO Users + users, err := userRepo.FindAll(ctx, false) + if err != nil { + logger.Error(err, "Error finding all users") + return err + } + + for _, u := range users { + // Generate a secret key for the Minio User + secretKey, err := randomGenerator.GenerateRandomString(40) + if err != nil { + logger.Error(err, "Error creating a MinIO User secret key", "username", u.Username) + return err + } + + // create the User in MinIO + accessKey, err := minioAdminService.CreateUser(ctx, u.UsernameSlug(), secretKey) + if err != nil { + logger.Error(err, "Error creating a MinIO User", "username", u.Username) + return err + } + + // Update MinIO User in database + err = userRepo.UpdateMinioAccess(ctx, u.Username, accessKey, secretKey) + if err != nil { + logger.Error(err, "Error updating user with MinIO access", "username", u.Username) + return err + } + } + + return nil +} + +func SyncProjects( + logger logr.Logger, + projectRepo project.Repository, + userRepo user.Repository, + minioAdminService minioadminservice.MinioAdminInterface, + randomGenerator kdlutil.RandomGenerator, +) error { + ctx := context.Background() + // find all projects in database and create MinIO Projects and Policies + projects, err := projectRepo.FindAll(ctx) + if err != nil { + logger.Error(err, "Error finding all projects") + return err + } + + for _, p := range projects { + // Create MinIO Policy for the Project + err = minioAdminService.CreateProjectPolicy(ctx, p.ID) + if err != nil { + logger.Error(err, "Error creating a MinIO Project Policy", "projectID", p.ID) + return err + } + + // Generate a secret key for the MinIO project user + secretKey, err := randomGenerator.GenerateRandomString(40) + if err != nil { + logger.Error(err, "Error creating a MinIO Project User secret key", "projectID", p.ID) + return err + } + + // Create MinIO Project User + accessKey, err := minioAdminService.CreateProjectUser(ctx, p.ID, secretKey) + if err != nil { + logger.Error(err, "Error creating a MinIO Project User", "projectID", p.ID) + return err + } + + // Update Project in database + err = projectRepo.UpdateMinioAccess(ctx, p.ID, accessKey, secretKey) + if err != nil { + logger.Error(err, "Error updating project with MinIO access", "projectID", p.ID) + return err + } + + // Join all project members to the project + for _, member := range p.Members { + u, err := userRepo.Get(ctx, member.UserID) + if err != nil { + logger.Error(err, "Error getting user", "userID", member.UserID) + return err + } + + err = minioAdminService.JoinProject(ctx, u.UsernameSlug(), p.ID) + if err != nil { + logger.Error(err, "Error joining user to project", "username", u.Username, "projectID", p.ID) + return err + } + } + } + + return nil +} diff --git a/app/api/scripts/minio/sync_test.go b/app/api/scripts/minio/sync_test.go new file mode 100644 index 000000000..e4cbf3ed1 --- /dev/null +++ b/app/api/scripts/minio/sync_test.go @@ -0,0 +1,191 @@ +//go:build integration + +package minio_test + +import ( + "context" + "fmt" + "testing" + "time" + + "bou.ke/monkey" + "github.com/go-logr/logr" + "github.com/go-logr/zapr" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/suite" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.uber.org/zap" + + "github.com/konstellation-io/kdl-server/app/api/entity" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/mongodb" + "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" + "github.com/konstellation-io/kdl-server/app/api/pkg/mongodbutils" + "github.com/konstellation-io/kdl-server/app/api/scripts/minio" +) + +const ( + dbName = "kdl" + rootUsername = "root" + rootPassword = "root" +) + +var ( + userID = primitive.NewObjectID().Hex() + testTimeExample = time.Now().Add(-time.Hour).Truncate(time.Millisecond).UTC() + userEntity = entity.User{ + ID: userID, + Email: "email1", + Username: "user1", + Sub: "d5d70477-5192-4182-b80e-5d34550eb4fe", + LastActivity: &testTimeExample, + } + projectEntity = entity.Project{ + ID: primitive.NewObjectID().Hex(), + Name: "project1", + Description: "description1", + Members: []entity.Member{ + { + UserID: userID, + }, + }, + } +) + +type TestSuite struct { + suite.Suite + logger logr.Logger + mongoDBContainer testcontainers.Container + mongoClient *mongodbutils.MongoDB + projectRepo *mongodb.ProjectRepo + userRepo *mongodb.UserRepo + randomGenerator *kdlutil.RandomGeneratorImplementation + minioAdminService *minioadminservice.MockMinioAdminInterface +} + +func TestSyncMinioTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (s *TestSuite) SetupSuite() { + ctx := context.Background() + + zapLog, err := zap.NewDevelopment() + s.Require().NoError(err) + + s.logger = zapr.NewLogger(zapLog) + + req := testcontainers.ContainerRequest{ + Image: "mongo:8.0", + ExposedPorts: []string{"27017/tcp", "27018/tcp"}, + Env: map[string]string{ + "MONGO_INITDB_ROOT_USERNAME": rootUsername, + "MONGO_INITDB_ROOT_PASSWORD": rootPassword, + }, + WaitingFor: wait.ForLog("MongoDB starting"), + } + + mongoDBContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + s.Require().NoError(err) + + host, err := mongoDBContainer.Host(context.Background()) + s.Require().NoError(err) + p, err := mongoDBContainer.MappedPort(context.Background(), "27017/tcp") + s.Require().NoError(err) + + port := p.Int() + uri := fmt.Sprintf("mongodb://%v:%v@%v:%v/", rootUsername, rootPassword, host, port) // NOSONAR not used in secure contexts + mongoClient, err := mongodbutils.NewMongoDB(s.logger, uri) + s.Require().NoError(err) + + s.mongoDBContainer = mongoDBContainer + s.mongoClient = mongoClient + s.projectRepo = mongodb.NewProjectRepo(s.logger, mongoClient, dbName) + s.userRepo = mongodb.NewUserRepo(s.logger, mongoClient, dbName) + + // set random generator + s.randomGenerator = kdlutil.NewRandomGenerator() + + ctrl := gomock.NewController(s.T()) + defer ctrl.Finish() + + s.minioAdminService = minioadminservice.NewMockMinioAdminInterface(ctrl) + + monkey.Patch(time.Now, func() time.Time { + return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC) + }) +} + +func (s *TestSuite) TearDownSuite() { + monkey.UnpatchAll() + s.Require().NoError(s.mongoDBContainer.Terminate(context.Background())) +} + +func (s *TestSuite) TearDownTest() { + err := s.mongoClient.DropDatabase(dbName) + s.Require().NoError(err) +} + +func (s *TestSuite) TestSyncUsers() { + // assert that MinIO Access and Secret keys are empty + s.Require().Empty(userEntity.MinioAccessKey.AccessKey) + s.Require().Empty(userEntity.MinioAccessKey.SecretKey) + + ctx := context.Background() + + _, err := s.userRepo.Create(ctx, userEntity) + s.Require().NoError(err) + + s.minioAdminService.EXPECT().CreateUser(ctx, gomock.Any(), gomock.Any()).Return(gomock.Any().String(), nil) + + err = minio.SyncUsers(s.logger, s.userRepo, s.minioAdminService, s.randomGenerator) + s.Require().NoError(err) + + // Assert minIO Access and Secret keys were created for user + u, err := s.userRepo.Get(ctx, userEntity.ID) + s.Require().NoError(err) + s.Require().NotEmpty(u.MinioAccessKey.AccessKey) + s.Require().NotEmpty(u.MinioAccessKey.SecretKey) +} + +func (s *TestSuite) TestSyncUsers_NoUsers() { + err := minio.SyncUsers(s.logger, s.userRepo, s.minioAdminService, s.randomGenerator) + s.Require().NoError(err) +} + +func (s *TestSuite) TestSyncProjects() { + // assert that MinIO Access and Secret keys are empty + s.Require().Empty(projectEntity.MinioAccessKey.AccessKey) + s.Require().Empty(projectEntity.MinioAccessKey.SecretKey) + + ctx := context.Background() + + _, err := s.projectRepo.Create(ctx, projectEntity) + s.Require().NoError(err) + + _, err = s.userRepo.Create(ctx, userEntity) + s.Require().NoError(err) + + s.minioAdminService.EXPECT().CreateProjectPolicy(ctx, projectEntity.ID).Return(nil) + s.minioAdminService.EXPECT().CreateProjectUser(ctx, projectEntity.ID, gomock.Any()).Return(gomock.Any().String(), nil) + s.minioAdminService.EXPECT().JoinProject(ctx, gomock.Any(), projectEntity.ID).Return(nil) + + err = minio.SyncProjects(s.logger, s.projectRepo, s.userRepo, s.minioAdminService, s.randomGenerator) + s.Require().NoError(err) + + // Assert minIO Access and Secret keys were created for project + p, err := s.projectRepo.Get(ctx, projectEntity.ID) + s.Require().NoError(err) + s.Require().NotEmpty(p.MinioAccessKey.AccessKey) + s.Require().NotEmpty(p.MinioAccessKey.SecretKey) +} + +func (s *TestSuite) TestSyncProjects_NoProjects() { + err := minio.SyncProjects(s.logger, s.projectRepo, s.userRepo, s.minioAdminService, s.randomGenerator) + s.Require().NoError(err) +} diff --git a/app/api/scripts/sync_minio_data.go b/app/api/scripts/sync_minio_data.go new file mode 100644 index 000000000..cf11d6f64 --- /dev/null +++ b/app/api/scripts/sync_minio_data.go @@ -0,0 +1,67 @@ +package main + +import ( + "log" + "os" + + "github.com/go-logr/zapr" + "go.uber.org/zap" + + "github.com/konstellation-io/kdl-server/app/api/infrastructure/config" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/mongodb" + "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" + "github.com/konstellation-io/kdl-server/app/api/pkg/mongodbutils" + "github.com/konstellation-io/kdl-server/app/api/scripts/minio" +) + +func main() { + // Load configuration + cfg := config.NewConfig() + + // Setup zap logger + zapLog, err := zap.NewDevelopment() + if err != nil { + log.Fatal(err) + } + + logger := zapr.NewLogger(zapLog) + + // set random generator + randomGenerator := kdlutil.NewRandomGenerator() + + // set minio admin service + minioAdminService, err := minioadminservice.NewMinioAdminService( + logger, cfg.Minio.Endpoint, cfg.Minio.AccessKey, cfg.Minio.SecretKey, + ) + if err != nil { + logger.Error(err, "Error connecting to Minio for administration") + os.Exit(1) + } + + // set mongodb client + mongodbClient, err := mongodbutils.NewMongoDB(logger, cfg.MongoDB.URI) + if err != nil { + logger.Error(err, "Error connecting to MongoDB") + os.Exit(1) + } + defer mongodbClient.Disconnect() + + // set project and user repositories + projectRepo := mongodb.NewProjectRepo(logger, mongodbClient, cfg.MongoDB.DBName) + userRepo := mongodb.NewUserRepo(logger, mongodbClient, cfg.MongoDB.DBName) + + err = minio.SyncUsers(logger, userRepo, minioAdminService, randomGenerator) + if err != nil { + logger.Error(err, "Error syncing users") + + defer os.Exit(1) + } + + err = minio.SyncProjects(logger, projectRepo, userRepo, minioAdminService, randomGenerator) + if err != nil { + logger.Error(err, "Error syncing projects") + + defer os.Exit(1) + } +} diff --git a/app/api/usecase/capabilities/interactor.go b/app/api/usecase/capabilities/interactor.go index 681a4382e..756dda1d8 100644 --- a/app/api/usecase/capabilities/interactor.go +++ b/app/api/usecase/capabilities/interactor.go @@ -16,6 +16,9 @@ type interactor struct { k8sClient k8s.ClientInterface } +// Assure implementation adheres to interface. +var _ UseCase = (*interactor)(nil) + // NewInteractor factory function. func NewInteractor( logger logr.Logger, diff --git a/app/api/usecase/configmap/interactor.go b/app/api/usecase/configmap/interactor.go index 44c0a1531..67bb6827c 100644 --- a/app/api/usecase/configmap/interactor.go +++ b/app/api/usecase/configmap/interactor.go @@ -21,6 +21,9 @@ type interactor struct { user user.UseCase } +// Assure implementation adheres to interface. +var _ UseCase = (*interactor)(nil) + // NewInteractor factory function. func NewInteractor( logger logr.Logger, diff --git a/app/api/usecase/project/interactor.go b/app/api/usecase/project/interactor.go index b82136576..625cc830a 100644 --- a/app/api/usecase/project/interactor.go +++ b/app/api/usecase/project/interactor.go @@ -11,6 +11,7 @@ import ( "github.com/go-logr/logr" "github.com/konstellation-io/kdl-server/app/api/entity" "github.com/konstellation-io/kdl-server/app/api/infrastructure/k8s" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioservice" "github.com/konstellation-io/kdl-server/app/api/pkg/clock" "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" @@ -87,30 +88,39 @@ func (c CreateProjectOption) Validate() error { // interactor implements the UseCase interface. type interactor struct { - logger logr.Logger - projectRepo Repository - userActivityRepo UserActivityRepo - clock clock.Clock - minioService minioservice.MinioService - k8sClient k8s.ClientInterface + logger logr.Logger + projectRepo Repository + userActivityRepo UserActivityRepo + clock clock.Clock + minioService minioservice.MinioService + minioAdminService minioadminservice.MinioAdminInterface + k8sClient k8s.ClientInterface + randomGenerator kdlutil.RandomGenerator } +// Assure implementation adheres to interface. +var _ UseCase = (*interactor)(nil) + // NewInteractor is a constructor function. func NewInteractor( logger logr.Logger, k8sClient k8s.ClientInterface, minioService minioservice.MinioService, + minioAdminService minioadminservice.MinioAdminInterface, realClock clock.Clock, projectRepo Repository, userActivityRepo UserActivityRepo, + randomGenerator kdlutil.RandomGenerator, ) UseCase { return &interactor{ - logger: logger, - projectRepo: projectRepo, - userActivityRepo: userActivityRepo, - clock: realClock, - minioService: minioService, - k8sClient: k8sClient, + logger: logger, + projectRepo: projectRepo, + userActivityRepo: userActivityRepo, + clock: realClock, + minioService: minioService, + minioAdminService: minioAdminService, + k8sClient: k8sClient, + randomGenerator: randomGenerator, } } @@ -121,6 +131,8 @@ Depending on the repository type: - Create a k8s KDLProject containing a MLFLow instance - Create Minio bucket - Create Minio folders + - Create Minio project user + - Create Minio policy for the project user */ func (i *interactor) Create(ctx context.Context, opt CreateProjectOption) (entity.Project, error) { // Validate the creation input @@ -131,6 +143,12 @@ func (i *interactor) Create(ctx context.Context, opt CreateProjectOption) (entit now := i.clock.Now() + // Generate a secret key for the Minio project user + secretKey, err := i.randomGenerator.GenerateRandomString(40) + if err != nil { + return entity.Project{}, err + } + project := entity.NewProject(opt.ProjectID, opt.Name, opt.Description) project.CreationDate = now project.Members = []entity.Member{ @@ -147,12 +165,6 @@ func (i *interactor) Create(ctx context.Context, opt CreateProjectOption) (entit RepoName: opt.ProjectID, } - // Create a k8s KDLProject containing a MLFLow instance - err = i.k8sClient.CreateKDLProjectCR(ctx, opt.ProjectID) - if err != nil { - return entity.Project{}, err - } - // Create Minio bucket err = i.minioService.CreateBucket(ctx, opt.ProjectID) if err != nil { @@ -165,6 +177,29 @@ func (i *interactor) Create(ctx context.Context, opt CreateProjectOption) (entit return entity.Project{}, err } + // Create Minio policy for the project user + err = i.minioAdminService.CreateProjectPolicy(ctx, opt.ProjectID) + if err != nil { + return entity.Project{}, err + } + + // Create Minio project user + accessKey, err := i.minioAdminService.CreateProjectUser(ctx, opt.ProjectID, secretKey) + if err != nil { + return entity.Project{}, err + } + + project.MinioAccessKey = entity.MinioAccessKey{ + AccessKey: accessKey, + SecretKey: secretKey, + } + + // Create a k8s KDLProject containing a MLFLow instance + err = i.k8sClient.CreateKDLProjectCR(ctx, k8s.ProjectData{ProjectID: opt.ProjectID, MinioAccessKey: project.MinioAccessKey}) + if err != nil { + return entity.Project{}, err + } + // Store the project into the database insertedID, err := i.projectRepo.Create(ctx, project) if err != nil { @@ -290,6 +325,19 @@ func (i *interactor) Delete(ctx context.Context, opt DeleteProjectOption) (*enti return nil, err } + // Determine policy/user name + accessKey := fmt.Sprintf("project-%s", projectID) + + err = i.minioAdminService.DeleteProjectPolicy(ctx, accessKey) + if err != nil { + return nil, err + } + + err = i.minioAdminService.DeleteUser(ctx, accessKey) + if err != nil { + return nil, err + } + err = i.projectRepo.DeleteOne(ctx, projectID) if err != nil { return nil, err diff --git a/app/api/usecase/project/interactor_members.go b/app/api/usecase/project/interactor_members.go index 215bcd191..31f1a8b60 100644 --- a/app/api/usecase/project/interactor_members.go +++ b/app/api/usecase/project/interactor_members.go @@ -73,6 +73,14 @@ func (i *interactor) AddMembers(ctx context.Context, opt AddMembersOption) (enti } } + // Add new members to the project on MinIO + for _, u := range opt.Users { + err = i.minioAdminService.JoinProject(ctx, u.UsernameSlug(), opt.ProjectID) + if err != nil { + return entity.Project{}, fmt.Errorf("%w: user ID=%s", err, u.ID) + } + } + // Store new members into the DataBase now := i.clock.Now() newMembers := make([]entity.Member, len(opt.Users)) @@ -143,6 +151,14 @@ func (i *interactor) RemoveMembers(ctx context.Context, opt RemoveMembersOption) return entity.Project{}, ErrRemoveNoMoreAdmins } + // Remove members from the project on MinIO + for _, u := range opt.Users { + err = i.minioAdminService.LeaveProject(ctx, u.UsernameSlug(), opt.ProjectID) + if err != nil { + return entity.Project{}, fmt.Errorf("%w: user ID=%s", err, u.ID) + } + } + // Remove members from stored project in our DataBase err = i.projectRepo.RemoveMembers(ctx, opt.ProjectID, opt.Users) if err != nil { diff --git a/app/api/usecase/project/interactor_test.go b/app/api/usecase/project/interactor_test.go index 5a0c6d087..794c3995d 100644 --- a/app/api/usecase/project/interactor_test.go +++ b/app/api/usecase/project/interactor_test.go @@ -18,8 +18,10 @@ import ( "github.com/konstellation-io/kdl-server/app/api/entity" "github.com/konstellation-io/kdl-server/app/api/infrastructure/k8s" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioservice" "github.com/konstellation-io/kdl-server/app/api/pkg/clock" + "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" "github.com/konstellation-io/kdl-server/app/api/usecase/project" ) @@ -41,12 +43,14 @@ type projectSuite struct { } type projectMocks struct { - repo *project.MockRepository - userActivityRepo *project.MockUserActivityRepo - clock *clock.MockClock - minioService *minioservice.MockMinioService - k8sClient *k8s.MockClientInterface - logger logr.Logger + repo *project.MockRepository + userActivityRepo *project.MockUserActivityRepo + clock *clock.MockClock + minioService *minioservice.MockMinioService + minioAdminService *minioadminservice.MockMinioAdminInterface + k8sClient *k8s.MockClientInterface + logger logr.Logger + randomGenerator *kdlutil.MockRandomGenerator } func newProjectSuite(t *testing.T) *projectSuite { @@ -55,25 +59,29 @@ func newProjectSuite(t *testing.T) *projectSuite { userActivityRepo := project.NewMockUserActivityRepo(ctrl) clockMock := clock.NewMockClock(ctrl) minioService := minioservice.NewMockMinioService(ctrl) + minioAdminService := minioadminservice.NewMockMinioAdminInterface(ctrl) k8sClient := k8s.NewMockClientInterface(ctrl) + randomGenerator := kdlutil.NewMockRandomGenerator(ctrl) zapLog, err := zap.NewDevelopment() require.NoError(t, err) logger := zapr.NewLogger(zapLog) - interactor := project.NewInteractor(logger, k8sClient, minioService, clockMock, repo, userActivityRepo) + interactor := project.NewInteractor(logger, k8sClient, minioService, minioAdminService, clockMock, repo, userActivityRepo, randomGenerator) return &projectSuite{ ctrl: ctrl, interactor: interactor, mocks: projectMocks{ - logger: logger, - repo: repo, - userActivityRepo: userActivityRepo, - clock: clockMock, - minioService: minioService, - k8sClient: k8sClient, + logger: logger, + repo: repo, + userActivityRepo: userActivityRepo, + clock: clockMock, + minioService: minioService, + minioAdminService: minioAdminService, + k8sClient: k8sClient, + randomGenerator: randomGenerator, }, } } @@ -83,10 +91,12 @@ func TestInteractor_Create(t *testing.T) { defer s.ctrl.Finish() const ( - projectName = "The Project Y" - projectDesc = "The Project Y Description" - ownerUserID = "user.1234" - ownerUsername = "john" + projectName = "The Project Y" + projectDesc = "The Project Y Description" + projectMinioAccessKey = "project-test-project" // derived from project ID + projectMinioSecretKey = "projectY123" + ownerUserID = "user.1234" + ownerUsername = "john" ) url := "https://github.com/org/repo.git" @@ -108,6 +118,10 @@ func TestInteractor_Create(t *testing.T) { URL: url, RepoName: testProjectID, } + createProject.MinioAccessKey = entity.MinioAccessKey{ + AccessKey: projectMinioAccessKey, + SecretKey: projectMinioSecretKey, + } expectedProject := entity.Project{ ID: testProjectID, @@ -118,14 +132,22 @@ func TestInteractor_Create(t *testing.T) { URL: url, RepoName: testProjectID, }, + MinioAccessKey: entity.MinioAccessKey{ + AccessKey: projectMinioAccessKey, + SecretKey: projectMinioSecretKey, + }, } - s.mocks.k8sClient.EXPECT().CreateKDLProjectCR(ctx, testProjectID).Return(nil) + s.mocks.k8sClient.EXPECT().CreateKDLProjectCR(ctx, + k8s.ProjectData{ProjectID: testProjectID, MinioAccessKey: createProject.MinioAccessKey}).Return(nil) s.mocks.minioService.EXPECT().CreateBucket(ctx, testProjectID).Return(nil) s.mocks.minioService.EXPECT().CreateProjectDirs(ctx, testProjectID).Return(nil) s.mocks.clock.EXPECT().Now().Return(now) s.mocks.repo.EXPECT().Create(ctx, createProject).Return(testProjectID, nil) s.mocks.repo.EXPECT().Get(ctx, testProjectID).Return(expectedProject, nil) + s.mocks.randomGenerator.EXPECT().GenerateRandomString(40).Return(projectMinioSecretKey, nil) + s.mocks.minioAdminService.EXPECT().CreateProjectUser(ctx, testProjectID, projectMinioSecretKey).Return(projectMinioAccessKey, nil) + s.mocks.minioAdminService.EXPECT().CreateProjectPolicy(ctx, testProjectID).Return(nil) createdProject, err := s.interactor.Create(ctx, project.CreateProjectOption{ ProjectID: testProjectID, @@ -253,6 +275,8 @@ func TestInteractor_AddMembers(t *testing.T) { }, ).Return(nil) s.mocks.repo.EXPECT().Get(ctx, p.ID).Return(expectedProject, nil) + s.mocks.minioAdminService.EXPECT().JoinProject(ctx, "user-a", testProjectID).Return(nil) + s.mocks.minioAdminService.EXPECT().JoinProject(ctx, "user-b", testProjectID).Return(nil) p, err := s.interactor.AddMembers(ctx, project.AddMembersOption{ ProjectID: p.ID, @@ -345,6 +369,8 @@ func TestInteractor_RemoveMembers(t *testing.T) { }, ).Return(nil) s.mocks.repo.EXPECT().Get(ctx, p.ID).Return(expectedProject, nil) + s.mocks.minioAdminService.EXPECT().LeaveProject(ctx, "user-a", testProjectID).Return(nil) + s.mocks.minioAdminService.EXPECT().LeaveProject(ctx, "user-b", testProjectID).Return(nil) p, err := s.interactor.RemoveMembers(ctx, project.RemoveMembersOption{ ProjectID: p.ID, @@ -625,6 +651,10 @@ func TestInteractor_Delete(t *testing.T) { s := newProjectSuite(t) defer s.ctrl.Finish() + const ( + accessKey = "project-test-project" + ) + monkey.Patch(time.Now, func() time.Time { return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC) }) @@ -677,6 +707,9 @@ func TestInteractor_Delete(t *testing.T) { s.mocks.k8sClient.EXPECT().DeleteKDLProjectCR(ctx, testProjectID).Return(nil) s.mocks.repo.EXPECT().DeleteOne(ctx, testProjectID).Return(nil) s.mocks.minioService.EXPECT().DeleteBucket(ctx, testProjectID).Return(expectedMinioBackup, nil) + s.mocks.minioAdminService.EXPECT().DeleteProjectPolicy(ctx, accessKey).Return(nil) + s.mocks.minioAdminService.EXPECT().DeleteUser(ctx, accessKey).Return(nil) + s.mocks.userActivityRepo.EXPECT().Create(ctx, userActivity).Return(nil) result, err := s.interactor.Delete(ctx, project.DeleteProjectOption{ diff --git a/app/api/usecase/project/interface.go b/app/api/usecase/project/interface.go index 959424675..6f3db4443 100644 --- a/app/api/usecase/project/interface.go +++ b/app/api/usecase/project/interface.go @@ -19,6 +19,7 @@ type Repository interface { UpdateName(ctx context.Context, projectID, name string) error UpdateDescription(ctx context.Context, projectID, description string) error UpdateArchived(ctx context.Context, projectID string, archived bool) error + UpdateMinioAccess(ctx context.Context, projectID, accessKey, secretKey string) error DeleteOne(ctx context.Context, projectID string) error } diff --git a/app/api/usecase/project/mocks_interface.go b/app/api/usecase/project/mocks_interface.go index 2bd6d6943..26146b434 100644 --- a/app/api/usecase/project/mocks_interface.go +++ b/app/api/usecase/project/mocks_interface.go @@ -164,6 +164,20 @@ func (mr *MockRepositoryMockRecorder) UpdateMembersAccessLevel(ctx, projectID, u return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMembersAccessLevel", reflect.TypeOf((*MockRepository)(nil).UpdateMembersAccessLevel), ctx, projectID, users, accessLevel) } +// UpdateMinioAccess mocks base method. +func (m *MockRepository) UpdateMinioAccess(ctx context.Context, projectID, accessKey, secretKey string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateMinioAccess", ctx, projectID, accessKey, secretKey) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateMinioAccess indicates an expected call of UpdateMinioAccess. +func (mr *MockRepositoryMockRecorder) UpdateMinioAccess(ctx, projectID, accessKey, secretKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMinioAccess", reflect.TypeOf((*MockRepository)(nil).UpdateMinioAccess), ctx, projectID, accessKey, secretKey) +} + // UpdateName mocks base method. func (m *MockRepository) UpdateName(ctx context.Context, projectID, name string) error { m.ctrl.T.Helper() diff --git a/app/api/usecase/runtime/interactor.go b/app/api/usecase/runtime/interactor.go index 36689f567..5e697fcc8 100644 --- a/app/api/usecase/runtime/interactor.go +++ b/app/api/usecase/runtime/interactor.go @@ -15,6 +15,9 @@ type interactor struct { repo Repository } +// Assure implementation adheres to interface. +var _ UseCase = (*interactor)(nil) + // NewInteractor factory function. func NewInteractor( logger logr.Logger, diff --git a/app/api/usecase/user/interactor.go b/app/api/usecase/user/interactor.go index 87e330038..76a331361 100644 --- a/app/api/usecase/user/interactor.go +++ b/app/api/usecase/user/interactor.go @@ -12,6 +12,7 @@ import ( "github.com/konstellation-io/kdl-server/app/api/entity" "github.com/konstellation-io/kdl-server/app/api/infrastructure/config" "github.com/konstellation-io/kdl-server/app/api/infrastructure/k8s" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" "github.com/konstellation-io/kdl-server/app/api/pkg/clock" "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" "github.com/konstellation-io/kdl-server/app/api/pkg/sshhelper" @@ -21,21 +22,22 @@ import ( ) var ( - ErrStopUserTools = errors.New("cannot stop uninitialized user tools") - ErrUserToolsActive = errors.New("it is not possible to regenerate SSH keys with the usertools active") - errCreatingKDLUserTools = errors.New("error creating CRD KDLUserTools ") + ErrStopUserTools = errors.New("cannot stop uninitialized user tools") + ErrUserToolsActive = errors.New("it is not possible to regenerate SSH keys with the usertools active") ) type Interactor struct { - logger logr.Logger - cfg config.Config - repo Repository - userActivityRepo project.UserActivityRepo - repoRuntimes runtime.Repository - repoCapabilities capabilities.Repository - sshGenerator sshhelper.SSHKeyGenerator - clock clock.Clock - k8sClient k8s.ClientInterface + logger logr.Logger + cfg config.Config + repo Repository + userActivityRepo project.UserActivityRepo + repoRuntimes runtime.Repository + repoCapabilities capabilities.Repository + sshGenerator sshhelper.SSHKeyGenerator + clock clock.Clock + k8sClient k8s.ClientInterface + minioAdminService minioadminservice.MinioAdminInterface + randomGenerator kdlutil.RandomGenerator } // Interactor implements the UseCase interface. @@ -52,17 +54,21 @@ func NewInteractor( sshGenerator sshhelper.SSHKeyGenerator, c clock.Clock, k8sClient k8s.ClientInterface, + minioAdminService minioadminservice.MinioAdminInterface, + randomGenerator kdlutil.RandomGenerator, ) UseCase { return &Interactor{ - logger: logger, - cfg: cfg, - repo: repo, - userActivityRepo: userActivityRepo, - repoRuntimes: repoRuntimes, - repoCapabilities: repoCapabilities, - sshGenerator: sshGenerator, - clock: c, - k8sClient: k8sClient, + logger: logger, + cfg: cfg, + repo: repo, + userActivityRepo: userActivityRepo, + repoRuntimes: repoRuntimes, + repoCapabilities: repoCapabilities, + sshGenerator: sshGenerator, + clock: c, + k8sClient: k8sClient, + minioAdminService: minioAdminService, + randomGenerator: randomGenerator, } } @@ -148,22 +154,36 @@ func (i *Interactor) Create(ctx context.Context, email, sub string, accessLevel } // Created a service account for the user - _, err = i.k8sClient.CreateUserServiceAccount(ctx, user.UsernameSlug()) + slug := user.UsernameSlug() + + _, err = i.k8sClient.CreateUserServiceAccount(ctx, slug) if err != nil { i.logger.Error(err, "Error creating service account", "username", username) return entity.User{}, err } + user.MinioAccessKey.SecretKey, err = i.randomGenerator.GenerateRandomString(40) + if err != nil { + i.logger.Error(err, "Error creating a MinIO secret key", "username", username) + return entity.User{}, err + } + + user.MinioAccessKey.AccessKey, err = i.minioAdminService.CreateUser(ctx, slug, user.MinioAccessKey.SecretKey) + if err != nil { + i.logger.Error(err, "Error creating a MinIO user", "accessKey", user.MinioAccessKey.AccessKey) + return entity.User{}, err + } + // Save user creation in user activity createUserActVars := entity.NewActivityVarsWithUserID(insertedID) - createUserdAct := entity.UserActivity{ + createUserAct := entity.UserActivity{ Date: i.clock.Now(), UserID: insertedID, Type: entity.UserActivityTypeCreateUser, Vars: createUserActVars, } - err = i.SaveUserActivity(ctx, createUserdAct) + err = i.SaveUserActivity(ctx, createUserAct) if err != nil { return entity.User{}, err } @@ -232,7 +252,10 @@ func (i *Interactor) StartTools(ctx context.Context, email string, runtimeID, ca i.logger.Info("Creating user tools for user", "email", email) - err = i.k8sClient.CreateKDLUserToolsCR(ctx, user.Username, data) + data.Username = user.Username + data.SlugUsername = user.UsernameSlug() + + err = i.k8sClient.CreateKDLUserToolsCR(ctx, data) if err != nil { return entity.User{}, err } @@ -440,48 +463,7 @@ func (i *Interactor) UpdateKDLUserTools(ctx context.Context) error { for _, userTool := range kdlUserTools { resourceName := userTool.GetName() - spec, ok := userTool.Object["spec"].(map[string]interface{}) - if !ok { - i.logger.Error(errCreatingKDLUserTools, "Missing spec from KDL UserTools CR", "userToolName", userTool.GetName()) - continue - } - - podLabels, ok := spec["podLabels"].(map[string]interface{}) - if !ok { - i.logger.Error(errCreatingKDLUserTools, "Missing spec.podLabels from KDL UserTools CR", "userToolName", userTool.GetName()) - continue - } - - runtimeID, ok := podLabels["runtimeId"].(string) - if !ok || runtimeID == "" { - i.logger.Error(errCreatingKDLUserTools, "Runtime ID provided is not valid, skipping user tools update", "userToolName", resourceName) - continue - } - - capabilitiesID, ok := podLabels["capabilityId"].(string) - if !ok || capabilitiesID == "" { - i.logger.Error(errCreatingKDLUserTools, "Capability ID provided is not valid, skipping user tools update", "userToolName", resourceName) - continue - } - - r, err := i.repoRuntimes.Get(ctx, runtimeID) - if err != nil { - i.logger.Error(err, "Error getting runtime", "runtimeID", runtimeID) - continue - } - - var data = k8s.UserToolsData{} - data.RuntimeID = r.ID - data.RuntimeImage = r.DockerImage - data.RuntimeTag = r.DockerTag - - data.Capabilities, err = i.repoCapabilities.Get(ctx, capabilitiesID) - if err != nil { - i.logger.Error(err, "Error getting capability", "capabilitiesID", capabilitiesID) - continue - } - - err = i.k8sClient.UpdateKDLUserToolsCR(ctx, resourceName, data, &crd) + err = i.k8sClient.UpdateKDLUserToolsCR(ctx, resourceName, &crd) if err != nil { i.logger.Error(err, "Error updating KDL UserTools CR in k8s", "userToolName", resourceName) } diff --git a/app/api/usecase/user/interactor_test.go b/app/api/usecase/user/interactor_test.go index 1113c7d8b..4af913ffd 100644 --- a/app/api/usecase/user/interactor_test.go +++ b/app/api/usecase/user/interactor_test.go @@ -19,7 +19,9 @@ import ( "github.com/konstellation-io/kdl-server/app/api/entity" "github.com/konstellation-io/kdl-server/app/api/infrastructure/config" "github.com/konstellation-io/kdl-server/app/api/infrastructure/k8s" + "github.com/konstellation-io/kdl-server/app/api/infrastructure/minioadminservice" "github.com/konstellation-io/kdl-server/app/api/pkg/clock" + "github.com/konstellation-io/kdl-server/app/api/pkg/kdlutil" "github.com/konstellation-io/kdl-server/app/api/pkg/sshhelper" "github.com/konstellation-io/kdl-server/app/api/usecase/capabilities" "github.com/konstellation-io/kdl-server/app/api/usecase/project" @@ -43,26 +45,30 @@ type userSuite struct { } type userMocks struct { - repo *user.MockRepository - runtimeRepo *runtime.MockRepository - capabilitiesRepo *capabilities.MockRepository - sshGenerator *sshhelper.MockSSHKeyGenerator - clock *clock.MockClock - k8sClientMock *k8s.MockClientInterface - logger logr.Logger - cfg config.Config - userActivityRepo *project.MockUserActivityRepo + repo *user.MockRepository + userActivityRepo *project.MockUserActivityRepo + runtimeRepo *runtime.MockRepository + capabilitiesRepo *capabilities.MockRepository + sshGenerator *sshhelper.MockSSHKeyGenerator + clock *clock.MockClock + k8sClientMock *k8s.MockClientInterface + logger logr.Logger + cfg config.Config + minioAdminService *minioadminservice.MockMinioAdminInterface + randomGenerator *kdlutil.MockRandomGenerator } func newUserSuite(t *testing.T) *userSuite { ctrl := gomock.NewController(t) repo := user.NewMockRepository(ctrl) + userActivityRepo := project.NewMockUserActivityRepo(ctrl) repoRuntimes := runtime.NewMockRepository(ctrl) repoCapabilities := capabilities.NewMockRepository(ctrl) clockMock := clock.NewMockClock(ctrl) sshGenerator := sshhelper.NewMockSSHKeyGenerator(ctrl) k8sClientMock := k8s.NewMockClientInterface(ctrl) - userActivityRepo := project.NewMockUserActivityRepo(ctrl) + minioAdminService := minioadminservice.NewMockMinioAdminInterface(ctrl) + randomGenerator := kdlutil.NewMockRandomGenerator(ctrl) zapLog, err := zap.NewDevelopment() require.NoError(t, err) @@ -71,22 +77,24 @@ func newUserSuite(t *testing.T) *userSuite { cfg := &config.Config{} - interactor := user.NewInteractor(logger, *cfg, repo, userActivityRepo, repoRuntimes, repoCapabilities, - sshGenerator, clockMock, k8sClientMock) + interactor := user.NewInteractor(logger, *cfg, repo, userActivityRepo, repoRuntimes, repoCapabilities, sshGenerator, + clockMock, k8sClientMock, minioAdminService, randomGenerator) return &userSuite{ ctrl: ctrl, interactor: interactor, mocks: userMocks{ - logger: logger, - cfg: *cfg, - repo: repo, - userActivityRepo: userActivityRepo, - runtimeRepo: repoRuntimes, - capabilitiesRepo: repoCapabilities, - sshGenerator: sshGenerator, - clock: clockMock, - k8sClientMock: k8sClientMock, + logger: logger, + cfg: *cfg, + repo: repo, + userActivityRepo: userActivityRepo, + runtimeRepo: repoRuntimes, + capabilitiesRepo: repoCapabilities, + sshGenerator: sshGenerator, + clock: clockMock, + k8sClientMock: k8sClientMock, + minioAdminService: minioAdminService, + randomGenerator: randomGenerator, }, } } @@ -96,13 +104,15 @@ func TestInteractor_Create(t *testing.T) { defer s.ctrl.Finish() const ( - id = "user.1234" - email = "user@email.com" - username = "user" - sub = "f6717d2b-ac1f-40da-ade6-00037512933b" - accessLevel = entity.AccessLevelAdmin - publicSSHKey = "test-ssh-key-public" - privateSSHKey = "test-ssh-key-private" + id = "user.1234" + email = "user@email.com" + username = "user" + sub = "f6717d2b-ac1f-40da-ade6-00037512933b" + accessLevel = entity.AccessLevelAdmin + publicSSHKey = "test-ssh-key-public" + privateSSHKey = "test-ssh-key-private" + minioAccessKey = "user-user" // derived from username + minioSecretKey = "test-minio-secret-key" // #nosec G101 ) ctx := context.Background() @@ -147,6 +157,8 @@ func TestInteractor_Create(t *testing.T) { s.mocks.sshGenerator.EXPECT().NewKeys().Return(sshKey, nil) s.mocks.repo.EXPECT().Create(ctx, u).Return(id, nil) s.mocks.repo.EXPECT().Get(ctx, id).Return(expectedUser, nil) + s.mocks.randomGenerator.EXPECT().GenerateRandomString(40).Return(minioSecretKey, nil) + s.mocks.minioAdminService.EXPECT().CreateUser(ctx, u.UsernameSlug(), minioSecretKey).Return(minioAccessKey, nil) s.mocks.k8sClientMock.EXPECT().CreateUserSSHKeySecret(ctx, u, publicSSHKey, privateSSHKey) s.mocks.k8sClientMock.EXPECT().CreateUserServiceAccount(ctx, u.UsernameSlug()) s.mocks.clock.EXPECT().Now().Return(now) @@ -300,6 +312,7 @@ func TestInteractor_StartTools(t *testing.T) { const ( username = "john" + slugUsername = "john" email = "john@doe.com" toolsRunning = false runtimeImage = "konstellation/image" @@ -319,6 +332,8 @@ func TestInteractor_StartTools(t *testing.T) { runtimeID := "12345" data := k8s.UserToolsData{ + Username: username, + SlugUsername: slugUsername, RuntimeID: runtimeID, RuntimeImage: runtimeImage, RuntimeTag: runtimeTag, @@ -333,7 +348,7 @@ func TestInteractor_StartTools(t *testing.T) { s.mocks.runtimeRepo.EXPECT().Get(ctx, runtimeID).Return(expectedRuntime, nil) s.mocks.capabilitiesRepo.EXPECT().Get(ctx, capability.ID).Return(capability, nil) s.mocks.k8sClientMock.EXPECT().IsUserToolPODRunning(ctx, username).Return(toolsRunning, nil) - s.mocks.k8sClientMock.EXPECT().CreateKDLUserToolsCR(ctx, username, data).Return(nil) + s.mocks.k8sClientMock.EXPECT().CreateKDLUserToolsCR(ctx, data).Return(nil) returnedUser, err := s.interactor.StartTools(ctx, email, &runtimeID, &capability.ID) @@ -348,6 +363,7 @@ func TestInteractor_StartTools_DefaultRuntime(t *testing.T) { const ( username = "john" + slugUsername = "john" email = "john@doe.com" toolsRunning = false ) @@ -363,6 +379,8 @@ func TestInteractor_StartTools_DefaultRuntime(t *testing.T) { } data := k8s.UserToolsData{ + Username: username, + SlugUsername: slugUsername, Capabilities: capability, } @@ -376,7 +394,7 @@ func TestInteractor_StartTools_DefaultRuntime(t *testing.T) { s.mocks.capabilitiesRepo.EXPECT().Get(ctx, capability.ID).Return(capability, nil) // AND the CR creation does not return any error - s.mocks.k8sClientMock.EXPECT().CreateKDLUserToolsCR(ctx, username, data).Return(nil) + s.mocks.k8sClientMock.EXPECT().CreateKDLUserToolsCR(ctx, data).Return(nil) // WHEN the tools are started returnedUser, err := s.interactor.StartTools(ctx, email, nil, &capability.ID) @@ -399,6 +417,7 @@ func TestInteractor_StartTools_Replace(t *testing.T) { const ( username = "john" + slugUsername = "john" email = "john@doe.com" toolsRunning = true dockerImage = "image" @@ -420,6 +439,8 @@ func TestInteractor_StartTools_Replace(t *testing.T) { } data := k8s.UserToolsData{ + Username: username, + SlugUsername: slugUsername, RuntimeID: runtimeID, RuntimeImage: dockerImage, RuntimeTag: dockerTag, @@ -439,7 +460,7 @@ func TestInteractor_StartTools_Replace(t *testing.T) { s.mocks.capabilitiesRepo.EXPECT().Get(ctx, capability.ID).Return(capability, nil) // AND the CR creation does not return any error - s.mocks.k8sClientMock.EXPECT().CreateKDLUserToolsCR(ctx, username, data).Return(nil) + s.mocks.k8sClientMock.EXPECT().CreateKDLUserToolsCR(ctx, data).Return(nil) // WHEN the tools are started returnedUser, err := s.interactor.StartTools(ctx, email, &runtimeID, &capability.ID) @@ -842,7 +863,6 @@ func TestInteractor_UpdateKDLUserTools(t *testing.T) { configMap.Data["template"] = "" crd := map[string]interface{}{} - data := k8s.UserToolsData{} listKDLUserTools := []unstructured.Unstructured{ { @@ -862,10 +882,8 @@ func TestInteractor_UpdateKDLUserTools(t *testing.T) { s.mocks.k8sClientMock.EXPECT().GetConfigMapTemplateNameKDLUserTools().Return(templateConfigMap) s.mocks.k8sClientMock.EXPECT().GetConfigMap(ctx, templateConfigMap).Return(&configMap, nil) - s.mocks.runtimeRepo.EXPECT().Get(ctx, "12345").Return(entity.Runtime{}, nil) - s.mocks.capabilitiesRepo.EXPECT().Get(ctx, "54321").Return(entity.Capabilities{}, nil) s.mocks.k8sClientMock.EXPECT().ListKDLUserToolsCR(ctx).Return(listKDLUserTools, nil) - s.mocks.k8sClientMock.EXPECT().UpdateKDLUserToolsCR(ctx, "kdlusertools-v1", data, &crd).Return(nil) + s.mocks.k8sClientMock.EXPECT().UpdateKDLUserToolsCR(ctx, "kdlusertools-v1", &crd).Return(nil) err := s.interactor.UpdateKDLUserTools(ctx) require.NoError(t, err) @@ -883,7 +901,6 @@ func TestInteractor_UpdateKDLUserTools_UpdateKDLUserToolsCR_Error(t *testing.T) configMap.Data["template"] = "" crd := map[string]interface{}{} - data := k8s.UserToolsData{} listKDLUserTools := []unstructured.Unstructured{ { @@ -904,141 +921,7 @@ func TestInteractor_UpdateKDLUserTools_UpdateKDLUserToolsCR_Error(t *testing.T) s.mocks.k8sClientMock.EXPECT().GetConfigMapTemplateNameKDLUserTools().Return(templateConfigMap) s.mocks.k8sClientMock.EXPECT().GetConfigMap(ctx, templateConfigMap).Return(&configMap, nil) s.mocks.k8sClientMock.EXPECT().ListKDLUserToolsCR(ctx).Return(listKDLUserTools, nil) - s.mocks.runtimeRepo.EXPECT().Get(ctx, "12345").Return(entity.Runtime{}, nil) - s.mocks.capabilitiesRepo.EXPECT().Get(ctx, "54321").Return(entity.Capabilities{}, nil) - s.mocks.k8sClientMock.EXPECT().UpdateKDLUserToolsCR(ctx, "kdlusertools-v1", data, &crd).Return(errUpdatingCrd) - - // even if there is an error updating the CRD, - // the function should return no error to allow updating the next CRD - err := s.interactor.UpdateKDLUserTools(ctx) - require.NoError(t, err) -} -func TestInteractor_UpdateKDLUserTools_NoSpec(t *testing.T) { - s := newUserSuite(t) - defer s.ctrl.Finish() - - ctx := context.Background() - - configMap := v1.ConfigMap{ - Data: map[string]string{}, - } - configMap.Data["template"] = "" - - listKDLUserTools := []unstructured.Unstructured{ - { - Object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "name": "kdlusertools-v1", - }, - }, - }, - } - - s.mocks.k8sClientMock.EXPECT().GetConfigMapTemplateNameKDLUserTools().Return(templateConfigMap) - s.mocks.k8sClientMock.EXPECT().GetConfigMap(ctx, templateConfigMap).Return(&configMap, nil) - s.mocks.k8sClientMock.EXPECT().ListKDLUserToolsCR(ctx).Return(listKDLUserTools, nil) - - // even if there is an error updating the CRD, - // the function should return no error to allow updating the next CRD - err := s.interactor.UpdateKDLUserTools(ctx) - require.NoError(t, err) -} - -func TestInteractor_UpdateKDLUserTools_NoPodLabels(t *testing.T) { - s := newUserSuite(t) - defer s.ctrl.Finish() - - ctx := context.Background() - - configMap := v1.ConfigMap{ - Data: map[string]string{}, - } - configMap.Data["template"] = "" - - listKDLUserTools := []unstructured.Unstructured{ - { - Object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "name": "kdlusertools-v1", - }, - "spec": map[string]interface{}{}, - }, - }, - } - - s.mocks.k8sClientMock.EXPECT().GetConfigMapTemplateNameKDLUserTools().Return(templateConfigMap) - s.mocks.k8sClientMock.EXPECT().GetConfigMap(ctx, templateConfigMap).Return(&configMap, nil) - s.mocks.k8sClientMock.EXPECT().ListKDLUserToolsCR(ctx).Return(listKDLUserTools, nil) - - // even if there is an error updating the CRD, - // the function should return no error to allow updating the next CRD - err := s.interactor.UpdateKDLUserTools(ctx) - require.NoError(t, err) -} - -func TestInteractor_UpdateKDLUserTools_NoRuntimeId(t *testing.T) { - s := newUserSuite(t) - defer s.ctrl.Finish() - - ctx := context.Background() - - configMap := v1.ConfigMap{ - Data: map[string]string{}, - } - configMap.Data["template"] = "" - - listKDLUserTools := []unstructured.Unstructured{ - { - Object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "name": "kdlusertools-v1", - }, - "spec": map[string]interface{}{ - "podLabels": map[string]interface{}{}, - }, - }, - }, - } - - s.mocks.k8sClientMock.EXPECT().GetConfigMapTemplateNameKDLUserTools().Return(templateConfigMap) - s.mocks.k8sClientMock.EXPECT().GetConfigMap(ctx, templateConfigMap).Return(&configMap, nil) - s.mocks.k8sClientMock.EXPECT().ListKDLUserToolsCR(ctx).Return(listKDLUserTools, nil) - - // even if there is an error updating the CRD, - // the function should return no error to allow updating the next CRD - err := s.interactor.UpdateKDLUserTools(ctx) - require.NoError(t, err) -} - -func TestInteractor_UpdateKDLUserTools_NoCapabilityId(t *testing.T) { - s := newUserSuite(t) - defer s.ctrl.Finish() - - ctx := context.Background() - - configMap := v1.ConfigMap{ - Data: map[string]string{}, - } - configMap.Data["template"] = "" - - listKDLUserTools := []unstructured.Unstructured{ - { - Object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "name": "kdlusertools-v1", - }, - "spec": map[string]interface{}{ - "podLabels": map[string]interface{}{ - "runtimeId": "12345", - }, - }, - }, - }, - } - - s.mocks.k8sClientMock.EXPECT().GetConfigMapTemplateNameKDLUserTools().Return(templateConfigMap) - s.mocks.k8sClientMock.EXPECT().GetConfigMap(ctx, templateConfigMap).Return(&configMap, nil) - s.mocks.k8sClientMock.EXPECT().ListKDLUserToolsCR(ctx).Return(listKDLUserTools, nil) + s.mocks.k8sClientMock.EXPECT().UpdateKDLUserToolsCR(ctx, "kdlusertools-v1", &crd).Return(errUpdatingCrd) // even if there is an error updating the CRD, // the function should return no error to allow updating the next CRD diff --git a/app/api/usecase/user/interface.go b/app/api/usecase/user/interface.go index 71a0681ce..fd8660d34 100644 --- a/app/api/usecase/user/interface.go +++ b/app/api/usecase/user/interface.go @@ -26,6 +26,7 @@ type Repository interface { UpdateUsername(ctx context.Context, email, userID string) error UpdateDeleted(ctx context.Context, userID string, deleted bool) error UpdateLastActivity(ctx context.Context, userID string, lastActivity time.Time) error + UpdateMinioAccess(ctx context.Context, username, accessKey, secretKey string) error } // UseCase interface to manage all operations related with users. diff --git a/app/api/usecase/user/mocks_interface.go b/app/api/usecase/user/mocks_interface.go index 9764400b9..893594ba8 100644 --- a/app/api/usecase/user/mocks_interface.go +++ b/app/api/usecase/user/mocks_interface.go @@ -211,6 +211,20 @@ func (mr *MockRepositoryMockRecorder) UpdateLastActivity(ctx, userID, lastActivi return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLastActivity", reflect.TypeOf((*MockRepository)(nil).UpdateLastActivity), ctx, userID, lastActivity) } +// UpdateMinioAccess mocks base method. +func (m *MockRepository) UpdateMinioAccess(ctx context.Context, username, accessKey, secretKey string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateMinioAccess", ctx, username, accessKey, secretKey) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateMinioAccess indicates an expected call of UpdateMinioAccess. +func (mr *MockRepositoryMockRecorder) UpdateMinioAccess(ctx, username, accessKey, secretKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMinioAccess", reflect.TypeOf((*MockRepository)(nil).UpdateMinioAccess), ctx, username, accessKey, secretKey) +} + // UpdateSSHKey mocks base method. func (m *MockRepository) UpdateSSHKey(ctx context.Context, username string, SSHKey entity.SSHKey) error { m.ctrl.T.Helper() diff --git a/app/graphql/schema.graphqls b/app/graphql/schema.graphqls index 6fe0c81ca..97357abfa 100644 --- a/app/graphql/schema.graphqls +++ b/app/graphql/schema.graphqls @@ -86,6 +86,7 @@ type Project { toolUrls: ToolUrls! needAccess: Boolean! archived: Boolean! + minioAccessKey: MinioAccessKey! } type Repository { @@ -99,6 +100,11 @@ type Member { addedDate: String! } +type MinioAccessKey { + accessKey: String! + secretKey: String! +} + type ToolUrls { knowledgeGalaxy: String! filebrowser: String! diff --git a/app/ui/README.md b/app/ui/README.md index 0b080d3c6..9ff84d096 100644 --- a/app/ui/README.md +++ b/app/ui/README.md @@ -1,5 +1,7 @@ # KDL Server UI + ## Contributing + You can access contributing documentation [here](./docs/contributing.md). ## Development @@ -7,7 +9,10 @@ You can access contributing documentation [here](./docs/contributing.md). To initialize the development environment: 1. Run the mock-server -- The mock-server uses the same graphql schema as the real server, but it returns mock data. To update the schema check "Other scripts" section. + +- The mock-server uses the same graphql schema as the real server, but it + returns mock data. To update the schema check "Other scripts" section. + ```bash # Go to kdl-server/app/ui/mock-server cd mock-server @@ -20,6 +25,7 @@ yarn start ``` 2. Check if `SERVER_URL` setting on `config.json` has value `http://localhost:4000` + ```bash # Go to kdl-server/app/ui cd .. @@ -39,6 +45,7 @@ vim public/config.json ``` 3. Run React app + ```bash # (optional) install deps yarn install @@ -47,35 +54,64 @@ yarn install yarn start ``` +## Generate types for graphql + +Any change in the graphql schema should be reflected in the frontend. To do +this, you need to do the following after changing the schema: + +- Update the queries/mutation affected in folder `src/Graphql/**/*.ts` to match + the schema. +- Regenerate all `interfaces`, `types` and `enums` for the graphql queries and + schema used. This schema is used to generate types for the frontend and is + used by the real backend and mock-server (without doing anything in the + mock-server code). + + ```bash + yarn gen:types + ``` + ## Tests + ### Unit tests -* Launch the test runner in the interactive watch mode: + +- Launch the test runner in the interactive watch mode: + ```bash yarn test ``` -* Launch the test runner in the interactive watch mode. It also launches the coverage report: + +- Launch the test runner in the interactive watch mode. It also launches the coverage report: + ```bash yarn test:cov ``` + ### Integration tests + The integration tests are made with `Cypress`. It accepts headed and headless execution. -* Headed: +- Headed: + ```bash yarn cy ``` -* Headless: +- Headless: + ```bash yarn cy:run ``` + ## Linter -* Pass eslint to all files inside the React application: +- Pass eslint to all files inside the React application: + ```bash yarn lint ``` -* Pass eslint to all files and fix errors: + +- Pass eslint to all files and fix errors: + ```bash yarn lint:fix ``` @@ -83,22 +119,17 @@ yarn lint:fix ## Fix vulnerable dependencies This is equivalent to `npm audit --fix` but with `yarn`: + ```bash yarn audit:fix ``` -## Other scripts +## Other available scripts -## Available Scripts +- Builds the app for production to the `build` folder. It correctly bundles + React in production mode and optimizes the build for the best performance. The + build is minified, and the filenames include the hashes. -* Regenerate all `interfaces`, `types` and `enums` for the graphql queries and schema used. This schema is used to generate types for the frontend and is used by the real backend and mock-server (without doing anything in the mock-server code). -```bash -yarn gen:types -``` - -* Builds the app for production to the `build` folder.\ - It correctly bundles React in production mode and optimizes the build for the best performance. - The build is minified, and the filenames include the hashes. -```bash -yarn build -``` + ```bash + yarn build + ``` diff --git a/app/ui/mock-server/src/mocks/projectMock.js b/app/ui/mock-server/src/mocks/projectMock.js index 95dbb2295..0de9b2664 100644 --- a/app/ui/mock-server/src/mocks/projectMock.js +++ b/app/ui/mock-server/src/mocks/projectMock.js @@ -21,6 +21,10 @@ function buildProject(_, index) { filebrowser: 'https://filebrowser.org/', mlflow: 'https://mlflow.org/', }, + minioAccessKey: { + accessKey: casual.uuid, + secretKey: casual.uuid, + }, }; } diff --git a/app/ui/src/Graphql/client/models/SettingsTab.ts b/app/ui/src/Graphql/client/models/SettingsTab.ts index 255ea95ff..aa86eab3f 100644 --- a/app/ui/src/Graphql/client/models/SettingsTab.ts +++ b/app/ui/src/Graphql/client/models/SettingsTab.ts @@ -1,6 +1,7 @@ export enum SettingsTab { INFO, REPOSITORY, + MINIO, MEMBERS, DANGER, } diff --git a/app/ui/src/Graphql/queries/getProjects.ts b/app/ui/src/Graphql/queries/getProjects.ts index 589e54d99..77cb565f9 100644 --- a/app/ui/src/Graphql/queries/getProjects.ts +++ b/app/ui/src/Graphql/queries/getProjects.ts @@ -26,6 +26,10 @@ export default gql` members { ...MemberFields } + minioAccessKey { + accessKey + secretKey + } } } `; diff --git a/app/ui/src/Graphql/queries/types/GetProjects.ts b/app/ui/src/Graphql/queries/types/GetProjects.ts index 88ce91695..e50362016 100644 --- a/app/ui/src/Graphql/queries/types/GetProjects.ts +++ b/app/ui/src/Graphql/queries/types/GetProjects.ts @@ -36,6 +36,12 @@ export interface GetProjects_projects_members { addedDate: string; } +export interface GetProjects_projects_minioAccessKey { + __typename: "MinioAccessKey"; + accessKey: string; + secretKey: string; +} + export interface GetProjects_projects { __typename: "Project"; id: string; @@ -50,6 +56,7 @@ export interface GetProjects_projects { error: string | null; toolUrls: GetProjects_projects_toolUrls; members: GetProjects_projects_members[]; + minioAccessKey: GetProjects_projects_minioAccessKey; } export interface GetProjects { diff --git a/app/ui/src/Pages/Project/panels/ProjectSettings/ProjectSettings.tsx b/app/ui/src/Pages/Project/panels/ProjectSettings/ProjectSettings.tsx index e92ddf0b1..a00fc811c 100644 --- a/app/ui/src/Pages/Project/panels/ProjectSettings/ProjectSettings.tsx +++ b/app/ui/src/Pages/Project/panels/ProjectSettings/ProjectSettings.tsx @@ -7,6 +7,7 @@ import TabDangerZone from './components/TabDangerZone/TabDangerZone'; import TabGit from './components/TabGit/TabGit'; import TabInfo from './components/TabInfo/TabInfo'; import TabMembers from './components/TabMembers/TabMembers'; +import TabMinio from './components/TabMinio/TabMinio'; import cx from 'classnames'; import styles from './ProjectSettings.module.scss'; import { GetProjects_projects } from 'Graphql/queries/types/GetProjects'; @@ -40,8 +41,9 @@ function ProjectSettings({ project }: Props) { INFO GIT - MEMBERS - {hasAccess && DANGER ZONE} + MINIO + USERS + {hasAccess && ADMIN}
@@ -51,6 +53,9 @@ function ProjectSettings({ project }: Props) { + + + diff --git a/app/ui/src/Pages/Project/panels/ProjectSettings/components/TabMinio/TabMinio.module.scss b/app/ui/src/Pages/Project/panels/ProjectSettings/components/TabMinio/TabMinio.module.scss new file mode 100644 index 000000000..f7c8cc8b8 --- /dev/null +++ b/app/ui/src/Pages/Project/panels/ProjectSettings/components/TabMinio/TabMinio.module.scss @@ -0,0 +1,62 @@ +@import 'Styles/variables'; +@import 'Styles/mixins'; +@import 'Styles/colors'; + +.container { + position: relative; + padding: 3 * $grid-unit 2 * $grid-unit; + height: 100%; + display: flex; + flex-direction: column; +} + +.edit { + position: absolute; + top: 2 * $grid-unit; + right: 2 * $grid-unit; +} + +.accessKey { + @include font-body; + display: flex; + align-items: center; + margin-top: 1 - $grid-unit; +} + +.fieldLabel { + @include font-caption; + color: font-color(regular); +} + +.key { + display: flex; + align-items: center; + + .keyValue { + @include font-title; + color: font-color(light); + + padding: 2 * $grid-unit; + border: solid 1px font-color(regular); + max-width: 30 * $grid-unit; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + } + + .copyPre { + @include font-caption; + color: font-color(regular); + margin-left: 1 * $grid-unit; + } + + .copy { + margin-left: 1 * $grid-unit; + background-color: palette(base, 300); + color: font-color(light); + + &:hover { + background-color: palette(base, 200); + } + } +} diff --git a/app/ui/src/Pages/Project/panels/ProjectSettings/components/TabMinio/TabMinio.tsx b/app/ui/src/Pages/Project/panels/ProjectSettings/components/TabMinio/TabMinio.tsx new file mode 100644 index 000000000..0488a3ee5 --- /dev/null +++ b/app/ui/src/Pages/Project/panels/ProjectSettings/components/TabMinio/TabMinio.tsx @@ -0,0 +1,49 @@ +import RepositoryIcon, { LOCATION, SIZE } from 'Pages/NewProject/pages/RepositoryIcon/RepositoryIcon'; + +import { ErrorMessage } from 'kwc'; +import CopyToClipboard from 'Components/CopyToClipboard/CopyToClipboard'; +import { GetProjects_projects } from 'Graphql/queries/types/GetProjects'; +import React from 'react'; +import styles from './TabMinio.module.scss'; +import { useQuery } from '@apollo/client'; +import GetMeQuery from 'Graphql/queries/getMe'; +import { GetMe } from 'Graphql/queries/types/GetMe'; +import { Button } from 'kwc'; +import { copyAndToast } from 'Utils/clipboard'; + +type Props = { + project: GetProjects_projects; +}; +function TabGit({ project }: Props) { + const { minioAccessKey } = project; + const { data: loading, error } = useQuery(GetMeQuery); + + if (!minioAccessKey || error || !loading) return ; + + return ( +
+

ACCESS KEY

+
+

{minioAccessKey.accessKey}

+ {minioAccessKey.accessKey} +
+
+

SECRET KEY

+
+
+ {minioAccessKey.secretKey} +
+

TO SEE THE KEY, JUST

+
+
+
+ ); +} + +export default TabGit; diff --git a/app/ui/src/Styles/react-tabs.scss b/app/ui/src/Styles/react-tabs.scss index 32c0e60a9..99d1b8012 100644 --- a/app/ui/src/Styles/react-tabs.scss +++ b/app/ui/src/Styles/react-tabs.scss @@ -2,7 +2,7 @@ @import './colors'; @import './mixins'; -$tab-min-width: 13 * $grid-unit; +$tab-min-width: 11 * $grid-unit; $tab-height: 3.5 * $grid-unit; .react-tabs { @@ -38,6 +38,7 @@ $tab-height: 3.5 * $grid-unit; border-radius: 0; text-align: center; transition: background-color ease 0.4s, color ease 0.4s; + margin-bottom: $grid-unit; &--selected { background-color: palette(lowlight, 900); diff --git a/hack/scripts/helmfile/values/initial-resources/values.yaml b/hack/scripts/helmfile/values/initial-resources/values.yaml index af64a0310..eed5b3ed0 100644 --- a/hack/scripts/helmfile/values/initial-resources/values.yaml +++ b/hack/scripts/helmfile/values/initial-resources/values.yaml @@ -395,6 +395,67 @@ resources: "microprofile-jwt" ] }, + { + "clientId": "minio", + "name": "MinIO client", + "description": "", + "rootUrl": "https://minio-console.kdl.10.0.1.1.nip.io/", + "adminUrl": "https://minio-console.kdl.10.0.1.1.nip.io/", + "baseUrl": "", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": true, + "clientAuthenticatorType": "client-secret", + "secret": "minio123456", + "redirectUris": [ + "https://minio-console.kdl.10.0.1.1.nip.io/oauth_callback" + ], + "webOrigins": [ + "*" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": true, + "protocol": "openid-connect", + "attributes": { + "realm_client": "false", + "oidc.ciba.grant.enabled": "false", + "client.secret.creation.time": "1733141642", + "backchannel.logout.session.required": "true", + "display.on.consent.screen": "false", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "basic", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "organization", + "offline_access", + "microprofile-jwt" + ], + "access": { + "view": true, + "configure": true, + "manage": true + } + }, { "id": "07ef6dca-5bcb-49a1-9b28-a4a9b7a52500", "clientId": "proxy",