diff --git a/Pipfile b/Pipfile index 12be9c965..c16ea2232 100644 --- a/Pipfile +++ b/Pipfile @@ -8,21 +8,25 @@ django-environ = "==0.10.0" djangorestframework = "==3.15.2" gunicorn = "==22.0.0" whitenoise = "==6.4.0" -django = "==4.2.13" +django = "==4.2.14" django-filter = "==22.1" requests = "==2.32.3" django-tenants = "==3.5.0" django-cors-headers = "==3.13.0" djangorestframework-csv = "==2.1.1" +grpcio = "==1.64.1" +grpcio-status = "==1.64.1" pytz = "==2022.2.1" tzdata = "==2022.2" django-prometheus = "==2.2.0" prometheus-client = "==0.15.0" +protoc-gen-validate = "==1.0.4" urllib3 = "==1.26.19" watchtower = "==3.0.0" boto3 = "==1.24.24" celery = "==5.3.0b2" redis = "==5.0.0" +relations-grpc-clients-python-kessel-project = "==0.2.1" sqlparse = "==0.5.0" django-extensions = "==3.2.1" python-dateutil = "==2.8.2" diff --git a/Pipfile.lock b/Pipfile.lock index c07e1d292..8a4a444e9 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "49137a4483071f1a6ff5917ab819ab42db6e9e843d38c04e9889658edc56ab49" + "sha256": "12d8e169213816e791713cc5003c6af3bb7d8e143d6418ddbfea464e4d0ce497" }, "pipfile-spec": 6, "requires": { @@ -40,6 +40,13 @@ "markers": "python_version >= '3.8'", "version": "==3.8.1" }, + "astunparse": { + "hashes": [ + "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", + "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8" + ], + "version": "==1.6.3" + }, "async-timeout": { "hashes": [ "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f", @@ -330,12 +337,12 @@ }, "django": { "hashes": [ - "sha256:837e3cf1f6c31347a1396a3f6b65688f2b4bb4a11c580dcb628b5afe527b68a5", - "sha256:a17fcba2aad3fc7d46fdb23215095dbbd64e6174bf4589171e732b18b07e426a" + "sha256:3ec32bc2c616ab02834b9cac93143a7dc1cdcd5b822d78ac95fc20a38c534240", + "sha256:fc6919875a6226c7ffcae1a7d51e0f2ceaf6f160393180818f6c95f51b1e7b96" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.2.13" + "version": "==4.2.14" }, "django-cors-headers": { "hashes": [ @@ -413,6 +420,128 @@ "markers": "python_version >= '3.6'", "version": "==2.0.0" }, + "googleapis-common-protos": { + "hashes": [ + "sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945", + "sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87" + ], + "markers": "python_version >= '3.7'", + "version": "==1.63.2" + }, + "grpcio": { + "hashes": [ + "sha256:03b43d0ccf99c557ec671c7dede64f023c7da9bb632ac65dbc57f166e4970040", + "sha256:0a12ddb1678ebc6a84ec6b0487feac020ee2b1659cbe69b80f06dbffdb249122", + "sha256:0a2813093ddb27418a4c99f9b1c223fab0b053157176a64cc9db0f4557b69bd9", + "sha256:0cc79c982ccb2feec8aad0e8fb0d168bcbca85bc77b080d0d3c5f2f15c24ea8f", + "sha256:1257b76748612aca0f89beec7fa0615727fd6f2a1ad580a9638816a4b2eb18fd", + "sha256:1262402af5a511c245c3ae918167eca57342c72320dffae5d9b51840c4b2f86d", + "sha256:19264fc964576ddb065368cae953f8d0514ecc6cb3da8903766d9fb9d4554c33", + "sha256:198908f9b22e2672a998870355e226a725aeab327ac4e6ff3a1399792ece4762", + "sha256:1de403fc1305fd96cfa75e83be3dee8538f2413a6b1685b8452301c7ba33c294", + "sha256:20405cb8b13fd779135df23fabadc53b86522d0f1cba8cca0e87968587f50650", + "sha256:2981c7365a9353f9b5c864595c510c983251b1ab403e05b1ccc70a3d9541a73b", + "sha256:2c3c1b90ab93fed424e454e93c0ed0b9d552bdf1b0929712b094f5ecfe7a23ad", + "sha256:39b9d0acaa8d835a6566c640f48b50054f422d03e77e49716d4c4e8e279665a1", + "sha256:3b64ae304c175671efdaa7ec9ae2cc36996b681eb63ca39c464958396697daff", + "sha256:4657d24c8063e6095f850b68f2d1ba3b39f2b287a38242dcabc166453e950c59", + "sha256:4d6dab6124225496010bd22690f2d9bd35c7cbb267b3f14e7a3eb05c911325d4", + "sha256:55260032b95c49bee69a423c2f5365baa9369d2f7d233e933564d8a47b893027", + "sha256:55697ecec192bc3f2f3cc13a295ab670f51de29884ca9ae6cd6247df55df2502", + "sha256:5841dd1f284bd1b3d8a6eca3a7f062b06f1eec09b184397e1d1d43447e89a7ae", + "sha256:58b1041e7c870bb30ee41d3090cbd6f0851f30ae4eb68228955d973d3efa2e61", + "sha256:5e42634a989c3aa6049f132266faf6b949ec2a6f7d302dbb5c15395b77d757eb", + "sha256:5e56462b05a6f860b72f0fa50dca06d5b26543a4e88d0396259a07dc30f4e5aa", + "sha256:5f8b75f64d5d324c565b263c67dbe4f0af595635bbdd93bb1a88189fc62ed2e5", + "sha256:62b4e6eb7bf901719fce0ca83e3ed474ae5022bb3827b0a501e056458c51c0a1", + "sha256:6503b64c8b2dfad299749cad1b595c650c91e5b2c8a1b775380fcf8d2cbba1e9", + "sha256:6c024ffc22d6dc59000faf8ad781696d81e8e38f4078cb0f2630b4a3cf231a90", + "sha256:73819689c169417a4f978e562d24f2def2be75739c4bed1992435d007819da1b", + "sha256:75dbbf415026d2862192fe1b28d71f209e2fd87079d98470db90bebe57b33179", + "sha256:8caee47e970b92b3dd948371230fcceb80d3f2277b3bf7fbd7c0564e7d39068e", + "sha256:8d51dd1c59d5fa0f34266b80a3805ec29a1f26425c2a54736133f6d87fc4968a", + "sha256:940e3ec884520155f68a3b712d045e077d61c520a195d1a5932c531f11883489", + "sha256:a011ac6c03cfe162ff2b727bcb530567826cec85eb8d4ad2bfb4bd023287a52d", + "sha256:a3a035c37ce7565b8f4f35ff683a4db34d24e53dc487e47438e434eb3f701b2a", + "sha256:a5e771d0252e871ce194d0fdcafd13971f1aae0ddacc5f25615030d5df55c3a2", + "sha256:ac15b6c2c80a4d1338b04d42a02d376a53395ddf0ec9ab157cbaf44191f3ffdd", + "sha256:b1a82e0b9b3022799c336e1fc0f6210adc019ae84efb7321d668129d28ee1efb", + "sha256:bac71b4b28bc9af61efcdc7630b166440bbfbaa80940c9a697271b5e1dabbc61", + "sha256:bbc5b1d78a7822b0a84c6f8917faa986c1a744e65d762ef6d8be9d75677af2ca", + "sha256:c1a786ac592b47573a5bb7e35665c08064a5d77ab88a076eec11f8ae86b3e3f6", + "sha256:c84ad903d0d94311a2b7eea608da163dace97c5fe9412ea311e72c3684925602", + "sha256:d4d29cc612e1332237877dfa7fe687157973aab1d63bd0f84cf06692f04c0367", + "sha256:e3d9f8d1221baa0ced7ec7322a981e28deb23749c76eeeb3d33e18b72935ab62", + "sha256:e7cd5c1325f6808b8ae31657d281aadb2a51ac11ab081ae335f4f7fc44c1721d", + "sha256:ed6091fa0adcc7e4ff944090cf203a52da35c37a130efa564ded02b7aff63bcd", + "sha256:ee73a2f5ca4ba44fa33b4d7d2c71e2c8a9e9f78d53f6507ad68e7d2ad5f64a22", + "sha256:f10193c69fc9d3d726e83bbf0f3d316f1847c3071c8c93d8090cf5f326b14309" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==1.64.1" + }, + "grpcio-status": { + "hashes": [ + "sha256:2ec6e0777958831484a517e32b6ffe0a4272242eae81bff2f5c3707fa58b40e3", + "sha256:c50bd14eb6506d8580a6c553bea463d7c08499b2c0e93f6d1864c5e8eabb1066" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==1.64.1" + }, + "grpcio-tools": { + "hashes": [ + "sha256:0a72a2d87529329117fca6493d948489f1963e3f645d27a785a27b54b05c38cb", + "sha256:0d3572b62453342f4164cb245c434053c6991ee7bf883eb94f15e45f3121967b", + "sha256:1c0db41e812e1741b59844c71c8dfc8c3076eb4472b4c30165aefacf609c81bf", + "sha256:1f467bd03d70de23e7d68d3465fd9bfd5167d15168a569edacee730b4ec105bf", + "sha256:22efb9a167da6fd051c76f1a00c4275b5d15e8b7842364c84dc4cc88def8fd4c", + "sha256:23e6c847647e338b6ed139c7d10ed783dbb37d8ce078ce9ab0a3f7e6a518ff4e", + "sha256:23eef1138065edf360ed649381cf1d9c9123b3a8c003a4b28bf0c4a5b025087a", + "sha256:24340327f7fb85f7406910c9484f98dd9588bdf639578b9341920d67f64306a0", + "sha256:35efe38dd8cc5e05f44e67bcc2ae40f461862549b5d2590c1b644c5d4d93c390", + "sha256:37664461c8da4777c78772f79914ddd59914a4c1dc0bdd11ba86b569477a9d25", + "sha256:3cc3036589e416cf8516802d3e6c37fd7de1b6c4defc292a1859848515c67ab5", + "sha256:4a75c8f13762dc323b38e8dc7186d80a61c0d1321062850e3056221a4db779a4", + "sha256:5532fbac9e1229d3290a512d4253bd311ed742d3b77d634ce7240e97b4af32ac", + "sha256:58eee15287eb54d1ba27d4e73fcd7e7a9f819e529a74dabc9cf3933fbe3bef07", + "sha256:59db889e5f698e00ea65032d3fddbfdbec72b22b285a57c167fb7a48bce2ca27", + "sha256:5ecfecf1da38fa9f0f95dd5f3314c04974be5af40264c520fbc1a9f4f5b1acca", + "sha256:625cc1493d4672af90d23f9909bbc0c4041cfa9fa21f9228abe433f5ad9b356f", + "sha256:646828eec31218e1314b04d7c62c78534d3478cae6348909b6a39ee880a757b2", + "sha256:6711f3e3fbfae9313e15f9abc47241d881772f3fb4e4d0257918bff24363139e", + "sha256:6c8181318e3a21827c2f834fd0505040aa8f24fb568a154ff1c95c9802c0e3f5", + "sha256:6cef267289e3a1257ef79c399a4a244a2b508c4f8d28faf9b061983187b8c2ff", + "sha256:72b3550b91adb8354656ecf0f6d1d4611299044bae11fb1e7cc1d1bb66b8c1eb", + "sha256:759982ba0f942995bf170386559679b9d9f3b00caf103f346f3c33b8703c3057", + "sha256:7e67903fba7b122cad7e41b1347c71f2d8e484f51f5c91cacc52249b4ab274bf", + "sha256:85808e3c020d6e08975be00521ec8841885740ffd84a48375305fe7198d8b9e5", + "sha256:8c7d0633b1177fafaeb76e9b0c7b8b14221eb1086874a79925879b298843f8a0", + "sha256:984ed040f13efcede72c4dfb298274df3877573ca305f51f5cb24249463d6a77", + "sha256:9d3fd5f43503ac594872ad4deb80c08353a3d73c9304afe0226bcb077d5dacca", + "sha256:a078af44be25363f55cbedc33c560513f2b2928a0594c8069da0bc65917ef1a1", + "sha256:a29163cbc6ecaf6f853711420ddab7e35f24d9ee014a5e35b0a6b31c328d1c63", + "sha256:a5208855046a338c5663ca39f59fb167e24514f1287c266db42fbf2057373aa0", + "sha256:a653002b287accf79b0924bb1a76b33ea83774be57cef14e6ec383a965999ad5", + "sha256:a808aaa308e26fc5026b15008aec45bea8aa2f2662989cbaffa300601ac98fae", + "sha256:a8fb6a4438ef1ce619bd6695799b0a06c049a0be3e10ecf0b5fc5d72929a9f02", + "sha256:acf9a8bce188bb760c138327a89f64be8bbeb062634d151c77bbcd138d60bdc6", + "sha256:b740e136a12a992c3c75dafe12d96c65e9249daa71e6b75f17aac5459c64f165", + "sha256:ba0758d779bc2134219c9ee392d7d30a7ff7f788fd68bf4f56bb4a0213e5d2e4", + "sha256:be39db97d13f3bd0b2ff9bf8d0e68f590f4877cf2c4db201a2f9d4d39724e137", + "sha256:c8cb5567cd5836b29d37ea12c8ccb753a19712ec459c4dbc05c084ca57b84b3b", + "sha256:cf3fbad6312bb61f48eab8ae5d2b31dcb007653282d5901982e17111773104e1", + "sha256:d0f42307f951851894a1ddcbed2e2403fdb0ac0920bbb4ec5c80a2959a1d328d", + "sha256:d9a470f9e72bccc8994b025fa40cb1a7202db17a5f8e1869f4c2079ded869ac2", + "sha256:e28cfaede2a243452252c94b72378f1d939b786689cb11d218fdae6a8421940f", + "sha256:e8ffaa1972e64d968a706c954f6614e718abd10068b107727028ffb9506503d2", + "sha256:fd4a596ec2b34c8a6b15c6581ef7ea91c9b85f68099004da656db79e5a2b7a8c", + "sha256:ff9b631788573bfbecfe8cb647d484dfac9cfbad4a7bb640a9e5dcfb24a1b3c5" + ], + "markers": "python_version >= '3.8'", + "version": "==1.64.1" + }, "gunicorn": { "hashes": [ "sha256:350679f91b24062c86e386e198a15438d53a7a8207235a78ba1b53df4c4378d9", @@ -563,6 +692,32 @@ "markers": "python_full_version >= '3.7.0'", "version": "==3.0.47" }, + "protobuf": { + "hashes": [ + "sha256:0e341109c609749d501986b835f667c6e1e24531096cff9d34ae411595e26505", + "sha256:176c12b1f1c880bf7a76d9f7c75822b6a2bc3db2d28baa4d300e8ce4cde7409b", + "sha256:354d84fac2b0d76062e9b3221f4abbbacdfd2a4d8af36bab0474f3a0bb30ab38", + "sha256:4fadd8d83e1992eed0248bc50a4a6361dc31bcccc84388c54c86e530b7f58863", + "sha256:54330f07e4949d09614707c48b06d1a22f8ffb5763c159efd5c0928326a91470", + "sha256:610e700f02469c4a997e58e328cac6f305f649826853813177e6290416e846c6", + "sha256:7fc3add9e6003e026da5fc9e59b131b8f22b428b991ccd53e2af8071687b4fce", + "sha256:9e8f199bf7f97bd7ecebffcae45ebf9527603549b2b562df0fbc6d4d688f14ca", + "sha256:a109916aaac42bff84702fb5187f3edadbc7c97fc2c99c5ff81dd15dcce0d1e5", + "sha256:b848dbe1d57ed7c191dfc4ea64b8b004a3f9ece4bf4d0d80a367b76df20bf36e", + "sha256:f3ecdef226b9af856075f28227ff2c90ce3a594d092c39bee5513573f25e2714" + ], + "markers": "python_version >= '3.8'", + "version": "==5.27.2" + }, + "protoc-gen-validate": { + "hashes": [ + "sha256:5154f763a12d41e656643c5002bca996c2a470eff914ed9d2b2c9f2d7b0f1cc7", + "sha256:e38953396924d8948001bea118e92874eff55f93acbb999497fb18209464581f" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==1.0.4" + }, "psycopg2": { "hashes": [ "sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955", @@ -695,6 +850,15 @@ "markers": "python_version >= '3.7'", "version": "==5.0.0" }, + "relations-grpc-clients-python-kessel-project": { + "hashes": [ + "sha256:e44704c28e1f1ad20aff30965c20b82dbd621742a05e418f58529c13b3c316f9", + "sha256:fd5dd66c76f8deca5c7803f09ba16bd33c6f16751042d5065a483d563b521f15" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==0.2.1" + }, "requests": { "hashes": [ "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", @@ -720,6 +884,14 @@ "index": "pypi", "version": "==1.18.0" }, + "setuptools": { + "hashes": [ + "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5", + "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc" + ], + "markers": "python_version >= '3.8'", + "version": "==70.3.0" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", @@ -776,6 +948,12 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==1.26.19" }, + "validate-email": { + "hashes": [ + "sha256:784719dc5f780be319cdd185dc85dd93afebdb6ebb943811bc4c7c5f9c72aeaf" + ], + "version": "==1.3" + }, "vine": { "hashes": [ "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", @@ -800,6 +978,14 @@ ], "version": "==0.2.13" }, + "wheel": { + "hashes": [ + "sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85", + "sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81" + ], + "markers": "python_version >= '3.8'", + "version": "==0.43.0" + }, "whitenoise": { "hashes": [ "sha256:599dc6ca57e48929dfeffb2e8e187879bfe2aed0d49ca419577005b7f2cc930b", @@ -1231,11 +1417,11 @@ }, "identify": { "hashes": [ - "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa", - "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d" + "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf", + "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0" ], "markers": "python_version >= '3.8'", - "version": "==2.5.36" + "version": "==2.6.0" }, "idna": { "hashes": [ @@ -1597,11 +1783,11 @@ }, "setuptools": { "hashes": [ - "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05", - "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1" + "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5", + "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc" ], "markers": "python_version >= '3.8'", - "version": "==70.2.0" + "version": "==70.3.0" }, "six": { "hashes": [ diff --git a/deploy/rbac-clowdapp.yml b/deploy/rbac-clowdapp.yml index dc704f9db..9ba278753 100644 --- a/deploy/rbac-clowdapp.yml +++ b/deploy/rbac-clowdapp.yml @@ -209,6 +209,8 @@ objects: value: ${UMB_PORT} - name: SA_NAME value: ${SA_NAME} + - name: RELATION_API_SERVER + value: ${RELATION_API_SERVER} - name: scheduler-service minReplicas: ${{MIN_SCHEDULER_REPLICAS}} @@ -874,3 +876,6 @@ parameters: - name: BONFIRE_DEPENDENCIES description: A comma separated list of non ClowdApp dependencies for bonfire to deploy value: "model-access-permissions-yml-stage,rbac-config-yml-stage" +- name: RELATION_API_SERVER + description: The gRPC API server to use for the relation + value: "localhost:9000" diff --git a/rbac/internal/views.py b/rbac/internal/views.py index b38c13b3f..6b162f0b2 100644 --- a/rbac/internal/views.py +++ b/rbac/internal/views.py @@ -477,7 +477,7 @@ def get_param_list(request, param_name): def role_migration(request): """View method for running role migrations from V1 to V2 spiceDB schema. - POST /_private/api/utils/role_migration/?exclude_apps=cost_management,rbac&orgs=id_1,id_2 + POST /_private/api/utils/role_migration/?exclude_apps=cost_management,rbac&orgs=id_1,id_2&write_db=True """ if request.method != "POST": return HttpResponse('Invalid method, only "POST" is allowed.', status=405) @@ -486,6 +486,7 @@ def role_migration(request): args = { "exclude_apps": get_param_list(request, "exclude_apps"), "orgs": get_param_list(request, "orgs"), + "write_db": request.GET.get("write_db", "False") == "True", } migrate_roles_in_worker.delay(args) return HttpResponse("Role migration from V1 to V2 are running in a background worker.", status=202) diff --git a/rbac/migration_tool/migrate.py b/rbac/migration_tool/migrate.py index ade987372..27f73ee25 100644 --- a/rbac/migration_tool/migrate.py +++ b/rbac/migration_tool/migrate.py @@ -21,89 +21,77 @@ from management.role.model import Role from migration_tool.migrator import Migrator -from migration_tool.models import Relationship, V1group, V2rolebinding +from migration_tool.models import V1group, V2rolebinding from migration_tool.sharedSystemRolesReplicatedRoleBindings import ( shared_system_role_replicated_role_bindings_v1_to_v2_mapping, ) +from migration_tool.utils import create_relationship, write_relationships +from relations.v0 import common_pb2 from api.models import Tenant from .ingest import extract_info_into_v1_role + logger = logging.getLogger(__name__) # pylint: disable=invalid-name def spicedb_relationships(v2_role_bindings: FrozenSet[V2rolebinding]): """Generate a set of relationships for the given set of v2 role bindings.""" - relationships = set[Relationship]() + relationships = list() for v2_role_binding in v2_role_bindings: - relationships.add( - Relationship( - "role_binding", - v2_role_binding.id, - "granted", - "role", - v2_role_binding.role.id, - ) + relationships.append( + create_relationship("role_binding", v2_role_binding.id, "role", v2_role_binding.role.id, "granted") ) - relationships.add( - Relationship("rbac/v1role", v2_role_binding.originalRole.id, "binding", "role_binding", v2_role_binding.id) + relationships.append( + create_relationship( + "rbac/v1role", v2_role_binding.originalRole.id, "role_binding", v2_role_binding.id, "binding" + ) ) for perm in v2_role_binding.role.permissions: - relationships.add(Relationship("role", v2_role_binding.role.id, perm, "user", "*")) + relationships.append(create_relationship("role", v2_role_binding.role.id, "user", "*", perm)) if not v2_role_binding.role.is_system: - relationships.add( - Relationship( - "rbac/v1role", v2_role_binding.originalRole.id, "customrole", "role", v2_role_binding.role.id + relationships.append( + create_relationship( + "rbac/v1role", v2_role_binding.originalRole.id, "role", v2_role_binding.role.id, "customrole" ) ) for group in v2_role_binding.groups: # These might be duplicate but it is OK, spiceDB will handle duplication through touch for user in group.users: - relationships.add(Relationship("group", group.id, "member", "user", user)) - relationships.add( - Relationship( - "role_binding", - v2_role_binding.id, - "member", - "group", - group.id, - ) - ) + relationships.append(create_relationship("group", group.id, user, "user", "member")) + relationships.append(create_relationship("role_binding", v2_role_binding.id, "group", group.id, "member")) for bound_resource in v2_role_binding.resources: parent_relation = "parent" if bound_resource.resource_type == "workspace" else "workspace" # TODO: create root workspace and replace it if not bound_resource.resource_type == "workspace" and bound_resource.resourceId == "org_migration_root": - relationships.add( - Relationship( - bound_resource.resource_type, - bound_resource.resourceId, - parent_relation, - "workspace", - "org_migration_root", + relationships.append( + create_relationship( + "workspace", "org_migration_root", "workspace", bound_resource.resourceId, parent_relation ) ) - relationships.add( - Relationship( + relationships.append( + create_relationship( bound_resource.resource_type, bound_resource.resourceId, - "user_grant", "role_binding", v2_role_binding.id, + "user_grant", ) ) return relationships -def stringify_spicedb_relationship(rel: Relationship): +def stringify_spicedb_relationship(rel: common_pb2.Relationship): """Stringify a relationship for logging.""" return ( - rel.resource_type + ":" + rel.resource_id + "#" + rel.relation + "@" + rel.subject_type + ":" + rel.subject_id + f"{rel.resource.type.name}:{rel.resource.id}#{rel.relation}@{rel.subject.subject.type.name}:" + f"{rel.subject.subject.id}" ) -def migrate_role(role: Role): +def migrate_role(role: Role, write_db: bool): """Migrate a role from v1 to v2.""" v1_role = extract_info_into_v1_role(role) # With the replicated role bindings algorithm, role bindings are scoped by group, so we need to add groups @@ -119,12 +107,14 @@ def migrate_role(role: Role): v1_to_v2_mapping = shared_system_role_replicated_role_bindings_v1_to_v2_mapping permissioned_role_migrator = Migrator(v1_to_v2_mapping) v2_roles = [v2_role for v2_role in permissioned_role_migrator.migrate_v1_roles(v1_role)] - spicedb_rel_summary = spicedb_relationships(frozenset(v2_roles)) - for rel in spicedb_rel_summary: + relationships = spicedb_relationships(frozenset(v2_roles)) + for rel in relationships: logger.info(stringify_spicedb_relationship(rel)) + if write_db: + write_relationships(relationships) -def migrate_roles_for_tenant(tenant: Tenant, app_list: list): +def migrate_roles_for_tenant(tenant: Tenant, app_list: list, write_db: bool): """Migrate all roles for a given tenant.""" roles = tenant.role_set.all() if app_list: @@ -132,12 +122,12 @@ def migrate_roles_for_tenant(tenant: Tenant, app_list: list): for role in roles: logger.info(f"Migrating role: {role.name} with UUID {role.uuid}.") - migrate_role(role) + migrate_role(role, write_db) logger.info(f"Migration completed for role: {role.name} with UUID {role.uuid}.") logger.info(f"Migrated {roles.count()} roles for tenant: {tenant.org_id}") -def migrate_roles(exclude_apps: list = [], orgs: list = []): +def migrate_roles(exclude_apps: list = [], orgs: list = [], write_db: bool = False): """Migrate all roles for all tenants.""" count = 0 tenants = Tenant.objects.exclude(tenant_name="public") @@ -147,7 +137,7 @@ def migrate_roles(exclude_apps: list = [], orgs: list = []): for tenant in tenants.iterator(): logger.info(f"Migrating roles for tenant: {tenant.org_id}") try: - migrate_roles_for_tenant(tenant, exclude_apps) + migrate_roles_for_tenant(tenant, exclude_apps, write_db) except Exception as e: logger.error(f"Failed to migrate roles for tenant: {tenant.org_id}. Error: {e}") raise e diff --git a/rbac/migration_tool/utils.py b/rbac/migration_tool/utils.py new file mode 100644 index 000000000..e887b7fdf --- /dev/null +++ b/rbac/migration_tool/utils.py @@ -0,0 +1,82 @@ +"""Utilities for working with the relation API server.""" +import json +import logging + +import grpc +from django.conf import settings +from google.rpc import error_details_pb2 +from grpc_status import rpc_status +from protoc_gen_validate.validator import ValidationFailed, validate_all +from relations.v0 import common_pb2 +from relations.v0 import relation_tuples_pb2 +from relations.v0 import relation_tuples_pb2_grpc + + +logger = logging.getLogger(__name__) + + +class GRPCError: + """A wrapper for a gRPC error.""" + + code: grpc.StatusCode + reason: str + message: str + metadata: dict + + def __init__(self, error: grpc.RpcError): + """Initialize the error.""" + self.code = error.code() + self.message = error.details() + + status = rpc_status.from_call(error) + if status is not None: + detail = status.details[0] + info = error_details_pb2.ErrorInfo() + detail.Unpack(info) + self.reason = info.reason + self.metadata = json.loads(str(info.metadata).replace("'", '"')) + + +def validate_and_create_obj_ref(obj_name, obj_id): + """Validate and create a resource.""" + object_type = common_pb2.ObjectType(name=obj_name) + try: + validate_all(object_type) + except ValidationFailed as err: + logger.error(err) + + obj_ref = common_pb2.ObjectReference(type=object_type, id=obj_id) + try: + validate_all(obj_ref) + except ValidationFailed as err: + logger.error(err) + return obj_ref + + +def create_relationship(resource_name, resource_id, subject_name, subject_id, relation): + """Create a relationship between a resource and a subject.""" + return common_pb2.Relationship( + resource=validate_and_create_obj_ref(resource_name, resource_id), + relation=relation, + subject=common_pb2.SubjectReference(subject=validate_and_create_obj_ref(subject_name, subject_id)), + ) + + +def write_relationships(relationships): + """Write relationships to the relation API server.""" + with grpc.insecure_channel(settings.RELATION_API_SERVER) as channel: + stub = relation_tuples_pb2_grpc.KesselTupleServiceStub(channel) + + request = relation_tuples_pb2.CreateTuplesRequest( + upsert=True, + tuples=relationships, + ) + try: + stub.CreateTuples(request) + except grpc.RpcError as err: + error = GRPCError(err.value) + logger.error( + "Failed to write relationships to the relation API server: " + f"error code {error.code}, reason {error.reason}" + f"relationships: {relationships}" + ) diff --git a/rbac/rbac/settings.py b/rbac/rbac/settings.py index 374bf09fb..5a9047c3d 100644 --- a/rbac/rbac/settings.py +++ b/rbac/rbac/settings.py @@ -468,3 +468,5 @@ UMB_PORT = ENVIRONMENT.get_value("UMB_PORT", default="61612") # Service account name SA_NAME = ENVIRONMENT.get_value("SA_NAME", default="nonprod-hcc-rbac") + +RELATION_API_SERVER = ENVIRONMENT.get_value("RELATION_API_SERVER", default="localhost:9000") diff --git a/requirements.txt b/requirements.txt index b09b81656..94c7b6d11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ click-didyoumean==0.3.1; python_full_version >= '3.6.2' click-plugins==1.1.1 click-repl==0.3.0; python_version >= '3.6' cryptography==42.0.8; python_version >= '3.7' -django==4.2.13; python_version >= '3.8' +django==4.2.14; python_version >= '3.8' django-cors-headers==3.13.0; python_version >= '3.7' django-environ==0.10.0; python_version >= '3.5' and python_version < '4' django-extensions==3.2.1; python_version >= '3.6' @@ -25,6 +25,8 @@ django-tenants==3.5.0 djangorestframework==3.15.2; python_version >= '3.8' djangorestframework-csv==2.1.1 ecs-logging==2.0.0; python_version >= '3.6' +grpcio==1.64.1 +grpcio-status==1.64.1 gunicorn==22.0.0; python_version >= '3.7' idna==3.7; python_version >= '3.5' jinja2==3.1.4; python_version >= '3.7' @@ -36,12 +38,14 @@ markupsafe==2.1.5; python_version >= '3.7' packaging==24.1; python_version >= '3.8' prometheus-client==0.15.0; python_version >= '3.6' prompt-toolkit==3.0.47; python_full_version >= '3.7.0' +protoc-gen-validate==1.0.4 psycopg2==2.9.5; python_version >= '3.6' psycopg2-binary==2.9.5; python_version >= '3.6' pycparser==2.22; python_version >= '3.8' python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pytz==2022.2.1 redis==5.0.0; python_version >= '3.7' +relations-grpc-clients-python-kessel-project==0.2.1 requests==2.32.3; python_version >= '3.8' s3transfer==0.6.2; python_version >= '3.7' sentry-sdk==1.18.0 diff --git a/tests/internal/test_views.py b/tests/internal/test_views.py index 2a56ae96f..6f34138a7 100644 --- a/tests/internal/test_views.py +++ b/tests/internal/test_views.py @@ -464,7 +464,7 @@ def test_run_migrations_of_roles(self, migration_mock): **self.request.META, ) migration_mock.assert_called_once_with( - {"exclude_apps": ["rbac", "costmanagement"], "orgs": ["acct00001", "acct00002"]} + {"exclude_apps": ["rbac", "costmanagement"], "orgs": ["acct00001", "acct00002"], "write_db": False} ) self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual( @@ -478,7 +478,7 @@ def test_run_migrations_of_roles(self, migration_mock): f"/_private/api/utils/role_migration/", **self.request.META, ) - migration_mock.assert_called_once_with({"exclude_apps": [], "orgs": []}) + migration_mock.assert_called_once_with({"exclude_apps": [], "orgs": [], "write_db": False}) self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual( response.content.decode(), diff --git a/tests/migration_tool/tests_migrate.py b/tests/migration_tool/tests_migrate.py index 8b4e95273..1402fcf7a 100644 --- a/tests/migration_tool/tests_migrate.py +++ b/tests/migration_tool/tests_migrate.py @@ -83,5 +83,5 @@ def test_migration_of_roles(self, logger_mock): migrate_roles(**kwargs) self.assertEqual( len(logger_mock.info.call_args_list), - 18, + 20, )