Merge pull request RedHatInsights#1136 from astrozzc/migration

Consume gRPC python client

Pipfile

@@ -8,21 +8,25 @@ django-environ = "==0.10.0"
 djangorestframework = "==3.15.2"
 gunicorn = "==22.0.0"
 whitenoise = "==6.4.0"
-django = "==4.2.13"
+django = "==4.2.14"
 django-filter = "==22.1"
 requests = "==2.32.3"
 django-tenants = "==3.5.0"
 django-cors-headers = "==3.13.0"
 djangorestframework-csv = "==2.1.1"
+grpcio = "==1.64.1"
+grpcio-status = "==1.64.1"
 pytz = "==2022.2.1"
 tzdata = "==2022.2"
 django-prometheus = "==2.2.0"
 prometheus-client = "==0.15.0"
+protoc-gen-validate = "==1.0.4"
 urllib3 = "==1.26.19"
 watchtower = "==3.0.0"
 boto3 = "==1.24.24"
 celery = "==5.3.0b2"
 redis = "==5.0.0"
+relations-grpc-clients-python-kessel-project = "==0.2.1"
 sqlparse = "==0.5.0"
 django-extensions = "==3.2.1"
 python-dateutil = "==2.8.2"

deploy/rbac-clowdapp.yml

@@ -209,6 +209,8 @@ objects:
             value: ${UMB_PORT}
           - name: SA_NAME
             value: ${SA_NAME}
+          - name: RELATION_API_SERVER
+            value: ${RELATION_API_SERVER}
 
     - name: scheduler-service
       minReplicas: ${{MIN_SCHEDULER_REPLICAS}}
@@ -874,3 +876,6 @@ parameters:
 - name: BONFIRE_DEPENDENCIES
   description: A comma separated list of non ClowdApp dependencies for bonfire to deploy
   value: "model-access-permissions-yml-stage,rbac-config-yml-stage"
+- name: RELATION_API_SERVER
+  description: The gRPC API server to use for the relation
+  value: "localhost:9000"

rbac/internal/views.py

@@ -477,7 +477,7 @@ def get_param_list(request, param_name):
 def role_migration(request):
     """View method for running role migrations from V1 to V2 spiceDB schema.
 
-    POST /_private/api/utils/role_migration/?exclude_apps=cost_management,rbac&orgs=id_1,id_2
+    POST /_private/api/utils/role_migration/?exclude_apps=cost_management,rbac&orgs=id_1,id_2&write_db=True
     """
     if request.method != "POST":
         return HttpResponse('Invalid method, only "POST" is allowed.', status=405)
@@ -486,6 +486,7 @@ def role_migration(request):
     args = {
         "exclude_apps": get_param_list(request, "exclude_apps"),
         "orgs": get_param_list(request, "orgs"),
+        "write_db": request.GET.get("write_db", "False") == "True",
     }
     migrate_roles_in_worker.delay(args)
     return HttpResponse("Role migration from V1 to V2 are running in a background worker.", status=202)

rbac/migration_tool/migrate.py

@@ -21,89 +21,77 @@
 
 from management.role.model import Role
 from migration_tool.migrator import Migrator
-from migration_tool.models import Relationship, V1group, V2rolebinding
+from migration_tool.models import V1group, V2rolebinding
 from migration_tool.sharedSystemRolesReplicatedRoleBindings import (
     shared_system_role_replicated_role_bindings_v1_to_v2_mapping,
 )
+from migration_tool.utils import create_relationship, write_relationships
+from relations.v0 import common_pb2
 
 from api.models import Tenant
 from .ingest import extract_info_into_v1_role
 
+
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
 def spicedb_relationships(v2_role_bindings: FrozenSet[V2rolebinding]):
     """Generate a set of relationships for the given set of v2 role bindings."""
-    relationships = set[Relationship]()
+    relationships = list()
     for v2_role_binding in v2_role_bindings:
-        relationships.add(
-            Relationship(
-                "role_binding",
-                v2_role_binding.id,
-                "granted",
-                "role",
-                v2_role_binding.role.id,
-            )
+        relationships.append(
+            create_relationship("role_binding", v2_role_binding.id, "role", v2_role_binding.role.id, "granted")
         )
-        relationships.add(
-            Relationship("rbac/v1role", v2_role_binding.originalRole.id, "binding", "role_binding", v2_role_binding.id)
+        relationships.append(
+            create_relationship(
+                "rbac/v1role", v2_role_binding.originalRole.id, "role_binding", v2_role_binding.id, "binding"
+            )
         )
         for perm in v2_role_binding.role.permissions:
-            relationships.add(Relationship("role", v2_role_binding.role.id, perm, "user", "*"))
+            relationships.append(create_relationship("role", v2_role_binding.role.id, "user", "*", perm))
         if not v2_role_binding.role.is_system:
-            relationships.add(
-                Relationship(
-                    "rbac/v1role", v2_role_binding.originalRole.id, "customrole", "role", v2_role_binding.role.id
+            relationships.append(
+                create_relationship(
+                    "rbac/v1role", v2_role_binding.originalRole.id, "role", v2_role_binding.role.id, "customrole"
                 )
             )
         for group in v2_role_binding.groups:
             # These might be duplicate but it is OK, spiceDB will handle duplication through touch
             for user in group.users:
-                relationships.add(Relationship("group", group.id, "member", "user", user))
-            relationships.add(
-                Relationship(
-                    "role_binding",
-                    v2_role_binding.id,
-                    "member",
-                    "group",
-                    group.id,
-                )
-            )
+                relationships.append(create_relationship("group", group.id, user, "user", "member"))
+            relationships.append(create_relationship("role_binding", v2_role_binding.id, "group", group.id, "member"))
 
         for bound_resource in v2_role_binding.resources:
             parent_relation = "parent" if bound_resource.resource_type == "workspace" else "workspace"
             # TODO: create root workspace and replace it
             if not bound_resource.resource_type == "workspace" and bound_resource.resourceId == "org_migration_root":
-                relationships.add(
-                    Relationship(
-                        bound_resource.resource_type,
-                        bound_resource.resourceId,
-                        parent_relation,
-                        "workspace",
-                        "org_migration_root",
+                relationships.append(
+                    create_relationship(
+                        "workspace", "org_migration_root", "workspace", bound_resource.resourceId, parent_relation
                     )
                 )
-            relationships.add(
-                Relationship(
+            relationships.append(
+                create_relationship(
                     bound_resource.resource_type,
                     bound_resource.resourceId,
-                    "user_grant",
                     "role_binding",
                     v2_role_binding.id,
+                    "user_grant",
                 )
             )
 
     return relationships
 
 
-def stringify_spicedb_relationship(rel: Relationship):
+def stringify_spicedb_relationship(rel: common_pb2.Relationship):
     """Stringify a relationship for logging."""
     return (
-        rel.resource_type + ":" + rel.resource_id + "#" + rel.relation + "@" + rel.subject_type + ":" + rel.subject_id
+        f"{rel.resource.type.name}:{rel.resource.id}#{rel.relation}@{rel.subject.subject.type.name}:"
+        f"{rel.subject.subject.id}"
     )
 
 
-def migrate_role(role: Role):
+def migrate_role(role: Role, write_db: bool):
     """Migrate a role from v1 to v2."""
     v1_role = extract_info_into_v1_role(role)
     # With the replicated role bindings algorithm, role bindings are scoped by group, so we need to add groups
@@ -119,25 +107,27 @@ def migrate_role(role: Role):
     v1_to_v2_mapping = shared_system_role_replicated_role_bindings_v1_to_v2_mapping
     permissioned_role_migrator = Migrator(v1_to_v2_mapping)
     v2_roles = [v2_role for v2_role in permissioned_role_migrator.migrate_v1_roles(v1_role)]
-    spicedb_rel_summary = spicedb_relationships(frozenset(v2_roles))
-    for rel in spicedb_rel_summary:
+    relationships = spicedb_relationships(frozenset(v2_roles))
+    for rel in relationships:
         logger.info(stringify_spicedb_relationship(rel))
+    if write_db:
+        write_relationships(relationships)
 
 
-def migrate_roles_for_tenant(tenant: Tenant, app_list: list):
+def migrate_roles_for_tenant(tenant: Tenant, app_list: list, write_db: bool):
     """Migrate all roles for a given tenant."""
     roles = tenant.role_set.all()
     if app_list:
         roles = roles.exclude(access__permission__application__in=app_list)
 
     for role in roles:
         logger.info(f"Migrating role: {role.name} with UUID {role.uuid}.")
-        migrate_role(role)
+        migrate_role(role, write_db)
         logger.info(f"Migration completed for role: {role.name} with UUID {role.uuid}.")
     logger.info(f"Migrated {roles.count()} roles for tenant: {tenant.org_id}")
 
 
-def migrate_roles(exclude_apps: list = [], orgs: list = []):
+def migrate_roles(exclude_apps: list = [], orgs: list = [], write_db: bool = False):
     """Migrate all roles for all tenants."""
     count = 0
     tenants = Tenant.objects.exclude(tenant_name="public")
@@ -147,7 +137,7 @@ def migrate_roles(exclude_apps: list = [], orgs: list = []):
     for tenant in tenants.iterator():
         logger.info(f"Migrating roles for tenant: {tenant.org_id}")
         try:
-            migrate_roles_for_tenant(tenant, exclude_apps)
+            migrate_roles_for_tenant(tenant, exclude_apps, write_db)
         except Exception as e:
             logger.error(f"Failed to migrate roles for tenant: {tenant.org_id}. Error: {e}")
             raise e

rbac/migration_tool/utils.py

@@ -0,0 +1,82 @@
+"""Utilities for working with the relation API server."""
+import json
+import logging
+
+import grpc
+from django.conf import settings
+from google.rpc import error_details_pb2
+from grpc_status import rpc_status
+from protoc_gen_validate.validator import ValidationFailed, validate_all
+from relations.v0 import common_pb2
+from relations.v0 import relation_tuples_pb2
+from relations.v0 import relation_tuples_pb2_grpc
+
+
+logger = logging.getLogger(__name__)
+
+
+class GRPCError:
+    """A wrapper for a gRPC error."""
+
+    code: grpc.StatusCode
+    reason: str
+    message: str
+    metadata: dict
+
+    def __init__(self, error: grpc.RpcError):
+        """Initialize the error."""
+        self.code = error.code()
+        self.message = error.details()
+
+        status = rpc_status.from_call(error)
+        if status is not None:
+            detail = status.details[0]
+            info = error_details_pb2.ErrorInfo()
+            detail.Unpack(info)
+            self.reason = info.reason
+            self.metadata = json.loads(str(info.metadata).replace("'", '"'))
+
+
+def validate_and_create_obj_ref(obj_name, obj_id):
+    """Validate and create a resource."""
+    object_type = common_pb2.ObjectType(name=obj_name)
+    try:
+        validate_all(object_type)
+    except ValidationFailed as err:
+        logger.error(err)
+
+    obj_ref = common_pb2.ObjectReference(type=object_type, id=obj_id)
+    try:
+        validate_all(obj_ref)
+    except ValidationFailed as err:
+        logger.error(err)
+    return obj_ref
+
+
+def create_relationship(resource_name, resource_id, subject_name, subject_id, relation):
+    """Create a relationship between a resource and a subject."""
+    return common_pb2.Relationship(
+        resource=validate_and_create_obj_ref(resource_name, resource_id),
+        relation=relation,
+        subject=common_pb2.SubjectReference(subject=validate_and_create_obj_ref(subject_name, subject_id)),
+    )
+
+
+def write_relationships(relationships):
+    """Write relationships to the relation API server."""
+    with grpc.insecure_channel(settings.RELATION_API_SERVER) as channel:
+        stub = relation_tuples_pb2_grpc.KesselTupleServiceStub(channel)
+
+        request = relation_tuples_pb2.CreateTuplesRequest(
+            upsert=True,
+            tuples=relationships,
+        )
+        try:
+            stub.CreateTuples(request)
+        except grpc.RpcError as err:
+            error = GRPCError(err.value)
+            logger.error(
+                "Failed to write relationships to the relation API server: "
+                f"error code {error.code}, reason {error.reason}"
+                f"relationships: {relationships}"
+            )

rbac/rbac/settings.py

@@ -468,3 +468,5 @@
 UMB_PORT = ENVIRONMENT.get_value("UMB_PORT", default="61612")
 # Service account name
 SA_NAME = ENVIRONMENT.get_value("SA_NAME", default="nonprod-hcc-rbac")
+
+RELATION_API_SERVER = ENVIRONMENT.get_value("RELATION_API_SERVER", default="localhost:9000")

requirements.txt

@@ -15,7 +15,7 @@ click-didyoumean==0.3.1; python_full_version >= '3.6.2'
 click-plugins==1.1.1
 click-repl==0.3.0; python_version >= '3.6'
 cryptography==42.0.8; python_version >= '3.7'
-django==4.2.13; python_version >= '3.8'
+django==4.2.14; python_version >= '3.8'
 django-cors-headers==3.13.0; python_version >= '3.7'
 django-environ==0.10.0; python_version >= '3.5' and python_version < '4'
 django-extensions==3.2.1; python_version >= '3.6'
@@ -25,6 +25,8 @@ django-tenants==3.5.0
 djangorestframework==3.15.2; python_version >= '3.8'
 djangorestframework-csv==2.1.1
 ecs-logging==2.0.0; python_version >= '3.6'
+grpcio==1.64.1
+grpcio-status==1.64.1
 gunicorn==22.0.0; python_version >= '3.7'
 idna==3.7; python_version >= '3.5'
 jinja2==3.1.4; python_version >= '3.7'
@@ -36,12 +38,14 @@ markupsafe==2.1.5; python_version >= '3.7'
 packaging==24.1; python_version >= '3.8'
 prometheus-client==0.15.0; python_version >= '3.6'
 prompt-toolkit==3.0.47; python_full_version >= '3.7.0'
+protoc-gen-validate==1.0.4
 psycopg2==2.9.5; python_version >= '3.6'
 psycopg2-binary==2.9.5; python_version >= '3.6'
 pycparser==2.22; python_version >= '3.8'
 python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pytz==2022.2.1
 redis==5.0.0; python_version >= '3.7'
+relations-grpc-clients-python-kessel-project==0.2.1
 requests==2.32.3; python_version >= '3.8'
 s3transfer==0.6.2; python_version >= '3.7'
 sentry-sdk==1.18.0

tests/internal/test_views.py

@@ -464,7 +464,7 @@ def test_run_migrations_of_roles(self, migration_mock):
             **self.request.META,
         )
         migration_mock.assert_called_once_with(
-            {"exclude_apps": ["rbac", "costmanagement"], "orgs": ["acct00001", "acct00002"]}
+            {"exclude_apps": ["rbac", "costmanagement"], "orgs": ["acct00001", "acct00002"], "write_db": False}
         )
         self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
         self.assertEqual(
@@ -478,7 +478,7 @@ def test_run_migrations_of_roles(self, migration_mock):
             f"/_private/api/utils/role_migration/",
             **self.request.META,
         )
-        migration_mock.assert_called_once_with({"exclude_apps": [], "orgs": []})
+        migration_mock.assert_called_once_with({"exclude_apps": [], "orgs": [], "write_db": False})
         self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
         self.assertEqual(
             response.content.decode(),

tests/migration_tool/tests_migrate.py

@@ -83,5 +83,5 @@ def test_migration_of_roles(self, logger_mock):
         migrate_roles(**kwargs)
         self.assertEqual(
             len(logger_mock.info.call_args_list),
-            18,
+            20,
         )