Merge pull request #19 from keystone-scim/mongodb-store

Add MongoDB store

.github/workflows/unit_tests.yaml

@@ -35,7 +35,14 @@ jobs:
           poetry-version: 1.1.13
       - name: Install dependencies
         run: poetry install
+      - name: Start MongoDB
+        uses: supercharge/[email protected]
+        with:
+          mongodb-version: 5.0
+          mongodb-username: root
+          mongodb-password: example
+          mongodb-db: scim2UnitTest
+          mongodb-port: 27017
       - name: Unit tests
         run: |
-          ISTORE_PG_SCHEMA_IGNORE_THIS=unit_test_$(date +%s) \
           poetry run pytest tests/unit -p no:warnings --asyncio-mode=strict

Makefile

@@ -31,6 +31,13 @@ integration-tests-cosmos-store:
 	$(POETRY_BIN) run pytest tests/integration -p no:warnings --verbose --asyncio-mode=strict ; \
 	$(POETRY_BIN) run tests/integration/scripts/cleanup.py
 
+.PHONY: integration-tests-mongo-store
+integration-tests-mongo-store: export CONFIG_PATH=./config/integration-tests-mongo-store.yaml
+integration-tests-mongo-store: export STORE_MONGO_DATABASE=scim_int_tsts_$(shell date +%s)
+integration-tests-mongo-store:
+	$(POETRY_BIN) run pytest tests/integration -p no:warnings --verbose --asyncio-mode=strict ; \
+	$(POETRY_BIN) run tests/integration/scripts/cleanup.py
+
 .PHONY: security-tests
 security-tests:
 	$(POETRY_BIN) run bandit -r ./keystone

README.md

@@ -49,20 +49,23 @@ operations with an identity manager that supports user provisioning (e.g., Azure
 you can use **Keystone** to persist directory changes. Keystone v0.1.0 supports two
 persistence layers: PostgreSQL and Azure Cosmos DB.
 
+<div align="center">
+    <img src="./logo/how-it-works.png" alt="logo" />
+</div>
+
+
 Key features:
 
 * A compliant [SCIM 2.0 REST API](https://datatracker.ietf.org/doc/html/rfc7644)
   implementation for Users and Groups.
-* Stateless container - deploy it anywhere you want (e.g., Kubernetes).
+* Stateless container - deploy it anywhere you want (e.g., Kubernetes) and bring your own storage.
 * Pluggable store for users and groups. Current supported storage technologies:
   * [Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db/introduction)
   * [PostgreSQL](https://www.postgresql.org) (version 10 or higher)
-
+  * [MongoDB](https://www.mongodb.com/docs/) (version 3.6 or higher)
 * Azure Key Vault bearer token retrieval.
-* Extensible stores.
-
-Can't use Cosmos DB or PostgreSQL?  Open an issue and/or consider
-[becoming a contributor](./CONTRIBUTING.md).
+* Extensible store: Can't use MongoDB, Cosmos DB, or PostgreSQL?  Open an issue and/or consider
+  [becoming a contributor](./CONTRIBUTING.md) by implementing your own data store.
 
 ## Configure the API
 

config/dev-cosmos.yaml

@@ -1,6 +1,7 @@
 store:
   type: CosmosDB
-  cosmos_account_uri: <YOUR_COSMOS_ACCOUNT_URI>
-  cosmos_account_key: <YOUR_COSMOS_ACCOUNT_KEY>
+  cosmos:
+    account_uri: <YOUR_COSMOS_ACCOUNT_URI>
+    account_key: <YOUR_COSMOS_ACCOUNT_KEY>
 authentication:
   secret: not-so-secret

config/integration-tests-mongo-store.yaml

@@ -0,0 +1,8 @@
+store:
+  type: MongoDb
+  mongo:
+    host: localhost
+    port: 27017
+    tls: false
+authentication:
+  secret: not-so-secret

config/integration-tests-pg-store.yaml

@@ -1,5 +1,6 @@
 store:
   type: PostgreSQL
-  pg_schema: public
+  pg:
+    schema: public
 authentication:
   secret: not-so-secret

docker-compose.yaml

@@ -0,0 +1,23 @@
+# Use root/example as user/password credentials
+version: '3.1'
+
+services:
+
+  mongo:
+    image: mongo
+    restart: always
+    ports:
+      - 27017:27017
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: example
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: root
+      ME_CONFIG_MONGODB_ADMINPASSWORD: example
+      ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/

keystone/cmd.py

@@ -3,6 +3,7 @@
 import os
 
 from keystone import VERSION, LOGO, InterceptHandler
+from keystone.store.mongodb_store import set_up
 from keystone.store.postgresql_store import set_up_schema
 from keystone.util.logger import get_log_handler
 
@@ -46,8 +47,10 @@ async def print_logo(_logger):
 
 
 async def serve(host: str = "0.0.0.0", port: int = 5001):
-    if CONFIG.get("store.type") == "PostgreSQL":
+    if CONFIG.get("store.pg.host") is not None:
         set_up_schema()
+    elif CONFIG.get("store.mongo.host") or CONFIG.get("store.mongo.dsn"):
+        await set_up()
 
     error_handling_mw = await get_error_handling_mw()
 

keystone/rest/__init__.py

@@ -1,6 +1,7 @@
 from aiohttp_catcher import Catcher, canned, catch
 from azure.cosmos.exceptions import CosmosResourceNotFoundError
 from psycopg2.errors import UniqueViolation
+from pymongo.errors import DuplicateKeyError
 
 from keystone.models import DEFAULT_ERROR_SCHEMA
 from keystone.util.exc import ResourceNotFound, ResourceAlreadyExists, UnauthorizedRequest
@@ -9,14 +10,17 @@
 async def get_error_handling_mw():
     catcher = Catcher(code="status", envelope="detail")
     err_schemas = {"schemas": [DEFAULT_ERROR_SCHEMA]}
-    await catcher.add_scenarios(*[sc.with_additional_fields(err_schemas) for sc in canned.AIOHTTP_SCENARIOS])
     await catcher.add_scenarios(
+        *[sc.with_additional_fields(err_schemas) for sc in canned.AIOHTTP_SCENARIOS],
+
         catch(ResourceNotFound).with_status_code(404).and_stringify().with_additional_fields(err_schemas),
+
         catch(CosmosResourceNotFoundError).with_status_code(404).and_return(
             "Resource not found").with_additional_fields(err_schemas),
-        catch(UniqueViolation).with_status_code(409).and_return(
+
+        catch(UniqueViolation, DuplicateKeyError, ResourceAlreadyExists).with_status_code(409).and_return(
             "Resource already exists").with_additional_fields(err_schemas),
-        catch(ResourceAlreadyExists).with_status_code(409).and_stringify().with_additional_fields(err_schemas),
+
         catch(UnauthorizedRequest).with_status_code(401).and_return("Unauthorized request").with_additional_fields(
             err_schemas)
     )

keystone/rest/group.py

@@ -12,7 +12,7 @@
 
 from keystone.models import ListQueryParams, ErrorResponse, DEFAULT_LIST_SCHEMA
 from keystone.models.group import Group, PatchGroupOp, ListGroupsResponse
-from keystone.store import BaseStore, RDBMSStore
+from keystone.store import BaseStore, RDBMSStore, DocumentStore, DatabaseStore
 from keystone.util.store_util import Stores
 
 LOGGER = logging.getLogger(__name__)
@@ -97,7 +97,9 @@ async def _execute_group_operation(self, operation: Dict) -> Dict:
                     ]
                 }
             """
-            is_rdbms = isinstance(group_store, RDBMSStore)
+            is_dbs = isinstance(group_store, DatabaseStore)
+            is_rdbmss = isinstance(group_store, RDBMSStore)
+            is_docs = isinstance(group_store, DocumentStore)
             group_id = self.request.match_info["group_id"]
             if hasattr(group_store, "resource_db"):
                 group = group_store.resource_db[group_id]
@@ -113,20 +115,23 @@ async def _execute_group_operation(self, operation: Dict) -> Dict:
             if op_path and op_path.startswith("members[") and not op_value:
                 # Remove members with a path:
                 _filter = op_path.strip("members[").strip("]")
-                if is_rdbms:
+                if is_dbs:
                     if op_type == "remove":
-                        selected_members = await group_store.search_members(
-                            _filter=_filter, group_id=group_id
-                        )
-                        _ = await group_store.remove_users_from_group(
-                            user_ids=[m.get("id") for m in selected_members], group_id=group_id
-                        )
+                        if is_rdbmss:
+                            selected_members = await group_store.search_members(
+                                _filter=_filter, group_id=group_id
+                            )
+                            _ = await group_store.remove_users_from_group(
+                                user_ids=[m.get("id") for m in selected_members], group_id=group_id
+                            )
+                        else:
+                            _filter = _filter.replace("value", "id").replace("display", "userName")
+                            u, _ = await user_store.search(_filter=_filter)
+                            _ = await group_store.remove_users_from_group(
+                                user_ids=[m.get("id") for m in u], group_id=group_id
+                            )
                     elif op_type == "add":
                         selected_members, _ = await user_store.search(_filter=_filter.replace("value", "id"))
-                        LOGGER.debug(str(selected_members))
-                        LOGGER.debug(op_type)
-                        LOGGER.debug(_filter)
-                        # TODO: Need to handle:
                         for m in selected_members:
                             _ = await group_store.add_user_to_group(
                                 user_id=m.get("id"), group_id=group_id
@@ -143,15 +148,15 @@ async def _execute_group_operation(self, operation: Dict) -> Dict:
                             _ = await group["members_store"].delete(member.get("value"))
                         return group
             if op_path == "members" and op_type == "replace" and op_value:
-                if is_rdbms:
+                if is_dbs:
                     _ = await group_store.set_group_members(users=op_value, group_id=group_id)
                     return group
                 else:
                     return await group_store.update(group_id, **{"members": op_value})
             if op_path == "members" and op_value:
                 for member in op_value:
                     member_id = member.get("value")
-                    if is_rdbms:
+                    if is_dbs:
                         if op_type == "add":
                             _ = await group_store.add_user_to_group(member.get("value"), group_id)
                         elif op_type == "remove":

keystone/store/__init__.py

@@ -50,8 +50,7 @@ async def _sanitize(self, resource: Dict) -> Dict:
         return s_resource
 
 
-class RDBMSStore(BaseStore, ABC):
-
+class DatabaseStore(BaseStore, ABC):
     async def remove_users_from_group(self, user_ids: List[str], group_id: str):
         raise NotImplementedError("Method 'remove_user_from_group' not implemented")
 
@@ -61,5 +60,12 @@ async def add_user_to_group(self, user_id: str, group_id: str):
     async def set_group_members(self, users: List[Dict], group_id: str):
         raise NotImplementedError("Method 'set_group_members' not implemented")
 
+
+class RDBMSStore(DatabaseStore, ABC):
+
     async def search_members(self, _filter: str, group_id: str):
         raise NotImplementedError("Method 'search_members' not implemented")
+
+
+class DocumentStore(DatabaseStore, ABC):
+    pass