From 1380888cbc5dad92bc4ab47ff836898d4295da1b Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Mon, 25 Dec 2023 23:23:38 +0100 Subject: [PATCH 01/16] Configure taxii2 server --- .gitignore | 1 + opentaxii/defaults.yml | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index e42b1ae0..e3f2e0d6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ # Distribution / packaging .Python +.venv/ env/ build/ develop-eggs/ diff --git a/opentaxii/defaults.yml b/opentaxii/defaults.yml index 214e866a..bb30c3ed 100644 --- a/opentaxii/defaults.yml +++ b/opentaxii/defaults.yml @@ -26,6 +26,16 @@ taxii1: create_tables: yes taxii2: + public_discovery: true + allow_custom_properties: true + description: "TAXII2 Server" + title: "Taxii2 Service" + max_content_length: 2048 + persistence_api: + class: opentaxii.persistence.sqldb.Taxii2SQLDatabaseAPI + parameters: + db_connection: sqlite:////tmp/data2.db + create_tables: yes logging: opentaxii: info From b8a5b75583a84dd7e00023bb90a8ed7c30dfca0b Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Wed, 27 Dec 2023 21:22:21 +0100 Subject: [PATCH 02/16] Fix problem with encoding of UUIDs --- .gitignore | 1 + opentaxii/taxii2/http.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index e3f2e0d6..5c718ea3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python .venv/ +.vscode/ env/ build/ develop-eggs/ diff --git a/opentaxii/taxii2/http.py b/opentaxii/taxii2/http.py index a0408d33..3dafbec0 100644 --- a/opentaxii/taxii2/http.py +++ b/opentaxii/taxii2/http.py @@ -3,12 +3,22 @@ from typing import Dict, Optional from flask import Response, make_response +from uuid import UUID -def make_taxii2_response(data, status: Optional[int] = 200, extra_headers: Optional[Dict] = None) -> Response: +class JSONEncoderWithUUID(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, UUID): + return str(obj) + return super().default(obj) + + +def make_taxii2_response( + data, status: Optional[int] = 200, extra_headers: Optional[Dict] = None +) -> Response: """Turn input data into valid taxii2 response.""" if not isinstance(data, str): - data = json.dumps(data) + data = json.dumps(data, cls=JSONEncoderWithUUID) response = make_response((data, status)) response.content_type = "application/taxii+json;version=2.1" response.headers.update(extra_headers or {}) From 067831a3ca901cddbce05b5ff41515d931a28eb3 Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Wed, 27 Dec 2023 21:23:13 +0100 Subject: [PATCH 03/16] Add public argument to create public api roots --- opentaxii/cli/persistence.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/opentaxii/cli/persistence.py b/opentaxii/cli/persistence.py index 9efca24a..ec0e1cdb 100644 --- a/opentaxii/cli/persistence.py +++ b/opentaxii/cli/persistence.py @@ -105,10 +105,16 @@ def add_api_root(): "--default", action="store_true", help="Set as default api root" ) + parser.add_argument( + "--public", action="store_true", help="Create a public api root" + ) args = parser.parse_args() with app.app_context(): app.taxii_server.servers.taxii2.persistence.api.add_api_root( - title=args.title, description=args.description, default=args.default + title=args.title, + description=args.description, + default=args.default, + is_public=args.public, ) From 18d7a8936260804f2cd648d58ec69d0427ec851b Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Thu, 28 Dec 2023 00:58:10 +0100 Subject: [PATCH 04/16] Add authentication for taxii2 - Added example on how to define accounts with permissions for taxii2 and taxii1. - Fixed authentication for taxii2. Now accounts can be defined to access private collections with 'read' or 'modify' access. --- examples/data-configuration-accounts.yml | 13 +++++++++++++ opentaxii/auth/manager.py | 20 ++++++++++++++++++-- opentaxii/taxii2/entities.py | 5 +++-- 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 examples/data-configuration-accounts.yml diff --git a/examples/data-configuration-accounts.yml b/examples/data-configuration-accounts.yml new file mode 100644 index 00000000..ddc6f70b --- /dev/null +++ b/examples/data-configuration-accounts.yml @@ -0,0 +1,13 @@ +--- +accounts: + - username: user + password: user + permissions: + taxii1: + firstcollection: read + taxii2: + ec5f3df5-2b43-4037-9ae1-fe168554ed08: + PrivCollectionAlias: read + - username: admin + password: admin + is_admin: yes diff --git a/opentaxii/auth/manager.py b/opentaxii/auth/manager.py index aa4774ab..ad89db15 100644 --- a/opentaxii/auth/manager.py +++ b/opentaxii/auth/manager.py @@ -44,13 +44,29 @@ def update_account(self, account, password): NOTE: Additional method that is only used in the helper scripts shipped with OpenTAXII. ''' - for colname, permission in list(account.permissions.items()): + permission_collections = {} + # Check for taxii1 collections + for colname, permission in list(account.permissions.get("taxii1", {}).items()): collection = self.server.servers.taxii1.persistence.get_collection(colname) if not collection: log.warning( "update_account.unknown_collection", collection=colname) - account.permissions.pop(colname) + else: + permission_collections[colname] = permission + + # Check for taxii2 collections + for api_root, collections in list(account.permissions.get("taxii2", {}).items()): + for colname, permission in collections.items(): + collection = self.server.servers.taxii2.persistence.get_collection(api_root, colname) + if not collection: + log.warning( + "update_account.unknown_collection", + collection=colname) + else: + permission_collections[str(collection.id)] = permission + + account.permissions = permission_collections account = self.api.update_account(account, password) return account diff --git a/opentaxii/taxii2/entities.py b/opentaxii/taxii2/entities.py index e660f225..bbe8573a 100644 --- a/opentaxii/taxii2/entities.py +++ b/opentaxii/taxii2/entities.py @@ -63,10 +63,11 @@ def __init__( def can_read(self, account: Optional[Account]): """Determine if `account` is allowed to read from this collection.""" + permission = account.permissions.get(str(self.id), "") return self.is_public or ( account and ( - account.is_admin or "read" in set(account.permissions.get(self.id, [])) + account.is_admin or "read" in permission or "modify" in permission ) ) @@ -75,7 +76,7 @@ def can_write(self, account: Optional[Account]): return self.is_public_write or ( account and ( - account.is_admin or "write" in set(account.permissions.get(self.id, [])) + account.is_admin or "modify" in account.permissions.get(self.id, "") ) ) From 171e99bfb47b9280818a0f488ea3c6945271f150 Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Fri, 29 Dec 2023 17:10:14 +0100 Subject: [PATCH 05/16] Fix error for opentaxii-sync-data When a collection that does not exist is used in a data-configuration file, a warning is produced --- opentaxii/auth/manager.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/opentaxii/auth/manager.py b/opentaxii/auth/manager.py index ad89db15..89c9f657 100644 --- a/opentaxii/auth/manager.py +++ b/opentaxii/auth/manager.py @@ -1,4 +1,5 @@ import structlog +from opentaxii.persistence.exceptions import DoesNotExistError log = structlog.getLogger(__name__) @@ -58,11 +59,12 @@ def update_account(self, account, password): # Check for taxii2 collections for api_root, collections in list(account.permissions.get("taxii2", {}).items()): for colname, permission in collections.items(): - collection = self.server.servers.taxii2.persistence.get_collection(api_root, colname) - if not collection: + try: + collection = self.server.servers.taxii2.persistence.get_collection(api_root, colname) + except DoesNotExistError: log.warning( "update_account.unknown_collection", - collection=colname) + api_root=api_root, collection=colname) else: permission_collections[str(collection.id)] = permission From 8c60407a48b08c0e629c67908eabec8d499f8f5b Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Fri, 29 Dec 2023 17:45:08 +0100 Subject: [PATCH 06/16] Fix problem with can_read function If no credentials are provided and you try to access a public collection, there is an error. Now it has been fixed. --- opentaxii/taxii2/entities.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/opentaxii/taxii2/entities.py b/opentaxii/taxii2/entities.py index bbe8573a..113cf857 100644 --- a/opentaxii/taxii2/entities.py +++ b/opentaxii/taxii2/entities.py @@ -63,11 +63,12 @@ def __init__( def can_read(self, account: Optional[Account]): """Determine if `account` is allowed to read from this collection.""" - permission = account.permissions.get(str(self.id), "") return self.is_public or ( account and ( - account.is_admin or "read" in permission or "modify" in permission + account.is_admin or + "read" in account.permissions.get(str(self.id), "") or + "modify" in account.permissions.get(str(self.id), "") ) ) From 63fae86f7fba83c1ea213eb68acd4707b6b160b7 Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Fri, 29 Dec 2023 18:22:40 +0100 Subject: [PATCH 07/16] Fixed write access to collections Also expanded the example for taxii2 data-configuration-accounts --- examples/data-configuration-accounts.yml | 14 ++++++++++---- opentaxii/taxii2/entities.py | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/examples/data-configuration-accounts.yml b/examples/data-configuration-accounts.yml index ddc6f70b..721a6b15 100644 --- a/examples/data-configuration-accounts.yml +++ b/examples/data-configuration-accounts.yml @@ -1,13 +1,19 @@ --- accounts: - - username: user - password: user + - username: user_read + password: user_read permissions: taxii1: firstcollection: read taxii2: - ec5f3df5-2b43-4037-9ae1-fe168554ed08: - PrivCollectionAlias: read + ea9cdf30-dc20-4ec3-b308-bf658d865cae: + privCollectionAlias: read + - username: user_write + password: user_write + permissions: + taxii2: + ea9cdf30-dc20-4ec3-b308-bf658d865cae: + privCollectionAlias: modify - username: admin password: admin is_admin: yes diff --git a/opentaxii/taxii2/entities.py b/opentaxii/taxii2/entities.py index 113cf857..c2787fbb 100644 --- a/opentaxii/taxii2/entities.py +++ b/opentaxii/taxii2/entities.py @@ -77,7 +77,8 @@ def can_write(self, account: Optional[Account]): return self.is_public_write or ( account and ( - account.is_admin or "modify" in account.permissions.get(self.id, "") + account.is_admin or + "modify" in account.permissions.get(str(self.id), "") ) ) From d54a7543e89b30f6c624700cdb7f81057fce97e4 Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Tue, 2 Jan 2024 17:20:06 +0100 Subject: [PATCH 08/16] Add script for push, pull, and subscribe The script can be used to test push, pull, and subscribe functionality with taxii2-client --- examples/data-configuration-accounts.yml | 4 +- examples/pullpushsub.py | 175 +++++++++++++++++++++++ examples/stix/nettool.stix.json | 11 ++ 3 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 examples/pullpushsub.py create mode 100644 examples/stix/nettool.stix.json diff --git a/examples/data-configuration-accounts.yml b/examples/data-configuration-accounts.yml index 721a6b15..896cd718 100644 --- a/examples/data-configuration-accounts.yml +++ b/examples/data-configuration-accounts.yml @@ -6,13 +6,13 @@ accounts: taxii1: firstcollection: read taxii2: - ea9cdf30-dc20-4ec3-b308-bf658d865cae: + ea9cdf30-root-idc3-b308-bf658d865cae: privCollectionAlias: read - username: user_write password: user_write permissions: taxii2: - ea9cdf30-dc20-4ec3-b308-bf658d865cae: + ea9cdf30-root-idc3-b308-bf658d865cae: privCollectionAlias: modify - username: admin password: admin diff --git a/examples/pullpushsub.py b/examples/pullpushsub.py new file mode 100644 index 00000000..b6e5ab0d --- /dev/null +++ b/examples/pullpushsub.py @@ -0,0 +1,175 @@ +import json +import sys +import requests +from taxii2client.v21 import Server +from taxii2client.exceptions import AccessError +from uuid import uuid4 +from time import sleep + +# Define your TAXII server and collection details +OPENTAXII_URL = "http://localhost:9000/" +TAXII2_SERVER = OPENTAXII_URL + "taxii2/" +USERNAME = "user_write" +PASSWORD = "user_write" + + +def pull_data(api_root_url, collection): + # Pull data from the TAXII collection + try: + # Pull data from the collection + data = collection.get_objects() + print(f"Num objects pulled: {len(data.get('objects', []))}") + except AccessError: + print("[Pull Error] The user does not have write access") + return None + + return data + + +def push_data(api_root_url, collection): + # load stix data and push it + with open("stix/nettool.stix.json", "r") as f: + stix_loaded = json.load(f) + + stix_type = stix_loaded["type"] + stix_id = stix_type + "--" + str(uuid4()) + stix_loaded["id"] = stix_id + + envelope_data = { + "more": False, + "objects": [stix_loaded], + } + try: + # Push data to the collection + collection.add_objects(envelope_data) + print("Data pushed successfully.") + except AccessError: + print("[Push Error] The user does not have write access") + + +def subscribe(api_root_url, collection): + total_objects_pulled = 0 + added_after = None + + # Get Authentication Token + response = requests.post( + OPENTAXII_URL + "management/auth", + headers={ + "Content-Type": "application/json", + }, + json={ + "username": USERNAME, + "password": PASSWORD, + }, + ) + auth_token = response.json().get("token", None) + + while True: + if added_after is None: + url = api_root_url + "collections/" + collection.id + "/objects/" + else: + url = ( + api_root_url + + "collections/" + + collection.id + + f"/objects/?added_after={added_after}" + ) + + # Get all objects from added_after + response = requests.get( + url=url, + headers={ + "Authorization": f"Bearer {auth_token}", + }, + ) + taxii_env = response.json() + objects = taxii_env.get("objects", []) + + print(f"Read {len(objects)} objects from the TAXII2 server") + if len(objects) > 0: + added_after = response.headers.get("X-TAXII-Date-Added-Last", "") + + sleep(3) + + +def not_an_action(collection): + print("That is not an option!") + + +def main(): + server = Server( + TAXII2_SERVER, + user=USERNAME, + password=PASSWORD, + ) + print(server.title) + print("=" * len(server.title)) + + print("Select an API Root:") + print(server.api_roots) + print() + for index, aroot in enumerate(server.api_roots, start=1): + print(f"{index}.") + try: + print(f"Title: {aroot.title}") + print(f"Description: {aroot.description}") + print(f"Versions: {aroot.versions}") + except Exception: + print( + "This API Root is not public.\nYou need to identify to see this API Root" + ) + print() + + aroot_choice = input("Enter the number of your choice: ") + try: + aroot_choice = int(aroot_choice) + selected_api_root = server.api_roots[aroot_choice - 1] + collections_l = selected_api_root.collections + except (ValueError, IndexError): + print("Invalid choice. Please enter a valid number.") + sys.exit() + except Exception as e: + print(e) + print("You cannot access this API Root. You need to authenticate.") + sys.exit() + + for index, coll in enumerate(collections_l, start=1): + print(f"{index}.") + print(f"\tId: {coll.id}") + print(f"\tTitle: {coll.title}") + print(f"\tAlias: {coll.alias}") + print(f"\tDescription: {coll.description}") + print(f"\tMedia Types: {coll.media_types}") + print(f"\tCan Read: {coll.can_read}") + print(f"\tCan Write: {coll.can_write}") + print(f"\tObjects URL: {coll.objects_url}") + print(f"\tCustom Properties: {coll.custom_properties}") + print() + + coll_choice = input("Enter the number of your choice: ") + try: + coll_choice = int(coll_choice) + selected_collection = selected_api_root.collections[coll_choice - 1] + except (ValueError, IndexError): + print("Invalid choice. Please enter a valid number.") + sys.exit() + + actions_d = { + 1: pull_data, + 2: push_data, + 3: subscribe, + } + + while True: + print() + print("1: Pull") + print("2: Push") + print("3: Subscribe") + action_choice = int(input("Enter the number of your choice: ")) + action_func = actions_d.get(action_choice, not_an_action) + action_func(selected_api_root.url, selected_collection) + print() + + +if __name__ == "__main__": + main() diff --git a/examples/stix/nettool.stix.json b/examples/stix/nettool.stix.json new file mode 100644 index 00000000..5ce5f615 --- /dev/null +++ b/examples/stix/nettool.stix.json @@ -0,0 +1,11 @@ +{ + "modified": "2023-07-25T19:25:59.767Z", + "name": "Net", + "description": "The [Net](https://attack.mitre.org/software/S0039) utility is a component of the Windows operating system. It is used in command-line operations for control of users, groups, services, and network connections. (Citation: Microsoft Net Utility)\n\n[Net](https://attack.mitre.org/software/S0039) has a great deal of functionality, (Citation: Savill 1999) much of which is useful for an adversary, such as gathering system and network information for Discovery, moving laterally through [SMB/Windows Admin Shares](https://attack.mitre.org/techniques/T1021/002) using net use commands, and interacting with services. The net1.exe utility is executed for certain functionality when net.exe is run and can be used directly in commands such as net1 user.", + "type": "tool", + "id": "tool--03342581-f790-4f03-ba41-e82e67392e25", + "created": "2017-05-31T21:32:31.601Z", + "revoked": false, + "external_references": [], + "spec_version": "2.1" +} From 056eb8d48e8d0cd424a727ffe3638a5d14aed73b Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Tue, 2 Jan 2024 17:25:24 +0100 Subject: [PATCH 09/16] Fix docker-compose.yml docker-compose.yml was missing version and services. --- examples/docker-compose.yml | 105 ++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml index d865d20d..d7673585 100644 --- a/examples/docker-compose.yml +++ b/examples/docker-compose.yml @@ -1,54 +1,57 @@ -db: - image: postgres:9.4 - environment: - POSTGRES_USER: user - POSTGRES_PASSWORD: password - POSTGRES_DB: opentaxii +version: '3' -authdb: - image: postgres:9.4 - environment: - POSTGRES_USER: user1 - POSTGRES_PASSWORD: password1 - POSTGRES_DB: opentaxii1 +services: + db: + image: postgres:9.4 + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: opentaxii -opentaxii: - image: eclecticiq/opentaxii - environment: - OPENTAXII_AUTH_SECRET: secret - OPENTAXII_DOMAIN: 192.168.59.103:9000 - OPENTAXII_USER: user - OPENTAXII_PASS: pass - DATABASE_HOST: db - DATABASE_NAME: opentaxii - DATABASE_USER: user - DATABASE_PASS: password - AUTH_DATABASE_HOST: authdb - AUTH_DATABASE_NAME: opentaxii1 - AUTH_DATABASE_USER: user1 - AUTH_DATABASE_PASS: password1 - volumes: - - ./:/input:ro - ports: - - 9000:9000 - links: - - db:db - - authdb:authdb + authdb: + image: postgres:9.4 + environment: + POSTGRES_USER: user1 + POSTGRES_PASSWORD: password1 + POSTGRES_DB: opentaxii1 -opentaxii2: - image: eclecticiq/opentaxii - environment: - OPENTAXII_AUTH_SECRET: secrettwo - OPENTAXII_DOMAIN: 192.168.59.103 - OPENTAXII_USER: user1 - OPENTAXII_PASS: pass1 - DATABASE_HOST: authdb - DATABASE_NAME: opentaxii1 - DATABASE_USER: user1 - DATABASE_PASS: password1 - volumes: - - ./:/input:ro - ports: - - 9001:9000 - links: - - authdb:authdb + opentaxii: + image: eclecticiq/opentaxii + environment: + OPENTAXII_AUTH_SECRET: secret + OPENTAXII_DOMAIN: 192.168.59.103:9000 + OPENTAXII_USER: user + OPENTAXII_PASS: pass + DATABASE_HOST: db + DATABASE_NAME: opentaxii + DATABASE_USER: user + DATABASE_PASS: password + AUTH_DATABASE_HOST: authdb + AUTH_DATABASE_NAME: opentaxii1 + AUTH_DATABASE_USER: user1 + AUTH_DATABASE_PASS: password1 + volumes: + - ./:/input:ro + ports: + - 9000:9000 + links: + - db:db + - authdb:authdb + + opentaxii2: + image: eclecticiq/opentaxii + environment: + OPENTAXII_AUTH_SECRET: secrettwo + OPENTAXII_DOMAIN: 192.168.59.103 + OPENTAXII_USER: user1 + OPENTAXII_PASS: pass1 + DATABASE_HOST: authdb + DATABASE_NAME: opentaxii1 + DATABASE_USER: user1 + DATABASE_PASS: password1 + volumes: + - ./:/input:ro + ports: + - 9001:9000 + links: + - authdb:authdb From 1e5658dea3b2cb03d4e56d2e4a14674e3d4f831e Mon Sep 17 00:00:00 2001 From: "Alejandro A. Moreno Sancho" Date: Sun, 14 Jan 2024 00:00:26 +0100 Subject: [PATCH 10/16] Deleting unused var --- examples/pullpushsub.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/pullpushsub.py b/examples/pullpushsub.py index b6e5ab0d..ab10b4f1 100644 --- a/examples/pullpushsub.py +++ b/examples/pullpushsub.py @@ -48,7 +48,6 @@ def push_data(api_root_url, collection): def subscribe(api_root_url, collection): - total_objects_pulled = 0 added_after = None # Get Authentication Token From 5b7128e9e51e02b0f280ec0bdfdcdd1c72d200b2 Mon Sep 17 00:00:00 2001 From: Priyank Bhuva Date: Tue, 5 Nov 2024 18:28:52 +0530 Subject: [PATCH 11/16] Fixes in create account command and taxii2 delete objects API --- opentaxii/cli/auth.py | 1 + opentaxii/persistence/sqldb/api.py | 1 + 2 files changed, 2 insertions(+) diff --git a/opentaxii/cli/auth.py b/opentaxii/cli/auth.py index 428b81fe..5630a4b6 100644 --- a/opentaxii/cli/auth.py +++ b/opentaxii/cli/auth.py @@ -21,6 +21,7 @@ def create_account(argv=None): account = app.taxii_server.auth.api.create_account( username=args.username, password=args.password, + is_admin=args.admin ) token = app.taxii_server.auth.authenticate( username=account.username, diff --git a/opentaxii/persistence/sqldb/api.py b/opentaxii/persistence/sqldb/api.py index 4ce29538..7b7cb5ae 100644 --- a/opentaxii/persistence/sqldb/api.py +++ b/opentaxii/persistence/sqldb/api.py @@ -1109,6 +1109,7 @@ def delete_object( ordered=False, ) query.delete("fetch") + self.db.session.commit() def get_versions( self, From 9d5438c5ca234a9958919d845de1969853a94275 Mon Sep 17 00:00:00 2001 From: Priyank Bhuva Date: Wed, 6 Nov 2024 12:29:21 +0530 Subject: [PATCH 12/16] Minor changes --- opentaxii/config.py | 2 ++ opentaxii/persistence/sqldb/api.py | 2 ++ opentaxii/server.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/opentaxii/config.py b/opentaxii/config.py index b8f2f87a..548cb723 100644 --- a/opentaxii/config.py +++ b/opentaxii/config.py @@ -64,6 +64,8 @@ class ServerConfig(dict): "title", "public_discovery", "allow_custom_properties", + "max_pagination_limit", + "default_pagination_limit" ) ALL_VALID_OPTIONS = VALID_BASE_OPTIONS + VALID_TAXII_OPTIONS + VALID_TAXII1_OPTIONS diff --git a/opentaxii/persistence/sqldb/api.py b/opentaxii/persistence/sqldb/api.py index 7b7cb5ae..8b113bb9 100644 --- a/opentaxii/persistence/sqldb/api.py +++ b/opentaxii/persistence/sqldb/api.py @@ -983,6 +983,8 @@ def add_objects( self.db.session.commit() job_details = [] for obj in objects: + if not obj.get("modified"): + obj["modified"] = obj.get("created") or datetime.datetime.now(datetime.timezone.utc).isoformat() version = datetime.datetime.strptime( obj["modified"], DATETIMEFORMAT ).replace(tzinfo=datetime.timezone.utc) diff --git a/opentaxii/server.py b/opentaxii/server.py index 928ae033..fefb236f 100644 --- a/opentaxii/server.py +++ b/opentaxii/server.py @@ -599,6 +599,7 @@ def collection_handler(self, api_root_id, collection_id_or_alias): ) def manifest_handler(self, api_root_id, collection_id_or_alias): filter_params = validate_list_filter_params(request.args, self.persistence.api) + filter_params["limit"] = self.config.get("max_pagination_limit") if filter_params.get("limit", self.config.get("max_pagination_limit")) > self.config.get("max_pagination_limit") else filter_params.get("limit", self.config.get("default_pagination_limit")) try: manifest, more = self.persistence.get_manifest( api_root_id=api_root_id, @@ -652,6 +653,7 @@ def objects_handler(self, api_root_id, collection_id_or_alias): def objects_get_handler(self, api_root_id, collection_id_or_alias): filter_params = validate_list_filter_params(request.args, self.persistence.api) + filter_params["limit"] = self.config.get("max_pagination_limit") if filter_params.get("limit", self.config.get("max_pagination_limit")) > self.config.get("max_pagination_limit") else filter_params.get("limit", self.config.get("default_pagination_limit")) try: objects, more, next_param = self.persistence.get_objects( api_root_id=api_root_id, From 54673b0f649ca9ab0dc4478d86d5d0de6598dcfc Mon Sep 17 00:00:00 2001 From: Priyank Bhuva Date: Wed, 6 Nov 2024 13:52:24 +0530 Subject: [PATCH 13/16] Minor fixes --- opentaxii/defaults.yml | 2 ++ opentaxii/persistence/sqldb/api.py | 4 ++-- opentaxii/server.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/opentaxii/defaults.yml b/opentaxii/defaults.yml index 214e866a..d1cda737 100644 --- a/opentaxii/defaults.yml +++ b/opentaxii/defaults.yml @@ -26,6 +26,8 @@ taxii1: create_tables: yes taxii2: + default_pagination_limit: 10 + max_pagination_limit: 1000 logging: opentaxii: info diff --git a/opentaxii/persistence/sqldb/api.py b/opentaxii/persistence/sqldb/api.py index 8b113bb9..bd22f534 100644 --- a/opentaxii/persistence/sqldb/api.py +++ b/opentaxii/persistence/sqldb/api.py @@ -984,7 +984,7 @@ def add_objects( job_details = [] for obj in objects: if not obj.get("modified"): - obj["modified"] = obj.get("created") or datetime.datetime.now(datetime.timezone.utc).isoformat() + obj["modified"] = obj.get("created") or datetime.datetime.now(datetime.timezone.utc).strftime(DATETIMEFORMAT) version = datetime.datetime.strptime( obj["modified"], DATETIMEFORMAT ).replace(tzinfo=datetime.timezone.utc) @@ -1006,7 +1006,7 @@ def add_objects( id=obj["id"], collection_id=collection_id, type=obj["id"].split("--")[0], - spec_version=obj["spec_version"], + spec_version=obj.get("spec_version", 2.1), date_added=datetime.datetime.now(datetime.timezone.utc), version=version, serialized_data={ diff --git a/opentaxii/server.py b/opentaxii/server.py index fefb236f..c5edad4f 100644 --- a/opentaxii/server.py +++ b/opentaxii/server.py @@ -671,7 +671,7 @@ def objects_get_handler(self, api_root_id, collection_id_or_alias): { "id": obj.id, "type": obj.type, - "spec_version": obj.type, + "spec_version": obj.spec_version, **obj.serialized_data, } for obj in objects @@ -754,7 +754,7 @@ def object_get_handler(self, api_root_id, collection_id_or_alias, object_id): { "id": obj.id, "type": obj.type, - "spec_version": obj.type, + "spec_version": obj.spec_version, **obj.serialized_data, } for obj in versions From 0bc2f59801dbce33859ea44ca04e11a6f8aa79f6 Mon Sep 17 00:00:00 2001 From: Priyank Bhuva Date: Tue, 12 Nov 2024 10:50:41 +0530 Subject: [PATCH 14/16] Fixed DB issues --- opentaxii/common/sqldb.py | 25 ++++++++++++++++--------- opentaxii/sqldb_helper.py | 22 +++++++++++++++++++--- opentaxii/utils.py | 15 ++++++++------- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/opentaxii/common/sqldb.py b/opentaxii/common/sqldb.py index 65720a1b..cb8842fc 100644 --- a/opentaxii/common/sqldb.py +++ b/opentaxii/common/sqldb.py @@ -7,21 +7,28 @@ except ImportError: from sqlalchemy.ext.declarative import DeclarativeMeta +selected_db = None + class BaseSQLDatabaseAPI: BASEMODEL: ClassVar[Type[DeclarativeMeta]] def __init__(self, db_connection, create_tables=False, **engine_parameters): super().__init__() - self.db = SQLAlchemyDB( - db_connection, - self.BASEMODEL, - session_options={ - "autocommit": False, - "autoflush": True, - }, - **engine_parameters - ) + global selected_db + if not selected_db: + selected_db = SQLAlchemyDB( + db_connection, + self.BASEMODEL, + session_options={ + "autocommit": False, + "autoflush": True, + }, + **engine_parameters + ) + # Use same db object in auth and taxii persistent to keep exact track of connection pools + self.db = selected_db + self.db.Model = self.db.extend_base_model(self.BASEMODEL) if create_tables: self.db.create_all_tables() diff --git a/opentaxii/sqldb_helper.py b/opentaxii/sqldb_helper.py index a80b0b55..1ead8110 100644 --- a/opentaxii/sqldb_helper.py +++ b/opentaxii/sqldb_helper.py @@ -1,7 +1,8 @@ from threading import get_ident -from sqlalchemy import engine, orm +from sqlalchemy import engine, orm, event, exc from sqlalchemy.orm.exc import UnmappedClassError +import os class _QueryProperty: @@ -31,6 +32,20 @@ def __init__(self, db_connection, base_model, session_options=None, **kwargs): self.Model = self.extend_base_model(base_model) self._session = None + @event.listens_for(self.engine, "connect") + def connect(dbapi_connection, connection_record): + connection_record.info["pid"] = os.getpid() + + @event.listens_for(self.engine, "checkout") + def checkout(dbapi_connection, connection_record, connection_proxy): + pid = os.getpid() + if connection_record.info["pid"] != pid: + connection_record.dbapi_connection = connection_proxy.dbapi_connection = None + raise exc.DisconnectionError( + "Connection record belongs to pid %s, " + "attempting to check out in pid %s" % (connection_record.info["pid"], pid) + ) + def extend_base_model(self, base): if not getattr(base, 'query_class', None): base.query_class = self.Query @@ -70,5 +85,6 @@ def create_all_tables(self): def init_app(self, app): @app.teardown_appcontext def shutdown_session(response_or_exc): - self.session.remove() - return response_or_exc + if self._session: + self._session.remove() + return response_or_exc \ No newline at end of file diff --git a/opentaxii/utils.py b/opentaxii/utils.py index 43e2e45a..cb8e973a 100644 --- a/opentaxii/utils.py +++ b/opentaxii/utils.py @@ -162,12 +162,13 @@ def emit(self, record): def sync_conf_dict_into_db(server, config, force_collection_deletion=False): - services = config.get("services", []) - sync_services(server.servers.taxii1, services) - collections = config.get("collections", []) - sync_collections( - server.servers.taxii1, collections, force_deletion=force_collection_deletion - ) + if server.servers.taxii1: + services = config.get("services", []) + sync_services(server.servers.taxii1, services) + collections = config.get("collections", []) + sync_collections( + server.servers.taxii1, collections, force_deletion=force_collection_deletion + ) accounts = config.get("accounts", []) sync_accounts(server, accounts) @@ -349,4 +350,4 @@ def inner(*args, **kwargs): inner.handles_own_auth = handles_own_auth return inner - return inner_decorator + return inner_decorator \ No newline at end of file From 2eee81e2d3da419141c7fed405ede94be332b185 Mon Sep 17 00:00:00 2001 From: meetghodasara-crest Date: Mon, 2 Dec 2024 14:11:00 +0530 Subject: [PATCH 15/16] Fixing the bug for the data versioning and data fields duplication by direct storing the serialized data --- opentaxii/persistence/sqldb/api.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/opentaxii/persistence/sqldb/api.py b/opentaxii/persistence/sqldb/api.py index bd22f534..3320fa1d 100644 --- a/opentaxii/persistence/sqldb/api.py +++ b/opentaxii/persistence/sqldb/api.py @@ -983,14 +983,13 @@ def add_objects( self.db.session.commit() job_details = [] for obj in objects: - if not obj.get("modified"): - obj["modified"] = obj.get("created") or datetime.datetime.now(datetime.timezone.utc).strftime(DATETIMEFORMAT) + modified_date = obj.get("modified") or obj.get("created") or datetime.datetime.now(datetime.timezone.utc).strftime(DATETIMEFORMAT) version = datetime.datetime.strptime( - obj["modified"], DATETIMEFORMAT + modified_date, DATETIMEFORMAT ).replace(tzinfo=datetime.timezone.utc) - if ( - not self.db.session.query(literal(True)) - .filter( + modified_object = True + if obj.get("modified") or obj.get("created"): + modified_object = not self.db.session.query(literal(True)).filter( self.db.session.query(taxii2models.STIXObject) .filter( taxii2models.STIXObject.id == obj["id"], @@ -998,9 +997,15 @@ def add_objects( taxii2models.STIXObject.version == version, ) .exists() - ) - .scalar() - ): + ).scalar() + else: + stored_object = self.db.session.query(taxii2models.STIXObject).filter( + taxii2models.STIXObject.id == obj["id"], + taxii2models.STIXObject.collection_id == collection_id, + ).order_by(taxii2models.STIXObject.date_added.desc()).first() + if stored_object and stored_object.serialized_data == obj: + modified_object = False + if modified_object: self.db.session.add( taxii2models.STIXObject( id=obj["id"], @@ -1009,11 +1014,7 @@ def add_objects( spec_version=obj.get("spec_version", 2.1), date_added=datetime.datetime.now(datetime.timezone.utc), version=version, - serialized_data={ - key: value - for (key, value) in obj.items() - if key not in ["id", "type", "spec_version"] - }, + serialized_data=obj, ) ) job_detail = taxii2models.JobDetail( From 482a56beb6acfdb6d266a9df574618b5a8c0532b Mon Sep 17 00:00:00 2001 From: meetghodasara-crest Date: Mon, 2 Dec 2024 15:38:54 +0530 Subject: [PATCH 16/16] Minor updates. --- opentaxii/server.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/opentaxii/server.py b/opentaxii/server.py index c5edad4f..80e77f5b 100644 --- a/opentaxii/server.py +++ b/opentaxii/server.py @@ -669,9 +669,6 @@ def objects_get_handler(self, api_root_id, collection_id_or_alias): "more": more, "objects": [ { - "id": obj.id, - "type": obj.type, - "spec_version": obj.spec_version, **obj.serialized_data, } for obj in objects @@ -752,9 +749,6 @@ def object_get_handler(self, api_root_id, collection_id_or_alias, object_id): "more": more, "objects": [ { - "id": obj.id, - "type": obj.type, - "spec_version": obj.spec_version, **obj.serialized_data, } for obj in versions