diff --git a/.gitignore b/.gitignore index e42b1ae0..5c718ea3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ __pycache__/ # Distribution / packaging .Python +.venv/ +.vscode/ env/ build/ develop-eggs/ diff --git a/examples/data-configuration-accounts.yml b/examples/data-configuration-accounts.yml new file mode 100644 index 00000000..896cd718 --- /dev/null +++ b/examples/data-configuration-accounts.yml @@ -0,0 +1,19 @@ +--- +accounts: + - username: user_read + password: user_read + permissions: + taxii1: + firstcollection: read + taxii2: + ea9cdf30-root-idc3-b308-bf658d865cae: + privCollectionAlias: read + - username: user_write + password: user_write + permissions: + taxii2: + ea9cdf30-root-idc3-b308-bf658d865cae: + privCollectionAlias: modify + - username: admin + password: admin + is_admin: yes diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml index d865d20d..d7673585 100644 --- a/examples/docker-compose.yml +++ b/examples/docker-compose.yml @@ -1,54 +1,57 @@ -db: - image: postgres:9.4 - environment: - POSTGRES_USER: user - POSTGRES_PASSWORD: password - POSTGRES_DB: opentaxii +version: '3' -authdb: - image: postgres:9.4 - environment: - POSTGRES_USER: user1 - POSTGRES_PASSWORD: password1 - POSTGRES_DB: opentaxii1 +services: + db: + image: postgres:9.4 + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: opentaxii -opentaxii: - image: eclecticiq/opentaxii - environment: - OPENTAXII_AUTH_SECRET: secret - OPENTAXII_DOMAIN: 192.168.59.103:9000 - OPENTAXII_USER: user - OPENTAXII_PASS: pass - DATABASE_HOST: db - DATABASE_NAME: opentaxii - DATABASE_USER: user - DATABASE_PASS: password - AUTH_DATABASE_HOST: authdb - AUTH_DATABASE_NAME: opentaxii1 - AUTH_DATABASE_USER: user1 - AUTH_DATABASE_PASS: password1 - volumes: - - ./:/input:ro - ports: - - 9000:9000 - links: - - db:db - - authdb:authdb + authdb: + image: postgres:9.4 + environment: + POSTGRES_USER: user1 + POSTGRES_PASSWORD: password1 + POSTGRES_DB: opentaxii1 -opentaxii2: - image: eclecticiq/opentaxii - environment: - OPENTAXII_AUTH_SECRET: secrettwo - OPENTAXII_DOMAIN: 192.168.59.103 - OPENTAXII_USER: user1 - OPENTAXII_PASS: pass1 - DATABASE_HOST: authdb - DATABASE_NAME: opentaxii1 - DATABASE_USER: user1 - DATABASE_PASS: password1 - volumes: - - ./:/input:ro - ports: - - 9001:9000 - links: - - authdb:authdb + opentaxii: + image: eclecticiq/opentaxii + environment: + OPENTAXII_AUTH_SECRET: secret + OPENTAXII_DOMAIN: 192.168.59.103:9000 + OPENTAXII_USER: user + OPENTAXII_PASS: pass + DATABASE_HOST: db + DATABASE_NAME: opentaxii + DATABASE_USER: user + DATABASE_PASS: password + AUTH_DATABASE_HOST: authdb + AUTH_DATABASE_NAME: opentaxii1 + AUTH_DATABASE_USER: user1 + AUTH_DATABASE_PASS: password1 + volumes: + - ./:/input:ro + ports: + - 9000:9000 + links: + - db:db + - authdb:authdb + + opentaxii2: + image: eclecticiq/opentaxii + environment: + OPENTAXII_AUTH_SECRET: secrettwo + OPENTAXII_DOMAIN: 192.168.59.103 + OPENTAXII_USER: user1 + OPENTAXII_PASS: pass1 + DATABASE_HOST: authdb + DATABASE_NAME: opentaxii1 + DATABASE_USER: user1 + DATABASE_PASS: password1 + volumes: + - ./:/input:ro + ports: + - 9001:9000 + links: + - authdb:authdb diff --git a/examples/pullpushsub.py b/examples/pullpushsub.py new file mode 100644 index 00000000..ab10b4f1 --- /dev/null +++ b/examples/pullpushsub.py @@ -0,0 +1,174 @@ +import json +import sys +import requests +from taxii2client.v21 import Server +from taxii2client.exceptions import AccessError +from uuid import uuid4 +from time import sleep + +# Define your TAXII server and collection details +OPENTAXII_URL = "http://localhost:9000/" +TAXII2_SERVER = OPENTAXII_URL + "taxii2/" +USERNAME = "user_write" +PASSWORD = "user_write" + + +def pull_data(api_root_url, collection): + # Pull data from the TAXII collection + try: + # Pull data from the collection + data = collection.get_objects() + print(f"Num objects pulled: {len(data.get('objects', []))}") + except AccessError: + print("[Pull Error] The user does not have write access") + return None + + return data + + +def push_data(api_root_url, collection): + # load stix data and push it + with open("stix/nettool.stix.json", "r") as f: + stix_loaded = json.load(f) + + stix_type = stix_loaded["type"] + stix_id = stix_type + "--" + str(uuid4()) + stix_loaded["id"] = stix_id + + envelope_data = { + "more": False, + "objects": [stix_loaded], + } + try: + # Push data to the collection + collection.add_objects(envelope_data) + print("Data pushed successfully.") + except AccessError: + print("[Push Error] The user does not have write access") + + +def subscribe(api_root_url, collection): + added_after = None + + # Get Authentication Token + response = requests.post( + OPENTAXII_URL + "management/auth", + headers={ + "Content-Type": "application/json", + }, + json={ + "username": USERNAME, + "password": PASSWORD, + }, + ) + auth_token = response.json().get("token", None) + + while True: + if added_after is None: + url = api_root_url + "collections/" + collection.id + "/objects/" + else: + url = ( + api_root_url + + "collections/" + + collection.id + + f"/objects/?added_after={added_after}" + ) + + # Get all objects from added_after + response = requests.get( + url=url, + headers={ + "Authorization": f"Bearer {auth_token}", + }, + ) + taxii_env = response.json() + objects = taxii_env.get("objects", []) + + print(f"Read {len(objects)} objects from the TAXII2 server") + if len(objects) > 0: + added_after = response.headers.get("X-TAXII-Date-Added-Last", "") + + sleep(3) + + +def not_an_action(collection): + print("That is not an option!") + + +def main(): + server = Server( + TAXII2_SERVER, + user=USERNAME, + password=PASSWORD, + ) + print(server.title) + print("=" * len(server.title)) + + print("Select an API Root:") + print(server.api_roots) + print() + for index, aroot in enumerate(server.api_roots, start=1): + print(f"{index}.") + try: + print(f"Title: {aroot.title}") + print(f"Description: {aroot.description}") + print(f"Versions: {aroot.versions}") + except Exception: + print( + "This API Root is not public.\nYou need to identify to see this API Root" + ) + print() + + aroot_choice = input("Enter the number of your choice: ") + try: + aroot_choice = int(aroot_choice) + selected_api_root = server.api_roots[aroot_choice - 1] + collections_l = selected_api_root.collections + except (ValueError, IndexError): + print("Invalid choice. Please enter a valid number.") + sys.exit() + except Exception as e: + print(e) + print("You cannot access this API Root. You need to authenticate.") + sys.exit() + + for index, coll in enumerate(collections_l, start=1): + print(f"{index}.") + print(f"\tId: {coll.id}") + print(f"\tTitle: {coll.title}") + print(f"\tAlias: {coll.alias}") + print(f"\tDescription: {coll.description}") + print(f"\tMedia Types: {coll.media_types}") + print(f"\tCan Read: {coll.can_read}") + print(f"\tCan Write: {coll.can_write}") + print(f"\tObjects URL: {coll.objects_url}") + print(f"\tCustom Properties: {coll.custom_properties}") + print() + + coll_choice = input("Enter the number of your choice: ") + try: + coll_choice = int(coll_choice) + selected_collection = selected_api_root.collections[coll_choice - 1] + except (ValueError, IndexError): + print("Invalid choice. Please enter a valid number.") + sys.exit() + + actions_d = { + 1: pull_data, + 2: push_data, + 3: subscribe, + } + + while True: + print() + print("1: Pull") + print("2: Push") + print("3: Subscribe") + action_choice = int(input("Enter the number of your choice: ")) + action_func = actions_d.get(action_choice, not_an_action) + action_func(selected_api_root.url, selected_collection) + print() + + +if __name__ == "__main__": + main() diff --git a/examples/stix/nettool.stix.json b/examples/stix/nettool.stix.json new file mode 100644 index 00000000..5ce5f615 --- /dev/null +++ b/examples/stix/nettool.stix.json @@ -0,0 +1,11 @@ +{ + "modified": "2023-07-25T19:25:59.767Z", + "name": "Net", + "description": "The [Net](https://attack.mitre.org/software/S0039) utility is a component of the Windows operating system. It is used in command-line operations for control of users, groups, services, and network connections. (Citation: Microsoft Net Utility)\n\n[Net](https://attack.mitre.org/software/S0039) has a great deal of functionality, (Citation: Savill 1999) much of which is useful for an adversary, such as gathering system and network information for Discovery, moving laterally through [SMB/Windows Admin Shares](https://attack.mitre.org/techniques/T1021/002) using net use commands, and interacting with services. The net1.exe utility is executed for certain functionality when net.exe is run and can be used directly in commands such as net1 user.", + "type": "tool", + "id": "tool--03342581-f790-4f03-ba41-e82e67392e25", + "created": "2017-05-31T21:32:31.601Z", + "revoked": false, + "external_references": [], + "spec_version": "2.1" +} diff --git a/opentaxii/auth/manager.py b/opentaxii/auth/manager.py index aa4774ab..89c9f657 100644 --- a/opentaxii/auth/manager.py +++ b/opentaxii/auth/manager.py @@ -1,4 +1,5 @@ import structlog +from opentaxii.persistence.exceptions import DoesNotExistError log = structlog.getLogger(__name__) @@ -44,13 +45,30 @@ def update_account(self, account, password): NOTE: Additional method that is only used in the helper scripts shipped with OpenTAXII. ''' - for colname, permission in list(account.permissions.items()): + permission_collections = {} + # Check for taxii1 collections + for colname, permission in list(account.permissions.get("taxii1", {}).items()): collection = self.server.servers.taxii1.persistence.get_collection(colname) if not collection: log.warning( "update_account.unknown_collection", collection=colname) - account.permissions.pop(colname) + else: + permission_collections[colname] = permission + + # Check for taxii2 collections + for api_root, collections in list(account.permissions.get("taxii2", {}).items()): + for colname, permission in collections.items(): + try: + collection = self.server.servers.taxii2.persistence.get_collection(api_root, colname) + except DoesNotExistError: + log.warning( + "update_account.unknown_collection", + api_root=api_root, collection=colname) + else: + permission_collections[str(collection.id)] = permission + + account.permissions = permission_collections account = self.api.update_account(account, password) return account diff --git a/opentaxii/cli/auth.py b/opentaxii/cli/auth.py index 428b81fe..5630a4b6 100644 --- a/opentaxii/cli/auth.py +++ b/opentaxii/cli/auth.py @@ -21,6 +21,7 @@ def create_account(argv=None): account = app.taxii_server.auth.api.create_account( username=args.username, password=args.password, + is_admin=args.admin ) token = app.taxii_server.auth.authenticate( username=account.username, diff --git a/opentaxii/cli/persistence.py b/opentaxii/cli/persistence.py index 9efca24a..ec0e1cdb 100644 --- a/opentaxii/cli/persistence.py +++ b/opentaxii/cli/persistence.py @@ -105,10 +105,16 @@ def add_api_root(): "--default", action="store_true", help="Set as default api root" ) + parser.add_argument( + "--public", action="store_true", help="Create a public api root" + ) args = parser.parse_args() with app.app_context(): app.taxii_server.servers.taxii2.persistence.api.add_api_root( - title=args.title, description=args.description, default=args.default + title=args.title, + description=args.description, + default=args.default, + is_public=args.public, ) diff --git a/opentaxii/common/sqldb.py b/opentaxii/common/sqldb.py index 65720a1b..cb8842fc 100644 --- a/opentaxii/common/sqldb.py +++ b/opentaxii/common/sqldb.py @@ -7,21 +7,28 @@ except ImportError: from sqlalchemy.ext.declarative import DeclarativeMeta +selected_db = None + class BaseSQLDatabaseAPI: BASEMODEL: ClassVar[Type[DeclarativeMeta]] def __init__(self, db_connection, create_tables=False, **engine_parameters): super().__init__() - self.db = SQLAlchemyDB( - db_connection, - self.BASEMODEL, - session_options={ - "autocommit": False, - "autoflush": True, - }, - **engine_parameters - ) + global selected_db + if not selected_db: + selected_db = SQLAlchemyDB( + db_connection, + self.BASEMODEL, + session_options={ + "autocommit": False, + "autoflush": True, + }, + **engine_parameters + ) + # Use same db object in auth and taxii persistent to keep exact track of connection pools + self.db = selected_db + self.db.Model = self.db.extend_base_model(self.BASEMODEL) if create_tables: self.db.create_all_tables() diff --git a/opentaxii/config.py b/opentaxii/config.py index b8f2f87a..548cb723 100644 --- a/opentaxii/config.py +++ b/opentaxii/config.py @@ -64,6 +64,8 @@ class ServerConfig(dict): "title", "public_discovery", "allow_custom_properties", + "max_pagination_limit", + "default_pagination_limit" ) ALL_VALID_OPTIONS = VALID_BASE_OPTIONS + VALID_TAXII_OPTIONS + VALID_TAXII1_OPTIONS diff --git a/opentaxii/defaults.yml b/opentaxii/defaults.yml index 214e866a..f7d37a4c 100644 --- a/opentaxii/defaults.yml +++ b/opentaxii/defaults.yml @@ -26,6 +26,18 @@ taxii1: create_tables: yes taxii2: + public_discovery: true + allow_custom_properties: true + description: "TAXII2 Server" + title: "Taxii2 Service" + max_content_length: 2048 + persistence_api: + class: opentaxii.persistence.sqldb.Taxii2SQLDatabaseAPI + parameters: + db_connection: sqlite:////tmp/data2.db + create_tables: yes + default_pagination_limit: 10 + max_pagination_limit: 1000 logging: opentaxii: info diff --git a/opentaxii/persistence/sqldb/api.py b/opentaxii/persistence/sqldb/api.py index 4ce29538..3320fa1d 100644 --- a/opentaxii/persistence/sqldb/api.py +++ b/opentaxii/persistence/sqldb/api.py @@ -983,12 +983,13 @@ def add_objects( self.db.session.commit() job_details = [] for obj in objects: + modified_date = obj.get("modified") or obj.get("created") or datetime.datetime.now(datetime.timezone.utc).strftime(DATETIMEFORMAT) version = datetime.datetime.strptime( - obj["modified"], DATETIMEFORMAT + modified_date, DATETIMEFORMAT ).replace(tzinfo=datetime.timezone.utc) - if ( - not self.db.session.query(literal(True)) - .filter( + modified_object = True + if obj.get("modified") or obj.get("created"): + modified_object = not self.db.session.query(literal(True)).filter( self.db.session.query(taxii2models.STIXObject) .filter( taxii2models.STIXObject.id == obj["id"], @@ -996,22 +997,24 @@ def add_objects( taxii2models.STIXObject.version == version, ) .exists() - ) - .scalar() - ): + ).scalar() + else: + stored_object = self.db.session.query(taxii2models.STIXObject).filter( + taxii2models.STIXObject.id == obj["id"], + taxii2models.STIXObject.collection_id == collection_id, + ).order_by(taxii2models.STIXObject.date_added.desc()).first() + if stored_object and stored_object.serialized_data == obj: + modified_object = False + if modified_object: self.db.session.add( taxii2models.STIXObject( id=obj["id"], collection_id=collection_id, type=obj["id"].split("--")[0], - spec_version=obj["spec_version"], + spec_version=obj.get("spec_version", 2.1), date_added=datetime.datetime.now(datetime.timezone.utc), version=version, - serialized_data={ - key: value - for (key, value) in obj.items() - if key not in ["id", "type", "spec_version"] - }, + serialized_data=obj, ) ) job_detail = taxii2models.JobDetail( @@ -1109,6 +1112,7 @@ def delete_object( ordered=False, ) query.delete("fetch") + self.db.session.commit() def get_versions( self, diff --git a/opentaxii/server.py b/opentaxii/server.py index 928ae033..80e77f5b 100644 --- a/opentaxii/server.py +++ b/opentaxii/server.py @@ -599,6 +599,7 @@ def collection_handler(self, api_root_id, collection_id_or_alias): ) def manifest_handler(self, api_root_id, collection_id_or_alias): filter_params = validate_list_filter_params(request.args, self.persistence.api) + filter_params["limit"] = self.config.get("max_pagination_limit") if filter_params.get("limit", self.config.get("max_pagination_limit")) > self.config.get("max_pagination_limit") else filter_params.get("limit", self.config.get("default_pagination_limit")) try: manifest, more = self.persistence.get_manifest( api_root_id=api_root_id, @@ -652,6 +653,7 @@ def objects_handler(self, api_root_id, collection_id_or_alias): def objects_get_handler(self, api_root_id, collection_id_or_alias): filter_params = validate_list_filter_params(request.args, self.persistence.api) + filter_params["limit"] = self.config.get("max_pagination_limit") if filter_params.get("limit", self.config.get("max_pagination_limit")) > self.config.get("max_pagination_limit") else filter_params.get("limit", self.config.get("default_pagination_limit")) try: objects, more, next_param = self.persistence.get_objects( api_root_id=api_root_id, @@ -667,9 +669,6 @@ def objects_get_handler(self, api_root_id, collection_id_or_alias): "more": more, "objects": [ { - "id": obj.id, - "type": obj.type, - "spec_version": obj.type, **obj.serialized_data, } for obj in objects @@ -750,9 +749,6 @@ def object_get_handler(self, api_root_id, collection_id_or_alias, object_id): "more": more, "objects": [ { - "id": obj.id, - "type": obj.type, - "spec_version": obj.type, **obj.serialized_data, } for obj in versions diff --git a/opentaxii/sqldb_helper.py b/opentaxii/sqldb_helper.py index a80b0b55..1ead8110 100644 --- a/opentaxii/sqldb_helper.py +++ b/opentaxii/sqldb_helper.py @@ -1,7 +1,8 @@ from threading import get_ident -from sqlalchemy import engine, orm +from sqlalchemy import engine, orm, event, exc from sqlalchemy.orm.exc import UnmappedClassError +import os class _QueryProperty: @@ -31,6 +32,20 @@ def __init__(self, db_connection, base_model, session_options=None, **kwargs): self.Model = self.extend_base_model(base_model) self._session = None + @event.listens_for(self.engine, "connect") + def connect(dbapi_connection, connection_record): + connection_record.info["pid"] = os.getpid() + + @event.listens_for(self.engine, "checkout") + def checkout(dbapi_connection, connection_record, connection_proxy): + pid = os.getpid() + if connection_record.info["pid"] != pid: + connection_record.dbapi_connection = connection_proxy.dbapi_connection = None + raise exc.DisconnectionError( + "Connection record belongs to pid %s, " + "attempting to check out in pid %s" % (connection_record.info["pid"], pid) + ) + def extend_base_model(self, base): if not getattr(base, 'query_class', None): base.query_class = self.Query @@ -70,5 +85,6 @@ def create_all_tables(self): def init_app(self, app): @app.teardown_appcontext def shutdown_session(response_or_exc): - self.session.remove() - return response_or_exc + if self._session: + self._session.remove() + return response_or_exc \ No newline at end of file diff --git a/opentaxii/taxii2/entities.py b/opentaxii/taxii2/entities.py index e660f225..c2787fbb 100644 --- a/opentaxii/taxii2/entities.py +++ b/opentaxii/taxii2/entities.py @@ -66,7 +66,9 @@ def can_read(self, account: Optional[Account]): return self.is_public or ( account and ( - account.is_admin or "read" in set(account.permissions.get(self.id, [])) + account.is_admin or + "read" in account.permissions.get(str(self.id), "") or + "modify" in account.permissions.get(str(self.id), "") ) ) @@ -75,7 +77,8 @@ def can_write(self, account: Optional[Account]): return self.is_public_write or ( account and ( - account.is_admin or "write" in set(account.permissions.get(self.id, [])) + account.is_admin or + "modify" in account.permissions.get(str(self.id), "") ) ) diff --git a/opentaxii/taxii2/http.py b/opentaxii/taxii2/http.py index a0408d33..3dafbec0 100644 --- a/opentaxii/taxii2/http.py +++ b/opentaxii/taxii2/http.py @@ -3,12 +3,22 @@ from typing import Dict, Optional from flask import Response, make_response +from uuid import UUID -def make_taxii2_response(data, status: Optional[int] = 200, extra_headers: Optional[Dict] = None) -> Response: +class JSONEncoderWithUUID(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, UUID): + return str(obj) + return super().default(obj) + + +def make_taxii2_response( + data, status: Optional[int] = 200, extra_headers: Optional[Dict] = None +) -> Response: """Turn input data into valid taxii2 response.""" if not isinstance(data, str): - data = json.dumps(data) + data = json.dumps(data, cls=JSONEncoderWithUUID) response = make_response((data, status)) response.content_type = "application/taxii+json;version=2.1" response.headers.update(extra_headers or {}) diff --git a/opentaxii/utils.py b/opentaxii/utils.py index 43e2e45a..cb8e973a 100644 --- a/opentaxii/utils.py +++ b/opentaxii/utils.py @@ -162,12 +162,13 @@ def emit(self, record): def sync_conf_dict_into_db(server, config, force_collection_deletion=False): - services = config.get("services", []) - sync_services(server.servers.taxii1, services) - collections = config.get("collections", []) - sync_collections( - server.servers.taxii1, collections, force_deletion=force_collection_deletion - ) + if server.servers.taxii1: + services = config.get("services", []) + sync_services(server.servers.taxii1, services) + collections = config.get("collections", []) + sync_collections( + server.servers.taxii1, collections, force_deletion=force_collection_deletion + ) accounts = config.get("accounts", []) sync_accounts(server, accounts) @@ -349,4 +350,4 @@ def inner(*args, **kwargs): inner.handles_own_auth = handles_own_auth return inner - return inner_decorator + return inner_decorator \ No newline at end of file