diff --git a/contentctl/actions/build.py b/contentctl/actions/build.py index 97c0296c..feb0351b 100644 --- a/contentctl/actions/build.py +++ b/contentctl/actions/build.py @@ -51,7 +51,9 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto: updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups)) updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros)) updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards)) - updated_conf_files.update(conf_output.writeAppConf()) + updated_conf_files.update(conf_output.writeMiscellaneousAppFiles()) + + #Ensure that the conf file we just generated/update is syntactically valid for conf_file in updated_conf_files: diff --git a/contentctl/actions/deploy_acs.py b/contentctl/actions/deploy_acs.py index 0bc0054f..8451751b 100644 --- a/contentctl/actions/deploy_acs.py +++ b/contentctl/actions/deploy_acs.py @@ -1,38 +1,55 @@ -from dataclasses import dataclass -from contentctl.input.director import DirectorInputDto -from contentctl.output.conf_output import ConfOutput - - -from typing import Union - -@dataclass(frozen=True) -class ACSDeployInputDto: - director_input_dto: DirectorInputDto - splunk_api_username: str - splunk_api_password: str - splunk_cloud_jwt_token: str - splunk_cloud_stack: str - stack_type: str +from contentctl.objects.config import deploy_acs, StackType +from requests import post +import pprint class Deploy: - def execute(self, input_dto: ACSDeployInputDto) -> None: - - conf_output = ConfOutput(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config) + def execute(self, config: deploy_acs, appinspect_token:str) -> None: - appinspect_token = conf_output.inspectAppAPI(input_dto.splunk_api_username, input_dto.splunk_api_password, input_dto.stack_type) + #The following common headers are used by both Clasic and Victoria + headers = { + 'Authorization': f'Bearer {config.splunk_cloud_jwt_token}', + 'ACS-Legal-Ack': 'Y' + } + try: + + with open(config.getPackageFilePath(include_version=False),'rb') as app_data: + #request_data = app_data.read() + if config.stack_type == StackType.classic: + # Classic instead uses a form to store token and package + # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience + address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps" + + form_data = { + 'token': (None, appinspect_token), + 'package': app_data + } + res = post(address, headers=headers, files = form_data) + elif config.stack_type == StackType.victoria: + # Victoria uses the X-Splunk-Authorization Header + # It also uses --data-binary for the app content + # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience + headers.update({'X-Splunk-Authorization': appinspect_token}) + address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria" + res = post(address, headers=headers, data=app_data.read()) + else: + raise Exception(f"Unsupported stack type: '{config.stack_type}'") + except Exception as e: + raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}") - - if input_dto.splunk_cloud_jwt_token is None or input_dto.splunk_cloud_stack is None: - if input_dto.splunk_cloud_jwt_token is None: - raise Exception("Cannot deploy app via ACS, --splunk_cloud_jwt_token was not defined on command line.") - else: - raise Exception("Cannot deploy app via ACS, --splunk_cloud_stack was not defined on command line.") - - conf_output.deploy_via_acs(input_dto.splunk_cloud_jwt_token, - input_dto.splunk_cloud_stack, - appinspect_token, - input_dto.stack_type) - + try: + # Request went through and completed, but may have returned a non-successful error code. + # This likely includes a more verbose response describing the error + res.raise_for_status() + print(res.json()) + except Exception as e: + try: + error_text = res.json() + except Exception as e: + error_text = "No error text - request failed" + formatted_error_text = pprint.pformat(error_text) + print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n" + "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.") + raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}") - \ No newline at end of file + print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!") \ No newline at end of file diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index bca81521..0a54cf11 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -29,8 +29,7 @@ def buildDetection(self)->dict[str,Any]: answers['date'] = datetime.today().strftime('%Y-%m-%d') answers['author'] = answers['detection_author'] del answers['detection_author'] - answers['data_sources'] = answers['data_source'] - del answers['data_source'] + answers['data_source'] = answers['data_source'] answers['type'] = answers['detection_type'] del answers['detection_type'] answers['status'] = "production" #start everything as production since that's what we INTEND the content to become diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index dbf434a7..efef5853 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -19,6 +19,7 @@ from contentctl.actions.reporting import ReportingInputDto, Reporting from contentctl.actions.inspect import Inspect from contentctl.input.yml_reader import YmlReader +from contentctl.actions.deploy_acs import Deploy from contentctl.actions.release_notes import ReleaseNotes # def print_ascii_art(): @@ -95,8 +96,11 @@ def new_func(config:new): def deploy_acs_func(config:deploy_acs): - #This is a bit challenging to get to work with the default values. - raise Exception("deploy acs not yet implemented") + print("Building and inspecting app...") + token = inspect_func(config) + print("App successfully built and inspected.") + print("Deploying app...") + Deploy().execute(config, token) def test_common_func(config:test_common): if type(config) == test: diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 9057a4c4..659d1113 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -294,6 +294,7 @@ class StackType(StrEnum): class inspect(build): + splunk_api_username: str = Field( description="Splunk API username used for appinspect and Splunkbase downloads." ) diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index 0d00cf64..e53aeba0 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -57,19 +57,26 @@ def writeHeaders(self) -> set[pathlib.Path]: pass - def writeAppConf(self)->set[pathlib.Path]: + + + def writeMiscellaneousAppFiles(self)->set[pathlib.Path]: written_files:set[pathlib.Path] = set() - for output_app_path, template_name in [ ("default/app.conf", "app.conf.j2"), - ("default/content-version.conf", "content-version.j2")]: - written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path), - template_name, - self.config, - [self.config.app])) + + written_files.add(ConfWriter.writeConfFile(pathlib.Path("default/content-version.conf"), + "content-version.j2", + self.config, + [self.config.app])) written_files.add(ConfWriter.writeManifestFile(pathlib.Path("app.manifest"), "app.manifest.j2", self.config, [self.config.app])) + + written_files.add(ConfWriter.writeServerConf(self.config)) + + written_files.add(ConfWriter.writeAppConf(self.config)) + + return written_files diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 4d2e0490..410ce4f6 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -12,6 +12,76 @@ from contentctl.objects.config import build import xml.etree.ElementTree as ET +# This list is not exhaustive of all default conf files, but should be +# sufficient for our purposes. +DEFAULT_CONF_FILES = [ + "alert_actions.conf", + "app.conf", + "audit.conf", + "authentication.conf", + "authorize.conf", + "bookmarks.conf", + "checklist.conf", + "collections.conf", + "commands.conf", + "conf.conf", + "datamodels.conf", + "datatypesbnf.conf", + "default-mode.conf", + "deploymentclient.conf", + "distsearch.conf", + "event_renderers.conf", + "eventdiscoverer.conf", + "eventtypes.conf", + "federated.conf", + "fields.conf", + "global-banner.conf", + "health.conf", + "indexes.conf", + "inputs.conf", + "limits.conf", + "literals.conf", + "livetail.conf", + "macros.conf", + "messages.conf", + "metric_alerts.conf", + "metric_rollups.conf", + "multikv.conf", + "outputs.conf", + "passwords.conf", + "procmon-filters.conf", + "props.conf", + "pubsub.conf", + "restmap.conf", + "rolling_upgrade.conf", + "savedsearches.conf", + "searchbnf.conf", + "segmenters.conf", + "server.conf", + "serverclass.conf", + "serverclass.seed.xml.conf", + "source-classifier.conf", + "sourcetypes.conf", + "tags.conf", + "telemetry.conf", + "times.conf", + "transactiontypes.conf", + "transforms.conf", + "ui-prefs.conf", + "ui-tour.conf", + "user-prefs.conf", + "user-seed.conf", + "viewstates.conf", + "visualizations.conf", + "web-features.conf", + "web.conf", + "wmi.conf", + "workflow_actions.conf", + "workload_policy.conf", + "workload_pools.conf", + "workload_rules.conf", +] + class ConfWriter(): @staticmethod @@ -57,6 +127,52 @@ def writeConfFileHeader(app_output_path:pathlib.Path, config: build) -> pathlib. ConfWriter.validateConfFile(output_path) return output_path + @staticmethod + def getCustomConfFileStems(config:build)->list[str]: + # Get all the conf files in the default directory. We must make a reload.conf_file = simple key/value for them if + # they are custom conf files + default_path = config.getPackageDirectoryPath()/"default" + conf_files = default_path.glob("*.conf") + + custom_conf_file_stems = [conf_file.stem for conf_file in conf_files if conf_file.name not in DEFAULT_CONF_FILES] + return sorted(custom_conf_file_stems) + + @staticmethod + def writeServerConf(config: build) -> pathlib.Path: + app_output_path = pathlib.Path("default/server.conf") + template_name = "server.conf.j2" + + j2_env = ConfWriter.getJ2Environment() + template = j2_env.get_template(template_name) + + output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config)) + + output_path = config.getPackageDirectoryPath()/app_output_path + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, 'a') as f: + output = output.encode('utf-8', 'ignore').decode('utf-8') + f.write(output) + return output_path + + + @staticmethod + def writeAppConf(config: build) -> pathlib.Path: + app_output_path = pathlib.Path("default/app.conf") + template_name = "app.conf.j2" + + j2_env = ConfWriter.getJ2Environment() + template = j2_env.get_template(template_name) + + output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config), + app=config.app) + + output_path = config.getPackageDirectoryPath()/app_output_path + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, 'a') as f: + output = output.encode('utf-8', 'ignore').decode('utf-8') + f.write(output) + return output_path + @staticmethod def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() @@ -70,6 +186,7 @@ def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: output = output.encode('utf-8', 'ignore').decode('utf-8') f.write(output) return output_path + @staticmethod @@ -218,8 +335,3 @@ def validateManifestFile(path:pathlib.Path): _ = json.load(manifestFile) except Exception as e: raise Exception(f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}") - - - - - diff --git a/contentctl/output/templates/app.conf.j2 b/contentctl/output/templates/app.conf.j2 index 51734792..63310711 100644 --- a/contentctl/output/templates/app.conf.j2 +++ b/contentctl/output/templates/app.conf.j2 @@ -4,31 +4,33 @@ is_configured = false state = enabled state_change_requires_restart = false -build = {{ objects[0].build }} +build = {{ app.build }} [triggers] -reload.analytic_stories = simple -reload.usage_searches = simple -reload.use_case_library = simple -reload.correlationsearches = simple -reload.analyticstories = simple -reload.governance = simple -reload.managed_configurations = simple -reload.postprocess = simple -reload.content-version = simple -reload.es_investigations = simple +{% for custom_conf_file in custom_conf_files%} +reload.{{custom_conf_file}} = simple +{% endfor %} [launcher] -author = {{ objects[0].author_company }} -version = {{ objects[0].version }} -description = {{ objects[0].description | escapeNewlines() }} +author = {{ app.author_company }} +version = {{ app.version }} +description = {{ app.description | escapeNewlines() }} [ui] is_visible = true -label = {{ objects[0].title }} +label = {{ app.title }} [package] -id = {{ objects[0].appid }} +id = {{ app.appid }} +{% if app.uid == 3449 %} +check_for_updates = true +{% else %} +check_for_updates = false +{% endif %} + +[id] +version = {{ app.version }} +name = {{ app.appid }} diff --git a/contentctl/output/templates/app.manifest.j2 b/contentctl/output/templates/app.manifest.j2 index 7891f52b..408eb1ef 100644 --- a/contentctl/output/templates/app.manifest.j2 +++ b/contentctl/output/templates/app.manifest.j2 @@ -1,5 +1,6 @@ { - "schemaVersion": "1.0.0", + "schemaVersion": "1.0.0", + "targetWorkloads": ["_search_heads"], "info": { "title": "{{ objects[0].title }}", "id": { diff --git a/contentctl/output/templates/server.conf.j2 b/contentctl/output/templates/server.conf.j2 new file mode 100644 index 00000000..28a78f98 --- /dev/null +++ b/contentctl/output/templates/server.conf.j2 @@ -0,0 +1,4 @@ +[shclustering] +{% for custom_conf_file in custom_conf_files%} +conf_replication_include.{{custom_conf_file}} = true +{% endfor %} \ No newline at end of file diff --git a/contentctl/templates/app_template/default/app.conf b/contentctl/templates/app_template/default/app.conf deleted file mode 100644 index c6991ff5..00000000 --- a/contentctl/templates/app_template/default/app.conf +++ /dev/null @@ -1,30 +0,0 @@ -## Splunk app configuration file - -[install] -is_configured = false -state = enabled -state_change_requires_restart = false -build = 16367 - -[triggers] -reload.analytic_stories = simple -reload.use_case_library = simple -reload.correlationsearches = simple -reload.analyticstories = simple -reload.governance = simple -reload.managed_configurations = simple -reload.postprocess = simple -reload.content-version = simple -reload.es_investigations = simple - -[launcher] -author = Splunk -version = 4.9.0 -description = Explore the Analytic Stories included with ES Content Updates. - -[ui] -is_visible = true -label = ES Content Updates - -[package] -id = DA-ESS-ContentUpdate diff --git a/contentctl/templates/app_template/default/content-version.conf b/contentctl/templates/app_template/default/content-version.conf deleted file mode 100644 index 4bbba5eb..00000000 --- a/contentctl/templates/app_template/default/content-version.conf +++ /dev/null @@ -1,2 +0,0 @@ -[content-version] -version = 4.9.0 diff --git a/contentctl/templates/app_template/metadata/default.meta b/contentctl/templates/app_template/metadata/default.meta index b9b933bf..7d137480 100644 --- a/contentctl/templates/app_template/metadata/default.meta +++ b/contentctl/templates/app_template/metadata/default.meta @@ -1,6 +1,6 @@ ## shared Application-level permissions [] -access = read : [ * ], write : [ admin ] +access = read : [ * ], write : [ admin, sc_admin ] export = system [savedsearches] diff --git a/pyproject.toml b/pyproject.toml index 70bbf071..f9f17b1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "contentctl" -version = "4.4.3" +version = "4.4.4" description = "Splunk Content Control Tool" authors = ["STRT "]