Merge branch 'main' into obs_to_rba

contentctl/actions/build.py

@@ -51,7 +51,9 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards))
-            updated_conf_files.update(conf_output.writeAppConf())
+            updated_conf_files.update(conf_output.writeMiscellaneousAppFiles())
+
+
 
             #Ensure that the conf file we just generated/update is syntactically valid
             for conf_file in updated_conf_files:

contentctl/actions/deploy_acs.py

@@ -1,38 +1,55 @@
-from dataclasses import dataclass
-from contentctl.input.director import DirectorInputDto
-from contentctl.output.conf_output import ConfOutput
-
-
-from typing import Union
-
-@dataclass(frozen=True)
-class ACSDeployInputDto:
-    director_input_dto: DirectorInputDto
-    splunk_api_username: str
-    splunk_api_password: str
-    splunk_cloud_jwt_token: str
-    splunk_cloud_stack: str
-    stack_type: str
+from contentctl.objects.config import deploy_acs, StackType
+from requests import post
+import pprint
 
 
 class Deploy:
-    def execute(self, input_dto: ACSDeployInputDto) -> None:
-
-        conf_output = ConfOutput(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config)
+    def execute(self, config: deploy_acs, appinspect_token:str) -> None:
 
-        appinspect_token = conf_output.inspectAppAPI(input_dto.splunk_api_username, input_dto.splunk_api_password, input_dto.stack_type)
+        #The following common headers are used by both Clasic and Victoria
+        headers = {
+            'Authorization': f'Bearer {config.splunk_cloud_jwt_token}',
+            'ACS-Legal-Ack': 'Y'
+        }
+        try:
+
+            with open(config.getPackageFilePath(include_version=False),'rb') as app_data:
+                #request_data = app_data.read()
+                if config.stack_type == StackType.classic:
+                    # Classic instead uses a form to store token and package
+                    # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience
+                    address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps"
+
+                    form_data = {
+                        'token': (None, appinspect_token),
+                        'package': app_data
+                    }
+                    res = post(address, headers=headers, files = form_data)
+                elif config.stack_type == StackType.victoria:
+                    # Victoria uses the X-Splunk-Authorization Header
+                    # It also uses --data-binary for the app content
+                    # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience
+                    headers.update({'X-Splunk-Authorization':  appinspect_token})
+                    address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria"
+                    res = post(address, headers=headers, data=app_data.read())
+                else:
+                    raise Exception(f"Unsupported stack type: '{config.stack_type}'")
+        except Exception as e:
+            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}")
 
-
-        if input_dto.splunk_cloud_jwt_token is None or input_dto.splunk_cloud_stack is None:
-            if input_dto.splunk_cloud_jwt_token is None:
-                raise Exception("Cannot deploy app via ACS, --splunk_cloud_jwt_token was not defined on command line.")
-            else:
-                raise Exception("Cannot deploy app via ACS, --splunk_cloud_stack was not defined on command line.")
-
-        conf_output.deploy_via_acs(input_dto.splunk_cloud_jwt_token,
-                                input_dto.splunk_cloud_stack, 
-                                appinspect_token,
-                                input_dto.stack_type)
-
+        try:
+            # Request went through and completed, but may have returned a non-successful error code.
+            # This likely includes a more verbose response describing the error
+            res.raise_for_status()
+            print(res.json())
+        except Exception as e:
+            try:
+                error_text = res.json()
+            except Exception as e:
+                error_text = "No error text - request failed"
+            formatted_error_text = pprint.pformat(error_text)
+            print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
+                  "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.")
+            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}")
 
-
+        print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!")

contentctl/actions/new_content.py

@@ -29,8 +29,7 @@ def buildDetection(self)->dict[str,Any]:
         answers['date'] = datetime.today().strftime('%Y-%m-%d')
         answers['author'] = answers['detection_author']
         del answers['detection_author']
-        answers['data_sources'] = answers['data_source']
-        del answers['data_source']
+        answers['data_source'] = answers['data_source']
         answers['type'] = answers['detection_type']
         del answers['detection_type']
         answers['status'] = "production" #start everything as production since that's what we INTEND the content to become   

contentctl/contentctl.py

@@ -19,6 +19,7 @@
 from contentctl.actions.reporting import ReportingInputDto, Reporting
 from contentctl.actions.inspect import Inspect
 from contentctl.input.yml_reader import YmlReader
+from contentctl.actions.deploy_acs import Deploy
 from contentctl.actions.release_notes import ReleaseNotes
 
 # def print_ascii_art():
@@ -95,8 +96,11 @@ def new_func(config:new):
 
 
 def deploy_acs_func(config:deploy_acs):
-    #This is a bit challenging to get to work with the default values.
-    raise Exception("deploy acs not yet implemented") 
+    print("Building and inspecting app...")
+    token = inspect_func(config)
+    print("App successfully built and inspected.")
+    print("Deploying app...")
+    Deploy().execute(config, token)
 
 def test_common_func(config:test_common):
     if type(config) == test:

contentctl/objects/config.py

@@ -294,6 +294,7 @@ class StackType(StrEnum):
 
 
 class inspect(build):
+
     splunk_api_username: str = Field(
         description="Splunk API username used for appinspect and Splunkbase downloads."
     )

contentctl/output/conf_output.py

@@ -57,19 +57,26 @@ def writeHeaders(self) -> set[pathlib.Path]:
             pass
 
 
-    def writeAppConf(self)->set[pathlib.Path]:
+
+
+    def writeMiscellaneousAppFiles(self)->set[pathlib.Path]:
         written_files:set[pathlib.Path] = set()
-        for output_app_path, template_name in [ ("default/app.conf", "app.conf.j2"),
-                                                ("default/content-version.conf", "content-version.j2")]:
-            written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path),
-                                    template_name,
-                                    self.config,
-                                    [self.config.app]))
+
+        written_files.add(ConfWriter.writeConfFile(pathlib.Path("default/content-version.conf"),
+                                "content-version.j2",
+                                self.config,
+                                [self.config.app]))
 
         written_files.add(ConfWriter.writeManifestFile(pathlib.Path("app.manifest"),
                                               "app.manifest.j2",
                                               self.config,
                                               [self.config.app]))
+
+        written_files.add(ConfWriter.writeServerConf(self.config))
+
+        written_files.add(ConfWriter.writeAppConf(self.config))
+
+
         return written_files
 
 

contentctl/output/conf_writer.py

@@ -12,6 +12,76 @@
 from contentctl.objects.config import build
 import xml.etree.ElementTree as ET
 
+# This list is not exhaustive of all default conf files, but should be
+# sufficient for our purposes.
+DEFAULT_CONF_FILES = [
+    "alert_actions.conf",
+    "app.conf",
+    "audit.conf",
+    "authentication.conf",
+    "authorize.conf",
+    "bookmarks.conf",
+    "checklist.conf",
+    "collections.conf",
+    "commands.conf",
+    "conf.conf",
+    "datamodels.conf",
+    "datatypesbnf.conf",
+    "default-mode.conf",
+    "deploymentclient.conf",
+    "distsearch.conf",
+    "event_renderers.conf",
+    "eventdiscoverer.conf",
+    "eventtypes.conf",
+    "federated.conf",
+    "fields.conf",
+    "global-banner.conf",
+    "health.conf",
+    "indexes.conf",
+    "inputs.conf",
+    "limits.conf",
+    "literals.conf",
+    "livetail.conf",
+    "macros.conf",
+    "messages.conf",
+    "metric_alerts.conf",
+    "metric_rollups.conf",
+    "multikv.conf",
+    "outputs.conf",
+    "passwords.conf",
+    "procmon-filters.conf",
+    "props.conf",
+    "pubsub.conf",
+    "restmap.conf",
+    "rolling_upgrade.conf",
+    "savedsearches.conf",
+    "searchbnf.conf",
+    "segmenters.conf",
+    "server.conf",
+    "serverclass.conf",
+    "serverclass.seed.xml.conf",
+    "source-classifier.conf",
+    "sourcetypes.conf",
+    "tags.conf",
+    "telemetry.conf",
+    "times.conf",
+    "transactiontypes.conf",
+    "transforms.conf",
+    "ui-prefs.conf",
+    "ui-tour.conf",
+    "user-prefs.conf",
+    "user-seed.conf",
+    "viewstates.conf",
+    "visualizations.conf",
+    "web-features.conf",
+    "web.conf",
+    "wmi.conf",
+    "workflow_actions.conf",
+    "workload_policy.conf",
+    "workload_pools.conf",
+    "workload_rules.conf",
+]
+
 class ConfWriter():
 
     @staticmethod
@@ -57,6 +127,52 @@ def writeConfFileHeader(app_output_path:pathlib.Path, config: build) -> pathlib.
         ConfWriter.validateConfFile(output_path)        
         return output_path
 
+    @staticmethod
+    def getCustomConfFileStems(config:build)->list[str]:
+        # Get all the conf files in the default directory. We must make a reload.conf_file = simple key/value for them if
+        # they are custom conf files
+        default_path = config.getPackageDirectoryPath()/"default"
+        conf_files = default_path.glob("*.conf")
+
+        custom_conf_file_stems = [conf_file.stem for conf_file in conf_files if conf_file.name not in DEFAULT_CONF_FILES]
+        return sorted(custom_conf_file_stems)
+
+    @staticmethod
+    def writeServerConf(config: build) -> pathlib.Path:
+        app_output_path = pathlib.Path("default/server.conf")
+        template_name = "server.conf.j2"
+
+        j2_env = ConfWriter.getJ2Environment()
+        template = j2_env.get_template(template_name)
+
+        output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config))
+
+        output_path = config.getPackageDirectoryPath()/app_output_path
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(output_path, 'a') as f:
+            output = output.encode('utf-8', 'ignore').decode('utf-8')
+            f.write(output)
+        return output_path
+
+
+    @staticmethod
+    def writeAppConf(config: build) -> pathlib.Path:
+        app_output_path = pathlib.Path("default/app.conf")
+        template_name = "app.conf.j2"
+
+        j2_env = ConfWriter.getJ2Environment()
+        template = j2_env.get_template(template_name)
+
+        output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config), 
+                                 app=config.app)
+
+        output_path = config.getPackageDirectoryPath()/app_output_path
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(output_path, 'a') as f:
+            output = output.encode('utf-8', 'ignore').decode('utf-8')
+            f.write(output)
+        return output_path
+
     @staticmethod
     def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> pathlib.Path:
         j2_env = ConfWriter.getJ2Environment()
@@ -70,6 +186,7 @@ def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config:
             output = output.encode('utf-8', 'ignore').decode('utf-8')
             f.write(output)
         return output_path
+
 
 
     @staticmethod
@@ -218,8 +335,3 @@ def validateManifestFile(path:pathlib.Path):
                 _ = json.load(manifestFile)
         except Exception as e:
             raise Exception(f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}")
-
-
-
-
-

contentctl/output/templates/app.conf.j2

@@ -4,31 +4,33 @@
 is_configured = false
 state = enabled
 state_change_requires_restart = false
-build = {{ objects[0].build }}
+build = {{ app.build }}
 
 [triggers]
-reload.analytic_stories = simple
-reload.usage_searches = simple
-reload.use_case_library = simple
-reload.correlationsearches = simple
-reload.analyticstories = simple
-reload.governance = simple
-reload.managed_configurations = simple
-reload.postprocess = simple
-reload.content-version = simple
-reload.es_investigations = simple
+{% for custom_conf_file in custom_conf_files%}
+reload.{{custom_conf_file}} = simple
+{% endfor %}
 
 [launcher]
-author = {{ objects[0].author_company }}
-version = {{ objects[0].version }} 
-description = {{ objects[0].description | escapeNewlines() }}
+author = {{ app.author_company }}
+version = {{ app.version }} 
+description = {{ app.description | escapeNewlines() }}
 
 [ui]
 is_visible = true
-label = {{ objects[0].title }}
+label = {{ app.title }}
 
 [package]
-id = {{ objects[0].appid }}
+id = {{ app.appid }}
+{% if app.uid == 3449 %}
+check_for_updates = true
+{% else %}
+check_for_updates = false
+{% endif %}
+
+[id]
+version = {{ app.version }}
+name = {{ app.appid }}
 
 
 

contentctl/output/templates/app.manifest.j2

@@ -1,5 +1,6 @@
 {
-  "schemaVersion": "1.0.0", 
+  "schemaVersion": "1.0.0",
+  "targetWorkloads": ["_search_heads"], 
   "info": {
     "title": "{{ objects[0].title }}", 
     "id": {

contentctl/output/templates/server.conf.j2

@@ -0,0 +1,4 @@
+[shclustering]
+{% for custom_conf_file in custom_conf_files%}
+conf_replication_include.{{custom_conf_file}} = true
+{% endfor %}