From d2cfdaccb7d99a2220733ead51397a0b5e31a7c2 Mon Sep 17 00:00:00 2001 From: James Barnett Date: Fri, 9 Aug 2024 14:02:52 -0400 Subject: [PATCH 1/4] feat: enable configuration of various collection options New options are configured with a minimum and maximum value which is checked against the configured value. If these values are exceeded, the default value will be used. --- client/rest/http.go | 5 +- cmd/list-app-owners.go | 3 +- cmd/list-app-role-assignments.go | 3 +- ...ist-automation-account-role-assignments.go | 3 +- cmd/list-automation-accounts.go | 3 +- cmd/list-container-registries.go | 3 +- ...ist-container-registry-role-assignments.go | 3 +- cmd/list-device-owners.go | 3 +- cmd/list-function-app-role-assignments.go | 3 +- cmd/list-function-apps.go | 3 +- cmd/list-group-members.go | 3 +- cmd/list-group-owners.go | 3 +- cmd/list-key-vault-role-assignments.go | 3 +- cmd/list-key-vaults.go | 3 +- cmd/list-logic-app-role-assignments.go | 3 +- cmd/list-logic-apps.go | 3 +- cmd/list-managed-cluster-role-assignments.go | 3 +- cmd/list-managed-clusters.go | 3 +- cmd/list-management-group-descendants.go | 3 +- cmd/list-management-group-role-assignments.go | 3 +- cmd/list-resource-group-role-assignments.go | 3 +- cmd/list-resource-groups.go | 3 +- cmd/list-role-assignments.go | 3 +- cmd/list-service-principal-owners.go | 3 +- cmd/list-storage-account-role-assignments.go | 3 +- cmd/list-storage-accounts.go | 3 +- cmd/list-storage-containers.go | 3 +- cmd/list-subscription-role-assignments.go | 3 +- cmd/list-virtual-machine-role-assignments.go | 3 +- cmd/list-virtual-machines.go | 3 +- cmd/list-vm-scale-set-role-assignments.go | 3 +- cmd/list-vm-scale-sets.go | 3 +- cmd/list-web-app-role-assignments.go | 3 +- cmd/list-web-apps.go | 3 +- cmd/start.go | 3 +- cmd/svc_windows.go | 1 + cmd/utils.go | 1 + config/config.go | 51 +++++++++++++++++++ config/internal/config.go | 2 + config/utils.go | 19 +++++++ 40 files changed, 145 insertions(+), 36 deletions(-) diff --git a/client/rest/http.go b/client/rest/http.go index 5fa256a..cc729fe 100644 --- a/client/rest/http.go +++ b/client/rest/http.go @@ -29,13 +29,14 @@ import ( "strings" "time" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/constants" ) func NewHTTPClient(proxyUrl string) (*http.Client, error) { transport := http.DefaultTransport.(*http.Transport).Clone() - transport.MaxConnsPerHost = 200 - transport.MaxIdleConnsPerHost = 200 + transport.MaxConnsPerHost = config.ColMaxConnsPerHost.Value().(int) + transport.MaxIdleConnsPerHost = config.ColMaxIdleConnsPerHost.Value().(int) transport.DisableKeepAlives = false // defaults to TLS 1.0 which is not favorable diff --git a/cmd/list-app-owners.go b/cmd/list-app-owners.go index 5fe7336..5bb4b73 100644 --- a/cmd/list-app-owners.go +++ b/cmd/list-app-owners.go @@ -25,6 +25,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -61,7 +62,7 @@ func listAppOwnersCmdImpl(cmd *cobra.Command, args []string) { func listAppOwners(ctx context.Context, client client.AzureClient, apps <-chan azureWrapper[models.App]) <-chan azureWrapper[models.AppOwners] { var ( out = make(chan azureWrapper[models.AppOwners]) - streams = pipeline.Demux(ctx.Done(), apps, 25) + streams = pipeline.Demux(ctx.Done(), apps, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-app-role-assignments.go b/cmd/list-app-role-assignments.go index 9e3aa4b..42b8ab2 100644 --- a/cmd/list-app-role-assignments.go +++ b/cmd/list-app-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listAppRoleAssignments(ctx context.Context, client client.AzureClient, serv var ( out = make(chan interface{}) filteredSPs = make(chan models.ServicePrincipal) - streams = pipeline.Demux(ctx.Done(), filteredSPs, 25) + streams = pipeline.Demux(ctx.Done(), filteredSPs, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-automation-account-role-assignments.go b/cmd/list-automation-account-role-assignments.go index 7b25179..de4af3c 100644 --- a/cmd/list-automation-account-role-assignments.go +++ b/cmd/list-automation-account-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listAutomationAccountRoleAssignments(ctx context.Context, client client.Azu var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-automation-accounts.go b/cmd/list-automation-accounts.go index bd51f48..ab42972 100644 --- a/cmd/list-automation-accounts.go +++ b/cmd/list-automation-accounts.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listAutomationAccounts(ctx context.Context, client client.AzureClient, subs var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-container-registries.go b/cmd/list-container-registries.go index 93a0932..faf8ddf 100644 --- a/cmd/list-container-registries.go +++ b/cmd/list-container-registries.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -68,7 +69,7 @@ func listContainerRegistries(ctx context.Context, client client.AzureClient, sub var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-container-registry-role-assignments.go b/cmd/list-container-registry-role-assignments.go index 31c3806..091c6dc 100644 --- a/cmd/list-container-registry-role-assignments.go +++ b/cmd/list-container-registry-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listContainerRegistryRoleAssignments(ctx context.Context, client client.Azu var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-device-owners.go b/cmd/list-device-owners.go index 1d0c2e6..29b2e35 100644 --- a/cmd/list-device-owners.go +++ b/cmd/list-device-owners.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listDeviceOwners(ctx context.Context, client client.AzureClient, devices <- var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-function-app-role-assignments.go b/cmd/list-function-app-role-assignments.go index d881936..dd44df5 100644 --- a/cmd/list-function-app-role-assignments.go +++ b/cmd/list-function-app-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listFunctionAppRoleAssignments(ctx context.Context, client client.AzureClie var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-function-apps.go b/cmd/list-function-apps.go index d4e9b56..8e9c836 100644 --- a/cmd/list-function-apps.go +++ b/cmd/list-function-apps.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listFunctionApps(ctx context.Context, client client.AzureClient, subscripti var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-group-members.go b/cmd/list-group-members.go index 2f534a0..80f30cc 100644 --- a/cmd/list-group-members.go +++ b/cmd/list-group-members.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -62,7 +63,7 @@ func listGroupMembers(ctx context.Context, client client.AzureClient, groups <-c var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-group-owners.go b/cmd/list-group-owners.go index e24d13b..5ff04a1 100644 --- a/cmd/list-group-owners.go +++ b/cmd/list-group-owners.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -62,7 +63,7 @@ func listGroupOwners(ctx context.Context, client client.AzureClient, groups <-ch var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-key-vault-role-assignments.go b/cmd/list-key-vault-role-assignments.go index 05b67d2..97f6c22 100644 --- a/cmd/list-key-vault-role-assignments.go +++ b/cmd/list-key-vault-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listKeyVaultRoleAssignments(ctx context.Context, client client.AzureClient, var ( out = make(chan azureWrapper[models.KeyVaultRoleAssignments]) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-key-vaults.go b/cmd/list-key-vaults.go index 66626d4..14f475c 100644 --- a/cmd/list-key-vaults.go +++ b/cmd/list-key-vaults.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listKeyVaults(ctx context.Context, client client.AzureClient, subscriptions var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-logic-app-role-assignments.go b/cmd/list-logic-app-role-assignments.go index 0dd33aa..020fe0f 100644 --- a/cmd/list-logic-app-role-assignments.go +++ b/cmd/list-logic-app-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listLogicAppRoleAssignments(ctx context.Context, client client.AzureClient, var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-logic-apps.go b/cmd/list-logic-apps.go index 415168d..cbc161d 100644 --- a/cmd/list-logic-apps.go +++ b/cmd/list-logic-apps.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -68,7 +69,7 @@ func listLogicApps(ctx context.Context, client client.AzureClient, subscriptions var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-managed-cluster-role-assignments.go b/cmd/list-managed-cluster-role-assignments.go index 102db09..0b8b291 100644 --- a/cmd/list-managed-cluster-role-assignments.go +++ b/cmd/list-managed-cluster-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listManagedClusterRoleAssignments(ctx context.Context, client client.AzureC var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-managed-clusters.go b/cmd/list-managed-clusters.go index d186666..d58fd97 100644 --- a/cmd/list-managed-clusters.go +++ b/cmd/list-managed-clusters.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -68,7 +69,7 @@ func listManagedClusters(ctx context.Context, client client.AzureClient, subscri var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-management-group-descendants.go b/cmd/list-management-group-descendants.go index 404c56e..fbfa860 100644 --- a/cmd/list-management-group-descendants.go +++ b/cmd/list-management-group-descendants.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listManagementGroupDescendants(ctx context.Context, client client.AzureClie var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-management-group-role-assignments.go b/cmd/list-management-group-role-assignments.go index 6ef9a51..60005e5 100644 --- a/cmd/list-management-group-role-assignments.go +++ b/cmd/list-management-group-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listManagementGroupRoleAssignments(ctx context.Context, client client.Azure var ( out = make(chan azureWrapper[models.ManagementGroupRoleAssignments]) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-resource-group-role-assignments.go b/cmd/list-resource-group-role-assignments.go index a6d2fb7..06e533e 100644 --- a/cmd/list-resource-group-role-assignments.go +++ b/cmd/list-resource-group-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listResourceGroupRoleAssignments(ctx context.Context, client client.AzureCl var ( out = make(chan azureWrapper[models.ResourceGroupRoleAssignments]) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-resource-groups.go b/cmd/list-resource-groups.go index c3ae948..a111d1e 100644 --- a/cmd/list-resource-groups.go +++ b/cmd/list-resource-groups.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listResourceGroups(ctx context.Context, client client.AzureClient, subscrip var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-role-assignments.go b/cmd/list-role-assignments.go index 6e7ca32..d3c3df2 100644 --- a/cmd/list-role-assignments.go +++ b/cmd/list-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listRoleAssignments(ctx context.Context, client client.AzureClient, roles < var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-service-principal-owners.go b/cmd/list-service-principal-owners.go index 7fd4c56..0c4adfe 100644 --- a/cmd/list-service-principal-owners.go +++ b/cmd/list-service-principal-owners.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listServicePrincipalOwners(ctx context.Context, client client.AzureClient, var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-storage-account-role-assignments.go b/cmd/list-storage-account-role-assignments.go index 71df35a..8f336ec 100644 --- a/cmd/list-storage-account-role-assignments.go +++ b/cmd/list-storage-account-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listStorageAccountRoleAssignments(ctx context.Context, client client.AzureC var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-storage-accounts.go b/cmd/list-storage-accounts.go index 8c91c71..035da36 100644 --- a/cmd/list-storage-accounts.go +++ b/cmd/list-storage-accounts.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listStorageAccounts(ctx context.Context, client client.AzureClient, subscri var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-storage-containers.go b/cmd/list-storage-containers.go index 322f456..7c0b5cb 100644 --- a/cmd/list-storage-containers.go +++ b/cmd/list-storage-containers.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listStorageContainers(ctx context.Context, client client.AzureClient, stora // The error message with higher values for size is // "The request was throttled." // See issue #7: https://github.com/bloodhoundad/azurehound/issues/7 - streams = pipeline.Demux(ctx.Done(), ids, 2) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-subscription-role-assignments.go b/cmd/list-subscription-role-assignments.go index 94279e3..31347df 100644 --- a/cmd/list-subscription-role-assignments.go +++ b/cmd/list-subscription-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listSubscriptionRoleAssignments(ctx context.Context, client client.AzureCli var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-virtual-machine-role-assignments.go b/cmd/list-virtual-machine-role-assignments.go index 097055e..9f80f4d 100644 --- a/cmd/list-virtual-machine-role-assignments.go +++ b/cmd/list-virtual-machine-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listVirtualMachineRoleAssignments(ctx context.Context, client client.AzureC var ( out = make(chan azureWrapper[models.VirtualMachineRoleAssignments]) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-virtual-machines.go b/cmd/list-virtual-machines.go index d28d6e5..d5bb0d4 100644 --- a/cmd/list-virtual-machines.go +++ b/cmd/list-virtual-machines.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listVirtualMachines(ctx context.Context, client client.AzureClient, subscri var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-vm-scale-set-role-assignments.go b/cmd/list-vm-scale-set-role-assignments.go index 4a82b2e..dd201db 100644 --- a/cmd/list-vm-scale-set-role-assignments.go +++ b/cmd/list-vm-scale-set-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listVMScaleSetRoleAssignments(ctx context.Context, client client.AzureClien var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-vm-scale-sets.go b/cmd/list-vm-scale-sets.go index c55f4b2..0edc59b 100644 --- a/cmd/list-vm-scale-sets.go +++ b/cmd/list-vm-scale-sets.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -69,7 +70,7 @@ func listVMScaleSets(ctx context.Context, client client.AzureClient, subscriptio var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-web-app-role-assignments.go b/cmd/list-web-app-role-assignments.go index 90ce0ef..1f0e72e 100644 --- a/cmd/list-web-app-role-assignments.go +++ b/cmd/list-web-app-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listWebAppRoleAssignments(ctx context.Context, client client.AzureClient, w var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-web-apps.go b/cmd/list-web-apps.go index ac2aaec..782a3eb 100644 --- a/cmd/list-web-apps.go +++ b/cmd/list-web-apps.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -68,7 +69,7 @@ func listWebApps(ctx context.Context, client client.AzureClient, subscriptions < var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/start.go b/cmd/start.go index e817d42..01e450b 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -53,6 +53,7 @@ var ErrExceededRetryLimit = errors.New("exceeded max retry limit for ingest batc func init() { configs := append(config.AzureConfig, config.BloodHoundEnterpriseConfig...) + configs = append(configs, config.CollectionConfig...) config.Init(startCmd, configs) rootCmd.AddCommand(startCmd) } @@ -154,7 +155,7 @@ func start(ctx context.Context) { // Batch data out for ingestion stream := listAll(ctx, azClient) - batches := pipeline.Batch(ctx.Done(), stream, 256, 10*time.Second) + batches := pipeline.Batch(ctx.Done(), stream, config.ColBatchSize.Value().(int), 10*time.Second) hasIngestErr := ingest(ctx, *bheInstance, bheClient, batches) // Notify BHE instance of job end diff --git a/cmd/svc_windows.go b/cmd/svc_windows.go index 3831993..a3109f4 100644 --- a/cmd/svc_windows.go +++ b/cmd/svc_windows.go @@ -46,6 +46,7 @@ func (s *azurehoundSvc) Init(env svc.Environment) error { return err } else { log = *logr + config.CheckCollectionConfigSanity(log) if config.ConfigFileUsed() != "" { log.V(1).Info(fmt.Sprintf("Config File: %v", config.ConfigFileUsed())) diff --git a/cmd/utils.go b/cmd/utils.go index fd59f82..6d4792a 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -72,6 +72,7 @@ func persistentPreRunE(cmd *cobra.Command, args []string) error { return err } else { log = *logr + config.CheckCollectionConfigSanity(log) if config.ConfigFileUsed() != "" { log.V(1).Info(fmt.Sprintf("Config File: %v", config.ConfigFileUsed())) diff --git a/config/config.go b/config/config.go index 578262f..64d5db0 100644 --- a/config/config.go +++ b/config/config.go @@ -264,6 +264,50 @@ var ( Default: "", } + ColBatchSize = Config{ + Name: "batchSize", + Shorthand: "", + Usage: "The number of resources to send in a single batch sent to the server.", + Persistent: true, + Required: false, + Default: 100, + MinValue: 1, + MaxValue: 256, + } + + ColMaxConnsPerHost = Config{ + Name: "maxConnsPerHost", + Shorthand: "", + Usage: "The maximum number of connections made during collection.", + Persistent: true, + Required: false, + Default: 20, + MinValue: 1, + MaxValue: 200, + } + + ColMaxIdleConnsPerHost = Config{ + Name: "maxIdleConnsPerHost", + Shorthand: "", + Usage: "The maximum number of idle connections allowed during collection.", + Persistent: true, + Required: false, + Default: 20, + MinValue: 1, + MaxValue: 200, + } + + ColStreamCount = Config{ + Name: "streamCount", + Shorthand: "", + Usage: "The number of threads to use when collecting various resources.", + Persistent: true, + Required: true, + Default: 2, + MinValue: 1, + MaxValue: 25, + } + // Command specific configurations KeyVaultAccessTypes = Config{ Name: "access-types", @@ -314,6 +358,13 @@ var ( BHETokenId, BHEToken, } + + CollectionConfig = []Config{ + ColBatchSize, + ColMaxConnsPerHost, + ColMaxIdleConnsPerHost, + ColStreamCount, + } ) func ConfigFileUsed() string { diff --git a/config/internal/config.go b/config/internal/config.go index 83517ac..5cff72f 100644 --- a/config/internal/config.go +++ b/config/internal/config.go @@ -36,6 +36,8 @@ type Config struct { Required bool Persistent bool Default interface{} + MinValue int + MaxValue int } func (s Config) Value() interface{} { diff --git a/config/utils.go b/config/utils.go index 27c36c1..a7c36a1 100644 --- a/config/utils.go +++ b/config/utils.go @@ -24,6 +24,7 @@ import ( client "github.com/bloodhoundad/azurehound/v2/client/config" config "github.com/bloodhoundad/azurehound/v2/config/internal" "github.com/bloodhoundad/azurehound/v2/constants" + "github.com/go-logr/logr" ) var Init = config.Init @@ -49,6 +50,24 @@ func SetAzureDefaults() { } } +func CheckCollectionConfigSanity(log logr.Logger) { + useSaneIntValues(ColBatchSize, log) + useSaneIntValues(ColMaxConnsPerHost, log) + useSaneIntValues(ColMaxIdleConnsPerHost, log) + useSaneIntValues(ColStreamCount, log) +} + +func useSaneIntValues(c config.Config, log logr.Logger) { + val := c.Value().(int) + if val < c.MinValue { + log.V(1).Info(fmt.Sprintf("Provided value %d for config option %s is less than minimum value %d. Using default value %d.", val, c.Name, c.MinValue, c.Default)) + c.Set(c.Default) + } else if val > c.MaxValue { + log.V(1).Info(fmt.Sprintf("Provided value %d for config option %s is greater than maximum value %d. Using default value %d.", val, c.Name, c.MaxValue, c.Default)) + c.Set(c.Default) + } +} + func ValidateURL(input string) error { if parsedURL, err := url.Parse(input); err != nil { return err From 98a7af9cd91970883292cd47faadc5c7e6eea829 Mon Sep 17 00:00:00 2001 From: James Barnett Date: Mon, 12 Aug 2024 12:00:25 -0400 Subject: [PATCH 2/4] fix: shouldnt have been marked required --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 64d5db0..3e48f94 100644 --- a/config/config.go +++ b/config/config.go @@ -302,7 +302,7 @@ var ( Shorthand: "", Usage: "The number of threads to use when collecting various resources.", Persistent: true, - Required: true, + Required: false, Default: 2, MinValue: 1, MaxValue: 25, From 575907c1d6a6be819b943faaaf071519eab19310 Mon Sep 17 00:00:00 2001 From: James Barnett Date: Mon, 12 Aug 2024 12:06:46 -0400 Subject: [PATCH 3/4] test: Add tests for sanity checks --- config/utils_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 config/utils_test.go diff --git a/config/utils_test.go b/config/utils_test.go new file mode 100644 index 0000000..9fafccc --- /dev/null +++ b/config/utils_test.go @@ -0,0 +1,75 @@ +// Copyright (C) 2022 Specter Ops, Inc. +// +// This file is part of AzureHound. +// +// AzureHound is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// AzureHound is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package config_test + +import ( + "testing" + + "github.com/bloodhoundad/azurehound/v2/config" + "github.com/bloodhoundad/azurehound/v2/logger" +) + +func TestCheckCollectionConfigSanity(t *testing.T) { + config.JsonLogs.Set(true) + + if logr, err := logger.GetLogger(); err != nil { + t.Errorf("Error creating logger: %v", err) + } else { + log := *logr + config.CheckCollectionConfigSanity(log) + + if config.ColBatchSize.Value().(int) != config.ColBatchSize.Default { + t.Errorf("ColBatchSize did not have the default value of %d. Actual: %d", config.ColBatchSize.Default, config.ColBatchSize.Value()) + } + + if config.ColMaxConnsPerHost.Value().(int) != config.ColMaxConnsPerHost.Default { + t.Errorf("ColMaxConnsPerHost did not have the default value of %d. Actual: %d", config.ColMaxConnsPerHost.Default, config.ColMaxConnsPerHost.Value()) + } + + if config.ColMaxIdleConnsPerHost.Value().(int) != config.ColMaxIdleConnsPerHost.Default { + t.Errorf("ColMaxIdleConnsPerHost did not have the default value of %d. Actual: %d", config.ColMaxIdleConnsPerHost.Default, config.ColMaxIdleConnsPerHost.Value()) + } + + if config.ColStreamCount.Value().(int) != config.ColStreamCount.Default { + t.Errorf("ColStreamCount did not have the default value of %d. Actual: %d", config.ColStreamCount.Default, config.ColStreamCount.Value()) + } + } +} + +func TestCheckCollectionConfigSanityOutOfBounds(t *testing.T) { + config.JsonLogs.Set(true) + + if logr, err := logger.GetLogger(); err != nil { + t.Errorf("Error creating logger: %v", err) + } else { + log := *logr + + config.ColBatchSize.Set(9999) + config.ColMaxConnsPerHost.Set(-9999) + + config.CheckCollectionConfigSanity(log) + + if config.ColBatchSize.Value().(int) != config.ColBatchSize.Default { + t.Errorf("ColBatchSize should have reverted to the default value of %d. Actual: %d", config.ColBatchSize.Default, config.ColBatchSize.Value()) + } + + if config.ColMaxConnsPerHost.Value().(int) != config.ColMaxConnsPerHost.Default { + t.Errorf("ColMaxConnsPerHost should have reverted to the default value of %d. Actual: %d", config.ColMaxConnsPerHost.Default, config.ColMaxConnsPerHost.Value()) + } + } +} From 4379121bcee67912d5559dc739730c8a1c467a77 Mon Sep 17 00:00:00 2001 From: James Barnett Date: Thu, 29 Aug 2024 11:16:09 -0400 Subject: [PATCH 4/4] chore: commit gofmt changes --- cmd/list-storage-containers.go | 2 +- cmd/list-vm-scale-sets.go | 2 +- models/azure/app_scope.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/list-storage-containers.go b/cmd/list-storage-containers.go index 4fd5fb6..8578bf6 100644 --- a/cmd/list-storage-containers.go +++ b/cmd/list-storage-containers.go @@ -105,7 +105,7 @@ func listStorageContainers(ctx context.Context, client client.AzureClient, stora storageContainer := models.StorageContainer{ StorageContainer: item.Ok, StorageAccountId: stAccount.(models.StorageAccount).StorageAccount.Id, - SubscriptionId: "/subscriptions/"+stAccount.(models.StorageAccount).SubscriptionId, + SubscriptionId: "/subscriptions/" + stAccount.(models.StorageAccount).SubscriptionId, ResourceGroupId: item.Ok.ResourceGroupId(), ResourceGroupName: item.Ok.ResourceGroupName(), TenantId: client.TenantInfo().TenantId, diff --git a/cmd/list-vm-scale-sets.go b/cmd/list-vm-scale-sets.go index 93cc648..e4c42e8 100644 --- a/cmd/list-vm-scale-sets.go +++ b/cmd/list-vm-scale-sets.go @@ -103,7 +103,7 @@ func listVMScaleSets(ctx context.Context, client client.AzureClient, subscriptio } else { vmScaleSet := models.VMScaleSet{ VMScaleSet: item.Ok, - SubscriptionId: "/subscriptions/"+id, + SubscriptionId: "/subscriptions/" + id, ResourceGroupId: item.Ok.ResourceGroupId(), TenantId: client.TenantInfo().TenantId, } diff --git a/models/azure/app_scope.go b/models/azure/app_scope.go index 45908ac..ff2da07 100644 --- a/models/azure/app_scope.go +++ b/models/azure/app_scope.go @@ -23,8 +23,8 @@ package azure // // This may be in both the following principal and scope scenarios: // -// A single principal and a single scope -// Multiple principals and multiple scopes. +// A single principal and a single scope +// Multiple principals and multiple scopes. type AppScope struct { Entity