From 8374bc3b6226c54f6bc05a0bede99a16937eb1a0 Mon Sep 17 00:00:00 2001 From: Hayden Roszell Date: Tue, 19 Nov 2024 14:08:26 -0700 Subject: [PATCH 1/2] chore(paginate): Paginate returned Cert Stores in Discovery job & Script Signed-off-by: Hayden Roszell --- .../Client/GraphClient.cs | 178 +++++++++++++++--- CHANGELOG.md | 3 + Scripts/DefineDiscoveredStores.ps1 | 85 +++++++-- integration-manifest.json | 2 +- 4 files changed, 219 insertions(+), 49 deletions(-) diff --git a/AzureEnterpriseApplicationOrchestrator/Client/GraphClient.cs b/AzureEnterpriseApplicationOrchestrator/Client/GraphClient.cs index 467aafc..b5288df 100644 --- a/AzureEnterpriseApplicationOrchestrator/Client/GraphClient.cs +++ b/AzureEnterpriseApplicationOrchestrator/Client/GraphClient.cs @@ -485,14 +485,44 @@ public OperationResult> DiscoverApplicationObjectIds() List oids = new(); OperationResult> result = new(oids); - _logger.LogDebug($"Retrieving application registrations for tenant ID \"{_tenantId}\""); - ApplicationCollectionResponse apps; + _logger.LogDebug($"Retrieving Applications for tenant ID \"{_tenantId}\""); + List allApplications = new List(); try { - apps = _graphClient.Applications.GetAsync((requestConfiguration) => + var spsResponse = _graphClient.Applications.GetAsync(requestConfiguration => + { + requestConfiguration.QueryParameters.Top = 500; + }).Result; + + if (spsResponse?.Value == null) + { + _logger.LogWarning("No Applications found."); + return result; + } + + // Create the PageIterator to handle pagination + var pageIterator = PageIterator + .CreatePageIterator( + _graphClient, + spsResponse, + // Callback executed for each Application in the collection + (application) => { - requestConfiguration.QueryParameters.Top = 999; - }).Result; + // Add each application to the list + allApplications.Add(application); + return true; // Continue to the next item + }, + // Configure subsequent page requests + (request) => + { + _logger.LogDebug("Fetching the next page of applications..."); + return request; + }); + + // Execute the pagination + pageIterator.IterateAsync().Wait(); + + _logger.LogInformation($"Successfully retrieved {allApplications.Count} applications."); } catch (AggregateException e) { @@ -500,15 +530,15 @@ public OperationResult> DiscoverApplicationObjectIds() throw; } - if (apps?.Value == null || apps.Value.Count == 0) + if (allApplications.Count == 0) { - _logger.LogWarning($"No application registrations found for tenant ID \"{_tenantId}\""); + _logger.LogWarning($"No Applications found for tenant ID \"{_tenantId}\""); return result; } - foreach (Application app in apps.Value) + foreach (Application app in allApplications) { - _logger.LogDebug($"Found application \"{app.DisplayName}\" ({app.Id})"); + _logger.LogDebug($"Found Application \"{app.DisplayName}\" ({app.Id})"); if (string.IsNullOrEmpty(app.Id)) { @@ -529,13 +559,43 @@ public OperationResult> DiscoverServicePrincipalObjectIds() OperationResult> result = new(oids); _logger.LogDebug($"Retrieving Service Principals for tenant ID \"{_tenantId}\""); - ServicePrincipalCollectionResponse sps; + List allServicePrincipals = new List(); try { - sps = _graphClient.ServicePrincipals.GetAsync((requestConfiguration) => + var spsResponse = _graphClient.ServicePrincipals.GetAsync(requestConfiguration => { - requestConfiguration.QueryParameters.Top = 999; + requestConfiguration.QueryParameters.Top = 500; }).Result; + + if (spsResponse?.Value == null) + { + _logger.LogWarning("No service principals found."); + return result; + } + + // Create the PageIterator to handle pagination + var pageIterator = PageIterator + .CreatePageIterator( + _graphClient, + spsResponse, + // Callback executed for each Service Principal in the collection + (servicePrincipal) => + { + // Add each service principal to the list + allServicePrincipals.Add(servicePrincipal); + return true; // Continue to the next item + }, + // Configure subsequent page requests + (request) => + { + _logger.LogDebug("Fetching the next page of service principals..."); + return request; + }); + + // Execute the pagination + pageIterator.IterateAsync().Wait(); + + _logger.LogInformation($"Successfully retrieved {allServicePrincipals.Count} service principals."); } catch (AggregateException e) { @@ -543,13 +603,13 @@ public OperationResult> DiscoverServicePrincipalObjectIds() throw; } - if (sps?.Value == null || sps.Value.Count == 0) + if (allServicePrincipals.Count == 0) { _logger.LogWarning($"No Service Principals found for tenant ID \"{_tenantId}\""); return result; } - foreach (ServicePrincipal sp in sps.Value) + foreach (ServicePrincipal sp in allServicePrincipals) { _logger.LogDebug($"Found SP \"{sp.DisplayName}\" ({sp.Id})"); @@ -571,30 +631,60 @@ public OperationResult> DiscoverApplicationApplicationIds() List appIds = new(); OperationResult> result = new(appIds); - _logger.LogDebug($"Retrieving application registrations for tenant ID \"{_tenantId}\""); - ApplicationCollectionResponse apps; + _logger.LogDebug($"Retrieving Applications in tenant ID \"{_tenantId}\""); + List allApplications = new List(); try { - apps = _graphClient.Applications.GetAsync((requestConfiguration) => + var spsResponse = _graphClient.Applications.GetAsync(requestConfiguration => + { + requestConfiguration.QueryParameters.Top = 500; + }).Result; + + if (spsResponse?.Value == null) + { + _logger.LogWarning("No applications found."); + return result; + } + + // Create the PageIterator to handle pagination + var pageIterator = PageIterator + .CreatePageIterator( + _graphClient, + spsResponse, + // Callback executed for each Application in the collection + (application) => { - requestConfiguration.QueryParameters.Top = 999; - }).Result; + // Add each application to the list + allApplications.Add(application); + return true; // Continue to the next item + }, + // Configure subsequent page requests + (request) => + { + _logger.LogDebug("Fetching the next page of applications..."); + return request; + }); + + // Execute the pagination + pageIterator.IterateAsync().Wait(); + + _logger.LogInformation($"Successfully retrieved {allApplications.Count} applications."); } catch (AggregateException e) { - _logger.LogError($"Unable to retrieve application registrations for tenant ID \"{_tenantId}\": {e}"); + _logger.LogError($"Unable to retrieve Applications for tenant ID \"{_tenantId}\": {e}"); throw; } - if (apps?.Value == null || apps.Value.Count == 0) + if (allApplications.Count == 0) { - _logger.LogWarning($"No application registrations found for tenant ID \"{_tenantId}\""); + _logger.LogWarning($"No Applications found for tenant ID \"{_tenantId}\""); return result; } - foreach (Application app in apps.Value) + foreach (Application app in allApplications) { - _logger.LogDebug($"Found application \"{app.DisplayName}\" ({app.Id})"); + _logger.LogDebug($"Found Application \"{app.DisplayName}\" ({app.Id})"); if (string.IsNullOrEmpty(app.AppId)) { @@ -615,13 +705,43 @@ public OperationResult> DiscoverServicePrincipalApplicationI OperationResult> result = new(appIds); _logger.LogDebug($"Retrieving Service Principals for tenant ID \"{_tenantId}\""); - ServicePrincipalCollectionResponse sps; + List allServicePrincipals = new List(); try { - sps = _graphClient.ServicePrincipals.GetAsync((requestConfiguration) => + var spsResponse = _graphClient.ServicePrincipals.GetAsync(requestConfiguration => { - requestConfiguration.QueryParameters.Top = 999; + requestConfiguration.QueryParameters.Top = 500; }).Result; + + if (spsResponse?.Value == null) + { + _logger.LogWarning("No service principals found."); + return result; + } + + // Create the PageIterator to handle pagination + var pageIterator = PageIterator + .CreatePageIterator( + _graphClient, + spsResponse, + // Callback executed for each Service Principal in the collection + (servicePrincipal) => + { + // Add each service principal to the list + allServicePrincipals.Add(servicePrincipal); + return true; // Continue to the next item + }, + // Configure subsequent page requests + (request) => + { + _logger.LogDebug("Fetching the next page of service principals..."); + return request; + }); + + // Execute the pagination + pageIterator.IterateAsync().Wait(); + + _logger.LogInformation($"Successfully retrieved {allServicePrincipals.Count} service principals."); } catch (AggregateException e) { @@ -629,13 +749,13 @@ public OperationResult> DiscoverServicePrincipalApplicationI throw; } - if (sps?.Value == null || sps.Value.Count == 0) + if (allServicePrincipals.Count == 0) { _logger.LogWarning($"No Service Principals found for tenant ID \"{_tenantId}\""); return result; } - foreach (ServicePrincipal sp in sps.Value) + foreach (ServicePrincipal sp in allServicePrincipals) { _logger.LogDebug($"Found SP \"{sp.DisplayName}\" ({sp.Id})"); diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ca1d0e..d21cf8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,3 +22,6 @@ - Discovery job modified to return available Certificate Stores with Store Path in the format ` ()`. - Before other jobs operate on Certificate Stores, the contents after the ID GUID will be truncated, maintaining backward compatibility. +- 4.1.0 + - Paginate discovery method to download all available App Registrations and Enterprise Applications. + - Fix DefineDiscoveredStores.ps1 to paginate Certificate Stores that come back from Command. diff --git a/Scripts/DefineDiscoveredStores.ps1 b/Scripts/DefineDiscoveredStores.ps1 index ab7acc5..2b9a044 100644 --- a/Scripts/DefineDiscoveredStores.ps1 +++ b/Scripts/DefineDiscoveredStores.ps1 @@ -187,7 +187,9 @@ function Submit-RESTRequest return $apiResponse } -# Step 1: Get the available Certificate Store Types +Write-Host "============================================================================================" +Write-Host "Step 1: Get the Store Type ID that corresponds to $CertificateStoreType" +Write-Host "============================================================================================" $certificateStoreTypes = Submit-RESTRequest -Method GET -Path "CertificateStoreTypes" $desiredStoreType = $certificateStoreTypes | Where-Object { $_.ShortName -eq $CertificateStoreType } @@ -201,26 +203,71 @@ if (-not $desiredStoreType) $certStoreTypeId = $desiredStoreType.StoreType Write-Host "$CertificateStoreType has Type ID $certStoreTypeId" -# Step 3: Fetch the Certificate Stores -$certificateStores = Submit-RESTRequest -Method GET -Path "CertificateStores" +Write-Host "============================================================================================" +Write-Host "Step 2: Download all Certificate Stores from Command with pagination" +Write-Host "============================================================================================" +$pageSize = 100 +$currentPage = 1 +$allPagesDownloaded = $false -# Step 4: Process the Certificate Stores -$storesToProcess = $certificateStores | Where-Object { $_.Approved -eq $false -and $_.CertStoreType -eq $certStoreTypeId } -$storesToProcessLength = $storesToProcess.Length +$totalDownloadedCertStores +$allCertificateStores = @() +do +{ + Write-Host "Downloading Certificate Stores page $currentPage (page size $pageSize)" + try + { + $certificateStores = Submit-RESTRequest -Method GET -Path "CertificateStores?ReturnLimit=$pageSize&PageReturned=$currentPage" + + # Check if any certificate stores are returned + if ($certificateStores.Count -eq 0) + { + $allPagesDownloaded = $true + } else + { + Write-Host "Fetched $($certificateStores.Count) certificate stores." + $totalDownloadedCertStores += $certificateStores.Count + + $allCertificateStores += $certificateStores + } + + # Move to the next page + $currentPage++ + + } catch + { + Write-Error "Failed to fetch certificate stores: $_" + $allPagesDownloaded = $true # Exit the loop on error + } + +} while (!$allPagesDownloaded) -Write-Host "Found $storesToProcessLength Discovered Certificate Stores of type $CertificateStoreType" +Write-Host "Finished downloading $totalDownloadedCertStores total certificate stores in $($currentPage - 2) pages" +Write-Host "============================================================================================" +Write-Host "Step 3: Filter the downloaded Certificate Stores for ones that came back in Discovery" +Write-Host "============================================================================================" +$storesToProcess = $allCertificateStores | Where-Object { $_.Approved -eq $false -and $_.CertStoreType -eq $certStoreTypeId } + +Write-Host "$($storesToProcess.Length)/$totalDownloadedCertStores downloaded Certificate Stores are Discovered Certificate Stores of type $CertificateStoreType ($certStoreTypeId) (only exist on the Discovery page; haven't been defined in Command)" + +Write-Host "============================================================================================" +Write-Host "Step 4: Update (define) Certificate Stores that are on the Whitelist" +Write-Host "============================================================================================" foreach ($store in $storesToProcess) { # Truncate Storepath to extract GUID - $storePathGuid = $store.Storepath.Split(" ")[0] + $storePathParts = $store.Storepath.Split(" ") + $storePathGuid = $storePathParts[0] if (-not ($whitelistGuids -contains $storePathGuid)) { - Write-Host "Skipping store with Id $($store.Id) as its Storepath GUID '$storePathGuid' is not in the whitelist." + Write-Host "Skipping store with Path '$($store.Storepath)' as its Storepath GUID '$storePathGuid' is not in the whitelist." continue } + Write-Host "Certificate Store with Path '$($store.Storepath)' was found in the whitelist - adding" + # Add/update the properties $properties = @{ ServerUsername = @{ @@ -233,16 +280,16 @@ foreach ($store in $storesToProcess) SecretValue = $ServicePrincipalClientSecret } } - ClientCertificate = @{ - value = @{ - SecretValue = "" - } - } - ClientCertificatePassword = @{ - value = @{ - SecretValue = "" - } - } + # ClientCertificate = @{ + # value = @{ + # SecretValue = "" + # } + # } + # ClientCertificatePassword = @{ + # value = @{ + # SecretValue = "" + # } + # } ServerUseSsl = @{ value = "true" } diff --git a/integration-manifest.json b/integration-manifest.json index 6eabd4a..0fb8326 100644 --- a/integration-manifest.json +++ b/integration-manifest.json @@ -292,4 +292,4 @@ ] } } -} \ No newline at end of file +} From 6a1b9da038691293995d79bdcac425b64510f2a4 Mon Sep 17 00:00:00 2001 From: Keyfactor Date: Tue, 19 Nov 2024 21:09:47 +0000 Subject: [PATCH 2/2] Update generated docs --- README.md | 4 ++++ integration-manifest.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6123f6c..91534db 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ The Azure App Registration and Enterprise Application Universal Orchestrator ext
Azure App Registration (Application) (AzureApp) + ### AzureApp > **WARNING** AzureApp "Azure App Registration (Application)" is **Depricated**. Please use **AzureApp2** "Azure App Registration 2 (Application)" instead. @@ -48,6 +49,7 @@ Azure [App Registration/Application certificates](https://learn.microsoft.com/en
Azure Enterprise Application (Service Principal) (AzureSP) + ### AzureSP > **WARNING** AzureSP "Azure Enterprise Application (Service Principal)" is **Depricated**. Please use **AzureSP2** "Azure Enterprise Application 2 (Service Principal)" instead. @@ -57,6 +59,7 @@ The Azure Enterprise Application/Service Principal certificate operations are im
Azure App Registration 2 (Application) (AzureApp2) + ### AzureApp2 Azure [App Registration/Application certificates](https://learn.microsoft.com/en-us/entra/identity-platform/certificate-credentials) are typically used for client authentication by applications and are typically public key only in Azure. The general model by which these credentials are consumed is that the certificate and private key are accessible by the Application using the App Registration, and are passed to the service that is authenticating the Application. The Azure App Registration and Enterprise Application Orchestrator extension implements the Inventory, Management Add, Management Remove, and Discovery job types for managing these certificates. @@ -64,6 +67,7 @@ Azure [App Registration/Application certificates](https://learn.microsoft.com/en
Azure Enterprise Application 2 (Service Principal) (AzureSP2) + ### AzureSP2 The Azure Enterprise Application/Service Principal certificate operations are implemented by the `AzureSP` store type, and supports the management of a single certificate for use in SSO/SAML assertion signing. The Management Add operation is only supported with the certificate replacement option, since adding a new certificate will replace the existing certificate. The Add operation will also set newly added certificates as the active certificate for SSO/SAML usage. The Management Remove operation removes the certificate from the Enterprise Application/Service Principal, which is the same as removing the SSO/SAML signing certificate. The Discovery operation discovers all Enterprise Applications/Service Principals in the tenant. diff --git a/integration-manifest.json b/integration-manifest.json index 0fb8326..6eabd4a 100644 --- a/integration-manifest.json +++ b/integration-manifest.json @@ -292,4 +292,4 @@ ] } } -} +} \ No newline at end of file