wip: wip

src/CommonLib/LDAPUtilsNew.cs

@@ -40,37 +40,51 @@ public class LDAPUtilsNew {
     private readonly object _lockObj = new();
     private readonly ManualResetEvent _connectionResetEvent = new(false);
 
-    public async IAsyncEnumerable<ISearchResultEntry> PagedQuery(LdapQueryParameters queryParameters,
+    public async IAsyncEnumerable<LdapResult<ISearchResultEntry>> PagedQuery(LdapQueryParameters queryParameters,
         [EnumeratorCancellation] CancellationToken cancellationToken = new()) {
         //Always force create a new connection
         var (success, connectionWrapper, message) = await GetLdapConnection(queryParameters.DomainName,
             queryParameters.GlobalCatalog, true);
         if (!success) {
             _log.LogDebug("PagedQuery failure: unable to create a connection: {Reason}\n{Info}", message,
                 queryParameters.GetQueryInfo());
+            yield return new LdapResult<ISearchResultEntry> {
+                Error = $"Unable to create a connection: {message}",
+                QueryInfo = queryParameters.GetQueryInfo()
+            };
             yield break;
         }
 
         //This should never happen as far as I know, so just checking for safety
         if (connectionWrapper == null) {
-            _log.LogWarning("PagedQuery failure: ldap connection is null\n{Info}", queryParameters.GetQueryInfo());
+            _log.LogError("PagedQuery failure: ldap connection is null\n{Info}", queryParameters.GetQueryInfo());
+            yield return new LdapResult<ISearchResultEntry> {
+                Error = "Connection is null",
+                QueryInfo = queryParameters.GetQueryInfo()
+            };
             yield break;
         }
 
         //Pull the server name from the connection for retry logic later
         if (!connectionWrapper.GetServer(out var serverName)) {
+            _log.LogDebug("PagedQuery: Failed to get server value");
             serverName = null;
         }
 
         if (!CreateSearchRequest(queryParameters, ref connectionWrapper, out var searchRequest)) {
             _log.LogError("PagedQuery failure: unable to resolve search base\n{Info}", queryParameters.GetQueryInfo());
+            yield return new LdapResult<ISearchResultEntry> {
+                Error = "Unable to create search request",
+                QueryInfo = queryParameters.GetQueryInfo()
+            };
             yield break;
         }
 
         var pageControl = new PageResultRequestControl(500);
         searchRequest.Controls.Add(pageControl);
 
         PageResultResponseControl pageResponse = null;
+        var busyRetryCount = 0;
 
         while (true) {
             if (cancellationToken.IsCancellationRequested) {
@@ -96,58 +110,65 @@ public async IAsyncEnumerable<ISearchResultEntry> PagedQuery(LdapQueryParameters
                         queryParameters.GetQueryInfo());
                     yield break;
                 }
-                /*A ServerDown exception indicates that our connection is no longer valid for one of many reasons.
-                    However, this function is generally called by multiple threads, so we need to be careful in recreating
-                    the connection. Using a semaphore, we can ensure that only one thread is actually recreating the connection
-                    while the other threads that hit the ServerDown exception simply wait. The initial caller will hold the semaphore
-                    and do a backoff delay before trying to make a new connection which will replace the existing connection in the
-                    _ldapConnections cache. Other threads will retrieve the new connection from the cache instead of making a new one
-                    This minimizes overhead of new connections while still fixing our core problem.*/
-
-                var backoffDelay = MinBackoffDelay;
-                var retryCount = 0;
-
-                //Attempt to acquire a lock
-                if (Monitor.TryEnter(_lockObj)) {
-                    //If we've acquired the lock, we want to immediately signal our reset event so everyone else waits
-                    _connectionResetEvent.Reset();
-                    try {
-                        //Sleep for our backoff
-                        Thread.Sleep(backoffDelay);
-                        //Explicitly skip the cache so we don't get the same connection back
-                        connectionWrapper = GetLdapConnectionForServer(serverName)
-                        conn = CreateNewConnection(domainName, globalCatalog, true).Connection;
-                        if (conn == null) {
-                            _log.LogError(
-                                "Unable to create replacement ldap connection for ServerDown exception. Breaking loop");
-                            yield break;
-                        }
-
-                        _log.LogInformation("Created new LDAP connection after receiving ServerDown from server");
+
+                /*
+                 * Paged queries will not use the cached ldap connections, as the intention is to only have 1 or a couple of these queries running at once.
+                 * The connection logic here is simplified accordingly
+                 */
+                for (var retryCount = 0; retryCount < MaxRetries; retryCount++) {
+                    var backoffDelay = GetNextBackoff(retryCount);
+                    await Task.Delay(backoffDelay, cancellationToken);
+                    if (GetLdapConnectionForServer(serverName, out var newConnectionWrapper,
+                            queryParameters.GlobalCatalog,
+                            true)) {
+                        newConnectionWrapper.CopyContexts(connectionWrapper);
+                        connectionWrapper.Connection.Dispose();
+                        connectionWrapper = newConnectionWrapper;
+                        _log.LogDebug(
+                            "PagedQuery - Successfully created new ldap connection to {Server} after ServerDown",
+                            serverName);
+                        break;
                     }
-                    finally {
-                        //Reset our event + release the lock
-                        _connectionResetEvent.Set();
-                        Monitor.Exit(_lockObj);
+
+                    if (retryCount == MaxRetries - 1) {
+                        _log.LogError("PagedQuery - Failed to get a new connection after ServerDown.\n{Info}",
+                            queryParameters.GetQueryInfo());
+                        yield break;
                     }
                 }
-                else {
-                    //If someone else is holding the reset event, we want to just wait and then pull the newly created connection out of the cache
-                    //This event will be released after the first entrant thread is done making a new connection
-                    //The thread.sleep is to prevent a potential, very unlikely race
-                    Thread.Sleep(50);
-                    _connectionResetEvent.WaitOne();
-                    conn = CreateNewConnection(domainName, globalCatalog).Connection;
+            }
+            catch (LdapException le) when (le.ErrorCode == (int)ResultCode.Busy && busyRetryCount < MaxRetries) {
+                /*
+                 * If we get a busy error, we want to do an exponential backoff, but maintain the current connection
+                 * The expectation is that given enough time, the server should stop being busy and service our query appropriately
+                 */
+                busyRetryCount++;
+                var backoffDelay = GetNextBackoff(busyRetryCount);
+                await Task.Delay(backoffDelay, cancellationToken);
+            }
+            catch (LdapException le) {
+                //No point in printing local exceptions because they're literally worthless
+                if (le.ErrorCode != (int)LdapErrorCodes.LocalError)
+                {
+                    _log.LogWarning(le,
+                        "LDAP Exception in Loop: {ErrorCode}. {ServerErrorMessage}. {Message}. Filter: {Filter}. Domain: {Domain}",
+                        le.ErrorCode, le.ServerErrorMessage, le.Message, ldapFilter, domainName);
                 }
 
-                backoffDelay = GetNextBackoff(retryCount);
-                continue;
+                yield break;
             }
         }
     }
+
+    private static TimeSpan GetNextBackoff(int retryCount)
+    {
+        return TimeSpan.FromSeconds(Math.Min(
+            MinBackoffDelay.TotalSeconds * Math.Pow(BackoffDelayMultiplier, retryCount),
+            MaxBackoffDelay.TotalSeconds));
+    }
 
     private bool CreateSearchRequest(LdapQueryParameters queryParameters,
-        ref LdapConnectionWrapperNew connectionWrapper, out SearchRequest searchRequest, bool paged = false) {
+        ref LdapConnectionWrapperNew connectionWrapper, out SearchRequest searchRequest) {
         string basePath;
         if (!string.IsNullOrWhiteSpace(queryParameters.SearchBase)) {
             basePath = queryParameters.SearchBase;

src/CommonLib/LdapConnectionWrapperNew.cs

@@ -20,6 +20,13 @@ public LdapConnectionWrapperNew(LdapConnection connection, ISearchResultEntry en
         _searchResultEntry = entry;
     }
 
+    public void CopyContexts(LdapConnectionWrapperNew other) {
+        _domainSearchBase = other._domainSearchBase;
+        _configurationSearchBase = other._configurationSearchBase;
+        _schemaSearchBase = other._schemaSearchBase;
+        _server = other._server;
+    }
+
     public bool GetServer(out string server) {
         if (_server != null) {
             server = _server;

src/CommonLib/LdapQueryParameters.cs

@@ -15,6 +15,7 @@ public class LdapQueryParameters
     public bool IncludeDeleted { get; set; } = false;
     public string SearchBase { get; set; }
     public NamingContext NamingContext { get; set; } = NamingContext.Default;
+    public bool ThrowException { get; set; } = false;
 
     public string GetQueryInfo()
     {

src/CommonLib/LdapResult.cs

@@ -0,0 +1,11 @@
+using System;
+
+namespace SharpHoundCommonLib;
+
+public class LdapResult<T>
+{
+    public T Value { get; set; }
+    public string Error { get; set; }
+    public bool IsSuccess => Error == null;
+    public string QueryInfo { get; set; }
+}

src/CommonLib/Processors/CertAbuseProcessor.cs

@@ -384,15 +384,15 @@ await SendComputerStatus(new CSVComputerStatus
         // TODO: Copied from URA processor. Find a way to have this function in a shared spot
 
 
-        public virtual Result<ISAMServer> OpenSamServer(string computerName)
+        public virtual LdapResult<ISAMServer> OpenSamServer(string computerName)
         {
             var result = SAMServer.OpenServer(computerName);
             if (result.IsFailed)
             {
-                return Result<ISAMServer>.Fail(result.SError);
+                return LdapResult<ISAMServer>.Fail(result.SError);
             }
 
-            return Result<ISAMServer>.Ok(result.Value);
+            return LdapResult<ISAMServer>.Ok(result.Value);
         }
 
         private async Task SendComputerStatus(CSVComputerStatus status)

src/CommonLib/Processors/LocalGroupProcessor.cs

@@ -26,15 +26,15 @@ public LocalGroupProcessor(ILDAPUtils utils, ILogger log = null)
 
         public event ComputerStatusDelegate ComputerStatusEvent;
 
-        public virtual Result<ISAMServer> OpenSamServer(string computerName)
+        public virtual LdapResult<ISAMServer> OpenSamServer(string computerName)
         {
             var result = SAMServer.OpenServer(computerName);
             if (result.IsFailed)
             {
-                return Result<ISAMServer>.Fail(result.SError);
+                return LdapResult<ISAMServer>.Fail(result.SError);
             }
 
-            return Result<ISAMServer>.Ok(result.Value);
+            return LdapResult<ISAMServer>.Ok(result.Value);
         }
 
         public IAsyncEnumerable<LocalGroupAPIResult> GetLocalGroups(ResolvedSearchResult result)

src/CommonLib/Processors/UserRightsAssignmentProcessor.cs

@@ -25,12 +25,12 @@ public UserRightsAssignmentProcessor(ILDAPUtils utils, ILogger log = null)
 
         public event ComputerStatusDelegate ComputerStatusEvent;
 
-        public virtual Result<ILSAPolicy> OpenLSAPolicy(string computerName)
+        public virtual LdapResult<ILSAPolicy> OpenLSAPolicy(string computerName)
         {
             var result = LSAPolicy.OpenPolicy(computerName);
-            if (result.IsFailed) return Result<ILSAPolicy>.Fail(result.SError);
+            if (result.IsFailed) return LdapResult<ILSAPolicy>.Fail(result.SError);
 
-            return Result<ILSAPolicy>.Ok(result.Value);
+            return LdapResult<ILSAPolicy>.Ok(result.Value);
         }
 
         public IAsyncEnumerable<UserRightsAssignmentAPIResult> GetUserRightsAssignments(ResolvedSearchResult result,