Merge remote-tracking branch 'origin/main' into fix_swallow_send_request

Tim-Brooks · May 15, 2024 · 09f4c7c · 09f4c7c
2 parents f7e2ae8 + 639eee5
commit 09f4c7c
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 9 deletions.
diff --git a/docs/changelog/108687.yaml b/docs/changelog/108687.yaml
@@ -0,0 +1,5 @@
+pr: 108687
+summary: Adding `user_type` support for the enterprise database for the geoip processor
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
@@ -67,8 +67,8 @@ depend on what has been found and which properties were configured in `propertie
 * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
 `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, `residential_proxy`,
-`isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added depend on what has been found
-and which properties were configured in `properties`.
+`isp`, `isp_organization`, `mobile_country_code`, `mobile_network_code`, and `user_type`. The fields actually added depend on what has been
+found and which properties were configured in `properties`.
 
 
 Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:

diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java
@@ -100,7 +100,8 @@ enum Database {
             Property.ISP,
             Property.ISP_ORGANIZATION_NAME,
             Property.MOBILE_COUNTRY_CODE,
-            Property.MOBILE_NETWORK_CODE
+            Property.MOBILE_NETWORK_CODE,
+            Property.USER_TYPE
         ),
         Set.of(
             Property.COUNTRY_ISO_CODE,
@@ -248,7 +249,8 @@ enum Property {
         ISP,
         ISP_ORGANIZATION_NAME,
         MOBILE_COUNTRY_CODE,
-        MOBILE_NETWORK_CODE;
+        MOBILE_NETWORK_CODE,
+        USER_TYPE;
 
         /**
          * Parses a string representation of a property into an actual Property instance. Not all properties that exist are

diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -438,6 +438,8 @@ private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas
         boolean isPublicProxy = response.getTraits().isPublicProxy();
         boolean isResidentialProxy = response.getTraits().isResidentialProxy();
 
+        String userType = response.getTraits().getUserType();
+
         String domain = response.getTraits().getDomain();
 
         Map<String, Object> geoData = new HashMap<>();
@@ -558,6 +560,11 @@ private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas
                         geoData.put("mobile_network_code", mobileNetworkCode);
                     }
                 }
+                case USER_TYPE -> {
+                    if (userType != null) {
+                        geoData.put("user_type", userType);
+                    }
+                }
             }
         }
         return geoData;

diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -387,7 +387,7 @@ public void testEnterprise() throws Exception {
         assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData.size(), equalTo(21));
+        assertThat(geoData.size(), equalTo(22));
         assertThat(geoData.get("ip"), equalTo(ip));
         assertThat(geoData.get("country_iso_code"), equalTo("US"));
         assertThat(geoData.get("country_name"), equalTo("United States"));
@@ -412,6 +412,7 @@ public void testEnterprise() throws Exception {
         assertThat(geoData.get("domain"), equalTo("frpt.net"));
         assertThat(geoData.get("isp"), equalTo("Fairpoint Communications"));
         assertThat(geoData.get("isp_organization"), equalTo("Fairpoint Communications"));
+        assertThat(geoData.get("user_type"), equalTo("residential"));
     }
 
     public void testIsp() throws Exception {

diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java
@@ -227,7 +227,8 @@ public class MaxMindSupportTests extends ESTestCase {
         "traits.organization",
         "traits.publicProxy",
         "traits.residentialProxy",
-        "traits.torExitNode"
+        "traits.torExitNode",
+        "traits.userType"
     );
     private static final Set<String> ENTERPRISE_UNSUPPORTED_FIELDS = Set.of(
         "city.confidence",
@@ -280,8 +281,7 @@ public class MaxMindSupportTests extends ESTestCase {
         "traits.legitimateProxy",
         "traits.satelliteProvider",
         "traits.staticIpScore",
-        "traits.userCount",
-        "traits.userType"
+        "traits.userCount"
     );
 
     private static final Set<String> ISP_SUPPORTED_FIELDS = Set.of(

diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java
@@ -47,7 +47,10 @@
  * threads that wait on a phaser. This lets us verify that operations on system indices
  * are being directed to other thread pools.</p>
  */
-@TestLogging(reason = "investigate", value = "org.elasticsearch.kibana.KibanaThreadPoolIT:DEBUG")
+@TestLogging(
+    reason = "investigate",
+    value = "org.elasticsearch.kibana.KibanaThreadPoolIT:DEBUG,org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor:TRACE"
+)
 public class KibanaThreadPoolIT extends ESIntegTestCase {
     private static final Logger logger = LogManager.getLogger(KibanaThreadPoolIT.class);
 

diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
@@ -137,6 +137,12 @@ public final String toString() {
         return b.toString();
     }
 
+    @Override
+    public boolean remove(Runnable task) {
+        logger.trace(() -> "task is removed " + task);
+        return super.remove(task);
+    }
+
     /**
      * Append details about this thread pool to the specified {@link StringBuilder}. All details should be appended as key/value pairs in
      * the form "%s = %s, "