From 7f1976dd2ed1d7fe01b7cc051897a92b7c1af7a8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Wed, 15 May 2024 19:17:00 +0200 Subject: [PATCH 1/2] Trace logging for ESThreadPoolExecutor#remove (#108688) this is just to find out if we ever remove the tasks from the threadPoolExecutor in order to make the Kibana(System) ThreadPool tests reliable --- .../java/org/elasticsearch/kibana/KibanaThreadPoolIT.java | 5 ++++- .../common/util/concurrent/EsThreadPoolExecutor.java | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java index 98eb69aa9e21e..b48b2941e6097 100644 --- a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java +++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java @@ -47,7 +47,10 @@ * threads that wait on a phaser. This lets us verify that operations on system indices * are being directed to other thread pools.

*/ -@TestLogging(reason = "investigate", value = "org.elasticsearch.kibana.KibanaThreadPoolIT:DEBUG") +@TestLogging( + reason = "investigate", + value = "org.elasticsearch.kibana.KibanaThreadPoolIT:DEBUG,org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor:TRACE" +) public class KibanaThreadPoolIT extends ESIntegTestCase { private static final Logger logger = LogManager.getLogger(KibanaThreadPoolIT.class); diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java index 6b49fd80e8665..39297146825a1 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java @@ -137,6 +137,12 @@ public final String toString() { return b.toString(); } + @Override + public boolean remove(Runnable task) { + logger.trace(() -> "task is removed " + task); + return super.remove(task); + } + /** * Append details about this thread pool to the specified {@link StringBuilder}. All details should be appended as key/value pairs in * the form "%s = %s, " From 639eee577e6cc3092ffafbaa6e13e8b77e119862 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 15 May 2024 12:23:52 -0500 Subject: [PATCH 2/2] Adding user_type support for the enterprise database for the geoip processor (#108687) --- docs/changelog/108687.yaml | 5 +++++ docs/reference/ingest/processors/geoip.asciidoc | 4 ++-- .../main/java/org/elasticsearch/ingest/geoip/Database.java | 6 ++++-- .../org/elasticsearch/ingest/geoip/GeoIpProcessor.java | 7 +++++++ .../elasticsearch/ingest/geoip/GeoIpProcessorTests.java | 3 ++- .../elasticsearch/ingest/geoip/MaxMindSupportTests.java | 6 +++--- 6 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/108687.yaml diff --git a/docs/changelog/108687.yaml b/docs/changelog/108687.yaml new file mode 100644 index 0000000000000..771516d551567 --- /dev/null +++ b/docs/changelog/108687.yaml @@ -0,0 +1,5 @@ +pr: 108687 +summary: Adding `user_type` support for the enterprise database for the geoip processor +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 4fbf9678f2fc7..b18205ddbad95 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -67,8 +67,8 @@ depend on what has been found and which properties were configured in `propertie * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`, `country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`, `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, `residential_proxy`, -`isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added depend on what has been found -and which properties were configured in `properties`. +`isp`, `isp_organization`, `mobile_country_code`, `mobile_network_code`, and `user_type`. The fields actually added depend on what has been +found and which properties were configured in `properties`. Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field: diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index 641d8bad2b135..429a04a225aa4 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -100,7 +100,8 @@ enum Database { Property.ISP, Property.ISP_ORGANIZATION_NAME, Property.MOBILE_COUNTRY_CODE, - Property.MOBILE_NETWORK_CODE + Property.MOBILE_NETWORK_CODE, + Property.USER_TYPE ), Set.of( Property.COUNTRY_ISO_CODE, @@ -248,7 +249,8 @@ enum Property { ISP, ISP_ORGANIZATION_NAME, MOBILE_COUNTRY_CODE, - MOBILE_NETWORK_CODE; + MOBILE_NETWORK_CODE, + USER_TYPE; /** * Parses a string representation of a property into an actual Property instance. Not all properties that exist are diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 6d420b0547293..5e1d3dd42a49c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -438,6 +438,8 @@ private Map retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas boolean isPublicProxy = response.getTraits().isPublicProxy(); boolean isResidentialProxy = response.getTraits().isResidentialProxy(); + String userType = response.getTraits().getUserType(); + String domain = response.getTraits().getDomain(); Map geoData = new HashMap<>(); @@ -558,6 +560,11 @@ private Map retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas geoData.put("mobile_network_code", mobileNetworkCode); } } + case USER_TYPE -> { + if (userType != null) { + geoData.put("user_type", userType); + } + } } } return geoData; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 3f1216c515f5d..dd6f2fe8e8d5b 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -387,7 +387,7 @@ public void testEnterprise() throws Exception { assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(21)); + assertThat(geoData.size(), equalTo(22)); assertThat(geoData.get("ip"), equalTo(ip)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); @@ -412,6 +412,7 @@ public void testEnterprise() throws Exception { assertThat(geoData.get("domain"), equalTo("frpt.net")); assertThat(geoData.get("isp"), equalTo("Fairpoint Communications")); assertThat(geoData.get("isp_organization"), equalTo("Fairpoint Communications")); + assertThat(geoData.get("user_type"), equalTo("residential")); } public void testIsp() throws Exception { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index a396995663da7..0db8e30de3dd4 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -227,7 +227,8 @@ public class MaxMindSupportTests extends ESTestCase { "traits.organization", "traits.publicProxy", "traits.residentialProxy", - "traits.torExitNode" + "traits.torExitNode", + "traits.userType" ); private static final Set ENTERPRISE_UNSUPPORTED_FIELDS = Set.of( "city.confidence", @@ -280,8 +281,7 @@ public class MaxMindSupportTests extends ESTestCase { "traits.legitimateProxy", "traits.satelliteProvider", "traits.staticIpScore", - "traits.userCount", - "traits.userType" + "traits.userCount" ); private static final Set ISP_SUPPORTED_FIELDS = Set.of(