Add support for the 'Domain' database to the geoip processor (#108639)

docs/changelog/108639.yaml

@@ -0,0 +1,28 @@
+pr: 108639
+summary: Add support for the 'Domain' database to the geoip processor
+area: Ingest Node
+type: enhancement
+issues: []
+highlight:
+  title: Add support for the 'Domain' database to the geoip processor
+  body: |-
+    Follow on to #107287 and #107377
+
+    Adds support for the ['GeoIP2
+    Domain'](https://dev.maxmind.com/geoip/docs/databases/domain) database
+    from MaxMind to the `geoip` processor.
+
+    The `geoip` processor will automatically download the [various
+    'GeoLite2'
+    databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data),
+    but the 'GeoIP2 Domain' database is not a 'GeoLite2' database -- it's a
+    commercial database available to those with a suitable license from
+    MaxMind.
+
+    The support that is being added for it in this PR is in line with the
+    support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2
+    Country' databases -- that is, one would need to arrange their own
+    download management via some custom endpoint or otherwise arrange for
+    the relevant file(s) to be in the `$ES_CONFIG/ingest-geoip` directory on
+    the nodes of the cluster.
+  notable: true

docs/reference/ingest/processors/geoip.asciidoc

@@ -59,6 +59,7 @@ in `properties`.
 * If the GeoIP2 Anonymous IP database is used, then the following fields may be added under the `target_field`: `ip`,
 `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added
 depend on what has been found and which properties were configured in `properties`.
+* If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`.
 * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
 `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`.

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java

@@ -75,6 +75,7 @@ enum Database {
             Property.RESIDENTIAL_PROXY
         )
     ),
+    Domain(Set.of(Property.IP, Property.DOMAIN), Set.of(Property.DOMAIN)),
     Enterprise(
         Set.of(
             Property.IP,
@@ -94,7 +95,8 @@ enum Database {
             Property.ANONYMOUS_VPN,
             Property.ANONYMOUS,
             Property.PUBLIC_PROXY,
-            Property.RESIDENTIAL_PROXY
+            Property.RESIDENTIAL_PROXY,
+            Property.DOMAIN
         ),
         Set.of(
             Property.COUNTRY_ISO_CODE,
@@ -111,6 +113,7 @@ enum Database {
     private static final String COUNTRY_DB_SUFFIX = "-Country";
     private static final String ASN_DB_SUFFIX = "-ASN";
     private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
+    private static final String DOMAIN_DB_SUFFIX = "-Domain";
     private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
 
     /**
@@ -133,6 +136,8 @@ enum Database {
                 database = Database.Asn;
             } else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) {
                 database = Database.AnonymousIp;
+            } else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) {
+                database = Database.Domain;
             } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) {
                 database = Database.Enterprise;
             }
@@ -209,7 +214,8 @@ enum Database {
         ANONYMOUS_VPN,
         ANONYMOUS,
         PUBLIC_PROXY,
-        RESIDENTIAL_PROXY;
+        RESIDENTIAL_PROXY,
+        DOMAIN;
 
         /**
          * Parses a string representation of a property into an actual Property instance. Not all properties that exist are

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java

@@ -16,6 +16,7 @@ import com.maxmind.geoip2.model.AnonymousIpResponse;
 import com.maxmind.geoip2.model.AsnResponse;
 import com.maxmind.geoip2.model.CityResponse;
 import com.maxmind.geoip2.model.CountryResponse;
+import com.maxmind.geoip2.model.DomainResponse;
 import com.maxmind.geoip2.model.EnterpriseResponse;
 
 import org.apache.logging.log4j.LogManager;
@@ -177,6 +178,12 @@ class DatabaseReaderLazyLoader implements GeoIpDatabase, Closeable {
         return getResponse(ipAddress, DatabaseReader::tryAnonymousIp);
     }
 
+    @Nullable
+    @Override
+    public DomainResponse getDomain(InetAddress ipAddress) {
+        return getResponse(ipAddress, DatabaseReader::tryDomain);
+    }
+
     @Nullable
     @Override
     public EnterpriseResponse getEnterprise(InetAddress ipAddress) {

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java

@@ -12,6 +12,7 @@ import com.maxmind.geoip2.model.AnonymousIpResponse;
 import com.maxmind.geoip2.model.AsnResponse;
 import com.maxmind.geoip2.model.CityResponse;
 import com.maxmind.geoip2.model.CountryResponse;
+import com.maxmind.geoip2.model.DomainResponse;
 import com.maxmind.geoip2.model.EnterpriseResponse;
 
 import org.elasticsearch.core.Nullable;
@@ -58,6 +59,9 @@ public interface GeoIpDatabase {
     @Nullable
     AnonymousIpResponse getAnonymousIp(InetAddress ipAddress);
 
+    @Nullable
+    DomainResponse getDomain(InetAddress ipAddress);
+
     @Nullable
     EnterpriseResponse getEnterprise(InetAddress ipAddress);
 

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java

@@ -13,6 +13,7 @@ import com.maxmind.geoip2.model.AnonymousIpResponse;
 import com.maxmind.geoip2.model.AsnResponse;
 import com.maxmind.geoip2.model.CityResponse;
 import com.maxmind.geoip2.model.CountryResponse;
+import com.maxmind.geoip2.model.DomainResponse;
 import com.maxmind.geoip2.model.EnterpriseResponse;
 import com.maxmind.geoip2.record.City;
 import com.maxmind.geoip2.record.Continent;
@@ -175,6 +176,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
             case Country -> retrieveCountryGeoData(geoIpDatabase, ipAddress);
             case Asn -> retrieveAsnGeoData(geoIpDatabase, ipAddress);
             case AnonymousIp -> retrieveAnonymousIpGeoData(geoIpDatabase, ipAddress);
+            case Domain -> retrieveDomainGeoData(geoIpDatabase, ipAddress);
             case Enterprise -> retrieveEnterpriseGeoData(geoIpDatabase, ipAddress);
         };
     }
@@ -384,6 +386,28 @@ public final class GeoIpProcessor extends AbstractProcessor {
         return geoData;
     }
 
+    private Map<String, Object> retrieveDomainGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
+        DomainResponse response = geoIpDatabase.getDomain(ipAddress);
+        if (response == null) {
+            return Map.of();
+        }
+
+        String domain = response.getDomain();
+
+        Map<String, Object> geoData = new HashMap<>();
+        for (Property property : this.properties) {
+            switch (property) {
+                case IP -> geoData.put("ip", NetworkAddress.format(ipAddress));
+                case DOMAIN -> {
+                    if (domain != null) {
+                        geoData.put("domain", domain);
+                    }
+                }
+            }
+        }
+        return geoData;
+    }
+
     private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
         EnterpriseResponse response = geoIpDatabase.getEnterprise(ipAddress);
         if (response == null) {
@@ -407,6 +431,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
         boolean isPublicProxy = response.getTraits().isPublicProxy();
         boolean isResidentialProxy = response.getTraits().isResidentialProxy();
 
+        String domain = response.getTraits().getDomain();
+
         Map<String, Object> geoData = new HashMap<>();
         for (Property property : this.properties) {
             switch (property) {
@@ -500,6 +526,11 @@ public final class GeoIpProcessor extends AbstractProcessor {
                 case RESIDENTIAL_PROXY -> {
                     geoData.put("residential_proxy", isResidentialProxy);
                 }
+                case DOMAIN -> {
+                    if (domain != null) {
+                        geoData.put("domain", domain);
+                    }
+                }
             }
         }
         return geoData;

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java

@@ -336,8 +336,36 @@ public class GeoIpProcessorTests extends ESTestCase {
         assertThat(geoData.get("residential_proxy"), equalTo(true));
     }
 
+    public void testDomain() throws Exception {
+        String ip = "69.219.64.2";
+        GeoIpProcessor processor = new GeoIpProcessor(
+            randomAlphaOfLength(10),
+            null,
+            "source_field",
+            loader("/GeoIP2-Domain-Test.mmdb"),
+            () -> true,
+            "target_field",
+            ALL_PROPERTIES,
+            false,
+            false,
+            "filename"
+        );
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field", ip);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
+        @SuppressWarnings("unchecked")
+        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(geoData.size(), equalTo(2));
+        assertThat(geoData.get("ip"), equalTo(ip));
+        assertThat(geoData.get("domain"), equalTo("ameritech.net"));
+    }
+
     public void testEnterprise() throws Exception {
-        String ip = "2.125.160.216";
+        String ip = "74.209.24.4";
         GeoIpProcessor processor = new GeoIpProcessor(
             randomAlphaOfLength(10),
             null,
@@ -359,26 +387,29 @@ public class GeoIpProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData.size(), equalTo(16));
+        assertThat(geoData.size(), equalTo(19));
         assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("country_iso_code"), equalTo("GB"));
+        assertThat(geoData.get("country_iso_code"), equalTo("US"));
-        assertThat(geoData.get("country_name"), equalTo("United Kingdom"));
+        assertThat(geoData.get("country_name"), equalTo("United States"));
-        assertThat(geoData.get("continent_name"), equalTo("Europe"));
+        assertThat(geoData.get("continent_name"), equalTo("North America"));
-        assertThat(geoData.get("region_iso_code"), equalTo("GB-WBK"));
+        assertThat(geoData.get("region_iso_code"), equalTo("US-NY"));
-        assertThat(geoData.get("region_name"), equalTo("West Berkshire"));
+        assertThat(geoData.get("region_name"), equalTo("New York"));
-        assertThat(geoData.get("city_name"), equalTo("Boxford"));
+        assertThat(geoData.get("city_name"), equalTo("Chatham"));
-        assertThat(geoData.get("timezone"), equalTo("Europe/London"));
+        assertThat(geoData.get("timezone"), equalTo("America/New_York"));
         Map<String, Object> location = new HashMap<>();
-        location.put("lat", 51.75);
+        location.put("lat", 42.3478);
-        location.put("lon", -1.25);
+        location.put("lon", -73.5549);
         assertThat(geoData.get("location"), equalTo(location));
-        assertThat(geoData.get("network"), equalTo("2.125.160.216/29"));
+        assertThat(geoData.get("asn"), equalTo(14671L));
+        assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications"));
+        assertThat(geoData.get("network"), equalTo("74.209.16.0/20"));
         assertThat(geoData.get("hosting_provider"), equalTo(false));
         assertThat(geoData.get("tor_exit_node"), equalTo(false));
         assertThat(geoData.get("anonymous_vpn"), equalTo(false));
         assertThat(geoData.get("anonymous"), equalTo(false));
         assertThat(geoData.get("public_proxy"), equalTo(false));
         assertThat(geoData.get("residential_proxy"), equalTo(false));
+        assertThat(geoData.get("domain"), equalTo("frpt.net"));
     }
 
     public void testAddressIsNotInTheDatabase() throws Exception {

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java

@@ -201,6 +201,9 @@ public class MaxMindSupportTests extends ESTestCase {
         "traits.userType"
     );
 
+    private static final Set<String> DOMAIN_SUPPORTED_FIELDS = Set.of("domain");
+    private static final Set<String> DOMAIN_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network");
+
     private static final Set<String> ENTERPRISE_SUPPORTED_FIELDS = Set.of(
         "city.name",
         "continent.name",
@@ -215,6 +218,7 @@ public class MaxMindSupportTests extends ESTestCase {
         "traits.anonymousVpn",
         "traits.autonomousSystemNumber",
         "traits.autonomousSystemOrganization",
+        "traits.domain",
         "traits.hostingProvider",
         "traits.network",
         "traits.publicProxy",
@@ -268,7 +272,6 @@ public class MaxMindSupportTests extends ESTestCase {
         "traits.anonymousProxy",
         "traits.anycast",
         "traits.connectionType",
-        "traits.domain",
         "traits.ipAddress",
         "traits.isp",
         "traits.legitimateProxy",
@@ -290,6 +293,8 @@ public class MaxMindSupportTests extends ESTestCase {
         CITY_SUPPORTED_FIELDS,
         Database.Country,
         COUNTRY_SUPPORTED_FIELDS,
+        Database.Domain,
+        DOMAIN_SUPPORTED_FIELDS,
         Database.Enterprise,
         ENTERPRISE_SUPPORTED_FIELDS
     );
@@ -302,6 +307,8 @@ public class MaxMindSupportTests extends ESTestCase {
         CITY_UNSUPPORTED_FIELDS,
         Database.Country,
         COUNTRY_UNSUPPORTED_FIELDS,
+        Database.Domain,
+        DOMAIN_UNSUPPORTED_FIELDS,
         Database.Enterprise,
         ENTERPRISE_UNSUPPORTED_FIELDS
     );
@@ -314,13 +321,14 @@ public class MaxMindSupportTests extends ESTestCase {
         CityResponse.class,
         Database.Country,
         CountryResponse.class,
+        Database.Domain,
+        DomainResponse.class,
         Database.Enterprise,
         EnterpriseResponse.class
     );
 
     private static final Set<Class<? extends AbstractResponse>> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(
         ConnectionTypeResponse.class,
-        DomainResponse.class,
         IspResponse.class,
         IpRiskResponse.class
     );