github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-24 23:27:25 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add support for the 'Domain' database to the geoip processor (#108639)

Browse source
This commit is contained in:
Joe Gallo 2024-05-14 17:49:05 -04:00 • committed by GitHub
parent 229100851d
commit cc6597df23
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 132 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

28
docs/changelog/108639.yaml Normal file
View file

@ -0,0 +1,28 @@
pr: 108639
summary: Add support for the 'Domain' database to the geoip processor
area: Ingest Node
type: enhancement
issues: []
highlight:
title: Add support for the 'Domain' database to the geoip processor
body: |-
Follow on to #107287 and #107377
Adds support for the ['GeoIP2
Domain'](https://dev.maxmind.com/geoip/docs/databases/domain) database
from MaxMind to the `geoip` processor.
The `geoip` processor will automatically download the [various
'GeoLite2'
databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data),
but the 'GeoIP2 Domain' database is not a 'GeoLite2' database -- it's a
commercial database available to those with a suitable license from
MaxMind.
The support that is being added for it in this PR is in line with the
support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2
Country' databases -- that is, one would need to arrange their own
download management via some custom endpoint or otherwise arrange for
the relevant file(s) to be in the `$ES_CONFIG/ingest-geoip` directory on
the nodes of the cluster.
notable: true

1
docs/reference/ingest/processors/geoip.asciidoc
View file

@ -59,6 +59,7 @@ in `properties`.
* If the GeoIP2 Anonymous IP database is used, then the following fields may be added under the `target_field`: `ip`,
`hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added
depend on what has been found and which properties were configured in `properties`.
* If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`.
* If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`.

10
modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java
View file

@ -75,6 +75,7 @@ enum Database {
Property.RESIDENTIAL_PROXY
)
),
Domain(Set.of(Property.IP, Property.DOMAIN), Set.of(Property.DOMAIN)),
Enterprise(
Set.of(
Property.IP,
@ -94,7 +95,8 @@ enum Database {
Property.ANONYMOUS_VPN,
Property.ANONYMOUS,
Property.PUBLIC_PROXY,
Property.RESIDENTIAL_PROXY
Property.RESIDENTIAL_PROXY,
Property.DOMAIN
),
Set.of(
Property.COUNTRY_ISO_CODE,
@ -111,6 +113,7 @@ enum Database {
private static final String COUNTRY_DB_SUFFIX = "-Country";
private static final String ASN_DB_SUFFIX = "-ASN";
private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
private static final String DOMAIN_DB_SUFFIX = "-Domain";
private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
/**
@ -133,6 +136,8 @@ enum Database {
database = Database.Asn;
} else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) {
database = Database.AnonymousIp;
} else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) {
database = Database.Domain;
} else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) {
database = Database.Enterprise;
}
@ -209,7 +214,8 @@ enum Database {
ANONYMOUS_VPN,
ANONYMOUS,
PUBLIC_PROXY,
RESIDENTIAL_PROXY;
RESIDENTIAL_PROXY,
DOMAIN;
/**
* Parses a string representation of a property into an actual Property instance. Not all properties that exist are

7
modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
View file

@ -16,6 +16,7 @@ import com.maxmind.geoip2.model.AnonymousIpResponse;
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.model.DomainResponse;
import com.maxmind.geoip2.model.EnterpriseResponse;
import org.apache.logging.log4j.LogManager;
@ -177,6 +178,12 @@ class DatabaseReaderLazyLoader implements GeoIpDatabase, Closeable {
return getResponse(ipAddress, DatabaseReader::tryAnonymousIp);
}
@Nullable
@Override
public DomainResponse getDomain(InetAddress ipAddress) {
return getResponse(ipAddress, DatabaseReader::tryDomain);
}
@Nullable
@Override
public EnterpriseResponse getEnterprise(InetAddress ipAddress) {

4
modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java
View file

@ -12,6 +12,7 @@ import com.maxmind.geoip2.model.AnonymousIpResponse;
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.model.DomainResponse;
import com.maxmind.geoip2.model.EnterpriseResponse;
import org.elasticsearch.core.Nullable;
@ -58,6 +59,9 @@ public interface GeoIpDatabase {
@Nullable
AnonymousIpResponse getAnonymousIp(InetAddress ipAddress);
@Nullable
DomainResponse getDomain(InetAddress ipAddress);
@Nullable
EnterpriseResponse getEnterprise(InetAddress ipAddress);

31
modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
View file

@ -13,6 +13,7 @@ import com.maxmind.geoip2.model.AnonymousIpResponse;
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.model.DomainResponse;
import com.maxmind.geoip2.model.EnterpriseResponse;
import com.maxmind.geoip2.record.City;
import com.maxmind.geoip2.record.Continent;
@ -175,6 +176,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
case Country -> retrieveCountryGeoData(geoIpDatabase, ipAddress);
case Asn -> retrieveAsnGeoData(geoIpDatabase, ipAddress);
case AnonymousIp -> retrieveAnonymousIpGeoData(geoIpDatabase, ipAddress);
case Domain -> retrieveDomainGeoData(geoIpDatabase, ipAddress);
case Enterprise -> retrieveEnterpriseGeoData(geoIpDatabase, ipAddress);
};
}
@ -384,6 +386,28 @@ public final class GeoIpProcessor extends AbstractProcessor {
return geoData;
}
private Map<String, Object> retrieveDomainGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
DomainResponse response = geoIpDatabase.getDomain(ipAddress);
if (response == null) {
return Map.of();
}
String domain = response.getDomain();
Map<String, Object> geoData = new HashMap<>();
for (Property property : this.properties) {
switch (property) {
case IP -> geoData.put("ip", NetworkAddress.format(ipAddress));
case DOMAIN -> {
if (domain != null) {
geoData.put("domain", domain);
}
}
}
}
return geoData;
}
private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
EnterpriseResponse response = geoIpDatabase.getEnterprise(ipAddress);
if (response == null) {
@ -407,6 +431,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
boolean isPublicProxy = response.getTraits().isPublicProxy();
boolean isResidentialProxy = response.getTraits().isResidentialProxy();
String domain = response.getTraits().getDomain();
Map<String, Object> geoData = new HashMap<>();
for (Property property : this.properties) {
switch (property) {
@ -500,6 +526,11 @@ public final class GeoIpProcessor extends AbstractProcessor {
case RESIDENTIAL_PROXY -> {
geoData.put("residential_proxy", isResidentialProxy);
}
case DOMAIN -> {
if (domain != null) {
geoData.put("domain", domain);
}
}
}
}
return geoData;

55
modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
View file

@ -336,8 +336,36 @@ public class GeoIpProcessorTests extends ESTestCase {
assertThat(geoData.get("residential_proxy"), equalTo(true));
}
public void testDomain() throws Exception {
String ip = "69.219.64.2";
GeoIpProcessor processor = new GeoIpProcessor(
randomAlphaOfLength(10),
null,
"source_field",
loader("/GeoIP2-Domain-Test.mmdb"),
() -> true,
"target_field",
ALL_PROPERTIES,
false,
false,
"filename"
);
Map<String, Object> document = new HashMap<>();
document.put("source_field", ip);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
@SuppressWarnings("unchecked")
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(2));
assertThat(geoData.get("ip"), equalTo(ip));
assertThat(geoData.get("domain"), equalTo("ameritech.net"));
}
public void testEnterprise() throws Exception {
String ip = "2.125.160.216";
String ip = "74.209.24.4";
GeoIpProcessor processor = new GeoIpProcessor(
randomAlphaOfLength(10),
null,
@ -359,26 +387,29 @@ public class GeoIpProcessorTests extends ESTestCase {
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
@SuppressWarnings("unchecked")
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(16));
assertThat(geoData.size(), equalTo(19));
assertThat(geoData.get("ip"), equalTo(ip));
assertThat(geoData.get("country_iso_code"), equalTo("GB"));
assertThat(geoData.get("country_name"), equalTo("United Kingdom"));
assertThat(geoData.get("continent_name"), equalTo("Europe"));
assertThat(geoData.get("region_iso_code"), equalTo("GB-WBK"));
assertThat(geoData.get("region_name"), equalTo("West Berkshire"));
assertThat(geoData.get("city_name"), equalTo("Boxford"));
assertThat(geoData.get("timezone"), equalTo("Europe/London"));
assertThat(geoData.get("country_iso_code"), equalTo("US"));
assertThat(geoData.get("country_name"), equalTo("United States"));
assertThat(geoData.get("continent_name"), equalTo("North America"));
assertThat(geoData.get("region_iso_code"), equalTo("US-NY"));
assertThat(geoData.get("region_name"), equalTo("New York"));
assertThat(geoData.get("city_name"), equalTo("Chatham"));
assertThat(geoData.get("timezone"), equalTo("America/New_York"));
Map<String, Object> location = new HashMap<>();
location.put("lat", 51.75);
location.put("lon", -1.25);
location.put("lat", 42.3478);
location.put("lon", -73.5549);
assertThat(geoData.get("location"), equalTo(location));
assertThat(geoData.get("network"), equalTo("2.125.160.216/29"));
assertThat(geoData.get("asn"), equalTo(14671L));
assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications"));
assertThat(geoData.get("network"), equalTo("74.209.16.0/20"));
assertThat(geoData.get("hosting_provider"), equalTo(false));
assertThat(geoData.get("tor_exit_node"), equalTo(false));
assertThat(geoData.get("anonymous_vpn"), equalTo(false));
assertThat(geoData.get("anonymous"), equalTo(false));
assertThat(geoData.get("public_proxy"), equalTo(false));
assertThat(geoData.get("residential_proxy"), equalTo(false));
assertThat(geoData.get("domain"), equalTo("frpt.net"));
}
public void testAddressIsNotInTheDatabase() throws Exception {

12
modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java
View file

@ -201,6 +201,9 @@ public class MaxMindSupportTests extends ESTestCase {
"traits.userType"
);
private static final Set<String> DOMAIN_SUPPORTED_FIELDS = Set.of("domain");
private static final Set<String> DOMAIN_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network");
private static final Set<String> ENTERPRISE_SUPPORTED_FIELDS = Set.of(
"city.name",
"continent.name",
@ -215,6 +218,7 @@ public class MaxMindSupportTests extends ESTestCase {
"traits.anonymousVpn",
"traits.autonomousSystemNumber",
"traits.autonomousSystemOrganization",
"traits.domain",
"traits.hostingProvider",
"traits.network",
"traits.publicProxy",
@ -268,7 +272,6 @@ public class MaxMindSupportTests extends ESTestCase {
"traits.anonymousProxy",
"traits.anycast",
"traits.connectionType",
"traits.domain",
"traits.ipAddress",
"traits.isp",
"traits.legitimateProxy",
@ -290,6 +293,8 @@ public class MaxMindSupportTests extends ESTestCase {
CITY_SUPPORTED_FIELDS,
Database.Country,
COUNTRY_SUPPORTED_FIELDS,
Database.Domain,
DOMAIN_SUPPORTED_FIELDS,
Database.Enterprise,
ENTERPRISE_SUPPORTED_FIELDS
);
@ -302,6 +307,8 @@ public class MaxMindSupportTests extends ESTestCase {
CITY_UNSUPPORTED_FIELDS,
Database.Country,
COUNTRY_UNSUPPORTED_FIELDS,
Database.Domain,
DOMAIN_UNSUPPORTED_FIELDS,
Database.Enterprise,
ENTERPRISE_UNSUPPORTED_FIELDS
);
@ -314,13 +321,14 @@ public class MaxMindSupportTests extends ESTestCase {
CityResponse.class,
Database.Country,
CountryResponse.class,
Database.Domain,
DomainResponse.class,
Database.Enterprise,
EnterpriseResponse.class
);
private static final Set<Class<? extends AbstractResponse>> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(
ConnectionTypeResponse.class,
DomainResponse.class,
IspResponse.class,
IpRiskResponse.class
);

BIN
modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 6.3 KiB