mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-24 23:27:25 -04:00
Add continent_code support to the geoip processor (#108780)
This commit is contained in:
Joe Gallo
GitHub
parent
0f147d4aff
commit
e1b2b599de
7 changed files with 53 additions and 18 deletions
6
docs/changelog/108780.yaml
Normal file
6
docs/changelog/108780.yaml
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
pr: 108780
|
||||
summary: Add `continent_code` support to the geoip processor
|
||||
area: Ingest Node
|
||||
type: enhancement
|
||||
issues:
|
||||
- 85820
|
||||
|
|
@ -48,11 +48,11 @@ field instead.
|
|||
*Depends on what is available in `database_file`:
|
||||
|
||||
* If a GeoLite2 City or GeoIP2 City database is used, then the following fields may be added under the `target_field`: `ip`,
|
||||
`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`,
|
||||
`country_iso_code`, `country_name`, `continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`,
|
||||
and `location`. The fields actually added depend on what has been found and which properties were configured in `properties`.
|
||||
* If a GeoLite2 Country or GeoIP2 Country database is used, then the following fields may be added under the `target_field`: `ip`,
|
||||
`country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties
|
||||
were configured in `properties`.
|
||||
`country_iso_code`, `country_name`, `continent_code`, and `continent_name`. The fields actually added depend on what has been found
|
||||
and which properties were configured in `properties`.
|
||||
* If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`,
|
||||
`asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured
|
||||
in `properties`.
|
||||
|
|
@ -67,10 +67,10 @@ The fields actually added depend on what has been found and which properties wer
|
|||
`organization_name`, `network`, `isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added
|
||||
depend on what has been found and which properties were configured in `properties`.
|
||||
* If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
|
||||
`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
|
||||
`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, `residential_proxy`,
|
||||
`domain`, `isp`, `isp_organization`, `mobile_country_code`, `mobile_network_code`, `user_type`, and `connection_type`. The fields
|
||||
actually added depend on what has been found and which properties were configured in `properties`.
|
||||
`country_iso_code`, `country_name`, `continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`,
|
||||
`location`, `asn`, `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`,
|
||||
`residential_proxy`, `domain`, `isp`, `isp_organization`, `mobile_country_code`, `mobile_network_code`, `user_type`, and
|
||||
`connection_type`. The fields actually added depend on what has been found and which properties were configured in `properties`.
|
||||
|
||||
|
||||
Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ enum Database {
|
|||
Set.of(
|
||||
Property.IP,
|
||||
Property.COUNTRY_ISO_CODE,
|
||||
Property.CONTINENT_CODE,
|
||||
Property.COUNTRY_NAME,
|
||||
Property.CONTINENT_NAME,
|
||||
Property.REGION_ISO_CODE,
|
||||
|
|
@ -49,7 +50,7 @@ enum Database {
|
|||
)
|
||||
),
|
||||
Country(
|
||||
Set.of(Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE),
|
||||
Set.of(Property.IP, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE),
|
||||
Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE)
|
||||
),
|
||||
Asn(
|
||||
|
|
@ -82,6 +83,7 @@ enum Database {
|
|||
Property.IP,
|
||||
Property.COUNTRY_ISO_CODE,
|
||||
Property.COUNTRY_NAME,
|
||||
Property.CONTINENT_CODE,
|
||||
Property.CONTINENT_NAME,
|
||||
Property.REGION_ISO_CODE,
|
||||
Property.REGION_NAME,
|
||||
|
|
@ -235,6 +237,7 @@ enum Database {
|
|||
IP,
|
||||
COUNTRY_ISO_CODE,
|
||||
COUNTRY_NAME,
|
||||
CONTINENT_CODE,
|
||||
CONTINENT_NAME,
|
||||
REGION_ISO_CODE,
|
||||
REGION_NAME,
|
||||
|
|
|
|||
|
|
@ -234,6 +234,12 @@ public final class GeoIpProcessor extends AbstractProcessor {
|
|||
geoData.put("country_name", countryName);
|
||||
}
|
||||
}
|
||||
case CONTINENT_CODE -> {
|
||||
String continentCode = continent.getCode();
|
||||
if (continentCode != null) {
|
||||
geoData.put("continent_code", continentCode);
|
||||
}
|
||||
}
|
||||
case CONTINENT_NAME -> {
|
||||
String continentName = continent.getName();
|
||||
if (continentName != null) {
|
||||
|
|
@ -307,6 +313,12 @@ public final class GeoIpProcessor extends AbstractProcessor {
|
|||
geoData.put("country_name", countryName);
|
||||
}
|
||||
}
|
||||
case CONTINENT_CODE -> {
|
||||
String continentCode = continent.getCode();
|
||||
if (continentCode != null) {
|
||||
geoData.put("continent_code", continentCode);
|
||||
}
|
||||
}
|
||||
case CONTINENT_NAME -> {
|
||||
String continentName = continent.getName();
|
||||
if (continentName != null) {
|
||||
|
|
@ -485,6 +497,12 @@ public final class GeoIpProcessor extends AbstractProcessor {
|
|||
geoData.put("country_name", countryName);
|
||||
}
|
||||
}
|
||||
case CONTINENT_CODE -> {
|
||||
String continentCode = continent.getCode();
|
||||
if (continentCode != null) {
|
||||
geoData.put("continent_code", continentCode);
|
||||
}
|
||||
}
|
||||
case CONTINENT_NAME -> {
|
||||
String continentName = continent.getName();
|
||||
if (continentName != null) {
|
||||
|
|
|
|||
|
|
@ -196,7 +196,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
equalTo(
|
||||
"[properties] illegal property value ["
|
||||
+ asnProperty
|
||||
+ "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME]"
|
||||
+ "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
|
@ -278,7 +278,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
e.getMessage(),
|
||||
equalTo(
|
||||
"[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, "
|
||||
+ "COUNTRY_NAME, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"
|
||||
+ "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"
|
||||
)
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -80,10 +80,11 @@ public class GeoIpProcessorTests extends ESTestCase {
|
|||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("8.8.8.8"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
||||
assertThat(geoData.size(), equalTo(6));
|
||||
assertThat(geoData.size(), equalTo(7));
|
||||
assertThat(geoData.get("ip"), equalTo("8.8.8.8"));
|
||||
assertThat(geoData.get("country_iso_code"), equalTo("US"));
|
||||
assertThat(geoData.get("country_name"), equalTo("United States"));
|
||||
assertThat(geoData.get("continent_code"), equalTo("NA"));
|
||||
assertThat(geoData.get("continent_name"), equalTo("North America"));
|
||||
assertThat(geoData.get("timezone"), equalTo("America/Chicago"));
|
||||
Map<String, Object> location = new HashMap<>();
|
||||
|
|
@ -197,10 +198,11 @@ public class GeoIpProcessorTests extends ESTestCase {
|
|||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(address));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
||||
assertThat(geoData.size(), equalTo(9));
|
||||
assertThat(geoData.size(), equalTo(10));
|
||||
assertThat(geoData.get("ip"), equalTo(address));
|
||||
assertThat(geoData.get("country_iso_code"), equalTo("US"));
|
||||
assertThat(geoData.get("country_name"), equalTo("United States"));
|
||||
assertThat(geoData.get("continent_code"), equalTo("NA"));
|
||||
assertThat(geoData.get("continent_name"), equalTo("North America"));
|
||||
assertThat(geoData.get("region_iso_code"), equalTo("US-FL"));
|
||||
assertThat(geoData.get("region_name"), equalTo("Florida"));
|
||||
|
|
@ -260,10 +262,11 @@ public class GeoIpProcessorTests extends ESTestCase {
|
|||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("82.170.213.79"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
||||
assertThat(geoData.size(), equalTo(4));
|
||||
assertThat(geoData.size(), equalTo(5));
|
||||
assertThat(geoData.get("ip"), equalTo("82.170.213.79"));
|
||||
assertThat(geoData.get("country_iso_code"), equalTo("NL"));
|
||||
assertThat(geoData.get("country_name"), equalTo("Netherlands"));
|
||||
assertThat(geoData.get("continent_code"), equalTo("EU"));
|
||||
assertThat(geoData.get("continent_name"), equalTo("Europe"));
|
||||
}
|
||||
|
||||
|
|
@ -435,10 +438,11 @@ public class GeoIpProcessorTests extends ESTestCase {
|
|||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
||||
assertThat(geoData.size(), equalTo(23));
|
||||
assertThat(geoData.size(), equalTo(24));
|
||||
assertThat(geoData.get("ip"), equalTo(ip));
|
||||
assertThat(geoData.get("country_iso_code"), equalTo("US"));
|
||||
assertThat(geoData.get("country_name"), equalTo("United States"));
|
||||
assertThat(geoData.get("continent_code"), equalTo("NA"));
|
||||
assertThat(geoData.get("continent_name"), equalTo("North America"));
|
||||
assertThat(geoData.get("region_iso_code"), equalTo("US-NY"));
|
||||
assertThat(geoData.get("region_name"), equalTo("New York"));
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ public class MaxMindSupportTests extends ESTestCase {
|
|||
|
||||
private static final Set<String> CITY_SUPPORTED_FIELDS = Set.of(
|
||||
"city.name",
|
||||
"continent.code",
|
||||
"continent.name",
|
||||
"country.isoCode",
|
||||
"country.name",
|
||||
|
|
@ -88,7 +89,6 @@ public class MaxMindSupportTests extends ESTestCase {
|
|||
"city.confidence",
|
||||
"city.geoNameId",
|
||||
"city.names",
|
||||
"continent.code",
|
||||
"continent.geoNameId",
|
||||
"continent.names",
|
||||
"country.confidence",
|
||||
|
|
@ -156,9 +156,13 @@ public class MaxMindSupportTests extends ESTestCase {
|
|||
private static final Set<String> CONNECT_TYPE_SUPPORTED_FIELDS = Set.of("connectionType");
|
||||
private static final Set<String> CONNECT_TYPE_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network");
|
||||
|
||||
private static final Set<String> COUNTRY_SUPPORTED_FIELDS = Set.of("continent.name", "country.isoCode", "country.name");
|
||||
private static final Set<String> COUNTRY_UNSUPPORTED_FIELDS = Set.of(
|
||||
private static final Set<String> COUNTRY_SUPPORTED_FIELDS = Set.of(
|
||||
"continent.name",
|
||||
"country.isoCode",
|
||||
"continent.code",
|
||||
"country.name"
|
||||
);
|
||||
private static final Set<String> COUNTRY_UNSUPPORTED_FIELDS = Set.of(
|
||||
"continent.geoNameId",
|
||||
"continent.names",
|
||||
"country.confidence",
|
||||
|
|
@ -209,6 +213,7 @@ public class MaxMindSupportTests extends ESTestCase {
|
|||
|
||||
private static final Set<String> ENTERPRISE_SUPPORTED_FIELDS = Set.of(
|
||||
"city.name",
|
||||
"continent.code",
|
||||
"continent.name",
|
||||
"country.isoCode",
|
||||
"country.name",
|
||||
|
|
@ -238,7 +243,6 @@ public class MaxMindSupportTests extends ESTestCase {
|
|||
"city.confidence",
|
||||
"city.geoNameId",
|
||||
"city.names",
|
||||
"continent.code",
|
||||
"continent.geoNameId",
|
||||
"continent.names",
|
||||
"country.confidence",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue