Add new-style block loader tests for constant_keyword, version, wildcard (#126968)

This commit is contained in:
Oleksandr Kolomiiets 2025-04-17 13:22:09 -07:00 committed by GitHub
parent af6dac5c05
commit 62c0629da6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 424 additions and 31 deletions

View file

@ -32,9 +32,12 @@ dependencies {
api "org.elasticsearch:mocksocket:${versions.mocksocket}"
testImplementation project(":modules:mapper-extras")
testImplementation project(':x-pack:plugin:core')
testImplementation project(':x-pack:plugin:mapper-unsigned-long')
testImplementation project(':x-pack:plugin:mapper-counted-keyword')
testImplementation project(":modules:mapper-extras")
testImplementation project(':x-pack:plugin:mapper-constant-keyword')
testImplementation project(':x-pack:plugin:wildcard')
}
sourceSets {

View file

@ -12,6 +12,7 @@ package org.elasticsearch.logsdb.datageneration;
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
import org.elasticsearch.logsdb.datageneration.fields.leaf.BooleanFieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.fields.leaf.ConstantKeywordFieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.fields.leaf.CountedKeywordFieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.fields.leaf.DateFieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator;
@ -26,6 +27,7 @@ import org.elasticsearch.logsdb.datageneration.fields.leaf.ScaledFloatFieldDataG
import org.elasticsearch.logsdb.datageneration.fields.leaf.ShortFieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.fields.leaf.TextFieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.fields.leaf.WildcardFieldDataGenerator;
/**
* Lists all leaf field types that are supported for data generation by default.
@ -46,7 +48,9 @@ public enum FieldType {
DATE("date"),
GEO_POINT("geo_point"),
TEXT("text"),
IP("ip");
IP("ip"),
CONSTANT_KEYWORD("constant_keyword"),
WILDCARD("wildcard");
private final String name;
@ -56,7 +60,7 @@ public enum FieldType {
public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
return switch (this) {
case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource);
case KEYWORD -> new KeywordFieldDataGenerator(dataSource);
case LONG -> new LongFieldDataGenerator(fieldName, dataSource);
case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource);
case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource);
@ -72,6 +76,8 @@ public enum FieldType {
case GEO_POINT -> new GeoPointFieldDataGenerator(dataSource);
case TEXT -> new TextFieldDataGenerator(dataSource);
case IP -> new IpFieldDataGenerator(dataSource);
case CONSTANT_KEYWORD -> new ConstantKeywordFieldDataGenerator();
case WILDCARD -> new WildcardFieldDataGenerator(dataSource);
};
}
@ -93,6 +99,8 @@ public enum FieldType {
case "geo_point" -> FieldType.GEO_POINT;
case "text" -> FieldType.TEXT;
case "ip" -> FieldType.IP;
case "constant_keyword" -> FieldType.CONSTANT_KEYWORD;
case "wildcard" -> FieldType.WILDCARD;
default -> null;
};
}

View file

@ -78,6 +78,10 @@ public interface DataSourceHandler {
return null;
}
default DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) {
return null;
}
default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
return null;
}

View file

@ -126,6 +126,12 @@ public interface DataSourceRequest<TResponse extends DataSourceResponse> {
}
}
record VersionStringGenerator() implements DataSourceRequest<DataSourceResponse.VersionStringGenerator> {
public DataSourceResponse.VersionStringGenerator accept(DataSourceHandler handler) {
return handler.handle(this);
}
}
record NullWrapper() implements DataSourceRequest<DataSourceResponse.NullWrapper> {
public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) {
return handler.handle(this);

View file

@ -53,6 +53,8 @@ public interface DataSourceResponse {
record IpGenerator(Supplier<InetAddress> generator) implements DataSourceResponse {}
record VersionStringGenerator(Supplier<String> generator) implements DataSourceResponse {}
record NullWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}

View file

@ -51,6 +51,8 @@ public class DefaultMappingParametersHandler implements DataSourceHandler {
case GEO_POINT -> geoPointMapping(map);
case TEXT -> textMapping(request, new HashMap<>());
case IP -> ipMapping(map);
case CONSTANT_KEYWORD -> constantKeywordMapping(new HashMap<>());
case WILDCARD -> wildcardMapping(new HashMap<>());
});
}
@ -225,6 +227,29 @@ public class DefaultMappingParametersHandler implements DataSourceHandler {
};
}
private Supplier<Map<String, Object>> constantKeywordMapping(Map<String, Object> injected) {
return () -> {
// value is optional and can be set from the first document
// we don't cover this case here
injected.put("value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
return injected;
};
}
private Supplier<Map<String, Object>> wildcardMapping(Map<String, Object> injected) {
return () -> {
if (ESTestCase.randomDouble() <= 0.2) {
injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100));
}
if (ESTestCase.randomDouble() <= 0.2) {
injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
}
return injected;
};
}
private static HashMap<String, Object> commonMappingParameters() {
var map = new HashMap<String, Object>();
map.put("store", ESTestCase.randomBoolean());

View file

@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.logsdb.datageneration.fields.leaf;
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
import java.util.Map;
public class ConstantKeywordFieldDataGenerator implements FieldDataGenerator {
@Override
public Object generateValue(Map<String, Object> fieldMapping) {
if (fieldMapping == null) {
// Dynamically mapped, skip it because it will be mapped as text, and we cover this case already
return null;
}
var value = fieldMapping.get("value");
assert value != null;
return value;
}
}

View file

@ -13,14 +13,11 @@ import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
public class CountedKeywordFieldDataGenerator implements FieldDataGenerator {
private final Supplier<Object> valueGenerator;
private final Set<String> previousStrings = new HashSet<>();
public CountedKeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
var strings = dataSource.get(new DataSourceRequest.StringGenerator());

View file

@ -19,7 +19,7 @@ import java.util.function.Supplier;
public class KeywordFieldDataGenerator implements FieldDataGenerator {
private final Supplier<Object> valueGenerator;
public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
public KeywordFieldDataGenerator(DataSource dataSource) {
var strings = dataSource.get(new DataSourceRequest.StringGenerator());
var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());

View file

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.logsdb.datageneration.fields.leaf;
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
import java.util.Map;
public class WildcardFieldDataGenerator implements FieldDataGenerator {
private final FieldDataGenerator keywordGenerator;
public WildcardFieldDataGenerator(DataSource dataSource) {
this.keywordGenerator = new KeywordFieldDataGenerator(dataSource);
}
@Override
public Object generateValue(Map<String, Object> fieldMapping) {
return keywordGenerator.generateValue(fieldMapping);
}
}

View file

@ -46,7 +46,6 @@ interface FieldSpecificMatcher {
return new HashMap<>() {
{
put("keyword", new KeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("date", new DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("long", new NumberMatcher("long", actualMappings, actualSettings, expectedMappings, expectedSettings));
put("unsigned_long", new UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("integer", new NumberMatcher("integer", actualMappings, actualSettings, expectedMappings, expectedSettings));
@ -58,11 +57,14 @@ interface FieldSpecificMatcher {
put("scaled_float", new ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("counted_keyword", new CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("boolean", new BooleanMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("date", new DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("geo_shape", new ExactMatcher("geo_shape", actualMappings, actualSettings, expectedMappings, expectedSettings));
put("shape", new ExactMatcher("shape", actualMappings, actualSettings, expectedMappings, expectedSettings));
put("geo_point", new GeoPointMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("text", new TextMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("ip", new IpMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("constant_keyword", new ConstantKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
put("wildcard", new WildcardMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
}
};
}
@ -691,6 +693,46 @@ interface FieldSpecificMatcher {
}
}
class ConstantKeywordMatcher extends GenericMappingAwareMatcher {
ConstantKeywordMatcher(
XContentBuilder actualMappings,
Settings.Builder actualSettings,
XContentBuilder expectedMappings,
Settings.Builder expectedSettings
) {
super("constant_keyword", actualMappings, actualSettings, expectedMappings, expectedSettings);
}
@Override
Object convert(Object value, Object nullValue) {
// We just need to get rid of literal `null`s which is done in the caller.
return value;
}
}
class WildcardMatcher extends GenericMappingAwareMatcher {
WildcardMatcher(
XContentBuilder actualMappings,
Settings.Builder actualSettings,
XContentBuilder expectedMappings,
Settings.Builder expectedSettings
) {
super("wildcard", actualMappings, actualSettings, expectedMappings, expectedSettings);
}
@Override
Object convert(Object value, Object nullValue) {
if (value == null) {
if (nullValue != null) {
return nullValue;
}
return null;
}
return value;
}
}
/**
* Generic matcher that supports common matching logic like null values.
*/

View file

@ -20,8 +20,10 @@ import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin;
import org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin;
import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin;
import org.elasticsearch.xpack.wildcard.Wildcard;
import java.io.IOException;
import java.util.Collection;
@ -111,7 +113,13 @@ public class DataGenerationTests extends ESTestCase {
var mappingService = new MapperServiceTestCase() {
@Override
protected Collection<? extends Plugin> getPlugins() {
return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin(), new CountedKeywordMapperPlugin());
return List.of(
new UnsignedLongMapperPlugin(),
new MapperExtrasPlugin(),
new CountedKeywordMapperPlugin(),
new ConstantKeywordMapperPlugin(),
new Wildcard()
);
}
}.createMapperService(mappingXContent);

View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.constantkeyword.mapper;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin;
import java.util.Collection;
import java.util.List;
import java.util.Map;
public class ConstantKeywordFieldBlockLoaderTests extends BlockLoaderTestCase {
public ConstantKeywordFieldBlockLoaderTests(Params params) {
super(FieldType.CONSTANT_KEYWORD.toString(), params);
}
@Override
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
return new BytesRef((String) fieldMapping.get("value"));
}
@Override
protected Collection<Plugin> getPlugins() {
return List.of(new ConstantKeywordMapperPlugin());
}
}

View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.versionfield;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.xpack.versionfield.datageneration.VersionStringDataSourceHandler;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public class VersionStringFieldBlockLoaderTests extends BlockLoaderTestCase {
public VersionStringFieldBlockLoaderTests(Params params) {
super("version", List.of(new VersionStringDataSourceHandler()), params);
}
@Override
@SuppressWarnings("unchecked")
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
if (value == null) {
return null;
}
if (value instanceof String s) {
return convert(s);
}
var resultList = ((List<String>) value).stream().map(this::convert).filter(Objects::nonNull).distinct().sorted().toList();
return maybeFoldList(resultList);
}
private BytesRef convert(String value) {
if (value == null) {
return null;
}
return VersionEncoder.encodeVersion(value).bytesRef;
}
@Override
protected Collection<? extends Plugin> getPlugins() {
return Collections.singletonList(new VersionFieldPlugin(getIndexSettings()));
}
}

View file

@ -148,27 +148,7 @@ public class VersionStringFieldMapperTests extends MapperTestCase {
@Override
protected String generateRandomInputValue(MappedFieldType ft) {
return randomValue();
}
protected static String randomValue() {
return randomVersionNumber() + (randomBoolean() ? "" : randomPrerelease());
}
private static String randomVersionNumber() {
int numbers = between(1, 3);
String v = Integer.toString(between(0, 100));
for (int i = 1; i < numbers; i++) {
v += "." + between(0, 100);
}
return v;
}
private static String randomPrerelease() {
if (rarely()) {
return randomFrom("alpha", "beta", "prerelease", "whatever");
}
return randomFrom("alpha", "beta", "") + randomVersionNumber();
return VersionStringTestUtils.randomVersionString();
}
@Override
@ -212,7 +192,7 @@ public class VersionStringFieldMapperTests extends MapperTestCase {
}
private Tuple<String, String> generateValue() {
String v = randomValue();
String v = VersionStringTestUtils.randomVersionString();
return Tuple.tuple(v, v);
}

View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.versionfield;
import org.elasticsearch.test.ESTestCase;
public class VersionStringTestUtils {
public static String randomVersionString() {
return randomVersionNumber() + (ESTestCase.randomBoolean() ? "" : randomPrerelease());
}
private static String randomVersionNumber() {
int numbers = ESTestCase.between(1, 3);
String v = Integer.toString(ESTestCase.between(0, 100));
for (int i = 1; i < numbers; i++) {
v += "." + ESTestCase.between(0, 100);
}
return v;
}
private static String randomPrerelease() {
if (ESTestCase.rarely()) {
return ESTestCase.randomFrom("alpha", "beta", "prerelease", "whatever");
}
return ESTestCase.randomFrom("alpha", "beta", "") + randomVersionNumber();
}
}

View file

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.versionfield.datageneration;
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
import org.elasticsearch.xpack.versionfield.VersionStringTestUtils;
import java.util.HashMap;
public class VersionStringDataSourceHandler implements DataSourceHandler {
@Override
public DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) {
return new DataSourceResponse.VersionStringGenerator(VersionStringTestUtils::randomVersionString);
}
@Override
public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
if (request.fieldType().equals("version") == false) {
return null;
}
return new DataSourceResponse.LeafMappingParametersGenerator(HashMap::new);
}
@Override
public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) {
if (request.fieldType().equals("version") == false) {
return null;
}
return new DataSourceResponse.FieldDataGenerator(new VersionStringFieldDataGenerator(request.dataSource()));
}
}

View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.versionfield.datageneration;
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
import java.util.Map;
import java.util.function.Supplier;
public class VersionStringFieldDataGenerator implements FieldDataGenerator {
private final Supplier<Object> values;
public VersionStringFieldDataGenerator(DataSource dataSource) {
var nullWrapper = dataSource.get(new DataSourceRequest.NullWrapper());
var versionStrings = dataSource.get(new DataSourceRequest.VersionStringGenerator());
this.values = nullWrapper.wrapper().apply(versionStrings.generator()::get);
}
@Override
public Object generateValue(Map<String, Object> fieldMapping) {
return this.values.get();
}
}

View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.wildcard.mapper;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.xpack.wildcard.Wildcard;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public class WildcardFieldBlockLoaderTests extends BlockLoaderTestCase {
public WildcardFieldBlockLoaderTests(Params params) {
super(FieldType.WILDCARD.toString(), params);
}
@Override
@SuppressWarnings("unchecked")
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
var nullValue = (String) fieldMapping.get("null_value");
var ignoreAbove = fieldMapping.get("ignore_above") == null
? Integer.MAX_VALUE
: ((Number) fieldMapping.get("ignore_above")).intValue();
if (value == null) {
return convert(null, nullValue, ignoreAbove);
}
if (value instanceof String s) {
return convert(s, nullValue, ignoreAbove);
}
var resultList = ((List<String>) value).stream()
.map(s -> convert(s, nullValue, ignoreAbove))
.filter(Objects::nonNull)
.distinct()
.sorted()
.toList();
return maybeFoldList(resultList);
}
private static BytesRef convert(String value, String nullValue, int ignoreAbove) {
if (value == null) {
if (nullValue != null) {
value = nullValue;
} else {
return null;
}
}
return value.length() <= ignoreAbove ? new BytesRef(value) : null;
}
@Override
protected Collection<? extends Plugin> getPlugins() {
return Collections.singleton(new Wildcard());
}
}