From b8266e5fb2ea20331d9e354ceabf4577a8709279 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 7 Apr 2021 19:37:57 +0300 Subject: [PATCH] Adds `runtime_mappings` to EQL and SQL requests (#71356) * Adds `runtime_mappings` to EQL and SQL requests allowing users to define search time runtime fields which will be used in queries --- client/rest-high-level/build.gradle | 1 + .../client/eql/EqlSearchRequest.java | 23 +- .../client/eql/EqlSearchRequestTests.java | 5 + .../test/eql/20_runtime_mappings.yml | 192 +++++++++++++ .../xpack/eql/action/EqlSearchRequest.java | 45 +++- .../execution/search/BasicQueryClient.java | 3 + .../eql/execution/search/SourceGenerator.java | 9 +- .../xpack/eql/plan/physical/EsQueryExec.java | 2 +- .../eql/plugin/TransportEqlSearchAction.java | 5 +- .../querydsl/container/QueryContainer.java | 2 +- .../xpack/eql/session/EqlConfiguration.java | 12 +- .../xpack/eql/session/EqlSession.java | 2 +- .../elasticsearch/xpack/eql/EqlTestUtils.java | 10 +- .../eql/action/EqlSearchRequestTests.java | 5 +- .../xpack/ql/index/IndexResolver.java | 21 +- .../org/elasticsearch/xpack/ql/TestUtils.java | 16 ++ .../xpack/sql/jdbc/JdbcHttpClient.java | 4 +- .../xpack/sql/qa/jdbc/DataLoader.java | 17 +- .../sql/qa/rest/BaseRestSqlTestCase.java | 8 +- .../xpack/sql/qa/rest/RestSqlTestCase.java | 91 ++++++- .../server/src/main/resources/alias.csv-spec | 102 +++---- .../src/main/resources/command.csv-spec | 139 +++++----- .../src/main/resources/conditionals.csv-spec | 15 ++ .../src/main/resources/datetime.csv-spec | 30 +++ .../src/main/resources/docs/docs.csv-spec | 92 +++---- .../server/src/main/resources/filter.csv-spec | 18 ++ .../src/main/resources/fulltext.csv-spec | 11 + .../server/src/main/resources/nested.csv-spec | 37 +-- .../server/src/main/resources/pivot.csv-spec | 48 ++-- .../server/src/main/resources/select.csv-spec | 38 ++- .../src/main/resources/setup_test_emp.sql | 3 +- .../single-node-only/command-sys.csv-spec | 254 +++++++++--------- x-pack/plugin/sql/sql-action/build.gradle | 1 + .../sql/action/AbstractSqlQueryRequest.java | 66 ++++- .../xpack/sql/action/SqlQueryRequest.java | 9 +- .../sql/action/SqlQueryRequestBuilder.java | 20 +- .../xpack/sql/action/SqlTranslateRequest.java | 10 +- .../action/SqlTranslateRequestBuilder.java | 17 +- .../sql/action/SqlQueryRequestTests.java | 9 +- .../sql/action/SqlTranslateRequestTests.java | 17 +- .../xpack/sql/client/HttpClient.java | 9 +- .../sql/client/HttpClientRequestTests.java | 4 +- .../xpack/sql/proto/Protocol.java | 1 + .../xpack/sql/proto/SqlQueryRequest.java | 26 +- .../xpack/sql/execution/search/Querier.java | 5 + .../sql/plan/logical/command/ShowColumns.java | 3 +- .../plan/logical/command/sys/SysColumns.java | 5 +- .../plugin/TransportSqlClearCursorAction.java | 5 +- .../sql/plugin/TransportSqlQueryAction.java | 2 +- .../plugin/TransportSqlTranslateAction.java | 2 +- .../xpack/sql/session/SqlConfiguration.java | 11 + .../xpack/sql/session/SqlSession.java | 2 +- .../elasticsearch/xpack/sql/SqlTestUtils.java | 5 +- .../scalar/DatabaseFunctionTests.java | 2 +- .../function/scalar/UserFunctionTests.java | 2 +- .../logical/command/sys/SysColumnsTests.java | 12 +- .../logical/command/sys/SysTablesTests.java | 2 +- .../logical/command/sys/SysTypesTests.java | 2 +- .../sql/plugin/SqlMediaTypeParserTests.java | 2 +- 59 files changed, 1071 insertions(+), 440 deletions(-) create mode 100644 x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 145dbdc944b2..6d8dc6f1aa60 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -54,6 +54,7 @@ dependencies { exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations' } testImplementation(project(':x-pack:plugin:eql')) + testImplementation(project(':x-pack:plugin:ql:test-fixtures')) } tasks.named('forbiddenApisMain').configure { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java index ff3ec273a863..fd1241149bb2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java @@ -20,8 +20,11 @@ import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; + public class EqlSearchRequest implements Validatable, ToXContentObject { private String[] indices; @@ -32,6 +35,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { private String eventCategoryField = "event.category"; private String resultPosition = "tail"; private List fetchFields; + private Map runtimeMappings = emptyMap(); private int size = 10; private int fetchSize = 1000; @@ -55,6 +59,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { static final String KEY_KEEP_ALIVE = "keep_alive"; static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion"; static final String KEY_FETCH_FIELDS = "fields"; + static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings"; public EqlSearchRequest(String indices, String query) { indices(indices); @@ -87,6 +92,9 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { if (fetchFields != null) { builder.field(KEY_FETCH_FIELDS, fetchFields); } + if (runtimeMappings != null && runtimeMappings.isEmpty() == false) { + builder.field(KEY_RUNTIME_MAPPINGS, runtimeMappings); + } builder.endObject(); return builder; } @@ -161,6 +169,15 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { return this; } + public Map runtimeMappings() { + return runtimeMappings; + } + + public EqlSearchRequest runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + return this; + } + public int size() { return this.size; } @@ -243,7 +260,8 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { Objects.equals(keepAlive, that.keepAlive) && Objects.equals(keepOnCompletion, that.keepOnCompletion) && Objects.equals(resultPosition, that.resultPosition) && - Objects.equals(fetchFields, that.fetchFields); + Objects.equals(fetchFields, that.fetchFields) && + Objects.equals(runtimeMappings, that.runtimeMappings); } @Override @@ -262,7 +280,8 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { keepAlive, keepOnCompletion, resultPosition, - fetchFields); + fetchFields, + runtimeMappings); } public String[] indices() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java index 9cc4a68bea39..25a210df3077 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.search.SearchModule; import java.io.IOException; import java.util.List; +import static org.elasticsearch.xpack.ql.TestUtils.randomRuntimeMappings; import static org.hamcrest.Matchers.equalTo; public class EqlSearchRequestTests extends AbstractRequestTestCase { @@ -50,6 +51,9 @@ public class EqlSearchRequestTests extends AbstractRequestTestCase fetchFields; + private Map runtimeMappings = emptyMap(); // Async settings private TimeValue waitForCompletionTimeout = null; @@ -76,6 +78,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion"; static final String KEY_RESULT_POSITION = "result_position"; static final String KEY_FETCH_FIELDS = "fields"; + static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings"; static final ParseField FILTER = new ParseField(KEY_FILTER); static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD); @@ -119,6 +122,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re if (in.readBoolean()) { fetchFields = in.readList(FieldAndFormat::new); } + runtimeMappings = in.readMap(); } } @@ -166,6 +170,29 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re addValidationError("[keep_alive] must be greater than 1 minute, got:" + keepAlive.toString(), validationException); } + if (runtimeMappings != null) { + validationException = validateRuntimeMappings(runtimeMappings, validationException); + } + + return validationException; + } + + private static ActionRequestValidationException validateRuntimeMappings(Map runtimeMappings, + ActionRequestValidationException validationException) { + for (Map.Entry entry : runtimeMappings.entrySet()) { + // top level objects are fields + String fieldName = entry.getKey(); + if (entry.getValue() instanceof Map) { + @SuppressWarnings("unchecked") + Map propNode = (Map) entry.getValue(); + if (propNode.get("type") == null) { + return addValidationError("No type specified for runtime field [" + fieldName + "]", validationException); + } + } else { + return addValidationError("Expected map for runtime field [" + fieldName + "] definition but got [" + + fieldName.getClass().getSimpleName() + "]", validationException); + } + } return validationException; } @@ -193,6 +220,9 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re if (fetchFields != null && fetchFields.isEmpty() == false) { builder.field(KEY_FETCH_FIELDS, fetchFields); } + if (runtimeMappings != null && runtimeMappings.isEmpty() == false) { + builder.field(KEY_RUNTIME_MAPPINGS, runtimeMappings); + } return builder; } @@ -219,6 +249,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re parser.declareBoolean(EqlSearchRequest::keepOnCompletion, KEEP_ON_COMPLETION); parser.declareString(EqlSearchRequest::resultPosition, RESULT_POSITION); parser.declareField(EqlSearchRequest::fetchFields, EqlSearchRequest::parseFetchFields, FETCH_FIELDS_FIELD, ValueType.VALUE_ARRAY); + parser.declareObject(EqlSearchRequest::runtimeMappings, (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD); return parser; } @@ -235,6 +266,13 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re return this; } + public Map runtimeMappings() { return this.runtimeMappings; } + + public EqlSearchRequest runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + return this; + } + public String timestampField() { return this.timestampField; } public EqlSearchRequest timestampField(String timestampField) { @@ -368,6 +406,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re if (fetchFields != null) { out.writeList(fetchFields); } + out.writeMap(runtimeMappings); } } @@ -392,7 +431,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && Objects.equals(keepAlive, that.keepAlive) && Objects.equals(resultPosition, that.resultPosition) && - Objects.equals(fetchFields, that.fetchFields); + Objects.equals(fetchFields, that.fetchFields) && + Objects.equals(runtimeMappings, that.runtimeMappings); } @@ -411,7 +451,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re waitForCompletionTimeout, keepAlive, resultPosition, - fetchFields); + fetchFields, + runtimeMappings); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java index 3caa8271f61d..1b5325cb8c87 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java @@ -143,6 +143,9 @@ public class BasicQueryClient implements QueryClient { if (fetchFields != null) { fetchFields.forEach(builder::fetchField); } + if (cfg.runtimeMappings() != null) { + builder.runtimeMappings(cfg.runtimeMappings()); + } SearchRequest search = prepareRequest(builder, false, entry.getKey()); multiSearchBuilder.add(search); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java index 1a244129487b..0c126fe11c6c 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.ql.querydsl.container.ScriptSort; import org.elasticsearch.xpack.ql.querydsl.container.Sort; import java.util.List; +import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.search.sort.SortBuilders.fieldSort; @@ -31,7 +32,8 @@ public abstract class SourceGenerator { private SourceGenerator() {} - public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryBuilder filter, List fetchFields) { + public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryBuilder filter, List fetchFields, + Map runtimeMappings) { QueryBuilder finalQuery = null; // add the source if (container.query() != null) { @@ -69,6 +71,11 @@ public abstract class SourceGenerator { fetchFields.forEach(source::fetchField); } + // add the runtime fields + if (runtimeMappings != null) { + source.runtimeMappings(runtimeMappings); + } + if (container.limit() != null) { // add size and from source.size(container.limit().absLimit()); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java index f79c95ded870..906de0c4add0 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java @@ -55,7 +55,7 @@ public class EsQueryExec extends LeafExec { public SearchSourceBuilder source(EqlSession session) { EqlConfiguration cfg = session.configuration(); // by default use the configuration size - return SourceGenerator.sourceBuilder(queryContainer, cfg.filter(), cfg.fetchFields()); + return SourceGenerator.sourceBuilder(queryContainer, cfg.filter(), cfg.fetchFields(), cfg.runtimeMappings()); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index e38fe2995365..b2cf7f51e94b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -131,8 +131,9 @@ public class TransportEqlSearchAction extends HandledTransportAction planExecutor.eql(cfg, request.query(), params, wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())), onFailure)), diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java index bb0d4b1484c8..f3ab326b4c9a 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java @@ -169,7 +169,7 @@ public class QueryContainer { public String toString() { try (XContentBuilder builder = JsonXContent.contentBuilder()) { builder.humanReadable(true).prettyPrint(); - SourceGenerator.sourceBuilder(this, null, null).toXContent(builder, ToXContent.EMPTY_PARAMS); + SourceGenerator.sourceBuilder(this, null, null, null).toXContent(builder, ToXContent.EMPTY_PARAMS); return Strings.toString(builder); } catch (IOException e) { throw new EqlIllegalArgumentException("error rendering", e); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java index 69f07b70a074..d4184ce98c3a 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.eql.action.EqlSearchTask; import java.time.ZoneId; import java.util.List; +import java.util.Map; public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configuration { @@ -33,14 +34,17 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu private final QueryBuilder filter; @Nullable private final List fetchFields; + @Nullable + private Map runtimeMappings; public EqlConfiguration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter, - List fetchFields, TimeValue requestTimeout, IndicesOptions indicesOptions, int fetchSize, - String clientId, TaskId taskId, EqlSearchTask task) { + Map runtimeMappings, List fetchFields, TimeValue requestTimeout, + IndicesOptions indicesOptions, int fetchSize, String clientId, TaskId taskId, EqlSearchTask task) { super(zi, username, clusterName); this.indices = indices; this.filter = filter; + this.runtimeMappings = runtimeMappings; this.fetchFields = fetchFields; this.requestTimeout = requestTimeout; this.clientId = clientId; @@ -70,6 +74,10 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu return filter; } + public Map runtimeMappings() { + return runtimeMappings; + } + public List fetchFields() { return fetchFields; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index ccde1cebf6fb..44b79f5f76f3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -101,7 +101,7 @@ public class EqlSession { listener.onFailure(new TaskCancelledException("cancelled")); return; } - indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.indicesOptions(), + indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.indicesOptions(), configuration.runtimeMappings(), map(listener, r -> preAnalyzer.preAnalyze(parsed, r)) ); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java index cfaf5769aaae..5719661f17bf 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java @@ -21,8 +21,7 @@ import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.Inse import org.elasticsearch.xpack.eql.session.EqlConfiguration; import org.elasticsearch.xpack.ql.expression.Expression; -import java.util.Collections; - +import static java.util.Collections.emptyMap; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomIntBetween; @@ -39,8 +38,8 @@ public final class EqlTestUtils { } public static final EqlConfiguration TEST_CFG = new EqlConfiguration(new String[] {"none"}, - org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, null, TimeValue.timeValueSeconds(30), null, - 123, "", new TaskId("test", 123), null); + org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, emptyMap(), null, + TimeValue.timeValueSeconds(30), null, 123, "", new TaskId("test", 123), null); public static EqlConfiguration randomConfiguration() { return new EqlConfiguration(new String[]{randomAlphaOfLength(16)}, @@ -48,6 +47,7 @@ public final class EqlTestUtils { randomAlphaOfLength(16), randomAlphaOfLength(16), null, + emptyMap(), null, new TimeValue(randomNonNegativeLong()), randomIndicesOptions(), @@ -58,7 +58,7 @@ public final class EqlTestUtils { } public static EqlSearchTask randomTask() { - return new EqlSearchTask(randomLong(), "transport", EqlSearchAction.NAME, "", null, Collections.emptyMap(), Collections.emptyMap(), + return new EqlSearchTask(randomLong(), "transport", EqlSearchAction.NAME, "", null, emptyMap(), emptyMap(), new AsyncExecutionId("", new TaskId(randomAlphaOfLength(10), 1)), TimeValue.timeValueDays(5)); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java index 8e7372ce5b11..3cba7869934a 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; +import static org.elasticsearch.xpack.ql.TestUtils.randomRuntimeMappings; public class EqlSearchRequestTests extends AbstractBWCSerializationTestCase { @@ -73,7 +74,8 @@ public class EqlSearchRequestTests extends AbstractBWCSerializationTestCase listener) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions); + Map runtimeMappings, ActionListener listener) { + FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions, runtimeMappings); client.fieldCaps(fieldRequest, ActionListener.wrap( response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), @@ -289,9 +289,9 @@ public class IndexResolver { /** * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. */ - public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boolean includeFrozen, + public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boolean includeFrozen, Map runtimeMappings, ActionListener listener) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); + FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, runtimeMappings); client.fieldCaps(fieldRequest, ActionListener.wrap( response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), @@ -455,27 +455,30 @@ public class IndexResolver { return new EsField(fieldName, esType, props, isAggregateable, isAlias); } - private static FieldCapabilitiesRequest createFieldCapsRequest(String index, IndicesOptions indicesOptions) { + private static FieldCapabilitiesRequest createFieldCapsRequest(String index, IndicesOptions indicesOptions, + Map runtimeMappings) { return new FieldCapabilitiesRequest() .indices(Strings.commaDelimitedListToStringArray(index)) .fields("*") .includeUnmapped(true) + .runtimeFields(runtimeMappings) //lenient because we throw our own errors looking at the response e.g. if something was not resolved //also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable .indicesOptions(indicesOptions); } - private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boolean includeFrozen) { + private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boolean includeFrozen, + Map runtimeMappings) { IndicesOptions indicesOptions = includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS; - return createFieldCapsRequest(index, indicesOptions); + return createFieldCapsRequest(index, indicesOptions, runtimeMappings); } /** * Resolves a pattern to multiple, separate indices. Doesn't perform validation. */ public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen, - ActionListener> listener) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); + Map runtimeMappings, ActionListener> listener) { + FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, runtimeMappings); client.fieldCaps(fieldRequest, wrap(response -> { client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases -> listener.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases())), diff --git a/x-pack/plugin/ql/test-fixtures/src/main/java/org/elasticsearch/xpack/ql/TestUtils.java b/x-pack/plugin/ql/test-fixtures/src/main/java/org/elasticsearch/xpack/ql/TestUtils.java index bc933c8b1fe9..619ea9e2f2e9 100644 --- a/x-pack/plugin/ql/test-fixtures/src/main/java/org/elasticsearch/xpack/ql/TestUtils.java +++ b/x-pack/plugin/ql/test-fixtures/src/main/java/org/elasticsearch/xpack/ql/TestUtils.java @@ -55,12 +55,14 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.ZoneId; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.jar.JarInputStream; import java.util.zip.ZipEntry; import static java.util.Collections.emptyMap; +import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomFrom; @@ -300,4 +302,18 @@ public final class TestUtils { return builder.toString(); } } + + public static Map randomRuntimeMappings() { + int count = between(1, 100); + Map runtimeFields = new HashMap<>(count); + while (runtimeFields.size() < count) { + int size = between(1, 10); + Map config = new HashMap<>(size); + while (config.size() < size) { + config.put(randomAlphaOfLength(5), randomAlphaOfLength(5)); + } + runtimeFields.put(randomAlphaOfLength(5), config); + } + return runtimeFields; + } } diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java index 315b5b1241f4..276799811bc8 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java @@ -23,6 +23,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.sql.client.StringUtils.EMPTY; /** @@ -67,7 +68,8 @@ class JdbcHttpClient { new RequestInfo(Mode.JDBC, ClientVersion.CURRENT), conCfg.fieldMultiValueLeniency(), conCfg.indexIncludeFrozen(), - conCfg.binaryCommunication()); + conCfg.binaryCommunication(), + emptyMap()); SqlQueryResponse response = httpClient.query(sqlRequest); return new DefaultCursor(this, response.cursor(), toJdbcColumnInfo(response.columns()), response.rows(), meta); } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java index 790762cb907d..b2ed00321d7c 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java @@ -9,7 +9,6 @@ package org.elasticsearch.xpack.sql.qa.jdbc; import org.apache.http.HttpHost; import org.apache.logging.log4j.LogManager; import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Strings; @@ -151,6 +150,20 @@ public class DataLoader { } } createIndex.endObject(); + // define the runtime field + createIndex.startObject("runtime"); + { + createIndex.startObject("birth_date_day_of_week").field("type", "keyword"); + createIndex.startObject("script") + .field( + "source", + "if (doc['birth_date'].size()==0) return; " + + "else emit(doc['birth_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + ); + createIndex.endObject(); + createIndex.endObject(); + } + createIndex.endObject(); } createIndex.endObject().endObject(); request.setJsonEntity(Strings.toString(createIndex)); @@ -369,7 +382,7 @@ public class DataLoader { bulk.append("}\n"); }); request.setJsonEntity(bulk.toString()); - Response response = client.performRequest(request); + client.performRequest(request); } public static void makeAlias(RestClient client, String aliasName, String... indices) throws Exception { diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java index bcb0c5d7a2df..fdcb6f936b95 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java @@ -33,6 +33,7 @@ import static org.elasticsearch.xpack.sql.proto.Protocol.FILTER_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.MODE_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.PARAMS_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.QUERY_NAME; +import static org.elasticsearch.xpack.sql.proto.Protocol.RUNTIME_MAPPINGS_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.TIME_ZONE_NAME; public abstract class BaseRestSqlTestCase extends ESRestTestCase { @@ -111,6 +112,11 @@ public abstract class BaseRestSqlTestCase extends ESRestTestCase { return this; } + public RequestObjectBuilder runtimeMappings(String runtimeMappings) { + request.append(field(RUNTIME_MAPPINGS_NAME, runtimeMappings)); + return this; + } + private static String field(String name, Object value) { if (value == null) { return StringUtils.EMPTY; @@ -122,7 +128,7 @@ public abstract class BaseRestSqlTestCase extends ESRestTestCase { return StringUtils.EMPTY; } String lowerName = name.toLowerCase(Locale.ROOT); - if (lowerName.equals(PARAMS_NAME) || lowerName.equals(FILTER_NAME)) { + if (lowerName.equals(PARAMS_NAME) || lowerName.equals(FILTER_NAME) || lowerName.equals(RUNTIME_MAPPINGS_NAME)) { field += value; } else { field += "\"" + value + "\""; diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index 037de36cdcd6..b901dde7704d 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -45,6 +45,7 @@ import java.util.Locale; import java.util.Map; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static java.util.Collections.unmodifiableMap; @@ -482,7 +483,7 @@ public abstract class RestSqlTestCase extends BaseRestSqlTestCase implements Err ); expectBadRequest(() -> { client().performRequest(request); - return Collections.emptyMap(); + return emptyMap(); }, containsString("Invalid use of [columnar] argument: cannot be used in combination with txt, csv or tsv formats")); } @@ -494,10 +495,94 @@ public abstract class RestSqlTestCase extends BaseRestSqlTestCase implements Err request.setEntity(new StringEntity(query("SELECT * FROM test").mode(mode).columnar(true).toString(), ContentType.APPLICATION_JSON)); expectBadRequest(() -> { client().performRequest(request); - return Collections.emptyMap(); + return emptyMap(); }, containsString("unknown field [columnar]")); } + public void testValidateRuntimeMappingsInSqlQuery() throws IOException { + testValidateRuntimeMappingsInQuery(SQL_QUERY_REST_ENDPOINT); + + String mode = randomMode(); + Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT); + index("{\"test\":true}", "{\"test\":false}"); + String runtimeMappings = "{\"bool_as_long\": {\"type\":\"long\", \"script\": {\"source\":\"if(doc['test'].value == true) emit(1);" + + "else emit(0);\"}}}"; + request.setEntity( + new StringEntity( + query("SELECT * FROM test").mode(mode).runtimeMappings(runtimeMappings).toString(), + ContentType.APPLICATION_JSON + ) + ); + Map expected = new HashMap<>(); + expected.put( + "columns", + Arrays.asList( + columnInfo(mode, "bool_as_long", "long", JDBCType.BIGINT, 20), + columnInfo(mode, "test", "boolean", JDBCType.BOOLEAN, 1) + ) + ); + expected.put("rows", Arrays.asList(Arrays.asList(1, true), Arrays.asList(0, false))); + assertResponse( + expected, + runSql( + new StringEntity( + query("SELECT * FROM test").mode(mode).runtimeMappings(runtimeMappings).toString(), + ContentType.APPLICATION_JSON + ), + StringUtils.EMPTY, + mode + ) + ); + } + + public void testValidateRuntimeMappingsInTranslateQuery() throws IOException { + testValidateRuntimeMappingsInQuery(SQL_TRANSLATE_REST_ENDPOINT); + + index("{\"test\":true}", "{\"test\":false}"); + String runtimeMappings = "{\"bool_as_long\": {\"type\":\"long\", \"script\": {\"source\":\"if(doc['test'].value == true) emit(1);" + + "else emit(0);\"}}}"; + Map response = runTranslateSql(query("SELECT * FROM test").runtimeMappings(runtimeMappings).toString()); + assertEquals(response.get("size"), 1000); + assertFalse((Boolean) response.get("_source")); + @SuppressWarnings("unchecked") + List> source = (List>) response.get("fields"); + assertEquals(Arrays.asList(singletonMap("field", "bool_as_long"), singletonMap("field", "test")), source); + + assertNull(response.get("query")); + + @SuppressWarnings("unchecked") + List> sort = (List>) response.get("sort"); + assertEquals(singletonList(singletonMap("_doc", singletonMap("order", "asc"))), sort); + } + + private static void testValidateRuntimeMappingsInQuery(String queryTypeEndpoint) { + String mode = randomMode(); + String runtimeMappings = "{\"address\": {\"script\": \"return\"}}"; + Request request = new Request("POST", queryTypeEndpoint); + request.setEntity( + new StringEntity( + query("SELECT * FROM test").mode(mode).runtimeMappings(runtimeMappings).toString(), + ContentType.APPLICATION_JSON + ) + ); + expectBadRequest(() -> { + client().performRequest(request); + return emptyMap(); + }, containsString("No type specified for runtime field [address]")); + + runtimeMappings = "{\"address\": [{\"script\": \"return\"}]}"; + request.setEntity( + new StringEntity( + query("SELECT * FROM test").mode(mode).runtimeMappings(runtimeMappings).toString(), + ContentType.APPLICATION_JSON + ) + ); + expectBadRequest(() -> { + client().performRequest(request); + return emptyMap(); + }, containsString("Expected map for runtime field [address] definition but got [String]")); + } + public static void expectBadRequest(CheckedSupplier, Exception> code, Matcher errorMessageMatcher) { try { Map result = code.get(); @@ -917,7 +1002,7 @@ public abstract class RestSqlTestCase extends BaseRestSqlTestCase implements Err ); expectBadRequest(() -> { client().performRequest(badRequest); - return Collections.emptyMap(); + return emptyMap(); }, containsString("request [/_sql] contains unrecognized parameter: [delimiter]")); Request csvRequest = new Request("POST", SQL_QUERY_REST_ENDPOINT + "?format=csv&delimiter=%3B"); diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec index 3c72cb630b0c..d5b27f215869 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec @@ -28,61 +28,63 @@ emp_no:i | first_name:s describeAlias DESCRIBE test_alias; - column | type | mapping ---------------------+---------------+--------------- -birth_date |TIMESTAMP |datetime -dep |STRUCT |nested -dep.dep_id |VARCHAR |keyword -dep.dep_name |VARCHAR |text -dep.dep_name.keyword|VARCHAR |keyword -dep.from_date |TIMESTAMP |datetime -dep.to_date |TIMESTAMP |datetime -emp_no |INTEGER |integer -extra |STRUCT |object -extra.info |STRUCT |object -extra.info.gender |VARCHAR |keyword -extra_gender |VARCHAR |keyword -extra_no |INTEGER |integer -first_name |VARCHAR |text -first_name.keyword |VARCHAR |keyword -gender |VARCHAR |keyword -hire_date |TIMESTAMP |datetime -languages |TINYINT |byte -last_name |VARCHAR |text -last_name.keyword |VARCHAR |keyword -null_constant |VARCHAR |keyword -salary |INTEGER |integer -wildcard_name |VARCHAR |keyword + column | type | mapping +----------------------+---------------+--------------- +birth_date |TIMESTAMP |datetime +birth_date_day_of_week|VARCHAR |keyword +dep |STRUCT |nested +dep.dep_id |VARCHAR |keyword +dep.dep_name |VARCHAR |text +dep.dep_name.keyword |VARCHAR |keyword +dep.from_date |TIMESTAMP |datetime +dep.to_date |TIMESTAMP |datetime +emp_no |INTEGER |integer +extra |STRUCT |object +extra.info |STRUCT |object +extra.info.gender |VARCHAR |keyword +extra_gender |VARCHAR |keyword +extra_no |INTEGER |integer +first_name |VARCHAR |text +first_name.keyword |VARCHAR |keyword +gender |VARCHAR |keyword +hire_date |TIMESTAMP |datetime +languages |TINYINT |byte +last_name |VARCHAR |text +last_name.keyword |VARCHAR |keyword +null_constant |VARCHAR |keyword +salary |INTEGER |integer +wildcard_name |VARCHAR |keyword ; describePattern DESCRIBE "test_*"; - column | type | mapping ---------------------+---------------+--------------- -birth_date |TIMESTAMP |datetime -dep |STRUCT |nested -dep.dep_id |VARCHAR |keyword -dep.dep_name |VARCHAR |text -dep.dep_name.keyword|VARCHAR |keyword -dep.from_date |TIMESTAMP |datetime -dep.to_date |TIMESTAMP |datetime -emp_no |INTEGER |integer -extra |STRUCT |object -extra.info |STRUCT |object -extra.info.gender |VARCHAR |keyword -extra_gender |VARCHAR |keyword -extra_no |INTEGER |integer -first_name |VARCHAR |text -first_name.keyword |VARCHAR |keyword -gender |VARCHAR |keyword -hire_date |TIMESTAMP |datetime -languages |TINYINT |byte -last_name |VARCHAR |text -last_name.keyword |VARCHAR |keyword -null_constant |VARCHAR |keyword -salary |INTEGER |integer -wildcard_name |VARCHAR |keyword + column | type | mapping +----------------------+---------------+--------------- +birth_date |TIMESTAMP |datetime +birth_date_day_of_week|VARCHAR |keyword +dep |STRUCT |nested +dep.dep_id |VARCHAR |keyword +dep.dep_name |VARCHAR |text +dep.dep_name.keyword |VARCHAR |keyword +dep.from_date |TIMESTAMP |datetime +dep.to_date |TIMESTAMP |datetime +emp_no |INTEGER |integer +extra |STRUCT |object +extra.info |STRUCT |object +extra.info.gender |VARCHAR |keyword +extra_gender |VARCHAR |keyword +extra_no |INTEGER |integer +first_name |VARCHAR |text +first_name.keyword |VARCHAR |keyword +gender |VARCHAR |keyword +hire_date |TIMESTAMP |datetime +languages |TINYINT |byte +last_name |VARCHAR |text +last_name.keyword |VARCHAR |keyword +null_constant |VARCHAR |keyword +salary |INTEGER |integer +wildcard_name |VARCHAR |keyword ; showAlias diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec index d8170b71347d..7431097c7d02 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec @@ -274,84 +274,87 @@ test_alias_emp |VIEW |ALIAS describeSimpleLike DESCRIBE LIKE 'test_emp'; - column | type | mapping ---------------------+---------------+--------------- -birth_date |TIMESTAMP |datetime -dep |STRUCT |nested -dep.dep_id |VARCHAR |keyword -dep.dep_name |VARCHAR |text -dep.dep_name.keyword|VARCHAR |keyword -dep.from_date |TIMESTAMP |datetime -dep.to_date |TIMESTAMP |datetime -emp_no |INTEGER |integer -extra |STRUCT |object -extra.info |STRUCT |object -extra.info.gender |VARCHAR |keyword -extra_gender |VARCHAR |keyword -extra_no |INTEGER |integer -first_name |VARCHAR |text -first_name.keyword |VARCHAR |keyword -gender |VARCHAR |keyword -hire_date |TIMESTAMP |datetime -languages |TINYINT |byte -last_name |VARCHAR |text -last_name.keyword |VARCHAR |keyword -null_constant |VARCHAR |keyword -salary |INTEGER |integer -wildcard_name |VARCHAR |keyword + column | type | mapping +----------------------+---------------+--------------- +birth_date |TIMESTAMP |datetime +birth_date_day_of_week|VARCHAR |keyword +dep |STRUCT |nested +dep.dep_id |VARCHAR |keyword +dep.dep_name |VARCHAR |text +dep.dep_name.keyword |VARCHAR |keyword +dep.from_date |TIMESTAMP |datetime +dep.to_date |TIMESTAMP |datetime +emp_no |INTEGER |integer +extra |STRUCT |object +extra.info |STRUCT |object +extra.info.gender |VARCHAR |keyword +extra_gender |VARCHAR |keyword +extra_no |INTEGER |integer +first_name |VARCHAR |text +first_name.keyword |VARCHAR |keyword +gender |VARCHAR |keyword +hire_date |TIMESTAMP |datetime +languages |TINYINT |byte +last_name |VARCHAR |text +last_name.keyword |VARCHAR |keyword +null_constant |VARCHAR |keyword +salary |INTEGER |integer +wildcard_name |VARCHAR |keyword ; describeMultiLike DESCRIBE LIKE 'test_emp%'; - column | type | mapping ---------------------+---------------+--------------- -birth_date |TIMESTAMP |datetime -dep |STRUCT |nested -dep.dep_id |VARCHAR |keyword -dep.dep_name |VARCHAR |text -dep.dep_name.keyword|VARCHAR |keyword -dep.from_date |TIMESTAMP |datetime -dep.to_date |TIMESTAMP |datetime -emp_no |INTEGER |integer -extra |STRUCT |object -extra.info |STRUCT |object -extra.info.gender |VARCHAR |keyword -extra_gender |VARCHAR |keyword -extra_no |INTEGER |integer -first_name |VARCHAR |text -first_name.keyword |VARCHAR |keyword -gender |VARCHAR |keyword -hire_date |TIMESTAMP |datetime -languages |TINYINT |byte -last_name |VARCHAR |text -last_name.keyword |VARCHAR |keyword -null_constant |VARCHAR |keyword -salary |INTEGER |integer -wildcard_name |VARCHAR |keyword + column | type | mapping +----------------------+---------------+--------------- +birth_date |TIMESTAMP |datetime +birth_date_day_of_week|VARCHAR |keyword +dep |STRUCT |nested +dep.dep_id |VARCHAR |keyword +dep.dep_name |VARCHAR |text +dep.dep_name.keyword |VARCHAR |keyword +dep.from_date |TIMESTAMP |datetime +dep.to_date |TIMESTAMP |datetime +emp_no |INTEGER |integer +extra |STRUCT |object +extra.info |STRUCT |object +extra.info.gender |VARCHAR |keyword +extra_gender |VARCHAR |keyword +extra_no |INTEGER |integer +first_name |VARCHAR |text +first_name.keyword |VARCHAR |keyword +gender |VARCHAR |keyword +hire_date |TIMESTAMP |datetime +languages |TINYINT |byte +last_name |VARCHAR |text +last_name.keyword |VARCHAR |keyword +null_constant |VARCHAR |keyword +salary |INTEGER |integer +wildcard_name |VARCHAR |keyword ; describeSimpleIdentifier DESCRIBE "test_emp"; - column | type | mapping ---------------------+---------------+--------------- -birth_date |TIMESTAMP |datetime -dep |STRUCT |nested -dep.dep_id |VARCHAR |keyword -dep.dep_name |VARCHAR |text -dep.dep_name.keyword|VARCHAR |keyword -dep.from_date |TIMESTAMP |datetime -dep.to_date |TIMESTAMP |datetime -emp_no |INTEGER |integer -first_name |VARCHAR |text -first_name.keyword |VARCHAR |keyword -gender |VARCHAR |keyword -hire_date |TIMESTAMP |datetime -languages |TINYINT |byte -last_name |VARCHAR |text -last_name.keyword |VARCHAR |keyword -salary |INTEGER |integer + column | type | mapping +----------------------+---------------+--------------- +birth_date |TIMESTAMP |datetime +birth_date_day_of_week|VARCHAR |keyword +dep |STRUCT |nested +dep.dep_id |VARCHAR |keyword +dep.dep_name |VARCHAR |text +dep.dep_name.keyword |VARCHAR |keyword +dep.from_date |TIMESTAMP |datetime +dep.to_date |TIMESTAMP |datetime +emp_no |INTEGER |integer +first_name |VARCHAR |text +first_name.keyword |VARCHAR |keyword +gender |VARCHAR |keyword +hire_date |TIMESTAMP |datetime +languages |TINYINT |byte +last_name |VARCHAR |text +last_name.keyword |VARCHAR |keyword +salary |INTEGER |integer ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/conditionals.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/conditionals.csv-spec index d0b764e7a4fc..893076a4ddb7 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/conditionals.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/conditionals.csv-spec @@ -532,3 +532,18 @@ SELECT GREATEST(null, null, birth_date + INTERVAL 25 YEARS, hire_date + INTERVAL 1986-02-28T00:00:00.000Z|1952-11-13T00:00:00.000Z|1986-02-26T00:00:00.000Z 1986-05-30T00:00:00.000Z|1961-05-30T00:00:00.000Z|1986-03-14T00:00:00.000Z ; + +ifNullWithRuntimeField +SELECT COUNT(*) c, IFNULL(birth_date_day_of_week, 'no day') AS day FROM test_emp GROUP BY day ORDER BY c; + + c:l | day:s +---------------+--------------- +8 |Monday +10 |Sunday +10 |no day +12 |Friday +13 |Saturday +14 |Wednesday +15 |Thursday +18 |Tuesday +; \ No newline at end of file diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec index 6a1a9b5d6a57..7e6085f28980 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec @@ -131,6 +131,36 @@ SELECT WEEK(birth_date) week, birth_date FROM test_emp ORDER BY WEEK(birth_date) 44 |1961-11-02T00:00:00.000Z ; +isoDayOfWeek +SELECT ISO_DAY_OF_WEEK(birth_date) AS d, SUM(salary) s FROM test_emp GROUP BY d ORDER BY d DESC; + + d:i | s:i +---------------+--------------- +7 |386466 +6 |643304 +5 |653130 +4 |740669 +3 |655169 +2 |888011 +1 |428067 +null |430039 +; + +runtimeFieldDayOfWeek +SELECT birth_date_day_of_week, SUM(salary) s FROM test_emp GROUP BY birth_date_day_of_week ORDER BY birth_date_day_of_week DESC; + +birth_date_day_of_week:s| s:i +------------------------+--------------- +Wednesday |655169 +Tuesday |888011 +Thursday |740669 +Sunday |386466 +Saturday |643304 +Monday |428067 +Friday |653130 +null |430039 +; + isoWeekOfYear schema::birth_date:ts|iso_week:i|week:i SELECT birth_date, IW(birth_date) iso_week, WEEK(birth_date) week FROM test_emp WHERE IW(birth_date) < 8 AND week >2 ORDER BY iso_week; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec index c96777647ef5..9fbef26fdcdd 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec @@ -14,24 +14,25 @@ describeTable // tag::describeTable DESCRIBE emp; - column | type | mapping ---------------------+---------------+--------------- -birth_date |TIMESTAMP |datetime -dep |STRUCT |nested -dep.dep_id |VARCHAR |keyword -dep.dep_name |VARCHAR |text -dep.dep_name.keyword|VARCHAR |keyword -dep.from_date |TIMESTAMP |datetime -dep.to_date |TIMESTAMP |datetime -emp_no |INTEGER |integer -first_name |VARCHAR |text -first_name.keyword |VARCHAR |keyword -gender |VARCHAR |keyword -hire_date |TIMESTAMP |datetime -languages |TINYINT |byte -last_name |VARCHAR |text -last_name.keyword |VARCHAR |keyword -salary |INTEGER |integer + column | type | mapping +----------------------+---------------+--------------- +birth_date |TIMESTAMP |datetime +birth_date_day_of_week|VARCHAR |keyword +dep |STRUCT |nested +dep.dep_id |VARCHAR |keyword +dep.dep_name |VARCHAR |text +dep.dep_name.keyword |VARCHAR |keyword +dep.from_date |TIMESTAMP |datetime +dep.to_date |TIMESTAMP |datetime +emp_no |INTEGER |integer +first_name |VARCHAR |text +first_name.keyword |VARCHAR |keyword +gender |VARCHAR |keyword +hire_date |TIMESTAMP |datetime +languages |TINYINT |byte +last_name |VARCHAR |text +last_name.keyword |VARCHAR |keyword +salary |INTEGER |integer // end::describeTable ; @@ -53,24 +54,25 @@ showColumns // tag::showColumns SHOW COLUMNS IN emp; - column | type | mapping ---------------------+---------------+--------------- -birth_date |TIMESTAMP |datetime -dep |STRUCT |nested -dep.dep_id |VARCHAR |keyword -dep.dep_name |VARCHAR |text -dep.dep_name.keyword|VARCHAR |keyword -dep.from_date |TIMESTAMP |datetime -dep.to_date |TIMESTAMP |datetime -emp_no |INTEGER |integer -first_name |VARCHAR |text -first_name.keyword |VARCHAR |keyword -gender |VARCHAR |keyword -hire_date |TIMESTAMP |datetime -languages |TINYINT |byte -last_name |VARCHAR |text -last_name.keyword |VARCHAR |keyword -salary |INTEGER |integer + column | type | mapping +----------------------+---------------+--------------- +birth_date |TIMESTAMP |datetime +birth_date_day_of_week|VARCHAR |keyword +dep |STRUCT |nested +dep.dep_id |VARCHAR |keyword +dep.dep_name |VARCHAR |text +dep.dep_name.keyword |VARCHAR |keyword +dep.from_date |TIMESTAMP |datetime +dep.to_date |TIMESTAMP |datetime +emp_no |INTEGER |integer +first_name |VARCHAR |text +first_name.keyword |VARCHAR |keyword +gender |VARCHAR |keyword +hire_date |TIMESTAMP |datetime +languages |TINYINT |byte +last_name |VARCHAR |text +last_name.keyword |VARCHAR |keyword +salary |INTEGER |integer // end::showColumns ; @@ -485,9 +487,9 @@ wildcardWithOrder // tag::wildcardWithOrder SELECT * FROM emp LIMIT 1; - birth_date | emp_no | first_name | gender | hire_date | languages | last_name | salary ---------------------+---------------+---------------+---------------+--------------------+---------------+---------------+--------------- -1953-09-02T00:00:00Z|10001 |Georgi |M |1986-06-26T00:00:00Z|2 |Facello |57305 + birth_date |birth_date_day_of_week| emp_no | first_name | gender | hire_date | languages | last_name | salary +--------------------+----------------------+---------------+---------------+---------------+--------------------+---------------+---------------+--------------- +1953-09-02T00:00:00Z|Wednesday |10001 |Georgi |M |1986-06-26T00:00:00Z|2 |Facello |57305 // end::wildcardWithOrder ; @@ -496,9 +498,9 @@ fromTable // tag::fromTable SELECT * FROM emp LIMIT 1; - birth_date | emp_no | first_name | gender | hire_date | languages | last_name | salary ---------------------+---------------+---------------+---------------+--------------------+---------------+---------------+--------------- -1953-09-02T00:00:00Z|10001 |Georgi |M |1986-06-26T00:00:00Z|2 |Facello |57305 + birth_date |birth_date_day_of_week| emp_no | first_name | gender | hire_date | languages | last_name | salary +--------------------+----------------------+---------------+---------------+---------------+--------------------+---------------+---------------+--------------- +1953-09-02T00:00:00Z|Wednesday |10001 |Georgi |M |1986-06-26T00:00:00Z|2 |Facello |57305 // end::fromTable @@ -508,9 +510,9 @@ fromTableQuoted // tag::fromTableQuoted SELECT * FROM "emp" LIMIT 1; - birth_date | emp_no | first_name | gender | hire_date | languages | last_name | salary ---------------------+---------------+---------------+---------------+--------------------+---------------+---------------+--------------- -1953-09-02T00:00:00Z|10001 |Georgi |M |1986-06-26T00:00:00Z|2 |Facello |57305 + birth_date |birth_date_day_of_week| emp_no | first_name | gender | hire_date | languages | last_name | salary +--------------------+----------------------+---------------+---------------+---------------+--------------------+---------------+---------------+--------------- +1953-09-02T00:00:00Z|Wednesday |10001 |Georgi |M |1986-06-26T00:00:00Z|2 |Facello |57305 // end::fromTableQuoted ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/filter.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/filter.csv-spec index dd7d50d937da..63fabdffdd8d 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/filter.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/filter.csv-spec @@ -158,3 +158,21 @@ SELECT count(*) AS c FROM test_emp WHERE birth_date::time IN ('00:00:00Z'::TIME, --------------- 90 ; + +inWithRuntimeField +SELECT COUNT(*) AS c, birth_date_day_of_week FROM test_emp WHERE birth_date_day_of_week IN ('Saturday', 'Sunday') GROUP BY birth_date_day_of_week; + + c:l |birth_date_day_of_week:s +---------------+------------------------ +13 |Saturday +10 |Sunday +; + +inWithDayName +SELECT COUNT(*) AS c, DAY_NAME(birth_date) FROM test_emp WHERE DAY_NAME(birth_date) IN ('Saturday', 'Sunday') GROUP BY DAY_NAME(birth_date); + + c:l |DAY_NAME(birth_date):s +---------------+---------------------- +13 |Saturday +10 |Sunday +; \ No newline at end of file diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/fulltext.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/fulltext.csv-spec index 933347ff7afd..b25085d88c50 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/fulltext.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/fulltext.csv-spec @@ -202,3 +202,14 @@ null |1.9161749939033146|0.1480828817161133 |74999 |28336 3 |1.0732551278666582|0.05483979801873433|65030 |38376 |4 5 |1.322529094661261 |0.24501477738153868|66817 |37137 |4 ; + +runtimeFieldWithQuery +SELECT COUNT(*) AS c, birth_date_day_of_week FROM test_emp WHERE QUERY('T*','default_field=birth_date_day_of_week') OR QUERY('S*','default_field=birth_date_day_of_week') GROUP BY birth_date_day_of_week ORDER BY c; + + c:l |birth_date_day_of_week:s +---------------+------------------------ +10 |Sunday +13 |Saturday +15 |Thursday +18 |Tuesday +; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/nested.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/nested.csv-spec index ccc864bc1425..950b8baa14fd 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/nested.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/nested.csv-spec @@ -8,24 +8,25 @@ describeParent DESCRIBE test_emp; - column | type | mapping ---------------------+---------------+--------------- -birth_date |TIMESTAMP |datetime -dep |STRUCT |nested -dep.dep_id |VARCHAR |keyword -dep.dep_name |VARCHAR |text -dep.dep_name.keyword|VARCHAR |keyword -dep.from_date |TIMESTAMP |datetime -dep.to_date |TIMESTAMP |datetime -emp_no |INTEGER |integer -first_name |VARCHAR |text -first_name.keyword |VARCHAR |keyword -gender |VARCHAR |keyword -hire_date |TIMESTAMP |datetime -languages |TINYINT |byte -last_name |VARCHAR |text -last_name.keyword |VARCHAR |keyword -salary |INTEGER |integer + column | type | mapping +----------------------+---------------+--------------- +birth_date |TIMESTAMP |datetime +birth_date_day_of_week|VARCHAR |keyword +dep |STRUCT |nested +dep.dep_id |VARCHAR |keyword +dep.dep_name |VARCHAR |text +dep.dep_name.keyword |VARCHAR |keyword +dep.from_date |TIMESTAMP |datetime +dep.to_date |TIMESTAMP |datetime +emp_no |INTEGER |integer +first_name |VARCHAR |text +first_name.keyword |VARCHAR |keyword +gender |VARCHAR |keyword +hire_date |TIMESTAMP |datetime +languages |TINYINT |byte +last_name |VARCHAR |text +last_name.keyword |VARCHAR |keyword +salary |INTEGER |integer ; nestedStar diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/pivot.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/pivot.csv-spec index 3265c1038a47..c594b7b2cf5d 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/pivot.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/pivot.csv-spec @@ -186,17 +186,17 @@ null |48396.28571428572|62140.666666666664 ; sumWithoutSubquery -schema::birth_date:ts|emp_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|1:i|2:i +schema::birth_date:ts|birth_date_day_of_week:s|emp_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|1:i|2:i // tag::sumWithoutSubquery SELECT * FROM test_emp PIVOT (SUM(salary) FOR languages IN (1, 2)) LIMIT 5; - birth_date | emp_no | first_name | gender | hire_date | last_name | 1 | 2 ----------------------+---------------+---------------+---------------+---------------------+---------------+---------------+--------------- -null |10041 |Uri |F |1989-11-12 00:00:00.0|Lenart |56415 |null -null |10043 |Yishay |M |1990-10-20 00:00:00.0|Tzvieli |34341 |null -null |10044 |Mingsen |F |1994-05-21 00:00:00.0|Casley |39728 |null -1952-04-19 00:00:00.0|10009 |Sumant |F |1985-02-18 00:00:00.0|Peac |66174 |null -1953-01-07 00:00:00.0|10067 |Claudi |M |1987-03-04 00:00:00.0|Stavenow |null |52044 + birth_date |birth_date_day_of_week| emp_no | first_name | gender | hire_date | last_name | 1 | 2 +---------------------+----------------------+---------------+---------------+---------------+---------------------+---------------+---------------+--------------- +null |null |10041 |Uri |F |1989-11-12 00:00:00.0|Lenart |56415 |null +null |null |10043 |Yishay |M |1990-10-20 00:00:00.0|Tzvieli |34341 |null +null |null |10044 |Mingsen |F |1994-05-21 00:00:00.0|Casley |39728 |null +1952-04-19 00:00:00.0|Saturday |10009 |Sumant |F |1985-02-18 00:00:00.0|Peac |66174 |null +1953-01-07 00:00:00.0|Wednesday |10067 |Claudi |M |1987-03-04 00:00:00.0|Stavenow |null |52044 // end::sumWithoutSubquery ; @@ -213,29 +213,29 @@ PIVOT (SUM(bytes_in) FOR status IN ('OK','Error')); ; sumWithInnerAggregateSumOfSquares -schema::birth_date:ts|emp_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|1:d|2:d +schema::birth_date:ts|birth_date_day_of_week:s|emp_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|1:d|2:d SELECT * FROM test_emp PIVOT (SUM_OF_SQUARES(salary) FOR languages IN (1, 2)) LIMIT 5; - birth_date | emp_no | first_name | gender | hire_date | last_name | 1 | 2 ----------------------+---------------+---------------+---------------+------------------------+---------------+---------------+--------------- -null |10041 |Uri |F |1989-11-12T00:00:00.000Z|Lenart |3.182652225E9 |null -null |10043 |Yishay |M |1990-10-20T00:00:00.000Z|Tzvieli |1.179304281E9 |null -null |10044 |Mingsen |F |1994-05-21T00:00:00.000Z|Casley |1.578313984E9 |null -1952-04-19T00:00:00Z |10009 |Sumant |F |1985-02-18T00:00:00.000Z|Peac |4.378998276E9 |null -1953-01-07T00:00:00Z |10067 |Claudi |M |1987-03-04T00:00:00.000Z|Stavenow |null |2.708577936E9 + birth_date |birth_date_day_of_week| emp_no | first_name | gender | hire_date | last_name | 1 | 2 +---------------------+----------------------+---------------+---------------+---------------+---------------------+---------------+---------------+--------------- +null |null |10041 |Uri |F |1989-11-12 00:00:00.0|Lenart |3.182652225E9 |null +null |null |10043 |Yishay |M |1990-10-20 00:00:00.0|Tzvieli |1.179304281E9 |null +null |null |10044 |Mingsen |F |1994-05-21 00:00:00.0|Casley |1.578313984E9 |null +1952-04-19 00:00:00.0|Saturday |10009 |Sumant |F |1985-02-18 00:00:00.0|Peac |4.378998276E9 |null +1953-01-07 00:00:00.0|Wednesday |10067 |Claudi |M |1987-03-04 00:00:00.0|Stavenow |null |2.708577936E9 ; sumWithInnerAggregateSumOfSquaresRound -schema::birth_date:ts|emp_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|1:d|2:d +schema::birth_date:ts|birth_date_day_of_week:s|emp_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|1:d|2:d SELECT * FROM test_emp PIVOT (ROUND(SUM_OF_SQUARES(salary)/1E6, 2) FOR languages IN (1, 2)) LIMIT 5; - birth_date | emp_no | first_name | gender | hire_date | last_name | 1 | 2 ----------------------+---------------+---------------+---------------+------------------------+---------------+---------------+--------------- -null |10041 |Uri |F |1989-11-12T00:00:00.000Z|Lenart |3182.65 |null -null |10043 |Yishay |M |1990-10-20T00:00:00.000Z|Tzvieli |1179.30 |null -null |10044 |Mingsen |F |1994-05-21T00:00:00.000Z|Casley |1578.31 |null -1952-04-19T00:00:00Z |10009 |Sumant |F |1985-02-18T00:00:00.000Z|Peac |4379.00 |null -1953-01-07T00:00:00Z |10067 |Claudi |M |1987-03-04T00:00:00.000Z|Stavenow |null |2708.58 + birth_date |birth_date_day_of_week| emp_no | first_name | gender | hire_date | last_name | 1 | 2 +---------------------+----------------------+---------------+---------------+---------------+---------------------+---------------+---------------+--------------- +null |null |10041 |Uri |F |1989-11-12 00:00:00.0|Lenart |3182.65 |null +null |null |10043 |Yishay |M |1990-10-20 00:00:00.0|Tzvieli |1179.3 |null +null |null |10044 |Mingsen |F |1994-05-21 00:00:00.0|Casley |1578.31 |null +1952-04-19 00:00:00.0|Saturday |10009 |Sumant |F |1985-02-18 00:00:00.0|Peac |4379.0 |null +1953-01-07 00:00:00.0|Wednesday |10067 |Claudi |M |1987-03-04 00:00:00.0|Stavenow |null |2708.58 ; sumWithInnerAggregateKurtosis diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/select.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/select.csv-spec index 32c2f4053725..c03c34dcc10f 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/select.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/select.csv-spec @@ -189,16 +189,16 @@ SELECT emp_no, CAST(languages NOT IN (2, 3) AS STRING), CAST(NOT languages IN (2 ; topWithWildCard -schema::birth_date:ts|emp_no:i|first_name:s|gender:s|hire_date:ts|languages:byte|last_name:s|salary:i +schema::birth_date:ts|birth_date_day_of_week:s|emp_no:i|first_name:s|gender:s|hire_date:ts|languages:byte|last_name:s|salary:i SELECT TOP 5 * FROM test_emp ORDER BY emp_no; - birth_date | emp_no | first_name | gender | hire_date | languages | last_name | salary -------------------------+---------------+---------------+---------------+------------------------+---------------+---------------+--------------- -1953-09-02T00:00:00.000Z|10001 |Georgi |M |1986-06-26T00:00:00.000Z|2 |Facello |57305 -1964-06-02T00:00:00.000Z|10002 |Bezalel |F |1985-11-21T00:00:00.000Z|5 |Simmel |56371 -1959-12-03T00:00:00.000Z|10003 |Parto |M |1986-08-28T00:00:00.000Z|4 |Bamford |61805 -1954-05-01T00:00:00.000Z|10004 |Chirstian |M |1986-12-01T00:00:00.000Z|5 |Koblick |36174 -1955-01-21T00:00:00.000Z|10005 |Kyoichi |M |1989-09-12T00:00:00.000Z|1 |Maliniak |63528 + birth_date |birth_date_day_of_week| emp_no | first_name | gender | hire_date | languages | last_name | salary +------------------------+----------------------+---------------+---------------+---------------+------------------------+---------------+---------------+--------------- +1953-09-02T00:00:00.000Z|Wednesday |10001 |Georgi |M |1986-06-26T00:00:00.000Z|2 |Facello |57305 +1964-06-02T00:00:00.000Z|Tuesday |10002 |Bezalel |F |1985-11-21T00:00:00.000Z|5 |Simmel |56371 +1959-12-03T00:00:00.000Z|Thursday |10003 |Parto |M |1986-08-28T00:00:00.000Z|4 |Bamford |61805 +1954-05-01T00:00:00.000Z|Saturday |10004 |Chirstian |M |1986-12-01T00:00:00.000Z|5 |Koblick |36174 +1955-01-21T00:00:00.000Z|Friday |10005 |Kyoichi |M |1989-09-12T00:00:00.000Z|1 |Maliniak |63528 ; topWithColumnNames @@ -225,3 +225,25 @@ SELECT TOP 3 count(*) AS cnt, emp_no % languages FROM test_emp GROUP BY 2 ORDER 24 |1 16 |2 ; + +runtimeFieldWithFunctions +SELECT emp_no, CONCAT(CONCAT(SUBSTRING(birth_date_day_of_week, 0, LENGTH(birth_date_day_of_week) - 3), SPACE(11 - LENGTH(birth_date_day_of_week))), 'DAY') AS c, ISO_DAY_OF_WEEK(birth_date) AS iso FROM test_emp ORDER BY emp_no DESC LIMIT 15; + + emp_no | c | iso +---------------+---------------+--------------- +10100 |Tues DAY |2 +10099 |Fri DAY |5 +10098 |Satur DAY |6 +10097 |Wednes DAY |3 +10096 |Thurs DAY |4 +10095 |Sun DAY |7 +10094 |Satur DAY |6 +10093 |Thurs DAY |4 +10092 |Sun DAY |7 +10091 |Tues DAY |2 +10090 |Tues DAY |2 +10089 |Thurs DAY |4 +10088 |Thurs DAY |4 +10087 |Thurs DAY |4 +10086 |Mon DAY |1 +; \ No newline at end of file diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/setup_test_emp.sql b/x-pack/plugin/sql/qa/server/src/main/resources/setup_test_emp.sql index 3b79b3037f11..bfaa2376230b 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/setup_test_emp.sql +++ b/x-pack/plugin/sql/qa/server/src/main/resources/setup_test_emp.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS "test_emp"; CREATE TABLE "test_emp" ( "birth_date" TIMESTAMP WITH TIME ZONE, + "birth_date_day_of_week" VARCHAR(50), "emp_no" INT, "first_name" VARCHAR(50), "gender" VARCHAR(1), @@ -9,4 +10,4 @@ CREATE TABLE "test_emp" ( "last_name" VARCHAR(50), "salary" INT ) - AS SELECT * FROM CSVREAD('classpath:/employees.csv'); \ No newline at end of file + AS SELECT birth_date, DAYNAME(birth_date) AS birth_date_day_of_week, emp_no, first_name, gender, hire_date, languages, last_name, salary FROM CSVREAD('classpath:/employees.csv'); \ No newline at end of file diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec index 95d3d46c4564..da45d6093051 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec @@ -7,18 +7,19 @@ sysColumnsWithTableLikeWithEscape SYS COLUMNS TABLE LIKE 'test\_emp' ESCAPE '\'; - TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ----------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +-----------------+---------------+---------------+----------------------+---------------+---------------+---------------+---------------+-----------------+-----------------+-----------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +integTest |null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date_day_of_week|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |2 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |12 |YES |null |null |null |null |NO |NO ; sysColumnsWithTableLikeNoEscape @@ -29,21 +30,22 @@ SYS COLUMNS TABLE LIKE 'test_emp'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date_day_of_week|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |2 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; sysColumnsWithCatalogAndLike @@ -51,21 +53,22 @@ SYS COLUMNS CATALOG 'integTest' TABLE LIKE 'test\_emp\_copy' ESCAPE '\'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ---------------+---------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |test_emp_copy|birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date_day_of_week|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |2 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; sysColumnsOnAliasWithTableLike @@ -73,21 +76,22 @@ SYS COLUMNS TABLE LIKE 'test\_alias' ESCAPE '\'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |test_alias |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date_day_of_week|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |2 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; sysColumnsAllTables @@ -95,70 +99,74 @@ SYS COLUMNS TABLE LIKE '%'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i|BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s -----------------+---------------+---------------+------------------+---------------+----------------+---------------+---------------+----------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |logs |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO -integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO -integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO -integTest |null |logs_nanos |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |logs_nanos |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -integTest |null |logs_nanos |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO +integTest |null |logs |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO +integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO +integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |logs_nanos |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs_nanos |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs_nanos |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date_day_of_week|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |2 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |birth_date_day_of_week|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |2 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date_day_of_week|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |2 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date_day_of_week|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |2 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle index bf362745e1b9..f9db4e702611 100644 --- a/x-pack/plugin/sql/sql-action/build.gradle +++ b/x-pack/plugin/sql/sql-action/build.gradle @@ -26,6 +26,7 @@ dependencies { runtimeOnly "org.apache.logging.log4j:log4j-core:${versions.log4j}" testImplementation project(":test:framework") + testImplementation(project(xpackModule('ql:test-fixtures'))) } tasks.named('forbiddenApisMain').configure { diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java index 20e6f0ce34f4..f6032674c6c4 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.sql.action; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.common.Nullable; @@ -23,6 +24,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.Protocol; import org.elasticsearch.xpack.sql.proto.RequestInfo; @@ -32,15 +34,16 @@ import org.elasticsearch.xpack.sql.proto.SqlVersion; import java.io.IOException; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Supplier; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static org.elasticsearch.Version.CURRENT; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.sql.proto.Protocol.CLIENT_ID_NAME; -import static org.elasticsearch.xpack.sql.proto.Protocol.VERSION_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.CURSOR_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.FETCH_SIZE_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.FILTER_NAME; @@ -50,6 +53,7 @@ import static org.elasticsearch.xpack.sql.proto.Protocol.PARAMS_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.QUERY_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.REQUEST_TIMEOUT_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.TIME_ZONE_NAME; +import static org.elasticsearch.xpack.sql.proto.Protocol.VERSION_NAME; /** * Base class for requests that contain sql queries (Query and Translate) @@ -63,7 +67,8 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme private TimeValue pageTimeout = Protocol.PAGE_TIMEOUT; @Nullable private QueryBuilder filter = null; - private List params = Collections.emptyList(); + private List params = emptyList(); + private Map runtimeMappings = emptyMap(); static final ParseField QUERY = new ParseField(QUERY_NAME); static final ParseField CURSOR = new ParseField(CURSOR_NAME); @@ -81,8 +86,8 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme super(); } - public AbstractSqlQueryRequest(String query, List params, QueryBuilder filter, ZoneId zoneId, - int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) { + public AbstractSqlQueryRequest(String query, List params, QueryBuilder filter, Map runtimeMappings, + ZoneId zoneId, int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) { super(requestInfo); this.query = query; this.params = params; @@ -91,6 +96,7 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme this.requestTimeout = requestTimeout; this.pageTimeout = pageTimeout; this.filter = filter; + this.runtimeMappings = runtimeMappings; } protected static ObjectParser objectParser(Supplier supplier) { @@ -109,8 +115,8 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme parser.declareString( (request, timeout) -> request.pageTimeout(TimeValue.parseTimeValue(timeout, Protocol.PAGE_TIMEOUT, PAGE_TIMEOUT_NAME)), PAGE_TIMEOUT); - parser.declareObject(AbstractSqlQueryRequest::filter, - (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), FILTER); + parser.declareObject(AbstractSqlQueryRequest::filter, (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), FILTER); + parser.declareObject(AbstractSqlQueryRequest::runtimeMappings, (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD); return parser; } @@ -242,6 +248,29 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme validationException); } } + if (runtimeMappings != null) { + // Perform a superficial check for runtime_mappings structure. The full check cannot happen until the actual search request + validationException = validateRuntimeMappings(runtimeMappings, validationException); + } + return validationException; + } + + private static ActionRequestValidationException validateRuntimeMappings(Map runtimeMappings, + ActionRequestValidationException validationException) { + for (Map.Entry entry : runtimeMappings.entrySet()) { + // top level objects are fields + String fieldName = entry.getKey(); + if (entry.getValue() instanceof Map) { + @SuppressWarnings("unchecked") + Map propNode = (Map) entry.getValue(); + if (propNode.get("type") == null) { + return addValidationError("No type specified for runtime field [" + fieldName + "]", validationException); + } + } else { + return addValidationError("Expected map for runtime field [" + fieldName + "] definition but got [" + + fieldName.getClass().getSimpleName() + "]", validationException); + } + } return validationException; } @@ -317,6 +346,18 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme return filter; } + public Map runtimeMappings() { + return runtimeMappings; + } + + /** + * An optional list of runtime fields that can be added to the request similar to the way it is done in Query DSL search requests + */ + public AbstractSqlQueryRequest runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + return this; + } + public AbstractSqlQueryRequest(StreamInput in) throws IOException { super(in); query = in.readString(); @@ -326,6 +367,9 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme requestTimeout = in.readTimeValue(); pageTimeout = in.readTimeValue(); filter = in.readOptionalNamedWriteable(QueryBuilder.class); + if (in.getVersion().onOrAfter(Version.V_7_13_0)) { + runtimeMappings = in.readMap(); + } } public static void writeSqlTypedParamValue(StreamOutput out, SqlTypedParamValue value) throws IOException { @@ -351,6 +395,9 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme out.writeTimeValue(requestTimeout); out.writeTimeValue(pageTimeout); out.writeOptionalNamedWriteable(filter); + if (out.getVersion().onOrAfter(Version.V_7_13_0)) { + out.writeMap(runtimeMappings); + } } @Override @@ -371,11 +418,12 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme Objects.equals(zoneId, that.zoneId) && Objects.equals(requestTimeout, that.requestTimeout) && Objects.equals(pageTimeout, that.pageTimeout) && - Objects.equals(filter, that.filter); + Objects.equals(filter, that.filter) && + Objects.equals(runtimeMappings, that.runtimeMappings); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), query, params, zoneId, fetchSize, requestTimeout, pageTimeout, filter); + return Objects.hash(super.hashCode(), query, params, zoneId, fetchSize, requestTimeout, pageTimeout, filter, runtimeMappings); } } diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java index b1064ba9dee8..d0d29b1f6d9d 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import java.io.IOException; import java.time.ZoneId; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -65,10 +66,10 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest { super(); } - public SqlQueryRequest(String query, List params, QueryBuilder filter, ZoneId zoneId, - int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, Boolean columnar, + public SqlQueryRequest(String query, List params, QueryBuilder filter, Map runtimeMappings, + ZoneId zoneId, int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, Boolean columnar, String cursor, RequestInfo requestInfo, boolean fieldMultiValueLeniency, boolean indexIncludeFrozen) { - super(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, requestInfo); + super(query, params, filter, runtimeMappings, zoneId, fetchSize, requestTimeout, pageTimeout, requestInfo); this.cursor = cursor; this.columnar = columnar; this.fieldMultiValueLeniency = fieldMultiValueLeniency; @@ -189,7 +190,7 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest { // This is needed just to test round-trip compatibility with proto.SqlQueryRequest return new org.elasticsearch.xpack.sql.proto.SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(), pageTimeout(), filter(), columnar(), cursor(), requestInfo(), fieldMultiValueLeniency(), indexIncludeFrozen(), - binaryCommunication()).toXContent(builder, params); + binaryCommunication(), runtimeMappings()).toXContent(builder, params); } public static SqlQueryRequest fromXContent(XContentParser parser) { diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java index 47573ba403ce..0398e2109176 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java @@ -16,8 +16,11 @@ import org.elasticsearch.xpack.sql.proto.RequestInfo; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import java.time.ZoneId; -import java.util.Collections; import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; /** * The builder to build sql request @@ -25,17 +28,17 @@ import java.util.List; public class SqlQueryRequestBuilder extends ActionRequestBuilder { public SqlQueryRequestBuilder(ElasticsearchClient client, SqlQueryAction action) { - this(client, action, "", Collections.emptyList(), null, Protocol.TIME_ZONE, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, false, "", new RequestInfo(Mode.PLAIN), Protocol.FIELD_MULTI_VALUE_LENIENCY, + this(client, action, "", emptyList(), null, emptyMap(), Protocol.TIME_ZONE, Protocol.FETCH_SIZE, + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, false, "", new RequestInfo(Mode.PLAIN), Protocol.FIELD_MULTI_VALUE_LENIENCY, Protocol.INDEX_INCLUDE_FROZEN); } public SqlQueryRequestBuilder(ElasticsearchClient client, SqlQueryAction action, String query, List params, - QueryBuilder filter, ZoneId zoneId, int fetchSize, TimeValue requestTimeout, + QueryBuilder filter, Map runtimeMappings, ZoneId zoneId, int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, boolean columnar, String nextPageInfo, RequestInfo requestInfo, boolean multiValueFieldLeniency, boolean indexIncludeFrozen) { - super(client, action, new SqlQueryRequest(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, columnar, - nextPageInfo, requestInfo, multiValueFieldLeniency, indexIncludeFrozen)); + super(client, action, new SqlQueryRequest(query, params, filter, runtimeMappings, zoneId, fetchSize, requestTimeout, pageTimeout, + columnar, nextPageInfo, requestInfo, multiValueFieldLeniency, indexIncludeFrozen)); } public SqlQueryRequestBuilder query(String query) { @@ -68,6 +71,11 @@ public class SqlQueryRequestBuilder extends ActionRequestBuilder runtimeMappings) { + request.runtimeMappings(runtimeMappings); + return this; + } + public SqlQueryRequestBuilder zoneId(ZoneId zoneId) { request.zoneId(zoneId); return this; diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java index 7b705f6090df..90456eff075f 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import java.io.IOException; import java.time.ZoneId; import java.util.List; +import java.util.Map; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -34,9 +35,9 @@ public class SqlTranslateRequest extends AbstractSqlQueryRequest { super(); } - public SqlTranslateRequest(String query, List params, QueryBuilder filter, ZoneId zoneId, - int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) { - super(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, requestInfo); + public SqlTranslateRequest(String query, List params, QueryBuilder filter, Map runtimeMappings, + ZoneId zoneId, int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) { + super(query, params, filter, runtimeMappings, zoneId, fetchSize, requestTimeout, pageTimeout, requestInfo); } public SqlTranslateRequest(StreamInput in) throws IOException { @@ -73,6 +74,7 @@ public class SqlTranslateRequest extends AbstractSqlQueryRequest { requestInfo(), false, false, - null).toXContent(builder, params); + null, + runtimeMappings()).toXContent(builder, params); } } diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestBuilder.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestBuilder.java index cec0aa08342a..d1dfa8e0e47b 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestBuilder.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestBuilder.java @@ -16,23 +16,26 @@ import org.elasticsearch.xpack.sql.proto.RequestInfo; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import java.time.ZoneId; -import java.util.Collections; import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; /** * Builder for the request for the sql action for translating SQL queries into ES requests */ public class SqlTranslateRequestBuilder extends ActionRequestBuilder { public SqlTranslateRequestBuilder(ElasticsearchClient client, SqlTranslateAction action) { - this(client, action, null, null, Collections.emptyList(), Protocol.TIME_ZONE, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, new RequestInfo(Mode.PLAIN)); + this(client, action, null, null, emptyMap(), emptyList(), Protocol.TIME_ZONE, Protocol.FETCH_SIZE, + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, new RequestInfo(Mode.PLAIN)); } public SqlTranslateRequestBuilder(ElasticsearchClient client, SqlTranslateAction action, String query, QueryBuilder filter, - List params, ZoneId zoneId, int fetchSize, TimeValue requestTimeout, - TimeValue pageTimeout, RequestInfo requestInfo) { - super(client, action, - new SqlTranslateRequest(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, requestInfo)); + Map runtimeMappings, List params, ZoneId zoneId, int fetchSize, TimeValue requestTimeout, + TimeValue pageTimeout, RequestInfo requestInfo) { + super(client, action, new SqlTranslateRequest(query, params, filter, runtimeMappings, zoneId, fetchSize, requestTimeout, + pageTimeout, requestInfo)); } public SqlTranslateRequestBuilder query(String query) { diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java index 05c50375827c..2814105d9270 100644 --- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java +++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java @@ -32,6 +32,7 @@ import java.util.function.Consumer; import java.util.function.Supplier; import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; +import static org.elasticsearch.xpack.ql.TestUtils.randomRuntimeMappings; import static org.elasticsearch.xpack.sql.action.SqlTestUtils.randomFilter; import static org.elasticsearch.xpack.sql.action.SqlTestUtils.randomFilterOrNull; import static org.elasticsearch.xpack.sql.proto.Protocol.BINARY_FORMAT_NAME; @@ -49,6 +50,7 @@ import static org.elasticsearch.xpack.sql.proto.Protocol.PARAMS_TYPE_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.PARAMS_VALUE_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.QUERY_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.REQUEST_TIMEOUT_NAME; +import static org.elasticsearch.xpack.sql.proto.Protocol.RUNTIME_MAPPINGS_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.TIME_ZONE_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.VERSION_NAME; import static org.elasticsearch.xpack.sql.proto.RequestInfo.CLIENT_IDS; @@ -78,7 +80,7 @@ public class SqlQueryRequestTests extends AbstractWireSerializingTestCase request.columnar(randomValueOtherThan(request.columnar(), () -> randomBoolean())), request -> request.cursor(randomValueOtherThan(request.cursor(), SqlQueryResponseTests::randomStringCursor)) ); - SqlQueryRequest newRequest = new SqlQueryRequest(instance.query(), instance.params(), instance.filter(), + SqlQueryRequest newRequest = new SqlQueryRequest(instance.query(), instance.params(), instance.filter(), instance.runtimeMappings(), instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), instance.columnar(), instance.cursor(), instance.requestInfo(), instance.fieldMultiValueLeniency(), instance.indexIncludeFrozen()); mutator.accept(newRequest); @@ -242,6 +244,9 @@ public class SqlQueryRequestTests extends AbstractWireSerializingTestCase request.fetchSize(randomValueOtherThan(request.fetchSize(), () -> between(1, Integer.MAX_VALUE))), request -> request.requestTimeout(randomValueOtherThan(request.requestTimeout(), this::randomTV)), request -> request.filter(randomValueOtherThan(request.filter(), - () -> request.filter() == null ? randomFilter(random()) : randomFilterOrNull(random()))) + () -> request.filter() == null ? randomFilter(random()) : randomFilterOrNull(random()))), + request -> request.runtimeMappings(randomValueOtherThan(request.runtimeMappings(), () -> randomRuntimeMappings())) ); SqlTranslateRequest newRequest = new SqlTranslateRequest(instance.query(), instance.params(), instance.filter(), - instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), instance.requestInfo()); + instance.runtimeMappings(), instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), + instance.requestInfo()); mutator.accept(newRequest); return newRequest; } diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java index eed15fbf27cf..4ce5d1f979f7 100644 --- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java +++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java @@ -33,9 +33,11 @@ import java.io.InputStream; import java.security.AccessController; import java.security.PrivilegedAction; import java.sql.SQLException; -import java.util.Collections; import java.util.function.Function; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + /** * A specialized high-level REST client with support for SQL-related functions. * Similar to JDBC and the underlying HTTP connection, this class is not thread-safe @@ -64,7 +66,7 @@ public class HttpClient { public SqlQueryResponse basicQuery(String query, int fetchSize) throws SQLException { // TODO allow customizing the time zone - this is what session set/reset/get should be about // method called only from CLI - SqlQueryRequest sqlRequest = new SqlQueryRequest(query, Collections.emptyList(), Protocol.TIME_ZONE, + SqlQueryRequest sqlRequest = new SqlQueryRequest(query, emptyList(), Protocol.TIME_ZONE, fetchSize, TimeValue.timeValueMillis(cfg.queryTimeout()), TimeValue.timeValueMillis(cfg.pageTimeout()), @@ -74,7 +76,8 @@ public class HttpClient { new RequestInfo(Mode.CLI, ClientVersion.CURRENT), false, false, - cfg.binaryCommunication()); + cfg.binaryCommunication(), + emptyMap()); return query(sqlRequest); } diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java index df4b1d7e6925..71847fedbfe2 100644 --- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java +++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java @@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; @@ -171,7 +172,8 @@ public class HttpClientRequestTests extends ESTestCase { new RequestInfo(mode, ClientVersion.CURRENT), randomBoolean(), randomBoolean(), - isBinary); + isBinary, + Collections.emptyMap()); prepareMockResponse(); try { diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Protocol.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Protocol.java index 6ed0720b5c42..adc69bafd97c 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Protocol.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Protocol.java @@ -33,6 +33,7 @@ public final class Protocol { public static final String BINARY_FORMAT_NAME = "binary_format"; public static final String FIELD_MULTI_VALUE_LENIENCY_NAME = "field_multi_value_leniency"; public static final String INDEX_INCLUDE_FROZEN_NAME = "index_include_frozen"; + public static final String RUNTIME_MAPPINGS_NAME = "runtime_mappings"; // params public static final String PARAMS_NAME = "params"; public static final String PARAMS_TYPE_NAME = "type"; diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java index 9f26f471a904..fb9dae502669 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java @@ -14,10 +14,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.time.ZoneId; -import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.sql.proto.Protocol.BINARY_FORMAT_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.CLIENT_ID_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.COLUMNAR_NAME; @@ -31,6 +33,7 @@ import static org.elasticsearch.xpack.sql.proto.Protocol.PAGE_TIMEOUT_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.PARAMS_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.QUERY_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.REQUEST_TIMEOUT_NAME; +import static org.elasticsearch.xpack.sql.proto.Protocol.RUNTIME_MAPPINGS_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.TIME_ZONE_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.VERSION_NAME; @@ -52,11 +55,13 @@ public class SqlQueryRequest extends AbstractSqlRequest { private final boolean fieldMultiValueLeniency; private final boolean indexIncludeFrozen; private final Boolean binaryCommunication; + @Nullable + private final Map runtimeMappings; public SqlQueryRequest(String query, List params, ZoneId zoneId, int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, ToXContent filter, Boolean columnar, String cursor, RequestInfo requestInfo, boolean fieldMultiValueLeniency, boolean indexIncludeFrozen, - Boolean binaryCommunication) { + Boolean binaryCommunication, Map runtimeMappings) { super(requestInfo); this.query = query; this.params = params; @@ -70,12 +75,13 @@ public class SqlQueryRequest extends AbstractSqlRequest { this.fieldMultiValueLeniency = fieldMultiValueLeniency; this.indexIncludeFrozen = indexIncludeFrozen; this.binaryCommunication = binaryCommunication; + this.runtimeMappings = runtimeMappings; } public SqlQueryRequest(String cursor, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo, boolean binaryCommunication) { - this("", Collections.emptyList(), Protocol.TIME_ZONE, Protocol.FETCH_SIZE, requestTimeout, pageTimeout, - null, false, cursor, requestInfo, Protocol.FIELD_MULTI_VALUE_LENIENCY, Protocol.INDEX_INCLUDE_FROZEN, binaryCommunication); + this("", emptyList(), Protocol.TIME_ZONE, Protocol.FETCH_SIZE, requestTimeout, pageTimeout, null, false, + cursor, requestInfo, Protocol.FIELD_MULTI_VALUE_LENIENCY, Protocol.INDEX_INCLUDE_FROZEN, binaryCommunication, emptyMap()); } /** @@ -156,6 +162,10 @@ public class SqlQueryRequest extends AbstractSqlRequest { return binaryCommunication; } + public Map runtimeMappings() { + return runtimeMappings; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -179,13 +189,14 @@ public class SqlQueryRequest extends AbstractSqlRequest { && Objects.equals(cursor, that.cursor) && fieldMultiValueLeniency == that.fieldMultiValueLeniency && indexIncludeFrozen == that.indexIncludeFrozen - && Objects.equals(binaryCommunication, that.binaryCommunication); + && Objects.equals(binaryCommunication, that.binaryCommunication) + && Objects.equals(runtimeMappings, that.runtimeMappings); } @Override public int hashCode() { return Objects.hash(super.hashCode(), query, zoneId, fetchSize, requestTimeout, pageTimeout, - filter, columnar, cursor, fieldMultiValueLeniency, indexIncludeFrozen, binaryCommunication); + filter, columnar, cursor, fieldMultiValueLeniency, indexIncludeFrozen, binaryCommunication, runtimeMappings); } @Override @@ -238,6 +249,9 @@ public class SqlQueryRequest extends AbstractSqlRequest { if (cursor != null) { builder.field(CURSOR_NAME, cursor); } + if (runtimeMappings.isEmpty() == false) { + builder.field(RUNTIME_MAPPINGS_NAME, runtimeMappings); + } return builder; } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java index 69eb2f08a1db..9f78301ced37 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java @@ -116,6 +116,11 @@ public class Querier { sourceBuilder.timeout(timeout); } + // set runtime mappings + if (this.cfg.runtimeMappings() != null) { + sourceBuilder.runtimeMappings(this.cfg.runtimeMappings()); + } + if (log.isTraceEnabled()) { log.trace("About to execute query {} on {}", StringUtils.toString(sourceBuilder), index); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index 6f91e944e0bf..d6a59055e15c 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -27,6 +27,7 @@ import java.util.Objects; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; public class ShowColumns extends Command { @@ -67,7 +68,7 @@ public class ShowColumns extends Command { String regex = pattern != null ? pattern.asJavaRegex() : null; boolean withFrozen = includeFrozen || session.configuration().includeFrozen(); - session.indexResolver().resolveAsMergedMapping(idx, regex, withFrozen, ActionListener.wrap( + session.indexResolver().resolveAsMergedMapping(idx, regex, withFrozen, emptyMap(), ActionListener.wrap( indexResult -> { List> rows = emptyList(); if (indexResult.isValid()) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 10fc96a1bc93..039df83ee419 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -32,6 +32,7 @@ import java.util.Objects; import java.util.regex.Pattern; import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.ql.type.DataTypes.BINARY; import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED; @@ -129,7 +130,7 @@ public class SysColumns extends Command { // special case for '%' (translated to *) if ("*".equals(idx)) { - session.indexResolver().resolveAsSeparateMappings(idx, regex, includeFrozen, + session.indexResolver().resolveAsSeparateMappings(idx, regex, includeFrozen, emptyMap(), ActionListener.wrap(esIndices -> { List> rows = new ArrayList<>(); for (EsIndex esIndex : esIndices) { @@ -140,7 +141,7 @@ public class SysColumns extends Command { } // otherwise use a merged mapping else { - session.indexResolver().resolveAsMergedMapping(idx, regex, includeFrozen, + session.indexResolver().resolveAsMergedMapping(idx, regex, includeFrozen, emptyMap(), ActionListener.wrap(r -> { List> rows = new ArrayList<>(); // populate the data only when a target is found diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java index 183515aa023a..10ab94c0ec9b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java @@ -17,11 +17,12 @@ import org.elasticsearch.xpack.sql.action.SqlClearCursorRequest; import org.elasticsearch.xpack.sql.action.SqlClearCursorResponse; import org.elasticsearch.xpack.sql.execution.PlanExecutor; import org.elasticsearch.xpack.sql.proto.Protocol; -import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.session.Cursor; import org.elasticsearch.xpack.sql.session.Cursors; +import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.util.DateUtils; +import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.sql.action.SqlClearCursorAction.NAME; public class TransportSqlClearCursorAction extends HandledTransportAction { @@ -47,7 +48,7 @@ public class TransportSqlClearCursorAction extends HandledTransportAction listener.onResponse(new SqlClearCursorResponse(success)), listener::onFailure)); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java index e2f067ed95f3..f26f8783a479 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java @@ -88,7 +88,7 @@ public class TransportSqlQueryAction extends HandledTransportAction runtimeMappings; + public SqlConfiguration(ZoneId zi, int pageSize, TimeValue requestTimeout, TimeValue pageTimeout, QueryBuilder filter, + Map runtimeMappings, Mode mode, String clientId, SqlVersion version, String username, String clusterName, boolean multiValueFieldLeniency, @@ -42,6 +47,7 @@ public class SqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu this.requestTimeout = requestTimeout; this.pageTimeout = pageTimeout; this.filter = filter; + this.runtimeMappings = runtimeMappings; this.mode = mode == null ? Mode.PLAIN : mode; this.clientId = clientId; this.version = version != null ? version : SqlVersion.fromId(Version.CURRENT.id); @@ -64,6 +70,11 @@ public class SqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu public QueryBuilder filter() { return filter; } + + public Map runtimeMappings() { + return runtimeMappings; + } + public Mode mode() { return mode; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index 23adbddff80b..7fa9069fa244 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -141,7 +141,7 @@ public class SqlSession implements Session { } boolean includeFrozen = configuration.includeFrozen() || tableInfo.isFrozen(); - indexResolver.resolveAsMergedMapping(table.index(), null, includeFrozen, + indexResolver.resolveAsMergedMapping(table.index(), null, includeFrozen, configuration.runtimeMappings(), wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure)); } else { try { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java index 5e2d17d7d283..84a9270b266f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java @@ -39,7 +39,7 @@ public final class SqlTestUtils { private SqlTestUtils() {} public static final SqlConfiguration TEST_CFG = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, - Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, null, Mode.PLAIN, null, null, null, null, false, false); public static SqlConfiguration randomConfiguration() { @@ -48,6 +48,7 @@ public final class SqlTestUtils { new TimeValue(randomNonNegativeLong()), new TimeValue(randomNonNegativeLong()), null, + null, randomFrom(Mode.values()), randomAlphaOfLength(10), null, @@ -63,6 +64,7 @@ public final class SqlTestUtils { new TimeValue(randomNonNegativeLong()), new TimeValue(randomNonNegativeLong()), null, + null, randomFrom(Mode.values()), randomAlphaOfLength(10), null, @@ -78,6 +80,7 @@ public final class SqlTestUtils { new TimeValue(randomNonNegativeLong()), new TimeValue(randomNonNegativeLong()), null, + null, randomFrom(Mode.values()), randomAlphaOfLength(10), version, diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java index a517ea98d7e5..fb15cf81ebc8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java @@ -31,7 +31,7 @@ public class DatabaseFunctionTests extends ESTestCase { SqlParser parser = new SqlParser(); EsIndex test = new EsIndex("test", SqlTypesTests.loadMapping("mapping-basic.json", true)); SqlConfiguration sqlConfig = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, + Protocol.PAGE_TIMEOUT, null, null, randomFrom(Mode.values()), randomAlphaOfLength(10), null, null, clusterName, randomBoolean(), randomBoolean()); Analyzer analyzer = new Analyzer( diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java index a4ff40f177a1..adddfeb7d2b6 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java @@ -30,7 +30,7 @@ public class UserFunctionTests extends ESTestCase { SqlParser parser = new SqlParser(); EsIndex test = new EsIndex("test", SqlTypesTests.loadMapping("mapping-basic.json", true)); SqlConfiguration sqlConfig = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, + Protocol.PAGE_TIMEOUT, null, null, randomFrom(Mode.values()), randomAlphaOfLength(10), null, null, randomAlphaOfLengthBetween(1, 15), randomBoolean(), randomBoolean()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index 3ef7dae4f7c7..278e53cb0d66 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -231,7 +231,7 @@ public class SysColumnsTests extends ESTestCase { private int executeCommandInOdbcModeAndCountRows(String sql) { final SqlConfiguration config = new SqlConfiguration(DateUtils.UTC, randomIntBetween(1, 15), Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, Mode.ODBC, null, SqlVersion.fromId(Version.CURRENT.id), null, null, false, false); + Protocol.PAGE_TIMEOUT, null, null, Mode.ODBC, null, SqlVersion.fromId(Version.CURRENT.id), null, null, false, false); Tuple tuple = sql(sql, emptyList(), config, MAPPING1); int[] rowCount = {0}; @@ -256,7 +256,7 @@ public class SysColumnsTests extends ESTestCase { private void executeCommand(String sql, List params, Mode mode, Consumer consumer, Map mapping) { final SqlConfiguration config = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, mode, null, SqlVersion.fromId(Version.CURRENT.id), null, null, false, false); + Protocol.PAGE_TIMEOUT, null, null, mode, null, SqlVersion.fromId(Version.CURRENT.id), null, null, false, false); Tuple tuple = sql(sql, params, config, mapping); tuple.v1().execute(tuple.v2(), wrap(p -> consumer.accept((SchemaRowSet) p.rowSet()), ex -> fail(ex.getMessage()))); @@ -277,13 +277,13 @@ public class SysColumnsTests extends ESTestCase { IndexResolver resolver = mock(IndexResolver.class); when(resolver.clusterName()).thenReturn(CLUSTER_NAME); doAnswer(invocation -> { - ((ActionListener) invocation.getArguments()[3]).onResponse(IndexResolution.valid(test)); + ((ActionListener) invocation.getArguments()[4]).onResponse(IndexResolution.valid(test)); return Void.TYPE; - }).when(resolver).resolveAsMergedMapping(any(), any(), anyBoolean(), any()); + }).when(resolver).resolveAsMergedMapping(any(), any(), anyBoolean(), any(), any()); doAnswer(invocation -> { - ((ActionListener>) invocation.getArguments()[3]).onResponse(singletonList(test)); + ((ActionListener>) invocation.getArguments()[4]).onResponse(singletonList(test)); return Void.TYPE; - }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any()); + }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any(), any()); SqlSession session = new SqlSession(config, null, null, resolver, null, null, null, null, null); return new Tuple<>(cmd, session); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index 0cb38a2ae531..9668cd899dc7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -60,7 +60,7 @@ public class SysTablesTests extends ESTestCase { private final IndexInfo frozen = new IndexInfo("frozen", IndexType.FROZEN_INDEX); private final SqlConfiguration FROZEN_CFG = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null, null, false, true); + Protocol.PAGE_TIMEOUT, null, null, Mode.PLAIN, null, null, null, null, false, true); // // catalog enumeration diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index 3f032c394de3..da3420c2dad7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -40,7 +40,7 @@ public class SysTypesTests extends ESTestCase { private Tuple sql(String sql, Mode mode, SqlVersion version) { SqlConfiguration configuration = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, - Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, mode, null, version, null, null, false, false); + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, null, mode, null, version, null, null, false, false); EsIndex test = new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json", true)); Analyzer analyzer = new Analyzer(configuration, new FunctionRegistry(), IndexResolution.valid(test), null); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql), false); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlMediaTypeParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlMediaTypeParserTests.java index e6243a9bae84..3ac2b705778f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlMediaTypeParserTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlMediaTypeParserTests.java @@ -86,7 +86,7 @@ public class SqlMediaTypeParserTests extends ESTestCase { } protected SqlQueryRequest createTestInstance(boolean binaryCommunication, Mode mode, boolean columnar) { - return new SqlQueryRequest(randomAlphaOfLength(10), Collections.emptyList(), null, + return new SqlQueryRequest(randomAlphaOfLength(10), Collections.emptyList(), null, null, randomZone(), between(1, Integer.MAX_VALUE), TimeValue.parseTimeValue(randomTimeValue(), null, "test"), TimeValue.parseTimeValue(randomTimeValue(), null, "test"), columnar, randomAlphaOfLength(10), new RequestInfo(mode, randomFrom(randomFrom(CLIENT_IDS), randomAlphaOfLengthBetween(10, 20))),