From 23e4db85580f966d3a41fe2c8f0fb628707c9297 Mon Sep 17 00:00:00 2001 From: Mikhail Berezovskiy Date: Tue, 11 Feb 2025 19:29:00 -0800 Subject: [PATCH 01/93] Unmute #122104 & #122103 --- muted-tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 06e913bd8631..b6401c11ca8c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -372,12 +372,6 @@ tests: - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously} issue: https://github.com/elastic/elasticsearch/issues/122102 -- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT - method: test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates} - issue: https://github.com/elastic/elasticsearch/issues/122103 -- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT - method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously} - issue: https://github.com/elastic/elasticsearch/issues/122104 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT method: testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsBoolean issue: https://github.com/elastic/elasticsearch/issues/121680 From e44e5a3b925cc48f970f2831548ea10fdc05a897 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 12 Feb 2025 16:36:36 +1100 Subject: [PATCH 02/93] Mute org.elasticsearch.xpack.searchablesnapshots.hdfs.HdfsSearchableSnapshotsIT org.elasticsearch.xpack.searchablesnapshots.hdfs.HdfsSearchableSnapshotsIT #122024 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index b6401c11ca8c..3829cda8cafa 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -412,6 +412,8 @@ tests: - class: org.elasticsearch.blocks.SimpleBlocksIT method: testConcurrentAddBlock issue: https://github.com/elastic/elasticsearch/issues/122324 +- class: org.elasticsearch.xpack.searchablesnapshots.hdfs.HdfsSearchableSnapshotsIT + issue: https://github.com/elastic/elasticsearch/issues/122024 # Examples: # From 6d25cbfd7a64749bffa89464e3a5b9edcb6a858e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 12 Feb 2025 17:08:45 +1100 Subject: [PATCH 03/93] Mute org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT test {yaml=reference/cat/health/cat-health-example} #122335 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3829cda8cafa..be974ca2860f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -414,6 +414,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122324 - class: org.elasticsearch.xpack.searchablesnapshots.hdfs.HdfsSearchableSnapshotsIT issue: https://github.com/elastic/elasticsearch/issues/122024 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/cat/health/cat-health-example} + issue: https://github.com/elastic/elasticsearch/issues/122335 # Examples: # From 9112aba6018dbb4bb0feee7460b8aff34e88846c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 12 Feb 2025 17:41:43 +1100 Subject: [PATCH 04/93] Mute org.elasticsearch.xpack.esql.action.CrossClusterCancellationIT testCloseSkipUnavailable #122336 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index be974ca2860f..0090b6fe9008 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -417,6 +417,9 @@ tests: - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/cat/health/cat-health-example} issue: https://github.com/elastic/elasticsearch/issues/122335 +- class: org.elasticsearch.xpack.esql.action.CrossClusterCancellationIT + method: testCloseSkipUnavailable + issue: https://github.com/elastic/elasticsearch/issues/122336 # Examples: # From 2b54de1e4bad8db288f487eca7988bff463ff28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Wed, 12 Feb 2025 08:14:36 +0100 Subject: [PATCH 05/93] Skip SM policy parsing and validation for Java24+ (#122233) --- .../plugins/cli/InstallPluginAction.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java index f4601d70a7f0..0803d24c3914 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.jdk.JarHell; +import org.elasticsearch.jdk.RuntimeVersionFeature; import org.elasticsearch.plugin.scanner.ClassReaders; import org.elasticsearch.plugin.scanner.NamedComponentScanner; import org.elasticsearch.plugins.Platforms; @@ -922,10 +923,12 @@ public class InstallPluginAction implements Closeable { */ private PluginDescriptor installPlugin(InstallablePlugin descriptor, Path tmpRoot, List deleteOnFailure) throws Exception { final PluginDescriptor info = loadPluginInfo(tmpRoot); - PluginPolicyInfo pluginPolicy = PolicyUtil.getPluginPolicyInfo(tmpRoot, env.tmpDir()); - if (pluginPolicy != null) { - Set permissions = PluginSecurity.getPermissionDescriptions(pluginPolicy, env.tmpDir()); - PluginSecurity.confirmPolicyExceptions(terminal, permissions, batch); + if (RuntimeVersionFeature.isSecurityManagerAvailable()) { + PluginPolicyInfo pluginPolicy = PolicyUtil.getPluginPolicyInfo(tmpRoot, env.tmpDir()); + if (pluginPolicy != null) { + Set permissions = PluginSecurity.getPermissionDescriptions(pluginPolicy, env.tmpDir()); + PluginSecurity.confirmPolicyExceptions(terminal, permissions, batch); + } } // Validate that the downloaded plugin's ID matches what we expect from the descriptor. The From a0029cd83329c6279684258f1c7ba5bd0963424c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Wed, 12 Feb 2025 09:16:42 +0100 Subject: [PATCH 06/93] Ensure that IndexShard is mutable before force merges (#122275) Closes ES-10787 --- .../forcemerge/TransportForceMergeAction.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java index da08b78d711c..7a866792d167 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java @@ -12,6 +12,7 @@ package org.elasticsearch.action.admin.indices.forcemerge; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.elasticsearch.cluster.ClusterState; @@ -92,12 +93,16 @@ public class TransportForceMergeAction extends TransportBroadcastByNodeAction< ActionListener listener ) { assert (task instanceof CancellableTask) == false; // TODO: add cancellation handling here once the task supports it - threadPool.executor(ThreadPool.Names.FORCE_MERGE).execute(ActionRunnable.supply(listener, () -> { + SubscribableListener.newForked(l -> { IndexShard indexShard = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()) .getShard(shardRouting.shardId().id()); - indexShard.forceMerge(request); - return EmptyResult.INSTANCE; - })); + indexShard.ensureMutable(l.map(unused -> indexShard)); + }).andThen((l, indexShard) -> { + threadPool.executor(ThreadPool.Names.FORCE_MERGE).execute(ActionRunnable.supply(l, () -> { + indexShard.forceMerge(request); + return EmptyResult.INSTANCE; + })); + }).addListener(listener); } /** From a57b985487955886769375eaf9a69a2d9b5b3c80 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Wed, 12 Feb 2025 09:17:15 +0100 Subject: [PATCH 07/93] Enforce aggs interface (#121898) --- .../compute/gen/AggregatorImplementer.java | 3 +-- .../compute/gen/GroupingAggregatorImplementer.java | 3 +-- .../java/org/elasticsearch/compute/gen/Types.java | 3 +++ .../compute/aggregation/RateDoubleAggregator.java | 3 ++- .../compute/aggregation/RateFloatAggregator.java | 3 ++- .../compute/aggregation/RateIntAggregator.java | 3 ++- .../compute/aggregation/RateLongAggregator.java | 3 ++- .../compute/aggregation/TopBooleanAggregator.java | 14 ++++++++------ .../compute/aggregation/TopBytesRefAggregator.java | 14 ++++++++------ .../compute/aggregation/TopDoubleAggregator.java | 14 ++++++++------ .../compute/aggregation/TopFloatAggregator.java | 14 ++++++++------ .../compute/aggregation/TopIntAggregator.java | 14 ++++++++------ .../compute/aggregation/TopIpAggregator.java | 14 ++++++++------ .../compute/aggregation/TopLongAggregator.java | 14 ++++++++------ .../aggregation/ValuesBytesRefAggregator.java | 14 ++++++++------ .../aggregation/ValuesDoubleAggregator.java | 14 ++++++++------ .../compute/aggregation/ValuesFloatAggregator.java | 14 ++++++++------ .../compute/aggregation/ValuesIntAggregator.java | 14 ++++++++------ .../compute/aggregation/ValuesLongAggregator.java | 14 ++++++++------ .../compute/aggregation/AbstractArrayState.java | 1 + .../compute/aggregation/BytesRefArrayState.java | 3 ++- .../aggregation/GroupingAggregatorState.java | 1 + .../compute/aggregation/HllStates.java | 3 ++- .../compute/aggregation/MaxBytesRefAggregator.java | 14 ++++++++------ .../compute/aggregation/MaxIpAggregator.java | 14 ++++++++------ .../compute/aggregation/MinBytesRefAggregator.java | 14 ++++++++------ .../compute/aggregation/MinIpAggregator.java | 14 ++++++++------ .../compute/aggregation/QuantileStates.java | 3 ++- .../compute/aggregation/StdDevStates.java | 3 ++- .../aggregation/ValuesBooleanAggregator.java | 14 ++++++++------ .../compute/aggregation/X-RateAggregator.java.st | 3 ++- .../compute/aggregation/X-TopAggregator.java.st | 14 ++++++++------ .../compute/aggregation/X-ValuesAggregator.java.st | 14 ++++++++------ .../spatial/CentroidPointAggregator.java | 3 ++- 34 files changed, 179 insertions(+), 128 deletions(-) diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index 8a02f8bc4c69..d775a4610921 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -95,8 +95,7 @@ public class AggregatorImplementer { this.init = requireStaticMethod( declarationType, - // This should be more restrictive and require org.elasticsearch.compute.aggregation.AggregatorState - requirePrimitiveOrImplements(elements, Types.RELEASABLE), + requirePrimitiveOrImplements(elements, Types.AGGREGATOR_STATE), requireName("init", "initSingle"), requireAnyArgs("") ); diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index fc377f22bbbc..d2b6a0e01168 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -99,8 +99,7 @@ public class GroupingAggregatorImplementer { this.init = requireStaticMethod( declarationType, - // This should be more restrictive and require org.elasticsearch.compute.aggregation.GroupingAggregatorState - requirePrimitiveOrImplements(elements, Types.RELEASABLE), + requirePrimitiveOrImplements(elements, Types.GROUPING_AGGREGATOR_STATE), requireName("init", "initGrouping"), requireAnyArgs("") ); diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index fcd9c64be767..35c42153f9ad 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -79,6 +79,9 @@ public class Types { static final ClassName DOUBLE_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "DoubleVector", "FixedBuilder"); static final ClassName FLOAT_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "FloatVector", "FixedBuilder"); + static final ClassName AGGREGATOR_STATE = ClassName.get(AGGREGATION_PACKAGE, "AggregatorState"); + static final ClassName GROUPING_AGGREGATOR_STATE = ClassName.get(AGGREGATION_PACKAGE, "GroupingAggregatorState"); + static final ClassName AGGREGATOR_FUNCTION = ClassName.get(AGGREGATION_PACKAGE, "AggregatorFunction"); static final ClassName AGGREGATOR_FUNCTION_SUPPLIER = ClassName.get(AGGREGATION_PACKAGE, "AggregatorFunctionSupplier"); static final ClassName GROUPING_AGGREGATOR_FUNCTION = ClassName.get(AGGREGATION_PACKAGE, "GroupingAggregatorFunction"); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateDoubleAggregator.java index cbd20f15c651..deec1ef04f62 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateDoubleAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateDoubleAggregator.java @@ -333,7 +333,8 @@ public class RateDoubleAggregator { } } - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { // noop - we handle the null states inside `toIntermediate` and `evaluateFinal` } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java index b50b125d9833..94ad5254bc72 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java @@ -334,7 +334,8 @@ public class RateFloatAggregator { } } - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { // noop - we handle the null states inside `toIntermediate` and `evaluateFinal` } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateIntAggregator.java index 01c3e3d7fb8e..011291dd08c5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateIntAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateIntAggregator.java @@ -334,7 +334,8 @@ public class RateIntAggregator { } } - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { // noop - we handle the null states inside `toIntermediate` and `evaluateFinal` } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateLongAggregator.java index c84985b703ae..9ccb5d3bd1b1 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateLongAggregator.java @@ -333,7 +333,8 @@ public class RateLongAggregator { } } - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { // noop - we handle the null states inside `toIntermediate` and `evaluateFinal` } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java index 32391c482730..a2e86b3b0934 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.sort.BooleanBucketedSort; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.sort.SortOrder; @@ -74,7 +73,7 @@ class TopBooleanAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final BooleanBucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { @@ -89,7 +88,8 @@ class TopBooleanAggregator { sort.merge(groupId, other.sort, otherGroupId); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -97,7 +97,8 @@ class TopBooleanAggregator { return sort.toBlock(blockFactory, selected); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } @@ -107,7 +108,7 @@ class TopBooleanAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final GroupingState internalState; private SingleState(BigArrays bigArrays, int limit, boolean ascending) { @@ -122,7 +123,8 @@ class TopBooleanAggregator { internalState.merge(0, other, 0); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBytesRefAggregator.java index c9b0e679b3e6..0a965899c077 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBytesRefAggregator.java @@ -19,7 +19,6 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.sort.BytesRefBucketedSort; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.sort.SortOrder; @@ -78,7 +77,7 @@ class TopBytesRefAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final BytesRefBucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { @@ -95,7 +94,8 @@ class TopBytesRefAggregator { sort.merge(groupId, other.sort, otherGroupId); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -103,7 +103,8 @@ class TopBytesRefAggregator { return sort.toBlock(blockFactory, selected); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } @@ -113,7 +114,7 @@ class TopBytesRefAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final GroupingState internalState; private SingleState(BigArrays bigArrays, int limit, boolean ascending) { @@ -128,7 +129,8 @@ class TopBytesRefAggregator { internalState.merge(0, other, 0); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java index d9a7a302f07c..6a20ed99bc23 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.sort.DoubleBucketedSort; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.sort.SortOrder; @@ -74,7 +73,7 @@ class TopDoubleAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final DoubleBucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { @@ -89,7 +88,8 @@ class TopDoubleAggregator { sort.merge(groupId, other.sort, otherGroupId); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -97,7 +97,8 @@ class TopDoubleAggregator { return sort.toBlock(blockFactory, selected); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } @@ -107,7 +108,7 @@ class TopDoubleAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final GroupingState internalState; private SingleState(BigArrays bigArrays, int limit, boolean ascending) { @@ -122,7 +123,8 @@ class TopDoubleAggregator { internalState.merge(0, other, 0); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java index 8b65261e10f4..cf6ad0f9017d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.sort.FloatBucketedSort; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.sort.SortOrder; @@ -74,7 +73,7 @@ class TopFloatAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final FloatBucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { @@ -89,7 +88,8 @@ class TopFloatAggregator { sort.merge(groupId, other.sort, otherGroupId); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -97,7 +97,8 @@ class TopFloatAggregator { return sort.toBlock(blockFactory, selected); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } @@ -107,7 +108,7 @@ class TopFloatAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final GroupingState internalState; private SingleState(BigArrays bigArrays, int limit, boolean ascending) { @@ -122,7 +123,8 @@ class TopFloatAggregator { internalState.merge(0, other, 0); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java index 5c6b79f710af..f4ac83c43806 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.sort.IntBucketedSort; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.sort.SortOrder; @@ -74,7 +73,7 @@ class TopIntAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final IntBucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { @@ -89,7 +88,8 @@ class TopIntAggregator { sort.merge(groupId, other.sort, otherGroupId); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -97,7 +97,8 @@ class TopIntAggregator { return sort.toBlock(blockFactory, selected); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } @@ -107,7 +108,7 @@ class TopIntAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final GroupingState internalState; private SingleState(BigArrays bigArrays, int limit, boolean ascending) { @@ -122,7 +123,8 @@ class TopIntAggregator { internalState.merge(0, other, 0); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java index 219f7385b56d..292dd539edeb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java @@ -18,7 +18,6 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.sort.IpBucketedSort; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.sort.SortOrder; @@ -77,7 +76,7 @@ class TopIpAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final IpBucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { @@ -92,7 +91,8 @@ class TopIpAggregator { sort.merge(groupId, other.sort, otherGroupId); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -100,7 +100,8 @@ class TopIpAggregator { return sort.toBlock(blockFactory, selected); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } @@ -110,7 +111,7 @@ class TopIpAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final GroupingState internalState; private SingleState(BigArrays bigArrays, int limit, boolean ascending) { @@ -125,7 +126,8 @@ class TopIpAggregator { internalState.merge(0, other, 0); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java index 44cef8df7257..c5af92956bec 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.sort.LongBucketedSort; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.sort.SortOrder; @@ -74,7 +73,7 @@ class TopLongAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final LongBucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { @@ -89,7 +88,8 @@ class TopLongAggregator { sort.merge(groupId, other.sort, otherGroupId); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -97,7 +97,8 @@ class TopLongAggregator { return sort.toBlock(blockFactory, selected); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } @@ -107,7 +108,7 @@ class TopLongAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final GroupingState internalState; private SingleState(BigArrays bigArrays, int limit, boolean ascending) { @@ -122,7 +123,8 @@ class TopLongAggregator { internalState.merge(0, other, 0); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java index bd77bd7ff1e4..ad0ab2f7189f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java @@ -20,7 +20,6 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; /** @@ -83,14 +82,15 @@ class ValuesBytesRefAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final BytesRefHash values; private SingleState(BigArrays bigArrays) { values = new BytesRefHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } @@ -125,7 +125,7 @@ class ValuesBytesRefAggregator { * an {@code O(n^2)} operation for collection to support a {@code O(1)} * collector operation. But at least it's fairly simple. */ - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final LongLongHash values; private final BytesRefHash bytes; @@ -146,7 +146,8 @@ class ValuesBytesRefAggregator { } } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -190,7 +191,8 @@ class ValuesBytesRefAggregator { } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java index a8409367bc09..271d7120092c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java @@ -18,7 +18,6 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; /** * Aggregates field values for double. @@ -77,14 +76,15 @@ class ValuesDoubleAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final LongHash values; private SingleState(BigArrays bigArrays) { values = new LongHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } @@ -118,14 +118,15 @@ class ValuesDoubleAggregator { * an {@code O(n^2)} operation for collection to support a {@code O(1)} * collector operation. But at least it's fairly simple. */ - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final LongLongHash values; private GroupingState(BigArrays bigArrays) { values = new LongLongHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -168,7 +169,8 @@ class ValuesDoubleAggregator { } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java index f9e5e1b7b283..b44cad807fba 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; /** * Aggregates field values for float. @@ -82,14 +81,15 @@ class ValuesFloatAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final LongHash values; private SingleState(BigArrays bigArrays) { values = new LongHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } @@ -123,14 +123,15 @@ class ValuesFloatAggregator { * an {@code O(n^2)} operation for collection to support a {@code O(1)} * collector operation. But at least it's fairly simple. */ - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final LongHash values; private GroupingState(BigArrays bigArrays) { values = new LongHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -175,7 +176,8 @@ class ValuesFloatAggregator { } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesIntAggregator.java index 2420dcee7071..4d0c51824569 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesIntAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesIntAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; /** * Aggregates field values for int. @@ -82,14 +81,15 @@ class ValuesIntAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final LongHash values; private SingleState(BigArrays bigArrays) { values = new LongHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } @@ -123,14 +123,15 @@ class ValuesIntAggregator { * an {@code O(n^2)} operation for collection to support a {@code O(1)} * collector operation. But at least it's fairly simple. */ - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final LongHash values; private GroupingState(BigArrays bigArrays) { values = new LongHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -175,7 +176,8 @@ class ValuesIntAggregator { } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesLongAggregator.java index 4938b8f15edb..5471c90147ec 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesLongAggregator.java @@ -18,7 +18,6 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; /** * Aggregates field values for long. @@ -77,14 +76,15 @@ class ValuesLongAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final LongHash values; private SingleState(BigArrays bigArrays) { values = new LongHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } @@ -118,14 +118,15 @@ class ValuesLongAggregator { * an {@code O(n^2)} operation for collection to support a {@code O(1)} * collector operation. But at least it's fairly simple. */ - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final LongLongHash values; private GroupingState(BigArrays bigArrays) { values = new LongLongHash(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -168,7 +169,8 @@ class ValuesLongAggregator { } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java index 5fa1394e8cf9..9886e0c1af30 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java @@ -37,6 +37,7 @@ public abstract class AbstractArrayState implements Releasable, GroupingAggregat * idempotent and fast if already tracking so it's safe to, say, call it once * for every block of values that arrives containing {@code null}. */ + @Override public final void enableGroupIdTracking(SeenGroupIds seenGroupIds) { if (seen == null) { seen = seenGroupIds.seenGroupIds(bigArrays); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java index eb0a992c8610..18b92c544707 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java @@ -138,7 +138,8 @@ public final class BytesRefArrayState implements GroupingAggregatorState, Releas * stores a flag to know if optimizations can be made. *

*/ - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { this.groupIdTrackingEnabled = true; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/GroupingAggregatorState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/GroupingAggregatorState.java index 7c644342598d..0e6516466580 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/GroupingAggregatorState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/GroupingAggregatorState.java @@ -17,4 +17,5 @@ public interface GroupingAggregatorState extends Releasable { /** Extracts an intermediate view of the contents of this state. */ void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext); + void enableGroupIdTracking(SeenGroupIds seenGroupIds); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/HllStates.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/HllStates.java index 3d8d04d7dc7e..64a970c2acc0 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/HllStates.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/HllStates.java @@ -138,7 +138,8 @@ final class HllStates { this.hll = new HyperLogLogPlusPlus(HyperLogLogPlusPlus.precisionFromThreshold(precision), bigArrays, 1); } - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { // Nothing to do } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java index 144214f93571..049642c35091 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; /** @@ -71,7 +70,7 @@ class MaxBytesRefAggregator { return state.toBlock(selected, driverContext); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final BytesRefArrayState internalState; private GroupingState(BigArrays bigArrays, CircuitBreaker breaker) { @@ -90,7 +89,8 @@ class MaxBytesRefAggregator { } } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { internalState.toIntermediate(blocks, offset, selected, driverContext); } @@ -98,7 +98,8 @@ class MaxBytesRefAggregator { return internalState.toValuesBlock(selected, driverContext); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { internalState.enableGroupIdTracking(seen); } @@ -108,7 +109,7 @@ class MaxBytesRefAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final BreakingBytesRefBuilder internalState; private boolean seen; @@ -128,7 +129,8 @@ class MaxBytesRefAggregator { } } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxIpAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxIpAggregator.java index 1ddce7674ae7..43b4a4a2fe0a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxIpAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxIpAggregator.java @@ -15,7 +15,6 @@ import org.elasticsearch.compute.ann.IntermediateState; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @Aggregator({ @IntermediateState(name = "max", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") }) @@ -67,7 +66,7 @@ class MaxIpAggregator { return state.toBlock(selected, driverContext); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final BytesRef scratch = new BytesRef(); private final IpArrayState internalState; @@ -87,7 +86,8 @@ class MaxIpAggregator { } } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { internalState.toIntermediate(blocks, offset, selected, driverContext); } @@ -95,7 +95,8 @@ class MaxIpAggregator { return internalState.toValuesBlock(selected, driverContext); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { internalState.enableGroupIdTracking(seen); } @@ -105,7 +106,7 @@ class MaxIpAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final BytesRef internalState; private boolean seen; @@ -121,7 +122,8 @@ class MaxIpAggregator { } } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState, 1); blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java index 830900702a37..677b38a9af3a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; /** @@ -71,7 +70,7 @@ class MinBytesRefAggregator { return state.toBlock(selected, driverContext); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final BytesRefArrayState internalState; private GroupingState(BigArrays bigArrays, CircuitBreaker breaker) { @@ -90,7 +89,8 @@ class MinBytesRefAggregator { } } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { internalState.toIntermediate(blocks, offset, selected, driverContext); } @@ -98,7 +98,8 @@ class MinBytesRefAggregator { return internalState.toValuesBlock(selected, driverContext); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { internalState.enableGroupIdTracking(seen); } @@ -108,7 +109,7 @@ class MinBytesRefAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final BreakingBytesRefBuilder internalState; private boolean seen; @@ -128,7 +129,8 @@ class MinBytesRefAggregator { } } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinIpAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinIpAggregator.java index 8313756851c1..c4ee93db89cf 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinIpAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinIpAggregator.java @@ -15,7 +15,6 @@ import org.elasticsearch.compute.ann.IntermediateState; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @Aggregator({ @IntermediateState(name = "max", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") }) @@ -67,7 +66,7 @@ class MinIpAggregator { return state.toBlock(selected, driverContext); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final BytesRef scratch = new BytesRef(); private final IpArrayState internalState; @@ -87,7 +86,8 @@ class MinIpAggregator { } } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { internalState.toIntermediate(blocks, offset, selected, driverContext); } @@ -95,7 +95,8 @@ class MinIpAggregator { return internalState.toValuesBlock(selected, driverContext); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { internalState.enableGroupIdTracking(seen); } @@ -105,7 +106,7 @@ class MinIpAggregator { } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final BytesRef internalState; private boolean seen; @@ -121,7 +122,8 @@ class MinIpAggregator { } } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState, 1); blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java index 329e798dcb3f..d5ea72ed23e5 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java @@ -146,7 +146,8 @@ public final class QuantileStates { } } - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { // We always enable. } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/StdDevStates.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/StdDevStates.java index bff8903fd3be..5b48498d8329 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/StdDevStates.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/StdDevStates.java @@ -204,7 +204,8 @@ public final class StdDevStates { Releasables.close(states); } - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { // noop - we handle the null states inside `toIntermediate` and `evaluateFinal` } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/ValuesBooleanAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/ValuesBooleanAggregator.java index 252436ad9634..e19d3107172e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/ValuesBooleanAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/ValuesBooleanAggregator.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; /** @@ -84,11 +83,12 @@ class ValuesBooleanAggregator { return state.toBlock(driverContext.blockFactory(), selected); } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private boolean seenFalse; private boolean seenTrue; - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } @@ -113,14 +113,15 @@ class ValuesBooleanAggregator { public void close() {} } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final BitArray values; private GroupingState(BigArrays bigArrays) { values = new BitArray(1, bigArrays); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -155,7 +156,8 @@ class ValuesBooleanAggregator { } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we don't need to track which values have been seen because we don't do anything special for groups without values } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st index 2581d3ebbf80..a0b4ed8bd633 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st @@ -338,7 +338,8 @@ public class Rate$Type$Aggregator { } } - void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + @Override + public void enableGroupIdTracking(SeenGroupIds seenGroupIds) { // noop - we handle the null states inside `toIntermediate` and `evaluateFinal` } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st index 18d573eea4a4..761b70791e94 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st @@ -28,7 +28,6 @@ import org.elasticsearch.compute.data.$Type$Block; $endif$ import org.elasticsearch.compute.data.sort.$Name$BucketedSort; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.sort.SortOrder; @@ -99,7 +98,7 @@ $endif$ return state.toBlock(driverContext.blockFactory(), selected); } - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { private final $Name$BucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { @@ -120,7 +119,8 @@ $endif$ sort.merge(groupId, other.sort, otherGroupId); } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -128,7 +128,8 @@ $endif$ return sort.toBlock(blockFactory, selected); } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } @@ -138,7 +139,7 @@ $endif$ } } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { private final GroupingState internalState; private SingleState(BigArrays bigArrays, int limit, boolean ascending) { @@ -153,7 +154,8 @@ $endif$ internalState.merge(0, other, 0); } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st index 1cef234b2238..3006af595be1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st @@ -35,7 +35,6 @@ $if(long)$ import org.elasticsearch.compute.data.LongBlock; $endif$ import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; $if(BytesRef)$ import org.elasticsearch.core.Releasables; @@ -155,7 +154,7 @@ $endif$ return state.toBlock(driverContext.blockFactory(), selected); } - public static class SingleState implements Releasable { + public static class SingleState implements AggregatorState { $if(BytesRef)$ private final BytesRefHash values; @@ -171,7 +170,8 @@ $else$ $endif$ } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory()); } @@ -228,7 +228,7 @@ $endif$ * an {@code O(n^2)} operation for collection to support a {@code O(1)} * collector operation. But at least it's fairly simple. */ - public static class GroupingState implements Releasable { + public static class GroupingState implements GroupingAggregatorState { $if(long||double)$ private final LongLongHash values; @@ -263,7 +263,8 @@ $elseif(int||float)$ $endif$ } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { blocks[offset] = toBlock(driverContext.blockFactory(), selected); } @@ -324,7 +325,8 @@ $endif$ } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java index 47d927fda91b..c3b07d069cf1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java @@ -260,7 +260,8 @@ abstract class CentroidPointAggregator { } /** Needed for generated code that does null tracking, which we do not need because we use count */ - final void enableGroupIdTracking(SeenGroupIds ignore) {} + @Override + public final void enableGroupIdTracking(SeenGroupIds ignore) {} private void ensureCapacity(int groupId) { if (groupId >= xValues.size()) { From bc6731b4672fb2f083476b0fea8b090458f0a09b Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 12 Feb 2025 20:00:49 +1100 Subject: [PATCH 08/93] Mute org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT testUnsupportedTypesOrdinalGrouping #122342 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0090b6fe9008..6b2546cebc88 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -420,6 +420,9 @@ tests: - class: org.elasticsearch.xpack.esql.action.CrossClusterCancellationIT method: testCloseSkipUnavailable issue: https://github.com/elastic/elasticsearch/issues/122336 +- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT + method: testUnsupportedTypesOrdinalGrouping + issue: https://github.com/elastic/elasticsearch/issues/122342 # Examples: # From 49eccbe52da2ce88481368e9be0eac52dadc070c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 12 Feb 2025 20:07:05 +1100 Subject: [PATCH 09/93] Mute org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT test {yaml=reference/alias/line_260} #122343 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 6b2546cebc88..7a87efd0b158 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -423,6 +423,9 @@ tests: - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT method: testUnsupportedTypesOrdinalGrouping issue: https://github.com/elastic/elasticsearch/issues/122342 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/alias/line_260} + issue: https://github.com/elastic/elasticsearch/issues/122343 # Examples: # From f3b3297f9eecac5cbbd036cfbbbbab647cd1be40 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 12 Feb 2025 11:09:53 +0100 Subject: [PATCH 10/93] Sets the new engine before closing the previous one in IndexShard.resetEngine (#122255) Current code closes the previous engine instance, creates the new engine and then updates the currentEngineReference: IOUtils.close(currentEngine); var newEngine = createEngine(engineConfig); currentEngineReference.set(newEngine); This leaves more room for callers of getEngineOrNull() to pick a closed instance. Instead we can create the new engine first and atomically update the reference to the new instance. --- .../java/org/elasticsearch/index/shard/IndexShard.java | 10 ++++------ .../org/elasticsearch/index/shard/IndexShardTests.java | 6 ++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index e5085ce9944b..d56d7471d498 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -4313,17 +4313,15 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl assert waitForEngineOrClosedShardListeners.isDone(); try { synchronized (engineMutex) { - final var currentEngine = getEngine(); - currentEngine.prepareForEngineReset(); - var engineConfig = newEngineConfig(replicationTracker); verifyNotClosed(); - IOUtils.close(currentEngine); - var newEngine = createEngine(engineConfig); - currentEngineReference.set(newEngine); + getEngine().prepareForEngineReset(); + var newEngine = createEngine(newEngineConfig(replicationTracker)); + IOUtils.close(currentEngineReference.getAndSet(newEngine)); onNewEngine(newEngine); } onSettingsChanged(); } catch (Exception e) { + // we want to fail the shard in the case prepareForEngineReset throws failShard("unable to reset engine", e); } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 4549a329d499..c07b396626c4 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -4563,11 +4563,9 @@ public class IndexShardTests extends IndexShardTestCase { var newEngineCreated = new CountDownLatch(2); var indexShard = newStartedShard(true, Settings.EMPTY, config -> { try { - return new ReadOnlyEngine(config, null, null, true, Function.identity(), true, true) { + return new ReadOnlyEngine(config, null, new TranslogStats(), false, Function.identity(), true, true) { @Override - public void prepareForEngineReset() throws IOException { - ; - } + public void prepareForEngineReset() throws IOException {} }; } finally { newEngineCreated.countDown(); From 656b54bc74ee46994e14150459aed7ac50876f30 Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Wed, 12 Feb 2025 11:25:03 +0100 Subject: [PATCH 11/93] Unmute changepoint tests (#122351) --- muted-tests.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 7a87efd0b158..240efdcf8d0a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -381,15 +381,6 @@ tests: - class: org.elasticsearch.search.SearchCancellationIT method: testCancelFailedSearchWhenPartialResultDisallowed issue: https://github.com/elastic/elasticsearch/issues/121719 -- class: org.elasticsearch.xpack.esql.analysis.VerifierTests - method: testChangePoint - issue: https://github.com/elastic/elasticsearch/issues/122179 -- class: org.elasticsearch.xpack.esql.analysis.VerifierTests - method: testChangePoint_keySortable - issue: https://github.com/elastic/elasticsearch/issues/122180 -- class: org.elasticsearch.xpack.esql.analysis.VerifierTests - method: testChangePoint_valueNumeric - issue: https://github.com/elastic/elasticsearch/issues/122181 - class: org.elasticsearch.datastreams.TSDBPassthroughIndexingIT issue: https://github.com/elastic/elasticsearch/issues/121716 - class: org.elasticsearch.smoketest.SmokeTestMonitoringWithSecurityIT From 858769ddb65d0f22c3bc602f1b3196313963677a Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 12 Feb 2025 11:40:48 +0100 Subject: [PATCH 12/93] Simplify comparators in InternalOrder (#122330) We can build slightly more compact (and likely also faster) iterators while using less code for these. Also, no need to create method references as a way of casting. --- .../search/aggregations/InternalOrder.java | 55 +++++++------------ 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java index afeeaa9bd675..ead1374fb9b4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -188,33 +189,22 @@ public abstract class InternalOrder extends BucketOrder { @Override public Comparator> partiallyBuiltBucketComparator(Aggregator aggregator) { - List>> comparators = new ArrayList<>(orderElements.size()); - for (BucketOrder order : orderElements) { - comparators.add(order.partiallyBuiltBucketComparator(aggregator)); + Iterator iterator = orderElements.iterator(); + Comparator> comparator = iterator.next().partiallyBuiltBucketComparator(aggregator); + while (iterator.hasNext()) { + comparator = comparator.thenComparing(iterator.next().partiallyBuiltBucketComparator(aggregator)); } - return (lhs, rhs) -> { - for (Comparator> c : comparators) { - int result = c.compare(lhs, rhs); - if (result != 0) { - return result; - } - } - return 0; - }; + return comparator; } @Override public Comparator comparator() { - List> comparators = orderElements.stream().map(BucketOrder::comparator).toList(); - return (lhs, rhs) -> { - for (Comparator c : comparators) { - int result = c.compare(lhs, rhs); - if (result != 0) { - return result; - } - } - return 0; - }; + Iterator iterator = orderElements.iterator(); + Comparator comparator = iterator.next().comparator(); + while (iterator.hasNext()) { + comparator = comparator.thenComparing(iterator.next().comparator()); + } + return comparator; } @Override @@ -222,18 +212,12 @@ public abstract class InternalOrder extends BucketOrder { BiFunction, AggregationReduceContext, B> reduce, AggregationReduceContext reduceContext ) { - List>> comparators = orderElements.stream() - .map(b -> b.delayedBucketComparator(reduce, reduceContext)) - .toList(); - return (lhs, rhs) -> { - for (Comparator> c : comparators) { - int result = c.compare(lhs, rhs); - if (result != 0) { - return result; - } - } - return 0; - }; + Iterator iterator = orderElements.iterator(); + Comparator> comparator = iterator.next().delayedBucketComparator(reduce, reduceContext); + while (iterator.hasNext()) { + comparator = comparator.thenComparing(iterator.next().delayedBucketComparator(reduce, reduceContext)); + } + return comparator; } @Override @@ -285,12 +269,13 @@ public abstract class InternalOrder extends BucketOrder { return comparator; } + @SuppressWarnings({ "rawtypes", "unchecked" }) @Override Comparator> delayedBucketComparator( BiFunction, AggregationReduceContext, B> reduce, AggregationReduceContext reduceContext ) { - return delayedBucketCompator::compare; + return (Comparator) delayedBucketCompator; } @Override From e8bbf27146aa978cd5edce42729f277b9f4c7999 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 12 Feb 2025 12:09:08 +0100 Subject: [PATCH 13/93] [ML] Unmute fixed Classification IT tests #122352 Unmute tests following #122268 Closes #120071 Closes #121680 Closes #121492 Closes #121415 Closes #121236 --- muted-tests.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 240efdcf8d0a..9fc54786d579 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -254,9 +254,6 @@ tests: - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT method: testFileSettingsReprocessedOnRestartWithoutVersionChange issue: https://github.com/elastic/elasticsearch/issues/120964 -- class: org.elasticsearch.xpack.ml.integration.ClassificationIT - method: testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsKeyword - issue: https://github.com/elastic/elasticsearch/issues/120071 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests method: testGetUsersWithProfileUidWhenProfileIndexDoesNotExists issue: https://github.com/elastic/elasticsearch/issues/121179 @@ -265,9 +262,6 @@ tests: - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests method: testSetEnabled issue: https://github.com/elastic/elasticsearch/issues/121183 -- class: org.elasticsearch.xpack.ml.integration.ClassificationIT - method: testWithDatastreams - issue: https://github.com/elastic/elasticsearch/issues/121236 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/*} issue: https://github.com/elastic/elasticsearch/issues/120816 @@ -317,9 +311,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/121151 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/121407 -- class: org.elasticsearch.xpack.ml.integration.ClassificationIT - method: testDependentVariableIsAliasToNested - issue: https://github.com/elastic/elasticsearch/issues/121415 - class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests method: testClientSecretRotation issue: https://github.com/elastic/elasticsearch/issues/120985 @@ -347,9 +338,6 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only} issue: https://github.com/elastic/elasticsearch/issues/121412 -- class: org.elasticsearch.xpack.ml.integration.ClassificationIT - method: testDependentVariableIsAliasToKeyword - issue: https://github.com/elastic/elasticsearch/issues/121492 - class: org.elasticsearch.search.CrossClusterSearchUnavailableClusterIT method: testSearchSkipUnavailable issue: https://github.com/elastic/elasticsearch/issues/121497 @@ -372,9 +360,6 @@ tests: - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously} issue: https://github.com/elastic/elasticsearch/issues/122102 -- class: org.elasticsearch.xpack.ml.integration.ClassificationIT - method: testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsBoolean - issue: https://github.com/elastic/elasticsearch/issues/121680 - class: org.elasticsearch.xpack.downsample.DownsampleActionSingleNodeTests method: testDuplicateDownsampleRequest issue: https://github.com/elastic/elasticsearch/issues/122158 From ec7f4ccb04fe033a1e12795b84767a44fc7a1512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Wed, 12 Feb 2025 14:04:18 +0100 Subject: [PATCH 14/93] [Entitlements] Add FileStore instrumentation + tests (#122348) --- .../bridge/EntitlementChecker.java | 20 +++++ .../entitlement/qa/test/FileCheckActions.java | 10 +-- .../entitlement/qa/test/FileStoreActions.java | 71 ++++++++++++++++ .../qa/test/RestEntitlementsCheckAction.java | 3 +- .../EntitlementInitialization.java | 84 +++++++++++++++---- .../api/ElasticsearchEntitlementChecker.java | 46 ++++++++++ .../runtime/policy/PolicyManager.java | 9 ++ .../ReadStoreAttributesEntitlement.java | 15 ++++ 8 files changed, 236 insertions(+), 22 deletions(-) create mode 100644 libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileStoreActions.java create mode 100644 libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ReadStoreAttributesEntitlement.java diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 28306cc0e660..6e6e5f306c80 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -50,6 +50,7 @@ import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.nio.channels.spi.SelectorProvider; import java.nio.charset.Charset; +import java.nio.file.FileStore; import java.nio.file.LinkOption; import java.nio.file.OpenOption; import java.nio.file.Path; @@ -523,4 +524,23 @@ public interface EntitlementChecker { // file system providers void checkNewInputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options); + + // file store + void checkGetFileStoreAttributeView(Class callerClass, FileStore that, Class type); + + void checkGetAttribute(Class callerClass, FileStore that, String attribute); + + void checkGetBlockSize(Class callerClass, FileStore that); + + void checkGetTotalSpace(Class callerClass, FileStore that); + + void checkGetUnallocatedSpace(Class callerClass, FileStore that); + + void checkGetUsableSpace(Class callerClass, FileStore that); + + void checkIsReadOnly(Class callerClass, FileStore that); + + void checkName(Class callerClass, FileStore that); + + void checkType(Class callerClass, FileStore that); } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java index c62f6c003fe7..4a15d63f99ec 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java @@ -27,21 +27,21 @@ import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAcce @SuppressForbidden(reason = "Explicitly checking APIs that are forbidden") class FileCheckActions { - private static Path testRootDir = Paths.get(System.getProperty("es.entitlements.testdir")); + static Path testRootDir = Paths.get(System.getProperty("es.entitlements.testdir")); - private static Path readDir() { + static Path readDir() { return testRootDir.resolve("read_dir"); } - private static Path readWriteDir() { + static Path readWriteDir() { return testRootDir.resolve("read_write_dir"); } - private static Path readFile() { + static Path readFile() { return testRootDir.resolve("read_file"); } - private static Path readWriteFile() { + static Path readWriteFile() { return testRootDir.resolve("read_write_file"); } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileStoreActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileStoreActions.java new file mode 100644 index 000000000000..0c8026ea9fee --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileStoreActions.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.FileStoreAttributeView; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY; + +class FileStoreActions { + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkGetFileStoreAttributeView() throws IOException { + Files.getFileStore(FileCheckActions.readWriteFile()).getFileStoreAttributeView(FileStoreAttributeView.class); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetAttribute() throws IOException { + try { + Files.getFileStore(FileCheckActions.readFile()).getAttribute("zfs:compression"); + } catch (UnsupportedOperationException e) { + // It's OK if the attribute view is not available or it does not support reading the attribute + } + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetBlockSize() throws IOException { + Files.getFileStore(FileCheckActions.readWriteFile()).getBlockSize(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetTotalSpace() throws IOException { + Files.getFileStore(FileCheckActions.readWriteFile()).getTotalSpace(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetUnallocatedSpace() throws IOException { + Files.getFileStore(FileCheckActions.readWriteFile()).getUnallocatedSpace(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetUsableSpace() throws IOException { + Files.getFileStore(FileCheckActions.readFile()).getUsableSpace(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkIsReadOnly() throws IOException { + Files.getFileStore(FileCheckActions.readFile()).isReadOnly(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkName() throws IOException { + Files.getFileStore(FileCheckActions.readFile()).name(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkType() throws IOException { + Files.getFileStore(FileCheckActions.readFile()).type(); + } + + private FileStoreActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java index dbc9a7692b70..315ff9d2b526 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java @@ -185,7 +185,8 @@ public class RestEntitlementsCheckAction extends BaseRestHandler { getTestEntries(FileCheckActions.class), getTestEntries(SpiActions.class), getTestEntries(SystemActions.class), - getTestEntries(NativeActions.class) + getTestEntries(NativeActions.class), + getTestEntries(FileStoreActions.class) ) .flatMap(Function.identity()) .filter(entry -> entry.getValue().fromJavaVersion() == null || Runtime.version().feature() >= entry.getValue().fromJavaVersion()) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 8935d718b830..0b0caa0745fc 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -27,11 +27,13 @@ import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitleme import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement; import java.lang.instrument.Instrumentation; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.channels.spi.SelectorProvider; +import java.nio.file.FileStore; import java.nio.file.FileSystems; import java.nio.file.OpenOption; import java.nio.file.Path; @@ -43,6 +45,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; /** * Called by the agent during {@code agentmain} to configure the entitlement system, @@ -58,6 +61,11 @@ public class EntitlementInitialization { private static ElasticsearchEntitlementChecker manager; + interface InstrumentationInfoFunction { + InstrumentationService.InstrumentationInfo of(String methodName, Class... parameterTypes) throws ClassNotFoundException, + NoSuchMethodException; + } + // Note: referenced by bridge reflectively public static EntitlementChecker checker() { return manager; @@ -71,22 +79,26 @@ public class EntitlementInitialization { Map checkMethods = new HashMap<>(INSTRUMENTATION_SERVICE.lookupMethods(latestCheckerInterface)); var fileSystemProviderClass = FileSystems.getDefault().provider().getClass(); - Stream.of( - INSTRUMENTATION_SERVICE.lookupImplementationMethod( - FileSystemProvider.class, - "newInputStream", - fileSystemProviderClass, - EntitlementChecker.class, - "checkNewInputStream", - Path.class, - OpenOption[].class - ), - INSTRUMENTATION_SERVICE.lookupImplementationMethod( - SelectorProvider.class, - "inheritedChannel", - SelectorProvider.provider().getClass(), - EntitlementChecker.class, - "checkSelectorProviderInheritedChannel" + + Stream.concat( + fileStoreChecks(), + Stream.of( + INSTRUMENTATION_SERVICE.lookupImplementationMethod( + FileSystemProvider.class, + "newInputStream", + fileSystemProviderClass, + EntitlementChecker.class, + "checkNewInputStream", + Path.class, + OpenOption[].class + ), + INSTRUMENTATION_SERVICE.lookupImplementationMethod( + SelectorProvider.class, + "inheritedChannel", + SelectorProvider.provider().getClass(), + EntitlementChecker.class, + "checkSelectorProviderInheritedChannel" + ) ) ).forEach(instrumentation -> checkMethods.put(instrumentation.targetMethod(), instrumentation.checkMethod())); @@ -120,6 +132,7 @@ public class EntitlementInitialization { "org.elasticsearch.server", List.of( new ExitVMEntitlement(), + new ReadStoreAttributesEntitlement(), new CreateClassLoaderEntitlement(), new InboundNetworkEntitlement(), new OutboundNetworkEntitlement(), @@ -139,6 +152,45 @@ public class EntitlementInitialization { return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, AGENTS_PACKAGE_NAME, ENTITLEMENTS_MODULE); } + private static Stream fileStoreChecks() { + var fileStoreClasses = StreamSupport.stream(FileSystems.getDefault().getFileStores().spliterator(), false) + .map(FileStore::getClass) + .distinct(); + return fileStoreClasses.flatMap(fileStoreClass -> { + var instrumentation = new InstrumentationInfoFunction() { + @Override + public InstrumentationService.InstrumentationInfo of(String methodName, Class... parameterTypes) + throws ClassNotFoundException, NoSuchMethodException { + return INSTRUMENTATION_SERVICE.lookupImplementationMethod( + FileStore.class, + methodName, + fileStoreClass, + EntitlementChecker.class, + "check" + Character.toUpperCase(methodName.charAt(0)) + methodName.substring(1), + parameterTypes + ); + } + }; + + try { + return Stream.of( + instrumentation.of("getFileStoreAttributeView", Class.class), + instrumentation.of("getAttribute", String.class), + instrumentation.of("getBlockSize"), + instrumentation.of("getTotalSpace"), + instrumentation.of("getUnallocatedSpace"), + instrumentation.of("getUsableSpace"), + instrumentation.of("isReadOnly"), + instrumentation.of("name"), + instrumentation.of("type") + + ); + } catch (NoSuchMethodException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + }); + } + /** * Returns the "most recent" checker class compatible with the current runtime Java version. * For checkers, we have (optionally) version specific classes, each with a prefix (e.g. Java23). diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index 50e3e6d9c55e..156ae9cc4a35 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -55,6 +55,7 @@ import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.nio.channels.spi.SelectorProvider; import java.nio.charset.Charset; +import java.nio.file.FileStore; import java.nio.file.LinkOption; import java.nio.file.OpenOption; import java.nio.file.Path; @@ -998,4 +999,49 @@ public class ElasticsearchEntitlementChecker implements EntitlementChecker { public void checkNewInputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options) { // TODO: policyManger.checkFileSystemRead(path); } + + @Override + public void checkGetFileStoreAttributeView(Class callerClass, FileStore that, Class type) { + policyManager.checkWriteStoreAttributes(callerClass); + } + + @Override + public void checkGetAttribute(Class callerClass, FileStore that, String attribute) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkGetBlockSize(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkGetTotalSpace(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkGetUnallocatedSpace(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkGetUsableSpace(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkIsReadOnly(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkName(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkType(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 393eb93478e6..b4d099007866 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -20,6 +20,7 @@ import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlemen import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement; import org.elasticsearch.logging.LogManager; @@ -181,6 +182,14 @@ public class PolicyManager { neverEntitled(callerClass, () -> "start process"); } + public void checkWriteStoreAttributes(Class callerClass) { + neverEntitled(callerClass, () -> "change file store attributes"); + } + + public void checkReadStoreAttributes(Class callerClass) { + checkEntitlementPresent(callerClass, ReadStoreAttributesEntitlement.class); + } + /** * @param operationDescription is only called when the operation is not trivially allowed, meaning the check is about to fail; * therefore, its performance is not a major concern. diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ReadStoreAttributesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ReadStoreAttributesEntitlement.java new file mode 100644 index 000000000000..ccb84c4a68c9 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ReadStoreAttributesEntitlement.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +/** + * Describes an entitlement for reading file store attributes (e.g. disk space) + */ +public record ReadStoreAttributesEntitlement() implements Entitlement {} From 65f6b44b2a0759fa6c950e8646d2e8aed0937aab Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 12 Feb 2025 15:07:38 +0200 Subject: [PATCH 15/93] Handle error in concurrent downsample actions (#122251) --- muted-tests.yml | 3 --- .../xpack/downsample/DownsampleActionSingleNodeTests.java | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 9fc54786d579..1f42941579cf 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -360,9 +360,6 @@ tests: - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously} issue: https://github.com/elastic/elasticsearch/issues/122102 -- class: org.elasticsearch.xpack.downsample.DownsampleActionSingleNodeTests - method: testDuplicateDownsampleRequest - issue: https://github.com/elastic/elasticsearch/issues/122158 - class: org.elasticsearch.search.SearchCancellationIT method: testCancelFailedSearchWhenPartialResultDisallowed issue: https://github.com/elastic/elasticsearch/issues/121719 diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index ce9b60938526..2c759f150e57 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -1760,7 +1760,7 @@ public class DownsampleActionSingleNodeTests extends ESSingleNodeTestCase { new Thread(() -> { try { downsample(sourceIndex, targetIndex, config); - } catch (ResourceAlreadyExistsException e) { + } catch (ElasticsearchException e) { firstFailed.set(true); } finally { downsampleComplete.countDown(); @@ -1770,7 +1770,7 @@ public class DownsampleActionSingleNodeTests extends ESSingleNodeTestCase { new Thread(() -> { try { downsample(sourceIndex, targetIndex, config); - } catch (ResourceAlreadyExistsException e) { + } catch (ElasticsearchException e) { secondFailed.set(true); } finally { downsampleComplete.countDown(); From 4b3acd4f5755bb98c5d020af459a198d891d6b3f Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 12 Feb 2025 15:12:20 +0200 Subject: [PATCH 16/93] ESQL: revive inlinestats (#122257) --- docs/changelog/122257.yaml | 5 +++ .../xpack/esql/ccq/MultiClusterSpecIT.java | 2 + ....csv-spec-ignored => inlinestats.csv-spec} | 44 +++++++++---------- .../xpack/esql/action/EsqlCapabilities.java | 8 +++- .../esql/plan/logical/join/InlineJoin.java | 8 ++-- .../xpack/esql/planner/mapper/Mapper.java | 5 +++ 6 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 docs/changelog/122257.yaml rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{inlinestats.csv-spec-ignored => inlinestats.csv-spec} (97%) diff --git a/docs/changelog/122257.yaml b/docs/changelog/122257.yaml new file mode 100644 index 000000000000..24078170eb6b --- /dev/null +++ b/docs/changelog/122257.yaml @@ -0,0 +1,5 @@ +pr: 122257 +summary: Revive inlinestats +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 84bf34b70372..723c5e2dfd1a 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -48,6 +48,7 @@ import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.ENRICH_SOURCE_INDI import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V3; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; @@ -124,6 +125,7 @@ public class MultiClusterSpecIT extends EsqlSpecTestCase { assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); + assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V3.capabilityName())); assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec-ignored b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec similarity index 97% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec-ignored rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index 91075691a6a1..cf2d44665bd5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec-ignored +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -2,8 +2,8 @@ // TODO: re-enable the commented tests once the Join functionality stabilizes // -maxOfInt-Ignore -required_capability: join_planning_v1 +maxOfInt +required_capability: inlinestats_v3 // tag::max-languages[] FROM employees | KEEP emp_no, languages @@ -25,7 +25,7 @@ emp_no:integer | languages:integer | max_lang:integer ; maxOfIntByKeyword -required_capability: join_planning_v1 +required_capability: inlinestats_v3 FROM employees | KEEP emp_no, languages, gender @@ -43,7 +43,7 @@ emp_no:integer | languages:integer | gender:keyword | max_lang:integer ; maxOfLongByKeyword -required_capability: join_planning_v1 +required_capability: inlinestats_v3 FROM employees | KEEP emp_no, avg_worked_seconds, gender @@ -57,8 +57,8 @@ emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_secon 10030 | 394597613 | M | 394597613 ; -maxOfLong-Ignore -required_capability: join_planning_v1 +maxOfLong +required_capability: inlinestats_v3 FROM employees | KEEP emp_no, avg_worked_seconds, gender @@ -71,7 +71,7 @@ emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_secon ; maxOfLongByCalculatedKeyword -required_capability: join_planning_v1 +required_capability: inlinestats_v3 // tag::longest-tenured-by-first[] FROM employees @@ -94,7 +94,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | SUBSTRING(last_na ; maxOfLongByCalculatedNamedKeyword -required_capability: join_planning_v1 +required_capability: inlinestats_v3 FROM employees | KEEP emp_no, avg_worked_seconds, last_name @@ -112,7 +112,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | l:keyword | max_a 10087 | 305782871 | Eugenio | E | 305782871 ; -maxOfLongByCalculatedDroppedKeyword +maxOfLongByCalculatedDroppedKeyword-Ignore required_capability: join_planning_v1 FROM employees @@ -132,7 +132,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se ; maxOfLongByEvaledKeyword -required_capability: join_planning_v1 +required_capability: inlinestats_v3 FROM employees | EVAL l = SUBSTRING(last_name, 0, 1) @@ -152,7 +152,7 @@ emp_no:integer | avg_worked_seconds:long | l:keyword | max_avg_worked_seconds:lo ; maxOfLongByInt -required_capability: join_planning_v1 +required_capability: inlinestats_v3 FROM employees | KEEP emp_no, avg_worked_seconds, languages @@ -170,7 +170,7 @@ emp_no:integer | avg_worked_seconds:long | languages:integer | max_avg_worked_se ; maxOfLongByIntDouble -required_capability: join_planning_v1 +required_capability: inlinestats_v3 FROM employees | KEEP emp_no, avg_worked_seconds, languages, height @@ -205,7 +205,7 @@ emp_no:integer | languages:integer | avg_worked_seconds:long | gender:keyword | 10007 | 4 | 393084805 | F | 2.863684210555556E8 | 5 ; -byMultivaluedSimple +byMultivaluedSimple-Ignore required_capability: join_planning_v1 // tag::mv-group[] @@ -223,7 +223,7 @@ abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer // end::mv-group-result[] ; -byMultivaluedMvExpand +byMultivaluedMvExpand-Ignore required_capability: join_planning_v1 // tag::mv-expand[] @@ -243,7 +243,7 @@ abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer // end::mv-expand-result[] ; -byMvExpand +byMvExpand-Ignore required_capability: join_planning_v1 // tag::extreme-airports[] @@ -307,7 +307,7 @@ count:long | country:keyword | avg:double 17 | United Kingdom | 4.455 ; -afterWhere +afterWhere-Ignore required_capability: join_planning_v1 FROM airports @@ -366,8 +366,8 @@ abbrev:keyword | city:keyword | region:text | "COUNT(*)":long FUK | Fukuoka | 中央区 | 2 ; -beforeStats-Ignore -required_capability: join_planning_v1 +beforeStats +required_capability: inlinestats_v3 FROM airports | EVAL lat = ST_Y(location) @@ -379,7 +379,7 @@ northern:long | southern:long 520 | 371 ; -beforeKeepSort +beforeKeepSort-Ignore required_capability: join_planning_v1 FROM employees @@ -394,7 +394,7 @@ emp_no:integer | languages:integer | max_salary:integer 10003 | 4 | 74572 ; -beforeKeepWhere +beforeKeepWhere-Ignore required_capability: join_planning_v1 FROM employees @@ -537,8 +537,8 @@ emp_no:integer | one:integer 10005 | 1 ; -percentile-Ignore -required_capability: join_planning_v1 +percentile +required_capability: inlinestats_v3 FROM employees | KEEP emp_no, salary diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 0b9b0995d3ba..59e52a409e66 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -808,7 +808,13 @@ public class EsqlCapabilities { * and https://github.com/elastic/elasticsearch/issues/120803 * Support for queries that have multiple SORTs that cannot become TopN */ - REMOVE_REDUNDANT_SORT; + REMOVE_REDUNDANT_SORT, + + /** + * Fixes a series of issues with inlinestats which had an incomplete implementation after lookup and inlinestats + * were refactored. + */ + INLINESTATS_V3; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java index 87c9db1db480..e3daa4fcbfb9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java @@ -16,10 +16,9 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; @@ -71,10 +70,9 @@ public class InlineJoin extends Join { List aliases = new ArrayList<>(schema.size()); for (int i = 0; i < schema.size(); i++) { Attribute attr = schema.get(i); - aliases.add(new Alias(attr.source(), attr.name(), Literal.of(attr, BlockUtils.toJavaObject(blocks[i], 0)))); + aliases.add(new Alias(attr.source(), attr.name(), Literal.of(attr, BlockUtils.toJavaObject(blocks[i], 0)), attr.id())); } - LogicalPlan left = target.left(); - return new Project(target.source(), left, CollectionUtils.combine(left.output(), aliases)); + return new Eval(target.source(), target.left(), aliases); } else { return target.replaceRight(data); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java index 8ea19f545e67..b4560b2e3355 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.join.InlineJoin; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; @@ -178,6 +179,10 @@ public class Mapper { throw new EsqlIllegalArgumentException("unsupported join type [" + config.type() + "]"); } + if (join instanceof InlineJoin) { + return new FragmentExec(bp); + } + PhysicalPlan left = map(bp.left()); // only broadcast joins supported for now - hence push down as a streaming operator From b9d122205ab2de0128148df824ccb895fc01c3da Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Wed, 12 Feb 2025 08:16:48 -0500 Subject: [PATCH 17/93] [ML] Adding elser default endpoint for EIS (#122066) * Adding elser default endpoint * [CI] Auto commit changes from spotless * Fixing test and allowing duplicate calls * [CI] Auto commit changes from spotless * Update docs/changelog/122066.yaml --------- Co-authored-by: elasticsearchmachine --- docs/changelog/122066.yaml | 5 +++ ...etModelsWithElasticInferenceServiceIT.java | 13 +++++++- .../InferenceRevokeDefaultEndpointsIT.java | 14 ++++++-- .../inference/registry/ModelRegistry.java | 15 +++++++-- .../elastic/ElasticInferenceService.java | 33 +++++++++++++++++-- ...erviceSparseEmbeddingsServiceSettings.java | 2 +- .../elastic/ElasticInferenceServiceTests.java | 14 ++++++-- 7 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 docs/changelog/122066.yaml diff --git a/docs/changelog/122066.yaml b/docs/changelog/122066.yaml new file mode 100644 index 000000000000..79a9129bd542 --- /dev/null +++ b/docs/changelog/122066.yaml @@ -0,0 +1,5 @@ +pr: 122066 +summary: Adding elser default endpoint for EIS +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetModelsWithElasticInferenceServiceIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetModelsWithElasticInferenceServiceIT.java index 3a2a003636b1..42289c50864e 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetModelsWithElasticInferenceServiceIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetModelsWithElasticInferenceServiceIT.java @@ -12,10 +12,13 @@ package org.elasticsearch.xpack.inference; import org.elasticsearch.inference.TaskType; import java.io.IOException; +import java.util.List; +import java.util.Map; import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.getAllModels; import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.getModels; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; public class InferenceGetModelsWithElasticInferenceServiceIT extends BaseMockEISAuthServerTest { @@ -23,12 +26,20 @@ public class InferenceGetModelsWithElasticInferenceServiceIT extends BaseMockEIS var allModels = getAllModels(); var chatCompletionModels = getModels("_all", TaskType.CHAT_COMPLETION); - assertThat(allModels, hasSize(4)); + assertThat(allModels, hasSize(5)); assertThat(chatCompletionModels, hasSize(1)); for (var model : chatCompletionModels) { assertEquals("chat_completion", model.get("task_type")); } + assertInferenceIdTaskType(allModels, ".rainbow-sprinkles-elastic", TaskType.CHAT_COMPLETION); + assertInferenceIdTaskType(allModels, ".elser-v2-elastic", TaskType.SPARSE_EMBEDDING); + } + + private static void assertInferenceIdTaskType(List> models, String inferenceId, TaskType taskType) { + var model = models.stream().filter(m -> m.get("inference_id").equals(inferenceId)).findFirst(); + assertTrue("could not find inference id: " + inferenceId, model.isPresent()); + assertThat(model.get().get("task_type"), is(taskType.toString())); } } diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java index 0fa8d39d4c89..5205ce07a067 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java @@ -204,6 +204,7 @@ public class InferenceRevokeDefaultEndpointsIT extends ESSingleNodeTestCase { service.defaultConfigIds(), is( List.of( + new InferenceService.DefaultConfigId(".elser-v2-elastic", MinimalServiceSettings.sparseEmbedding(), service), new InferenceService.DefaultConfigId( ".rainbow-sprinkles-elastic", MinimalServiceSettings.chatCompletion(), @@ -216,7 +217,8 @@ public class InferenceRevokeDefaultEndpointsIT extends ESSingleNodeTestCase { PlainActionFuture> listener = new PlainActionFuture<>(); service.defaultConfigs(listener); - assertThat(listener.actionGet(TIMEOUT).get(0).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic")); + assertThat(listener.actionGet(TIMEOUT).get(0).getConfigurations().getInferenceEntityId(), is(".elser-v2-elastic")); + assertThat(listener.actionGet(TIMEOUT).get(1).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic")); var getModelListener = new PlainActionFuture(); // persists the default endpoints @@ -244,12 +246,18 @@ public class InferenceRevokeDefaultEndpointsIT extends ESSingleNodeTestCase { try (var service = createElasticInferenceService()) { service.waitForAuthorizationToComplete(TIMEOUT); assertThat(service.supportedStreamingTasks(), is(EnumSet.noneOf(TaskType.class))); - assertTrue(service.defaultConfigIds().isEmpty()); + assertThat( + service.defaultConfigIds(), + is( + List.of( + new InferenceService.DefaultConfigId(".elser-v2-elastic", MinimalServiceSettings.sparseEmbedding(), service) + ) + ) + ); assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING))); var getModelListener = new PlainActionFuture(); modelRegistry.getModel(".rainbow-sprinkles-elastic", getModelListener); - var exception = expectThrows(ResourceNotFoundException.class, () -> getModelListener.actionGet(TIMEOUT)); assertThat(exception.getMessage(), is("Inference endpoint not found [.rainbow-sprinkles-elastic]")); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java index 2bcb130ddccb..ca7595f78da0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java @@ -127,10 +127,19 @@ public class ModelRegistry { } /** - * Set the default inference ids provided by the services - * @param defaultConfigId The default + * Adds the default configuration information if it does not already exist internally. + * @param defaultConfigId the default endpoint information */ - public synchronized void addDefaultIds(InferenceService.DefaultConfigId defaultConfigId) { + public synchronized void putDefaultIdIfAbsent(InferenceService.DefaultConfigId defaultConfigId) { + defaultConfigIds.putIfAbsent(defaultConfigId.inferenceId(), defaultConfigId); + } + + /** + * Set the default inference ids provided by the services + * @param defaultConfigId The default endpoint information + * @throws IllegalStateException if the {@link InferenceService.DefaultConfigId#inferenceId()} already exists internally + */ + public synchronized void addDefaultIds(InferenceService.DefaultConfigId defaultConfigId) throws IllegalStateException { var config = defaultConfigIds.get(defaultConfigId.inferenceId()); if (config != null) { throw new IllegalStateException( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java index c4e961d9e9f6..fee66a9f84ac 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java @@ -57,6 +57,7 @@ import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; import org.elasticsearch.xpack.inference.telemetry.TraceContext; import java.util.ArrayList; +import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -65,6 +66,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -90,14 +92,24 @@ public class ElasticInferenceService extends SenderService { private static final Logger logger = LogManager.getLogger(ElasticInferenceService.class); private static final EnumSet IMPLEMENTED_TASK_TYPES = EnumSet.of(TaskType.SPARSE_EMBEDDING, TaskType.CHAT_COMPLETION); private static final String SERVICE_NAME = "Elastic"; + + // rainbow-sprinkles static final String DEFAULT_CHAT_COMPLETION_MODEL_ID_V1 = "rainbow-sprinkles"; - static final String DEFAULT_CHAT_COMPLETION_ENDPOINT_ID_V1 = Strings.format(".%s-elastic", DEFAULT_CHAT_COMPLETION_MODEL_ID_V1); + static final String DEFAULT_CHAT_COMPLETION_ENDPOINT_ID_V1 = defaultEndpointId(DEFAULT_CHAT_COMPLETION_MODEL_ID_V1); + + // elser-v2 + static final String DEFAULT_ELSER_MODEL_ID_V2 = "elser-v2"; + static final String DEFAULT_ELSER_ENDPOINT_ID_V2 = defaultEndpointId(DEFAULT_ELSER_MODEL_ID_V2); /** * The task types that the {@link InferenceAction.Request} can accept. */ private static final EnumSet SUPPORTED_INFERENCE_ACTION_TASK_TYPES = EnumSet.of(TaskType.SPARSE_EMBEDDING); + private static String defaultEndpointId(String modelId) { + return Strings.format(".%s-elastic", modelId); + } + private final ElasticInferenceServiceComponents elasticInferenceServiceComponents; private Configuration configuration; private final AtomicReference authRef = new AtomicReference<>(AuthorizedContent.empty()); @@ -142,6 +154,19 @@ public class ElasticInferenceService extends SenderService { elasticInferenceServiceComponents ), MinimalServiceSettings.chatCompletion() + ), + DEFAULT_ELSER_MODEL_ID_V2, + new DefaultModelConfig( + new ElasticInferenceServiceSparseEmbeddingsModel( + DEFAULT_ELSER_ENDPOINT_ID_V2, + TaskType.SPARSE_EMBEDDING, + NAME, + new ElasticInferenceServiceSparseEmbeddingsServiceSettings(DEFAULT_ELSER_MODEL_ID_V2, null, null), + EmptyTaskSettings.INSTANCE, + EmptySecretSettings.INSTANCE, + elasticInferenceServiceComponents + ), + MinimalServiceSettings.sparseEmbedding() ) ); } @@ -184,13 +209,13 @@ public class ElasticInferenceService extends SenderService { configuration = new Configuration(authRef.get().taskTypesAndModels.getAuthorizedTaskTypes()); - defaultConfigIds().forEach(modelRegistry::addDefaultIds); + defaultConfigIds().forEach(modelRegistry::putDefaultIdIfAbsent); handleRevokedDefaultConfigs(authorizedDefaultModelIds); } private Set getAuthorizedDefaultModelIds(ElasticInferenceServiceAuthorization auth) { var authorizedModels = auth.getAuthorizedModelIds(); - var authorizedDefaultModelIds = new HashSet<>(defaultModelsConfigs.keySet()); + var authorizedDefaultModelIds = new TreeSet<>(defaultModelsConfigs.keySet()); authorizedDefaultModelIds.retainAll(authorizedModels); return authorizedDefaultModelIds; @@ -218,6 +243,7 @@ public class ElasticInferenceService extends SenderService { } } + authorizedConfigIds.sort(Comparator.comparing(DefaultConfigId::inferenceId)); return authorizedConfigIds; } @@ -230,6 +256,7 @@ public class ElasticInferenceService extends SenderService { } } + authorizedModels.sort(Comparator.comparing(modelConfig -> modelConfig.model.getInferenceEntityId())); return authorizedModels; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java index 175f03f14673..9ac42f66a0c4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java @@ -75,7 +75,7 @@ public class ElasticInferenceServiceSparseEmbeddingsServiceSettings extends Filt public ElasticInferenceServiceSparseEmbeddingsServiceSettings( String modelId, @Nullable Integer maxInputTokens, - RateLimitSettings rateLimitSettings + @Nullable RateLimitSettings rateLimitSettings ) { this.modelId = Objects.requireNonNull(modelId); this.maxInputTokens = maxInputTokens; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index 46c2435ed1fe..bf610577ab08 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -934,13 +934,17 @@ public class ElasticInferenceServiceTests extends ESTestCase { } } - public void testDefaultConfigs_Returns_DefaultChatCompletion_V1_WhenTaskTypeIsCorrect() throws Exception { + public void testDefaultConfigs_Returns_DefaultEndpoints_WhenTaskTypeIsCorrect() throws Exception { String responseJson = """ { "models": [ { "model_name": "rainbow-sprinkles", "task_types": ["chat"] + }, + { + "model_name": "elser-v2", + "task_types": ["embed/text/sparse"] } ] } @@ -957,15 +961,19 @@ public class ElasticInferenceServiceTests extends ESTestCase { service.defaultConfigIds(), is( List.of( + new InferenceService.DefaultConfigId(".elser-v2-elastic", MinimalServiceSettings.sparseEmbedding(), service), new InferenceService.DefaultConfigId(".rainbow-sprinkles-elastic", MinimalServiceSettings.chatCompletion(), service) ) ) ); - assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.CHAT_COMPLETION))); + assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.SPARSE_EMBEDDING))); PlainActionFuture> listener = new PlainActionFuture<>(); service.defaultConfigs(listener); - assertThat(listener.actionGet(TIMEOUT).get(0).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic")); + var models = listener.actionGet(TIMEOUT); + assertThat(models.size(), is(2)); + assertThat(models.get(0).getConfigurations().getInferenceEntityId(), is(".elser-v2-elastic")); + assertThat(models.get(1).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic")); } } From b8babb663af2a794eb9f05d9cdbf344eccd91c6d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:37:06 +1100 Subject: [PATCH 18/93] Mute org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_488} #121611 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1f42941579cf..83325f0c2c68 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -399,6 +399,9 @@ tests: - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/alias/line_260} issue: https://github.com/elastic/elasticsearch/issues/122343 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_488} + issue: https://github.com/elastic/elasticsearch/issues/121611 # Examples: # From 55f5d68409944c51693ca9e083a6e5c320f1fa54 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:43:25 +1100 Subject: [PATCH 19/93] Mute org.elasticsearch.repositories.blobstore.testkit.analyze.SecureHdfsRepositoryAnalysisRestIT org.elasticsearch.repositories.blobstore.testkit.analyze.SecureHdfsRepositoryAnalysisRestIT #122377 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 83325f0c2c68..ee56c67f67bb 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -402,6 +402,8 @@ tests: - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_488} issue: https://github.com/elastic/elasticsearch/issues/121611 +- class: org.elasticsearch.repositories.blobstore.testkit.analyze.SecureHdfsRepositoryAnalysisRestIT + issue: https://github.com/elastic/elasticsearch/issues/122377 # Examples: # From a7957a8cc6617a581aa1e212bde3f35912b52431 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:43:36 +1100 Subject: [PATCH 20/93] Mute org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT #122378 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index ee56c67f67bb..25ff3ab6ac93 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -404,6 +404,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/121611 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.SecureHdfsRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/122377 +- class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT + issue: https://github.com/elastic/elasticsearch/issues/122378 # Examples: # From 05a2003a9f363ce379af17d8393682f67b8c13e8 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Wed, 12 Feb 2025 08:00:16 -0600 Subject: [PATCH 21/93] Fix ReindexDataStreamIndexAction timestamp validation bug in tests (#122274) Fix race condition test bugs related to the reindex-data-stream-pipeline. For tests that add doc without timestamp, then add mapping with timestamp, ensure green between adding doc and adding mapping. This makes sure that doc has been written to all shards and thus that timestamp validation does not occur while doc is being written to a shard. Delete pipeline in Before method, then wait for it to be re-created by the MigrateTemplateRegistry. --- muted-tests.yml | 2 -- ...indexDatastreamIndexTransportActionIT.java | 28 ++++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 25ff3ab6ac93..ef820b14770c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -374,8 +374,6 @@ tests: - class: org.elasticsearch.xpack.security.authz.IndexAliasesTests method: testRemoveIndex issue: https://github.com/elastic/elasticsearch/issues/122221 -- class: org.elasticsearch.xpack.migrate.action.ReindexDatastreamIndexTransportActionIT - issue: https://github.com/elastic/elasticsearch/issues/121737 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT method: testGroupingMultiValueByOrdinals issue: https://github.com/elastic/elasticsearch/issues/122228 diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index a3642ddb664d..e3b73d0aaa5c 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -49,7 +49,7 @@ import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.migrate.MigratePlugin; import org.elasticsearch.xpack.migrate.MigrateTemplateRegistry; -import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.time.Instant; @@ -67,9 +67,11 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { - @After - private void cleanup() { + + @Before + private void setup() throws Exception { deletePipeline(MigrateTemplateRegistry.REINDEX_DATA_STREAM_PIPELINE_NAME); + assertBusy(() -> { assertTrue(getPipelines(MigrateTemplateRegistry.REINDEX_DATA_STREAM_PIPELINE_NAME).isFound()); }); } private static final String MAPPING = """ @@ -114,6 +116,9 @@ public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { // add doc without timestamp addDoc(sourceIndex, "{\"foo\":\"baz\"}"); + // wait until doc is written to all shards before adding mapping + ensureHealth(sourceIndex); + // add timestamp to source mapping indicesAdmin().preparePutMapping(sourceIndex).setSource(DATA_STREAM_MAPPING, XContentType.JSON).get(); @@ -129,6 +134,7 @@ public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { } public void testTimestampNotAddedIfExists() { + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); @@ -137,6 +143,9 @@ public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { var doc = String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, time); addDoc(sourceIndex, doc); + // wait until doc is written to all shards before adding mapping + ensureHealth(sourceIndex); + // add timestamp to source mapping indicesAdmin().preparePutMapping(sourceIndex).setSource(DATA_STREAM_MAPPING, XContentType.JSON).get(); @@ -184,6 +193,9 @@ public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { var doc = String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, time); addDoc(sourceIndex, doc); + // wait until doc is written to all shards before adding mapping + ensureHealth(sourceIndex); + // add timestamp to source mapping indicesAdmin().preparePutMapping(sourceIndex).setSource(DATA_STREAM_MAPPING, XContentType.JSON).get(); @@ -293,7 +305,7 @@ public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { ); } - public void testSettingsAddedBeforeReindex() throws Exception { + public void testSettingsAddedBeforeReindex() { // start with a static setting var numShards = randomIntBetween(1, 10); var staticSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build(); @@ -604,4 +616,12 @@ public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { bulkRequest.add(new IndexRequest(index).opType(DocWriteRequest.OpType.CREATE).source(doc, XContentType.JSON)); safeGet(client().bulk(bulkRequest)); } + + private void ensureHealth(String index) { + if (cluster().numDataNodes() > 1) { + ensureGreen(index); + } else { + ensureYellow(index); + } + } } From 5f00b64ec77c86f01e743dcf0ab173d565c632d2 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 12 Feb 2025 06:20:34 -0800 Subject: [PATCH 22/93] Instrument methods on File that require write permission (#122109) This commit adds instrumentation for File methods that require write permission. No server or plugins use these methods, so no policy changes were necessary. Note that since we are not planning to restrict temp file creation, the bootstrap self test on file writing was removed, which failed with these changes. --- .../bridge/EntitlementChecker.java | 30 +++++++ .../qa/entitled/EntitledActions.java | 4 + .../entitlement/qa/test/FileCheckActions.java | 86 +++++++++++++++++++ .../bootstrap/EntitlementBootstrap.java | 35 -------- .../EntitlementInitialization.java | 14 ++- .../api/ElasticsearchEntitlementChecker.java | 76 ++++++++++++++++ 6 files changed, 209 insertions(+), 36 deletions(-) diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 6e6e5f306c80..cbedccc2a37b 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -501,6 +501,36 @@ public interface EntitlementChecker { // // old io (ie File) + void check$java_io_File$createNewFile(Class callerClass, File file); + + void check$java_io_File$$createTempFile(Class callerClass, String prefix, String suffix, File directory); + + void check$java_io_File$delete(Class callerClass, File file); + + void check$java_io_File$deleteOnExit(Class callerClass, File file); + + void check$java_io_File$mkdir(Class callerClass, File file); + + void check$java_io_File$mkdirs(Class callerClass, File file); + + void check$java_io_File$renameTo(Class callerClass, File file, File dest); + + void check$java_io_File$setExecutable(Class callerClass, File file, boolean executable); + + void check$java_io_File$setExecutable(Class callerClass, File file, boolean executable, boolean ownerOnly); + + void check$java_io_File$setLastModified(Class callerClass, File file, long time); + + void check$java_io_File$setReadable(Class callerClass, File file, boolean readable); + + void check$java_io_File$setReadable(Class callerClass, File file, boolean readable, boolean ownerOnly); + + void check$java_io_File$setReadOnly(Class callerClass, File file); + + void check$java_io_File$setWritable(Class callerClass, File file, boolean writable); + + void check$java_io_File$setWritable(Class callerClass, File file, boolean writable, boolean ownerOnly); + void check$java_io_FileOutputStream$(Class callerClass, File file); void check$java_io_FileOutputStream$(Class callerClass, File file, boolean append); diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java index 24d7472e07c6..640cc75d7ac7 100644 --- a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java +++ b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java @@ -27,4 +27,8 @@ public final class EntitledActions { public static UserPrincipal getFileOwner(Path path) throws IOException { return Files.getOwner(path); } + + public static void createFile(Path path) throws IOException { + Files.createFile(path); + } } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java index 4a15d63f99ec..29736a46040e 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java @@ -12,6 +12,7 @@ package org.elasticsearch.entitlement.qa.test; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.qa.entitled.EntitledActions; +import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -45,6 +46,91 @@ class FileCheckActions { return testRootDir.resolve("read_write_file"); } + @EntitlementTest(expectedAccess = PLUGINS) + static void fileCreateNewFile() throws IOException { + readWriteDir().resolve("new_file").toFile().createNewFile(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileCreateTempFile() throws IOException { + File.createTempFile("prefix", "suffix", readWriteDir().toFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileDelete() throws IOException { + Path toDelete = readWriteDir().resolve("to_delete"); + EntitledActions.createFile(toDelete); + toDelete.toFile().delete(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileDeleteOnExit() throws IOException { + Path toDelete = readWriteDir().resolve("to_delete_on_exit"); + EntitledActions.createFile(toDelete); + toDelete.toFile().deleteOnExit(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileMkdir() throws IOException { + Path mkdir = readWriteDir().resolve("mkdir"); + mkdir.toFile().mkdir(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileMkdirs() throws IOException { + Path mkdir = readWriteDir().resolve("mkdirs"); + mkdir.toFile().mkdirs(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileRenameTo() throws IOException { + Path toRename = readWriteDir().resolve("to_rename"); + EntitledActions.createFile(toRename); + toRename.toFile().renameTo(readWriteDir().resolve("renamed").toFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetExecutable() throws IOException { + readWriteFile().toFile().setExecutable(false); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetExecutableOwner() throws IOException { + readWriteFile().toFile().setExecutable(false, false); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetLastModified() throws IOException { + readWriteFile().toFile().setLastModified(System.currentTimeMillis()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetReadable() throws IOException { + readWriteFile().toFile().setReadable(true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetReadableOwner() throws IOException { + readWriteFile().toFile().setReadable(true, false); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetReadOnly() throws IOException { + Path readOnly = readWriteDir().resolve("read_only"); + EntitledActions.createFile(readOnly); + readOnly.toFile().setReadOnly(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetWritable() throws IOException { + readWriteFile().toFile().setWritable(true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetWritableOwner() throws IOException { + readWriteFile().toFile().setWritable(true, false); + } + @EntitlementTest(expectedAccess = PLUGINS) static void createScannerFile() throws FileNotFoundException { new Scanner(readFile().toFile()); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java index 4badc4bb3a44..19acd0decdca 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java @@ -15,7 +15,6 @@ import com.sun.tools.attach.AttachNotSupportedException; import com.sun.tools.attach.VirtualMachine; import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.core.CheckedSupplier; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.initialization.EntitlementInitialization; import org.elasticsearch.entitlement.runtime.api.NotEntitledException; @@ -27,7 +26,6 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; import java.util.Map; import java.util.function.Function; @@ -149,11 +147,8 @@ public class EntitlementBootstrap { */ private static void selfTest() { ensureCannotStartProcess(ProcessBuilder::start); - ensureCanCreateTempFile(EntitlementBootstrap::createTempFile); - // Try again with reflection ensureCannotStartProcess(EntitlementBootstrap::reflectiveStartProcess); - ensureCanCreateTempFile(EntitlementBootstrap::reflectiveCreateTempFile); } private static void ensureCannotStartProcess(CheckedConsumer startProcess) { @@ -169,31 +164,6 @@ public class EntitlementBootstrap { throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); } - @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test") - private static void ensureCanCreateTempFile(CheckedSupplier createTempFile) { - try { - Path p = createTempFile.get(); - p.toFile().deleteOnExit(); - - // Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally. - try { - Files.delete(p); - } catch (IOException ignored) { - // Can be caused by virus scanner - } - } catch (NotEntitledException e) { - throw new IllegalStateException("Entitlement protection self-test was incorrectly forbidden", e); - } catch (Exception e) { - throw new IllegalStateException("Unable to perform entitlement protection self-test", e); - } - logger.debug("Success: Entitlement protection correctly permitted temp file creation"); - } - - @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test") - private static Path createTempFile() throws Exception { - return Files.createTempFile(null, null); - } - private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception { try { var start = ProcessBuilder.class.getMethod("start"); @@ -203,10 +173,5 @@ public class EntitlementBootstrap { } } - private static Path reflectiveCreateTempFile() throws Exception { - return (Path) Files.class.getMethod("createTempFile", String.class, String.class, FileAttribute[].class) - .invoke(null, null, null, new FileAttribute[0]); - } - private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 0b0caa0745fc..f99ba1040436 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -24,6 +24,8 @@ import org.elasticsearch.entitlement.runtime.policy.Scope; import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.FileData; import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; @@ -39,6 +41,7 @@ import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.spi.FileSystemProvider; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,6 +50,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE; + /** * Called by the agent during {@code agentmain} to configure the entitlement system, * instantiate and configure an {@link EntitlementChecker}, @@ -121,6 +126,7 @@ public class EntitlementInitialization { private static PolicyManager createPolicyManager() { Map pluginPolicies = EntitlementBootstrap.bootstrapArgs().pluginPolicies(); + Path[] dataDirs = EntitlementBootstrap.bootstrapArgs().dataDirs(); // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it var serverPolicy = new Policy( @@ -142,7 +148,13 @@ public class EntitlementInitialization { new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())), new Scope("io.netty.transport", List.of(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement())), new Scope("org.apache.lucene.core", List.of(new LoadNativeLibrariesEntitlement())), - new Scope("org.elasticsearch.nativeaccess", List.of(new LoadNativeLibrariesEntitlement())) + new Scope( + "org.elasticsearch.nativeaccess", + List.of( + new LoadNativeLibrariesEntitlement(), + new FilesEntitlement(Arrays.stream(dataDirs).map(d -> new FileData(d.toString(), READ_WRITE)).toList()) + ) + ) ) ); // agents run without a module, so this is a special hack for the apm agent diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index 156ae9cc4a35..632be24d83cd 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -941,6 +941,82 @@ public class ElasticsearchEntitlementChecker implements EntitlementChecker { // old io (ie File) + @Override + public void check$java_io_File$createNewFile(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$$createTempFile(Class callerClass, String prefix, String suffix, File directory) { + policyManager.checkFileWrite(callerClass, directory); + } + + @Override + public void check$java_io_File$delete(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$deleteOnExit(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$mkdir(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$mkdirs(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$renameTo(Class callerClass, File file, File dest) { + policyManager.checkFileRead(callerClass, file); + policyManager.checkFileWrite(callerClass, dest); + } + + @Override + public void check$java_io_File$setExecutable(Class callerClass, File file, boolean executable) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setExecutable(Class callerClass, File file, boolean executable, boolean ownerOnly) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setLastModified(Class callerClass, File file, long time) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setReadable(Class callerClass, File file, boolean readable) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setReadable(Class callerClass, File file, boolean readable, boolean ownerOnly) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setReadOnly(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setWritable(Class callerClass, File file, boolean writable) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setWritable(Class callerClass, File file, boolean writable, boolean ownerOnly) { + policyManager.checkFileWrite(callerClass, file); + } + @Override public void check$java_io_FileOutputStream$(Class callerClass, String name) { policyManager.checkFileWrite(callerClass, new File(name)); From d1bbc4ceb87b958b019a730e1a1b5651a41152f5 Mon Sep 17 00:00:00 2001 From: Adam Demjen Date: Wed, 12 Feb 2025 09:23:31 -0500 Subject: [PATCH 23/93] [Inference API] Rename model_id prop to model in EIS sparse inference request body (#122272) * Rename model_id prop to model in EIS sparse inference request body * Update docs/changelog/122272.yaml * Fix broken tests --- docs/changelog/122272.yaml | 6 ++++++ ...sticInferenceServiceSparseEmbeddingsRequestEntity.java | 4 ++-- .../ElasticInferenceServiceActionCreatorTests.java | 4 ++-- ...nferenceServiceSparseEmbeddingsRequestEntityTests.java | 8 ++++---- ...asticInferenceServiceSparseEmbeddingsRequestTests.java | 4 ++-- .../services/elastic/ElasticInferenceServiceTests.java | 4 ++-- 6 files changed, 18 insertions(+), 12 deletions(-) create mode 100644 docs/changelog/122272.yaml diff --git a/docs/changelog/122272.yaml b/docs/changelog/122272.yaml new file mode 100644 index 000000000000..62e576917940 --- /dev/null +++ b/docs/changelog/122272.yaml @@ -0,0 +1,6 @@ +pr: 122272 +summary: "[Inference API] Rename `model_id` prop to model in EIS sparse inference\ + \ request body" +area: Inference +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java index 77ae48e6ccdc..0ba6b46da05e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java @@ -23,7 +23,7 @@ public record ElasticInferenceServiceSparseEmbeddingsRequestEntity( ) implements ToXContentObject { private static final String INPUT_FIELD = "input"; - private static final String MODEL_ID_FIELD = "model_id"; + private static final String MODEL_FIELD = "model"; private static final String USAGE_CONTEXT = "usage_context"; public ElasticInferenceServiceSparseEmbeddingsRequestEntity { @@ -42,7 +42,7 @@ public record ElasticInferenceServiceSparseEmbeddingsRequestEntity( builder.endArray(); - builder.field(MODEL_ID_FIELD, modelId); + builder.field(MODEL_FIELD, modelId); // optional field if ((usageContext == ElasticInferenceServiceUsageContext.UNSPECIFIED) == false) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java index e1d2ee56733e..28e182aa2d43 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java @@ -124,7 +124,7 @@ public class ElasticInferenceServiceActionCreatorTests extends ESTestCase { assertThat(requestMap.get("input"), instanceOf(List.class)); var inputList = (List) requestMap.get("input"); assertThat(inputList, contains("hello world")); - assertThat(requestMap.get("model_id"), is("my-model-id")); + assertThat(requestMap.get("model"), is("my-model-id")); } } @@ -179,7 +179,7 @@ public class ElasticInferenceServiceActionCreatorTests extends ESTestCase { assertThat(requestMap.get("input"), instanceOf(List.class)); var inputList = (List) requestMap.get("input"); assertThat(inputList, contains("hello world")); - assertThat(requestMap.get("model_id"), is("my-model-id")); + assertThat(requestMap.get("model"), is("my-model-id")); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java index c0ebaf8668c5..f81f6e58964f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java @@ -31,7 +31,7 @@ public class ElasticInferenceServiceSparseEmbeddingsRequestEntityTests extends E assertThat(xContentString, equalToIgnoringWhitespaceInJsonString(""" { "input": ["abc"], - "model_id": "my-model-id" + "model": "my-model-id" }""")); } @@ -48,7 +48,7 @@ public class ElasticInferenceServiceSparseEmbeddingsRequestEntityTests extends E "abc", "def" ], - "model_id": "my-model-id" + "model": "my-model-id" } """)); } @@ -63,7 +63,7 @@ public class ElasticInferenceServiceSparseEmbeddingsRequestEntityTests extends E assertThat(xContentString, equalToIgnoringWhitespaceInJsonString(""" { "input": ["abc"], - "model_id": "my-model-id", + "model": "my-model-id", "usage_context": "search" } """)); @@ -79,7 +79,7 @@ public class ElasticInferenceServiceSparseEmbeddingsRequestEntityTests extends E assertThat(xContentString, equalToIgnoringWhitespaceInJsonString(""" { "input": ["abc"], - "model_id": "my-model-id", + "model": "my-model-id", "usage_context": "ingest" } """)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java index abcc94640981..9211b55236b1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java @@ -46,7 +46,7 @@ public class ElasticInferenceServiceSparseEmbeddingsRequestTests extends ESTestC var requestMap = entityAsMap(httpPost.getEntity().getContent()); assertThat(requestMap.size(), equalTo(3)); assertThat(requestMap.get("input"), is(List.of(input))); - assertThat(requestMap.get("model_id"), is(modelId)); + assertThat(requestMap.get("model"), is(modelId)); assertThat(requestMap.get("usage_context"), equalTo("search")); } @@ -83,7 +83,7 @@ public class ElasticInferenceServiceSparseEmbeddingsRequestTests extends ESTestC var requestMap = entityAsMap(httpPost.getEntity().getContent()); assertThat(requestMap, aMapWithSize(2)); assertThat(requestMap.get("input"), is(List.of("ab"))); - assertThat(requestMap.get("model_id"), is(modelId)); + assertThat(requestMap.get("model"), is(modelId)); } public void testIsTruncated_ReturnsTrue() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index bf610577ab08..414c2a3f943d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -504,7 +504,7 @@ public class ElasticInferenceServiceTests extends ESTestCase { assertThat(request.getHeader(HttpHeaders.CONTENT_TYPE), Matchers.equalTo(XContentType.JSON.mediaType())); var requestMap = entityAsMap(request.getBody()); - assertThat(requestMap, is(Map.of("input", List.of("input text"), "model_id", "my-model-id", "usage_context", "search"))); + assertThat(requestMap, is(Map.of("input", List.of("input text"), "model", "my-model-id", "usage_context", "search"))); } } @@ -562,7 +562,7 @@ public class ElasticInferenceServiceTests extends ESTestCase { ); var requestMap = entityAsMap(webServer.requests().get(0).getBody()); - assertThat(requestMap, is(Map.of("input", List.of("input text"), "model_id", "my-model-id", "usage_context", "ingest"))); + assertThat(requestMap, is(Map.of("input", List.of("input text"), "model", "my-model-id", "usage_context", "ingest"))); } } From dfcf47920443d418bd06a6eb822905f47c9e4da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Wed, 12 Feb 2025 15:45:26 +0100 Subject: [PATCH 24/93] ESQL: Emit warnings on LOOKUP JOIN multivalues (#121219) Fixes https://github.com/elastic/elasticsearch/issues/118780 --- .../org/elasticsearch/TransportVersions.java | 3 + .../compute/operator/AsyncOperator.java | 12 ++- .../compute/operator/Warnings.java | 13 ++- .../compute/operator/lookup/QueryList.java | 74 +++++++++++----- .../querydsl/query/SingleValueMatchQuery.java | 17 ++-- .../compute/operator/AsyncOperatorTests.java | 8 +- .../compute/operator/DriverTests.java | 2 +- .../EnrichQuerySourceOperatorTests.java | 19 +++-- .../src/main/resources/lookup-join.csv-spec | 67 ++++++++++++++- .../xpack/esql/action/EsqlCapabilities.java | 9 +- .../esql/enrich/AbstractLookupService.java | 10 ++- .../esql/enrich/EnrichLookupOperator.java | 12 +-- .../esql/enrich/EnrichLookupService.java | 9 +- .../esql/enrich/LookupFromIndexOperator.java | 2 +- .../esql/enrich/LookupFromIndexService.java | 25 +++++- .../xpack/esql/plan/logical/join/Join.java | 2 +- .../esql/querydsl/query/SingleValueQuery.java | 3 +- .../plan/logical/JoinSerializationTests.java | 5 -- .../query/SingleValueMathQueryTests.java | 6 +- .../test/esql/190_lookup_join.yml | 85 ++++++++++++++++++- 20 files changed, 302 insertions(+), 81 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 37473c565189..058251e3c269 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -185,6 +185,7 @@ public class TransportVersions { public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION = def(9_005_0_00); public static final TransportVersion ESQL_RETRY_ON_SHARD_LEVEL_FAILURE = def(9_006_0_00); public static final TransportVersion ESQL_PROFILE_ASYNC_NANOS = def(9_007_00_0); + public static final TransportVersion ESQL_LOOKUP_JOIN_SOURCE_TEXT = def(9_008_0_00); /* * STOP! READ THIS FIRST! No, really, @@ -197,6 +198,8 @@ public class TransportVersions { * A new transport version should be added EVERY TIME a change is made to the serialization protocol of one or more classes. Each * transport version should only be used in a single merged commit (apart from the BwC versions copied from o.e.Version, ≤V_8_8_1). * + * More information about versions and backporting at docs/internal/Versioning.md + * * ADDING A TRANSPORT VERSION * To add a new transport version, add a new constant at the bottom of the list, above this comment. Don't add other lines, * comments, etc. The version id has the following layout: diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java index 112fc44311a5..0d5d86fb186e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.seqno.LocalCheckpointTracker; @@ -34,6 +35,11 @@ import java.util.concurrent.atomic.LongAdder; * to reduce communication overhead and fetches a {@code Fetched} at a time. * It's the responsibility of subclasses to transform that {@code Fetched} into * output. + *

+ * This operator will also take care of merging response headers from the thread context into the main thread, + * which must be the one that closes this. + *

+ * * @see #performAsync(Page, ActionListener) */ public abstract class AsyncOperator implements Operator { @@ -45,6 +51,7 @@ public abstract class AsyncOperator implements Operator { private final DriverContext driverContext; private final int maxOutstandingRequests; + private final ResponseHeadersCollector responseHeadersCollector; private final LongAdder processNanos = new LongAdder(); private boolean finished = false; @@ -66,9 +73,10 @@ public abstract class AsyncOperator implements Operator { * * @param maxOutstandingRequests the maximum number of outstanding requests */ - public AsyncOperator(DriverContext driverContext, int maxOutstandingRequests) { + public AsyncOperator(DriverContext driverContext, ThreadContext threadContext, int maxOutstandingRequests) { this.driverContext = driverContext; this.maxOutstandingRequests = maxOutstandingRequests; + this.responseHeadersCollector = new ResponseHeadersCollector(threadContext); } @Override @@ -97,6 +105,7 @@ public abstract class AsyncOperator implements Operator { }); final long startNanos = System.nanoTime(); performAsync(input, ActionListener.runAfter(listener, () -> { + responseHeadersCollector.collect(); driverContext.removeAsyncAction(); processNanos.add(System.nanoTime() - startNanos); })); @@ -172,6 +181,7 @@ public abstract class AsyncOperator implements Operator { finish(); closed = true; discardResults(); + responseHeadersCollector.finish(); doClose(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Warnings.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Warnings.java index 999ecca91619..9ecb83853ec2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Warnings.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Warnings.java @@ -102,12 +102,23 @@ public class Warnings { } public void registerException(Exception exception) { + registerException(exception.getClass(), exception.getMessage()); + } + + /** + * Register an exception to be included in the warnings. + *

+ * This overload avoids the need to instantiate the exception, which can be expensive. + * Instead, it asks only the required pieces to build the warning. + *

+ */ + public void registerException(Class exceptionClass, String message) { if (addedWarnings < MAX_ADDED_WARNINGS) { if (addedWarnings == 0) { addWarning(first); } // location needs to be added to the exception too, since the headers are deduplicated - addWarning(location + exception.getClass().getName() + ": " + exception.getMessage()); + addWarning(location + exceptionClass.getName() + ": " + message); addedWarnings++; } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index 5d359e2fb612..f05d552c3e62 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; import java.util.function.IntFunction; @@ -47,13 +48,19 @@ public abstract class QueryList { protected final SearchExecutionContext searchExecutionContext; protected final MappedFieldType field; protected final Block block; - protected final boolean onlySingleValues; + @Nullable + protected final OnlySingleValueParams onlySingleValueParams; - protected QueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block, boolean onlySingleValues) { + protected QueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + Block block, + OnlySingleValueParams onlySingleValueParams + ) { this.searchExecutionContext = searchExecutionContext; this.field = field; this.block = block; - this.onlySingleValues = onlySingleValues; + this.onlySingleValueParams = onlySingleValueParams; } /** @@ -66,19 +73,27 @@ public abstract class QueryList { /** * Returns a copy of this query list that only returns queries for single-valued positions. * That is, it returns `null` queries for either multivalued or null positions. + *

+ * Whenever a multi-value position is encountered, whether in the input block or in the queried index, a warning is emitted. + *

*/ - public abstract QueryList onlySingleValues(); + public abstract QueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage); final Query getQuery(int position) { final int valueCount = block.getValueCount(position); - if (onlySingleValues && valueCount != 1) { + if (onlySingleValueParams != null && valueCount != 1) { + if (valueCount > 1) { + onlySingleValueParams.warnings.registerException( + new IllegalArgumentException(onlySingleValueParams.multiValueWarningMessage) + ); + } return null; } final int firstValueIndex = block.getFirstValueIndex(position); Query query = doGetQuery(position, firstValueIndex, valueCount); - if (onlySingleValues) { + if (onlySingleValueParams != null) { query = wrapSingleValueQuery(query); } @@ -92,13 +107,16 @@ public abstract class QueryList { abstract Query doGetQuery(int position, int firstValueIndex, int valueCount); private Query wrapSingleValueQuery(Query query) { + assert onlySingleValueParams != null : "Requested to wrap single value query without single value params"; + SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery( searchExecutionContext.getForField(field, MappedFieldType.FielddataOperation.SEARCH), // Not emitting warnings for multivalued fields not matching - Warnings.NOOP_WARNINGS + onlySingleValueParams.warnings, + onlySingleValueParams.multiValueWarningMessage ); - Query rewrite = singleValueQuery; + Query rewrite; try { rewrite = singleValueQuery.rewrite(searchExecutionContext.searcher()); if (rewrite instanceof MatchAllDocsQuery) { @@ -106,8 +124,7 @@ public abstract class QueryList { return query; } } catch (IOException e) { - // ignore - // TODO: Should we do something with the exception? + throw new UncheckedIOException("Error while rewriting SingleValueQuery", e); } BooleanQuery.Builder builder = new BooleanQuery.Builder(); @@ -152,7 +169,7 @@ public abstract class QueryList { case COMPOSITE -> throw new IllegalArgumentException("can't read values from [composite] block"); case UNKNOWN -> throw new IllegalArgumentException("can't read values from [" + block + "]"); }; - return new TermQueryList(field, searchExecutionContext, block, false, blockToJavaObject); + return new TermQueryList(field, searchExecutionContext, block, null, blockToJavaObject); } /** @@ -162,7 +179,7 @@ public abstract class QueryList { public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, BytesRefBlock block) { BytesRef scratch = new BytesRef(); byte[] ipBytes = new byte[InetAddressPoint.BYTES]; - return new TermQueryList(field, searchExecutionContext, block, false, offset -> { + return new TermQueryList(field, searchExecutionContext, block, null, offset -> { final var bytes = block.getBytesRef(offset, scratch); if (ipBytes.length != bytes.length) { // Lucene only support 16-byte IP addresses, even IPv4 is encoded in 16 bytes @@ -182,7 +199,7 @@ public abstract class QueryList { field, searchExecutionContext, block, - false, + null, field instanceof RangeFieldMapper.RangeFieldType rangeFieldType ? offset -> rangeFieldType.dateTimeFormatter().formatMillis(block.getLong(offset)) : block::getLong @@ -193,7 +210,7 @@ public abstract class QueryList { * Returns a list of geo_shape queries for the given field and the input block. */ public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) { - return new GeoShapeQueryList(field, searchExecutionContext, block, false); + return new GeoShapeQueryList(field, searchExecutionContext, block, null); } private static class TermQueryList extends QueryList { @@ -203,16 +220,22 @@ public abstract class QueryList { MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block, - boolean onlySingleValues, + OnlySingleValueParams onlySingleValueParams, IntFunction blockValueReader ) { - super(field, searchExecutionContext, block, onlySingleValues); + super(field, searchExecutionContext, block, onlySingleValueParams); this.blockValueReader = blockValueReader; } @Override - public TermQueryList onlySingleValues() { - return new TermQueryList(field, searchExecutionContext, block, true, blockValueReader); + public TermQueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage) { + return new TermQueryList( + field, + searchExecutionContext, + block, + new OnlySingleValueParams(warnings, multiValueWarningMessage), + blockValueReader + ); } @Override @@ -241,17 +264,22 @@ public abstract class QueryList { MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block, - boolean onlySingleValues + OnlySingleValueParams onlySingleValueParams ) { - super(field, searchExecutionContext, block, onlySingleValues); + super(field, searchExecutionContext, block, onlySingleValueParams); this.blockValueReader = blockToGeometry(block); this.shapeQuery = shapeQuery(); } @Override - public GeoShapeQueryList onlySingleValues() { - return new GeoShapeQueryList(field, searchExecutionContext, block, true); + public GeoShapeQueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage) { + return new GeoShapeQueryList( + field, + searchExecutionContext, + block, + new OnlySingleValueParams(warnings, multiValueWarningMessage) + ); } @Override @@ -295,4 +323,6 @@ public abstract class QueryList { throw new IllegalArgumentException("Unsupported field type for geo_match ENRICH: " + field.typeName()); } } + + protected record OnlySingleValueParams(Warnings warnings, String multiValueWarningMessage) {} } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/querydsl/query/SingleValueMatchQuery.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/querydsl/query/SingleValueMatchQuery.java index b948d0f409db..65ec5765e873 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/querydsl/query/SingleValueMatchQuery.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/querydsl/query/SingleValueMatchQuery.java @@ -46,15 +46,14 @@ public final class SingleValueMatchQuery extends Query { * This avoids reporting warnings when queries are not matching multi-values */ private static final int MULTI_VALUE_MATCH_COST = 1000; - private static final IllegalArgumentException MULTI_VALUE_EXCEPTION = new IllegalArgumentException( - "single-value function encountered multi-value" - ); private final IndexFieldData fieldData; private final Warnings warnings; + private final String multiValueExceptionMessage; - public SingleValueMatchQuery(IndexFieldData fieldData, Warnings warnings) { + public SingleValueMatchQuery(IndexFieldData fieldData, Warnings warnings, String multiValueExceptionMessage) { this.fieldData = fieldData; this.warnings = warnings; + this.multiValueExceptionMessage = multiValueExceptionMessage; } @Override @@ -123,7 +122,7 @@ public final class SingleValueMatchQuery extends Query { return false; } if (sortedNumerics.docValueCount() != 1) { - warnings.registerException(MULTI_VALUE_EXCEPTION); + registerMultiValueException(); return false; } return true; @@ -158,7 +157,7 @@ public final class SingleValueMatchQuery extends Query { return false; } if (sortedSetDocValues.docValueCount() != 1) { - warnings.registerException(MULTI_VALUE_EXCEPTION); + registerMultiValueException(); return false; } return true; @@ -187,7 +186,7 @@ public final class SingleValueMatchQuery extends Query { return false; } if (sortedBinaryDocValues.docValueCount() != 1) { - warnings.registerException(MULTI_VALUE_EXCEPTION); + registerMultiValueException(); return false; } return true; @@ -267,6 +266,10 @@ public final class SingleValueMatchQuery extends Query { } } + private void registerMultiValueException() { + warnings.registerException(IllegalArgumentException.class, multiValueExceptionMessage); + } + private static class PredicateScorerSupplier extends ScorerSupplier { private final float score; private final ScoreMode scoreMode; diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java index e94864b9530b..acc62de0884c 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java @@ -112,7 +112,7 @@ public class AsyncOperatorTests extends ESTestCase { } }; int maxConcurrentRequests = randomIntBetween(1, 10); - AsyncOperator asyncOperator = new AsyncOperator(driverContext, maxConcurrentRequests) { + AsyncOperator asyncOperator = new AsyncOperator(driverContext, threadPool.getThreadContext(), maxConcurrentRequests) { final LookupService lookupService = new LookupService(threadPool, globalBlockFactory, dict, maxConcurrentRequests); @Override @@ -182,7 +182,7 @@ public class AsyncOperatorTests extends ESTestCase { Map> handlers = new HashMap<>(); TestOp(DriverContext driverContext, int maxOutstandingRequests) { - super(driverContext, maxOutstandingRequests); + super(driverContext, threadPool.getThreadContext(), maxOutstandingRequests); } @Override @@ -262,7 +262,7 @@ public class AsyncOperatorTests extends ESTestCase { ); int maxConcurrentRequests = randomIntBetween(1, 10); AtomicBoolean failed = new AtomicBoolean(); - AsyncOperator asyncOperator = new AsyncOperator(driverContext, maxConcurrentRequests) { + AsyncOperator asyncOperator = new AsyncOperator(driverContext, threadPool.getThreadContext(), maxConcurrentRequests) { @Override protected void performAsync(Page inputPage, ActionListener listener) { ActionRunnable command = new ActionRunnable<>(listener) { @@ -324,7 +324,7 @@ public class AsyncOperatorTests extends ESTestCase { for (int i = 0; i < iters; i++) { DriverContext driverContext = new DriverContext(blockFactory.bigArrays(), blockFactory); CyclicBarrier barrier = new CyclicBarrier(2); - AsyncOperator asyncOperator = new AsyncOperator(driverContext, between(1, 10)) { + AsyncOperator asyncOperator = new AsyncOperator(driverContext, threadPool.getThreadContext(), between(1, 10)) { @Override protected void performAsync(Page inputPage, ActionListener listener) { ActionRunnable command = new ActionRunnable<>(listener) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java index 48a566994b2f..f1f1f7762322 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java @@ -379,7 +379,7 @@ public class DriverTests extends ESTestCase { private final ThreadPool threadPool; SwitchContextOperator(DriverContext driverContext, ThreadPool threadPool) { - super(driverContext, between(1, 3)); + super(driverContext, threadPool.getThreadContext(), between(1, 3)); this.threadPool = threadPool; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java index 454088c1751e..df0a31965055 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java @@ -99,13 +99,12 @@ public class EnrichQuerySourceOperatorTests extends ESTestCase { // 3 -> [] -> [] // 4 -> [a3] -> [3] // 5 -> [] -> [] - var warnings = Warnings.createWarnings(DriverContext.WarningsMode.IGNORE, 0, 0, "test enrich"); EnrichQuerySourceOperator queryOperator = new EnrichQuerySourceOperator( blockFactory, 128, queryList, directoryData.reader, - warnings + warnings() ); Page page = queryOperator.getOutput(); assertNotNull(page); @@ -156,13 +155,12 @@ public class EnrichQuerySourceOperatorTests extends ESTestCase { try (var directoryData = makeDirectoryWith(directoryTermsList); var inputTerms = makeTermsBlock(inputTermsList)) { var queryList = QueryList.rawTermQueryList(directoryData.field, directoryData.searchExecutionContext, inputTerms); int maxPageSize = between(1, 256); - var warnings = Warnings.createWarnings(DriverContext.WarningsMode.IGNORE, 0, 0, "test enrich"); EnrichQuerySourceOperator queryOperator = new EnrichQuerySourceOperator( blockFactory, maxPageSize, queryList, directoryData.reader, - warnings + warnings() ); Map> actualPositions = new HashMap<>(); while (queryOperator.isFinished() == false) { @@ -193,7 +191,7 @@ public class EnrichQuerySourceOperatorTests extends ESTestCase { ) ) { QueryList queryList = QueryList.rawTermQueryList(directoryData.field, directoryData.searchExecutionContext, inputTerms) - .onlySingleValues(); + .onlySingleValues(warnings(), "multi-value found"); // pos -> terms -> docs // ----------------------------- // 0 -> [b2] -> [] @@ -202,13 +200,12 @@ public class EnrichQuerySourceOperatorTests extends ESTestCase { // 3 -> [] -> [] // 4 -> [a3] -> [3] // 5 -> [a3, a2, z2, xx] -> [] - var warnings = Warnings.createWarnings(DriverContext.WarningsMode.IGNORE, 0, 0, "test lookup"); EnrichQuerySourceOperator queryOperator = new EnrichQuerySourceOperator( blockFactory, 128, queryList, directoryData.reader, - warnings + warnings() ); Page page = queryOperator.getOutput(); assertNotNull(page); @@ -220,6 +217,10 @@ public class EnrichQuerySourceOperatorTests extends ESTestCase { assertThat(BlockUtils.toJavaObject(positions, 0), equalTo(4)); page.releaseBlocks(); assertTrue(queryOperator.isFinished()); + assertWarnings( + "Line -1:-1: evaluation of [test] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: multi-value found" + ); } } @@ -228,6 +229,10 @@ public class EnrichQuerySourceOperatorTests extends ESTestCase { return doc.asVector().docs(); } + private static Warnings warnings() { + return Warnings.createWarnings(DriverContext.WarningsMode.COLLECT, -1, -1, "test"); + } + private record DirectoryData( DirectoryReader reader, MockDirectoryWrapper dir, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 8ca4292f97fa..1b5de3283fe6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -353,7 +353,7 @@ emp_no:integer | language_code:integer | language_name:keyword mvJoinKeyOnTheLookupIndex required_capability: join_lookup_v12 -required_capability: join_lookup_skip_mv_on_lookup_key +required_capability: join_lookup_skip_mv_warnings FROM employees | WHERE 10003 < emp_no AND emp_no < 10008 @@ -363,6 +363,9 @@ FROM employees | KEEP emp_no, language_code, language_name ; +warning:Line 4:3: evaluation of [LOOKUP JOIN languages_lookup_non_unique_key ON language_code] failed, treating result as null. Only first 20 failures recorded. +warning:Line 4:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + emp_no:integer | language_code:integer | language_name:keyword 10004 | 4 | Quenya 10005 | 5 | null @@ -372,7 +375,7 @@ emp_no:integer | language_code:integer | language_name:keyword mvJoinKeyOnFrom required_capability: join_lookup_v12 -required_capability: join_lookup_skip_mv +required_capability: join_lookup_skip_mv_warnings FROM employees | WHERE emp_no < 10006 @@ -382,6 +385,56 @@ FROM employees | KEEP emp_no, language_code, language_name ; +warning:Line 4:3: evaluation of [LOOKUP JOIN languages_lookup ON language_code] failed, treating result as null. Only first 20 failures recorded. +warning:Line 4:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +emp_no:integer | language_code:integer | language_name:keyword +10001 | 1 | English +10002 | [-7, 11] | null +10003 | [12, 14] | null +10004 | [0, 1, 3, 13] | null +10005 | [-2, 13] | null +; + +mvJoinKeyOnTheLookupIndexAfterStats +required_capability: join_lookup_v12 +required_capability: join_lookup_skip_mv_warnings + +FROM employees +| WHERE 10003 < emp_no AND emp_no < 10008 +| EVAL language_code = emp_no % 10 +| STATS BY emp_no, language_code +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| SORT emp_no, language_name +| KEEP emp_no, language_code, language_name +; + +warning:Line 5:3: evaluation of [LOOKUP JOIN languages_lookup_non_unique_key ON language_code] failed, treating result as null. Only first 20 failures recorded. +warning:Line 5:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +emp_no:integer | language_code:integer | language_name:keyword +10004 | 4 | Quenya +10005 | 5 | null +10006 | 6 | null +10007 | 7 | null +; + +mvJoinKeyOnFromAfterStats +required_capability: join_lookup_v12 +required_capability: join_lookup_skip_mv_warnings + +FROM employees +| WHERE emp_no < 10006 +| EVAL language_code = salary_change.int +| STATS language_code = VALUES(language_code) BY emp_no +| LOOKUP JOIN languages_lookup ON language_code +| SORT emp_no +| KEEP emp_no, language_code, language_name +; + +warning:Line 5:3: evaluation of [LOOKUP JOIN languages_lookup ON language_code] failed, treating result as null. Only first 20 failures recorded. +warning:Line 5:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + emp_no:integer | language_code:integer | language_name:keyword 10001 | 1 | English 10002 | [-7, 11] | null @@ -392,7 +445,7 @@ emp_no:integer | language_code:integer | language_name:keyword mvJoinKeyFromRow required_capability: join_lookup_v12 -required_capability: join_lookup_skip_mv +required_capability: join_lookup_skip_mv_warnings ROW language_code = [4, 5, 6, 7] | LOOKUP JOIN languages_lookup_non_unique_key ON language_code @@ -400,13 +453,16 @@ ROW language_code = [4, 5, 6, 7] | SORT language_code, language_name, country ; +warning:Line 2:3: evaluation of [LOOKUP JOIN languages_lookup_non_unique_key ON language_code] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + language_code:integer | language_name:keyword | country:text [4, 5, 6, 7] | null | null ; mvJoinKeyFromRowExpanded required_capability: join_lookup_v12 -required_capability: join_lookup_skip_mv_on_lookup_key +required_capability: join_lookup_skip_mv_warnings ROW language_code = [4, 5, 6, 7, 8] | MV_EXPAND language_code @@ -415,6 +471,9 @@ ROW language_code = [4, 5, 6, 7, 8] | SORT language_code, language_name, country ; +warning:Line 3:3: evaluation of [LOOKUP JOIN languages_lookup_non_unique_key ON language_code] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + language_code:integer | language_name:keyword | country:text 4 | Quenya | null 5 | null | Atlantis diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 59e52a409e66..d88b68ece210 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -709,14 +709,9 @@ public class EsqlCapabilities { LOOKUP_JOIN_TEXT(JOIN_LOOKUP_V12.isEnabled()), /** - * LOOKUP JOIN without MV matching (https://github.com/elastic/elasticsearch/issues/118780) + * LOOKUP JOIN skipping MVs and sending warnings (https://github.com/elastic/elasticsearch/issues/118780) */ - JOIN_LOOKUP_SKIP_MV(JOIN_LOOKUP_V12.isEnabled()), - - /** - * LOOKUP JOIN without MV matching on lookup index key (https://github.com/elastic/elasticsearch/issues/118780) - */ - JOIN_LOOKUP_SKIP_MV_ON_LOOKUP_KEY(JOIN_LOOKUP_V12.isEnabled()), + JOIN_LOOKUP_SKIP_MV_WARNINGS(JOIN_LOOKUP_V12.isEnabled()), /** * Fix pushing down LIMIT past LOOKUP JOIN in case of multiple matching join keys. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java index 3184b16257b5..4542fc196a19 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java @@ -176,7 +176,13 @@ public abstract class AbstractLookupService { private final String matchType; private final String matchField; private final List enrichFields; - private final ResponseHeadersCollector responseHeadersCollector; private final Source source; private long totalTerms = 0L; @@ -101,7 +99,7 @@ public final class EnrichLookupOperator extends AsyncOperator { List enrichFields, Source source ) { - super(driverContext, maxOutstandingRequests); + super(driverContext, enrichLookupService.getThreadContext(), maxOutstandingRequests); this.sessionId = sessionId; this.parentTask = parentTask; this.inputChannel = inputChannel; @@ -112,7 +110,6 @@ public final class EnrichLookupOperator extends AsyncOperator { this.matchField = matchField; this.enrichFields = enrichFields; this.source = source; - this.responseHeadersCollector = new ResponseHeadersCollector(enrichLookupService.getThreadContext()); } @Override @@ -135,11 +132,7 @@ public final class EnrichLookupOperator extends AsyncOperator { } return inputPage.appendPage(pages.getFirst()); }; - enrichLookupService.lookupAsync( - request, - parentTask, - ActionListener.runBefore(listener.map(handleResponse), responseHeadersCollector::collect) - ); + enrichLookupService.lookupAsync(request, parentTask, listener.map(handleResponse)); } @Override @@ -171,7 +164,6 @@ public final class EnrichLookupOperator extends AsyncOperator { protected void doClose() { // TODO: Maybe create a sub-task as the parent task of all the lookup tasks // then cancel it when this operator terminates early (e.g., have enough result). - responseHeadersCollector.finish(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 480b69ecd8e6..1dc18c090c1d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -22,6 +22,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockStreamInput; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.Warnings; import org.elasticsearch.compute.operator.lookup.QueryList; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.mapper.MappedFieldType; @@ -98,7 +99,13 @@ public class EnrichLookupService extends AbstractLookupService loadFields, Source source ) { - super(driverContext, maxOutstandingRequests); + super(driverContext, lookupService.getThreadContext(), maxOutstandingRequests); this.sessionId = sessionId; this.parentTask = parentTask; this.inputChannel = inputChannel; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java index 131d8ddfa5cc..62d9733a0458 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockStreamInput; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.Warnings; import org.elasticsearch.compute.operator.lookup.QueryList; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.query.SearchExecutionContext; @@ -34,6 +35,8 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import static java.lang.System.in; + /** * {@link LookupFromIndexService} performs lookup against a Lookup index for * a given input page. See {@link AbstractLookupService} for how it works @@ -76,8 +79,17 @@ public class LookupFromIndexService extends AbstractLookupService Date: Wed, 12 Feb 2025 15:55:37 +0100 Subject: [PATCH 25/93] Simplify InternalOrder.isOrder a little (#122372) No need for recursion here. We don't allow nested compound order instances so this thing works exactly as the JavaDoc states and we only need to check one level down. --- .../search/aggregations/InternalOrder.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java index ead1374fb9b4..2bc00aeedb83 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java @@ -438,16 +438,7 @@ public abstract class InternalOrder extends BucketOrder { * @return {@code true} if the order matches, {@code false} otherwise. */ private static boolean isOrder(BucketOrder order, BucketOrder expected) { - if (order == expected) { - return true; - } else if (order instanceof CompoundOrder) { - // check if its a compound order with the first element that matches - List orders = ((CompoundOrder) order).orderElements; - if (orders.size() >= 1) { - return isOrder(orders.get(0), expected); - } - } - return false; + return order == expected || (order instanceof CompoundOrder compoundOrder && compoundOrder.orderElements.getFirst() == expected); } /** From fd97007803e64e14051ebefe61820b0edd0417c6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 02:07:42 +1100 Subject: [PATCH 26/93] Mute org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT testStatsMissingFieldWithStats #122327 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index ef820b14770c..185a1e16d04c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -404,6 +404,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122377 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/122378 +- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT + method: testStatsMissingFieldWithStats + issue: https://github.com/elastic/elasticsearch/issues/122327 # Examples: # From 39da0749b053e55bd4a70d2e6da39a29453c2edc Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Wed, 12 Feb 2025 10:16:16 -0500 Subject: [PATCH 27/93] [Transform] Reset plugin after tests (#122252) There is a race condition where the test is trying to clean up while the Transform auditor is still writing messages - resetting the plugin will stop the auditor (and properly reset it). Fix #12148 --- muted-tests.yml | 3 --- .../transform/checkpoint/TransformCCSCanMatchIT.java | 8 ++++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 185a1e16d04c..043e261d9da5 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -320,9 +320,6 @@ tests: - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests method: testGetUsersWithProfileUid issue: https://github.com/elastic/elasticsearch/issues/121483 -- class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT - method: testTransformLifecycle_RangeQueryThatMatchesNoShards - issue: https://github.com/elastic/elasticsearch/issues/121480 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests method: testSuggestProfilesWithHint issue: https://github.com/elastic/elasticsearch/issues/121116 diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java index e4e577299d0d..4ce17b46805e 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java @@ -12,6 +12,8 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.PointValues; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateAction; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -61,6 +63,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformStats; import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; import org.elasticsearch.xpack.transform.LocalStateTransform; +import org.junit.After; import org.junit.Before; import java.io.IOException; @@ -136,6 +139,11 @@ public class TransformCCSCanMatchIT extends AbstractMultiClustersTestCase { remoteNewDocs = createIndexAndIndexDocs(REMOTE_CLUSTER, "remote_new_index", newRemoteNumShards, timestamp, randomBoolean()); } + @After + public void cleanup() { + client().execute(ResetFeatureStateAction.INSTANCE, new ResetFeatureStateRequest(TEST_REQUEST_TIMEOUT)).actionGet(); + } + private int createIndexAndIndexDocs(String cluster, String index, int numberOfShards, long timestamp, boolean exposeTimestamp) throws Exception { Client client = client(cluster); From 99c53981374841296778026e0f6fe12d91a7f39a Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Wed, 12 Feb 2025 10:24:07 -0500 Subject: [PATCH 28/93] [Transform] Delete Alias Write Index (#122074) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the Transform is configured to write to an alias, specifying `DELETE _transform/?delete_dest_index` will follow the alias to the concrete destination index. Fix #121913 Co-authored-by: Przemysław Witek --- docs/changelog/122074.yaml | 8 ++ .../integration/TransformDeleteIT.java | 99 ++++++++++++++++--- .../integration/TransformRestTestCase.java | 2 +- .../TransportDeleteTransformAction.java | 99 +++++++++++++++---- 4 files changed, 176 insertions(+), 32 deletions(-) create mode 100644 docs/changelog/122074.yaml diff --git a/docs/changelog/122074.yaml b/docs/changelog/122074.yaml new file mode 100644 index 000000000000..21e171d0eb5e --- /dev/null +++ b/docs/changelog/122074.yaml @@ -0,0 +1,8 @@ +pr: 122074 +summary: If the Transform is configured to write to an alias as its destination index, + when the delete_dest_index parameter is set to true, then the Delete API will now + delete the write index backing the alias +area: Transform +type: bug +issues: + - 121913 diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java index bb68c7b84da5..b5064c46c95a 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java @@ -110,7 +110,7 @@ public class TransformDeleteIT extends TransformRestTestCase { deleteTransform(transformId, false, true); assertFalse(indexExists(transformDest)); - assertFalse(aliasExists(transformDest)); + assertFalse(aliasExists(transformDestAlias)); } public void testDeleteWithParamDeletesManuallyCreatedDestinationIndex() throws Exception { @@ -139,7 +139,7 @@ public class TransformDeleteIT extends TransformRestTestCase { assertFalse(aliasExists(transformDestAlias)); } - public void testDeleteWithParamDoesNotDeleteManuallySetUpAlias() throws Exception { + public void testDeleteWithManuallyCreatedIndexAndManuallyCreatedAlias() throws Exception { String transformId = "transform-4"; String transformDest = transformId + "_idx"; String transformDestAlias = transformId + "_alias"; @@ -158,16 +158,9 @@ public class TransformDeleteIT extends TransformRestTestCase { assertTrue(indexExists(transformDest)); assertTrue(aliasExists(transformDestAlias)); - ResponseException e = expectThrows(ResponseException.class, () -> deleteTransform(transformId, false, true)); - assertThat( - e.getMessage(), - containsString( - Strings.format( - "The provided expression [%s] matches an alias, specify the corresponding concrete indices instead.", - transformDestAlias - ) - ) - ); + deleteTransform(transformId, false, true); + assertFalse(indexExists(transformDest)); + assertFalse(aliasExists(transformDestAlias)); } public void testDeleteDestinationIndexIsNoOpWhenNoDestinationIndexExists() throws Exception { @@ -185,6 +178,88 @@ public class TransformDeleteIT extends TransformRestTestCase { assertFalse(aliasExists(transformDestAlias)); } + public void testDeleteWithAliasPointingToManyIndices() throws Exception { + var transformId = "transform-6"; + var transformDest = transformId + "_idx"; + var otherIndex = "some-other-index-6"; + String transformDestAlias = transformId + "_alias"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, otherIndex, transformDestAlias); + + createIndex(transformDest, null, null, "\"" + transformDestAlias + "\": { \"is_write_index\": true }"); + createIndex(otherIndex, null, null, "\"" + transformDestAlias + "\": {}"); + + assertTrue(indexExists(transformDest)); + assertTrue(indexExists(otherIndex)); + assertTrue(aliasExists(transformDestAlias)); + + createTransform(transformId, transformDestAlias, null); + + startTransform(transformId); + waitForTransformCheckpoint(transformId, 1); + + stopTransform(transformId, false); + + assertTrue(indexExists(transformDest)); + assertTrue(indexExists(otherIndex)); + assertTrue(aliasExists(transformDestAlias)); + + deleteTransform(transformId, false, true); + + assertFalse(indexExists(transformDest)); + assertTrue(indexExists(otherIndex)); + assertTrue(aliasExists(transformDestAlias)); + } + + public void testDeleteWithNoWriteIndexThrowsException() throws Exception { + var transformId = "transform-7"; + var transformDest = transformId + "_idx"; + var otherIndex = "some-other-index-7"; + String transformDestAlias = transformId + "_alias"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, otherIndex, transformDestAlias); + + createIndex(transformDest, null, null, "\"" + transformDestAlias + "\": {}"); + + assertTrue(indexExists(transformDest)); + assertTrue(aliasExists(transformDestAlias)); + + createTransform(transformId, transformDestAlias, null); + + createIndex(otherIndex, null, null, "\"" + transformDestAlias + "\": {}"); + assertTrue(indexExists(otherIndex)); + + ResponseException e = expectThrows(ResponseException.class, () -> deleteTransform(transformId, false, true)); + assertThat( + e.getMessage(), + containsString( + Strings.format( + "Cannot disambiguate destination index alias [%s]. Alias points to many indices with no clear write alias." + + " Retry with delete_dest_index=false and manually clean up destination index.", + transformDestAlias + ) + ) + ); + } + + public void testDeleteWithAlreadyDeletedIndex() throws Exception { + var transformId = "transform-8"; + var transformDest = transformId + "_idx"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest); + + createIndex(transformDest); + + assertTrue(indexExists(transformDest)); + + createTransform(transformId, transformDest, null); + + deleteIndex(transformDest); + + assertFalse(indexExists(transformDest)); + + deleteTransform(transformId, false, true); + + assertFalse(indexExists(transformDest)); + } + private void createTransform(String transformId, String destIndex, String destAlias) throws IOException { final Request createTransformRequest = createRequestWithAuth( "PUT", diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java index 537f50a30b5d..20ec649f7481 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java @@ -412,7 +412,7 @@ public abstract class TransformRestTestCase extends TransformCommonRestTestCase } updateTransformRequest.setJsonEntity(update); - client().performRequest(updateTransformRequest); + assertOKAndConsume(client().performRequest(updateTransformRequest)); } protected void startTransform(String transformId) throws IOException { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java index 41b683a7965c..619e72581cb5 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java @@ -10,9 +10,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; import org.elasticsearch.client.internal.Client; @@ -27,6 +31,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -42,6 +47,8 @@ import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex; import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.transforms.TransformTask; +import java.util.Objects; + import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.executeWithHeadersAsync; @@ -146,20 +153,31 @@ public class TransportDeleteTransformAction extends AcknowledgedTransportMasterN TimeValue timeout, ActionListener listener ) { - // <3> Check if the error is "index not found" error. If so, just move on. The index is already deleted. - ActionListener deleteDestIndexListener = ActionListener.wrap(listener::onResponse, e -> { - if (e instanceof IndexNotFoundException) { - listener.onResponse(AcknowledgedResponse.TRUE); - } else { - listener.onFailure(e); - } - }); + getTransformConfig(transformId).andThen((l, r) -> deleteDestinationIndex(r.v1(), parentTaskId, timeout, l)) + .addListener(listener.delegateResponse((l, e) -> { + if (e instanceof IndexNotFoundException) { + l.onResponse(AcknowledgedResponse.TRUE); + } else { + l.onFailure(e); + } + })); + } - // <2> Delete destination index - ActionListener> getTransformConfigurationListener = ActionListener.wrap( - transformConfigAndVersion -> { - TransformConfig config = transformConfigAndVersion.v1(); - String destIndex = config.getDestination().getIndex(); + private SubscribableListener> getTransformConfig(String transformId) { + return SubscribableListener.newForked(l -> transformConfigManager.getTransformConfigurationForUpdate(transformId, l)); + } + + /** + * Delete the destination index. If the Transform is configured to write to an alias, then follow that alias to the concrete index. + */ + private void deleteDestinationIndex( + TransformConfig config, + TaskId parentTaskId, + TimeValue timeout, + ActionListener listener + ) { + SubscribableListener.newForked(l -> resolveDestinationIndex(config, parentTaskId, timeout, l)) + .andThen((l, destIndex) -> { DeleteIndexRequest deleteDestIndexRequest = new DeleteIndexRequest(destIndex); deleteDestIndexRequest.ackTimeout(timeout); deleteDestIndexRequest.setParentTask(parentTaskId); @@ -169,14 +187,57 @@ public class TransportDeleteTransformAction extends AcknowledgedTransportMasterN client, TransportDeleteIndexAction.TYPE, deleteDestIndexRequest, - deleteDestIndexListener + l ); - }, - listener::onFailure - ); + }) + .addListener(listener); + } - // <1> Fetch transform configuration - transformConfigManager.getTransformConfigurationForUpdate(transformId, getTransformConfigurationListener); + private void resolveDestinationIndex(TransformConfig config, TaskId parentTaskId, TimeValue timeout, ActionListener listener) { + var destIndex = config.getDestination().getIndex(); + var responseListener = ActionListener.wrap(r -> findDestinationIndexInAliases(r, destIndex, listener), e -> { + if (e instanceof AliasesNotFoundException) { + // no alias == the destIndex is our concrete index + listener.onResponse(destIndex); + } else { + listener.onFailure(e); + } + }); + + GetAliasesRequest request = new GetAliasesRequest(timeout, destIndex); + request.setParentTask(parentTaskId); + executeWithHeadersAsync(config.getHeaders(), TRANSFORM_ORIGIN, client, GetAliasesAction.INSTANCE, request, responseListener); + } + + private static void findDestinationIndexInAliases(GetAliasesResponse aliases, String destIndex, ActionListener listener) { + var indexToAliases = aliases.getAliases(); + if (indexToAliases.isEmpty()) { + // if the alias list is empty, that means the index is a concrete index + listener.onResponse(destIndex); + } else if (indexToAliases.size() == 1) { + // if there is one value, the alias will treat it as the write index, so it's our destination index + listener.onResponse(indexToAliases.keySet().iterator().next()); + } else { + // if there is more than one index, there may be more than one alias for each index + // we have to search for the alias that matches our destination index name AND is declared the write index for that alias + indexToAliases.entrySet().stream().map(entry -> { + if (entry.getValue().stream().anyMatch(md -> destIndex.equals(md.getAlias()) && Boolean.TRUE.equals(md.writeIndex()))) { + return entry.getKey(); + } else { + return null; + } + }).filter(Objects::nonNull).findFirst().ifPresentOrElse(listener::onResponse, () -> { + listener.onFailure( + new ElasticsearchStatusException( + "Cannot disambiguate destination index alias [" + + destIndex + + "]. Alias points to many indices with no clear write alias. Retry with delete_dest_index=false and manually" + + " clean up destination index.", + RestStatus.CONFLICT + ) + ); + }); + } } @Override From 8f28bc2febe0c850baf21809c29dd7984187ae48 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 12 Feb 2025 16:41:40 +0100 Subject: [PATCH 29/93] Fix handling of auto expand replicas for stateless indices (#122365) Auto expand replicas is meant to be entirely disabled for stateless indices. The only scenario where a change needs to be applied is when the number of replicas is initialized to 0, in which case 0 needs to be turned into 1. Otherwise, no changes should be applied in stateless indices despite auto expand replicas is used. The current handling for this was missing an early exit of the indices loop in the case where 0 shoudl be turned into 1, that leads to a potentially higher number of copies being allocated (effectively auto-expand gets applied by mistake). --- docs/changelog/122365.yaml | 5 ++ .../cluster/metadata/AutoExpandReplicas.java | 3 +- .../metadata/AutoExpandReplicasTests.java | 49 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/122365.yaml diff --git a/docs/changelog/122365.yaml b/docs/changelog/122365.yaml new file mode 100644 index 000000000000..1229cd8754ca --- /dev/null +++ b/docs/changelog/122365.yaml @@ -0,0 +1,5 @@ +pr: 122365 +summary: Fix handling of auto expand replicas for stateless indices +area: "Search" +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java index ef28a46d423d..896381ba185e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java @@ -156,9 +156,8 @@ public record AutoExpandReplicas(int minReplicas, int maxReplicas, boolean enabl )) { if (indexMetadata.getNumberOfReplicas() == 0) { nrReplicasChanged.computeIfAbsent(1, ArrayList::new).add(indexMetadata.getIndex().getName()); - } else { - continue; } + continue; } if (allocation == null) { allocation = allocationSupplier.get(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java index efb5df7d7a4f..eb550223617e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java @@ -20,8 +20,10 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RoutingNodesHelper; import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.cluster.ClusterStateChanges; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; @@ -31,11 +33,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.everyItem; @@ -221,4 +226,48 @@ public class AutoExpandReplicasTests extends ESTestCase { assertThat(autoExpandReplicas.calculateDesiredNumberOfReplicas(matchingNodes), equalTo(Math.max(lowerBound, matchingNodes - 1))); assertThat(autoExpandReplicas.calculateDesiredNumberOfReplicas(max + 1), equalTo(max)); } + + public void testGetAutoExpandReplicaChangesStatelessIndices() { + { + // number of replicas is adjusted to 1 when it is initialized to 0 + Metadata metadata = Metadata.builder() + .put( + IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey(), "stateless") + .put("index.version.created", IndexVersion.current()) + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-all") + ) + ) + .build(); + Map> autoExpandReplicaChanges = AutoExpandReplicas.getAutoExpandReplicaChanges(metadata, null); + assertEquals(1, autoExpandReplicaChanges.size()); + List indices = autoExpandReplicaChanges.get(1); + assertEquals(1, indices.size()); + assertEquals("test", indices.getFirst()); + } + { + // no changes when number of replicas is set to anything other than 0 + Metadata metadata = Metadata.builder() + .put( + IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey(), "stateless") + .put("index.version.created", IndexVersion.current()) + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, randomIntBetween(1, 10)) + .put(INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-all") + ) + ) + .build(); + Map> autoExpandReplicaChanges = AutoExpandReplicas.getAutoExpandReplicaChanges(metadata, () -> { + throw new UnsupportedOperationException(); + }); + assertEquals(0, autoExpandReplicaChanges.size()); + } + } } From 2c846e735192d12cdb6fb85c5e711fc49a9cd6bd Mon Sep 17 00:00:00 2001 From: Stanislav Malyshev Date: Wed, 12 Feb 2025 09:46:28 -0700 Subject: [PATCH 30/93] Add metadata checking to RequestIndexFilteringIT (#122322) --- .../xpack/esql/ccq/MultiClustersIT.java | 5 +- .../esql/ccq/RequestIndexFilteringIT.java | 51 ++++++++++++++++++- .../rest/RequestIndexFilteringTestCase.java | 18 ++++--- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java index b838d8ae284a..791f5dacdce6 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java @@ -38,7 +38,10 @@ import java.util.stream.Stream; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.xpack.esql.ccq.Clusters.REMOTE_CLUSTER_NAME; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.any; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasKey; @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class MultiClustersIT extends ESRestTestCase { diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java index 7c81f97714a6..d8c68dd5281a 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java @@ -14,9 +14,12 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.TestClustersThreadFilter; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.xpack.esql.qa.rest.RequestIndexFilteringTestCase; +import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; +import org.hamcrest.Matcher; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -25,6 +28,12 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase { @@ -49,6 +58,8 @@ public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase { } } + private boolean isCCSRequest; + @AfterClass public static void closeRemoteClients() throws IOException { try { @@ -66,13 +77,20 @@ public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase { @Override protected String from(String... indexName) { - if (randomBoolean()) { + isCCSRequest = randomBoolean(); + if (isCCSRequest) { return "FROM *:" + String.join(",*:", indexName); } else { return "FROM " + String.join(",", indexName); } } + @Override + public Map runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject) throws IOException { + requestObject.includeCCSMetadata(true); + return super.runEsql(requestObject); + } + @After public void wipeRemoteTestData() throws IOException { try { @@ -82,4 +100,35 @@ public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase { assertEquals(404, re.getResponse().getStatusLine().getStatusCode()); } } + + private MapMatcher getClustersMetadataMatcher() { + MapMatcher mapMatcher = matchesMap(); + mapMatcher = mapMatcher.entry("running", 0); + mapMatcher = mapMatcher.entry("total", 1); + mapMatcher = mapMatcher.entry("failed", 0); + mapMatcher = mapMatcher.entry("partial", 0); + mapMatcher = mapMatcher.entry("successful", 1); + mapMatcher = mapMatcher.entry("skipped", 0); + mapMatcher = mapMatcher.entry( + "details", + matchesMap().entry( + Clusters.REMOTE_CLUSTER_NAME, + matchesMap().entry("_shards", matchesMap().extraOk()) + .entry("took", greaterThanOrEqualTo(0)) + .entry("indices", instanceOf(String.class)) + .entry("status", "successful") + ) + ); + return mapMatcher; + } + + @Override + protected void assertQueryResult(Map result, Matcher columnMatcher, Matcher valuesMatcher) { + var matcher = getResultMatcher(result).entry("columns", columnMatcher).entry("values", valuesMatcher); + if (isCCSRequest) { + matcher = matcher.entry("_clusters", getClustersMetadataMatcher()); + } + assertMap(result, matcher); + } + } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java index ad61c52775eb..1fdc11174ee0 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java @@ -17,6 +17,7 @@ import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.AssertWarnings; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.hamcrest.Matcher; import org.junit.After; import org.junit.Assert; @@ -62,7 +63,7 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { // filter includes both indices in the result (all columns, all rows) RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query(from("test*")); - assertResultMap( + assertQueryResult( runEsql(builder), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -73,7 +74,7 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)! builder = timestampFilter("gte", "2024-01-01").query(from("test*")); - assertResultMap( + assertQueryResult( runEsql(builder), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -84,7 +85,7 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { // filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds // after eliminating the index filter. All columns are returned. builder = timestampFilter("gte", "2025-01-01").query(from("test*")); - assertResultMap( + assertQueryResult( runEsql(builder), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -102,7 +103,7 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { // filter includes only test1. Columns and rows of test2 are filtered out RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(from("test*")); - assertResultMap( + assertQueryResult( runEsql(builder), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -113,7 +114,7 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)! builder = existsFilter("id1").query(from("test*") + " METADATA _index | KEEP _index, id*"); Map result = runEsql(builder); - assertResultMap( + assertQueryResult( result, matchesList().item(matchesMap().entry("name", "_index").entry("type", "keyword")) .item(matchesMap().entry("name", "id1").entry("type", "integer")), @@ -138,7 +139,7 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { from("test*") + " METADATA _index | SORT id2 | KEEP _index, id*" ); Map result = runEsql(builder); - assertResultMap( + assertQueryResult( result, matchesList().item(matchesMap().entry("name", "_index").entry("type", "keyword")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -298,4 +299,9 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); } } + + protected void assertQueryResult(Map result, Matcher columnMatcher, Matcher valuesMatcher) { + assertResultMap(result, columnMatcher, valuesMatcher); + } + } From bd242cccbc7720d9e4c6e6bd5462d4e2e741b128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Wed, 12 Feb 2025 18:56:13 +0100 Subject: [PATCH 31/93] [Entitlements] Instrumentation for FileSystemProvider (#122232) --- .../bridge/EntitlementChecker.java | 79 ++++++ .../qa/entitled/EntitledActions.java | 32 ++- .../qa/test/DummyImplementations.java | 108 ++++++++ .../qa/test/NioFileSystemActions.java | 230 ++++++++++++++++++ .../qa/test/RestEntitlementsCheckAction.java | 1 + .../EntitlementInitialization.java | 84 +++++-- .../api/ElasticsearchEntitlementChecker.java | 204 +++++++++++++++- .../runtime/policy/PolicyManager.java | 9 + 8 files changed, 729 insertions(+), 18 deletions(-) create mode 100644 libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFileSystemActions.java diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index cbedccc2a37b..39671e298cb9 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -35,6 +35,7 @@ import java.net.ServerSocket; import java.net.Socket; import java.net.SocketAddress; import java.net.SocketImplFactory; +import java.net.URI; import java.net.URL; import java.net.URLStreamHandler; import java.net.URLStreamHandlerFactory; @@ -50,17 +51,24 @@ import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.nio.channels.spi.SelectorProvider; import java.nio.charset.Charset; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; import java.nio.file.FileStore; import java.nio.file.LinkOption; import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.UserPrincipal; import java.nio.file.spi.FileSystemProvider; import java.security.cert.CertStoreParameters; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.TimeZone; +import java.util.concurrent.ExecutorService; import java.util.function.Consumer; import javax.net.ssl.HostnameVerifier; @@ -553,8 +561,79 @@ public interface EntitlementChecker { void check$java_nio_file_Files$$setOwner(Class callerClass, Path path, UserPrincipal principal); // file system providers + void check$java_nio_file_spi_FileSystemProvider$(Class callerClass); + + void checkNewFileSystem(Class callerClass, FileSystemProvider that, URI uri, Map env); + + void checkNewFileSystem(Class callerClass, FileSystemProvider that, Path path, Map env); + void checkNewInputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options); + void checkNewOutputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options); + + void checkNewFileChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + FileAttribute... attrs + ); + + void checkNewAsynchronousFileChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + ExecutorService executor, + FileAttribute... attrs + ); + + void checkNewByteChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + FileAttribute... attrs + ); + + void checkNewDirectoryStream(Class callerClass, FileSystemProvider that, Path dir, DirectoryStream.Filter filter); + + void checkCreateDirectory(Class callerClass, FileSystemProvider that, Path dir, FileAttribute... attrs); + + void checkCreateSymbolicLink(Class callerClass, FileSystemProvider that, Path link, Path target, FileAttribute... attrs); + + void checkCreateLink(Class callerClass, FileSystemProvider that, Path link, Path existing); + + void checkDelete(Class callerClass, FileSystemProvider that, Path path); + + void checkDeleteIfExists(Class callerClass, FileSystemProvider that, Path path); + + void checkReadSymbolicLink(Class callerClass, FileSystemProvider that, Path link); + + void checkCopy(Class callerClass, FileSystemProvider that, Path source, Path target, CopyOption... options); + + void checkMove(Class callerClass, FileSystemProvider that, Path source, Path target, CopyOption... options); + + void checkIsSameFile(Class callerClass, FileSystemProvider that, Path path, Path path2); + + void checkIsHidden(Class callerClass, FileSystemProvider that, Path path); + + void checkGetFileStore(Class callerClass, FileSystemProvider that, Path path); + + void checkCheckAccess(Class callerClass, FileSystemProvider that, Path path, AccessMode... modes); + + void checkGetFileAttributeView(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options); + + void checkReadAttributes(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options); + + void checkReadAttributes(Class callerClass, FileSystemProvider that, Path path, String attributes, LinkOption... options); + + void checkReadAttributesIfExists(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options); + + void checkSetAttribute(Class callerClass, FileSystemProvider that, Path path, String attribute, Object value, LinkOption... options); + + void checkExists(Class callerClass, FileSystemProvider that, Path path, LinkOption... options); + // file store void checkGetFileStoreAttributeView(Class callerClass, FileStore that, Class type); diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java index 640cc75d7ac7..f8451e84449f 100644 --- a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java +++ b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java @@ -14,12 +14,26 @@ import org.elasticsearch.core.SuppressForbidden; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.UserPrincipal; +import java.security.SecureRandom; +@SuppressForbidden(reason = "Exposes forbidden APIs for testing purposes") public final class EntitledActions { private EntitledActions() {} - @SuppressForbidden(reason = "Exposes forbidden APIs for testing purposes") + private static final SecureRandom random = new SecureRandom(); + + private static final Path testRootDir = Paths.get(System.getProperty("es.entitlements.testdir")); + + private static Path readDir() { + return testRootDir.resolve("read_dir"); + } + + private static Path readWriteDir() { + return testRootDir.resolve("read_write_dir"); + } + static void System_clearProperty(String key) { System.clearProperty(key); } @@ -31,4 +45,20 @@ public final class EntitledActions { public static void createFile(Path path) throws IOException { Files.createFile(path); } + + public static Path createTempFileForRead() throws IOException { + return Files.createFile(readDir().resolve("entitlements-" + random.nextLong() + ".tmp")); + } + + public static Path createTempFileForWrite() throws IOException { + return Files.createFile(readWriteDir().resolve("entitlements-" + random.nextLong() + ".tmp")); + } + + public static Path createTempDirectoryForWrite() throws IOException { + return Files.createDirectory(readWriteDir().resolve("entitlements-dir-" + random.nextLong())); + } + + public static Path createTempSymbolicLink() throws IOException { + return Files.createSymbolicLink(readDir().resolve("entitlements-link-" + random.nextLong()), readWriteDir()); + } } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java index 67e06c836f7b..ca0301463407 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java @@ -23,11 +23,13 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.SocketException; import java.net.SocketImpl; +import java.net.URI; import java.nio.channels.AsynchronousChannelGroup; import java.nio.channels.AsynchronousServerSocketChannel; import java.nio.channels.AsynchronousSocketChannel; import java.nio.channels.DatagramChannel; import java.nio.channels.Pipe; +import java.nio.channels.SeekableByteChannel; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.nio.channels.spi.AbstractSelector; @@ -35,6 +37,18 @@ import java.nio.channels.spi.AsynchronousChannelProvider; import java.nio.channels.spi.SelectorProvider; import java.nio.charset.Charset; import java.nio.charset.spi.CharsetProvider; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.spi.FileSystemProvider; import java.security.cert.Certificate; import java.text.BreakIterator; import java.text.Collator; @@ -51,6 +65,7 @@ import java.text.spi.NumberFormatProvider; import java.util.Iterator; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.spi.CalendarDataProvider; @@ -568,4 +583,97 @@ class DummyImplementations { return null; } } + + static class DummyFileSystemProvider extends FileSystemProvider { + @Override + public String getScheme() { + return ""; + } + + @Override + public FileSystem newFileSystem(URI uri, Map env) throws IOException { + return null; + } + + @Override + public FileSystem getFileSystem(URI uri) { + return null; + } + + @Override + public Path getPath(URI uri) { + return null; + } + + @Override + public SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) + throws IOException { + return null; + } + + @Override + public DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) throws IOException { + return null; + } + + @Override + public void createDirectory(Path dir, FileAttribute... attrs) throws IOException { + + } + + @Override + public void delete(Path path) throws IOException { + + } + + @Override + public void copy(Path source, Path target, CopyOption... options) throws IOException { + + } + + @Override + public void move(Path source, Path target, CopyOption... options) throws IOException { + + } + + @Override + public boolean isSameFile(Path path, Path path2) throws IOException { + return false; + } + + @Override + public boolean isHidden(Path path) throws IOException { + return false; + } + + @Override + public FileStore getFileStore(Path path) throws IOException { + return null; + } + + @Override + public void checkAccess(Path path, AccessMode... modes) throws IOException { + + } + + @Override + public V getFileAttributeView(Path path, Class type, LinkOption... options) { + return null; + } + + @Override + public A readAttributes(Path path, Class type, LinkOption... options) throws IOException { + return null; + } + + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) throws IOException { + return Map.of(); + } + + @Override + public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException { + + } + } } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFileSystemActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFileSystemActions.java new file mode 100644 index 000000000000..9dc36bda840e --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFileSystemActions.java @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.entitlement.qa.entitled.EntitledActions; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystemException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileOwnerAttributeView; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY; + +class NioFileSystemActions { + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void createFileSystemProvider() { + new DummyImplementations.DummyFileSystemProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkNewFileSystemFromUri() throws IOException { + try (var fs = FileSystems.getDefault().provider().newFileSystem(URI.create("/dummy/path"), Map.of())) {} + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkNewFileSystemFromPath() { + var fs = FileSystems.getDefault().provider(); + try (var newFs = fs.newFileSystem(Path.of("/dummy/path"), Map.of())) {} catch (IOException e) { + // When entitled, we expect to throw IOException, as the path is not valid - we don't really want to create a FS + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewInputStream() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var is = fs.newInputStream(FileCheckActions.readFile())) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewOutputStream() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var os = fs.newOutputStream(FileCheckActions.readWriteFile())) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewFileChannelRead() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var fc = fs.newFileChannel(FileCheckActions.readFile(), Set.of(StandardOpenOption.READ))) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewFileChannelWrite() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var fc = fs.newFileChannel(FileCheckActions.readWriteFile(), Set.of(StandardOpenOption.WRITE))) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewAsynchronousFileChannel() throws IOException { + var fs = FileSystems.getDefault().provider(); + try ( + var fc = fs.newAsynchronousFileChannel( + FileCheckActions.readWriteFile(), + Set.of(StandardOpenOption.WRITE), + EsExecutors.DIRECT_EXECUTOR_SERVICE + ) + ) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewByteChannel() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var bc = fs.newByteChannel(FileCheckActions.readWriteFile(), Set.of(StandardOpenOption.WRITE))) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewDirectoryStream() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var bc = fs.newDirectoryStream(FileCheckActions.readDir(), entry -> false)) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCreateDirectory() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + fs.createDirectory(directory.resolve("subdir")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCreateSymbolicLink() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + try { + fs.createSymbolicLink(directory.resolve("link"), FileCheckActions.readFile()); + } catch (UnsupportedOperationException | FileSystemException e) { + // OK not to implement symbolic link in the filesystem + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCreateLink() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + try { + fs.createLink(directory.resolve("link"), FileCheckActions.readFile()); + } catch (UnsupportedOperationException | FileSystemException e) { + // OK not to implement symbolic link in the filesystem + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkDelete() throws IOException { + var fs = FileSystems.getDefault().provider(); + var file = EntitledActions.createTempFileForWrite(); + fs.delete(file); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkDeleteIfExists() throws IOException { + var fs = FileSystems.getDefault().provider(); + var file = EntitledActions.createTempFileForWrite(); + fs.deleteIfExists(file); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkReadSymbolicLink() throws IOException { + var fs = FileSystems.getDefault().provider(); + var link = EntitledActions.createTempSymbolicLink(); + fs.readSymbolicLink(link); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCopy() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + fs.copy(FileCheckActions.readFile(), directory.resolve("copied")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkMove() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + var file = EntitledActions.createTempFileForWrite(); + fs.move(file, directory.resolve("moved")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkIsSameFile() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.isSameFile(FileCheckActions.readWriteFile(), FileCheckActions.readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkIsHidden() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.isHidden(FileCheckActions.readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkGetFileStore() throws IOException { + var fs = FileSystems.getDefault().provider(); + var file = EntitledActions.createTempFileForRead(); + var store = fs.getFileStore(file); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCheckAccess() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.checkAccess(FileCheckActions.readFile()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkGetFileAttributeView() { + var fs = FileSystems.getDefault().provider(); + fs.getFileAttributeView(FileCheckActions.readFile(), FileOwnerAttributeView.class); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkReadAttributesWithClass() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.readAttributes(FileCheckActions.readFile(), BasicFileAttributes.class); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkReadAttributesWithString() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.readAttributes(FileCheckActions.readFile(), "*"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkReadAttributesIfExists() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.readAttributesIfExists(FileCheckActions.readFile(), BasicFileAttributes.class); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkSetAttribute() throws IOException { + var fs = FileSystems.getDefault().provider(); + var file = EntitledActions.createTempFileForWrite(); + try { + fs.setAttribute(file, "dos:hidden", true); + } catch (UnsupportedOperationException | IllegalArgumentException | FileSystemException e) { + // OK if the file does not have/does not support the attribute + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkExists() { + var fs = FileSystems.getDefault().provider(); + fs.exists(FileCheckActions.readFile()); + } + + private NioFileSystemActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java index 315ff9d2b526..18709465fe1a 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java @@ -186,6 +186,7 @@ public class RestEntitlementsCheckAction extends BaseRestHandler { getTestEntries(SpiActions.class), getTestEntries(SystemActions.class), getTestEntries(NativeActions.class), + getTestEntries(NioFileSystemActions.class), getTestEntries(FileStoreActions.class) ) .flatMap(Function.identity()) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index f99ba1040436..331d47e561f1 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -34,11 +34,17 @@ import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttrib import java.lang.instrument.Instrumentation; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.net.URI; import java.nio.channels.spi.SelectorProvider; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; import java.nio.file.FileStore; import java.nio.file.FileSystems; +import java.nio.file.LinkOption; import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; import java.nio.file.spi.FileSystemProvider; import java.util.ArrayList; import java.util.Arrays; @@ -46,6 +52,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -66,7 +74,7 @@ public class EntitlementInitialization { private static ElasticsearchEntitlementChecker manager; - interface InstrumentationInfoFunction { + interface InstrumentationInfoFactory { InstrumentationService.InstrumentationInfo of(String methodName, Class... parameterTypes) throws ClassNotFoundException, NoSuchMethodException; } @@ -83,20 +91,10 @@ public class EntitlementInitialization { var latestCheckerInterface = getVersionSpecificCheckerClass(EntitlementChecker.class); Map checkMethods = new HashMap<>(INSTRUMENTATION_SERVICE.lookupMethods(latestCheckerInterface)); - var fileSystemProviderClass = FileSystems.getDefault().provider().getClass(); - - Stream.concat( + Stream.of( + fileSystemProviderChecks(), fileStoreChecks(), Stream.of( - INSTRUMENTATION_SERVICE.lookupImplementationMethod( - FileSystemProvider.class, - "newInputStream", - fileSystemProviderClass, - EntitlementChecker.class, - "checkNewInputStream", - Path.class, - OpenOption[].class - ), INSTRUMENTATION_SERVICE.lookupImplementationMethod( SelectorProvider.class, "inheritedChannel", @@ -105,7 +103,9 @@ public class EntitlementInitialization { "checkSelectorProviderInheritedChannel" ) ) - ).forEach(instrumentation -> checkMethods.put(instrumentation.targetMethod(), instrumentation.checkMethod())); + ) + .flatMap(Function.identity()) + .forEach(instrumentation -> checkMethods.put(instrumentation.targetMethod(), instrumentation.checkMethod())); var classesToTransform = checkMethods.keySet().stream().map(MethodKey::className).collect(Collectors.toSet()); @@ -142,7 +142,10 @@ public class EntitlementInitialization { new CreateClassLoaderEntitlement(), new InboundNetworkEntitlement(), new OutboundNetworkEntitlement(), - new LoadNativeLibrariesEntitlement() + new LoadNativeLibrariesEntitlement(), + new FilesEntitlement( + List.of(new FilesEntitlement.FileData(EntitlementBootstrap.bootstrapArgs().tempDir().toString(), READ_WRITE)) + ) ) ), new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())), @@ -164,12 +167,61 @@ public class EntitlementInitialization { return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, AGENTS_PACKAGE_NAME, ENTITLEMENTS_MODULE); } + private static Stream fileSystemProviderChecks() throws ClassNotFoundException, + NoSuchMethodException { + var fileSystemProviderClass = FileSystems.getDefault().provider().getClass(); + + var instrumentation = new InstrumentationInfoFactory() { + @Override + public InstrumentationService.InstrumentationInfo of(String methodName, Class... parameterTypes) + throws ClassNotFoundException, NoSuchMethodException { + return INSTRUMENTATION_SERVICE.lookupImplementationMethod( + FileSystemProvider.class, + methodName, + fileSystemProviderClass, + EntitlementChecker.class, + "check" + Character.toUpperCase(methodName.charAt(0)) + methodName.substring(1), + parameterTypes + ); + } + }; + + return Stream.of( + instrumentation.of("newFileSystem", URI.class, Map.class), + instrumentation.of("newFileSystem", Path.class, Map.class), + instrumentation.of("newInputStream", Path.class, OpenOption[].class), + instrumentation.of("newOutputStream", Path.class, OpenOption[].class), + instrumentation.of("newFileChannel", Path.class, Set.class, FileAttribute[].class), + instrumentation.of("newAsynchronousFileChannel", Path.class, Set.class, ExecutorService.class, FileAttribute[].class), + instrumentation.of("newByteChannel", Path.class, Set.class, FileAttribute[].class), + instrumentation.of("newDirectoryStream", Path.class, DirectoryStream.Filter.class), + instrumentation.of("createDirectory", Path.class, FileAttribute[].class), + instrumentation.of("createSymbolicLink", Path.class, Path.class, FileAttribute[].class), + instrumentation.of("createLink", Path.class, Path.class), + instrumentation.of("delete", Path.class), + instrumentation.of("deleteIfExists", Path.class), + instrumentation.of("readSymbolicLink", Path.class), + instrumentation.of("copy", Path.class, Path.class, CopyOption[].class), + instrumentation.of("move", Path.class, Path.class, CopyOption[].class), + instrumentation.of("isSameFile", Path.class, Path.class), + instrumentation.of("isHidden", Path.class), + instrumentation.of("getFileStore", Path.class), + instrumentation.of("checkAccess", Path.class, AccessMode[].class), + instrumentation.of("getFileAttributeView", Path.class, Class.class, LinkOption[].class), + instrumentation.of("readAttributes", Path.class, Class.class, LinkOption[].class), + instrumentation.of("readAttributes", Path.class, String.class, LinkOption[].class), + instrumentation.of("readAttributesIfExists", Path.class, Class.class, LinkOption[].class), + instrumentation.of("setAttribute", Path.class, String.class, Object.class, LinkOption[].class), + instrumentation.of("exists", Path.class, LinkOption[].class) + ); + } + private static Stream fileStoreChecks() { var fileStoreClasses = StreamSupport.stream(FileSystems.getDefault().getFileStores().spliterator(), false) .map(FileStore::getClass) .distinct(); return fileStoreClasses.flatMap(fileStoreClass -> { - var instrumentation = new InstrumentationInfoFunction() { + var instrumentation = new InstrumentationInfoFactory() { @Override public InstrumentationService.InstrumentationInfo of(String methodName, Class... parameterTypes) throws ClassNotFoundException, NoSuchMethodException { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index 632be24d83cd..1ca03fa876b1 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -40,6 +40,7 @@ import java.net.ServerSocket; import java.net.Socket; import java.net.SocketAddress; import java.net.SocketImplFactory; +import java.net.URI; import java.net.URL; import java.net.URLStreamHandler; import java.net.URLStreamHandlerFactory; @@ -55,17 +56,25 @@ import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.nio.channels.spi.SelectorProvider; import java.nio.charset.Charset; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; import java.nio.file.FileStore; import java.nio.file.LinkOption; import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.UserPrincipal; import java.nio.file.spi.FileSystemProvider; import java.security.cert.CertStoreParameters; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.TimeZone; +import java.util.concurrent.ExecutorService; import java.util.function.Consumer; import javax.net.ssl.HostnameVerifier; @@ -1071,9 +1080,202 @@ public class ElasticsearchEntitlementChecker implements EntitlementChecker { // file system providers + @Override + public void check$java_nio_file_spi_FileSystemProvider$(Class callerClass) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + @Override + public void checkNewFileSystem(Class callerClass, FileSystemProvider that, URI uri, Map env) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + @Override + public void checkNewFileSystem(Class callerClass, FileSystemProvider that, Path path, Map env) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + @Override public void checkNewInputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options) { - // TODO: policyManger.checkFileSystemRead(path); + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkNewOutputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + private static boolean isOpenForWrite(Set options) { + return options.contains(StandardOpenOption.WRITE) + || options.contains(StandardOpenOption.APPEND) + || options.contains(StandardOpenOption.CREATE) + || options.contains(StandardOpenOption.CREATE_NEW) + || options.contains(StandardOpenOption.DELETE_ON_CLOSE); + } + + @Override + public void checkNewFileChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + FileAttribute... attrs + ) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void checkNewAsynchronousFileChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + ExecutorService executor, + FileAttribute... attrs + ) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void checkNewByteChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + FileAttribute... attrs + ) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void checkNewDirectoryStream( + Class callerClass, + FileSystemProvider that, + Path dir, + DirectoryStream.Filter filter + ) { + policyManager.checkFileRead(callerClass, dir); + } + + @Override + public void checkCreateDirectory(Class callerClass, FileSystemProvider that, Path dir, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, dir); + } + + @Override + public void checkCreateSymbolicLink(Class callerClass, FileSystemProvider that, Path link, Path target, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, link); + policyManager.checkFileRead(callerClass, target); + } + + @Override + public void checkCreateLink(Class callerClass, FileSystemProvider that, Path link, Path existing) { + policyManager.checkFileWrite(callerClass, link); + policyManager.checkFileRead(callerClass, existing); + } + + @Override + public void checkDelete(Class callerClass, FileSystemProvider that, Path path) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void checkDeleteIfExists(Class callerClass, FileSystemProvider that, Path path) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void checkReadSymbolicLink(Class callerClass, FileSystemProvider that, Path link) { + policyManager.checkFileRead(callerClass, link); + } + + @Override + public void checkCopy(Class callerClass, FileSystemProvider that, Path source, Path target, CopyOption... options) { + policyManager.checkFileWrite(callerClass, target); + policyManager.checkFileRead(callerClass, source); + } + + @Override + public void checkMove(Class callerClass, FileSystemProvider that, Path source, Path target, CopyOption... options) { + policyManager.checkFileWrite(callerClass, target); + policyManager.checkFileWrite(callerClass, source); + } + + @Override + public void checkIsSameFile(Class callerClass, FileSystemProvider that, Path path, Path path2) { + policyManager.checkFileRead(callerClass, path); + policyManager.checkFileRead(callerClass, path2); + } + + @Override + public void checkIsHidden(Class callerClass, FileSystemProvider that, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkGetFileStore(Class callerClass, FileSystemProvider that, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkCheckAccess(Class callerClass, FileSystemProvider that, Path path, AccessMode... modes) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkGetFileAttributeView(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options) { + policyManager.checkGetFileAttributeView(callerClass); + } + + @Override + public void checkReadAttributes(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkReadAttributes(Class callerClass, FileSystemProvider that, Path path, String attributes, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkReadAttributesIfExists( + Class callerClass, + FileSystemProvider that, + Path path, + Class type, + LinkOption... options + ) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkSetAttribute( + Class callerClass, + FileSystemProvider that, + Path path, + String attribute, + Object value, + LinkOption... options + ) { + policyManager.checkFileWrite(callerClass, path); + + } + + @Override + public void checkExists(Class callerClass, FileSystemProvider that, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); } @Override diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index b4d099007866..0d6905f64bd7 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -294,6 +294,15 @@ public class PolicyManager { } } + /** + * Invoked when we try to get an arbitrary {@code FileAttributeView} class. Such a class can modify attributes, like owner etc.; + * we could think about introducing checks for each of the operations, but for now we over-approximate this and simply deny when it is + * used directly. + */ + public void checkGetFileAttributeView(Class callerClass) { + neverEntitled(callerClass, () -> "get file attribute view"); + } + /** * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions */ From a644202e9244671d372dace583c3f8749556d421 Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Wed, 12 Feb 2025 14:28:42 -0500 Subject: [PATCH 32/93] Extract metric handling from DesiredBalanceReconciler (#121771) The DesiredBalanceReconciler is responsible for applying updates to the cluster states that reflect shard allocation changes towards a DesiredBalance. It isn't the Reconciler's responsibility to handle pushing APM metrics. This patch cleans up the Reconciler constructor and logic by extracting metric handling, modularizing metric updates in the Allocator level of the code instead of being split across the two components. This will facilitate testing. This patch also contains general improvements in documentation around the desired balance Reconciler and Metrics related code. Relates ES-10581 --- .../allocator/DesiredBalanceMetrics.java | 51 ++++-- .../allocator/DesiredBalanceReconciler.java | 158 +++++++++--------- .../DesiredBalanceShardsAllocator.java | 43 ++++- .../allocator/OrderedShardsIterator.java | 21 +++ .../DesiredBalanceReconcilerTests.java | 17 +- 5 files changed, 176 insertions(+), 114 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java index fddf9267cdbb..0eb4d89bd6d3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java @@ -11,6 +11,7 @@ package org.elasticsearch.cluster.routing.allocation.allocator; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator.NodeAllocationStatsAndWeight; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.telemetry.metric.DoubleWithAttributes; import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -28,17 +29,28 @@ import java.util.concurrent.atomic.AtomicReference; */ public class DesiredBalanceMetrics { + /** + * @param unassignedShards Shards that are not assigned to any node. + * @param totalAllocations Shards that are assigned to a node. + * @param undesiredAllocationsExcludingShuttingDownNodes Shards that are assigned to a node but must move to alleviate a resource + * constraint per the {@link AllocationDeciders}. Excludes shards that must move + * because of a node shutting down. + */ public record AllocationStats(long unassignedShards, long totalAllocations, long undesiredAllocationsExcludingShuttingDownNodes) {} public record NodeWeightStats(long shardCount, double diskUsageInBytes, double writeLoad, double nodeWeight) {} - public static final DesiredBalanceMetrics NOOP = new DesiredBalanceMetrics(MeterRegistry.NOOP); - + // Reconciliation metrics. + /** See {@link #unassignedShards} */ public static final String UNASSIGNED_SHARDS_METRIC_NAME = "es.allocator.desired_balance.shards.unassigned.current"; + /** See {@link #totalAllocations} */ public static final String TOTAL_SHARDS_METRIC_NAME = "es.allocator.desired_balance.shards.current"; + /** See {@link #undesiredAllocationsExcludingShuttingDownNodes} */ public static final String UNDESIRED_ALLOCATION_COUNT_METRIC_NAME = "es.allocator.desired_balance.allocations.undesired.current"; + /** {@link #UNDESIRED_ALLOCATION_COUNT_METRIC_NAME} / {@link #TOTAL_SHARDS_METRIC_NAME} */ public static final String UNDESIRED_ALLOCATION_RATIO_METRIC_NAME = "es.allocator.desired_balance.allocations.undesired.ratio"; + // Desired balance node metrics. public static final String DESIRED_BALANCE_NODE_WEIGHT_METRIC_NAME = "es.allocator.desired_balance.allocations.node_weight.current"; public static final String DESIRED_BALANCE_NODE_SHARD_COUNT_METRIC_NAME = "es.allocator.desired_balance.allocations.node_shard_count.current"; @@ -47,6 +59,7 @@ public class DesiredBalanceMetrics { public static final String DESIRED_BALANCE_NODE_DISK_USAGE_METRIC_NAME = "es.allocator.desired_balance.allocations.node_disk_usage_bytes.current"; + // Node weight metrics. public static final String CURRENT_NODE_WEIGHT_METRIC_NAME = "es.allocator.allocations.node.weight.current"; public static final String CURRENT_NODE_SHARD_COUNT_METRIC_NAME = "es.allocator.allocations.node.shard_count.current"; public static final String CURRENT_NODE_WRITE_LOAD_METRIC_NAME = "es.allocator.allocations.node.write_load.current"; @@ -59,6 +72,7 @@ public class DesiredBalanceMetrics { public static final AllocationStats EMPTY_ALLOCATION_STATS = new AllocationStats(-1, -1, -1); private volatile boolean nodeIsMaster = false; + /** * Number of unassigned shards during last reconciliation */ @@ -70,9 +84,10 @@ public class DesiredBalanceMetrics { private volatile long totalAllocations; /** - * Number of assigned shards during last reconciliation that are not allocated on desired node and need to be moved + * Number of assigned shards during last reconciliation that are not allocated on a desired node and need to be moved. + * This excludes shards that must be reassigned due to a shutting down node. */ - private volatile long undesiredAllocations; + private volatile long undesiredAllocationsExcludingShuttingDownNodes; private final AtomicReference> weightStatsPerNodeRef = new AtomicReference<>(Map.of()); private final AtomicReference> allocationStatsPerNodeRef = new AtomicReference<>( @@ -89,7 +104,7 @@ public class DesiredBalanceMetrics { if (allocationStats != EMPTY_ALLOCATION_STATS) { this.unassignedShards = allocationStats.unassignedShards; this.totalAllocations = allocationStats.totalAllocations; - this.undesiredAllocations = allocationStats.undesiredAllocationsExcludingShuttingDownNodes; + this.undesiredAllocationsExcludingShuttingDownNodes = allocationStats.undesiredAllocationsExcludingShuttingDownNodes; } weightStatsPerNodeRef.set(weightStatsPerNode); allocationStatsPerNodeRef.set(nodeAllocationStats); @@ -107,7 +122,7 @@ public class DesiredBalanceMetrics { UNDESIRED_ALLOCATION_COUNT_METRIC_NAME, "Total number of shards allocated on undesired nodes excluding shutting down nodes", "{shard}", - this::getUndesiredAllocationsMetrics + this::getUndesiredAllocationsExcludingShuttingDownNodesMetrics ); meterRegistry.registerDoublesGauge( UNDESIRED_ALLOCATION_RATIO_METRIC_NAME, @@ -115,6 +130,7 @@ public class DesiredBalanceMetrics { "1", this::getUndesiredAllocationsRatioMetrics ); + meterRegistry.registerDoublesGauge( DESIRED_BALANCE_NODE_WEIGHT_METRIC_NAME, "Weight of nodes in the computed desired balance", @@ -133,18 +149,19 @@ public class DesiredBalanceMetrics { "bytes", this::getDesiredBalanceNodeDiskUsageMetrics ); - meterRegistry.registerDoublesGauge( - CURRENT_NODE_WEIGHT_METRIC_NAME, - "The weight of nodes based on the current allocation state", - "unit", - this::getCurrentNodeWeightMetrics - ); meterRegistry.registerLongsGauge( DESIRED_BALANCE_NODE_SHARD_COUNT_METRIC_NAME, "Shard count of nodes in the computed desired balance", "unit", this::getDesiredBalanceNodeShardCountMetrics ); + + meterRegistry.registerDoublesGauge( + CURRENT_NODE_WEIGHT_METRIC_NAME, + "The weight of nodes based on the current allocation state", + "unit", + this::getCurrentNodeWeightMetrics + ); meterRegistry.registerDoublesGauge( CURRENT_NODE_WRITE_LOAD_METRIC_NAME, "The current write load of nodes", @@ -194,7 +211,7 @@ public class DesiredBalanceMetrics { } public long undesiredAllocations() { - return undesiredAllocations; + return undesiredAllocationsExcludingShuttingDownNodes; } private List getUnassignedShardsMetrics() { @@ -330,8 +347,8 @@ public class DesiredBalanceMetrics { return getIfPublishing(totalAllocations); } - private List getUndesiredAllocationsMetrics() { - return getIfPublishing(undesiredAllocations); + private List getUndesiredAllocationsExcludingShuttingDownNodesMetrics() { + return getIfPublishing(undesiredAllocationsExcludingShuttingDownNodes); } private List getIfPublishing(long value) { @@ -344,7 +361,7 @@ public class DesiredBalanceMetrics { private List getUndesiredAllocationsRatioMetrics() { if (nodeIsMaster) { var total = totalAllocations; - var undesired = undesiredAllocations; + var undesired = undesiredAllocationsExcludingShuttingDownNodes; return List.of(new DoubleWithAttributes(total != 0 ? (double) undesired / total : 0.0)); } return List.of(); @@ -357,7 +374,7 @@ public class DesiredBalanceMetrics { public void zeroAllMetrics() { unassignedShards = 0; totalAllocations = 0; - undesiredAllocations = 0; + undesiredAllocationsExcludingShuttingDownNodes = 0; weightStatsPerNodeRef.set(Map.of()); allocationStatsPerNodeRef.set(Map.of()); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java index 909a7a7a99a6..dd7216758c3b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java @@ -20,10 +20,7 @@ import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus; -import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator; -import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator.NodeAllocationStatsAndWeight; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; -import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceMetrics.AllocationStats; import org.elasticsearch.cluster.routing.allocation.decider.Decision; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.ClusterSettings; @@ -36,9 +33,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.threadpool.ThreadPool; import java.util.Comparator; -import java.util.HashMap; import java.util.Iterator; -import java.util.Map; import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -82,16 +77,8 @@ public class DesiredBalanceReconciler { private double undesiredAllocationsLogThreshold; private final NodeAllocationOrdering allocationOrdering = new NodeAllocationOrdering(); private final NodeAllocationOrdering moveOrdering = new NodeAllocationOrdering(); - private final DesiredBalanceMetrics desiredBalanceMetrics; - private final NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator; - public DesiredBalanceReconciler( - ClusterSettings clusterSettings, - ThreadPool threadPool, - DesiredBalanceMetrics desiredBalanceMetrics, - NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator - ) { - this.desiredBalanceMetrics = desiredBalanceMetrics; + public DesiredBalanceReconciler(ClusterSettings clusterSettings, ThreadPool threadPool) { this.undesiredAllocationLogInterval = new FrequencyCappedAction( threadPool.relativeTimeInMillisSupplier(), TimeValue.timeValueMinutes(5) @@ -101,7 +88,6 @@ public class DesiredBalanceReconciler { UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING, value -> this.undesiredAllocationsLogThreshold = value ); - this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator; } /** @@ -110,12 +96,13 @@ public class DesiredBalanceReconciler { * @param desiredBalance The new desired cluster shard allocation * @param allocation Cluster state information with which to make decisions, contains routing table metadata that will be modified to * reach the given desired balance. + * @return {@link DesiredBalanceMetrics.AllocationStats} for this round of reconciliation changes. */ - public void reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) { + public DesiredBalanceMetrics.AllocationStats reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) { var nodeIds = allocation.routingNodes().getAllNodeIds(); allocationOrdering.retainNodes(nodeIds); moveOrdering.retainNodes(nodeIds); - new Reconciliation(desiredBalance, allocation).run(); + return new Reconciliation(desiredBalance, allocation).run(); } public void clear() { @@ -123,6 +110,11 @@ public class DesiredBalanceReconciler { moveOrdering.clear(); } + /** + * Handles updating the {@code RoutingNodes} to reflect the next steps towards the new {@code DesiredBalance}. Updates are limited by + * throttling (there are limits on the number of concurrent shard moves) or resource constraints (some shard moves might not be + * immediately possible until other shards move first). + */ private class Reconciliation { private final DesiredBalance desiredBalance; @@ -135,7 +127,7 @@ public class DesiredBalanceReconciler { this.routingNodes = allocation.routingNodes(); } - void run() { + DesiredBalanceMetrics.AllocationStats run() { try (var ignored = allocation.withReconcilingFlag()) { logger.debug("Reconciling desired balance for [{}]", desiredBalance.lastConvergedIndex()); @@ -144,13 +136,13 @@ public class DesiredBalanceReconciler { // no data nodes, so fail allocation to report red health failAllocationOfNewPrimaries(allocation); logger.trace("no nodes available, nothing to reconcile"); - return; + return DesiredBalanceMetrics.EMPTY_ALLOCATION_STATS; } if (desiredBalance.assignments().isEmpty()) { // no desired state yet but it is on its way and we'll reroute again when it is ready logger.trace("desired balance is empty, nothing to reconcile"); - return; + return DesiredBalanceMetrics.EMPTY_ALLOCATION_STATS; } // compute next moves towards current desired balance: @@ -163,38 +155,22 @@ public class DesiredBalanceReconciler { // 2. move any shards that cannot remain where they are logger.trace("Reconciler#moveShards"); moveShards(); + // 3. move any other shards that are desired elsewhere + // This is the rebalancing work. The previous calls were necessary, to assign unassigned shard copies, and move shards that + // violate resource thresholds. Now we run moves to improve the relative node resource loads. logger.trace("Reconciler#balance"); - var allocationStats = balance(); + DesiredBalanceMetrics.AllocationStats allocationStats = balance(); logger.debug("Reconciliation is complete"); - - updateDesireBalanceMetrics(allocationStats); + return allocationStats; } } - private void updateDesireBalanceMetrics(AllocationStats allocationStats) { - var nodesStatsAndWeights = nodeAllocationStatsAndWeightsCalculator.nodesAllocationStatsAndWeights( - allocation.metadata(), - allocation.routingNodes(), - allocation.clusterInfo(), - desiredBalance - ); - Map filteredNodeAllocationStatsAndWeights = new HashMap<>( - nodesStatsAndWeights.size() - ); - for (var nodeStatsAndWeight : nodesStatsAndWeights.entrySet()) { - var node = allocation.nodes().get(nodeStatsAndWeight.getKey()); - if (node != null) { - filteredNodeAllocationStatsAndWeights.put(node, nodeStatsAndWeight.getValue()); - } - } - desiredBalanceMetrics.updateMetrics(allocationStats, desiredBalance.weightsPerNode(), filteredNodeAllocationStatsAndWeights); - } - + /** + * Checks whether every shard is either assigned or ignored. Expected to be called after {@link #allocateUnassigned()}. + */ private boolean allocateUnassignedInvariant() { - // after allocateUnassigned, every shard must be either assigned or ignored - assert routingNodes.unassigned().isEmpty(); final var shardCounts = allocation.metadata().stream().filter(indexMetadata -> @@ -263,45 +239,55 @@ public class DesiredBalanceReconciler { } /* + * Create some comparators to sort the unassigned shard copies in priority to allocate order. * TODO: We could be smarter here and group the shards by index and then * use the sorter to save some iterations. */ - final PriorityComparator secondaryComparator = PriorityComparator.getAllocationComparator(allocation); - final Comparator comparator = (o1, o2) -> { + final PriorityComparator indexPriorityComparator = PriorityComparator.getAllocationComparator(allocation); + final Comparator shardAllocationPriorityComparator = (o1, o2) -> { + // Prioritize assigning a primary shard copy, if one is a primary and the other is not. if (o1.primary() ^ o2.primary()) { return o1.primary() ? -1 : 1; } + + // Then order shards in the same index arbitrarily by shard ID. if (o1.getIndexName().compareTo(o2.getIndexName()) == 0) { return o1.getId() - o2.getId(); } + + // Lastly, prioritize system indices, then use index priority of non-system indices, then by age, etc. + // // this comparator is more expensive than all the others up there // that's why it's added last even though it could be easier to read // if we'd apply it earlier. this comparator will only differentiate across // indices all shards of the same index is treated equally. - final int secondary = secondaryComparator.compare(o1, o2); - assert secondary != 0 : "Index names are equal, should be returned early."; - return secondary; + final int secondaryComparison = indexPriorityComparator.compare(o1, o2); + assert secondaryComparison != 0 : "Index names are equal, should be returned early."; + return secondaryComparison; }; + /* * we use 2 arrays and move replicas to the second array once we allocated an identical * replica in the current iteration to make sure all indices get allocated in the same manner. - * The arrays are sorted by primaries first and then by index and shard ID so a 2 indices with + * The arrays are sorted by primaries first and then by index and shard ID so 2 indices with * 2 replica and 1 shard would look like: * [(0,P,IDX1), (0,P,IDX2), (0,R,IDX1), (0,R,IDX1), (0,R,IDX2), (0,R,IDX2)] * if we allocate for instance (0, R, IDX1) we move the second replica to the secondary array and proceed with * the next replica. If we could not find a node to allocate (0,R,IDX1) we move all it's replicas to ignoreUnassigned. */ - ShardRouting[] primary = unassigned.drain(); - ShardRouting[] secondary = new ShardRouting[primary.length]; - int secondaryLength = 0; - int primaryLength = primary.length; - ArrayUtil.timSort(primary, comparator); + ShardRouting[] orderedShardAllocationList = unassigned.drain(); + ShardRouting[] deferredShardAllocationList = new ShardRouting[orderedShardAllocationList.length]; + int deferredShardAllocationListLength = 0; + int orderedShardAllocationListLength = orderedShardAllocationList.length; + ArrayUtil.timSort(orderedShardAllocationList, shardAllocationPriorityComparator); do { - nextShard: for (int i = 0; i < primaryLength; i++) { - final var shard = primary[i]; + nextShard: for (int i = 0; i < orderedShardAllocationListLength; i++) { + final var shard = orderedShardAllocationList[i]; final var assignment = desiredBalance.getAssignment(shard.shardId()); + // An ignored shard copy is one that has no desired balance assignment. final boolean ignored = assignment == null || isIgnored(routingNodes, shard, assignment); + AllocationStatus unallocatedStatus; if (ignored) { unallocatedStatus = AllocationStatus.NO_ATTEMPT; @@ -331,8 +317,13 @@ public class DesiredBalanceReconciler { if (shard.primary() == false) { // copy over the same replica shards to the secondary array so they will get allocated // in a subsequent iteration, allowing replicas of other shards to be allocated first - while (i < primaryLength - 1 && comparator.compare(primary[i], primary[i + 1]) == 0) { - secondary[secondaryLength++] = primary[++i]; + while (i < orderedShardAllocationListLength - 1 + && shardAllocationPriorityComparator.compare( + orderedShardAllocationList[i], + orderedShardAllocationList[i + 1] + ) == 0) { + deferredShardAllocationList[deferredShardAllocationListLength++] = + orderedShardAllocationList[++i]; } } continue nextShard; @@ -352,18 +343,23 @@ public class DesiredBalanceReconciler { logger.debug("No eligible node found to assign shard [{}]", shard); unassigned.ignoreShard(shard, unallocatedStatus, allocation.changes()); if (shard.primary() == false) { - // we could not allocate it and we are a replica - check if we can ignore the other replicas - while (i < primaryLength - 1 && comparator.compare(primary[i], primary[i + 1]) == 0) { - unassigned.ignoreShard(primary[++i], unallocatedStatus, allocation.changes()); + // We could not allocate the shard copy and the copy is a replica: check if we can ignore the other unassigned + // replicas. + while (i < orderedShardAllocationListLength - 1 + && shardAllocationPriorityComparator.compare( + orderedShardAllocationList[i], + orderedShardAllocationList[i + 1] + ) == 0) { + unassigned.ignoreShard(orderedShardAllocationList[++i], unallocatedStatus, allocation.changes()); } } } - primaryLength = secondaryLength; - ShardRouting[] tmp = primary; - primary = secondary; - secondary = tmp; - secondaryLength = 0; - } while (primaryLength > 0); + ShardRouting[] tmp = orderedShardAllocationList; + orderedShardAllocationList = deferredShardAllocationList; + deferredShardAllocationList = tmp; + orderedShardAllocationListLength = deferredShardAllocationListLength; + deferredShardAllocationListLength = 0; + } while (orderedShardAllocationListLength > 0); } private final class NodeIdsIterator implements Iterator { @@ -371,11 +367,7 @@ public class DesiredBalanceReconciler { private final ShardRouting shard; private final RoutingNodes routingNodes; /** - * Contains the source of the nodeIds used for shard assignment. It could be: - * * desired - when using desired nodes - * * forced initial allocation - when initial allocation is forced to certain nodes by shrink/split/clone index operation - * * fallback - when assigning the primary shard is temporarily not possible on desired nodes, - * and it is assigned elsewhere in the cluster + * Contains the source of the nodeIds used for shard assignment. */ private NodeIdSource source; private Iterator nodeIds; @@ -431,11 +423,21 @@ public class DesiredBalanceReconciler { } private enum NodeIdSource { + // Using desired nodes. DESIRED, + // Initial allocation is forced to certain nodes by shrink/split/clone index operation. FORCED_INITIAL_ALLOCATION, + // Assigning the primary shard is temporarily not possible on desired nodes, and it is assigned elsewhere in the cluster. FALLBACK; } + /** + * Checks whether the {@code shard} copy has been assigned to a node or not in {@code assignment}. + * @param routingNodes The current routing information + * @param shard A particular shard copy + * @param assignment The assignments for shard primary and replica copies + * @return Whether the shard has a node assignment. + */ private boolean isIgnored(RoutingNodes routingNodes, ShardRouting shard, ShardAssignment assignment) { if (assignment.ignored() == 0) { // no shards are ignored @@ -512,7 +514,8 @@ public class DesiredBalanceReconciler { } } - private AllocationStats balance() { + private DesiredBalanceMetrics.AllocationStats balance() { + // Check if rebalancing is disabled. if (allocation.deciders().canRebalance(allocation).type() != Decision.Type.YES) { return DesiredBalanceMetrics.EMPTY_ALLOCATION_STATS; } @@ -581,8 +584,11 @@ public class DesiredBalanceReconciler { } maybeLogUndesiredAllocationsWarning(totalAllocations, undesiredAllocationsExcludingShuttingDownNodes, routingNodes.size()); - - return new AllocationStats(unassignedShards, totalAllocations, undesiredAllocationsExcludingShuttingDownNodes); + return new DesiredBalanceMetrics.AllocationStats( + unassignedShards, + totalAllocations, + undesiredAllocationsExcludingShuttingDownNodes + ); } private void maybeLogUndesiredAllocationsWarning(int totalAllocations, int undesiredAllocations, int nodeCount) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java index d9fba492fb9d..cb3f3b306d80 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.AllocationService.RerouteStrategy; @@ -39,6 +40,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -87,6 +89,7 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { private final AtomicReference currentDesiredBalanceRef = new AtomicReference<>(DesiredBalance.NOT_MASTER); private volatile boolean resetCurrentDesiredBalance = false; private final Set processedNodeShutdowns = new HashSet<>(); + private final NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator; private final DesiredBalanceMetrics desiredBalanceMetrics; /** * Manages balancer round results in order to report on the balancer activity in a configurable manner. @@ -136,17 +139,13 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator ) { this.desiredBalanceMetrics = new DesiredBalanceMetrics(telemetryProvider.getMeterRegistry()); + this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator; this.balancerRoundSummaryService = new AllocationBalancingRoundSummaryService(threadPool, clusterService.getClusterSettings()); this.delegateAllocator = delegateAllocator; this.threadPool = threadPool; this.reconciler = reconciler; this.desiredBalanceComputer = desiredBalanceComputer; - this.desiredBalanceReconciler = new DesiredBalanceReconciler( - clusterService.getClusterSettings(), - threadPool, - desiredBalanceMetrics, - nodeAllocationStatsAndWeightsCalculator - ); + this.desiredBalanceReconciler = new DesiredBalanceReconciler(clusterService.getClusterSettings(), threadPool); this.desiredBalanceComputation = new ContinuousComputation<>(threadPool.generic()) { @Override @@ -347,6 +346,10 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { return new BalancingRoundSummary(DesiredBalance.shardMovements(oldDesiredBalance, newDesiredBalance)); } + /** + * Submits the desired balance to be reconciled (applies the desired changes to the routing table) and creates and publishes a new + * cluster state. The data nodes will receive and apply the new cluster state to start/move/remove shards. + */ protected void submitReconcileTask(DesiredBalance desiredBalance) { masterServiceTaskQueue.submitTask("reconcile-desired-balance", new ReconcileDesiredBalanceTask(desiredBalance), null); } @@ -357,7 +360,11 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { } else { logger.debug("Reconciling desired balance for [{}]", desiredBalance.lastConvergedIndex()); } - recordTime(cumulativeReconciliationTime, () -> desiredBalanceReconciler.reconcile(desiredBalance, allocation)); + recordTime(cumulativeReconciliationTime, () -> { + DesiredBalanceMetrics.AllocationStats allocationStats = desiredBalanceReconciler.reconcile(desiredBalance, allocation); + updateDesireBalanceMetrics(desiredBalance, allocation, allocationStats); + }); + if (logger.isTraceEnabled()) { logger.trace("Reconciled desired balance: {}", desiredBalance); } else { @@ -391,6 +398,28 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { resetCurrentDesiredBalance = true; } + private void updateDesireBalanceMetrics( + DesiredBalance desiredBalance, + RoutingAllocation routingAllocation, + DesiredBalanceMetrics.AllocationStats allocationStats + ) { + var nodesStatsAndWeights = nodeAllocationStatsAndWeightsCalculator.nodesAllocationStatsAndWeights( + routingAllocation.metadata(), + routingAllocation.routingNodes(), + routingAllocation.clusterInfo(), + desiredBalance + ); + Map filteredNodeAllocationStatsAndWeights = + new HashMap<>(nodesStatsAndWeights.size()); + for (var nodeStatsAndWeight : nodesStatsAndWeights.entrySet()) { + var node = routingAllocation.nodes().get(nodeStatsAndWeight.getKey()); + if (node != null) { + filteredNodeAllocationStatsAndWeights.put(node, nodeStatsAndWeight.getValue()); + } + } + desiredBalanceMetrics.updateMetrics(allocationStats, desiredBalance.weightsPerNode(), filteredNodeAllocationStatsAndWeights); + } + public DesiredBalanceStats getStats() { return new DesiredBalanceStats( Math.max(currentDesiredBalanceRef.get().lastConvergedIndex(), 0L), diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/OrderedShardsIterator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/OrderedShardsIterator.java index 8229c2142356..3b8c4403c4c8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/OrderedShardsIterator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/OrderedShardsIterator.java @@ -33,10 +33,28 @@ public class OrderedShardsIterator implements Iterator { private final ArrayDeque queue; + /** + * This iterator will progress through the shards node by node, each node's shards ordered from most write active to least. + * + * @param allocation + * @param ordering + * @return An iterator over all shards in the {@link RoutingNodes} held by {@code allocation} (all shards assigned to a node). The + * iterator will progress node by node, where each node's shards are ordered from data stream write indices, to regular indices and + * lastly to data stream read indices. + */ public static OrderedShardsIterator createForNecessaryMoves(RoutingAllocation allocation, NodeAllocationOrdering ordering) { return create(allocation.routingNodes(), createShardsComparator(allocation.metadata()), ordering); } + /** + * This iterator will progress through the shards node by node, each node's shards ordered from least write active to most. + * + * @param allocation + * @param ordering + * @return An iterator over all shards in the {@link RoutingNodes} held by {@code allocation} (all shards assigned to a node). The + * iterator will progress node by node, where each node's shards are ordered from data stream read indices, to regular indices and + * lastly to data stream write indices. + */ public static OrderedShardsIterator createForBalancing(RoutingAllocation allocation, NodeAllocationOrdering ordering) { return create(allocation.routingNodes(), createShardsComparator(allocation.metadata()).reversed(), ordering); } @@ -61,6 +79,9 @@ public class OrderedShardsIterator implements Iterator { return Iterators.forArray(shards); } + /** + * Prioritizes write indices of data streams, and deprioritizes data stream read indices, relative to regular indices. + */ private static Comparator createShardsComparator(Metadata metadata) { return Comparator.comparing(shard -> { var lookup = metadata.getIndicesLookup().get(shard.getIndexName()); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java index cd94c87bb4b5..81aa1a60eb45 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java @@ -1212,12 +1212,7 @@ public class DesiredBalanceReconcilerTests extends ESAllocationTestCase { new ConcurrentRebalanceAllocationDecider(clusterSettings), new ThrottlingAllocationDecider(clusterSettings) }; - var reconciler = new DesiredBalanceReconciler( - clusterSettings, - new DeterministicTaskQueue().getThreadPool(), - DesiredBalanceMetrics.NOOP, - EMPTY_NODE_ALLOCATION_STATS - ); + var reconciler = new DesiredBalanceReconciler(clusterSettings, new DeterministicTaskQueue().getThreadPool()); var totalOutgoingMoves = new HashMap(); for (int i = 0; i < numberOfNodes; i++) { @@ -1299,12 +1294,7 @@ public class DesiredBalanceReconcilerTests extends ESAllocationTestCase { final var timeInMillisSupplier = new AtomicLong(); when(threadPool.relativeTimeInMillisSupplier()).thenReturn(timeInMillisSupplier::incrementAndGet); - var reconciler = new DesiredBalanceReconciler( - createBuiltInClusterSettings(), - threadPool, - DesiredBalanceMetrics.NOOP, - EMPTY_NODE_ALLOCATION_STATS - ); + var reconciler = new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool); final long initialDelayInMillis = TimeValue.timeValueMinutes(5).getMillis(); timeInMillisSupplier.addAndGet(randomLongBetween(initialDelayInMillis, 2 * initialDelayInMillis)); @@ -1356,8 +1346,7 @@ public class DesiredBalanceReconcilerTests extends ESAllocationTestCase { private static void reconcile(RoutingAllocation routingAllocation, DesiredBalance desiredBalance) { final var threadPool = mock(ThreadPool.class); when(threadPool.relativeTimeInMillisSupplier()).thenReturn(new AtomicLong()::incrementAndGet); - new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool, DesiredBalanceMetrics.NOOP, EMPTY_NODE_ALLOCATION_STATS) - .reconcile(desiredBalance, routingAllocation); + new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool).reconcile(desiredBalance, routingAllocation); } private static boolean isReconciled(RoutingNode node, DesiredBalance balance) { From a7cf0f6875087e3f734ae18d7b082f2a15fd7b84 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 12 Feb 2025 22:27:54 +0200 Subject: [PATCH 33/93] [Failure store] Remove `::*` selector (#121900) --- .../org/elasticsearch/TransportVersions.java | 1 + .../indices/resolve/ResolveIndexAction.java | 11 -- .../indices/rollover/RolloverRequest.java | 15 +- .../support/IndexComponentSelector.java | 18 +- .../metadata/IndexAbstractionResolver.java | 23 +-- .../metadata/IndexNameExpressionResolver.java | 69 ++------ .../transport/RemoteClusterAware.java | 6 +- .../rollover/RolloverRequestTests.java | 2 +- .../support/IndexComponentSelectorTests.java | 7 +- .../IndexAbstractionResolverTests.java | 24 +-- .../IndexNameExpressionResolverTests.java | 47 ++++- .../metadata/SelectorResolverTests.java | 9 +- .../WildcardExpressionResolverTests.java | 165 ++++-------------- .../authz/IndicesAndAliasesResolverTests.java | 16 -- 14 files changed, 130 insertions(+), 283 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 058251e3c269..1a4bb7fde5ef 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -186,6 +186,7 @@ public class TransportVersions { public static final TransportVersion ESQL_RETRY_ON_SHARD_LEVEL_FAILURE = def(9_006_0_00); public static final TransportVersion ESQL_PROFILE_ASYNC_NANOS = def(9_007_00_0); public static final TransportVersion ESQL_LOOKUP_JOIN_SOURCE_TEXT = def(9_008_0_00); + public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR = def(9_009_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 0f1b77af0242..d61901b246d0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -646,10 +646,6 @@ public class ResolveIndexAction extends ActionType : switch (resolvedExpression.selector()) { case DATA -> dataStream.getDataComponent().getIndices().stream(); case FAILURES -> dataStream.getFailureIndices().stream(); - case ALL_APPLICABLE -> Stream.concat( - dataStream.getIndices().stream(), - dataStream.getFailureIndices().stream() - ); }; String[] backingIndices = dataStreamIndices.map(Index::getName).toArray(String[]::new); dataStreams.add(new ResolvedDataStream(dataStream.getName(), backingIndices, DataStream.TIMESTAMP_FIELD_NAME)); @@ -670,13 +666,6 @@ public class ResolveIndexAction extends ActionType assert ia.isDataStreamRelated() : "Illegal selector [failures] used on non data stream alias"; yield ia.getFailureIndices(metadata).stream(); } - case ALL_APPLICABLE -> { - if (ia.isDataStreamRelated()) { - yield Stream.concat(ia.getIndices().stream(), ia.getFailureIndices(metadata).stream()); - } else { - yield ia.getIndices().stream(); - } - } }; } return aliasIndices; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 608d32d50a85..cb46d039c5b3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -13,14 +13,13 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -126,14 +125,12 @@ public class RolloverRequest extends AcknowledgedRequest implem ); } + // Ensure we have a valid selector in the request if (rolloverTarget != null) { - ResolvedExpression resolvedExpression = SelectorResolver.parseExpression(rolloverTarget, indicesOptions); - IndexComponentSelector selector = resolvedExpression.selector(); - if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { - validationException = addValidationError( - "rollover cannot be applied to both regular and failure indices at the same time", - validationException - ); + try { + SelectorResolver.parseExpression(rolloverTarget, indicesOptions); + } catch (InvalidIndexNameException exception) { + validationException = addValidationError(exception.getMessage(), validationException); } } diff --git a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java index 910be151d1bf..d41042f4c657 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.support; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -23,14 +24,11 @@ import java.util.Map; * We define as index components the two different sets of indices a data stream could consist of: * - DATA: represents the backing indices * - FAILURES: represent the failing indices - * - ALL: represents all available in this expression components, meaning if it's a data stream both backing and failure indices and if it's - * an index only the index itself. * Note: An index is its own DATA component, but it cannot have a FAILURE component. */ public enum IndexComponentSelector implements Writeable { DATA("data", (byte) 0), - FAILURES("failures", (byte) 1), - ALL_APPLICABLE("*", (byte) 2); + FAILURES("failures", (byte) 1); private final String key; private final byte id; @@ -75,7 +73,13 @@ public enum IndexComponentSelector implements Writeable { } public static IndexComponentSelector read(StreamInput in) throws IOException { - return getById(in.readByte()); + byte id = in.readByte(); + if (in.getTransportVersion().onOrAfter(TransportVersions.REMOVE_ALL_APPLICABLE_SELECTOR)) { + return getById(id); + } else { + // Legacy value ::*, converted to ::data + return id == 2 ? DATA : getById(id); + } } // Visible for testing @@ -95,10 +99,10 @@ public enum IndexComponentSelector implements Writeable { } public boolean shouldIncludeData() { - return this == ALL_APPLICABLE || this == DATA; + return this == DATA; } public boolean shouldIncludeFailures() { - return this == ALL_APPLICABLE || this == FAILURES; + return this == FAILURES; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java index d83bde9542d9..fe7199f8332d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java @@ -81,8 +81,7 @@ public class IndexAbstractionResolver { indexNameExpressionResolver, includeDataStreams )) { - // Resolve any ::* suffixes on the expression. We need to resolve them all to their final valid selectors - resolveSelectorsAndCombine(authorizedIndex, selectorString, indicesOptions, resolvedIndices, metadata); + resolveSelectorsAndCollect(authorizedIndex, selectorString, indicesOptions, resolvedIndices, metadata); } } if (resolvedIndices.isEmpty()) { @@ -98,9 +97,8 @@ public class IndexAbstractionResolver { } } } else { - // Resolve any ::* suffixes on the expression. We need to resolve them all to their final valid selectors Set resolvedIndices = new HashSet<>(); - resolveSelectorsAndCombine(indexAbstraction, selectorString, indicesOptions, resolvedIndices, metadata); + resolveSelectorsAndCollect(indexAbstraction, selectorString, indicesOptions, resolvedIndices, metadata); if (minus) { finalIndices.removeAll(resolvedIndices); } else if (indicesOptions.ignoreUnavailable() == false || isAuthorized.test(indexAbstraction)) { @@ -114,7 +112,7 @@ public class IndexAbstractionResolver { return finalIndices; } - private static void resolveSelectorsAndCombine( + private static void resolveSelectorsAndCollect( String indexAbstraction, String selectorString, IndicesOptions indicesOptions, @@ -132,19 +130,8 @@ public class IndexAbstractionResolver { selectorString = IndexComponentSelector.DATA.getKey(); } - if (Regex.isMatchAllPattern(selectorString)) { - // Always accept data - collect.add(IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, IndexComponentSelector.DATA.getKey())); - // Only put failures on the expression if the abstraction supports it. - if (acceptsAllSelectors) { - collect.add( - IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, IndexComponentSelector.FAILURES.getKey()) - ); - } - } else { - // A non-wildcard selector is always passed along as-is, it's validity for this kind of abstraction is tested later - collect.add(IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, selectorString)); - } + // A selector is always passed along as-is, it's validity for this kind of abstraction is tested later + collect.add(IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, selectorString)); } else { assert selectorString == null : "A selector string [" + selectorString + "] is present but selectors are disabled in this context"; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index cb074b143704..d28049f2a631 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -364,21 +364,9 @@ public class IndexNameExpressionResolver { } } else { if (isExclusion) { - if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { - resources.remove(new ResolvedExpression(baseExpression, IndexComponentSelector.DATA)); - resources.remove(new ResolvedExpression(baseExpression, IndexComponentSelector.FAILURES)); - } else { - resources.remove(new ResolvedExpression(baseExpression, selector)); - } + resources.remove(new ResolvedExpression(baseExpression, selector)); } else if (ensureAliasOrIndexExists(context, baseExpression, selector)) { - if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { - resources.add(new ResolvedExpression(baseExpression, IndexComponentSelector.DATA)); - if (context.getState().getMetadata().getIndicesLookup().get(baseExpression).isDataStreamRelated()) { - resources.add(new ResolvedExpression(baseExpression, IndexComponentSelector.FAILURES)); - } - } else { - resources.add(new ResolvedExpression(baseExpression, selector)); - } + resources.add(new ResolvedExpression(baseExpression, selector)); } } } @@ -1046,8 +1034,7 @@ public class IndexNameExpressionResolver { private static boolean resolvedExpressionsContainsAbstraction(Set resolvedExpressions, String abstractionName) { return resolvedExpressions.contains(new ResolvedExpression(abstractionName)) - || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA)) - || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.ALL_APPLICABLE)); + || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA)); } /** @@ -1342,8 +1329,7 @@ public class IndexNameExpressionResolver { if (context.options.allowSelectors()) { // Ensure that the selectors are present and that they are compatible with the abstractions they are used with assert selector != null : "Earlier logic should have parsed selectors or added the default selectors already"; - // Check if ::failures has been explicitly requested, since requesting ::* for non-data-stream abstractions would just - // return their data components. + // Check if ::failures has been explicitly requested if (IndexComponentSelector.FAILURES.equals(selector) && indexAbstraction.isDataStreamRelated() == false) { // If requested abstraction is not data stream related, then you cannot use ::failures if (ignoreUnavailable) { @@ -1700,9 +1686,9 @@ public class IndexNameExpressionResolver { final IndexMetadata.State excludeState = excludeState(context.getOptions()); Set resources = new HashSet<>(); if (context.isPreserveAliases() && indexAbstraction.getType() == Type.ALIAS) { - expandToApplicableSelectors(indexAbstraction, selector, resources); + resources.add(new ResolvedExpression(indexAbstraction.getName(), selector)); } else if (context.isPreserveDataStreams() && indexAbstraction.getType() == Type.DATA_STREAM) { - expandToApplicableSelectors(indexAbstraction, selector, resources); + resources.add(new ResolvedExpression(indexAbstraction.getName(), selector)); } else { if (shouldIncludeRegularIndices(context.getOptions(), selector)) { for (int i = 0, n = indexAbstraction.getIndices().size(); i < n; i++) { @@ -1729,31 +1715,6 @@ public class IndexNameExpressionResolver { return resources; } - /** - * Adds the abstraction and selector to the results when preserving data streams and aliases at wildcard resolution. If a selector - * is provided, the result is only added if the selector is applicable to the abstraction provided. If - * {@link IndexComponentSelector#ALL_APPLICABLE} is given, the selectors are expanded only to those which are applicable to the - * provided abstraction. - * @param indexAbstraction abstraction to add - * @param selector The selector to add - * @param resources Result collector which is updated with all applicable resolved expressions for a given abstraction and selector - * pair. - */ - private static void expandToApplicableSelectors( - IndexAbstraction indexAbstraction, - IndexComponentSelector selector, - Set resources - ) { - if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { - resources.add(new ResolvedExpression(indexAbstraction.getName(), IndexComponentSelector.DATA)); - if (indexAbstraction.isDataStreamRelated()) { - resources.add(new ResolvedExpression(indexAbstraction.getName(), IndexComponentSelector.FAILURES)); - } - } else if (selector == null || indexAbstraction.isDataStreamRelated() || selector.shouldIncludeFailures() == false) { - resources.add(new ResolvedExpression(indexAbstraction.getName(), selector)); - } - } - private static List resolveEmptyOrTrivialWildcard(Context context, IndexComponentSelector selector) { final String[] allIndices = resolveEmptyOrTrivialWildcardToAllIndices( context.getOptions(), @@ -2150,20 +2111,10 @@ public class IndexNameExpressionResolver { String suffix = expression.substring(lastDoubleColon + SELECTOR_SEPARATOR.length()); IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix); if (selector == null) { - // Do some work to surface a helpful error message for likely errors - if (Regex.isSimpleMatchPattern(suffix)) { - throw new InvalidIndexNameException( - expression, - "Invalid usage of :: separator, [" - + suffix - + "] contains a wildcard, but only the match all wildcard [*] is supported in a selector" - ); - } else { - throw new InvalidIndexNameException( - expression, - "Invalid usage of :: separator, [" + suffix + "] is not a recognized selector" - ); - } + throw new InvalidIndexNameException( + expression, + "invalid usage of :: separator, [" + suffix + "] is not a recognized selector" + ); } String expressionBase = expression.substring(0, lastDoubleColon); ensureNoMoreSelectorSeparators(expressionBase, expression); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 8399f5dd72f7..95e507f70d7a 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -157,15 +157,15 @@ public abstract class RemoteClusterAware { if (indexName.equals("*") == false) { throw new IllegalArgumentException( Strings.format( - "To exclude a cluster you must specify the '*' wildcard for " + "the index expression, but found: [%s]", + "To exclude a cluster you must specify the '*' wildcard for the index expression, but found: [%s]", indexName ) ); } - if (selectorString != null && selectorString.equals("*") == false) { + if (selectorString != null) { throw new IllegalArgumentException( Strings.format( - "To exclude a cluster you must specify the '::*' selector or leave it off, but found: [%s]", + "To exclude a cluster you must not specify the a selector, but found selector: [%s]", selectorString ) ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index ee95f7ffb5b9..515d571243a7 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -253,7 +253,7 @@ public class RolloverRequestTests extends ESTestCase { assertNotNull(validationException); assertEquals(1, validationException.validationErrors().size()); assertEquals( - "rollover cannot be applied to both regular and failure indices at the same time", + "Invalid index name [alias-index::*], invalid usage of :: separator, [*] is not a recognized selector", validationException.validationErrors().get(0) ); } diff --git a/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java b/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java index 73d4ab59ce47..585d660917e4 100644 --- a/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java @@ -20,7 +20,7 @@ public class IndexComponentSelectorTests extends ESTestCase { public void testIndexComponentSelectorFromKey() { assertThat(IndexComponentSelector.getByKey("data"), equalTo(IndexComponentSelector.DATA)); assertThat(IndexComponentSelector.getByKey("failures"), equalTo(IndexComponentSelector.FAILURES)); - assertThat(IndexComponentSelector.getByKey("*"), equalTo(IndexComponentSelector.ALL_APPLICABLE)); + assertThat(IndexComponentSelector.getByKey("*"), nullValue()); assertThat(IndexComponentSelector.getByKey("d*ta"), nullValue()); assertThat(IndexComponentSelector.getByKey("_all"), nullValue()); assertThat(IndexComponentSelector.getByKey("**"), nullValue()); @@ -30,11 +30,10 @@ public class IndexComponentSelectorTests extends ESTestCase { public void testIndexComponentSelectorFromId() { assertThat(IndexComponentSelector.getById((byte) 0), equalTo(IndexComponentSelector.DATA)); assertThat(IndexComponentSelector.getById((byte) 1), equalTo(IndexComponentSelector.FAILURES)); - assertThat(IndexComponentSelector.getById((byte) 2), equalTo(IndexComponentSelector.ALL_APPLICABLE)); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> IndexComponentSelector.getById((byte) 3)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> IndexComponentSelector.getById((byte) 2)); assertThat( exception.getMessage(), - containsString("Unknown id of index component selector [3], available options are: {0=DATA, 1=FAILURES, 2=ALL_APPLICABLE}") + containsString("Unknown id of index component selector [2], available options are: {0=DATA, 1=FAILURES}") ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java index 286e1d3afaee..a3ac361f5b05 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java @@ -81,11 +81,8 @@ public class IndexAbstractionResolverTests extends ESTestCase { expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("index1::data"))); // Selectors allowed, valid selector given, data selector stripped off in result since it is the default assertThat(resolveAbstractionsSelectorAllowed(List.of("index1::data")), contains("index1")); - // Selectors allowed, wildcard selector provided, data selector stripped off in result since it is the default - // ** only returns ::data since expression is an index - assertThat(resolveAbstractionsSelectorAllowed(List.of("index1::*")), contains("index1")); // Selectors allowed, invalid selector given - expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("index1::custom"))); + expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("index1::*"))); // == Single Date Math Expressions == @@ -125,7 +122,7 @@ public class IndexAbstractionResolverTests extends ESTestCase { assertThat(resolveAbstractionsSelectorAllowed(List.of("index*::data")), containsInAnyOrder("index1", "index2")); // Selectors allowed, wildcard selector provided, data selector stripped off in result since it is the default // ** only returns ::data since expression is an index - assertThat(resolveAbstractionsSelectorAllowed(List.of("index*::*")), containsInAnyOrder("index1", "index2")); + assertThat(resolveAbstractionsSelectorAllowed(List.of("index*")), containsInAnyOrder("index1", "index2")); // Selectors allowed, invalid selector given expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("index*::custom"))); @@ -137,11 +134,9 @@ public class IndexAbstractionResolverTests extends ESTestCase { expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("data-stream1::data"))); // Selectors allowed, valid selector given assertThat(resolveAbstractionsSelectorAllowed(List.of("data-stream1::failures")), contains("data-stream1::failures")); - // Selectors allowed, wildcard selector provided - // ** returns both ::data and ::failures since expression is a data stream - // ** data selector stripped off in result since it is the default + // Selectors allowed, data selector is not added in result since it is the default assertThat( - resolveAbstractionsSelectorAllowed(List.of("data-stream1::*")), + resolveAbstractionsSelectorAllowed(List.of("data-stream1", "data-stream1::failures")), containsInAnyOrder("data-stream1", "data-stream1::failures") ); // Selectors allowed, invalid selector given @@ -155,10 +150,9 @@ public class IndexAbstractionResolverTests extends ESTestCase { expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("data-stream*::data"))); // Selectors allowed, valid selector given assertThat(resolveAbstractionsSelectorAllowed(List.of("data-stream*::failures")), contains("data-stream1::failures")); - // Selectors allowed, wildcard selector provided - // ** returns both ::data and ::failures since expression is a data stream + // Selectors allowed, both ::data and ::failures are returned assertThat( - resolveAbstractionsSelectorAllowed(List.of("data-stream*::*")), + resolveAbstractionsSelectorAllowed(List.of("data-stream*", "data-stream*::failures")), containsInAnyOrder("data-stream1", "data-stream1::failures") ); // Selectors allowed, invalid selector given @@ -179,7 +173,7 @@ public class IndexAbstractionResolverTests extends ESTestCase { // Selectors allowed, wildcard selector provided // ** returns both ::data and ::failures for applicable abstractions assertThat( - resolveAbstractionsSelectorAllowed(List.of("*::*")), + resolveAbstractionsSelectorAllowed(List.of("*", "*::failures")), containsInAnyOrder("index1", "index2", "data-stream1", "data-stream1::failures") ); // Selectors allowed, invalid selector given @@ -194,11 +188,11 @@ public class IndexAbstractionResolverTests extends ESTestCase { // Selectors allowed, wildcard selector provided // ** returns both ::data and ::failures for applicable abstractions // ** limits the returned values based on selectors - assertThat(resolveAbstractionsSelectorAllowed(List.of("*::*", "-*::data")), contains("data-stream1::failures")); + assertThat(resolveAbstractionsSelectorAllowed(List.of("*", "*::failures", "-*::data")), contains("data-stream1::failures")); // Selectors allowed, wildcard selector provided // ** limits the returned values based on selectors assertThat( - resolveAbstractionsSelectorAllowed(List.of("*::*", "-*::failures")), + resolveAbstractionsSelectorAllowed(List.of("*", "*::failures", "-*::failures")), containsInAnyOrder("index1", "index2", "data-stream1") ); // Selectors allowed, none given, default to both selectors diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index f5e4c3d8f2d0..293bdb2c5389 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -2765,10 +2765,27 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis))); } + // Test default with an exact data stream name and include failures true + { + IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_NO_SELECTORS; + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream"); + assertThat(result.length, equalTo(4)); + assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis))); + assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis))); + assertThat(result[2].getName(), equalTo(DataStream.getDefaultFailureStoreName(dataStreamName, 1, epochMillis))); + assertThat(result[3].getName(), equalTo(DataStream.getDefaultFailureStoreName(dataStreamName, 2, epochMillis))); + } + // Test explicit include failure store with an exact data stream name { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; - Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::*"); + Index[] result = indexNameExpressionResolver.concreteIndices( + state, + indicesOptions, + true, + "my-data-stream::data", + "my-data-stream::failures" + ); assertThat(result.length, equalTo(4)); assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis))); assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis))); @@ -2784,7 +2801,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { .build(); expectThrows( IllegalArgumentException.class, - () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::*") + () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::failures") ); } @@ -2813,6 +2830,26 @@ public class IndexNameExpressionResolverTests extends ESTestCase { ); } + // Test default without any expressions and include failures + { + IndicesOptions indicesOptions = IndicesOptions.builder() + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowSelectors(false).includeFailureIndices(true).build()) + .build(); + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true); + assertThat(result.length, equalTo(5)); + List indexNames = Arrays.stream(result).map(Index::getName).toList(); + assertThat( + indexNames, + containsInAnyOrder( + DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis), + DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis), + DataStream.getDefaultFailureStoreName(dataStreamName, 1, epochMillis), + DataStream.getDefaultFailureStoreName(dataStreamName, 2, epochMillis), + otherIndex.getIndex().getName() + ) + ); + } + // Test default with wildcard expression { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; @@ -2832,7 +2869,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { // Test explicit include failure store with wildcard expression { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; - Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*::*"); + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*::data", "my-*::failures"); assertThat(result.length, equalTo(5)); List indexNames = Arrays.stream(result).map(Index::getName).toList(); assertThat( @@ -3225,8 +3262,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(streams, containsInAnyOrder(new ResolvedExpression(dataStream1, DATA), new ResolvedExpression(dataStream2, DATA))); assertThat(names, containsInAnyOrder(dataStream1, dataStream2)); - streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "*foobar::*"); - names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*foobar::*"); + streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "*foobar::data", "*foobar::failures"); + names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*foobar::data", "*foobar::failures"); assertThat( streams, containsInAnyOrder( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java index 2bf34dcfd2a3..dd3876afd3c7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.action.support.IndexComponentSelector.ALL_APPLICABLE; import static org.elasticsearch.action.support.IndexComponentSelector.DATA; import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES; import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context; @@ -38,7 +37,6 @@ public class SelectorResolverTests extends ESTestCase { assertThat(resolve(selectorsAllowed, "testXXX"), equalTo(new ResolvedExpression("testXXX", DATA))); assertThat(resolve(selectorsAllowed, "testXXX::data"), equalTo(new ResolvedExpression("testXXX", DATA))); assertThat(resolve(selectorsAllowed, "testXXX::failures"), equalTo(new ResolvedExpression("testXXX", FAILURES))); - assertThat(resolve(selectorsAllowed, "testXXX::*"), equalTo(new ResolvedExpression("testXXX", ALL_APPLICABLE))); // Disallow selectors (example: creating, modifying, or deleting indices/data streams/aliases). // Accepts standard expressions but throws when selectors are specified. @@ -47,7 +45,6 @@ public class SelectorResolverTests extends ESTestCase { assertThat(resolve(noSelectors, "testXXX"), equalTo(new ResolvedExpression("testXXX"))); expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::data")); expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::failures")); - expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::*")); // === Errors // Only recognized components can be selected @@ -116,9 +113,7 @@ public class SelectorResolverTests extends ESTestCase { assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", null), is(equalTo("a"))); assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", ""), is(equalTo("a::"))); assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", "b"), is(equalTo("a::b"))); - assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", "*"), is(equalTo("a::*"))); assertThat(IndexNameExpressionResolver.combineSelectorExpression("*", "b"), is(equalTo("*::b"))); - assertThat(IndexNameExpressionResolver.combineSelectorExpression("*", "*"), is(equalTo("*::*"))); } public void testHasSelectorSuffix() { @@ -151,14 +146,14 @@ public class SelectorResolverTests extends ESTestCase { assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::data"), is(equalTo(new Tuple<>("a", "data")))); assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::failures"), is(equalTo(new Tuple<>("a", "failures")))); - assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::*"), is(equalTo(new Tuple<>("a", "*")))); + expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::*")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::random")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::d*ta")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::*ailures")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::**")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("index::data::*")); - assertThat(IndexNameExpressionResolver.splitSelectorExpression("::*"), is(equalTo(new Tuple<>("", "*")))); + expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("::*")); } private static IndicesOptions getOptionsForSelectors() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java index 9d9a5ebd3721..1eeaef473521 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -25,7 +25,6 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.elasticsearch.action.support.IndexComponentSelector.ALL_APPLICABLE; import static org.elasticsearch.action.support.IndexComponentSelector.DATA; import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; @@ -54,19 +53,19 @@ public class WildcardExpressionResolverTests extends ESTestCase { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "ku*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "ku*", DATA)), equalTo(resolvedExpressionsSet("kuku")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY", "kuku")) ); } @@ -87,7 +86,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY")) ); context = new IndexNameExpressionResolver.Context( @@ -96,7 +95,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", DATA)), equalTo(resolvedExpressionsSet("testXYY")) ); context = new IndexNameExpressionResolver.Context( @@ -105,7 +104,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXXY")) ); } @@ -128,31 +127,27 @@ public class WildcardExpressionResolverTests extends ESTestCase { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY")) ); assertThat( - newHashSet( - IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*Y", ALL_APPLICABLE) - ), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*Y", DATA)), equalTo(resolvedExpressionsSet("testXXY", "testXYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "kuku*Y*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "kuku*Y*", DATA)), equalTo(resolvedExpressionsSet("kukuYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*", DATA)), equalTo(resolvedExpressionsSet("testXXY", "testXYY", "testYYY", "kukuYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*Y*X", ALL_APPLICABLE)) - .size(), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*Y*X", DATA)).size(), equalTo(0) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*X", ALL_APPLICABLE)) - .size(), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*X", DATA)).size(), equalTo(0) ); } @@ -171,7 +166,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); } @@ -189,7 +184,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, null)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(resolvedExpressionsNoSelectorSet("testXXX", "testXYY", "testYYY")) ); } @@ -212,10 +207,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { IndicesOptions.lenientExpandOpen(), // don't include hidden SystemIndexAccessLevel.NONE ); - assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), - equalTo(newHashSet()) - ); + assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(newHashSet())); } { @@ -235,7 +227,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(resolvedExpressionsSet("index-visible-alias")) ); } @@ -290,13 +282,8 @@ public class WildcardExpressionResolverTests extends ESTestCase { equalTo(resolvedExpressionsSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis))) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), - equalTo( - resolvedExpressionsSet( - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis) - ) - ) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, FAILURES)), + equalTo(resolvedExpressionsSet(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis))) ); } @@ -328,10 +315,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { ); assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(Set.of())); - assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), - equalTo(Set.of()) - ); + assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, FAILURES)), equalTo(Set.of())); } } @@ -455,7 +439,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo_a*", - ALL_APPLICABLE + DATA ); assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_index", DATA), new ResolvedExpression("bar_index", DATA))); } @@ -463,7 +447,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesLenientContext, "foo_a*", - ALL_APPLICABLE + DATA ); assertEquals(0, indices.size()); } @@ -471,7 +455,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Set indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesStrictContext, "foo_a*", - ALL_APPLICABLE + DATA ); assertThat(indices, empty()); } @@ -479,7 +463,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo*", - ALL_APPLICABLE + DATA ); assertThat( indices, @@ -494,7 +478,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesLenientContext, "foo*", - ALL_APPLICABLE + DATA ); assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA))); } @@ -502,7 +486,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesStrictContext, "foo*", - ALL_APPLICABLE + DATA ); assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA))); } @@ -556,7 +540,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo_*", - ALL_APPLICABLE + DATA ); assertThat( indices, @@ -571,7 +555,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "bar_*", - ALL_APPLICABLE + DATA ); assertThat(indices, containsInAnyOrder(new ResolvedExpression("bar_bar", DATA), new ResolvedExpression("bar_index", DATA))); } @@ -602,7 +586,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAliasesAndDataStreamsContext, "foo_*", - ALL_APPLICABLE + DATA ); assertThat( indices, @@ -611,9 +595,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { new ResolvedExpression("bar_index", DATA), new ResolvedExpression("foo_foo", DATA), new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA) + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA) ) ); @@ -632,26 +614,6 @@ public class WildcardExpressionResolverTests extends ESTestCase { ) ) ); - - // include all wildcard adds the data stream's backing indices - indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - indicesAliasesAndDataStreamsContext, - "*", - ALL_APPLICABLE - ); - assertThat( - indices, - containsInAnyOrder( - new ResolvedExpression("foo_index", DATA), - new ResolvedExpression("bar_index", DATA), - new ResolvedExpression("foo_foo", DATA), - new ResolvedExpression("bar_bar", DATA), - new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA) - ) - ); } { @@ -681,7 +643,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAliasesDataStreamsAndHiddenIndices, "foo_*", - ALL_APPLICABLE + DATA ); assertThat( indices, @@ -690,9 +652,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { new ResolvedExpression("bar_index", DATA), new ResolvedExpression("foo_foo", DATA), new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA) + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA) ) ); @@ -712,32 +672,11 @@ public class WildcardExpressionResolverTests extends ESTestCase { ) ); - // Resolve both backing and failure indices - indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - indicesAliasesDataStreamsAndHiddenIndices, - "foo_*", - ALL_APPLICABLE - ); - assertThat( - newHashSet(indices), - equalTo( - resolvedExpressionsSet( - "foo_index", - "bar_index", - "foo_foo", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis) - ) - ) - ); - // include all wildcard adds the data stream's backing indices indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAliasesDataStreamsAndHiddenIndices, "*", - ALL_APPLICABLE + DATA ); assertThat( newHashSet(indices), @@ -770,28 +709,6 @@ public class WildcardExpressionResolverTests extends ESTestCase { ) ) ); - - // include all wildcard adds the data stream's backing and failure indices - indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - indicesAliasesDataStreamsAndHiddenIndices, - "*", - ALL_APPLICABLE - ); - assertThat( - newHashSet(indices), - equalTo( - resolvedExpressionsSet( - "foo_index", - "bar_index", - "foo_foo", - "bar_bar", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis) - ) - ) - ); } } @@ -824,7 +741,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { Collection matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "*", - ALL_APPLICABLE + DATA ); assertThat( matches, @@ -835,7 +752,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { new ResolvedExpression("bar_index", DATA) ) ); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "*", ALL_APPLICABLE); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "*", DATA); assertThat( matches, containsInAnyOrder( @@ -845,11 +762,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { new ResolvedExpression("bar_index", DATA) ) ); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - indicesAndAliasesContext, - "foo*", - ALL_APPLICABLE - ); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(indicesAndAliasesContext, "foo*", DATA); assertThat( matches, containsInAnyOrder( @@ -858,11 +771,7 @@ public class WildcardExpressionResolverTests extends ESTestCase { new ResolvedExpression("bar_index", DATA) ) ); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - onlyIndicesContext, - "foo*", - ALL_APPLICABLE - ); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "foo*", DATA); assertThat(matches, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA))); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index f7dc725c3f07..6099b7351cf7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -465,22 +465,6 @@ public class IndicesAndAliasesResolverTests extends ESTestCase { ); } - public void testWildcardSelectorsAreNotAllowedInShardLevelRequests() { - ShardSearchRequest request = mock(ShardSearchRequest.class); - when(request.indices()).thenReturn(new String[] { "index10::*" }); - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> defaultIndicesResolver.resolveIndicesAndAliasesWithoutWildcards(TransportSearchAction.TYPE.name() + "[s]", request) - ); - assertThat( - exception, - throwableWithMessage( - "the action indices:data/read/search[s] does not support wildcard selectors;" - + " the provided index expression(s) [index10::*] are not allowed" - ) - ); - } - public void testAllIsNotAllowedInShardLevelRequests() { ShardSearchRequest request = mock(ShardSearchRequest.class); final boolean literalAll = randomBoolean(); From f5c901e68c8203e732334075a6c98ff7038ac4e1 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 12 Feb 2025 16:20:13 -0500 Subject: [PATCH 34/93] Fix synthetic source bug that would mishandle nested dense_vector fields (#122425) When utilizing synthetic source with nested fields, we attempt to rebuild the child values in addition to all the parent values. While this generally works well, its potential that certain values might be missing from various child docs. Consequently, we will attempt to iterate the vector values strangely, resulting in seemingly missing values or potentially exceptions indicating EOFs. closes: #122383 --- docs/changelog/122425.yaml | 5 + .../indices.create/20_synthetic_source.yml | 140 ++++++++++++++++++ .../vectors/DenseVectorFieldMapper.java | 18 +++ .../indices/CreateIndexCapabilities.java | 8 +- 4 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/122425.yaml diff --git a/docs/changelog/122425.yaml b/docs/changelog/122425.yaml new file mode 100644 index 000000000000..a0e590dcdc36 --- /dev/null +++ b/docs/changelog/122425.yaml @@ -0,0 +1,5 @@ +pr: 122425 +summary: Fix synthetic source bug that would mishandle nested `dense_vector` fields +area: Mapping +type: bug +issues: [] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index d1c492caf9b4..dc476147c960 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -2008,3 +2008,143 @@ create index with use_synthetic_source: flush: false - gt: { test.store_size_in_bytes: 0 } - is_false: test.fields._recovery_source +--- +"Nested synthetic source with indexed dense vectors": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ synthetic_nested_dense_vector_bug_fix ] + reason: "Requires synthetic source bugfix for dense vectors in nested objects" + - do: + indices.create: + index: nested_dense_vector_synthetic_test + body: + mappings: + properties: + parent: + type: nested + properties: + vector: + type: dense_vector + index: true + similarity: l2_norm + text: + type: text + settings: + index: + mapping: + source: + mode: synthetic + - do: + index: + index: nested_dense_vector_synthetic_test + id: 0 + refresh: true + body: { "parent": [ { "vector": [ 1, 2 ],"text": "foo" }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + - do: + index: + index: nested_dense_vector_synthetic_test + id: 1 + refresh: true + body: { "parent": [ { "text": "foo" }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + - do: + index: + index: nested_dense_vector_synthetic_test + id: 2 + refresh: true + body: { "parent": [ { "vector": [ 1, 2 ] }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + + - do: + search: + index: nested_dense_vector_synthetic_test + body: + query: + match_all: {} + + - match: { hits.hits.0._source.parent.0.vector: [ 1.0, 2.0 ] } + - match: { hits.hits.0._source.parent.0.text: "foo" } + - match: { hits.hits.0._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.0._source.parent.1.text: "bar" } + - is_false: hits.hits.1._source.parent.0.vector + - match: { hits.hits.1._source.parent.0.text: "foo" } + - match: { hits.hits.1._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.1._source.parent.1.text: "bar" } + - match: {hits.hits.2._source.parent.0.vector: [ 1.0, 2.0 ] } + - is_false: hits.hits.2._source.parent.0.text + - match: { hits.hits.2._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.2._source.parent.1.text: "bar" } +--- +"Nested synthetic source with un-indexed dense vectors": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ synthetic_nested_dense_vector_bug_fix ] + reason: "Requires synthetic source bugfix for dense vectors in nested objects" + - do: + indices.create: + index: nested_dense_vector_synthetic_test + body: + mappings: + properties: + parent: + type: nested + properties: + vector: + type: dense_vector + index: false + text: + type: text + settings: + index: + mapping: + source: + mode: synthetic + - do: + index: + index: nested_dense_vector_synthetic_test + id: 0 + refresh: true + body: { "parent": [ { "vector": [ 1, 2 ],"text": "foo" }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + - do: + index: + index: nested_dense_vector_synthetic_test + id: 1 + refresh: true + body: { "parent": [ { "text": "foo" }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + - do: + index: + index: nested_dense_vector_synthetic_test + id: 2 + refresh: true + body: { "parent": [ { "vector": [ 1, 2 ] }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + + - do: + search: + index: nested_dense_vector_synthetic_test + body: + query: + match_all: {} + + - match: { hits.hits.0._source.parent.0.vector: [ 1.0, 2.0 ] } + - match: { hits.hits.0._source.parent.0.text: "foo" } + - match: { hits.hits.0._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.0._source.parent.1.text: "bar" } + - is_false: hits.hits.1._source.parent.0.vector + - match: { hits.hits.1._source.parent.0.text: "foo" } + - match: { hits.hits.1._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.1._source.parent.1.text: "bar" } + - match: {hits.hits.2._source.parent.0.vector: [ 1.0, 2.0 ] } + - is_false: hits.hits.2._source.parent.0.text + - match: { hits.hits.2._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.2._source.parent.1.text: "bar" } + diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 0d514408c912..ce41c2164e20 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2404,6 +2404,12 @@ public class DenseVectorFieldMapper extends FieldMapper { } KnnVectorValues.DocIndexIterator iterator = values.iterator(); return docId -> { + if (iterator.docID() > docId) { + return hasValue = false; + } + if (iterator.docID() == docId) { + return hasValue = true; + } hasValue = docId == iterator.advance(docId); hasMagnitude = hasValue && magnitudeReader != null && magnitudeReader.advanceExact(docId); ord = iterator.index(); @@ -2414,6 +2420,12 @@ public class DenseVectorFieldMapper extends FieldMapper { if (byteVectorValues != null) { KnnVectorValues.DocIndexIterator iterator = byteVectorValues.iterator(); return docId -> { + if (iterator.docID() > docId) { + return hasValue = false; + } + if (iterator.docID() == docId) { + return hasValue = true; + } hasValue = docId == iterator.advance(docId); ord = iterator.index(); return hasValue; @@ -2476,6 +2488,12 @@ public class DenseVectorFieldMapper extends FieldMapper { return null; } return docId -> { + if (values.docID() > docId) { + return hasValue = false; + } + if (values.docID() == docId) { + return hasValue = true; + } hasValue = docId == values.advance(docId); return hasValue; }; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java index 9083c781ae16..334e68648d85 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java @@ -26,5 +26,11 @@ public class CreateIndexCapabilities { */ private static final String LOOKUP_INDEX_MODE_CAPABILITY = "lookup_index_mode"; - public static final Set CAPABILITIES = Set.of(LOGSDB_INDEX_MODE_CAPABILITY, LOOKUP_INDEX_MODE_CAPABILITY); + private static final String NESTED_DENSE_VECTOR_SYNTHETIC_TEST = "nested_dense_vector_synthetic_test"; + + public static final Set CAPABILITIES = Set.of( + LOGSDB_INDEX_MODE_CAPABILITY, + LOOKUP_INDEX_MODE_CAPABILITY, + NESTED_DENSE_VECTOR_SYNTHETIC_TEST + ); } From f8fb3c3d847c93ee5e8a95799e808735cb280680 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 13 Feb 2025 00:20:16 +0200 Subject: [PATCH 35/93] Don't run the csv tests for inlinestats in non-snapshot tests (#122407) --- .../org/elasticsearch/xpack/esql/action/EsqlCapabilities.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index d88b68ece210..9d550ad32804 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -809,7 +809,7 @@ public class EsqlCapabilities { * Fixes a series of issues with inlinestats which had an incomplete implementation after lookup and inlinestats * were refactored. */ - INLINESTATS_V3; + INLINESTATS_V3(EsqlPlugin.INLINESTATS_FEATURE_FLAG); private final boolean enabled; From ecde1cad9d2f2083630b7269480f0cbbff7e5617 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 09:27:41 +1100 Subject: [PATCH 36/93] Mute org.elasticsearch.xpack.esql.CsvTests org.elasticsearch.xpack.esql.CsvTests #122440 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 043e261d9da5..17c307d1bea5 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -404,6 +404,8 @@ tests: - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT method: testStatsMissingFieldWithStats issue: https://github.com/elastic/elasticsearch/issues/122327 +- class: org.elasticsearch.xpack.esql.CsvTests + issue: https://github.com/elastic/elasticsearch/issues/122440 # Examples: # From 88550f61265c81db6674b18ba0280e58c31315b3 Mon Sep 17 00:00:00 2001 From: Stanislav Malyshev Date: Wed, 12 Feb 2025 15:37:44 -0700 Subject: [PATCH 37/93] Improve CrossClusterAsyncEnrichStopIT test (#122432) --- .../action/AbstractCrossClusterTestCase.java | 6 +++++ .../action/CrossClusterAsyncEnrichStopIT.java | 22 +++++++++++++++++++ .../action/CrossClusterAsyncQueryStopIT.java | 6 ----- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClusterTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClusterTestCase.java index 571e89c3fa50..510f5945f745 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClusterTestCase.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClusterTestCase.java @@ -14,9 +14,11 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.compute.operator.DriverTaskRunner; import org.elasticsearch.compute.operator.exchange.ExchangeService; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.test.FailingFieldPlugin; import org.elasticsearch.test.XContentTestUtils; @@ -273,4 +275,8 @@ public abstract class AbstractCrossClusterTestCase extends AbstractMultiClusters } return runQuery(request); } + + static List getDriverTasks(Client client) { + return client.admin().cluster().prepareListTasks().setActions(DriverTaskRunner.ACTION_NAME).setDetailed(true).get().getTasks(); + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncEnrichStopIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncEnrichStopIT.java index 99a81c60a9ad..1d6acf51db03 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncEnrichStopIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncEnrichStopIT.java @@ -10,7 +10,9 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.compute.operator.DriverStatus; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.async.AsyncStopRequest; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -28,10 +30,13 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.elasticsearch.xpack.esql.action.AbstractCrossClusterTestCase.getDriverTasks; import static org.elasticsearch.xpack.esql.action.EsqlAsyncTestUtils.deleteAsyncId; import static org.elasticsearch.xpack.esql.action.EsqlAsyncTestUtils.startAsyncQuery; import static org.elasticsearch.xpack.esql.action.EsqlAsyncTestUtils.waitForCluster; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; // This tests if enrich after stop works correctly public class CrossClusterAsyncEnrichStopIT extends AbstractEnrichBasedCrossClusterTestCase { @@ -87,10 +92,27 @@ public class CrossClusterAsyncEnrichStopIT extends AbstractEnrichBasedCrossClust // wait until c1 is done waitForCluster(client(), "c1", asyncExecutionId); waitForCluster(client(), LOCAL_CLUSTER, asyncExecutionId); + // wait until remote reduce task starts on c2 + assertBusy(() -> { + List tasks = getDriverTasks(client(REMOTE_CLUSTER_2)); + List reduceTasks = tasks.stream() + .filter(t -> t.status() instanceof DriverStatus ds && ds.taskDescription().equals("remote_reduce")) + .toList(); + assertThat(reduceTasks, not(empty())); + }); // Run the stop request var stopRequest = new AsyncStopRequest(asyncExecutionId); var stopAction = client().execute(EsqlAsyncStopAction.INSTANCE, stopRequest); + // wait until remote reduce tasks are gone + assertBusy(() -> { + List tasks = getDriverTasks(client(REMOTE_CLUSTER_2)); + List reduceTasks = tasks.stream() + .filter(t -> t.status() instanceof DriverStatus ds && ds.taskDescription().equals("remote_reduce")) + .toList(); + assertThat(reduceTasks, empty()); + }); + // Allow the processing to proceed SimplePauseFieldPlugin.allowEmitting.countDown(); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryStopIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryStopIT.java index 8a09c4af8ca5..7401a2838ae8 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryStopIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryStopIT.java @@ -9,8 +9,6 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.Build; import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.compute.operator.DriverTaskRunner; import org.elasticsearch.core.Tuple; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.xpack.core.async.AsyncStopRequest; @@ -244,8 +242,4 @@ public class CrossClusterAsyncQueryStopIT extends AbstractCrossClusterTestCase { assertAcked(deleteAsyncId(client(), asyncExecutionId)); } } - - private static List getDriverTasks(Client client) { - return client.admin().cluster().prepareListTasks().setActions(DriverTaskRunner.ACTION_NAME).setDetailed(true).get().getTasks(); - } } From 8c0ab4eab391c7e121ac92ef2b6267bf0cd9929c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 12 Feb 2025 17:23:23 -0600 Subject: [PATCH 38/93] Making reindex data streams actions cancellable (#122438) --- .../action/CreateIndexFromSourceAction.java | 13 +++++++++++++ .../migrate/action/ReindexDataStreamAction.java | 14 ++++++++++++++ .../action/ReindexDataStreamIndexAction.java | 14 ++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java index 14e5e8cccd91..5ab009decd38 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java @@ -15,6 +15,9 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -191,5 +194,15 @@ public class CreateIndexFromSourceAction extends ActionType headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public String getDescription() { + return "creating index " + destIndex + " from " + sourceIndex; + } } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java index faf8982b79bf..5ebd2040fbcb 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java @@ -16,6 +16,9 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -24,6 +27,7 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.function.Predicate; @@ -144,5 +148,15 @@ public class ReindexDataStreamAction extends ActionType { builder.endObject(); return builder; } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public String getDescription() { + return "reindexing data stream " + sourceDataStream; + } } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java index 2e3fd1b76ed3..dec3cf2901fc 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java @@ -14,8 +14,12 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; public class ReindexDataStreamIndexAction extends ActionType { @@ -78,6 +82,16 @@ public class ReindexDataStreamIndexAction extends ActionType headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public String getDescription() { + return "reindexing data stream index " + sourceIndex; + } } public static class Response extends ActionResponse { From 6c6e8d8768b792c9a7cbb42a7de6d1426bf510ad Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 12 Feb 2025 15:59:51 -0800 Subject: [PATCH 39/93] Add release tooling for adding new transport versions (#122426) --- .../internal/release/UpdateVersionsTask.java | 96 ++++++++++++++++++- .../release/UpdateVersionsTaskTests.java | 90 +++++++++++++++++ 2 files changed, 184 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java index a6ead34b1107..ebd316d7f042 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java @@ -15,6 +15,7 @@ import com.github.javaparser.ast.NodeList; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; import com.github.javaparser.ast.body.FieldDeclaration; import com.github.javaparser.ast.body.VariableDeclarator; +import com.github.javaparser.ast.expr.Expression; import com.github.javaparser.ast.expr.NameExpr; import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter; import com.google.common.annotations.VisibleForTesting; @@ -33,6 +34,7 @@ import java.util.NavigableMap; import java.util.Objects; import java.util.Optional; import java.util.TreeMap; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -51,6 +53,8 @@ public class UpdateVersionsTask extends AbstractVersionsTask { private boolean setCurrent; @Nullable private Version removeVersion; + @Nullable + private String addTransportVersion; @Inject public UpdateVersionsTask(BuildLayout layout) { @@ -62,6 +66,11 @@ public class UpdateVersionsTask extends AbstractVersionsTask { this.addVersion = Version.fromString(version); } + @Option(option = "add-transport-version", description = "Specifies transport version to add") + public void addTransportVersion(String transportVersion) { + this.addTransportVersion = transportVersion; + } + @Option(option = "set-current", description = "Set the 'current' constant to the new version") public void setCurrent(boolean setCurrent) { this.setCurrent = setCurrent; @@ -87,15 +96,18 @@ public class UpdateVersionsTask extends AbstractVersionsTask { @TaskAction public void executeTask() throws IOException { - if (addVersion == null && removeVersion == null) { + if (addVersion == null && removeVersion == null && addTransportVersion == null) { throw new IllegalArgumentException("No versions to add or remove specified"); } if (setCurrent && addVersion == null) { throw new IllegalArgumentException("No new version added to set as the current version"); } - if (Objects.equals(addVersion, removeVersion)) { + if (addVersion != null && removeVersion != null && Objects.equals(addVersion, removeVersion)) { throw new IllegalArgumentException("Same version specified to add and remove"); } + if (addTransportVersion != null && addTransportVersion.split(":").length != 2) { + throw new IllegalArgumentException("Transport version specified must be in the format ':'"); + } Path versionJava = rootDir.resolve(VERSION_FILE_PATH); CompilationUnit file = LexicalPreservingPrinter.setup(StaticJavaParser.parse(versionJava)); @@ -115,6 +127,18 @@ public class UpdateVersionsTask extends AbstractVersionsTask { modifiedFile = removed; } } + if (addTransportVersion != null) { + var constant = addTransportVersion.split(":")[0]; + var versionId = Integer.parseInt(addTransportVersion.split(":")[1]); + LOGGER.lifecycle("Adding transport version constant [{}] with id [{}]", constant, versionId); + + var transportVersionsFile = rootDir.resolve(TRANSPORT_VERSIONS_FILE_PATH); + var transportVersions = LexicalPreservingPrinter.setup(StaticJavaParser.parse(transportVersionsFile)); + var modified = addTransportVersionConstant(transportVersions, constant, versionId); + if (modified.isPresent()) { + writeOutNewContents(transportVersionsFile, modified.get()); + } + } if (modifiedFile.isPresent()) { writeOutNewContents(versionJava, modifiedFile.get()); @@ -161,6 +185,51 @@ public class UpdateVersionsTask extends AbstractVersionsTask { return Optional.of(versionJava); } + @VisibleForTesting + static Optional addTransportVersionConstant(CompilationUnit transportVersions, String constant, int versionId) { + ClassOrInterfaceDeclaration transportVersionsClass = transportVersions.getClassByName("TransportVersions").get(); + if (transportVersionsClass.getFieldByName(constant).isPresent()) { + LOGGER.lifecycle("New transport version constant [{}] already present, skipping", constant); + return Optional.empty(); + } + + TreeMap versions = transportVersionsClass.getFields() + .stream() + .filter(f -> f.getElementType().asString().equals("TransportVersion")) + .filter( + f -> f.getVariables().stream().limit(1).allMatch(v -> v.getInitializer().filter(Expression::isMethodCallExpr).isPresent()) + ) + .filter(f -> f.getVariable(0).getInitializer().get().asMethodCallExpr().getNameAsString().endsWith("def")) + .collect( + Collectors.toMap( + f -> f.getVariable(0) + .getInitializer() + .get() + .asMethodCallExpr() + .getArgument(0) + .asIntegerLiteralExpr() + .asNumber() + .intValue(), + Function.identity(), + (f1, f2) -> { + throw new IllegalStateException("Duplicate version constant " + f1); + }, + TreeMap::new + ) + ); + + // find the version this should be inserted after + Map.Entry previousVersion = versions.lowerEntry(versionId); + if (previousVersion == null) { + throw new IllegalStateException(String.format("Could not find previous version to [%s]", versionId)); + } + + FieldDeclaration newTransportVersion = createNewTransportVersionConstant(previousVersion.getValue(), constant, versionId); + transportVersionsClass.getMembers().addAfter(newTransportVersion, previousVersion.getValue()); + + return Optional.of(transportVersions); + } + private static FieldDeclaration createNewVersionConstant(FieldDeclaration lastVersion, String newName, String newExpr) { return new FieldDeclaration( new NodeList<>(lastVersion.getModifiers()), @@ -172,6 +241,29 @@ public class UpdateVersionsTask extends AbstractVersionsTask { ); } + private static FieldDeclaration createNewTransportVersionConstant(FieldDeclaration lastVersion, String newName, int newId) { + return new FieldDeclaration( + new NodeList<>(lastVersion.getModifiers()), + new VariableDeclarator( + lastVersion.getCommonType(), + newName, + StaticJavaParser.parseExpression(String.format("def(%s)", formatTransportVersionId(newId))) + ) + ); + } + + private static String formatTransportVersionId(int id) { + String idString = Integer.toString(id); + + return new StringBuilder(idString.substring(idString.length() - 2, idString.length())).insert(0, "_") + .insert(0, idString.substring(idString.length() - 3, idString.length() - 2)) + .insert(0, "_") + .insert(0, idString.substring(idString.length() - 6, idString.length() - 3)) + .insert(0, "_") + .insert(0, idString.substring(0, idString.length() - 6)) + .toString(); + } + @VisibleForTesting static Optional removeVersionConstant(CompilationUnit versionJava, Version version) { String removeFieldName = toVersionField(version); diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java index 9e4f1cd3a913..d5060a2e6236 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java @@ -239,6 +239,96 @@ public class UpdateVersionsTaskTests { assertThat(field.isPresent(), is(false)); } + @Test + public void addTransportVersion() throws Exception { + var transportVersions = """ + public class TransportVersions { + public static final TransportVersion V_1_0_0 = def(1_000_0_00); + public static final TransportVersion V_1_1_0 = def(1_001_0_00); + public static final TransportVersion V_1_2_0 = def(1_002_0_00); + public static final TransportVersion V_1_2_1 = def(1_002_0_01); + public static final TransportVersion V_1_2_2 = def(1_002_0_02); + public static final TransportVersion SOME_OTHER_VERSION = def(1_003_0_00); + public static final TransportVersion YET_ANOTHER_VERSION = def(1_004_0_00); + public static final TransportVersion MINIMUM_COMPATIBLE = V_1_0_0; + } + """; + + var expectedTransportVersions = """ + public class TransportVersions { + + public static final TransportVersion V_1_0_0 = def(1_000_0_00); + + public static final TransportVersion V_1_1_0 = def(1_001_0_00); + + public static final TransportVersion V_1_2_0 = def(1_002_0_00); + + public static final TransportVersion V_1_2_1 = def(1_002_0_01); + + public static final TransportVersion V_1_2_2 = def(1_002_0_02); + + public static final TransportVersion SOME_OTHER_VERSION = def(1_003_0_00); + + public static final TransportVersion YET_ANOTHER_VERSION = def(1_004_0_00); + + public static final TransportVersion NEXT_TRANSPORT_VERSION = def(1_005_0_00); + + public static final TransportVersion MINIMUM_COMPATIBLE = V_1_0_0; + } + """; + + var unit = StaticJavaParser.parse(transportVersions); + var result = UpdateVersionsTask.addTransportVersionConstant(unit, "NEXT_TRANSPORT_VERSION", 1_005_0_00); + + assertThat(result.isPresent(), is(true)); + assertThat(result.get(), hasToString(expectedTransportVersions)); + } + + @Test + public void addTransportVersionPatch() throws Exception { + var transportVersions = """ + public class TransportVersions { + public static final TransportVersion V_1_0_0 = def(1_000_0_00); + public static final TransportVersion V_1_1_0 = def(1_001_0_00); + public static final TransportVersion V_1_2_0 = def(1_002_0_00); + public static final TransportVersion V_1_2_1 = def(1_002_0_01); + public static final TransportVersion V_1_2_2 = def(1_002_0_02); + public static final TransportVersion SOME_OTHER_VERSION = def(1_003_0_00); + public static final TransportVersion YET_ANOTHER_VERSION = def(1_004_0_00); + public static final TransportVersion MINIMUM_COMPATIBLE = V_1_0_0; + } + """; + + var expectedTransportVersions = """ + public class TransportVersions { + + public static final TransportVersion V_1_0_0 = def(1_000_0_00); + + public static final TransportVersion V_1_1_0 = def(1_001_0_00); + + public static final TransportVersion V_1_2_0 = def(1_002_0_00); + + public static final TransportVersion V_1_2_1 = def(1_002_0_01); + + public static final TransportVersion V_1_2_2 = def(1_002_0_02); + + public static final TransportVersion SOME_OTHER_VERSION = def(1_003_0_00); + + public static final TransportVersion PATCH_TRANSPORT_VERSION = def(1_003_0_01); + + public static final TransportVersion YET_ANOTHER_VERSION = def(1_004_0_00); + + public static final TransportVersion MINIMUM_COMPATIBLE = V_1_0_0; + } + """; + + var unit = StaticJavaParser.parse(transportVersions); + var result = UpdateVersionsTask.addTransportVersionConstant(unit, "PATCH_TRANSPORT_VERSION", 1_003_0_01); + + assertThat(result.isPresent(), is(true)); + assertThat(result.get(), hasToString(expectedTransportVersions)); + } + private static Optional findFirstField(Node node, String name) { return node.findFirst(FieldDeclaration.class, f -> f.getVariable(0).getName().getIdentifier().equals(name)); } From b8d7e99cb9eb6f3644a4269a01bb3ddc6651b073 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 12 Feb 2025 16:12:19 -0800 Subject: [PATCH 40/93] Use FallbackSyntheticSourceBlockLoader for number fields (#122280) --- .../operator/ValuesSourceReaderBenchmark.java | 3 +- .../script/ScriptScoreBenchmark.java | 2 +- docs/changelog/122280.yaml | 5 + .../mapper/extras/TokenCountFieldMapper.java | 9 +- .../index/mapper/size/SizeFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 177 +++++++++++++++++- .../fielddata/IndexFieldDataServiceTests.java | 6 +- .../index/mapper/NumberFieldTypeTests.java | 3 +- .../ByteFieldBlockLoaderTests.java | 24 +++ .../DoubleFieldBlockLoaderTests.java | 23 +++ .../FloatFieldBlockLoaderTests.java | 24 +++ .../HalfFloatFieldBlockLoaderTests.java | 25 +++ .../IntegerFieldBlockLoaderTests.java | 23 +++ .../KeywordFieldBlockLoaderTests.java | 12 -- .../LongFieldBlockLoaderTests.java | 23 +++ .../NumberFieldBlockLoaderTestCase.java | 62 ++++++ .../ShortFieldBlockLoaderTests.java | 24 +++ .../aggregations/AggregatorBaseTests.java | 3 +- .../bucket/filter/FiltersAggregatorTests.java | 6 +- .../bucket/range/RangeAggregatorTests.java | 9 +- .../search/collapse/CollapseBuilderTests.java | 6 +- .../index/mapper/BlockLoaderTestCase.java | 12 ++ .../rate/TimeSeriesRateAggregatorTests.java | 3 +- 23 files changed, 447 insertions(+), 39 deletions(-) create mode 100644 docs/changelog/122280.yaml create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java index 9ed5e1accef5..fba3c752bb23 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java @@ -261,7 +261,8 @@ public class ValuesSourceReaderBenchmark { null, false, null, - null + null, + false ).blockLoader(null); } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 4085e74d35db..e61171aeff02 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -83,7 +83,7 @@ public class ScriptScoreBenchmark { private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList()); private final Map fieldTypes = Map.ofEntries( - Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null)) + Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null, false)) ); private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None(); private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); diff --git a/docs/changelog/122280.yaml b/docs/changelog/122280.yaml new file mode 100644 index 000000000000..93a7e4e1aaf5 --- /dev/null +++ b/docs/changelog/122280.yaml @@ -0,0 +1,5 @@ +pr: 122280 +summary: Use `FallbackSyntheticSourceBlockLoader` for number fields +area: Mapping +type: enhancement +issues: [] diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index db2762a028e6..a1174c444831 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -86,7 +86,8 @@ public class TokenCountFieldMapper extends FieldMapper { store.getValue(), hasDocValues.getValue(), nullValue.getValue(), - meta.getValue() + meta.getValue(), + context.isSourceSynthetic() ); return new TokenCountFieldMapper(leafName(), ft, builderParams(this, context), this); } @@ -100,7 +101,8 @@ public class TokenCountFieldMapper extends FieldMapper { boolean isStored, boolean hasDocValues, Number nullValue, - Map meta + Map meta, + boolean isSyntheticSource ) { super( name, @@ -114,7 +116,8 @@ public class TokenCountFieldMapper extends FieldMapper { null, false, null, - null + null, + isSyntheticSource ); } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 68b93db39646..11dbf34f6c79 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -50,7 +50,7 @@ public class SizeFieldMapper extends MetadataFieldMapper { private static class SizeFieldType extends NumberFieldType { SizeFieldType() { - super(NAME, NumberType.INTEGER, true, true, true, false, null, Collections.emptyMap(), null, false, null, null); + super(NAME, NumberType.INTEGER, true, true, true, false, null, Collections.emptyMap(), null, false, null, null, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 76528ccf0667..078faa25938f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -269,7 +269,7 @@ public class NumberFieldMapper extends FieldMapper { dimension.setValue(true); } - MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this); + MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this, context.isSourceSynthetic()); hasScript = script.get() != null; onScriptError = onScriptErrorParam.getValue(); return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this); @@ -463,6 +463,11 @@ public class NumberFieldMapper extends FieldMapper { BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } + + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } }, FLOAT("float", NumericType.FLOAT) { @Override @@ -647,6 +652,11 @@ public class NumberFieldMapper extends FieldMapper { BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } + + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } }, DOUBLE("double", NumericType.DOUBLE) { @Override @@ -797,6 +807,11 @@ public class NumberFieldMapper extends FieldMapper { BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } + + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } }, BYTE("byte", NumericType.BYTE) { @Override @@ -911,6 +926,11 @@ public class NumberFieldMapper extends FieldMapper { return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } + private boolean isOutOfRange(Object value) { double doubleValue = objectToDouble(value); return doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE; @@ -1024,6 +1044,11 @@ public class NumberFieldMapper extends FieldMapper { return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } + private boolean isOutOfRange(Object value) { double doubleValue = objectToDouble(value); return doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE; @@ -1210,6 +1235,11 @@ public class NumberFieldMapper extends FieldMapper { BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } + + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } }, LONG("long", NumericType.LONG) { @Override @@ -1358,6 +1388,26 @@ public class NumberFieldMapper extends FieldMapper { return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup); } + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + var reader = new NumberFallbackSyntheticSourceReader(this, nullValue, coerce) { + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var builder = (BlockLoader.LongBuilder) blockBuilder; + for (var value : values) { + builder.appendLong(value.longValue()); + } + } + }; + + return new FallbackSyntheticSourceBlockLoader(reader, fieldName) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + }; + } + private boolean isOutOfRange(Object value) { if (value instanceof Long) { return false; @@ -1626,6 +1676,106 @@ public class NumberFieldMapper extends FieldMapper { abstract BlockLoader blockLoaderFromDocValues(String fieldName); abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup); + + abstract BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce); + + // All values that fit into integer are returned as integers + private static BlockLoader integerBlockLoaderFromFallbackSyntheticSource( + NumberType type, + String fieldName, + Number nullValue, + boolean coerce + ) { + var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) { + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var builder = (BlockLoader.IntBuilder) blockBuilder; + for (var value : values) { + builder.appendInt(value.intValue()); + } + } + }; + + return new FallbackSyntheticSourceBlockLoader(reader, fieldName) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.ints(expectedCount); + } + }; + } + + // All floating point values are returned as doubles + private static BlockLoader floatingPointBlockLoaderFromFallbackSyntheticSource( + NumberType type, + String fieldName, + Number nullValue, + boolean coerce + ) { + var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) { + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var builder = (BlockLoader.DoubleBuilder) blockBuilder; + for (var value : values) { + builder.appendDouble(value.doubleValue()); + } + } + }; + + return new FallbackSyntheticSourceBlockLoader(reader, fieldName) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.doubles(expectedCount); + } + }; + } + + abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport< + Number> { + private final NumberType type; + private final Number nullValue; + private final boolean coerce; + + NumberFallbackSyntheticSourceReader(NumberType type, Number nullValue, boolean coerce) { + super(nullValue); + this.type = type; + this.nullValue = nullValue; + this.coerce = coerce; + } + + @Override + public void convertValue(Object value, List accumulator) { + if (coerce && value.equals("")) { + if (nullValue != null) { + accumulator.add(nullValue); + } + } + + try { + var converted = type.parse(value, coerce); + accumulator.add(converted); + } catch (Exception e) { + // Malformed value, skip it + } + } + + @Override + public void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { + // Aligned with implementation of `value(XContentParser)` + if (coerce && parser.currentToken() == Token.VALUE_STRING && parser.textLength() == 0) { + if (nullValue != null) { + accumulator.add(nullValue); + } + } + + try { + Number rawValue = type.parse(parser, coerce); + // Transform number to correct type (e.g. reduce precision) + accumulator.add(type.parse(rawValue, coerce)); + } catch (Exception e) { + // Malformed value, skip it + } + } + }; } public static class NumberFieldType extends SimpleMappedFieldType { @@ -1637,6 +1787,7 @@ public class NumberFieldMapper extends FieldMapper { private final boolean isDimension; private final MetricType metricType; private final IndexMode indexMode; + private final boolean isSyntheticSource; public NumberFieldType( String name, @@ -1650,7 +1801,8 @@ public class NumberFieldMapper extends FieldMapper { FieldValues script, boolean isDimension, MetricType metricType, - IndexMode indexMode + IndexMode indexMode, + boolean isSyntheticSource ) { super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); this.type = Objects.requireNonNull(type); @@ -1660,9 +1812,10 @@ public class NumberFieldMapper extends FieldMapper { this.isDimension = isDimension; this.metricType = metricType; this.indexMode = indexMode; + this.isSyntheticSource = isSyntheticSource; } - NumberFieldType(String name, Builder builder) { + NumberFieldType(String name, Builder builder, boolean isSyntheticSource) { this( name, builder.type, @@ -1675,7 +1828,8 @@ public class NumberFieldMapper extends FieldMapper { builder.scriptValues(), builder.dimension.getValue(), builder.metric.getValue(), - builder.indexMode + builder.indexMode, + isSyntheticSource ); } @@ -1684,7 +1838,7 @@ public class NumberFieldMapper extends FieldMapper { } public NumberFieldType(String name, NumberType type, boolean isIndexed) { - this(name, type, isIndexed, false, true, true, null, Collections.emptyMap(), null, false, null, null); + this(name, type, isIndexed, false, true, true, null, Collections.emptyMap(), null, false, null, null, false); } @Override @@ -1761,6 +1915,11 @@ public class NumberFieldMapper extends FieldMapper { if (hasDocValues()) { return type.blockLoaderFromDocValues(name()); } + + if (isSyntheticSource) { + return type.blockLoaderFromFallbackSyntheticSource(name(), nullValue, coerce); + } + BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); @@ -1876,7 +2035,7 @@ public class NumberFieldMapper extends FieldMapper { private final MetricType metricType; private boolean allowMultipleValues; private final IndexVersion indexCreatedVersion; - private final boolean storeMalformedFields; + private final boolean isSyntheticSource; private final IndexMode indexMode; @@ -1884,7 +2043,7 @@ public class NumberFieldMapper extends FieldMapper { String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams, - boolean storeMalformedFields, + boolean isSyntheticSource, Builder builder ) { super(simpleName, mappedFieldType, builderParams); @@ -1904,7 +2063,7 @@ public class NumberFieldMapper extends FieldMapper { this.metricType = builder.metric.getValue(); this.allowMultipleValues = builder.allowMultipleValues; this.indexCreatedVersion = builder.indexCreatedVersion; - this.storeMalformedFields = storeMalformedFields; + this.isSyntheticSource = isSyntheticSource; this.indexMode = builder.indexMode; } @@ -1939,7 +2098,7 @@ public class NumberFieldMapper extends FieldMapper { } catch (IllegalArgumentException e) { if (ignoreMalformed.value() && context.parser().currentToken().isValue()) { context.addIgnoredField(mappedFieldType.name()); - if (storeMalformedFields) { + if (isSyntheticSource) { // Save a copy of the field so synthetic source can load it context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser())); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 36c25b352a79..172545ab459c 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -333,7 +333,8 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { null, false, null, - null + null, + false ) ); } @@ -353,7 +354,8 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { null, false, null, - null + null, + false ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index a8b935c79ccc..164e0232bf40 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -140,7 +140,8 @@ public class NumberFieldTypeTests extends FieldTypeTestCase { null, false, null, - null + null, + false ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java new file mode 100644 index 000000000000..28d7cbcfb42d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class ByteFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public ByteFieldBlockLoaderTests() { + super(FieldType.BYTE); + } + + @Override + protected Integer convert(Number value) { + // All values that fit into int are represented as ints + return value.intValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java new file mode 100644 index 000000000000..e0b62b21ad87 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class DoubleFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public DoubleFieldBlockLoaderTests() { + super(FieldType.DOUBLE); + } + + @Override + protected Double convert(Number value) { + return value.doubleValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java new file mode 100644 index 000000000000..63439a97d7c9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class FloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public FloatFieldBlockLoaderTests() { + super(FieldType.FLOAT); + } + + @Override + protected Double convert(Number value) { + // All float values are represented as double + return value.doubleValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java new file mode 100644 index 000000000000..1e8cedb734af --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class HalfFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public HalfFloatFieldBlockLoaderTests() { + super(FieldType.HALF_FLOAT); + } + + @Override + protected Double convert(Number value) { + // All float values are represented as double + return (double) HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value.floatValue())); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java new file mode 100644 index 000000000000..5d7b9d78442c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class IntegerFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public IntegerFieldBlockLoaderTests() { + super(FieldType.INTEGER); + } + + @Override + protected Integer convert(Number value) { + return value.intValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java index 4d5eb2ea641a..909cccf9e7d5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java @@ -59,18 +59,6 @@ public class KeywordFieldBlockLoaderTests extends BlockLoaderTestCase { return maybeFoldList(resultList); } - private Object maybeFoldList(List list) { - if (list.isEmpty()) { - return null; - } - - if (list.size() == 1) { - return list.get(0); - } - - return list; - } - private BytesRef convert(String value, String nullValue, int ignoreAbove) { if (value == null) { if (nullValue != null) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java new file mode 100644 index 000000000000..ff953294fb61 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class LongFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public LongFieldBlockLoaderTests() { + super(FieldType.LONG); + } + + @Override + protected Long convert(Number value) { + return value.longValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java new file mode 100644 index 000000000000..e523d011c3ab --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.logsdb.datageneration.FieldType; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public abstract class NumberFieldBlockLoaderTestCase extends BlockLoaderTestCase { + public NumberFieldBlockLoaderTestCase(FieldType fieldType) { + super(fieldType); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value, boolean syntheticSource) { + var nullValue = fieldMapping.get("null_value") != null ? convert((Number) fieldMapping.get("null_value")) : null; + + if (value instanceof List == false) { + return convert(value, nullValue); + } + + if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { + // Sorted and no duplicates + var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).sorted().toList(); + return maybeFoldList(resultList); + } + + // parsing from source + var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList(); + return maybeFoldList(resultList); + } + + @SuppressWarnings("unchecked") + private T convert(Object value, T nullValue) { + if (value == null) { + return nullValue; + } + // String coercion is true by default + if (value instanceof String s && s.isEmpty()) { + return nullValue; + } + if (value instanceof Number n) { + return convert(n); + } + + // Malformed values are excluded + return null; + } + + protected abstract T convert(Number value); +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java new file mode 100644 index 000000000000..a40bc1c404f4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class ShortFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public ShortFieldBlockLoaderTests() { + super(FieldType.SHORT); + } + + @Override + protected Integer convert(Number value) { + // All values that fit into int are represented as ints + return value.intValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java index 2d0622dbb632..230495db7327 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java @@ -92,7 +92,8 @@ public class AggregatorBaseTests extends MapperServiceTestCase { null, false, null, - null + null, + false ); return ValuesSourceConfig.resolveFieldOnly(ft, context); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index ba186695bcda..e7d19c0f56db 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -1494,7 +1494,8 @@ public class FiltersAggregatorTests extends AggregatorTestCase { null, false, null, - null + null, + false ); docValuesFieldExistsTestCase(new ExistsQueryBuilder("f"), ft, true, i -> { final LuceneDocument document = new LuceneDocument(); @@ -1517,7 +1518,8 @@ public class FiltersAggregatorTests extends AggregatorTestCase { null, false, null, - null + null, + false ) ); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index 32831f46c7a1..ab92ea859344 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -311,7 +311,8 @@ public class RangeAggregatorTests extends AggregatorTestCase { null, false, null, - null + null, + false ) ) ); @@ -426,7 +427,8 @@ public class RangeAggregatorTests extends AggregatorTestCase { null, false, null, - null + null, + false ); long start = 2L << 54; // Double stores 53 bits of mantissa, so we aggregate a bunch of bigger values @@ -707,7 +709,8 @@ public class RangeAggregatorTests extends AggregatorTestCase { null, false, null, - null + null, + false ); RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("test_range_agg"); aggregationBuilder.field(NUMBER_FIELD_NAME); diff --git a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java index 1261c8300902..336fae2cfec2 100644 --- a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java @@ -154,7 +154,8 @@ public class CollapseBuilderTests extends AbstractXContentSerializingTestCase builder.build(searchExecutionContext)); @@ -172,7 +173,8 @@ public class CollapseBuilderTests extends AbstractXContentSerializingTestCase list) { + if (list.isEmpty()) { + return null; + } + + if (list.size() == 1) { + return list.get(0); + } + + return list; + } + private Object setupAndInvokeBlockLoader(MapperService mapperService, XContentBuilder document, String fieldName) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index e68409209994..563828527afb 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -210,7 +210,8 @@ public class TimeSeriesRateAggregatorTests extends AggregatorTestCase { null, false, TimeSeriesParams.MetricType.COUNTER, - IndexMode.TIME_SERIES + IndexMode.TIME_SERIES, + false ); } From 03d7d34f01a5ad219545848fb6f66eca6a2f2167 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 12 Feb 2025 17:57:55 -0800 Subject: [PATCH 41/93] Remove completion listener from ExchangeSourceHandler (#122446) With #117410, each remote sink now has its own listener, and the main query won't finish until all remote sink listeners have completed. As a result, we no longer need to wait for the exchange source to finish. This change removes the completion listener to simplify the exchange service. The completion listener could previously return prematurely while remote sinks were still being registered. Closes #122408 --- .../exchange/ExchangeSourceHandler.java | 29 +-- .../compute/operator/DriverTests.java | 4 +- .../operator/ForkingOperatorTestCase.java | 6 +- .../exchange/ExchangeServiceTests.java | 204 +++++++++--------- .../esql/plugin/ClusterComputeHandler.java | 3 +- .../xpack/esql/plugin/ComputeService.java | 12 +- .../esql/plugin/DataNodeComputeHandler.java | 4 +- .../elasticsearch/xpack/esql/CsvTests.java | 2 +- .../optimizer/PhysicalPlanOptimizerTests.java | 2 +- 9 files changed, 120 insertions(+), 146 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java index ebbddaeeb0d2..68f684cdf9dc 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java @@ -8,8 +8,6 @@ package org.elasticsearch.compute.operator.exchange; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRunnable; -import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; @@ -19,7 +17,6 @@ import org.elasticsearch.compute.operator.IsBlockedResult; import org.elasticsearch.core.Releasable; import org.elasticsearch.tasks.TaskCancelledException; -import java.util.List; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -51,28 +48,12 @@ public final class ExchangeSourceHandler { * @param maxBufferSize the maximum size of the exchange buffer. A larger buffer reduces ``pauses`` but uses more memory, * which could otherwise be allocated for other purposes. * @param fetchExecutor the executor used to fetch pages. - * @param completionListener a listener that will be notified when the exchange source handler completes */ - public ExchangeSourceHandler(int maxBufferSize, Executor fetchExecutor, ActionListener completionListener) { + public ExchangeSourceHandler(int maxBufferSize, Executor fetchExecutor) { this.buffer = new ExchangeBuffer(maxBufferSize); this.fetchExecutor = fetchExecutor; this.outstandingSinks = new PendingInstances(() -> buffer.finish(false)); - final PendingInstances closingSinks = new PendingInstances(() -> {}); - closingSinks.trackNewInstance(); - this.outstandingSources = new PendingInstances(() -> finishEarly(true, ActionListener.running(closingSinks::finishInstance))); - buffer.addCompletionListener(ActionListener.running(() -> { - final ActionListener listener = ActionListener.assertAtLeastOnce(completionListener); - try (RefCountingRunnable refs = new RefCountingRunnable(ActionRunnable.run(listener, this::checkFailure))) { - closingSinks.completion.addListener(refs.acquireListener()); - for (PendingInstances pending : List.of(outstandingSinks, outstandingSources)) { - // Create an outstanding instance and then finish to complete the completionListener - // if we haven't registered any instances of exchange sinks or exchange sources before. - pending.trackNewInstance(); - pending.completion.addListener(refs.acquireListener()); - pending.finishInstance(); - } - } - })); + this.outstandingSources = new PendingInstances(() -> finishEarly(true, ActionListener.noop())); } private void checkFailure() { @@ -271,7 +252,13 @@ public final class ExchangeSourceHandler { final ActionListener sinkListener = ActionListener.assertAtLeastOnce( ActionListener.notifyOnce(ActionListener.runBefore(listener, () -> remoteSinks.remove(sinkId))) ); + final Releasable emptySink = addEmptySink(); fetchExecutor.execute(new AbstractRunnable() { + @Override + public void onAfter() { + emptySink.close(); + } + @Override public void onFailure(Exception e) { if (failFast) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java index f1f1f7762322..35ccf0da4296 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java @@ -334,8 +334,7 @@ public class DriverTests extends ESTestCase { DriverContext driverContext = driverContext(); ThreadPool threadPool = threadPool(); try { - PlainActionFuture sourceFuture = new PlainActionFuture<>(); - var sourceHandler = new ExchangeSourceHandler(between(1, 5), threadPool.executor("esql"), sourceFuture); + var sourceHandler = new ExchangeSourceHandler(between(1, 5), threadPool.executor("esql")); var sinkHandler = new ExchangeSinkHandler(driverContext.blockFactory(), between(1, 5), System::currentTimeMillis); var sourceOperator = new ExchangeSourceOperator(sourceHandler.createExchangeSource()); var sinkOperator = new ExchangeSinkOperator(sinkHandler.createExchangeSink(() -> {}), Function.identity()); @@ -351,7 +350,6 @@ public class DriverTests extends ESTestCase { sinkHandler.fetchPageAsync(true, ActionListener.noop()); future.actionGet(5, TimeUnit.SECONDS); assertThat(driver.status().status(), equalTo(DriverStatus.Status.DONE)); - sourceFuture.actionGet(5, TimeUnit.SECONDS); } finally { terminate(threadPool); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java index 6b036dea5f74..f08552913963 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java @@ -216,11 +216,7 @@ public abstract class ForkingOperatorTestCase extends OperatorTestCase { randomIntBetween(2, 10), threadPool.relativeTimeInMillisSupplier() ); - ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler( - randomIntBetween(1, 4), - threadPool.executor(ESQL_TEST_EXECUTOR), - ActionListener.noop() - ); + ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler(randomIntBetween(1, 4), threadPool.executor(ESQL_TEST_EXECUTOR)); sourceExchanger.addRemoteSink( sinkExchanger::fetchPageAsync, randomBoolean(), diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java index 2927bc5439af..57dfe65ca485 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.VersionInformation; @@ -105,16 +106,16 @@ public class ExchangeServiceTests extends ESTestCase { AtomicInteger pagesAddedToSink = new AtomicInteger(); ExchangeSink sink1 = sinkExchanger.createExchangeSink(pagesAddedToSink::incrementAndGet); ExchangeSink sink2 = sinkExchanger.createExchangeSink(pagesAddedToSink::incrementAndGet); - PlainActionFuture sourceCompletion = new PlainActionFuture<>(); - ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler(3, threadPool.executor(ESQL_TEST_EXECUTOR), sourceCompletion); + ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler(3, threadPool.executor(ESQL_TEST_EXECUTOR)); ExchangeSource source = sourceExchanger.createExchangeSource(); AtomicInteger pagesAddedToSource = new AtomicInteger(); + PlainActionFuture remoteSinkFuture = new PlainActionFuture<>(); sourceExchanger.addRemoteSink( sinkExchanger::fetchPageAsync, randomBoolean(), pagesAddedToSource::incrementAndGet, 1, - ActionListener.noop() + remoteSinkFuture ); SubscribableListener waitForReading = source.waitForReading().listener(); assertFalse(waitForReading.isDone()); @@ -161,13 +162,12 @@ public class ExchangeServiceTests extends ESTestCase { sink2.finish(); assertTrue(sink2.isFinished()); assertTrue(source.isFinished()); - assertFalse(sourceCompletion.isDone()); source.finish(); - sourceCompletion.actionGet(10, TimeUnit.SECONDS); ESTestCase.terminate(threadPool); for (Page page : pages) { page.releaseBlocks(); } + safeGet(remoteSinkFuture); } /** @@ -350,47 +350,45 @@ public class ExchangeServiceTests extends ESTestCase { public void testConcurrentWithHandlers() { BlockFactory blockFactory = blockFactory(); - PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); - var sourceExchanger = new ExchangeSourceHandler( - randomExchangeBuffer(), - threadPool.executor(ESQL_TEST_EXECUTOR), - sourceCompletionFuture - ); - List sinkHandlers = new ArrayList<>(); - Supplier exchangeSink = () -> { - final ExchangeSinkHandler sinkHandler; - if (sinkHandlers.isEmpty() == false && randomBoolean()) { - sinkHandler = randomFrom(sinkHandlers); - } else { - sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier()); - sourceExchanger.addRemoteSink( - sinkHandler::fetchPageAsync, - randomBoolean(), - () -> {}, - randomIntBetween(1, 3), - ActionListener.noop() - ); - sinkHandlers.add(sinkHandler); - } - return sinkHandler.createExchangeSink(() -> {}); - }; - final int maxInputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); - final int maxOutputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); - Set actualSeqNos = runConcurrentTest(maxInputSeqNo, maxOutputSeqNo, sourceExchanger::createExchangeSource, exchangeSink); - var expectedSeqNos = IntStream.range(0, Math.min(maxInputSeqNo, maxOutputSeqNo)).boxed().collect(Collectors.toSet()); - assertThat(actualSeqNos, hasSize(expectedSeqNos.size())); - assertThat(actualSeqNos, equalTo(expectedSeqNos)); - sourceCompletionFuture.actionGet(10, TimeUnit.SECONDS); + var sourceExchanger = new ExchangeSourceHandler(randomExchangeBuffer(), threadPool.executor(ESQL_TEST_EXECUTOR)); + PlainActionFuture remoteSinksFuture = new PlainActionFuture<>(); + try (RefCountingListener refs = new RefCountingListener(remoteSinksFuture)) { + List sinkHandlers = new ArrayList<>(); + Supplier exchangeSink = () -> { + final ExchangeSinkHandler sinkHandler; + if (sinkHandlers.isEmpty() == false && randomBoolean()) { + sinkHandler = randomFrom(sinkHandlers); + } else { + sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier()); + sourceExchanger.addRemoteSink( + sinkHandler::fetchPageAsync, + randomBoolean(), + () -> {}, + randomIntBetween(1, 3), + refs.acquire() + ); + sinkHandlers.add(sinkHandler); + } + return sinkHandler.createExchangeSink(() -> {}); + }; + final int maxInputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); + final int maxOutputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); + Set actualSeqNos = runConcurrentTest( + maxInputSeqNo, + maxOutputSeqNo, + sourceExchanger::createExchangeSource, + exchangeSink + ); + var expectedSeqNos = IntStream.range(0, Math.min(maxInputSeqNo, maxOutputSeqNo)).boxed().collect(Collectors.toSet()); + assertThat(actualSeqNos, hasSize(expectedSeqNos.size())); + assertThat(actualSeqNos, equalTo(expectedSeqNos)); + } + safeGet(remoteSinksFuture); } public void testExchangeSourceContinueOnFailure() { BlockFactory blockFactory = blockFactory(); - PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); - var exchangeSourceHandler = new ExchangeSourceHandler( - randomExchangeBuffer(), - threadPool.executor(ESQL_TEST_EXECUTOR), - sourceCompletionFuture - ); + var exchangeSourceHandler = new ExchangeSourceHandler(randomExchangeBuffer(), threadPool.executor(ESQL_TEST_EXECUTOR)); final int maxInputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); final int maxOutputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); Set expectedSeqNos = ConcurrentCollections.newConcurrentSet(); @@ -398,57 +396,65 @@ public class ExchangeServiceTests extends ESTestCase { AtomicInteger totalSinks = new AtomicInteger(); AtomicInteger failedSinks = new AtomicInteger(); AtomicInteger completedSinks = new AtomicInteger(); - Supplier exchangeSink = () -> { - var sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier()); - int failAfter = randomBoolean() ? Integer.MAX_VALUE : randomIntBetween(0, 100); - AtomicInteger fetched = new AtomicInteger(); - int instance = randomIntBetween(1, 3); - totalSinks.incrementAndGet(); - AtomicBoolean sinkFailed = new AtomicBoolean(); - exchangeSourceHandler.addRemoteSink((allSourcesFinished, listener) -> { - if (fetched.incrementAndGet() > failAfter) { - sinkHandler.fetchPageAsync(true, listener.delegateFailure((l, r) -> { - failedRequests.incrementAndGet(); - sinkFailed.set(true); - listener.onFailure(new CircuitBreakingException("simulated", CircuitBreaker.Durability.PERMANENT)); - })); - } else { - sinkHandler.fetchPageAsync(allSourcesFinished, listener.delegateFailure((l, r) -> { - Page page = r.takePage(); - if (page != null) { - IntBlock block = page.getBlock(0); - for (int i = 0; i < block.getPositionCount(); i++) { - int v = block.getInt(i); - if (v < maxOutputSeqNo) { - expectedSeqNos.add(v); + PlainActionFuture remoteSinksFuture = new PlainActionFuture<>(); + try (RefCountingListener refs = new RefCountingListener(remoteSinksFuture)) { + Supplier exchangeSink = () -> { + var sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier()); + int failAfter = randomBoolean() ? Integer.MAX_VALUE : randomIntBetween(0, 100); + AtomicInteger fetched = new AtomicInteger(); + int instance = randomIntBetween(1, 3); + totalSinks.incrementAndGet(); + AtomicBoolean sinkFailed = new AtomicBoolean(); + ActionListener oneSinkListener = refs.acquire(); + exchangeSourceHandler.addRemoteSink((allSourcesFinished, listener) -> { + if (fetched.incrementAndGet() > failAfter) { + sinkHandler.fetchPageAsync(true, listener.delegateFailure((l, r) -> { + failedRequests.incrementAndGet(); + sinkFailed.set(true); + listener.onFailure(new CircuitBreakingException("simulated", CircuitBreaker.Durability.PERMANENT)); + })); + } else { + sinkHandler.fetchPageAsync(allSourcesFinished, listener.delegateFailure((l, r) -> { + Page page = r.takePage(); + if (page != null) { + IntBlock block = page.getBlock(0); + for (int i = 0; i < block.getPositionCount(); i++) { + int v = block.getInt(i); + if (v < maxOutputSeqNo) { + expectedSeqNos.add(v); + } } } - } - l.onResponse(new ExchangeResponse(blockFactory, page, r.finished())); - })); - } - }, false, () -> {}, instance, ActionListener.wrap(r -> { - assertFalse(sinkFailed.get()); - completedSinks.incrementAndGet(); - }, e -> { - assertTrue(sinkFailed.get()); - failedSinks.incrementAndGet(); - })); - return sinkHandler.createExchangeSink(() -> {}); - }; - Set actualSeqNos = runConcurrentTest( - maxInputSeqNo, - maxOutputSeqNo, - exchangeSourceHandler::createExchangeSource, - exchangeSink - ); - assertThat(actualSeqNos, equalTo(expectedSeqNos)); - safeGet(sourceCompletionFuture); - assertThat(completedSinks.get() + failedSinks.get(), equalTo(totalSinks.get())); + l.onResponse(new ExchangeResponse(blockFactory, page, r.finished())); + })); + } + }, false, () -> {}, instance, ActionListener.wrap(r -> { + assertFalse(sinkFailed.get()); + completedSinks.incrementAndGet(); + oneSinkListener.onResponse(null); + }, e -> { + assertTrue(sinkFailed.get()); + failedSinks.incrementAndGet(); + oneSinkListener.onFailure(e); + })); + return sinkHandler.createExchangeSink(() -> {}); + }; + Set actualSeqNos = runConcurrentTest( + maxInputSeqNo, + maxOutputSeqNo, + exchangeSourceHandler::createExchangeSource, + exchangeSink + ); + assertThat(actualSeqNos, equalTo(expectedSeqNos)); + } if (failedRequests.get() > 0) { + expectThrows(CircuitBreakingException.class, () -> remoteSinksFuture.actionGet(1, TimeUnit.MINUTES)); assertThat(failedSinks.get(), greaterThan(0)); + assertThat(completedSinks.get() + failedSinks.get(), equalTo(totalSinks.get())); } else { + safeGet(remoteSinksFuture); assertThat(failedSinks.get(), equalTo(0)); + assertThat(completedSinks.get(), equalTo(totalSinks.get())); } } @@ -465,7 +471,7 @@ public class ExchangeServiceTests extends ESTestCase { assertFalse(sink.waitForWriting().listener().isDone()); PlainActionFuture future = new PlainActionFuture<>(); sinkExchanger.fetchPageAsync(true, future); - ExchangeResponse resp = future.actionGet(); + ExchangeResponse resp = safeGet(future); assertTrue(resp.finished()); assertNull(resp.takePage()); assertTrue(sink.waitForWriting().listener().isDone()); @@ -473,7 +479,7 @@ public class ExchangeServiceTests extends ESTestCase { } public void testFinishEarly() throws Exception { - ExchangeSourceHandler sourceHandler = new ExchangeSourceHandler(20, threadPool.generic(), ActionListener.noop()); + ExchangeSourceHandler sourceHandler = new ExchangeSourceHandler(20, threadPool.generic()); Semaphore permits = new Semaphore(between(1, 5)); BlockFactory blockFactory = blockFactory(); Queue pages = ConcurrentCollections.newQueue(); @@ -544,12 +550,7 @@ public class ExchangeServiceTests extends ESTestCase { try (exchange0; exchange1; node0; node1) { String exchangeId = "exchange"; Task task = new Task(1, "", "", "", null, Collections.emptyMap()); - PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); - var sourceHandler = new ExchangeSourceHandler( - randomExchangeBuffer(), - threadPool.executor(ESQL_TEST_EXECUTOR), - sourceCompletionFuture - ); + var sourceHandler = new ExchangeSourceHandler(randomExchangeBuffer(), threadPool.executor(ESQL_TEST_EXECUTOR)); ExchangeSinkHandler sinkHandler = exchange1.createSinkHandler(exchangeId, randomExchangeBuffer()); Transport.Connection connection = node0.getConnection(node1.getLocalNode()); sourceHandler.addRemoteSink( @@ -570,7 +571,6 @@ public class ExchangeServiceTests extends ESTestCase { var expectedSeqNos = IntStream.range(0, Math.min(maxInputSeqNo, maxOutputSeqNo)).boxed().collect(Collectors.toSet()); assertThat(actualSeqNos, hasSize(expectedSeqNos.size())); assertThat(actualSeqNos, equalTo(expectedSeqNos)); - sourceCompletionFuture.actionGet(10, TimeUnit.SECONDS); } } @@ -620,12 +620,7 @@ public class ExchangeServiceTests extends ESTestCase { try (exchange0; exchange1; node0; node1) { String exchangeId = "exchange"; Task task = new Task(1, "", "", "", null, Collections.emptyMap()); - PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); - var sourceHandler = new ExchangeSourceHandler( - randomIntBetween(1, 128), - threadPool.executor(ESQL_TEST_EXECUTOR), - sourceCompletionFuture - ); + var sourceHandler = new ExchangeSourceHandler(randomIntBetween(1, 128), threadPool.executor(ESQL_TEST_EXECUTOR)); ExchangeSinkHandler sinkHandler = exchange1.createSinkHandler(exchangeId, randomIntBetween(1, 128)); Transport.Connection connection = node0.getConnection(node1.getLocalNode()); PlainActionFuture remoteSinkFuture = new PlainActionFuture<>(); @@ -652,15 +647,14 @@ public class ExchangeServiceTests extends ESTestCase { assertThat(cause.getMessage(), equalTo("page is too large")); PlainActionFuture sinkCompletionFuture = new PlainActionFuture<>(); sinkHandler.addCompletionListener(sinkCompletionFuture); - assertBusy(() -> assertTrue(sinkCompletionFuture.isDone())); - expectThrows(Exception.class, () -> sourceCompletionFuture.actionGet(10, TimeUnit.SECONDS)); + safeGet(sinkCompletionFuture); } } public void testNoCyclicException() throws Exception { PlainActionFuture future = new PlainActionFuture<>(); try (EsqlRefCountingListener refs = new EsqlRefCountingListener(future)) { - var exchangeSourceHandler = new ExchangeSourceHandler(between(10, 100), threadPool.generic(), refs.acquire()); + var exchangeSourceHandler = new ExchangeSourceHandler(between(10, 100), threadPool.generic()); int numSinks = between(5, 10); for (int i = 0; i < numSinks; i++) { RemoteSink remoteSink = (allSourcesFinished, listener) -> threadPool.schedule( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java index a2a5e5175c4e..e41dd42c7579 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java @@ -222,8 +222,7 @@ final class ClusterComputeHandler implements TransportRequestHandler outputAttributes = physicalPlan.output(); + var exchangeSource = new ExchangeSourceHandler( + queryPragmas.exchangeBufferSize(), + transportService.getThreadPool().executor(ThreadPool.Names.SEARCH) + ); + listener = ActionListener.runBefore(listener, () -> exchangeService.removeExchangeSourceHandler(sessionId)); + exchangeService.addExchangeSourceHandler(sessionId, exchangeSource); try (var computeListener = new ComputeListener(transportService.getThreadPool(), cancelQueryOnFailure, listener.map(profiles -> { execInfo.markEndQuery(); // TODO: revisit this time recording model as part of INLINESTATS improvements return new Result(outputAttributes, collectedPages, profiles, execInfo); }))) { - var exchangeSource = new ExchangeSourceHandler( - queryPragmas.exchangeBufferSize(), - transportService.getThreadPool().executor(ThreadPool.Names.SEARCH), - ActionListener.runBefore(computeListener.acquireAvoid(), () -> exchangeService.removeExchangeSourceHandler(sessionId)) - ); - exchangeService.addExchangeSourceHandler(sessionId, exchangeSource); try (Releasable ignored = exchangeSource.addEmptySink()) { // run compute on the coordinator final AtomicBoolean localClusterWasInterrupted = new AtomicBoolean(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java index ba2f3c5dfdc2..ee5b192bf328 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java @@ -171,7 +171,7 @@ final class DataNodeComputeHandler implements TransportRequestHandler exchangeService.finishSinkHandler(externalId, new TaskCancelledException(task.getReasonCancelled())) ); - var exchangeSource = new ExchangeSourceHandler(1, esqlExecutor, computeListener.acquireAvoid()); + var exchangeSource = new ExchangeSourceHandler(1, esqlExecutor); exchangeSource.addRemoteSink(internalSink::fetchPageAsync, true, () -> {}, 1, ActionListener.noop()); var reductionListener = computeListener.acquireCompute(); computeService.runCompute( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 263c076c2331..1e21aa3774af 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -613,7 +613,7 @@ public class CsvTests extends ESTestCase { bigArrays, ByteSizeValue.ofBytes(randomLongBetween(1, BlockFactory.DEFAULT_MAX_BLOCK_PRIMITIVE_ARRAY_SIZE.getBytes() * 2)) ); - ExchangeSourceHandler exchangeSource = new ExchangeSourceHandler(between(1, 64), executor, ActionListener.noop()); + ExchangeSourceHandler exchangeSource = new ExchangeSourceHandler(between(1, 64), executor); ExchangeSinkHandler exchangeSink = new ExchangeSinkHandler(blockFactory, between(1, 64), threadPool::relativeTimeInMillis); LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index b877937d6397..55448b7ceaf4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -7596,7 +7596,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase { TestBlockFactory.getNonBreakingInstance(), Settings.EMPTY, config, - new ExchangeSourceHandler(10, null, null)::createExchangeSource, + new ExchangeSourceHandler(10, null)::createExchangeSource, () -> exchangeSinkHandler.createExchangeSink(() -> {}), null, null, From 8e2322c2efd872905551605bfa7d4fb840c90bd9 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 12 Feb 2025 17:58:53 -0800 Subject: [PATCH 42/93] Fix listener leak in exchange service (#122417) If we hit the circuit breaker exception before fetching pages, we fail to notify the listener. Closes #122271 --- docs/changelog/122417.yaml | 6 ++++++ .../elasticsearch/compute/EsqlRefCountingListener.java | 3 ++- .../compute/operator/exchange/ExchangeService.java | 9 ++++++++- .../elasticsearch/xpack/esql/plugin/ComputeListener.java | 3 ++- 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/122417.yaml diff --git a/docs/changelog/122417.yaml b/docs/changelog/122417.yaml new file mode 100644 index 000000000000..f9e33df2a523 --- /dev/null +++ b/docs/changelog/122417.yaml @@ -0,0 +1,6 @@ +pr: 122417 +summary: Fix listener leak in exchange service +area: ES|QL +type: bug +issues: + - 122271 diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java index 69df0fb8ceff..2dfc60744be2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java @@ -34,7 +34,8 @@ public final class EsqlRefCountingListener implements Releasable { } public ActionListener acquire() { - return refs.acquireListener().delegateResponse((l, e) -> { + var listener = ActionListener.assertAtLeastOnce(refs.acquireListener()); + return listener.delegateResponse((l, e) -> { failureCollector.unwrapAndCollect(e); l.onFailure(e); }); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index ac02273a48ee..dd36a6f455e8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -366,7 +367,13 @@ public final class ExchangeService extends AbstractLifecycleComponent { final long reservedBytes = allSourcesFinished ? 0 : estimatedPageSizeInBytes.get(); if (reservedBytes > 0) { // This doesn't fully protect ESQL from OOM, but reduces the likelihood. - blockFactory.breaker().addEstimateBytesAndMaybeBreak(reservedBytes, "fetch page"); + try { + blockFactory.breaker().addEstimateBytesAndMaybeBreak(reservedBytes, "fetch page"); + } catch (Exception e) { + assert e instanceof CircuitBreakingException : new AssertionError(e); + listener.onFailure(e); + return; + } listener = ActionListener.runAfter(listener, () -> blockFactory.breaker().addWithoutBreaking(-reservedBytes)); } transportService.sendChildRequest( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java index 3d358b8c7a8a..c8b8e84fd247 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java @@ -47,7 +47,8 @@ final class ComputeListener implements Releasable { * Acquires a new listener that doesn't collect result */ ActionListener acquireAvoid() { - return refs.acquire().delegateResponse((l, e) -> { + var listener = ActionListener.assertAtLeastOnce(refs.acquire()); + return listener.delegateResponse((l, e) -> { try { runOnFailure.run(); } finally { From 3c18ea6e4454c5977c20eb853e41fb8d80edf8dc Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Thu, 13 Feb 2025 12:19:00 +1000 Subject: [PATCH 43/93] Unmute `SetSingleNodeAllocateStepTests` (#122318) This failing test was already fixed by #121266 Fixes #121495 --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 17c307d1bea5..ce6a0a197b6e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -329,9 +329,6 @@ tests: - class: org.elasticsearch.ingest.geoip.FullClusterRestartIT method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/121115 -- class: org.elasticsearch.xpack.core.ilm.SetSingleNodeAllocateStepTests - method: testPerformActionSomeShardsOnlyOnNewNodesButNewNodesInvalidAttrs - issue: https://github.com/elastic/elasticsearch/issues/121495 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only} issue: https://github.com/elastic/elasticsearch/issues/121412 From ee3542a27c8f327f48c4efa65ec51366d4335b6c Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:39:05 +1000 Subject: [PATCH 44/93] Log hot threads after cluster cleanup timeout (#122341) In addition to logging the pending cluster tasks after the cluster health request times out during cluster cleanup in REST tests, we should log the hot threads to help identify any issues that could cause tasks to get stuck. Follow-up of #119186 Relates #111632 Relates #111431 Relates #111662 --- .../test/rest/ESRestTestCase.java | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 40b2bae2fc4b..677924a553ec 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1014,14 +1014,21 @@ public abstract class ESRestTestCase extends ESTestCase { private void waitForClusterUpdates() throws Exception { logger.info("Waiting for all cluster updates up to this moment to be processed"); + try { assertOK(adminClient().performRequest(new Request("GET", "_cluster/health?wait_for_events=languid"))); } catch (ResponseException e) { if (e.getResponse().getStatusLine().getStatusCode() == HttpStatus.SC_REQUEST_TIMEOUT) { + StringBuilder logMessage = new StringBuilder("Timed out waiting for cluster updates to be processed."); final var pendingTasks = getPendingClusterStateTasks(); if (pendingTasks != null) { - logger.error("Timed out waiting for cluster updates to be processed, {}", pendingTasks); + logMessage.append('\n').append(pendingTasks); } + final var hotThreads = getHotThreads(); + if (hotThreads != null) { + logMessage.append("\nHot threads: ").append(hotThreads); + } + logger.error(logMessage.toString()); } throw e; } @@ -1031,8 +1038,8 @@ public abstract class ESRestTestCase extends ESTestCase { try { Response response = adminClient().performRequest(new Request("GET", "/_cluster/pending_tasks")); List tasks = (List) entityAsMap(response).get("tasks"); - if (false == tasks.isEmpty()) { - StringBuilder message = new StringBuilder("there are still running tasks:"); + if (tasks.isEmpty() == false) { + StringBuilder message = new StringBuilder("There are still running tasks:"); for (Object task : tasks) { message.append('\n').append(task.toString()); } @@ -1044,6 +1051,18 @@ public abstract class ESRestTestCase extends ESTestCase { return null; } + private String getHotThreads() { + try { + Response response = adminClient().performRequest( + new Request("GET", "/_nodes/hot_threads?ignore_idle_threads=false&threads=9999") + ); + return EntityUtils.toString(response.getEntity()); + } catch (IOException e) { + logger.error("Failed to retrieve hot threads in the cluster during cleanup", e); + } + return null; + } + /** * This method checks whether ILM policies or templates get recreated after they have been deleted. If so, we are probably deleting * them unnecessarily, potentially causing test performance problems. This could happen for example if someone adds a new standard ILM From f7892776774a09a16e64226b8473a6817e2e4116 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 12 Feb 2025 19:41:37 -0800 Subject: [PATCH 45/93] Prefer client errors while collecting ES|QL failures (#122290) Currently, the ES|QL failure collectors categorize errors into non-cancellation and cancellation errors, preferring to return non-cancellation errors to users. With the retry on shard-level failure, the failure collector can now collect more categories of errors: client errors, server errors, shard-unavailable errors, and cancellation errors. For easier diagnostics and operations (especially on serverless), the failure collectors prefer returning client (4xx) errors over server (5xx) errors, shard-unavailable errors, and cancellation errors. Relates #120774 --- muted-tests.yml | 12 --- .../compute/operator/FailureCollector.java | 90 +++++++++++-------- .../operator/FailureCollectorTests.java | 21 ++++- .../esql/plugin/DataNodeRequestSender.java | 14 ++- 4 files changed, 78 insertions(+), 59 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index ce6a0a197b6e..3f4b11bf7602 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -368,12 +368,6 @@ tests: - class: org.elasticsearch.xpack.security.authz.IndexAliasesTests method: testRemoveIndex issue: https://github.com/elastic/elasticsearch/issues/122221 -- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT - method: testGroupingMultiValueByOrdinals - issue: https://github.com/elastic/elasticsearch/issues/122228 -- class: org.elasticsearch.xpack.esql.action.EsqlNodeFailureIT - method: testFailureLoadingFields - issue: https://github.com/elastic/elasticsearch/issues/122132 - class: org.elasticsearch.blocks.SimpleBlocksIT method: testConcurrentAddBlock issue: https://github.com/elastic/elasticsearch/issues/122324 @@ -385,9 +379,6 @@ tests: - class: org.elasticsearch.xpack.esql.action.CrossClusterCancellationIT method: testCloseSkipUnavailable issue: https://github.com/elastic/elasticsearch/issues/122336 -- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT - method: testUnsupportedTypesOrdinalGrouping - issue: https://github.com/elastic/elasticsearch/issues/122342 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/alias/line_260} issue: https://github.com/elastic/elasticsearch/issues/122343 @@ -398,9 +389,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122377 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/122378 -- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT - method: testStatsMissingFieldWithStats - issue: https://github.com/elastic/elasticsearch/issues/122327 - class: org.elasticsearch.xpack.esql.CsvTests issue: https://github.com/elastic/elasticsearch/issues/122440 diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java index c492ba679635..7040f8712e61 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java @@ -9,26 +9,35 @@ package org.elasticsearch.compute.operator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.transport.TransportException; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; import java.util.Queue; -import java.util.concurrent.Semaphore; +import java.util.concurrent.ArrayBlockingQueue; /** * {@code FailureCollector} is responsible for collecting exceptions that occur in the compute engine. - * The collected exceptions are categorized into task-cancelled and non-task-cancelled exceptions. - * To limit memory usage, this class collects only the first 10 exceptions in each category by default. - * When returning the accumulated failure to the caller, this class prefers non-task-cancelled exceptions - * over task-cancelled ones as they are more useful for diagnosing issues. + * The collected exceptions are categorized into client (4xx), server (5xx), shard-unavailable errors, + * and cancellation errors. To limit memory usage, this class collects only the first 10 exceptions in + * each category by default. When returning the accumulated failures to the caller, this class prefers + * client (4xx) errors over server (5xx) errors, shard-unavailable errors, and cancellation errors, + * as they are more useful for diagnosing issues. */ public final class FailureCollector { - private final Queue cancelledExceptions = ConcurrentCollections.newQueue(); - private final Semaphore cancelledExceptionsPermits; - private final Queue nonCancelledExceptions = ConcurrentCollections.newQueue(); - private final Semaphore nonCancelledExceptionsPermits; + private enum Category { + CLIENT, + SERVER, + SHARD_UNAVAILABLE, + CANCELLATION + } + + private final Map> categories; + private final int maxExceptions; private volatile boolean hasFailure = false; private Exception finalFailure = null; @@ -41,8 +50,11 @@ public final class FailureCollector { if (maxExceptions <= 0) { throw new IllegalArgumentException("maxExceptions must be at least one"); } - this.cancelledExceptionsPermits = new Semaphore(maxExceptions); - this.nonCancelledExceptionsPermits = new Semaphore(maxExceptions); + this.maxExceptions = maxExceptions; + this.categories = new EnumMap<>(Category.class); + for (Category c : Category.values()) { + this.categories.put(c, new ArrayBlockingQueue<>(maxExceptions)); + } } public static Exception unwrapTransportException(TransportException te) { @@ -56,16 +68,24 @@ public final class FailureCollector { } } + private static Category getErrorCategory(Exception e) { + if (ExceptionsHelper.unwrap(e, TaskCancelledException.class) != null) { + return Category.CANCELLATION; + } else if (TransportActions.isShardNotAvailableException(e)) { + return Category.SHARD_UNAVAILABLE; + } else { + final int status = ExceptionsHelper.status(e).getStatus(); + if (400 <= status && status < 500) { + return Category.CLIENT; + } else { + return Category.SERVER; + } + } + } + public void unwrapAndCollect(Exception e) { e = e instanceof TransportException te ? unwrapTransportException(te) : e; - if (ExceptionsHelper.unwrap(e, TaskCancelledException.class) != null) { - if (nonCancelledExceptions.isEmpty() && cancelledExceptionsPermits.tryAcquire()) { - cancelledExceptions.add(e); - } - } else if (nonCancelledExceptionsPermits.tryAcquire()) { - nonCancelledExceptions.add(e); - cancelledExceptions.clear(); - } + categories.get(getErrorCategory(e)).offer(e); hasFailure = true; } @@ -77,8 +97,8 @@ public final class FailureCollector { } /** - * Returns the accumulated failure, preferring non-task-cancelled exceptions over task-cancelled ones. - * Once this method builds the failure, incoming failures are discarded. + * Returns the accumulated failure, preferring client (4xx) errors over server (5xx) errors and cancellation errors, + * as they are more useful for diagnosing issues. Once this method builds the failure, incoming failures are discarded. * * @return the accumulated failure, or {@code null} if no failure has been collected */ @@ -98,21 +118,19 @@ public final class FailureCollector { assert hasFailure; assert Thread.holdsLock(this); Exception first = null; - for (Exception e : nonCancelledExceptions) { - if (first == null) { - first = e; - } else if (first != e) { - first.addSuppressed(e); + int collected = 0; + for (Category category : List.of(Category.CLIENT, Category.SERVER, Category.SHARD_UNAVAILABLE, Category.CANCELLATION)) { + if (first != null && category == Category.CANCELLATION) { + continue; // do not add cancellation errors if other errors present } - } - if (first != null) { - return first; - } - for (Exception e : cancelledExceptions) { - if (first == null) { - first = e; - } else if (first != e) { - first.addSuppressed(e); + for (Exception e : categories.get(category)) { + if (++collected <= maxExceptions) { + if (first == null) { + first = e; + } else if (first != e) { + first.addSuppressed(e); + } + } } } assert first != null; diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java index 5fec82b32dda..4007d4d433f5 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java @@ -8,10 +8,12 @@ package org.elasticsearch.compute.operator; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.NodeDisconnectedException; @@ -106,13 +108,28 @@ public class FailureCollectorTests extends ESTestCase { public void testTransportExceptions() { FailureCollector collector = new FailureCollector(5); collector.unwrapAndCollect(new NodeDisconnectedException(DiscoveryNodeUtils.builder("node-1").build(), "/field_caps")); - collector.unwrapAndCollect(new TransportException(new CircuitBreakingException("too large", CircuitBreaker.Durability.TRANSIENT))); + collector.unwrapAndCollect(new TransportException(new IOException("disk issue"))); Exception failure = collector.getFailure(); assertNotNull(failure); assertThat(failure, instanceOf(NodeDisconnectedException.class)); assertThat(failure.getMessage(), equalTo("[][0.0.0.0:1][/field_caps] disconnected")); Throwable[] suppressed = failure.getSuppressed(); assertThat(suppressed, arrayWithSize(1)); - assertThat(suppressed[0], instanceOf(CircuitBreakingException.class)); + assertThat(suppressed[0], instanceOf(IOException.class)); + } + + public void testErrorCategory() { + FailureCollector collector = new FailureCollector(5); + collector.unwrapAndCollect(new NoShardAvailableActionException(new ShardId("test", "n/a", 1), "not ready")); + collector.unwrapAndCollect( + new TransportException(new CircuitBreakingException("request is too large", CircuitBreaker.Durability.TRANSIENT)) + ); + Exception failure = collector.getFailure(); + assertNotNull(failure); + assertThat(failure, instanceOf(CircuitBreakingException.class)); + assertThat(failure.getMessage(), equalTo("request is too large")); + assertThat(failure.getSuppressed(), arrayWithSize(1)); + assertThat(failure.getSuppressed()[0], instanceOf(NoShardAvailableActionException.class)); + assertThat(failure.getSuppressed()[0].getMessage(), equalTo("not ready")); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java index 8c2a6bb06da9..2d5b4169c021 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java @@ -124,10 +124,6 @@ abstract class DataNodeRequestSender { reportedFailure = true; reportFailures(computeListener); } else { - pendingShardIds.removeIf(shr -> { - var failure = shardFailures.get(shr); - return failure != null && failure.fatal; - }); var nodeRequests = selectNodeRequests(targetShards); for (NodeRequest request : nodeRequests) { sendOneNodeRequest(targetShards, computeListener, request); @@ -238,10 +234,6 @@ abstract class DataNodeRequestSender { TargetShard getShard(ShardId shardId) { return shards.get(shardId); } - - Set shardIds() { - return shards.keySet(); - } } /** @@ -270,7 +262,11 @@ abstract class DataNodeRequestSender { final Iterator shardsIt = pendingShardIds.iterator(); while (shardsIt.hasNext()) { ShardId shardId = shardsIt.next(); - assert shardFailures.get(shardId) == null || shardFailures.get(shardId).fatal == false; + ShardFailure failure = shardFailures.get(shardId); + if (failure != null && failure.fatal) { + shardsIt.remove(); + continue; + } TargetShard shard = targetShards.getShard(shardId); Iterator nodesIt = shard.remainingNodes.iterator(); DiscoveryNode selectedNode = null; From 71e45ae138c79ee3d81038daad44a1cc1cbd4e17 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 13 Feb 2025 14:45:23 +1100 Subject: [PATCH 46/93] Make persistent task name and ID available in update source (#122334) Add the task name (when possible) and ID as part of the source string for updating cluster state. This helps better identifying the source of a task. The updatePersistentTaskState method already does it. This PR ensures it is the case in other places. --- .../persistent/PersistentTaskCreationFailureIT.java | 4 +++- .../persistent/PersistentTasksClusterService.java | 10 +++++----- .../TransportSetTransformUpgradeModeActionTests.java | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskCreationFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskCreationFailureIT.java index 6452968f2467..d6da940b0188 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskCreationFailureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskCreationFailureIT.java @@ -77,7 +77,9 @@ public class PersistentTaskCreationFailureIT extends ESIntegTestCase { .pendingTasks() .stream() .filter( - pendingClusterTask -> pendingClusterTask.getSource().string().equals("finish persistent task (failed)") + pendingClusterTask -> pendingClusterTask.getSource() + .string() + .matches("finish persistent task \\[.*] \\(failed\\)") ) .count(); assertThat(completePersistentTaskPendingTasksCount, lessThanOrEqualTo(1L)); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 003aaa20d6ac..bfec7b372323 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -113,7 +113,7 @@ public final class PersistentTasksClusterService implements ClusterStateListener Params taskParams, ActionListener> listener ) { - submitUnbatchedTask("create persistent task", new ClusterStateUpdateTask() { + submitUnbatchedTask("create persistent task " + taskName + " [" + taskId + "]", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { PersistentTasksCustomMetadata.Builder builder = builder(currentState); @@ -166,9 +166,9 @@ public final class PersistentTasksClusterService implements ClusterStateListener final String source; if (failure != null) { logger.warn("persistent task " + id + " failed", failure); - source = "finish persistent task (failed)"; + source = "finish persistent task [" + id + "] (failed)"; } else { - source = "finish persistent task (success)"; + source = "finish persistent task [" + id + "] (success)"; } submitUnbatchedTask(source, new ClusterStateUpdateTask() { @Override @@ -212,7 +212,7 @@ public final class PersistentTasksClusterService implements ClusterStateListener * @param listener the listener that will be called when task is removed */ public void removePersistentTask(String id, ActionListener> listener) { - submitUnbatchedTask("remove persistent task", new ClusterStateUpdateTask() { + submitUnbatchedTask("remove persistent task [" + id + "]", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { PersistentTasksCustomMetadata.Builder tasksInProgress = builder(currentState); @@ -295,7 +295,7 @@ public final class PersistentTasksClusterService implements ClusterStateListener final String reason, final ActionListener> listener ) { - submitUnbatchedTask("unassign persistent task from any node", new ClusterStateUpdateTask() { + submitUnbatchedTask("unassign persistent task [" + taskId + "] from any node", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { PersistentTasksCustomMetadata.Builder tasksInProgress = builder(currentState); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeActionTests.java index 97ef367dca8e..f31922d5b69f 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeActionTests.java @@ -38,6 +38,7 @@ import static org.elasticsearch.xpack.core.action.AbstractTransportSetUpgradeMod import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.matches; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -176,7 +177,7 @@ public class TransportSetTransformUpgradeModeActionTests extends ESTestCase { upgradeModeSuccessfullyChanged(stateWithTransformTask(), assertNoFailureListener(r -> { assertThat(r, is(AcknowledgedResponse.TRUE)); - verify(clusterService).submitUnbatchedStateUpdateTask(eq("unassign persistent task from any node"), any()); + verify(clusterService).submitUnbatchedStateUpdateTask(matches("unassign persistent task \\[.*\\] from any node"), any()); })); } From 49ecbcaf27208fd7246eb204bdd2904e03976826 Mon Sep 17 00:00:00 2001 From: Mikhail Berezovskiy Date: Wed, 12 Feb 2025 20:36:48 -0800 Subject: [PATCH 47/93] Test-fix testAcceptsMismatchedServerlessBuildHash #121869 (#122332) --- .../java/org/elasticsearch/transport/TransportService.java | 2 +- .../transport/TransportServiceHandshakeTests.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 0fb767c5789f..b546b8cdd0f5 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -83,7 +83,7 @@ public class TransportService extends AbstractLifecycleComponent /** * A feature flag enabling transport upgrades for serverless. */ - private static final String SERVERLESS_TRANSPORT_SYSTEM_PROPERTY = "es.serverless_transport"; + static final String SERVERLESS_TRANSPORT_SYSTEM_PROPERTY = "es.serverless_transport"; private static final boolean SERVERLESS_TRANSPORT_FEATURE_FLAG = Booleans.parseBoolean( System.getProperty(SERVERLESS_TRANSPORT_SYSTEM_PROPERTY), false diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java index c686329c4154..d663e2d4c354 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java @@ -377,7 +377,8 @@ public class TransportServiceHandshakeTests extends ESTestCase { public void testAcceptsMismatchedServerlessBuildHash() { assumeTrue("Current build needs to be a snapshot", Build.current().isSnapshot()); assumeTrue("Security manager needs to be disabled", System.getSecurityManager() == null); - System.setProperty("es.serverless", Boolean.TRUE.toString()); // security manager blocks this + System.setProperty(TransportService.SERVERLESS_TRANSPORT_SYSTEM_PROPERTY, Boolean.TRUE.toString()); // security manager blocks + // this try { final DisruptingTransportInterceptor transportInterceptorA = new DisruptingTransportInterceptor(); final DisruptingTransportInterceptor transportInterceptorB = new DisruptingTransportInterceptor(); @@ -404,7 +405,7 @@ public class TransportServiceHandshakeTests extends ESTestCase { AbstractSimpleTransportTestCase.connectToNode(transportServiceA, transportServiceB.getLocalNode(), TestProfiles.LIGHT_PROFILE); assertTrue(transportServiceA.nodeConnected(transportServiceB.getLocalNode())); } finally { - System.clearProperty("es.serverless"); + System.clearProperty(TransportService.SERVERLESS_TRANSPORT_SYSTEM_PROPERTY); } } From 1bafbc1d3d715b42877434ab120c50d301a65f29 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 13 Feb 2025 09:09:15 +0100 Subject: [PATCH 48/93] [main][ML] Delete obsolete snapshot stats after upgrade (#121661)(#121661) (#122400) Ensure that the old snapshot model_size_stats document is removed after the snapshot upgrade. --- .../autodetect/JobModelSnapshotUpgrader.java | 125 ++++++++++++++---- .../upgrades/MlJobSnapshotUpgradeIT.java | 10 +- 2 files changed, 105 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java index d69acab30451..d42eb8f748b5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java @@ -12,19 +12,27 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.process.autodetect.output.FlushAcknowledgement; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.snapshot.upgrade.SnapshotUpgradeState; import org.elasticsearch.xpack.core.ml.job.snapshot.upgrade.SnapshotUpgradeTaskState; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -44,9 +52,7 @@ import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -153,6 +159,55 @@ public final class JobModelSnapshotUpgrader { executor.execute(); } + private void removeDuplicateModelSnapshotDoc(Consumer runAfter) { + String snapshotDocId = jobId + "_model_snapshot_" + snapshotId; + client.prepareSearch(AnomalyDetectorsIndex.jobResultsIndexPattern()) + .setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(snapshotDocId))) + .setSize(2) + .addSort(ModelSnapshot.MIN_VERSION.getPreferredName(), org.elasticsearch.search.sort.SortOrder.ASC) + .execute(ActionListener.wrap(searchResponse -> { + if (searchResponse.getHits().getTotalHits().value() > 1) { + deleteOlderSnapshotDoc(searchResponse, runAfter); + } else { + onFinish.accept(null); + } + }, e -> { + logger.warn(() -> format("[%s] [%s] error during search for model snapshot documents", jobId, snapshotId), e); + onFinish.accept(null); + })); + } + + private void deleteOlderSnapshotDoc(SearchResponse searchResponse, Consumer runAfter) { + SearchHit firstHit = searchResponse.getHits().getAt(0); + logger.debug(() -> format("[%s] deleting duplicate model snapshot doc [%s]", jobId, firstHit.getId())); + client.prepareDelete() + .setIndex(firstHit.getIndex()) + .setId(firstHit.getId()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute(ActionListener.runAfter(ActionListener.wrap(deleteResponse -> { + if ((deleteResponse.getResult() == DocWriteResponse.Result.DELETED) == false) { + logger.warn( + () -> format( + "[%s] [%s] failed to delete old snapshot [%s] result document, document not found", + jobId, + snapshotId, + ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName() + ) + ); + } + }, e -> { + logger.warn( + () -> format( + "[%s] [%s] failed to delete old snapshot [%s] result document", + jobId, + snapshotId, + ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName() + ), + e + ); + }), () -> runAfter.accept(null))); + } + void setTaskToFailed(String reason, ActionListener> listener) { SnapshotUpgradeTaskState taskState = new SnapshotUpgradeTaskState(SnapshotUpgradeState.FAILED, task.getAllocationId(), reason); task.updatePersistentTaskState(taskState, ActionListener.wrap(listener::onResponse, f -> { @@ -259,7 +314,7 @@ public final class JobModelSnapshotUpgrader { logger.error(() -> format("[%s] [%s] failed to write old state", jobId, snapshotId), e); setTaskToFailed( "Failed to write old state due to: " + e.getMessage(), - ActionListener.wrap(t -> shutdown(e), f -> shutdown(e)) + ActionListener.running(() -> shutdownWithFailure(e)) ); return; } @@ -273,7 +328,7 @@ public final class JobModelSnapshotUpgrader { logger.error(() -> format("[%s] [%s] failed to flush after writing old state", jobId, snapshotId), e); nextStep = () -> setTaskToFailed( "Failed to flush after writing old state due to: " + e.getMessage(), - ActionListener.wrap(t -> shutdown(e), f -> shutdown(e)) + ActionListener.running(() -> shutdownWithFailure(e)) ); } else { logger.debug( @@ -295,7 +350,7 @@ public final class JobModelSnapshotUpgrader { new SnapshotUpgradeTaskState(SnapshotUpgradeState.SAVING_NEW_STATE, task.getAllocationId(), ""), ActionListener.wrap(readingNewState -> { if (continueRunning.get() == false) { - shutdown(null); + shutdownWithFailure(null); return; } submitOperation(() -> { @@ -310,12 +365,12 @@ public final class JobModelSnapshotUpgrader { // Execute callback in the UTILITY thread pool, as the current thread in the callback will be one in the // autodetectWorkerExecutor. Trying to run the callback in that executor will cause a dead lock as that // executor has a single processing queue. - (aVoid, e) -> threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> shutdown(e)) + (aVoid, e) -> threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> handlePersistingState(e)) ); logger.debug("[{}] [{}] asked for state to be persisted", jobId, snapshotId); }, f -> { logger.error(() -> format("[%s] [%s] failed to update snapshot upgrader task to started", jobId, snapshotId), f); - shutdown( + shutdownWithFailure( new ElasticsearchStatusException( "Failed to start snapshot upgrade [{}] for job [{}]", RestStatus.INTERNAL_SERVER_ERROR, @@ -378,17 +433,45 @@ public final class JobModelSnapshotUpgrader { } } - void shutdown(Exception e) { + private void handlePersistingState(@Nullable Exception exception) { + assert Thread.currentThread().getName().contains(UTILITY_THREAD_POOL_NAME); + + if (exception != null) { + shutdownWithFailure(exception); + } else { + stopProcess((aVoid, e) -> { + threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> { + autodetectWorkerExecutor.shutdownNow(); + // If there are two snapshot documents in the results indices with the same snapshot id, + // remove the old one. This can happen when the result index has been rolled over and + // the write alias is pointing to the new index. + removeDuplicateModelSnapshotDoc(onFinish); + }); + + }); + } + } + + void shutdownWithFailure(Exception e) { + stopProcess((aVoid, ignored) -> { + threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> { + onFinish.accept(e); + autodetectWorkerExecutor.shutdownNow(); + }); + }); + } + + private void stopProcess(BiConsumer, Exception> runNext) { logger.debug("[{}] [{}] shutdown initiated", jobId, snapshotId); // No point in sending an action to the executor if the process has died if (process.isProcessAlive() == false) { logger.debug("[{}] [{}] process is dead, no need to shutdown", jobId, snapshotId); - onFinish.accept(e); - autodetectWorkerExecutor.shutdownNow(); stateStreamer.cancel(); + runNext.accept(null, null); return; } - Future future = autodetectWorkerExecutor.submit(() -> { + + submitOperation(() -> { try { logger.debug("[{}] [{}] shutdown is now occurring", jobId, snapshotId); if (process.isReady()) { @@ -401,24 +484,10 @@ public final class JobModelSnapshotUpgrader { processor.awaitCompletion(); } catch (IOException | TimeoutException exc) { logger.warn(() -> format("[%s] [%s] failed to shutdown process", jobId, snapshotId), exc); - } finally { - onFinish.accept(e); } logger.debug("[{}] [{}] connection for upgrade has been closed, process is shutdown", jobId, snapshotId); - }); - try { - future.get(); - autodetectWorkerExecutor.shutdownNow(); - } catch (InterruptedException interrupt) { - Thread.currentThread().interrupt(); - } catch (ExecutionException executionException) { - if (processor.isProcessKilled()) { - // In this case the original exception is spurious and highly misleading - throw ExceptionsHelper.conflictStatusException("close snapshot upgrade interrupted by kill request"); - } else { - throw FutureUtils.rethrowExecutionException(executionException); - } - } + return Void.TYPE; + }, runNext); } } } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java index 75613c7e3ebc..78f6bcd8ac9a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java @@ -65,7 +65,6 @@ public class MlJobSnapshotUpgradeIT extends AbstractUpgradeTestCase { * The purpose of this test is to ensure that when a job is open through a rolling upgrade we upgrade the results * index mappings when it is assigned to an upgraded node even if no other ML endpoint is called after the upgrade */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98560") public void testSnapshotUpgrader() throws Exception { Request adjustLoggingLevels = new Request("PUT", "/_cluster/settings"); adjustLoggingLevels.setJsonEntity(""" @@ -98,6 +97,13 @@ public class MlJobSnapshotUpgradeIT extends AbstractUpgradeTestCase { @SuppressWarnings("unchecked") private void testSnapshotUpgradeFailsOnMixedCluster() throws Exception { + // TODO the mixed cluster assertions sometimes fail because the code that + // detects the mixed cluster relies on the transport versions being different. + // This assumption does not hold immediately after a version bump and new + // branch being cut as the new branch will have the same transport version + // See https://github.com/elastic/elasticsearch/issues/98560 + + assumeTrue("The mixed cluster is not always detected correctly, see https://github.com/elastic/elasticsearch/issues/98560", false); Map jobs = entityAsMap(getJob(JOB_ID)); String currentSnapshot = ((List) XContentMapValues.extractValue("jobs.model_snapshot_id", jobs)).get(0); @@ -154,7 +160,7 @@ public class MlJobSnapshotUpgradeIT extends AbstractUpgradeTestCase { List> upgradedSnapshot = (List>) entityAsMap(getModelSnapshots(JOB_ID, snapshotToUpgradeId)) .get("model_snapshots"); - assertThat(upgradedSnapshot, hasSize(1)); + assertThat(upgradedSnapshot.toString(), upgradedSnapshot, hasSize(1)); assertThat(upgradedSnapshot.get(0).get("latest_record_time_stamp"), equalTo(snapshotToUpgrade.get("latest_record_time_stamp"))); // Does the snapshot still work? From 93d601c39154717614101891fbefb4b5075c81bc Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 13 Feb 2025 09:41:30 +0100 Subject: [PATCH 49/93] Develocity setup does not allow overwriting server url (#122470) This is a prerequisite to have https://github.com/elastic/elasticsearch/pull/122296 passing in bwc tests --- .../src/main/groovy/elasticsearch.build-scan.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index 797dc8bd0641..8702f5a9bf0e 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -32,7 +32,9 @@ develocity { // Automatically publish scans from Elasticsearch CI if (onCI) { publishing.onlyIf { true } - server = 'https://gradle-enterprise.elastic.co' + if(server.isPresent() == false) { + server = 'https://gradle-enterprise.elastic.co' + } } else if( server.isPresent() == false) { publishing.onlyIf { false } } From 81b73b630cf9cc185e95c85e73f009788cfcf96d Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Thu, 13 Feb 2025 10:03:46 +0100 Subject: [PATCH 50/93] Un-mute CsvTests (#122469) --- muted-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 3f4b11bf7602..35e010618f37 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -389,8 +389,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122377 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/122378 -- class: org.elasticsearch.xpack.esql.CsvTests - issue: https://github.com/elastic/elasticsearch/issues/122440 # Examples: # From bb4915947f2f80daa6092a0bc0548f379741785a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 20:06:28 +1100 Subject: [PATCH 51/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testClosedIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} #122481 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 35e010618f37..0e058c6f45af 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -389,6 +389,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122377 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/122378 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testClosedIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122481 # Examples: # From 260a2ee463cac46b5e077d2b0feff94b009b4b7f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 20:06:36 +1100 Subject: [PATCH 52/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testRestoreIndex {p0=[9.1.0, 9.1.0, 8.19.0]} #122482 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0e058c6f45af..e7f427b3f651 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -392,6 +392,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testClosedIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122481 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testRestoreIndex {p0=[9.1.0, 9.1.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122482 # Examples: # From da024dc942a9980ee3614ca0525507bb7803354e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 20:06:44 +1100 Subject: [PATCH 53/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testRestoreIndex {p0=[9.1.0, 8.19.0, 8.19.0]} #122483 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e7f427b3f651..11887e16a8a4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -395,6 +395,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testRestoreIndex {p0=[9.1.0, 9.1.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122482 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testRestoreIndex {p0=[9.1.0, 8.19.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122483 # Examples: # From bfaffe9d51029dc220e2173100910042b3afa388 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 20:06:52 +1100 Subject: [PATCH 54/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} #122484 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 11887e16a8a4..36ce57c27fb2 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -398,6 +398,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testRestoreIndex {p0=[9.1.0, 8.19.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122483 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122484 # Examples: # From 22e1a67d6f1805adb2ae5d7b0abda5b6c8e3daf0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 21:16:58 +1100 Subject: [PATCH 55/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} #122487 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 36ce57c27fb2..6d7ca3025d3f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -401,6 +401,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122484 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} + issue: https://github.com/elastic/elasticsearch/issues/122487 # Examples: # From f0c3bc9ad297274ab5939d2aab9f53ad36d15c45 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 21:17:06 +1100 Subject: [PATCH 56/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} #122488 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 6d7ca3025d3f..0c1a9c4082cf 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -404,6 +404,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} issue: https://github.com/elastic/elasticsearch/issues/122487 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122488 # Examples: # From b40465152799e69931b584ae682b0d7589c12bb6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 21:19:03 +1100 Subject: [PATCH 57/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} #122489 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0c1a9c4082cf..260d46ec5a98 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -407,6 +407,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122488 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122489 # Examples: # From 9d50ceaf59c26f3c37427a1da26414b0f3d85335 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 21:27:11 +1100 Subject: [PATCH 58/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testRestoreIndex {p0=[9.1.0, 9.1.0, 9.1.0]} #122490 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 260d46ec5a98..0bd08da9d786 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -410,6 +410,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122489 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testRestoreIndex {p0=[9.1.0, 9.1.0, 9.1.0]} + issue: https://github.com/elastic/elasticsearch/issues/122490 # Examples: # From 0c9c9103c93a9a8cb7065c6b809d367f2b21efdc Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 22:25:51 +1100 Subject: [PATCH 59/93] Mute org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} #122495 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0bd08da9d786..693db1072db4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -413,6 +413,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testRestoreIndex {p0=[9.1.0, 9.1.0, 9.1.0]} issue: https://github.com/elastic/elasticsearch/issues/122490 +- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase + method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} + issue: https://github.com/elastic/elasticsearch/issues/122495 # Examples: # From d5b6c265bfde9e7ebeaee8fbee3189618749988e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:40:48 +1100 Subject: [PATCH 60/93] Mute org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT testSearchableSnapshotUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} #122500 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 693db1072db4..4239e6c8ad38 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -416,6 +416,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} issue: https://github.com/elastic/elasticsearch/issues/122495 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testSearchableSnapshotUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122500 # Examples: # From 6e0541e08061d6efe449abd624f014b2b5ae94ae Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:40:55 +1100 Subject: [PATCH 61/93] Mute org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 8.19.0]} #122501 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 4239e6c8ad38..caa29211d1a3 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -419,6 +419,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT method: testSearchableSnapshotUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122500 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122501 # Examples: # From 28797290ade3d434bce49512669f31dc83a2bc74 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:41:04 +1100 Subject: [PATCH 62/93] Mute org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT testMountSearchableSnapshot {p0=[9.1.0, 8.19.0, 8.19.0]} #122502 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index caa29211d1a3..c0816e7ef92a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -422,6 +422,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122501 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testMountSearchableSnapshot {p0=[9.1.0, 8.19.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122502 # Examples: # From 4cdb1e3cdd7c6c396a60fcd36cdb6ffea0f3ba0a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:41:11 +1100 Subject: [PATCH 63/93] Mute org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} #122503 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index c0816e7ef92a..2c4fc4d421cc 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -425,6 +425,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT method: testMountSearchableSnapshot {p0=[9.1.0, 8.19.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122502 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} + issue: https://github.com/elastic/elasticsearch/issues/122503 # Examples: # From aa5ee2c433362446a1c06e2cc0d5af2070cb33a4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 14 Feb 2025 00:50:32 +1100 Subject: [PATCH 64/93] Mute org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 9.1.0]} #122504 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 2c4fc4d421cc..71bad1bb33bb 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -428,6 +428,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} issue: https://github.com/elastic/elasticsearch/issues/122503 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 9.1.0]} + issue: https://github.com/elastic/elasticsearch/issues/122504 # Examples: # From 41be8b940f6ea36eda5c0c8952597b4599585345 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 14 Feb 2025 00:50:40 +1100 Subject: [PATCH 65/93] Mute org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} #122505 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 71bad1bb33bb..e87e4cc83e96 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -431,6 +431,9 @@ tests: - class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 9.1.0]} issue: https://github.com/elastic/elasticsearch/issues/122504 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} + issue: https://github.com/elastic/elasticsearch/issues/122505 # Examples: # From cd74d18c5e81a4d870e3dde6a2c5b60959a19fcb Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Thu, 13 Feb 2025 16:18:42 +0200 Subject: [PATCH 66/93] Add missing transport versions (#122506) --- muted-tests.yml | 45 ------------------- .../org/elasticsearch/TransportVersions.java | 2 + .../support/IndexComponentSelector.java | 4 +- 3 files changed, 5 insertions(+), 46 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index e87e4cc83e96..35e010618f37 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -389,51 +389,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122377 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/122378 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testClosedIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122481 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testRestoreIndex {p0=[9.1.0, 9.1.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122482 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testRestoreIndex {p0=[9.1.0, 8.19.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122483 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122484 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} - issue: https://github.com/elastic/elasticsearch/issues/122487 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122488 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122489 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testRestoreIndex {p0=[9.1.0, 9.1.0, 9.1.0]} - issue: https://github.com/elastic/elasticsearch/issues/122490 -- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase - method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} - issue: https://github.com/elastic/elasticsearch/issues/122495 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122500 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122501 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.1.0, 8.19.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122502 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]} - issue: https://github.com/elastic/elasticsearch/issues/122503 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 9.1.0]} - issue: https://github.com/elastic/elasticsearch/issues/122504 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]} - issue: https://github.com/elastic/elasticsearch/issues/122505 # Examples: # diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 1a4bb7fde5ef..e4c83dc50fb4 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -173,11 +173,13 @@ public class TransportVersions { public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_0_00); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00); public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_840_0_01); + public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_X = def(8_840_0_02); public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_0_00); public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01); public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02); public static final TransportVersion REMOVE_DESIRED_NODE_VERSION_90 = def(9_000_0_03); public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION_90 = def(9_000_0_04); + public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_9_0 = def(9_000_0_05); public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00); public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00); public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED = def(9_003_0_00); diff --git a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java index d41042f4c657..c2cc8d98cca3 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java @@ -74,7 +74,9 @@ public enum IndexComponentSelector implements Writeable { public static IndexComponentSelector read(StreamInput in) throws IOException { byte id = in.readByte(); - if (in.getTransportVersion().onOrAfter(TransportVersions.REMOVE_ALL_APPLICABLE_SELECTOR)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.REMOVE_ALL_APPLICABLE_SELECTOR) + || in.getTransportVersion().isPatchFrom(TransportVersions.REMOVE_ALL_APPLICABLE_SELECTOR_9_0) + || in.getTransportVersion().isPatchFrom(TransportVersions.REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_X)) { return getById(id); } else { // Legacy value ::*, converted to ::data From fda7fc71e68259f313d6fcfd6b43ca9fec97b983 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Thu, 13 Feb 2025 10:36:50 -0500 Subject: [PATCH 67/93] [Deprecation] Refine Transform Destination Index message (#122192) When we detect that a Transform writes to the index and the index is incompatible with the next version, change the message, detail, and URL to help the user take the necessary steps to migrate the destination index. --- .../deprecation/IndexDeprecationChecker.java | 84 ++++++--- .../IndexDeprecationCheckerTests.java | 163 +++++++++++++++--- 2 files changed, 202 insertions(+), 45 deletions(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index 844988c7a489..6bed9143175c 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.LegacyFormatNames; +import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -95,25 +96,39 @@ public class IndexDeprecationChecker implements ResourceDeprecationChecker { IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false) && isNotDataStreamIndex(indexMetadata, clusterState)) { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 9.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", - "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), - false, - meta(indexMetadata, indexToTransformIds) - ); + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "One or more Transforms write to this index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + Strings.format( + "This index was created in version [%s] and requires action before upgrading to 9.0. The following transforms are " + + "configured to write to this index: [%s]. Refer to the migration guide to learn more about how to prepare " + + "transforms destination indices for your upgrade.", + currentCompatibilityVersion.toReleaseVersion(), + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Old index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", + "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + false, + Map.of("reindex_required", true) + ); + } } return null; } - private Map meta(IndexMetadata indexMetadata, Map> indexToTransformIds) { - var transforms = indexToTransformIds.getOrDefault(indexMetadata.getIndex().getName(), List.of()); - if (transforms.isEmpty()) { - return Map.of("reindex_required", true); - } else { - return Map.of("reindex_required", true, "transform_ids", transforms); - } + private List transformIdsForIndex(IndexMetadata indexMetadata, Map> indexToTransformIds) { + return indexToTransformIds.getOrDefault(indexMetadata.getIndex().getName(), List.of()); } private DeprecationIssue ignoredOldIndicesCheck( @@ -124,16 +139,35 @@ public class IndexDeprecationChecker implements ResourceDeprecationChecker { IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true) && isNotDataStreamIndex(indexMetadata, clusterState)) { - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < 9.0 Has Been Ignored", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", - "This read-only index has version: " - + currentCompatibilityVersion.toReleaseVersion() - + " and will be supported as read-only in 9.0", - false, - meta(indexMetadata, indexToTransformIds) - ); + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + Strings.format( + "This index was created in version [%s] and will be supported as a read-only index in 9.0. The following " + + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to handle your transforms destination indices.", + currentCompatibilityVersion.toReleaseVersion(), + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "Old index with a compatibility version < 9.0 has been ignored", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", + "This read-only index has version: " + + currentCompatibilityVersion.toReleaseVersion() + + " and will be supported as read-only in 9.0", + false, + Map.of("reindex_required", true) + ); + } } return null; } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 296ae31cd9f2..44a7d4bf57bd 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -102,9 +102,14 @@ public class IndexDeprecationCheckerTests extends ESTestCase { .build(); var expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 9.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", - "This index has version: " + OLD_VERSION.toReleaseVersion(), + "One or more Transforms write to this index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and requires action before upgrading to 9.0. " + + "The following transforms are configured to write to this index: [test-transform]. Refer to the " + + "migration guide to learn more about how to prepare transforms destination indices for your upgrade.", false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform")) ); @@ -124,9 +129,14 @@ public class IndexDeprecationCheckerTests extends ESTestCase { .build(); var expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 9.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", - "This index has version: " + OLD_VERSION.toReleaseVersion(), + "One or more Transforms write to this index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and requires action before upgrading to 9.0. " + + "The following transforms are configured to write to this index: [test-transform1, test-transform2]. Refer to the " + + "migration guide to learn more about how to prepare transforms destination indices for your upgrade.", false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform1", "test-transform2")) ); @@ -150,9 +160,14 @@ public class IndexDeprecationCheckerTests extends ESTestCase { List.of( new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 9.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", - "This index has version: " + OLD_VERSION.toReleaseVersion(), + "One or more Transforms write to this index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and requires action before upgrading to 9.0. " + + "The following transforms are configured to write to this index: [test-transform1]. Refer to the " + + "migration guide to learn more about how to prepare transforms destination indices for your upgrade.", false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform1")) ) @@ -161,9 +176,14 @@ public class IndexDeprecationCheckerTests extends ESTestCase { List.of( new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 9.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", - "This index has version: " + OLD_VERSION.toReleaseVersion(), + "One or more Transforms write to this index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and requires action before upgrading to 9.0. " + + "The following transforms are configured to write to this index: [test-transform2]. Refer to the " + + "migration guide to learn more about how to prepare transforms destination indices for your upgrade.", false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform2")) ) @@ -256,20 +276,14 @@ public class IndexDeprecationCheckerTests extends ESTestCase { } public void testOldIndicesIgnoredWarningCheck() { - Settings.Builder settings = settings(OLD_VERSION).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); - IndexMetadata indexMetadata = IndexMetadata.builder("test") - .settings(settings) - .numberOfShards(1) - .numberOfReplicas(0) - .state(indexMetdataState) - .build(); + IndexMetadata indexMetadata = readonlyIndexMetadata("test", OLD_VERSION); ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder().put(indexMetadata, true)) .blocks(clusterBlocksForIndices(indexMetadata)) .build(); DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < 9.0 Has Been Ignored", + "Old index with a compatibility version < 9.0 has been ignored", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html", "This read-only index has version: " + OLD_VERSION.toReleaseVersion() + " and will be supported as read-only in 9.0", false, @@ -284,6 +298,115 @@ public class IndexDeprecationCheckerTests extends ESTestCase { assertEquals(List.of(expected), issuesByIndex.get("test")); } + private IndexMetadata readonlyIndexMetadata(String indexName, IndexVersion indexVersion) { + Settings.Builder settings = settings(indexVersion).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); + return IndexMetadata.builder(indexName).settings(settings).numberOfShards(1).numberOfReplicas(0).state(indexMetdataState).build(); + } + + public void testOldTransformIndicesIgnoredCheck() { + var checker = new IndexDeprecationChecker(indexNameExpressionResolver); + var indexMetadata = readonlyIndexMetadata("test", OLD_VERSION); + var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + var expected = new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and will be supported as a read-only index in 9.0. " + + "The following transforms are no longer able to write to this index: [test-transform]. Refer to the " + + "migration guide to learn more about how to handle your transforms destination indices.", + false, + Map.of("reindex_required", true, "transform_ids", List.of("test-transform")) + ); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform"))) + ); + assertEquals(singletonList(expected), issuesByIndex.get("test")); + } + + public void testOldIndicesIgnoredCheckWithMultipleTransforms() { + var indexMetadata = readonlyIndexMetadata("test", OLD_VERSION); + var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + var expected = new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and will be supported as a read-only index in 9.0. " + + "The following transforms are no longer able to write to this index: [test-transform1, test-transform2]. Refer to the " + + "migration guide to learn more about how to handle your transforms destination indices.", + false, + Map.of("reindex_required", true, "transform_ids", List.of("test-transform1", "test-transform2")) + ); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform1", "test-transform2"))) + ); + assertEquals(singletonList(expected), issuesByIndex.get("test")); + } + + public void testMultipleOldIndicesIgnoredCheckWithTransforms() { + var indexMetadata1 = readonlyIndexMetadata("test1", OLD_VERSION); + var indexMetadata2 = readonlyIndexMetadata("test2", OLD_VERSION); + var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata1, true).put(indexMetadata2, true)) + .blocks(clusterBlocksForIndices(indexMetadata1, indexMetadata2)) + .build(); + var expected = Map.of( + "test1", + List.of( + new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and will be supported as a read-only index in 9.0. " + + "The following transforms are no longer able to write to this index: [test-transform1]. Refer to the " + + "migration guide to learn more about how to handle your transforms destination indices.", + false, + Map.of("reindex_required", true, "transform_ids", List.of("test-transform1")) + ) + ), + "test2", + List.of( + new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 9.0", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and will be supported as a read-only index in 9.0. " + + "The following transforms are no longer able to write to this index: [test-transform2]. Refer to the " + + "migration guide to learn more about how to handle your transforms destination indices.", + false, + Map.of("reindex_required", true, "transform_ids", List.of("test-transform2")) + ) + ) + ); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test1", List.of("test-transform1"), "test2", List.of("test-transform2"))) + ); + assertEquals(expected, issuesByIndex); + } + public void testTranslogRetentionSettings() { Settings.Builder settings = settings(IndexVersion.current()); settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); From 1b452c719a66b2e5498c3bfc579dfa5e2e447992 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 13 Feb 2025 17:06:05 +0100 Subject: [PATCH 68/93] Logsdb and source only snapshots. (#122199) Addresses a few issues with logsdb and source only snapshots: * Avoid initializing index sorting, because sort fields will not have doc values. * Also disable doc value skippers when doc values get disabled. * As part of source only validation figure out what the nested parent field is. Also added a few more tests that snapshot and restore logsdb data streams. --- docs/changelog/122199.yaml | 5 + .../elasticsearch/common/lucene/Lucene.java | 14 + .../org/elasticsearch/index/IndexService.java | 3 +- .../sourceonly/SourceOnlySnapshot.java | 3 +- .../xpack/logsdb/LogsdbSnapshotRestoreIT.java | 372 ++++++++++++++++++ 5 files changed, 395 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/122199.yaml create mode 100644 x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java diff --git a/docs/changelog/122199.yaml b/docs/changelog/122199.yaml new file mode 100644 index 000000000000..172ae900bdab --- /dev/null +++ b/docs/changelog/122199.yaml @@ -0,0 +1,5 @@ +pr: 122199 +summary: Fix issues that prevents using search only snapshots for indices that use index sorting. This is includes Logsdb and time series indices. +area: Logs +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 073000979918..2aa87d808fc9 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -20,6 +20,8 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FilterCodecReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.FilterLeafReader; @@ -190,7 +192,18 @@ public class Lucene { throw new IllegalStateException("no commit found in the directory"); } } + // Need to figure out what the parent field is that, so that validation in IndexWriter doesn't fail + // if no parent field is configured, but FieldInfo says there is a parent field. + String parentField = null; final IndexCommit cp = getIndexCommit(si, directory); + try (var reader = DirectoryReader.open(cp)) { + var topLevelFieldInfos = FieldInfos.getMergedFieldInfos(reader); + for (FieldInfo fieldInfo : topLevelFieldInfos) { + if (fieldInfo.isParentField()) { + parentField = fieldInfo.getName(); + } + } + } try ( IndexWriter writer = new IndexWriter( directory, @@ -198,6 +211,7 @@ public class Lucene { .setIndexCommit(cp) .setCommitOnClose(false) .setOpenMode(IndexWriterConfig.OpenMode.APPEND) + .setParentField(parentField) ) ) { // do nothing and close this will kick off IndexFileDeleter which will remove all pending files diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 5512dffdda53..baba9e94db7a 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -232,7 +232,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust mapperMetrics ); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService); - if (indexSettings.getIndexSortConfig().hasIndexSort()) { + boolean sourceOnly = Boolean.parseBoolean(indexSettings.getSettings().get("index.source_only")); + if (indexSettings.getIndexSortConfig().hasIndexSort() && sourceOnly == false) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. // The sort order is validated right after the merge of the mapping later in the process. this.indexSortSupplier = () -> indexSettings.getIndexSortConfig() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java index c76af6b0cfa0..731ab1500141 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java @@ -9,6 +9,7 @@ package org.elasticsearch.snapshots.sourceonly; import org.apache.lucene.codecs.Codec; import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -252,7 +253,7 @@ public class SourceOnlySnapshot { false, IndexOptions.NONE, DocValuesType.NONE, - fieldInfo.docValuesSkipIndexType(), + DocValuesSkipIndexType.NONE, -1, fieldInfo.attributes(), 0, diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java new file mode 100644 index 000000000000..0b57d0ed8c4f --- /dev/null +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java @@ -0,0 +1,372 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentType; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.net.InetAddress; +import java.time.Instant; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasSize; + +public class LogsdbSnapshotRestoreIT extends ESRestTestCase { + + private static TemporaryFolder repoDirectory = new TemporaryFolder(); + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("path.repo", () -> getRepoPath()) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + // TODO: remove when initializing / serializing default SourceFieldMapper instance have been fixed: + // (SFM's mode attribute often gets initialized, even when mode attribute isn't set) + .jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper") + .jvmArg("-da:org.elasticsearch.index.mapper.MapperService") + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); + + static final String LOGS_TEMPLATE = """ + { + "index_patterns": [ "logs-*-*" ], + "data_stream": {}, + "priority": 1000, + "template": { + "settings": { + "index": { + "mapping": { + "source":{ + "mode": "{{source_mode}}" + } + } + } + }, + "mappings": { + "properties": { + "@timestamp" : { + "type": "date" + }, + "host": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "pid": { + "type": "integer" + }, + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "ip_address": { + "type": "ip" + }, + "my_object_array": { + "type": "{{array_type}}" + } + } + } + } + }"""; + + static final String DOC_TEMPLATE = """ + { + "@timestamp": "%s", + "host": { "name": "%s"}, + "pid": %d, + "method": "%s", + "message": "%s", + "ip_address": "%s", + "memory_usage_bytes": "%d", + "my_object_array": [ + { + "field_1": "a", + "field_2": "b" + }, + { + "field_1": "c", + "field_2": "d" + } + ] + } + """; + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testSnapshotRestore() throws Exception { + snapshotAndRestore("synthetic", "object", false); + } + + public void testSnapshotRestoreWithSourceOnlyRepository() throws Exception { + snapshotAndFail("object"); + } + + public void testSnapshotRestoreNested() throws Exception { + snapshotAndRestore("synthetic", "nested", false); + } + + public void testSnapshotRestoreNestedWithSourceOnlyRepository() throws Exception { + snapshotAndFail("nested"); + } + + public void testSnapshotRestoreStoredSource() throws Exception { + snapshotAndRestore("stored", "object", false); + } + + public void testSnapshotRestoreStoredSourceWithSourceOnlyRepository() throws Exception { + snapshotAndRestore("stored", "object", true); + } + + public void testSnapshotRestoreStoredSourceNested() throws Exception { + snapshotAndRestore("stored", "nested", false); + } + + public void testSnapshotRestoreStoredSourceNestedWithSourceOnlyRepository() throws Exception { + snapshotAndRestore("stored", "nested", true); + } + + @After + public void cleanup() throws Exception { + deleteSnapshot("my-repository", "my-snapshot", true); + deleteRepository("my-repository"); + deleteDataStream("logs-my-test"); + } + + static void snapshotAndRestore(String sourceMode, String arrayType, boolean sourceOnly) throws IOException { + String dataStreamName = "logs-my-test"; + String repositoryName = "my-repository"; + if (sourceOnly) { + var repositorySettings = Settings.builder().put("delegate_type", "fs").put("location", getRepoPath()).build(); + registerRepository(repositoryName, "source", true, repositorySettings); + } else { + var repositorySettings = Settings.builder().put("location", getRepoPath()).build(); + registerRepository(repositoryName, FsRepository.TYPE, true, repositorySettings); + } + + putTemplate("my-template", LOGS_TEMPLATE.replace("{{source_mode}}", sourceMode).replace("{{array_type}}", arrayType)); + String[] docs = new String[100]; + for (int i = 0; i < 100; i++) { + docs[i] = document( + Instant.now(), + String.format(Locale.ROOT, "host-%03d", i), + randomNonNegativeInt(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(32), + randomIp(randomBoolean()), + randomLongBetween(1_000_000L, 2_000_000L) + ); + indexDocument(dataStreamName, docs[i]); + } + refresh(dataStreamName); + assertDocCount(client(), dataStreamName, 100); + assertSource(dataStreamName, docs); + assertDataStream(dataStreamName, sourceMode); + + String snapshotName = "my-snapshot"; + var snapshotResponse = performSnapshot(repositoryName, dataStreamName, snapshotName, true); + assertOK(snapshotResponse); + var snapshotResponseBody = entityAsMap(snapshotResponse); + Map snapshotItem = (Map) snapshotResponseBody.get("snapshot"); + List failures = (List) snapshotItem.get("failures"); + assertThat(failures, empty()); + deleteDataStream(dataStreamName); + assertDocCount(dataStreamName, 0); + + restoreSnapshot(repositoryName, snapshotName, true); + assertDataStream(dataStreamName, sourceMode); + assertDocCount(dataStreamName, 100); + assertSource(dataStreamName, docs); + } + + static void snapshotAndFail(String arrayType) throws IOException { + String dataStreamName = "logs-my-test"; + String repositoryName = "my-repository"; + var repositorySettings = Settings.builder().put("delegate_type", "fs").put("location", getRepoPath()).build(); + registerRepository(repositoryName, "source", true, repositorySettings); + + putTemplate("my-template", LOGS_TEMPLATE.replace("{{source_mode}}", "synthetic").replace("{{array_type}}", arrayType)); + for (int i = 0; i < 100; i++) { + indexDocument( + dataStreamName, + document( + Instant.now(), + randomAlphaOfLength(10), + randomNonNegativeLong(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(32), + randomIp(randomBoolean()), + randomIntBetween(1_000_000, 2_000_000) + ) + ); + } + refresh(dataStreamName); + assertDocCount(client(), dataStreamName, 100); + assertDataStream(dataStreamName, "synthetic"); + + String snapshotName = "my-snapshot"; + var snapshotResponse = performSnapshot(repositoryName, dataStreamName, snapshotName, true); + assertOK(snapshotResponse); + var snapshotResponseBody = entityAsMap(snapshotResponse); + Map snapshotItem = (Map) snapshotResponseBody.get("snapshot"); + List failures = (List) snapshotItem.get("failures"); + assertThat(failures, hasSize(1)); + Map failure = (Map) failures.get(0); + assertThat( + (String) failure.get("reason"), + containsString( + "Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source" + ) + ); + } + + static void deleteDataStream(String dataStreamName) throws IOException { + assertOK(client().performRequest(new Request("DELETE", "/_data_stream/" + dataStreamName))); + } + + static void putTemplate(String templateName, String template) throws IOException { + final Request request = new Request("PUT", "/_index_template/" + templateName); + request.setJsonEntity(template); + assertOK(client().performRequest(request)); + } + + static void indexDocument(String indexOrtDataStream, String doc) throws IOException { + final Request request = new Request("POST", "/" + indexOrtDataStream + "/_doc?refresh=true"); + request.setJsonEntity(doc); + final Response response = client().performRequest(request); + assertOK(response); + assertThat(entityAsMap(response).get("result"), equalTo("created")); + } + + static String document( + final Instant timestamp, + final String hostname, + long pid, + final String method, + final String message, + final InetAddress ipAddress, + long memoryUsageBytes + ) { + return String.format( + Locale.ROOT, + DOC_TEMPLATE, + DateFormatter.forPattern(FormatNames.DATE_TIME.getName()).format(timestamp), + hostname, + pid, + method, + message, + InetAddresses.toAddrString(ipAddress), + memoryUsageBytes + ); + } + + static Response performSnapshot(String repository, String dataStreamName, String snapshot, boolean waitForCompletion) + throws IOException { + final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); + request.setJsonEntity(""" + { + "indices": "{{dataStreamName}}" + } + """.replace("{{dataStreamName}}", dataStreamName)); + request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); + + return client().performRequest(request); + } + + static void assertDataStream(String dataStreamName, final String sourceMode) throws IOException { + String indexName = getWriteBackingIndex(dataStreamName, 0); + var flatSettings = (Map) ((Map) getIndexSettings(indexName).get(indexName)).get("settings"); + assertThat(flatSettings, hasEntry("index.mode", "logsdb")); + assertThat(flatSettings, hasEntry("index.mapping.source.mode", sourceMode)); + } + + static String getWriteBackingIndex(String dataStreamName, int backingIndex) throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client().performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List backingIndices = (List) dataStream.get("indices"); + return (String) ((Map) backingIndices.get(backingIndex)).get("index_name"); + } + + static void assertDocCount(String indexName, long docCount) throws IOException { + Request countReq = new Request("GET", "/" + indexName + "/_count"); + countReq.addParameter("ignore_unavailable", "true"); + ObjectPath resp = ObjectPath.createFromResponse(client().performRequest(countReq)); + assertEquals( + "expected " + docCount + " documents but it was a different number", + docCount, + Long.parseLong(resp.evaluate("count").toString()) + ); + } + + static void assertSource(String indexName, String[] docs) throws IOException { + Request searchReq = new Request("GET", "/" + indexName + "/_search"); + searchReq.addParameter("size", String.valueOf(docs.length)); + var response = client().performRequest(searchReq); + assertOK(response); + var responseBody = entityAsMap(response); + List hits = (List) ((Map) responseBody.get("hits")).get("hits"); + assertThat(hits, hasSize(docs.length)); + for (Object hit : hits) { + Map actualSource = (Map) ((Map) hit).get("_source"); + String actualHost = (String) ((Map) actualSource.get("host")).get("name"); + Map expectedSource = null; + for (String doc : docs) { + expectedSource = XContentHelper.convertToMap(XContentType.JSON.xContent(), doc, false); + String expectedHost = (String) ((Map) expectedSource.get("host")).get("name"); + if (expectedHost.equals(actualHost)) { + break; + } + } + + assertMap(actualSource, matchesMap(expectedSource)); + } + } + + @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File") + private static String getRepoPath() { + return repoDirectory.getRoot().getPath(); + } + +} From 33a2bc9a31859f363111cbe50f0c9c998ea0cbc6 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 13 Feb 2025 17:09:09 +0100 Subject: [PATCH 69/93] Deduplicate IngestStats and IngestStats.Stats identity records when deserializing (#122496) This commit makes sure we reuse the existing static instance when deserializing to avoid excessive heap usage. --- docs/changelog/122496.yaml | 5 +++ .../org/elasticsearch/ingest/IngestStats.java | 31 +++++++++++++------ .../ingest/IngestStatsTests.java | 6 ++++ 3 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 docs/changelog/122496.yaml diff --git a/docs/changelog/122496.yaml b/docs/changelog/122496.yaml new file mode 100644 index 000000000000..37ce70977112 --- /dev/null +++ b/docs/changelog/122496.yaml @@ -0,0 +1,5 @@ +pr: 122496 +summary: Deduplicate `IngestStats` and `IngestStats.Stats` identity records when deserializing +area: Ingest Node +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java index c75cd3a022cb..da1b99f4f075 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java @@ -57,14 +57,17 @@ public record IngestStats(Stats totalStats, List pipelineStats, Ma * Read from a stream. */ public static IngestStats read(StreamInput in) throws IOException { - var stats = new Stats(in); + var stats = readStats(in); var size = in.readVInt(); + if (stats == Stats.IDENTITY && size == 0) { + return IDENTITY; + } var pipelineStats = new ArrayList(size); var processorStats = Maps.>newMapWithExpectedSize(size); for (var i = 0; i < size; i++) { var pipelineId = in.readString(); - var pipelineStat = new Stats(in); + var pipelineStat = readStats(in); var byteStat = in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? new ByteStats(in) : new ByteStats(0, 0); pipelineStats.add(new PipelineStat(pipelineId, pipelineStat, byteStat)); int processorsSize = in.readVInt(); @@ -72,7 +75,7 @@ public record IngestStats(Stats totalStats, List pipelineStats, Ma for (var j = 0; j < processorsSize; j++) { var processorName = in.readString(); var processorType = in.readString(); - var processorStat = new Stats(in); + var processorStat = readStats(in); processorStatsPerPipeline.add(new ProcessorStat(processorName, processorType, processorStat)); } processorStats.put(pipelineId, Collections.unmodifiableList(processorStatsPerPipeline)); @@ -167,6 +170,21 @@ public record IngestStats(Stats totalStats, List pipelineStats, Ma return totalsPerPipelineProcessor; } + /** + * Read {@link Stats} from a stream. + */ + private static Stats readStats(StreamInput in) throws IOException { + long ingestCount = in.readVLong(); + long ingestTimeInMillis = in.readVLong(); + long ingestCurrent = in.readVLong(); + long ingestFailedCount = in.readVLong(); + if (ingestCount == 0 && ingestTimeInMillis == 0 && ingestCurrent == 0 && ingestFailedCount == 0) { + return Stats.IDENTITY; + } else { + return new Stats(ingestCount, ingestTimeInMillis, ingestCurrent, ingestFailedCount); + } + } + public record Stats(long ingestCount, long ingestTimeInMillis, long ingestCurrent, long ingestFailedCount) implements Writeable, @@ -174,13 +192,6 @@ public record IngestStats(Stats totalStats, List pipelineStats, Ma public static final Stats IDENTITY = new Stats(0, 0, 0, 0); - /** - * Read from a stream. - */ - public Stats(StreamInput in) throws IOException { - this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong()); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(ingestCount); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java index dc3fb2a473f4..d9189c56e668 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.sameInstance; public class IngestStatsTests extends ESTestCase { @@ -31,6 +32,11 @@ public class IngestStatsTests extends ESTestCase { assertIngestStats(ingestStats, serializedStats); } + public void testIdentitySerialization() throws IOException { + IngestStats serializedStats = serialize(IngestStats.IDENTITY); + assertThat(serializedStats, sameInstance(IngestStats.IDENTITY)); + } + public void testStatsMerge() { var first = randomStats(); var second = randomStats(); From 54613656c395c43f73db7659f5bb28731b8e4743 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 13 Feb 2025 17:57:18 +0100 Subject: [PATCH 70/93] Remove another outdated assertion from SearchQueryThenFetchAsyncActionTests (#122086) Same reasoning as in 2b410c44eb8fc684d3986247a49f9696b44f2d4c, with the now weaker ordering guarantees from optimizations this assertion stopped being reliable. --- muted-tests.yml | 3 --- .../search/SearchQueryThenFetchAsyncActionTests.java | 8 ++------ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 35e010618f37..c90c6b60b128 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -343,9 +343,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/121625 - class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT issue: https://github.com/elastic/elasticsearch/issues/121967 -- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests - method: testBottomFieldSort - issue: https://github.com/elastic/elasticsearch/issues/121503 - class: org.elasticsearch.xpack.application.CohereServiceUpgradeIT issue: https://github.com/elastic/elasticsearch/issues/121537 - class: org.elasticsearch.xpack.restart.FullClusterRestartIT diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java index be693a2d7d29..227239481a55 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -219,12 +219,8 @@ public class SearchQueryThenFetchAsyncActionTests extends ESTestCase { if (withScroll) { assertFalse(canReturnNullResponse.get()); assertThat(numWithTopDocs.get(), equalTo(0)); - } else { - if (withCollapse) { - assertThat(numWithTopDocs.get(), equalTo(0)); - } else { - assertThat(numWithTopDocs.get(), greaterThanOrEqualTo(1)); - } + } else if (withCollapse) { + assertThat(numWithTopDocs.get(), equalTo(0)); } SearchPhaseController.ReducedQueryPhase phase = action.results.reduce(); assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1)); From b117651322e7fd318e4693799672fdcf2f7b0076 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 13 Feb 2025 18:17:39 +0100 Subject: [PATCH 71/93] Unmute org.elasticsearch.search.CrossClusterSearchUnavailableClusterIT.testSearchSkipUnavailable (#122515) Long fixed, forgot to unmute. closes #121497 --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index c90c6b60b128..1810e98f6ca0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -332,9 +332,6 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only} issue: https://github.com/elastic/elasticsearch/issues/121412 -- class: org.elasticsearch.search.CrossClusterSearchUnavailableClusterIT - method: testSearchSkipUnavailable - issue: https://github.com/elastic/elasticsearch/issues/121497 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/cat/health/cat-health-no-timestamp-example} issue: https://github.com/elastic/elasticsearch/issues/121867 From 610722d53925b10e2ad0f47b3af5a72aa8ecd49c Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 13 Feb 2025 18:26:46 +0100 Subject: [PATCH 72/93] Handle search timeout in SuggestPhase (#122357) Whenever a search timeout is set to a search request, the timeout may be triggered by the suggest phase via exitable directory reader. In that case, the exception that is thrown by the timeout check needs to be handled, instead of returned back to the user. Instead of handling the timeout in each phase, this commit handles it as part of QueryPhase for both SuggestPhase and RescorePhase. For rescore phase, one integration test that is time dependent is also rewritten to remove the time dependency and moved from QueryRescorerIT to SearchTimeoutIT. Closes #122186 --- docs/changelog/122357.yaml | 6 + .../elasticsearch/search/SearchTimeoutIT.java | 241 +++++++++++++++++- .../search/functionscore/QueryRescorerIT.java | 17 -- .../search/query/QueryPhase.java | 27 +- .../search/rescore/RescorePhase.java | 10 +- .../search/suggest/SuggestPhase.java | 1 - .../search/query/QueryPhaseTimeoutTests.java | 127 +++++++++ 7 files changed, 394 insertions(+), 35 deletions(-) create mode 100644 docs/changelog/122357.yaml diff --git a/docs/changelog/122357.yaml b/docs/changelog/122357.yaml new file mode 100644 index 000000000000..7648002c9356 --- /dev/null +++ b/docs/changelog/122357.yaml @@ -0,0 +1,6 @@ +pr: 122357 +summary: Handle search timeout in `SuggestPhase` +area: Search +type: bug +issues: + - 122186 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java index f45c37715f7c..f79321ef8d0d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Query; @@ -22,8 +23,10 @@ import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -33,12 +36,23 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.internal.ContextIndexSearcher; +import org.elasticsearch.search.rescore.RescoreContext; +import org.elasticsearch.search.rescore.Rescorer; +import org.elasticsearch.search.rescore.RescorerBuilder; +import org.elasticsearch.search.suggest.SortBy; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.Suggester; +import org.elasticsearch.search.suggest.SuggestionSearchContext; +import org.elasticsearch.search.suggest.term.TermSuggestion; +import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.xcontent.XContentBuilder; @@ -58,7 +72,7 @@ public class SearchTimeoutIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(BulkScorerTimeoutQueryPlugin.class); + return Collections.singleton(SearchTimeoutPlugin.class); } @Override @@ -72,6 +86,9 @@ public class SearchTimeoutIT extends ESIntegTestCase { indexRandom(true, "test", randomIntBetween(20, 50)); } + /** + * Test the scenario where the query times out before starting to collect documents, verify that partial hits are not returned + */ public void testTopHitsTimeoutBeforeCollecting() { // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.SECONDS)) @@ -88,6 +105,9 @@ public class SearchTimeoutIT extends ESIntegTestCase { }); } + /** + * Test the scenario where the query times out while collecting documents, verify that partial hits results are returned + */ public void testTopHitsTimeoutWhileCollecting() { // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.SECONDS)) @@ -103,6 +123,9 @@ public class SearchTimeoutIT extends ESIntegTestCase { }); } + /** + * Test the scenario where the query times out before starting to collect documents, verify that partial aggs results are not returned + */ public void testAggsTimeoutBeforeCollecting() { SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setSize(0) // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set @@ -123,6 +146,9 @@ public class SearchTimeoutIT extends ESIntegTestCase { }); } + /** + * Test the scenario where the query times out while collecting documents, verify that partial aggs results are returned + */ public void testAggsTimeoutWhileCollecting() { SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setSize(0) // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set @@ -145,6 +171,56 @@ public class SearchTimeoutIT extends ESIntegTestCase { }); } + /** + * Test the scenario where the suggest phase (part of the query phase) times out, yet there are results + * available coming from executing the query and aggs on each shard. + */ + public void testSuggestTimeoutWithPartialResults() { + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.setGlobalText("text"); + TimeoutSuggestionBuilder timeoutSuggestionBuilder = new TimeoutSuggestionBuilder(); + suggestBuilder.addSuggestion("suggest", timeoutSuggestionBuilder); + SearchRequestBuilder searchRequestBuilder = prepareSearch("test").suggest(suggestBuilder) + .addAggregation(new TermsAggregationBuilder("terms").field("field.keyword")); + ElasticsearchAssertions.assertResponse(searchRequestBuilder, searchResponse -> { + assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertEquals(0, searchResponse.getShardFailures().length); + assertEquals(0, searchResponse.getFailedShards()); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); + assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); + assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L)); + assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); + StringTerms terms = searchResponse.getAggregations().get("terms"); + assertEquals(1, terms.getBuckets().size()); + StringTerms.Bucket bucket = terms.getBuckets().get(0); + assertEquals("value", bucket.getKeyAsString()); + assertThat(bucket.getDocCount(), greaterThan(0L)); + }); + } + + /** + * Test the scenario where the rescore phase (part of the query phase) times out, yet there are results + * available coming from executing the query and aggs on each shard. + */ + public void testRescoreTimeoutWithPartialResults() { + SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setRescorer(new TimeoutRescorerBuilder()) + .addAggregation(new TermsAggregationBuilder("terms").field("field.keyword")); + ElasticsearchAssertions.assertResponse(searchRequestBuilder, searchResponse -> { + assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertEquals(0, searchResponse.getShardFailures().length); + assertEquals(0, searchResponse.getFailedShards()); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); + assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); + assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L)); + assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); + StringTerms terms = searchResponse.getAggregations().get("terms"); + assertEquals(1, terms.getBuckets().size()); + StringTerms.Bucket bucket = terms.getBuckets().get(0); + assertEquals("value", bucket.getKeyAsString()); + assertThat(bucket.getDocCount(), greaterThan(0L)); + }); + } + public void testPartialResultsIntolerantTimeoutBeforeCollecting() { ElasticsearchException ex = expectThrows( ElasticsearchException.class, @@ -171,13 +247,67 @@ public class SearchTimeoutIT extends ESIntegTestCase { assertEquals(429, ex.status().getStatus()); } - public static final class BulkScorerTimeoutQueryPlugin extends Plugin implements SearchPlugin { + public void testPartialResultsIntolerantTimeoutWhileSuggestingOnly() { + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.setGlobalText("text"); + TimeoutSuggestionBuilder timeoutSuggestionBuilder = new TimeoutSuggestionBuilder(); + suggestBuilder.addSuggestion("suggest", timeoutSuggestionBuilder); + ElasticsearchException ex = expectThrows( + ElasticsearchException.class, + prepareSearch("test").suggest(suggestBuilder).setAllowPartialSearchResults(false) // this line causes timeouts to report + // failures + ); + assertTrue(ex.toString().contains("Time exceeded")); + assertEquals(429, ex.status().getStatus()); + } + + public void testPartialResultsIntolerantTimeoutWhileSuggesting() { + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.setGlobalText("text"); + TimeoutSuggestionBuilder timeoutSuggestionBuilder = new TimeoutSuggestionBuilder(); + suggestBuilder.addSuggestion("suggest", timeoutSuggestionBuilder); + ElasticsearchException ex = expectThrows( + ElasticsearchException.class, + prepareSearch("test").setQuery(new TermQueryBuilder("field", "value")) + .suggest(suggestBuilder) + .setAllowPartialSearchResults(false) // this line causes timeouts to report failures + ); + assertTrue(ex.toString().contains("Time exceeded")); + assertEquals(429, ex.status().getStatus()); + } + + public void testPartialResultsIntolerantTimeoutWhileRescoring() { + ElasticsearchException ex = expectThrows( + ElasticsearchException.class, + prepareSearch("test").setQuery(new TermQueryBuilder("field", "value")) + .setRescorer(new TimeoutRescorerBuilder()) + .setAllowPartialSearchResults(false) // this line causes timeouts to report failures + ); + assertTrue(ex.toString().contains("Time exceeded")); + assertEquals(429, ex.status().getStatus()); + } + + public static final class SearchTimeoutPlugin extends Plugin implements SearchPlugin { @Override public List> getQueries() { return Collections.singletonList(new QuerySpec("timeout", BulkScorerTimeoutQuery::new, parser -> { throw new UnsupportedOperationException(); })); } + + @Override + public List> getSuggesters() { + return Collections.singletonList(new SuggesterSpec<>("timeout", TimeoutSuggestionBuilder::new, parser -> { + throw new UnsupportedOperationException(); + }, TermSuggestion::new)); + } + + @Override + public List> getRescorers() { + return Collections.singletonList(new RescorerSpec<>("timeout", TimeoutRescorerBuilder::new, parser -> { + throw new UnsupportedOperationException(); + })); + } } /** @@ -315,4 +445,111 @@ public class SearchTimeoutIT extends ESIntegTestCase { return null; } } + + /** + * Suggestion builder that triggers a timeout as part of its execution + */ + private static final class TimeoutSuggestionBuilder extends TermSuggestionBuilder { + TimeoutSuggestionBuilder() { + super("field"); + } + + TimeoutSuggestionBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return "timeout"; + } + + @Override + public SuggestionSearchContext.SuggestionContext build(SearchExecutionContext context) { + return new TimeoutSuggestionContext(new TimeoutSuggester((ContextIndexSearcher) context.searcher()), context); + } + } + + private static final class TimeoutSuggester extends Suggester { + private final ContextIndexSearcher contextIndexSearcher; + + TimeoutSuggester(ContextIndexSearcher contextIndexSearcher) { + this.contextIndexSearcher = contextIndexSearcher; + } + + @Override + protected TermSuggestion innerExecute( + String name, + TimeoutSuggestionContext suggestion, + IndexSearcher searcher, + CharsRefBuilder spare + ) { + contextIndexSearcher.throwTimeExceededException(); + assert false; + return new TermSuggestion(name, suggestion.getSize(), SortBy.SCORE); + } + + @Override + protected TermSuggestion emptySuggestion(String name, TimeoutSuggestionContext suggestion, CharsRefBuilder spare) { + return new TermSuggestion(name, suggestion.getSize(), SortBy.SCORE); + } + } + + private static final class TimeoutSuggestionContext extends SuggestionSearchContext.SuggestionContext { + TimeoutSuggestionContext(Suggester suggester, SearchExecutionContext searchExecutionContext) { + super(suggester, searchExecutionContext); + } + } + + private static final class TimeoutRescorerBuilder extends RescorerBuilder { + TimeoutRescorerBuilder() { + super(); + } + + TimeoutRescorerBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + protected void doWriteTo(StreamOutput out) {} + + @Override + protected void doXContent(XContentBuilder builder, Params params) {} + + @Override + protected RescoreContext innerBuildContext(int windowSize, SearchExecutionContext context) throws IOException { + return new RescoreContext(10, new Rescorer() { + @Override + public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext rescoreContext) { + ((ContextIndexSearcher) context.searcher()).throwTimeExceededException(); + assert false; + return null; + } + + @Override + public Explanation explain( + int topLevelDocId, + IndexSearcher searcher, + RescoreContext rescoreContext, + Explanation sourceExplanation + ) { + throw new UnsupportedOperationException(); + } + }); + } + + @Override + public String getWriteableName() { + return "timeout"; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return null; + } + + @Override + public RescorerBuilder rewrite(QueryRewriteContext ctx) { + return this; + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index fbdcfe26d28e..0ba4c13c352c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.lucene.search.function.LeafScoreFunction; import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -994,22 +993,6 @@ public class QueryRescorerIT extends ESIntegTestCase { }); } - public void testRescoreWithTimeout() throws Exception { - // no dummy docs since merges can change scores while we run queries. - int numDocs = indexRandomNumbers("whitespace", -1, false); - - String intToEnglish = English.intToEnglish(between(0, numDocs - 1)); - String query = intToEnglish.split(" ")[0]; - assertResponse( - prepareSearch().setSearchType(SearchType.QUERY_THEN_FETCH) - .setQuery(QueryBuilders.matchQuery("field1", query).operator(Operator.OR)) - .setSize(10) - .addRescorer(new QueryRescorerBuilder(functionScoreQuery(new TestTimedScoreFunctionBuilder())).windowSize(100)) - .setTimeout(TimeValue.timeValueMillis(10)), - r -> assertTrue(r.isTimedOut()) - ); - } - @Override protected Collection> nodePlugins() { return List.of(TestTimedQueryPlugin.class); diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 3036a295d459..8ad52c4f9bb5 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -126,7 +126,15 @@ public class QueryPhase { static void executeQuery(SearchContext searchContext) throws QueryPhaseExecutionException { if (searchContext.hasOnlySuggest()) { - SuggestPhase.execute(searchContext); + try { + SuggestPhase.execute(searchContext); + } catch (ContextIndexSearcher.TimeExceededException timeExceededException) { + SearchTimeoutException.handleTimeout( + searchContext.request().allowPartialSearchResults(), + searchContext.shardTarget(), + searchContext.queryResult() + ); + } searchContext.queryResult().topDocs(new TopDocsAndMaxScore(Lucene.EMPTY_TOP_DOCS, Float.NaN), new DocValueFormat[0]); return; } @@ -142,11 +150,18 @@ public class QueryPhase { addCollectorsAndSearch(searchContext); - RescorePhase.execute(searchContext); - SuggestPhase.execute(searchContext); - - if (searchContext.getProfilers() != null) { - searchContext.queryResult().profileResults(searchContext.getProfilers().buildQueryPhaseResults()); + try { + RescorePhase.execute(searchContext); + SuggestPhase.execute(searchContext); + if (searchContext.getProfilers() != null) { + searchContext.queryResult().profileResults(searchContext.getProfilers().buildQueryPhaseResults()); + } + } catch (ContextIndexSearcher.TimeExceededException timeExceededException) { + SearchTimeoutException.handleTimeout( + searchContext.request().allowPartialSearchResults(), + searchContext.shardTarget(), + searchContext.queryResult() + ); } } diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java index f8b348b383f0..7223da3c6101 100644 --- a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java +++ b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java @@ -18,9 +18,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.util.Maps; import org.elasticsearch.lucene.grouping.TopFieldGroups; -import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.query.SearchTimeoutException; import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.search.sort.SortAndFormats; @@ -72,7 +70,7 @@ public class RescorePhase { assert topDocsSortedByScore(topDocs) : "topdocs should be sorted after rescore"; ctx.setCancellationChecker(null); } - /** + /* * Since rescorers are building top docs with score only, we must reconstruct the {@link TopFieldGroups} * or {@link TopFieldDocs} using their original version before rescoring. */ @@ -86,12 +84,6 @@ public class RescorePhase { .topDocs(new TopDocsAndMaxScore(topDocs, topDocs.scoreDocs[0].score), context.queryResult().sortValueFormats()); } catch (IOException e) { throw new ElasticsearchException("Rescore Phase Failed", e); - } catch (ContextIndexSearcher.TimeExceededException e) { - SearchTimeoutException.handleTimeout( - context.request().allowPartialSearchResults(), - context.shardTarget(), - context.queryResult() - ); } } diff --git a/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java index d63e0717ca7a..272855bacd54 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java @@ -56,5 +56,4 @@ public class SuggestPhase { throw new ElasticsearchException("I/O exception during suggest phase", e); } } - } diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java index b417f7adbc8b..6b38e05bdc4e 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.Scorable; @@ -38,14 +39,29 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.CharsRefBuilder; +import org.elasticsearch.action.OriginalIndices; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchShardTask; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.text.Text; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.Suggester; +import org.elasticsearch.search.suggest.SuggestionSearchContext; import org.elasticsearch.test.TestSearchContext; +import org.hamcrest.Matchers; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -275,6 +291,117 @@ public class QueryPhaseTimeoutTests extends IndexShardTestCase { return context; } + public void testSuggestOnlyWithTimeout() throws Exception { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().suggest(new SuggestBuilder()); + try (SearchContext context = createSearchContextWithSuggestTimeout(searchSourceBuilder)) { + assertTrue(context.hasOnlySuggest()); + QueryPhase.execute(context); + assertTrue(context.queryResult().searchTimedOut()); + assertNull(context.queryResult().suggest()); + assertNotNull(context.queryResult().topDocs()); + assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value()); + } + } + + public void testSuggestAndQueryWithSuggestTimeout() throws Exception { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().suggest(new SuggestBuilder()).query(new MatchAllQueryBuilder()); + try (SearchContext context = createSearchContextWithSuggestTimeout(searchSourceBuilder)) { + context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); + assertFalse(context.hasOnlySuggest()); + QueryPhase.execute(context); + assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), Matchers.greaterThan(0L)); + assertTrue(context.queryResult().searchTimedOut()); + assertNull(context.queryResult().suggest()); + } + } + + private TestSearchContext createSearchContextWithSuggestTimeout(SearchSourceBuilder searchSourceBuilder) throws IOException { + ContextIndexSearcher contextIndexSearcher = newContextSearcher(reader); + SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); + suggestionSearchContext.addSuggestion("suggestion", new TestSuggestionContext(new TestSuggester(contextIndexSearcher), null)); + TestSearchContext context = new TestSearchContext(null, indexShard, contextIndexSearcher) { + @Override + public SuggestionSearchContext suggest() { + return suggestionSearchContext; + } + + @Override + public ShardSearchRequest request() { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.allowPartialSearchResults(true); + searchRequest.source(searchSourceBuilder); + return new ShardSearchRequest( + OriginalIndices.NONE, + searchRequest, + indexShard.shardId(), + 0, + 1, + AliasFilter.EMPTY, + 1F, + 0, + null + ); + } + }; + context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); + return context; + } + + private static final class TestSuggester extends Suggester { + private final ContextIndexSearcher contextIndexSearcher; + + TestSuggester(ContextIndexSearcher contextIndexSearcher) { + this.contextIndexSearcher = contextIndexSearcher; + } + + @Override + protected TestSuggestion innerExecute( + String name, + TestSuggestionContext suggestion, + IndexSearcher searcher, + CharsRefBuilder spare + ) { + contextIndexSearcher.throwTimeExceededException(); + throw new AssertionError("should have thrown TimeExceededException"); + } + + @Override + protected TestSuggestion emptySuggestion(String name, TestSuggestionContext suggestion, CharsRefBuilder spare) { + return new TestSuggestion(); + } + } + + private static final class TestSuggestionContext extends SuggestionSearchContext.SuggestionContext { + TestSuggestionContext(Suggester suggester, SearchExecutionContext searchExecutionContext) { + super(suggester, searchExecutionContext); + } + } + + private static final class TestSuggestion extends Suggest.Suggestion< + Suggest.Suggestion.Entry> { + TestSuggestion() { + super("suggestion", 10); + } + + @Override + protected Entry newEntry(StreamInput in) { + return new TestSuggestionEntry(); + } + + @Override + public String getWriteableName() { + return "suggestion"; + } + } + + private static final class TestSuggestionEntry extends Suggest.Suggestion.Entry { + @Override + protected Option newOption(StreamInput in) { + return new Option(new Text("text"), 1f) { + }; + } + } + private static class Score extends Scorable { float score; From 6a57b70f16756b5369a5906f34eded51fd4eb9b6 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 13 Feb 2025 18:30:23 +0100 Subject: [PATCH 73/93] Remove Countable interface and dry up ShardsIterator (#121923) The size method is needed only in SearchShardIterator and PlainIterator. An interface is not really needed, we can just rather add the size method where needed. Also, there's a couple of methods in the ShardsIterator interface that are not needed that this commit removes: size, reset and remaining. --- .../action/search/SearchShardIterator.java | 4 +--- .../cluster/routing/ShardsIterator.java | 23 +------------------ .../elasticsearch/common/util/Countable.java | 14 ----------- .../common/util/PlainIterator.java | 3 +-- .../cluster/routing/RoutingTableTests.java | 13 +++++++---- 5 files changed, 11 insertions(+), 46 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/common/util/Countable.java diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java index 9cd635e9e2b0..bf020cbd309e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java @@ -12,7 +12,6 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardsIterator; -import org.elasticsearch.common.util.Countable; import org.elasticsearch.common.util.PlainIterator; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -29,7 +28,7 @@ import java.util.Objects; * the cluster alias. * @see OriginalIndices */ -public final class SearchShardIterator implements Comparable, Countable { +public final class SearchShardIterator implements Comparable { private final OriginalIndices originalIndices; private final String clusterAlias; @@ -171,7 +170,6 @@ public final class SearchShardIterator implements Comparable, Countable { - - /** - * Resets the iterator to its initial state. - */ - void reset(); - - /** - * The number of shard routing instances. - * - * @return number of shard routing instances in this iterator - */ - int size(); +public interface ShardsIterator extends Iterable { /** * The number of active shard routing instances @@ -41,13 +27,6 @@ public interface ShardsIterator extends Iterable, Countable { */ ShardRouting nextOrNull(); - /** - * Return the number of shards remaining in this {@link ShardsIterator} - * - * @return number of shard remaining - */ - int remaining(); - @Override int hashCode(); diff --git a/server/src/main/java/org/elasticsearch/common/util/Countable.java b/server/src/main/java/org/elasticsearch/common/util/Countable.java deleted file mode 100644 index 27ac9cd0c938..000000000000 --- a/server/src/main/java/org/elasticsearch/common/util/Countable.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.common.util; - -public interface Countable { - int size(); -} diff --git a/server/src/main/java/org/elasticsearch/common/util/PlainIterator.java b/server/src/main/java/org/elasticsearch/common/util/PlainIterator.java index f4e90e002ac9..b4a049b34a13 100644 --- a/server/src/main/java/org/elasticsearch/common/util/PlainIterator.java +++ b/server/src/main/java/org/elasticsearch/common/util/PlainIterator.java @@ -13,7 +13,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -public class PlainIterator implements Iterable, Countable { +public class PlainIterator implements Iterable { private final List elements; // Calls to nextOrNull might be performed on different threads in the transport actions so we need the volatile @@ -43,7 +43,6 @@ public class PlainIterator implements Iterable, Countable { } } - @Override public int size() { return elements.size(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index 27caa5b53c96..0001ad175f5c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -312,18 +312,21 @@ public class RoutingTableTests extends ESAllocationTestCase { } public void testAllShardsForMultipleIndices() { - assertThat(this.emptyRoutingTable.allShards(new String[0]).size(), is(0)); + assertThat(this.emptyRoutingTable.allShards(new String[0]).getShardRoutings().size(), is(0)); - assertThat(clusterState.routingTable().allShards(new String[] { TEST_INDEX_1 }).size(), is(this.shardsPerIndex)); + assertThat(clusterState.routingTable().allShards(new String[] { TEST_INDEX_1 }).getShardRoutings().size(), is(this.shardsPerIndex)); initPrimaries(); - assertThat(clusterState.routingTable().allShards(new String[] { TEST_INDEX_1 }).size(), is(this.shardsPerIndex)); + assertThat(clusterState.routingTable().allShards(new String[] { TEST_INDEX_1 }).getShardRoutings().size(), is(this.shardsPerIndex)); startInitializingShards(TEST_INDEX_1); - assertThat(clusterState.routingTable().allShards(new String[] { TEST_INDEX_1 }).size(), is(this.shardsPerIndex)); + assertThat(clusterState.routingTable().allShards(new String[] { TEST_INDEX_1 }).getShardRoutings().size(), is(this.shardsPerIndex)); startInitializingShards(TEST_INDEX_2); - assertThat(clusterState.routingTable().allShards(new String[] { TEST_INDEX_1, TEST_INDEX_2 }).size(), is(this.totalNumberOfShards)); + assertThat( + clusterState.routingTable().allShards(new String[] { TEST_INDEX_1, TEST_INDEX_2 }).getShardRoutings().size(), + is(this.totalNumberOfShards) + ); try { clusterState.routingTable().allShards(new String[] { TEST_INDEX_1, "not_exists" }); From 53150881e92b23ef92107de76f8564bb20f6d242 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 13 Feb 2025 09:33:04 -0800 Subject: [PATCH 74/93] Enable the use of nested field type with index.mode=time_series (#122224) This patch removes the check that fails requests that attempt to use fields of type: nested within indices with mode time_series. This patch also updates TimeSeriesIdFieldMapper#postParse to set the _id field on child documents once it's calculated. Closes #120874 --- docs/changelog/122224.yaml | 6 + rest-api-spec/build.gradle | 1 + .../test/tsdb/160_nested_fields.yml | 233 ++++++++++++++++++ .../rest-api-spec/test/tsdb/20_mapping.yml | 31 --- .../org/elasticsearch/index/IndexMode.java | 4 - .../index/mapper/DocumentParserContext.java | 8 +- .../index/mapper/MapperFeatures.java | 2 + .../index/mapper/TimeSeriesIdFieldMapper.java | 12 +- .../mapper/TsidExtractingIdFieldMapper.java | 7 +- .../mapper/SemanticTextFieldMapperTests.java | 5 - 10 files changed, 266 insertions(+), 43 deletions(-) create mode 100644 docs/changelog/122224.yaml create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml diff --git a/docs/changelog/122224.yaml b/docs/changelog/122224.yaml new file mode 100644 index 000000000000..41ae8c657860 --- /dev/null +++ b/docs/changelog/122224.yaml @@ -0,0 +1,6 @@ +pr: 122224 +summary: Enable the use of nested field type with index.mode=time_series +area: Mapping +type: enhancement +issues: + - 120874 diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 3bdaf029c364..205b02a8936b 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -82,4 +82,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> "cluster.desired_nodes/10_basic/Test update desired nodes with node_version generates a warning", "node_version warning is removed in 9.0" ) + task.skipTest("tsdb/20_mapping/nested fields", "nested field support in tsdb indices is now supported") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml new file mode 100644 index 000000000000..f4aca5ab264e --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml @@ -0,0 +1,233 @@ +setup: + - requires: + cluster_features: ["mapper.tsdb_nested_field_support"] + reason: "tsdb index with nested field support enabled" + +--- +"Create TSDB index with field of nested type": + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 1 + routing_path: [department] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + department: + type: keyword + time_series_dimension: true + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + + - do: + index: + index: test + body: { "@timestamp": "2021-04-28T01:00:00Z", "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4 } ] } + + - do: + index: + index: test + body: { "@timestamp": "2021-04-28T02:00:00Z", "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1 }, { "name": "Linear Algebra", "credits": 3 } ] } + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + size: 0 + query: + nested: + path: "courses" + query: + bool: + must: + - term: + courses.name: Precalculus + - term: + courses.credits: 3 + + - match: { hits.total.value: 0 } + + - do: + search: + index: test + body: + query: + nested: + path: "courses" + query: + bool: + must: + - term: + courses.name: "Object Oriented Programming" + - term: + courses.credits: 3 + + - match: { hits.total.value: 1 } + - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } + - match: { hits.hits.0._source.department: "compsci" } + - match: { hits.hits.0._source.courses: [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] } + +--- + +"TSDB index with multi-level nested fields": + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 1 + routing_path: [department] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + department: + type: keyword + time_series_dimension: true + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + students: + type: nested + properties: + name: + type: text + major: + type: keyword + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T01:00:00Z" + department: "compsci" + staff: 12 + courses: + - name: "Object Oriented Programming" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garrett" + major: "Software Engineering" + - name: "Theory of Computation" + credits: 4 + students: + - name: "Elliott Booker" + major: "Computer Engineering" + - name: "Kimora Tanner" + major: "Software Engineering" + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T02:00:00Z" + department: "math" + staff: 20 + courses: + - name: "Precalculus" + credits: 4 + students: + - name: "Elliott Ayers" + major: "Software Engineering" + - name: "Sylvie Howe" + major: "Computer Engineering" + - name: "Linear Algebra" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garett" + major: "Software Engineering" + - name: "Amelia Booker" + major: "Psychology" + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T03:00:00Z" + department: "compsci" + staff: 12 + courses: + - name: "Object Oriented Programming" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garrett" + major: "Software Engineering" + - name: "Elliott Booker" + major: "Computer Engineering" + - name: "Theory of Computation" + credits: 4 + students: + - name: "Kimora Tanner" + major: "Software Engineering" + - name: "Elliott Ayers" + major: "Software Engineering" + - name: "Apollo Pittman" + major: "Computer Engineering" + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + query: + nested: + path: "courses" + query: + bool: + must: + - nested: + path: "courses.students" + query: + bool: + must: + - match: + courses.students.name: "Elliott" + - term: + courses.students.major: "Computer Engineering" + - term: + courses.name: "Theory of Computation" + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.department: "compsci" } + - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index f25601fc2e22..5963ddb46e0b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -344,37 +344,6 @@ nested dimensions: type: keyword time_series_dimension: true ---- -nested fields: - - requires: - cluster_features: ["gte_v8.2.0"] - reason: message changed in 8.2.0 - - - do: - catch: /cannot have nested fields when index is in \[index.mode=time_series\]/ - indices.create: - index: test - body: - settings: - index: - mode: time_series - routing_path: [dim] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - mappings: - properties: - "@timestamp": - type: date - dim: - type: keyword - time_series_dimension: true - nested: - type: nested - properties: - foo: - type: keyword - --- "Unable to define a metric type for a runtime field": - requires: diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 7287a0bf307b..04bf82092666 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -29,7 +29,6 @@ import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.RoutingFields; @@ -156,9 +155,6 @@ public enum IndexMode { @Override public void validateMapping(MappingLookup lookup) { - if (lookup.nestedLookup() != NestedLookup.EMPTY) { - throw new IllegalArgumentException("cannot have nested fields when index is in " + tsdbMode()); - } if (((RoutingFieldMapper) lookup.getMapper(RoutingFieldMapper.NAME)).required()) { throw new IllegalArgumentException(routingRequiredBad()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 127ec05b25e6..7b5e28dc1fbe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -665,8 +665,14 @@ public abstract class DocumentParserContext { if (idField != null) { // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then // delete it when the root document is deleted too. - // NOTE: we don't support nested fields in tsdb so it's safe to assume the standard id mapper. doc.add(new StringField(IdFieldMapper.NAME, idField.binaryValue(), Field.Store.NO)); + } else if (indexSettings().getMode() == IndexMode.TIME_SERIES) { + // For time series indices, the _id is generated from the _tsid, which in turn is generated from the values of the configured + // routing fields. At this point in document parsing, we can't guarantee that we've parsed all the routing fields yet, so the + // parent document's _id is not yet available. + // So we just add the child document without the parent _id, then in TimeSeriesIdFieldMapper#postParse we set the _id on all + // child documents once we've calculated it. + assert getRoutingFields().equals(RoutingFields.Noop.INSTANCE) == false; } else { throw new IllegalStateException("The root document of a nested document should have an _id field"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 7567fae7d73e..42d35cb2ef09 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -27,6 +27,7 @@ public class MapperFeatures implements FeatureSpecification { "mapper.counted_keyword.synthetic_source_native_support" ); + public static final NodeFeature TSDB_NESTED_FIELD_SUPPORT = new NodeFeature("mapper.tsdb_nested_field_support"); public static final NodeFeature META_FETCH_FIELDS_ERROR_CODE_CHANGED = new NodeFeature("meta_fetch_fields_error_code_changed"); public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support"); public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix"); @@ -49,6 +50,7 @@ public class MapperFeatures implements FeatureSpecification { COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, DYNAMIC_HANDLING_IN_COPY_TO, + TSDB_NESTED_FIELD_SUPPORT, SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 8af3c3e6ec27..3325ad4e96eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -9,7 +9,9 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StringField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; @@ -135,13 +137,21 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { } context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); - TsidExtractingIdFieldMapper.createField( + BytesRef uidEncoded = TsidExtractingIdFieldMapper.createField( context, getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) ? routingPathFields.routingBuilder() : null, timeSeriesId ); + + // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are + // also deleted. Usually this happens when the nested document is created (in DocumentParserContext#createNestedContext), but + // for time-series indices the _id isn't available at that point. + for (LuceneDocument doc : context.nonRootDocuments()) { + assert doc.getField(IdFieldMapper.NAME) == null; + doc.add(new StringField(IdFieldMapper.NAME, uidEncoded, Field.Store.NO)); + } } private IndexVersion getIndexVersionCreated(final DocumentParserContext context) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 821b11410f93..8aca00494920 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -46,7 +46,11 @@ public class TsidExtractingIdFieldMapper extends IdFieldMapper { private static final long SEED = 0; - public static void createField(DocumentParserContext context, IndexRouting.ExtractFromSource.Builder routingBuilder, BytesRef tsid) { + public static BytesRef createField( + DocumentParserContext context, + IndexRouting.ExtractFromSource.Builder routingBuilder, + BytesRef tsid + ) { final long timestamp = DataStreamTimestampFieldMapper.extractTimestampValue(context.doc()); String id; if (routingBuilder != null) { @@ -94,6 +98,7 @@ public class TsidExtractingIdFieldMapper extends IdFieldMapper { BytesRef uidEncoded = Uid.encodeId(context.id()); context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); + return uidEncoded; } public static String createId(int routingHash, BytesRef tsid, long timestamp) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index e837e1b0db98..5d1c058c89da 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -136,11 +136,6 @@ public class SemanticTextFieldMapperTests extends MapperTestCase { b.field("type", "semantic_text"); } - @Override - protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - return "cannot have nested fields when index is in [index.mode=time_series]"; - } - @Override protected void metaMapping(XContentBuilder b) throws IOException { super.metaMapping(b); From 6ff8bedafd120db82ab2fac26cad560f5bc5d5d1 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 13 Feb 2025 19:38:54 +0100 Subject: [PATCH 75/93] Ensure shard is mutable before proceeding with updates (#122392) An update operations should make sure that the shard is mutable before proceeding further with a Get to retrieve the document and then any of the create/update/delete/noop operation. Relates ES-10708 --- .../action/update/TransportUpdateAction.java | 264 ++++++++++-------- 1 file changed, 140 insertions(+), 124 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index ee84bcd15824..bd76ef81d2c4 100644 --- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction; import org.elasticsearch.client.internal.node.NodeClient; @@ -96,7 +97,10 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio @Override protected Executor executor(ShardId shardId) { - final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); + return executor(indicesService.indexServiceSafe(shardId.getIndex())); + } + + private Executor executor(IndexService indexService) { return threadPool.executor(indexService.getIndexSettings().getIndexMetadata().isSystem() ? Names.SYSTEM_WRITE : Names.WRITE); } @@ -189,136 +193,148 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); final IndexShard indexShard = indexService.getShard(shardId.getId()); final MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); - final UpdateHelper.Result result = deleteInferenceResults( - request, - updateHelper.prepare(request, indexShard, threadPool::absoluteTimeInMillis), - indexService.getMetadata(), - mappingLookup - ); - switch (result.getResponseResult()) { - case CREATED -> { - IndexRequest upsertRequest = result.action(); - // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request - final BytesReference upsertSourceBytes = upsertRequest.source(); - client.bulk( - toSingleItemBulkRequest(upsertRequest), - unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { - UpdateResponse update = new UpdateResponse( - response.getShardInfo(), - response.getShardId(), - response.getId(), - response.getSeqNo(), - response.getPrimaryTerm(), - response.getVersion(), - response.getResult() - ); - if (request.fetchSource() != null && request.fetchSource().fetchSource()) { - Tuple> sourceAndContent = XContentHelper.convertToMap( - upsertSourceBytes, - true, - upsertRequest.getContentType() - ); - update.setGetResult( - UpdateHelper.extractGetResult( - request, - request.concreteIndex(), - mappingLookup, + var executor = executor(indexService); + assert ThreadPool.assertCurrentThreadPool(Names.SYSTEM_WRITE, Names.WRITE); + + SubscribableListener.newForked(indexShard::ensureMutable) + // Make sure to fork back to a `write` thread pool if necessary + .andThen(executor, threadPool.getThreadContext(), (l, unused) -> ActionListener.completeWith(l, () -> { + assert ThreadPool.assertCurrentThreadPool(Names.SYSTEM_WRITE, Names.WRITE); + return deleteInferenceResults( + request, + updateHelper.prepare(request, indexShard, threadPool::absoluteTimeInMillis), // Gets the doc using the engine + indexService.getMetadata(), + mappingLookup + ); + })) + // Proceed with a single item bulk request + .andThen((l, result) -> { + switch (result.getResponseResult()) { + case CREATED -> { + IndexRequest upsertRequest = result.action(); + // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request + final BytesReference upsertSourceBytes = upsertRequest.source(); + client.bulk( + toSingleItemBulkRequest(upsertRequest), + unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { + UpdateResponse update = new UpdateResponse( + response.getShardInfo(), + response.getShardId(), + response.getId(), response.getSeqNo(), response.getPrimaryTerm(), response.getVersion(), - sourceAndContent.v2(), - sourceAndContent.v1(), - upsertSourceBytes - ) - ); - } else { - update.setGetResult(null); - } - update.setForcedRefresh(response.forcedRefresh()); - listener.onResponse(update); - }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount))) - ); - } - case UPDATED -> { - IndexRequest indexRequest = result.action(); - // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request - final BytesReference indexSourceBytes = indexRequest.source(); - client.bulk( - toSingleItemBulkRequest(indexRequest), - unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { - UpdateResponse update = new UpdateResponse( - response.getShardInfo(), - response.getShardId(), - response.getId(), - response.getSeqNo(), - response.getPrimaryTerm(), - response.getVersion(), - response.getResult() + response.getResult() + ); + if (request.fetchSource() != null && request.fetchSource().fetchSource()) { + Tuple> sourceAndContent = XContentHelper.convertToMap( + upsertSourceBytes, + true, + upsertRequest.getContentType() + ); + update.setGetResult( + UpdateHelper.extractGetResult( + request, + request.concreteIndex(), + mappingLookup, + response.getSeqNo(), + response.getPrimaryTerm(), + response.getVersion(), + sourceAndContent.v2(), + sourceAndContent.v1(), + upsertSourceBytes + ) + ); + } else { + update.setGetResult(null); + } + update.setForcedRefresh(response.forcedRefresh()); + l.onResponse(update); + }, exception -> handleUpdateFailureWithRetry(l, request, exception, retryCount))) ); - update.setGetResult( - UpdateHelper.extractGetResult( - request, - request.concreteIndex(), - mappingLookup, - response.getSeqNo(), - response.getPrimaryTerm(), - response.getVersion(), - result.updatedSourceAsMap(), - result.updateSourceContentType(), - indexSourceBytes - ) - ); - update.setForcedRefresh(response.forcedRefresh()); - listener.onResponse(update); - }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount))) - ); - } - case DELETED -> { - DeleteRequest deleteRequest = result.action(); - client.bulk( - toSingleItemBulkRequest(deleteRequest), - unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { - UpdateResponse update = new UpdateResponse( - response.getShardInfo(), - response.getShardId(), - response.getId(), - response.getSeqNo(), - response.getPrimaryTerm(), - response.getVersion(), - response.getResult() - ); - update.setGetResult( - UpdateHelper.extractGetResult( - request, - request.concreteIndex(), - mappingLookup, - response.getSeqNo(), - response.getPrimaryTerm(), - response.getVersion(), - result.updatedSourceAsMap(), - result.updateSourceContentType(), - null - ) - ); - update.setForcedRefresh(response.forcedRefresh()); - listener.onResponse(update); - }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount))) - ); - } - case NOOP -> { - UpdateResponse update = result.action(); - IndexService indexServiceOrNull = indicesService.indexService(shardId.getIndex()); - if (indexServiceOrNull != null) { - IndexShard shard = indexService.getShardOrNull(shardId.getId()); - if (shard != null) { - shard.noopUpdate(); } + case UPDATED -> { + IndexRequest indexRequest = result.action(); + // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request + final BytesReference indexSourceBytes = indexRequest.source(); + client.bulk( + toSingleItemBulkRequest(indexRequest), + unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { + UpdateResponse update = new UpdateResponse( + response.getShardInfo(), + response.getShardId(), + response.getId(), + response.getSeqNo(), + response.getPrimaryTerm(), + response.getVersion(), + response.getResult() + ); + update.setGetResult( + UpdateHelper.extractGetResult( + request, + request.concreteIndex(), + mappingLookup, + response.getSeqNo(), + response.getPrimaryTerm(), + response.getVersion(), + result.updatedSourceAsMap(), + result.updateSourceContentType(), + indexSourceBytes + ) + ); + update.setForcedRefresh(response.forcedRefresh()); + l.onResponse(update); + }, exception -> handleUpdateFailureWithRetry(l, request, exception, retryCount))) + ); + } + case DELETED -> { + DeleteRequest deleteRequest = result.action(); + client.bulk( + toSingleItemBulkRequest(deleteRequest), + unwrappingSingleItemBulkResponse(ActionListener.wrap(response -> { + UpdateResponse update = new UpdateResponse( + response.getShardInfo(), + response.getShardId(), + response.getId(), + response.getSeqNo(), + response.getPrimaryTerm(), + response.getVersion(), + response.getResult() + ); + update.setGetResult( + UpdateHelper.extractGetResult( + request, + request.concreteIndex(), + mappingLookup, + response.getSeqNo(), + response.getPrimaryTerm(), + response.getVersion(), + result.updatedSourceAsMap(), + result.updateSourceContentType(), + null + ) + ); + update.setForcedRefresh(response.forcedRefresh()); + l.onResponse(update); + }, exception -> handleUpdateFailureWithRetry(l, request, exception, retryCount))) + ); + } + case NOOP -> { + UpdateResponse update = result.action(); + IndexService indexServiceOrNull = indicesService.indexService(shardId.getIndex()); + if (indexServiceOrNull != null) { + IndexShard shard = indexService.getShardOrNull(shardId.getId()); + if (shard != null) { + shard.noopUpdate(); + } + } + l.onResponse(update); + } + default -> throw new IllegalStateException("Illegal result " + result.getResponseResult()); } - listener.onResponse(update); - } - default -> throw new IllegalStateException("Illegal result " + result.getResponseResult()); - } + }) + .addListener(listener); } private void handleUpdateFailureWithRetry( From f8aa04799418e0efd8285847718fcff50400b552 Mon Sep 17 00:00:00 2001 From: Patrick Doyle <810052+prdoyle@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:45:51 -0500 Subject: [PATCH 76/93] Entitlements: manage_threads (#122261) * Refactor: protected -> private * Initial thread-related entitlements * Entitlements from manual test runs * Refactor: notEntitled method * Entitlements reporting mode * Entitlements from CI * Revert "Entitlements reporting mode" This reverts commit 443ca767333269a73accd0cedf9ca6c6ac51698a. * Remove unnecessary EntitledActions.newThread * Don't log in entitlements ITs by default * Import SuppressForbidden * Respond to PR comments * Move manage_threads tests to their own file --- .../impl/InstrumenterImpl.java | 5 +- .../bridge/EntitlementChecker.java | 23 +++++++ .../qa/entitled/EntitledActions.java | 4 -- .../qa/entitled/EntitledPlugin.java | 16 +++-- .../qa/test/ManageThreadsActions.java | 69 +++++++++++++++++++ .../qa/test/RestEntitlementsCheckAction.java | 12 +++- .../qa/AbstractEntitlementsIT.java | 1 + .../entitlement/qa/EntitlementsTestRule.java | 4 +- .../EntitlementInitialization.java | 7 +- .../api/ElasticsearchEntitlementChecker.java | 47 +++++++++++++ .../runtime/policy/PolicyManager.java | 53 ++++++++------ .../runtime/policy/PolicyParser.java | 26 ++++--- .../ManageThreadsEntitlement.java | 17 +++++ .../plugin-metadata/entitlement-policy.yaml | 1 + .../plugin-metadata/entitlement-policy.yaml | 1 + .../plugin-metadata/entitlement-policy.yaml | 1 + .../plugin-metadata/entitlement-policy.yaml | 2 + .../plugin-metadata/entitlement-policy.yaml | 1 + .../plugin-metadata/entitlement-policy.yaml | 2 + .../plugin-metadata/entitlement-policy.yaml | 5 ++ .../plugin-metadata/entitlement-policy.yaml | 2 + .../plugin-metadata/entitlement-policy.yaml | 2 + .../plugin-metadata/entitlement-policy.yaml | 2 + 23 files changed, 256 insertions(+), 47 deletions(-) create mode 100644 libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/ManageThreadsActions.java create mode 100644 libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ManageThreadsEntitlement.java create mode 100644 x-pack/plugin/ml/src/main/plugin-metadata/entitlement-policy.yaml create mode 100644 x-pack/plugin/watcher/src/main/plugin-metadata/entitlement-policy.yaml diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java index 06408941ac96..b10c58afacb1 100644 --- a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java +++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java @@ -152,14 +152,13 @@ public class InstrumenterImpl implements Instrumenter { if (isAnnotationPresent == false) { boolean isStatic = (access & ACC_STATIC) != 0; boolean isCtor = "".equals(name); - boolean hasReceiver = (isStatic || isCtor) == false; var key = new MethodKey(className, name, Stream.of(Type.getArgumentTypes(descriptor)).map(Type::getInternalName).toList()); var instrumentationMethod = checkMethods.get(key); if (instrumentationMethod != null) { - // LOGGER.debug("Will instrument method {}", key); + // System.out.println("Will instrument method " + key); return new EntitlementMethodVisitor(Opcodes.ASM9, mv, isStatic, isCtor, descriptor, instrumentationMethod); } else { - // LOGGER.trace("Will not instrument method {}", key); + // System.out.println("Will not instrument method " + key); } } return mv; diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 39671e298cb9..978e4e93e375 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -69,6 +69,7 @@ import java.util.Properties; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ForkJoinPool; import java.util.function.Consumer; import javax.net.ssl.HostnameVerifier; @@ -652,4 +653,26 @@ public interface EntitlementChecker { void checkName(Class callerClass, FileStore that); void checkType(Class callerClass, FileStore that); + + //////////////////// + // + // Thread management + // + + void check$java_lang_Thread$start(Class callerClass, Thread thread); + + void check$java_lang_Thread$setDaemon(Class callerClass, Thread thread, boolean on); + + void check$java_lang_ThreadGroup$setDaemon(Class callerClass, ThreadGroup threadGroup, boolean daemon); + + void check$java_util_concurrent_ForkJoinPool$setParallelism(Class callerClass, ForkJoinPool forkJoinPool, int size); + + void check$java_lang_Thread$setName(Class callerClass, Thread thread, String name); + + void check$java_lang_Thread$setPriority(Class callerClass, Thread thread, int newPriority); + + void check$java_lang_Thread$setUncaughtExceptionHandler(Class callerClass, Thread thread, Thread.UncaughtExceptionHandler ueh); + + void check$java_lang_ThreadGroup$setMaxPriority(Class callerClass, ThreadGroup threadGroup, int pri); + } diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java index f8451e84449f..58bafdc47a0b 100644 --- a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java +++ b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java @@ -34,10 +34,6 @@ public final class EntitledActions { return testRootDir.resolve("read_write_dir"); } - static void System_clearProperty(String key) { - System.clearProperty(key); - } - public static UserPrincipal getFileOwner(Path path) throws IOException { return Files.getOwner(path); } diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledPlugin.java b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledPlugin.java index 7a60d92ecc55..cec48ac16867 100644 --- a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledPlugin.java +++ b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledPlugin.java @@ -15,7 +15,7 @@ import org.elasticsearch.logging.Logger; import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; -import static org.elasticsearch.entitlement.qa.entitled.EntitledActions.System_clearProperty; +import java.util.concurrent.atomic.AtomicBoolean; public class EntitledPlugin extends Plugin implements ExtensiblePlugin { @@ -28,11 +28,19 @@ public class EntitledPlugin extends Plugin implements ExtensiblePlugin { selfTestNotEntitled(); } - private static final String SELF_TEST_PROPERTY = "org.elasticsearch.entitlement.qa.selfTest"; - private static void selfTestEntitled() { logger.debug("selfTestEntitled"); - System_clearProperty(SELF_TEST_PROPERTY); + AtomicBoolean threadRan = new AtomicBoolean(false); + try { + Thread testThread = new Thread(() -> threadRan.set(true), "testThread"); + testThread.start(); + testThread.join(); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + if (threadRan.get() == false) { + throw new AssertionError("Self-test thread did not run"); + } } private static void selfTestNotEntitled() { diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/ManageThreadsActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/ManageThreadsActions.java new file mode 100644 index 000000000000..53f17faf0699 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/ManageThreadsActions.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.core.SuppressForbidden; + +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.lang.Thread.currentThread; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +@SuppressForbidden(reason = "testing entitlements") +@SuppressWarnings("unused") // used via reflection +class ManageThreadsActions { + private ManageThreadsActions() {} + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$start() throws InterruptedException { + AtomicBoolean threadRan = new AtomicBoolean(false); + Thread thread = new Thread(() -> threadRan.set(true), "test"); + thread.start(); + thread.join(); + assert threadRan.get(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$setDaemon() { + new Thread().setDaemon(true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_ThreadGroup$setDaemon() { + currentThread().getThreadGroup().setDaemon(currentThread().getThreadGroup().isDaemon()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_util_concurrent_ForkJoinPool$setParallelism() { + ForkJoinPool.commonPool().setParallelism(ForkJoinPool.commonPool().getParallelism()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$setName() { + currentThread().setName(currentThread().getName()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$setPriority() { + currentThread().setPriority(currentThread().getPriority()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$setUncaughtExceptionHandler() { + currentThread().setUncaughtExceptionHandler(currentThread().getUncaughtExceptionHandler()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_ThreadGroup$setMaxPriority() { + currentThread().getThreadGroup().setMaxPriority(currentThread().getThreadGroup().getMaxPriority()); + } + +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java index 18709465fe1a..7e8adc473668 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java @@ -181,13 +181,17 @@ public class RestEntitlementsCheckAction extends BaseRestHandler { entry("runtime_load_library", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoadLibrary)), entry("system_load", forPlugins(LoadNativeLibrariesCheckActions::systemLoad)), entry("system_load_library", forPlugins(LoadNativeLibrariesCheckActions::systemLoadLibrary)) + + // MAINTENANCE NOTE: Please don't add any more entries to this map. + // Put new tests into their own "Actions" class using the @EntitlementTest annotation. ), getTestEntries(FileCheckActions.class), - getTestEntries(SpiActions.class), - getTestEntries(SystemActions.class), + getTestEntries(FileStoreActions.class), + getTestEntries(ManageThreadsActions.class), getTestEntries(NativeActions.class), getTestEntries(NioFileSystemActions.class), - getTestEntries(FileStoreActions.class) + getTestEntries(SpiActions.class), + getTestEntries(SystemActions.class) ) .flatMap(Function.identity()) .filter(entry -> entry.getValue().fromJavaVersion() == null || Runtime.version().feature() >= entry.getValue().fromJavaVersion()) @@ -424,7 +428,9 @@ public class RestEntitlementsCheckAction extends BaseRestHandler { return channel -> { logger.info("Calling check action [{}]", actionName); checkAction.action().run(); + logger.debug("Check action [{}] returned", actionName); channel.sendResponse(new RestResponse(RestStatus.OK, Strings.format("Succesfully executed action [%s]", actionName))); }; } + } diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java index 29e616855f27..bd88c23fc5b9 100644 --- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java @@ -29,6 +29,7 @@ public abstract class AbstractEntitlementsIT extends ESRestTestCase { builder.value("inbound_network"); builder.value("outbound_network"); builder.value("load_native_libraries"); + builder.value("manage_threads"); builder.value( Map.of( "write_system_properties", diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java index fd702d60e723..9dc1028148a3 100644 --- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java @@ -33,7 +33,7 @@ class EntitlementsTestRule implements TestRule { // entitlements that test methods may use, see EntitledActions private static final PolicyBuilder ENTITLED_POLICY = (builder, tempDir) -> { - builder.value(Map.of("write_system_properties", Map.of("properties", List.of("org.elasticsearch.entitlement.qa.selfTest")))); + builder.value("manage_threads"); builder.value( Map.of( "files", @@ -74,6 +74,8 @@ class EntitlementsTestRule implements TestRule { .systemProperty("es.entitlements.enabled", "true") .systemProperty("es.entitlements.testdir", () -> testDir.getRoot().getAbsolutePath()) .setting("xpack.security.enabled", "false") + // Logs in libs/entitlement/qa/build/test-results/javaRestTest/TEST-org.elasticsearch.entitlement.qa.EntitlementsXXX.xml + // .setting("logger.org.elasticsearch.entitlement", "DEBUG") .build(); ruleChain = RuleChain.outerRule(testDir).around(tempDirSetup).around(cluster); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 331d47e561f1..12af77de248e 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -28,6 +28,7 @@ import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlemen import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.FileData; import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement; @@ -143,6 +144,7 @@ public class EntitlementInitialization { new InboundNetworkEntitlement(), new OutboundNetworkEntitlement(), new LoadNativeLibrariesEntitlement(), + new ManageThreadsEntitlement(), new FilesEntitlement( List.of(new FilesEntitlement.FileData(EntitlementBootstrap.bootstrapArgs().tempDir().toString(), READ_WRITE)) ) @@ -150,7 +152,8 @@ public class EntitlementInitialization { ), new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())), new Scope("io.netty.transport", List.of(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement())), - new Scope("org.apache.lucene.core", List.of(new LoadNativeLibrariesEntitlement())), + new Scope("org.apache.lucene.core", List.of(new LoadNativeLibrariesEntitlement(), new ManageThreadsEntitlement())), + new Scope("org.apache.logging.log4j.core", List.of(new ManageThreadsEntitlement())), new Scope( "org.elasticsearch.nativeaccess", List.of( @@ -162,7 +165,7 @@ public class EntitlementInitialization { ); // agents run without a module, so this is a special hack for the apm agent // this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed - List agentEntitlements = List.of(new CreateClassLoaderEntitlement()); + List agentEntitlements = List.of(new CreateClassLoaderEntitlement(), new ManageThreadsEntitlement()); var resolver = EntitlementBootstrap.bootstrapArgs().pluginResolver(); return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, AGENTS_PACKAGE_NAME, ENTITLEMENTS_MODULE); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index 1ca03fa876b1..986d8bee5bf2 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -75,6 +75,7 @@ import java.util.Properties; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ForkJoinPool; import java.util.function.Consumer; import javax.net.ssl.HostnameVerifier; @@ -1278,6 +1279,52 @@ public class ElasticsearchEntitlementChecker implements EntitlementChecker { policyManager.checkFileRead(callerClass, path); } + // Thread management + + @Override + public void check$java_lang_Thread$start(Class callerClass, Thread thread) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_Thread$setDaemon(Class callerClass, Thread thread, boolean on) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_ThreadGroup$setDaemon(Class callerClass, ThreadGroup threadGroup, boolean daemon) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_util_concurrent_ForkJoinPool$setParallelism(Class callerClass, ForkJoinPool forkJoinPool, int size) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_Thread$setName(Class callerClass, Thread thread, String name) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_Thread$setPriority(Class callerClass, Thread thread, int newPriority) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_Thread$setUncaughtExceptionHandler( + Class callerClass, + Thread thread, + Thread.UncaughtExceptionHandler ueh + ) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_ThreadGroup$setMaxPriority(Class callerClass, ThreadGroup threadGroup, int pri) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + @Override public void checkGetFileStoreAttributeView(Class callerClass, FileStore that, Class type) { policyManager.checkWriteStoreAttributes(callerClass); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 0d6905f64bd7..008983f099be 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -19,6 +19,7 @@ import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitleme import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; @@ -102,9 +103,9 @@ public class PolicyManager { final Map moduleEntitlementsMap = new ConcurrentHashMap<>(); - protected final Map> serverEntitlements; - protected final List apmAgentEntitlements; - protected final Map>> pluginsEntitlements; + private final Map> serverEntitlements; + private final List apmAgentEntitlements; + private final Map>> pluginsEntitlements; private final Function, String> pluginResolver; public static final String ALL_UNNAMED = "ALL-UNNAMED"; @@ -200,7 +201,7 @@ public class PolicyManager { return; } - throw new NotEntitledException( + notEntitled( Strings.format( "Not entitled: component [%s], module [%s], class [%s], operation [%s]", getEntitlements(requestingClass).componentName(), @@ -224,17 +225,19 @@ public class PolicyManager { } public void checkChangeJVMGlobalState(Class callerClass) { - neverEntitled(callerClass, () -> { - // Look up the check$ method to compose an informative error message. - // This way, we don't need to painstakingly describe every individual global-state change. - Optional checkMethodName = StackWalker.getInstance() - .walk( - frames -> frames.map(StackFrame::getMethodName) - .dropWhile(not(methodName -> methodName.startsWith(InstrumentationService.CHECK_METHOD_PREFIX))) - .findFirst() - ); - return checkMethodName.map(this::operationDescription).orElse("change JVM global state"); - }); + neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("change JVM global state")); + } + + private Optional walkStackForCheckMethodName() { + // Look up the check$ method to compose an informative error message. + // This way, we don't need to painstakingly describe every individual global-state change. + return StackWalker.getInstance() + .walk( + frames -> frames.map(StackFrame::getMethodName) + .dropWhile(not(methodName -> methodName.startsWith(InstrumentationService.CHECK_METHOD_PREFIX))) + .findFirst() + ) + .map(this::operationDescription); } /** @@ -257,11 +260,11 @@ public class PolicyManager { ModuleEntitlements entitlements = getEntitlements(requestingClass); if (entitlements.fileAccess().canRead(path) == false) { - throw new NotEntitledException( + notEntitled( Strings.format( "Not entitled: component [%s], module [%s], class [%s], entitlement [file], operation [read], path [%s]", entitlements.componentName(), - requestingClass.getModule(), + requestingClass.getModule().getName(), requestingClass, path ) @@ -282,11 +285,11 @@ public class PolicyManager { ModuleEntitlements entitlements = getEntitlements(requestingClass); if (entitlements.fileAccess().canWrite(path) == false) { - throw new NotEntitledException( + notEntitled( Strings.format( "Not entitled: component [%s], module [%s], class [%s], entitlement [file], operation [write], path [%s]", entitlements.componentName(), - requestingClass.getModule(), + requestingClass.getModule().getName(), requestingClass, path ) @@ -340,7 +343,7 @@ public class PolicyManager { Class requestingClass ) { if (classEntitlements.hasEntitlement(entitlementClass) == false) { - throw new NotEntitledException( + notEntitled( Strings.format( "Not entitled: component [%s], module [%s], class [%s], entitlement [%s]", classEntitlements.componentName(), @@ -380,7 +383,7 @@ public class PolicyManager { ); return; } - throw new NotEntitledException( + notEntitled( Strings.format( "Not entitled: component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]", entitlements.componentName(), @@ -391,6 +394,14 @@ public class PolicyManager { ); } + private static void notEntitled(String message) { + throw new NotEntitledException(message); + } + + public void checkManageThreadsEntitlement(Class callerClass) { + checkEntitlementPresent(callerClass, ManageThreadsEntitlement.class); + } + private void checkEntitlementPresent(Class callerClass, Class entitlementClass) { var requestingClass = requestingClass(callerClass); if (isTriviallyAllowed(requestingClass)) { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java index fead4fc1f629..9698b9e86704 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -14,8 +14,10 @@ import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteAllSystemPropertiesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement; import org.elasticsearch.xcontent.XContentLocation; import org.elasticsearch.xcontent.XContentParser; @@ -45,20 +47,22 @@ import java.util.stream.Stream; */ public class PolicyParser { - private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of( - FilesEntitlement.class, + private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of( CreateClassLoaderEntitlement.class, - SetHttpsConnectionPropertiesEntitlement.class, - OutboundNetworkEntitlement.class, + FilesEntitlement.class, InboundNetworkEntitlement.class, - WriteSystemPropertiesEntitlement.class, - LoadNativeLibrariesEntitlement.class + LoadNativeLibrariesEntitlement.class, + ManageThreadsEntitlement.class, + OutboundNetworkEntitlement.class, + SetHttpsConnectionPropertiesEntitlement.class, + WriteAllSystemPropertiesEntitlement.class, + WriteSystemPropertiesEntitlement.class ).collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity())); protected final XContentParser policyParser; protected final String policyName; private final boolean isExternalPlugin; - private final Map> externalEntitlements; + private final Map> externalEntitlements; static String getEntitlementTypeName(Class entitlementClass) { var entitlementClassName = entitlementClass.getSimpleName(); @@ -81,8 +85,12 @@ public class PolicyParser { } // package private for tests - PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin, Map> externalEntitlements) - throws IOException { + PolicyParser( + InputStream inputStream, + String policyName, + boolean isExternalPlugin, + Map> externalEntitlements + ) throws IOException { this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream)); this.policyName = policyName; this.isExternalPlugin = isExternalPlugin; diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ManageThreadsEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ManageThreadsEntitlement.java new file mode 100644 index 000000000000..c75ccf26d143 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ManageThreadsEntitlement.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; + +public record ManageThreadsEntitlement() implements Entitlement { + @ExternalEntitlement(esModulesOnly = false) + public ManageThreadsEntitlement {} +} diff --git a/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml b/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml index df557f994425..394e5e38d9f5 100644 --- a/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,3 @@ ALL-UNNAMED: + - manage_threads - outbound_network diff --git a/modules/repository-azure/src/main/plugin-metadata/entitlement-policy.yaml b/modules/repository-azure/src/main/plugin-metadata/entitlement-policy.yaml index 74197fb3ed9a..53049c74b6a4 100644 --- a/modules/repository-azure/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/repository-azure/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,3 @@ io.netty.common: - outbound_network + - manage_threads diff --git a/modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml b/modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml index df557f994425..394e5e38d9f5 100644 --- a/modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,3 @@ ALL-UNNAMED: + - manage_threads - outbound_network diff --git a/modules/transport-netty4/src/main/plugin-metadata/entitlement-policy.yaml b/modules/transport-netty4/src/main/plugin-metadata/entitlement-policy.yaml index eb772a06423a..1562b806a82d 100644 --- a/modules/transport-netty4/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/transport-netty4/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,6 +1,8 @@ io.netty.transport: - inbound_network - outbound_network + - manage_threads io.netty.common: - inbound_network - outbound_network + - manage_threads diff --git a/plugins/discovery-ec2/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/discovery-ec2/src/main/plugin-metadata/entitlement-policy.yaml index df557f994425..394e5e38d9f5 100644 --- a/plugins/discovery-ec2/src/main/plugin-metadata/entitlement-policy.yaml +++ b/plugins/discovery-ec2/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,3 @@ ALL-UNNAMED: + - manage_threads - outbound_network diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/repository-hdfs/src/main/plugin-metadata/entitlement-policy.yaml index b5020dc1b746..30e61739a063 100644 --- a/plugins/repository-hdfs/src/main/plugin-metadata/entitlement-policy.yaml +++ b/plugins/repository-hdfs/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,5 +1,7 @@ ALL-UNNAMED: + - manage_threads - outbound_network + - load_native_libraries - write_system_properties: properties: - hadoop.home.dir diff --git a/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml index 4e0266b06bbb..a069e1b4ce4c 100644 --- a/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,8 +1,13 @@ org.apache.httpcomponents.httpclient: - outbound_network # For SamlRealm + - manage_threads org.apache.httpcomponents.httpcore.nio: - outbound_network + - manage_threads +org.apache.httpcomponents.httpasyncclient: + - manage_threads unboundid.ldapsdk: + - manage_threads - write_system_properties: properties: - java.security.auth.login.config diff --git a/x-pack/plugin/ml/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/ml/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 000000000000..ff8f2a8f73ea --- /dev/null +++ b/x-pack/plugin/ml/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +org.elasticsearch.ml: + - manage_threads diff --git a/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml index 636627240bf4..0695c8e5766f 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,9 +1,11 @@ org.elasticsearch.security: - set_https_connection_properties # for CommandLineHttpClient io.netty.transport: + - manage_threads - inbound_network - outbound_network io.netty.common: + - manage_threads - inbound_network - outbound_network org.opensaml.xmlsec.impl: diff --git a/x-pack/plugin/watcher/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/watcher/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 000000000000..2eb0d0dbd988 --- /dev/null +++ b/x-pack/plugin/watcher/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +ALL-UNNAMED: + - manage_threads From 482a49fce184a138debeefaa67d0e02e1d141a92 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 13 Feb 2025 11:28:22 -0800 Subject: [PATCH 77/93] Improve size limiting string message (#122427) SizeLimitingStringWriter is used to limit the output size of mustache scripts. When the size is limited, the resulting exception lacks detail needed to identify the problem. In particular, the actual size that would result is not given. Additionally, the excerpt lacks any of the new string being added. This commit tweaks the exception to include both of these details. --- docs/changelog/122427.yaml | 5 ++ .../mustache/MustacheScriptEngineTests.java | 2 +- .../common/text/SizeLimitingStringWriter.java | 46 +++++++++++++------ .../text/SizeLimitingStringWriterTests.java | 9 ++++ 4 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 docs/changelog/122427.yaml diff --git a/docs/changelog/122427.yaml b/docs/changelog/122427.yaml new file mode 100644 index 000000000000..2444a0ec894a --- /dev/null +++ b/docs/changelog/122427.yaml @@ -0,0 +1,5 @@ +pr: 122427 +summary: Improve size limiting string message +area: Infra/Core +type: enhancement +issues: [] diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index bc1cd30ad45b..c327ba49e6d1 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -423,7 +423,7 @@ public class MustacheScriptEngineTests extends ESTestCase { ex.getCause().getCause(), allOf( instanceOf(SizeLimitingStringWriter.SizeLimitExceededException.class), - transformedMatch(Throwable::getMessage, endsWith("has exceeded the size limit [1024]")) + transformedMatch(Throwable::getMessage, endsWith("has size [1030] which exceeds the size limit [1024]")) ) ); } diff --git a/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java b/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java index 2df7e6537c60..3aa7c67a14c6 100644 --- a/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java +++ b/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java @@ -30,18 +30,29 @@ public class SizeLimitingStringWriter extends StringWriter { this.sizeLimit = sizeLimit; } - private void checkSizeLimit(int additionalChars) { - int bufLen = getBuffer().length(); - if (bufLen + additionalChars > sizeLimit) { - throw new SizeLimitExceededException( - Strings.format("String [%s...] has exceeded the size limit [%s]", getBuffer().substring(0, Math.min(bufLen, 20)), sizeLimit) - ); + private int limitSize(int additionalChars) { + int neededSize = getBuffer().length() + additionalChars; + if (neededSize > sizeLimit) { + return additionalChars - (neededSize - sizeLimit); } + return additionalChars; + } + + private void throwSizeLimitExceeded(int limitedChars, int requestedChars) { + assert limitedChars < requestedChars; + int bufLen = getBuffer().length(); + int foundSize = bufLen - limitedChars + requestedChars; // reconstitute original + String selection = getBuffer().substring(0, Math.min(bufLen, 20)); + throw new SizeLimitExceededException( + Strings.format("String [%s...] has size [%d] which exceeds the size limit [%d]", selection, foundSize, sizeLimit) + ); } @Override public void write(int c) { - checkSizeLimit(1); + if (limitSize(1) != 1) { + throwSizeLimitExceeded(0, 1); + } super.write(c); } @@ -49,20 +60,29 @@ public class SizeLimitingStringWriter extends StringWriter { @Override public void write(char[] cbuf, int off, int len) { - checkSizeLimit(len); - super.write(cbuf, off, len); + int limitedLen = limitSize(len); + if (limitedLen > 0) { + super.write(cbuf, off, limitedLen); + } + if (limitedLen != len) { + throwSizeLimitExceeded(limitedLen, len); + } } @Override public void write(String str) { - checkSizeLimit(str.length()); - super.write(str); + this.write(str, 0, str.length()); } @Override public void write(String str, int off, int len) { - checkSizeLimit(len); - super.write(str, off, len); + int limitedLen = limitSize(len); + if (limitedLen > 0) { + super.write(str, off, limitedLen); + } + if (limitedLen != len) { + throwSizeLimitExceeded(limitedLen, len); + } } // append(...) delegates to write(...) methods diff --git a/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java b/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java index 32a8de20df9a..0874a106e59e 100644 --- a/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java +++ b/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java @@ -11,6 +11,8 @@ package org.elasticsearch.common.text; import org.elasticsearch.test.ESTestCase; +import static org.hamcrest.Matchers.equalTo; + public class SizeLimitingStringWriterTests extends ESTestCase { public void testSizeIsLimited() { SizeLimitingStringWriter writer = new SizeLimitingStringWriter(10); @@ -26,4 +28,11 @@ public class SizeLimitingStringWriterTests extends ESTestCase { expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.append("a")); expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.append("a", 0, 1)); } + + public void testLimitMessage() { + SizeLimitingStringWriter writer = new SizeLimitingStringWriter(3); + + var e = expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.write("abcdefgh")); + assertThat(e.getMessage(), equalTo("String [abc...] has size [8] which exceeds the size limit [3]")); + } } From 9076ac43033fb301e6d579ea3006a0992a893ea6 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 13 Feb 2025 11:49:14 -0800 Subject: [PATCH 78/93] System Index Migration Failure Results in a Non-Recoverable State (#122326) This PR changes the code to no-longer rely on the persistent task state for the cleanup logic of existing indices. --- docs/changelog/122326.yaml | 5 + .../migration/FeatureMigrationIT.java | 1 - .../upgrades/SystemIndexMigrator.java | 91 +++++++++---------- 3 files changed, 46 insertions(+), 51 deletions(-) create mode 100644 docs/changelog/122326.yaml diff --git a/docs/changelog/122326.yaml b/docs/changelog/122326.yaml new file mode 100644 index 000000000000..91c71041d58f --- /dev/null +++ b/docs/changelog/122326.yaml @@ -0,0 +1,5 @@ +pr: 122326 +summary: System Index Migration Failure Results in a Non-Recoverable State +area: Infra/Core +type: bug +issues: [] diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java index ee95ce551382..efca437d14eb 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java @@ -277,7 +277,6 @@ public class FeatureMigrationIT extends AbstractFeatureMigrationIntegTest { }); } - @AwaitsFix(bugUrl = "ES-10666") // This test uncovered an existing issue public void testIndexBlockIsRemovedWhenAliasRequestFails() throws Exception { createSystemIndexForDescriptor(INTERNAL_UNMANAGED); ensureGreen(); diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java index cdd466c567e8..994760647017 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java +++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java @@ -207,49 +207,14 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { } // Kick off our callback "loop" - finishIndexAndLoop calls back into prepareNextIndex - cleanUpPreviousMigration( - taskState, - clusterState, - state -> prepareNextIndex(state, state2 -> migrateSingleIndex(state2, this::finishIndexAndLoop), stateFeatureName) - ); - } - - private void cleanUpPreviousMigration( - SystemIndexMigrationTaskState taskState, - ClusterState currentState, - Consumer listener - ) { logger.debug("cleaning up previous migration, task state: [{}]", taskState == null ? "null" : Strings.toString(taskState)); - if (taskState != null && taskState.getCurrentIndex() != null) { - SystemIndexMigrationInfo migrationInfo; - try { - migrationInfo = SystemIndexMigrationInfo.fromTaskState( - taskState, - systemIndices, - currentState.metadata(), - indexScopedSettings - ); - } catch (Exception e) { - markAsFailed(e); - return; - } - final String newIndexName = migrationInfo.getNextIndexName(); - logger.info("removing index [{}] from previous incomplete migration", newIndexName); - - migrationInfo.createClient(baseClient) - .admin() - .indices() - .prepareDelete(newIndexName) - .execute(ActionListener.wrap(ackedResponse -> { - if (ackedResponse.isAcknowledged()) { - logger.debug("successfully removed index [{}]", newIndexName); - clearResults(clusterService, ActionListener.wrap(listener::accept, this::markAsFailed)); - } - }, this::markAsFailed)); - } else { - logger.debug("no incomplete index to remove"); - clearResults(clusterService, ActionListener.wrap(listener::accept, this::markAsFailed)); - } + clearResults( + clusterService, + ActionListener.wrap( + state -> prepareNextIndex(state2 -> migrateSingleIndex(state2, this::finishIndexAndLoop), stateFeatureName), + this::markAsFailed + ) + ); } private void finishIndexAndLoop(BulkByScrollResponse bulkResponse) { @@ -289,11 +254,7 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { }, this::markAsFailed) ); } else { - prepareNextIndex( - clusterService.state(), - state2 -> migrateSingleIndex(state2, this::finishIndexAndLoop), - lastMigrationInfo.getFeatureName() - ); + prepareNextIndex(state2 -> migrateSingleIndex(state2, this::finishIndexAndLoop), lastMigrationInfo.getFeatureName()); } } @@ -303,7 +264,6 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { SingleFeatureMigrationResult.success(), ActionListener.wrap(state -> { prepareNextIndex( - state, clusterState -> migrateSingleIndex(clusterState, this::finishIndexAndLoop), lastMigrationInfo.getFeatureName() ); @@ -312,7 +272,7 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { updateTask.submit(clusterService); } - private void prepareNextIndex(ClusterState clusterState, Consumer listener, String lastFeatureName) { + private void prepareNextIndex(Consumer listener, String lastFeatureName) { synchronized (migrationQueue) { assert migrationQueue != null; if (migrationQueue.isEmpty()) { @@ -424,7 +384,7 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { logger.info("migrating index [{}] from feature [{}] to new index [{}]", oldIndexName, migrationInfo.getFeatureName(), newIndexName); ActionListener innerListener = ActionListener.wrap(listener::accept, this::markAsFailed); try { - createIndex(migrationInfo, innerListener.delegateFailureAndWrap((delegate, shardsAcknowledgedResponse) -> { + createIndexRetryOnFailure(migrationInfo, innerListener.delegateFailureAndWrap((delegate, shardsAcknowledgedResponse) -> { logger.debug( "while migrating [{}] , got create index response: [{}]", oldIndexName, @@ -509,6 +469,8 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { } private void createIndex(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { + logger.info("creating new system index [{}] from feature [{}]", migrationInfo.getNextIndexName(), migrationInfo.getFeatureName()); + final CreateIndexClusterStateUpdateRequest createRequest = new CreateIndexClusterStateUpdateRequest( "migrate-system-index", migrationInfo.getNextIndexName(), @@ -534,6 +496,35 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { ); } + private void createIndexRetryOnFailure(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { + createIndex(migrationInfo, listener.delegateResponse((l, e) -> { + logger.warn("createIndex failed, retrying after removing index [{}] from previous attempt", migrationInfo.getNextIndexName()); + deleteIndex(migrationInfo, ActionListener.wrap(cleanupResponse -> createIndex(migrationInfo, l.delegateResponse((l3, e3) -> { + logger.error( + "createIndex failed after retrying, aborting system index migration. index: " + migrationInfo.getNextIndexName(), + e3 + ); + l.onFailure(e3); + })), e2 -> { + logger.error("deleteIndex failed, aborting system index migration. index: " + migrationInfo.getNextIndexName(), e2); + l.onFailure(e2); + })); + })); + } + + private void deleteIndex(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { + logger.info("removing index [{}] from feature [{}]", migrationInfo.getNextIndexName(), migrationInfo.getFeatureName()); + String newIndexName = migrationInfo.getNextIndexName(); + baseClient.admin().indices().prepareDelete(newIndexName).execute(ActionListener.wrap(ackedResponse -> { + if (ackedResponse.isAcknowledged()) { + logger.info("successfully removed index [{}]", newIndexName); + listener.onResponse(ackedResponse); + } else { + listener.onFailure(new ElasticsearchException("Failed to acknowledge index deletion for [" + newIndexName + "]")); + } + }, listener::onFailure)); + } + private void setAliasAndRemoveOldIndex(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { final IndicesAliasesRequestBuilder aliasesRequest = migrationInfo.createClient(baseClient).admin().indices().prepareAliases(); aliasesRequest.removeIndex(migrationInfo.getCurrentIndexName()); From 556281ace83ace816d0bbc77e6f6f352e23d4123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 13 Feb 2025 22:08:09 +0100 Subject: [PATCH 79/93] Removed unused "else" branch for no entitlements/no SM (#122354) --- .../java/org/elasticsearch/bootstrap/Elasticsearch.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index ba978f09dfef..ea7a0b5dcf47 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -249,7 +249,8 @@ class Elasticsearch { nodeEnv.configDir(), nodeEnv.tmpDir() ); - } else if (RuntimeVersionFeature.isSecurityManagerAvailable()) { + } else { + assert RuntimeVersionFeature.isSecurityManagerAvailable(); // no need to explicitly enable native access for legacy code pluginsLoader = PluginsLoader.createPluginsLoader(modulesBundles, pluginsBundles, Map.of()); // install SM after natives, shutdown hooks, etc. @@ -259,10 +260,6 @@ class Elasticsearch { SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(args.nodeSettings()), args.pidFile() ); - } else { - // TODO: should we throw/interrupt startup in this case? - pluginsLoader = PluginsLoader.createPluginsLoader(modulesBundles, pluginsBundles, Map.of()); - LogManager.getLogger(Elasticsearch.class).warn("Bootstrapping without any protection"); } bootstrap.setPluginsLoader(pluginsLoader); From 0810e66802e3b243f3201fe8c375155b3a6b0d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 13 Feb 2025 22:08:29 +0100 Subject: [PATCH 80/93] [Entitlements] Make `lookupImplementationMethod` inheritance-aware (#122474) --- .../impl/InstrumentationServiceImpl.java | 134 ++++++++---------- .../impl/InstrumentationServiceImplTests.java | 53 ++++++- .../InstrumentationService.java | 3 +- 3 files changed, 110 insertions(+), 80 deletions(-) diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java index 05a5af374e5d..ffcc23e16d1f 100644 --- a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java +++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java @@ -44,10 +44,11 @@ public class InstrumentationServiceImpl implements InstrumentationService { return InstrumenterImpl.create(clazz, methods); } - @Override - public Map lookupMethods(Class checkerClass) throws IOException { - Map methodsToInstrument = new HashMap<>(); + private interface CheckerMethodVisitor { + void visit(Class currentClass, int access, String checkerMethodName, String checkerMethodDescriptor); + } + private void visitClassAndSupers(Class checkerClass, CheckerMethodVisitor checkerMethodVisitor) throws ClassNotFoundException { Set> visitedClasses = new HashSet<>(); ArrayDeque> classesToVisit = new ArrayDeque<>(Collections.singleton(checkerClass)); while (classesToVisit.isEmpty() == false) { @@ -57,52 +58,61 @@ public class InstrumentationServiceImpl implements InstrumentationService { } visitedClasses.add(currentClass); - var classFileInfo = InstrumenterImpl.getClassFileInfo(currentClass); - ClassReader reader = new ClassReader(classFileInfo.bytecodes()); - ClassVisitor visitor = new ClassVisitor(Opcodes.ASM9) { + try { + var classFileInfo = InstrumenterImpl.getClassFileInfo(currentClass); + ClassReader reader = new ClassReader(classFileInfo.bytecodes()); + ClassVisitor visitor = new ClassVisitor(Opcodes.ASM9) { - @Override - public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { - super.visit(version, access, name, signature, superName, interfaces); - try { - if (OBJECT_INTERNAL_NAME.equals(superName) == false) { - classesToVisit.add(Class.forName(Type.getObjectType(superName).getClassName())); + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + try { + if (OBJECT_INTERNAL_NAME.equals(superName) == false) { + classesToVisit.add(Class.forName(Type.getObjectType(superName).getClassName())); + } + for (var interfaceName : interfaces) { + classesToVisit.add(Class.forName(Type.getObjectType(interfaceName).getClassName())); + } + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Cannot inspect checker class " + currentClass.getName(), e); } - for (var interfaceName : interfaces) { - classesToVisit.add(Class.forName(Type.getObjectType(interfaceName).getClassName())); - } - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("Cannot inspect checker class " + checkerClass.getName(), e); } - } - @Override - public MethodVisitor visitMethod( - int access, - String checkerMethodName, - String checkerMethodDescriptor, - String signature, - String[] exceptions - ) { - var mv = super.visitMethod(access, checkerMethodName, checkerMethodDescriptor, signature, exceptions); - if (checkerMethodName.startsWith(InstrumentationService.CHECK_METHOD_PREFIX)) { - var checkerMethodArgumentTypes = Type.getArgumentTypes(checkerMethodDescriptor); - var methodToInstrument = parseCheckerMethodSignature(checkerMethodName, checkerMethodArgumentTypes); - - var checkerParameterDescriptors = Arrays.stream(checkerMethodArgumentTypes).map(Type::getDescriptor).toList(); - var checkMethod = new CheckMethod( - Type.getInternalName(currentClass), - checkerMethodName, - checkerParameterDescriptors - ); - - methodsToInstrument.putIfAbsent(methodToInstrument, checkMethod); + @Override + public MethodVisitor visitMethod( + int access, + String checkerMethodName, + String checkerMethodDescriptor, + String signature, + String[] exceptions + ) { + var mv = super.visitMethod(access, checkerMethodName, checkerMethodDescriptor, signature, exceptions); + checkerMethodVisitor.visit(currentClass, access, checkerMethodName, checkerMethodDescriptor); + return mv; } - return mv; - } - }; - reader.accept(visitor, 0); + }; + reader.accept(visitor, 0); + } catch (IOException e) { + throw new ClassNotFoundException("Cannot find a definition for class [" + checkerClass.getName() + "]", e); + } } + } + + @Override + public Map lookupMethods(Class checkerClass) throws ClassNotFoundException { + Map methodsToInstrument = new HashMap<>(); + + visitClassAndSupers(checkerClass, (currentClass, access, checkerMethodName, checkerMethodDescriptor) -> { + if (checkerMethodName.startsWith(InstrumentationService.CHECK_METHOD_PREFIX)) { + var checkerMethodArgumentTypes = Type.getArgumentTypes(checkerMethodDescriptor); + var methodToInstrument = parseCheckerMethodSignature(checkerMethodName, checkerMethodArgumentTypes); + + var checkerParameterDescriptors = Arrays.stream(checkerMethodArgumentTypes).map(Type::getDescriptor).toList(); + var checkMethod = new CheckMethod(Type.getInternalName(currentClass), checkerMethodName, checkerParameterDescriptors); + methodsToInstrument.putIfAbsent(methodToInstrument, checkMethod); + } + }); + return methodsToInstrument; } @@ -110,14 +120,14 @@ public class InstrumentationServiceImpl implements InstrumentationService { @Override public InstrumentationInfo lookupImplementationMethod( Class targetSuperclass, - String methodName, + String targetMethodName, Class implementationClass, Class checkerClass, String checkMethodName, Class... parameterTypes ) throws NoSuchMethodException, ClassNotFoundException { - var targetMethod = targetSuperclass.getDeclaredMethod(methodName, parameterTypes); + var targetMethod = targetSuperclass.getDeclaredMethod(targetMethodName, parameterTypes); var implementationMethod = implementationClass.getMethod(targetMethod.getName(), targetMethod.getParameterTypes()); validateTargetMethod(implementationClass, targetMethod, implementationMethod); @@ -128,33 +138,15 @@ public class InstrumentationServiceImpl implements InstrumentationService { CheckMethod[] checkMethod = new CheckMethod[1]; - try { - InstrumenterImpl.ClassFileInfo classFileInfo = InstrumenterImpl.getClassFileInfo(checkerClass); - ClassReader reader = new ClassReader(classFileInfo.bytecodes()); - ClassVisitor visitor = new ClassVisitor(Opcodes.ASM9) { - @Override - public MethodVisitor visitMethod( - int access, - String methodName, - String methodDescriptor, - String signature, - String[] exceptions - ) { - var mv = super.visitMethod(access, methodName, methodDescriptor, signature, exceptions); - if (methodName.equals(checkMethodName)) { - var methodArgumentTypes = Type.getArgumentTypes(methodDescriptor); - if (Arrays.equals(methodArgumentTypes, checkMethodArgumentTypes)) { - var checkerParameterDescriptors = Arrays.stream(methodArgumentTypes).map(Type::getDescriptor).toList(); - checkMethod[0] = new CheckMethod(Type.getInternalName(checkerClass), methodName, checkerParameterDescriptors); - } - } - return mv; + visitClassAndSupers(checkerClass, (currentClass, access, methodName, methodDescriptor) -> { + if (methodName.equals(checkMethodName)) { + var methodArgumentTypes = Type.getArgumentTypes(methodDescriptor); + if (Arrays.equals(methodArgumentTypes, checkMethodArgumentTypes)) { + var checkerParameterDescriptors = Arrays.stream(methodArgumentTypes).map(Type::getDescriptor).toList(); + checkMethod[0] = new CheckMethod(Type.getInternalName(currentClass), methodName, checkerParameterDescriptors); } - }; - reader.accept(visitor, 0); - } catch (IOException e) { - throw new ClassNotFoundException("Cannot find a definition for class [" + checkerClass.getName() + "]", e); - } + } + }); if (checkMethod[0] == null) { throw new NoSuchMethodException( diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java index 2b9b70d46c0e..25689f0b8a63 100644 --- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.entitlement.instrumentation.MethodKey; import org.elasticsearch.test.ESTestCase; import org.objectweb.asm.Type; -import java.io.IOException; import java.util.List; import java.util.Map; @@ -90,7 +89,9 @@ public class InstrumentationServiceImplTests extends ESTestCase { void checkInstanceMethodManual(Class clazz, TestTargetBaseClass that, int x, String y); } - public void testInstrumentationTargetLookup() throws IOException { + interface TestCheckerDerived3 extends TestCheckerMixed {} + + public void testInstrumentationTargetLookup() throws ClassNotFoundException { Map checkMethods = instrumentationService.lookupMethods(TestChecker.class); assertThat(checkMethods, aMapWithSize(3)); @@ -143,7 +144,7 @@ public class InstrumentationServiceImplTests extends ESTestCase { ); } - public void testInstrumentationTargetLookupWithOverloads() throws IOException { + public void testInstrumentationTargetLookupWithOverloads() throws ClassNotFoundException { Map checkMethods = instrumentationService.lookupMethods(TestCheckerOverloads.class); assertThat(checkMethods, aMapWithSize(2)); @@ -175,7 +176,7 @@ public class InstrumentationServiceImplTests extends ESTestCase { ); } - public void testInstrumentationTargetLookupWithDerivedClass() throws IOException { + public void testInstrumentationTargetLookupWithDerivedClass() throws ClassNotFoundException { Map checkMethods = instrumentationService.lookupMethods(TestCheckerDerived2.class); assertThat(checkMethods, aMapWithSize(4)); @@ -244,7 +245,7 @@ public class InstrumentationServiceImplTests extends ESTestCase { ); } - public void testInstrumentationTargetLookupWithCtors() throws IOException { + public void testInstrumentationTargetLookupWithCtors() throws ClassNotFoundException { Map checkMethods = instrumentationService.lookupMethods(TestCheckerCtors.class); assertThat(checkMethods, aMapWithSize(2)); @@ -276,7 +277,7 @@ public class InstrumentationServiceImplTests extends ESTestCase { ); } - public void testInstrumentationTargetLookupWithExtraMethods() throws IOException { + public void testInstrumentationTargetLookupWithExtraMethods() throws ClassNotFoundException { Map checkMethods = instrumentationService.lookupMethods(TestCheckerMixed.class); assertThat(checkMethods, aMapWithSize(1)); @@ -371,7 +372,7 @@ public class InstrumentationServiceImplTests extends ESTestCase { ); } - public void testLookupImplementationMethodWithInheritance() throws ClassNotFoundException, NoSuchMethodException { + public void testLookupImplementationMethodWithInheritanceOnTarget() throws ClassNotFoundException, NoSuchMethodException { var info = instrumentationService.lookupImplementationMethod( TestTargetBaseClass.class, "instanceMethod2", @@ -409,6 +410,44 @@ public class InstrumentationServiceImplTests extends ESTestCase { ); } + public void testLookupImplementationMethodWithInheritanceOnChecker() throws ClassNotFoundException, NoSuchMethodException { + var info = instrumentationService.lookupImplementationMethod( + TestTargetBaseClass.class, + "instanceMethod2", + TestTargetImplementationClass.class, + TestCheckerDerived3.class, + "checkInstanceMethodManual", + int.class, + String.class + ); + + assertThat( + info.targetMethod(), + equalTo( + new MethodKey( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetIntermediateClass", + "instanceMethod2", + List.of("I", "java/lang/String") + ) + ) + ); + assertThat( + info.checkMethod(), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerMixed", + "checkInstanceMethodManual", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetBaseClass;", + "I", + "Ljava/lang/String;" + ) + ) + ) + ); + } + public void testParseCheckerMethodSignatureStaticMethod() { var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( "check$org_example_TestClass$$staticMethod", diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java index 79673418eb32..ece51a8414b7 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java @@ -9,7 +9,6 @@ package org.elasticsearch.entitlement.instrumentation; -import java.io.IOException; import java.util.Map; /** @@ -23,7 +22,7 @@ public interface InstrumentationService { Instrumenter newInstrumenter(Class clazz, Map methods); - Map lookupMethods(Class clazz) throws IOException; + Map lookupMethods(Class clazz) throws ClassNotFoundException; InstrumentationInfo lookupImplementationMethod( Class targetSuperclass, From 38cc72c06672f94bd5a275c568fa7f997821e0fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Thu, 13 Feb 2025 22:31:30 +0100 Subject: [PATCH 81/93] Fix ShuffleForcedMergePolicyTests#testDiagnostics (#122473) This commit ensures that opening the DirectoryReader won't merge all the segments into one, making the tested force merge a no-op. Closes #121336 --- muted-tests.yml | 3 --- .../index/engine/ShuffleForcedMergePolicyTests.java | 10 +++++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 1810e98f6ca0..535ef6095e80 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -291,9 +291,6 @@ tests: - class: org.elasticsearch.env.NodeEnvironmentTests method: testGetBestDowngradeVersion issue: https://github.com/elastic/elasticsearch/issues/121316 -- class: org.elasticsearch.index.engine.ShuffleForcedMergePolicyTests - method: testDiagnostics - issue: https://github.com/elastic/elasticsearch/issues/121336 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/rest-api/security/invalidate-tokens/line_194} issue: https://github.com/elastic/elasticsearch/issues/121337 diff --git a/server/src/test/java/org/elasticsearch/index/engine/ShuffleForcedMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/ShuffleForcedMergePolicyTests.java index 165472842d2a..4af179360a4f 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/ShuffleForcedMergePolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/ShuffleForcedMergePolicyTests.java @@ -20,6 +20,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.store.Directory; @@ -37,7 +38,14 @@ public class ShuffleForcedMergePolicyTests extends BaseMergePolicyTestCase { IndexWriterConfig iwc = newIndexWriterConfig(); // Disable merging on flush. iwc.setMaxFullFlushMergeWaitMillis(0L); - MergePolicy mp = new ShuffleForcedMergePolicy(newTieredMergePolicy()); + // Even though we set setMaxFullFlushMergeWaitMillis=0, opening the DirectoryReader + // might trigger a merge after flushing the in-memory documents. Therefore, we set + // a high enough number of maxSegmentsPerTier (we index at most 300 documents, and we flush + // a new segment per 10 documents) that would prevent merging all the segments into one and + // making the force merge a no-op. + var tieredMergePolicy = new TieredMergePolicy(); + tieredMergePolicy.setSegmentsPerTier(100); + MergePolicy mp = new ShuffleForcedMergePolicy(tieredMergePolicy); iwc.setMergePolicy(mp); boolean sorted = random().nextBoolean(); if (sorted) { From 94bf12a4b26392d20a26f80dcdca7123394c404c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 14 Feb 2025 08:35:53 +1100 Subject: [PATCH 82/93] Mute org.elasticsearch.telemetry.apm.ApmAgentSettingsIT org.elasticsearch.telemetry.apm.ApmAgentSettingsIT #122546 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 535ef6095e80..e86d0d9de7d3 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -380,6 +380,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122377 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/122378 +- class: org.elasticsearch.telemetry.apm.ApmAgentSettingsIT + issue: https://github.com/elastic/elasticsearch/issues/122546 # Examples: # From 8b1397fb87cc26648d15649530490de5678bdc10 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 14 Feb 2025 08:57:25 +1100 Subject: [PATCH 83/93] Mute org.elasticsearch.search.SearchTimeoutIT testSuggestTimeoutWithPartialResults #122548 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e86d0d9de7d3..6b163b5fffd0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -382,6 +382,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122378 - class: org.elasticsearch.telemetry.apm.ApmAgentSettingsIT issue: https://github.com/elastic/elasticsearch/issues/122546 +- class: org.elasticsearch.search.SearchTimeoutIT + method: testSuggestTimeoutWithPartialResults + issue: https://github.com/elastic/elasticsearch/issues/122548 # Examples: # From 61c8fb299efe6d8950bdacf58e6508f0fe796ead Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 14 Feb 2025 09:19:09 +1100 Subject: [PATCH 84/93] Mute org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests testSnapshotRecovery {p0=false p1=false} #122549 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 6b163b5fffd0..1ebe20b11038 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -385,6 +385,9 @@ tests: - class: org.elasticsearch.search.SearchTimeoutIT method: testSuggestTimeoutWithPartialResults issue: https://github.com/elastic/elasticsearch/issues/122548 +- class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests + method: testSnapshotRecovery {p0=false p1=false} + issue: https://github.com/elastic/elasticsearch/issues/122549 # Examples: # From 45167d5a381f3708526015902b15060c4cc76c48 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 14 Feb 2025 09:19:18 +1100 Subject: [PATCH 85/93] Mute org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests testSnapshotRecovery {p0=true p1=false} #122550 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1ebe20b11038..9c222a7d384f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -388,6 +388,9 @@ tests: - class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests method: testSnapshotRecovery {p0=false p1=false} issue: https://github.com/elastic/elasticsearch/issues/122549 +- class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests + method: testSnapshotRecovery {p0=true p1=false} + issue: https://github.com/elastic/elasticsearch/issues/122550 # Examples: # From b0ded613aed758d252eb03a8723f34d0b713c986 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 14 Feb 2025 09:19:26 +1100 Subject: [PATCH 86/93] Mute org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests testSnapshotRecovery {p0=false p1=true} #122551 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9c222a7d384f..e20d1cff07c8 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -391,6 +391,9 @@ tests: - class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests method: testSnapshotRecovery {p0=true p1=false} issue: https://github.com/elastic/elasticsearch/issues/122550 +- class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests + method: testSnapshotRecovery {p0=false p1=true} + issue: https://github.com/elastic/elasticsearch/issues/122551 # Examples: # From d3ed999356cc7ee178710ea093fbb8a3e6ed69a6 Mon Sep 17 00:00:00 2001 From: Gal Lalouche Date: Fri, 14 Feb 2025 00:28:35 +0200 Subject: [PATCH 87/93] ESQL: Initial support for unmapped fields (#119886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds initial support for unmapped fields, using the INSIST clause. For starters, this unmapped fields without a cast. Note that the INSIST keyword is potentially a placeholder, as the method of defining an unmapped field might change in the future, e.g., use a special magic function. As this is currently under development, the actual syntax is INSIST_🐔. First stage of #120072. Specifically, the following features are implemented in this PR: * Support for INSIST keyword without a cast. In particular, if the type being INSISTed upon is mapped to anything other than KEYWORD, it will result in an InvalidMappedField. There is no support for union type resolution on top of INSIST. Future PRs will handle these conflicts. There is support for multiple parameters, or INSIST on top of INSIST which is equivalent. * Enforcing that INSIST must always be on top of a FROM or another INSIST. While this may change in the future, e.g., handling cases like `FROM foo | EVAL x = 1 | INSIST bar` will not be done in this PR, as it makes handling INSIST too complicated. --- docs/changelog/119886.yaml | 5 + .../index/mapper/BlockSourceReader.java | 3 +- .../xpack/esql/core/type/EsField.java | 3 +- .../esql/core/type/InvalidMappedField.java | 22 +- .../PotentiallyUnmappedKeywordEsField.java | 30 + .../esql/qa/mixed/MixedClusterEsqlSpecIT.java | 6 + .../xpack/esql/ccq/MultiClusterSpecIT.java | 8 + .../xpack/esql/qa/multi_node/EsqlSpecIT.java | 6 +- .../xpack/esql/qa/single_node/EsqlSpecIT.java | 5 + .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 15 +- .../rest/generative/GenerativeRestTest.java | 2 +- .../xpack/esql/CsvTestsDataLoader.java | 92 +- .../xpack/esql/EsqlTestUtils.java | 8 + .../data/partial_mapping_sample_data.csv | 8 + .../mapping-no_mapping_sample_data.json | 4 + ...l_mapping_excluded_source_sample_data.json | 13 + ...partial_mapping_no_source_sample_data.json | 11 + .../mapping-partial_mapping_sample_data.json | 17 + .../main/resources/unmapped_fields.csv-spec | 582 +++++ .../esql/src/main/antlr/EsqlBaseLexer.g4 | 15 +- .../esql/src/main/antlr/EsqlBaseLexer.tokens | 328 +-- .../esql/src/main/antlr/EsqlBaseParser.g4 | 5 + .../esql/src/main/antlr/EsqlBaseParser.tokens | 328 +-- .../xpack/esql/action/EsqlCapabilities.java | 10 + .../xpack/esql/analysis/Analyzer.java | 63 +- .../xpack/esql/analysis/Verifier.java | 14 +- .../xpack/esql/index/EsIndex.java | 23 +- .../esql/optimizer/LogicalPlanOptimizer.java | 3 +- .../rules/logical/PropgateUnmappedFields.java | 43 + .../local/ReplaceMissingFieldWithNull.java | 10 +- .../xpack/esql/parser/EsqlBaseLexer.interp | 17 +- .../xpack/esql/parser/EsqlBaseLexer.java | 2321 +++++++++-------- .../xpack/esql/parser/EsqlBaseParser.interp | 11 +- .../xpack/esql/parser/EsqlBaseParser.java | 1696 ++++++------ .../parser/EsqlBaseParserBaseListener.java | 12 + .../parser/EsqlBaseParserBaseVisitor.java | 7 + .../esql/parser/EsqlBaseParserListener.java | 10 + .../esql/parser/EsqlBaseParserVisitor.java | 6 + .../xpack/esql/parser/LogicalPlanBuilder.java | 17 + .../xpack/esql/plan/PlanWritables.java | 2 +- .../xpack/esql/plan/logical/EsRelation.java | 4 + .../xpack/esql/plan/logical/Insist.java | 91 + .../xpack/esql/plan/physical/EnrichExec.java | 5 +- .../planner/EsPhysicalOperationProviders.java | 66 +- .../xpack/esql/session/IndexResolver.java | 28 +- .../elasticsearch/xpack/esql/CsvTests.java | 25 +- .../xpack/esql/analysis/AnalyzerTests.java | 179 +- .../xpack/esql/analysis/VerifierTests.java | 9 + .../optimizer/LogicalPlanOptimizerTests.java | 74 +- .../esql/parser/StatementParserTests.java | 6 + .../TestPhysicalOperationProviders.java | 90 +- .../session/IndexResolverFieldNamesTests.java | 78 + .../esql/type/EsqlDataTypeRegistryTests.java | 2 +- 53 files changed, 4034 insertions(+), 2404 deletions(-) create mode 100644 docs/changelog/119886.yaml create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/PotentiallyUnmappedKeywordEsField.java create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/partial_mapping_sample_data.csv create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-no_mapping_sample_data.json create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_excluded_source_sample_data.json create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_no_source_sample_data.json create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_sample_data.json create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped_fields.csv-spec create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropgateUnmappedFields.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Insist.java diff --git a/docs/changelog/119886.yaml b/docs/changelog/119886.yaml new file mode 100644 index 000000000000..8b866637ddc4 --- /dev/null +++ b/docs/changelog/119886.yaml @@ -0,0 +1,5 @@ +pr: 119886 +summary: Initial support for unmapped fields +area: ES|QL +type: feature +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 19a1cce74617..7b3ecf365e44 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -22,6 +22,7 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * Loads values from {@code _source}. This whole process is very slow and cast-tastic, @@ -230,7 +231,7 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader { @Override protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, (String) v)); + ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, Objects.toString(v))); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index 73e2d5ec626a..321c79ee13a8 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -27,11 +27,12 @@ import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCache public class EsField implements Writeable { private static Map> readers = Map.ofEntries( - Map.entry("EsField", EsField::new), Map.entry("DateEsField", DateEsField::new), + Map.entry("EsField", EsField::new), Map.entry("InvalidMappedField", InvalidMappedField::new), Map.entry("KeywordEsField", KeywordEsField::new), Map.entry("MultiTypeEsField", MultiTypeEsField::new), + Map.entry("PotentiallyUnmappedKeywordEsField", PotentiallyUnmappedKeywordEsField::new), Map.entry("TextEsField", TextEsField::new), Map.entry("UnsupportedEsField", UnsupportedEsField::new) ); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java index f83e4652ebeb..f8337d0decae 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java @@ -45,7 +45,7 @@ public class InvalidMappedField extends EsField { * Constructor supporting union types, used in ES|QL. */ public InvalidMappedField(String name, Map> typesToIndices) { - this(name, makeErrorMessage(typesToIndices), new TreeMap<>(), typesToIndices); + this(name, makeErrorMessage(typesToIndices, false), new TreeMap<>(), typesToIndices); } private InvalidMappedField(String name, String errorMessage, Map properties, Map> typesToIndices) { @@ -107,12 +107,21 @@ public class InvalidMappedField extends EsField { return typesToIndices; } - private static String makeErrorMessage(Map> typesToIndices) { + public static String makeErrorsMessageIncludingInsistKeyword(Map> typesToIndices) { + return makeErrorMessage(typesToIndices, true); + } + + private static String makeErrorMessage(Map> typesToIndices, boolean includeInsistKeyword) { StringBuilder errorMessage = new StringBuilder(); + var isInsistKeywordOnlyKeyword = includeInsistKeyword && typesToIndices.containsKey(DataType.KEYWORD.typeName()) == false; errorMessage.append("mapped as ["); - errorMessage.append(typesToIndices.size()); + errorMessage.append(typesToIndices.size() + (isInsistKeywordOnlyKeyword ? 1 : 0)); errorMessage.append("] incompatible types: "); boolean first = true; + if (isInsistKeywordOnlyKeyword) { + first = false; + errorMessage.append("[keyword] enforced by INSIST command"); + } for (Map.Entry> e : typesToIndices.entrySet()) { if (first) { first = false; @@ -121,7 +130,12 @@ public class InvalidMappedField extends EsField { } errorMessage.append("["); errorMessage.append(e.getKey()); - errorMessage.append("] in "); + errorMessage.append("] "); + if (e.getKey().equals(DataType.KEYWORD.typeName()) && includeInsistKeyword) { + errorMessage.append("enforced by INSIST command and in "); + } else { + errorMessage.append("in "); + } if (e.getValue().size() <= 3) { errorMessage.append(e.getValue()); } else { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/PotentiallyUnmappedKeywordEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/PotentiallyUnmappedKeywordEsField.java new file mode 100644 index 000000000000..8672b6b61dee --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/PotentiallyUnmappedKeywordEsField.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.core.type; + +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * This class is used as a marker for fields that may be unmapped, where an unmapped field is a field which exists in the _source but is not + * mapped in the index. Note that this field may be mapped for some indices, but is unmapped in at least one of them. + * For indices where the field is unmapped, we will try to load them directly from _source. + */ +public class PotentiallyUnmappedKeywordEsField extends KeywordEsField { + public PotentiallyUnmappedKeywordEsField(String name) { + super(name); + } + + public PotentiallyUnmappedKeywordEsField(StreamInput in) throws IOException { + super(in); + } + + public String getWriteableName() { + return "PotentiallyUnmappedKeywordEsField"; + } +} diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java index 9623d6071d32..e12598a3dc03 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java @@ -21,6 +21,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SOURCE_FIELD_MAPPING; public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase { @ClassRule @@ -90,6 +91,11 @@ public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase { return hasCapabilities(List.of(JOIN_LOOKUP_V12.capabilityName())); } + @Override + protected boolean supportsSourceFieldMapping() throws IOException { + return hasCapabilities(List.of(SOURCE_FIELD_MAPPING.capabilityName())); + } + @Override protected boolean deduplicateExactWarnings() { /* diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 723c5e2dfd1a..6ffd9d8c9231 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -52,6 +52,7 @@ import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTA import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS; import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -127,6 +128,8 @@ public class MultiClusterSpecIT extends EsqlSpecTestCase { assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V3.capabilityName())); assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())); + // Unmapped fields require a coorect capability response from every cluster, which isn't currently implemented. + assumeFalse("UNMAPPED FIELDS not yet supported in CCS", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName())); } @Override @@ -294,4 +297,9 @@ public class MultiClusterSpecIT extends EsqlSpecTestCase { // return hasCapabilities(List.of(JOIN_LOOKUP_V10.capabilityName())); return false; } + + @Override + protected boolean supportsSourceFieldMapping() { + return false; + } } diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java index c0e82a455a4f..09083aa0445e 100644 --- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java @@ -12,6 +12,8 @@ import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase; import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase; import org.junit.ClassRule; +import java.io.IOException; + public class EsqlSpecIT extends EsqlSpecTestCase { @ClassRule public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test")); @@ -39,7 +41,7 @@ public class EsqlSpecIT extends EsqlSpecTestCase { } @Override - protected boolean shouldSkipTestsWithSemanticTextFields() { - return true; + protected boolean supportsSourceFieldMapping() throws IOException { + return false; } } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java index 42974795a77d..8b83e7689c7d 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java @@ -47,4 +47,9 @@ public class EsqlSpecIT extends EsqlSpecTestCase { protected boolean shouldSkipTestsWithSemanticTextFields() { return cluster.getNumNodes() > 1; } + + @Override + protected boolean supportsSourceFieldMapping() { + return cluster.getNumNodes() == 1; + } } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index d5e4651a847c..98cad3103222 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -71,6 +71,7 @@ import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.deleteInferenceEnd import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs; import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SEMANTIC_TEXT_TYPE; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SOURCE_FIELD_MAPPING; // This test can run very long in serverless configurations @TimeoutSuite(millis = 30 * TimeUnits.MINUTE) @@ -135,8 +136,11 @@ public abstract class EsqlSpecTestCase extends ESRestTestCase { if (supportsInferenceTestService() && clusterHasInferenceEndpoint(client()) == false) { createInferenceEndpoint(client()); } - if (indexExists(availableDatasetsForEs(client(), supportsIndexModeLookup()).iterator().next().indexName()) == false) { - loadDataSetIntoEs(client(), supportsIndexModeLookup()); + + boolean supportsLookup = supportsIndexModeLookup(); + boolean supportsSourceMapping = supportsSourceFieldMapping(); + if (indexExists(availableDatasetsForEs(client(), supportsLookup, supportsSourceMapping).iterator().next().indexName()) == false) { + loadDataSetIntoEs(client(), supportsLookup, supportsSourceMapping); } } @@ -183,6 +187,9 @@ public abstract class EsqlSpecTestCase extends ESRestTestCase { if (shouldSkipTestsWithSemanticTextFields()) { assumeFalse("semantic_text tests are muted", testCase.requiredCapabilities.contains(SEMANTIC_TEXT_TYPE.capabilityName())); } + if (supportsSourceFieldMapping() == false) { + assumeFalse("source mapping tests are muted", testCase.requiredCapabilities.contains(SOURCE_FIELD_MAPPING.capabilityName())); + } } protected static void checkCapabilities(RestClient client, TestFeatureService testFeatureService, String testName, CsvTestCase testCase) @@ -240,6 +247,10 @@ public abstract class EsqlSpecTestCase extends ESRestTestCase { return true; } + protected boolean supportsSourceFieldMapping() throws IOException { + return true; + } + protected final void doTest() throws Throwable { RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values())); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index a841c2fc9995..0ceeb132f5b5 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -57,7 +57,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase { @Before public void setup() throws IOException { if (indexExists(CSV_DATASET_MAP.keySet().iterator().next()) == false) { - loadDataSetIntoEs(client(), true); + loadDataSetIntoEs(client(), true, true); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index fbd4f9feca78..c66ffb37184e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -42,6 +42,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.CsvTestUtils.COMMA_ESCAPING_REGEX; @@ -83,6 +85,22 @@ public class CsvTestsDataLoader { .withData("sample_data_ts_nanos.csv") .withTypeMapping(Map.of("@timestamp", "date_nanos")); private static final TestDataset MISSING_IP_SAMPLE_DATA = new TestDataset("missing_ip_sample_data"); + private static final TestDataset SAMPLE_DATA_PARTIAL_MAPPING = new TestDataset("partial_mapping_sample_data"); + private static final TestDataset SAMPLE_DATA_NO_MAPPING = new TestDataset( + "no_mapping_sample_data", + "mapping-no_mapping_sample_data.json", + "partial_mapping_sample_data.csv" + ).withTypeMapping(Stream.of("timestamp", "client_ip", "event_duration").collect(Collectors.toMap(k -> k, k -> "keyword"))); + private static final TestDataset SAMPLE_DATA_PARTIAL_MAPPING_NO_SOURCE = new TestDataset( + "partial_mapping_no_source_sample_data", + "mapping-partial_mapping_no_source_sample_data.json", + "partial_mapping_sample_data.csv" + ).withSetting("source_parameters-settings.json"); + private static final TestDataset SAMPLE_DATA_PARTIAL_MAPPING_EXCLUDED_SOURCE = new TestDataset( + "partial_mapping_excluded_source_sample_data", + "mapping-partial_mapping_excluded_source_sample_data.json", + "partial_mapping_sample_data.csv" + ).withSetting("source_parameters-settings.json"); private static final TestDataset CLIENT_IPS = new TestDataset("clientips"); private static final TestDataset CLIENT_IPS_LOOKUP = CLIENT_IPS.withIndex("clientips_lookup") .withSetting("clientips_lookup-settings.json"); @@ -128,6 +146,10 @@ public class CsvTestsDataLoader { Map.entry(LANGUAGES_NESTED_FIELDS.indexName, LANGUAGES_NESTED_FIELDS), Map.entry(UL_LOGS.indexName, UL_LOGS), Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA), + Map.entry(SAMPLE_DATA_PARTIAL_MAPPING.indexName, SAMPLE_DATA_PARTIAL_MAPPING), + Map.entry(SAMPLE_DATA_NO_MAPPING.indexName, SAMPLE_DATA_NO_MAPPING), + Map.entry(SAMPLE_DATA_PARTIAL_MAPPING_NO_SOURCE.indexName, SAMPLE_DATA_PARTIAL_MAPPING_NO_SOURCE), + Map.entry(SAMPLE_DATA_PARTIAL_MAPPING_EXCLUDED_SOURCE.indexName, SAMPLE_DATA_PARTIAL_MAPPING_EXCLUDED_SOURCE), Map.entry(MV_SAMPLE_DATA.indexName, MV_SAMPLE_DATA), Map.entry(ALERTS.indexName, ALERTS), Map.entry(SAMPLE_DATA_STR.indexName, SAMPLE_DATA_STR), @@ -248,7 +270,7 @@ public class CsvTestsDataLoader { } try (RestClient client = builder.build()) { - loadDataSetIntoEs(client, true, (restClient, indexName, indexMapping, indexSettings) -> { + loadDataSetIntoEs(client, true, true, (restClient, indexName, indexMapping, indexSettings) -> { // don't use ESRestTestCase methods here or, if you do, test running the main method before making the change StringBuilder jsonBody = new StringBuilder("{"); if (indexSettings != null && indexSettings.isEmpty() == false) { @@ -267,14 +289,19 @@ public class CsvTestsDataLoader { } } - public static Set availableDatasetsForEs(RestClient client, boolean supportsIndexModeLookup) throws IOException { + public static Set availableDatasetsForEs( + RestClient client, + boolean supportsIndexModeLookup, + boolean supportsSourceFieldMapping + ) throws IOException { boolean inferenceEnabled = clusterHasInferenceEndpoint(client); Set testDataSets = new HashSet<>(); for (TestDataset dataset : CSV_DATASET_MAP.values()) { if ((inferenceEnabled || dataset.requiresInferenceEndpoint == false) - && (supportsIndexModeLookup || isLookupDataset(dataset) == false)) { + && (supportsIndexModeLookup || isLookupDataset(dataset) == false) + && (supportsSourceFieldMapping || isSourceMappingDataset(dataset) == false)) { testDataSets.add(dataset); } } @@ -282,24 +309,44 @@ public class CsvTestsDataLoader { return testDataSets; } - public static boolean isLookupDataset(TestDataset dataset) throws IOException { + private static boolean isLookupDataset(TestDataset dataset) throws IOException { Settings settings = dataset.readSettingsFile(); String mode = settings.get("index.mode"); return (mode != null && mode.equalsIgnoreCase("lookup")); } - public static void loadDataSetIntoEs(RestClient client, boolean supportsIndexModeLookup) throws IOException { - loadDataSetIntoEs(client, supportsIndexModeLookup, (restClient, indexName, indexMapping, indexSettings) -> { - ESRestTestCase.createIndex(restClient, indexName, indexSettings, indexMapping, null); - }); + private static boolean isSourceMappingDataset(TestDataset dataset) throws IOException { + if (dataset.mappingFileName() == null) { + return true; + } + String mappingJsonText = readTextFile(getResource("/" + dataset.mappingFileName())); + JsonNode mappingNode = new ObjectMapper().readTree(mappingJsonText); + // BWC tests don't support _source field mappings, so don't load those datasets. + return mappingNode.get("_source") != null; } - private static void loadDataSetIntoEs(RestClient client, boolean supportsIndexModeLookup, IndexCreator indexCreator) + public static void loadDataSetIntoEs(RestClient client, boolean supportsIndexModeLookup, boolean supportsSourceFieldMapping) throws IOException { + loadDataSetIntoEs( + client, + supportsIndexModeLookup, + supportsSourceFieldMapping, + (restClient, indexName, indexMapping, indexSettings) -> { + ESRestTestCase.createIndex(restClient, indexName, indexSettings, indexMapping, null); + } + ); + } + + private static void loadDataSetIntoEs( + RestClient client, + boolean supportsIndexModeLookup, + boolean supportsSourceFieldMapping, + IndexCreator indexCreator + ) throws IOException { Logger logger = LogManager.getLogger(CsvTestsDataLoader.class); Set loadedDatasets = new HashSet<>(); - for (var dataset : availableDatasetsForEs(client, supportsIndexModeLookup)) { + for (var dataset : availableDatasetsForEs(client, supportsIndexModeLookup, supportsSourceFieldMapping)) { load(client, dataset, logger, indexCreator); loadedDatasets.add(dataset.indexName); } @@ -351,10 +398,7 @@ public class CsvTestsDataLoader { } private static void loadEnrichPolicy(RestClient client, String policyName, String policyFileName, Logger logger) throws IOException { - URL policyMapping = CsvTestsDataLoader.class.getResource("/" + policyFileName); - if (policyMapping == null) { - throw new IllegalArgumentException("Cannot find resource " + policyFileName); - } + URL policyMapping = getResource("/" + policyFileName); String entity = readTextFile(policyMapping); Request request = new Request("PUT", "/_enrich/policy/" + policyName); request.setJsonEntity(entity); @@ -364,17 +408,17 @@ public class CsvTestsDataLoader { client.performRequest(request); } + private static URL getResource(String name) { + URL result = CsvTestsDataLoader.class.getResource(name); + if (result == null) { + throw new IllegalArgumentException("Cannot find resource " + name); + } + return result; + } + private static void load(RestClient client, TestDataset dataset, Logger logger, IndexCreator indexCreator) throws IOException { - final String mappingName = "/" + dataset.mappingFileName; - URL mapping = CsvTestsDataLoader.class.getResource(mappingName); - if (mapping == null) { - throw new IllegalArgumentException("Cannot find resource " + mappingName); - } - final String dataName = "/data/" + dataset.dataFileName; - URL data = CsvTestsDataLoader.class.getResource(dataName); - if (data == null) { - throw new IllegalArgumentException("Cannot find resource " + dataName); - } + URL mapping = getResource("/" + dataset.mappingFileName); + URL data = getResource("/data/" + dataset.dataFileName); Settings indexSettings = dataset.readSettingsFile(); indexCreator.createIndex(client, dataset.indexName, readMappingFile(mapping, dataset.typeMapping), indexSettings); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 6deda725dcad..dc9bda8581f2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -100,6 +100,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Period; import java.util.ArrayList; +import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -141,6 +142,8 @@ import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassification.IDENTIFIER; import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassification.PATTERN; import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassification.VALUE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -841,4 +844,9 @@ public final class EsqlTestUtils { ExceptionsHelper.unwrapCausesAndSuppressed(e, t -> t instanceof RemoteTransportException) .ifPresent(transportFailure -> assertNull("remote transport exception must be unwrapped", transportFailure.getCause())); } + + public static T singleValue(Collection collection) { + assertThat(collection, hasSize(1)); + return collection.iterator().next(); + } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/partial_mapping_sample_data.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/partial_mapping_sample_data.csv new file mode 100644 index 000000000000..a7782a3c429a --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/partial_mapping_sample_data.csv @@ -0,0 +1,8 @@ +@timestamp:date,client_ip:ip,event_duration:long,message:keyword,unmapped_message:keyword,unmapped_event_duration:keyword,unmapped.nested:keyword +2024-10-23T13:55:01.543Z,173.21.3.15,1756466,Connected to 10.1.0.1!,Disconnected from 10.1.0.1,1756468,a +2024-10-23T13:53:55.832Z,173.21.3.15,5033754,Connection error?,Disconnection error,5033756,b +2024-10-23T13:52:55.015Z,173.21.3.15,8268152,Connection error?,Disconnection error,8268154,c +2024-10-23T13:51:54.732Z,173.21.3.15,725447,Connection error?,Disconnection error,725449,d +2024-10-23T13:33:34.937Z,173.21.0.5,1232381,42,43,1232383,e +2024-10-23T12:27:28.948Z,173.21.2.113,2764888,Connected to 10.1.0.2!,Disconnected from 10.1.0.2,2764890,f +2024-10-23T12:15:03.360Z,173.21.2.162,3450232,Connected to 10.1.0.3!,Disconnected from 10.1.0.3,3450234,g diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-no_mapping_sample_data.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-no_mapping_sample_data.json new file mode 100644 index 000000000000..d2ae900835e4 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-no_mapping_sample_data.json @@ -0,0 +1,4 @@ +{ + "dynamic": "false", + "properties": {} +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_excluded_source_sample_data.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_excluded_source_sample_data.json new file mode 100644 index 000000000000..0f77e59f4dc3 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_excluded_source_sample_data.json @@ -0,0 +1,13 @@ +{ + "dynamic": "false", + "properties": { + "@timestamp": { + "type": "date" + } + }, + "_source": { + "excludes": [ + "message" + ] + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_no_source_sample_data.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_no_source_sample_data.json new file mode 100644 index 000000000000..64f209e8d64a --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_no_source_sample_data.json @@ -0,0 +1,11 @@ +{ + "dynamic": "false", + "properties": { + "@timestamp": { + "type": "date" + } + }, + "_source": { + "enabled": false + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_sample_data.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_sample_data.json new file mode 100644 index 000000000000..bb86a1428f59 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-partial_mapping_sample_data.json @@ -0,0 +1,17 @@ +{ + "dynamic": "false", + "properties": { + "@timestamp": { + "type": "date" + }, + "client_ip": { + "type": "ip" + }, + "event_duration": { + "type": "long" + }, + "message": { + "type": "keyword" + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped_fields.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped_fields.csv-spec new file mode 100644 index 000000000000..a0828ff628a6 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped_fields.csv-spec @@ -0,0 +1,582 @@ +###################### +# Single index tests # +###################### + +// This one is more of a test of the configuration than the unmapped fields feature. +doesNotLoadUnmappedFields +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| SORT @timestamp DESC +; + +@timestamp:datetime | client_ip:ip | event_duration:long | message:keyword +2024-10-23T13:55:01.543Z | 173.21.3.15 | 1756466 | Connected to 10.1.0.1! +2024-10-23T13:53:55.832Z | 173.21.3.15 | 5033754 | Connection error? +2024-10-23T13:52:55.015Z | 173.21.3.15 | 8268152 | Connection error? +2024-10-23T13:51:54.732Z | 173.21.3.15 | 725447 | Connection error? +2024-10-23T13:33:34.937Z | 173.21.0.5 | 1232381 | 42 +2024-10-23T12:27:28.948Z | 173.21.2.113 | 2764888 | Connected to 10.1.0.2! +2024-10-23T12:15:03.360Z | 173.21.2.162 | 3450232 | Connected to 10.1.0.3! +; + +fieldIsMappedToNonKeywordSingleIndex +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 client_ip +| KEEP @timestamp, client_ip +| SORT @timestamp DESC +; + +@timestamp:date | client_ip:ip +2024-10-23T13:55:01.543Z | 173.21.3.15 +2024-10-23T13:53:55.832Z | 173.21.3.15 +2024-10-23T13:52:55.015Z | 173.21.3.15 +2024-10-23T13:51:54.732Z | 173.21.3.15 +2024-10-23T13:33:34.937Z | 173.21.0.5 +2024-10-23T12:27:28.948Z | 173.21.2.113 +2024-10-23T12:15:03.360Z | 173.21.2.162 +; + +fieldIsMappedToKeywordSingleIndex +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 message +| KEEP @timestamp, message +| SORT @timestamp DESC +; + +@timestamp:datetime | message:keyword +2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! +2024-10-23T13:53:55.832Z | Connection error? +2024-10-23T13:52:55.015Z | Connection error? +2024-10-23T13:51:54.732Z | Connection error? +2024-10-23T13:33:34.937Z | 42 +2024-10-23T12:27:28.948Z | Connected to 10.1.0.2! +2024-10-23T12:15:03.360Z | Connected to 10.1.0.3! +; + +unmappedFieldAppearsLast +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 event_duration +| SORT @timestamp DESC +| Limit 1 +; + +@timestamp:date | client_ip:ip | message:keyword | event_duration:long +2024-10-23T13:55:01.543Z | 173.21.3.15 | Connected to 10.1.0.1! | 1756466 +; + +fieldDoesNotExistSingleIndex +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 foo +| KEEP @timestamp, foo +| SORT @timestamp DESC +; + +@timestamp:date | foo:keyword +2024-10-23T13:55:01.543Z | null +2024-10-23T13:53:55.832Z | null +2024-10-23T13:52:55.015Z | null +2024-10-23T13:51:54.732Z | null +2024-10-23T13:33:34.937Z | null +2024-10-23T12:27:28.948Z | null +2024-10-23T12:15:03.360Z | null +; + +fieldIsUnmappedSingleIndex +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 unmapped_message +| KEEP @timestamp, message, unmapped_message +| SORT @timestamp DESC +; + +@timestamp:date | message:keyword | unmapped_message:keyword +2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! | Disconnected from 10.1.0.1 +2024-10-23T13:53:55.832Z | Connection error? | Disconnection error +2024-10-23T13:52:55.015Z | Connection error? | Disconnection error +2024-10-23T13:51:54.732Z | Connection error? | Disconnection error +2024-10-23T13:33:34.937Z | 42 | 43 +2024-10-23T12:27:28.948Z | Connected to 10.1.0.2! | Disconnected from 10.1.0.2 +2024-10-23T12:15:03.360Z | Connected to 10.1.0.3! | Disconnected from 10.1.0.3 +; + +fieldIsUnmappedButSourceIsDisabledSingleIndex +required_capability: source_field_mapping +required_capability: unmapped_fields +FROM partial_mapping_no_source_sample_data +| INSIST_🐔 message +; + +@timestamp:date | message:keyword +2024-10-23T13:55:01.543Z | null +2024-10-23T13:53:55.832Z | null +2024-10-23T13:52:55.015Z | null +2024-10-23T13:51:54.732Z | null +2024-10-23T13:33:34.937Z | null +2024-10-23T12:27:28.948Z | null +2024-10-23T12:15:03.360Z | null +; + +fieldIsUnmappedButExcludedFromSourceSingleIndex +required_capability: source_field_mapping +required_capability: unmapped_fields +FROM partial_mapping_excluded_source_sample_data +| INSIST_🐔 message +| SORT @timestamp DESC +; + +@timestamp:date | message:keyword +2024-10-23T13:55:01.543Z | null +2024-10-23T13:53:55.832Z | null +2024-10-23T13:52:55.015Z | null +2024-10-23T13:51:54.732Z | null +2024-10-23T13:33:34.937Z | null +2024-10-23T12:27:28.948Z | null +2024-10-23T12:15:03.360Z | null +; + +fieldIsNestedAndMapped +required_capability: unmapped_fields +FROM addresses +| INSIST_🐔 city.name +| KEEP city.name +| SORT city.name DESC +; + +city.name:keyword +Tokyo +San Francisco +Amsterdam +; + +fieldIsNestedAndUnmapped +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 unmapped.nested +| KEEP @timestamp, unmapped.nested +| SORT @timestamp +; + +@timestamp:date | unmapped.nested:keyword +2024-10-23T12:15:03.360Z | g +2024-10-23T12:27:28.948Z | f +2024-10-23T13:33:34.937Z | e +2024-10-23T13:51:54.732Z | d +2024-10-23T13:52:55.015Z | c +2024-10-23T13:53:55.832Z | b +2024-10-23T13:55:01.543Z | a +; + +fieldIsNestedAndNonExistent +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 unmapped.nested.nonexistent +| KEEP @timestamp, unmapped.nested.nonexistent +| SORT @timestamp +; + +@timestamp:date | unmapped.nested.nonexistent:keyword +2024-10-23T12:15:03.360Z | null +2024-10-23T12:27:28.948Z | null +2024-10-23T13:33:34.937Z | null +2024-10-23T13:51:54.732Z | null +2024-10-23T13:52:55.015Z | null +2024-10-23T13:53:55.832Z | null +2024-10-23T13:55:01.543Z | null +; + +######################### +# Multi-parameter tests # +######################### + +noFieldExistsMultiParametersSingleIndex +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 foo, bar, bazz +| KEEP @timestamp, foo, bar, bazz +| SORT @timestamp DESC +; + +@timestamp:date | foo:keyword | bar:keyword | bazz:keyword +2024-10-23T13:55:01.543Z | null | null | null +2024-10-23T13:53:55.832Z | null | null | null +2024-10-23T13:52:55.015Z | null | null | null +2024-10-23T13:51:54.732Z | null | null | null +2024-10-23T13:33:34.937Z | null | null | null +2024-10-23T12:27:28.948Z | null | null | null +2024-10-23T12:15:03.360Z | null | null | null +; + +mixedFieldsMultiParametersSingleIndex +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 foo, message, unmapped_message +| KEEP @timestamp, foo, message, unmapped_message +| SORT @timestamp DESC +; + +@timestamp:date | foo:keyword | message:keyword | unmapped_message:keyword +2024-10-23T13:55:01.543Z | null | Connected to 10.1.0.1! | Disconnected from 10.1.0.1 +2024-10-23T13:53:55.832Z | null | Connection error? | Disconnection error +2024-10-23T13:52:55.015Z | null | Connection error? | Disconnection error +2024-10-23T13:51:54.732Z | null | Connection error? | Disconnection error +2024-10-23T13:33:34.937Z | null | 42 | 43 +2024-10-23T12:27:28.948Z | null | Connected to 10.1.0.2! | Disconnected from 10.1.0.2 +2024-10-23T12:15:03.360Z | null | Connected to 10.1.0.3! | Disconnected from 10.1.0.3 +; + +repeatedInsistFieldsUseTheLastEntry +required_capability: unmapped_fields +FROM partial_mapping_sample_data +| INSIST_🐔 unmapped_message, foo, message, foo, message, unmapped_message +| KEEP @timestamp, foo, message, unmapped_message +| SORT @timestamp DESC +; + +@timestamp:date | foo:keyword | message:keyword | unmapped_message:keyword +2024-10-23T13:55:01.543Z | null | Connected to 10.1.0.1! | Disconnected from 10.1.0.1 +2024-10-23T13:53:55.832Z | null | Connection error? | Disconnection error +2024-10-23T13:52:55.015Z | null | Connection error? | Disconnection error +2024-10-23T13:51:54.732Z | null | Connection error? | Disconnection error +2024-10-23T13:33:34.937Z | null | 42 | 43 +2024-10-23T12:27:28.948Z | null | Connected to 10.1.0.2! | Disconnected from 10.1.0.2 +2024-10-23T12:15:03.360Z | null | Connected to 10.1.0.3! | Disconnected from 10.1.0.3 +; + +##################### +# Multi index tests # +##################### + +mixedFieldsMultiParametersMultiIndex +required_capability: unmapped_fields +required_capability: index_metadata_field +FROM partial_mapping_sample_data, sample_data METADATA _index +| INSIST_🐔 foo, message, unmapped_message +| KEEP _index, @timestamp, foo, message, unmapped_message +| SORT @timestamp DESC +; + +_index:keyword | @timestamp:datetime | foo:keyword | message:keyword | unmapped_message:keyword +partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | null | Connected to 10.1.0.1! | Disconnected from 10.1.0.1 +partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | null | Connection error? | Disconnection error +partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | null | Connection error? | Disconnection error +partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | null | Connection error? | Disconnection error +partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | null | 42 | 43 +partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | null | Connected to 10.1.0.2! | Disconnected from 10.1.0.2 +partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | null | Connected to 10.1.0.3! | Disconnected from 10.1.0.3 +sample_data | 2023-10-23T13:55:01.543Z | null | Connected to 10.1.0.1 | null +sample_data | 2023-10-23T13:53:55.832Z | null | Connection error | null +sample_data | 2023-10-23T13:52:55.015Z | null | Connection error | null +sample_data | 2023-10-23T13:51:54.732Z | null | Connection error | null +sample_data | 2023-10-23T13:33:34.937Z | null | Disconnected | null +sample_data | 2023-10-23T12:27:28.948Z | null | Connected to 10.1.0.2 | null +sample_data | 2023-10-23T12:15:03.360Z | null | Connected to 10.1.0.3 | null +; + +insistOnTopOfInsistMultiIndex +required_capability: unmapped_fields +required_capability: index_metadata_field +FROM partial_mapping_sample_data, sample_data METADATA _index +| INSIST_🐔 foo, message +| INSIST_🐔 unmapped_message +| KEEP _index, @timestamp, foo, message, unmapped_message +| SORT @timestamp DESC +; + +_index:keyword | @timestamp:datetime | foo:keyword | message:keyword | unmapped_message:keyword +partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | null | Connected to 10.1.0.1! | Disconnected from 10.1.0.1 +partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | null | Connection error? | Disconnection error +partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | null | Connection error? | Disconnection error +partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | null | Connection error? | Disconnection error +partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | null | 42 | 43 +partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | null | Connected to 10.1.0.2! | Disconnected from 10.1.0.2 +partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | null | Connected to 10.1.0.3! | Disconnected from 10.1.0.3 +sample_data | 2023-10-23T13:55:01.543Z | null | Connected to 10.1.0.1 | null +sample_data | 2023-10-23T13:53:55.832Z | null | Connection error | null +sample_data | 2023-10-23T13:52:55.015Z | null | Connection error | null +sample_data | 2023-10-23T13:51:54.732Z | null | Connection error | null +sample_data | 2023-10-23T13:33:34.937Z | null | Disconnected | null +sample_data | 2023-10-23T12:27:28.948Z | null | Connected to 10.1.0.2 | null +sample_data | 2023-10-23T12:15:03.360Z | null | Connected to 10.1.0.3 | null +; + +fieldDoesNotExistMultiIndex +required_capability: index_metadata_field +required_capability: unmapped_fields +FROM partial_mapping_sample_data, sample_data METADATA _index +| INSIST_🐔 foo +| KEEP _index, @timestamp, foo +| SORT @timestamp DESC +; + +_index:keyword | @timestamp:date | foo:keyword +partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | null +partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | null +partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | null +partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | null +partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | null +partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | null +partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | null +sample_data | 2023-10-23T13:55:01.543Z | null +sample_data | 2023-10-23T13:53:55.832Z | null +sample_data | 2023-10-23T13:52:55.015Z | null +sample_data | 2023-10-23T13:51:54.732Z | null +sample_data | 2023-10-23T13:33:34.937Z | null +sample_data | 2023-10-23T12:27:28.948Z | null +sample_data | 2023-10-23T12:15:03.360Z | null +; + +fieldIsUnmappedMultiIndex +required_capability: index_metadata_field +required_capability: unmapped_fields +FROM partial_mapping_sample_data, sample_data METADATA _index +| INSIST_🐔 unmapped_message +| KEEP @timestamp, message, unmapped_message, _index +| SORT @timestamp DESC +; + +@timestamp:date | message:keyword | unmapped_message:keyword | _index:keyword +2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! | Disconnected from 10.1.0.1 | partial_mapping_sample_data +2024-10-23T13:53:55.832Z | Connection error? | Disconnection error | partial_mapping_sample_data +2024-10-23T13:52:55.015Z | Connection error? | Disconnection error | partial_mapping_sample_data +2024-10-23T13:51:54.732Z | Connection error? | Disconnection error | partial_mapping_sample_data +2024-10-23T13:33:34.937Z | 42 | 43 | partial_mapping_sample_data +2024-10-23T12:27:28.948Z | Connected to 10.1.0.2! | Disconnected from 10.1.0.2 | partial_mapping_sample_data +2024-10-23T12:15:03.360Z | Connected to 10.1.0.3! | Disconnected from 10.1.0.3 | partial_mapping_sample_data +2023-10-23T13:55:01.543Z | Connected to 10.1.0.1 | null | sample_data +2023-10-23T13:53:55.832Z | Connection error | null | sample_data +2023-10-23T13:52:55.015Z | Connection error | null | sample_data +2023-10-23T13:51:54.732Z | Connection error | null | sample_data +2023-10-23T13:33:34.937Z | Disconnected | null | sample_data +2023-10-23T12:27:28.948Z | Connected to 10.1.0.2 | null | sample_data +2023-10-23T12:15:03.360Z | Connected to 10.1.0.3 | null | sample_data +; + + +fieldIsMappedToDifferentTypesMultiIndex +required_capability: index_metadata_field +required_capability: unmapped_fields +FROM sample_data_ts_long, sample_data METADATA _index +| INSIST_🐔 @timestamp +| KEEP _index, @timestamp +| SORT _index +; + +_index:keyword | @timestamp:unsupported +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +; + +fieldIsMappedToDifferentTypesButDropped +required_capability: index_metadata_field +required_capability: unmapped_fields +FROM sample_data_ts_long, sample_data METADATA _index +| INSIST_🐔 @timestamp +| KEEP _index, @timestamp +| DROP @timestamp +| EVAL @timestamp = 42 +| SORT _index +; + +_index:keyword | @timestamp:integer +sample_data | 42 +sample_data | 42 +sample_data | 42 +sample_data | 42 +sample_data | 42 +sample_data | 42 +sample_data | 42 +sample_data_ts_long | 42 +sample_data_ts_long | 42 +sample_data_ts_long | 42 +sample_data_ts_long | 42 +sample_data_ts_long | 42 +sample_data_ts_long | 42 +sample_data_ts_long | 42 +; + +fieldIsPartiallyUnmappedMultiIndex +required_capability: index_metadata_field +required_capability: unmapped_fields +FROM sample_data, no_mapping_sample_data METADATA _index +| INSIST_🐔 message +| KEEP _index, message +| SORT _index, message DESC +; + +_index:keyword | message:keyword +no_mapping_sample_data | Connection error? +no_mapping_sample_data | Connection error? +no_mapping_sample_data | Connection error? +no_mapping_sample_data | Connected to 10.1.0.3! +no_mapping_sample_data | Connected to 10.1.0.2! +no_mapping_sample_data | Connected to 10.1.0.1! +no_mapping_sample_data | 42 +sample_data | Disconnected +sample_data | Connection error +sample_data | Connection error +sample_data | Connection error +sample_data | Connected to 10.1.0.3 +sample_data | Connected to 10.1.0.2 +sample_data | Connected to 10.1.0.1 +; + +fieldIsPartiallyUnmappedAndRenamedMultiIndex +required_capability: unmapped_fields +FROM sample_data, no_mapping_sample_data +| INSIST_🐔 message +| KEEP message +| RENAME message AS msg +| SORT msg DESC +; + +msg:keyword +Disconnected +Connection error? +Connection error? +Connection error? +Connection error +Connection error +Connection error +Connected to 10.1.0.3! +Connected to 10.1.0.3 +Connected to 10.1.0.2! +Connected to 10.1.0.2 +Connected to 10.1.0.1! +Connected to 10.1.0.1 +42 +; + +fieldIsPartiallyUnmappedPartiallySourceIsDisabledMultiIndex +required_capability: index_metadata_field +required_capability: source_field_mapping +required_capability: unmapped_fields +FROM partial_mapping_sample_data,partial_mapping_no_source_sample_data METADATA _index +| INSIST_🐔 message +| KEEP _index, @timestamp, message +| SORT _index, @timestamp +; + +_index:keyword | @timestamp:date | message:keyword +partial_mapping_no_source_sample_data | 2024-10-23T12:15:03.360Z | null +partial_mapping_no_source_sample_data | 2024-10-23T12:27:28.948Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:33:34.937Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:51:54.732Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:52:55.015Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:53:55.832Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:55:01.543Z | null +partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | Connected to 10.1.0.3! +partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | Connected to 10.1.0.2! +partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | 42 +partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! +; + +partialMappingStats +required_capability: index_metadata_field +required_capability: source_field_mapping +required_capability: unmapped_fields +FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data METADATA _index +| INSIST_🐔 message +| SORT message, @timestamp +| STATS max(@timestamp), count(*) BY message +; + +max(@timestamp):date | count(*):long | message:keyword +2024-10-23T13:55:01.543Z | 7 | null +2024-10-23T13:33:34.937Z | 1 | 42 +2024-10-23T13:55:01.543Z | 1 | Connected to 10.1.0.1! +2024-10-23T12:27:28.948Z | 1 | Connected to 10.1.0.2! +2024-10-23T12:15:03.360Z | 1 | Connected to 10.1.0.3! +2024-10-23T13:53:55.832Z | 3 | Connection error? +; + +partialMappingCoalesce +required_capability: index_metadata_field +required_capability: source_field_mapping +required_capability: unmapped_fields +FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data METADATA _index +| INSIST_🐔 message +| EVAL actual_value = COALESCE(message, "no _source") +| DROP message +| KEEP @timestamp, _index, actual_value +| SORT _index, @timestamp ASC +; + +@timestamp:date | _index:keyword | actual_value:keyword +2024-10-23T12:15:03.360Z | partial_mapping_excluded_source_sample_data | no _source +2024-10-23T12:27:28.948Z | partial_mapping_excluded_source_sample_data | no _source +2024-10-23T13:33:34.937Z | partial_mapping_excluded_source_sample_data | no _source +2024-10-23T13:51:54.732Z | partial_mapping_excluded_source_sample_data | no _source +2024-10-23T13:52:55.015Z | partial_mapping_excluded_source_sample_data | no _source +2024-10-23T13:53:55.832Z | partial_mapping_excluded_source_sample_data | no _source +2024-10-23T13:55:01.543Z | partial_mapping_excluded_source_sample_data | no _source +2024-10-23T12:15:03.360Z | partial_mapping_sample_data | Connected to 10.1.0.3! +2024-10-23T12:27:28.948Z | partial_mapping_sample_data | Connected to 10.1.0.2! +2024-10-23T13:33:34.937Z | partial_mapping_sample_data | 42 +2024-10-23T13:51:54.732Z | partial_mapping_sample_data | Connection error? +2024-10-23T13:52:55.015Z | partial_mapping_sample_data | Connection error? +2024-10-23T13:53:55.832Z | partial_mapping_sample_data | Connection error? +2024-10-23T13:55:01.543Z | partial_mapping_sample_data | Connected to 10.1.0.1! +; + +partialMappingUnionTypes +required_capability: index_metadata_field +required_capability: source_field_mapping +required_capability: unmapped_fields +FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data METADATA _index +| INSIST_🐔 message +| EVAL actual_value = message::STRING +| KEEP @timestamp, _index, actual_value +| SORT actual_value, @timestamp ASC +; + +@timestamp:date | _index:keyword | actual_value:string +2024-10-23T13:33:34.937Z | partial_mapping_sample_data | 42 +2024-10-23T13:55:01.543Z | partial_mapping_sample_data | Connected to 10.1.0.1! +2024-10-23T12:27:28.948Z | partial_mapping_sample_data | Connected to 10.1.0.2! +2024-10-23T12:15:03.360Z | partial_mapping_sample_data | Connected to 10.1.0.3! +2024-10-23T13:51:54.732Z | partial_mapping_sample_data | Connection error? +2024-10-23T13:52:55.015Z | partial_mapping_sample_data | Connection error? +2024-10-23T13:53:55.832Z | partial_mapping_sample_data | Connection error? +2024-10-23T12:15:03.360Z | partial_mapping_excluded_source_sample_data | null +2024-10-23T12:27:28.948Z | partial_mapping_excluded_source_sample_data | null +2024-10-23T13:33:34.937Z | partial_mapping_excluded_source_sample_data | null +2024-10-23T13:51:54.732Z | partial_mapping_excluded_source_sample_data | null +2024-10-23T13:52:55.015Z | partial_mapping_excluded_source_sample_data | null +2024-10-23T13:53:55.832Z | partial_mapping_excluded_source_sample_data | null +2024-10-23T13:55:01.543Z | partial_mapping_excluded_source_sample_data | null +; + +partialMappingStatsAfterCast +required_capability: index_metadata_field +required_capability: source_field_mapping +required_capability: unmapped_fields +FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data +| INSIST_🐔 message +| STATS count(*) BY message::INT +; +warningRegex: Line 3:21: evaluation of \[message::INT\] failed, treating result as null. Only first 20 failures recorded. +warningRegex: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number \[.*\] + +count(*):long | message::INT:integer +13 | null +1 | 42 +; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 9f900200d5b2..790c63d47609 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -87,6 +87,7 @@ JOIN_LOOKUP : 'lookup' -> pushMode(JOIN_MODE); // MYCOMMAND : 'mycommand' -> ... DEV_CHANGE_POINT : {this.isDevVersion()}? 'change_point' -> pushMode(CHANGE_POINT_MODE); DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats' -> pushMode(EXPRESSION_MODE); +DEV_INSIST : {this.isDevVersion()}? 'insist_🐔' -> pushMode(PROJECT_MODE); DEV_LOOKUP : {this.isDevVersion()}? 'lookup_🐔' -> pushMode(LOOKUP_MODE); DEV_METRICS : {this.isDevVersion()}? 'metrics' -> pushMode(METRICS_MODE); // list of all JOIN commands @@ -308,8 +309,9 @@ FROM_MULTILINE_COMMENT FROM_WS : WS -> channel(HIDDEN) ; + // -// DROP, KEEP +// DROP, KEEP, INSIST // mode PROJECT_MODE; PROJECT_PIPE : PIPE -> type(PIPE), popMode; @@ -655,3 +657,14 @@ CHANGE_POINT_UNQUOTED_IDENTIFIER: UNQUOTED_IDENTIFIER -> type(UNQUOTED_IDENTIFIE CHANGE_POINT_LINE_COMMENT: LINE_COMMENT -> channel(HIDDEN); CHANGE_POINT_MULTILINE_COMMENT: MULTILINE_COMMENT -> channel(HIDDEN); CHANGE_POINT_WS: WS -> channel(HIDDEN); + +// +// INSIST command +// +mode INSIST_MODE; +INSIST_PIPE : PIPE -> type(PIPE), popMode; +INSIST_IDENTIFIER: UNQUOTED_IDENTIFIER -> type(UNQUOTED_IDENTIFIER); + +INSIST_WS : WS -> channel(HIDDEN); +INSIST_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN); +INSIST_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN); diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index 7ab99c293bc9..67105e31fac8 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -17,121 +17,125 @@ WHERE=16 JOIN_LOOKUP=17 DEV_CHANGE_POINT=18 DEV_INLINESTATS=19 -DEV_LOOKUP=20 -DEV_METRICS=21 -DEV_JOIN_FULL=22 -DEV_JOIN_LEFT=23 -DEV_JOIN_RIGHT=24 -UNKNOWN_CMD=25 -LINE_COMMENT=26 -MULTILINE_COMMENT=27 -WS=28 -PIPE=29 -QUOTED_STRING=30 -INTEGER_LITERAL=31 -DECIMAL_LITERAL=32 -BY=33 -AND=34 -ASC=35 -ASSIGN=36 -CAST_OP=37 -COLON=38 -COMMA=39 -DESC=40 -DOT=41 -FALSE=42 -FIRST=43 -IN=44 -IS=45 -LAST=46 -LIKE=47 -LP=48 -NOT=49 -NULL=50 -NULLS=51 -OR=52 -PARAM=53 -RLIKE=54 -RP=55 -TRUE=56 -EQ=57 -CIEQ=58 -NEQ=59 -LT=60 -LTE=61 -GT=62 -GTE=63 -PLUS=64 -MINUS=65 -ASTERISK=66 -SLASH=67 -PERCENT=68 -LEFT_BRACES=69 -RIGHT_BRACES=70 -NAMED_OR_POSITIONAL_PARAM=71 -OPENING_BRACKET=72 -CLOSING_BRACKET=73 -UNQUOTED_IDENTIFIER=74 -QUOTED_IDENTIFIER=75 -EXPR_LINE_COMMENT=76 -EXPR_MULTILINE_COMMENT=77 -EXPR_WS=78 -EXPLAIN_WS=79 -EXPLAIN_LINE_COMMENT=80 -EXPLAIN_MULTILINE_COMMENT=81 -METADATA=82 -UNQUOTED_SOURCE=83 -FROM_LINE_COMMENT=84 -FROM_MULTILINE_COMMENT=85 -FROM_WS=86 -ID_PATTERN=87 -PROJECT_LINE_COMMENT=88 -PROJECT_MULTILINE_COMMENT=89 -PROJECT_WS=90 -AS=91 -RENAME_LINE_COMMENT=92 -RENAME_MULTILINE_COMMENT=93 -RENAME_WS=94 -ON=95 -WITH=96 -ENRICH_POLICY_NAME=97 -ENRICH_LINE_COMMENT=98 -ENRICH_MULTILINE_COMMENT=99 -ENRICH_WS=100 -ENRICH_FIELD_LINE_COMMENT=101 -ENRICH_FIELD_MULTILINE_COMMENT=102 -ENRICH_FIELD_WS=103 -MVEXPAND_LINE_COMMENT=104 -MVEXPAND_MULTILINE_COMMENT=105 -MVEXPAND_WS=106 -INFO=107 -SHOW_LINE_COMMENT=108 -SHOW_MULTILINE_COMMENT=109 -SHOW_WS=110 -SETTING=111 -SETTING_LINE_COMMENT=112 -SETTTING_MULTILINE_COMMENT=113 -SETTING_WS=114 -LOOKUP_LINE_COMMENT=115 -LOOKUP_MULTILINE_COMMENT=116 -LOOKUP_WS=117 -LOOKUP_FIELD_LINE_COMMENT=118 -LOOKUP_FIELD_MULTILINE_COMMENT=119 -LOOKUP_FIELD_WS=120 -JOIN=121 -USING=122 -JOIN_LINE_COMMENT=123 -JOIN_MULTILINE_COMMENT=124 -JOIN_WS=125 -METRICS_LINE_COMMENT=126 -METRICS_MULTILINE_COMMENT=127 -METRICS_WS=128 -CLOSING_METRICS_LINE_COMMENT=129 -CLOSING_METRICS_MULTILINE_COMMENT=130 -CLOSING_METRICS_WS=131 -CHANGE_POINT_LINE_COMMENT=132 -CHANGE_POINT_MULTILINE_COMMENT=133 -CHANGE_POINT_WS=134 +DEV_INSIST=20 +DEV_LOOKUP=21 +DEV_METRICS=22 +DEV_JOIN_FULL=23 +DEV_JOIN_LEFT=24 +DEV_JOIN_RIGHT=25 +UNKNOWN_CMD=26 +LINE_COMMENT=27 +MULTILINE_COMMENT=28 +WS=29 +PIPE=30 +QUOTED_STRING=31 +INTEGER_LITERAL=32 +DECIMAL_LITERAL=33 +BY=34 +AND=35 +ASC=36 +ASSIGN=37 +CAST_OP=38 +COLON=39 +COMMA=40 +DESC=41 +DOT=42 +FALSE=43 +FIRST=44 +IN=45 +IS=46 +LAST=47 +LIKE=48 +LP=49 +NOT=50 +NULL=51 +NULLS=52 +OR=53 +PARAM=54 +RLIKE=55 +RP=56 +TRUE=57 +EQ=58 +CIEQ=59 +NEQ=60 +LT=61 +LTE=62 +GT=63 +GTE=64 +PLUS=65 +MINUS=66 +ASTERISK=67 +SLASH=68 +PERCENT=69 +LEFT_BRACES=70 +RIGHT_BRACES=71 +NAMED_OR_POSITIONAL_PARAM=72 +OPENING_BRACKET=73 +CLOSING_BRACKET=74 +UNQUOTED_IDENTIFIER=75 +QUOTED_IDENTIFIER=76 +EXPR_LINE_COMMENT=77 +EXPR_MULTILINE_COMMENT=78 +EXPR_WS=79 +EXPLAIN_WS=80 +EXPLAIN_LINE_COMMENT=81 +EXPLAIN_MULTILINE_COMMENT=82 +METADATA=83 +UNQUOTED_SOURCE=84 +FROM_LINE_COMMENT=85 +FROM_MULTILINE_COMMENT=86 +FROM_WS=87 +ID_PATTERN=88 +PROJECT_LINE_COMMENT=89 +PROJECT_MULTILINE_COMMENT=90 +PROJECT_WS=91 +AS=92 +RENAME_LINE_COMMENT=93 +RENAME_MULTILINE_COMMENT=94 +RENAME_WS=95 +ON=96 +WITH=97 +ENRICH_POLICY_NAME=98 +ENRICH_LINE_COMMENT=99 +ENRICH_MULTILINE_COMMENT=100 +ENRICH_WS=101 +ENRICH_FIELD_LINE_COMMENT=102 +ENRICH_FIELD_MULTILINE_COMMENT=103 +ENRICH_FIELD_WS=104 +MVEXPAND_LINE_COMMENT=105 +MVEXPAND_MULTILINE_COMMENT=106 +MVEXPAND_WS=107 +INFO=108 +SHOW_LINE_COMMENT=109 +SHOW_MULTILINE_COMMENT=110 +SHOW_WS=111 +SETTING=112 +SETTING_LINE_COMMENT=113 +SETTTING_MULTILINE_COMMENT=114 +SETTING_WS=115 +LOOKUP_LINE_COMMENT=116 +LOOKUP_MULTILINE_COMMENT=117 +LOOKUP_WS=118 +LOOKUP_FIELD_LINE_COMMENT=119 +LOOKUP_FIELD_MULTILINE_COMMENT=120 +LOOKUP_FIELD_WS=121 +JOIN=122 +USING=123 +JOIN_LINE_COMMENT=124 +JOIN_MULTILINE_COMMENT=125 +JOIN_WS=126 +METRICS_LINE_COMMENT=127 +METRICS_MULTILINE_COMMENT=128 +METRICS_WS=129 +CLOSING_METRICS_LINE_COMMENT=130 +CLOSING_METRICS_MULTILINE_COMMENT=131 +CLOSING_METRICS_WS=132 +CHANGE_POINT_LINE_COMMENT=133 +CHANGE_POINT_MULTILINE_COMMENT=134 +CHANGE_POINT_WS=135 +INSIST_WS=136 +INSIST_LINE_COMMENT=137 +INSIST_MULTILINE_COMMENT=138 'dissect'=1 'drop'=2 'enrich'=3 @@ -149,50 +153,50 @@ CHANGE_POINT_WS=134 'stats'=15 'where'=16 'lookup'=17 -'|'=29 -'by'=33 -'and'=34 -'asc'=35 -'='=36 -'::'=37 -':'=38 -','=39 -'desc'=40 -'.'=41 -'false'=42 -'first'=43 -'in'=44 -'is'=45 -'last'=46 -'like'=47 -'('=48 -'not'=49 -'null'=50 -'nulls'=51 -'or'=52 -'?'=53 -'rlike'=54 -')'=55 -'true'=56 -'=='=57 -'=~'=58 -'!='=59 -'<'=60 -'<='=61 -'>'=62 -'>='=63 -'+'=64 -'-'=65 -'*'=66 -'/'=67 -'%'=68 -'{'=69 -'}'=70 -']'=73 -'metadata'=82 -'as'=91 -'on'=95 -'with'=96 -'info'=107 -'join'=121 -'USING'=122 +'|'=30 +'by'=34 +'and'=35 +'asc'=36 +'='=37 +'::'=38 +':'=39 +','=40 +'desc'=41 +'.'=42 +'false'=43 +'first'=44 +'in'=45 +'is'=46 +'last'=47 +'like'=48 +'('=49 +'not'=50 +'null'=51 +'nulls'=52 +'or'=53 +'?'=54 +'rlike'=55 +')'=56 +'true'=57 +'=='=58 +'=~'=59 +'!='=60 +'<'=61 +'<='=62 +'>'=63 +'>='=64 +'+'=65 +'-'=66 +'*'=67 +'/'=68 +'%'=69 +'{'=70 +'}'=71 +']'=74 +'metadata'=83 +'as'=92 +'on'=96 +'with'=97 +'info'=108 +'join'=122 +'USING'=123 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index 3e30dd0cb4a0..3cc769f2ed85 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -56,6 +56,7 @@ processingCommand | {this.isDevVersion()}? inlinestatsCommand | {this.isDevVersion()}? lookupCommand | {this.isDevVersion()}? changePointCommand + | {this.isDevVersion()}? insistCommand ; whereCommand @@ -344,3 +345,7 @@ joinPredicate changePointCommand : DEV_CHANGE_POINT value=qualifiedName (ON key=qualifiedName)? (AS targetType=qualifiedName COMMA targetPvalue=qualifiedName)? ; + +insistCommand + : DEV_INSIST qualifiedNamePatterns + ; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens index 7ab99c293bc9..67105e31fac8 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens @@ -17,121 +17,125 @@ WHERE=16 JOIN_LOOKUP=17 DEV_CHANGE_POINT=18 DEV_INLINESTATS=19 -DEV_LOOKUP=20 -DEV_METRICS=21 -DEV_JOIN_FULL=22 -DEV_JOIN_LEFT=23 -DEV_JOIN_RIGHT=24 -UNKNOWN_CMD=25 -LINE_COMMENT=26 -MULTILINE_COMMENT=27 -WS=28 -PIPE=29 -QUOTED_STRING=30 -INTEGER_LITERAL=31 -DECIMAL_LITERAL=32 -BY=33 -AND=34 -ASC=35 -ASSIGN=36 -CAST_OP=37 -COLON=38 -COMMA=39 -DESC=40 -DOT=41 -FALSE=42 -FIRST=43 -IN=44 -IS=45 -LAST=46 -LIKE=47 -LP=48 -NOT=49 -NULL=50 -NULLS=51 -OR=52 -PARAM=53 -RLIKE=54 -RP=55 -TRUE=56 -EQ=57 -CIEQ=58 -NEQ=59 -LT=60 -LTE=61 -GT=62 -GTE=63 -PLUS=64 -MINUS=65 -ASTERISK=66 -SLASH=67 -PERCENT=68 -LEFT_BRACES=69 -RIGHT_BRACES=70 -NAMED_OR_POSITIONAL_PARAM=71 -OPENING_BRACKET=72 -CLOSING_BRACKET=73 -UNQUOTED_IDENTIFIER=74 -QUOTED_IDENTIFIER=75 -EXPR_LINE_COMMENT=76 -EXPR_MULTILINE_COMMENT=77 -EXPR_WS=78 -EXPLAIN_WS=79 -EXPLAIN_LINE_COMMENT=80 -EXPLAIN_MULTILINE_COMMENT=81 -METADATA=82 -UNQUOTED_SOURCE=83 -FROM_LINE_COMMENT=84 -FROM_MULTILINE_COMMENT=85 -FROM_WS=86 -ID_PATTERN=87 -PROJECT_LINE_COMMENT=88 -PROJECT_MULTILINE_COMMENT=89 -PROJECT_WS=90 -AS=91 -RENAME_LINE_COMMENT=92 -RENAME_MULTILINE_COMMENT=93 -RENAME_WS=94 -ON=95 -WITH=96 -ENRICH_POLICY_NAME=97 -ENRICH_LINE_COMMENT=98 -ENRICH_MULTILINE_COMMENT=99 -ENRICH_WS=100 -ENRICH_FIELD_LINE_COMMENT=101 -ENRICH_FIELD_MULTILINE_COMMENT=102 -ENRICH_FIELD_WS=103 -MVEXPAND_LINE_COMMENT=104 -MVEXPAND_MULTILINE_COMMENT=105 -MVEXPAND_WS=106 -INFO=107 -SHOW_LINE_COMMENT=108 -SHOW_MULTILINE_COMMENT=109 -SHOW_WS=110 -SETTING=111 -SETTING_LINE_COMMENT=112 -SETTTING_MULTILINE_COMMENT=113 -SETTING_WS=114 -LOOKUP_LINE_COMMENT=115 -LOOKUP_MULTILINE_COMMENT=116 -LOOKUP_WS=117 -LOOKUP_FIELD_LINE_COMMENT=118 -LOOKUP_FIELD_MULTILINE_COMMENT=119 -LOOKUP_FIELD_WS=120 -JOIN=121 -USING=122 -JOIN_LINE_COMMENT=123 -JOIN_MULTILINE_COMMENT=124 -JOIN_WS=125 -METRICS_LINE_COMMENT=126 -METRICS_MULTILINE_COMMENT=127 -METRICS_WS=128 -CLOSING_METRICS_LINE_COMMENT=129 -CLOSING_METRICS_MULTILINE_COMMENT=130 -CLOSING_METRICS_WS=131 -CHANGE_POINT_LINE_COMMENT=132 -CHANGE_POINT_MULTILINE_COMMENT=133 -CHANGE_POINT_WS=134 +DEV_INSIST=20 +DEV_LOOKUP=21 +DEV_METRICS=22 +DEV_JOIN_FULL=23 +DEV_JOIN_LEFT=24 +DEV_JOIN_RIGHT=25 +UNKNOWN_CMD=26 +LINE_COMMENT=27 +MULTILINE_COMMENT=28 +WS=29 +PIPE=30 +QUOTED_STRING=31 +INTEGER_LITERAL=32 +DECIMAL_LITERAL=33 +BY=34 +AND=35 +ASC=36 +ASSIGN=37 +CAST_OP=38 +COLON=39 +COMMA=40 +DESC=41 +DOT=42 +FALSE=43 +FIRST=44 +IN=45 +IS=46 +LAST=47 +LIKE=48 +LP=49 +NOT=50 +NULL=51 +NULLS=52 +OR=53 +PARAM=54 +RLIKE=55 +RP=56 +TRUE=57 +EQ=58 +CIEQ=59 +NEQ=60 +LT=61 +LTE=62 +GT=63 +GTE=64 +PLUS=65 +MINUS=66 +ASTERISK=67 +SLASH=68 +PERCENT=69 +LEFT_BRACES=70 +RIGHT_BRACES=71 +NAMED_OR_POSITIONAL_PARAM=72 +OPENING_BRACKET=73 +CLOSING_BRACKET=74 +UNQUOTED_IDENTIFIER=75 +QUOTED_IDENTIFIER=76 +EXPR_LINE_COMMENT=77 +EXPR_MULTILINE_COMMENT=78 +EXPR_WS=79 +EXPLAIN_WS=80 +EXPLAIN_LINE_COMMENT=81 +EXPLAIN_MULTILINE_COMMENT=82 +METADATA=83 +UNQUOTED_SOURCE=84 +FROM_LINE_COMMENT=85 +FROM_MULTILINE_COMMENT=86 +FROM_WS=87 +ID_PATTERN=88 +PROJECT_LINE_COMMENT=89 +PROJECT_MULTILINE_COMMENT=90 +PROJECT_WS=91 +AS=92 +RENAME_LINE_COMMENT=93 +RENAME_MULTILINE_COMMENT=94 +RENAME_WS=95 +ON=96 +WITH=97 +ENRICH_POLICY_NAME=98 +ENRICH_LINE_COMMENT=99 +ENRICH_MULTILINE_COMMENT=100 +ENRICH_WS=101 +ENRICH_FIELD_LINE_COMMENT=102 +ENRICH_FIELD_MULTILINE_COMMENT=103 +ENRICH_FIELD_WS=104 +MVEXPAND_LINE_COMMENT=105 +MVEXPAND_MULTILINE_COMMENT=106 +MVEXPAND_WS=107 +INFO=108 +SHOW_LINE_COMMENT=109 +SHOW_MULTILINE_COMMENT=110 +SHOW_WS=111 +SETTING=112 +SETTING_LINE_COMMENT=113 +SETTTING_MULTILINE_COMMENT=114 +SETTING_WS=115 +LOOKUP_LINE_COMMENT=116 +LOOKUP_MULTILINE_COMMENT=117 +LOOKUP_WS=118 +LOOKUP_FIELD_LINE_COMMENT=119 +LOOKUP_FIELD_MULTILINE_COMMENT=120 +LOOKUP_FIELD_WS=121 +JOIN=122 +USING=123 +JOIN_LINE_COMMENT=124 +JOIN_MULTILINE_COMMENT=125 +JOIN_WS=126 +METRICS_LINE_COMMENT=127 +METRICS_MULTILINE_COMMENT=128 +METRICS_WS=129 +CLOSING_METRICS_LINE_COMMENT=130 +CLOSING_METRICS_MULTILINE_COMMENT=131 +CLOSING_METRICS_WS=132 +CHANGE_POINT_LINE_COMMENT=133 +CHANGE_POINT_MULTILINE_COMMENT=134 +CHANGE_POINT_WS=135 +INSIST_WS=136 +INSIST_LINE_COMMENT=137 +INSIST_MULTILINE_COMMENT=138 'dissect'=1 'drop'=2 'enrich'=3 @@ -149,50 +153,50 @@ CHANGE_POINT_WS=134 'stats'=15 'where'=16 'lookup'=17 -'|'=29 -'by'=33 -'and'=34 -'asc'=35 -'='=36 -'::'=37 -':'=38 -','=39 -'desc'=40 -'.'=41 -'false'=42 -'first'=43 -'in'=44 -'is'=45 -'last'=46 -'like'=47 -'('=48 -'not'=49 -'null'=50 -'nulls'=51 -'or'=52 -'?'=53 -'rlike'=54 -')'=55 -'true'=56 -'=='=57 -'=~'=58 -'!='=59 -'<'=60 -'<='=61 -'>'=62 -'>='=63 -'+'=64 -'-'=65 -'*'=66 -'/'=67 -'%'=68 -'{'=69 -'}'=70 -']'=73 -'metadata'=82 -'as'=91 -'on'=95 -'with'=96 -'info'=107 -'join'=121 -'USING'=122 +'|'=30 +'by'=34 +'and'=35 +'asc'=36 +'='=37 +'::'=38 +':'=39 +','=40 +'desc'=41 +'.'=42 +'false'=43 +'first'=44 +'in'=45 +'is'=46 +'last'=47 +'like'=48 +'('=49 +'not'=50 +'null'=51 +'nulls'=52 +'or'=53 +'?'=54 +'rlike'=55 +')'=56 +'true'=57 +'=='=58 +'=~'=59 +'!='=60 +'<'=61 +'<='=62 +'>'=63 +'>='=64 +'+'=65 +'-'=66 +'*'=67 +'/'=68 +'%'=69 +'{'=70 +'}'=71 +']'=74 +'metadata'=83 +'as'=92 +'on'=96 +'with'=97 +'info'=108 +'join'=122 +'USING'=123 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 9d550ad32804..57c8f033c883 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -303,6 +303,11 @@ public class EsqlCapabilities { */ UNION_TYPES, + /** + * Support unmapped using the INSIST keyword. + */ + UNMAPPED_FIELDS(Build.current().isSnapshot()), + /** * Support for function {@code ST_DISTANCE}. Done in #108764. */ @@ -622,6 +627,11 @@ public class EsqlCapabilities { */ SORT_RETURNING_SOURCE_OK, + /** + * _source field mapping directives: https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html + */ + SOURCE_FIELD_MAPPING, + /** * Allow filter per individual aggregation. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 1351b5ce51f4..74242cf9be3f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -9,13 +9,13 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.compute.data.Block; +import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.Column; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.VerificationException; -import org.elasticsearch.xpack.esql.analysis.AnalyzerRules.BaseAnalyzerRule; import org.elasticsearch.xpack.esql.analysis.AnalyzerRules.ParameterizedAnalyzerRule; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; @@ -39,6 +39,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -70,6 +71,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -430,9 +432,9 @@ public class Analyzer extends ParameterizedRuleExecutor { @Override - protected LogicalPlan doRule(LogicalPlan plan) { + protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) { if (plan.childrenResolved() == false) { return plan; } @@ -479,6 +481,10 @@ public class Analyzer extends ParameterizedRuleExecutor maybeResolveAttribute(ua, childrenOutput)); } @@ -662,13 +668,13 @@ public class Analyzer extends ParameterizedRuleExecutor resolved = new ArrayList<>(cols.size()); for (Attribute col : cols) { if (col instanceof UnresolvedAttribute ua) { - Attribute resolvedCol = maybeResolveAttribute(ua, output); - if (resolvedCol instanceof UnresolvedAttribute ucol) { + Attribute resolvedField = maybeResolveAttribute(ua, output); + if (resolvedField instanceof UnresolvedAttribute ucol) { String message = ua.unresolvedMessage(); String match = "column [" + ucol.name() + "]"; - resolvedCol = ucol.withUnresolvedMessage(message.replace(match, match + " in " + side + " side of join")); + resolvedField = ucol.withUnresolvedMessage(message.replace(match, match + " in " + side + " side of join")); } - resolved.add(resolvedCol); + resolved.add(resolvedField); } else { throw new IllegalStateException( "Surprised to discover column [ " + col.name() + "] already resolved when resolving JOIN keys" @@ -678,6 +684,49 @@ public class Analyzer extends ParameterizedRuleExecutor childrenOutput, IndexResolution indexResolution) { + List list = new ArrayList<>(); + for (Attribute a : insist.insistedAttributes()) { + list.add(resolveInsistAttribute(a, childrenOutput, indexResolution)); + } + return insist.withAttributes(list); + } + + private Attribute resolveInsistAttribute(Attribute attribute, List childrenOutput, IndexResolution indexResolution) { + Attribute resolvedCol = maybeResolveAttribute((UnresolvedAttribute) attribute, childrenOutput); + // Field isn't mapped anywhere. + if (resolvedCol instanceof UnresolvedAttribute) { + return insistKeyword(attribute); + } + + // Field is partially unmapped. + if (resolvedCol instanceof FieldAttribute fa && indexResolution.get().isPartiallyUnmappedField(fa.name())) { + return fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa); + } + + // Either the field is mapped everywhere and we can just use the resolved column, or the INSIST clause isn't on top of a FROM + // clause—for example, it might be on top of a ROW clause—so the verifier will catch it and fail. + return resolvedCol; + } + + private static Attribute invalidInsistAttribute(FieldAttribute fa) { + var name = fa.name(); + EsField field = fa.field() instanceof InvalidMappedField imf + ? new InvalidMappedField(name, InvalidMappedField.makeErrorsMessageIncludingInsistKeyword(imf.getTypesToIndices())) + : new InvalidMappedField( + name, + Strings.format( + "mapped as [2] incompatible types: [keyword] enforced by INSIST command, and [%s] in index mappings", + fa.dataType().typeName() + ) + ); + return new FieldAttribute(fa.source(), name, field); + } + + private static FieldAttribute insistKeyword(Attribute attribute) { + return new FieldAttribute(attribute.source(), attribute.name(), new PotentiallyUnmappedKeywordEsField(attribute.name())); + } + private Attribute maybeResolveAttribute(UnresolvedAttribute ua, List childrenOutput) { return maybeResolveAttribute(ua, childrenOutput, log); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index c2663650685e..87e555e8d2f7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -29,6 +29,8 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.Project; @@ -95,6 +97,7 @@ public class Verifier { checkOperationsOnUnsignedLong(p, failures); checkBinaryComparison(p, failures); + checkInsist(p, failures); }); if (failures.hasFailures() == false) { @@ -132,7 +135,7 @@ public class Verifier { e.forEachUp(ae -> { // Special handling for Project and unsupported/union types: disallow renaming them but pass them through otherwise. - if (p instanceof Project) { + if (p instanceof Project || p instanceof Insist) { if (ae instanceof Alias as && as.child() instanceof UnsupportedAttribute ua) { failures.add(fail(ae, ua.unresolvedMessage())); } @@ -231,6 +234,15 @@ public class Verifier { }); } + private static void checkInsist(LogicalPlan p, Failures failures) { + if (p instanceof Insist i) { + LogicalPlan child = i.child(); + if ((child instanceof EsRelation || child instanceof Insist) == false) { + failures.add(fail(i, "[insist] can only be used after [from] or [insist] commands, but was [{}]", child.sourceText())); + } + } + } + private void licenseCheck(LogicalPlan plan, Failures failures) { Consumer> licenseCheck = n -> { if (n instanceof LicenseAware la && la.licenseCheck(licenseState) == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java index 1edab8ce0e4a..09c3eda3dd69 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java @@ -19,18 +19,29 @@ import java.util.Set; import static java.util.stream.Collectors.toMap; -public record EsIndex(String name, Map mapping, Map indexNameWithModes) implements Writeable { +public record EsIndex( + String name, + Map mapping, + Map indexNameWithModes, + /** Fields mapped only in some (but *not* all) indices. Since this is only used by the analyzer, it is not serialized. */ + Set partiallyUnmappedFields +) implements Writeable { public EsIndex { assert name != null; assert mapping != null; + assert partiallyUnmappedFields != null; + } + + public EsIndex(String name, Map mapping, Map indexNameWithModes) { + this(name, mapping, indexNameWithModes, Set.of()); } /** * Intended for tests. Returns an index with an empty index mode map. */ public EsIndex(String name, Map mapping) { - this(name, mapping, Map.of()); + this(name, mapping, Map.of(), Set.of()); } public static EsIndex readFrom(StreamInput in) throws IOException { @@ -45,7 +56,8 @@ public record EsIndex(String name, Map mapping, Map e, e -> IndexMode.STANDARD)); } - return new EsIndex(name, mapping, indexNameWithModes); + // partially unmapped fields shouldn't pass the coordinator node anyway, since they are only used by the Analyzer. + return new EsIndex(name, mapping, indexNameWithModes, Set.of()); } @Override @@ -57,6 +69,11 @@ public record EsIndex(String name, Map mapping, Map concreteIndices() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index bc32945d73eb..5fcf7d35b476 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateEquals; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateEvalFoldables; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateInlineEvals; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateNullable; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropgateUnmappedFields; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneColumns; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneEmptyPlans; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneFilters; @@ -193,6 +194,6 @@ public class LogicalPlanOptimizer extends ParameterizedRuleExecutor cleanup() { - return new Batch<>("Clean Up", new ReplaceLimitAndSortAsTopN(), new ReplaceRowAsLocalRelation()); + return new Batch<>("Clean Up", new ReplaceLimitAndSortAsTopN(), new ReplaceRowAsLocalRelation(), new PropgateUnmappedFields()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropgateUnmappedFields.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropgateUnmappedFields.java new file mode 100644 index 000000000000..570b5b7e82be --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropgateUnmappedFields.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules.logical; + +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; +import org.elasticsearch.xpack.esql.expression.NamedExpressions; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.rule.Rule; + +import java.util.ArrayList; + +/** + * Merges unmapped fields into the output of the ES relation. This marking is necessary for the block loaders to force loading from _source + * if the field is unmapped. + */ +public class PropgateUnmappedFields extends Rule { + @Override + public LogicalPlan apply(LogicalPlan logicalPlan) { + if (logicalPlan instanceof EsRelation) { + return logicalPlan; + } + var unmappedFields = new AttributeSet(); + logicalPlan.forEachExpressionDown(FieldAttribute.class, fa -> { + if (fa.field() instanceof PotentiallyUnmappedKeywordEsField) { + unmappedFields.add(fa); + } + }); + return unmappedFields.isEmpty() + ? logicalPlan + : logicalPlan.transformUp( + EsRelation.class, + er -> er.withAttributes(NamedExpressions.mergeOutputAttributes(new ArrayList<>(unmappedFields), er.output())) + ); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java index e41e500aad11..d36fae54f516 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; @@ -70,7 +71,10 @@ public class ReplaceMissingFieldWithNull extends ParameterizedRule stats.exists(f.fieldName()) || lookupFields.contains(f) ? f : Literal.of(f, null) + f -> f.field() instanceof PotentiallyUnmappedKeywordEsField || (stats.exists(f.fieldName()) || lookupFields.contains(f)) + ? f + : Literal.of(f, null) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index 9afa5dcbb095..140054634ddb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -28,6 +28,7 @@ null null null null +null '|' null null @@ -134,6 +135,9 @@ null null null null +null +null +null token symbolic names: null @@ -156,6 +160,7 @@ WHERE JOIN_LOOKUP DEV_CHANGE_POINT DEV_INLINESTATS +DEV_INSIST DEV_LOOKUP DEV_METRICS DEV_JOIN_FULL @@ -271,6 +276,9 @@ CLOSING_METRICS_WS CHANGE_POINT_LINE_COMMENT CHANGE_POINT_MULTILINE_COMMENT CHANGE_POINT_WS +INSIST_WS +INSIST_LINE_COMMENT +INSIST_MULTILINE_COMMENT rule names: DISSECT @@ -292,6 +300,7 @@ WHERE JOIN_LOOKUP DEV_CHANGE_POINT DEV_INLINESTATS +DEV_INSIST DEV_LOOKUP DEV_METRICS DEV_JOIN_FULL @@ -501,6 +510,11 @@ CHANGE_POINT_UNQUOTED_IDENTIFIER CHANGE_POINT_LINE_COMMENT CHANGE_POINT_MULTILINE_COMMENT CHANGE_POINT_WS +INSIST_PIPE +INSIST_IDENTIFIER +INSIST_WS +INSIST_LINE_COMMENT +INSIST_MULTILINE_COMMENT channel names: DEFAULT_TOKEN_CHANNEL @@ -524,6 +538,7 @@ JOIN_MODE METRICS_MODE CLOSING_METRICS_MODE CHANGE_POINT_MODE +INSIST_MODE atn: -[4, 0, 134, 1689, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 2, 198, 7, 198, 2, 199, 7, 199, 2, 200, 7, 200, 2, 201, 7, 201, 2, 202, 7, 202, 2, 203, 7, 203, 2, 204, 7, 204, 2, 205, 7, 205, 2, 206, 7, 206, 2, 207, 7, 207, 2, 208, 7, 208, 2, 209, 7, 209, 2, 210, 7, 210, 2, 211, 7, 211, 2, 212, 7, 212, 2, 213, 7, 213, 2, 214, 7, 214, 2, 215, 7, 215, 2, 216, 7, 216, 2, 217, 7, 217, 2, 218, 7, 218, 2, 219, 7, 219, 2, 220, 7, 220, 2, 221, 7, 221, 2, 222, 7, 222, 2, 223, 7, 223, 2, 224, 7, 224, 2, 225, 7, 225, 2, 226, 7, 226, 2, 227, 7, 227, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 4, 24, 692, 8, 24, 11, 24, 12, 24, 693, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 702, 8, 25, 10, 25, 12, 25, 705, 9, 25, 1, 25, 3, 25, 708, 8, 25, 1, 25, 3, 25, 711, 8, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 5, 26, 720, 8, 26, 10, 26, 12, 26, 723, 9, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 4, 27, 731, 8, 27, 11, 27, 12, 27, 732, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 3, 33, 752, 8, 33, 1, 33, 4, 33, 755, 8, 33, 11, 33, 12, 33, 756, 1, 34, 1, 34, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 3, 36, 766, 8, 36, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 3, 38, 773, 8, 38, 1, 39, 1, 39, 1, 39, 5, 39, 778, 8, 39, 10, 39, 12, 39, 781, 9, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 5, 39, 789, 8, 39, 10, 39, 12, 39, 792, 9, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 799, 8, 39, 1, 39, 3, 39, 802, 8, 39, 3, 39, 804, 8, 39, 1, 40, 4, 40, 807, 8, 40, 11, 40, 12, 40, 808, 1, 41, 4, 41, 812, 8, 41, 11, 41, 12, 41, 813, 1, 41, 1, 41, 5, 41, 818, 8, 41, 10, 41, 12, 41, 821, 9, 41, 1, 41, 1, 41, 4, 41, 825, 8, 41, 11, 41, 12, 41, 826, 1, 41, 4, 41, 830, 8, 41, 11, 41, 12, 41, 831, 1, 41, 1, 41, 5, 41, 836, 8, 41, 10, 41, 12, 41, 839, 9, 41, 3, 41, 841, 8, 41, 1, 41, 1, 41, 1, 41, 1, 41, 4, 41, 847, 8, 41, 11, 41, 12, 41, 848, 1, 41, 1, 41, 3, 41, 853, 8, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 3, 81, 985, 8, 81, 1, 81, 5, 81, 988, 8, 81, 10, 81, 12, 81, 991, 9, 81, 1, 81, 1, 81, 4, 81, 995, 8, 81, 11, 81, 12, 81, 996, 3, 81, 999, 8, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 5, 84, 1013, 8, 84, 10, 84, 12, 84, 1016, 9, 84, 1, 84, 1, 84, 3, 84, 1020, 8, 84, 1, 84, 4, 84, 1023, 8, 84, 11, 84, 12, 84, 1024, 3, 84, 1027, 8, 84, 1, 85, 1, 85, 4, 85, 1031, 8, 85, 11, 85, 12, 85, 1032, 1, 85, 1, 85, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 3, 102, 1110, 8, 102, 1, 103, 4, 103, 1113, 8, 103, 11, 103, 12, 103, 1114, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 3, 114, 1162, 8, 114, 1, 115, 1, 115, 3, 115, 1166, 8, 115, 1, 115, 5, 115, 1169, 8, 115, 10, 115, 12, 115, 1172, 9, 115, 1, 115, 1, 115, 3, 115, 1176, 8, 115, 1, 115, 4, 115, 1179, 8, 115, 11, 115, 12, 115, 1180, 3, 115, 1183, 8, 115, 1, 116, 1, 116, 4, 116, 1187, 8, 116, 11, 116, 12, 116, 1188, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 136, 4, 136, 1272, 8, 136, 11, 136, 12, 136, 1273, 1, 136, 1, 136, 3, 136, 1278, 8, 136, 1, 136, 4, 136, 1281, 8, 136, 11, 136, 12, 136, 1282, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 4, 169, 1424, 8, 169, 11, 169, 12, 169, 1425, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 198, 1, 198, 1, 198, 1, 198, 1, 199, 1, 199, 1, 199, 1, 199, 1, 200, 1, 200, 1, 200, 1, 200, 1, 201, 1, 201, 1, 201, 1, 201, 1, 202, 1, 202, 1, 202, 1, 202, 1, 203, 1, 203, 1, 203, 1, 203, 1, 203, 1, 204, 1, 204, 1, 204, 1, 204, 1, 204, 1, 204, 1, 205, 1, 205, 1, 205, 1, 205, 1, 205, 1, 205, 1, 206, 1, 206, 1, 206, 1, 206, 1, 207, 1, 207, 1, 207, 1, 207, 1, 208, 1, 208, 1, 208, 1, 208, 1, 209, 1, 209, 1, 209, 1, 209, 1, 209, 1, 209, 1, 210, 1, 210, 1, 210, 1, 210, 1, 210, 1, 210, 1, 211, 1, 211, 1, 211, 1, 211, 1, 212, 1, 212, 1, 212, 1, 212, 1, 213, 1, 213, 1, 213, 1, 213, 1, 214, 1, 214, 1, 214, 1, 214, 1, 214, 1, 214, 1, 215, 1, 215, 1, 215, 1, 215, 1, 215, 1, 215, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 1, 217, 1, 217, 1, 217, 1, 217, 1, 217, 1, 218, 1, 218, 1, 218, 1, 218, 1, 218, 1, 219, 1, 219, 1, 219, 1, 219, 1, 220, 1, 220, 1, 220, 1, 220, 1, 221, 1, 221, 1, 221, 1, 221, 1, 222, 1, 222, 1, 222, 1, 222, 1, 223, 1, 223, 1, 223, 1, 223, 1, 224, 1, 224, 1, 224, 1, 224, 1, 225, 1, 225, 1, 225, 1, 225, 1, 226, 1, 226, 1, 226, 1, 226, 1, 227, 1, 227, 1, 227, 1, 227, 2, 721, 790, 0, 228, 17, 1, 19, 2, 21, 3, 23, 4, 25, 5, 27, 6, 29, 7, 31, 8, 33, 9, 35, 10, 37, 11, 39, 12, 41, 13, 43, 14, 45, 15, 47, 16, 49, 17, 51, 18, 53, 19, 55, 20, 57, 21, 59, 22, 61, 23, 63, 24, 65, 25, 67, 26, 69, 27, 71, 28, 73, 29, 75, 0, 77, 0, 79, 0, 81, 0, 83, 0, 85, 0, 87, 0, 89, 0, 91, 0, 93, 0, 95, 30, 97, 31, 99, 32, 101, 33, 103, 34, 105, 35, 107, 36, 109, 37, 111, 38, 113, 39, 115, 40, 117, 41, 119, 42, 121, 43, 123, 44, 125, 45, 127, 46, 129, 47, 131, 48, 133, 49, 135, 50, 137, 51, 139, 52, 141, 53, 143, 54, 145, 55, 147, 56, 149, 57, 151, 58, 153, 59, 155, 60, 157, 61, 159, 62, 161, 63, 163, 64, 165, 65, 167, 66, 169, 67, 171, 68, 173, 69, 175, 70, 177, 0, 179, 71, 181, 72, 183, 73, 185, 74, 187, 0, 189, 75, 191, 76, 193, 77, 195, 78, 197, 0, 199, 0, 201, 79, 203, 80, 205, 81, 207, 0, 209, 0, 211, 0, 213, 0, 215, 0, 217, 0, 219, 82, 221, 0, 223, 83, 225, 0, 227, 0, 229, 84, 231, 85, 233, 86, 235, 0, 237, 0, 239, 0, 241, 0, 243, 0, 245, 0, 247, 0, 249, 87, 251, 88, 253, 89, 255, 90, 257, 0, 259, 0, 261, 0, 263, 0, 265, 0, 267, 0, 269, 91, 271, 0, 273, 92, 275, 93, 277, 94, 279, 0, 281, 0, 283, 95, 285, 96, 287, 0, 289, 97, 291, 0, 293, 98, 295, 99, 297, 100, 299, 0, 301, 0, 303, 0, 305, 0, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 101, 319, 102, 321, 103, 323, 0, 325, 0, 327, 0, 329, 0, 331, 0, 333, 0, 335, 104, 337, 105, 339, 106, 341, 0, 343, 107, 345, 108, 347, 109, 349, 110, 351, 0, 353, 0, 355, 111, 357, 112, 359, 113, 361, 114, 363, 0, 365, 0, 367, 0, 369, 0, 371, 0, 373, 0, 375, 0, 377, 115, 379, 116, 381, 117, 383, 0, 385, 0, 387, 0, 389, 0, 391, 118, 393, 119, 395, 120, 397, 0, 399, 121, 401, 0, 403, 0, 405, 122, 407, 0, 409, 0, 411, 0, 413, 0, 415, 0, 417, 123, 419, 124, 421, 125, 423, 0, 425, 0, 427, 0, 429, 126, 431, 127, 433, 128, 435, 0, 437, 0, 439, 129, 441, 130, 443, 131, 445, 0, 447, 0, 449, 0, 451, 0, 453, 0, 455, 0, 457, 0, 459, 0, 461, 0, 463, 0, 465, 0, 467, 132, 469, 133, 471, 134, 17, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 36, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 2, 0, 74, 74, 106, 106, 1715, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 0, 63, 1, 0, 0, 0, 0, 65, 1, 0, 0, 0, 0, 67, 1, 0, 0, 0, 0, 69, 1, 0, 0, 0, 0, 71, 1, 0, 0, 0, 1, 73, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 171, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 1, 181, 1, 0, 0, 0, 1, 183, 1, 0, 0, 0, 1, 185, 1, 0, 0, 0, 1, 189, 1, 0, 0, 0, 1, 191, 1, 0, 0, 0, 1, 193, 1, 0, 0, 0, 1, 195, 1, 0, 0, 0, 2, 197, 1, 0, 0, 0, 2, 199, 1, 0, 0, 0, 2, 201, 1, 0, 0, 0, 2, 203, 1, 0, 0, 0, 2, 205, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 3, 219, 1, 0, 0, 0, 3, 223, 1, 0, 0, 0, 3, 225, 1, 0, 0, 0, 3, 227, 1, 0, 0, 0, 3, 229, 1, 0, 0, 0, 3, 231, 1, 0, 0, 0, 3, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 4, 239, 1, 0, 0, 0, 4, 241, 1, 0, 0, 0, 4, 243, 1, 0, 0, 0, 4, 249, 1, 0, 0, 0, 4, 251, 1, 0, 0, 0, 4, 253, 1, 0, 0, 0, 4, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 5, 261, 1, 0, 0, 0, 5, 263, 1, 0, 0, 0, 5, 265, 1, 0, 0, 0, 5, 267, 1, 0, 0, 0, 5, 269, 1, 0, 0, 0, 5, 271, 1, 0, 0, 0, 5, 273, 1, 0, 0, 0, 5, 275, 1, 0, 0, 0, 5, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 6, 281, 1, 0, 0, 0, 6, 283, 1, 0, 0, 0, 6, 285, 1, 0, 0, 0, 6, 289, 1, 0, 0, 0, 6, 291, 1, 0, 0, 0, 6, 293, 1, 0, 0, 0, 6, 295, 1, 0, 0, 0, 6, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 7, 307, 1, 0, 0, 0, 7, 309, 1, 0, 0, 0, 7, 311, 1, 0, 0, 0, 7, 313, 1, 0, 0, 0, 7, 315, 1, 0, 0, 0, 7, 317, 1, 0, 0, 0, 7, 319, 1, 0, 0, 0, 7, 321, 1, 0, 0, 0, 8, 323, 1, 0, 0, 0, 8, 325, 1, 0, 0, 0, 8, 327, 1, 0, 0, 0, 8, 329, 1, 0, 0, 0, 8, 331, 1, 0, 0, 0, 8, 333, 1, 0, 0, 0, 8, 335, 1, 0, 0, 0, 8, 337, 1, 0, 0, 0, 8, 339, 1, 0, 0, 0, 9, 341, 1, 0, 0, 0, 9, 343, 1, 0, 0, 0, 9, 345, 1, 0, 0, 0, 9, 347, 1, 0, 0, 0, 9, 349, 1, 0, 0, 0, 10, 351, 1, 0, 0, 0, 10, 353, 1, 0, 0, 0, 10, 355, 1, 0, 0, 0, 10, 357, 1, 0, 0, 0, 10, 359, 1, 0, 0, 0, 10, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 11, 365, 1, 0, 0, 0, 11, 367, 1, 0, 0, 0, 11, 369, 1, 0, 0, 0, 11, 371, 1, 0, 0, 0, 11, 373, 1, 0, 0, 0, 11, 375, 1, 0, 0, 0, 11, 377, 1, 0, 0, 0, 11, 379, 1, 0, 0, 0, 11, 381, 1, 0, 0, 0, 12, 383, 1, 0, 0, 0, 12, 385, 1, 0, 0, 0, 12, 387, 1, 0, 0, 0, 12, 389, 1, 0, 0, 0, 12, 391, 1, 0, 0, 0, 12, 393, 1, 0, 0, 0, 12, 395, 1, 0, 0, 0, 13, 397, 1, 0, 0, 0, 13, 399, 1, 0, 0, 0, 13, 401, 1, 0, 0, 0, 13, 403, 1, 0, 0, 0, 13, 405, 1, 0, 0, 0, 13, 407, 1, 0, 0, 0, 13, 409, 1, 0, 0, 0, 13, 411, 1, 0, 0, 0, 13, 413, 1, 0, 0, 0, 13, 415, 1, 0, 0, 0, 13, 417, 1, 0, 0, 0, 13, 419, 1, 0, 0, 0, 13, 421, 1, 0, 0, 0, 14, 423, 1, 0, 0, 0, 14, 425, 1, 0, 0, 0, 14, 427, 1, 0, 0, 0, 14, 429, 1, 0, 0, 0, 14, 431, 1, 0, 0, 0, 14, 433, 1, 0, 0, 0, 15, 435, 1, 0, 0, 0, 15, 437, 1, 0, 0, 0, 15, 439, 1, 0, 0, 0, 15, 441, 1, 0, 0, 0, 15, 443, 1, 0, 0, 0, 15, 445, 1, 0, 0, 0, 15, 447, 1, 0, 0, 0, 15, 449, 1, 0, 0, 0, 15, 451, 1, 0, 0, 0, 16, 453, 1, 0, 0, 0, 16, 455, 1, 0, 0, 0, 16, 457, 1, 0, 0, 0, 16, 459, 1, 0, 0, 0, 16, 461, 1, 0, 0, 0, 16, 463, 1, 0, 0, 0, 16, 465, 1, 0, 0, 0, 16, 467, 1, 0, 0, 0, 16, 469, 1, 0, 0, 0, 16, 471, 1, 0, 0, 0, 17, 473, 1, 0, 0, 0, 19, 483, 1, 0, 0, 0, 21, 490, 1, 0, 0, 0, 23, 499, 1, 0, 0, 0, 25, 506, 1, 0, 0, 0, 27, 516, 1, 0, 0, 0, 29, 523, 1, 0, 0, 0, 31, 530, 1, 0, 0, 0, 33, 537, 1, 0, 0, 0, 35, 545, 1, 0, 0, 0, 37, 557, 1, 0, 0, 0, 39, 566, 1, 0, 0, 0, 41, 572, 1, 0, 0, 0, 43, 579, 1, 0, 0, 0, 45, 586, 1, 0, 0, 0, 47, 594, 1, 0, 0, 0, 49, 602, 1, 0, 0, 0, 51, 611, 1, 0, 0, 0, 53, 627, 1, 0, 0, 0, 55, 642, 1, 0, 0, 0, 57, 654, 1, 0, 0, 0, 59, 665, 1, 0, 0, 0, 61, 673, 1, 0, 0, 0, 63, 681, 1, 0, 0, 0, 65, 691, 1, 0, 0, 0, 67, 697, 1, 0, 0, 0, 69, 714, 1, 0, 0, 0, 71, 730, 1, 0, 0, 0, 73, 736, 1, 0, 0, 0, 75, 740, 1, 0, 0, 0, 77, 742, 1, 0, 0, 0, 79, 744, 1, 0, 0, 0, 81, 747, 1, 0, 0, 0, 83, 749, 1, 0, 0, 0, 85, 758, 1, 0, 0, 0, 87, 760, 1, 0, 0, 0, 89, 765, 1, 0, 0, 0, 91, 767, 1, 0, 0, 0, 93, 772, 1, 0, 0, 0, 95, 803, 1, 0, 0, 0, 97, 806, 1, 0, 0, 0, 99, 852, 1, 0, 0, 0, 101, 854, 1, 0, 0, 0, 103, 857, 1, 0, 0, 0, 105, 861, 1, 0, 0, 0, 107, 865, 1, 0, 0, 0, 109, 867, 1, 0, 0, 0, 111, 870, 1, 0, 0, 0, 113, 872, 1, 0, 0, 0, 115, 874, 1, 0, 0, 0, 117, 879, 1, 0, 0, 0, 119, 881, 1, 0, 0, 0, 121, 887, 1, 0, 0, 0, 123, 893, 1, 0, 0, 0, 125, 896, 1, 0, 0, 0, 127, 899, 1, 0, 0, 0, 129, 904, 1, 0, 0, 0, 131, 909, 1, 0, 0, 0, 133, 911, 1, 0, 0, 0, 135, 915, 1, 0, 0, 0, 137, 920, 1, 0, 0, 0, 139, 926, 1, 0, 0, 0, 141, 929, 1, 0, 0, 0, 143, 931, 1, 0, 0, 0, 145, 937, 1, 0, 0, 0, 147, 939, 1, 0, 0, 0, 149, 944, 1, 0, 0, 0, 151, 947, 1, 0, 0, 0, 153, 950, 1, 0, 0, 0, 155, 953, 1, 0, 0, 0, 157, 955, 1, 0, 0, 0, 159, 958, 1, 0, 0, 0, 161, 960, 1, 0, 0, 0, 163, 963, 1, 0, 0, 0, 165, 965, 1, 0, 0, 0, 167, 967, 1, 0, 0, 0, 169, 969, 1, 0, 0, 0, 171, 971, 1, 0, 0, 0, 173, 973, 1, 0, 0, 0, 175, 975, 1, 0, 0, 0, 177, 977, 1, 0, 0, 0, 179, 998, 1, 0, 0, 0, 181, 1000, 1, 0, 0, 0, 183, 1005, 1, 0, 0, 0, 185, 1026, 1, 0, 0, 0, 187, 1028, 1, 0, 0, 0, 189, 1036, 1, 0, 0, 0, 191, 1038, 1, 0, 0, 0, 193, 1042, 1, 0, 0, 0, 195, 1046, 1, 0, 0, 0, 197, 1050, 1, 0, 0, 0, 199, 1055, 1, 0, 0, 0, 201, 1060, 1, 0, 0, 0, 203, 1064, 1, 0, 0, 0, 205, 1068, 1, 0, 0, 0, 207, 1072, 1, 0, 0, 0, 209, 1077, 1, 0, 0, 0, 211, 1081, 1, 0, 0, 0, 213, 1085, 1, 0, 0, 0, 215, 1089, 1, 0, 0, 0, 217, 1093, 1, 0, 0, 0, 219, 1097, 1, 0, 0, 0, 221, 1109, 1, 0, 0, 0, 223, 1112, 1, 0, 0, 0, 225, 1116, 1, 0, 0, 0, 227, 1120, 1, 0, 0, 0, 229, 1124, 1, 0, 0, 0, 231, 1128, 1, 0, 0, 0, 233, 1132, 1, 0, 0, 0, 235, 1136, 1, 0, 0, 0, 237, 1141, 1, 0, 0, 0, 239, 1145, 1, 0, 0, 0, 241, 1149, 1, 0, 0, 0, 243, 1153, 1, 0, 0, 0, 245, 1161, 1, 0, 0, 0, 247, 1182, 1, 0, 0, 0, 249, 1186, 1, 0, 0, 0, 251, 1190, 1, 0, 0, 0, 253, 1194, 1, 0, 0, 0, 255, 1198, 1, 0, 0, 0, 257, 1202, 1, 0, 0, 0, 259, 1207, 1, 0, 0, 0, 261, 1211, 1, 0, 0, 0, 263, 1215, 1, 0, 0, 0, 265, 1219, 1, 0, 0, 0, 267, 1223, 1, 0, 0, 0, 269, 1227, 1, 0, 0, 0, 271, 1230, 1, 0, 0, 0, 273, 1234, 1, 0, 0, 0, 275, 1238, 1, 0, 0, 0, 277, 1242, 1, 0, 0, 0, 279, 1246, 1, 0, 0, 0, 281, 1251, 1, 0, 0, 0, 283, 1256, 1, 0, 0, 0, 285, 1261, 1, 0, 0, 0, 287, 1268, 1, 0, 0, 0, 289, 1277, 1, 0, 0, 0, 291, 1284, 1, 0, 0, 0, 293, 1288, 1, 0, 0, 0, 295, 1292, 1, 0, 0, 0, 297, 1296, 1, 0, 0, 0, 299, 1300, 1, 0, 0, 0, 301, 1306, 1, 0, 0, 0, 303, 1310, 1, 0, 0, 0, 305, 1314, 1, 0, 0, 0, 307, 1318, 1, 0, 0, 0, 309, 1322, 1, 0, 0, 0, 311, 1326, 1, 0, 0, 0, 313, 1330, 1, 0, 0, 0, 315, 1334, 1, 0, 0, 0, 317, 1338, 1, 0, 0, 0, 319, 1342, 1, 0, 0, 0, 321, 1346, 1, 0, 0, 0, 323, 1350, 1, 0, 0, 0, 325, 1355, 1, 0, 0, 0, 327, 1359, 1, 0, 0, 0, 329, 1363, 1, 0, 0, 0, 331, 1367, 1, 0, 0, 0, 333, 1371, 1, 0, 0, 0, 335, 1375, 1, 0, 0, 0, 337, 1379, 1, 0, 0, 0, 339, 1383, 1, 0, 0, 0, 341, 1387, 1, 0, 0, 0, 343, 1392, 1, 0, 0, 0, 345, 1397, 1, 0, 0, 0, 347, 1401, 1, 0, 0, 0, 349, 1405, 1, 0, 0, 0, 351, 1409, 1, 0, 0, 0, 353, 1414, 1, 0, 0, 0, 355, 1423, 1, 0, 0, 0, 357, 1427, 1, 0, 0, 0, 359, 1431, 1, 0, 0, 0, 361, 1435, 1, 0, 0, 0, 363, 1439, 1, 0, 0, 0, 365, 1444, 1, 0, 0, 0, 367, 1448, 1, 0, 0, 0, 369, 1452, 1, 0, 0, 0, 371, 1456, 1, 0, 0, 0, 373, 1461, 1, 0, 0, 0, 375, 1465, 1, 0, 0, 0, 377, 1469, 1, 0, 0, 0, 379, 1473, 1, 0, 0, 0, 381, 1477, 1, 0, 0, 0, 383, 1481, 1, 0, 0, 0, 385, 1487, 1, 0, 0, 0, 387, 1491, 1, 0, 0, 0, 389, 1495, 1, 0, 0, 0, 391, 1499, 1, 0, 0, 0, 393, 1503, 1, 0, 0, 0, 395, 1507, 1, 0, 0, 0, 397, 1511, 1, 0, 0, 0, 399, 1516, 1, 0, 0, 0, 401, 1521, 1, 0, 0, 0, 403, 1525, 1, 0, 0, 0, 405, 1531, 1, 0, 0, 0, 407, 1540, 1, 0, 0, 0, 409, 1544, 1, 0, 0, 0, 411, 1548, 1, 0, 0, 0, 413, 1552, 1, 0, 0, 0, 415, 1556, 1, 0, 0, 0, 417, 1560, 1, 0, 0, 0, 419, 1564, 1, 0, 0, 0, 421, 1568, 1, 0, 0, 0, 423, 1572, 1, 0, 0, 0, 425, 1577, 1, 0, 0, 0, 427, 1583, 1, 0, 0, 0, 429, 1589, 1, 0, 0, 0, 431, 1593, 1, 0, 0, 0, 433, 1597, 1, 0, 0, 0, 435, 1601, 1, 0, 0, 0, 437, 1607, 1, 0, 0, 0, 439, 1613, 1, 0, 0, 0, 441, 1617, 1, 0, 0, 0, 443, 1621, 1, 0, 0, 0, 445, 1625, 1, 0, 0, 0, 447, 1631, 1, 0, 0, 0, 449, 1637, 1, 0, 0, 0, 451, 1643, 1, 0, 0, 0, 453, 1648, 1, 0, 0, 0, 455, 1653, 1, 0, 0, 0, 457, 1657, 1, 0, 0, 0, 459, 1661, 1, 0, 0, 0, 461, 1665, 1, 0, 0, 0, 463, 1669, 1, 0, 0, 0, 465, 1673, 1, 0, 0, 0, 467, 1677, 1, 0, 0, 0, 469, 1681, 1, 0, 0, 0, 471, 1685, 1, 0, 0, 0, 473, 474, 7, 0, 0, 0, 474, 475, 7, 1, 0, 0, 475, 476, 7, 2, 0, 0, 476, 477, 7, 2, 0, 0, 477, 478, 7, 3, 0, 0, 478, 479, 7, 4, 0, 0, 479, 480, 7, 5, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 0, 0, 0, 482, 18, 1, 0, 0, 0, 483, 484, 7, 0, 0, 0, 484, 485, 7, 6, 0, 0, 485, 486, 7, 7, 0, 0, 486, 487, 7, 8, 0, 0, 487, 488, 1, 0, 0, 0, 488, 489, 6, 1, 1, 0, 489, 20, 1, 0, 0, 0, 490, 491, 7, 3, 0, 0, 491, 492, 7, 9, 0, 0, 492, 493, 7, 6, 0, 0, 493, 494, 7, 1, 0, 0, 494, 495, 7, 4, 0, 0, 495, 496, 7, 10, 0, 0, 496, 497, 1, 0, 0, 0, 497, 498, 6, 2, 2, 0, 498, 22, 1, 0, 0, 0, 499, 500, 7, 3, 0, 0, 500, 501, 7, 11, 0, 0, 501, 502, 7, 12, 0, 0, 502, 503, 7, 13, 0, 0, 503, 504, 1, 0, 0, 0, 504, 505, 6, 3, 0, 0, 505, 24, 1, 0, 0, 0, 506, 507, 7, 3, 0, 0, 507, 508, 7, 14, 0, 0, 508, 509, 7, 8, 0, 0, 509, 510, 7, 13, 0, 0, 510, 511, 7, 12, 0, 0, 511, 512, 7, 1, 0, 0, 512, 513, 7, 9, 0, 0, 513, 514, 1, 0, 0, 0, 514, 515, 6, 4, 3, 0, 515, 26, 1, 0, 0, 0, 516, 517, 7, 15, 0, 0, 517, 518, 7, 6, 0, 0, 518, 519, 7, 7, 0, 0, 519, 520, 7, 16, 0, 0, 520, 521, 1, 0, 0, 0, 521, 522, 6, 5, 4, 0, 522, 28, 1, 0, 0, 0, 523, 524, 7, 17, 0, 0, 524, 525, 7, 6, 0, 0, 525, 526, 7, 7, 0, 0, 526, 527, 7, 18, 0, 0, 527, 528, 1, 0, 0, 0, 528, 529, 6, 6, 0, 0, 529, 30, 1, 0, 0, 0, 530, 531, 7, 18, 0, 0, 531, 532, 7, 3, 0, 0, 532, 533, 7, 3, 0, 0, 533, 534, 7, 8, 0, 0, 534, 535, 1, 0, 0, 0, 535, 536, 6, 7, 1, 0, 536, 32, 1, 0, 0, 0, 537, 538, 7, 13, 0, 0, 538, 539, 7, 1, 0, 0, 539, 540, 7, 16, 0, 0, 540, 541, 7, 1, 0, 0, 541, 542, 7, 5, 0, 0, 542, 543, 1, 0, 0, 0, 543, 544, 6, 8, 0, 0, 544, 34, 1, 0, 0, 0, 545, 546, 7, 16, 0, 0, 546, 547, 7, 11, 0, 0, 547, 548, 5, 95, 0, 0, 548, 549, 7, 3, 0, 0, 549, 550, 7, 14, 0, 0, 550, 551, 7, 8, 0, 0, 551, 552, 7, 12, 0, 0, 552, 553, 7, 9, 0, 0, 553, 554, 7, 0, 0, 0, 554, 555, 1, 0, 0, 0, 555, 556, 6, 9, 5, 0, 556, 36, 1, 0, 0, 0, 557, 558, 7, 6, 0, 0, 558, 559, 7, 3, 0, 0, 559, 560, 7, 9, 0, 0, 560, 561, 7, 12, 0, 0, 561, 562, 7, 16, 0, 0, 562, 563, 7, 3, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 6, 10, 6, 0, 565, 38, 1, 0, 0, 0, 566, 567, 7, 6, 0, 0, 567, 568, 7, 7, 0, 0, 568, 569, 7, 19, 0, 0, 569, 570, 1, 0, 0, 0, 570, 571, 6, 11, 0, 0, 571, 40, 1, 0, 0, 0, 572, 573, 7, 2, 0, 0, 573, 574, 7, 10, 0, 0, 574, 575, 7, 7, 0, 0, 575, 576, 7, 19, 0, 0, 576, 577, 1, 0, 0, 0, 577, 578, 6, 12, 7, 0, 578, 42, 1, 0, 0, 0, 579, 580, 7, 2, 0, 0, 580, 581, 7, 7, 0, 0, 581, 582, 7, 6, 0, 0, 582, 583, 7, 5, 0, 0, 583, 584, 1, 0, 0, 0, 584, 585, 6, 13, 0, 0, 585, 44, 1, 0, 0, 0, 586, 587, 7, 2, 0, 0, 587, 588, 7, 5, 0, 0, 588, 589, 7, 12, 0, 0, 589, 590, 7, 5, 0, 0, 590, 591, 7, 2, 0, 0, 591, 592, 1, 0, 0, 0, 592, 593, 6, 14, 0, 0, 593, 46, 1, 0, 0, 0, 594, 595, 7, 19, 0, 0, 595, 596, 7, 10, 0, 0, 596, 597, 7, 3, 0, 0, 597, 598, 7, 6, 0, 0, 598, 599, 7, 3, 0, 0, 599, 600, 1, 0, 0, 0, 600, 601, 6, 15, 0, 0, 601, 48, 1, 0, 0, 0, 602, 603, 7, 13, 0, 0, 603, 604, 7, 7, 0, 0, 604, 605, 7, 7, 0, 0, 605, 606, 7, 18, 0, 0, 606, 607, 7, 20, 0, 0, 607, 608, 7, 8, 0, 0, 608, 609, 1, 0, 0, 0, 609, 610, 6, 16, 8, 0, 610, 50, 1, 0, 0, 0, 611, 612, 4, 17, 0, 0, 612, 613, 7, 4, 0, 0, 613, 614, 7, 10, 0, 0, 614, 615, 7, 12, 0, 0, 615, 616, 7, 9, 0, 0, 616, 617, 7, 17, 0, 0, 617, 618, 7, 3, 0, 0, 618, 619, 5, 95, 0, 0, 619, 620, 7, 8, 0, 0, 620, 621, 7, 7, 0, 0, 621, 622, 7, 1, 0, 0, 622, 623, 7, 9, 0, 0, 623, 624, 7, 5, 0, 0, 624, 625, 1, 0, 0, 0, 625, 626, 6, 17, 9, 0, 626, 52, 1, 0, 0, 0, 627, 628, 4, 18, 1, 0, 628, 629, 7, 1, 0, 0, 629, 630, 7, 9, 0, 0, 630, 631, 7, 13, 0, 0, 631, 632, 7, 1, 0, 0, 632, 633, 7, 9, 0, 0, 633, 634, 7, 3, 0, 0, 634, 635, 7, 2, 0, 0, 635, 636, 7, 5, 0, 0, 636, 637, 7, 12, 0, 0, 637, 638, 7, 5, 0, 0, 638, 639, 7, 2, 0, 0, 639, 640, 1, 0, 0, 0, 640, 641, 6, 18, 0, 0, 641, 54, 1, 0, 0, 0, 642, 643, 4, 19, 2, 0, 643, 644, 7, 13, 0, 0, 644, 645, 7, 7, 0, 0, 645, 646, 7, 7, 0, 0, 646, 647, 7, 18, 0, 0, 647, 648, 7, 20, 0, 0, 648, 649, 7, 8, 0, 0, 649, 650, 5, 95, 0, 0, 650, 651, 5, 128020, 0, 0, 651, 652, 1, 0, 0, 0, 652, 653, 6, 19, 10, 0, 653, 56, 1, 0, 0, 0, 654, 655, 4, 20, 3, 0, 655, 656, 7, 16, 0, 0, 656, 657, 7, 3, 0, 0, 657, 658, 7, 5, 0, 0, 658, 659, 7, 6, 0, 0, 659, 660, 7, 1, 0, 0, 660, 661, 7, 4, 0, 0, 661, 662, 7, 2, 0, 0, 662, 663, 1, 0, 0, 0, 663, 664, 6, 20, 11, 0, 664, 58, 1, 0, 0, 0, 665, 666, 4, 21, 4, 0, 666, 667, 7, 15, 0, 0, 667, 668, 7, 20, 0, 0, 668, 669, 7, 13, 0, 0, 669, 670, 7, 13, 0, 0, 670, 671, 1, 0, 0, 0, 671, 672, 6, 21, 8, 0, 672, 60, 1, 0, 0, 0, 673, 674, 4, 22, 5, 0, 674, 675, 7, 13, 0, 0, 675, 676, 7, 3, 0, 0, 676, 677, 7, 15, 0, 0, 677, 678, 7, 5, 0, 0, 678, 679, 1, 0, 0, 0, 679, 680, 6, 22, 8, 0, 680, 62, 1, 0, 0, 0, 681, 682, 4, 23, 6, 0, 682, 683, 7, 6, 0, 0, 683, 684, 7, 1, 0, 0, 684, 685, 7, 17, 0, 0, 685, 686, 7, 10, 0, 0, 686, 687, 7, 5, 0, 0, 687, 688, 1, 0, 0, 0, 688, 689, 6, 23, 8, 0, 689, 64, 1, 0, 0, 0, 690, 692, 8, 21, 0, 0, 691, 690, 1, 0, 0, 0, 692, 693, 1, 0, 0, 0, 693, 691, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 695, 1, 0, 0, 0, 695, 696, 6, 24, 0, 0, 696, 66, 1, 0, 0, 0, 697, 698, 5, 47, 0, 0, 698, 699, 5, 47, 0, 0, 699, 703, 1, 0, 0, 0, 700, 702, 8, 22, 0, 0, 701, 700, 1, 0, 0, 0, 702, 705, 1, 0, 0, 0, 703, 701, 1, 0, 0, 0, 703, 704, 1, 0, 0, 0, 704, 707, 1, 0, 0, 0, 705, 703, 1, 0, 0, 0, 706, 708, 5, 13, 0, 0, 707, 706, 1, 0, 0, 0, 707, 708, 1, 0, 0, 0, 708, 710, 1, 0, 0, 0, 709, 711, 5, 10, 0, 0, 710, 709, 1, 0, 0, 0, 710, 711, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0, 712, 713, 6, 25, 12, 0, 713, 68, 1, 0, 0, 0, 714, 715, 5, 47, 0, 0, 715, 716, 5, 42, 0, 0, 716, 721, 1, 0, 0, 0, 717, 720, 3, 69, 26, 0, 718, 720, 9, 0, 0, 0, 719, 717, 1, 0, 0, 0, 719, 718, 1, 0, 0, 0, 720, 723, 1, 0, 0, 0, 721, 722, 1, 0, 0, 0, 721, 719, 1, 0, 0, 0, 722, 724, 1, 0, 0, 0, 723, 721, 1, 0, 0, 0, 724, 725, 5, 42, 0, 0, 725, 726, 5, 47, 0, 0, 726, 727, 1, 0, 0, 0, 727, 728, 6, 26, 12, 0, 728, 70, 1, 0, 0, 0, 729, 731, 7, 23, 0, 0, 730, 729, 1, 0, 0, 0, 731, 732, 1, 0, 0, 0, 732, 730, 1, 0, 0, 0, 732, 733, 1, 0, 0, 0, 733, 734, 1, 0, 0, 0, 734, 735, 6, 27, 12, 0, 735, 72, 1, 0, 0, 0, 736, 737, 5, 124, 0, 0, 737, 738, 1, 0, 0, 0, 738, 739, 6, 28, 13, 0, 739, 74, 1, 0, 0, 0, 740, 741, 7, 24, 0, 0, 741, 76, 1, 0, 0, 0, 742, 743, 7, 25, 0, 0, 743, 78, 1, 0, 0, 0, 744, 745, 5, 92, 0, 0, 745, 746, 7, 26, 0, 0, 746, 80, 1, 0, 0, 0, 747, 748, 8, 27, 0, 0, 748, 82, 1, 0, 0, 0, 749, 751, 7, 3, 0, 0, 750, 752, 7, 28, 0, 0, 751, 750, 1, 0, 0, 0, 751, 752, 1, 0, 0, 0, 752, 754, 1, 0, 0, 0, 753, 755, 3, 75, 29, 0, 754, 753, 1, 0, 0, 0, 755, 756, 1, 0, 0, 0, 756, 754, 1, 0, 0, 0, 756, 757, 1, 0, 0, 0, 757, 84, 1, 0, 0, 0, 758, 759, 5, 64, 0, 0, 759, 86, 1, 0, 0, 0, 760, 761, 5, 96, 0, 0, 761, 88, 1, 0, 0, 0, 762, 766, 8, 29, 0, 0, 763, 764, 5, 96, 0, 0, 764, 766, 5, 96, 0, 0, 765, 762, 1, 0, 0, 0, 765, 763, 1, 0, 0, 0, 766, 90, 1, 0, 0, 0, 767, 768, 5, 95, 0, 0, 768, 92, 1, 0, 0, 0, 769, 773, 3, 77, 30, 0, 770, 773, 3, 75, 29, 0, 771, 773, 3, 91, 37, 0, 772, 769, 1, 0, 0, 0, 772, 770, 1, 0, 0, 0, 772, 771, 1, 0, 0, 0, 773, 94, 1, 0, 0, 0, 774, 779, 5, 34, 0, 0, 775, 778, 3, 79, 31, 0, 776, 778, 3, 81, 32, 0, 777, 775, 1, 0, 0, 0, 777, 776, 1, 0, 0, 0, 778, 781, 1, 0, 0, 0, 779, 777, 1, 0, 0, 0, 779, 780, 1, 0, 0, 0, 780, 782, 1, 0, 0, 0, 781, 779, 1, 0, 0, 0, 782, 804, 5, 34, 0, 0, 783, 784, 5, 34, 0, 0, 784, 785, 5, 34, 0, 0, 785, 786, 5, 34, 0, 0, 786, 790, 1, 0, 0, 0, 787, 789, 8, 22, 0, 0, 788, 787, 1, 0, 0, 0, 789, 792, 1, 0, 0, 0, 790, 791, 1, 0, 0, 0, 790, 788, 1, 0, 0, 0, 791, 793, 1, 0, 0, 0, 792, 790, 1, 0, 0, 0, 793, 794, 5, 34, 0, 0, 794, 795, 5, 34, 0, 0, 795, 796, 5, 34, 0, 0, 796, 798, 1, 0, 0, 0, 797, 799, 5, 34, 0, 0, 798, 797, 1, 0, 0, 0, 798, 799, 1, 0, 0, 0, 799, 801, 1, 0, 0, 0, 800, 802, 5, 34, 0, 0, 801, 800, 1, 0, 0, 0, 801, 802, 1, 0, 0, 0, 802, 804, 1, 0, 0, 0, 803, 774, 1, 0, 0, 0, 803, 783, 1, 0, 0, 0, 804, 96, 1, 0, 0, 0, 805, 807, 3, 75, 29, 0, 806, 805, 1, 0, 0, 0, 807, 808, 1, 0, 0, 0, 808, 806, 1, 0, 0, 0, 808, 809, 1, 0, 0, 0, 809, 98, 1, 0, 0, 0, 810, 812, 3, 75, 29, 0, 811, 810, 1, 0, 0, 0, 812, 813, 1, 0, 0, 0, 813, 811, 1, 0, 0, 0, 813, 814, 1, 0, 0, 0, 814, 815, 1, 0, 0, 0, 815, 819, 3, 117, 50, 0, 816, 818, 3, 75, 29, 0, 817, 816, 1, 0, 0, 0, 818, 821, 1, 0, 0, 0, 819, 817, 1, 0, 0, 0, 819, 820, 1, 0, 0, 0, 820, 853, 1, 0, 0, 0, 821, 819, 1, 0, 0, 0, 822, 824, 3, 117, 50, 0, 823, 825, 3, 75, 29, 0, 824, 823, 1, 0, 0, 0, 825, 826, 1, 0, 0, 0, 826, 824, 1, 0, 0, 0, 826, 827, 1, 0, 0, 0, 827, 853, 1, 0, 0, 0, 828, 830, 3, 75, 29, 0, 829, 828, 1, 0, 0, 0, 830, 831, 1, 0, 0, 0, 831, 829, 1, 0, 0, 0, 831, 832, 1, 0, 0, 0, 832, 840, 1, 0, 0, 0, 833, 837, 3, 117, 50, 0, 834, 836, 3, 75, 29, 0, 835, 834, 1, 0, 0, 0, 836, 839, 1, 0, 0, 0, 837, 835, 1, 0, 0, 0, 837, 838, 1, 0, 0, 0, 838, 841, 1, 0, 0, 0, 839, 837, 1, 0, 0, 0, 840, 833, 1, 0, 0, 0, 840, 841, 1, 0, 0, 0, 841, 842, 1, 0, 0, 0, 842, 843, 3, 83, 33, 0, 843, 853, 1, 0, 0, 0, 844, 846, 3, 117, 50, 0, 845, 847, 3, 75, 29, 0, 846, 845, 1, 0, 0, 0, 847, 848, 1, 0, 0, 0, 848, 846, 1, 0, 0, 0, 848, 849, 1, 0, 0, 0, 849, 850, 1, 0, 0, 0, 850, 851, 3, 83, 33, 0, 851, 853, 1, 0, 0, 0, 852, 811, 1, 0, 0, 0, 852, 822, 1, 0, 0, 0, 852, 829, 1, 0, 0, 0, 852, 844, 1, 0, 0, 0, 853, 100, 1, 0, 0, 0, 854, 855, 7, 30, 0, 0, 855, 856, 7, 31, 0, 0, 856, 102, 1, 0, 0, 0, 857, 858, 7, 12, 0, 0, 858, 859, 7, 9, 0, 0, 859, 860, 7, 0, 0, 0, 860, 104, 1, 0, 0, 0, 861, 862, 7, 12, 0, 0, 862, 863, 7, 2, 0, 0, 863, 864, 7, 4, 0, 0, 864, 106, 1, 0, 0, 0, 865, 866, 5, 61, 0, 0, 866, 108, 1, 0, 0, 0, 867, 868, 5, 58, 0, 0, 868, 869, 5, 58, 0, 0, 869, 110, 1, 0, 0, 0, 870, 871, 5, 58, 0, 0, 871, 112, 1, 0, 0, 0, 872, 873, 5, 44, 0, 0, 873, 114, 1, 0, 0, 0, 874, 875, 7, 0, 0, 0, 875, 876, 7, 3, 0, 0, 876, 877, 7, 2, 0, 0, 877, 878, 7, 4, 0, 0, 878, 116, 1, 0, 0, 0, 879, 880, 5, 46, 0, 0, 880, 118, 1, 0, 0, 0, 881, 882, 7, 15, 0, 0, 882, 883, 7, 12, 0, 0, 883, 884, 7, 13, 0, 0, 884, 885, 7, 2, 0, 0, 885, 886, 7, 3, 0, 0, 886, 120, 1, 0, 0, 0, 887, 888, 7, 15, 0, 0, 888, 889, 7, 1, 0, 0, 889, 890, 7, 6, 0, 0, 890, 891, 7, 2, 0, 0, 891, 892, 7, 5, 0, 0, 892, 122, 1, 0, 0, 0, 893, 894, 7, 1, 0, 0, 894, 895, 7, 9, 0, 0, 895, 124, 1, 0, 0, 0, 896, 897, 7, 1, 0, 0, 897, 898, 7, 2, 0, 0, 898, 126, 1, 0, 0, 0, 899, 900, 7, 13, 0, 0, 900, 901, 7, 12, 0, 0, 901, 902, 7, 2, 0, 0, 902, 903, 7, 5, 0, 0, 903, 128, 1, 0, 0, 0, 904, 905, 7, 13, 0, 0, 905, 906, 7, 1, 0, 0, 906, 907, 7, 18, 0, 0, 907, 908, 7, 3, 0, 0, 908, 130, 1, 0, 0, 0, 909, 910, 5, 40, 0, 0, 910, 132, 1, 0, 0, 0, 911, 912, 7, 9, 0, 0, 912, 913, 7, 7, 0, 0, 913, 914, 7, 5, 0, 0, 914, 134, 1, 0, 0, 0, 915, 916, 7, 9, 0, 0, 916, 917, 7, 20, 0, 0, 917, 918, 7, 13, 0, 0, 918, 919, 7, 13, 0, 0, 919, 136, 1, 0, 0, 0, 920, 921, 7, 9, 0, 0, 921, 922, 7, 20, 0, 0, 922, 923, 7, 13, 0, 0, 923, 924, 7, 13, 0, 0, 924, 925, 7, 2, 0, 0, 925, 138, 1, 0, 0, 0, 926, 927, 7, 7, 0, 0, 927, 928, 7, 6, 0, 0, 928, 140, 1, 0, 0, 0, 929, 930, 5, 63, 0, 0, 930, 142, 1, 0, 0, 0, 931, 932, 7, 6, 0, 0, 932, 933, 7, 13, 0, 0, 933, 934, 7, 1, 0, 0, 934, 935, 7, 18, 0, 0, 935, 936, 7, 3, 0, 0, 936, 144, 1, 0, 0, 0, 937, 938, 5, 41, 0, 0, 938, 146, 1, 0, 0, 0, 939, 940, 7, 5, 0, 0, 940, 941, 7, 6, 0, 0, 941, 942, 7, 20, 0, 0, 942, 943, 7, 3, 0, 0, 943, 148, 1, 0, 0, 0, 944, 945, 5, 61, 0, 0, 945, 946, 5, 61, 0, 0, 946, 150, 1, 0, 0, 0, 947, 948, 5, 61, 0, 0, 948, 949, 5, 126, 0, 0, 949, 152, 1, 0, 0, 0, 950, 951, 5, 33, 0, 0, 951, 952, 5, 61, 0, 0, 952, 154, 1, 0, 0, 0, 953, 954, 5, 60, 0, 0, 954, 156, 1, 0, 0, 0, 955, 956, 5, 60, 0, 0, 956, 957, 5, 61, 0, 0, 957, 158, 1, 0, 0, 0, 958, 959, 5, 62, 0, 0, 959, 160, 1, 0, 0, 0, 960, 961, 5, 62, 0, 0, 961, 962, 5, 61, 0, 0, 962, 162, 1, 0, 0, 0, 963, 964, 5, 43, 0, 0, 964, 164, 1, 0, 0, 0, 965, 966, 5, 45, 0, 0, 966, 166, 1, 0, 0, 0, 967, 968, 5, 42, 0, 0, 968, 168, 1, 0, 0, 0, 969, 970, 5, 47, 0, 0, 970, 170, 1, 0, 0, 0, 971, 972, 5, 37, 0, 0, 972, 172, 1, 0, 0, 0, 973, 974, 5, 123, 0, 0, 974, 174, 1, 0, 0, 0, 975, 976, 5, 125, 0, 0, 976, 176, 1, 0, 0, 0, 977, 978, 3, 47, 15, 0, 978, 979, 1, 0, 0, 0, 979, 980, 6, 80, 14, 0, 980, 178, 1, 0, 0, 0, 981, 984, 3, 141, 62, 0, 982, 985, 3, 77, 30, 0, 983, 985, 3, 91, 37, 0, 984, 982, 1, 0, 0, 0, 984, 983, 1, 0, 0, 0, 985, 989, 1, 0, 0, 0, 986, 988, 3, 93, 38, 0, 987, 986, 1, 0, 0, 0, 988, 991, 1, 0, 0, 0, 989, 987, 1, 0, 0, 0, 989, 990, 1, 0, 0, 0, 990, 999, 1, 0, 0, 0, 991, 989, 1, 0, 0, 0, 992, 994, 3, 141, 62, 0, 993, 995, 3, 75, 29, 0, 994, 993, 1, 0, 0, 0, 995, 996, 1, 0, 0, 0, 996, 994, 1, 0, 0, 0, 996, 997, 1, 0, 0, 0, 997, 999, 1, 0, 0, 0, 998, 981, 1, 0, 0, 0, 998, 992, 1, 0, 0, 0, 999, 180, 1, 0, 0, 0, 1000, 1001, 5, 91, 0, 0, 1001, 1002, 1, 0, 0, 0, 1002, 1003, 6, 82, 0, 0, 1003, 1004, 6, 82, 0, 0, 1004, 182, 1, 0, 0, 0, 1005, 1006, 5, 93, 0, 0, 1006, 1007, 1, 0, 0, 0, 1007, 1008, 6, 83, 13, 0, 1008, 1009, 6, 83, 13, 0, 1009, 184, 1, 0, 0, 0, 1010, 1014, 3, 77, 30, 0, 1011, 1013, 3, 93, 38, 0, 1012, 1011, 1, 0, 0, 0, 1013, 1016, 1, 0, 0, 0, 1014, 1012, 1, 0, 0, 0, 1014, 1015, 1, 0, 0, 0, 1015, 1027, 1, 0, 0, 0, 1016, 1014, 1, 0, 0, 0, 1017, 1020, 3, 91, 37, 0, 1018, 1020, 3, 85, 34, 0, 1019, 1017, 1, 0, 0, 0, 1019, 1018, 1, 0, 0, 0, 1020, 1022, 1, 0, 0, 0, 1021, 1023, 3, 93, 38, 0, 1022, 1021, 1, 0, 0, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1022, 1, 0, 0, 0, 1024, 1025, 1, 0, 0, 0, 1025, 1027, 1, 0, 0, 0, 1026, 1010, 1, 0, 0, 0, 1026, 1019, 1, 0, 0, 0, 1027, 186, 1, 0, 0, 0, 1028, 1030, 3, 87, 35, 0, 1029, 1031, 3, 89, 36, 0, 1030, 1029, 1, 0, 0, 0, 1031, 1032, 1, 0, 0, 0, 1032, 1030, 1, 0, 0, 0, 1032, 1033, 1, 0, 0, 0, 1033, 1034, 1, 0, 0, 0, 1034, 1035, 3, 87, 35, 0, 1035, 188, 1, 0, 0, 0, 1036, 1037, 3, 187, 85, 0, 1037, 190, 1, 0, 0, 0, 1038, 1039, 3, 67, 25, 0, 1039, 1040, 1, 0, 0, 0, 1040, 1041, 6, 87, 12, 0, 1041, 192, 1, 0, 0, 0, 1042, 1043, 3, 69, 26, 0, 1043, 1044, 1, 0, 0, 0, 1044, 1045, 6, 88, 12, 0, 1045, 194, 1, 0, 0, 0, 1046, 1047, 3, 71, 27, 0, 1047, 1048, 1, 0, 0, 0, 1048, 1049, 6, 89, 12, 0, 1049, 196, 1, 0, 0, 0, 1050, 1051, 3, 181, 82, 0, 1051, 1052, 1, 0, 0, 0, 1052, 1053, 6, 90, 15, 0, 1053, 1054, 6, 90, 16, 0, 1054, 198, 1, 0, 0, 0, 1055, 1056, 3, 73, 28, 0, 1056, 1057, 1, 0, 0, 0, 1057, 1058, 6, 91, 17, 0, 1058, 1059, 6, 91, 13, 0, 1059, 200, 1, 0, 0, 0, 1060, 1061, 3, 71, 27, 0, 1061, 1062, 1, 0, 0, 0, 1062, 1063, 6, 92, 12, 0, 1063, 202, 1, 0, 0, 0, 1064, 1065, 3, 67, 25, 0, 1065, 1066, 1, 0, 0, 0, 1066, 1067, 6, 93, 12, 0, 1067, 204, 1, 0, 0, 0, 1068, 1069, 3, 69, 26, 0, 1069, 1070, 1, 0, 0, 0, 1070, 1071, 6, 94, 12, 0, 1071, 206, 1, 0, 0, 0, 1072, 1073, 3, 73, 28, 0, 1073, 1074, 1, 0, 0, 0, 1074, 1075, 6, 95, 17, 0, 1075, 1076, 6, 95, 13, 0, 1076, 208, 1, 0, 0, 0, 1077, 1078, 3, 181, 82, 0, 1078, 1079, 1, 0, 0, 0, 1079, 1080, 6, 96, 15, 0, 1080, 210, 1, 0, 0, 0, 1081, 1082, 3, 183, 83, 0, 1082, 1083, 1, 0, 0, 0, 1083, 1084, 6, 97, 18, 0, 1084, 212, 1, 0, 0, 0, 1085, 1086, 3, 111, 47, 0, 1086, 1087, 1, 0, 0, 0, 1087, 1088, 6, 98, 19, 0, 1088, 214, 1, 0, 0, 0, 1089, 1090, 3, 113, 48, 0, 1090, 1091, 1, 0, 0, 0, 1091, 1092, 6, 99, 20, 0, 1092, 216, 1, 0, 0, 0, 1093, 1094, 3, 107, 45, 0, 1094, 1095, 1, 0, 0, 0, 1095, 1096, 6, 100, 21, 0, 1096, 218, 1, 0, 0, 0, 1097, 1098, 7, 16, 0, 0, 1098, 1099, 7, 3, 0, 0, 1099, 1100, 7, 5, 0, 0, 1100, 1101, 7, 12, 0, 0, 1101, 1102, 7, 0, 0, 0, 1102, 1103, 7, 12, 0, 0, 1103, 1104, 7, 5, 0, 0, 1104, 1105, 7, 12, 0, 0, 1105, 220, 1, 0, 0, 0, 1106, 1110, 8, 32, 0, 0, 1107, 1108, 5, 47, 0, 0, 1108, 1110, 8, 33, 0, 0, 1109, 1106, 1, 0, 0, 0, 1109, 1107, 1, 0, 0, 0, 1110, 222, 1, 0, 0, 0, 1111, 1113, 3, 221, 102, 0, 1112, 1111, 1, 0, 0, 0, 1113, 1114, 1, 0, 0, 0, 1114, 1112, 1, 0, 0, 0, 1114, 1115, 1, 0, 0, 0, 1115, 224, 1, 0, 0, 0, 1116, 1117, 3, 223, 103, 0, 1117, 1118, 1, 0, 0, 0, 1118, 1119, 6, 104, 22, 0, 1119, 226, 1, 0, 0, 0, 1120, 1121, 3, 95, 39, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 105, 23, 0, 1123, 228, 1, 0, 0, 0, 1124, 1125, 3, 67, 25, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 106, 12, 0, 1127, 230, 1, 0, 0, 0, 1128, 1129, 3, 69, 26, 0, 1129, 1130, 1, 0, 0, 0, 1130, 1131, 6, 107, 12, 0, 1131, 232, 1, 0, 0, 0, 1132, 1133, 3, 71, 27, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 108, 12, 0, 1135, 234, 1, 0, 0, 0, 1136, 1137, 3, 73, 28, 0, 1137, 1138, 1, 0, 0, 0, 1138, 1139, 6, 109, 17, 0, 1139, 1140, 6, 109, 13, 0, 1140, 236, 1, 0, 0, 0, 1141, 1142, 3, 117, 50, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 110, 24, 0, 1144, 238, 1, 0, 0, 0, 1145, 1146, 3, 113, 48, 0, 1146, 1147, 1, 0, 0, 0, 1147, 1148, 6, 111, 20, 0, 1148, 240, 1, 0, 0, 0, 1149, 1150, 3, 141, 62, 0, 1150, 1151, 1, 0, 0, 0, 1151, 1152, 6, 112, 25, 0, 1152, 242, 1, 0, 0, 0, 1153, 1154, 3, 179, 81, 0, 1154, 1155, 1, 0, 0, 0, 1155, 1156, 6, 113, 26, 0, 1156, 244, 1, 0, 0, 0, 1157, 1162, 3, 77, 30, 0, 1158, 1162, 3, 75, 29, 0, 1159, 1162, 3, 91, 37, 0, 1160, 1162, 3, 167, 75, 0, 1161, 1157, 1, 0, 0, 0, 1161, 1158, 1, 0, 0, 0, 1161, 1159, 1, 0, 0, 0, 1161, 1160, 1, 0, 0, 0, 1162, 246, 1, 0, 0, 0, 1163, 1166, 3, 77, 30, 0, 1164, 1166, 3, 167, 75, 0, 1165, 1163, 1, 0, 0, 0, 1165, 1164, 1, 0, 0, 0, 1166, 1170, 1, 0, 0, 0, 1167, 1169, 3, 245, 114, 0, 1168, 1167, 1, 0, 0, 0, 1169, 1172, 1, 0, 0, 0, 1170, 1168, 1, 0, 0, 0, 1170, 1171, 1, 0, 0, 0, 1171, 1183, 1, 0, 0, 0, 1172, 1170, 1, 0, 0, 0, 1173, 1176, 3, 91, 37, 0, 1174, 1176, 3, 85, 34, 0, 1175, 1173, 1, 0, 0, 0, 1175, 1174, 1, 0, 0, 0, 1176, 1178, 1, 0, 0, 0, 1177, 1179, 3, 245, 114, 0, 1178, 1177, 1, 0, 0, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1178, 1, 0, 0, 0, 1180, 1181, 1, 0, 0, 0, 1181, 1183, 1, 0, 0, 0, 1182, 1165, 1, 0, 0, 0, 1182, 1175, 1, 0, 0, 0, 1183, 248, 1, 0, 0, 0, 1184, 1187, 3, 247, 115, 0, 1185, 1187, 3, 187, 85, 0, 1186, 1184, 1, 0, 0, 0, 1186, 1185, 1, 0, 0, 0, 1187, 1188, 1, 0, 0, 0, 1188, 1186, 1, 0, 0, 0, 1188, 1189, 1, 0, 0, 0, 1189, 250, 1, 0, 0, 0, 1190, 1191, 3, 67, 25, 0, 1191, 1192, 1, 0, 0, 0, 1192, 1193, 6, 117, 12, 0, 1193, 252, 1, 0, 0, 0, 1194, 1195, 3, 69, 26, 0, 1195, 1196, 1, 0, 0, 0, 1196, 1197, 6, 118, 12, 0, 1197, 254, 1, 0, 0, 0, 1198, 1199, 3, 71, 27, 0, 1199, 1200, 1, 0, 0, 0, 1200, 1201, 6, 119, 12, 0, 1201, 256, 1, 0, 0, 0, 1202, 1203, 3, 73, 28, 0, 1203, 1204, 1, 0, 0, 0, 1204, 1205, 6, 120, 17, 0, 1205, 1206, 6, 120, 13, 0, 1206, 258, 1, 0, 0, 0, 1207, 1208, 3, 107, 45, 0, 1208, 1209, 1, 0, 0, 0, 1209, 1210, 6, 121, 21, 0, 1210, 260, 1, 0, 0, 0, 1211, 1212, 3, 113, 48, 0, 1212, 1213, 1, 0, 0, 0, 1213, 1214, 6, 122, 20, 0, 1214, 262, 1, 0, 0, 0, 1215, 1216, 3, 117, 50, 0, 1216, 1217, 1, 0, 0, 0, 1217, 1218, 6, 123, 24, 0, 1218, 264, 1, 0, 0, 0, 1219, 1220, 3, 141, 62, 0, 1220, 1221, 1, 0, 0, 0, 1221, 1222, 6, 124, 25, 0, 1222, 266, 1, 0, 0, 0, 1223, 1224, 3, 179, 81, 0, 1224, 1225, 1, 0, 0, 0, 1225, 1226, 6, 125, 26, 0, 1226, 268, 1, 0, 0, 0, 1227, 1228, 7, 12, 0, 0, 1228, 1229, 7, 2, 0, 0, 1229, 270, 1, 0, 0, 0, 1230, 1231, 3, 249, 116, 0, 1231, 1232, 1, 0, 0, 0, 1232, 1233, 6, 127, 27, 0, 1233, 272, 1, 0, 0, 0, 1234, 1235, 3, 67, 25, 0, 1235, 1236, 1, 0, 0, 0, 1236, 1237, 6, 128, 12, 0, 1237, 274, 1, 0, 0, 0, 1238, 1239, 3, 69, 26, 0, 1239, 1240, 1, 0, 0, 0, 1240, 1241, 6, 129, 12, 0, 1241, 276, 1, 0, 0, 0, 1242, 1243, 3, 71, 27, 0, 1243, 1244, 1, 0, 0, 0, 1244, 1245, 6, 130, 12, 0, 1245, 278, 1, 0, 0, 0, 1246, 1247, 3, 73, 28, 0, 1247, 1248, 1, 0, 0, 0, 1248, 1249, 6, 131, 17, 0, 1249, 1250, 6, 131, 13, 0, 1250, 280, 1, 0, 0, 0, 1251, 1252, 3, 181, 82, 0, 1252, 1253, 1, 0, 0, 0, 1253, 1254, 6, 132, 15, 0, 1254, 1255, 6, 132, 28, 0, 1255, 282, 1, 0, 0, 0, 1256, 1257, 7, 7, 0, 0, 1257, 1258, 7, 9, 0, 0, 1258, 1259, 1, 0, 0, 0, 1259, 1260, 6, 133, 29, 0, 1260, 284, 1, 0, 0, 0, 1261, 1262, 7, 19, 0, 0, 1262, 1263, 7, 1, 0, 0, 1263, 1264, 7, 5, 0, 0, 1264, 1265, 7, 10, 0, 0, 1265, 1266, 1, 0, 0, 0, 1266, 1267, 6, 134, 29, 0, 1267, 286, 1, 0, 0, 0, 1268, 1269, 8, 34, 0, 0, 1269, 288, 1, 0, 0, 0, 1270, 1272, 3, 287, 135, 0, 1271, 1270, 1, 0, 0, 0, 1272, 1273, 1, 0, 0, 0, 1273, 1271, 1, 0, 0, 0, 1273, 1274, 1, 0, 0, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 3, 111, 47, 0, 1276, 1278, 1, 0, 0, 0, 1277, 1271, 1, 0, 0, 0, 1277, 1278, 1, 0, 0, 0, 1278, 1280, 1, 0, 0, 0, 1279, 1281, 3, 287, 135, 0, 1280, 1279, 1, 0, 0, 0, 1281, 1282, 1, 0, 0, 0, 1282, 1280, 1, 0, 0, 0, 1282, 1283, 1, 0, 0, 0, 1283, 290, 1, 0, 0, 0, 1284, 1285, 3, 289, 136, 0, 1285, 1286, 1, 0, 0, 0, 1286, 1287, 6, 137, 30, 0, 1287, 292, 1, 0, 0, 0, 1288, 1289, 3, 67, 25, 0, 1289, 1290, 1, 0, 0, 0, 1290, 1291, 6, 138, 12, 0, 1291, 294, 1, 0, 0, 0, 1292, 1293, 3, 69, 26, 0, 1293, 1294, 1, 0, 0, 0, 1294, 1295, 6, 139, 12, 0, 1295, 296, 1, 0, 0, 0, 1296, 1297, 3, 71, 27, 0, 1297, 1298, 1, 0, 0, 0, 1298, 1299, 6, 140, 12, 0, 1299, 298, 1, 0, 0, 0, 1300, 1301, 3, 73, 28, 0, 1301, 1302, 1, 0, 0, 0, 1302, 1303, 6, 141, 17, 0, 1303, 1304, 6, 141, 13, 0, 1304, 1305, 6, 141, 13, 0, 1305, 300, 1, 0, 0, 0, 1306, 1307, 3, 107, 45, 0, 1307, 1308, 1, 0, 0, 0, 1308, 1309, 6, 142, 21, 0, 1309, 302, 1, 0, 0, 0, 1310, 1311, 3, 113, 48, 0, 1311, 1312, 1, 0, 0, 0, 1312, 1313, 6, 143, 20, 0, 1313, 304, 1, 0, 0, 0, 1314, 1315, 3, 117, 50, 0, 1315, 1316, 1, 0, 0, 0, 1316, 1317, 6, 144, 24, 0, 1317, 306, 1, 0, 0, 0, 1318, 1319, 3, 285, 134, 0, 1319, 1320, 1, 0, 0, 0, 1320, 1321, 6, 145, 31, 0, 1321, 308, 1, 0, 0, 0, 1322, 1323, 3, 249, 116, 0, 1323, 1324, 1, 0, 0, 0, 1324, 1325, 6, 146, 27, 0, 1325, 310, 1, 0, 0, 0, 1326, 1327, 3, 189, 86, 0, 1327, 1328, 1, 0, 0, 0, 1328, 1329, 6, 147, 32, 0, 1329, 312, 1, 0, 0, 0, 1330, 1331, 3, 141, 62, 0, 1331, 1332, 1, 0, 0, 0, 1332, 1333, 6, 148, 25, 0, 1333, 314, 1, 0, 0, 0, 1334, 1335, 3, 179, 81, 0, 1335, 1336, 1, 0, 0, 0, 1336, 1337, 6, 149, 26, 0, 1337, 316, 1, 0, 0, 0, 1338, 1339, 3, 67, 25, 0, 1339, 1340, 1, 0, 0, 0, 1340, 1341, 6, 150, 12, 0, 1341, 318, 1, 0, 0, 0, 1342, 1343, 3, 69, 26, 0, 1343, 1344, 1, 0, 0, 0, 1344, 1345, 6, 151, 12, 0, 1345, 320, 1, 0, 0, 0, 1346, 1347, 3, 71, 27, 0, 1347, 1348, 1, 0, 0, 0, 1348, 1349, 6, 152, 12, 0, 1349, 322, 1, 0, 0, 0, 1350, 1351, 3, 73, 28, 0, 1351, 1352, 1, 0, 0, 0, 1352, 1353, 6, 153, 17, 0, 1353, 1354, 6, 153, 13, 0, 1354, 324, 1, 0, 0, 0, 1355, 1356, 3, 117, 50, 0, 1356, 1357, 1, 0, 0, 0, 1357, 1358, 6, 154, 24, 0, 1358, 326, 1, 0, 0, 0, 1359, 1360, 3, 141, 62, 0, 1360, 1361, 1, 0, 0, 0, 1361, 1362, 6, 155, 25, 0, 1362, 328, 1, 0, 0, 0, 1363, 1364, 3, 179, 81, 0, 1364, 1365, 1, 0, 0, 0, 1365, 1366, 6, 156, 26, 0, 1366, 330, 1, 0, 0, 0, 1367, 1368, 3, 189, 86, 0, 1368, 1369, 1, 0, 0, 0, 1369, 1370, 6, 157, 32, 0, 1370, 332, 1, 0, 0, 0, 1371, 1372, 3, 185, 84, 0, 1372, 1373, 1, 0, 0, 0, 1373, 1374, 6, 158, 33, 0, 1374, 334, 1, 0, 0, 0, 1375, 1376, 3, 67, 25, 0, 1376, 1377, 1, 0, 0, 0, 1377, 1378, 6, 159, 12, 0, 1378, 336, 1, 0, 0, 0, 1379, 1380, 3, 69, 26, 0, 1380, 1381, 1, 0, 0, 0, 1381, 1382, 6, 160, 12, 0, 1382, 338, 1, 0, 0, 0, 1383, 1384, 3, 71, 27, 0, 1384, 1385, 1, 0, 0, 0, 1385, 1386, 6, 161, 12, 0, 1386, 340, 1, 0, 0, 0, 1387, 1388, 3, 73, 28, 0, 1388, 1389, 1, 0, 0, 0, 1389, 1390, 6, 162, 17, 0, 1390, 1391, 6, 162, 13, 0, 1391, 342, 1, 0, 0, 0, 1392, 1393, 7, 1, 0, 0, 1393, 1394, 7, 9, 0, 0, 1394, 1395, 7, 15, 0, 0, 1395, 1396, 7, 7, 0, 0, 1396, 344, 1, 0, 0, 0, 1397, 1398, 3, 67, 25, 0, 1398, 1399, 1, 0, 0, 0, 1399, 1400, 6, 164, 12, 0, 1400, 346, 1, 0, 0, 0, 1401, 1402, 3, 69, 26, 0, 1402, 1403, 1, 0, 0, 0, 1403, 1404, 6, 165, 12, 0, 1404, 348, 1, 0, 0, 0, 1405, 1406, 3, 71, 27, 0, 1406, 1407, 1, 0, 0, 0, 1407, 1408, 6, 166, 12, 0, 1408, 350, 1, 0, 0, 0, 1409, 1410, 3, 183, 83, 0, 1410, 1411, 1, 0, 0, 0, 1411, 1412, 6, 167, 18, 0, 1412, 1413, 6, 167, 13, 0, 1413, 352, 1, 0, 0, 0, 1414, 1415, 3, 111, 47, 0, 1415, 1416, 1, 0, 0, 0, 1416, 1417, 6, 168, 19, 0, 1417, 354, 1, 0, 0, 0, 1418, 1424, 3, 85, 34, 0, 1419, 1424, 3, 75, 29, 0, 1420, 1424, 3, 117, 50, 0, 1421, 1424, 3, 77, 30, 0, 1422, 1424, 3, 91, 37, 0, 1423, 1418, 1, 0, 0, 0, 1423, 1419, 1, 0, 0, 0, 1423, 1420, 1, 0, 0, 0, 1423, 1421, 1, 0, 0, 0, 1423, 1422, 1, 0, 0, 0, 1424, 1425, 1, 0, 0, 0, 1425, 1423, 1, 0, 0, 0, 1425, 1426, 1, 0, 0, 0, 1426, 356, 1, 0, 0, 0, 1427, 1428, 3, 67, 25, 0, 1428, 1429, 1, 0, 0, 0, 1429, 1430, 6, 170, 12, 0, 1430, 358, 1, 0, 0, 0, 1431, 1432, 3, 69, 26, 0, 1432, 1433, 1, 0, 0, 0, 1433, 1434, 6, 171, 12, 0, 1434, 360, 1, 0, 0, 0, 1435, 1436, 3, 71, 27, 0, 1436, 1437, 1, 0, 0, 0, 1437, 1438, 6, 172, 12, 0, 1438, 362, 1, 0, 0, 0, 1439, 1440, 3, 73, 28, 0, 1440, 1441, 1, 0, 0, 0, 1441, 1442, 6, 173, 17, 0, 1442, 1443, 6, 173, 13, 0, 1443, 364, 1, 0, 0, 0, 1444, 1445, 3, 111, 47, 0, 1445, 1446, 1, 0, 0, 0, 1446, 1447, 6, 174, 19, 0, 1447, 366, 1, 0, 0, 0, 1448, 1449, 3, 113, 48, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1451, 6, 175, 20, 0, 1451, 368, 1, 0, 0, 0, 1452, 1453, 3, 117, 50, 0, 1453, 1454, 1, 0, 0, 0, 1454, 1455, 6, 176, 24, 0, 1455, 370, 1, 0, 0, 0, 1456, 1457, 3, 283, 133, 0, 1457, 1458, 1, 0, 0, 0, 1458, 1459, 6, 177, 34, 0, 1459, 1460, 6, 177, 35, 0, 1460, 372, 1, 0, 0, 0, 1461, 1462, 3, 223, 103, 0, 1462, 1463, 1, 0, 0, 0, 1463, 1464, 6, 178, 22, 0, 1464, 374, 1, 0, 0, 0, 1465, 1466, 3, 95, 39, 0, 1466, 1467, 1, 0, 0, 0, 1467, 1468, 6, 179, 23, 0, 1468, 376, 1, 0, 0, 0, 1469, 1470, 3, 67, 25, 0, 1470, 1471, 1, 0, 0, 0, 1471, 1472, 6, 180, 12, 0, 1472, 378, 1, 0, 0, 0, 1473, 1474, 3, 69, 26, 0, 1474, 1475, 1, 0, 0, 0, 1475, 1476, 6, 181, 12, 0, 1476, 380, 1, 0, 0, 0, 1477, 1478, 3, 71, 27, 0, 1478, 1479, 1, 0, 0, 0, 1479, 1480, 6, 182, 12, 0, 1480, 382, 1, 0, 0, 0, 1481, 1482, 3, 73, 28, 0, 1482, 1483, 1, 0, 0, 0, 1483, 1484, 6, 183, 17, 0, 1484, 1485, 6, 183, 13, 0, 1485, 1486, 6, 183, 13, 0, 1486, 384, 1, 0, 0, 0, 1487, 1488, 3, 113, 48, 0, 1488, 1489, 1, 0, 0, 0, 1489, 1490, 6, 184, 20, 0, 1490, 386, 1, 0, 0, 0, 1491, 1492, 3, 117, 50, 0, 1492, 1493, 1, 0, 0, 0, 1493, 1494, 6, 185, 24, 0, 1494, 388, 1, 0, 0, 0, 1495, 1496, 3, 249, 116, 0, 1496, 1497, 1, 0, 0, 0, 1497, 1498, 6, 186, 27, 0, 1498, 390, 1, 0, 0, 0, 1499, 1500, 3, 67, 25, 0, 1500, 1501, 1, 0, 0, 0, 1501, 1502, 6, 187, 12, 0, 1502, 392, 1, 0, 0, 0, 1503, 1504, 3, 69, 26, 0, 1504, 1505, 1, 0, 0, 0, 1505, 1506, 6, 188, 12, 0, 1506, 394, 1, 0, 0, 0, 1507, 1508, 3, 71, 27, 0, 1508, 1509, 1, 0, 0, 0, 1509, 1510, 6, 189, 12, 0, 1510, 396, 1, 0, 0, 0, 1511, 1512, 3, 73, 28, 0, 1512, 1513, 1, 0, 0, 0, 1513, 1514, 6, 190, 17, 0, 1514, 1515, 6, 190, 13, 0, 1515, 398, 1, 0, 0, 0, 1516, 1517, 7, 35, 0, 0, 1517, 1518, 7, 7, 0, 0, 1518, 1519, 7, 1, 0, 0, 1519, 1520, 7, 9, 0, 0, 1520, 400, 1, 0, 0, 0, 1521, 1522, 3, 269, 126, 0, 1522, 1523, 1, 0, 0, 0, 1523, 1524, 6, 192, 36, 0, 1524, 402, 1, 0, 0, 0, 1525, 1526, 3, 283, 133, 0, 1526, 1527, 1, 0, 0, 0, 1527, 1528, 6, 193, 34, 0, 1528, 1529, 6, 193, 13, 0, 1529, 1530, 6, 193, 0, 0, 1530, 404, 1, 0, 0, 0, 1531, 1532, 7, 20, 0, 0, 1532, 1533, 7, 2, 0, 0, 1533, 1534, 7, 1, 0, 0, 1534, 1535, 7, 9, 0, 0, 1535, 1536, 7, 17, 0, 0, 1536, 1537, 1, 0, 0, 0, 1537, 1538, 6, 194, 13, 0, 1538, 1539, 6, 194, 0, 0, 1539, 406, 1, 0, 0, 0, 1540, 1541, 3, 223, 103, 0, 1541, 1542, 1, 0, 0, 0, 1542, 1543, 6, 195, 22, 0, 1543, 408, 1, 0, 0, 0, 1544, 1545, 3, 95, 39, 0, 1545, 1546, 1, 0, 0, 0, 1546, 1547, 6, 196, 23, 0, 1547, 410, 1, 0, 0, 0, 1548, 1549, 3, 111, 47, 0, 1549, 1550, 1, 0, 0, 0, 1550, 1551, 6, 197, 19, 0, 1551, 412, 1, 0, 0, 0, 1552, 1553, 3, 185, 84, 0, 1553, 1554, 1, 0, 0, 0, 1554, 1555, 6, 198, 33, 0, 1555, 414, 1, 0, 0, 0, 1556, 1557, 3, 189, 86, 0, 1557, 1558, 1, 0, 0, 0, 1558, 1559, 6, 199, 32, 0, 1559, 416, 1, 0, 0, 0, 1560, 1561, 3, 67, 25, 0, 1561, 1562, 1, 0, 0, 0, 1562, 1563, 6, 200, 12, 0, 1563, 418, 1, 0, 0, 0, 1564, 1565, 3, 69, 26, 0, 1565, 1566, 1, 0, 0, 0, 1566, 1567, 6, 201, 12, 0, 1567, 420, 1, 0, 0, 0, 1568, 1569, 3, 71, 27, 0, 1569, 1570, 1, 0, 0, 0, 1570, 1571, 6, 202, 12, 0, 1571, 422, 1, 0, 0, 0, 1572, 1573, 3, 73, 28, 0, 1573, 1574, 1, 0, 0, 0, 1574, 1575, 6, 203, 17, 0, 1575, 1576, 6, 203, 13, 0, 1576, 424, 1, 0, 0, 0, 1577, 1578, 3, 223, 103, 0, 1578, 1579, 1, 0, 0, 0, 1579, 1580, 6, 204, 22, 0, 1580, 1581, 6, 204, 13, 0, 1581, 1582, 6, 204, 37, 0, 1582, 426, 1, 0, 0, 0, 1583, 1584, 3, 95, 39, 0, 1584, 1585, 1, 0, 0, 0, 1585, 1586, 6, 205, 23, 0, 1586, 1587, 6, 205, 13, 0, 1587, 1588, 6, 205, 37, 0, 1588, 428, 1, 0, 0, 0, 1589, 1590, 3, 67, 25, 0, 1590, 1591, 1, 0, 0, 0, 1591, 1592, 6, 206, 12, 0, 1592, 430, 1, 0, 0, 0, 1593, 1594, 3, 69, 26, 0, 1594, 1595, 1, 0, 0, 0, 1595, 1596, 6, 207, 12, 0, 1596, 432, 1, 0, 0, 0, 1597, 1598, 3, 71, 27, 0, 1598, 1599, 1, 0, 0, 0, 1599, 1600, 6, 208, 12, 0, 1600, 434, 1, 0, 0, 0, 1601, 1602, 3, 111, 47, 0, 1602, 1603, 1, 0, 0, 0, 1603, 1604, 6, 209, 19, 0, 1604, 1605, 6, 209, 13, 0, 1605, 1606, 6, 209, 11, 0, 1606, 436, 1, 0, 0, 0, 1607, 1608, 3, 113, 48, 0, 1608, 1609, 1, 0, 0, 0, 1609, 1610, 6, 210, 20, 0, 1610, 1611, 6, 210, 13, 0, 1611, 1612, 6, 210, 11, 0, 1612, 438, 1, 0, 0, 0, 1613, 1614, 3, 67, 25, 0, 1614, 1615, 1, 0, 0, 0, 1615, 1616, 6, 211, 12, 0, 1616, 440, 1, 0, 0, 0, 1617, 1618, 3, 69, 26, 0, 1618, 1619, 1, 0, 0, 0, 1619, 1620, 6, 212, 12, 0, 1620, 442, 1, 0, 0, 0, 1621, 1622, 3, 71, 27, 0, 1622, 1623, 1, 0, 0, 0, 1623, 1624, 6, 213, 12, 0, 1624, 444, 1, 0, 0, 0, 1625, 1626, 3, 189, 86, 0, 1626, 1627, 1, 0, 0, 0, 1627, 1628, 6, 214, 13, 0, 1628, 1629, 6, 214, 0, 0, 1629, 1630, 6, 214, 32, 0, 1630, 446, 1, 0, 0, 0, 1631, 1632, 3, 185, 84, 0, 1632, 1633, 1, 0, 0, 0, 1633, 1634, 6, 215, 13, 0, 1634, 1635, 6, 215, 0, 0, 1635, 1636, 6, 215, 33, 0, 1636, 448, 1, 0, 0, 0, 1637, 1638, 3, 101, 42, 0, 1638, 1639, 1, 0, 0, 0, 1639, 1640, 6, 216, 13, 0, 1640, 1641, 6, 216, 0, 0, 1641, 1642, 6, 216, 38, 0, 1642, 450, 1, 0, 0, 0, 1643, 1644, 3, 73, 28, 0, 1644, 1645, 1, 0, 0, 0, 1645, 1646, 6, 217, 17, 0, 1646, 1647, 6, 217, 13, 0, 1647, 452, 1, 0, 0, 0, 1648, 1649, 3, 73, 28, 0, 1649, 1650, 1, 0, 0, 0, 1650, 1651, 6, 218, 17, 0, 1651, 1652, 6, 218, 13, 0, 1652, 454, 1, 0, 0, 0, 1653, 1654, 3, 283, 133, 0, 1654, 1655, 1, 0, 0, 0, 1655, 1656, 6, 219, 34, 0, 1656, 456, 1, 0, 0, 0, 1657, 1658, 3, 269, 126, 0, 1658, 1659, 1, 0, 0, 0, 1659, 1660, 6, 220, 36, 0, 1660, 458, 1, 0, 0, 0, 1661, 1662, 3, 117, 50, 0, 1662, 1663, 1, 0, 0, 0, 1663, 1664, 6, 221, 24, 0, 1664, 460, 1, 0, 0, 0, 1665, 1666, 3, 113, 48, 0, 1666, 1667, 1, 0, 0, 0, 1667, 1668, 6, 222, 20, 0, 1668, 462, 1, 0, 0, 0, 1669, 1670, 3, 189, 86, 0, 1670, 1671, 1, 0, 0, 0, 1671, 1672, 6, 223, 32, 0, 1672, 464, 1, 0, 0, 0, 1673, 1674, 3, 185, 84, 0, 1674, 1675, 1, 0, 0, 0, 1675, 1676, 6, 224, 33, 0, 1676, 466, 1, 0, 0, 0, 1677, 1678, 3, 67, 25, 0, 1678, 1679, 1, 0, 0, 0, 1679, 1680, 6, 225, 12, 0, 1680, 468, 1, 0, 0, 0, 1681, 1682, 3, 69, 26, 0, 1682, 1683, 1, 0, 0, 0, 1683, 1684, 6, 226, 12, 0, 1684, 470, 1, 0, 0, 0, 1685, 1686, 3, 71, 27, 0, 1686, 1687, 1, 0, 0, 0, 1687, 1688, 6, 227, 12, 0, 1688, 472, 1, 0, 0, 0, 67, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 693, 703, 707, 710, 719, 721, 732, 751, 756, 765, 772, 777, 779, 790, 798, 801, 803, 808, 813, 819, 826, 831, 837, 840, 848, 852, 984, 989, 996, 998, 1014, 1019, 1024, 1026, 1032, 1109, 1114, 1161, 1165, 1170, 1175, 1180, 1182, 1186, 1188, 1273, 1277, 1282, 1423, 1425, 39, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 13, 0, 5, 16, 0, 5, 11, 0, 5, 14, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 72, 0, 5, 0, 0, 7, 29, 0, 7, 73, 0, 7, 38, 0, 7, 39, 0, 7, 36, 0, 7, 83, 0, 7, 30, 0, 7, 41, 0, 7, 53, 0, 7, 71, 0, 7, 87, 0, 5, 10, 0, 5, 7, 0, 7, 97, 0, 7, 96, 0, 7, 75, 0, 7, 74, 0, 7, 95, 0, 5, 12, 0, 7, 91, 0, 5, 15, 0, 7, 33, 0] \ No newline at end of file +[4, 0, 138, 1735, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 2, 198, 7, 198, 2, 199, 7, 199, 2, 200, 7, 200, 2, 201, 7, 201, 2, 202, 7, 202, 2, 203, 7, 203, 2, 204, 7, 204, 2, 205, 7, 205, 2, 206, 7, 206, 2, 207, 7, 207, 2, 208, 7, 208, 2, 209, 7, 209, 2, 210, 7, 210, 2, 211, 7, 211, 2, 212, 7, 212, 2, 213, 7, 213, 2, 214, 7, 214, 2, 215, 7, 215, 2, 216, 7, 216, 2, 217, 7, 217, 2, 218, 7, 218, 2, 219, 7, 219, 2, 220, 7, 220, 2, 221, 7, 221, 2, 222, 7, 222, 2, 223, 7, 223, 2, 224, 7, 224, 2, 225, 7, 225, 2, 226, 7, 226, 2, 227, 7, 227, 2, 228, 7, 228, 2, 229, 7, 229, 2, 230, 7, 230, 2, 231, 7, 231, 2, 232, 7, 232, 2, 233, 7, 233, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 4, 25, 717, 8, 25, 11, 25, 12, 25, 718, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 5, 26, 727, 8, 26, 10, 26, 12, 26, 730, 9, 26, 1, 26, 3, 26, 733, 8, 26, 1, 26, 3, 26, 736, 8, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 745, 8, 27, 10, 27, 12, 27, 748, 9, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 4, 28, 756, 8, 28, 11, 28, 12, 28, 757, 1, 28, 1, 28, 1, 29, 1, 29, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 34, 1, 34, 3, 34, 777, 8, 34, 1, 34, 4, 34, 780, 8, 34, 11, 34, 12, 34, 781, 1, 35, 1, 35, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 3, 37, 791, 8, 37, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 3, 39, 798, 8, 39, 1, 40, 1, 40, 1, 40, 5, 40, 803, 8, 40, 10, 40, 12, 40, 806, 9, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 5, 40, 814, 8, 40, 10, 40, 12, 40, 817, 9, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 3, 40, 824, 8, 40, 1, 40, 3, 40, 827, 8, 40, 3, 40, 829, 8, 40, 1, 41, 4, 41, 832, 8, 41, 11, 41, 12, 41, 833, 1, 42, 4, 42, 837, 8, 42, 11, 42, 12, 42, 838, 1, 42, 1, 42, 5, 42, 843, 8, 42, 10, 42, 12, 42, 846, 9, 42, 1, 42, 1, 42, 4, 42, 850, 8, 42, 11, 42, 12, 42, 851, 1, 42, 4, 42, 855, 8, 42, 11, 42, 12, 42, 856, 1, 42, 1, 42, 5, 42, 861, 8, 42, 10, 42, 12, 42, 864, 9, 42, 3, 42, 866, 8, 42, 1, 42, 1, 42, 1, 42, 1, 42, 4, 42, 872, 8, 42, 11, 42, 12, 42, 873, 1, 42, 1, 42, 3, 42, 878, 8, 42, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 3, 82, 1010, 8, 82, 1, 82, 5, 82, 1013, 8, 82, 10, 82, 12, 82, 1016, 9, 82, 1, 82, 1, 82, 4, 82, 1020, 8, 82, 11, 82, 12, 82, 1021, 3, 82, 1024, 8, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 5, 85, 1038, 8, 85, 10, 85, 12, 85, 1041, 9, 85, 1, 85, 1, 85, 3, 85, 1045, 8, 85, 1, 85, 4, 85, 1048, 8, 85, 11, 85, 12, 85, 1049, 3, 85, 1052, 8, 85, 1, 86, 1, 86, 4, 86, 1056, 8, 86, 11, 86, 12, 86, 1057, 1, 86, 1, 86, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 3, 103, 1135, 8, 103, 1, 104, 4, 104, 1138, 8, 104, 11, 104, 12, 104, 1139, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 3, 115, 1187, 8, 115, 1, 116, 1, 116, 3, 116, 1191, 8, 116, 1, 116, 5, 116, 1194, 8, 116, 10, 116, 12, 116, 1197, 9, 116, 1, 116, 1, 116, 3, 116, 1201, 8, 116, 1, 116, 4, 116, 1204, 8, 116, 11, 116, 12, 116, 1205, 3, 116, 1208, 8, 116, 1, 117, 1, 117, 4, 117, 1212, 8, 117, 11, 117, 12, 117, 1213, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 137, 4, 137, 1297, 8, 137, 11, 137, 12, 137, 1298, 1, 137, 1, 137, 3, 137, 1303, 8, 137, 1, 137, 4, 137, 1306, 8, 137, 11, 137, 12, 137, 1307, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 4, 170, 1449, 8, 170, 11, 170, 12, 170, 1450, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 198, 1, 198, 1, 198, 1, 198, 1, 199, 1, 199, 1, 199, 1, 199, 1, 200, 1, 200, 1, 200, 1, 200, 1, 201, 1, 201, 1, 201, 1, 201, 1, 202, 1, 202, 1, 202, 1, 202, 1, 203, 1, 203, 1, 203, 1, 203, 1, 204, 1, 204, 1, 204, 1, 204, 1, 204, 1, 205, 1, 205, 1, 205, 1, 205, 1, 205, 1, 205, 1, 206, 1, 206, 1, 206, 1, 206, 1, 206, 1, 206, 1, 207, 1, 207, 1, 207, 1, 207, 1, 208, 1, 208, 1, 208, 1, 208, 1, 209, 1, 209, 1, 209, 1, 209, 1, 210, 1, 210, 1, 210, 1, 210, 1, 210, 1, 210, 1, 211, 1, 211, 1, 211, 1, 211, 1, 211, 1, 211, 1, 212, 1, 212, 1, 212, 1, 212, 1, 213, 1, 213, 1, 213, 1, 213, 1, 214, 1, 214, 1, 214, 1, 214, 1, 215, 1, 215, 1, 215, 1, 215, 1, 215, 1, 215, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 1, 217, 1, 217, 1, 217, 1, 217, 1, 217, 1, 217, 1, 218, 1, 218, 1, 218, 1, 218, 1, 218, 1, 219, 1, 219, 1, 219, 1, 219, 1, 219, 1, 220, 1, 220, 1, 220, 1, 220, 1, 221, 1, 221, 1, 221, 1, 221, 1, 222, 1, 222, 1, 222, 1, 222, 1, 223, 1, 223, 1, 223, 1, 223, 1, 224, 1, 224, 1, 224, 1, 224, 1, 225, 1, 225, 1, 225, 1, 225, 1, 226, 1, 226, 1, 226, 1, 226, 1, 227, 1, 227, 1, 227, 1, 227, 1, 228, 1, 228, 1, 228, 1, 228, 1, 229, 1, 229, 1, 229, 1, 229, 1, 229, 1, 230, 1, 230, 1, 230, 1, 230, 1, 231, 1, 231, 1, 231, 1, 231, 1, 232, 1, 232, 1, 232, 1, 232, 1, 233, 1, 233, 1, 233, 1, 233, 2, 746, 815, 0, 234, 18, 1, 20, 2, 22, 3, 24, 4, 26, 5, 28, 6, 30, 7, 32, 8, 34, 9, 36, 10, 38, 11, 40, 12, 42, 13, 44, 14, 46, 15, 48, 16, 50, 17, 52, 18, 54, 19, 56, 20, 58, 21, 60, 22, 62, 23, 64, 24, 66, 25, 68, 26, 70, 27, 72, 28, 74, 29, 76, 30, 78, 0, 80, 0, 82, 0, 84, 0, 86, 0, 88, 0, 90, 0, 92, 0, 94, 0, 96, 0, 98, 31, 100, 32, 102, 33, 104, 34, 106, 35, 108, 36, 110, 37, 112, 38, 114, 39, 116, 40, 118, 41, 120, 42, 122, 43, 124, 44, 126, 45, 128, 46, 130, 47, 132, 48, 134, 49, 136, 50, 138, 51, 140, 52, 142, 53, 144, 54, 146, 55, 148, 56, 150, 57, 152, 58, 154, 59, 156, 60, 158, 61, 160, 62, 162, 63, 164, 64, 166, 65, 168, 66, 170, 67, 172, 68, 174, 69, 176, 70, 178, 71, 180, 0, 182, 72, 184, 73, 186, 74, 188, 75, 190, 0, 192, 76, 194, 77, 196, 78, 198, 79, 200, 0, 202, 0, 204, 80, 206, 81, 208, 82, 210, 0, 212, 0, 214, 0, 216, 0, 218, 0, 220, 0, 222, 83, 224, 0, 226, 84, 228, 0, 230, 0, 232, 85, 234, 86, 236, 87, 238, 0, 240, 0, 242, 0, 244, 0, 246, 0, 248, 0, 250, 0, 252, 88, 254, 89, 256, 90, 258, 91, 260, 0, 262, 0, 264, 0, 266, 0, 268, 0, 270, 0, 272, 92, 274, 0, 276, 93, 278, 94, 280, 95, 282, 0, 284, 0, 286, 96, 288, 97, 290, 0, 292, 98, 294, 0, 296, 99, 298, 100, 300, 101, 302, 0, 304, 0, 306, 0, 308, 0, 310, 0, 312, 0, 314, 0, 316, 0, 318, 0, 320, 102, 322, 103, 324, 104, 326, 0, 328, 0, 330, 0, 332, 0, 334, 0, 336, 0, 338, 105, 340, 106, 342, 107, 344, 0, 346, 108, 348, 109, 350, 110, 352, 111, 354, 0, 356, 0, 358, 112, 360, 113, 362, 114, 364, 115, 366, 0, 368, 0, 370, 0, 372, 0, 374, 0, 376, 0, 378, 0, 380, 116, 382, 117, 384, 118, 386, 0, 388, 0, 390, 0, 392, 0, 394, 119, 396, 120, 398, 121, 400, 0, 402, 122, 404, 0, 406, 0, 408, 123, 410, 0, 412, 0, 414, 0, 416, 0, 418, 0, 420, 124, 422, 125, 424, 126, 426, 0, 428, 0, 430, 0, 432, 127, 434, 128, 436, 129, 438, 0, 440, 0, 442, 130, 444, 131, 446, 132, 448, 0, 450, 0, 452, 0, 454, 0, 456, 0, 458, 0, 460, 0, 462, 0, 464, 0, 466, 0, 468, 0, 470, 133, 472, 134, 474, 135, 476, 0, 478, 0, 480, 136, 482, 137, 484, 138, 18, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 36, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 2, 0, 74, 74, 106, 106, 1760, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 58, 1, 0, 0, 0, 0, 60, 1, 0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 64, 1, 0, 0, 0, 0, 66, 1, 0, 0, 0, 0, 68, 1, 0, 0, 0, 0, 70, 1, 0, 0, 0, 0, 72, 1, 0, 0, 0, 0, 74, 1, 0, 0, 0, 1, 76, 1, 0, 0, 0, 1, 98, 1, 0, 0, 0, 1, 100, 1, 0, 0, 0, 1, 102, 1, 0, 0, 0, 1, 104, 1, 0, 0, 0, 1, 106, 1, 0, 0, 0, 1, 108, 1, 0, 0, 0, 1, 110, 1, 0, 0, 0, 1, 112, 1, 0, 0, 0, 1, 114, 1, 0, 0, 0, 1, 116, 1, 0, 0, 0, 1, 118, 1, 0, 0, 0, 1, 120, 1, 0, 0, 0, 1, 122, 1, 0, 0, 0, 1, 124, 1, 0, 0, 0, 1, 126, 1, 0, 0, 0, 1, 128, 1, 0, 0, 0, 1, 130, 1, 0, 0, 0, 1, 132, 1, 0, 0, 0, 1, 134, 1, 0, 0, 0, 1, 136, 1, 0, 0, 0, 1, 138, 1, 0, 0, 0, 1, 140, 1, 0, 0, 0, 1, 142, 1, 0, 0, 0, 1, 144, 1, 0, 0, 0, 1, 146, 1, 0, 0, 0, 1, 148, 1, 0, 0, 0, 1, 150, 1, 0, 0, 0, 1, 152, 1, 0, 0, 0, 1, 154, 1, 0, 0, 0, 1, 156, 1, 0, 0, 0, 1, 158, 1, 0, 0, 0, 1, 160, 1, 0, 0, 0, 1, 162, 1, 0, 0, 0, 1, 164, 1, 0, 0, 0, 1, 166, 1, 0, 0, 0, 1, 168, 1, 0, 0, 0, 1, 170, 1, 0, 0, 0, 1, 172, 1, 0, 0, 0, 1, 174, 1, 0, 0, 0, 1, 176, 1, 0, 0, 0, 1, 178, 1, 0, 0, 0, 1, 180, 1, 0, 0, 0, 1, 182, 1, 0, 0, 0, 1, 184, 1, 0, 0, 0, 1, 186, 1, 0, 0, 0, 1, 188, 1, 0, 0, 0, 1, 192, 1, 0, 0, 0, 1, 194, 1, 0, 0, 0, 1, 196, 1, 0, 0, 0, 1, 198, 1, 0, 0, 0, 2, 200, 1, 0, 0, 0, 2, 202, 1, 0, 0, 0, 2, 204, 1, 0, 0, 0, 2, 206, 1, 0, 0, 0, 2, 208, 1, 0, 0, 0, 3, 210, 1, 0, 0, 0, 3, 212, 1, 0, 0, 0, 3, 214, 1, 0, 0, 0, 3, 216, 1, 0, 0, 0, 3, 218, 1, 0, 0, 0, 3, 220, 1, 0, 0, 0, 3, 222, 1, 0, 0, 0, 3, 226, 1, 0, 0, 0, 3, 228, 1, 0, 0, 0, 3, 230, 1, 0, 0, 0, 3, 232, 1, 0, 0, 0, 3, 234, 1, 0, 0, 0, 3, 236, 1, 0, 0, 0, 4, 238, 1, 0, 0, 0, 4, 240, 1, 0, 0, 0, 4, 242, 1, 0, 0, 0, 4, 244, 1, 0, 0, 0, 4, 246, 1, 0, 0, 0, 4, 252, 1, 0, 0, 0, 4, 254, 1, 0, 0, 0, 4, 256, 1, 0, 0, 0, 4, 258, 1, 0, 0, 0, 5, 260, 1, 0, 0, 0, 5, 262, 1, 0, 0, 0, 5, 264, 1, 0, 0, 0, 5, 266, 1, 0, 0, 0, 5, 268, 1, 0, 0, 0, 5, 270, 1, 0, 0, 0, 5, 272, 1, 0, 0, 0, 5, 274, 1, 0, 0, 0, 5, 276, 1, 0, 0, 0, 5, 278, 1, 0, 0, 0, 5, 280, 1, 0, 0, 0, 6, 282, 1, 0, 0, 0, 6, 284, 1, 0, 0, 0, 6, 286, 1, 0, 0, 0, 6, 288, 1, 0, 0, 0, 6, 292, 1, 0, 0, 0, 6, 294, 1, 0, 0, 0, 6, 296, 1, 0, 0, 0, 6, 298, 1, 0, 0, 0, 6, 300, 1, 0, 0, 0, 7, 302, 1, 0, 0, 0, 7, 304, 1, 0, 0, 0, 7, 306, 1, 0, 0, 0, 7, 308, 1, 0, 0, 0, 7, 310, 1, 0, 0, 0, 7, 312, 1, 0, 0, 0, 7, 314, 1, 0, 0, 0, 7, 316, 1, 0, 0, 0, 7, 318, 1, 0, 0, 0, 7, 320, 1, 0, 0, 0, 7, 322, 1, 0, 0, 0, 7, 324, 1, 0, 0, 0, 8, 326, 1, 0, 0, 0, 8, 328, 1, 0, 0, 0, 8, 330, 1, 0, 0, 0, 8, 332, 1, 0, 0, 0, 8, 334, 1, 0, 0, 0, 8, 336, 1, 0, 0, 0, 8, 338, 1, 0, 0, 0, 8, 340, 1, 0, 0, 0, 8, 342, 1, 0, 0, 0, 9, 344, 1, 0, 0, 0, 9, 346, 1, 0, 0, 0, 9, 348, 1, 0, 0, 0, 9, 350, 1, 0, 0, 0, 9, 352, 1, 0, 0, 0, 10, 354, 1, 0, 0, 0, 10, 356, 1, 0, 0, 0, 10, 358, 1, 0, 0, 0, 10, 360, 1, 0, 0, 0, 10, 362, 1, 0, 0, 0, 10, 364, 1, 0, 0, 0, 11, 366, 1, 0, 0, 0, 11, 368, 1, 0, 0, 0, 11, 370, 1, 0, 0, 0, 11, 372, 1, 0, 0, 0, 11, 374, 1, 0, 0, 0, 11, 376, 1, 0, 0, 0, 11, 378, 1, 0, 0, 0, 11, 380, 1, 0, 0, 0, 11, 382, 1, 0, 0, 0, 11, 384, 1, 0, 0, 0, 12, 386, 1, 0, 0, 0, 12, 388, 1, 0, 0, 0, 12, 390, 1, 0, 0, 0, 12, 392, 1, 0, 0, 0, 12, 394, 1, 0, 0, 0, 12, 396, 1, 0, 0, 0, 12, 398, 1, 0, 0, 0, 13, 400, 1, 0, 0, 0, 13, 402, 1, 0, 0, 0, 13, 404, 1, 0, 0, 0, 13, 406, 1, 0, 0, 0, 13, 408, 1, 0, 0, 0, 13, 410, 1, 0, 0, 0, 13, 412, 1, 0, 0, 0, 13, 414, 1, 0, 0, 0, 13, 416, 1, 0, 0, 0, 13, 418, 1, 0, 0, 0, 13, 420, 1, 0, 0, 0, 13, 422, 1, 0, 0, 0, 13, 424, 1, 0, 0, 0, 14, 426, 1, 0, 0, 0, 14, 428, 1, 0, 0, 0, 14, 430, 1, 0, 0, 0, 14, 432, 1, 0, 0, 0, 14, 434, 1, 0, 0, 0, 14, 436, 1, 0, 0, 0, 15, 438, 1, 0, 0, 0, 15, 440, 1, 0, 0, 0, 15, 442, 1, 0, 0, 0, 15, 444, 1, 0, 0, 0, 15, 446, 1, 0, 0, 0, 15, 448, 1, 0, 0, 0, 15, 450, 1, 0, 0, 0, 15, 452, 1, 0, 0, 0, 15, 454, 1, 0, 0, 0, 16, 456, 1, 0, 0, 0, 16, 458, 1, 0, 0, 0, 16, 460, 1, 0, 0, 0, 16, 462, 1, 0, 0, 0, 16, 464, 1, 0, 0, 0, 16, 466, 1, 0, 0, 0, 16, 468, 1, 0, 0, 0, 16, 470, 1, 0, 0, 0, 16, 472, 1, 0, 0, 0, 16, 474, 1, 0, 0, 0, 17, 476, 1, 0, 0, 0, 17, 478, 1, 0, 0, 0, 17, 480, 1, 0, 0, 0, 17, 482, 1, 0, 0, 0, 17, 484, 1, 0, 0, 0, 18, 486, 1, 0, 0, 0, 20, 496, 1, 0, 0, 0, 22, 503, 1, 0, 0, 0, 24, 512, 1, 0, 0, 0, 26, 519, 1, 0, 0, 0, 28, 529, 1, 0, 0, 0, 30, 536, 1, 0, 0, 0, 32, 543, 1, 0, 0, 0, 34, 550, 1, 0, 0, 0, 36, 558, 1, 0, 0, 0, 38, 570, 1, 0, 0, 0, 40, 579, 1, 0, 0, 0, 42, 585, 1, 0, 0, 0, 44, 592, 1, 0, 0, 0, 46, 599, 1, 0, 0, 0, 48, 607, 1, 0, 0, 0, 50, 615, 1, 0, 0, 0, 52, 624, 1, 0, 0, 0, 54, 640, 1, 0, 0, 0, 56, 655, 1, 0, 0, 0, 58, 667, 1, 0, 0, 0, 60, 679, 1, 0, 0, 0, 62, 690, 1, 0, 0, 0, 64, 698, 1, 0, 0, 0, 66, 706, 1, 0, 0, 0, 68, 716, 1, 0, 0, 0, 70, 722, 1, 0, 0, 0, 72, 739, 1, 0, 0, 0, 74, 755, 1, 0, 0, 0, 76, 761, 1, 0, 0, 0, 78, 765, 1, 0, 0, 0, 80, 767, 1, 0, 0, 0, 82, 769, 1, 0, 0, 0, 84, 772, 1, 0, 0, 0, 86, 774, 1, 0, 0, 0, 88, 783, 1, 0, 0, 0, 90, 785, 1, 0, 0, 0, 92, 790, 1, 0, 0, 0, 94, 792, 1, 0, 0, 0, 96, 797, 1, 0, 0, 0, 98, 828, 1, 0, 0, 0, 100, 831, 1, 0, 0, 0, 102, 877, 1, 0, 0, 0, 104, 879, 1, 0, 0, 0, 106, 882, 1, 0, 0, 0, 108, 886, 1, 0, 0, 0, 110, 890, 1, 0, 0, 0, 112, 892, 1, 0, 0, 0, 114, 895, 1, 0, 0, 0, 116, 897, 1, 0, 0, 0, 118, 899, 1, 0, 0, 0, 120, 904, 1, 0, 0, 0, 122, 906, 1, 0, 0, 0, 124, 912, 1, 0, 0, 0, 126, 918, 1, 0, 0, 0, 128, 921, 1, 0, 0, 0, 130, 924, 1, 0, 0, 0, 132, 929, 1, 0, 0, 0, 134, 934, 1, 0, 0, 0, 136, 936, 1, 0, 0, 0, 138, 940, 1, 0, 0, 0, 140, 945, 1, 0, 0, 0, 142, 951, 1, 0, 0, 0, 144, 954, 1, 0, 0, 0, 146, 956, 1, 0, 0, 0, 148, 962, 1, 0, 0, 0, 150, 964, 1, 0, 0, 0, 152, 969, 1, 0, 0, 0, 154, 972, 1, 0, 0, 0, 156, 975, 1, 0, 0, 0, 158, 978, 1, 0, 0, 0, 160, 980, 1, 0, 0, 0, 162, 983, 1, 0, 0, 0, 164, 985, 1, 0, 0, 0, 166, 988, 1, 0, 0, 0, 168, 990, 1, 0, 0, 0, 170, 992, 1, 0, 0, 0, 172, 994, 1, 0, 0, 0, 174, 996, 1, 0, 0, 0, 176, 998, 1, 0, 0, 0, 178, 1000, 1, 0, 0, 0, 180, 1002, 1, 0, 0, 0, 182, 1023, 1, 0, 0, 0, 184, 1025, 1, 0, 0, 0, 186, 1030, 1, 0, 0, 0, 188, 1051, 1, 0, 0, 0, 190, 1053, 1, 0, 0, 0, 192, 1061, 1, 0, 0, 0, 194, 1063, 1, 0, 0, 0, 196, 1067, 1, 0, 0, 0, 198, 1071, 1, 0, 0, 0, 200, 1075, 1, 0, 0, 0, 202, 1080, 1, 0, 0, 0, 204, 1085, 1, 0, 0, 0, 206, 1089, 1, 0, 0, 0, 208, 1093, 1, 0, 0, 0, 210, 1097, 1, 0, 0, 0, 212, 1102, 1, 0, 0, 0, 214, 1106, 1, 0, 0, 0, 216, 1110, 1, 0, 0, 0, 218, 1114, 1, 0, 0, 0, 220, 1118, 1, 0, 0, 0, 222, 1122, 1, 0, 0, 0, 224, 1134, 1, 0, 0, 0, 226, 1137, 1, 0, 0, 0, 228, 1141, 1, 0, 0, 0, 230, 1145, 1, 0, 0, 0, 232, 1149, 1, 0, 0, 0, 234, 1153, 1, 0, 0, 0, 236, 1157, 1, 0, 0, 0, 238, 1161, 1, 0, 0, 0, 240, 1166, 1, 0, 0, 0, 242, 1170, 1, 0, 0, 0, 244, 1174, 1, 0, 0, 0, 246, 1178, 1, 0, 0, 0, 248, 1186, 1, 0, 0, 0, 250, 1207, 1, 0, 0, 0, 252, 1211, 1, 0, 0, 0, 254, 1215, 1, 0, 0, 0, 256, 1219, 1, 0, 0, 0, 258, 1223, 1, 0, 0, 0, 260, 1227, 1, 0, 0, 0, 262, 1232, 1, 0, 0, 0, 264, 1236, 1, 0, 0, 0, 266, 1240, 1, 0, 0, 0, 268, 1244, 1, 0, 0, 0, 270, 1248, 1, 0, 0, 0, 272, 1252, 1, 0, 0, 0, 274, 1255, 1, 0, 0, 0, 276, 1259, 1, 0, 0, 0, 278, 1263, 1, 0, 0, 0, 280, 1267, 1, 0, 0, 0, 282, 1271, 1, 0, 0, 0, 284, 1276, 1, 0, 0, 0, 286, 1281, 1, 0, 0, 0, 288, 1286, 1, 0, 0, 0, 290, 1293, 1, 0, 0, 0, 292, 1302, 1, 0, 0, 0, 294, 1309, 1, 0, 0, 0, 296, 1313, 1, 0, 0, 0, 298, 1317, 1, 0, 0, 0, 300, 1321, 1, 0, 0, 0, 302, 1325, 1, 0, 0, 0, 304, 1331, 1, 0, 0, 0, 306, 1335, 1, 0, 0, 0, 308, 1339, 1, 0, 0, 0, 310, 1343, 1, 0, 0, 0, 312, 1347, 1, 0, 0, 0, 314, 1351, 1, 0, 0, 0, 316, 1355, 1, 0, 0, 0, 318, 1359, 1, 0, 0, 0, 320, 1363, 1, 0, 0, 0, 322, 1367, 1, 0, 0, 0, 324, 1371, 1, 0, 0, 0, 326, 1375, 1, 0, 0, 0, 328, 1380, 1, 0, 0, 0, 330, 1384, 1, 0, 0, 0, 332, 1388, 1, 0, 0, 0, 334, 1392, 1, 0, 0, 0, 336, 1396, 1, 0, 0, 0, 338, 1400, 1, 0, 0, 0, 340, 1404, 1, 0, 0, 0, 342, 1408, 1, 0, 0, 0, 344, 1412, 1, 0, 0, 0, 346, 1417, 1, 0, 0, 0, 348, 1422, 1, 0, 0, 0, 350, 1426, 1, 0, 0, 0, 352, 1430, 1, 0, 0, 0, 354, 1434, 1, 0, 0, 0, 356, 1439, 1, 0, 0, 0, 358, 1448, 1, 0, 0, 0, 360, 1452, 1, 0, 0, 0, 362, 1456, 1, 0, 0, 0, 364, 1460, 1, 0, 0, 0, 366, 1464, 1, 0, 0, 0, 368, 1469, 1, 0, 0, 0, 370, 1473, 1, 0, 0, 0, 372, 1477, 1, 0, 0, 0, 374, 1481, 1, 0, 0, 0, 376, 1486, 1, 0, 0, 0, 378, 1490, 1, 0, 0, 0, 380, 1494, 1, 0, 0, 0, 382, 1498, 1, 0, 0, 0, 384, 1502, 1, 0, 0, 0, 386, 1506, 1, 0, 0, 0, 388, 1512, 1, 0, 0, 0, 390, 1516, 1, 0, 0, 0, 392, 1520, 1, 0, 0, 0, 394, 1524, 1, 0, 0, 0, 396, 1528, 1, 0, 0, 0, 398, 1532, 1, 0, 0, 0, 400, 1536, 1, 0, 0, 0, 402, 1541, 1, 0, 0, 0, 404, 1546, 1, 0, 0, 0, 406, 1550, 1, 0, 0, 0, 408, 1556, 1, 0, 0, 0, 410, 1565, 1, 0, 0, 0, 412, 1569, 1, 0, 0, 0, 414, 1573, 1, 0, 0, 0, 416, 1577, 1, 0, 0, 0, 418, 1581, 1, 0, 0, 0, 420, 1585, 1, 0, 0, 0, 422, 1589, 1, 0, 0, 0, 424, 1593, 1, 0, 0, 0, 426, 1597, 1, 0, 0, 0, 428, 1602, 1, 0, 0, 0, 430, 1608, 1, 0, 0, 0, 432, 1614, 1, 0, 0, 0, 434, 1618, 1, 0, 0, 0, 436, 1622, 1, 0, 0, 0, 438, 1626, 1, 0, 0, 0, 440, 1632, 1, 0, 0, 0, 442, 1638, 1, 0, 0, 0, 444, 1642, 1, 0, 0, 0, 446, 1646, 1, 0, 0, 0, 448, 1650, 1, 0, 0, 0, 450, 1656, 1, 0, 0, 0, 452, 1662, 1, 0, 0, 0, 454, 1668, 1, 0, 0, 0, 456, 1673, 1, 0, 0, 0, 458, 1678, 1, 0, 0, 0, 460, 1682, 1, 0, 0, 0, 462, 1686, 1, 0, 0, 0, 464, 1690, 1, 0, 0, 0, 466, 1694, 1, 0, 0, 0, 468, 1698, 1, 0, 0, 0, 470, 1702, 1, 0, 0, 0, 472, 1706, 1, 0, 0, 0, 474, 1710, 1, 0, 0, 0, 476, 1714, 1, 0, 0, 0, 478, 1719, 1, 0, 0, 0, 480, 1723, 1, 0, 0, 0, 482, 1727, 1, 0, 0, 0, 484, 1731, 1, 0, 0, 0, 486, 487, 7, 0, 0, 0, 487, 488, 7, 1, 0, 0, 488, 489, 7, 2, 0, 0, 489, 490, 7, 2, 0, 0, 490, 491, 7, 3, 0, 0, 491, 492, 7, 4, 0, 0, 492, 493, 7, 5, 0, 0, 493, 494, 1, 0, 0, 0, 494, 495, 6, 0, 0, 0, 495, 19, 1, 0, 0, 0, 496, 497, 7, 0, 0, 0, 497, 498, 7, 6, 0, 0, 498, 499, 7, 7, 0, 0, 499, 500, 7, 8, 0, 0, 500, 501, 1, 0, 0, 0, 501, 502, 6, 1, 1, 0, 502, 21, 1, 0, 0, 0, 503, 504, 7, 3, 0, 0, 504, 505, 7, 9, 0, 0, 505, 506, 7, 6, 0, 0, 506, 507, 7, 1, 0, 0, 507, 508, 7, 4, 0, 0, 508, 509, 7, 10, 0, 0, 509, 510, 1, 0, 0, 0, 510, 511, 6, 2, 2, 0, 511, 23, 1, 0, 0, 0, 512, 513, 7, 3, 0, 0, 513, 514, 7, 11, 0, 0, 514, 515, 7, 12, 0, 0, 515, 516, 7, 13, 0, 0, 516, 517, 1, 0, 0, 0, 517, 518, 6, 3, 0, 0, 518, 25, 1, 0, 0, 0, 519, 520, 7, 3, 0, 0, 520, 521, 7, 14, 0, 0, 521, 522, 7, 8, 0, 0, 522, 523, 7, 13, 0, 0, 523, 524, 7, 12, 0, 0, 524, 525, 7, 1, 0, 0, 525, 526, 7, 9, 0, 0, 526, 527, 1, 0, 0, 0, 527, 528, 6, 4, 3, 0, 528, 27, 1, 0, 0, 0, 529, 530, 7, 15, 0, 0, 530, 531, 7, 6, 0, 0, 531, 532, 7, 7, 0, 0, 532, 533, 7, 16, 0, 0, 533, 534, 1, 0, 0, 0, 534, 535, 6, 5, 4, 0, 535, 29, 1, 0, 0, 0, 536, 537, 7, 17, 0, 0, 537, 538, 7, 6, 0, 0, 538, 539, 7, 7, 0, 0, 539, 540, 7, 18, 0, 0, 540, 541, 1, 0, 0, 0, 541, 542, 6, 6, 0, 0, 542, 31, 1, 0, 0, 0, 543, 544, 7, 18, 0, 0, 544, 545, 7, 3, 0, 0, 545, 546, 7, 3, 0, 0, 546, 547, 7, 8, 0, 0, 547, 548, 1, 0, 0, 0, 548, 549, 6, 7, 1, 0, 549, 33, 1, 0, 0, 0, 550, 551, 7, 13, 0, 0, 551, 552, 7, 1, 0, 0, 552, 553, 7, 16, 0, 0, 553, 554, 7, 1, 0, 0, 554, 555, 7, 5, 0, 0, 555, 556, 1, 0, 0, 0, 556, 557, 6, 8, 0, 0, 557, 35, 1, 0, 0, 0, 558, 559, 7, 16, 0, 0, 559, 560, 7, 11, 0, 0, 560, 561, 5, 95, 0, 0, 561, 562, 7, 3, 0, 0, 562, 563, 7, 14, 0, 0, 563, 564, 7, 8, 0, 0, 564, 565, 7, 12, 0, 0, 565, 566, 7, 9, 0, 0, 566, 567, 7, 0, 0, 0, 567, 568, 1, 0, 0, 0, 568, 569, 6, 9, 5, 0, 569, 37, 1, 0, 0, 0, 570, 571, 7, 6, 0, 0, 571, 572, 7, 3, 0, 0, 572, 573, 7, 9, 0, 0, 573, 574, 7, 12, 0, 0, 574, 575, 7, 16, 0, 0, 575, 576, 7, 3, 0, 0, 576, 577, 1, 0, 0, 0, 577, 578, 6, 10, 6, 0, 578, 39, 1, 0, 0, 0, 579, 580, 7, 6, 0, 0, 580, 581, 7, 7, 0, 0, 581, 582, 7, 19, 0, 0, 582, 583, 1, 0, 0, 0, 583, 584, 6, 11, 0, 0, 584, 41, 1, 0, 0, 0, 585, 586, 7, 2, 0, 0, 586, 587, 7, 10, 0, 0, 587, 588, 7, 7, 0, 0, 588, 589, 7, 19, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 6, 12, 7, 0, 591, 43, 1, 0, 0, 0, 592, 593, 7, 2, 0, 0, 593, 594, 7, 7, 0, 0, 594, 595, 7, 6, 0, 0, 595, 596, 7, 5, 0, 0, 596, 597, 1, 0, 0, 0, 597, 598, 6, 13, 0, 0, 598, 45, 1, 0, 0, 0, 599, 600, 7, 2, 0, 0, 600, 601, 7, 5, 0, 0, 601, 602, 7, 12, 0, 0, 602, 603, 7, 5, 0, 0, 603, 604, 7, 2, 0, 0, 604, 605, 1, 0, 0, 0, 605, 606, 6, 14, 0, 0, 606, 47, 1, 0, 0, 0, 607, 608, 7, 19, 0, 0, 608, 609, 7, 10, 0, 0, 609, 610, 7, 3, 0, 0, 610, 611, 7, 6, 0, 0, 611, 612, 7, 3, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 6, 15, 0, 0, 614, 49, 1, 0, 0, 0, 615, 616, 7, 13, 0, 0, 616, 617, 7, 7, 0, 0, 617, 618, 7, 7, 0, 0, 618, 619, 7, 18, 0, 0, 619, 620, 7, 20, 0, 0, 620, 621, 7, 8, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 6, 16, 8, 0, 623, 51, 1, 0, 0, 0, 624, 625, 4, 17, 0, 0, 625, 626, 7, 4, 0, 0, 626, 627, 7, 10, 0, 0, 627, 628, 7, 12, 0, 0, 628, 629, 7, 9, 0, 0, 629, 630, 7, 17, 0, 0, 630, 631, 7, 3, 0, 0, 631, 632, 5, 95, 0, 0, 632, 633, 7, 8, 0, 0, 633, 634, 7, 7, 0, 0, 634, 635, 7, 1, 0, 0, 635, 636, 7, 9, 0, 0, 636, 637, 7, 5, 0, 0, 637, 638, 1, 0, 0, 0, 638, 639, 6, 17, 9, 0, 639, 53, 1, 0, 0, 0, 640, 641, 4, 18, 1, 0, 641, 642, 7, 1, 0, 0, 642, 643, 7, 9, 0, 0, 643, 644, 7, 13, 0, 0, 644, 645, 7, 1, 0, 0, 645, 646, 7, 9, 0, 0, 646, 647, 7, 3, 0, 0, 647, 648, 7, 2, 0, 0, 648, 649, 7, 5, 0, 0, 649, 650, 7, 12, 0, 0, 650, 651, 7, 5, 0, 0, 651, 652, 7, 2, 0, 0, 652, 653, 1, 0, 0, 0, 653, 654, 6, 18, 0, 0, 654, 55, 1, 0, 0, 0, 655, 656, 4, 19, 2, 0, 656, 657, 7, 1, 0, 0, 657, 658, 7, 9, 0, 0, 658, 659, 7, 2, 0, 0, 659, 660, 7, 1, 0, 0, 660, 661, 7, 2, 0, 0, 661, 662, 7, 5, 0, 0, 662, 663, 5, 95, 0, 0, 663, 664, 5, 128020, 0, 0, 664, 665, 1, 0, 0, 0, 665, 666, 6, 19, 1, 0, 666, 57, 1, 0, 0, 0, 667, 668, 4, 20, 3, 0, 668, 669, 7, 13, 0, 0, 669, 670, 7, 7, 0, 0, 670, 671, 7, 7, 0, 0, 671, 672, 7, 18, 0, 0, 672, 673, 7, 20, 0, 0, 673, 674, 7, 8, 0, 0, 674, 675, 5, 95, 0, 0, 675, 676, 5, 128020, 0, 0, 676, 677, 1, 0, 0, 0, 677, 678, 6, 20, 10, 0, 678, 59, 1, 0, 0, 0, 679, 680, 4, 21, 4, 0, 680, 681, 7, 16, 0, 0, 681, 682, 7, 3, 0, 0, 682, 683, 7, 5, 0, 0, 683, 684, 7, 6, 0, 0, 684, 685, 7, 1, 0, 0, 685, 686, 7, 4, 0, 0, 686, 687, 7, 2, 0, 0, 687, 688, 1, 0, 0, 0, 688, 689, 6, 21, 11, 0, 689, 61, 1, 0, 0, 0, 690, 691, 4, 22, 5, 0, 691, 692, 7, 15, 0, 0, 692, 693, 7, 20, 0, 0, 693, 694, 7, 13, 0, 0, 694, 695, 7, 13, 0, 0, 695, 696, 1, 0, 0, 0, 696, 697, 6, 22, 8, 0, 697, 63, 1, 0, 0, 0, 698, 699, 4, 23, 6, 0, 699, 700, 7, 13, 0, 0, 700, 701, 7, 3, 0, 0, 701, 702, 7, 15, 0, 0, 702, 703, 7, 5, 0, 0, 703, 704, 1, 0, 0, 0, 704, 705, 6, 23, 8, 0, 705, 65, 1, 0, 0, 0, 706, 707, 4, 24, 7, 0, 707, 708, 7, 6, 0, 0, 708, 709, 7, 1, 0, 0, 709, 710, 7, 17, 0, 0, 710, 711, 7, 10, 0, 0, 711, 712, 7, 5, 0, 0, 712, 713, 1, 0, 0, 0, 713, 714, 6, 24, 8, 0, 714, 67, 1, 0, 0, 0, 715, 717, 8, 21, 0, 0, 716, 715, 1, 0, 0, 0, 717, 718, 1, 0, 0, 0, 718, 716, 1, 0, 0, 0, 718, 719, 1, 0, 0, 0, 719, 720, 1, 0, 0, 0, 720, 721, 6, 25, 0, 0, 721, 69, 1, 0, 0, 0, 722, 723, 5, 47, 0, 0, 723, 724, 5, 47, 0, 0, 724, 728, 1, 0, 0, 0, 725, 727, 8, 22, 0, 0, 726, 725, 1, 0, 0, 0, 727, 730, 1, 0, 0, 0, 728, 726, 1, 0, 0, 0, 728, 729, 1, 0, 0, 0, 729, 732, 1, 0, 0, 0, 730, 728, 1, 0, 0, 0, 731, 733, 5, 13, 0, 0, 732, 731, 1, 0, 0, 0, 732, 733, 1, 0, 0, 0, 733, 735, 1, 0, 0, 0, 734, 736, 5, 10, 0, 0, 735, 734, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 738, 6, 26, 12, 0, 738, 71, 1, 0, 0, 0, 739, 740, 5, 47, 0, 0, 740, 741, 5, 42, 0, 0, 741, 746, 1, 0, 0, 0, 742, 745, 3, 72, 27, 0, 743, 745, 9, 0, 0, 0, 744, 742, 1, 0, 0, 0, 744, 743, 1, 0, 0, 0, 745, 748, 1, 0, 0, 0, 746, 747, 1, 0, 0, 0, 746, 744, 1, 0, 0, 0, 747, 749, 1, 0, 0, 0, 748, 746, 1, 0, 0, 0, 749, 750, 5, 42, 0, 0, 750, 751, 5, 47, 0, 0, 751, 752, 1, 0, 0, 0, 752, 753, 6, 27, 12, 0, 753, 73, 1, 0, 0, 0, 754, 756, 7, 23, 0, 0, 755, 754, 1, 0, 0, 0, 756, 757, 1, 0, 0, 0, 757, 755, 1, 0, 0, 0, 757, 758, 1, 0, 0, 0, 758, 759, 1, 0, 0, 0, 759, 760, 6, 28, 12, 0, 760, 75, 1, 0, 0, 0, 761, 762, 5, 124, 0, 0, 762, 763, 1, 0, 0, 0, 763, 764, 6, 29, 13, 0, 764, 77, 1, 0, 0, 0, 765, 766, 7, 24, 0, 0, 766, 79, 1, 0, 0, 0, 767, 768, 7, 25, 0, 0, 768, 81, 1, 0, 0, 0, 769, 770, 5, 92, 0, 0, 770, 771, 7, 26, 0, 0, 771, 83, 1, 0, 0, 0, 772, 773, 8, 27, 0, 0, 773, 85, 1, 0, 0, 0, 774, 776, 7, 3, 0, 0, 775, 777, 7, 28, 0, 0, 776, 775, 1, 0, 0, 0, 776, 777, 1, 0, 0, 0, 777, 779, 1, 0, 0, 0, 778, 780, 3, 78, 30, 0, 779, 778, 1, 0, 0, 0, 780, 781, 1, 0, 0, 0, 781, 779, 1, 0, 0, 0, 781, 782, 1, 0, 0, 0, 782, 87, 1, 0, 0, 0, 783, 784, 5, 64, 0, 0, 784, 89, 1, 0, 0, 0, 785, 786, 5, 96, 0, 0, 786, 91, 1, 0, 0, 0, 787, 791, 8, 29, 0, 0, 788, 789, 5, 96, 0, 0, 789, 791, 5, 96, 0, 0, 790, 787, 1, 0, 0, 0, 790, 788, 1, 0, 0, 0, 791, 93, 1, 0, 0, 0, 792, 793, 5, 95, 0, 0, 793, 95, 1, 0, 0, 0, 794, 798, 3, 80, 31, 0, 795, 798, 3, 78, 30, 0, 796, 798, 3, 94, 38, 0, 797, 794, 1, 0, 0, 0, 797, 795, 1, 0, 0, 0, 797, 796, 1, 0, 0, 0, 798, 97, 1, 0, 0, 0, 799, 804, 5, 34, 0, 0, 800, 803, 3, 82, 32, 0, 801, 803, 3, 84, 33, 0, 802, 800, 1, 0, 0, 0, 802, 801, 1, 0, 0, 0, 803, 806, 1, 0, 0, 0, 804, 802, 1, 0, 0, 0, 804, 805, 1, 0, 0, 0, 805, 807, 1, 0, 0, 0, 806, 804, 1, 0, 0, 0, 807, 829, 5, 34, 0, 0, 808, 809, 5, 34, 0, 0, 809, 810, 5, 34, 0, 0, 810, 811, 5, 34, 0, 0, 811, 815, 1, 0, 0, 0, 812, 814, 8, 22, 0, 0, 813, 812, 1, 0, 0, 0, 814, 817, 1, 0, 0, 0, 815, 816, 1, 0, 0, 0, 815, 813, 1, 0, 0, 0, 816, 818, 1, 0, 0, 0, 817, 815, 1, 0, 0, 0, 818, 819, 5, 34, 0, 0, 819, 820, 5, 34, 0, 0, 820, 821, 5, 34, 0, 0, 821, 823, 1, 0, 0, 0, 822, 824, 5, 34, 0, 0, 823, 822, 1, 0, 0, 0, 823, 824, 1, 0, 0, 0, 824, 826, 1, 0, 0, 0, 825, 827, 5, 34, 0, 0, 826, 825, 1, 0, 0, 0, 826, 827, 1, 0, 0, 0, 827, 829, 1, 0, 0, 0, 828, 799, 1, 0, 0, 0, 828, 808, 1, 0, 0, 0, 829, 99, 1, 0, 0, 0, 830, 832, 3, 78, 30, 0, 831, 830, 1, 0, 0, 0, 832, 833, 1, 0, 0, 0, 833, 831, 1, 0, 0, 0, 833, 834, 1, 0, 0, 0, 834, 101, 1, 0, 0, 0, 835, 837, 3, 78, 30, 0, 836, 835, 1, 0, 0, 0, 837, 838, 1, 0, 0, 0, 838, 836, 1, 0, 0, 0, 838, 839, 1, 0, 0, 0, 839, 840, 1, 0, 0, 0, 840, 844, 3, 120, 51, 0, 841, 843, 3, 78, 30, 0, 842, 841, 1, 0, 0, 0, 843, 846, 1, 0, 0, 0, 844, 842, 1, 0, 0, 0, 844, 845, 1, 0, 0, 0, 845, 878, 1, 0, 0, 0, 846, 844, 1, 0, 0, 0, 847, 849, 3, 120, 51, 0, 848, 850, 3, 78, 30, 0, 849, 848, 1, 0, 0, 0, 850, 851, 1, 0, 0, 0, 851, 849, 1, 0, 0, 0, 851, 852, 1, 0, 0, 0, 852, 878, 1, 0, 0, 0, 853, 855, 3, 78, 30, 0, 854, 853, 1, 0, 0, 0, 855, 856, 1, 0, 0, 0, 856, 854, 1, 0, 0, 0, 856, 857, 1, 0, 0, 0, 857, 865, 1, 0, 0, 0, 858, 862, 3, 120, 51, 0, 859, 861, 3, 78, 30, 0, 860, 859, 1, 0, 0, 0, 861, 864, 1, 0, 0, 0, 862, 860, 1, 0, 0, 0, 862, 863, 1, 0, 0, 0, 863, 866, 1, 0, 0, 0, 864, 862, 1, 0, 0, 0, 865, 858, 1, 0, 0, 0, 865, 866, 1, 0, 0, 0, 866, 867, 1, 0, 0, 0, 867, 868, 3, 86, 34, 0, 868, 878, 1, 0, 0, 0, 869, 871, 3, 120, 51, 0, 870, 872, 3, 78, 30, 0, 871, 870, 1, 0, 0, 0, 872, 873, 1, 0, 0, 0, 873, 871, 1, 0, 0, 0, 873, 874, 1, 0, 0, 0, 874, 875, 1, 0, 0, 0, 875, 876, 3, 86, 34, 0, 876, 878, 1, 0, 0, 0, 877, 836, 1, 0, 0, 0, 877, 847, 1, 0, 0, 0, 877, 854, 1, 0, 0, 0, 877, 869, 1, 0, 0, 0, 878, 103, 1, 0, 0, 0, 879, 880, 7, 30, 0, 0, 880, 881, 7, 31, 0, 0, 881, 105, 1, 0, 0, 0, 882, 883, 7, 12, 0, 0, 883, 884, 7, 9, 0, 0, 884, 885, 7, 0, 0, 0, 885, 107, 1, 0, 0, 0, 886, 887, 7, 12, 0, 0, 887, 888, 7, 2, 0, 0, 888, 889, 7, 4, 0, 0, 889, 109, 1, 0, 0, 0, 890, 891, 5, 61, 0, 0, 891, 111, 1, 0, 0, 0, 892, 893, 5, 58, 0, 0, 893, 894, 5, 58, 0, 0, 894, 113, 1, 0, 0, 0, 895, 896, 5, 58, 0, 0, 896, 115, 1, 0, 0, 0, 897, 898, 5, 44, 0, 0, 898, 117, 1, 0, 0, 0, 899, 900, 7, 0, 0, 0, 900, 901, 7, 3, 0, 0, 901, 902, 7, 2, 0, 0, 902, 903, 7, 4, 0, 0, 903, 119, 1, 0, 0, 0, 904, 905, 5, 46, 0, 0, 905, 121, 1, 0, 0, 0, 906, 907, 7, 15, 0, 0, 907, 908, 7, 12, 0, 0, 908, 909, 7, 13, 0, 0, 909, 910, 7, 2, 0, 0, 910, 911, 7, 3, 0, 0, 911, 123, 1, 0, 0, 0, 912, 913, 7, 15, 0, 0, 913, 914, 7, 1, 0, 0, 914, 915, 7, 6, 0, 0, 915, 916, 7, 2, 0, 0, 916, 917, 7, 5, 0, 0, 917, 125, 1, 0, 0, 0, 918, 919, 7, 1, 0, 0, 919, 920, 7, 9, 0, 0, 920, 127, 1, 0, 0, 0, 921, 922, 7, 1, 0, 0, 922, 923, 7, 2, 0, 0, 923, 129, 1, 0, 0, 0, 924, 925, 7, 13, 0, 0, 925, 926, 7, 12, 0, 0, 926, 927, 7, 2, 0, 0, 927, 928, 7, 5, 0, 0, 928, 131, 1, 0, 0, 0, 929, 930, 7, 13, 0, 0, 930, 931, 7, 1, 0, 0, 931, 932, 7, 18, 0, 0, 932, 933, 7, 3, 0, 0, 933, 133, 1, 0, 0, 0, 934, 935, 5, 40, 0, 0, 935, 135, 1, 0, 0, 0, 936, 937, 7, 9, 0, 0, 937, 938, 7, 7, 0, 0, 938, 939, 7, 5, 0, 0, 939, 137, 1, 0, 0, 0, 940, 941, 7, 9, 0, 0, 941, 942, 7, 20, 0, 0, 942, 943, 7, 13, 0, 0, 943, 944, 7, 13, 0, 0, 944, 139, 1, 0, 0, 0, 945, 946, 7, 9, 0, 0, 946, 947, 7, 20, 0, 0, 947, 948, 7, 13, 0, 0, 948, 949, 7, 13, 0, 0, 949, 950, 7, 2, 0, 0, 950, 141, 1, 0, 0, 0, 951, 952, 7, 7, 0, 0, 952, 953, 7, 6, 0, 0, 953, 143, 1, 0, 0, 0, 954, 955, 5, 63, 0, 0, 955, 145, 1, 0, 0, 0, 956, 957, 7, 6, 0, 0, 957, 958, 7, 13, 0, 0, 958, 959, 7, 1, 0, 0, 959, 960, 7, 18, 0, 0, 960, 961, 7, 3, 0, 0, 961, 147, 1, 0, 0, 0, 962, 963, 5, 41, 0, 0, 963, 149, 1, 0, 0, 0, 964, 965, 7, 5, 0, 0, 965, 966, 7, 6, 0, 0, 966, 967, 7, 20, 0, 0, 967, 968, 7, 3, 0, 0, 968, 151, 1, 0, 0, 0, 969, 970, 5, 61, 0, 0, 970, 971, 5, 61, 0, 0, 971, 153, 1, 0, 0, 0, 972, 973, 5, 61, 0, 0, 973, 974, 5, 126, 0, 0, 974, 155, 1, 0, 0, 0, 975, 976, 5, 33, 0, 0, 976, 977, 5, 61, 0, 0, 977, 157, 1, 0, 0, 0, 978, 979, 5, 60, 0, 0, 979, 159, 1, 0, 0, 0, 980, 981, 5, 60, 0, 0, 981, 982, 5, 61, 0, 0, 982, 161, 1, 0, 0, 0, 983, 984, 5, 62, 0, 0, 984, 163, 1, 0, 0, 0, 985, 986, 5, 62, 0, 0, 986, 987, 5, 61, 0, 0, 987, 165, 1, 0, 0, 0, 988, 989, 5, 43, 0, 0, 989, 167, 1, 0, 0, 0, 990, 991, 5, 45, 0, 0, 991, 169, 1, 0, 0, 0, 992, 993, 5, 42, 0, 0, 993, 171, 1, 0, 0, 0, 994, 995, 5, 47, 0, 0, 995, 173, 1, 0, 0, 0, 996, 997, 5, 37, 0, 0, 997, 175, 1, 0, 0, 0, 998, 999, 5, 123, 0, 0, 999, 177, 1, 0, 0, 0, 1000, 1001, 5, 125, 0, 0, 1001, 179, 1, 0, 0, 0, 1002, 1003, 3, 48, 15, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1005, 6, 81, 14, 0, 1005, 181, 1, 0, 0, 0, 1006, 1009, 3, 144, 63, 0, 1007, 1010, 3, 80, 31, 0, 1008, 1010, 3, 94, 38, 0, 1009, 1007, 1, 0, 0, 0, 1009, 1008, 1, 0, 0, 0, 1010, 1014, 1, 0, 0, 0, 1011, 1013, 3, 96, 39, 0, 1012, 1011, 1, 0, 0, 0, 1013, 1016, 1, 0, 0, 0, 1014, 1012, 1, 0, 0, 0, 1014, 1015, 1, 0, 0, 0, 1015, 1024, 1, 0, 0, 0, 1016, 1014, 1, 0, 0, 0, 1017, 1019, 3, 144, 63, 0, 1018, 1020, 3, 78, 30, 0, 1019, 1018, 1, 0, 0, 0, 1020, 1021, 1, 0, 0, 0, 1021, 1019, 1, 0, 0, 0, 1021, 1022, 1, 0, 0, 0, 1022, 1024, 1, 0, 0, 0, 1023, 1006, 1, 0, 0, 0, 1023, 1017, 1, 0, 0, 0, 1024, 183, 1, 0, 0, 0, 1025, 1026, 5, 91, 0, 0, 1026, 1027, 1, 0, 0, 0, 1027, 1028, 6, 83, 0, 0, 1028, 1029, 6, 83, 0, 0, 1029, 185, 1, 0, 0, 0, 1030, 1031, 5, 93, 0, 0, 1031, 1032, 1, 0, 0, 0, 1032, 1033, 6, 84, 13, 0, 1033, 1034, 6, 84, 13, 0, 1034, 187, 1, 0, 0, 0, 1035, 1039, 3, 80, 31, 0, 1036, 1038, 3, 96, 39, 0, 1037, 1036, 1, 0, 0, 0, 1038, 1041, 1, 0, 0, 0, 1039, 1037, 1, 0, 0, 0, 1039, 1040, 1, 0, 0, 0, 1040, 1052, 1, 0, 0, 0, 1041, 1039, 1, 0, 0, 0, 1042, 1045, 3, 94, 38, 0, 1043, 1045, 3, 88, 35, 0, 1044, 1042, 1, 0, 0, 0, 1044, 1043, 1, 0, 0, 0, 1045, 1047, 1, 0, 0, 0, 1046, 1048, 3, 96, 39, 0, 1047, 1046, 1, 0, 0, 0, 1048, 1049, 1, 0, 0, 0, 1049, 1047, 1, 0, 0, 0, 1049, 1050, 1, 0, 0, 0, 1050, 1052, 1, 0, 0, 0, 1051, 1035, 1, 0, 0, 0, 1051, 1044, 1, 0, 0, 0, 1052, 189, 1, 0, 0, 0, 1053, 1055, 3, 90, 36, 0, 1054, 1056, 3, 92, 37, 0, 1055, 1054, 1, 0, 0, 0, 1056, 1057, 1, 0, 0, 0, 1057, 1055, 1, 0, 0, 0, 1057, 1058, 1, 0, 0, 0, 1058, 1059, 1, 0, 0, 0, 1059, 1060, 3, 90, 36, 0, 1060, 191, 1, 0, 0, 0, 1061, 1062, 3, 190, 86, 0, 1062, 193, 1, 0, 0, 0, 1063, 1064, 3, 70, 26, 0, 1064, 1065, 1, 0, 0, 0, 1065, 1066, 6, 88, 12, 0, 1066, 195, 1, 0, 0, 0, 1067, 1068, 3, 72, 27, 0, 1068, 1069, 1, 0, 0, 0, 1069, 1070, 6, 89, 12, 0, 1070, 197, 1, 0, 0, 0, 1071, 1072, 3, 74, 28, 0, 1072, 1073, 1, 0, 0, 0, 1073, 1074, 6, 90, 12, 0, 1074, 199, 1, 0, 0, 0, 1075, 1076, 3, 184, 83, 0, 1076, 1077, 1, 0, 0, 0, 1077, 1078, 6, 91, 15, 0, 1078, 1079, 6, 91, 16, 0, 1079, 201, 1, 0, 0, 0, 1080, 1081, 3, 76, 29, 0, 1081, 1082, 1, 0, 0, 0, 1082, 1083, 6, 92, 17, 0, 1083, 1084, 6, 92, 13, 0, 1084, 203, 1, 0, 0, 0, 1085, 1086, 3, 74, 28, 0, 1086, 1087, 1, 0, 0, 0, 1087, 1088, 6, 93, 12, 0, 1088, 205, 1, 0, 0, 0, 1089, 1090, 3, 70, 26, 0, 1090, 1091, 1, 0, 0, 0, 1091, 1092, 6, 94, 12, 0, 1092, 207, 1, 0, 0, 0, 1093, 1094, 3, 72, 27, 0, 1094, 1095, 1, 0, 0, 0, 1095, 1096, 6, 95, 12, 0, 1096, 209, 1, 0, 0, 0, 1097, 1098, 3, 76, 29, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 6, 96, 17, 0, 1100, 1101, 6, 96, 13, 0, 1101, 211, 1, 0, 0, 0, 1102, 1103, 3, 184, 83, 0, 1103, 1104, 1, 0, 0, 0, 1104, 1105, 6, 97, 15, 0, 1105, 213, 1, 0, 0, 0, 1106, 1107, 3, 186, 84, 0, 1107, 1108, 1, 0, 0, 0, 1108, 1109, 6, 98, 18, 0, 1109, 215, 1, 0, 0, 0, 1110, 1111, 3, 114, 48, 0, 1111, 1112, 1, 0, 0, 0, 1112, 1113, 6, 99, 19, 0, 1113, 217, 1, 0, 0, 0, 1114, 1115, 3, 116, 49, 0, 1115, 1116, 1, 0, 0, 0, 1116, 1117, 6, 100, 20, 0, 1117, 219, 1, 0, 0, 0, 1118, 1119, 3, 110, 46, 0, 1119, 1120, 1, 0, 0, 0, 1120, 1121, 6, 101, 21, 0, 1121, 221, 1, 0, 0, 0, 1122, 1123, 7, 16, 0, 0, 1123, 1124, 7, 3, 0, 0, 1124, 1125, 7, 5, 0, 0, 1125, 1126, 7, 12, 0, 0, 1126, 1127, 7, 0, 0, 0, 1127, 1128, 7, 12, 0, 0, 1128, 1129, 7, 5, 0, 0, 1129, 1130, 7, 12, 0, 0, 1130, 223, 1, 0, 0, 0, 1131, 1135, 8, 32, 0, 0, 1132, 1133, 5, 47, 0, 0, 1133, 1135, 8, 33, 0, 0, 1134, 1131, 1, 0, 0, 0, 1134, 1132, 1, 0, 0, 0, 1135, 225, 1, 0, 0, 0, 1136, 1138, 3, 224, 103, 0, 1137, 1136, 1, 0, 0, 0, 1138, 1139, 1, 0, 0, 0, 1139, 1137, 1, 0, 0, 0, 1139, 1140, 1, 0, 0, 0, 1140, 227, 1, 0, 0, 0, 1141, 1142, 3, 226, 104, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 105, 22, 0, 1144, 229, 1, 0, 0, 0, 1145, 1146, 3, 98, 40, 0, 1146, 1147, 1, 0, 0, 0, 1147, 1148, 6, 106, 23, 0, 1148, 231, 1, 0, 0, 0, 1149, 1150, 3, 70, 26, 0, 1150, 1151, 1, 0, 0, 0, 1151, 1152, 6, 107, 12, 0, 1152, 233, 1, 0, 0, 0, 1153, 1154, 3, 72, 27, 0, 1154, 1155, 1, 0, 0, 0, 1155, 1156, 6, 108, 12, 0, 1156, 235, 1, 0, 0, 0, 1157, 1158, 3, 74, 28, 0, 1158, 1159, 1, 0, 0, 0, 1159, 1160, 6, 109, 12, 0, 1160, 237, 1, 0, 0, 0, 1161, 1162, 3, 76, 29, 0, 1162, 1163, 1, 0, 0, 0, 1163, 1164, 6, 110, 17, 0, 1164, 1165, 6, 110, 13, 0, 1165, 239, 1, 0, 0, 0, 1166, 1167, 3, 120, 51, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1169, 6, 111, 24, 0, 1169, 241, 1, 0, 0, 0, 1170, 1171, 3, 116, 49, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1173, 6, 112, 20, 0, 1173, 243, 1, 0, 0, 0, 1174, 1175, 3, 144, 63, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 113, 25, 0, 1177, 245, 1, 0, 0, 0, 1178, 1179, 3, 182, 82, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 114, 26, 0, 1181, 247, 1, 0, 0, 0, 1182, 1187, 3, 80, 31, 0, 1183, 1187, 3, 78, 30, 0, 1184, 1187, 3, 94, 38, 0, 1185, 1187, 3, 170, 76, 0, 1186, 1182, 1, 0, 0, 0, 1186, 1183, 1, 0, 0, 0, 1186, 1184, 1, 0, 0, 0, 1186, 1185, 1, 0, 0, 0, 1187, 249, 1, 0, 0, 0, 1188, 1191, 3, 80, 31, 0, 1189, 1191, 3, 170, 76, 0, 1190, 1188, 1, 0, 0, 0, 1190, 1189, 1, 0, 0, 0, 1191, 1195, 1, 0, 0, 0, 1192, 1194, 3, 248, 115, 0, 1193, 1192, 1, 0, 0, 0, 1194, 1197, 1, 0, 0, 0, 1195, 1193, 1, 0, 0, 0, 1195, 1196, 1, 0, 0, 0, 1196, 1208, 1, 0, 0, 0, 1197, 1195, 1, 0, 0, 0, 1198, 1201, 3, 94, 38, 0, 1199, 1201, 3, 88, 35, 0, 1200, 1198, 1, 0, 0, 0, 1200, 1199, 1, 0, 0, 0, 1201, 1203, 1, 0, 0, 0, 1202, 1204, 3, 248, 115, 0, 1203, 1202, 1, 0, 0, 0, 1204, 1205, 1, 0, 0, 0, 1205, 1203, 1, 0, 0, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1208, 1, 0, 0, 0, 1207, 1190, 1, 0, 0, 0, 1207, 1200, 1, 0, 0, 0, 1208, 251, 1, 0, 0, 0, 1209, 1212, 3, 250, 116, 0, 1210, 1212, 3, 190, 86, 0, 1211, 1209, 1, 0, 0, 0, 1211, 1210, 1, 0, 0, 0, 1212, 1213, 1, 0, 0, 0, 1213, 1211, 1, 0, 0, 0, 1213, 1214, 1, 0, 0, 0, 1214, 253, 1, 0, 0, 0, 1215, 1216, 3, 70, 26, 0, 1216, 1217, 1, 0, 0, 0, 1217, 1218, 6, 118, 12, 0, 1218, 255, 1, 0, 0, 0, 1219, 1220, 3, 72, 27, 0, 1220, 1221, 1, 0, 0, 0, 1221, 1222, 6, 119, 12, 0, 1222, 257, 1, 0, 0, 0, 1223, 1224, 3, 74, 28, 0, 1224, 1225, 1, 0, 0, 0, 1225, 1226, 6, 120, 12, 0, 1226, 259, 1, 0, 0, 0, 1227, 1228, 3, 76, 29, 0, 1228, 1229, 1, 0, 0, 0, 1229, 1230, 6, 121, 17, 0, 1230, 1231, 6, 121, 13, 0, 1231, 261, 1, 0, 0, 0, 1232, 1233, 3, 110, 46, 0, 1233, 1234, 1, 0, 0, 0, 1234, 1235, 6, 122, 21, 0, 1235, 263, 1, 0, 0, 0, 1236, 1237, 3, 116, 49, 0, 1237, 1238, 1, 0, 0, 0, 1238, 1239, 6, 123, 20, 0, 1239, 265, 1, 0, 0, 0, 1240, 1241, 3, 120, 51, 0, 1241, 1242, 1, 0, 0, 0, 1242, 1243, 6, 124, 24, 0, 1243, 267, 1, 0, 0, 0, 1244, 1245, 3, 144, 63, 0, 1245, 1246, 1, 0, 0, 0, 1246, 1247, 6, 125, 25, 0, 1247, 269, 1, 0, 0, 0, 1248, 1249, 3, 182, 82, 0, 1249, 1250, 1, 0, 0, 0, 1250, 1251, 6, 126, 26, 0, 1251, 271, 1, 0, 0, 0, 1252, 1253, 7, 12, 0, 0, 1253, 1254, 7, 2, 0, 0, 1254, 273, 1, 0, 0, 0, 1255, 1256, 3, 252, 117, 0, 1256, 1257, 1, 0, 0, 0, 1257, 1258, 6, 128, 27, 0, 1258, 275, 1, 0, 0, 0, 1259, 1260, 3, 70, 26, 0, 1260, 1261, 1, 0, 0, 0, 1261, 1262, 6, 129, 12, 0, 1262, 277, 1, 0, 0, 0, 1263, 1264, 3, 72, 27, 0, 1264, 1265, 1, 0, 0, 0, 1265, 1266, 6, 130, 12, 0, 1266, 279, 1, 0, 0, 0, 1267, 1268, 3, 74, 28, 0, 1268, 1269, 1, 0, 0, 0, 1269, 1270, 6, 131, 12, 0, 1270, 281, 1, 0, 0, 0, 1271, 1272, 3, 76, 29, 0, 1272, 1273, 1, 0, 0, 0, 1273, 1274, 6, 132, 17, 0, 1274, 1275, 6, 132, 13, 0, 1275, 283, 1, 0, 0, 0, 1276, 1277, 3, 184, 83, 0, 1277, 1278, 1, 0, 0, 0, 1278, 1279, 6, 133, 15, 0, 1279, 1280, 6, 133, 28, 0, 1280, 285, 1, 0, 0, 0, 1281, 1282, 7, 7, 0, 0, 1282, 1283, 7, 9, 0, 0, 1283, 1284, 1, 0, 0, 0, 1284, 1285, 6, 134, 29, 0, 1285, 287, 1, 0, 0, 0, 1286, 1287, 7, 19, 0, 0, 1287, 1288, 7, 1, 0, 0, 1288, 1289, 7, 5, 0, 0, 1289, 1290, 7, 10, 0, 0, 1290, 1291, 1, 0, 0, 0, 1291, 1292, 6, 135, 29, 0, 1292, 289, 1, 0, 0, 0, 1293, 1294, 8, 34, 0, 0, 1294, 291, 1, 0, 0, 0, 1295, 1297, 3, 290, 136, 0, 1296, 1295, 1, 0, 0, 0, 1297, 1298, 1, 0, 0, 0, 1298, 1296, 1, 0, 0, 0, 1298, 1299, 1, 0, 0, 0, 1299, 1300, 1, 0, 0, 0, 1300, 1301, 3, 114, 48, 0, 1301, 1303, 1, 0, 0, 0, 1302, 1296, 1, 0, 0, 0, 1302, 1303, 1, 0, 0, 0, 1303, 1305, 1, 0, 0, 0, 1304, 1306, 3, 290, 136, 0, 1305, 1304, 1, 0, 0, 0, 1306, 1307, 1, 0, 0, 0, 1307, 1305, 1, 0, 0, 0, 1307, 1308, 1, 0, 0, 0, 1308, 293, 1, 0, 0, 0, 1309, 1310, 3, 292, 137, 0, 1310, 1311, 1, 0, 0, 0, 1311, 1312, 6, 138, 30, 0, 1312, 295, 1, 0, 0, 0, 1313, 1314, 3, 70, 26, 0, 1314, 1315, 1, 0, 0, 0, 1315, 1316, 6, 139, 12, 0, 1316, 297, 1, 0, 0, 0, 1317, 1318, 3, 72, 27, 0, 1318, 1319, 1, 0, 0, 0, 1319, 1320, 6, 140, 12, 0, 1320, 299, 1, 0, 0, 0, 1321, 1322, 3, 74, 28, 0, 1322, 1323, 1, 0, 0, 0, 1323, 1324, 6, 141, 12, 0, 1324, 301, 1, 0, 0, 0, 1325, 1326, 3, 76, 29, 0, 1326, 1327, 1, 0, 0, 0, 1327, 1328, 6, 142, 17, 0, 1328, 1329, 6, 142, 13, 0, 1329, 1330, 6, 142, 13, 0, 1330, 303, 1, 0, 0, 0, 1331, 1332, 3, 110, 46, 0, 1332, 1333, 1, 0, 0, 0, 1333, 1334, 6, 143, 21, 0, 1334, 305, 1, 0, 0, 0, 1335, 1336, 3, 116, 49, 0, 1336, 1337, 1, 0, 0, 0, 1337, 1338, 6, 144, 20, 0, 1338, 307, 1, 0, 0, 0, 1339, 1340, 3, 120, 51, 0, 1340, 1341, 1, 0, 0, 0, 1341, 1342, 6, 145, 24, 0, 1342, 309, 1, 0, 0, 0, 1343, 1344, 3, 288, 135, 0, 1344, 1345, 1, 0, 0, 0, 1345, 1346, 6, 146, 31, 0, 1346, 311, 1, 0, 0, 0, 1347, 1348, 3, 252, 117, 0, 1348, 1349, 1, 0, 0, 0, 1349, 1350, 6, 147, 27, 0, 1350, 313, 1, 0, 0, 0, 1351, 1352, 3, 192, 87, 0, 1352, 1353, 1, 0, 0, 0, 1353, 1354, 6, 148, 32, 0, 1354, 315, 1, 0, 0, 0, 1355, 1356, 3, 144, 63, 0, 1356, 1357, 1, 0, 0, 0, 1357, 1358, 6, 149, 25, 0, 1358, 317, 1, 0, 0, 0, 1359, 1360, 3, 182, 82, 0, 1360, 1361, 1, 0, 0, 0, 1361, 1362, 6, 150, 26, 0, 1362, 319, 1, 0, 0, 0, 1363, 1364, 3, 70, 26, 0, 1364, 1365, 1, 0, 0, 0, 1365, 1366, 6, 151, 12, 0, 1366, 321, 1, 0, 0, 0, 1367, 1368, 3, 72, 27, 0, 1368, 1369, 1, 0, 0, 0, 1369, 1370, 6, 152, 12, 0, 1370, 323, 1, 0, 0, 0, 1371, 1372, 3, 74, 28, 0, 1372, 1373, 1, 0, 0, 0, 1373, 1374, 6, 153, 12, 0, 1374, 325, 1, 0, 0, 0, 1375, 1376, 3, 76, 29, 0, 1376, 1377, 1, 0, 0, 0, 1377, 1378, 6, 154, 17, 0, 1378, 1379, 6, 154, 13, 0, 1379, 327, 1, 0, 0, 0, 1380, 1381, 3, 120, 51, 0, 1381, 1382, 1, 0, 0, 0, 1382, 1383, 6, 155, 24, 0, 1383, 329, 1, 0, 0, 0, 1384, 1385, 3, 144, 63, 0, 1385, 1386, 1, 0, 0, 0, 1386, 1387, 6, 156, 25, 0, 1387, 331, 1, 0, 0, 0, 1388, 1389, 3, 182, 82, 0, 1389, 1390, 1, 0, 0, 0, 1390, 1391, 6, 157, 26, 0, 1391, 333, 1, 0, 0, 0, 1392, 1393, 3, 192, 87, 0, 1393, 1394, 1, 0, 0, 0, 1394, 1395, 6, 158, 32, 0, 1395, 335, 1, 0, 0, 0, 1396, 1397, 3, 188, 85, 0, 1397, 1398, 1, 0, 0, 0, 1398, 1399, 6, 159, 33, 0, 1399, 337, 1, 0, 0, 0, 1400, 1401, 3, 70, 26, 0, 1401, 1402, 1, 0, 0, 0, 1402, 1403, 6, 160, 12, 0, 1403, 339, 1, 0, 0, 0, 1404, 1405, 3, 72, 27, 0, 1405, 1406, 1, 0, 0, 0, 1406, 1407, 6, 161, 12, 0, 1407, 341, 1, 0, 0, 0, 1408, 1409, 3, 74, 28, 0, 1409, 1410, 1, 0, 0, 0, 1410, 1411, 6, 162, 12, 0, 1411, 343, 1, 0, 0, 0, 1412, 1413, 3, 76, 29, 0, 1413, 1414, 1, 0, 0, 0, 1414, 1415, 6, 163, 17, 0, 1415, 1416, 6, 163, 13, 0, 1416, 345, 1, 0, 0, 0, 1417, 1418, 7, 1, 0, 0, 1418, 1419, 7, 9, 0, 0, 1419, 1420, 7, 15, 0, 0, 1420, 1421, 7, 7, 0, 0, 1421, 347, 1, 0, 0, 0, 1422, 1423, 3, 70, 26, 0, 1423, 1424, 1, 0, 0, 0, 1424, 1425, 6, 165, 12, 0, 1425, 349, 1, 0, 0, 0, 1426, 1427, 3, 72, 27, 0, 1427, 1428, 1, 0, 0, 0, 1428, 1429, 6, 166, 12, 0, 1429, 351, 1, 0, 0, 0, 1430, 1431, 3, 74, 28, 0, 1431, 1432, 1, 0, 0, 0, 1432, 1433, 6, 167, 12, 0, 1433, 353, 1, 0, 0, 0, 1434, 1435, 3, 186, 84, 0, 1435, 1436, 1, 0, 0, 0, 1436, 1437, 6, 168, 18, 0, 1437, 1438, 6, 168, 13, 0, 1438, 355, 1, 0, 0, 0, 1439, 1440, 3, 114, 48, 0, 1440, 1441, 1, 0, 0, 0, 1441, 1442, 6, 169, 19, 0, 1442, 357, 1, 0, 0, 0, 1443, 1449, 3, 88, 35, 0, 1444, 1449, 3, 78, 30, 0, 1445, 1449, 3, 120, 51, 0, 1446, 1449, 3, 80, 31, 0, 1447, 1449, 3, 94, 38, 0, 1448, 1443, 1, 0, 0, 0, 1448, 1444, 1, 0, 0, 0, 1448, 1445, 1, 0, 0, 0, 1448, 1446, 1, 0, 0, 0, 1448, 1447, 1, 0, 0, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1448, 1, 0, 0, 0, 1450, 1451, 1, 0, 0, 0, 1451, 359, 1, 0, 0, 0, 1452, 1453, 3, 70, 26, 0, 1453, 1454, 1, 0, 0, 0, 1454, 1455, 6, 171, 12, 0, 1455, 361, 1, 0, 0, 0, 1456, 1457, 3, 72, 27, 0, 1457, 1458, 1, 0, 0, 0, 1458, 1459, 6, 172, 12, 0, 1459, 363, 1, 0, 0, 0, 1460, 1461, 3, 74, 28, 0, 1461, 1462, 1, 0, 0, 0, 1462, 1463, 6, 173, 12, 0, 1463, 365, 1, 0, 0, 0, 1464, 1465, 3, 76, 29, 0, 1465, 1466, 1, 0, 0, 0, 1466, 1467, 6, 174, 17, 0, 1467, 1468, 6, 174, 13, 0, 1468, 367, 1, 0, 0, 0, 1469, 1470, 3, 114, 48, 0, 1470, 1471, 1, 0, 0, 0, 1471, 1472, 6, 175, 19, 0, 1472, 369, 1, 0, 0, 0, 1473, 1474, 3, 116, 49, 0, 1474, 1475, 1, 0, 0, 0, 1475, 1476, 6, 176, 20, 0, 1476, 371, 1, 0, 0, 0, 1477, 1478, 3, 120, 51, 0, 1478, 1479, 1, 0, 0, 0, 1479, 1480, 6, 177, 24, 0, 1480, 373, 1, 0, 0, 0, 1481, 1482, 3, 286, 134, 0, 1482, 1483, 1, 0, 0, 0, 1483, 1484, 6, 178, 34, 0, 1484, 1485, 6, 178, 35, 0, 1485, 375, 1, 0, 0, 0, 1486, 1487, 3, 226, 104, 0, 1487, 1488, 1, 0, 0, 0, 1488, 1489, 6, 179, 22, 0, 1489, 377, 1, 0, 0, 0, 1490, 1491, 3, 98, 40, 0, 1491, 1492, 1, 0, 0, 0, 1492, 1493, 6, 180, 23, 0, 1493, 379, 1, 0, 0, 0, 1494, 1495, 3, 70, 26, 0, 1495, 1496, 1, 0, 0, 0, 1496, 1497, 6, 181, 12, 0, 1497, 381, 1, 0, 0, 0, 1498, 1499, 3, 72, 27, 0, 1499, 1500, 1, 0, 0, 0, 1500, 1501, 6, 182, 12, 0, 1501, 383, 1, 0, 0, 0, 1502, 1503, 3, 74, 28, 0, 1503, 1504, 1, 0, 0, 0, 1504, 1505, 6, 183, 12, 0, 1505, 385, 1, 0, 0, 0, 1506, 1507, 3, 76, 29, 0, 1507, 1508, 1, 0, 0, 0, 1508, 1509, 6, 184, 17, 0, 1509, 1510, 6, 184, 13, 0, 1510, 1511, 6, 184, 13, 0, 1511, 387, 1, 0, 0, 0, 1512, 1513, 3, 116, 49, 0, 1513, 1514, 1, 0, 0, 0, 1514, 1515, 6, 185, 20, 0, 1515, 389, 1, 0, 0, 0, 1516, 1517, 3, 120, 51, 0, 1517, 1518, 1, 0, 0, 0, 1518, 1519, 6, 186, 24, 0, 1519, 391, 1, 0, 0, 0, 1520, 1521, 3, 252, 117, 0, 1521, 1522, 1, 0, 0, 0, 1522, 1523, 6, 187, 27, 0, 1523, 393, 1, 0, 0, 0, 1524, 1525, 3, 70, 26, 0, 1525, 1526, 1, 0, 0, 0, 1526, 1527, 6, 188, 12, 0, 1527, 395, 1, 0, 0, 0, 1528, 1529, 3, 72, 27, 0, 1529, 1530, 1, 0, 0, 0, 1530, 1531, 6, 189, 12, 0, 1531, 397, 1, 0, 0, 0, 1532, 1533, 3, 74, 28, 0, 1533, 1534, 1, 0, 0, 0, 1534, 1535, 6, 190, 12, 0, 1535, 399, 1, 0, 0, 0, 1536, 1537, 3, 76, 29, 0, 1537, 1538, 1, 0, 0, 0, 1538, 1539, 6, 191, 17, 0, 1539, 1540, 6, 191, 13, 0, 1540, 401, 1, 0, 0, 0, 1541, 1542, 7, 35, 0, 0, 1542, 1543, 7, 7, 0, 0, 1543, 1544, 7, 1, 0, 0, 1544, 1545, 7, 9, 0, 0, 1545, 403, 1, 0, 0, 0, 1546, 1547, 3, 272, 127, 0, 1547, 1548, 1, 0, 0, 0, 1548, 1549, 6, 193, 36, 0, 1549, 405, 1, 0, 0, 0, 1550, 1551, 3, 286, 134, 0, 1551, 1552, 1, 0, 0, 0, 1552, 1553, 6, 194, 34, 0, 1553, 1554, 6, 194, 13, 0, 1554, 1555, 6, 194, 0, 0, 1555, 407, 1, 0, 0, 0, 1556, 1557, 7, 20, 0, 0, 1557, 1558, 7, 2, 0, 0, 1558, 1559, 7, 1, 0, 0, 1559, 1560, 7, 9, 0, 0, 1560, 1561, 7, 17, 0, 0, 1561, 1562, 1, 0, 0, 0, 1562, 1563, 6, 195, 13, 0, 1563, 1564, 6, 195, 0, 0, 1564, 409, 1, 0, 0, 0, 1565, 1566, 3, 226, 104, 0, 1566, 1567, 1, 0, 0, 0, 1567, 1568, 6, 196, 22, 0, 1568, 411, 1, 0, 0, 0, 1569, 1570, 3, 98, 40, 0, 1570, 1571, 1, 0, 0, 0, 1571, 1572, 6, 197, 23, 0, 1572, 413, 1, 0, 0, 0, 1573, 1574, 3, 114, 48, 0, 1574, 1575, 1, 0, 0, 0, 1575, 1576, 6, 198, 19, 0, 1576, 415, 1, 0, 0, 0, 1577, 1578, 3, 188, 85, 0, 1578, 1579, 1, 0, 0, 0, 1579, 1580, 6, 199, 33, 0, 1580, 417, 1, 0, 0, 0, 1581, 1582, 3, 192, 87, 0, 1582, 1583, 1, 0, 0, 0, 1583, 1584, 6, 200, 32, 0, 1584, 419, 1, 0, 0, 0, 1585, 1586, 3, 70, 26, 0, 1586, 1587, 1, 0, 0, 0, 1587, 1588, 6, 201, 12, 0, 1588, 421, 1, 0, 0, 0, 1589, 1590, 3, 72, 27, 0, 1590, 1591, 1, 0, 0, 0, 1591, 1592, 6, 202, 12, 0, 1592, 423, 1, 0, 0, 0, 1593, 1594, 3, 74, 28, 0, 1594, 1595, 1, 0, 0, 0, 1595, 1596, 6, 203, 12, 0, 1596, 425, 1, 0, 0, 0, 1597, 1598, 3, 76, 29, 0, 1598, 1599, 1, 0, 0, 0, 1599, 1600, 6, 204, 17, 0, 1600, 1601, 6, 204, 13, 0, 1601, 427, 1, 0, 0, 0, 1602, 1603, 3, 226, 104, 0, 1603, 1604, 1, 0, 0, 0, 1604, 1605, 6, 205, 22, 0, 1605, 1606, 6, 205, 13, 0, 1606, 1607, 6, 205, 37, 0, 1607, 429, 1, 0, 0, 0, 1608, 1609, 3, 98, 40, 0, 1609, 1610, 1, 0, 0, 0, 1610, 1611, 6, 206, 23, 0, 1611, 1612, 6, 206, 13, 0, 1612, 1613, 6, 206, 37, 0, 1613, 431, 1, 0, 0, 0, 1614, 1615, 3, 70, 26, 0, 1615, 1616, 1, 0, 0, 0, 1616, 1617, 6, 207, 12, 0, 1617, 433, 1, 0, 0, 0, 1618, 1619, 3, 72, 27, 0, 1619, 1620, 1, 0, 0, 0, 1620, 1621, 6, 208, 12, 0, 1621, 435, 1, 0, 0, 0, 1622, 1623, 3, 74, 28, 0, 1623, 1624, 1, 0, 0, 0, 1624, 1625, 6, 209, 12, 0, 1625, 437, 1, 0, 0, 0, 1626, 1627, 3, 114, 48, 0, 1627, 1628, 1, 0, 0, 0, 1628, 1629, 6, 210, 19, 0, 1629, 1630, 6, 210, 13, 0, 1630, 1631, 6, 210, 11, 0, 1631, 439, 1, 0, 0, 0, 1632, 1633, 3, 116, 49, 0, 1633, 1634, 1, 0, 0, 0, 1634, 1635, 6, 211, 20, 0, 1635, 1636, 6, 211, 13, 0, 1636, 1637, 6, 211, 11, 0, 1637, 441, 1, 0, 0, 0, 1638, 1639, 3, 70, 26, 0, 1639, 1640, 1, 0, 0, 0, 1640, 1641, 6, 212, 12, 0, 1641, 443, 1, 0, 0, 0, 1642, 1643, 3, 72, 27, 0, 1643, 1644, 1, 0, 0, 0, 1644, 1645, 6, 213, 12, 0, 1645, 445, 1, 0, 0, 0, 1646, 1647, 3, 74, 28, 0, 1647, 1648, 1, 0, 0, 0, 1648, 1649, 6, 214, 12, 0, 1649, 447, 1, 0, 0, 0, 1650, 1651, 3, 192, 87, 0, 1651, 1652, 1, 0, 0, 0, 1652, 1653, 6, 215, 13, 0, 1653, 1654, 6, 215, 0, 0, 1654, 1655, 6, 215, 32, 0, 1655, 449, 1, 0, 0, 0, 1656, 1657, 3, 188, 85, 0, 1657, 1658, 1, 0, 0, 0, 1658, 1659, 6, 216, 13, 0, 1659, 1660, 6, 216, 0, 0, 1660, 1661, 6, 216, 33, 0, 1661, 451, 1, 0, 0, 0, 1662, 1663, 3, 104, 43, 0, 1663, 1664, 1, 0, 0, 0, 1664, 1665, 6, 217, 13, 0, 1665, 1666, 6, 217, 0, 0, 1666, 1667, 6, 217, 38, 0, 1667, 453, 1, 0, 0, 0, 1668, 1669, 3, 76, 29, 0, 1669, 1670, 1, 0, 0, 0, 1670, 1671, 6, 218, 17, 0, 1671, 1672, 6, 218, 13, 0, 1672, 455, 1, 0, 0, 0, 1673, 1674, 3, 76, 29, 0, 1674, 1675, 1, 0, 0, 0, 1675, 1676, 6, 219, 17, 0, 1676, 1677, 6, 219, 13, 0, 1677, 457, 1, 0, 0, 0, 1678, 1679, 3, 286, 134, 0, 1679, 1680, 1, 0, 0, 0, 1680, 1681, 6, 220, 34, 0, 1681, 459, 1, 0, 0, 0, 1682, 1683, 3, 272, 127, 0, 1683, 1684, 1, 0, 0, 0, 1684, 1685, 6, 221, 36, 0, 1685, 461, 1, 0, 0, 0, 1686, 1687, 3, 120, 51, 0, 1687, 1688, 1, 0, 0, 0, 1688, 1689, 6, 222, 24, 0, 1689, 463, 1, 0, 0, 0, 1690, 1691, 3, 116, 49, 0, 1691, 1692, 1, 0, 0, 0, 1692, 1693, 6, 223, 20, 0, 1693, 465, 1, 0, 0, 0, 1694, 1695, 3, 192, 87, 0, 1695, 1696, 1, 0, 0, 0, 1696, 1697, 6, 224, 32, 0, 1697, 467, 1, 0, 0, 0, 1698, 1699, 3, 188, 85, 0, 1699, 1700, 1, 0, 0, 0, 1700, 1701, 6, 225, 33, 0, 1701, 469, 1, 0, 0, 0, 1702, 1703, 3, 70, 26, 0, 1703, 1704, 1, 0, 0, 0, 1704, 1705, 6, 226, 12, 0, 1705, 471, 1, 0, 0, 0, 1706, 1707, 3, 72, 27, 0, 1707, 1708, 1, 0, 0, 0, 1708, 1709, 6, 227, 12, 0, 1709, 473, 1, 0, 0, 0, 1710, 1711, 3, 74, 28, 0, 1711, 1712, 1, 0, 0, 0, 1712, 1713, 6, 228, 12, 0, 1713, 475, 1, 0, 0, 0, 1714, 1715, 3, 76, 29, 0, 1715, 1716, 1, 0, 0, 0, 1716, 1717, 6, 229, 17, 0, 1717, 1718, 6, 229, 13, 0, 1718, 477, 1, 0, 0, 0, 1719, 1720, 3, 188, 85, 0, 1720, 1721, 1, 0, 0, 0, 1721, 1722, 6, 230, 33, 0, 1722, 479, 1, 0, 0, 0, 1723, 1724, 3, 74, 28, 0, 1724, 1725, 1, 0, 0, 0, 1725, 1726, 6, 231, 12, 0, 1726, 481, 1, 0, 0, 0, 1727, 1728, 3, 70, 26, 0, 1728, 1729, 1, 0, 0, 0, 1729, 1730, 6, 232, 12, 0, 1730, 483, 1, 0, 0, 0, 1731, 1732, 3, 72, 27, 0, 1732, 1733, 1, 0, 0, 0, 1733, 1734, 6, 233, 12, 0, 1734, 485, 1, 0, 0, 0, 68, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 718, 728, 732, 735, 744, 746, 757, 776, 781, 790, 797, 802, 804, 815, 823, 826, 828, 833, 838, 844, 851, 856, 862, 865, 873, 877, 1009, 1014, 1021, 1023, 1039, 1044, 1049, 1051, 1057, 1134, 1139, 1186, 1190, 1195, 1200, 1205, 1207, 1211, 1213, 1298, 1302, 1307, 1448, 1450, 39, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 13, 0, 5, 16, 0, 5, 11, 0, 5, 14, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 73, 0, 5, 0, 0, 7, 30, 0, 7, 74, 0, 7, 39, 0, 7, 40, 0, 7, 37, 0, 7, 84, 0, 7, 31, 0, 7, 42, 0, 7, 54, 0, 7, 72, 0, 7, 88, 0, 5, 10, 0, 5, 7, 0, 7, 98, 0, 7, 97, 0, 7, 76, 0, 7, 75, 0, 7, 96, 0, 5, 12, 0, 7, 92, 0, 5, 15, 0, 7, 34, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index e4f8699993da..4401ea8a8d06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -27,37 +27,38 @@ public class EsqlBaseLexer extends LexerConfig { public static final int DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, - WHERE=16, JOIN_LOOKUP=17, DEV_CHANGE_POINT=18, DEV_INLINESTATS=19, DEV_LOOKUP=20, - DEV_METRICS=21, DEV_JOIN_FULL=22, DEV_JOIN_LEFT=23, DEV_JOIN_RIGHT=24, - UNKNOWN_CMD=25, LINE_COMMENT=26, MULTILINE_COMMENT=27, WS=28, PIPE=29, - QUOTED_STRING=30, INTEGER_LITERAL=31, DECIMAL_LITERAL=32, BY=33, AND=34, - ASC=35, ASSIGN=36, CAST_OP=37, COLON=38, COMMA=39, DESC=40, DOT=41, FALSE=42, - FIRST=43, IN=44, IS=45, LAST=46, LIKE=47, LP=48, NOT=49, NULL=50, NULLS=51, - OR=52, PARAM=53, RLIKE=54, RP=55, TRUE=56, EQ=57, CIEQ=58, NEQ=59, LT=60, - LTE=61, GT=62, GTE=63, PLUS=64, MINUS=65, ASTERISK=66, SLASH=67, PERCENT=68, - LEFT_BRACES=69, RIGHT_BRACES=70, NAMED_OR_POSITIONAL_PARAM=71, OPENING_BRACKET=72, - CLOSING_BRACKET=73, UNQUOTED_IDENTIFIER=74, QUOTED_IDENTIFIER=75, EXPR_LINE_COMMENT=76, - EXPR_MULTILINE_COMMENT=77, EXPR_WS=78, EXPLAIN_WS=79, EXPLAIN_LINE_COMMENT=80, - EXPLAIN_MULTILINE_COMMENT=81, METADATA=82, UNQUOTED_SOURCE=83, FROM_LINE_COMMENT=84, - FROM_MULTILINE_COMMENT=85, FROM_WS=86, ID_PATTERN=87, PROJECT_LINE_COMMENT=88, - PROJECT_MULTILINE_COMMENT=89, PROJECT_WS=90, AS=91, RENAME_LINE_COMMENT=92, - RENAME_MULTILINE_COMMENT=93, RENAME_WS=94, ON=95, WITH=96, ENRICH_POLICY_NAME=97, - ENRICH_LINE_COMMENT=98, ENRICH_MULTILINE_COMMENT=99, ENRICH_WS=100, ENRICH_FIELD_LINE_COMMENT=101, - ENRICH_FIELD_MULTILINE_COMMENT=102, ENRICH_FIELD_WS=103, MVEXPAND_LINE_COMMENT=104, - MVEXPAND_MULTILINE_COMMENT=105, MVEXPAND_WS=106, INFO=107, SHOW_LINE_COMMENT=108, - SHOW_MULTILINE_COMMENT=109, SHOW_WS=110, SETTING=111, SETTING_LINE_COMMENT=112, - SETTTING_MULTILINE_COMMENT=113, SETTING_WS=114, LOOKUP_LINE_COMMENT=115, - LOOKUP_MULTILINE_COMMENT=116, LOOKUP_WS=117, LOOKUP_FIELD_LINE_COMMENT=118, - LOOKUP_FIELD_MULTILINE_COMMENT=119, LOOKUP_FIELD_WS=120, JOIN=121, USING=122, - JOIN_LINE_COMMENT=123, JOIN_MULTILINE_COMMENT=124, JOIN_WS=125, METRICS_LINE_COMMENT=126, - METRICS_MULTILINE_COMMENT=127, METRICS_WS=128, CLOSING_METRICS_LINE_COMMENT=129, - CLOSING_METRICS_MULTILINE_COMMENT=130, CLOSING_METRICS_WS=131, CHANGE_POINT_LINE_COMMENT=132, - CHANGE_POINT_MULTILINE_COMMENT=133, CHANGE_POINT_WS=134; + WHERE=16, JOIN_LOOKUP=17, DEV_CHANGE_POINT=18, DEV_INLINESTATS=19, DEV_INSIST=20, + DEV_LOOKUP=21, DEV_METRICS=22, DEV_JOIN_FULL=23, DEV_JOIN_LEFT=24, DEV_JOIN_RIGHT=25, + UNKNOWN_CMD=26, LINE_COMMENT=27, MULTILINE_COMMENT=28, WS=29, PIPE=30, + QUOTED_STRING=31, INTEGER_LITERAL=32, DECIMAL_LITERAL=33, BY=34, AND=35, + ASC=36, ASSIGN=37, CAST_OP=38, COLON=39, COMMA=40, DESC=41, DOT=42, FALSE=43, + FIRST=44, IN=45, IS=46, LAST=47, LIKE=48, LP=49, NOT=50, NULL=51, NULLS=52, + OR=53, PARAM=54, RLIKE=55, RP=56, TRUE=57, EQ=58, CIEQ=59, NEQ=60, LT=61, + LTE=62, GT=63, GTE=64, PLUS=65, MINUS=66, ASTERISK=67, SLASH=68, PERCENT=69, + LEFT_BRACES=70, RIGHT_BRACES=71, NAMED_OR_POSITIONAL_PARAM=72, OPENING_BRACKET=73, + CLOSING_BRACKET=74, UNQUOTED_IDENTIFIER=75, QUOTED_IDENTIFIER=76, EXPR_LINE_COMMENT=77, + EXPR_MULTILINE_COMMENT=78, EXPR_WS=79, EXPLAIN_WS=80, EXPLAIN_LINE_COMMENT=81, + EXPLAIN_MULTILINE_COMMENT=82, METADATA=83, UNQUOTED_SOURCE=84, FROM_LINE_COMMENT=85, + FROM_MULTILINE_COMMENT=86, FROM_WS=87, ID_PATTERN=88, PROJECT_LINE_COMMENT=89, + PROJECT_MULTILINE_COMMENT=90, PROJECT_WS=91, AS=92, RENAME_LINE_COMMENT=93, + RENAME_MULTILINE_COMMENT=94, RENAME_WS=95, ON=96, WITH=97, ENRICH_POLICY_NAME=98, + ENRICH_LINE_COMMENT=99, ENRICH_MULTILINE_COMMENT=100, ENRICH_WS=101, ENRICH_FIELD_LINE_COMMENT=102, + ENRICH_FIELD_MULTILINE_COMMENT=103, ENRICH_FIELD_WS=104, MVEXPAND_LINE_COMMENT=105, + MVEXPAND_MULTILINE_COMMENT=106, MVEXPAND_WS=107, INFO=108, SHOW_LINE_COMMENT=109, + SHOW_MULTILINE_COMMENT=110, SHOW_WS=111, SETTING=112, SETTING_LINE_COMMENT=113, + SETTTING_MULTILINE_COMMENT=114, SETTING_WS=115, LOOKUP_LINE_COMMENT=116, + LOOKUP_MULTILINE_COMMENT=117, LOOKUP_WS=118, LOOKUP_FIELD_LINE_COMMENT=119, + LOOKUP_FIELD_MULTILINE_COMMENT=120, LOOKUP_FIELD_WS=121, JOIN=122, USING=123, + JOIN_LINE_COMMENT=124, JOIN_MULTILINE_COMMENT=125, JOIN_WS=126, METRICS_LINE_COMMENT=127, + METRICS_MULTILINE_COMMENT=128, METRICS_WS=129, CLOSING_METRICS_LINE_COMMENT=130, + CLOSING_METRICS_MULTILINE_COMMENT=131, CLOSING_METRICS_WS=132, CHANGE_POINT_LINE_COMMENT=133, + CHANGE_POINT_MULTILINE_COMMENT=134, CHANGE_POINT_WS=135, INSIST_WS=136, + INSIST_LINE_COMMENT=137, INSIST_MULTILINE_COMMENT=138; public static final int EXPRESSION_MODE=1, EXPLAIN_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9, SETTING_MODE=10, LOOKUP_MODE=11, LOOKUP_FIELD_MODE=12, JOIN_MODE=13, METRICS_MODE=14, CLOSING_METRICS_MODE=15, - CHANGE_POINT_MODE=16; + CHANGE_POINT_MODE=16, INSIST_MODE=17; public static String[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" }; @@ -66,22 +67,22 @@ public class EsqlBaseLexer extends LexerConfig { "DEFAULT_MODE", "EXPRESSION_MODE", "EXPLAIN_MODE", "FROM_MODE", "PROJECT_MODE", "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "MVEXPAND_MODE", "SHOW_MODE", "SETTING_MODE", "LOOKUP_MODE", "LOOKUP_FIELD_MODE", "JOIN_MODE", "METRICS_MODE", - "CLOSING_METRICS_MODE", "CHANGE_POINT_MODE" + "CLOSING_METRICS_MODE", "CHANGE_POINT_MODE", "INSIST_MODE" }; private static String[] makeRuleNames() { return new String[] { "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", - "JOIN_LOOKUP", "DEV_CHANGE_POINT", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", - "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "UNKNOWN_CMD", "LINE_COMMENT", - "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", - "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", - "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", "INTEGER_LITERAL", - "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "JOIN_LOOKUP", "DEV_CHANGE_POINT", "DEV_INLINESTATS", "DEV_INSIST", "DEV_LOOKUP", + "DEV_METRICS", "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "UNKNOWN_CMD", + "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", + "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", + "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", + "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", + "COLON", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", + "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", + "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", "NESTED_WHERE", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", @@ -120,7 +121,9 @@ public class EsqlBaseLexer extends LexerConfig { "CLOSING_METRICS_UNQUOTED_IDENTIFIER", "CLOSING_METRICS_BY", "CLOSING_METRICS_PIPE", "CHANGE_POINT_PIPE", "CHANGE_POINT_ON", "CHANGE_POINT_AS", "CHANGE_POINT_DOT", "CHANGE_POINT_COMMA", "CHANGE_POINT_QUOTED_IDENTIFIER", "CHANGE_POINT_UNQUOTED_IDENTIFIER", - "CHANGE_POINT_LINE_COMMENT", "CHANGE_POINT_MULTILINE_COMMENT", "CHANGE_POINT_WS" + "CHANGE_POINT_LINE_COMMENT", "CHANGE_POINT_MULTILINE_COMMENT", "CHANGE_POINT_WS", + "INSIST_PIPE", "INSIST_IDENTIFIER", "INSIST_WS", "INSIST_LINE_COMMENT", + "INSIST_MULTILINE_COMMENT" }; } public static final String[] ruleNames = makeRuleNames(); @@ -130,7 +133,7 @@ public class EsqlBaseLexer extends LexerConfig { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", "'lookup'", null, null, null, null, null, - null, null, null, null, null, null, "'|'", null, null, null, "'by'", + null, null, null, null, null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "':'", "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", "'=~'", @@ -147,13 +150,13 @@ public class EsqlBaseLexer extends LexerConfig { return new String[] { null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "JOIN_LOOKUP", "DEV_CHANGE_POINT", "DEV_INLINESTATS", "DEV_LOOKUP", - "DEV_METRICS", "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "UNKNOWN_CMD", - "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", - "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "WHERE", "JOIN_LOOKUP", "DEV_CHANGE_POINT", "DEV_INLINESTATS", "DEV_INSIST", + "DEV_LOOKUP", "DEV_METRICS", "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", + "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", + "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", + "COLON", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", + "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", + "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", @@ -171,7 +174,8 @@ public class EsqlBaseLexer extends LexerConfig { "JOIN", "USING", "JOIN_LINE_COMMENT", "JOIN_MULTILINE_COMMENT", "JOIN_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", "CLOSING_METRICS_WS", "CHANGE_POINT_LINE_COMMENT", - "CHANGE_POINT_MULTILINE_COMMENT", "CHANGE_POINT_WS" + "CHANGE_POINT_MULTILINE_COMMENT", "CHANGE_POINT_WS", "INSIST_WS", "INSIST_LINE_COMMENT", + "INSIST_MULTILINE_COMMENT" }; } private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); @@ -241,14 +245,16 @@ public class EsqlBaseLexer extends LexerConfig { case 18: return DEV_INLINESTATS_sempred((RuleContext)_localctx, predIndex); case 19: - return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex); + return DEV_INSIST_sempred((RuleContext)_localctx, predIndex); case 20: - return DEV_METRICS_sempred((RuleContext)_localctx, predIndex); + return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex); case 21: - return DEV_JOIN_FULL_sempred((RuleContext)_localctx, predIndex); + return DEV_METRICS_sempred((RuleContext)_localctx, predIndex); case 22: - return DEV_JOIN_LEFT_sempred((RuleContext)_localctx, predIndex); + return DEV_JOIN_FULL_sempred((RuleContext)_localctx, predIndex); case 23: + return DEV_JOIN_LEFT_sempred((RuleContext)_localctx, predIndex); + case 24: return DEV_JOIN_RIGHT_sempred((RuleContext)_localctx, predIndex); } return true; @@ -267,1143 +273,1180 @@ public class EsqlBaseLexer extends LexerConfig { } return true; } - private boolean DEV_LOOKUP_sempred(RuleContext _localctx, int predIndex) { + private boolean DEV_INSIST_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 2: return this.isDevVersion(); } return true; } - private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { + private boolean DEV_LOOKUP_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 3: return this.isDevVersion(); } return true; } - private boolean DEV_JOIN_FULL_sempred(RuleContext _localctx, int predIndex) { + private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 4: return this.isDevVersion(); } return true; } - private boolean DEV_JOIN_LEFT_sempred(RuleContext _localctx, int predIndex) { + private boolean DEV_JOIN_FULL_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 5: return this.isDevVersion(); } return true; } - private boolean DEV_JOIN_RIGHT_sempred(RuleContext _localctx, int predIndex) { + private boolean DEV_JOIN_LEFT_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 6: return this.isDevVersion(); } return true; } + private boolean DEV_JOIN_RIGHT_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 7: + return this.isDevVersion(); + } + return true; + } public static final String _serializedATN = - "\u0004\u0000\u0086\u0699\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ + "\u0004\u0000\u008a\u06c7\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ - "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002"+ - "\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002"+ - "\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002"+ - "\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002"+ - "\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e"+ - "\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011"+ - "\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014"+ - "\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017"+ - "\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a"+ - "\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d"+ - "\u0002\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!"+ - "\u0007!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002"+ - "&\u0007&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002"+ - "+\u0007+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002/\u0007/\u0002"+ - "0\u00070\u00021\u00071\u00022\u00072\u00023\u00073\u00024\u00074\u0002"+ - "5\u00075\u00026\u00076\u00027\u00077\u00028\u00078\u00029\u00079\u0002"+ - ":\u0007:\u0002;\u0007;\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002"+ - "?\u0007?\u0002@\u0007@\u0002A\u0007A\u0002B\u0007B\u0002C\u0007C\u0002"+ - "D\u0007D\u0002E\u0007E\u0002F\u0007F\u0002G\u0007G\u0002H\u0007H\u0002"+ - "I\u0007I\u0002J\u0007J\u0002K\u0007K\u0002L\u0007L\u0002M\u0007M\u0002"+ - "N\u0007N\u0002O\u0007O\u0002P\u0007P\u0002Q\u0007Q\u0002R\u0007R\u0002"+ - "S\u0007S\u0002T\u0007T\u0002U\u0007U\u0002V\u0007V\u0002W\u0007W\u0002"+ - "X\u0007X\u0002Y\u0007Y\u0002Z\u0007Z\u0002[\u0007[\u0002\\\u0007\\\u0002"+ - "]\u0007]\u0002^\u0007^\u0002_\u0007_\u0002`\u0007`\u0002a\u0007a\u0002"+ - "b\u0007b\u0002c\u0007c\u0002d\u0007d\u0002e\u0007e\u0002f\u0007f\u0002"+ - "g\u0007g\u0002h\u0007h\u0002i\u0007i\u0002j\u0007j\u0002k\u0007k\u0002"+ - "l\u0007l\u0002m\u0007m\u0002n\u0007n\u0002o\u0007o\u0002p\u0007p\u0002"+ - "q\u0007q\u0002r\u0007r\u0002s\u0007s\u0002t\u0007t\u0002u\u0007u\u0002"+ - "v\u0007v\u0002w\u0007w\u0002x\u0007x\u0002y\u0007y\u0002z\u0007z\u0002"+ - "{\u0007{\u0002|\u0007|\u0002}\u0007}\u0002~\u0007~\u0002\u007f\u0007\u007f"+ - "\u0002\u0080\u0007\u0080\u0002\u0081\u0007\u0081\u0002\u0082\u0007\u0082"+ - "\u0002\u0083\u0007\u0083\u0002\u0084\u0007\u0084\u0002\u0085\u0007\u0085"+ - "\u0002\u0086\u0007\u0086\u0002\u0087\u0007\u0087\u0002\u0088\u0007\u0088"+ - "\u0002\u0089\u0007\u0089\u0002\u008a\u0007\u008a\u0002\u008b\u0007\u008b"+ - "\u0002\u008c\u0007\u008c\u0002\u008d\u0007\u008d\u0002\u008e\u0007\u008e"+ - "\u0002\u008f\u0007\u008f\u0002\u0090\u0007\u0090\u0002\u0091\u0007\u0091"+ - "\u0002\u0092\u0007\u0092\u0002\u0093\u0007\u0093\u0002\u0094\u0007\u0094"+ - "\u0002\u0095\u0007\u0095\u0002\u0096\u0007\u0096\u0002\u0097\u0007\u0097"+ - "\u0002\u0098\u0007\u0098\u0002\u0099\u0007\u0099\u0002\u009a\u0007\u009a"+ - "\u0002\u009b\u0007\u009b\u0002\u009c\u0007\u009c\u0002\u009d\u0007\u009d"+ - "\u0002\u009e\u0007\u009e\u0002\u009f\u0007\u009f\u0002\u00a0\u0007\u00a0"+ - "\u0002\u00a1\u0007\u00a1\u0002\u00a2\u0007\u00a2\u0002\u00a3\u0007\u00a3"+ - "\u0002\u00a4\u0007\u00a4\u0002\u00a5\u0007\u00a5\u0002\u00a6\u0007\u00a6"+ - "\u0002\u00a7\u0007\u00a7\u0002\u00a8\u0007\u00a8\u0002\u00a9\u0007\u00a9"+ - "\u0002\u00aa\u0007\u00aa\u0002\u00ab\u0007\u00ab\u0002\u00ac\u0007\u00ac"+ - "\u0002\u00ad\u0007\u00ad\u0002\u00ae\u0007\u00ae\u0002\u00af\u0007\u00af"+ - "\u0002\u00b0\u0007\u00b0\u0002\u00b1\u0007\u00b1\u0002\u00b2\u0007\u00b2"+ - "\u0002\u00b3\u0007\u00b3\u0002\u00b4\u0007\u00b4\u0002\u00b5\u0007\u00b5"+ - "\u0002\u00b6\u0007\u00b6\u0002\u00b7\u0007\u00b7\u0002\u00b8\u0007\u00b8"+ - "\u0002\u00b9\u0007\u00b9\u0002\u00ba\u0007\u00ba\u0002\u00bb\u0007\u00bb"+ - "\u0002\u00bc\u0007\u00bc\u0002\u00bd\u0007\u00bd\u0002\u00be\u0007\u00be"+ - "\u0002\u00bf\u0007\u00bf\u0002\u00c0\u0007\u00c0\u0002\u00c1\u0007\u00c1"+ - "\u0002\u00c2\u0007\u00c2\u0002\u00c3\u0007\u00c3\u0002\u00c4\u0007\u00c4"+ - "\u0002\u00c5\u0007\u00c5\u0002\u00c6\u0007\u00c6\u0002\u00c7\u0007\u00c7"+ - "\u0002\u00c8\u0007\u00c8\u0002\u00c9\u0007\u00c9\u0002\u00ca\u0007\u00ca"+ - "\u0002\u00cb\u0007\u00cb\u0002\u00cc\u0007\u00cc\u0002\u00cd\u0007\u00cd"+ - "\u0002\u00ce\u0007\u00ce\u0002\u00cf\u0007\u00cf\u0002\u00d0\u0007\u00d0"+ - "\u0002\u00d1\u0007\u00d1\u0002\u00d2\u0007\u00d2\u0002\u00d3\u0007\u00d3"+ - "\u0002\u00d4\u0007\u00d4\u0002\u00d5\u0007\u00d5\u0002\u00d6\u0007\u00d6"+ - "\u0002\u00d7\u0007\u00d7\u0002\u00d8\u0007\u00d8\u0002\u00d9\u0007\u00d9"+ - "\u0002\u00da\u0007\u00da\u0002\u00db\u0007\u00db\u0002\u00dc\u0007\u00dc"+ - "\u0002\u00dd\u0007\u00dd\u0002\u00de\u0007\u00de\u0002\u00df\u0007\u00df"+ - "\u0002\u00e0\u0007\u00e0\u0002\u00e1\u0007\u00e1\u0002\u00e2\u0007\u00e2"+ - "\u0002\u00e3\u0007\u00e3\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006"+ - "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+ - "\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+ - "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+ - "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ - "\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"+ - "\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ - "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010"+ + "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0002\u0000"+ + "\u0007\u0000\u0002\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003"+ + "\u0007\u0003\u0002\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006"+ + "\u0007\u0006\u0002\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002"+ + "\n\u0007\n\u0002\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002"+ + "\u000e\u0007\u000e\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002"+ + "\u0011\u0007\u0011\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002"+ + "\u0014\u0007\u0014\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002"+ + "\u0017\u0007\u0017\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002"+ + "\u001a\u0007\u001a\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002"+ + "\u001d\u0007\u001d\u0002\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002"+ + " \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002"+ + "%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002"+ + "*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002"+ + "/\u0007/\u00020\u00070\u00021\u00071\u00022\u00072\u00023\u00073\u0002"+ + "4\u00074\u00025\u00075\u00026\u00076\u00027\u00077\u00028\u00078\u0002"+ + "9\u00079\u0002:\u0007:\u0002;\u0007;\u0002<\u0007<\u0002=\u0007=\u0002"+ + ">\u0007>\u0002?\u0007?\u0002@\u0007@\u0002A\u0007A\u0002B\u0007B\u0002"+ + "C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002F\u0007F\u0002G\u0007G\u0002"+ + "H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002K\u0007K\u0002L\u0007L\u0002"+ + "M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002P\u0007P\u0002Q\u0007Q\u0002"+ + "R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002U\u0007U\u0002V\u0007V\u0002"+ + "W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002Z\u0007Z\u0002[\u0007[\u0002"+ + "\\\u0007\\\u0002]\u0007]\u0002^\u0007^\u0002_\u0007_\u0002`\u0007`\u0002"+ + "a\u0007a\u0002b\u0007b\u0002c\u0007c\u0002d\u0007d\u0002e\u0007e\u0002"+ + "f\u0007f\u0002g\u0007g\u0002h\u0007h\u0002i\u0007i\u0002j\u0007j\u0002"+ + "k\u0007k\u0002l\u0007l\u0002m\u0007m\u0002n\u0007n\u0002o\u0007o\u0002"+ + "p\u0007p\u0002q\u0007q\u0002r\u0007r\u0002s\u0007s\u0002t\u0007t\u0002"+ + "u\u0007u\u0002v\u0007v\u0002w\u0007w\u0002x\u0007x\u0002y\u0007y\u0002"+ + "z\u0007z\u0002{\u0007{\u0002|\u0007|\u0002}\u0007}\u0002~\u0007~\u0002"+ + "\u007f\u0007\u007f\u0002\u0080\u0007\u0080\u0002\u0081\u0007\u0081\u0002"+ + "\u0082\u0007\u0082\u0002\u0083\u0007\u0083\u0002\u0084\u0007\u0084\u0002"+ + "\u0085\u0007\u0085\u0002\u0086\u0007\u0086\u0002\u0087\u0007\u0087\u0002"+ + "\u0088\u0007\u0088\u0002\u0089\u0007\u0089\u0002\u008a\u0007\u008a\u0002"+ + "\u008b\u0007\u008b\u0002\u008c\u0007\u008c\u0002\u008d\u0007\u008d\u0002"+ + "\u008e\u0007\u008e\u0002\u008f\u0007\u008f\u0002\u0090\u0007\u0090\u0002"+ + "\u0091\u0007\u0091\u0002\u0092\u0007\u0092\u0002\u0093\u0007\u0093\u0002"+ + "\u0094\u0007\u0094\u0002\u0095\u0007\u0095\u0002\u0096\u0007\u0096\u0002"+ + "\u0097\u0007\u0097\u0002\u0098\u0007\u0098\u0002\u0099\u0007\u0099\u0002"+ + "\u009a\u0007\u009a\u0002\u009b\u0007\u009b\u0002\u009c\u0007\u009c\u0002"+ + "\u009d\u0007\u009d\u0002\u009e\u0007\u009e\u0002\u009f\u0007\u009f\u0002"+ + "\u00a0\u0007\u00a0\u0002\u00a1\u0007\u00a1\u0002\u00a2\u0007\u00a2\u0002"+ + "\u00a3\u0007\u00a3\u0002\u00a4\u0007\u00a4\u0002\u00a5\u0007\u00a5\u0002"+ + "\u00a6\u0007\u00a6\u0002\u00a7\u0007\u00a7\u0002\u00a8\u0007\u00a8\u0002"+ + "\u00a9\u0007\u00a9\u0002\u00aa\u0007\u00aa\u0002\u00ab\u0007\u00ab\u0002"+ + "\u00ac\u0007\u00ac\u0002\u00ad\u0007\u00ad\u0002\u00ae\u0007\u00ae\u0002"+ + "\u00af\u0007\u00af\u0002\u00b0\u0007\u00b0\u0002\u00b1\u0007\u00b1\u0002"+ + "\u00b2\u0007\u00b2\u0002\u00b3\u0007\u00b3\u0002\u00b4\u0007\u00b4\u0002"+ + "\u00b5\u0007\u00b5\u0002\u00b6\u0007\u00b6\u0002\u00b7\u0007\u00b7\u0002"+ + "\u00b8\u0007\u00b8\u0002\u00b9\u0007\u00b9\u0002\u00ba\u0007\u00ba\u0002"+ + "\u00bb\u0007\u00bb\u0002\u00bc\u0007\u00bc\u0002\u00bd\u0007\u00bd\u0002"+ + "\u00be\u0007\u00be\u0002\u00bf\u0007\u00bf\u0002\u00c0\u0007\u00c0\u0002"+ + "\u00c1\u0007\u00c1\u0002\u00c2\u0007\u00c2\u0002\u00c3\u0007\u00c3\u0002"+ + "\u00c4\u0007\u00c4\u0002\u00c5\u0007\u00c5\u0002\u00c6\u0007\u00c6\u0002"+ + "\u00c7\u0007\u00c7\u0002\u00c8\u0007\u00c8\u0002\u00c9\u0007\u00c9\u0002"+ + "\u00ca\u0007\u00ca\u0002\u00cb\u0007\u00cb\u0002\u00cc\u0007\u00cc\u0002"+ + "\u00cd\u0007\u00cd\u0002\u00ce\u0007\u00ce\u0002\u00cf\u0007\u00cf\u0002"+ + "\u00d0\u0007\u00d0\u0002\u00d1\u0007\u00d1\u0002\u00d2\u0007\u00d2\u0002"+ + "\u00d3\u0007\u00d3\u0002\u00d4\u0007\u00d4\u0002\u00d5\u0007\u00d5\u0002"+ + "\u00d6\u0007\u00d6\u0002\u00d7\u0007\u00d7\u0002\u00d8\u0007\u00d8\u0002"+ + "\u00d9\u0007\u00d9\u0002\u00da\u0007\u00da\u0002\u00db\u0007\u00db\u0002"+ + "\u00dc\u0007\u00dc\u0002\u00dd\u0007\u00dd\u0002\u00de\u0007\u00de\u0002"+ + "\u00df\u0007\u00df\u0002\u00e0\u0007\u00e0\u0002\u00e1\u0007\u00e1\u0002"+ + "\u00e2\u0007\u00e2\u0002\u00e3\u0007\u00e3\u0002\u00e4\u0007\u00e4\u0002"+ + "\u00e5\u0007\u00e5\u0002\u00e6\u0007\u00e6\u0002\u00e7\u0007\u00e7\u0002"+ + "\u00e8\u0007\u00e8\u0002\u00e9\u0007\u00e9\u0001\u0000\u0001\u0000\u0001"+ + "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ + "\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+ + "\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ + "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ + "\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+ + "\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"+ + "\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ + "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010"+ "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ + "\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ - "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ - "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0001\u0012"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012"+ "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+ "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+ - "\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+ + "\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+ "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+ - "\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014"+ + "\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014"+ "\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014"+ + "\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015"+ "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015"+ - "\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016"+ - "\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017"+ - "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017"+ - "\u0001\u0017\u0001\u0018\u0004\u0018\u02b4\b\u0018\u000b\u0018\f\u0018"+ - "\u02b5\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001"+ - "\u0019\u0005\u0019\u02be\b\u0019\n\u0019\f\u0019\u02c1\t\u0019\u0001\u0019"+ - "\u0003\u0019\u02c4\b\u0019\u0001\u0019\u0003\u0019\u02c7\b\u0019\u0001"+ - "\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001"+ - "\u001a\u0005\u001a\u02d0\b\u001a\n\u001a\f\u001a\u02d3\t\u001a\u0001\u001a"+ - "\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b\u0004\u001b"+ - "\u02db\b\u001b\u000b\u001b\f\u001b\u02dc\u0001\u001b\u0001\u001b\u0001"+ - "\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0001"+ - "\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001"+ - "!\u0001!\u0003!\u02f0\b!\u0001!\u0004!\u02f3\b!\u000b!\f!\u02f4\u0001"+ - "\"\u0001\"\u0001#\u0001#\u0001$\u0001$\u0001$\u0003$\u02fe\b$\u0001%\u0001"+ - "%\u0001&\u0001&\u0001&\u0003&\u0305\b&\u0001\'\u0001\'\u0001\'\u0005\'"+ - "\u030a\b\'\n\'\f\'\u030d\t\'\u0001\'\u0001\'\u0001\'\u0001\'\u0001\'\u0001"+ - "\'\u0005\'\u0315\b\'\n\'\f\'\u0318\t\'\u0001\'\u0001\'\u0001\'\u0001\'"+ - "\u0001\'\u0003\'\u031f\b\'\u0001\'\u0003\'\u0322\b\'\u0003\'\u0324\b\'"+ - "\u0001(\u0004(\u0327\b(\u000b(\f(\u0328\u0001)\u0004)\u032c\b)\u000b)"+ - "\f)\u032d\u0001)\u0001)\u0005)\u0332\b)\n)\f)\u0335\t)\u0001)\u0001)\u0004"+ - ")\u0339\b)\u000b)\f)\u033a\u0001)\u0004)\u033e\b)\u000b)\f)\u033f\u0001"+ - ")\u0001)\u0005)\u0344\b)\n)\f)\u0347\t)\u0003)\u0349\b)\u0001)\u0001)"+ - "\u0001)\u0001)\u0004)\u034f\b)\u000b)\f)\u0350\u0001)\u0001)\u0003)\u0355"+ - "\b)\u0001*\u0001*\u0001*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001"+ - ",\u0001,\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001/\u00010\u0001"+ - "0\u00011\u00011\u00011\u00011\u00011\u00012\u00012\u00013\u00013\u0001"+ - "3\u00013\u00013\u00013\u00014\u00014\u00014\u00014\u00014\u00014\u0001"+ - "5\u00015\u00015\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u0001"+ - "7\u00018\u00018\u00018\u00018\u00018\u00019\u00019\u0001:\u0001:\u0001"+ - ":\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001"+ - "<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001>\u0001>\u0001?\u0001?\u0001"+ - "?\u0001?\u0001?\u0001?\u0001@\u0001@\u0001A\u0001A\u0001A\u0001A\u0001"+ - "A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001C\u0001D\u0001D\u0001D\u0001"+ - "E\u0001E\u0001F\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001H\u0001"+ - "I\u0001I\u0001J\u0001J\u0001K\u0001K\u0001L\u0001L\u0001M\u0001M\u0001"+ - "N\u0001N\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001"+ - "Q\u0003Q\u03d9\bQ\u0001Q\u0005Q\u03dc\bQ\nQ\fQ\u03df\tQ\u0001Q\u0001Q"+ - "\u0004Q\u03e3\bQ\u000bQ\fQ\u03e4\u0003Q\u03e7\bQ\u0001R\u0001R\u0001R"+ - "\u0001R\u0001R\u0001S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0005"+ - "T\u03f5\bT\nT\fT\u03f8\tT\u0001T\u0001T\u0003T\u03fc\bT\u0001T\u0004T"+ - "\u03ff\bT\u000bT\fT\u0400\u0003T\u0403\bT\u0001U\u0001U\u0004U\u0407\b"+ - "U\u000bU\fU\u0408\u0001U\u0001U\u0001V\u0001V\u0001W\u0001W\u0001W\u0001"+ - "W\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001Z\u0001"+ - "Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001"+ - "\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001"+ - "^\u0001_\u0001_\u0001_\u0001_\u0001_\u0001`\u0001`\u0001`\u0001`\u0001"+ - "a\u0001a\u0001a\u0001a\u0001b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001"+ - "c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001"+ - "e\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0003f\u0456\bf\u0001"+ - "g\u0004g\u0459\bg\u000bg\fg\u045a\u0001h\u0001h\u0001h\u0001h\u0001i\u0001"+ - "i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001k\u0001"+ - "k\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001m\u0001m\u0001"+ - "n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001"+ - "p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001r\u0003"+ - "r\u048a\br\u0001s\u0001s\u0003s\u048e\bs\u0001s\u0005s\u0491\bs\ns\fs"+ - "\u0494\ts\u0001s\u0001s\u0003s\u0498\bs\u0001s\u0004s\u049b\bs\u000bs"+ - "\fs\u049c\u0003s\u049f\bs\u0001t\u0001t\u0004t\u04a3\bt\u000bt\ft\u04a4"+ - "\u0001u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001"+ - "w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001"+ - "y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001"+ - "|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001"+ - "~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080"+ - "\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0081"+ - "\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083\u0001\u0083"+ + "\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016"+ + "\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017"+ + "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018"+ + "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018"+ + "\u0001\u0018\u0001\u0018\u0001\u0019\u0004\u0019\u02cd\b\u0019\u000b\u0019"+ + "\f\u0019\u02ce\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a"+ + "\u0001\u001a\u0005\u001a\u02d7\b\u001a\n\u001a\f\u001a\u02da\t\u001a\u0001"+ + "\u001a\u0003\u001a\u02dd\b\u001a\u0001\u001a\u0003\u001a\u02e0\b\u001a"+ + "\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"+ + "\u0001\u001b\u0005\u001b\u02e9\b\u001b\n\u001b\f\u001b\u02ec\t\u001b\u0001"+ + "\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001c\u0004"+ + "\u001c\u02f4\b\u001c\u000b\u001c\f\u001c\u02f5\u0001\u001c\u0001\u001c"+ + "\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e"+ + "\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001 \u0001!\u0001!\u0001\"\u0001"+ + "\"\u0003\"\u0309\b\"\u0001\"\u0004\"\u030c\b\"\u000b\"\f\"\u030d\u0001"+ + "#\u0001#\u0001$\u0001$\u0001%\u0001%\u0001%\u0003%\u0317\b%\u0001&\u0001"+ + "&\u0001\'\u0001\'\u0001\'\u0003\'\u031e\b\'\u0001(\u0001(\u0001(\u0005"+ + "(\u0323\b(\n(\f(\u0326\t(\u0001(\u0001(\u0001(\u0001(\u0001(\u0001(\u0005"+ + "(\u032e\b(\n(\f(\u0331\t(\u0001(\u0001(\u0001(\u0001(\u0001(\u0003(\u0338"+ + "\b(\u0001(\u0003(\u033b\b(\u0003(\u033d\b(\u0001)\u0004)\u0340\b)\u000b"+ + ")\f)\u0341\u0001*\u0004*\u0345\b*\u000b*\f*\u0346\u0001*\u0001*\u0005"+ + "*\u034b\b*\n*\f*\u034e\t*\u0001*\u0001*\u0004*\u0352\b*\u000b*\f*\u0353"+ + "\u0001*\u0004*\u0357\b*\u000b*\f*\u0358\u0001*\u0001*\u0005*\u035d\b*"+ + "\n*\f*\u0360\t*\u0003*\u0362\b*\u0001*\u0001*\u0001*\u0001*\u0004*\u0368"+ + "\b*\u000b*\f*\u0369\u0001*\u0001*\u0003*\u036e\b*\u0001+\u0001+\u0001"+ + "+\u0001,\u0001,\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001"+ + ".\u0001/\u0001/\u0001/\u00010\u00010\u00011\u00011\u00012\u00012\u0001"+ + "2\u00012\u00012\u00013\u00013\u00014\u00014\u00014\u00014\u00014\u0001"+ + "4\u00015\u00015\u00015\u00015\u00015\u00015\u00016\u00016\u00016\u0001"+ + "7\u00017\u00017\u00018\u00018\u00018\u00018\u00018\u00019\u00019\u0001"+ + "9\u00019\u00019\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001<\u0001"+ + "<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001=\u0001=\u0001=\u0001"+ + ">\u0001>\u0001>\u0001?\u0001?\u0001@\u0001@\u0001@\u0001@\u0001@\u0001"+ + "@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001B\u0001B\u0001C\u0001C\u0001"+ + "C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001E\u0001F\u0001F\u0001G\u0001"+ + "G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001I\u0001J\u0001J\u0001K\u0001"+ + "K\u0001L\u0001L\u0001M\u0001M\u0001N\u0001N\u0001O\u0001O\u0001P\u0001"+ + "P\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0003R\u03f2\bR\u0001"+ + "R\u0005R\u03f5\bR\nR\fR\u03f8\tR\u0001R\u0001R\u0004R\u03fc\bR\u000bR"+ + "\fR\u03fd\u0003R\u0400\bR\u0001S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001"+ + "T\u0001T\u0001T\u0001T\u0001U\u0001U\u0005U\u040e\bU\nU\fU\u0411\tU\u0001"+ + "U\u0001U\u0003U\u0415\bU\u0001U\u0004U\u0418\bU\u000bU\fU\u0419\u0003"+ + "U\u041c\bU\u0001V\u0001V\u0004V\u0420\bV\u000bV\fV\u0421\u0001V\u0001"+ + "V\u0001W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001"+ + "Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001[\u0001"+ + "\\\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^"+ + "\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001_\u0001`\u0001`\u0001"+ + "`\u0001`\u0001`\u0001a\u0001a\u0001a\u0001a\u0001b\u0001b\u0001b\u0001"+ + "b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001"+ + "e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001f\u0001f\u0001f\u0001"+ + "f\u0001f\u0001g\u0001g\u0001g\u0003g\u046f\bg\u0001h\u0004h\u0472\bh\u000b"+ + "h\fh\u0473\u0001i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001"+ + "k\u0001k\u0001k\u0001k\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001"+ + "m\u0001m\u0001n\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001"+ + "o\u0001p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001r\u0001"+ + "r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0003s\u04a3\bs\u0001t\u0001"+ + "t\u0003t\u04a7\bt\u0001t\u0005t\u04aa\bt\nt\ft\u04ad\tt\u0001t\u0001t"+ + "\u0003t\u04b1\bt\u0001t\u0004t\u04b4\bt\u000bt\ft\u04b5\u0003t\u04b8\b"+ + "t\u0001u\u0001u\u0004u\u04bc\bu\u000bu\fu\u04bd\u0001v\u0001v\u0001v\u0001"+ + "v\u0001w\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001"+ + "y\u0001y\u0001y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001"+ + "{\u0001{\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001"+ + "~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080"+ + "\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081\u0001\u0081"+ + "\u0001\u0081\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083"+ "\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001\u0084"+ "\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085"+ "\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086"+ - "\u0001\u0086\u0001\u0086\u0001\u0087\u0001\u0087\u0001\u0088\u0004\u0088"+ - "\u04f8\b\u0088\u000b\u0088\f\u0088\u04f9\u0001\u0088\u0001\u0088\u0003"+ - "\u0088\u04fe\b\u0088\u0001\u0088\u0004\u0088\u0501\b\u0088\u000b\u0088"+ - "\f\u0088\u0502\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a"+ - "\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001\u008b"+ - "\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008d"+ - "\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e"+ - "\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f"+ - "\u0001\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091"+ - "\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092"+ - "\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0094"+ - "\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001\u0095"+ - "\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097"+ - "\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001\u0098"+ - "\u0001\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099"+ - "\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b"+ - "\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c"+ - "\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e\u0001\u009e"+ - "\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f"+ - "\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a1\u0001\u00a1"+ - "\u0001\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2"+ + "\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087"+ + "\u0001\u0087\u0001\u0088\u0001\u0088\u0001\u0089\u0004\u0089\u0511\b\u0089"+ + "\u000b\u0089\f\u0089\u0512\u0001\u0089\u0001\u0089\u0003\u0089\u0517\b"+ + "\u0089\u0001\u0089\u0004\u0089\u051a\b\u0089\u000b\u0089\f\u0089\u051b"+ + "\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b"+ + "\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c"+ + "\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e"+ + "\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f"+ + "\u0001\u008f\u0001\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090"+ + "\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092"+ + "\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093"+ + "\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095"+ + "\u0001\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096"+ + "\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098"+ + "\u0001\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099"+ + "\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b"+ + "\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c"+ + "\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e"+ + "\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f"+ + "\u0001\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a1"+ + "\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2"+ "\u0001\u00a2\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3"+ - "\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5"+ - "\u0001\u00a5\u0001\u00a5\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6"+ - "\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8"+ - "\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9"+ - "\u0001\u00a9\u0001\u00a9\u0004\u00a9\u0590\b\u00a9\u000b\u00a9\f\u00a9"+ - "\u0591\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001"+ - "\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001"+ - "\u00ac\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001"+ - "\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001"+ - "\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001"+ - "\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001"+ - "\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001"+ - "\u00b3\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001"+ - "\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001"+ - "\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001"+ - "\u00b7\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9\u0001"+ - "\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001"+ - "\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bc\u0001"+ - "\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001"+ - "\u00bd\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001"+ + "\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5"+ + "\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a6\u0001\u00a6\u0001\u00a6"+ + "\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8"+ + "\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9\u0001\u00a9"+ + "\u0001\u00a9\u0001\u00a9\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa"+ + "\u0001\u00aa\u0004\u00aa\u05a9\b\u00aa\u000b\u00aa\f\u00aa\u05aa\u0001"+ + "\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001"+ + "\u00ac\u0001\u00ac\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001"+ + "\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001"+ + "\u00af\u0001\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001"+ + "\u00b0\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001"+ + "\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001"+ + "\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001"+ + "\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001"+ + "\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001"+ + "\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001"+ + "\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001"+ + "\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001"+ + "\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd\u0001"+ + "\u00bd\u0001\u00bd\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001"+ "\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001"+ - "\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001"+ - "\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001"+ - "\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001"+ - "\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4\u0001\u00c4\u0001"+ - "\u00c4\u0001\u00c4\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001"+ - "\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c7\u0001\u00c7\u0001"+ - "\u00c7\u0001\u00c7\u0001\u00c8\u0001\u00c8\u0001\u00c8\u0001\u00c8\u0001"+ - "\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00ca\u0001\u00ca\u0001"+ - "\u00ca\u0001\u00ca\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001"+ + "\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001"+ + "\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001"+ + "\u00c2\u0001\u00c2\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001"+ + "\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4\u0001"+ + "\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001"+ + "\u00c5\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c7\u0001"+ + "\u00c7\u0001\u00c7\u0001\u00c7\u0001\u00c8\u0001\u00c8\u0001\u00c8\u0001"+ + "\u00c8\u0001\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00ca\u0001"+ + "\u00ca\u0001\u00ca\u0001\u00ca\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001"+ "\u00cb\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001"+ - "\u00cc\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001"+ - "\u00cd\u0001\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00cf\u0001"+ - "\u00cf\u0001\u00cf\u0001\u00cf\u0001\u00d0\u0001\u00d0\u0001\u00d0\u0001"+ - "\u00d0\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001"+ - "\u00d1\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001"+ - "\u00d2\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d4\u0001"+ - "\u00d4\u0001\u00d4\u0001\u00d4\u0001\u00d5\u0001\u00d5\u0001\u00d5\u0001"+ - "\u00d5\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001"+ - "\u00d6\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001"+ - "\u00d7\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001"+ - "\u00d8\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001"+ + "\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001"+ + "\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00ce\u0001"+ + "\u00cf\u0001\u00cf\u0001\u00cf\u0001\u00cf\u0001\u00d0\u0001\u00d0\u0001"+ + "\u00d0\u0001\u00d0\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001"+ + "\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001"+ + "\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001"+ + "\u00d4\u0001\u00d4\u0001\u00d4\u0001\u00d4\u0001\u00d5\u0001\u00d5\u0001"+ + "\u00d5\u0001\u00d5\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001"+ + "\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001"+ + "\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001"+ + "\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001"+ "\u00da\u0001\u00da\u0001\u00da\u0001\u00da\u0001\u00da\u0001\u00db\u0001"+ - "\u00db\u0001\u00db\u0001\u00db\u0001\u00dc\u0001\u00dc\u0001\u00dc\u0001"+ - "\u00dc\u0001\u00dd\u0001\u00dd\u0001\u00dd\u0001\u00dd\u0001\u00de\u0001"+ - "\u00de\u0001\u00de\u0001\u00de\u0001\u00df\u0001\u00df\u0001\u00df\u0001"+ - "\u00df\u0001\u00e0\u0001\u00e0\u0001\u00e0\u0001\u00e0\u0001\u00e1\u0001"+ - "\u00e1\u0001\u00e1\u0001\u00e1\u0001\u00e2\u0001\u00e2\u0001\u00e2\u0001"+ - "\u00e2\u0001\u00e3\u0001\u00e3\u0001\u00e3\u0001\u00e3\u0002\u02d1\u0316"+ - "\u0000\u00e4\u0011\u0001\u0013\u0002\u0015\u0003\u0017\u0004\u0019\u0005"+ - "\u001b\u0006\u001d\u0007\u001f\b!\t#\n%\u000b\'\f)\r+\u000e-\u000f/\u0010"+ - "1\u00113\u00125\u00137\u00149\u0015;\u0016=\u0017?\u0018A\u0019C\u001a"+ - "E\u001bG\u001cI\u001dK\u0000M\u0000O\u0000Q\u0000S\u0000U\u0000W\u0000"+ - "Y\u0000[\u0000]\u0000_\u001ea\u001fc e!g\"i#k$m%o&q\'s(u)w*y+{,}-\u007f"+ - ".\u0081/\u00830\u00851\u00872\u00893\u008b4\u008d5\u008f6\u00917\u0093"+ - "8\u00959\u0097:\u0099;\u009b<\u009d=\u009f>\u00a1?\u00a3@\u00a5A\u00a7"+ - "B\u00a9C\u00abD\u00adE\u00afF\u00b1\u0000\u00b3G\u00b5H\u00b7I\u00b9J"+ - "\u00bb\u0000\u00bdK\u00bfL\u00c1M\u00c3N\u00c5\u0000\u00c7\u0000\u00c9"+ - "O\u00cbP\u00cdQ\u00cf\u0000\u00d1\u0000\u00d3\u0000\u00d5\u0000\u00d7"+ - "\u0000\u00d9\u0000\u00dbR\u00dd\u0000\u00dfS\u00e1\u0000\u00e3\u0000\u00e5"+ - "T\u00e7U\u00e9V\u00eb\u0000\u00ed\u0000\u00ef\u0000\u00f1\u0000\u00f3"+ - "\u0000\u00f5\u0000\u00f7\u0000\u00f9W\u00fbX\u00fdY\u00ffZ\u0101\u0000"+ - "\u0103\u0000\u0105\u0000\u0107\u0000\u0109\u0000\u010b\u0000\u010d[\u010f"+ - "\u0000\u0111\\\u0113]\u0115^\u0117\u0000\u0119\u0000\u011b_\u011d`\u011f"+ - "\u0000\u0121a\u0123\u0000\u0125b\u0127c\u0129d\u012b\u0000\u012d\u0000"+ - "\u012f\u0000\u0131\u0000\u0133\u0000\u0135\u0000\u0137\u0000\u0139\u0000"+ - "\u013b\u0000\u013de\u013ff\u0141g\u0143\u0000\u0145\u0000\u0147\u0000"+ - "\u0149\u0000\u014b\u0000\u014d\u0000\u014fh\u0151i\u0153j\u0155\u0000"+ - "\u0157k\u0159l\u015bm\u015dn\u015f\u0000\u0161\u0000\u0163o\u0165p\u0167"+ - "q\u0169r\u016b\u0000\u016d\u0000\u016f\u0000\u0171\u0000\u0173\u0000\u0175"+ - "\u0000\u0177\u0000\u0179s\u017bt\u017du\u017f\u0000\u0181\u0000\u0183"+ - "\u0000\u0185\u0000\u0187v\u0189w\u018bx\u018d\u0000\u018fy\u0191\u0000"+ - "\u0193\u0000\u0195z\u0197\u0000\u0199\u0000\u019b\u0000\u019d\u0000\u019f"+ - "\u0000\u01a1{\u01a3|\u01a5}\u01a7\u0000\u01a9\u0000\u01ab\u0000\u01ad"+ - "~\u01af\u007f\u01b1\u0080\u01b3\u0000\u01b5\u0000\u01b7\u0081\u01b9\u0082"+ - "\u01bb\u0083\u01bd\u0000\u01bf\u0000\u01c1\u0000\u01c3\u0000\u01c5\u0000"+ - "\u01c7\u0000\u01c9\u0000\u01cb\u0000\u01cd\u0000\u01cf\u0000\u01d1\u0000"+ - "\u01d3\u0084\u01d5\u0085\u01d7\u0086\u0011\u0000\u0001\u0002\u0003\u0004"+ - "\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010$\u0002\u0000DDdd"+ - "\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002\u0000CCcc\u0002"+ - "\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000PPpp\u0002\u0000"+ - "NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002\u0000LLll\u0002"+ - "\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000GGgg\u0002\u0000"+ - "KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r\r //[[]]\u0002"+ - "\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002\u0000AZaz\b\u0000"+ - "\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001"+ - "\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t\n\r\r \"\",,/"+ - "/::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r \"#,,//::<<>?\\\\||\u0002"+ - "\u0000JJjj\u06b3\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001"+ - "\u0000\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001"+ - "\u0000\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001"+ - "\u0000\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001"+ - "\u0000\u0000\u0000\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000\u0000"+ - "\u0000\u0000%\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000\u0000"+ - "\u0000)\u0001\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000-"+ - "\u0001\u0000\u0000\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001\u0000"+ - "\u0000\u0000\u00003\u0001\u0000\u0000\u0000\u00005\u0001\u0000\u0000\u0000"+ - "\u00007\u0001\u0000\u0000\u0000\u00009\u0001\u0000\u0000\u0000\u0000;"+ - "\u0001\u0000\u0000\u0000\u0000=\u0001\u0000\u0000\u0000\u0000?\u0001\u0000"+ - "\u0000\u0000\u0000A\u0001\u0000\u0000\u0000\u0000C\u0001\u0000\u0000\u0000"+ - "\u0000E\u0001\u0000\u0000\u0000\u0000G\u0001\u0000\u0000\u0000\u0001I"+ - "\u0001\u0000\u0000\u0000\u0001_\u0001\u0000\u0000\u0000\u0001a\u0001\u0000"+ - "\u0000\u0000\u0001c\u0001\u0000\u0000\u0000\u0001e\u0001\u0000\u0000\u0000"+ - "\u0001g\u0001\u0000\u0000\u0000\u0001i\u0001\u0000\u0000\u0000\u0001k"+ - "\u0001\u0000\u0000\u0000\u0001m\u0001\u0000\u0000\u0000\u0001o\u0001\u0000"+ - "\u0000\u0000\u0001q\u0001\u0000\u0000\u0000\u0001s\u0001\u0000\u0000\u0000"+ - "\u0001u\u0001\u0000\u0000\u0000\u0001w\u0001\u0000\u0000\u0000\u0001y"+ - "\u0001\u0000\u0000\u0000\u0001{\u0001\u0000\u0000\u0000\u0001}\u0001\u0000"+ - "\u0000\u0000\u0001\u007f\u0001\u0000\u0000\u0000\u0001\u0081\u0001\u0000"+ - "\u0000\u0000\u0001\u0083\u0001\u0000\u0000\u0000\u0001\u0085\u0001\u0000"+ - "\u0000\u0000\u0001\u0087\u0001\u0000\u0000\u0000\u0001\u0089\u0001\u0000"+ - "\u0000\u0000\u0001\u008b\u0001\u0000\u0000\u0000\u0001\u008d\u0001\u0000"+ - "\u0000\u0000\u0001\u008f\u0001\u0000\u0000\u0000\u0001\u0091\u0001\u0000"+ - "\u0000\u0000\u0001\u0093\u0001\u0000\u0000\u0000\u0001\u0095\u0001\u0000"+ - "\u0000\u0000\u0001\u0097\u0001\u0000\u0000\u0000\u0001\u0099\u0001\u0000"+ - "\u0000\u0000\u0001\u009b\u0001\u0000\u0000\u0000\u0001\u009d\u0001\u0000"+ - "\u0000\u0000\u0001\u009f\u0001\u0000\u0000\u0000\u0001\u00a1\u0001\u0000"+ - "\u0000\u0000\u0001\u00a3\u0001\u0000\u0000\u0000\u0001\u00a5\u0001\u0000"+ - "\u0000\u0000\u0001\u00a7\u0001\u0000\u0000\u0000\u0001\u00a9\u0001\u0000"+ - "\u0000\u0000\u0001\u00ab\u0001\u0000\u0000\u0000\u0001\u00ad\u0001\u0000"+ - "\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001\u00b1\u0001\u0000"+ - "\u0000\u0000\u0001\u00b3\u0001\u0000\u0000\u0000\u0001\u00b5\u0001\u0000"+ - "\u0000\u0000\u0001\u00b7\u0001\u0000\u0000\u0000\u0001\u00b9\u0001\u0000"+ - "\u0000\u0000\u0001\u00bd\u0001\u0000\u0000\u0000\u0001\u00bf\u0001\u0000"+ - "\u0000\u0000\u0001\u00c1\u0001\u0000\u0000\u0000\u0001\u00c3\u0001\u0000"+ - "\u0000\u0000\u0002\u00c5\u0001\u0000\u0000\u0000\u0002\u00c7\u0001\u0000"+ - "\u0000\u0000\u0002\u00c9\u0001\u0000\u0000\u0000\u0002\u00cb\u0001\u0000"+ - "\u0000\u0000\u0002\u00cd\u0001\u0000\u0000\u0000\u0003\u00cf\u0001\u0000"+ - "\u0000\u0000\u0003\u00d1\u0001\u0000\u0000\u0000\u0003\u00d3\u0001\u0000"+ - "\u0000\u0000\u0003\u00d5\u0001\u0000\u0000\u0000\u0003\u00d7\u0001\u0000"+ - "\u0000\u0000\u0003\u00d9\u0001\u0000\u0000\u0000\u0003\u00db\u0001\u0000"+ - "\u0000\u0000\u0003\u00df\u0001\u0000\u0000\u0000\u0003\u00e1\u0001\u0000"+ - "\u0000\u0000\u0003\u00e3\u0001\u0000\u0000\u0000\u0003\u00e5\u0001\u0000"+ - "\u0000\u0000\u0003\u00e7\u0001\u0000\u0000\u0000\u0003\u00e9\u0001\u0000"+ - "\u0000\u0000\u0004\u00eb\u0001\u0000\u0000\u0000\u0004\u00ed\u0001\u0000"+ - "\u0000\u0000\u0004\u00ef\u0001\u0000\u0000\u0000\u0004\u00f1\u0001\u0000"+ - "\u0000\u0000\u0004\u00f3\u0001\u0000\u0000\u0000\u0004\u00f9\u0001\u0000"+ - "\u0000\u0000\u0004\u00fb\u0001\u0000\u0000\u0000\u0004\u00fd\u0001\u0000"+ - "\u0000\u0000\u0004\u00ff\u0001\u0000\u0000\u0000\u0005\u0101\u0001\u0000"+ - "\u0000\u0000\u0005\u0103\u0001\u0000\u0000\u0000\u0005\u0105\u0001\u0000"+ - "\u0000\u0000\u0005\u0107\u0001\u0000\u0000\u0000\u0005\u0109\u0001\u0000"+ - "\u0000\u0000\u0005\u010b\u0001\u0000\u0000\u0000\u0005\u010d\u0001\u0000"+ - "\u0000\u0000\u0005\u010f\u0001\u0000\u0000\u0000\u0005\u0111\u0001\u0000"+ - "\u0000\u0000\u0005\u0113\u0001\u0000\u0000\u0000\u0005\u0115\u0001\u0000"+ - "\u0000\u0000\u0006\u0117\u0001\u0000\u0000\u0000\u0006\u0119\u0001\u0000"+ - "\u0000\u0000\u0006\u011b\u0001\u0000\u0000\u0000\u0006\u011d\u0001\u0000"+ - "\u0000\u0000\u0006\u0121\u0001\u0000\u0000\u0000\u0006\u0123\u0001\u0000"+ - "\u0000\u0000\u0006\u0125\u0001\u0000\u0000\u0000\u0006\u0127\u0001\u0000"+ - "\u0000\u0000\u0006\u0129\u0001\u0000\u0000\u0000\u0007\u012b\u0001\u0000"+ - "\u0000\u0000\u0007\u012d\u0001\u0000\u0000\u0000\u0007\u012f\u0001\u0000"+ - "\u0000\u0000\u0007\u0131\u0001\u0000\u0000\u0000\u0007\u0133\u0001\u0000"+ - "\u0000\u0000\u0007\u0135\u0001\u0000\u0000\u0000\u0007\u0137\u0001\u0000"+ - "\u0000\u0000\u0007\u0139\u0001\u0000\u0000\u0000\u0007\u013b\u0001\u0000"+ - "\u0000\u0000\u0007\u013d\u0001\u0000\u0000\u0000\u0007\u013f\u0001\u0000"+ - "\u0000\u0000\u0007\u0141\u0001\u0000\u0000\u0000\b\u0143\u0001\u0000\u0000"+ - "\u0000\b\u0145\u0001\u0000\u0000\u0000\b\u0147\u0001\u0000\u0000\u0000"+ - "\b\u0149\u0001\u0000\u0000\u0000\b\u014b\u0001\u0000\u0000\u0000\b\u014d"+ - "\u0001\u0000\u0000\u0000\b\u014f\u0001\u0000\u0000\u0000\b\u0151\u0001"+ - "\u0000\u0000\u0000\b\u0153\u0001\u0000\u0000\u0000\t\u0155\u0001\u0000"+ - "\u0000\u0000\t\u0157\u0001\u0000\u0000\u0000\t\u0159\u0001\u0000\u0000"+ - "\u0000\t\u015b\u0001\u0000\u0000\u0000\t\u015d\u0001\u0000\u0000\u0000"+ - "\n\u015f\u0001\u0000\u0000\u0000\n\u0161\u0001\u0000\u0000\u0000\n\u0163"+ - "\u0001\u0000\u0000\u0000\n\u0165\u0001\u0000\u0000\u0000\n\u0167\u0001"+ - "\u0000\u0000\u0000\n\u0169\u0001\u0000\u0000\u0000\u000b\u016b\u0001\u0000"+ - "\u0000\u0000\u000b\u016d\u0001\u0000\u0000\u0000\u000b\u016f\u0001\u0000"+ - "\u0000\u0000\u000b\u0171\u0001\u0000\u0000\u0000\u000b\u0173\u0001\u0000"+ - "\u0000\u0000\u000b\u0175\u0001\u0000\u0000\u0000\u000b\u0177\u0001\u0000"+ - "\u0000\u0000\u000b\u0179\u0001\u0000\u0000\u0000\u000b\u017b\u0001\u0000"+ - "\u0000\u0000\u000b\u017d\u0001\u0000\u0000\u0000\f\u017f\u0001\u0000\u0000"+ - "\u0000\f\u0181\u0001\u0000\u0000\u0000\f\u0183\u0001\u0000\u0000\u0000"+ - "\f\u0185\u0001\u0000\u0000\u0000\f\u0187\u0001\u0000\u0000\u0000\f\u0189"+ - "\u0001\u0000\u0000\u0000\f\u018b\u0001\u0000\u0000\u0000\r\u018d\u0001"+ - "\u0000\u0000\u0000\r\u018f\u0001\u0000\u0000\u0000\r\u0191\u0001\u0000"+ - "\u0000\u0000\r\u0193\u0001\u0000\u0000\u0000\r\u0195\u0001\u0000\u0000"+ - "\u0000\r\u0197\u0001\u0000\u0000\u0000\r\u0199\u0001\u0000\u0000\u0000"+ - "\r\u019b\u0001\u0000\u0000\u0000\r\u019d\u0001\u0000\u0000\u0000\r\u019f"+ - "\u0001\u0000\u0000\u0000\r\u01a1\u0001\u0000\u0000\u0000\r\u01a3\u0001"+ - "\u0000\u0000\u0000\r\u01a5\u0001\u0000\u0000\u0000\u000e\u01a7\u0001\u0000"+ - "\u0000\u0000\u000e\u01a9\u0001\u0000\u0000\u0000\u000e\u01ab\u0001\u0000"+ - "\u0000\u0000\u000e\u01ad\u0001\u0000\u0000\u0000\u000e\u01af\u0001\u0000"+ - "\u0000\u0000\u000e\u01b1\u0001\u0000\u0000\u0000\u000f\u01b3\u0001\u0000"+ - "\u0000\u0000\u000f\u01b5\u0001\u0000\u0000\u0000\u000f\u01b7\u0001\u0000"+ - "\u0000\u0000\u000f\u01b9\u0001\u0000\u0000\u0000\u000f\u01bb\u0001\u0000"+ - "\u0000\u0000\u000f\u01bd\u0001\u0000\u0000\u0000\u000f\u01bf\u0001\u0000"+ - "\u0000\u0000\u000f\u01c1\u0001\u0000\u0000\u0000\u000f\u01c3\u0001\u0000"+ - "\u0000\u0000\u0010\u01c5\u0001\u0000\u0000\u0000\u0010\u01c7\u0001\u0000"+ - "\u0000\u0000\u0010\u01c9\u0001\u0000\u0000\u0000\u0010\u01cb\u0001\u0000"+ - "\u0000\u0000\u0010\u01cd\u0001\u0000\u0000\u0000\u0010\u01cf\u0001\u0000"+ - "\u0000\u0000\u0010\u01d1\u0001\u0000\u0000\u0000\u0010\u01d3\u0001\u0000"+ - "\u0000\u0000\u0010\u01d5\u0001\u0000\u0000\u0000\u0010\u01d7\u0001\u0000"+ - "\u0000\u0000\u0011\u01d9\u0001\u0000\u0000\u0000\u0013\u01e3\u0001\u0000"+ - "\u0000\u0000\u0015\u01ea\u0001\u0000\u0000\u0000\u0017\u01f3\u0001\u0000"+ - "\u0000\u0000\u0019\u01fa\u0001\u0000\u0000\u0000\u001b\u0204\u0001\u0000"+ - "\u0000\u0000\u001d\u020b\u0001\u0000\u0000\u0000\u001f\u0212\u0001\u0000"+ - "\u0000\u0000!\u0219\u0001\u0000\u0000\u0000#\u0221\u0001\u0000\u0000\u0000"+ - "%\u022d\u0001\u0000\u0000\u0000\'\u0236\u0001\u0000\u0000\u0000)\u023c"+ - "\u0001\u0000\u0000\u0000+\u0243\u0001\u0000\u0000\u0000-\u024a\u0001\u0000"+ - "\u0000\u0000/\u0252\u0001\u0000\u0000\u00001\u025a\u0001\u0000\u0000\u0000"+ - "3\u0263\u0001\u0000\u0000\u00005\u0273\u0001\u0000\u0000\u00007\u0282"+ - "\u0001\u0000\u0000\u00009\u028e\u0001\u0000\u0000\u0000;\u0299\u0001\u0000"+ - "\u0000\u0000=\u02a1\u0001\u0000\u0000\u0000?\u02a9\u0001\u0000\u0000\u0000"+ - "A\u02b3\u0001\u0000\u0000\u0000C\u02b9\u0001\u0000\u0000\u0000E\u02ca"+ - "\u0001\u0000\u0000\u0000G\u02da\u0001\u0000\u0000\u0000I\u02e0\u0001\u0000"+ - "\u0000\u0000K\u02e4\u0001\u0000\u0000\u0000M\u02e6\u0001\u0000\u0000\u0000"+ - "O\u02e8\u0001\u0000\u0000\u0000Q\u02eb\u0001\u0000\u0000\u0000S\u02ed"+ - "\u0001\u0000\u0000\u0000U\u02f6\u0001\u0000\u0000\u0000W\u02f8\u0001\u0000"+ - "\u0000\u0000Y\u02fd\u0001\u0000\u0000\u0000[\u02ff\u0001\u0000\u0000\u0000"+ - "]\u0304\u0001\u0000\u0000\u0000_\u0323\u0001\u0000\u0000\u0000a\u0326"+ - "\u0001\u0000\u0000\u0000c\u0354\u0001\u0000\u0000\u0000e\u0356\u0001\u0000"+ - "\u0000\u0000g\u0359\u0001\u0000\u0000\u0000i\u035d\u0001\u0000\u0000\u0000"+ - "k\u0361\u0001\u0000\u0000\u0000m\u0363\u0001\u0000\u0000\u0000o\u0366"+ - "\u0001\u0000\u0000\u0000q\u0368\u0001\u0000\u0000\u0000s\u036a\u0001\u0000"+ - "\u0000\u0000u\u036f\u0001\u0000\u0000\u0000w\u0371\u0001\u0000\u0000\u0000"+ - "y\u0377\u0001\u0000\u0000\u0000{\u037d\u0001\u0000\u0000\u0000}\u0380"+ - "\u0001\u0000\u0000\u0000\u007f\u0383\u0001\u0000\u0000\u0000\u0081\u0388"+ - "\u0001\u0000\u0000\u0000\u0083\u038d\u0001\u0000\u0000\u0000\u0085\u038f"+ - "\u0001\u0000\u0000\u0000\u0087\u0393\u0001\u0000\u0000\u0000\u0089\u0398"+ - "\u0001\u0000\u0000\u0000\u008b\u039e\u0001\u0000\u0000\u0000\u008d\u03a1"+ - "\u0001\u0000\u0000\u0000\u008f\u03a3\u0001\u0000\u0000\u0000\u0091\u03a9"+ - "\u0001\u0000\u0000\u0000\u0093\u03ab\u0001\u0000\u0000\u0000\u0095\u03b0"+ - "\u0001\u0000\u0000\u0000\u0097\u03b3\u0001\u0000\u0000\u0000\u0099\u03b6"+ - "\u0001\u0000\u0000\u0000\u009b\u03b9\u0001\u0000\u0000\u0000\u009d\u03bb"+ - "\u0001\u0000\u0000\u0000\u009f\u03be\u0001\u0000\u0000\u0000\u00a1\u03c0"+ - "\u0001\u0000\u0000\u0000\u00a3\u03c3\u0001\u0000\u0000\u0000\u00a5\u03c5"+ - "\u0001\u0000\u0000\u0000\u00a7\u03c7\u0001\u0000\u0000\u0000\u00a9\u03c9"+ - "\u0001\u0000\u0000\u0000\u00ab\u03cb\u0001\u0000\u0000\u0000\u00ad\u03cd"+ - "\u0001\u0000\u0000\u0000\u00af\u03cf\u0001\u0000\u0000\u0000\u00b1\u03d1"+ - "\u0001\u0000\u0000\u0000\u00b3\u03e6\u0001\u0000\u0000\u0000\u00b5\u03e8"+ - "\u0001\u0000\u0000\u0000\u00b7\u03ed\u0001\u0000\u0000\u0000\u00b9\u0402"+ - "\u0001\u0000\u0000\u0000\u00bb\u0404\u0001\u0000\u0000\u0000\u00bd\u040c"+ - "\u0001\u0000\u0000\u0000\u00bf\u040e\u0001\u0000\u0000\u0000\u00c1\u0412"+ - "\u0001\u0000\u0000\u0000\u00c3\u0416\u0001\u0000\u0000\u0000\u00c5\u041a"+ - "\u0001\u0000\u0000\u0000\u00c7\u041f\u0001\u0000\u0000\u0000\u00c9\u0424"+ - "\u0001\u0000\u0000\u0000\u00cb\u0428\u0001\u0000\u0000\u0000\u00cd\u042c"+ - "\u0001\u0000\u0000\u0000\u00cf\u0430\u0001\u0000\u0000\u0000\u00d1\u0435"+ - "\u0001\u0000\u0000\u0000\u00d3\u0439\u0001\u0000\u0000\u0000\u00d5\u043d"+ - "\u0001\u0000\u0000\u0000\u00d7\u0441\u0001\u0000\u0000\u0000\u00d9\u0445"+ - "\u0001\u0000\u0000\u0000\u00db\u0449\u0001\u0000\u0000\u0000\u00dd\u0455"+ - "\u0001\u0000\u0000\u0000\u00df\u0458\u0001\u0000\u0000\u0000\u00e1\u045c"+ - "\u0001\u0000\u0000\u0000\u00e3\u0460\u0001\u0000\u0000\u0000\u00e5\u0464"+ - "\u0001\u0000\u0000\u0000\u00e7\u0468\u0001\u0000\u0000\u0000\u00e9\u046c"+ - "\u0001\u0000\u0000\u0000\u00eb\u0470\u0001\u0000\u0000\u0000\u00ed\u0475"+ - "\u0001\u0000\u0000\u0000\u00ef\u0479\u0001\u0000\u0000\u0000\u00f1\u047d"+ - "\u0001\u0000\u0000\u0000\u00f3\u0481\u0001\u0000\u0000\u0000\u00f5\u0489"+ - "\u0001\u0000\u0000\u0000\u00f7\u049e\u0001\u0000\u0000\u0000\u00f9\u04a2"+ - "\u0001\u0000\u0000\u0000\u00fb\u04a6\u0001\u0000\u0000\u0000\u00fd\u04aa"+ - "\u0001\u0000\u0000\u0000\u00ff\u04ae\u0001\u0000\u0000\u0000\u0101\u04b2"+ - "\u0001\u0000\u0000\u0000\u0103\u04b7\u0001\u0000\u0000\u0000\u0105\u04bb"+ - "\u0001\u0000\u0000\u0000\u0107\u04bf\u0001\u0000\u0000\u0000\u0109\u04c3"+ - "\u0001\u0000\u0000\u0000\u010b\u04c7\u0001\u0000\u0000\u0000\u010d\u04cb"+ - "\u0001\u0000\u0000\u0000\u010f\u04ce\u0001\u0000\u0000\u0000\u0111\u04d2"+ - "\u0001\u0000\u0000\u0000\u0113\u04d6\u0001\u0000\u0000\u0000\u0115\u04da"+ - "\u0001\u0000\u0000\u0000\u0117\u04de\u0001\u0000\u0000\u0000\u0119\u04e3"+ - "\u0001\u0000\u0000\u0000\u011b\u04e8\u0001\u0000\u0000\u0000\u011d\u04ed"+ - "\u0001\u0000\u0000\u0000\u011f\u04f4\u0001\u0000\u0000\u0000\u0121\u04fd"+ - "\u0001\u0000\u0000\u0000\u0123\u0504\u0001\u0000\u0000\u0000\u0125\u0508"+ - "\u0001\u0000\u0000\u0000\u0127\u050c\u0001\u0000\u0000\u0000\u0129\u0510"+ - "\u0001\u0000\u0000\u0000\u012b\u0514\u0001\u0000\u0000\u0000\u012d\u051a"+ - "\u0001\u0000\u0000\u0000\u012f\u051e\u0001\u0000\u0000\u0000\u0131\u0522"+ - "\u0001\u0000\u0000\u0000\u0133\u0526\u0001\u0000\u0000\u0000\u0135\u052a"+ - "\u0001\u0000\u0000\u0000\u0137\u052e\u0001\u0000\u0000\u0000\u0139\u0532"+ - "\u0001\u0000\u0000\u0000\u013b\u0536\u0001\u0000\u0000\u0000\u013d\u053a"+ - "\u0001\u0000\u0000\u0000\u013f\u053e\u0001\u0000\u0000\u0000\u0141\u0542"+ - "\u0001\u0000\u0000\u0000\u0143\u0546\u0001\u0000\u0000\u0000\u0145\u054b"+ - "\u0001\u0000\u0000\u0000\u0147\u054f\u0001\u0000\u0000\u0000\u0149\u0553"+ - "\u0001\u0000\u0000\u0000\u014b\u0557\u0001\u0000\u0000\u0000\u014d\u055b"+ - "\u0001\u0000\u0000\u0000\u014f\u055f\u0001\u0000\u0000\u0000\u0151\u0563"+ - "\u0001\u0000\u0000\u0000\u0153\u0567\u0001\u0000\u0000\u0000\u0155\u056b"+ - "\u0001\u0000\u0000\u0000\u0157\u0570\u0001\u0000\u0000\u0000\u0159\u0575"+ - "\u0001\u0000\u0000\u0000\u015b\u0579\u0001\u0000\u0000\u0000\u015d\u057d"+ - "\u0001\u0000\u0000\u0000\u015f\u0581\u0001\u0000\u0000\u0000\u0161\u0586"+ - "\u0001\u0000\u0000\u0000\u0163\u058f\u0001\u0000\u0000\u0000\u0165\u0593"+ - "\u0001\u0000\u0000\u0000\u0167\u0597\u0001\u0000\u0000\u0000\u0169\u059b"+ - "\u0001\u0000\u0000\u0000\u016b\u059f\u0001\u0000\u0000\u0000\u016d\u05a4"+ - "\u0001\u0000\u0000\u0000\u016f\u05a8\u0001\u0000\u0000\u0000\u0171\u05ac"+ - "\u0001\u0000\u0000\u0000\u0173\u05b0\u0001\u0000\u0000\u0000\u0175\u05b5"+ - "\u0001\u0000\u0000\u0000\u0177\u05b9\u0001\u0000\u0000\u0000\u0179\u05bd"+ - "\u0001\u0000\u0000\u0000\u017b\u05c1\u0001\u0000\u0000\u0000\u017d\u05c5"+ - "\u0001\u0000\u0000\u0000\u017f\u05c9\u0001\u0000\u0000\u0000\u0181\u05cf"+ - "\u0001\u0000\u0000\u0000\u0183\u05d3\u0001\u0000\u0000\u0000\u0185\u05d7"+ - "\u0001\u0000\u0000\u0000\u0187\u05db\u0001\u0000\u0000\u0000\u0189\u05df"+ - "\u0001\u0000\u0000\u0000\u018b\u05e3\u0001\u0000\u0000\u0000\u018d\u05e7"+ - "\u0001\u0000\u0000\u0000\u018f\u05ec\u0001\u0000\u0000\u0000\u0191\u05f1"+ - "\u0001\u0000\u0000\u0000\u0193\u05f5\u0001\u0000\u0000\u0000\u0195\u05fb"+ - "\u0001\u0000\u0000\u0000\u0197\u0604\u0001\u0000\u0000\u0000\u0199\u0608"+ - "\u0001\u0000\u0000\u0000\u019b\u060c\u0001\u0000\u0000\u0000\u019d\u0610"+ - "\u0001\u0000\u0000\u0000\u019f\u0614\u0001\u0000\u0000\u0000\u01a1\u0618"+ - "\u0001\u0000\u0000\u0000\u01a3\u061c\u0001\u0000\u0000\u0000\u01a5\u0620"+ - "\u0001\u0000\u0000\u0000\u01a7\u0624\u0001\u0000\u0000\u0000\u01a9\u0629"+ - "\u0001\u0000\u0000\u0000\u01ab\u062f\u0001\u0000\u0000\u0000\u01ad\u0635"+ - "\u0001\u0000\u0000\u0000\u01af\u0639\u0001\u0000\u0000\u0000\u01b1\u063d"+ - "\u0001\u0000\u0000\u0000\u01b3\u0641\u0001\u0000\u0000\u0000\u01b5\u0647"+ - "\u0001\u0000\u0000\u0000\u01b7\u064d\u0001\u0000\u0000\u0000\u01b9\u0651"+ - "\u0001\u0000\u0000\u0000\u01bb\u0655\u0001\u0000\u0000\u0000\u01bd\u0659"+ - "\u0001\u0000\u0000\u0000\u01bf\u065f\u0001\u0000\u0000\u0000\u01c1\u0665"+ - "\u0001\u0000\u0000\u0000\u01c3\u066b\u0001\u0000\u0000\u0000\u01c5\u0670"+ - "\u0001\u0000\u0000\u0000\u01c7\u0675\u0001\u0000\u0000\u0000\u01c9\u0679"+ - "\u0001\u0000\u0000\u0000\u01cb\u067d\u0001\u0000\u0000\u0000\u01cd\u0681"+ - "\u0001\u0000\u0000\u0000\u01cf\u0685\u0001\u0000\u0000\u0000\u01d1\u0689"+ - "\u0001\u0000\u0000\u0000\u01d3\u068d\u0001\u0000\u0000\u0000\u01d5\u0691"+ - "\u0001\u0000\u0000\u0000\u01d7\u0695\u0001\u0000\u0000\u0000\u01d9\u01da"+ - "\u0007\u0000\u0000\u0000\u01da\u01db\u0007\u0001\u0000\u0000\u01db\u01dc"+ - "\u0007\u0002\u0000\u0000\u01dc\u01dd\u0007\u0002\u0000\u0000\u01dd\u01de"+ - "\u0007\u0003\u0000\u0000\u01de\u01df\u0007\u0004\u0000\u0000\u01df\u01e0"+ - "\u0007\u0005\u0000\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e1\u01e2"+ - "\u0006\u0000\u0000\u0000\u01e2\u0012\u0001\u0000\u0000\u0000\u01e3\u01e4"+ - "\u0007\u0000\u0000\u0000\u01e4\u01e5\u0007\u0006\u0000\u0000\u01e5\u01e6"+ - "\u0007\u0007\u0000\u0000\u01e6\u01e7\u0007\b\u0000\u0000\u01e7\u01e8\u0001"+ - "\u0000\u0000\u0000\u01e8\u01e9\u0006\u0001\u0001\u0000\u01e9\u0014\u0001"+ - "\u0000\u0000\u0000\u01ea\u01eb\u0007\u0003\u0000\u0000\u01eb\u01ec\u0007"+ - "\t\u0000\u0000\u01ec\u01ed\u0007\u0006\u0000\u0000\u01ed\u01ee\u0007\u0001"+ - "\u0000\u0000\u01ee\u01ef\u0007\u0004\u0000\u0000\u01ef\u01f0\u0007\n\u0000"+ - "\u0000\u01f0\u01f1\u0001\u0000\u0000\u0000\u01f1\u01f2\u0006\u0002\u0002"+ - "\u0000\u01f2\u0016\u0001\u0000\u0000\u0000\u01f3\u01f4\u0007\u0003\u0000"+ - "\u0000\u01f4\u01f5\u0007\u000b\u0000\u0000\u01f5\u01f6\u0007\f\u0000\u0000"+ - "\u01f6\u01f7\u0007\r\u0000\u0000\u01f7\u01f8\u0001\u0000\u0000\u0000\u01f8"+ - "\u01f9\u0006\u0003\u0000\u0000\u01f9\u0018\u0001\u0000\u0000\u0000\u01fa"+ - "\u01fb\u0007\u0003\u0000\u0000\u01fb\u01fc\u0007\u000e\u0000\u0000\u01fc"+ - "\u01fd\u0007\b\u0000\u0000\u01fd\u01fe\u0007\r\u0000\u0000\u01fe\u01ff"+ - "\u0007\f\u0000\u0000\u01ff\u0200\u0007\u0001\u0000\u0000\u0200\u0201\u0007"+ - "\t\u0000\u0000\u0201\u0202\u0001\u0000\u0000\u0000\u0202\u0203\u0006\u0004"+ - "\u0003\u0000\u0203\u001a\u0001\u0000\u0000\u0000\u0204\u0205\u0007\u000f"+ - "\u0000\u0000\u0205\u0206\u0007\u0006\u0000\u0000\u0206\u0207\u0007\u0007"+ - "\u0000\u0000\u0207\u0208\u0007\u0010\u0000\u0000\u0208\u0209\u0001\u0000"+ - "\u0000\u0000\u0209\u020a\u0006\u0005\u0004\u0000\u020a\u001c\u0001\u0000"+ - "\u0000\u0000\u020b\u020c\u0007\u0011\u0000\u0000\u020c\u020d\u0007\u0006"+ - "\u0000\u0000\u020d\u020e\u0007\u0007\u0000\u0000\u020e\u020f\u0007\u0012"+ - "\u0000\u0000\u020f\u0210\u0001\u0000\u0000\u0000\u0210\u0211\u0006\u0006"+ - "\u0000\u0000\u0211\u001e\u0001\u0000\u0000\u0000\u0212\u0213\u0007\u0012"+ - "\u0000\u0000\u0213\u0214\u0007\u0003\u0000\u0000\u0214\u0215\u0007\u0003"+ - "\u0000\u0000\u0215\u0216\u0007\b\u0000\u0000\u0216\u0217\u0001\u0000\u0000"+ - "\u0000\u0217\u0218\u0006\u0007\u0001\u0000\u0218 \u0001\u0000\u0000\u0000"+ - "\u0219\u021a\u0007\r\u0000\u0000\u021a\u021b\u0007\u0001\u0000\u0000\u021b"+ - "\u021c\u0007\u0010\u0000\u0000\u021c\u021d\u0007\u0001\u0000\u0000\u021d"+ - "\u021e\u0007\u0005\u0000\u0000\u021e\u021f\u0001\u0000\u0000\u0000\u021f"+ - "\u0220\u0006\b\u0000\u0000\u0220\"\u0001\u0000\u0000\u0000\u0221\u0222"+ - "\u0007\u0010\u0000\u0000\u0222\u0223\u0007\u000b\u0000\u0000\u0223\u0224"+ - "\u0005_\u0000\u0000\u0224\u0225\u0007\u0003\u0000\u0000\u0225\u0226\u0007"+ - "\u000e\u0000\u0000\u0226\u0227\u0007\b\u0000\u0000\u0227\u0228\u0007\f"+ - "\u0000\u0000\u0228\u0229\u0007\t\u0000\u0000\u0229\u022a\u0007\u0000\u0000"+ - "\u0000\u022a\u022b\u0001\u0000\u0000\u0000\u022b\u022c\u0006\t\u0005\u0000"+ - "\u022c$\u0001\u0000\u0000\u0000\u022d\u022e\u0007\u0006\u0000\u0000\u022e"+ - "\u022f\u0007\u0003\u0000\u0000\u022f\u0230\u0007\t\u0000\u0000\u0230\u0231"+ - "\u0007\f\u0000\u0000\u0231\u0232\u0007\u0010\u0000\u0000\u0232\u0233\u0007"+ - "\u0003\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000\u0234\u0235\u0006"+ - "\n\u0006\u0000\u0235&\u0001\u0000\u0000\u0000\u0236\u0237\u0007\u0006"+ - "\u0000\u0000\u0237\u0238\u0007\u0007\u0000\u0000\u0238\u0239\u0007\u0013"+ - "\u0000\u0000\u0239\u023a\u0001\u0000\u0000\u0000\u023a\u023b\u0006\u000b"+ - "\u0000\u0000\u023b(\u0001\u0000\u0000\u0000\u023c\u023d\u0007\u0002\u0000"+ - "\u0000\u023d\u023e\u0007\n\u0000\u0000\u023e\u023f\u0007\u0007\u0000\u0000"+ - "\u023f\u0240\u0007\u0013\u0000\u0000\u0240\u0241\u0001\u0000\u0000\u0000"+ - "\u0241\u0242\u0006\f\u0007\u0000\u0242*\u0001\u0000\u0000\u0000\u0243"+ - "\u0244\u0007\u0002\u0000\u0000\u0244\u0245\u0007\u0007\u0000\u0000\u0245"+ - "\u0246\u0007\u0006\u0000\u0000\u0246\u0247\u0007\u0005\u0000\u0000\u0247"+ - "\u0248\u0001\u0000\u0000\u0000\u0248\u0249\u0006\r\u0000\u0000\u0249,"+ - "\u0001\u0000\u0000\u0000\u024a\u024b\u0007\u0002\u0000\u0000\u024b\u024c"+ - "\u0007\u0005\u0000\u0000\u024c\u024d\u0007\f\u0000\u0000\u024d\u024e\u0007"+ - "\u0005\u0000\u0000\u024e\u024f\u0007\u0002\u0000\u0000\u024f\u0250\u0001"+ - "\u0000\u0000\u0000\u0250\u0251\u0006\u000e\u0000\u0000\u0251.\u0001\u0000"+ - "\u0000\u0000\u0252\u0253\u0007\u0013\u0000\u0000\u0253\u0254\u0007\n\u0000"+ - "\u0000\u0254\u0255\u0007\u0003\u0000\u0000\u0255\u0256\u0007\u0006\u0000"+ - "\u0000\u0256\u0257\u0007\u0003\u0000\u0000\u0257\u0258\u0001\u0000\u0000"+ - "\u0000\u0258\u0259\u0006\u000f\u0000\u0000\u02590\u0001\u0000\u0000\u0000"+ - "\u025a\u025b\u0007\r\u0000\u0000\u025b\u025c\u0007\u0007\u0000\u0000\u025c"+ - "\u025d\u0007\u0007\u0000\u0000\u025d\u025e\u0007\u0012\u0000\u0000\u025e"+ - "\u025f\u0007\u0014\u0000\u0000\u025f\u0260\u0007\b\u0000\u0000\u0260\u0261"+ - "\u0001\u0000\u0000\u0000\u0261\u0262\u0006\u0010\b\u0000\u02622\u0001"+ - "\u0000\u0000\u0000\u0263\u0264\u0004\u0011\u0000\u0000\u0264\u0265\u0007"+ - "\u0004\u0000\u0000\u0265\u0266\u0007\n\u0000\u0000\u0266\u0267\u0007\f"+ - "\u0000\u0000\u0267\u0268\u0007\t\u0000\u0000\u0268\u0269\u0007\u0011\u0000"+ - "\u0000\u0269\u026a\u0007\u0003\u0000\u0000\u026a\u026b\u0005_\u0000\u0000"+ - "\u026b\u026c\u0007\b\u0000\u0000\u026c\u026d\u0007\u0007\u0000\u0000\u026d"+ - "\u026e\u0007\u0001\u0000\u0000\u026e\u026f\u0007\t\u0000\u0000\u026f\u0270"+ - "\u0007\u0005\u0000\u0000\u0270\u0271\u0001\u0000\u0000\u0000\u0271\u0272"+ - "\u0006\u0011\t\u0000\u02724\u0001\u0000\u0000\u0000\u0273\u0274\u0004"+ - "\u0012\u0001\u0000\u0274\u0275\u0007\u0001\u0000\u0000\u0275\u0276\u0007"+ - "\t\u0000\u0000\u0276\u0277\u0007\r\u0000\u0000\u0277\u0278\u0007\u0001"+ - "\u0000\u0000\u0278\u0279\u0007\t\u0000\u0000\u0279\u027a\u0007\u0003\u0000"+ - "\u0000\u027a\u027b\u0007\u0002\u0000\u0000\u027b\u027c\u0007\u0005\u0000"+ - "\u0000\u027c\u027d\u0007\f\u0000\u0000\u027d\u027e\u0007\u0005\u0000\u0000"+ - "\u027e\u027f\u0007\u0002\u0000\u0000\u027f\u0280\u0001\u0000\u0000\u0000"+ - "\u0280\u0281\u0006\u0012\u0000\u0000\u02816\u0001\u0000\u0000\u0000\u0282"+ - "\u0283\u0004\u0013\u0002\u0000\u0283\u0284\u0007\r\u0000\u0000\u0284\u0285"+ - "\u0007\u0007\u0000\u0000\u0285\u0286\u0007\u0007\u0000\u0000\u0286\u0287"+ - "\u0007\u0012\u0000\u0000\u0287\u0288\u0007\u0014\u0000\u0000\u0288\u0289"+ - "\u0007\b\u0000\u0000\u0289\u028a\u0005_\u0000\u0000\u028a\u028b\u0005"+ - "\u8001\uf414\u0000\u0000\u028b\u028c\u0001\u0000\u0000\u0000\u028c\u028d"+ - "\u0006\u0013\n\u0000\u028d8\u0001\u0000\u0000\u0000\u028e\u028f\u0004"+ - "\u0014\u0003\u0000\u028f\u0290\u0007\u0010\u0000\u0000\u0290\u0291\u0007"+ - "\u0003\u0000\u0000\u0291\u0292\u0007\u0005\u0000\u0000\u0292\u0293\u0007"+ - "\u0006\u0000\u0000\u0293\u0294\u0007\u0001\u0000\u0000\u0294\u0295\u0007"+ - "\u0004\u0000\u0000\u0295\u0296\u0007\u0002\u0000\u0000\u0296\u0297\u0001"+ - "\u0000\u0000\u0000\u0297\u0298\u0006\u0014\u000b\u0000\u0298:\u0001\u0000"+ - "\u0000\u0000\u0299\u029a\u0004\u0015\u0004\u0000\u029a\u029b\u0007\u000f"+ - "\u0000\u0000\u029b\u029c\u0007\u0014\u0000\u0000\u029c\u029d\u0007\r\u0000"+ - "\u0000\u029d\u029e\u0007\r\u0000\u0000\u029e\u029f\u0001\u0000\u0000\u0000"+ - "\u029f\u02a0\u0006\u0015\b\u0000\u02a0<\u0001\u0000\u0000\u0000\u02a1"+ - "\u02a2\u0004\u0016\u0005\u0000\u02a2\u02a3\u0007\r\u0000\u0000\u02a3\u02a4"+ - "\u0007\u0003\u0000\u0000\u02a4\u02a5\u0007\u000f\u0000\u0000\u02a5\u02a6"+ - "\u0007\u0005\u0000\u0000\u02a6\u02a7\u0001\u0000\u0000\u0000\u02a7\u02a8"+ - "\u0006\u0016\b\u0000\u02a8>\u0001\u0000\u0000\u0000\u02a9\u02aa\u0004"+ - "\u0017\u0006\u0000\u02aa\u02ab\u0007\u0006\u0000\u0000\u02ab\u02ac\u0007"+ - "\u0001\u0000\u0000\u02ac\u02ad\u0007\u0011\u0000\u0000\u02ad\u02ae\u0007"+ - "\n\u0000\u0000\u02ae\u02af\u0007\u0005\u0000\u0000\u02af\u02b0\u0001\u0000"+ - "\u0000\u0000\u02b0\u02b1\u0006\u0017\b\u0000\u02b1@\u0001\u0000\u0000"+ - "\u0000\u02b2\u02b4\b\u0015\u0000\u0000\u02b3\u02b2\u0001\u0000\u0000\u0000"+ - "\u02b4\u02b5\u0001\u0000\u0000\u0000\u02b5\u02b3\u0001\u0000\u0000\u0000"+ - "\u02b5\u02b6\u0001\u0000\u0000\u0000\u02b6\u02b7\u0001\u0000\u0000\u0000"+ - "\u02b7\u02b8\u0006\u0018\u0000\u0000\u02b8B\u0001\u0000\u0000\u0000\u02b9"+ - "\u02ba\u0005/\u0000\u0000\u02ba\u02bb\u0005/\u0000\u0000\u02bb\u02bf\u0001"+ - "\u0000\u0000\u0000\u02bc\u02be\b\u0016\u0000\u0000\u02bd\u02bc\u0001\u0000"+ - "\u0000\u0000\u02be\u02c1\u0001\u0000\u0000\u0000\u02bf\u02bd\u0001\u0000"+ - "\u0000\u0000\u02bf\u02c0\u0001\u0000\u0000\u0000\u02c0\u02c3\u0001\u0000"+ - "\u0000\u0000\u02c1\u02bf\u0001\u0000\u0000\u0000\u02c2\u02c4\u0005\r\u0000"+ - "\u0000\u02c3\u02c2\u0001\u0000\u0000\u0000\u02c3\u02c4\u0001\u0000\u0000"+ - "\u0000\u02c4\u02c6\u0001\u0000\u0000\u0000\u02c5\u02c7\u0005\n\u0000\u0000"+ - "\u02c6\u02c5\u0001\u0000\u0000\u0000\u02c6\u02c7\u0001\u0000\u0000\u0000"+ - "\u02c7\u02c8\u0001\u0000\u0000\u0000\u02c8\u02c9\u0006\u0019\f\u0000\u02c9"+ - "D\u0001\u0000\u0000\u0000\u02ca\u02cb\u0005/\u0000\u0000\u02cb\u02cc\u0005"+ - "*\u0000\u0000\u02cc\u02d1\u0001\u0000\u0000\u0000\u02cd\u02d0\u0003E\u001a"+ - "\u0000\u02ce\u02d0\t\u0000\u0000\u0000\u02cf\u02cd\u0001\u0000\u0000\u0000"+ - "\u02cf\u02ce\u0001\u0000\u0000\u0000\u02d0\u02d3\u0001\u0000\u0000\u0000"+ - "\u02d1\u02d2\u0001\u0000\u0000\u0000\u02d1\u02cf\u0001\u0000\u0000\u0000"+ - "\u02d2\u02d4\u0001\u0000\u0000\u0000\u02d3\u02d1\u0001\u0000\u0000\u0000"+ - "\u02d4\u02d5\u0005*\u0000\u0000\u02d5\u02d6\u0005/\u0000\u0000\u02d6\u02d7"+ - "\u0001\u0000\u0000\u0000\u02d7\u02d8\u0006\u001a\f\u0000\u02d8F\u0001"+ - "\u0000\u0000\u0000\u02d9\u02db\u0007\u0017\u0000\u0000\u02da\u02d9\u0001"+ - "\u0000\u0000\u0000\u02db\u02dc\u0001\u0000\u0000\u0000\u02dc\u02da\u0001"+ - "\u0000\u0000\u0000\u02dc\u02dd\u0001\u0000\u0000\u0000\u02dd\u02de\u0001"+ - "\u0000\u0000\u0000\u02de\u02df\u0006\u001b\f\u0000\u02dfH\u0001\u0000"+ - "\u0000\u0000\u02e0\u02e1\u0005|\u0000\u0000\u02e1\u02e2\u0001\u0000\u0000"+ - "\u0000\u02e2\u02e3\u0006\u001c\r\u0000\u02e3J\u0001\u0000\u0000\u0000"+ - "\u02e4\u02e5\u0007\u0018\u0000\u0000\u02e5L\u0001\u0000\u0000\u0000\u02e6"+ - "\u02e7\u0007\u0019\u0000\u0000\u02e7N\u0001\u0000\u0000\u0000\u02e8\u02e9"+ - "\u0005\\\u0000\u0000\u02e9\u02ea\u0007\u001a\u0000\u0000\u02eaP\u0001"+ - "\u0000\u0000\u0000\u02eb\u02ec\b\u001b\u0000\u0000\u02ecR\u0001\u0000"+ - "\u0000\u0000\u02ed\u02ef\u0007\u0003\u0000\u0000\u02ee\u02f0\u0007\u001c"+ - "\u0000\u0000\u02ef\u02ee\u0001\u0000\u0000\u0000\u02ef\u02f0\u0001\u0000"+ - "\u0000\u0000\u02f0\u02f2\u0001\u0000\u0000\u0000\u02f1\u02f3\u0003K\u001d"+ - "\u0000\u02f2\u02f1\u0001\u0000\u0000\u0000\u02f3\u02f4\u0001\u0000\u0000"+ - "\u0000\u02f4\u02f2\u0001\u0000\u0000\u0000\u02f4\u02f5\u0001\u0000\u0000"+ - "\u0000\u02f5T\u0001\u0000\u0000\u0000\u02f6\u02f7\u0005@\u0000\u0000\u02f7"+ - "V\u0001\u0000\u0000\u0000\u02f8\u02f9\u0005`\u0000\u0000\u02f9X\u0001"+ - "\u0000\u0000\u0000\u02fa\u02fe\b\u001d\u0000\u0000\u02fb\u02fc\u0005`"+ - "\u0000\u0000\u02fc\u02fe\u0005`\u0000\u0000\u02fd\u02fa\u0001\u0000\u0000"+ - "\u0000\u02fd\u02fb\u0001\u0000\u0000\u0000\u02feZ\u0001\u0000\u0000\u0000"+ - "\u02ff\u0300\u0005_\u0000\u0000\u0300\\\u0001\u0000\u0000\u0000\u0301"+ - "\u0305\u0003M\u001e\u0000\u0302\u0305\u0003K\u001d\u0000\u0303\u0305\u0003"+ - "[%\u0000\u0304\u0301\u0001\u0000\u0000\u0000\u0304\u0302\u0001\u0000\u0000"+ - "\u0000\u0304\u0303\u0001\u0000\u0000\u0000\u0305^\u0001\u0000\u0000\u0000"+ - "\u0306\u030b\u0005\"\u0000\u0000\u0307\u030a\u0003O\u001f\u0000\u0308"+ - "\u030a\u0003Q \u0000\u0309\u0307\u0001\u0000\u0000\u0000\u0309\u0308\u0001"+ - "\u0000\u0000\u0000\u030a\u030d\u0001\u0000\u0000\u0000\u030b\u0309\u0001"+ - "\u0000\u0000\u0000\u030b\u030c\u0001\u0000\u0000\u0000\u030c\u030e\u0001"+ - "\u0000\u0000\u0000\u030d\u030b\u0001\u0000\u0000\u0000\u030e\u0324\u0005"+ - "\"\u0000\u0000\u030f\u0310\u0005\"\u0000\u0000\u0310\u0311\u0005\"\u0000"+ - "\u0000\u0311\u0312\u0005\"\u0000\u0000\u0312\u0316\u0001\u0000\u0000\u0000"+ - "\u0313\u0315\b\u0016\u0000\u0000\u0314\u0313\u0001\u0000\u0000\u0000\u0315"+ - "\u0318\u0001\u0000\u0000\u0000\u0316\u0317\u0001\u0000\u0000\u0000\u0316"+ - "\u0314\u0001\u0000\u0000\u0000\u0317\u0319\u0001\u0000\u0000\u0000\u0318"+ - "\u0316\u0001\u0000\u0000\u0000\u0319\u031a\u0005\"\u0000\u0000\u031a\u031b"+ - "\u0005\"\u0000\u0000\u031b\u031c\u0005\"\u0000\u0000\u031c\u031e\u0001"+ - "\u0000\u0000\u0000\u031d\u031f\u0005\"\u0000\u0000\u031e\u031d\u0001\u0000"+ - "\u0000\u0000\u031e\u031f\u0001\u0000\u0000\u0000\u031f\u0321\u0001\u0000"+ - "\u0000\u0000\u0320\u0322\u0005\"\u0000\u0000\u0321\u0320\u0001\u0000\u0000"+ - "\u0000\u0321\u0322\u0001\u0000\u0000\u0000\u0322\u0324\u0001\u0000\u0000"+ - "\u0000\u0323\u0306\u0001\u0000\u0000\u0000\u0323\u030f\u0001\u0000\u0000"+ - "\u0000\u0324`\u0001\u0000\u0000\u0000\u0325\u0327\u0003K\u001d\u0000\u0326"+ - "\u0325\u0001\u0000\u0000\u0000\u0327\u0328\u0001\u0000\u0000\u0000\u0328"+ - "\u0326\u0001\u0000\u0000\u0000\u0328\u0329\u0001\u0000\u0000\u0000\u0329"+ - "b\u0001\u0000\u0000\u0000\u032a\u032c\u0003K\u001d\u0000\u032b\u032a\u0001"+ - "\u0000\u0000\u0000\u032c\u032d\u0001\u0000\u0000\u0000\u032d\u032b\u0001"+ - "\u0000\u0000\u0000\u032d\u032e\u0001\u0000\u0000\u0000\u032e\u032f\u0001"+ - "\u0000\u0000\u0000\u032f\u0333\u0003u2\u0000\u0330\u0332\u0003K\u001d"+ - "\u0000\u0331\u0330\u0001\u0000\u0000\u0000\u0332\u0335\u0001\u0000\u0000"+ - "\u0000\u0333\u0331\u0001\u0000\u0000\u0000\u0333\u0334\u0001\u0000\u0000"+ - "\u0000\u0334\u0355\u0001\u0000\u0000\u0000\u0335\u0333\u0001\u0000\u0000"+ - "\u0000\u0336\u0338\u0003u2\u0000\u0337\u0339\u0003K\u001d\u0000\u0338"+ - "\u0337\u0001\u0000\u0000\u0000\u0339\u033a\u0001\u0000\u0000\u0000\u033a"+ - "\u0338\u0001\u0000\u0000\u0000\u033a\u033b\u0001\u0000\u0000\u0000\u033b"+ - "\u0355\u0001\u0000\u0000\u0000\u033c\u033e\u0003K\u001d\u0000\u033d\u033c"+ - "\u0001\u0000\u0000\u0000\u033e\u033f\u0001\u0000\u0000\u0000\u033f\u033d"+ - "\u0001\u0000\u0000\u0000\u033f\u0340\u0001\u0000\u0000\u0000\u0340\u0348"+ - "\u0001\u0000\u0000\u0000\u0341\u0345\u0003u2\u0000\u0342\u0344\u0003K"+ - "\u001d\u0000\u0343\u0342\u0001\u0000\u0000\u0000\u0344\u0347\u0001\u0000"+ - "\u0000\u0000\u0345\u0343\u0001\u0000\u0000\u0000\u0345\u0346\u0001\u0000"+ - "\u0000\u0000\u0346\u0349\u0001\u0000\u0000\u0000\u0347\u0345\u0001\u0000"+ - "\u0000\u0000\u0348\u0341\u0001\u0000\u0000\u0000\u0348\u0349\u0001\u0000"+ - "\u0000\u0000\u0349\u034a\u0001\u0000\u0000\u0000\u034a\u034b\u0003S!\u0000"+ - "\u034b\u0355\u0001\u0000\u0000\u0000\u034c\u034e\u0003u2\u0000\u034d\u034f"+ - "\u0003K\u001d\u0000\u034e\u034d\u0001\u0000\u0000\u0000\u034f\u0350\u0001"+ - "\u0000\u0000\u0000\u0350\u034e\u0001\u0000\u0000\u0000\u0350\u0351\u0001"+ - "\u0000\u0000\u0000\u0351\u0352\u0001\u0000\u0000\u0000\u0352\u0353\u0003"+ - "S!\u0000\u0353\u0355\u0001\u0000\u0000\u0000\u0354\u032b\u0001\u0000\u0000"+ - "\u0000\u0354\u0336\u0001\u0000\u0000\u0000\u0354\u033d\u0001\u0000\u0000"+ - "\u0000\u0354\u034c\u0001\u0000\u0000\u0000\u0355d\u0001\u0000\u0000\u0000"+ - "\u0356\u0357\u0007\u001e\u0000\u0000\u0357\u0358\u0007\u001f\u0000\u0000"+ - "\u0358f\u0001\u0000\u0000\u0000\u0359\u035a\u0007\f\u0000\u0000\u035a"+ - "\u035b\u0007\t\u0000\u0000\u035b\u035c\u0007\u0000\u0000\u0000\u035ch"+ - "\u0001\u0000\u0000\u0000\u035d\u035e\u0007\f\u0000\u0000\u035e\u035f\u0007"+ - "\u0002\u0000\u0000\u035f\u0360\u0007\u0004\u0000\u0000\u0360j\u0001\u0000"+ - "\u0000\u0000\u0361\u0362\u0005=\u0000\u0000\u0362l\u0001\u0000\u0000\u0000"+ - "\u0363\u0364\u0005:\u0000\u0000\u0364\u0365\u0005:\u0000\u0000\u0365n"+ - "\u0001\u0000\u0000\u0000\u0366\u0367\u0005:\u0000\u0000\u0367p\u0001\u0000"+ - "\u0000\u0000\u0368\u0369\u0005,\u0000\u0000\u0369r\u0001\u0000\u0000\u0000"+ - "\u036a\u036b\u0007\u0000\u0000\u0000\u036b\u036c\u0007\u0003\u0000\u0000"+ - "\u036c\u036d\u0007\u0002\u0000\u0000\u036d\u036e\u0007\u0004\u0000\u0000"+ - "\u036et\u0001\u0000\u0000\u0000\u036f\u0370\u0005.\u0000\u0000\u0370v"+ - "\u0001\u0000\u0000\u0000\u0371\u0372\u0007\u000f\u0000\u0000\u0372\u0373"+ - "\u0007\f\u0000\u0000\u0373\u0374\u0007\r\u0000\u0000\u0374\u0375\u0007"+ - "\u0002\u0000\u0000\u0375\u0376\u0007\u0003\u0000\u0000\u0376x\u0001\u0000"+ - "\u0000\u0000\u0377\u0378\u0007\u000f\u0000\u0000\u0378\u0379\u0007\u0001"+ - "\u0000\u0000\u0379\u037a\u0007\u0006\u0000\u0000\u037a\u037b\u0007\u0002"+ - "\u0000\u0000\u037b\u037c\u0007\u0005\u0000\u0000\u037cz\u0001\u0000\u0000"+ - "\u0000\u037d\u037e\u0007\u0001\u0000\u0000\u037e\u037f\u0007\t\u0000\u0000"+ - "\u037f|\u0001\u0000\u0000\u0000\u0380\u0381\u0007\u0001\u0000\u0000\u0381"+ - "\u0382\u0007\u0002\u0000\u0000\u0382~\u0001\u0000\u0000\u0000\u0383\u0384"+ - "\u0007\r\u0000\u0000\u0384\u0385\u0007\f\u0000\u0000\u0385\u0386\u0007"+ - "\u0002\u0000\u0000\u0386\u0387\u0007\u0005\u0000\u0000\u0387\u0080\u0001"+ - "\u0000\u0000\u0000\u0388\u0389\u0007\r\u0000\u0000\u0389\u038a\u0007\u0001"+ - "\u0000\u0000\u038a\u038b\u0007\u0012\u0000\u0000\u038b\u038c\u0007\u0003"+ - "\u0000\u0000\u038c\u0082\u0001\u0000\u0000\u0000\u038d\u038e\u0005(\u0000"+ - "\u0000\u038e\u0084\u0001\u0000\u0000\u0000\u038f\u0390\u0007\t\u0000\u0000"+ - "\u0390\u0391\u0007\u0007\u0000\u0000\u0391\u0392\u0007\u0005\u0000\u0000"+ - "\u0392\u0086\u0001\u0000\u0000\u0000\u0393\u0394\u0007\t\u0000\u0000\u0394"+ - "\u0395\u0007\u0014\u0000\u0000\u0395\u0396\u0007\r\u0000\u0000\u0396\u0397"+ - "\u0007\r\u0000\u0000\u0397\u0088\u0001\u0000\u0000\u0000\u0398\u0399\u0007"+ - "\t\u0000\u0000\u0399\u039a\u0007\u0014\u0000\u0000\u039a\u039b\u0007\r"+ - "\u0000\u0000\u039b\u039c\u0007\r\u0000\u0000\u039c\u039d\u0007\u0002\u0000"+ - "\u0000\u039d\u008a\u0001\u0000\u0000\u0000\u039e\u039f\u0007\u0007\u0000"+ - "\u0000\u039f\u03a0\u0007\u0006\u0000\u0000\u03a0\u008c\u0001\u0000\u0000"+ - "\u0000\u03a1\u03a2\u0005?\u0000\u0000\u03a2\u008e\u0001\u0000\u0000\u0000"+ - "\u03a3\u03a4\u0007\u0006\u0000\u0000\u03a4\u03a5\u0007\r\u0000\u0000\u03a5"+ - "\u03a6\u0007\u0001\u0000\u0000\u03a6\u03a7\u0007\u0012\u0000\u0000\u03a7"+ - "\u03a8\u0007\u0003\u0000\u0000\u03a8\u0090\u0001\u0000\u0000\u0000\u03a9"+ - "\u03aa\u0005)\u0000\u0000\u03aa\u0092\u0001\u0000\u0000\u0000\u03ab\u03ac"+ - "\u0007\u0005\u0000\u0000\u03ac\u03ad\u0007\u0006\u0000\u0000\u03ad\u03ae"+ - "\u0007\u0014\u0000\u0000\u03ae\u03af\u0007\u0003\u0000\u0000\u03af\u0094"+ - "\u0001\u0000\u0000\u0000\u03b0\u03b1\u0005=\u0000\u0000\u03b1\u03b2\u0005"+ - "=\u0000\u0000\u03b2\u0096\u0001\u0000\u0000\u0000\u03b3\u03b4\u0005=\u0000"+ - "\u0000\u03b4\u03b5\u0005~\u0000\u0000\u03b5\u0098\u0001\u0000\u0000\u0000"+ - "\u03b6\u03b7\u0005!\u0000\u0000\u03b7\u03b8\u0005=\u0000\u0000\u03b8\u009a"+ - "\u0001\u0000\u0000\u0000\u03b9\u03ba\u0005<\u0000\u0000\u03ba\u009c\u0001"+ - "\u0000\u0000\u0000\u03bb\u03bc\u0005<\u0000\u0000\u03bc\u03bd\u0005=\u0000"+ - "\u0000\u03bd\u009e\u0001\u0000\u0000\u0000\u03be\u03bf\u0005>\u0000\u0000"+ - "\u03bf\u00a0\u0001\u0000\u0000\u0000\u03c0\u03c1\u0005>\u0000\u0000\u03c1"+ - "\u03c2\u0005=\u0000\u0000\u03c2\u00a2\u0001\u0000\u0000\u0000\u03c3\u03c4"+ - "\u0005+\u0000\u0000\u03c4\u00a4\u0001\u0000\u0000\u0000\u03c5\u03c6\u0005"+ - "-\u0000\u0000\u03c6\u00a6\u0001\u0000\u0000\u0000\u03c7\u03c8\u0005*\u0000"+ - "\u0000\u03c8\u00a8\u0001\u0000\u0000\u0000\u03c9\u03ca\u0005/\u0000\u0000"+ - "\u03ca\u00aa\u0001\u0000\u0000\u0000\u03cb\u03cc\u0005%\u0000\u0000\u03cc"+ - "\u00ac\u0001\u0000\u0000\u0000\u03cd\u03ce\u0005{\u0000\u0000\u03ce\u00ae"+ - "\u0001\u0000\u0000\u0000\u03cf\u03d0\u0005}\u0000\u0000\u03d0\u00b0\u0001"+ - "\u0000\u0000\u0000\u03d1\u03d2\u0003/\u000f\u0000\u03d2\u03d3\u0001\u0000"+ - "\u0000\u0000\u03d3\u03d4\u0006P\u000e\u0000\u03d4\u00b2\u0001\u0000\u0000"+ - "\u0000\u03d5\u03d8\u0003\u008d>\u0000\u03d6\u03d9\u0003M\u001e\u0000\u03d7"+ - "\u03d9\u0003[%\u0000\u03d8\u03d6\u0001\u0000\u0000\u0000\u03d8\u03d7\u0001"+ - "\u0000\u0000\u0000\u03d9\u03dd\u0001\u0000\u0000\u0000\u03da\u03dc\u0003"+ - "]&\u0000\u03db\u03da\u0001\u0000\u0000\u0000\u03dc\u03df\u0001\u0000\u0000"+ - "\u0000\u03dd\u03db\u0001\u0000\u0000\u0000\u03dd\u03de\u0001\u0000\u0000"+ - "\u0000\u03de\u03e7\u0001\u0000\u0000\u0000\u03df\u03dd\u0001\u0000\u0000"+ - "\u0000\u03e0\u03e2\u0003\u008d>\u0000\u03e1\u03e3\u0003K\u001d\u0000\u03e2"+ - "\u03e1\u0001\u0000\u0000\u0000\u03e3\u03e4\u0001\u0000\u0000\u0000\u03e4"+ - "\u03e2\u0001\u0000\u0000\u0000\u03e4\u03e5\u0001\u0000\u0000\u0000\u03e5"+ - "\u03e7\u0001\u0000\u0000\u0000\u03e6\u03d5\u0001\u0000\u0000\u0000\u03e6"+ - "\u03e0\u0001\u0000\u0000\u0000\u03e7\u00b4\u0001\u0000\u0000\u0000\u03e8"+ - "\u03e9\u0005[\u0000\u0000\u03e9\u03ea\u0001\u0000\u0000\u0000\u03ea\u03eb"+ - "\u0006R\u0000\u0000\u03eb\u03ec\u0006R\u0000\u0000\u03ec\u00b6\u0001\u0000"+ - "\u0000\u0000\u03ed\u03ee\u0005]\u0000\u0000\u03ee\u03ef\u0001\u0000\u0000"+ - "\u0000\u03ef\u03f0\u0006S\r\u0000\u03f0\u03f1\u0006S\r\u0000\u03f1\u00b8"+ - "\u0001\u0000\u0000\u0000\u03f2\u03f6\u0003M\u001e\u0000\u03f3\u03f5\u0003"+ - "]&\u0000\u03f4\u03f3\u0001\u0000\u0000\u0000\u03f5\u03f8\u0001\u0000\u0000"+ - "\u0000\u03f6\u03f4\u0001\u0000\u0000\u0000\u03f6\u03f7\u0001\u0000\u0000"+ - "\u0000\u03f7\u0403\u0001\u0000\u0000\u0000\u03f8\u03f6\u0001\u0000\u0000"+ - "\u0000\u03f9\u03fc\u0003[%\u0000\u03fa\u03fc\u0003U\"\u0000\u03fb\u03f9"+ - "\u0001\u0000\u0000\u0000\u03fb\u03fa\u0001\u0000\u0000\u0000\u03fc\u03fe"+ - "\u0001\u0000\u0000\u0000\u03fd\u03ff\u0003]&\u0000\u03fe\u03fd\u0001\u0000"+ - "\u0000\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400\u03fe\u0001\u0000"+ - "\u0000\u0000\u0400\u0401\u0001\u0000\u0000\u0000\u0401\u0403\u0001\u0000"+ - "\u0000\u0000\u0402\u03f2\u0001\u0000\u0000\u0000\u0402\u03fb\u0001\u0000"+ - "\u0000\u0000\u0403\u00ba\u0001\u0000\u0000\u0000\u0404\u0406\u0003W#\u0000"+ - "\u0405\u0407\u0003Y$\u0000\u0406\u0405\u0001\u0000\u0000\u0000\u0407\u0408"+ - "\u0001\u0000\u0000\u0000\u0408\u0406\u0001\u0000\u0000\u0000\u0408\u0409"+ - "\u0001\u0000\u0000\u0000\u0409\u040a\u0001\u0000\u0000\u0000\u040a\u040b"+ - "\u0003W#\u0000\u040b\u00bc\u0001\u0000\u0000\u0000\u040c\u040d\u0003\u00bb"+ - "U\u0000\u040d\u00be\u0001\u0000\u0000\u0000\u040e\u040f\u0003C\u0019\u0000"+ - "\u040f\u0410\u0001\u0000\u0000\u0000\u0410\u0411\u0006W\f\u0000\u0411"+ - "\u00c0\u0001\u0000\u0000\u0000\u0412\u0413\u0003E\u001a\u0000\u0413\u0414"+ - "\u0001\u0000\u0000\u0000\u0414\u0415\u0006X\f\u0000\u0415\u00c2\u0001"+ - "\u0000\u0000\u0000\u0416\u0417\u0003G\u001b\u0000\u0417\u0418\u0001\u0000"+ - "\u0000\u0000\u0418\u0419\u0006Y\f\u0000\u0419\u00c4\u0001\u0000\u0000"+ - "\u0000\u041a\u041b\u0003\u00b5R\u0000\u041b\u041c\u0001\u0000\u0000\u0000"+ - "\u041c\u041d\u0006Z\u000f\u0000\u041d\u041e\u0006Z\u0010\u0000\u041e\u00c6"+ - "\u0001\u0000\u0000\u0000\u041f\u0420\u0003I\u001c\u0000\u0420\u0421\u0001"+ - "\u0000\u0000\u0000\u0421\u0422\u0006[\u0011\u0000\u0422\u0423\u0006[\r"+ - "\u0000\u0423\u00c8\u0001\u0000\u0000\u0000\u0424\u0425\u0003G\u001b\u0000"+ - "\u0425\u0426\u0001\u0000\u0000\u0000\u0426\u0427\u0006\\\f\u0000\u0427"+ - "\u00ca\u0001\u0000\u0000\u0000\u0428\u0429\u0003C\u0019\u0000\u0429\u042a"+ - "\u0001\u0000\u0000\u0000\u042a\u042b\u0006]\f\u0000\u042b\u00cc\u0001"+ - "\u0000\u0000\u0000\u042c\u042d\u0003E\u001a\u0000\u042d\u042e\u0001\u0000"+ - "\u0000\u0000\u042e\u042f\u0006^\f\u0000\u042f\u00ce\u0001\u0000\u0000"+ - "\u0000\u0430\u0431\u0003I\u001c\u0000\u0431\u0432\u0001\u0000\u0000\u0000"+ - "\u0432\u0433\u0006_\u0011\u0000\u0433\u0434\u0006_\r\u0000\u0434\u00d0"+ - "\u0001\u0000\u0000\u0000\u0435\u0436\u0003\u00b5R\u0000\u0436\u0437\u0001"+ - "\u0000\u0000\u0000\u0437\u0438\u0006`\u000f\u0000\u0438\u00d2\u0001\u0000"+ - "\u0000\u0000\u0439\u043a\u0003\u00b7S\u0000\u043a\u043b\u0001\u0000\u0000"+ - "\u0000\u043b\u043c\u0006a\u0012\u0000\u043c\u00d4\u0001\u0000\u0000\u0000"+ - "\u043d\u043e\u0003o/\u0000\u043e\u043f\u0001\u0000\u0000\u0000\u043f\u0440"+ - "\u0006b\u0013\u0000\u0440\u00d6\u0001\u0000\u0000\u0000\u0441\u0442\u0003"+ - "q0\u0000\u0442\u0443\u0001\u0000\u0000\u0000\u0443\u0444\u0006c\u0014"+ - "\u0000\u0444\u00d8\u0001\u0000\u0000\u0000\u0445\u0446\u0003k-\u0000\u0446"+ - "\u0447\u0001\u0000\u0000\u0000\u0447\u0448\u0006d\u0015\u0000\u0448\u00da"+ - "\u0001\u0000\u0000\u0000\u0449\u044a\u0007\u0010\u0000\u0000\u044a\u044b"+ - "\u0007\u0003\u0000\u0000\u044b\u044c\u0007\u0005\u0000\u0000\u044c\u044d"+ - "\u0007\f\u0000\u0000\u044d\u044e\u0007\u0000\u0000\u0000\u044e\u044f\u0007"+ - "\f\u0000\u0000\u044f\u0450\u0007\u0005\u0000\u0000\u0450\u0451\u0007\f"+ - "\u0000\u0000\u0451\u00dc\u0001\u0000\u0000\u0000\u0452\u0456\b \u0000"+ - "\u0000\u0453\u0454\u0005/\u0000\u0000\u0454\u0456\b!\u0000\u0000\u0455"+ - "\u0452\u0001\u0000\u0000\u0000\u0455\u0453\u0001\u0000\u0000\u0000\u0456"+ - "\u00de\u0001\u0000\u0000\u0000\u0457\u0459\u0003\u00ddf\u0000\u0458\u0457"+ - "\u0001\u0000\u0000\u0000\u0459\u045a\u0001\u0000\u0000\u0000\u045a\u0458"+ - "\u0001\u0000\u0000\u0000\u045a\u045b\u0001\u0000\u0000\u0000\u045b\u00e0"+ - "\u0001\u0000\u0000\u0000\u045c\u045d\u0003\u00dfg\u0000\u045d\u045e\u0001"+ - "\u0000\u0000\u0000\u045e\u045f\u0006h\u0016\u0000\u045f\u00e2\u0001\u0000"+ - "\u0000\u0000\u0460\u0461\u0003_\'\u0000\u0461\u0462\u0001\u0000\u0000"+ - "\u0000\u0462\u0463\u0006i\u0017\u0000\u0463\u00e4\u0001\u0000\u0000\u0000"+ - "\u0464\u0465\u0003C\u0019\u0000\u0465\u0466\u0001\u0000\u0000\u0000\u0466"+ - "\u0467\u0006j\f\u0000\u0467\u00e6\u0001\u0000\u0000\u0000\u0468\u0469"+ - "\u0003E\u001a\u0000\u0469\u046a\u0001\u0000\u0000\u0000\u046a\u046b\u0006"+ - "k\f\u0000\u046b\u00e8\u0001\u0000\u0000\u0000\u046c\u046d\u0003G\u001b"+ - "\u0000\u046d\u046e\u0001\u0000\u0000\u0000\u046e\u046f\u0006l\f\u0000"+ - "\u046f\u00ea\u0001\u0000\u0000\u0000\u0470\u0471\u0003I\u001c\u0000\u0471"+ - "\u0472\u0001\u0000\u0000\u0000\u0472\u0473\u0006m\u0011\u0000\u0473\u0474"+ - "\u0006m\r\u0000\u0474\u00ec\u0001\u0000\u0000\u0000\u0475\u0476\u0003"+ - "u2\u0000\u0476\u0477\u0001\u0000\u0000\u0000\u0477\u0478\u0006n\u0018"+ - "\u0000\u0478\u00ee\u0001\u0000\u0000\u0000\u0479\u047a\u0003q0\u0000\u047a"+ - "\u047b\u0001\u0000\u0000\u0000\u047b\u047c\u0006o\u0014\u0000\u047c\u00f0"+ - "\u0001\u0000\u0000\u0000\u047d\u047e\u0003\u008d>\u0000\u047e\u047f\u0001"+ - "\u0000\u0000\u0000\u047f\u0480\u0006p\u0019\u0000\u0480\u00f2\u0001\u0000"+ - "\u0000\u0000\u0481\u0482\u0003\u00b3Q\u0000\u0482\u0483\u0001\u0000\u0000"+ - "\u0000\u0483\u0484\u0006q\u001a\u0000\u0484\u00f4\u0001\u0000\u0000\u0000"+ - "\u0485\u048a\u0003M\u001e\u0000\u0486\u048a\u0003K\u001d\u0000\u0487\u048a"+ - "\u0003[%\u0000\u0488\u048a\u0003\u00a7K\u0000\u0489\u0485\u0001\u0000"+ - "\u0000\u0000\u0489\u0486\u0001\u0000\u0000\u0000\u0489\u0487\u0001\u0000"+ - "\u0000\u0000\u0489\u0488\u0001\u0000\u0000\u0000\u048a\u00f6\u0001\u0000"+ - "\u0000\u0000\u048b\u048e\u0003M\u001e\u0000\u048c\u048e\u0003\u00a7K\u0000"+ - "\u048d\u048b\u0001\u0000\u0000\u0000\u048d\u048c\u0001\u0000\u0000\u0000"+ - "\u048e\u0492\u0001\u0000\u0000\u0000\u048f\u0491\u0003\u00f5r\u0000\u0490"+ - "\u048f\u0001\u0000\u0000\u0000\u0491\u0494\u0001\u0000\u0000\u0000\u0492"+ - "\u0490\u0001\u0000\u0000\u0000\u0492\u0493\u0001\u0000\u0000\u0000\u0493"+ - "\u049f\u0001\u0000\u0000\u0000\u0494\u0492\u0001\u0000\u0000\u0000\u0495"+ - "\u0498\u0003[%\u0000\u0496\u0498\u0003U\"\u0000\u0497\u0495\u0001\u0000"+ - "\u0000\u0000\u0497\u0496\u0001\u0000\u0000\u0000\u0498\u049a\u0001\u0000"+ - "\u0000\u0000\u0499\u049b\u0003\u00f5r\u0000\u049a\u0499\u0001\u0000\u0000"+ - "\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049a\u0001\u0000\u0000"+ - "\u0000\u049c\u049d\u0001\u0000\u0000\u0000\u049d\u049f\u0001\u0000\u0000"+ - "\u0000\u049e\u048d\u0001\u0000\u0000\u0000\u049e\u0497\u0001\u0000\u0000"+ - "\u0000\u049f\u00f8\u0001\u0000\u0000\u0000\u04a0\u04a3\u0003\u00f7s\u0000"+ - "\u04a1\u04a3\u0003\u00bbU\u0000\u04a2\u04a0\u0001\u0000\u0000\u0000\u04a2"+ - "\u04a1\u0001\u0000\u0000\u0000\u04a3\u04a4\u0001\u0000\u0000\u0000\u04a4"+ - "\u04a2\u0001\u0000\u0000\u0000\u04a4\u04a5\u0001\u0000\u0000\u0000\u04a5"+ - "\u00fa\u0001\u0000\u0000\u0000\u04a6\u04a7\u0003C\u0019\u0000\u04a7\u04a8"+ - "\u0001\u0000\u0000\u0000\u04a8\u04a9\u0006u\f\u0000\u04a9\u00fc\u0001"+ - "\u0000\u0000\u0000\u04aa\u04ab\u0003E\u001a\u0000\u04ab\u04ac\u0001\u0000"+ - "\u0000\u0000\u04ac\u04ad\u0006v\f\u0000\u04ad\u00fe\u0001\u0000\u0000"+ - "\u0000\u04ae\u04af\u0003G\u001b\u0000\u04af\u04b0\u0001\u0000\u0000\u0000"+ - "\u04b0\u04b1\u0006w\f\u0000\u04b1\u0100\u0001\u0000\u0000\u0000\u04b2"+ - "\u04b3\u0003I\u001c\u0000\u04b3\u04b4\u0001\u0000\u0000\u0000\u04b4\u04b5"+ - "\u0006x\u0011\u0000\u04b5\u04b6\u0006x\r\u0000\u04b6\u0102\u0001\u0000"+ - "\u0000\u0000\u04b7\u04b8\u0003k-\u0000\u04b8\u04b9\u0001\u0000\u0000\u0000"+ - "\u04b9\u04ba\u0006y\u0015\u0000\u04ba\u0104\u0001\u0000\u0000\u0000\u04bb"+ - "\u04bc\u0003q0\u0000\u04bc\u04bd\u0001\u0000\u0000\u0000\u04bd\u04be\u0006"+ - "z\u0014\u0000\u04be\u0106\u0001\u0000\u0000\u0000\u04bf\u04c0\u0003u2"+ - "\u0000\u04c0\u04c1\u0001\u0000\u0000\u0000\u04c1\u04c2\u0006{\u0018\u0000"+ - "\u04c2\u0108\u0001\u0000\u0000\u0000\u04c3\u04c4\u0003\u008d>\u0000\u04c4"+ - "\u04c5\u0001\u0000\u0000\u0000\u04c5\u04c6\u0006|\u0019\u0000\u04c6\u010a"+ - "\u0001\u0000\u0000\u0000\u04c7\u04c8\u0003\u00b3Q\u0000\u04c8\u04c9\u0001"+ - "\u0000\u0000\u0000\u04c9\u04ca\u0006}\u001a\u0000\u04ca\u010c\u0001\u0000"+ - "\u0000\u0000\u04cb\u04cc\u0007\f\u0000\u0000\u04cc\u04cd\u0007\u0002\u0000"+ - "\u0000\u04cd\u010e\u0001\u0000\u0000\u0000\u04ce\u04cf\u0003\u00f9t\u0000"+ - "\u04cf\u04d0\u0001\u0000\u0000\u0000\u04d0\u04d1\u0006\u007f\u001b\u0000"+ - "\u04d1\u0110\u0001\u0000\u0000\u0000\u04d2\u04d3\u0003C\u0019\u0000\u04d3"+ - "\u04d4\u0001\u0000\u0000\u0000\u04d4\u04d5\u0006\u0080\f\u0000\u04d5\u0112"+ - "\u0001\u0000\u0000\u0000\u04d6\u04d7\u0003E\u001a\u0000\u04d7\u04d8\u0001"+ - "\u0000\u0000\u0000\u04d8\u04d9\u0006\u0081\f\u0000\u04d9\u0114\u0001\u0000"+ - "\u0000\u0000\u04da\u04db\u0003G\u001b\u0000\u04db\u04dc\u0001\u0000\u0000"+ - "\u0000\u04dc\u04dd\u0006\u0082\f\u0000\u04dd\u0116\u0001\u0000\u0000\u0000"+ - "\u04de\u04df\u0003I\u001c\u0000\u04df\u04e0\u0001\u0000\u0000\u0000\u04e0"+ - "\u04e1\u0006\u0083\u0011\u0000\u04e1\u04e2\u0006\u0083\r\u0000\u04e2\u0118"+ - "\u0001\u0000\u0000\u0000\u04e3\u04e4\u0003\u00b5R\u0000\u04e4\u04e5\u0001"+ - "\u0000\u0000\u0000\u04e5\u04e6\u0006\u0084\u000f\u0000\u04e6\u04e7\u0006"+ - "\u0084\u001c\u0000\u04e7\u011a\u0001\u0000\u0000\u0000\u04e8\u04e9\u0007"+ - "\u0007\u0000\u0000\u04e9\u04ea\u0007\t\u0000\u0000\u04ea\u04eb\u0001\u0000"+ - "\u0000\u0000\u04eb\u04ec\u0006\u0085\u001d\u0000\u04ec\u011c\u0001\u0000"+ - "\u0000\u0000\u04ed\u04ee\u0007\u0013\u0000\u0000\u04ee\u04ef\u0007\u0001"+ - "\u0000\u0000\u04ef\u04f0\u0007\u0005\u0000\u0000\u04f0\u04f1\u0007\n\u0000"+ - "\u0000\u04f1\u04f2\u0001\u0000\u0000\u0000\u04f2\u04f3\u0006\u0086\u001d"+ - "\u0000\u04f3\u011e\u0001\u0000\u0000\u0000\u04f4\u04f5\b\"\u0000\u0000"+ - "\u04f5\u0120\u0001\u0000\u0000\u0000\u04f6\u04f8\u0003\u011f\u0087\u0000"+ - "\u04f7\u04f6\u0001\u0000\u0000\u0000\u04f8\u04f9\u0001\u0000\u0000\u0000"+ - "\u04f9\u04f7\u0001\u0000\u0000\u0000\u04f9\u04fa\u0001\u0000\u0000\u0000"+ - "\u04fa\u04fb\u0001\u0000\u0000\u0000\u04fb\u04fc\u0003o/\u0000\u04fc\u04fe"+ - "\u0001\u0000\u0000\u0000\u04fd\u04f7\u0001\u0000\u0000\u0000\u04fd\u04fe"+ - "\u0001\u0000\u0000\u0000\u04fe\u0500\u0001\u0000\u0000\u0000\u04ff\u0501"+ - "\u0003\u011f\u0087\u0000\u0500\u04ff\u0001\u0000\u0000\u0000\u0501\u0502"+ - "\u0001\u0000\u0000\u0000\u0502\u0500\u0001\u0000\u0000\u0000\u0502\u0503"+ - "\u0001\u0000\u0000\u0000\u0503\u0122\u0001\u0000\u0000\u0000\u0504\u0505"+ - "\u0003\u0121\u0088\u0000\u0505\u0506\u0001\u0000\u0000\u0000\u0506\u0507"+ - "\u0006\u0089\u001e\u0000\u0507\u0124\u0001\u0000\u0000\u0000\u0508\u0509"+ - "\u0003C\u0019\u0000\u0509\u050a\u0001\u0000\u0000\u0000\u050a\u050b\u0006"+ - "\u008a\f\u0000\u050b\u0126\u0001\u0000\u0000\u0000\u050c\u050d\u0003E"+ - "\u001a\u0000\u050d\u050e\u0001\u0000\u0000\u0000\u050e\u050f\u0006\u008b"+ - "\f\u0000\u050f\u0128\u0001\u0000\u0000\u0000\u0510\u0511\u0003G\u001b"+ - "\u0000\u0511\u0512\u0001\u0000\u0000\u0000\u0512\u0513\u0006\u008c\f\u0000"+ - "\u0513\u012a\u0001\u0000\u0000\u0000\u0514\u0515\u0003I\u001c\u0000\u0515"+ - "\u0516\u0001\u0000\u0000\u0000\u0516\u0517\u0006\u008d\u0011\u0000\u0517"+ - "\u0518\u0006\u008d\r\u0000\u0518\u0519\u0006\u008d\r\u0000\u0519\u012c"+ - "\u0001\u0000\u0000\u0000\u051a\u051b\u0003k-\u0000\u051b\u051c\u0001\u0000"+ - "\u0000\u0000\u051c\u051d\u0006\u008e\u0015\u0000\u051d\u012e\u0001\u0000"+ - "\u0000\u0000\u051e\u051f\u0003q0\u0000\u051f\u0520\u0001\u0000\u0000\u0000"+ - "\u0520\u0521\u0006\u008f\u0014\u0000\u0521\u0130\u0001\u0000\u0000\u0000"+ - "\u0522\u0523\u0003u2\u0000\u0523\u0524\u0001\u0000\u0000\u0000\u0524\u0525"+ - "\u0006\u0090\u0018\u0000\u0525\u0132\u0001\u0000\u0000\u0000\u0526\u0527"+ - "\u0003\u011d\u0086\u0000\u0527\u0528\u0001\u0000\u0000\u0000\u0528\u0529"+ - "\u0006\u0091\u001f\u0000\u0529\u0134\u0001\u0000\u0000\u0000\u052a\u052b"+ - "\u0003\u00f9t\u0000\u052b\u052c\u0001\u0000\u0000\u0000\u052c\u052d\u0006"+ - "\u0092\u001b\u0000\u052d\u0136\u0001\u0000\u0000\u0000\u052e\u052f\u0003"+ - "\u00bdV\u0000\u052f\u0530\u0001\u0000\u0000\u0000\u0530\u0531\u0006\u0093"+ - " \u0000\u0531\u0138\u0001\u0000\u0000\u0000\u0532\u0533\u0003\u008d>\u0000"+ - "\u0533\u0534\u0001\u0000\u0000\u0000\u0534\u0535\u0006\u0094\u0019\u0000"+ - "\u0535\u013a\u0001\u0000\u0000\u0000\u0536\u0537\u0003\u00b3Q\u0000\u0537"+ - "\u0538\u0001\u0000\u0000\u0000\u0538\u0539\u0006\u0095\u001a\u0000\u0539"+ - "\u013c\u0001\u0000\u0000\u0000\u053a\u053b\u0003C\u0019\u0000\u053b\u053c"+ - "\u0001\u0000\u0000\u0000\u053c\u053d\u0006\u0096\f\u0000\u053d\u013e\u0001"+ - "\u0000\u0000\u0000\u053e\u053f\u0003E\u001a\u0000\u053f\u0540\u0001\u0000"+ - "\u0000\u0000\u0540\u0541\u0006\u0097\f\u0000\u0541\u0140\u0001\u0000\u0000"+ - "\u0000\u0542\u0543\u0003G\u001b\u0000\u0543\u0544\u0001\u0000\u0000\u0000"+ - "\u0544\u0545\u0006\u0098\f\u0000\u0545\u0142\u0001\u0000\u0000\u0000\u0546"+ - "\u0547\u0003I\u001c\u0000\u0547\u0548\u0001\u0000\u0000\u0000\u0548\u0549"+ - "\u0006\u0099\u0011\u0000\u0549\u054a\u0006\u0099\r\u0000\u054a\u0144\u0001"+ - "\u0000\u0000\u0000\u054b\u054c\u0003u2\u0000\u054c\u054d\u0001\u0000\u0000"+ - "\u0000\u054d\u054e\u0006\u009a\u0018\u0000\u054e\u0146\u0001\u0000\u0000"+ - "\u0000\u054f\u0550\u0003\u008d>\u0000\u0550\u0551\u0001\u0000\u0000\u0000"+ - "\u0551\u0552\u0006\u009b\u0019\u0000\u0552\u0148\u0001\u0000\u0000\u0000"+ - "\u0553\u0554\u0003\u00b3Q\u0000\u0554\u0555\u0001\u0000\u0000\u0000\u0555"+ - "\u0556\u0006\u009c\u001a\u0000\u0556\u014a\u0001\u0000\u0000\u0000\u0557"+ - "\u0558\u0003\u00bdV\u0000\u0558\u0559\u0001\u0000\u0000\u0000\u0559\u055a"+ - "\u0006\u009d \u0000\u055a\u014c\u0001\u0000\u0000\u0000\u055b\u055c\u0003"+ - "\u00b9T\u0000\u055c\u055d\u0001\u0000\u0000\u0000\u055d\u055e\u0006\u009e"+ - "!\u0000\u055e\u014e\u0001\u0000\u0000\u0000\u055f\u0560\u0003C\u0019\u0000"+ - "\u0560\u0561\u0001\u0000\u0000\u0000\u0561\u0562\u0006\u009f\f\u0000\u0562"+ - "\u0150\u0001\u0000\u0000\u0000\u0563\u0564\u0003E\u001a\u0000\u0564\u0565"+ - "\u0001\u0000\u0000\u0000\u0565\u0566\u0006\u00a0\f\u0000\u0566\u0152\u0001"+ - "\u0000\u0000\u0000\u0567\u0568\u0003G\u001b\u0000\u0568\u0569\u0001\u0000"+ - "\u0000\u0000\u0569\u056a\u0006\u00a1\f\u0000\u056a\u0154\u0001\u0000\u0000"+ - "\u0000\u056b\u056c\u0003I\u001c\u0000\u056c\u056d\u0001\u0000\u0000\u0000"+ - "\u056d\u056e\u0006\u00a2\u0011\u0000\u056e\u056f\u0006\u00a2\r\u0000\u056f"+ - "\u0156\u0001\u0000\u0000\u0000\u0570\u0571\u0007\u0001\u0000\u0000\u0571"+ - "\u0572\u0007\t\u0000\u0000\u0572\u0573\u0007\u000f\u0000\u0000\u0573\u0574"+ - "\u0007\u0007\u0000\u0000\u0574\u0158\u0001\u0000\u0000\u0000\u0575\u0576"+ - "\u0003C\u0019\u0000\u0576\u0577\u0001\u0000\u0000\u0000\u0577\u0578\u0006"+ - "\u00a4\f\u0000\u0578\u015a\u0001\u0000\u0000\u0000\u0579\u057a\u0003E"+ - "\u001a\u0000\u057a\u057b\u0001\u0000\u0000\u0000\u057b\u057c\u0006\u00a5"+ - "\f\u0000\u057c\u015c\u0001\u0000\u0000\u0000\u057d\u057e\u0003G\u001b"+ - "\u0000\u057e\u057f\u0001\u0000\u0000\u0000\u057f\u0580\u0006\u00a6\f\u0000"+ - "\u0580\u015e\u0001\u0000\u0000\u0000\u0581\u0582\u0003\u00b7S\u0000\u0582"+ - "\u0583\u0001\u0000\u0000\u0000\u0583\u0584\u0006\u00a7\u0012\u0000\u0584"+ - "\u0585\u0006\u00a7\r\u0000\u0585\u0160\u0001\u0000\u0000\u0000\u0586\u0587"+ - "\u0003o/\u0000\u0587\u0588\u0001\u0000\u0000\u0000\u0588\u0589\u0006\u00a8"+ - "\u0013\u0000\u0589\u0162\u0001\u0000\u0000\u0000\u058a\u0590\u0003U\""+ - "\u0000\u058b\u0590\u0003K\u001d\u0000\u058c\u0590\u0003u2\u0000\u058d"+ - "\u0590\u0003M\u001e\u0000\u058e\u0590\u0003[%\u0000\u058f\u058a\u0001"+ - "\u0000\u0000\u0000\u058f\u058b\u0001\u0000\u0000\u0000\u058f\u058c\u0001"+ - "\u0000\u0000\u0000\u058f\u058d\u0001\u0000\u0000\u0000\u058f\u058e\u0001"+ - "\u0000\u0000\u0000\u0590\u0591\u0001\u0000\u0000\u0000\u0591\u058f\u0001"+ - "\u0000\u0000\u0000\u0591\u0592\u0001\u0000\u0000\u0000\u0592\u0164\u0001"+ - "\u0000\u0000\u0000\u0593\u0594\u0003C\u0019\u0000\u0594\u0595\u0001\u0000"+ - "\u0000\u0000\u0595\u0596\u0006\u00aa\f\u0000\u0596\u0166\u0001\u0000\u0000"+ - "\u0000\u0597\u0598\u0003E\u001a\u0000\u0598\u0599\u0001\u0000\u0000\u0000"+ - "\u0599\u059a\u0006\u00ab\f\u0000\u059a\u0168\u0001\u0000\u0000\u0000\u059b"+ - "\u059c\u0003G\u001b\u0000\u059c\u059d\u0001\u0000\u0000\u0000\u059d\u059e"+ - "\u0006\u00ac\f\u0000\u059e\u016a\u0001\u0000\u0000\u0000\u059f\u05a0\u0003"+ - "I\u001c\u0000\u05a0\u05a1\u0001\u0000\u0000\u0000\u05a1\u05a2\u0006\u00ad"+ - "\u0011\u0000\u05a2\u05a3\u0006\u00ad\r\u0000\u05a3\u016c\u0001\u0000\u0000"+ - "\u0000\u05a4\u05a5\u0003o/\u0000\u05a5\u05a6\u0001\u0000\u0000\u0000\u05a6"+ - "\u05a7\u0006\u00ae\u0013\u0000\u05a7\u016e\u0001\u0000\u0000\u0000\u05a8"+ - "\u05a9\u0003q0\u0000\u05a9\u05aa\u0001\u0000\u0000\u0000\u05aa\u05ab\u0006"+ - "\u00af\u0014\u0000\u05ab\u0170\u0001\u0000\u0000\u0000\u05ac\u05ad\u0003"+ - "u2\u0000\u05ad\u05ae\u0001\u0000\u0000\u0000\u05ae\u05af\u0006\u00b0\u0018"+ - "\u0000\u05af\u0172\u0001\u0000\u0000\u0000\u05b0\u05b1\u0003\u011b\u0085"+ - "\u0000\u05b1\u05b2\u0001\u0000\u0000\u0000\u05b2\u05b3\u0006\u00b1\"\u0000"+ - "\u05b3\u05b4\u0006\u00b1#\u0000\u05b4\u0174\u0001\u0000\u0000\u0000\u05b5"+ - "\u05b6\u0003\u00dfg\u0000\u05b6\u05b7\u0001\u0000\u0000\u0000\u05b7\u05b8"+ - "\u0006\u00b2\u0016\u0000\u05b8\u0176\u0001\u0000\u0000\u0000\u05b9\u05ba"+ - "\u0003_\'\u0000\u05ba\u05bb\u0001\u0000\u0000\u0000\u05bb\u05bc\u0006"+ - "\u00b3\u0017\u0000\u05bc\u0178\u0001\u0000\u0000\u0000\u05bd\u05be\u0003"+ - "C\u0019\u0000\u05be\u05bf\u0001\u0000\u0000\u0000\u05bf\u05c0\u0006\u00b4"+ - "\f\u0000\u05c0\u017a\u0001\u0000\u0000\u0000\u05c1\u05c2\u0003E\u001a"+ - "\u0000\u05c2\u05c3\u0001\u0000\u0000\u0000\u05c3\u05c4\u0006\u00b5\f\u0000"+ - "\u05c4\u017c\u0001\u0000\u0000\u0000\u05c5\u05c6\u0003G\u001b\u0000\u05c6"+ - "\u05c7\u0001\u0000\u0000\u0000\u05c7\u05c8\u0006\u00b6\f\u0000\u05c8\u017e"+ - "\u0001\u0000\u0000\u0000\u05c9\u05ca\u0003I\u001c\u0000\u05ca\u05cb\u0001"+ - "\u0000\u0000\u0000\u05cb\u05cc\u0006\u00b7\u0011\u0000\u05cc\u05cd\u0006"+ - "\u00b7\r\u0000\u05cd\u05ce\u0006\u00b7\r\u0000\u05ce\u0180\u0001\u0000"+ - "\u0000\u0000\u05cf\u05d0\u0003q0\u0000\u05d0\u05d1\u0001\u0000\u0000\u0000"+ - "\u05d1\u05d2\u0006\u00b8\u0014\u0000\u05d2\u0182\u0001\u0000\u0000\u0000"+ - "\u05d3\u05d4\u0003u2\u0000\u05d4\u05d5\u0001\u0000\u0000\u0000\u05d5\u05d6"+ - "\u0006\u00b9\u0018\u0000\u05d6\u0184\u0001\u0000\u0000\u0000\u05d7\u05d8"+ - "\u0003\u00f9t\u0000\u05d8\u05d9\u0001\u0000\u0000\u0000\u05d9\u05da\u0006"+ - "\u00ba\u001b\u0000\u05da\u0186\u0001\u0000\u0000\u0000\u05db\u05dc\u0003"+ - "C\u0019\u0000\u05dc\u05dd\u0001\u0000\u0000\u0000\u05dd\u05de\u0006\u00bb"+ - "\f\u0000\u05de\u0188\u0001\u0000\u0000\u0000\u05df\u05e0\u0003E\u001a"+ - "\u0000\u05e0\u05e1\u0001\u0000\u0000\u0000\u05e1\u05e2\u0006\u00bc\f\u0000"+ - "\u05e2\u018a\u0001\u0000\u0000\u0000\u05e3\u05e4\u0003G\u001b\u0000\u05e4"+ - "\u05e5\u0001\u0000\u0000\u0000\u05e5\u05e6\u0006\u00bd\f\u0000\u05e6\u018c"+ - "\u0001\u0000\u0000\u0000\u05e7\u05e8\u0003I\u001c\u0000\u05e8\u05e9\u0001"+ - "\u0000\u0000\u0000\u05e9\u05ea\u0006\u00be\u0011\u0000\u05ea\u05eb\u0006"+ - "\u00be\r\u0000\u05eb\u018e\u0001\u0000\u0000\u0000\u05ec\u05ed\u0007#"+ - "\u0000\u0000\u05ed\u05ee\u0007\u0007\u0000\u0000\u05ee\u05ef\u0007\u0001"+ - "\u0000\u0000\u05ef\u05f0\u0007\t\u0000\u0000\u05f0\u0190\u0001\u0000\u0000"+ - "\u0000\u05f1\u05f2\u0003\u010d~\u0000\u05f2\u05f3\u0001\u0000\u0000\u0000"+ - "\u05f3\u05f4\u0006\u00c0$\u0000\u05f4\u0192\u0001\u0000\u0000\u0000\u05f5"+ - "\u05f6\u0003\u011b\u0085\u0000\u05f6\u05f7\u0001\u0000\u0000\u0000\u05f7"+ - "\u05f8\u0006\u00c1\"\u0000\u05f8\u05f9\u0006\u00c1\r\u0000\u05f9\u05fa"+ - "\u0006\u00c1\u0000\u0000\u05fa\u0194\u0001\u0000\u0000\u0000\u05fb\u05fc"+ - "\u0007\u0014\u0000\u0000\u05fc\u05fd\u0007\u0002\u0000\u0000\u05fd\u05fe"+ - "\u0007\u0001\u0000\u0000\u05fe\u05ff\u0007\t\u0000\u0000\u05ff\u0600\u0007"+ - "\u0011\u0000\u0000\u0600\u0601\u0001\u0000\u0000\u0000\u0601\u0602\u0006"+ - "\u00c2\r\u0000\u0602\u0603\u0006\u00c2\u0000\u0000\u0603\u0196\u0001\u0000"+ - "\u0000\u0000\u0604\u0605\u0003\u00dfg\u0000\u0605\u0606\u0001\u0000\u0000"+ - "\u0000\u0606\u0607\u0006\u00c3\u0016\u0000\u0607\u0198\u0001\u0000\u0000"+ - "\u0000\u0608\u0609\u0003_\'\u0000\u0609\u060a\u0001\u0000\u0000\u0000"+ - "\u060a\u060b\u0006\u00c4\u0017\u0000\u060b\u019a\u0001\u0000\u0000\u0000"+ - "\u060c\u060d\u0003o/\u0000\u060d\u060e\u0001\u0000\u0000\u0000\u060e\u060f"+ - "\u0006\u00c5\u0013\u0000\u060f\u019c\u0001\u0000\u0000\u0000\u0610\u0611"+ - "\u0003\u00b9T\u0000\u0611\u0612\u0001\u0000\u0000\u0000\u0612\u0613\u0006"+ - "\u00c6!\u0000\u0613\u019e\u0001\u0000\u0000\u0000\u0614\u0615\u0003\u00bd"+ - "V\u0000\u0615\u0616\u0001\u0000\u0000\u0000\u0616\u0617\u0006\u00c7 \u0000"+ - "\u0617\u01a0\u0001\u0000\u0000\u0000\u0618\u0619\u0003C\u0019\u0000\u0619"+ - "\u061a\u0001\u0000\u0000\u0000\u061a\u061b\u0006\u00c8\f\u0000\u061b\u01a2"+ - "\u0001\u0000\u0000\u0000\u061c\u061d\u0003E\u001a\u0000\u061d\u061e\u0001"+ - "\u0000\u0000\u0000\u061e\u061f\u0006\u00c9\f\u0000\u061f\u01a4\u0001\u0000"+ - "\u0000\u0000\u0620\u0621\u0003G\u001b\u0000\u0621\u0622\u0001\u0000\u0000"+ - "\u0000\u0622\u0623\u0006\u00ca\f\u0000\u0623\u01a6\u0001\u0000\u0000\u0000"+ - "\u0624\u0625\u0003I\u001c\u0000\u0625\u0626\u0001\u0000\u0000\u0000\u0626"+ - "\u0627\u0006\u00cb\u0011\u0000\u0627\u0628\u0006\u00cb\r\u0000\u0628\u01a8"+ - "\u0001\u0000\u0000\u0000\u0629\u062a\u0003\u00dfg\u0000\u062a\u062b\u0001"+ - "\u0000\u0000\u0000\u062b\u062c\u0006\u00cc\u0016\u0000\u062c\u062d\u0006"+ - "\u00cc\r\u0000\u062d\u062e\u0006\u00cc%\u0000\u062e\u01aa\u0001\u0000"+ - "\u0000\u0000\u062f\u0630\u0003_\'\u0000\u0630\u0631\u0001\u0000\u0000"+ - "\u0000\u0631\u0632\u0006\u00cd\u0017\u0000\u0632\u0633\u0006\u00cd\r\u0000"+ - "\u0633\u0634\u0006\u00cd%\u0000\u0634\u01ac\u0001\u0000\u0000\u0000\u0635"+ - "\u0636\u0003C\u0019\u0000\u0636\u0637\u0001\u0000\u0000\u0000\u0637\u0638"+ - "\u0006\u00ce\f\u0000\u0638\u01ae\u0001\u0000\u0000\u0000\u0639\u063a\u0003"+ - "E\u001a\u0000\u063a\u063b\u0001\u0000\u0000\u0000\u063b\u063c\u0006\u00cf"+ - "\f\u0000\u063c\u01b0\u0001\u0000\u0000\u0000\u063d\u063e\u0003G\u001b"+ - "\u0000\u063e\u063f\u0001\u0000\u0000\u0000\u063f\u0640\u0006\u00d0\f\u0000"+ - "\u0640\u01b2\u0001\u0000\u0000\u0000\u0641\u0642\u0003o/\u0000\u0642\u0643"+ - "\u0001\u0000\u0000\u0000\u0643\u0644\u0006\u00d1\u0013\u0000\u0644\u0645"+ - "\u0006\u00d1\r\u0000\u0645\u0646\u0006\u00d1\u000b\u0000\u0646\u01b4\u0001"+ - "\u0000\u0000\u0000\u0647\u0648\u0003q0\u0000\u0648\u0649\u0001\u0000\u0000"+ - "\u0000\u0649\u064a\u0006\u00d2\u0014\u0000\u064a\u064b\u0006\u00d2\r\u0000"+ - "\u064b\u064c\u0006\u00d2\u000b\u0000\u064c\u01b6\u0001\u0000\u0000\u0000"+ - "\u064d\u064e\u0003C\u0019\u0000\u064e\u064f\u0001\u0000\u0000\u0000\u064f"+ - "\u0650\u0006\u00d3\f\u0000\u0650\u01b8\u0001\u0000\u0000\u0000\u0651\u0652"+ - "\u0003E\u001a\u0000\u0652\u0653\u0001\u0000\u0000\u0000\u0653\u0654\u0006"+ - "\u00d4\f\u0000\u0654\u01ba\u0001\u0000\u0000\u0000\u0655\u0656\u0003G"+ - "\u001b\u0000\u0656\u0657\u0001\u0000\u0000\u0000\u0657\u0658\u0006\u00d5"+ - "\f\u0000\u0658\u01bc\u0001\u0000\u0000\u0000\u0659\u065a\u0003\u00bdV"+ - "\u0000\u065a\u065b\u0001\u0000\u0000\u0000\u065b\u065c\u0006\u00d6\r\u0000"+ - "\u065c\u065d\u0006\u00d6\u0000\u0000\u065d\u065e\u0006\u00d6 \u0000\u065e"+ - "\u01be\u0001\u0000\u0000\u0000\u065f\u0660\u0003\u00b9T\u0000\u0660\u0661"+ - "\u0001\u0000\u0000\u0000\u0661\u0662\u0006\u00d7\r\u0000\u0662\u0663\u0006"+ - "\u00d7\u0000\u0000\u0663\u0664\u0006\u00d7!\u0000\u0664\u01c0\u0001\u0000"+ - "\u0000\u0000\u0665\u0666\u0003e*\u0000\u0666\u0667\u0001\u0000\u0000\u0000"+ - "\u0667\u0668\u0006\u00d8\r\u0000\u0668\u0669\u0006\u00d8\u0000\u0000\u0669"+ - "\u066a\u0006\u00d8&\u0000\u066a\u01c2\u0001\u0000\u0000\u0000\u066b\u066c"+ - "\u0003I\u001c\u0000\u066c\u066d\u0001\u0000\u0000\u0000\u066d\u066e\u0006"+ - "\u00d9\u0011\u0000\u066e\u066f\u0006\u00d9\r\u0000\u066f\u01c4\u0001\u0000"+ - "\u0000\u0000\u0670\u0671\u0003I\u001c\u0000\u0671\u0672\u0001\u0000\u0000"+ - "\u0000\u0672\u0673\u0006\u00da\u0011\u0000\u0673\u0674\u0006\u00da\r\u0000"+ - "\u0674\u01c6\u0001\u0000\u0000\u0000\u0675\u0676\u0003\u011b\u0085\u0000"+ - "\u0676\u0677\u0001\u0000\u0000\u0000\u0677\u0678\u0006\u00db\"\u0000\u0678"+ - "\u01c8\u0001\u0000\u0000\u0000\u0679\u067a\u0003\u010d~\u0000\u067a\u067b"+ - "\u0001\u0000\u0000\u0000\u067b\u067c\u0006\u00dc$\u0000\u067c\u01ca\u0001"+ - "\u0000\u0000\u0000\u067d\u067e\u0003u2\u0000\u067e\u067f\u0001\u0000\u0000"+ - "\u0000\u067f\u0680\u0006\u00dd\u0018\u0000\u0680\u01cc\u0001\u0000\u0000"+ - "\u0000\u0681\u0682\u0003q0\u0000\u0682\u0683\u0001\u0000\u0000\u0000\u0683"+ - "\u0684\u0006\u00de\u0014\u0000\u0684\u01ce\u0001\u0000\u0000\u0000\u0685"+ - "\u0686\u0003\u00bdV\u0000\u0686\u0687\u0001\u0000\u0000\u0000\u0687\u0688"+ - "\u0006\u00df \u0000\u0688\u01d0\u0001\u0000\u0000\u0000\u0689\u068a\u0003"+ - "\u00b9T\u0000\u068a\u068b\u0001\u0000\u0000\u0000\u068b\u068c\u0006\u00e0"+ - "!\u0000\u068c\u01d2\u0001\u0000\u0000\u0000\u068d\u068e\u0003C\u0019\u0000"+ - "\u068e\u068f\u0001\u0000\u0000\u0000\u068f\u0690\u0006\u00e1\f\u0000\u0690"+ - "\u01d4\u0001\u0000\u0000\u0000\u0691\u0692\u0003E\u001a\u0000\u0692\u0693"+ - "\u0001\u0000\u0000\u0000\u0693\u0694\u0006\u00e2\f\u0000\u0694\u01d6\u0001"+ - "\u0000\u0000\u0000\u0695\u0696\u0003G\u001b\u0000\u0696\u0697\u0001\u0000"+ - "\u0000\u0000\u0697\u0698\u0006\u00e3\f\u0000\u0698\u01d8\u0001\u0000\u0000"+ - "\u0000C\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f"+ - "\r\u000e\u000f\u0010\u02b5\u02bf\u02c3\u02c6\u02cf\u02d1\u02dc\u02ef\u02f4"+ - "\u02fd\u0304\u0309\u030b\u0316\u031e\u0321\u0323\u0328\u032d\u0333\u033a"+ - "\u033f\u0345\u0348\u0350\u0354\u03d8\u03dd\u03e4\u03e6\u03f6\u03fb\u0400"+ - "\u0402\u0408\u0455\u045a\u0489\u048d\u0492\u0497\u049c\u049e\u04a2\u04a4"+ - "\u04f9\u04fd\u0502\u058f\u0591\'\u0005\u0001\u0000\u0005\u0004\u0000\u0005"+ - "\u0006\u0000\u0005\u0002\u0000\u0005\u0003\u0000\u0005\b\u0000\u0005\u0005"+ - "\u0000\u0005\t\u0000\u0005\r\u0000\u0005\u0010\u0000\u0005\u000b\u0000"+ - "\u0005\u000e\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007\u0010\u0000"+ - "\u0007H\u0000\u0005\u0000\u0000\u0007\u001d\u0000\u0007I\u0000\u0007&"+ - "\u0000\u0007\'\u0000\u0007$\u0000\u0007S\u0000\u0007\u001e\u0000\u0007"+ - ")\u0000\u00075\u0000\u0007G\u0000\u0007W\u0000\u0005\n\u0000\u0005\u0007"+ - "\u0000\u0007a\u0000\u0007`\u0000\u0007K\u0000\u0007J\u0000\u0007_\u0000"+ - "\u0005\f\u0000\u0007[\u0000\u0005\u000f\u0000\u0007!\u0000"; + "\u00db\u0001\u00db\u0001\u00db\u0001\u00db\u0001\u00dc\u0001\u00dc\u0001"+ + "\u00dc\u0001\u00dc\u0001\u00dd\u0001\u00dd\u0001\u00dd\u0001\u00dd\u0001"+ + "\u00de\u0001\u00de\u0001\u00de\u0001\u00de\u0001\u00df\u0001\u00df\u0001"+ + "\u00df\u0001\u00df\u0001\u00e0\u0001\u00e0\u0001\u00e0\u0001\u00e0\u0001"+ + "\u00e1\u0001\u00e1\u0001\u00e1\u0001\u00e1\u0001\u00e2\u0001\u00e2\u0001"+ + "\u00e2\u0001\u00e2\u0001\u00e3\u0001\u00e3\u0001\u00e3\u0001\u00e3\u0001"+ + "\u00e4\u0001\u00e4\u0001\u00e4\u0001\u00e4\u0001\u00e5\u0001\u00e5\u0001"+ + "\u00e5\u0001\u00e5\u0001\u00e5\u0001\u00e6\u0001\u00e6\u0001\u00e6\u0001"+ + "\u00e6\u0001\u00e7\u0001\u00e7\u0001\u00e7\u0001\u00e7\u0001\u00e8\u0001"+ + "\u00e8\u0001\u00e8\u0001\u00e8\u0001\u00e9\u0001\u00e9\u0001\u00e9\u0001"+ + "\u00e9\u0002\u02ea\u032f\u0000\u00ea\u0012\u0001\u0014\u0002\u0016\u0003"+ + "\u0018\u0004\u001a\u0005\u001c\u0006\u001e\u0007 \b\"\t$\n&\u000b(\f*"+ + "\r,\u000e.\u000f0\u00102\u00114\u00126\u00138\u0014:\u0015<\u0016>\u0017"+ + "@\u0018B\u0019D\u001aF\u001bH\u001cJ\u001dL\u001eN\u0000P\u0000R\u0000"+ + "T\u0000V\u0000X\u0000Z\u0000\\\u0000^\u0000`\u0000b\u001fd f!h\"j#l$n"+ + "%p&r\'t(v)x*z+|,~-\u0080.\u0082/\u00840\u00861\u00882\u008a3\u008c4\u008e"+ + "5\u00906\u00927\u00948\u00969\u0098:\u009a;\u009c<\u009e=\u00a0>\u00a2"+ + "?\u00a4@\u00a6A\u00a8B\u00aaC\u00acD\u00aeE\u00b0F\u00b2G\u00b4\u0000"+ + "\u00b6H\u00b8I\u00baJ\u00bcK\u00be\u0000\u00c0L\u00c2M\u00c4N\u00c6O\u00c8"+ + "\u0000\u00ca\u0000\u00ccP\u00ceQ\u00d0R\u00d2\u0000\u00d4\u0000\u00d6"+ + "\u0000\u00d8\u0000\u00da\u0000\u00dc\u0000\u00deS\u00e0\u0000\u00e2T\u00e4"+ + "\u0000\u00e6\u0000\u00e8U\u00eaV\u00ecW\u00ee\u0000\u00f0\u0000\u00f2"+ + "\u0000\u00f4\u0000\u00f6\u0000\u00f8\u0000\u00fa\u0000\u00fcX\u00feY\u0100"+ + "Z\u0102[\u0104\u0000\u0106\u0000\u0108\u0000\u010a\u0000\u010c\u0000\u010e"+ + "\u0000\u0110\\\u0112\u0000\u0114]\u0116^\u0118_\u011a\u0000\u011c\u0000"+ + "\u011e`\u0120a\u0122\u0000\u0124b\u0126\u0000\u0128c\u012ad\u012ce\u012e"+ + "\u0000\u0130\u0000\u0132\u0000\u0134\u0000\u0136\u0000\u0138\u0000\u013a"+ + "\u0000\u013c\u0000\u013e\u0000\u0140f\u0142g\u0144h\u0146\u0000\u0148"+ + "\u0000\u014a\u0000\u014c\u0000\u014e\u0000\u0150\u0000\u0152i\u0154j\u0156"+ + "k\u0158\u0000\u015al\u015cm\u015en\u0160o\u0162\u0000\u0164\u0000\u0166"+ + "p\u0168q\u016ar\u016cs\u016e\u0000\u0170\u0000\u0172\u0000\u0174\u0000"+ + "\u0176\u0000\u0178\u0000\u017a\u0000\u017ct\u017eu\u0180v\u0182\u0000"+ + "\u0184\u0000\u0186\u0000\u0188\u0000\u018aw\u018cx\u018ey\u0190\u0000"+ + "\u0192z\u0194\u0000\u0196\u0000\u0198{\u019a\u0000\u019c\u0000\u019e\u0000"+ + "\u01a0\u0000\u01a2\u0000\u01a4|\u01a6}\u01a8~\u01aa\u0000\u01ac\u0000"+ + "\u01ae\u0000\u01b0\u007f\u01b2\u0080\u01b4\u0081\u01b6\u0000\u01b8\u0000"+ + "\u01ba\u0082\u01bc\u0083\u01be\u0084\u01c0\u0000\u01c2\u0000\u01c4\u0000"+ + "\u01c6\u0000\u01c8\u0000\u01ca\u0000\u01cc\u0000\u01ce\u0000\u01d0\u0000"+ + "\u01d2\u0000\u01d4\u0000\u01d6\u0085\u01d8\u0086\u01da\u0087\u01dc\u0000"+ + "\u01de\u0000\u01e0\u0088\u01e2\u0089\u01e4\u008a\u0012\u0000\u0001\u0002"+ + "\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011"+ + "$\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002"+ + "\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000"+ + "PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002"+ + "\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000"+ + "GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r"+ + "\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002"+ + "\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002"+ + "\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t"+ + "\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r \"#,,"+ + "//::<<>?\\\\||\u0002\u0000JJjj\u06e0\u0000\u0012\u0001\u0000\u0000\u0000"+ + "\u0000\u0014\u0001\u0000\u0000\u0000\u0000\u0016\u0001\u0000\u0000\u0000"+ + "\u0000\u0018\u0001\u0000\u0000\u0000\u0000\u001a\u0001\u0000\u0000\u0000"+ + "\u0000\u001c\u0001\u0000\u0000\u0000\u0000\u001e\u0001\u0000\u0000\u0000"+ + "\u0000 \u0001\u0000\u0000\u0000\u0000\"\u0001\u0000\u0000\u0000\u0000"+ + "$\u0001\u0000\u0000\u0000\u0000&\u0001\u0000\u0000\u0000\u0000(\u0001"+ + "\u0000\u0000\u0000\u0000*\u0001\u0000\u0000\u0000\u0000,\u0001\u0000\u0000"+ + "\u0000\u0000.\u0001\u0000\u0000\u0000\u00000\u0001\u0000\u0000\u0000\u0000"+ + "2\u0001\u0000\u0000\u0000\u00004\u0001\u0000\u0000\u0000\u00006\u0001"+ + "\u0000\u0000\u0000\u00008\u0001\u0000\u0000\u0000\u0000:\u0001\u0000\u0000"+ + "\u0000\u0000<\u0001\u0000\u0000\u0000\u0000>\u0001\u0000\u0000\u0000\u0000"+ + "@\u0001\u0000\u0000\u0000\u0000B\u0001\u0000\u0000\u0000\u0000D\u0001"+ + "\u0000\u0000\u0000\u0000F\u0001\u0000\u0000\u0000\u0000H\u0001\u0000\u0000"+ + "\u0000\u0000J\u0001\u0000\u0000\u0000\u0001L\u0001\u0000\u0000\u0000\u0001"+ + "b\u0001\u0000\u0000\u0000\u0001d\u0001\u0000\u0000\u0000\u0001f\u0001"+ + "\u0000\u0000\u0000\u0001h\u0001\u0000\u0000\u0000\u0001j\u0001\u0000\u0000"+ + "\u0000\u0001l\u0001\u0000\u0000\u0000\u0001n\u0001\u0000\u0000\u0000\u0001"+ + "p\u0001\u0000\u0000\u0000\u0001r\u0001\u0000\u0000\u0000\u0001t\u0001"+ + "\u0000\u0000\u0000\u0001v\u0001\u0000\u0000\u0000\u0001x\u0001\u0000\u0000"+ + "\u0000\u0001z\u0001\u0000\u0000\u0000\u0001|\u0001\u0000\u0000\u0000\u0001"+ + "~\u0001\u0000\u0000\u0000\u0001\u0080\u0001\u0000\u0000\u0000\u0001\u0082"+ + "\u0001\u0000\u0000\u0000\u0001\u0084\u0001\u0000\u0000\u0000\u0001\u0086"+ + "\u0001\u0000\u0000\u0000\u0001\u0088\u0001\u0000\u0000\u0000\u0001\u008a"+ + "\u0001\u0000\u0000\u0000\u0001\u008c\u0001\u0000\u0000\u0000\u0001\u008e"+ + "\u0001\u0000\u0000\u0000\u0001\u0090\u0001\u0000\u0000\u0000\u0001\u0092"+ + "\u0001\u0000\u0000\u0000\u0001\u0094\u0001\u0000\u0000\u0000\u0001\u0096"+ + "\u0001\u0000\u0000\u0000\u0001\u0098\u0001\u0000\u0000\u0000\u0001\u009a"+ + "\u0001\u0000\u0000\u0000\u0001\u009c\u0001\u0000\u0000\u0000\u0001\u009e"+ + "\u0001\u0000\u0000\u0000\u0001\u00a0\u0001\u0000\u0000\u0000\u0001\u00a2"+ + "\u0001\u0000\u0000\u0000\u0001\u00a4\u0001\u0000\u0000\u0000\u0001\u00a6"+ + "\u0001\u0000\u0000\u0000\u0001\u00a8\u0001\u0000\u0000\u0000\u0001\u00aa"+ + "\u0001\u0000\u0000\u0000\u0001\u00ac\u0001\u0000\u0000\u0000\u0001\u00ae"+ + "\u0001\u0000\u0000\u0000\u0001\u00b0\u0001\u0000\u0000\u0000\u0001\u00b2"+ + "\u0001\u0000\u0000\u0000\u0001\u00b4\u0001\u0000\u0000\u0000\u0001\u00b6"+ + "\u0001\u0000\u0000\u0000\u0001\u00b8\u0001\u0000\u0000\u0000\u0001\u00ba"+ + "\u0001\u0000\u0000\u0000\u0001\u00bc\u0001\u0000\u0000\u0000\u0001\u00c0"+ + "\u0001\u0000\u0000\u0000\u0001\u00c2\u0001\u0000\u0000\u0000\u0001\u00c4"+ + "\u0001\u0000\u0000\u0000\u0001\u00c6\u0001\u0000\u0000\u0000\u0002\u00c8"+ + "\u0001\u0000\u0000\u0000\u0002\u00ca\u0001\u0000\u0000\u0000\u0002\u00cc"+ + "\u0001\u0000\u0000\u0000\u0002\u00ce\u0001\u0000\u0000\u0000\u0002\u00d0"+ + "\u0001\u0000\u0000\u0000\u0003\u00d2\u0001\u0000\u0000\u0000\u0003\u00d4"+ + "\u0001\u0000\u0000\u0000\u0003\u00d6\u0001\u0000\u0000\u0000\u0003\u00d8"+ + "\u0001\u0000\u0000\u0000\u0003\u00da\u0001\u0000\u0000\u0000\u0003\u00dc"+ + "\u0001\u0000\u0000\u0000\u0003\u00de\u0001\u0000\u0000\u0000\u0003\u00e2"+ + "\u0001\u0000\u0000\u0000\u0003\u00e4\u0001\u0000\u0000\u0000\u0003\u00e6"+ + "\u0001\u0000\u0000\u0000\u0003\u00e8\u0001\u0000\u0000\u0000\u0003\u00ea"+ + "\u0001\u0000\u0000\u0000\u0003\u00ec\u0001\u0000\u0000\u0000\u0004\u00ee"+ + "\u0001\u0000\u0000\u0000\u0004\u00f0\u0001\u0000\u0000\u0000\u0004\u00f2"+ + "\u0001\u0000\u0000\u0000\u0004\u00f4\u0001\u0000\u0000\u0000\u0004\u00f6"+ + "\u0001\u0000\u0000\u0000\u0004\u00fc\u0001\u0000\u0000\u0000\u0004\u00fe"+ + "\u0001\u0000\u0000\u0000\u0004\u0100\u0001\u0000\u0000\u0000\u0004\u0102"+ + "\u0001\u0000\u0000\u0000\u0005\u0104\u0001\u0000\u0000\u0000\u0005\u0106"+ + "\u0001\u0000\u0000\u0000\u0005\u0108\u0001\u0000\u0000\u0000\u0005\u010a"+ + "\u0001\u0000\u0000\u0000\u0005\u010c\u0001\u0000\u0000\u0000\u0005\u010e"+ + "\u0001\u0000\u0000\u0000\u0005\u0110\u0001\u0000\u0000\u0000\u0005\u0112"+ + "\u0001\u0000\u0000\u0000\u0005\u0114\u0001\u0000\u0000\u0000\u0005\u0116"+ + "\u0001\u0000\u0000\u0000\u0005\u0118\u0001\u0000\u0000\u0000\u0006\u011a"+ + "\u0001\u0000\u0000\u0000\u0006\u011c\u0001\u0000\u0000\u0000\u0006\u011e"+ + "\u0001\u0000\u0000\u0000\u0006\u0120\u0001\u0000\u0000\u0000\u0006\u0124"+ + "\u0001\u0000\u0000\u0000\u0006\u0126\u0001\u0000\u0000\u0000\u0006\u0128"+ + "\u0001\u0000\u0000\u0000\u0006\u012a\u0001\u0000\u0000\u0000\u0006\u012c"+ + "\u0001\u0000\u0000\u0000\u0007\u012e\u0001\u0000\u0000\u0000\u0007\u0130"+ + "\u0001\u0000\u0000\u0000\u0007\u0132\u0001\u0000\u0000\u0000\u0007\u0134"+ + "\u0001\u0000\u0000\u0000\u0007\u0136\u0001\u0000\u0000\u0000\u0007\u0138"+ + "\u0001\u0000\u0000\u0000\u0007\u013a\u0001\u0000\u0000\u0000\u0007\u013c"+ + "\u0001\u0000\u0000\u0000\u0007\u013e\u0001\u0000\u0000\u0000\u0007\u0140"+ + "\u0001\u0000\u0000\u0000\u0007\u0142\u0001\u0000\u0000\u0000\u0007\u0144"+ + "\u0001\u0000\u0000\u0000\b\u0146\u0001\u0000\u0000\u0000\b\u0148\u0001"+ + "\u0000\u0000\u0000\b\u014a\u0001\u0000\u0000\u0000\b\u014c\u0001\u0000"+ + "\u0000\u0000\b\u014e\u0001\u0000\u0000\u0000\b\u0150\u0001\u0000\u0000"+ + "\u0000\b\u0152\u0001\u0000\u0000\u0000\b\u0154\u0001\u0000\u0000\u0000"+ + "\b\u0156\u0001\u0000\u0000\u0000\t\u0158\u0001\u0000\u0000\u0000\t\u015a"+ + "\u0001\u0000\u0000\u0000\t\u015c\u0001\u0000\u0000\u0000\t\u015e\u0001"+ + "\u0000\u0000\u0000\t\u0160\u0001\u0000\u0000\u0000\n\u0162\u0001\u0000"+ + "\u0000\u0000\n\u0164\u0001\u0000\u0000\u0000\n\u0166\u0001\u0000\u0000"+ + "\u0000\n\u0168\u0001\u0000\u0000\u0000\n\u016a\u0001\u0000\u0000\u0000"+ + "\n\u016c\u0001\u0000\u0000\u0000\u000b\u016e\u0001\u0000\u0000\u0000\u000b"+ + "\u0170\u0001\u0000\u0000\u0000\u000b\u0172\u0001\u0000\u0000\u0000\u000b"+ + "\u0174\u0001\u0000\u0000\u0000\u000b\u0176\u0001\u0000\u0000\u0000\u000b"+ + "\u0178\u0001\u0000\u0000\u0000\u000b\u017a\u0001\u0000\u0000\u0000\u000b"+ + "\u017c\u0001\u0000\u0000\u0000\u000b\u017e\u0001\u0000\u0000\u0000\u000b"+ + "\u0180\u0001\u0000\u0000\u0000\f\u0182\u0001\u0000\u0000\u0000\f\u0184"+ + "\u0001\u0000\u0000\u0000\f\u0186\u0001\u0000\u0000\u0000\f\u0188\u0001"+ + "\u0000\u0000\u0000\f\u018a\u0001\u0000\u0000\u0000\f\u018c\u0001\u0000"+ + "\u0000\u0000\f\u018e\u0001\u0000\u0000\u0000\r\u0190\u0001\u0000\u0000"+ + "\u0000\r\u0192\u0001\u0000\u0000\u0000\r\u0194\u0001\u0000\u0000\u0000"+ + "\r\u0196\u0001\u0000\u0000\u0000\r\u0198\u0001\u0000\u0000\u0000\r\u019a"+ + "\u0001\u0000\u0000\u0000\r\u019c\u0001\u0000\u0000\u0000\r\u019e\u0001"+ + "\u0000\u0000\u0000\r\u01a0\u0001\u0000\u0000\u0000\r\u01a2\u0001\u0000"+ + "\u0000\u0000\r\u01a4\u0001\u0000\u0000\u0000\r\u01a6\u0001\u0000\u0000"+ + "\u0000\r\u01a8\u0001\u0000\u0000\u0000\u000e\u01aa\u0001\u0000\u0000\u0000"+ + "\u000e\u01ac\u0001\u0000\u0000\u0000\u000e\u01ae\u0001\u0000\u0000\u0000"+ + "\u000e\u01b0\u0001\u0000\u0000\u0000\u000e\u01b2\u0001\u0000\u0000\u0000"+ + "\u000e\u01b4\u0001\u0000\u0000\u0000\u000f\u01b6\u0001\u0000\u0000\u0000"+ + "\u000f\u01b8\u0001\u0000\u0000\u0000\u000f\u01ba\u0001\u0000\u0000\u0000"+ + "\u000f\u01bc\u0001\u0000\u0000\u0000\u000f\u01be\u0001\u0000\u0000\u0000"+ + "\u000f\u01c0\u0001\u0000\u0000\u0000\u000f\u01c2\u0001\u0000\u0000\u0000"+ + "\u000f\u01c4\u0001\u0000\u0000\u0000\u000f\u01c6\u0001\u0000\u0000\u0000"+ + "\u0010\u01c8\u0001\u0000\u0000\u0000\u0010\u01ca\u0001\u0000\u0000\u0000"+ + "\u0010\u01cc\u0001\u0000\u0000\u0000\u0010\u01ce\u0001\u0000\u0000\u0000"+ + "\u0010\u01d0\u0001\u0000\u0000\u0000\u0010\u01d2\u0001\u0000\u0000\u0000"+ + "\u0010\u01d4\u0001\u0000\u0000\u0000\u0010\u01d6\u0001\u0000\u0000\u0000"+ + "\u0010\u01d8\u0001\u0000\u0000\u0000\u0010\u01da\u0001\u0000\u0000\u0000"+ + "\u0011\u01dc\u0001\u0000\u0000\u0000\u0011\u01de\u0001\u0000\u0000\u0000"+ + "\u0011\u01e0\u0001\u0000\u0000\u0000\u0011\u01e2\u0001\u0000\u0000\u0000"+ + "\u0011\u01e4\u0001\u0000\u0000\u0000\u0012\u01e6\u0001\u0000\u0000\u0000"+ + "\u0014\u01f0\u0001\u0000\u0000\u0000\u0016\u01f7\u0001\u0000\u0000\u0000"+ + "\u0018\u0200\u0001\u0000\u0000\u0000\u001a\u0207\u0001\u0000\u0000\u0000"+ + "\u001c\u0211\u0001\u0000\u0000\u0000\u001e\u0218\u0001\u0000\u0000\u0000"+ + " \u021f\u0001\u0000\u0000\u0000\"\u0226\u0001\u0000\u0000\u0000$\u022e"+ + "\u0001\u0000\u0000\u0000&\u023a\u0001\u0000\u0000\u0000(\u0243\u0001\u0000"+ + "\u0000\u0000*\u0249\u0001\u0000\u0000\u0000,\u0250\u0001\u0000\u0000\u0000"+ + ".\u0257\u0001\u0000\u0000\u00000\u025f\u0001\u0000\u0000\u00002\u0267"+ + "\u0001\u0000\u0000\u00004\u0270\u0001\u0000\u0000\u00006\u0280\u0001\u0000"+ + "\u0000\u00008\u028f\u0001\u0000\u0000\u0000:\u029b\u0001\u0000\u0000\u0000"+ + "<\u02a7\u0001\u0000\u0000\u0000>\u02b2\u0001\u0000\u0000\u0000@\u02ba"+ + "\u0001\u0000\u0000\u0000B\u02c2\u0001\u0000\u0000\u0000D\u02cc\u0001\u0000"+ + "\u0000\u0000F\u02d2\u0001\u0000\u0000\u0000H\u02e3\u0001\u0000\u0000\u0000"+ + "J\u02f3\u0001\u0000\u0000\u0000L\u02f9\u0001\u0000\u0000\u0000N\u02fd"+ + "\u0001\u0000\u0000\u0000P\u02ff\u0001\u0000\u0000\u0000R\u0301\u0001\u0000"+ + "\u0000\u0000T\u0304\u0001\u0000\u0000\u0000V\u0306\u0001\u0000\u0000\u0000"+ + "X\u030f\u0001\u0000\u0000\u0000Z\u0311\u0001\u0000\u0000\u0000\\\u0316"+ + "\u0001\u0000\u0000\u0000^\u0318\u0001\u0000\u0000\u0000`\u031d\u0001\u0000"+ + "\u0000\u0000b\u033c\u0001\u0000\u0000\u0000d\u033f\u0001\u0000\u0000\u0000"+ + "f\u036d\u0001\u0000\u0000\u0000h\u036f\u0001\u0000\u0000\u0000j\u0372"+ + "\u0001\u0000\u0000\u0000l\u0376\u0001\u0000\u0000\u0000n\u037a\u0001\u0000"+ + "\u0000\u0000p\u037c\u0001\u0000\u0000\u0000r\u037f\u0001\u0000\u0000\u0000"+ + "t\u0381\u0001\u0000\u0000\u0000v\u0383\u0001\u0000\u0000\u0000x\u0388"+ + "\u0001\u0000\u0000\u0000z\u038a\u0001\u0000\u0000\u0000|\u0390\u0001\u0000"+ + "\u0000\u0000~\u0396\u0001\u0000\u0000\u0000\u0080\u0399\u0001\u0000\u0000"+ + "\u0000\u0082\u039c\u0001\u0000\u0000\u0000\u0084\u03a1\u0001\u0000\u0000"+ + "\u0000\u0086\u03a6\u0001\u0000\u0000\u0000\u0088\u03a8\u0001\u0000\u0000"+ + "\u0000\u008a\u03ac\u0001\u0000\u0000\u0000\u008c\u03b1\u0001\u0000\u0000"+ + "\u0000\u008e\u03b7\u0001\u0000\u0000\u0000\u0090\u03ba\u0001\u0000\u0000"+ + "\u0000\u0092\u03bc\u0001\u0000\u0000\u0000\u0094\u03c2\u0001\u0000\u0000"+ + "\u0000\u0096\u03c4\u0001\u0000\u0000\u0000\u0098\u03c9\u0001\u0000\u0000"+ + "\u0000\u009a\u03cc\u0001\u0000\u0000\u0000\u009c\u03cf\u0001\u0000\u0000"+ + "\u0000\u009e\u03d2\u0001\u0000\u0000\u0000\u00a0\u03d4\u0001\u0000\u0000"+ + "\u0000\u00a2\u03d7\u0001\u0000\u0000\u0000\u00a4\u03d9\u0001\u0000\u0000"+ + "\u0000\u00a6\u03dc\u0001\u0000\u0000\u0000\u00a8\u03de\u0001\u0000\u0000"+ + "\u0000\u00aa\u03e0\u0001\u0000\u0000\u0000\u00ac\u03e2\u0001\u0000\u0000"+ + "\u0000\u00ae\u03e4\u0001\u0000\u0000\u0000\u00b0\u03e6\u0001\u0000\u0000"+ + "\u0000\u00b2\u03e8\u0001\u0000\u0000\u0000\u00b4\u03ea\u0001\u0000\u0000"+ + "\u0000\u00b6\u03ff\u0001\u0000\u0000\u0000\u00b8\u0401\u0001\u0000\u0000"+ + "\u0000\u00ba\u0406\u0001\u0000\u0000\u0000\u00bc\u041b\u0001\u0000\u0000"+ + "\u0000\u00be\u041d\u0001\u0000\u0000\u0000\u00c0\u0425\u0001\u0000\u0000"+ + "\u0000\u00c2\u0427\u0001\u0000\u0000\u0000\u00c4\u042b\u0001\u0000\u0000"+ + "\u0000\u00c6\u042f\u0001\u0000\u0000\u0000\u00c8\u0433\u0001\u0000\u0000"+ + "\u0000\u00ca\u0438\u0001\u0000\u0000\u0000\u00cc\u043d\u0001\u0000\u0000"+ + "\u0000\u00ce\u0441\u0001\u0000\u0000\u0000\u00d0\u0445\u0001\u0000\u0000"+ + "\u0000\u00d2\u0449\u0001\u0000\u0000\u0000\u00d4\u044e\u0001\u0000\u0000"+ + "\u0000\u00d6\u0452\u0001\u0000\u0000\u0000\u00d8\u0456\u0001\u0000\u0000"+ + "\u0000\u00da\u045a\u0001\u0000\u0000\u0000\u00dc\u045e\u0001\u0000\u0000"+ + "\u0000\u00de\u0462\u0001\u0000\u0000\u0000\u00e0\u046e\u0001\u0000\u0000"+ + "\u0000\u00e2\u0471\u0001\u0000\u0000\u0000\u00e4\u0475\u0001\u0000\u0000"+ + "\u0000\u00e6\u0479\u0001\u0000\u0000\u0000\u00e8\u047d\u0001\u0000\u0000"+ + "\u0000\u00ea\u0481\u0001\u0000\u0000\u0000\u00ec\u0485\u0001\u0000\u0000"+ + "\u0000\u00ee\u0489\u0001\u0000\u0000\u0000\u00f0\u048e\u0001\u0000\u0000"+ + "\u0000\u00f2\u0492\u0001\u0000\u0000\u0000\u00f4\u0496\u0001\u0000\u0000"+ + "\u0000\u00f6\u049a\u0001\u0000\u0000\u0000\u00f8\u04a2\u0001\u0000\u0000"+ + "\u0000\u00fa\u04b7\u0001\u0000\u0000\u0000\u00fc\u04bb\u0001\u0000\u0000"+ + "\u0000\u00fe\u04bf\u0001\u0000\u0000\u0000\u0100\u04c3\u0001\u0000\u0000"+ + "\u0000\u0102\u04c7\u0001\u0000\u0000\u0000\u0104\u04cb\u0001\u0000\u0000"+ + "\u0000\u0106\u04d0\u0001\u0000\u0000\u0000\u0108\u04d4\u0001\u0000\u0000"+ + "\u0000\u010a\u04d8\u0001\u0000\u0000\u0000\u010c\u04dc\u0001\u0000\u0000"+ + "\u0000\u010e\u04e0\u0001\u0000\u0000\u0000\u0110\u04e4\u0001\u0000\u0000"+ + "\u0000\u0112\u04e7\u0001\u0000\u0000\u0000\u0114\u04eb\u0001\u0000\u0000"+ + "\u0000\u0116\u04ef\u0001\u0000\u0000\u0000\u0118\u04f3\u0001\u0000\u0000"+ + "\u0000\u011a\u04f7\u0001\u0000\u0000\u0000\u011c\u04fc\u0001\u0000\u0000"+ + "\u0000\u011e\u0501\u0001\u0000\u0000\u0000\u0120\u0506\u0001\u0000\u0000"+ + "\u0000\u0122\u050d\u0001\u0000\u0000\u0000\u0124\u0516\u0001\u0000\u0000"+ + "\u0000\u0126\u051d\u0001\u0000\u0000\u0000\u0128\u0521\u0001\u0000\u0000"+ + "\u0000\u012a\u0525\u0001\u0000\u0000\u0000\u012c\u0529\u0001\u0000\u0000"+ + "\u0000\u012e\u052d\u0001\u0000\u0000\u0000\u0130\u0533\u0001\u0000\u0000"+ + "\u0000\u0132\u0537\u0001\u0000\u0000\u0000\u0134\u053b\u0001\u0000\u0000"+ + "\u0000\u0136\u053f\u0001\u0000\u0000\u0000\u0138\u0543\u0001\u0000\u0000"+ + "\u0000\u013a\u0547\u0001\u0000\u0000\u0000\u013c\u054b\u0001\u0000\u0000"+ + "\u0000\u013e\u054f\u0001\u0000\u0000\u0000\u0140\u0553\u0001\u0000\u0000"+ + "\u0000\u0142\u0557\u0001\u0000\u0000\u0000\u0144\u055b\u0001\u0000\u0000"+ + "\u0000\u0146\u055f\u0001\u0000\u0000\u0000\u0148\u0564\u0001\u0000\u0000"+ + "\u0000\u014a\u0568\u0001\u0000\u0000\u0000\u014c\u056c\u0001\u0000\u0000"+ + "\u0000\u014e\u0570\u0001\u0000\u0000\u0000\u0150\u0574\u0001\u0000\u0000"+ + "\u0000\u0152\u0578\u0001\u0000\u0000\u0000\u0154\u057c\u0001\u0000\u0000"+ + "\u0000\u0156\u0580\u0001\u0000\u0000\u0000\u0158\u0584\u0001\u0000\u0000"+ + "\u0000\u015a\u0589\u0001\u0000\u0000\u0000\u015c\u058e\u0001\u0000\u0000"+ + "\u0000\u015e\u0592\u0001\u0000\u0000\u0000\u0160\u0596\u0001\u0000\u0000"+ + "\u0000\u0162\u059a\u0001\u0000\u0000\u0000\u0164\u059f\u0001\u0000\u0000"+ + "\u0000\u0166\u05a8\u0001\u0000\u0000\u0000\u0168\u05ac\u0001\u0000\u0000"+ + "\u0000\u016a\u05b0\u0001\u0000\u0000\u0000\u016c\u05b4\u0001\u0000\u0000"+ + "\u0000\u016e\u05b8\u0001\u0000\u0000\u0000\u0170\u05bd\u0001\u0000\u0000"+ + "\u0000\u0172\u05c1\u0001\u0000\u0000\u0000\u0174\u05c5\u0001\u0000\u0000"+ + "\u0000\u0176\u05c9\u0001\u0000\u0000\u0000\u0178\u05ce\u0001\u0000\u0000"+ + "\u0000\u017a\u05d2\u0001\u0000\u0000\u0000\u017c\u05d6\u0001\u0000\u0000"+ + "\u0000\u017e\u05da\u0001\u0000\u0000\u0000\u0180\u05de\u0001\u0000\u0000"+ + "\u0000\u0182\u05e2\u0001\u0000\u0000\u0000\u0184\u05e8\u0001\u0000\u0000"+ + "\u0000\u0186\u05ec\u0001\u0000\u0000\u0000\u0188\u05f0\u0001\u0000\u0000"+ + "\u0000\u018a\u05f4\u0001\u0000\u0000\u0000\u018c\u05f8\u0001\u0000\u0000"+ + "\u0000\u018e\u05fc\u0001\u0000\u0000\u0000\u0190\u0600\u0001\u0000\u0000"+ + "\u0000\u0192\u0605\u0001\u0000\u0000\u0000\u0194\u060a\u0001\u0000\u0000"+ + "\u0000\u0196\u060e\u0001\u0000\u0000\u0000\u0198\u0614\u0001\u0000\u0000"+ + "\u0000\u019a\u061d\u0001\u0000\u0000\u0000\u019c\u0621\u0001\u0000\u0000"+ + "\u0000\u019e\u0625\u0001\u0000\u0000\u0000\u01a0\u0629\u0001\u0000\u0000"+ + "\u0000\u01a2\u062d\u0001\u0000\u0000\u0000\u01a4\u0631\u0001\u0000\u0000"+ + "\u0000\u01a6\u0635\u0001\u0000\u0000\u0000\u01a8\u0639\u0001\u0000\u0000"+ + "\u0000\u01aa\u063d\u0001\u0000\u0000\u0000\u01ac\u0642\u0001\u0000\u0000"+ + "\u0000\u01ae\u0648\u0001\u0000\u0000\u0000\u01b0\u064e\u0001\u0000\u0000"+ + "\u0000\u01b2\u0652\u0001\u0000\u0000\u0000\u01b4\u0656\u0001\u0000\u0000"+ + "\u0000\u01b6\u065a\u0001\u0000\u0000\u0000\u01b8\u0660\u0001\u0000\u0000"+ + "\u0000\u01ba\u0666\u0001\u0000\u0000\u0000\u01bc\u066a\u0001\u0000\u0000"+ + "\u0000\u01be\u066e\u0001\u0000\u0000\u0000\u01c0\u0672\u0001\u0000\u0000"+ + "\u0000\u01c2\u0678\u0001\u0000\u0000\u0000\u01c4\u067e\u0001\u0000\u0000"+ + "\u0000\u01c6\u0684\u0001\u0000\u0000\u0000\u01c8\u0689\u0001\u0000\u0000"+ + "\u0000\u01ca\u068e\u0001\u0000\u0000\u0000\u01cc\u0692\u0001\u0000\u0000"+ + "\u0000\u01ce\u0696\u0001\u0000\u0000\u0000\u01d0\u069a\u0001\u0000\u0000"+ + "\u0000\u01d2\u069e\u0001\u0000\u0000\u0000\u01d4\u06a2\u0001\u0000\u0000"+ + "\u0000\u01d6\u06a6\u0001\u0000\u0000\u0000\u01d8\u06aa\u0001\u0000\u0000"+ + "\u0000\u01da\u06ae\u0001\u0000\u0000\u0000\u01dc\u06b2\u0001\u0000\u0000"+ + "\u0000\u01de\u06b7\u0001\u0000\u0000\u0000\u01e0\u06bb\u0001\u0000\u0000"+ + "\u0000\u01e2\u06bf\u0001\u0000\u0000\u0000\u01e4\u06c3\u0001\u0000\u0000"+ + "\u0000\u01e6\u01e7\u0007\u0000\u0000\u0000\u01e7\u01e8\u0007\u0001\u0000"+ + "\u0000\u01e8\u01e9\u0007\u0002\u0000\u0000\u01e9\u01ea\u0007\u0002\u0000"+ + "\u0000\u01ea\u01eb\u0007\u0003\u0000\u0000\u01eb\u01ec\u0007\u0004\u0000"+ + "\u0000\u01ec\u01ed\u0007\u0005\u0000\u0000\u01ed\u01ee\u0001\u0000\u0000"+ + "\u0000\u01ee\u01ef\u0006\u0000\u0000\u0000\u01ef\u0013\u0001\u0000\u0000"+ + "\u0000\u01f0\u01f1\u0007\u0000\u0000\u0000\u01f1\u01f2\u0007\u0006\u0000"+ + "\u0000\u01f2\u01f3\u0007\u0007\u0000\u0000\u01f3\u01f4\u0007\b\u0000\u0000"+ + "\u01f4\u01f5\u0001\u0000\u0000\u0000\u01f5\u01f6\u0006\u0001\u0001\u0000"+ + "\u01f6\u0015\u0001\u0000\u0000\u0000\u01f7\u01f8\u0007\u0003\u0000\u0000"+ + "\u01f8\u01f9\u0007\t\u0000\u0000\u01f9\u01fa\u0007\u0006\u0000\u0000\u01fa"+ + "\u01fb\u0007\u0001\u0000\u0000\u01fb\u01fc\u0007\u0004\u0000\u0000\u01fc"+ + "\u01fd\u0007\n\u0000\u0000\u01fd\u01fe\u0001\u0000\u0000\u0000\u01fe\u01ff"+ + "\u0006\u0002\u0002\u0000\u01ff\u0017\u0001\u0000\u0000\u0000\u0200\u0201"+ + "\u0007\u0003\u0000\u0000\u0201\u0202\u0007\u000b\u0000\u0000\u0202\u0203"+ + "\u0007\f\u0000\u0000\u0203\u0204\u0007\r\u0000\u0000\u0204\u0205\u0001"+ + "\u0000\u0000\u0000\u0205\u0206\u0006\u0003\u0000\u0000\u0206\u0019\u0001"+ + "\u0000\u0000\u0000\u0207\u0208\u0007\u0003\u0000\u0000\u0208\u0209\u0007"+ + "\u000e\u0000\u0000\u0209\u020a\u0007\b\u0000\u0000\u020a\u020b\u0007\r"+ + "\u0000\u0000\u020b\u020c\u0007\f\u0000\u0000\u020c\u020d\u0007\u0001\u0000"+ + "\u0000\u020d\u020e\u0007\t\u0000\u0000\u020e\u020f\u0001\u0000\u0000\u0000"+ + "\u020f\u0210\u0006\u0004\u0003\u0000\u0210\u001b\u0001\u0000\u0000\u0000"+ + "\u0211\u0212\u0007\u000f\u0000\u0000\u0212\u0213\u0007\u0006\u0000\u0000"+ + "\u0213\u0214\u0007\u0007\u0000\u0000\u0214\u0215\u0007\u0010\u0000\u0000"+ + "\u0215\u0216\u0001\u0000\u0000\u0000\u0216\u0217\u0006\u0005\u0004\u0000"+ + "\u0217\u001d\u0001\u0000\u0000\u0000\u0218\u0219\u0007\u0011\u0000\u0000"+ + "\u0219\u021a\u0007\u0006\u0000\u0000\u021a\u021b\u0007\u0007\u0000\u0000"+ + "\u021b\u021c\u0007\u0012\u0000\u0000\u021c\u021d\u0001\u0000\u0000\u0000"+ + "\u021d\u021e\u0006\u0006\u0000\u0000\u021e\u001f\u0001\u0000\u0000\u0000"+ + "\u021f\u0220\u0007\u0012\u0000\u0000\u0220\u0221\u0007\u0003\u0000\u0000"+ + "\u0221\u0222\u0007\u0003\u0000\u0000\u0222\u0223\u0007\b\u0000\u0000\u0223"+ + "\u0224\u0001\u0000\u0000\u0000\u0224\u0225\u0006\u0007\u0001\u0000\u0225"+ + "!\u0001\u0000\u0000\u0000\u0226\u0227\u0007\r\u0000\u0000\u0227\u0228"+ + "\u0007\u0001\u0000\u0000\u0228\u0229\u0007\u0010\u0000\u0000\u0229\u022a"+ + "\u0007\u0001\u0000\u0000\u022a\u022b\u0007\u0005\u0000\u0000\u022b\u022c"+ + "\u0001\u0000\u0000\u0000\u022c\u022d\u0006\b\u0000\u0000\u022d#\u0001"+ + "\u0000\u0000\u0000\u022e\u022f\u0007\u0010\u0000\u0000\u022f\u0230\u0007"+ + "\u000b\u0000\u0000\u0230\u0231\u0005_\u0000\u0000\u0231\u0232\u0007\u0003"+ + "\u0000\u0000\u0232\u0233\u0007\u000e\u0000\u0000\u0233\u0234\u0007\b\u0000"+ + "\u0000\u0234\u0235\u0007\f\u0000\u0000\u0235\u0236\u0007\t\u0000\u0000"+ + "\u0236\u0237\u0007\u0000\u0000\u0000\u0237\u0238\u0001\u0000\u0000\u0000"+ + "\u0238\u0239\u0006\t\u0005\u0000\u0239%\u0001\u0000\u0000\u0000\u023a"+ + "\u023b\u0007\u0006\u0000\u0000\u023b\u023c\u0007\u0003\u0000\u0000\u023c"+ + "\u023d\u0007\t\u0000\u0000\u023d\u023e\u0007\f\u0000\u0000\u023e\u023f"+ + "\u0007\u0010\u0000\u0000\u023f\u0240\u0007\u0003\u0000\u0000\u0240\u0241"+ + "\u0001\u0000\u0000\u0000\u0241\u0242\u0006\n\u0006\u0000\u0242\'\u0001"+ + "\u0000\u0000\u0000\u0243\u0244\u0007\u0006\u0000\u0000\u0244\u0245\u0007"+ + "\u0007\u0000\u0000\u0245\u0246\u0007\u0013\u0000\u0000\u0246\u0247\u0001"+ + "\u0000\u0000\u0000\u0247\u0248\u0006\u000b\u0000\u0000\u0248)\u0001\u0000"+ + "\u0000\u0000\u0249\u024a\u0007\u0002\u0000\u0000\u024a\u024b\u0007\n\u0000"+ + "\u0000\u024b\u024c\u0007\u0007\u0000\u0000\u024c\u024d\u0007\u0013\u0000"+ + "\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u024f\u0006\f\u0007\u0000"+ + "\u024f+\u0001\u0000\u0000\u0000\u0250\u0251\u0007\u0002\u0000\u0000\u0251"+ + "\u0252\u0007\u0007\u0000\u0000\u0252\u0253\u0007\u0006\u0000\u0000\u0253"+ + "\u0254\u0007\u0005\u0000\u0000\u0254\u0255\u0001\u0000\u0000\u0000\u0255"+ + "\u0256\u0006\r\u0000\u0000\u0256-\u0001\u0000\u0000\u0000\u0257\u0258"+ + "\u0007\u0002\u0000\u0000\u0258\u0259\u0007\u0005\u0000\u0000\u0259\u025a"+ + "\u0007\f\u0000\u0000\u025a\u025b\u0007\u0005\u0000\u0000\u025b\u025c\u0007"+ + "\u0002\u0000\u0000\u025c\u025d\u0001\u0000\u0000\u0000\u025d\u025e\u0006"+ + "\u000e\u0000\u0000\u025e/\u0001\u0000\u0000\u0000\u025f\u0260\u0007\u0013"+ + "\u0000\u0000\u0260\u0261\u0007\n\u0000\u0000\u0261\u0262\u0007\u0003\u0000"+ + "\u0000\u0262\u0263\u0007\u0006\u0000\u0000\u0263\u0264\u0007\u0003\u0000"+ + "\u0000\u0264\u0265\u0001\u0000\u0000\u0000\u0265\u0266\u0006\u000f\u0000"+ + "\u0000\u02661\u0001\u0000\u0000\u0000\u0267\u0268\u0007\r\u0000\u0000"+ + "\u0268\u0269\u0007\u0007\u0000\u0000\u0269\u026a\u0007\u0007\u0000\u0000"+ + "\u026a\u026b\u0007\u0012\u0000\u0000\u026b\u026c\u0007\u0014\u0000\u0000"+ + "\u026c\u026d\u0007\b\u0000\u0000\u026d\u026e\u0001\u0000\u0000\u0000\u026e"+ + "\u026f\u0006\u0010\b\u0000\u026f3\u0001\u0000\u0000\u0000\u0270\u0271"+ + "\u0004\u0011\u0000\u0000\u0271\u0272\u0007\u0004\u0000\u0000\u0272\u0273"+ + "\u0007\n\u0000\u0000\u0273\u0274\u0007\f\u0000\u0000\u0274\u0275\u0007"+ + "\t\u0000\u0000\u0275\u0276\u0007\u0011\u0000\u0000\u0276\u0277\u0007\u0003"+ + "\u0000\u0000\u0277\u0278\u0005_\u0000\u0000\u0278\u0279\u0007\b\u0000"+ + "\u0000\u0279\u027a\u0007\u0007\u0000\u0000\u027a\u027b\u0007\u0001\u0000"+ + "\u0000\u027b\u027c\u0007\t\u0000\u0000\u027c\u027d\u0007\u0005\u0000\u0000"+ + "\u027d\u027e\u0001\u0000\u0000\u0000\u027e\u027f\u0006\u0011\t\u0000\u027f"+ + "5\u0001\u0000\u0000\u0000\u0280\u0281\u0004\u0012\u0001\u0000\u0281\u0282"+ + "\u0007\u0001\u0000\u0000\u0282\u0283\u0007\t\u0000\u0000\u0283\u0284\u0007"+ + "\r\u0000\u0000\u0284\u0285\u0007\u0001\u0000\u0000\u0285\u0286\u0007\t"+ + "\u0000\u0000\u0286\u0287\u0007\u0003\u0000\u0000\u0287\u0288\u0007\u0002"+ + "\u0000\u0000\u0288\u0289\u0007\u0005\u0000\u0000\u0289\u028a\u0007\f\u0000"+ + "\u0000\u028a\u028b\u0007\u0005\u0000\u0000\u028b\u028c\u0007\u0002\u0000"+ + "\u0000\u028c\u028d\u0001\u0000\u0000\u0000\u028d\u028e\u0006\u0012\u0000"+ + "\u0000\u028e7\u0001\u0000\u0000\u0000\u028f\u0290\u0004\u0013\u0002\u0000"+ + "\u0290\u0291\u0007\u0001\u0000\u0000\u0291\u0292\u0007\t\u0000\u0000\u0292"+ + "\u0293\u0007\u0002\u0000\u0000\u0293\u0294\u0007\u0001\u0000\u0000\u0294"+ + "\u0295\u0007\u0002\u0000\u0000\u0295\u0296\u0007\u0005\u0000\u0000\u0296"+ + "\u0297\u0005_\u0000\u0000\u0297\u0298\u0005\u8001\uf414\u0000\u0000\u0298"+ + "\u0299\u0001\u0000\u0000\u0000\u0299\u029a\u0006\u0013\u0001\u0000\u029a"+ + "9\u0001\u0000\u0000\u0000\u029b\u029c\u0004\u0014\u0003\u0000\u029c\u029d"+ + "\u0007\r\u0000\u0000\u029d\u029e\u0007\u0007\u0000\u0000\u029e\u029f\u0007"+ + "\u0007\u0000\u0000\u029f\u02a0\u0007\u0012\u0000\u0000\u02a0\u02a1\u0007"+ + "\u0014\u0000\u0000\u02a1\u02a2\u0007\b\u0000\u0000\u02a2\u02a3\u0005_"+ + "\u0000\u0000\u02a3\u02a4\u0005\u8001\uf414\u0000\u0000\u02a4\u02a5\u0001"+ + "\u0000\u0000\u0000\u02a5\u02a6\u0006\u0014\n\u0000\u02a6;\u0001\u0000"+ + "\u0000\u0000\u02a7\u02a8\u0004\u0015\u0004\u0000\u02a8\u02a9\u0007\u0010"+ + "\u0000\u0000\u02a9\u02aa\u0007\u0003\u0000\u0000\u02aa\u02ab\u0007\u0005"+ + "\u0000\u0000\u02ab\u02ac\u0007\u0006\u0000\u0000\u02ac\u02ad\u0007\u0001"+ + "\u0000\u0000\u02ad\u02ae\u0007\u0004\u0000\u0000\u02ae\u02af\u0007\u0002"+ + "\u0000\u0000\u02af\u02b0\u0001\u0000\u0000\u0000\u02b0\u02b1\u0006\u0015"+ + "\u000b\u0000\u02b1=\u0001\u0000\u0000\u0000\u02b2\u02b3\u0004\u0016\u0005"+ + "\u0000\u02b3\u02b4\u0007\u000f\u0000\u0000\u02b4\u02b5\u0007\u0014\u0000"+ + "\u0000\u02b5\u02b6\u0007\r\u0000\u0000\u02b6\u02b7\u0007\r\u0000\u0000"+ + "\u02b7\u02b8\u0001\u0000\u0000\u0000\u02b8\u02b9\u0006\u0016\b\u0000\u02b9"+ + "?\u0001\u0000\u0000\u0000\u02ba\u02bb\u0004\u0017\u0006\u0000\u02bb\u02bc"+ + "\u0007\r\u0000\u0000\u02bc\u02bd\u0007\u0003\u0000\u0000\u02bd\u02be\u0007"+ + "\u000f\u0000\u0000\u02be\u02bf\u0007\u0005\u0000\u0000\u02bf\u02c0\u0001"+ + "\u0000\u0000\u0000\u02c0\u02c1\u0006\u0017\b\u0000\u02c1A\u0001\u0000"+ + "\u0000\u0000\u02c2\u02c3\u0004\u0018\u0007\u0000\u02c3\u02c4\u0007\u0006"+ + "\u0000\u0000\u02c4\u02c5\u0007\u0001\u0000\u0000\u02c5\u02c6\u0007\u0011"+ + "\u0000\u0000\u02c6\u02c7\u0007\n\u0000\u0000\u02c7\u02c8\u0007\u0005\u0000"+ + "\u0000\u02c8\u02c9\u0001\u0000\u0000\u0000\u02c9\u02ca\u0006\u0018\b\u0000"+ + "\u02caC\u0001\u0000\u0000\u0000\u02cb\u02cd\b\u0015\u0000\u0000\u02cc"+ + "\u02cb\u0001\u0000\u0000\u0000\u02cd\u02ce\u0001\u0000\u0000\u0000\u02ce"+ + "\u02cc\u0001\u0000\u0000\u0000\u02ce\u02cf\u0001\u0000\u0000\u0000\u02cf"+ + "\u02d0\u0001\u0000\u0000\u0000\u02d0\u02d1\u0006\u0019\u0000\u0000\u02d1"+ + "E\u0001\u0000\u0000\u0000\u02d2\u02d3\u0005/\u0000\u0000\u02d3\u02d4\u0005"+ + "/\u0000\u0000\u02d4\u02d8\u0001\u0000\u0000\u0000\u02d5\u02d7\b\u0016"+ + "\u0000\u0000\u02d6\u02d5\u0001\u0000\u0000\u0000\u02d7\u02da\u0001\u0000"+ + "\u0000\u0000\u02d8\u02d6\u0001\u0000\u0000\u0000\u02d8\u02d9\u0001\u0000"+ + "\u0000\u0000\u02d9\u02dc\u0001\u0000\u0000\u0000\u02da\u02d8\u0001\u0000"+ + "\u0000\u0000\u02db\u02dd\u0005\r\u0000\u0000\u02dc\u02db\u0001\u0000\u0000"+ + "\u0000\u02dc\u02dd\u0001\u0000\u0000\u0000\u02dd\u02df\u0001\u0000\u0000"+ + "\u0000\u02de\u02e0\u0005\n\u0000\u0000\u02df\u02de\u0001\u0000\u0000\u0000"+ + "\u02df\u02e0\u0001\u0000\u0000\u0000\u02e0\u02e1\u0001\u0000\u0000\u0000"+ + "\u02e1\u02e2\u0006\u001a\f\u0000\u02e2G\u0001\u0000\u0000\u0000\u02e3"+ + "\u02e4\u0005/\u0000\u0000\u02e4\u02e5\u0005*\u0000\u0000\u02e5\u02ea\u0001"+ + "\u0000\u0000\u0000\u02e6\u02e9\u0003H\u001b\u0000\u02e7\u02e9\t\u0000"+ + "\u0000\u0000\u02e8\u02e6\u0001\u0000\u0000\u0000\u02e8\u02e7\u0001\u0000"+ + "\u0000\u0000\u02e9\u02ec\u0001\u0000\u0000\u0000\u02ea\u02eb\u0001\u0000"+ + "\u0000\u0000\u02ea\u02e8\u0001\u0000\u0000\u0000\u02eb\u02ed\u0001\u0000"+ + "\u0000\u0000\u02ec\u02ea\u0001\u0000\u0000\u0000\u02ed\u02ee\u0005*\u0000"+ + "\u0000\u02ee\u02ef\u0005/\u0000\u0000\u02ef\u02f0\u0001\u0000\u0000\u0000"+ + "\u02f0\u02f1\u0006\u001b\f\u0000\u02f1I\u0001\u0000\u0000\u0000\u02f2"+ + "\u02f4\u0007\u0017\u0000\u0000\u02f3\u02f2\u0001\u0000\u0000\u0000\u02f4"+ + "\u02f5\u0001\u0000\u0000\u0000\u02f5\u02f3\u0001\u0000\u0000\u0000\u02f5"+ + "\u02f6\u0001\u0000\u0000\u0000\u02f6\u02f7\u0001\u0000\u0000\u0000\u02f7"+ + "\u02f8\u0006\u001c\f\u0000\u02f8K\u0001\u0000\u0000\u0000\u02f9\u02fa"+ + "\u0005|\u0000\u0000\u02fa\u02fb\u0001\u0000\u0000\u0000\u02fb\u02fc\u0006"+ + "\u001d\r\u0000\u02fcM\u0001\u0000\u0000\u0000\u02fd\u02fe\u0007\u0018"+ + "\u0000\u0000\u02feO\u0001\u0000\u0000\u0000\u02ff\u0300\u0007\u0019\u0000"+ + "\u0000\u0300Q\u0001\u0000\u0000\u0000\u0301\u0302\u0005\\\u0000\u0000"+ + "\u0302\u0303\u0007\u001a\u0000\u0000\u0303S\u0001\u0000\u0000\u0000\u0304"+ + "\u0305\b\u001b\u0000\u0000\u0305U\u0001\u0000\u0000\u0000\u0306\u0308"+ + "\u0007\u0003\u0000\u0000\u0307\u0309\u0007\u001c\u0000\u0000\u0308\u0307"+ + "\u0001\u0000\u0000\u0000\u0308\u0309\u0001\u0000\u0000\u0000\u0309\u030b"+ + "\u0001\u0000\u0000\u0000\u030a\u030c\u0003N\u001e\u0000\u030b\u030a\u0001"+ + "\u0000\u0000\u0000\u030c\u030d\u0001\u0000\u0000\u0000\u030d\u030b\u0001"+ + "\u0000\u0000\u0000\u030d\u030e\u0001\u0000\u0000\u0000\u030eW\u0001\u0000"+ + "\u0000\u0000\u030f\u0310\u0005@\u0000\u0000\u0310Y\u0001\u0000\u0000\u0000"+ + "\u0311\u0312\u0005`\u0000\u0000\u0312[\u0001\u0000\u0000\u0000\u0313\u0317"+ + "\b\u001d\u0000\u0000\u0314\u0315\u0005`\u0000\u0000\u0315\u0317\u0005"+ + "`\u0000\u0000\u0316\u0313\u0001\u0000\u0000\u0000\u0316\u0314\u0001\u0000"+ + "\u0000\u0000\u0317]\u0001\u0000\u0000\u0000\u0318\u0319\u0005_\u0000\u0000"+ + "\u0319_\u0001\u0000\u0000\u0000\u031a\u031e\u0003P\u001f\u0000\u031b\u031e"+ + "\u0003N\u001e\u0000\u031c\u031e\u0003^&\u0000\u031d\u031a\u0001\u0000"+ + "\u0000\u0000\u031d\u031b\u0001\u0000\u0000\u0000\u031d\u031c\u0001\u0000"+ + "\u0000\u0000\u031ea\u0001\u0000\u0000\u0000\u031f\u0324\u0005\"\u0000"+ + "\u0000\u0320\u0323\u0003R \u0000\u0321\u0323\u0003T!\u0000\u0322\u0320"+ + "\u0001\u0000\u0000\u0000\u0322\u0321\u0001\u0000\u0000\u0000\u0323\u0326"+ + "\u0001\u0000\u0000\u0000\u0324\u0322\u0001\u0000\u0000\u0000\u0324\u0325"+ + "\u0001\u0000\u0000\u0000\u0325\u0327\u0001\u0000\u0000\u0000\u0326\u0324"+ + "\u0001\u0000\u0000\u0000\u0327\u033d\u0005\"\u0000\u0000\u0328\u0329\u0005"+ + "\"\u0000\u0000\u0329\u032a\u0005\"\u0000\u0000\u032a\u032b\u0005\"\u0000"+ + "\u0000\u032b\u032f\u0001\u0000\u0000\u0000\u032c\u032e\b\u0016\u0000\u0000"+ + "\u032d\u032c\u0001\u0000\u0000\u0000\u032e\u0331\u0001\u0000\u0000\u0000"+ + "\u032f\u0330\u0001\u0000\u0000\u0000\u032f\u032d\u0001\u0000\u0000\u0000"+ + "\u0330\u0332\u0001\u0000\u0000\u0000\u0331\u032f\u0001\u0000\u0000\u0000"+ + "\u0332\u0333\u0005\"\u0000\u0000\u0333\u0334\u0005\"\u0000\u0000\u0334"+ + "\u0335\u0005\"\u0000\u0000\u0335\u0337\u0001\u0000\u0000\u0000\u0336\u0338"+ + "\u0005\"\u0000\u0000\u0337\u0336\u0001\u0000\u0000\u0000\u0337\u0338\u0001"+ + "\u0000\u0000\u0000\u0338\u033a\u0001\u0000\u0000\u0000\u0339\u033b\u0005"+ + "\"\u0000\u0000\u033a\u0339\u0001\u0000\u0000\u0000\u033a\u033b\u0001\u0000"+ + "\u0000\u0000\u033b\u033d\u0001\u0000\u0000\u0000\u033c\u031f\u0001\u0000"+ + "\u0000\u0000\u033c\u0328\u0001\u0000\u0000\u0000\u033dc\u0001\u0000\u0000"+ + "\u0000\u033e\u0340\u0003N\u001e\u0000\u033f\u033e\u0001\u0000\u0000\u0000"+ + "\u0340\u0341\u0001\u0000\u0000\u0000\u0341\u033f\u0001\u0000\u0000\u0000"+ + "\u0341\u0342\u0001\u0000\u0000\u0000\u0342e\u0001\u0000\u0000\u0000\u0343"+ + "\u0345\u0003N\u001e\u0000\u0344\u0343\u0001\u0000\u0000\u0000\u0345\u0346"+ + "\u0001\u0000\u0000\u0000\u0346\u0344\u0001\u0000\u0000\u0000\u0346\u0347"+ + "\u0001\u0000\u0000\u0000\u0347\u0348\u0001\u0000\u0000\u0000\u0348\u034c"+ + "\u0003x3\u0000\u0349\u034b\u0003N\u001e\u0000\u034a\u0349\u0001\u0000"+ + "\u0000\u0000\u034b\u034e\u0001\u0000\u0000\u0000\u034c\u034a\u0001\u0000"+ + "\u0000\u0000\u034c\u034d\u0001\u0000\u0000\u0000\u034d\u036e\u0001\u0000"+ + "\u0000\u0000\u034e\u034c\u0001\u0000\u0000\u0000\u034f\u0351\u0003x3\u0000"+ + "\u0350\u0352\u0003N\u001e\u0000\u0351\u0350\u0001\u0000\u0000\u0000\u0352"+ + "\u0353\u0001\u0000\u0000\u0000\u0353\u0351\u0001\u0000\u0000\u0000\u0353"+ + "\u0354\u0001\u0000\u0000\u0000\u0354\u036e\u0001\u0000\u0000\u0000\u0355"+ + "\u0357\u0003N\u001e\u0000\u0356\u0355\u0001\u0000\u0000\u0000\u0357\u0358"+ + "\u0001\u0000\u0000\u0000\u0358\u0356\u0001\u0000\u0000\u0000\u0358\u0359"+ + "\u0001\u0000\u0000\u0000\u0359\u0361\u0001\u0000\u0000\u0000\u035a\u035e"+ + "\u0003x3\u0000\u035b\u035d\u0003N\u001e\u0000\u035c\u035b\u0001\u0000"+ + "\u0000\u0000\u035d\u0360\u0001\u0000\u0000\u0000\u035e\u035c\u0001\u0000"+ + "\u0000\u0000\u035e\u035f\u0001\u0000\u0000\u0000\u035f\u0362\u0001\u0000"+ + "\u0000\u0000\u0360\u035e\u0001\u0000\u0000\u0000\u0361\u035a\u0001\u0000"+ + "\u0000\u0000\u0361\u0362\u0001\u0000\u0000\u0000\u0362\u0363\u0001\u0000"+ + "\u0000\u0000\u0363\u0364\u0003V\"\u0000\u0364\u036e\u0001\u0000\u0000"+ + "\u0000\u0365\u0367\u0003x3\u0000\u0366\u0368\u0003N\u001e\u0000\u0367"+ + "\u0366\u0001\u0000\u0000\u0000\u0368\u0369\u0001\u0000\u0000\u0000\u0369"+ + "\u0367\u0001\u0000\u0000\u0000\u0369\u036a\u0001\u0000\u0000\u0000\u036a"+ + "\u036b\u0001\u0000\u0000\u0000\u036b\u036c\u0003V\"\u0000\u036c\u036e"+ + "\u0001\u0000\u0000\u0000\u036d\u0344\u0001\u0000\u0000\u0000\u036d\u034f"+ + "\u0001\u0000\u0000\u0000\u036d\u0356\u0001\u0000\u0000\u0000\u036d\u0365"+ + "\u0001\u0000\u0000\u0000\u036eg\u0001\u0000\u0000\u0000\u036f\u0370\u0007"+ + "\u001e\u0000\u0000\u0370\u0371\u0007\u001f\u0000\u0000\u0371i\u0001\u0000"+ + "\u0000\u0000\u0372\u0373\u0007\f\u0000\u0000\u0373\u0374\u0007\t\u0000"+ + "\u0000\u0374\u0375\u0007\u0000\u0000\u0000\u0375k\u0001\u0000\u0000\u0000"+ + "\u0376\u0377\u0007\f\u0000\u0000\u0377\u0378\u0007\u0002\u0000\u0000\u0378"+ + "\u0379\u0007\u0004\u0000\u0000\u0379m\u0001\u0000\u0000\u0000\u037a\u037b"+ + "\u0005=\u0000\u0000\u037bo\u0001\u0000\u0000\u0000\u037c\u037d\u0005:"+ + "\u0000\u0000\u037d\u037e\u0005:\u0000\u0000\u037eq\u0001\u0000\u0000\u0000"+ + "\u037f\u0380\u0005:\u0000\u0000\u0380s\u0001\u0000\u0000\u0000\u0381\u0382"+ + "\u0005,\u0000\u0000\u0382u\u0001\u0000\u0000\u0000\u0383\u0384\u0007\u0000"+ + "\u0000\u0000\u0384\u0385\u0007\u0003\u0000\u0000\u0385\u0386\u0007\u0002"+ + "\u0000\u0000\u0386\u0387\u0007\u0004\u0000\u0000\u0387w\u0001\u0000\u0000"+ + "\u0000\u0388\u0389\u0005.\u0000\u0000\u0389y\u0001\u0000\u0000\u0000\u038a"+ + "\u038b\u0007\u000f\u0000\u0000\u038b\u038c\u0007\f\u0000\u0000\u038c\u038d"+ + "\u0007\r\u0000\u0000\u038d\u038e\u0007\u0002\u0000\u0000\u038e\u038f\u0007"+ + "\u0003\u0000\u0000\u038f{\u0001\u0000\u0000\u0000\u0390\u0391\u0007\u000f"+ + "\u0000\u0000\u0391\u0392\u0007\u0001\u0000\u0000\u0392\u0393\u0007\u0006"+ + "\u0000\u0000\u0393\u0394\u0007\u0002\u0000\u0000\u0394\u0395\u0007\u0005"+ + "\u0000\u0000\u0395}\u0001\u0000\u0000\u0000\u0396\u0397\u0007\u0001\u0000"+ + "\u0000\u0397\u0398\u0007\t\u0000\u0000\u0398\u007f\u0001\u0000\u0000\u0000"+ + "\u0399\u039a\u0007\u0001\u0000\u0000\u039a\u039b\u0007\u0002\u0000\u0000"+ + "\u039b\u0081\u0001\u0000\u0000\u0000\u039c\u039d\u0007\r\u0000\u0000\u039d"+ + "\u039e\u0007\f\u0000\u0000\u039e\u039f\u0007\u0002\u0000\u0000\u039f\u03a0"+ + "\u0007\u0005\u0000\u0000\u03a0\u0083\u0001\u0000\u0000\u0000\u03a1\u03a2"+ + "\u0007\r\u0000\u0000\u03a2\u03a3\u0007\u0001\u0000\u0000\u03a3\u03a4\u0007"+ + "\u0012\u0000\u0000\u03a4\u03a5\u0007\u0003\u0000\u0000\u03a5\u0085\u0001"+ + "\u0000\u0000\u0000\u03a6\u03a7\u0005(\u0000\u0000\u03a7\u0087\u0001\u0000"+ + "\u0000\u0000\u03a8\u03a9\u0007\t\u0000\u0000\u03a9\u03aa\u0007\u0007\u0000"+ + "\u0000\u03aa\u03ab\u0007\u0005\u0000\u0000\u03ab\u0089\u0001\u0000\u0000"+ + "\u0000\u03ac\u03ad\u0007\t\u0000\u0000\u03ad\u03ae\u0007\u0014\u0000\u0000"+ + "\u03ae\u03af\u0007\r\u0000\u0000\u03af\u03b0\u0007\r\u0000\u0000\u03b0"+ + "\u008b\u0001\u0000\u0000\u0000\u03b1\u03b2\u0007\t\u0000\u0000\u03b2\u03b3"+ + "\u0007\u0014\u0000\u0000\u03b3\u03b4\u0007\r\u0000\u0000\u03b4\u03b5\u0007"+ + "\r\u0000\u0000\u03b5\u03b6\u0007\u0002\u0000\u0000\u03b6\u008d\u0001\u0000"+ + "\u0000\u0000\u03b7\u03b8\u0007\u0007\u0000\u0000\u03b8\u03b9\u0007\u0006"+ + "\u0000\u0000\u03b9\u008f\u0001\u0000\u0000\u0000\u03ba\u03bb\u0005?\u0000"+ + "\u0000\u03bb\u0091\u0001\u0000\u0000\u0000\u03bc\u03bd\u0007\u0006\u0000"+ + "\u0000\u03bd\u03be\u0007\r\u0000\u0000\u03be\u03bf\u0007\u0001\u0000\u0000"+ + "\u03bf\u03c0\u0007\u0012\u0000\u0000\u03c0\u03c1\u0007\u0003\u0000\u0000"+ + "\u03c1\u0093\u0001\u0000\u0000\u0000\u03c2\u03c3\u0005)\u0000\u0000\u03c3"+ + "\u0095\u0001\u0000\u0000\u0000\u03c4\u03c5\u0007\u0005\u0000\u0000\u03c5"+ + "\u03c6\u0007\u0006\u0000\u0000\u03c6\u03c7\u0007\u0014\u0000\u0000\u03c7"+ + "\u03c8\u0007\u0003\u0000\u0000\u03c8\u0097\u0001\u0000\u0000\u0000\u03c9"+ + "\u03ca\u0005=\u0000\u0000\u03ca\u03cb\u0005=\u0000\u0000\u03cb\u0099\u0001"+ + "\u0000\u0000\u0000\u03cc\u03cd\u0005=\u0000\u0000\u03cd\u03ce\u0005~\u0000"+ + "\u0000\u03ce\u009b\u0001\u0000\u0000\u0000\u03cf\u03d0\u0005!\u0000\u0000"+ + "\u03d0\u03d1\u0005=\u0000\u0000\u03d1\u009d\u0001\u0000\u0000\u0000\u03d2"+ + "\u03d3\u0005<\u0000\u0000\u03d3\u009f\u0001\u0000\u0000\u0000\u03d4\u03d5"+ + "\u0005<\u0000\u0000\u03d5\u03d6\u0005=\u0000\u0000\u03d6\u00a1\u0001\u0000"+ + "\u0000\u0000\u03d7\u03d8\u0005>\u0000\u0000\u03d8\u00a3\u0001\u0000\u0000"+ + "\u0000\u03d9\u03da\u0005>\u0000\u0000\u03da\u03db\u0005=\u0000\u0000\u03db"+ + "\u00a5\u0001\u0000\u0000\u0000\u03dc\u03dd\u0005+\u0000\u0000\u03dd\u00a7"+ + "\u0001\u0000\u0000\u0000\u03de\u03df\u0005-\u0000\u0000\u03df\u00a9\u0001"+ + "\u0000\u0000\u0000\u03e0\u03e1\u0005*\u0000\u0000\u03e1\u00ab\u0001\u0000"+ + "\u0000\u0000\u03e2\u03e3\u0005/\u0000\u0000\u03e3\u00ad\u0001\u0000\u0000"+ + "\u0000\u03e4\u03e5\u0005%\u0000\u0000\u03e5\u00af\u0001\u0000\u0000\u0000"+ + "\u03e6\u03e7\u0005{\u0000\u0000\u03e7\u00b1\u0001\u0000\u0000\u0000\u03e8"+ + "\u03e9\u0005}\u0000\u0000\u03e9\u00b3\u0001\u0000\u0000\u0000\u03ea\u03eb"+ + "\u00030\u000f\u0000\u03eb\u03ec\u0001\u0000\u0000\u0000\u03ec\u03ed\u0006"+ + "Q\u000e\u0000\u03ed\u00b5\u0001\u0000\u0000\u0000\u03ee\u03f1\u0003\u0090"+ + "?\u0000\u03ef\u03f2\u0003P\u001f\u0000\u03f0\u03f2\u0003^&\u0000\u03f1"+ + "\u03ef\u0001\u0000\u0000\u0000\u03f1\u03f0\u0001\u0000\u0000\u0000\u03f2"+ + "\u03f6\u0001\u0000\u0000\u0000\u03f3\u03f5\u0003`\'\u0000\u03f4\u03f3"+ + "\u0001\u0000\u0000\u0000\u03f5\u03f8\u0001\u0000\u0000\u0000\u03f6\u03f4"+ + "\u0001\u0000\u0000\u0000\u03f6\u03f7\u0001\u0000\u0000\u0000\u03f7\u0400"+ + "\u0001\u0000\u0000\u0000\u03f8\u03f6\u0001\u0000\u0000\u0000\u03f9\u03fb"+ + "\u0003\u0090?\u0000\u03fa\u03fc\u0003N\u001e\u0000\u03fb\u03fa\u0001\u0000"+ + "\u0000\u0000\u03fc\u03fd\u0001\u0000\u0000\u0000\u03fd\u03fb\u0001\u0000"+ + "\u0000\u0000\u03fd\u03fe\u0001\u0000\u0000\u0000\u03fe\u0400\u0001\u0000"+ + "\u0000\u0000\u03ff\u03ee\u0001\u0000\u0000\u0000\u03ff\u03f9\u0001\u0000"+ + "\u0000\u0000\u0400\u00b7\u0001\u0000\u0000\u0000\u0401\u0402\u0005[\u0000"+ + "\u0000\u0402\u0403\u0001\u0000\u0000\u0000\u0403\u0404\u0006S\u0000\u0000"+ + "\u0404\u0405\u0006S\u0000\u0000\u0405\u00b9\u0001\u0000\u0000\u0000\u0406"+ + "\u0407\u0005]\u0000\u0000\u0407\u0408\u0001\u0000\u0000\u0000\u0408\u0409"+ + "\u0006T\r\u0000\u0409\u040a\u0006T\r\u0000\u040a\u00bb\u0001\u0000\u0000"+ + "\u0000\u040b\u040f\u0003P\u001f\u0000\u040c\u040e\u0003`\'\u0000\u040d"+ + "\u040c\u0001\u0000\u0000\u0000\u040e\u0411\u0001\u0000\u0000\u0000\u040f"+ + "\u040d\u0001\u0000\u0000\u0000\u040f\u0410\u0001\u0000\u0000\u0000\u0410"+ + "\u041c\u0001\u0000\u0000\u0000\u0411\u040f\u0001\u0000\u0000\u0000\u0412"+ + "\u0415\u0003^&\u0000\u0413\u0415\u0003X#\u0000\u0414\u0412\u0001\u0000"+ + "\u0000\u0000\u0414\u0413\u0001\u0000\u0000\u0000\u0415\u0417\u0001\u0000"+ + "\u0000\u0000\u0416\u0418\u0003`\'\u0000\u0417\u0416\u0001\u0000\u0000"+ + "\u0000\u0418\u0419\u0001\u0000\u0000\u0000\u0419\u0417\u0001\u0000\u0000"+ + "\u0000\u0419\u041a\u0001\u0000\u0000\u0000\u041a\u041c\u0001\u0000\u0000"+ + "\u0000\u041b\u040b\u0001\u0000\u0000\u0000\u041b\u0414\u0001\u0000\u0000"+ + "\u0000\u041c\u00bd\u0001\u0000\u0000\u0000\u041d\u041f\u0003Z$\u0000\u041e"+ + "\u0420\u0003\\%\u0000\u041f\u041e\u0001\u0000\u0000\u0000\u0420\u0421"+ + "\u0001\u0000\u0000\u0000\u0421\u041f\u0001\u0000\u0000\u0000\u0421\u0422"+ + "\u0001\u0000\u0000\u0000\u0422\u0423\u0001\u0000\u0000\u0000\u0423\u0424"+ + "\u0003Z$\u0000\u0424\u00bf\u0001\u0000\u0000\u0000\u0425\u0426\u0003\u00be"+ + "V\u0000\u0426\u00c1\u0001\u0000\u0000\u0000\u0427\u0428\u0003F\u001a\u0000"+ + "\u0428\u0429\u0001\u0000\u0000\u0000\u0429\u042a\u0006X\f\u0000\u042a"+ + "\u00c3\u0001\u0000\u0000\u0000\u042b\u042c\u0003H\u001b\u0000\u042c\u042d"+ + "\u0001\u0000\u0000\u0000\u042d\u042e\u0006Y\f\u0000\u042e\u00c5\u0001"+ + "\u0000\u0000\u0000\u042f\u0430\u0003J\u001c\u0000\u0430\u0431\u0001\u0000"+ + "\u0000\u0000\u0431\u0432\u0006Z\f\u0000\u0432\u00c7\u0001\u0000\u0000"+ + "\u0000\u0433\u0434\u0003\u00b8S\u0000\u0434\u0435\u0001\u0000\u0000\u0000"+ + "\u0435\u0436\u0006[\u000f\u0000\u0436\u0437\u0006[\u0010\u0000\u0437\u00c9"+ + "\u0001\u0000\u0000\u0000\u0438\u0439\u0003L\u001d\u0000\u0439\u043a\u0001"+ + "\u0000\u0000\u0000\u043a\u043b\u0006\\\u0011\u0000\u043b\u043c\u0006\\"+ + "\r\u0000\u043c\u00cb\u0001\u0000\u0000\u0000\u043d\u043e\u0003J\u001c"+ + "\u0000\u043e\u043f\u0001\u0000\u0000\u0000\u043f\u0440\u0006]\f\u0000"+ + "\u0440\u00cd\u0001\u0000\u0000\u0000\u0441\u0442\u0003F\u001a\u0000\u0442"+ + "\u0443\u0001\u0000\u0000\u0000\u0443\u0444\u0006^\f\u0000\u0444\u00cf"+ + "\u0001\u0000\u0000\u0000\u0445\u0446\u0003H\u001b\u0000\u0446\u0447\u0001"+ + "\u0000\u0000\u0000\u0447\u0448\u0006_\f\u0000\u0448\u00d1\u0001\u0000"+ + "\u0000\u0000\u0449\u044a\u0003L\u001d\u0000\u044a\u044b\u0001\u0000\u0000"+ + "\u0000\u044b\u044c\u0006`\u0011\u0000\u044c\u044d\u0006`\r\u0000\u044d"+ + "\u00d3\u0001\u0000\u0000\u0000\u044e\u044f\u0003\u00b8S\u0000\u044f\u0450"+ + "\u0001\u0000\u0000\u0000\u0450\u0451\u0006a\u000f\u0000\u0451\u00d5\u0001"+ + "\u0000\u0000\u0000\u0452\u0453\u0003\u00baT\u0000\u0453\u0454\u0001\u0000"+ + "\u0000\u0000\u0454\u0455\u0006b\u0012\u0000\u0455\u00d7\u0001\u0000\u0000"+ + "\u0000\u0456\u0457\u0003r0\u0000\u0457\u0458\u0001\u0000\u0000\u0000\u0458"+ + "\u0459\u0006c\u0013\u0000\u0459\u00d9\u0001\u0000\u0000\u0000\u045a\u045b"+ + "\u0003t1\u0000\u045b\u045c\u0001\u0000\u0000\u0000\u045c\u045d\u0006d"+ + "\u0014\u0000\u045d\u00db\u0001\u0000\u0000\u0000\u045e\u045f\u0003n.\u0000"+ + "\u045f\u0460\u0001\u0000\u0000\u0000\u0460\u0461\u0006e\u0015\u0000\u0461"+ + "\u00dd\u0001\u0000\u0000\u0000\u0462\u0463\u0007\u0010\u0000\u0000\u0463"+ + "\u0464\u0007\u0003\u0000\u0000\u0464\u0465\u0007\u0005\u0000\u0000\u0465"+ + "\u0466\u0007\f\u0000\u0000\u0466\u0467\u0007\u0000\u0000\u0000\u0467\u0468"+ + "\u0007\f\u0000\u0000\u0468\u0469\u0007\u0005\u0000\u0000\u0469\u046a\u0007"+ + "\f\u0000\u0000\u046a\u00df\u0001\u0000\u0000\u0000\u046b\u046f\b \u0000"+ + "\u0000\u046c\u046d\u0005/\u0000\u0000\u046d\u046f\b!\u0000\u0000\u046e"+ + "\u046b\u0001\u0000\u0000\u0000\u046e\u046c\u0001\u0000\u0000\u0000\u046f"+ + "\u00e1\u0001\u0000\u0000\u0000\u0470\u0472\u0003\u00e0g\u0000\u0471\u0470"+ + "\u0001\u0000\u0000\u0000\u0472\u0473\u0001\u0000\u0000\u0000\u0473\u0471"+ + "\u0001\u0000\u0000\u0000\u0473\u0474\u0001\u0000\u0000\u0000\u0474\u00e3"+ + "\u0001\u0000\u0000\u0000\u0475\u0476\u0003\u00e2h\u0000\u0476\u0477\u0001"+ + "\u0000\u0000\u0000\u0477\u0478\u0006i\u0016\u0000\u0478\u00e5\u0001\u0000"+ + "\u0000\u0000\u0479\u047a\u0003b(\u0000\u047a\u047b\u0001\u0000\u0000\u0000"+ + "\u047b\u047c\u0006j\u0017\u0000\u047c\u00e7\u0001\u0000\u0000\u0000\u047d"+ + "\u047e\u0003F\u001a\u0000\u047e\u047f\u0001\u0000\u0000\u0000\u047f\u0480"+ + "\u0006k\f\u0000\u0480\u00e9\u0001\u0000\u0000\u0000\u0481\u0482\u0003"+ + "H\u001b\u0000\u0482\u0483\u0001\u0000\u0000\u0000\u0483\u0484\u0006l\f"+ + "\u0000\u0484\u00eb\u0001\u0000\u0000\u0000\u0485\u0486\u0003J\u001c\u0000"+ + "\u0486\u0487\u0001\u0000\u0000\u0000\u0487\u0488\u0006m\f\u0000\u0488"+ + "\u00ed\u0001\u0000\u0000\u0000\u0489\u048a\u0003L\u001d\u0000\u048a\u048b"+ + "\u0001\u0000\u0000\u0000\u048b\u048c\u0006n\u0011\u0000\u048c\u048d\u0006"+ + "n\r\u0000\u048d\u00ef\u0001\u0000\u0000\u0000\u048e\u048f\u0003x3\u0000"+ + "\u048f\u0490\u0001\u0000\u0000\u0000\u0490\u0491\u0006o\u0018\u0000\u0491"+ + "\u00f1\u0001\u0000\u0000\u0000\u0492\u0493\u0003t1\u0000\u0493\u0494\u0001"+ + "\u0000\u0000\u0000\u0494\u0495\u0006p\u0014\u0000\u0495\u00f3\u0001\u0000"+ + "\u0000\u0000\u0496\u0497\u0003\u0090?\u0000\u0497\u0498\u0001\u0000\u0000"+ + "\u0000\u0498\u0499\u0006q\u0019\u0000\u0499\u00f5\u0001\u0000\u0000\u0000"+ + "\u049a\u049b\u0003\u00b6R\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c"+ + "\u049d\u0006r\u001a\u0000\u049d\u00f7\u0001\u0000\u0000\u0000\u049e\u04a3"+ + "\u0003P\u001f\u0000\u049f\u04a3\u0003N\u001e\u0000\u04a0\u04a3\u0003^"+ + "&\u0000\u04a1\u04a3\u0003\u00aaL\u0000\u04a2\u049e\u0001\u0000\u0000\u0000"+ + "\u04a2\u049f\u0001\u0000\u0000\u0000\u04a2\u04a0\u0001\u0000\u0000\u0000"+ + "\u04a2\u04a1\u0001\u0000\u0000\u0000\u04a3\u00f9\u0001\u0000\u0000\u0000"+ + "\u04a4\u04a7\u0003P\u001f\u0000\u04a5\u04a7\u0003\u00aaL\u0000\u04a6\u04a4"+ + "\u0001\u0000\u0000\u0000\u04a6\u04a5\u0001\u0000\u0000\u0000\u04a7\u04ab"+ + "\u0001\u0000\u0000\u0000\u04a8\u04aa\u0003\u00f8s\u0000\u04a9\u04a8\u0001"+ + "\u0000\u0000\u0000\u04aa\u04ad\u0001\u0000\u0000\u0000\u04ab\u04a9\u0001"+ + "\u0000\u0000\u0000\u04ab\u04ac\u0001\u0000\u0000\u0000\u04ac\u04b8\u0001"+ + "\u0000\u0000\u0000\u04ad\u04ab\u0001\u0000\u0000\u0000\u04ae\u04b1\u0003"+ + "^&\u0000\u04af\u04b1\u0003X#\u0000\u04b0\u04ae\u0001\u0000\u0000\u0000"+ + "\u04b0\u04af\u0001\u0000\u0000\u0000\u04b1\u04b3\u0001\u0000\u0000\u0000"+ + "\u04b2\u04b4\u0003\u00f8s\u0000\u04b3\u04b2\u0001\u0000\u0000\u0000\u04b4"+ + "\u04b5\u0001\u0000\u0000\u0000\u04b5\u04b3\u0001\u0000\u0000\u0000\u04b5"+ + "\u04b6\u0001\u0000\u0000\u0000\u04b6\u04b8\u0001\u0000\u0000\u0000\u04b7"+ + "\u04a6\u0001\u0000\u0000\u0000\u04b7\u04b0\u0001\u0000\u0000\u0000\u04b8"+ + "\u00fb\u0001\u0000\u0000\u0000\u04b9\u04bc\u0003\u00fat\u0000\u04ba\u04bc"+ + "\u0003\u00beV\u0000\u04bb\u04b9\u0001\u0000\u0000\u0000\u04bb\u04ba\u0001"+ + "\u0000\u0000\u0000\u04bc\u04bd\u0001\u0000\u0000\u0000\u04bd\u04bb\u0001"+ + "\u0000\u0000\u0000\u04bd\u04be\u0001\u0000\u0000\u0000\u04be\u00fd\u0001"+ + "\u0000\u0000\u0000\u04bf\u04c0\u0003F\u001a\u0000\u04c0\u04c1\u0001\u0000"+ + "\u0000\u0000\u04c1\u04c2\u0006v\f\u0000\u04c2\u00ff\u0001\u0000\u0000"+ + "\u0000\u04c3\u04c4\u0003H\u001b\u0000\u04c4\u04c5\u0001\u0000\u0000\u0000"+ + "\u04c5\u04c6\u0006w\f\u0000\u04c6\u0101\u0001\u0000\u0000\u0000\u04c7"+ + "\u04c8\u0003J\u001c\u0000\u04c8\u04c9\u0001\u0000\u0000\u0000\u04c9\u04ca"+ + "\u0006x\f\u0000\u04ca\u0103\u0001\u0000\u0000\u0000\u04cb\u04cc\u0003"+ + "L\u001d\u0000\u04cc\u04cd\u0001\u0000\u0000\u0000\u04cd\u04ce\u0006y\u0011"+ + "\u0000\u04ce\u04cf\u0006y\r\u0000\u04cf\u0105\u0001\u0000\u0000\u0000"+ + "\u04d0\u04d1\u0003n.\u0000\u04d1\u04d2\u0001\u0000\u0000\u0000\u04d2\u04d3"+ + "\u0006z\u0015\u0000\u04d3\u0107\u0001\u0000\u0000\u0000\u04d4\u04d5\u0003"+ + "t1\u0000\u04d5\u04d6\u0001\u0000\u0000\u0000\u04d6\u04d7\u0006{\u0014"+ + "\u0000\u04d7\u0109\u0001\u0000\u0000\u0000\u04d8\u04d9\u0003x3\u0000\u04d9"+ + "\u04da\u0001\u0000\u0000\u0000\u04da\u04db\u0006|\u0018\u0000\u04db\u010b"+ + "\u0001\u0000\u0000\u0000\u04dc\u04dd\u0003\u0090?\u0000\u04dd\u04de\u0001"+ + "\u0000\u0000\u0000\u04de\u04df\u0006}\u0019\u0000\u04df\u010d\u0001\u0000"+ + "\u0000\u0000\u04e0\u04e1\u0003\u00b6R\u0000\u04e1\u04e2\u0001\u0000\u0000"+ + "\u0000\u04e2\u04e3\u0006~\u001a\u0000\u04e3\u010f\u0001\u0000\u0000\u0000"+ + "\u04e4\u04e5\u0007\f\u0000\u0000\u04e5\u04e6\u0007\u0002\u0000\u0000\u04e6"+ + "\u0111\u0001\u0000\u0000\u0000\u04e7\u04e8\u0003\u00fcu\u0000\u04e8\u04e9"+ + "\u0001\u0000\u0000\u0000\u04e9\u04ea\u0006\u0080\u001b\u0000\u04ea\u0113"+ + "\u0001\u0000\u0000\u0000\u04eb\u04ec\u0003F\u001a\u0000\u04ec\u04ed\u0001"+ + "\u0000\u0000\u0000\u04ed\u04ee\u0006\u0081\f\u0000\u04ee\u0115\u0001\u0000"+ + "\u0000\u0000\u04ef\u04f0\u0003H\u001b\u0000\u04f0\u04f1\u0001\u0000\u0000"+ + "\u0000\u04f1\u04f2\u0006\u0082\f\u0000\u04f2\u0117\u0001\u0000\u0000\u0000"+ + "\u04f3\u04f4\u0003J\u001c\u0000\u04f4\u04f5\u0001\u0000\u0000\u0000\u04f5"+ + "\u04f6\u0006\u0083\f\u0000\u04f6\u0119\u0001\u0000\u0000\u0000\u04f7\u04f8"+ + "\u0003L\u001d\u0000\u04f8\u04f9\u0001\u0000\u0000\u0000\u04f9\u04fa\u0006"+ + "\u0084\u0011\u0000\u04fa\u04fb\u0006\u0084\r\u0000\u04fb\u011b\u0001\u0000"+ + "\u0000\u0000\u04fc\u04fd\u0003\u00b8S\u0000\u04fd\u04fe\u0001\u0000\u0000"+ + "\u0000\u04fe\u04ff\u0006\u0085\u000f\u0000\u04ff\u0500\u0006\u0085\u001c"+ + "\u0000\u0500\u011d\u0001\u0000\u0000\u0000\u0501\u0502\u0007\u0007\u0000"+ + "\u0000\u0502\u0503\u0007\t\u0000\u0000\u0503\u0504\u0001\u0000\u0000\u0000"+ + "\u0504\u0505\u0006\u0086\u001d\u0000\u0505\u011f\u0001\u0000\u0000\u0000"+ + "\u0506\u0507\u0007\u0013\u0000\u0000\u0507\u0508\u0007\u0001\u0000\u0000"+ + "\u0508\u0509\u0007\u0005\u0000\u0000\u0509\u050a\u0007\n\u0000\u0000\u050a"+ + "\u050b\u0001\u0000\u0000\u0000\u050b\u050c\u0006\u0087\u001d\u0000\u050c"+ + "\u0121\u0001\u0000\u0000\u0000\u050d\u050e\b\"\u0000\u0000\u050e\u0123"+ + "\u0001\u0000\u0000\u0000\u050f\u0511\u0003\u0122\u0088\u0000\u0510\u050f"+ + "\u0001\u0000\u0000\u0000\u0511\u0512\u0001\u0000\u0000\u0000\u0512\u0510"+ + "\u0001\u0000\u0000\u0000\u0512\u0513\u0001\u0000\u0000\u0000\u0513\u0514"+ + "\u0001\u0000\u0000\u0000\u0514\u0515\u0003r0\u0000\u0515\u0517\u0001\u0000"+ + "\u0000\u0000\u0516\u0510\u0001\u0000\u0000\u0000\u0516\u0517\u0001\u0000"+ + "\u0000\u0000\u0517\u0519\u0001\u0000\u0000\u0000\u0518\u051a\u0003\u0122"+ + "\u0088\u0000\u0519\u0518\u0001\u0000\u0000\u0000\u051a\u051b\u0001\u0000"+ + "\u0000\u0000\u051b\u0519\u0001\u0000\u0000\u0000\u051b\u051c\u0001\u0000"+ + "\u0000\u0000\u051c\u0125\u0001\u0000\u0000\u0000\u051d\u051e\u0003\u0124"+ + "\u0089\u0000\u051e\u051f\u0001\u0000\u0000\u0000\u051f\u0520\u0006\u008a"+ + "\u001e\u0000\u0520\u0127\u0001\u0000\u0000\u0000\u0521\u0522\u0003F\u001a"+ + "\u0000\u0522\u0523\u0001\u0000\u0000\u0000\u0523\u0524\u0006\u008b\f\u0000"+ + "\u0524\u0129\u0001\u0000\u0000\u0000\u0525\u0526\u0003H\u001b\u0000\u0526"+ + "\u0527\u0001\u0000\u0000\u0000\u0527\u0528\u0006\u008c\f\u0000\u0528\u012b"+ + "\u0001\u0000\u0000\u0000\u0529\u052a\u0003J\u001c\u0000\u052a\u052b\u0001"+ + "\u0000\u0000\u0000\u052b\u052c\u0006\u008d\f\u0000\u052c\u012d\u0001\u0000"+ + "\u0000\u0000\u052d\u052e\u0003L\u001d\u0000\u052e\u052f\u0001\u0000\u0000"+ + "\u0000\u052f\u0530\u0006\u008e\u0011\u0000\u0530\u0531\u0006\u008e\r\u0000"+ + "\u0531\u0532\u0006\u008e\r\u0000\u0532\u012f\u0001\u0000\u0000\u0000\u0533"+ + "\u0534\u0003n.\u0000\u0534\u0535\u0001\u0000\u0000\u0000\u0535\u0536\u0006"+ + "\u008f\u0015\u0000\u0536\u0131\u0001\u0000\u0000\u0000\u0537\u0538\u0003"+ + "t1\u0000\u0538\u0539\u0001\u0000\u0000\u0000\u0539\u053a\u0006\u0090\u0014"+ + "\u0000\u053a\u0133\u0001\u0000\u0000\u0000\u053b\u053c\u0003x3\u0000\u053c"+ + "\u053d\u0001\u0000\u0000\u0000\u053d\u053e\u0006\u0091\u0018\u0000\u053e"+ + "\u0135\u0001\u0000\u0000\u0000\u053f\u0540\u0003\u0120\u0087\u0000\u0540"+ + "\u0541\u0001\u0000\u0000\u0000\u0541\u0542\u0006\u0092\u001f\u0000\u0542"+ + "\u0137\u0001\u0000\u0000\u0000\u0543\u0544\u0003\u00fcu\u0000\u0544\u0545"+ + "\u0001\u0000\u0000\u0000\u0545\u0546\u0006\u0093\u001b\u0000\u0546\u0139"+ + "\u0001\u0000\u0000\u0000\u0547\u0548\u0003\u00c0W\u0000\u0548\u0549\u0001"+ + "\u0000\u0000\u0000\u0549\u054a\u0006\u0094 \u0000\u054a\u013b\u0001\u0000"+ + "\u0000\u0000\u054b\u054c\u0003\u0090?\u0000\u054c\u054d\u0001\u0000\u0000"+ + "\u0000\u054d\u054e\u0006\u0095\u0019\u0000\u054e\u013d\u0001\u0000\u0000"+ + "\u0000\u054f\u0550\u0003\u00b6R\u0000\u0550\u0551\u0001\u0000\u0000\u0000"+ + "\u0551\u0552\u0006\u0096\u001a\u0000\u0552\u013f\u0001\u0000\u0000\u0000"+ + "\u0553\u0554\u0003F\u001a\u0000\u0554\u0555\u0001\u0000\u0000\u0000\u0555"+ + "\u0556\u0006\u0097\f\u0000\u0556\u0141\u0001\u0000\u0000\u0000\u0557\u0558"+ + "\u0003H\u001b\u0000\u0558\u0559\u0001\u0000\u0000\u0000\u0559\u055a\u0006"+ + "\u0098\f\u0000\u055a\u0143\u0001\u0000\u0000\u0000\u055b\u055c\u0003J"+ + "\u001c\u0000\u055c\u055d\u0001\u0000\u0000\u0000\u055d\u055e\u0006\u0099"+ + "\f\u0000\u055e\u0145\u0001\u0000\u0000\u0000\u055f\u0560\u0003L\u001d"+ + "\u0000\u0560\u0561\u0001\u0000\u0000\u0000\u0561\u0562\u0006\u009a\u0011"+ + "\u0000\u0562\u0563\u0006\u009a\r\u0000\u0563\u0147\u0001\u0000\u0000\u0000"+ + "\u0564\u0565\u0003x3\u0000\u0565\u0566\u0001\u0000\u0000\u0000\u0566\u0567"+ + "\u0006\u009b\u0018\u0000\u0567\u0149\u0001\u0000\u0000\u0000\u0568\u0569"+ + "\u0003\u0090?\u0000\u0569\u056a\u0001\u0000\u0000\u0000\u056a\u056b\u0006"+ + "\u009c\u0019\u0000\u056b\u014b\u0001\u0000\u0000\u0000\u056c\u056d\u0003"+ + "\u00b6R\u0000\u056d\u056e\u0001\u0000\u0000\u0000\u056e\u056f\u0006\u009d"+ + "\u001a\u0000\u056f\u014d\u0001\u0000\u0000\u0000\u0570\u0571\u0003\u00c0"+ + "W\u0000\u0571\u0572\u0001\u0000\u0000\u0000\u0572\u0573\u0006\u009e \u0000"+ + "\u0573\u014f\u0001\u0000\u0000\u0000\u0574\u0575\u0003\u00bcU\u0000\u0575"+ + "\u0576\u0001\u0000\u0000\u0000\u0576\u0577\u0006\u009f!\u0000\u0577\u0151"+ + "\u0001\u0000\u0000\u0000\u0578\u0579\u0003F\u001a\u0000\u0579\u057a\u0001"+ + "\u0000\u0000\u0000\u057a\u057b\u0006\u00a0\f\u0000\u057b\u0153\u0001\u0000"+ + "\u0000\u0000\u057c\u057d\u0003H\u001b\u0000\u057d\u057e\u0001\u0000\u0000"+ + "\u0000\u057e\u057f\u0006\u00a1\f\u0000\u057f\u0155\u0001\u0000\u0000\u0000"+ + "\u0580\u0581\u0003J\u001c\u0000\u0581\u0582\u0001\u0000\u0000\u0000\u0582"+ + "\u0583\u0006\u00a2\f\u0000\u0583\u0157\u0001\u0000\u0000\u0000\u0584\u0585"+ + "\u0003L\u001d\u0000\u0585\u0586\u0001\u0000\u0000\u0000\u0586\u0587\u0006"+ + "\u00a3\u0011\u0000\u0587\u0588\u0006\u00a3\r\u0000\u0588\u0159\u0001\u0000"+ + "\u0000\u0000\u0589\u058a\u0007\u0001\u0000\u0000\u058a\u058b\u0007\t\u0000"+ + "\u0000\u058b\u058c\u0007\u000f\u0000\u0000\u058c\u058d\u0007\u0007\u0000"+ + "\u0000\u058d\u015b\u0001\u0000\u0000\u0000\u058e\u058f\u0003F\u001a\u0000"+ + "\u058f\u0590\u0001\u0000\u0000\u0000\u0590\u0591\u0006\u00a5\f\u0000\u0591"+ + "\u015d\u0001\u0000\u0000\u0000\u0592\u0593\u0003H\u001b\u0000\u0593\u0594"+ + "\u0001\u0000\u0000\u0000\u0594\u0595\u0006\u00a6\f\u0000\u0595\u015f\u0001"+ + "\u0000\u0000\u0000\u0596\u0597\u0003J\u001c\u0000\u0597\u0598\u0001\u0000"+ + "\u0000\u0000\u0598\u0599\u0006\u00a7\f\u0000\u0599\u0161\u0001\u0000\u0000"+ + "\u0000\u059a\u059b\u0003\u00baT\u0000\u059b\u059c\u0001\u0000\u0000\u0000"+ + "\u059c\u059d\u0006\u00a8\u0012\u0000\u059d\u059e\u0006\u00a8\r\u0000\u059e"+ + "\u0163\u0001\u0000\u0000\u0000\u059f\u05a0\u0003r0\u0000\u05a0\u05a1\u0001"+ + "\u0000\u0000\u0000\u05a1\u05a2\u0006\u00a9\u0013\u0000\u05a2\u0165\u0001"+ + "\u0000\u0000\u0000\u05a3\u05a9\u0003X#\u0000\u05a4\u05a9\u0003N\u001e"+ + "\u0000\u05a5\u05a9\u0003x3\u0000\u05a6\u05a9\u0003P\u001f\u0000\u05a7"+ + "\u05a9\u0003^&\u0000\u05a8\u05a3\u0001\u0000\u0000\u0000\u05a8\u05a4\u0001"+ + "\u0000\u0000\u0000\u05a8\u05a5\u0001\u0000\u0000\u0000\u05a8\u05a6\u0001"+ + "\u0000\u0000\u0000\u05a8\u05a7\u0001\u0000\u0000\u0000\u05a9\u05aa\u0001"+ + "\u0000\u0000\u0000\u05aa\u05a8\u0001\u0000\u0000\u0000\u05aa\u05ab\u0001"+ + "\u0000\u0000\u0000\u05ab\u0167\u0001\u0000\u0000\u0000\u05ac\u05ad\u0003"+ + "F\u001a\u0000\u05ad\u05ae\u0001\u0000\u0000\u0000\u05ae\u05af\u0006\u00ab"+ + "\f\u0000\u05af\u0169\u0001\u0000\u0000\u0000\u05b0\u05b1\u0003H\u001b"+ + "\u0000\u05b1\u05b2\u0001\u0000\u0000\u0000\u05b2\u05b3\u0006\u00ac\f\u0000"+ + "\u05b3\u016b\u0001\u0000\u0000\u0000\u05b4\u05b5\u0003J\u001c\u0000\u05b5"+ + "\u05b6\u0001\u0000\u0000\u0000\u05b6\u05b7\u0006\u00ad\f\u0000\u05b7\u016d"+ + "\u0001\u0000\u0000\u0000\u05b8\u05b9\u0003L\u001d\u0000\u05b9\u05ba\u0001"+ + "\u0000\u0000\u0000\u05ba\u05bb\u0006\u00ae\u0011\u0000\u05bb\u05bc\u0006"+ + "\u00ae\r\u0000\u05bc\u016f\u0001\u0000\u0000\u0000\u05bd\u05be\u0003r"+ + "0\u0000\u05be\u05bf\u0001\u0000\u0000\u0000\u05bf\u05c0\u0006\u00af\u0013"+ + "\u0000\u05c0\u0171\u0001\u0000\u0000\u0000\u05c1\u05c2\u0003t1\u0000\u05c2"+ + "\u05c3\u0001\u0000\u0000\u0000\u05c3\u05c4\u0006\u00b0\u0014\u0000\u05c4"+ + "\u0173\u0001\u0000\u0000\u0000\u05c5\u05c6\u0003x3\u0000\u05c6\u05c7\u0001"+ + "\u0000\u0000\u0000\u05c7\u05c8\u0006\u00b1\u0018\u0000\u05c8\u0175\u0001"+ + "\u0000\u0000\u0000\u05c9\u05ca\u0003\u011e\u0086\u0000\u05ca\u05cb\u0001"+ + "\u0000\u0000\u0000\u05cb\u05cc\u0006\u00b2\"\u0000\u05cc\u05cd\u0006\u00b2"+ + "#\u0000\u05cd\u0177\u0001\u0000\u0000\u0000\u05ce\u05cf\u0003\u00e2h\u0000"+ + "\u05cf\u05d0\u0001\u0000\u0000\u0000\u05d0\u05d1\u0006\u00b3\u0016\u0000"+ + "\u05d1\u0179\u0001\u0000\u0000\u0000\u05d2\u05d3\u0003b(\u0000\u05d3\u05d4"+ + "\u0001\u0000\u0000\u0000\u05d4\u05d5\u0006\u00b4\u0017\u0000\u05d5\u017b"+ + "\u0001\u0000\u0000\u0000\u05d6\u05d7\u0003F\u001a\u0000\u05d7\u05d8\u0001"+ + "\u0000\u0000\u0000\u05d8\u05d9\u0006\u00b5\f\u0000\u05d9\u017d\u0001\u0000"+ + "\u0000\u0000\u05da\u05db\u0003H\u001b\u0000\u05db\u05dc\u0001\u0000\u0000"+ + "\u0000\u05dc\u05dd\u0006\u00b6\f\u0000\u05dd\u017f\u0001\u0000\u0000\u0000"+ + "\u05de\u05df\u0003J\u001c\u0000\u05df\u05e0\u0001\u0000\u0000\u0000\u05e0"+ + "\u05e1\u0006\u00b7\f\u0000\u05e1\u0181\u0001\u0000\u0000\u0000\u05e2\u05e3"+ + "\u0003L\u001d\u0000\u05e3\u05e4\u0001\u0000\u0000\u0000\u05e4\u05e5\u0006"+ + "\u00b8\u0011\u0000\u05e5\u05e6\u0006\u00b8\r\u0000\u05e6\u05e7\u0006\u00b8"+ + "\r\u0000\u05e7\u0183\u0001\u0000\u0000\u0000\u05e8\u05e9\u0003t1\u0000"+ + "\u05e9\u05ea\u0001\u0000\u0000\u0000\u05ea\u05eb\u0006\u00b9\u0014\u0000"+ + "\u05eb\u0185\u0001\u0000\u0000\u0000\u05ec\u05ed\u0003x3\u0000\u05ed\u05ee"+ + "\u0001\u0000\u0000\u0000\u05ee\u05ef\u0006\u00ba\u0018\u0000\u05ef\u0187"+ + "\u0001\u0000\u0000\u0000\u05f0\u05f1\u0003\u00fcu\u0000\u05f1\u05f2\u0001"+ + "\u0000\u0000\u0000\u05f2\u05f3\u0006\u00bb\u001b\u0000\u05f3\u0189\u0001"+ + "\u0000\u0000\u0000\u05f4\u05f5\u0003F\u001a\u0000\u05f5\u05f6\u0001\u0000"+ + "\u0000\u0000\u05f6\u05f7\u0006\u00bc\f\u0000\u05f7\u018b\u0001\u0000\u0000"+ + "\u0000\u05f8\u05f9\u0003H\u001b\u0000\u05f9\u05fa\u0001\u0000\u0000\u0000"+ + "\u05fa\u05fb\u0006\u00bd\f\u0000\u05fb\u018d\u0001\u0000\u0000\u0000\u05fc"+ + "\u05fd\u0003J\u001c\u0000\u05fd\u05fe\u0001\u0000\u0000\u0000\u05fe\u05ff"+ + "\u0006\u00be\f\u0000\u05ff\u018f\u0001\u0000\u0000\u0000\u0600\u0601\u0003"+ + "L\u001d\u0000\u0601\u0602\u0001\u0000\u0000\u0000\u0602\u0603\u0006\u00bf"+ + "\u0011\u0000\u0603\u0604\u0006\u00bf\r\u0000\u0604\u0191\u0001\u0000\u0000"+ + "\u0000\u0605\u0606\u0007#\u0000\u0000\u0606\u0607\u0007\u0007\u0000\u0000"+ + "\u0607\u0608\u0007\u0001\u0000\u0000\u0608\u0609\u0007\t\u0000\u0000\u0609"+ + "\u0193\u0001\u0000\u0000\u0000\u060a\u060b\u0003\u0110\u007f\u0000\u060b"+ + "\u060c\u0001\u0000\u0000\u0000\u060c\u060d\u0006\u00c1$\u0000\u060d\u0195"+ + "\u0001\u0000\u0000\u0000\u060e\u060f\u0003\u011e\u0086\u0000\u060f\u0610"+ + "\u0001\u0000\u0000\u0000\u0610\u0611\u0006\u00c2\"\u0000\u0611\u0612\u0006"+ + "\u00c2\r\u0000\u0612\u0613\u0006\u00c2\u0000\u0000\u0613\u0197\u0001\u0000"+ + "\u0000\u0000\u0614\u0615\u0007\u0014\u0000\u0000\u0615\u0616\u0007\u0002"+ + "\u0000\u0000\u0616\u0617\u0007\u0001\u0000\u0000\u0617\u0618\u0007\t\u0000"+ + "\u0000\u0618\u0619\u0007\u0011\u0000\u0000\u0619\u061a\u0001\u0000\u0000"+ + "\u0000\u061a\u061b\u0006\u00c3\r\u0000\u061b\u061c\u0006\u00c3\u0000\u0000"+ + "\u061c\u0199\u0001\u0000\u0000\u0000\u061d\u061e\u0003\u00e2h\u0000\u061e"+ + "\u061f\u0001\u0000\u0000\u0000\u061f\u0620\u0006\u00c4\u0016\u0000\u0620"+ + "\u019b\u0001\u0000\u0000\u0000\u0621\u0622\u0003b(\u0000\u0622\u0623\u0001"+ + "\u0000\u0000\u0000\u0623\u0624\u0006\u00c5\u0017\u0000\u0624\u019d\u0001"+ + "\u0000\u0000\u0000\u0625\u0626\u0003r0\u0000\u0626\u0627\u0001\u0000\u0000"+ + "\u0000\u0627\u0628\u0006\u00c6\u0013\u0000\u0628\u019f\u0001\u0000\u0000"+ + "\u0000\u0629\u062a\u0003\u00bcU\u0000\u062a\u062b\u0001\u0000\u0000\u0000"+ + "\u062b\u062c\u0006\u00c7!\u0000\u062c\u01a1\u0001\u0000\u0000\u0000\u062d"+ + "\u062e\u0003\u00c0W\u0000\u062e\u062f\u0001\u0000\u0000\u0000\u062f\u0630"+ + "\u0006\u00c8 \u0000\u0630\u01a3\u0001\u0000\u0000\u0000\u0631\u0632\u0003"+ + "F\u001a\u0000\u0632\u0633\u0001\u0000\u0000\u0000\u0633\u0634\u0006\u00c9"+ + "\f\u0000\u0634\u01a5\u0001\u0000\u0000\u0000\u0635\u0636\u0003H\u001b"+ + "\u0000\u0636\u0637\u0001\u0000\u0000\u0000\u0637\u0638\u0006\u00ca\f\u0000"+ + "\u0638\u01a7\u0001\u0000\u0000\u0000\u0639\u063a\u0003J\u001c\u0000\u063a"+ + "\u063b\u0001\u0000\u0000\u0000\u063b\u063c\u0006\u00cb\f\u0000\u063c\u01a9"+ + "\u0001\u0000\u0000\u0000\u063d\u063e\u0003L\u001d\u0000\u063e\u063f\u0001"+ + "\u0000\u0000\u0000\u063f\u0640\u0006\u00cc\u0011\u0000\u0640\u0641\u0006"+ + "\u00cc\r\u0000\u0641\u01ab\u0001\u0000\u0000\u0000\u0642\u0643\u0003\u00e2"+ + "h\u0000\u0643\u0644\u0001\u0000\u0000\u0000\u0644\u0645\u0006\u00cd\u0016"+ + "\u0000\u0645\u0646\u0006\u00cd\r\u0000\u0646\u0647\u0006\u00cd%\u0000"+ + "\u0647\u01ad\u0001\u0000\u0000\u0000\u0648\u0649\u0003b(\u0000\u0649\u064a"+ + "\u0001\u0000\u0000\u0000\u064a\u064b\u0006\u00ce\u0017\u0000\u064b\u064c"+ + "\u0006\u00ce\r\u0000\u064c\u064d\u0006\u00ce%\u0000\u064d\u01af\u0001"+ + "\u0000\u0000\u0000\u064e\u064f\u0003F\u001a\u0000\u064f\u0650\u0001\u0000"+ + "\u0000\u0000\u0650\u0651\u0006\u00cf\f\u0000\u0651\u01b1\u0001\u0000\u0000"+ + "\u0000\u0652\u0653\u0003H\u001b\u0000\u0653\u0654\u0001\u0000\u0000\u0000"+ + "\u0654\u0655\u0006\u00d0\f\u0000\u0655\u01b3\u0001\u0000\u0000\u0000\u0656"+ + "\u0657\u0003J\u001c\u0000\u0657\u0658\u0001\u0000\u0000\u0000\u0658\u0659"+ + "\u0006\u00d1\f\u0000\u0659\u01b5\u0001\u0000\u0000\u0000\u065a\u065b\u0003"+ + "r0\u0000\u065b\u065c\u0001\u0000\u0000\u0000\u065c\u065d\u0006\u00d2\u0013"+ + "\u0000\u065d\u065e\u0006\u00d2\r\u0000\u065e\u065f\u0006\u00d2\u000b\u0000"+ + "\u065f\u01b7\u0001\u0000\u0000\u0000\u0660\u0661\u0003t1\u0000\u0661\u0662"+ + "\u0001\u0000\u0000\u0000\u0662\u0663\u0006\u00d3\u0014\u0000\u0663\u0664"+ + "\u0006\u00d3\r\u0000\u0664\u0665\u0006\u00d3\u000b\u0000\u0665\u01b9\u0001"+ + "\u0000\u0000\u0000\u0666\u0667\u0003F\u001a\u0000\u0667\u0668\u0001\u0000"+ + "\u0000\u0000\u0668\u0669\u0006\u00d4\f\u0000\u0669\u01bb\u0001\u0000\u0000"+ + "\u0000\u066a\u066b\u0003H\u001b\u0000\u066b\u066c\u0001\u0000\u0000\u0000"+ + "\u066c\u066d\u0006\u00d5\f\u0000\u066d\u01bd\u0001\u0000\u0000\u0000\u066e"+ + "\u066f\u0003J\u001c\u0000\u066f\u0670\u0001\u0000\u0000\u0000\u0670\u0671"+ + "\u0006\u00d6\f\u0000\u0671\u01bf\u0001\u0000\u0000\u0000\u0672\u0673\u0003"+ + "\u00c0W\u0000\u0673\u0674\u0001\u0000\u0000\u0000\u0674\u0675\u0006\u00d7"+ + "\r\u0000\u0675\u0676\u0006\u00d7\u0000\u0000\u0676\u0677\u0006\u00d7 "+ + "\u0000\u0677\u01c1\u0001\u0000\u0000\u0000\u0678\u0679\u0003\u00bcU\u0000"+ + "\u0679\u067a\u0001\u0000\u0000\u0000\u067a\u067b\u0006\u00d8\r\u0000\u067b"+ + "\u067c\u0006\u00d8\u0000\u0000\u067c\u067d\u0006\u00d8!\u0000\u067d\u01c3"+ + "\u0001\u0000\u0000\u0000\u067e\u067f\u0003h+\u0000\u067f\u0680\u0001\u0000"+ + "\u0000\u0000\u0680\u0681\u0006\u00d9\r\u0000\u0681\u0682\u0006\u00d9\u0000"+ + "\u0000\u0682\u0683\u0006\u00d9&\u0000\u0683\u01c5\u0001\u0000\u0000\u0000"+ + "\u0684\u0685\u0003L\u001d\u0000\u0685\u0686\u0001\u0000\u0000\u0000\u0686"+ + "\u0687\u0006\u00da\u0011\u0000\u0687\u0688\u0006\u00da\r\u0000\u0688\u01c7"+ + "\u0001\u0000\u0000\u0000\u0689\u068a\u0003L\u001d\u0000\u068a\u068b\u0001"+ + "\u0000\u0000\u0000\u068b\u068c\u0006\u00db\u0011\u0000\u068c\u068d\u0006"+ + "\u00db\r\u0000\u068d\u01c9\u0001\u0000\u0000\u0000\u068e\u068f\u0003\u011e"+ + "\u0086\u0000\u068f\u0690\u0001\u0000\u0000\u0000\u0690\u0691\u0006\u00dc"+ + "\"\u0000\u0691\u01cb\u0001\u0000\u0000\u0000\u0692\u0693\u0003\u0110\u007f"+ + "\u0000\u0693\u0694\u0001\u0000\u0000\u0000\u0694\u0695\u0006\u00dd$\u0000"+ + "\u0695\u01cd\u0001\u0000\u0000\u0000\u0696\u0697\u0003x3\u0000\u0697\u0698"+ + "\u0001\u0000\u0000\u0000\u0698\u0699\u0006\u00de\u0018\u0000\u0699\u01cf"+ + "\u0001\u0000\u0000\u0000\u069a\u069b\u0003t1\u0000\u069b\u069c\u0001\u0000"+ + "\u0000\u0000\u069c\u069d\u0006\u00df\u0014\u0000\u069d\u01d1\u0001\u0000"+ + "\u0000\u0000\u069e\u069f\u0003\u00c0W\u0000\u069f\u06a0\u0001\u0000\u0000"+ + "\u0000\u06a0\u06a1\u0006\u00e0 \u0000\u06a1\u01d3\u0001\u0000\u0000\u0000"+ + "\u06a2\u06a3\u0003\u00bcU\u0000\u06a3\u06a4\u0001\u0000\u0000\u0000\u06a4"+ + "\u06a5\u0006\u00e1!\u0000\u06a5\u01d5\u0001\u0000\u0000\u0000\u06a6\u06a7"+ + "\u0003F\u001a\u0000\u06a7\u06a8\u0001\u0000\u0000\u0000\u06a8\u06a9\u0006"+ + "\u00e2\f\u0000\u06a9\u01d7\u0001\u0000\u0000\u0000\u06aa\u06ab\u0003H"+ + "\u001b\u0000\u06ab\u06ac\u0001\u0000\u0000\u0000\u06ac\u06ad\u0006\u00e3"+ + "\f\u0000\u06ad\u01d9\u0001\u0000\u0000\u0000\u06ae\u06af\u0003J\u001c"+ + "\u0000\u06af\u06b0\u0001\u0000\u0000\u0000\u06b0\u06b1\u0006\u00e4\f\u0000"+ + "\u06b1\u01db\u0001\u0000\u0000\u0000\u06b2\u06b3\u0003L\u001d\u0000\u06b3"+ + "\u06b4\u0001\u0000\u0000\u0000\u06b4\u06b5\u0006\u00e5\u0011\u0000\u06b5"+ + "\u06b6\u0006\u00e5\r\u0000\u06b6\u01dd\u0001\u0000\u0000\u0000\u06b7\u06b8"+ + "\u0003\u00bcU\u0000\u06b8\u06b9\u0001\u0000\u0000\u0000\u06b9\u06ba\u0006"+ + "\u00e6!\u0000\u06ba\u01df\u0001\u0000\u0000\u0000\u06bb\u06bc\u0003J\u001c"+ + "\u0000\u06bc\u06bd\u0001\u0000\u0000\u0000\u06bd\u06be\u0006\u00e7\f\u0000"+ + "\u06be\u01e1\u0001\u0000\u0000\u0000\u06bf\u06c0\u0003F\u001a\u0000\u06c0"+ + "\u06c1\u0001\u0000\u0000\u0000\u06c1\u06c2\u0006\u00e8\f\u0000\u06c2\u01e3"+ + "\u0001\u0000\u0000\u0000\u06c3\u06c4\u0003H\u001b\u0000\u06c4\u06c5\u0001"+ + "\u0000\u0000\u0000\u06c5\u06c6\u0006\u00e9\f\u0000\u06c6\u01e5\u0001\u0000"+ + "\u0000\u0000D\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b"+ + "\f\r\u000e\u000f\u0010\u0011\u02ce\u02d8\u02dc\u02df\u02e8\u02ea\u02f5"+ + "\u0308\u030d\u0316\u031d\u0322\u0324\u032f\u0337\u033a\u033c\u0341\u0346"+ + "\u034c\u0353\u0358\u035e\u0361\u0369\u036d\u03f1\u03f6\u03fd\u03ff\u040f"+ + "\u0414\u0419\u041b\u0421\u046e\u0473\u04a2\u04a6\u04ab\u04b0\u04b5\u04b7"+ + "\u04bb\u04bd\u0512\u0516\u051b\u05a8\u05aa\'\u0005\u0001\u0000\u0005\u0004"+ + "\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005\u0003\u0000\u0005\b\u0000"+ + "\u0005\u0005\u0000\u0005\t\u0000\u0005\r\u0000\u0005\u0010\u0000\u0005"+ + "\u000b\u0000\u0005\u000e\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007"+ + "\u0010\u0000\u0007I\u0000\u0005\u0000\u0000\u0007\u001e\u0000\u0007J\u0000"+ + "\u0007\'\u0000\u0007(\u0000\u0007%\u0000\u0007T\u0000\u0007\u001f\u0000"+ + "\u0007*\u0000\u00076\u0000\u0007H\u0000\u0007X\u0000\u0005\n\u0000\u0005"+ + "\u0007\u0000\u0007b\u0000\u0007a\u0000\u0007L\u0000\u0007K\u0000\u0007"+ + "`\u0000\u0005\f\u0000\u0007\\\u0000\u0005\u000f\u0000\u0007\"\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index 2957751d99f6..04440d5cbca7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -28,6 +28,7 @@ null null null null +null '|' null null @@ -134,6 +135,9 @@ null null null null +null +null +null token symbolic names: null @@ -156,6 +160,7 @@ WHERE JOIN_LOOKUP DEV_CHANGE_POINT DEV_INLINESTATS +DEV_INSIST DEV_LOOKUP DEV_METRICS DEV_JOIN_FULL @@ -271,6 +276,9 @@ CLOSING_METRICS_WS CHANGE_POINT_LINE_COMMENT CHANGE_POINT_MULTILINE_COMMENT CHANGE_POINT_WS +INSIST_WS +INSIST_LINE_COMMENT +INSIST_MULTILINE_COMMENT rule names: singleStatement @@ -340,7 +348,8 @@ joinTarget joinCondition joinPredicate changePointCommand +insistCommand atn: -[4, 1, 134, 659, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 144, 8, 1, 10, 1, 12, 1, 147, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 155, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 176, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 188, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 195, 8, 5, 10, 5, 12, 5, 198, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 205, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 210, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 218, 8, 5, 10, 5, 12, 5, 221, 9, 5, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 232, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 237, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 242, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 252, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 258, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 266, 8, 9, 10, 9, 12, 9, 269, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 279, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 284, 8, 10, 10, 10, 12, 10, 287, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 295, 8, 11, 10, 11, 12, 11, 298, 9, 11, 1, 11, 1, 11, 3, 11, 302, 8, 11, 3, 11, 304, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 5, 13, 314, 8, 13, 10, 13, 12, 13, 317, 9, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 5, 17, 333, 8, 17, 10, 17, 12, 17, 336, 9, 17, 1, 18, 1, 18, 1, 18, 3, 18, 341, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 349, 8, 19, 10, 19, 12, 19, 352, 9, 19, 1, 19, 3, 19, 355, 8, 19, 1, 20, 1, 20, 1, 20, 3, 20, 360, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 372, 8, 23, 10, 23, 12, 23, 375, 9, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 381, 8, 24, 10, 24, 12, 24, 384, 9, 24, 1, 24, 3, 24, 387, 8, 24, 1, 24, 1, 24, 3, 24, 391, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 398, 8, 26, 1, 26, 1, 26, 3, 26, 402, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 407, 8, 27, 10, 27, 12, 27, 410, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 415, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 420, 8, 29, 10, 29, 12, 29, 423, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 428, 8, 30, 10, 30, 12, 30, 431, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 436, 8, 31, 10, 31, 12, 31, 439, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 3, 33, 445, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 460, 8, 34, 10, 34, 12, 34, 463, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 471, 8, 34, 10, 34, 12, 34, 474, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 482, 8, 34, 10, 34, 12, 34, 485, 9, 34, 1, 34, 1, 34, 3, 34, 489, 8, 34, 1, 35, 1, 35, 3, 35, 493, 8, 35, 1, 36, 1, 36, 3, 36, 497, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 506, 8, 38, 10, 38, 12, 38, 509, 9, 38, 1, 39, 1, 39, 3, 39, 513, 8, 39, 1, 39, 1, 39, 3, 39, 517, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 529, 8, 42, 10, 42, 12, 42, 532, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 542, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 554, 8, 47, 10, 47, 12, 47, 557, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 567, 8, 50, 1, 51, 3, 51, 570, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 575, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 597, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 603, 8, 58, 10, 58, 12, 58, 606, 9, 58, 3, 58, 608, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 613, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 626, 8, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 639, 8, 64, 10, 64, 12, 64, 642, 9, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 66, 3, 66, 650, 8, 66, 1, 66, 1, 66, 1, 66, 1, 66, 1, 66, 3, 66, 657, 8, 66, 1, 66, 0, 4, 2, 10, 18, 20, 67, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 83, 83, 1, 0, 74, 75, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 2, 0, 17, 17, 23, 24, 686, 0, 134, 1, 0, 0, 0, 2, 137, 1, 0, 0, 0, 4, 154, 1, 0, 0, 0, 6, 175, 1, 0, 0, 0, 8, 177, 1, 0, 0, 0, 10, 209, 1, 0, 0, 0, 12, 236, 1, 0, 0, 0, 14, 238, 1, 0, 0, 0, 16, 251, 1, 0, 0, 0, 18, 257, 1, 0, 0, 0, 20, 278, 1, 0, 0, 0, 22, 288, 1, 0, 0, 0, 24, 307, 1, 0, 0, 0, 26, 309, 1, 0, 0, 0, 28, 320, 1, 0, 0, 0, 30, 324, 1, 0, 0, 0, 32, 326, 1, 0, 0, 0, 34, 329, 1, 0, 0, 0, 36, 340, 1, 0, 0, 0, 38, 344, 1, 0, 0, 0, 40, 359, 1, 0, 0, 0, 42, 363, 1, 0, 0, 0, 44, 365, 1, 0, 0, 0, 46, 367, 1, 0, 0, 0, 48, 376, 1, 0, 0, 0, 50, 392, 1, 0, 0, 0, 52, 395, 1, 0, 0, 0, 54, 403, 1, 0, 0, 0, 56, 411, 1, 0, 0, 0, 58, 416, 1, 0, 0, 0, 60, 424, 1, 0, 0, 0, 62, 432, 1, 0, 0, 0, 64, 440, 1, 0, 0, 0, 66, 444, 1, 0, 0, 0, 68, 488, 1, 0, 0, 0, 70, 492, 1, 0, 0, 0, 72, 496, 1, 0, 0, 0, 74, 498, 1, 0, 0, 0, 76, 501, 1, 0, 0, 0, 78, 510, 1, 0, 0, 0, 80, 518, 1, 0, 0, 0, 82, 521, 1, 0, 0, 0, 84, 524, 1, 0, 0, 0, 86, 533, 1, 0, 0, 0, 88, 537, 1, 0, 0, 0, 90, 543, 1, 0, 0, 0, 92, 547, 1, 0, 0, 0, 94, 550, 1, 0, 0, 0, 96, 558, 1, 0, 0, 0, 98, 562, 1, 0, 0, 0, 100, 566, 1, 0, 0, 0, 102, 569, 1, 0, 0, 0, 104, 574, 1, 0, 0, 0, 106, 578, 1, 0, 0, 0, 108, 580, 1, 0, 0, 0, 110, 582, 1, 0, 0, 0, 112, 585, 1, 0, 0, 0, 114, 589, 1, 0, 0, 0, 116, 592, 1, 0, 0, 0, 118, 612, 1, 0, 0, 0, 120, 616, 1, 0, 0, 0, 122, 621, 1, 0, 0, 0, 124, 627, 1, 0, 0, 0, 126, 632, 1, 0, 0, 0, 128, 634, 1, 0, 0, 0, 130, 643, 1, 0, 0, 0, 132, 645, 1, 0, 0, 0, 134, 135, 3, 2, 1, 0, 135, 136, 5, 0, 0, 1, 136, 1, 1, 0, 0, 0, 137, 138, 6, 1, -1, 0, 138, 139, 3, 4, 2, 0, 139, 145, 1, 0, 0, 0, 140, 141, 10, 1, 0, 0, 141, 142, 5, 29, 0, 0, 142, 144, 3, 6, 3, 0, 143, 140, 1, 0, 0, 0, 144, 147, 1, 0, 0, 0, 145, 143, 1, 0, 0, 0, 145, 146, 1, 0, 0, 0, 146, 3, 1, 0, 0, 0, 147, 145, 1, 0, 0, 0, 148, 155, 3, 110, 55, 0, 149, 155, 3, 38, 19, 0, 150, 155, 3, 32, 16, 0, 151, 155, 3, 114, 57, 0, 152, 153, 4, 2, 1, 0, 153, 155, 3, 48, 24, 0, 154, 148, 1, 0, 0, 0, 154, 149, 1, 0, 0, 0, 154, 150, 1, 0, 0, 0, 154, 151, 1, 0, 0, 0, 154, 152, 1, 0, 0, 0, 155, 5, 1, 0, 0, 0, 156, 176, 3, 50, 25, 0, 157, 176, 3, 8, 4, 0, 158, 176, 3, 80, 40, 0, 159, 176, 3, 74, 37, 0, 160, 176, 3, 52, 26, 0, 161, 176, 3, 76, 38, 0, 162, 176, 3, 82, 41, 0, 163, 176, 3, 84, 42, 0, 164, 176, 3, 88, 44, 0, 165, 176, 3, 90, 45, 0, 166, 176, 3, 116, 58, 0, 167, 176, 3, 92, 46, 0, 168, 176, 3, 124, 62, 0, 169, 170, 4, 3, 2, 0, 170, 176, 3, 122, 61, 0, 171, 172, 4, 3, 3, 0, 172, 176, 3, 120, 60, 0, 173, 174, 4, 3, 4, 0, 174, 176, 3, 132, 66, 0, 175, 156, 1, 0, 0, 0, 175, 157, 1, 0, 0, 0, 175, 158, 1, 0, 0, 0, 175, 159, 1, 0, 0, 0, 175, 160, 1, 0, 0, 0, 175, 161, 1, 0, 0, 0, 175, 162, 1, 0, 0, 0, 175, 163, 1, 0, 0, 0, 175, 164, 1, 0, 0, 0, 175, 165, 1, 0, 0, 0, 175, 166, 1, 0, 0, 0, 175, 167, 1, 0, 0, 0, 175, 168, 1, 0, 0, 0, 175, 169, 1, 0, 0, 0, 175, 171, 1, 0, 0, 0, 175, 173, 1, 0, 0, 0, 176, 7, 1, 0, 0, 0, 177, 178, 5, 16, 0, 0, 178, 179, 3, 10, 5, 0, 179, 9, 1, 0, 0, 0, 180, 181, 6, 5, -1, 0, 181, 182, 5, 49, 0, 0, 182, 210, 3, 10, 5, 8, 183, 210, 3, 16, 8, 0, 184, 210, 3, 12, 6, 0, 185, 187, 3, 16, 8, 0, 186, 188, 5, 49, 0, 0, 187, 186, 1, 0, 0, 0, 187, 188, 1, 0, 0, 0, 188, 189, 1, 0, 0, 0, 189, 190, 5, 44, 0, 0, 190, 191, 5, 48, 0, 0, 191, 196, 3, 16, 8, 0, 192, 193, 5, 39, 0, 0, 193, 195, 3, 16, 8, 0, 194, 192, 1, 0, 0, 0, 195, 198, 1, 0, 0, 0, 196, 194, 1, 0, 0, 0, 196, 197, 1, 0, 0, 0, 197, 199, 1, 0, 0, 0, 198, 196, 1, 0, 0, 0, 199, 200, 5, 55, 0, 0, 200, 210, 1, 0, 0, 0, 201, 202, 3, 16, 8, 0, 202, 204, 5, 45, 0, 0, 203, 205, 5, 49, 0, 0, 204, 203, 1, 0, 0, 0, 204, 205, 1, 0, 0, 0, 205, 206, 1, 0, 0, 0, 206, 207, 5, 50, 0, 0, 207, 210, 1, 0, 0, 0, 208, 210, 3, 14, 7, 0, 209, 180, 1, 0, 0, 0, 209, 183, 1, 0, 0, 0, 209, 184, 1, 0, 0, 0, 209, 185, 1, 0, 0, 0, 209, 201, 1, 0, 0, 0, 209, 208, 1, 0, 0, 0, 210, 219, 1, 0, 0, 0, 211, 212, 10, 5, 0, 0, 212, 213, 5, 34, 0, 0, 213, 218, 3, 10, 5, 6, 214, 215, 10, 4, 0, 0, 215, 216, 5, 52, 0, 0, 216, 218, 3, 10, 5, 5, 217, 211, 1, 0, 0, 0, 217, 214, 1, 0, 0, 0, 218, 221, 1, 0, 0, 0, 219, 217, 1, 0, 0, 0, 219, 220, 1, 0, 0, 0, 220, 11, 1, 0, 0, 0, 221, 219, 1, 0, 0, 0, 222, 224, 3, 16, 8, 0, 223, 225, 5, 49, 0, 0, 224, 223, 1, 0, 0, 0, 224, 225, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 227, 5, 47, 0, 0, 227, 228, 3, 106, 53, 0, 228, 237, 1, 0, 0, 0, 229, 231, 3, 16, 8, 0, 230, 232, 5, 49, 0, 0, 231, 230, 1, 0, 0, 0, 231, 232, 1, 0, 0, 0, 232, 233, 1, 0, 0, 0, 233, 234, 5, 54, 0, 0, 234, 235, 3, 106, 53, 0, 235, 237, 1, 0, 0, 0, 236, 222, 1, 0, 0, 0, 236, 229, 1, 0, 0, 0, 237, 13, 1, 0, 0, 0, 238, 241, 3, 58, 29, 0, 239, 240, 5, 37, 0, 0, 240, 242, 3, 30, 15, 0, 241, 239, 1, 0, 0, 0, 241, 242, 1, 0, 0, 0, 242, 243, 1, 0, 0, 0, 243, 244, 5, 38, 0, 0, 244, 245, 3, 68, 34, 0, 245, 15, 1, 0, 0, 0, 246, 252, 3, 18, 9, 0, 247, 248, 3, 18, 9, 0, 248, 249, 3, 108, 54, 0, 249, 250, 3, 18, 9, 0, 250, 252, 1, 0, 0, 0, 251, 246, 1, 0, 0, 0, 251, 247, 1, 0, 0, 0, 252, 17, 1, 0, 0, 0, 253, 254, 6, 9, -1, 0, 254, 258, 3, 20, 10, 0, 255, 256, 7, 0, 0, 0, 256, 258, 3, 18, 9, 3, 257, 253, 1, 0, 0, 0, 257, 255, 1, 0, 0, 0, 258, 267, 1, 0, 0, 0, 259, 260, 10, 2, 0, 0, 260, 261, 7, 1, 0, 0, 261, 266, 3, 18, 9, 3, 262, 263, 10, 1, 0, 0, 263, 264, 7, 0, 0, 0, 264, 266, 3, 18, 9, 2, 265, 259, 1, 0, 0, 0, 265, 262, 1, 0, 0, 0, 266, 269, 1, 0, 0, 0, 267, 265, 1, 0, 0, 0, 267, 268, 1, 0, 0, 0, 268, 19, 1, 0, 0, 0, 269, 267, 1, 0, 0, 0, 270, 271, 6, 10, -1, 0, 271, 279, 3, 68, 34, 0, 272, 279, 3, 58, 29, 0, 273, 279, 3, 22, 11, 0, 274, 275, 5, 48, 0, 0, 275, 276, 3, 10, 5, 0, 276, 277, 5, 55, 0, 0, 277, 279, 1, 0, 0, 0, 278, 270, 1, 0, 0, 0, 278, 272, 1, 0, 0, 0, 278, 273, 1, 0, 0, 0, 278, 274, 1, 0, 0, 0, 279, 285, 1, 0, 0, 0, 280, 281, 10, 1, 0, 0, 281, 282, 5, 37, 0, 0, 282, 284, 3, 30, 15, 0, 283, 280, 1, 0, 0, 0, 284, 287, 1, 0, 0, 0, 285, 283, 1, 0, 0, 0, 285, 286, 1, 0, 0, 0, 286, 21, 1, 0, 0, 0, 287, 285, 1, 0, 0, 0, 288, 289, 3, 24, 12, 0, 289, 303, 5, 48, 0, 0, 290, 304, 5, 66, 0, 0, 291, 296, 3, 10, 5, 0, 292, 293, 5, 39, 0, 0, 293, 295, 3, 10, 5, 0, 294, 292, 1, 0, 0, 0, 295, 298, 1, 0, 0, 0, 296, 294, 1, 0, 0, 0, 296, 297, 1, 0, 0, 0, 297, 301, 1, 0, 0, 0, 298, 296, 1, 0, 0, 0, 299, 300, 5, 39, 0, 0, 300, 302, 3, 26, 13, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 304, 1, 0, 0, 0, 303, 290, 1, 0, 0, 0, 303, 291, 1, 0, 0, 0, 303, 304, 1, 0, 0, 0, 304, 305, 1, 0, 0, 0, 305, 306, 5, 55, 0, 0, 306, 23, 1, 0, 0, 0, 307, 308, 3, 72, 36, 0, 308, 25, 1, 0, 0, 0, 309, 310, 5, 69, 0, 0, 310, 315, 3, 28, 14, 0, 311, 312, 5, 39, 0, 0, 312, 314, 3, 28, 14, 0, 313, 311, 1, 0, 0, 0, 314, 317, 1, 0, 0, 0, 315, 313, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 318, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 318, 319, 5, 70, 0, 0, 319, 27, 1, 0, 0, 0, 320, 321, 3, 106, 53, 0, 321, 322, 5, 38, 0, 0, 322, 323, 3, 68, 34, 0, 323, 29, 1, 0, 0, 0, 324, 325, 3, 64, 32, 0, 325, 31, 1, 0, 0, 0, 326, 327, 5, 12, 0, 0, 327, 328, 3, 34, 17, 0, 328, 33, 1, 0, 0, 0, 329, 334, 3, 36, 18, 0, 330, 331, 5, 39, 0, 0, 331, 333, 3, 36, 18, 0, 332, 330, 1, 0, 0, 0, 333, 336, 1, 0, 0, 0, 334, 332, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 35, 1, 0, 0, 0, 336, 334, 1, 0, 0, 0, 337, 338, 3, 58, 29, 0, 338, 339, 5, 36, 0, 0, 339, 341, 1, 0, 0, 0, 340, 337, 1, 0, 0, 0, 340, 341, 1, 0, 0, 0, 341, 342, 1, 0, 0, 0, 342, 343, 3, 10, 5, 0, 343, 37, 1, 0, 0, 0, 344, 345, 5, 6, 0, 0, 345, 350, 3, 40, 20, 0, 346, 347, 5, 39, 0, 0, 347, 349, 3, 40, 20, 0, 348, 346, 1, 0, 0, 0, 349, 352, 1, 0, 0, 0, 350, 348, 1, 0, 0, 0, 350, 351, 1, 0, 0, 0, 351, 354, 1, 0, 0, 0, 352, 350, 1, 0, 0, 0, 353, 355, 3, 46, 23, 0, 354, 353, 1, 0, 0, 0, 354, 355, 1, 0, 0, 0, 355, 39, 1, 0, 0, 0, 356, 357, 3, 42, 21, 0, 357, 358, 5, 38, 0, 0, 358, 360, 1, 0, 0, 0, 359, 356, 1, 0, 0, 0, 359, 360, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 362, 3, 44, 22, 0, 362, 41, 1, 0, 0, 0, 363, 364, 7, 2, 0, 0, 364, 43, 1, 0, 0, 0, 365, 366, 7, 2, 0, 0, 366, 45, 1, 0, 0, 0, 367, 368, 5, 82, 0, 0, 368, 373, 5, 83, 0, 0, 369, 370, 5, 39, 0, 0, 370, 372, 5, 83, 0, 0, 371, 369, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 371, 1, 0, 0, 0, 373, 374, 1, 0, 0, 0, 374, 47, 1, 0, 0, 0, 375, 373, 1, 0, 0, 0, 376, 377, 5, 21, 0, 0, 377, 382, 3, 40, 20, 0, 378, 379, 5, 39, 0, 0, 379, 381, 3, 40, 20, 0, 380, 378, 1, 0, 0, 0, 381, 384, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 386, 1, 0, 0, 0, 384, 382, 1, 0, 0, 0, 385, 387, 3, 54, 27, 0, 386, 385, 1, 0, 0, 0, 386, 387, 1, 0, 0, 0, 387, 390, 1, 0, 0, 0, 388, 389, 5, 33, 0, 0, 389, 391, 3, 34, 17, 0, 390, 388, 1, 0, 0, 0, 390, 391, 1, 0, 0, 0, 391, 49, 1, 0, 0, 0, 392, 393, 5, 4, 0, 0, 393, 394, 3, 34, 17, 0, 394, 51, 1, 0, 0, 0, 395, 397, 5, 15, 0, 0, 396, 398, 3, 54, 27, 0, 397, 396, 1, 0, 0, 0, 397, 398, 1, 0, 0, 0, 398, 401, 1, 0, 0, 0, 399, 400, 5, 33, 0, 0, 400, 402, 3, 34, 17, 0, 401, 399, 1, 0, 0, 0, 401, 402, 1, 0, 0, 0, 402, 53, 1, 0, 0, 0, 403, 408, 3, 56, 28, 0, 404, 405, 5, 39, 0, 0, 405, 407, 3, 56, 28, 0, 406, 404, 1, 0, 0, 0, 407, 410, 1, 0, 0, 0, 408, 406, 1, 0, 0, 0, 408, 409, 1, 0, 0, 0, 409, 55, 1, 0, 0, 0, 410, 408, 1, 0, 0, 0, 411, 414, 3, 36, 18, 0, 412, 413, 5, 16, 0, 0, 413, 415, 3, 10, 5, 0, 414, 412, 1, 0, 0, 0, 414, 415, 1, 0, 0, 0, 415, 57, 1, 0, 0, 0, 416, 421, 3, 72, 36, 0, 417, 418, 5, 41, 0, 0, 418, 420, 3, 72, 36, 0, 419, 417, 1, 0, 0, 0, 420, 423, 1, 0, 0, 0, 421, 419, 1, 0, 0, 0, 421, 422, 1, 0, 0, 0, 422, 59, 1, 0, 0, 0, 423, 421, 1, 0, 0, 0, 424, 429, 3, 66, 33, 0, 425, 426, 5, 41, 0, 0, 426, 428, 3, 66, 33, 0, 427, 425, 1, 0, 0, 0, 428, 431, 1, 0, 0, 0, 429, 427, 1, 0, 0, 0, 429, 430, 1, 0, 0, 0, 430, 61, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 437, 3, 60, 30, 0, 433, 434, 5, 39, 0, 0, 434, 436, 3, 60, 30, 0, 435, 433, 1, 0, 0, 0, 436, 439, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 437, 438, 1, 0, 0, 0, 438, 63, 1, 0, 0, 0, 439, 437, 1, 0, 0, 0, 440, 441, 7, 3, 0, 0, 441, 65, 1, 0, 0, 0, 442, 445, 5, 87, 0, 0, 443, 445, 3, 70, 35, 0, 444, 442, 1, 0, 0, 0, 444, 443, 1, 0, 0, 0, 445, 67, 1, 0, 0, 0, 446, 489, 5, 50, 0, 0, 447, 448, 3, 104, 52, 0, 448, 449, 5, 74, 0, 0, 449, 489, 1, 0, 0, 0, 450, 489, 3, 102, 51, 0, 451, 489, 3, 104, 52, 0, 452, 489, 3, 98, 49, 0, 453, 489, 3, 70, 35, 0, 454, 489, 3, 106, 53, 0, 455, 456, 5, 72, 0, 0, 456, 461, 3, 100, 50, 0, 457, 458, 5, 39, 0, 0, 458, 460, 3, 100, 50, 0, 459, 457, 1, 0, 0, 0, 460, 463, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 461, 462, 1, 0, 0, 0, 462, 464, 1, 0, 0, 0, 463, 461, 1, 0, 0, 0, 464, 465, 5, 73, 0, 0, 465, 489, 1, 0, 0, 0, 466, 467, 5, 72, 0, 0, 467, 472, 3, 98, 49, 0, 468, 469, 5, 39, 0, 0, 469, 471, 3, 98, 49, 0, 470, 468, 1, 0, 0, 0, 471, 474, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 472, 473, 1, 0, 0, 0, 473, 475, 1, 0, 0, 0, 474, 472, 1, 0, 0, 0, 475, 476, 5, 73, 0, 0, 476, 489, 1, 0, 0, 0, 477, 478, 5, 72, 0, 0, 478, 483, 3, 106, 53, 0, 479, 480, 5, 39, 0, 0, 480, 482, 3, 106, 53, 0, 481, 479, 1, 0, 0, 0, 482, 485, 1, 0, 0, 0, 483, 481, 1, 0, 0, 0, 483, 484, 1, 0, 0, 0, 484, 486, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 486, 487, 5, 73, 0, 0, 487, 489, 1, 0, 0, 0, 488, 446, 1, 0, 0, 0, 488, 447, 1, 0, 0, 0, 488, 450, 1, 0, 0, 0, 488, 451, 1, 0, 0, 0, 488, 452, 1, 0, 0, 0, 488, 453, 1, 0, 0, 0, 488, 454, 1, 0, 0, 0, 488, 455, 1, 0, 0, 0, 488, 466, 1, 0, 0, 0, 488, 477, 1, 0, 0, 0, 489, 69, 1, 0, 0, 0, 490, 493, 5, 53, 0, 0, 491, 493, 5, 71, 0, 0, 492, 490, 1, 0, 0, 0, 492, 491, 1, 0, 0, 0, 493, 71, 1, 0, 0, 0, 494, 497, 3, 64, 32, 0, 495, 497, 3, 70, 35, 0, 496, 494, 1, 0, 0, 0, 496, 495, 1, 0, 0, 0, 497, 73, 1, 0, 0, 0, 498, 499, 5, 9, 0, 0, 499, 500, 5, 31, 0, 0, 500, 75, 1, 0, 0, 0, 501, 502, 5, 14, 0, 0, 502, 507, 3, 78, 39, 0, 503, 504, 5, 39, 0, 0, 504, 506, 3, 78, 39, 0, 505, 503, 1, 0, 0, 0, 506, 509, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 77, 1, 0, 0, 0, 509, 507, 1, 0, 0, 0, 510, 512, 3, 10, 5, 0, 511, 513, 7, 4, 0, 0, 512, 511, 1, 0, 0, 0, 512, 513, 1, 0, 0, 0, 513, 516, 1, 0, 0, 0, 514, 515, 5, 51, 0, 0, 515, 517, 7, 5, 0, 0, 516, 514, 1, 0, 0, 0, 516, 517, 1, 0, 0, 0, 517, 79, 1, 0, 0, 0, 518, 519, 5, 8, 0, 0, 519, 520, 3, 62, 31, 0, 520, 81, 1, 0, 0, 0, 521, 522, 5, 2, 0, 0, 522, 523, 3, 62, 31, 0, 523, 83, 1, 0, 0, 0, 524, 525, 5, 11, 0, 0, 525, 530, 3, 86, 43, 0, 526, 527, 5, 39, 0, 0, 527, 529, 3, 86, 43, 0, 528, 526, 1, 0, 0, 0, 529, 532, 1, 0, 0, 0, 530, 528, 1, 0, 0, 0, 530, 531, 1, 0, 0, 0, 531, 85, 1, 0, 0, 0, 532, 530, 1, 0, 0, 0, 533, 534, 3, 60, 30, 0, 534, 535, 5, 91, 0, 0, 535, 536, 3, 60, 30, 0, 536, 87, 1, 0, 0, 0, 537, 538, 5, 1, 0, 0, 538, 539, 3, 20, 10, 0, 539, 541, 3, 106, 53, 0, 540, 542, 3, 94, 47, 0, 541, 540, 1, 0, 0, 0, 541, 542, 1, 0, 0, 0, 542, 89, 1, 0, 0, 0, 543, 544, 5, 7, 0, 0, 544, 545, 3, 20, 10, 0, 545, 546, 3, 106, 53, 0, 546, 91, 1, 0, 0, 0, 547, 548, 5, 10, 0, 0, 548, 549, 3, 58, 29, 0, 549, 93, 1, 0, 0, 0, 550, 555, 3, 96, 48, 0, 551, 552, 5, 39, 0, 0, 552, 554, 3, 96, 48, 0, 553, 551, 1, 0, 0, 0, 554, 557, 1, 0, 0, 0, 555, 553, 1, 0, 0, 0, 555, 556, 1, 0, 0, 0, 556, 95, 1, 0, 0, 0, 557, 555, 1, 0, 0, 0, 558, 559, 3, 64, 32, 0, 559, 560, 5, 36, 0, 0, 560, 561, 3, 68, 34, 0, 561, 97, 1, 0, 0, 0, 562, 563, 7, 6, 0, 0, 563, 99, 1, 0, 0, 0, 564, 567, 3, 102, 51, 0, 565, 567, 3, 104, 52, 0, 566, 564, 1, 0, 0, 0, 566, 565, 1, 0, 0, 0, 567, 101, 1, 0, 0, 0, 568, 570, 7, 0, 0, 0, 569, 568, 1, 0, 0, 0, 569, 570, 1, 0, 0, 0, 570, 571, 1, 0, 0, 0, 571, 572, 5, 32, 0, 0, 572, 103, 1, 0, 0, 0, 573, 575, 7, 0, 0, 0, 574, 573, 1, 0, 0, 0, 574, 575, 1, 0, 0, 0, 575, 576, 1, 0, 0, 0, 576, 577, 5, 31, 0, 0, 577, 105, 1, 0, 0, 0, 578, 579, 5, 30, 0, 0, 579, 107, 1, 0, 0, 0, 580, 581, 7, 7, 0, 0, 581, 109, 1, 0, 0, 0, 582, 583, 5, 5, 0, 0, 583, 584, 3, 112, 56, 0, 584, 111, 1, 0, 0, 0, 585, 586, 5, 72, 0, 0, 586, 587, 3, 2, 1, 0, 587, 588, 5, 73, 0, 0, 588, 113, 1, 0, 0, 0, 589, 590, 5, 13, 0, 0, 590, 591, 5, 107, 0, 0, 591, 115, 1, 0, 0, 0, 592, 593, 5, 3, 0, 0, 593, 596, 5, 97, 0, 0, 594, 595, 5, 95, 0, 0, 595, 597, 3, 60, 30, 0, 596, 594, 1, 0, 0, 0, 596, 597, 1, 0, 0, 0, 597, 607, 1, 0, 0, 0, 598, 599, 5, 96, 0, 0, 599, 604, 3, 118, 59, 0, 600, 601, 5, 39, 0, 0, 601, 603, 3, 118, 59, 0, 602, 600, 1, 0, 0, 0, 603, 606, 1, 0, 0, 0, 604, 602, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 608, 1, 0, 0, 0, 606, 604, 1, 0, 0, 0, 607, 598, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 608, 117, 1, 0, 0, 0, 609, 610, 3, 60, 30, 0, 610, 611, 5, 36, 0, 0, 611, 613, 1, 0, 0, 0, 612, 609, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 615, 3, 60, 30, 0, 615, 119, 1, 0, 0, 0, 616, 617, 5, 20, 0, 0, 617, 618, 3, 40, 20, 0, 618, 619, 5, 95, 0, 0, 619, 620, 3, 62, 31, 0, 620, 121, 1, 0, 0, 0, 621, 622, 5, 19, 0, 0, 622, 625, 3, 54, 27, 0, 623, 624, 5, 33, 0, 0, 624, 626, 3, 34, 17, 0, 625, 623, 1, 0, 0, 0, 625, 626, 1, 0, 0, 0, 626, 123, 1, 0, 0, 0, 627, 628, 7, 8, 0, 0, 628, 629, 5, 121, 0, 0, 629, 630, 3, 126, 63, 0, 630, 631, 3, 128, 64, 0, 631, 125, 1, 0, 0, 0, 632, 633, 3, 40, 20, 0, 633, 127, 1, 0, 0, 0, 634, 635, 5, 95, 0, 0, 635, 640, 3, 130, 65, 0, 636, 637, 5, 39, 0, 0, 637, 639, 3, 130, 65, 0, 638, 636, 1, 0, 0, 0, 639, 642, 1, 0, 0, 0, 640, 638, 1, 0, 0, 0, 640, 641, 1, 0, 0, 0, 641, 129, 1, 0, 0, 0, 642, 640, 1, 0, 0, 0, 643, 644, 3, 16, 8, 0, 644, 131, 1, 0, 0, 0, 645, 646, 5, 18, 0, 0, 646, 649, 3, 58, 29, 0, 647, 648, 5, 95, 0, 0, 648, 650, 3, 58, 29, 0, 649, 647, 1, 0, 0, 0, 649, 650, 1, 0, 0, 0, 650, 656, 1, 0, 0, 0, 651, 652, 5, 91, 0, 0, 652, 653, 3, 58, 29, 0, 653, 654, 5, 39, 0, 0, 654, 655, 3, 58, 29, 0, 655, 657, 1, 0, 0, 0, 656, 651, 1, 0, 0, 0, 656, 657, 1, 0, 0, 0, 657, 133, 1, 0, 0, 0, 63, 145, 154, 175, 187, 196, 204, 209, 217, 219, 224, 231, 236, 241, 251, 257, 265, 267, 278, 285, 296, 301, 303, 315, 334, 340, 350, 354, 359, 373, 382, 386, 390, 397, 401, 408, 414, 421, 429, 437, 444, 461, 472, 483, 488, 492, 496, 507, 512, 516, 530, 541, 555, 566, 569, 574, 596, 604, 607, 612, 625, 640, 649, 656] \ No newline at end of file +[4, 1, 138, 666, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 146, 8, 1, 10, 1, 12, 1, 149, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 157, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 180, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 192, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 199, 8, 5, 10, 5, 12, 5, 202, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 209, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 214, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 222, 8, 5, 10, 5, 12, 5, 225, 9, 5, 1, 6, 1, 6, 3, 6, 229, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 236, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 241, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 246, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 256, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 262, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 270, 8, 9, 10, 9, 12, 9, 273, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 283, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 288, 8, 10, 10, 10, 12, 10, 291, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 299, 8, 11, 10, 11, 12, 11, 302, 9, 11, 1, 11, 1, 11, 3, 11, 306, 8, 11, 3, 11, 308, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 5, 13, 318, 8, 13, 10, 13, 12, 13, 321, 9, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 5, 17, 337, 8, 17, 10, 17, 12, 17, 340, 9, 17, 1, 18, 1, 18, 1, 18, 3, 18, 345, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 353, 8, 19, 10, 19, 12, 19, 356, 9, 19, 1, 19, 3, 19, 359, 8, 19, 1, 20, 1, 20, 1, 20, 3, 20, 364, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 376, 8, 23, 10, 23, 12, 23, 379, 9, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 385, 8, 24, 10, 24, 12, 24, 388, 9, 24, 1, 24, 3, 24, 391, 8, 24, 1, 24, 1, 24, 3, 24, 395, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 402, 8, 26, 1, 26, 1, 26, 3, 26, 406, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 411, 8, 27, 10, 27, 12, 27, 414, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 419, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 424, 8, 29, 10, 29, 12, 29, 427, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 432, 8, 30, 10, 30, 12, 30, 435, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 440, 8, 31, 10, 31, 12, 31, 443, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 3, 33, 449, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 464, 8, 34, 10, 34, 12, 34, 467, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 475, 8, 34, 10, 34, 12, 34, 478, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 486, 8, 34, 10, 34, 12, 34, 489, 9, 34, 1, 34, 1, 34, 3, 34, 493, 8, 34, 1, 35, 1, 35, 3, 35, 497, 8, 35, 1, 36, 1, 36, 3, 36, 501, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 510, 8, 38, 10, 38, 12, 38, 513, 9, 38, 1, 39, 1, 39, 3, 39, 517, 8, 39, 1, 39, 1, 39, 3, 39, 521, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 533, 8, 42, 10, 42, 12, 42, 536, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 546, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 558, 8, 47, 10, 47, 12, 47, 561, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 571, 8, 50, 1, 51, 3, 51, 574, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 579, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 601, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 607, 8, 58, 10, 58, 12, 58, 610, 9, 58, 3, 58, 612, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 617, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 630, 8, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 643, 8, 64, 10, 64, 12, 64, 646, 9, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 66, 3, 66, 654, 8, 66, 1, 66, 1, 66, 1, 66, 1, 66, 1, 66, 3, 66, 661, 8, 66, 1, 67, 1, 67, 1, 67, 1, 67, 0, 4, 2, 10, 18, 20, 68, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 0, 9, 1, 0, 65, 66, 1, 0, 67, 69, 2, 0, 31, 31, 84, 84, 1, 0, 75, 76, 2, 0, 36, 36, 41, 41, 2, 0, 44, 44, 47, 47, 2, 0, 43, 43, 57, 57, 2, 0, 58, 58, 60, 64, 2, 0, 17, 17, 24, 25, 693, 0, 136, 1, 0, 0, 0, 2, 139, 1, 0, 0, 0, 4, 156, 1, 0, 0, 0, 6, 179, 1, 0, 0, 0, 8, 181, 1, 0, 0, 0, 10, 213, 1, 0, 0, 0, 12, 240, 1, 0, 0, 0, 14, 242, 1, 0, 0, 0, 16, 255, 1, 0, 0, 0, 18, 261, 1, 0, 0, 0, 20, 282, 1, 0, 0, 0, 22, 292, 1, 0, 0, 0, 24, 311, 1, 0, 0, 0, 26, 313, 1, 0, 0, 0, 28, 324, 1, 0, 0, 0, 30, 328, 1, 0, 0, 0, 32, 330, 1, 0, 0, 0, 34, 333, 1, 0, 0, 0, 36, 344, 1, 0, 0, 0, 38, 348, 1, 0, 0, 0, 40, 363, 1, 0, 0, 0, 42, 367, 1, 0, 0, 0, 44, 369, 1, 0, 0, 0, 46, 371, 1, 0, 0, 0, 48, 380, 1, 0, 0, 0, 50, 396, 1, 0, 0, 0, 52, 399, 1, 0, 0, 0, 54, 407, 1, 0, 0, 0, 56, 415, 1, 0, 0, 0, 58, 420, 1, 0, 0, 0, 60, 428, 1, 0, 0, 0, 62, 436, 1, 0, 0, 0, 64, 444, 1, 0, 0, 0, 66, 448, 1, 0, 0, 0, 68, 492, 1, 0, 0, 0, 70, 496, 1, 0, 0, 0, 72, 500, 1, 0, 0, 0, 74, 502, 1, 0, 0, 0, 76, 505, 1, 0, 0, 0, 78, 514, 1, 0, 0, 0, 80, 522, 1, 0, 0, 0, 82, 525, 1, 0, 0, 0, 84, 528, 1, 0, 0, 0, 86, 537, 1, 0, 0, 0, 88, 541, 1, 0, 0, 0, 90, 547, 1, 0, 0, 0, 92, 551, 1, 0, 0, 0, 94, 554, 1, 0, 0, 0, 96, 562, 1, 0, 0, 0, 98, 566, 1, 0, 0, 0, 100, 570, 1, 0, 0, 0, 102, 573, 1, 0, 0, 0, 104, 578, 1, 0, 0, 0, 106, 582, 1, 0, 0, 0, 108, 584, 1, 0, 0, 0, 110, 586, 1, 0, 0, 0, 112, 589, 1, 0, 0, 0, 114, 593, 1, 0, 0, 0, 116, 596, 1, 0, 0, 0, 118, 616, 1, 0, 0, 0, 120, 620, 1, 0, 0, 0, 122, 625, 1, 0, 0, 0, 124, 631, 1, 0, 0, 0, 126, 636, 1, 0, 0, 0, 128, 638, 1, 0, 0, 0, 130, 647, 1, 0, 0, 0, 132, 649, 1, 0, 0, 0, 134, 662, 1, 0, 0, 0, 136, 137, 3, 2, 1, 0, 137, 138, 5, 0, 0, 1, 138, 1, 1, 0, 0, 0, 139, 140, 6, 1, -1, 0, 140, 141, 3, 4, 2, 0, 141, 147, 1, 0, 0, 0, 142, 143, 10, 1, 0, 0, 143, 144, 5, 30, 0, 0, 144, 146, 3, 6, 3, 0, 145, 142, 1, 0, 0, 0, 146, 149, 1, 0, 0, 0, 147, 145, 1, 0, 0, 0, 147, 148, 1, 0, 0, 0, 148, 3, 1, 0, 0, 0, 149, 147, 1, 0, 0, 0, 150, 157, 3, 110, 55, 0, 151, 157, 3, 38, 19, 0, 152, 157, 3, 32, 16, 0, 153, 157, 3, 114, 57, 0, 154, 155, 4, 2, 1, 0, 155, 157, 3, 48, 24, 0, 156, 150, 1, 0, 0, 0, 156, 151, 1, 0, 0, 0, 156, 152, 1, 0, 0, 0, 156, 153, 1, 0, 0, 0, 156, 154, 1, 0, 0, 0, 157, 5, 1, 0, 0, 0, 158, 180, 3, 50, 25, 0, 159, 180, 3, 8, 4, 0, 160, 180, 3, 80, 40, 0, 161, 180, 3, 74, 37, 0, 162, 180, 3, 52, 26, 0, 163, 180, 3, 76, 38, 0, 164, 180, 3, 82, 41, 0, 165, 180, 3, 84, 42, 0, 166, 180, 3, 88, 44, 0, 167, 180, 3, 90, 45, 0, 168, 180, 3, 116, 58, 0, 169, 180, 3, 92, 46, 0, 170, 180, 3, 124, 62, 0, 171, 172, 4, 3, 2, 0, 172, 180, 3, 122, 61, 0, 173, 174, 4, 3, 3, 0, 174, 180, 3, 120, 60, 0, 175, 176, 4, 3, 4, 0, 176, 180, 3, 132, 66, 0, 177, 178, 4, 3, 5, 0, 178, 180, 3, 134, 67, 0, 179, 158, 1, 0, 0, 0, 179, 159, 1, 0, 0, 0, 179, 160, 1, 0, 0, 0, 179, 161, 1, 0, 0, 0, 179, 162, 1, 0, 0, 0, 179, 163, 1, 0, 0, 0, 179, 164, 1, 0, 0, 0, 179, 165, 1, 0, 0, 0, 179, 166, 1, 0, 0, 0, 179, 167, 1, 0, 0, 0, 179, 168, 1, 0, 0, 0, 179, 169, 1, 0, 0, 0, 179, 170, 1, 0, 0, 0, 179, 171, 1, 0, 0, 0, 179, 173, 1, 0, 0, 0, 179, 175, 1, 0, 0, 0, 179, 177, 1, 0, 0, 0, 180, 7, 1, 0, 0, 0, 181, 182, 5, 16, 0, 0, 182, 183, 3, 10, 5, 0, 183, 9, 1, 0, 0, 0, 184, 185, 6, 5, -1, 0, 185, 186, 5, 50, 0, 0, 186, 214, 3, 10, 5, 8, 187, 214, 3, 16, 8, 0, 188, 214, 3, 12, 6, 0, 189, 191, 3, 16, 8, 0, 190, 192, 5, 50, 0, 0, 191, 190, 1, 0, 0, 0, 191, 192, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193, 194, 5, 45, 0, 0, 194, 195, 5, 49, 0, 0, 195, 200, 3, 16, 8, 0, 196, 197, 5, 40, 0, 0, 197, 199, 3, 16, 8, 0, 198, 196, 1, 0, 0, 0, 199, 202, 1, 0, 0, 0, 200, 198, 1, 0, 0, 0, 200, 201, 1, 0, 0, 0, 201, 203, 1, 0, 0, 0, 202, 200, 1, 0, 0, 0, 203, 204, 5, 56, 0, 0, 204, 214, 1, 0, 0, 0, 205, 206, 3, 16, 8, 0, 206, 208, 5, 46, 0, 0, 207, 209, 5, 50, 0, 0, 208, 207, 1, 0, 0, 0, 208, 209, 1, 0, 0, 0, 209, 210, 1, 0, 0, 0, 210, 211, 5, 51, 0, 0, 211, 214, 1, 0, 0, 0, 212, 214, 3, 14, 7, 0, 213, 184, 1, 0, 0, 0, 213, 187, 1, 0, 0, 0, 213, 188, 1, 0, 0, 0, 213, 189, 1, 0, 0, 0, 213, 205, 1, 0, 0, 0, 213, 212, 1, 0, 0, 0, 214, 223, 1, 0, 0, 0, 215, 216, 10, 5, 0, 0, 216, 217, 5, 35, 0, 0, 217, 222, 3, 10, 5, 6, 218, 219, 10, 4, 0, 0, 219, 220, 5, 53, 0, 0, 220, 222, 3, 10, 5, 5, 221, 215, 1, 0, 0, 0, 221, 218, 1, 0, 0, 0, 222, 225, 1, 0, 0, 0, 223, 221, 1, 0, 0, 0, 223, 224, 1, 0, 0, 0, 224, 11, 1, 0, 0, 0, 225, 223, 1, 0, 0, 0, 226, 228, 3, 16, 8, 0, 227, 229, 5, 50, 0, 0, 228, 227, 1, 0, 0, 0, 228, 229, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 231, 5, 48, 0, 0, 231, 232, 3, 106, 53, 0, 232, 241, 1, 0, 0, 0, 233, 235, 3, 16, 8, 0, 234, 236, 5, 50, 0, 0, 235, 234, 1, 0, 0, 0, 235, 236, 1, 0, 0, 0, 236, 237, 1, 0, 0, 0, 237, 238, 5, 55, 0, 0, 238, 239, 3, 106, 53, 0, 239, 241, 1, 0, 0, 0, 240, 226, 1, 0, 0, 0, 240, 233, 1, 0, 0, 0, 241, 13, 1, 0, 0, 0, 242, 245, 3, 58, 29, 0, 243, 244, 5, 38, 0, 0, 244, 246, 3, 30, 15, 0, 245, 243, 1, 0, 0, 0, 245, 246, 1, 0, 0, 0, 246, 247, 1, 0, 0, 0, 247, 248, 5, 39, 0, 0, 248, 249, 3, 68, 34, 0, 249, 15, 1, 0, 0, 0, 250, 256, 3, 18, 9, 0, 251, 252, 3, 18, 9, 0, 252, 253, 3, 108, 54, 0, 253, 254, 3, 18, 9, 0, 254, 256, 1, 0, 0, 0, 255, 250, 1, 0, 0, 0, 255, 251, 1, 0, 0, 0, 256, 17, 1, 0, 0, 0, 257, 258, 6, 9, -1, 0, 258, 262, 3, 20, 10, 0, 259, 260, 7, 0, 0, 0, 260, 262, 3, 18, 9, 3, 261, 257, 1, 0, 0, 0, 261, 259, 1, 0, 0, 0, 262, 271, 1, 0, 0, 0, 263, 264, 10, 2, 0, 0, 264, 265, 7, 1, 0, 0, 265, 270, 3, 18, 9, 3, 266, 267, 10, 1, 0, 0, 267, 268, 7, 0, 0, 0, 268, 270, 3, 18, 9, 2, 269, 263, 1, 0, 0, 0, 269, 266, 1, 0, 0, 0, 270, 273, 1, 0, 0, 0, 271, 269, 1, 0, 0, 0, 271, 272, 1, 0, 0, 0, 272, 19, 1, 0, 0, 0, 273, 271, 1, 0, 0, 0, 274, 275, 6, 10, -1, 0, 275, 283, 3, 68, 34, 0, 276, 283, 3, 58, 29, 0, 277, 283, 3, 22, 11, 0, 278, 279, 5, 49, 0, 0, 279, 280, 3, 10, 5, 0, 280, 281, 5, 56, 0, 0, 281, 283, 1, 0, 0, 0, 282, 274, 1, 0, 0, 0, 282, 276, 1, 0, 0, 0, 282, 277, 1, 0, 0, 0, 282, 278, 1, 0, 0, 0, 283, 289, 1, 0, 0, 0, 284, 285, 10, 1, 0, 0, 285, 286, 5, 38, 0, 0, 286, 288, 3, 30, 15, 0, 287, 284, 1, 0, 0, 0, 288, 291, 1, 0, 0, 0, 289, 287, 1, 0, 0, 0, 289, 290, 1, 0, 0, 0, 290, 21, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 292, 293, 3, 24, 12, 0, 293, 307, 5, 49, 0, 0, 294, 308, 5, 67, 0, 0, 295, 300, 3, 10, 5, 0, 296, 297, 5, 40, 0, 0, 297, 299, 3, 10, 5, 0, 298, 296, 1, 0, 0, 0, 299, 302, 1, 0, 0, 0, 300, 298, 1, 0, 0, 0, 300, 301, 1, 0, 0, 0, 301, 305, 1, 0, 0, 0, 302, 300, 1, 0, 0, 0, 303, 304, 5, 40, 0, 0, 304, 306, 3, 26, 13, 0, 305, 303, 1, 0, 0, 0, 305, 306, 1, 0, 0, 0, 306, 308, 1, 0, 0, 0, 307, 294, 1, 0, 0, 0, 307, 295, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 310, 5, 56, 0, 0, 310, 23, 1, 0, 0, 0, 311, 312, 3, 72, 36, 0, 312, 25, 1, 0, 0, 0, 313, 314, 5, 70, 0, 0, 314, 319, 3, 28, 14, 0, 315, 316, 5, 40, 0, 0, 316, 318, 3, 28, 14, 0, 317, 315, 1, 0, 0, 0, 318, 321, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 319, 320, 1, 0, 0, 0, 320, 322, 1, 0, 0, 0, 321, 319, 1, 0, 0, 0, 322, 323, 5, 71, 0, 0, 323, 27, 1, 0, 0, 0, 324, 325, 3, 106, 53, 0, 325, 326, 5, 39, 0, 0, 326, 327, 3, 68, 34, 0, 327, 29, 1, 0, 0, 0, 328, 329, 3, 64, 32, 0, 329, 31, 1, 0, 0, 0, 330, 331, 5, 12, 0, 0, 331, 332, 3, 34, 17, 0, 332, 33, 1, 0, 0, 0, 333, 338, 3, 36, 18, 0, 334, 335, 5, 40, 0, 0, 335, 337, 3, 36, 18, 0, 336, 334, 1, 0, 0, 0, 337, 340, 1, 0, 0, 0, 338, 336, 1, 0, 0, 0, 338, 339, 1, 0, 0, 0, 339, 35, 1, 0, 0, 0, 340, 338, 1, 0, 0, 0, 341, 342, 3, 58, 29, 0, 342, 343, 5, 37, 0, 0, 343, 345, 1, 0, 0, 0, 344, 341, 1, 0, 0, 0, 344, 345, 1, 0, 0, 0, 345, 346, 1, 0, 0, 0, 346, 347, 3, 10, 5, 0, 347, 37, 1, 0, 0, 0, 348, 349, 5, 6, 0, 0, 349, 354, 3, 40, 20, 0, 350, 351, 5, 40, 0, 0, 351, 353, 3, 40, 20, 0, 352, 350, 1, 0, 0, 0, 353, 356, 1, 0, 0, 0, 354, 352, 1, 0, 0, 0, 354, 355, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 357, 359, 3, 46, 23, 0, 358, 357, 1, 0, 0, 0, 358, 359, 1, 0, 0, 0, 359, 39, 1, 0, 0, 0, 360, 361, 3, 42, 21, 0, 361, 362, 5, 39, 0, 0, 362, 364, 1, 0, 0, 0, 363, 360, 1, 0, 0, 0, 363, 364, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 366, 3, 44, 22, 0, 366, 41, 1, 0, 0, 0, 367, 368, 7, 2, 0, 0, 368, 43, 1, 0, 0, 0, 369, 370, 7, 2, 0, 0, 370, 45, 1, 0, 0, 0, 371, 372, 5, 83, 0, 0, 372, 377, 5, 84, 0, 0, 373, 374, 5, 40, 0, 0, 374, 376, 5, 84, 0, 0, 375, 373, 1, 0, 0, 0, 376, 379, 1, 0, 0, 0, 377, 375, 1, 0, 0, 0, 377, 378, 1, 0, 0, 0, 378, 47, 1, 0, 0, 0, 379, 377, 1, 0, 0, 0, 380, 381, 5, 22, 0, 0, 381, 386, 3, 40, 20, 0, 382, 383, 5, 40, 0, 0, 383, 385, 3, 40, 20, 0, 384, 382, 1, 0, 0, 0, 385, 388, 1, 0, 0, 0, 386, 384, 1, 0, 0, 0, 386, 387, 1, 0, 0, 0, 387, 390, 1, 0, 0, 0, 388, 386, 1, 0, 0, 0, 389, 391, 3, 54, 27, 0, 390, 389, 1, 0, 0, 0, 390, 391, 1, 0, 0, 0, 391, 394, 1, 0, 0, 0, 392, 393, 5, 34, 0, 0, 393, 395, 3, 34, 17, 0, 394, 392, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 49, 1, 0, 0, 0, 396, 397, 5, 4, 0, 0, 397, 398, 3, 34, 17, 0, 398, 51, 1, 0, 0, 0, 399, 401, 5, 15, 0, 0, 400, 402, 3, 54, 27, 0, 401, 400, 1, 0, 0, 0, 401, 402, 1, 0, 0, 0, 402, 405, 1, 0, 0, 0, 403, 404, 5, 34, 0, 0, 404, 406, 3, 34, 17, 0, 405, 403, 1, 0, 0, 0, 405, 406, 1, 0, 0, 0, 406, 53, 1, 0, 0, 0, 407, 412, 3, 56, 28, 0, 408, 409, 5, 40, 0, 0, 409, 411, 3, 56, 28, 0, 410, 408, 1, 0, 0, 0, 411, 414, 1, 0, 0, 0, 412, 410, 1, 0, 0, 0, 412, 413, 1, 0, 0, 0, 413, 55, 1, 0, 0, 0, 414, 412, 1, 0, 0, 0, 415, 418, 3, 36, 18, 0, 416, 417, 5, 16, 0, 0, 417, 419, 3, 10, 5, 0, 418, 416, 1, 0, 0, 0, 418, 419, 1, 0, 0, 0, 419, 57, 1, 0, 0, 0, 420, 425, 3, 72, 36, 0, 421, 422, 5, 42, 0, 0, 422, 424, 3, 72, 36, 0, 423, 421, 1, 0, 0, 0, 424, 427, 1, 0, 0, 0, 425, 423, 1, 0, 0, 0, 425, 426, 1, 0, 0, 0, 426, 59, 1, 0, 0, 0, 427, 425, 1, 0, 0, 0, 428, 433, 3, 66, 33, 0, 429, 430, 5, 42, 0, 0, 430, 432, 3, 66, 33, 0, 431, 429, 1, 0, 0, 0, 432, 435, 1, 0, 0, 0, 433, 431, 1, 0, 0, 0, 433, 434, 1, 0, 0, 0, 434, 61, 1, 0, 0, 0, 435, 433, 1, 0, 0, 0, 436, 441, 3, 60, 30, 0, 437, 438, 5, 40, 0, 0, 438, 440, 3, 60, 30, 0, 439, 437, 1, 0, 0, 0, 440, 443, 1, 0, 0, 0, 441, 439, 1, 0, 0, 0, 441, 442, 1, 0, 0, 0, 442, 63, 1, 0, 0, 0, 443, 441, 1, 0, 0, 0, 444, 445, 7, 3, 0, 0, 445, 65, 1, 0, 0, 0, 446, 449, 5, 88, 0, 0, 447, 449, 3, 70, 35, 0, 448, 446, 1, 0, 0, 0, 448, 447, 1, 0, 0, 0, 449, 67, 1, 0, 0, 0, 450, 493, 5, 51, 0, 0, 451, 452, 3, 104, 52, 0, 452, 453, 5, 75, 0, 0, 453, 493, 1, 0, 0, 0, 454, 493, 3, 102, 51, 0, 455, 493, 3, 104, 52, 0, 456, 493, 3, 98, 49, 0, 457, 493, 3, 70, 35, 0, 458, 493, 3, 106, 53, 0, 459, 460, 5, 73, 0, 0, 460, 465, 3, 100, 50, 0, 461, 462, 5, 40, 0, 0, 462, 464, 3, 100, 50, 0, 463, 461, 1, 0, 0, 0, 464, 467, 1, 0, 0, 0, 465, 463, 1, 0, 0, 0, 465, 466, 1, 0, 0, 0, 466, 468, 1, 0, 0, 0, 467, 465, 1, 0, 0, 0, 468, 469, 5, 74, 0, 0, 469, 493, 1, 0, 0, 0, 470, 471, 5, 73, 0, 0, 471, 476, 3, 98, 49, 0, 472, 473, 5, 40, 0, 0, 473, 475, 3, 98, 49, 0, 474, 472, 1, 0, 0, 0, 475, 478, 1, 0, 0, 0, 476, 474, 1, 0, 0, 0, 476, 477, 1, 0, 0, 0, 477, 479, 1, 0, 0, 0, 478, 476, 1, 0, 0, 0, 479, 480, 5, 74, 0, 0, 480, 493, 1, 0, 0, 0, 481, 482, 5, 73, 0, 0, 482, 487, 3, 106, 53, 0, 483, 484, 5, 40, 0, 0, 484, 486, 3, 106, 53, 0, 485, 483, 1, 0, 0, 0, 486, 489, 1, 0, 0, 0, 487, 485, 1, 0, 0, 0, 487, 488, 1, 0, 0, 0, 488, 490, 1, 0, 0, 0, 489, 487, 1, 0, 0, 0, 490, 491, 5, 74, 0, 0, 491, 493, 1, 0, 0, 0, 492, 450, 1, 0, 0, 0, 492, 451, 1, 0, 0, 0, 492, 454, 1, 0, 0, 0, 492, 455, 1, 0, 0, 0, 492, 456, 1, 0, 0, 0, 492, 457, 1, 0, 0, 0, 492, 458, 1, 0, 0, 0, 492, 459, 1, 0, 0, 0, 492, 470, 1, 0, 0, 0, 492, 481, 1, 0, 0, 0, 493, 69, 1, 0, 0, 0, 494, 497, 5, 54, 0, 0, 495, 497, 5, 72, 0, 0, 496, 494, 1, 0, 0, 0, 496, 495, 1, 0, 0, 0, 497, 71, 1, 0, 0, 0, 498, 501, 3, 64, 32, 0, 499, 501, 3, 70, 35, 0, 500, 498, 1, 0, 0, 0, 500, 499, 1, 0, 0, 0, 501, 73, 1, 0, 0, 0, 502, 503, 5, 9, 0, 0, 503, 504, 5, 32, 0, 0, 504, 75, 1, 0, 0, 0, 505, 506, 5, 14, 0, 0, 506, 511, 3, 78, 39, 0, 507, 508, 5, 40, 0, 0, 508, 510, 3, 78, 39, 0, 509, 507, 1, 0, 0, 0, 510, 513, 1, 0, 0, 0, 511, 509, 1, 0, 0, 0, 511, 512, 1, 0, 0, 0, 512, 77, 1, 0, 0, 0, 513, 511, 1, 0, 0, 0, 514, 516, 3, 10, 5, 0, 515, 517, 7, 4, 0, 0, 516, 515, 1, 0, 0, 0, 516, 517, 1, 0, 0, 0, 517, 520, 1, 0, 0, 0, 518, 519, 5, 52, 0, 0, 519, 521, 7, 5, 0, 0, 520, 518, 1, 0, 0, 0, 520, 521, 1, 0, 0, 0, 521, 79, 1, 0, 0, 0, 522, 523, 5, 8, 0, 0, 523, 524, 3, 62, 31, 0, 524, 81, 1, 0, 0, 0, 525, 526, 5, 2, 0, 0, 526, 527, 3, 62, 31, 0, 527, 83, 1, 0, 0, 0, 528, 529, 5, 11, 0, 0, 529, 534, 3, 86, 43, 0, 530, 531, 5, 40, 0, 0, 531, 533, 3, 86, 43, 0, 532, 530, 1, 0, 0, 0, 533, 536, 1, 0, 0, 0, 534, 532, 1, 0, 0, 0, 534, 535, 1, 0, 0, 0, 535, 85, 1, 0, 0, 0, 536, 534, 1, 0, 0, 0, 537, 538, 3, 60, 30, 0, 538, 539, 5, 92, 0, 0, 539, 540, 3, 60, 30, 0, 540, 87, 1, 0, 0, 0, 541, 542, 5, 1, 0, 0, 542, 543, 3, 20, 10, 0, 543, 545, 3, 106, 53, 0, 544, 546, 3, 94, 47, 0, 545, 544, 1, 0, 0, 0, 545, 546, 1, 0, 0, 0, 546, 89, 1, 0, 0, 0, 547, 548, 5, 7, 0, 0, 548, 549, 3, 20, 10, 0, 549, 550, 3, 106, 53, 0, 550, 91, 1, 0, 0, 0, 551, 552, 5, 10, 0, 0, 552, 553, 3, 58, 29, 0, 553, 93, 1, 0, 0, 0, 554, 559, 3, 96, 48, 0, 555, 556, 5, 40, 0, 0, 556, 558, 3, 96, 48, 0, 557, 555, 1, 0, 0, 0, 558, 561, 1, 0, 0, 0, 559, 557, 1, 0, 0, 0, 559, 560, 1, 0, 0, 0, 560, 95, 1, 0, 0, 0, 561, 559, 1, 0, 0, 0, 562, 563, 3, 64, 32, 0, 563, 564, 5, 37, 0, 0, 564, 565, 3, 68, 34, 0, 565, 97, 1, 0, 0, 0, 566, 567, 7, 6, 0, 0, 567, 99, 1, 0, 0, 0, 568, 571, 3, 102, 51, 0, 569, 571, 3, 104, 52, 0, 570, 568, 1, 0, 0, 0, 570, 569, 1, 0, 0, 0, 571, 101, 1, 0, 0, 0, 572, 574, 7, 0, 0, 0, 573, 572, 1, 0, 0, 0, 573, 574, 1, 0, 0, 0, 574, 575, 1, 0, 0, 0, 575, 576, 5, 33, 0, 0, 576, 103, 1, 0, 0, 0, 577, 579, 7, 0, 0, 0, 578, 577, 1, 0, 0, 0, 578, 579, 1, 0, 0, 0, 579, 580, 1, 0, 0, 0, 580, 581, 5, 32, 0, 0, 581, 105, 1, 0, 0, 0, 582, 583, 5, 31, 0, 0, 583, 107, 1, 0, 0, 0, 584, 585, 7, 7, 0, 0, 585, 109, 1, 0, 0, 0, 586, 587, 5, 5, 0, 0, 587, 588, 3, 112, 56, 0, 588, 111, 1, 0, 0, 0, 589, 590, 5, 73, 0, 0, 590, 591, 3, 2, 1, 0, 591, 592, 5, 74, 0, 0, 592, 113, 1, 0, 0, 0, 593, 594, 5, 13, 0, 0, 594, 595, 5, 108, 0, 0, 595, 115, 1, 0, 0, 0, 596, 597, 5, 3, 0, 0, 597, 600, 5, 98, 0, 0, 598, 599, 5, 96, 0, 0, 599, 601, 3, 60, 30, 0, 600, 598, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 611, 1, 0, 0, 0, 602, 603, 5, 97, 0, 0, 603, 608, 3, 118, 59, 0, 604, 605, 5, 40, 0, 0, 605, 607, 3, 118, 59, 0, 606, 604, 1, 0, 0, 0, 607, 610, 1, 0, 0, 0, 608, 606, 1, 0, 0, 0, 608, 609, 1, 0, 0, 0, 609, 612, 1, 0, 0, 0, 610, 608, 1, 0, 0, 0, 611, 602, 1, 0, 0, 0, 611, 612, 1, 0, 0, 0, 612, 117, 1, 0, 0, 0, 613, 614, 3, 60, 30, 0, 614, 615, 5, 37, 0, 0, 615, 617, 1, 0, 0, 0, 616, 613, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 619, 3, 60, 30, 0, 619, 119, 1, 0, 0, 0, 620, 621, 5, 21, 0, 0, 621, 622, 3, 40, 20, 0, 622, 623, 5, 96, 0, 0, 623, 624, 3, 62, 31, 0, 624, 121, 1, 0, 0, 0, 625, 626, 5, 19, 0, 0, 626, 629, 3, 54, 27, 0, 627, 628, 5, 34, 0, 0, 628, 630, 3, 34, 17, 0, 629, 627, 1, 0, 0, 0, 629, 630, 1, 0, 0, 0, 630, 123, 1, 0, 0, 0, 631, 632, 7, 8, 0, 0, 632, 633, 5, 122, 0, 0, 633, 634, 3, 126, 63, 0, 634, 635, 3, 128, 64, 0, 635, 125, 1, 0, 0, 0, 636, 637, 3, 40, 20, 0, 637, 127, 1, 0, 0, 0, 638, 639, 5, 96, 0, 0, 639, 644, 3, 130, 65, 0, 640, 641, 5, 40, 0, 0, 641, 643, 3, 130, 65, 0, 642, 640, 1, 0, 0, 0, 643, 646, 1, 0, 0, 0, 644, 642, 1, 0, 0, 0, 644, 645, 1, 0, 0, 0, 645, 129, 1, 0, 0, 0, 646, 644, 1, 0, 0, 0, 647, 648, 3, 16, 8, 0, 648, 131, 1, 0, 0, 0, 649, 650, 5, 18, 0, 0, 650, 653, 3, 58, 29, 0, 651, 652, 5, 96, 0, 0, 652, 654, 3, 58, 29, 0, 653, 651, 1, 0, 0, 0, 653, 654, 1, 0, 0, 0, 654, 660, 1, 0, 0, 0, 655, 656, 5, 92, 0, 0, 656, 657, 3, 58, 29, 0, 657, 658, 5, 40, 0, 0, 658, 659, 3, 58, 29, 0, 659, 661, 1, 0, 0, 0, 660, 655, 1, 0, 0, 0, 660, 661, 1, 0, 0, 0, 661, 133, 1, 0, 0, 0, 662, 663, 5, 20, 0, 0, 663, 664, 3, 62, 31, 0, 664, 135, 1, 0, 0, 0, 63, 147, 156, 179, 191, 200, 208, 213, 221, 223, 228, 235, 240, 245, 255, 261, 269, 271, 282, 289, 300, 305, 307, 319, 338, 344, 354, 358, 363, 377, 386, 390, 394, 401, 405, 412, 418, 425, 433, 441, 448, 465, 476, 487, 492, 496, 500, 511, 516, 520, 534, 545, 559, 570, 573, 578, 600, 608, 611, 616, 629, 644, 653, 660] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index 3691f2374408..d3df22deb3e9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -27,32 +27,33 @@ public class EsqlBaseParser extends ParserConfig { public static final int DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, - WHERE=16, JOIN_LOOKUP=17, DEV_CHANGE_POINT=18, DEV_INLINESTATS=19, DEV_LOOKUP=20, - DEV_METRICS=21, DEV_JOIN_FULL=22, DEV_JOIN_LEFT=23, DEV_JOIN_RIGHT=24, - UNKNOWN_CMD=25, LINE_COMMENT=26, MULTILINE_COMMENT=27, WS=28, PIPE=29, - QUOTED_STRING=30, INTEGER_LITERAL=31, DECIMAL_LITERAL=32, BY=33, AND=34, - ASC=35, ASSIGN=36, CAST_OP=37, COLON=38, COMMA=39, DESC=40, DOT=41, FALSE=42, - FIRST=43, IN=44, IS=45, LAST=46, LIKE=47, LP=48, NOT=49, NULL=50, NULLS=51, - OR=52, PARAM=53, RLIKE=54, RP=55, TRUE=56, EQ=57, CIEQ=58, NEQ=59, LT=60, - LTE=61, GT=62, GTE=63, PLUS=64, MINUS=65, ASTERISK=66, SLASH=67, PERCENT=68, - LEFT_BRACES=69, RIGHT_BRACES=70, NAMED_OR_POSITIONAL_PARAM=71, OPENING_BRACKET=72, - CLOSING_BRACKET=73, UNQUOTED_IDENTIFIER=74, QUOTED_IDENTIFIER=75, EXPR_LINE_COMMENT=76, - EXPR_MULTILINE_COMMENT=77, EXPR_WS=78, EXPLAIN_WS=79, EXPLAIN_LINE_COMMENT=80, - EXPLAIN_MULTILINE_COMMENT=81, METADATA=82, UNQUOTED_SOURCE=83, FROM_LINE_COMMENT=84, - FROM_MULTILINE_COMMENT=85, FROM_WS=86, ID_PATTERN=87, PROJECT_LINE_COMMENT=88, - PROJECT_MULTILINE_COMMENT=89, PROJECT_WS=90, AS=91, RENAME_LINE_COMMENT=92, - RENAME_MULTILINE_COMMENT=93, RENAME_WS=94, ON=95, WITH=96, ENRICH_POLICY_NAME=97, - ENRICH_LINE_COMMENT=98, ENRICH_MULTILINE_COMMENT=99, ENRICH_WS=100, ENRICH_FIELD_LINE_COMMENT=101, - ENRICH_FIELD_MULTILINE_COMMENT=102, ENRICH_FIELD_WS=103, MVEXPAND_LINE_COMMENT=104, - MVEXPAND_MULTILINE_COMMENT=105, MVEXPAND_WS=106, INFO=107, SHOW_LINE_COMMENT=108, - SHOW_MULTILINE_COMMENT=109, SHOW_WS=110, SETTING=111, SETTING_LINE_COMMENT=112, - SETTTING_MULTILINE_COMMENT=113, SETTING_WS=114, LOOKUP_LINE_COMMENT=115, - LOOKUP_MULTILINE_COMMENT=116, LOOKUP_WS=117, LOOKUP_FIELD_LINE_COMMENT=118, - LOOKUP_FIELD_MULTILINE_COMMENT=119, LOOKUP_FIELD_WS=120, JOIN=121, USING=122, - JOIN_LINE_COMMENT=123, JOIN_MULTILINE_COMMENT=124, JOIN_WS=125, METRICS_LINE_COMMENT=126, - METRICS_MULTILINE_COMMENT=127, METRICS_WS=128, CLOSING_METRICS_LINE_COMMENT=129, - CLOSING_METRICS_MULTILINE_COMMENT=130, CLOSING_METRICS_WS=131, CHANGE_POINT_LINE_COMMENT=132, - CHANGE_POINT_MULTILINE_COMMENT=133, CHANGE_POINT_WS=134; + WHERE=16, JOIN_LOOKUP=17, DEV_CHANGE_POINT=18, DEV_INLINESTATS=19, DEV_INSIST=20, + DEV_LOOKUP=21, DEV_METRICS=22, DEV_JOIN_FULL=23, DEV_JOIN_LEFT=24, DEV_JOIN_RIGHT=25, + UNKNOWN_CMD=26, LINE_COMMENT=27, MULTILINE_COMMENT=28, WS=29, PIPE=30, + QUOTED_STRING=31, INTEGER_LITERAL=32, DECIMAL_LITERAL=33, BY=34, AND=35, + ASC=36, ASSIGN=37, CAST_OP=38, COLON=39, COMMA=40, DESC=41, DOT=42, FALSE=43, + FIRST=44, IN=45, IS=46, LAST=47, LIKE=48, LP=49, NOT=50, NULL=51, NULLS=52, + OR=53, PARAM=54, RLIKE=55, RP=56, TRUE=57, EQ=58, CIEQ=59, NEQ=60, LT=61, + LTE=62, GT=63, GTE=64, PLUS=65, MINUS=66, ASTERISK=67, SLASH=68, PERCENT=69, + LEFT_BRACES=70, RIGHT_BRACES=71, NAMED_OR_POSITIONAL_PARAM=72, OPENING_BRACKET=73, + CLOSING_BRACKET=74, UNQUOTED_IDENTIFIER=75, QUOTED_IDENTIFIER=76, EXPR_LINE_COMMENT=77, + EXPR_MULTILINE_COMMENT=78, EXPR_WS=79, EXPLAIN_WS=80, EXPLAIN_LINE_COMMENT=81, + EXPLAIN_MULTILINE_COMMENT=82, METADATA=83, UNQUOTED_SOURCE=84, FROM_LINE_COMMENT=85, + FROM_MULTILINE_COMMENT=86, FROM_WS=87, ID_PATTERN=88, PROJECT_LINE_COMMENT=89, + PROJECT_MULTILINE_COMMENT=90, PROJECT_WS=91, AS=92, RENAME_LINE_COMMENT=93, + RENAME_MULTILINE_COMMENT=94, RENAME_WS=95, ON=96, WITH=97, ENRICH_POLICY_NAME=98, + ENRICH_LINE_COMMENT=99, ENRICH_MULTILINE_COMMENT=100, ENRICH_WS=101, ENRICH_FIELD_LINE_COMMENT=102, + ENRICH_FIELD_MULTILINE_COMMENT=103, ENRICH_FIELD_WS=104, MVEXPAND_LINE_COMMENT=105, + MVEXPAND_MULTILINE_COMMENT=106, MVEXPAND_WS=107, INFO=108, SHOW_LINE_COMMENT=109, + SHOW_MULTILINE_COMMENT=110, SHOW_WS=111, SETTING=112, SETTING_LINE_COMMENT=113, + SETTTING_MULTILINE_COMMENT=114, SETTING_WS=115, LOOKUP_LINE_COMMENT=116, + LOOKUP_MULTILINE_COMMENT=117, LOOKUP_WS=118, LOOKUP_FIELD_LINE_COMMENT=119, + LOOKUP_FIELD_MULTILINE_COMMENT=120, LOOKUP_FIELD_WS=121, JOIN=122, USING=123, + JOIN_LINE_COMMENT=124, JOIN_MULTILINE_COMMENT=125, JOIN_WS=126, METRICS_LINE_COMMENT=127, + METRICS_MULTILINE_COMMENT=128, METRICS_WS=129, CLOSING_METRICS_LINE_COMMENT=130, + CLOSING_METRICS_MULTILINE_COMMENT=131, CLOSING_METRICS_WS=132, CHANGE_POINT_LINE_COMMENT=133, + CHANGE_POINT_MULTILINE_COMMENT=134, CHANGE_POINT_WS=135, INSIST_WS=136, + INSIST_LINE_COMMENT=137, INSIST_MULTILINE_COMMENT=138; public static final int RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, @@ -74,7 +75,8 @@ public class EsqlBaseParser extends ParserConfig { RULE_explainCommand = 55, RULE_subqueryExpression = 56, RULE_showCommand = 57, RULE_enrichCommand = 58, RULE_enrichWithClause = 59, RULE_lookupCommand = 60, RULE_inlinestatsCommand = 61, RULE_joinCommand = 62, RULE_joinTarget = 63, - RULE_joinCondition = 64, RULE_joinPredicate = 65, RULE_changePointCommand = 66; + RULE_joinCondition = 64, RULE_joinPredicate = 65, RULE_changePointCommand = 66, + RULE_insistCommand = 67; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", @@ -91,7 +93,7 @@ public class EsqlBaseParser extends ParserConfig { "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", "inlinestatsCommand", "joinCommand", "joinTarget", "joinCondition", "joinPredicate", - "changePointCommand" + "changePointCommand", "insistCommand" }; } public static final String[] ruleNames = makeRuleNames(); @@ -101,7 +103,7 @@ public class EsqlBaseParser extends ParserConfig { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", "'lookup'", null, null, null, null, null, - null, null, null, null, null, null, "'|'", null, null, null, "'by'", + null, null, null, null, null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "':'", "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", "'=~'", @@ -118,13 +120,13 @@ public class EsqlBaseParser extends ParserConfig { return new String[] { null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "JOIN_LOOKUP", "DEV_CHANGE_POINT", "DEV_INLINESTATS", "DEV_LOOKUP", - "DEV_METRICS", "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "UNKNOWN_CMD", - "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", - "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "WHERE", "JOIN_LOOKUP", "DEV_CHANGE_POINT", "DEV_INLINESTATS", "DEV_INSIST", + "DEV_LOOKUP", "DEV_METRICS", "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", + "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", + "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", + "COLON", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", + "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", + "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", @@ -142,7 +144,8 @@ public class EsqlBaseParser extends ParserConfig { "JOIN", "USING", "JOIN_LINE_COMMENT", "JOIN_MULTILINE_COMMENT", "JOIN_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", "CLOSING_METRICS_WS", "CHANGE_POINT_LINE_COMMENT", - "CHANGE_POINT_MULTILINE_COMMENT", "CHANGE_POINT_WS" + "CHANGE_POINT_MULTILINE_COMMENT", "CHANGE_POINT_WS", "INSIST_WS", "INSIST_LINE_COMMENT", + "INSIST_MULTILINE_COMMENT" }; } private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); @@ -229,9 +232,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(134); + setState(136); query(0); - setState(135); + setState(137); match(EOF); } } @@ -327,11 +330,11 @@ public class EsqlBaseParser extends ParserConfig { _ctx = _localctx; _prevctx = _localctx; - setState(138); + setState(140); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(145); + setState(147); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -342,16 +345,16 @@ public class EsqlBaseParser extends ParserConfig { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(140); - if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(141); - match(PIPE); setState(142); + if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); + setState(143); + match(PIPE); + setState(144); processingCommand(); } } } - setState(147); + setState(149); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -409,43 +412,43 @@ public class EsqlBaseParser extends ParserConfig { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(154); + setState(156); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(148); + setState(150); explainCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(149); + setState(151); fromCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(150); + setState(152); rowCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(151); + setState(153); showCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(152); + setState(154); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(153); + setState(155); metricsCommand(); } break; @@ -512,6 +515,9 @@ public class EsqlBaseParser extends ParserConfig { public ChangePointCommandContext changePointCommand() { return getRuleContext(ChangePointCommandContext.class,0); } + public InsistCommandContext insistCommand() { + return getRuleContext(InsistCommandContext.class,0); + } @SuppressWarnings("this-escape") public ProcessingCommandContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -536,127 +542,136 @@ public class EsqlBaseParser extends ParserConfig { ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(175); + setState(179); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(156); + setState(158); evalCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(157); + setState(159); whereCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(158); + setState(160); keepCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(159); + setState(161); limitCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(160); + setState(162); statsCommand(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(161); + setState(163); sortCommand(); } break; case 7: enterOuterAlt(_localctx, 7); { - setState(162); + setState(164); dropCommand(); } break; case 8: enterOuterAlt(_localctx, 8); { - setState(163); + setState(165); renameCommand(); } break; case 9: enterOuterAlt(_localctx, 9); { - setState(164); + setState(166); dissectCommand(); } break; case 10: enterOuterAlt(_localctx, 10); { - setState(165); + setState(167); grokCommand(); } break; case 11: enterOuterAlt(_localctx, 11); { - setState(166); + setState(168); enrichCommand(); } break; case 12: enterOuterAlt(_localctx, 12); { - setState(167); + setState(169); mvExpandCommand(); } break; case 13: enterOuterAlt(_localctx, 13); { - setState(168); + setState(170); joinCommand(); } break; case 14: enterOuterAlt(_localctx, 14); { - setState(169); + setState(171); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(170); + setState(172); inlinestatsCommand(); } break; case 15: enterOuterAlt(_localctx, 15); { - setState(171); + setState(173); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(172); + setState(174); lookupCommand(); } break; case 16: enterOuterAlt(_localctx, 16); { - setState(173); + setState(175); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(174); + setState(176); changePointCommand(); } break; + case 17: + enterOuterAlt(_localctx, 17); + { + setState(177); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); + setState(178); + insistCommand(); + } + break; } } catch (RecognitionException re) { @@ -702,9 +717,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(177); + setState(181); match(WHERE); - setState(178); + setState(182); booleanExpression(0); } } @@ -920,7 +935,7 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(209); + setState(213); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -929,9 +944,9 @@ public class EsqlBaseParser extends ParserConfig { _ctx = _localctx; _prevctx = _localctx; - setState(181); + setState(185); match(NOT); - setState(182); + setState(186); booleanExpression(8); } break; @@ -940,7 +955,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(183); + setState(187); valueExpression(); } break; @@ -949,7 +964,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(184); + setState(188); regexBooleanExpression(); } break; @@ -958,41 +973,41 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(185); + setState(189); valueExpression(); - setState(187); + setState(191); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(186); + setState(190); match(NOT); } } - setState(189); + setState(193); match(IN); - setState(190); + setState(194); match(LP); - setState(191); + setState(195); valueExpression(); - setState(196); + setState(200); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(192); + setState(196); match(COMMA); - setState(193); + setState(197); valueExpression(); } } - setState(198); + setState(202); _errHandler.sync(this); _la = _input.LA(1); } - setState(199); + setState(203); match(RP); } break; @@ -1001,21 +1016,21 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(201); + setState(205); valueExpression(); - setState(202); + setState(206); match(IS); - setState(204); + setState(208); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(203); + setState(207); match(NOT); } } - setState(206); + setState(210); match(NULL); } break; @@ -1024,13 +1039,13 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new MatchExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(208); + setState(212); matchBooleanExpression(); } break; } _ctx.stop = _input.LT(-1); - setState(219); + setState(223); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1038,7 +1053,7 @@ public class EsqlBaseParser extends ParserConfig { if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(217); + setState(221); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -1046,11 +1061,11 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(211); + setState(215); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(212); + setState(216); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(213); + setState(217); ((LogicalBinaryContext)_localctx).right = booleanExpression(6); } break; @@ -1059,18 +1074,18 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(214); + setState(218); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(215); + setState(219); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(216); + setState(220); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; } } } - setState(221); + setState(225); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1125,48 +1140,48 @@ public class EsqlBaseParser extends ParserConfig { enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(236); + setState(240); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(222); + setState(226); valueExpression(); - setState(224); + setState(228); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(223); + setState(227); match(NOT); } } - setState(226); + setState(230); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(227); + setState(231); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(229); + setState(233); valueExpression(); - setState(231); + setState(235); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(230); + setState(234); match(NOT); } } - setState(233); + setState(237); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(234); + setState(238); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1226,23 +1241,23 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(238); + setState(242); ((MatchBooleanExpressionContext)_localctx).fieldExp = qualifiedName(); - setState(241); + setState(245); _errHandler.sync(this); _la = _input.LA(1); if (_la==CAST_OP) { { - setState(239); + setState(243); match(CAST_OP); - setState(240); + setState(244); ((MatchBooleanExpressionContext)_localctx).fieldType = dataType(); } } - setState(243); + setState(247); match(COLON); - setState(244); + setState(248); ((MatchBooleanExpressionContext)_localctx).matchQuery = constant(); } } @@ -1326,14 +1341,14 @@ public class EsqlBaseParser extends ParserConfig { ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(251); + setState(255); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(246); + setState(250); operatorExpression(0); } break; @@ -1341,11 +1356,11 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(247); + setState(251); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(248); + setState(252); comparisonOperator(); - setState(249); + setState(253); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1470,7 +1485,7 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(257); + setState(261); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1479,7 +1494,7 @@ public class EsqlBaseParser extends ParserConfig { _ctx = _localctx; _prevctx = _localctx; - setState(254); + setState(258); primaryExpression(0); } break; @@ -1488,7 +1503,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(255); + setState(259); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1499,13 +1514,13 @@ public class EsqlBaseParser extends ParserConfig { _errHandler.reportMatch(this); consume(); } - setState(256); + setState(260); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(267); + setState(271); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1513,7 +1528,7 @@ public class EsqlBaseParser extends ParserConfig { if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(265); + setState(269); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { case 1: @@ -1521,12 +1536,12 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(259); + setState(263); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(260); + setState(264); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & 7L) != 0)) ) { + if ( !(((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & 7L) != 0)) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { @@ -1534,7 +1549,7 @@ public class EsqlBaseParser extends ParserConfig { _errHandler.reportMatch(this); consume(); } - setState(261); + setState(265); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1543,9 +1558,9 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(262); + setState(266); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(263); + setState(267); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1556,14 +1571,14 @@ public class EsqlBaseParser extends ParserConfig { _errHandler.reportMatch(this); consume(); } - setState(264); + setState(268); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(269); + setState(273); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); } @@ -1721,7 +1736,7 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(278); + setState(282); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) { case 1: @@ -1730,7 +1745,7 @@ public class EsqlBaseParser extends ParserConfig { _ctx = _localctx; _prevctx = _localctx; - setState(271); + setState(275); constant(); } break; @@ -1739,7 +1754,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(272); + setState(276); qualifiedName(); } break; @@ -1748,7 +1763,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(273); + setState(277); functionExpression(); } break; @@ -1757,17 +1772,17 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(274); + setState(278); match(LP); - setState(275); + setState(279); booleanExpression(0); - setState(276); + setState(280); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(285); + setState(289); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1778,16 +1793,16 @@ public class EsqlBaseParser extends ParserConfig { { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(280); + setState(284); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(281); + setState(285); match(CAST_OP); - setState(282); + setState(286); dataType(); } } } - setState(287); + setState(291); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); } @@ -1853,16 +1868,16 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(288); + setState(292); functionName(); - setState(289); + setState(293); match(LP); - setState(303); + setState(307); _errHandler.sync(this); switch (_input.LA(1)) { case ASTERISK: { - setState(290); + setState(294); match(ASTERISK); } break; @@ -1883,34 +1898,34 @@ public class EsqlBaseParser extends ParserConfig { case QUOTED_IDENTIFIER: { { - setState(291); + setState(295); booleanExpression(0); - setState(296); + setState(300); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(292); + setState(296); match(COMMA); - setState(293); + setState(297); booleanExpression(0); } } } - setState(298); + setState(302); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); } - setState(301); + setState(305); _errHandler.sync(this); _la = _input.LA(1); if (_la==COMMA) { { - setState(299); + setState(303); match(COMMA); - setState(300); + setState(304); mapExpression(); } } @@ -1923,7 +1938,7 @@ public class EsqlBaseParser extends ParserConfig { default: break; } - setState(305); + setState(309); match(RP); } } @@ -1969,7 +1984,7 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(307); + setState(311); identifierOrParameter(); } } @@ -2025,27 +2040,27 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(309); + setState(313); match(LEFT_BRACES); - setState(310); + setState(314); entryExpression(); - setState(315); + setState(319); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(311); + setState(315); match(COMMA); - setState(312); + setState(316); entryExpression(); } } - setState(317); + setState(321); _errHandler.sync(this); _la = _input.LA(1); } - setState(318); + setState(322); match(RIGHT_BRACES); } } @@ -2097,11 +2112,11 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(320); + setState(324); ((EntryExpressionContext)_localctx).key = string(); - setState(321); + setState(325); match(COLON); - setState(322); + setState(326); ((EntryExpressionContext)_localctx).value = constant(); } } @@ -2159,7 +2174,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(324); + setState(328); identifier(); } } @@ -2206,9 +2221,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(326); + setState(330); match(ROW); - setState(327); + setState(331); fields(); } } @@ -2262,23 +2277,23 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(329); + setState(333); field(); - setState(334); + setState(338); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(330); + setState(334); match(COMMA); - setState(331); + setState(335); field(); } } } - setState(336); + setState(340); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } @@ -2330,19 +2345,19 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(340); + setState(344); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(337); + setState(341); qualifiedName(); - setState(338); + setState(342); match(ASSIGN); } break; } - setState(342); + setState(346); booleanExpression(0); } } @@ -2400,34 +2415,34 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(344); + setState(348); match(FROM); - setState(345); + setState(349); indexPattern(); - setState(350); + setState(354); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,25,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(346); + setState(350); match(COMMA); - setState(347); + setState(351); indexPattern(); } } } - setState(352); + setState(356); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,25,_ctx); } - setState(354); + setState(358); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) { case 1: { - setState(353); + setState(357); metadata(); } break; @@ -2480,19 +2495,19 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(359); + setState(363); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) { case 1: { - setState(356); + setState(360); clusterString(); - setState(357); + setState(361); match(COLON); } break; } - setState(361); + setState(365); indexString(); } } @@ -2538,7 +2553,7 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(363); + setState(367); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2592,7 +2607,7 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(365); + setState(369); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2653,25 +2668,25 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(367); + setState(371); match(METADATA); - setState(368); + setState(372); match(UNQUOTED_SOURCE); - setState(373); + setState(377); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,28,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(369); + setState(373); match(COMMA); - setState(370); + setState(374); match(UNQUOTED_SOURCE); } } } - setState(375); + setState(379); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,28,_ctx); } @@ -2737,46 +2752,46 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(376); + setState(380); match(DEV_METRICS); - setState(377); + setState(381); indexPattern(); - setState(382); + setState(386); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,29,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(378); + setState(382); match(COMMA); - setState(379); + setState(383); indexPattern(); } } } - setState(384); + setState(388); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,29,_ctx); } - setState(386); + setState(390); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: { - setState(385); + setState(389); ((MetricsCommandContext)_localctx).aggregates = aggFields(); } break; } - setState(390); + setState(394); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(388); + setState(392); match(BY); - setState(389); + setState(393); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2826,9 +2841,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(392); + setState(396); match(EVAL); - setState(393); + setState(397); fields(); } } @@ -2881,26 +2896,26 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(395); + setState(399); match(STATS); - setState(397); + setState(401); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { case 1: { - setState(396); + setState(400); ((StatsCommandContext)_localctx).stats = aggFields(); } break; } - setState(401); + setState(405); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: { - setState(399); + setState(403); match(BY); - setState(400); + setState(404); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2957,23 +2972,23 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(403); + setState(407); aggField(); - setState(408); + setState(412); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(404); + setState(408); match(COMMA); - setState(405); + setState(409); aggField(); } } } - setState(410); + setState(414); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } @@ -3025,16 +3040,16 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(411); + setState(415); field(); - setState(414); + setState(418); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) { case 1: { - setState(412); + setState(416); match(WHERE); - setState(413); + setState(417); booleanExpression(0); } break; @@ -3091,23 +3106,23 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(416); + setState(420); identifierOrParameter(); - setState(421); + setState(425); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,36,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(417); + setState(421); match(DOT); - setState(418); + setState(422); identifierOrParameter(); } } } - setState(423); + setState(427); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,36,_ctx); } @@ -3163,23 +3178,23 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(424); + setState(428); identifierPattern(); - setState(429); + setState(433); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,37,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(425); + setState(429); match(DOT); - setState(426); + setState(430); identifierPattern(); } } } - setState(431); + setState(435); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,37,_ctx); } @@ -3235,23 +3250,23 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(432); + setState(436); qualifiedNamePattern(); - setState(437); + setState(441); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,38,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(433); + setState(437); match(COMMA); - setState(434); + setState(438); qualifiedNamePattern(); } } } - setState(439); + setState(443); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,38,_ctx); } @@ -3299,7 +3314,7 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(440); + setState(444); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3352,13 +3367,13 @@ public class EsqlBaseParser extends ParserConfig { IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); enterRule(_localctx, 66, RULE_identifierPattern); try { - setState(444); + setState(448); _errHandler.sync(this); switch (_input.LA(1)) { case ID_PATTERN: enterOuterAlt(_localctx, 1); { - setState(442); + setState(446); match(ID_PATTERN); } break; @@ -3366,7 +3381,7 @@ public class EsqlBaseParser extends ParserConfig { case NAMED_OR_POSITIONAL_PARAM: enterOuterAlt(_localctx, 2); { - setState(443); + setState(447); parameter(); } break; @@ -3641,14 +3656,14 @@ public class EsqlBaseParser extends ParserConfig { enterRule(_localctx, 68, RULE_constant); int _la; try { - setState(488); + setState(492); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(446); + setState(450); match(NULL); } break; @@ -3656,9 +3671,9 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(447); + setState(451); integerValue(); - setState(448); + setState(452); match(UNQUOTED_IDENTIFIER); } break; @@ -3666,7 +3681,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(450); + setState(454); decimalValue(); } break; @@ -3674,7 +3689,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(451); + setState(455); integerValue(); } break; @@ -3682,7 +3697,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(452); + setState(456); booleanValue(); } break; @@ -3690,7 +3705,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(453); + setState(457); parameter(); } break; @@ -3698,7 +3713,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(454); + setState(458); string(); } break; @@ -3706,27 +3721,27 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(455); + setState(459); match(OPENING_BRACKET); - setState(456); + setState(460); numericValue(); - setState(461); + setState(465); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(457); + setState(461); match(COMMA); - setState(458); + setState(462); numericValue(); } } - setState(463); + setState(467); _errHandler.sync(this); _la = _input.LA(1); } - setState(464); + setState(468); match(CLOSING_BRACKET); } break; @@ -3734,27 +3749,27 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(466); + setState(470); match(OPENING_BRACKET); - setState(467); + setState(471); booleanValue(); - setState(472); + setState(476); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(468); + setState(472); match(COMMA); - setState(469); + setState(473); booleanValue(); } } - setState(474); + setState(478); _errHandler.sync(this); _la = _input.LA(1); } - setState(475); + setState(479); match(CLOSING_BRACKET); } break; @@ -3762,27 +3777,27 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(477); + setState(481); match(OPENING_BRACKET); - setState(478); + setState(482); string(); - setState(483); + setState(487); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(479); + setState(483); match(COMMA); - setState(480); + setState(484); string(); } } - setState(485); + setState(489); _errHandler.sync(this); _la = _input.LA(1); } - setState(486); + setState(490); match(CLOSING_BRACKET); } break; @@ -3856,14 +3871,14 @@ public class EsqlBaseParser extends ParserConfig { ParameterContext _localctx = new ParameterContext(_ctx, getState()); enterRule(_localctx, 70, RULE_parameter); try { - setState(492); + setState(496); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(490); + setState(494); match(PARAM); } break; @@ -3871,7 +3886,7 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(491); + setState(495); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3922,14 +3937,14 @@ public class EsqlBaseParser extends ParserConfig { IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); enterRule(_localctx, 72, RULE_identifierOrParameter); try { - setState(496); + setState(500); _errHandler.sync(this); switch (_input.LA(1)) { case UNQUOTED_IDENTIFIER: case QUOTED_IDENTIFIER: enterOuterAlt(_localctx, 1); { - setState(494); + setState(498); identifier(); } break; @@ -3937,7 +3952,7 @@ public class EsqlBaseParser extends ParserConfig { case NAMED_OR_POSITIONAL_PARAM: enterOuterAlt(_localctx, 2); { - setState(495); + setState(499); parameter(); } break; @@ -3986,9 +4001,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(498); + setState(502); match(LIMIT); - setState(499); + setState(503); match(INTEGER_LITERAL); } } @@ -4043,25 +4058,25 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(501); + setState(505); match(SORT); - setState(502); + setState(506); orderExpression(); - setState(507); + setState(511); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,46,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(503); + setState(507); match(COMMA); - setState(504); + setState(508); orderExpression(); } } } - setState(509); + setState(513); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,46,_ctx); } @@ -4117,14 +4132,14 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(510); + setState(514); booleanExpression(0); - setState(512); + setState(516); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { case 1: { - setState(511); + setState(515); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -4138,14 +4153,14 @@ public class EsqlBaseParser extends ParserConfig { } break; } - setState(516); + setState(520); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { case 1: { - setState(514); + setState(518); match(NULLS); - setState(515); + setState(519); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -4204,9 +4219,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(518); + setState(522); match(KEEP); - setState(519); + setState(523); qualifiedNamePatterns(); } } @@ -4253,9 +4268,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(525); match(DROP); - setState(522); + setState(526); qualifiedNamePatterns(); } } @@ -4310,25 +4325,25 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(524); + setState(528); match(RENAME); - setState(525); + setState(529); renameClause(); - setState(530); + setState(534); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,49,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(526); + setState(530); match(COMMA); - setState(527); + setState(531); renameClause(); } } } - setState(532); + setState(536); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,49,_ctx); } @@ -4382,11 +4397,11 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(533); + setState(537); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(534); + setState(538); match(AS); - setState(535); + setState(539); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4439,18 +4454,18 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(537); - match(DISSECT); - setState(538); - primaryExpression(0); - setState(539); - string(); setState(541); + match(DISSECT); + setState(542); + primaryExpression(0); + setState(543); + string(); + setState(545); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { case 1: { - setState(540); + setState(544); commandOptions(); } break; @@ -4503,11 +4518,11 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(543); + setState(547); match(GROK); - setState(544); + setState(548); primaryExpression(0); - setState(545); + setState(549); string(); } } @@ -4554,9 +4569,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(547); + setState(551); match(MV_EXPAND); - setState(548); + setState(552); qualifiedName(); } } @@ -4610,23 +4625,23 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(550); + setState(554); commandOption(); - setState(555); + setState(559); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,51,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(551); + setState(555); match(COMMA); - setState(552); + setState(556); commandOption(); } } } - setState(557); + setState(561); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,51,_ctx); } @@ -4678,11 +4693,11 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(558); + setState(562); identifier(); - setState(559); + setState(563); match(ASSIGN); - setState(560); + setState(564); constant(); } } @@ -4728,7 +4743,7 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(562); + setState(566); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4783,20 +4798,20 @@ public class EsqlBaseParser extends ParserConfig { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); enterRule(_localctx, 100, RULE_numericValue); try { - setState(566); + setState(570); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,52,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(564); + setState(568); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(565); + setState(569); integerValue(); } break; @@ -4845,12 +4860,12 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(569); + setState(573); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(568); + setState(572); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4863,7 +4878,7 @@ public class EsqlBaseParser extends ParserConfig { } } - setState(571); + setState(575); match(DECIMAL_LITERAL); } } @@ -4910,12 +4925,12 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(574); + setState(578); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(573); + setState(577); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4928,7 +4943,7 @@ public class EsqlBaseParser extends ParserConfig { } } - setState(576); + setState(580); match(INTEGER_LITERAL); } } @@ -4972,7 +4987,7 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(578); + setState(582); match(QUOTED_STRING); } } @@ -5022,9 +5037,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(580); + setState(584); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -432345564227567616L) != 0)) ) { + if ( !(((((_la - 58)) & ~0x3f) == 0 && ((1L << (_la - 58)) & 125L) != 0)) ) { _errHandler.recoverInline(this); } else { @@ -5077,9 +5092,9 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(582); + setState(586); match(EXPLAIN); - setState(583); + setState(587); subqueryExpression(); } } @@ -5127,11 +5142,11 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(585); + setState(589); match(OPENING_BRACKET); - setState(586); + setState(590); query(0); - setState(587); + setState(591); match(CLOSING_BRACKET); } } @@ -5188,9 +5203,9 @@ public class EsqlBaseParser extends ParserConfig { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(589); + setState(593); match(SHOW); - setState(590); + setState(594); match(INFO); } } @@ -5253,46 +5268,46 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(592); - match(ENRICH); - setState(593); - ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); setState(596); + match(ENRICH); + setState(597); + ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); + setState(600); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) { case 1: { - setState(594); + setState(598); match(ON); - setState(595); + setState(599); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(607); + setState(611); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { case 1: { - setState(598); + setState(602); match(WITH); - setState(599); + setState(603); enrichWithClause(); - setState(604); + setState(608); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,56,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(600); + setState(604); match(COMMA); - setState(601); + setState(605); enrichWithClause(); } } } - setState(606); + setState(610); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,56,_ctx); } @@ -5349,19 +5364,19 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(612); + setState(616); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) { case 1: { - setState(609); + setState(613); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(610); + setState(614); match(ASSIGN); } break; } - setState(614); + setState(618); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5414,13 +5429,13 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(616); + setState(620); match(DEV_LOOKUP); - setState(617); + setState(621); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(618); + setState(622); match(ON); - setState(619); + setState(623); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5473,18 +5488,18 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(621); - match(DEV_INLINESTATS); - setState(622); - ((InlinestatsCommandContext)_localctx).stats = aggFields(); setState(625); + match(DEV_INLINESTATS); + setState(626); + ((InlinestatsCommandContext)_localctx).stats = aggFields(); + setState(629); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,59,_ctx) ) { case 1: { - setState(623); + setState(627); match(BY); - setState(624); + setState(628); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5542,10 +5557,10 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(627); + setState(631); ((JoinCommandContext)_localctx).type = _input.LT(1); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 25296896L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 50462720L) != 0)) ) { ((JoinCommandContext)_localctx).type = (Token)_errHandler.recoverInline(this); } else { @@ -5553,11 +5568,11 @@ public class EsqlBaseParser extends ParserConfig { _errHandler.reportMatch(this); consume(); } - setState(628); + setState(632); match(JOIN); - setState(629); + setState(633); joinTarget(); - setState(630); + setState(634); joinCondition(); } } @@ -5604,7 +5619,7 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(632); + setState(636); ((JoinTargetContext)_localctx).index = indexPattern(); } } @@ -5659,25 +5674,25 @@ public class EsqlBaseParser extends ParserConfig { int _alt; enterOuterAlt(_localctx, 1); { - setState(634); + setState(638); match(ON); - setState(635); + setState(639); joinPredicate(); - setState(640); + setState(644); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,60,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(636); + setState(640); match(COMMA); - setState(637); + setState(641); joinPredicate(); } } } - setState(642); + setState(646); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,60,_ctx); } @@ -5725,7 +5740,7 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(643); + setState(647); valueExpression(); } } @@ -5782,34 +5797,34 @@ public class EsqlBaseParser extends ParserConfig { try { enterOuterAlt(_localctx, 1); { - setState(645); - match(DEV_CHANGE_POINT); - setState(646); - ((ChangePointCommandContext)_localctx).value = qualifiedName(); setState(649); + match(DEV_CHANGE_POINT); + setState(650); + ((ChangePointCommandContext)_localctx).value = qualifiedName(); + setState(653); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,61,_ctx) ) { case 1: { - setState(647); + setState(651); match(ON); - setState(648); + setState(652); ((ChangePointCommandContext)_localctx).key = qualifiedName(); } break; } - setState(656); + setState(660); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,62,_ctx) ) { case 1: { - setState(651); + setState(655); match(AS); - setState(652); + setState(656); ((ChangePointCommandContext)_localctx).targetType = qualifiedName(); - setState(653); + setState(657); match(COMMA); - setState(654); + setState(658); ((ChangePointCommandContext)_localctx).targetPvalue = qualifiedName(); } break; @@ -5827,6 +5842,55 @@ public class EsqlBaseParser extends ParserConfig { return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class InsistCommandContext extends ParserRuleContext { + public TerminalNode DEV_INSIST() { return getToken(EsqlBaseParser.DEV_INSIST, 0); } + public QualifiedNamePatternsContext qualifiedNamePatterns() { + return getRuleContext(QualifiedNamePatternsContext.class,0); + } + @SuppressWarnings("this-escape") + public InsistCommandContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_insistCommand; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterInsistCommand(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitInsistCommand(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitInsistCommand(this); + else return visitor.visitChildren(this); + } + } + + public final InsistCommandContext insistCommand() throws RecognitionException { + InsistCommandContext _localctx = new InsistCommandContext(_ctx, getState()); + enterRule(_localctx, 134, RULE_insistCommand); + try { + enterOuterAlt(_localctx, 1); + { + setState(662); + match(DEV_INSIST); + setState(663); + qualifiedNamePatterns(); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { case 1: @@ -5866,37 +5930,39 @@ public class EsqlBaseParser extends ParserConfig { return this.isDevVersion(); case 4: return this.isDevVersion(); + case 5: + return this.isDevVersion(); } return true; } private boolean booleanExpression_sempred(BooleanExpressionContext _localctx, int predIndex) { switch (predIndex) { - case 5: - return precpred(_ctx, 5); case 6: + return precpred(_ctx, 5); + case 7: return precpred(_ctx, 4); } return true; } private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, int predIndex) { switch (predIndex) { - case 7: - return precpred(_ctx, 2); case 8: + return precpred(_ctx, 2); + case 9: return precpred(_ctx, 1); } return true; } private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, int predIndex) { switch (predIndex) { - case 9: + case 10: return precpred(_ctx, 1); } return true; } public static final String _serializedATN = - "\u0004\u0001\u0086\u0293\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ + "\u0004\u0001\u008a\u029a\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+ "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+ "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+ @@ -5913,404 +5979,408 @@ public class EsqlBaseParser extends ParserConfig { "1\u00022\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u0007"+ "6\u00027\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007"+ ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007"+ - "@\u0002A\u0007A\u0002B\u0007B\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005"+ - "\u0001\u0090\b\u0001\n\u0001\f\u0001\u0093\t\u0001\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u009b\b\u0002"+ + "@\u0002A\u0007A\u0002B\u0007B\u0002C\u0007C\u0001\u0000\u0001\u0000\u0001"+ + "\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0005\u0001\u0092\b\u0001\n\u0001\f\u0001\u0095\t\u0001\u0001\u0002"+ + "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002"+ + "\u009d\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0003\u0003\u00b0\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0003\u0005\u00bc\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0005\u0005\u00c3\b\u0005\n\u0005\f\u0005\u00c6"+ - "\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+ - "\u0005\u00cd\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00d2"+ - "\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0005\u0005\u00da\b\u0005\n\u0005\f\u0005\u00dd\t\u0005\u0001\u0006"+ - "\u0001\u0006\u0003\u0006\u00e1\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ - "\u0001\u0006\u0001\u0006\u0003\u0006\u00e8\b\u0006\u0001\u0006\u0001\u0006"+ - "\u0001\u0006\u0003\u0006\u00ed\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007"+ - "\u0003\u0007\u00f2\b\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b"+ - "\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00fc\b\b\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0003\t\u0102\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0005\t\u010a\b\t\n\t\f\t\u010d\t\t\u0001\n\u0001\n\u0001\n\u0001\n"+ - "\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u0117\b\n\u0001\n\u0001\n\u0001"+ - "\n\u0005\n\u011c\b\n\n\n\f\n\u011f\t\n\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0127\b\u000b\n\u000b"+ - "\f\u000b\u012a\t\u000b\u0001\u000b\u0001\u000b\u0003\u000b\u012e\b\u000b"+ - "\u0003\u000b\u0130\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+ - "\r\u0001\r\u0001\r\u0001\r\u0005\r\u013a\b\r\n\r\f\r\u013d\t\r\u0001\r"+ - "\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001"+ - "\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001"+ - "\u0011\u0005\u0011\u014d\b\u0011\n\u0011\f\u0011\u0150\t\u0011\u0001\u0012"+ - "\u0001\u0012\u0001\u0012\u0003\u0012\u0155\b\u0012\u0001\u0012\u0001\u0012"+ - "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u015d\b\u0013"+ - "\n\u0013\f\u0013\u0160\t\u0013\u0001\u0013\u0003\u0013\u0163\b\u0013\u0001"+ - "\u0014\u0001\u0014\u0001\u0014\u0003\u0014\u0168\b\u0014\u0001\u0014\u0001"+ - "\u0014\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u0174\b\u0017\n\u0017\f\u0017"+ - "\u0177\t\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0005\u0018"+ - "\u017d\b\u0018\n\u0018\f\u0018\u0180\t\u0018\u0001\u0018\u0003\u0018\u0183"+ - "\b\u0018\u0001\u0018\u0001\u0018\u0003\u0018\u0187\b\u0018\u0001\u0019"+ - "\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0003\u001a\u018e\b\u001a"+ - "\u0001\u001a\u0001\u001a\u0003\u001a\u0192\b\u001a\u0001\u001b\u0001\u001b"+ - "\u0001\u001b\u0005\u001b\u0197\b\u001b\n\u001b\f\u001b\u019a\t\u001b\u0001"+ - "\u001c\u0001\u001c\u0001\u001c\u0003\u001c\u019f\b\u001c\u0001\u001d\u0001"+ - "\u001d\u0001\u001d\u0005\u001d\u01a4\b\u001d\n\u001d\f\u001d\u01a7\t\u001d"+ - "\u0001\u001e\u0001\u001e\u0001\u001e\u0005\u001e\u01ac\b\u001e\n\u001e"+ - "\f\u001e\u01af\t\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f"+ - "\u01b4\b\u001f\n\u001f\f\u001f\u01b7\t\u001f\u0001 \u0001 \u0001!\u0001"+ - "!\u0003!\u01bd\b!\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01cc\b\"\n"+ - "\"\f\"\u01cf\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005"+ - "\"\u01d7\b\"\n\"\f\"\u01da\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\""+ - "\u0001\"\u0005\"\u01e2\b\"\n\"\f\"\u01e5\t\"\u0001\"\u0001\"\u0003\"\u01e9"+ - "\b\"\u0001#\u0001#\u0003#\u01ed\b#\u0001$\u0001$\u0003$\u01f1\b$\u0001"+ - "%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0005&\u01fa\b&\n&\f&\u01fd"+ - "\t&\u0001\'\u0001\'\u0003\'\u0201\b\'\u0001\'\u0001\'\u0003\'\u0205\b"+ - "\'\u0001(\u0001(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001"+ - "*\u0005*\u0211\b*\n*\f*\u0214\t*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001"+ - ",\u0001,\u0001,\u0003,\u021e\b,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001"+ - ".\u0001.\u0001/\u0001/\u0001/\u0005/\u022a\b/\n/\f/\u022d\t/\u00010\u0001"+ - "0\u00010\u00010\u00011\u00011\u00012\u00012\u00032\u0237\b2\u00013\u0003"+ - "3\u023a\b3\u00013\u00013\u00014\u00034\u023f\b4\u00014\u00014\u00015\u0001"+ - "5\u00016\u00016\u00017\u00017\u00017\u00018\u00018\u00018\u00018\u0001"+ - "9\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u0255\b:\u0001:\u0001"+ - ":\u0001:\u0001:\u0005:\u025b\b:\n:\f:\u025e\t:\u0003:\u0260\b:\u0001;"+ - "\u0001;\u0001;\u0003;\u0265\b;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001"+ - "<\u0001<\u0001=\u0001=\u0001=\u0001=\u0003=\u0272\b=\u0001>\u0001>\u0001"+ - ">\u0001>\u0001>\u0001?\u0001?\u0001@\u0001@\u0001@\u0001@\u0005@\u027f"+ - "\b@\n@\f@\u0282\t@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001B\u0003B\u028a"+ - "\bB\u0001B\u0001B\u0001B\u0001B\u0001B\u0003B\u0291\bB\u0001B\u0000\u0004"+ - "\u0002\n\u0012\u0014C\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012"+ - "\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\"+ - "^`bdfhjlnprtvxz|~\u0080\u0082\u0084\u0000\t\u0001\u0000@A\u0001\u0000"+ - "BD\u0002\u0000\u001e\u001eSS\u0001\u0000JK\u0002\u0000##((\u0002\u0000"+ - "++..\u0002\u0000**88\u0002\u000099;?\u0002\u0000\u0011\u0011\u0017\u0018"+ - "\u02ae\u0000\u0086\u0001\u0000\u0000\u0000\u0002\u0089\u0001\u0000\u0000"+ - "\u0000\u0004\u009a\u0001\u0000\u0000\u0000\u0006\u00af\u0001\u0000\u0000"+ - "\u0000\b\u00b1\u0001\u0000\u0000\u0000\n\u00d1\u0001\u0000\u0000\u0000"+ - "\f\u00ec\u0001\u0000\u0000\u0000\u000e\u00ee\u0001\u0000\u0000\u0000\u0010"+ - "\u00fb\u0001\u0000\u0000\u0000\u0012\u0101\u0001\u0000\u0000\u0000\u0014"+ - "\u0116\u0001\u0000\u0000\u0000\u0016\u0120\u0001\u0000\u0000\u0000\u0018"+ - "\u0133\u0001\u0000\u0000\u0000\u001a\u0135\u0001\u0000\u0000\u0000\u001c"+ - "\u0140\u0001\u0000\u0000\u0000\u001e\u0144\u0001\u0000\u0000\u0000 \u0146"+ - "\u0001\u0000\u0000\u0000\"\u0149\u0001\u0000\u0000\u0000$\u0154\u0001"+ - "\u0000\u0000\u0000&\u0158\u0001\u0000\u0000\u0000(\u0167\u0001\u0000\u0000"+ - "\u0000*\u016b\u0001\u0000\u0000\u0000,\u016d\u0001\u0000\u0000\u0000."+ - "\u016f\u0001\u0000\u0000\u00000\u0178\u0001\u0000\u0000\u00002\u0188\u0001"+ - "\u0000\u0000\u00004\u018b\u0001\u0000\u0000\u00006\u0193\u0001\u0000\u0000"+ - "\u00008\u019b\u0001\u0000\u0000\u0000:\u01a0\u0001\u0000\u0000\u0000<"+ - "\u01a8\u0001\u0000\u0000\u0000>\u01b0\u0001\u0000\u0000\u0000@\u01b8\u0001"+ - "\u0000\u0000\u0000B\u01bc\u0001\u0000\u0000\u0000D\u01e8\u0001\u0000\u0000"+ - "\u0000F\u01ec\u0001\u0000\u0000\u0000H\u01f0\u0001\u0000\u0000\u0000J"+ - "\u01f2\u0001\u0000\u0000\u0000L\u01f5\u0001\u0000\u0000\u0000N\u01fe\u0001"+ - "\u0000\u0000\u0000P\u0206\u0001\u0000\u0000\u0000R\u0209\u0001\u0000\u0000"+ - "\u0000T\u020c\u0001\u0000\u0000\u0000V\u0215\u0001\u0000\u0000\u0000X"+ - "\u0219\u0001\u0000\u0000\u0000Z\u021f\u0001\u0000\u0000\u0000\\\u0223"+ - "\u0001\u0000\u0000\u0000^\u0226\u0001\u0000\u0000\u0000`\u022e\u0001\u0000"+ - "\u0000\u0000b\u0232\u0001\u0000\u0000\u0000d\u0236\u0001\u0000\u0000\u0000"+ - "f\u0239\u0001\u0000\u0000\u0000h\u023e\u0001\u0000\u0000\u0000j\u0242"+ - "\u0001\u0000\u0000\u0000l\u0244\u0001\u0000\u0000\u0000n\u0246\u0001\u0000"+ - "\u0000\u0000p\u0249\u0001\u0000\u0000\u0000r\u024d\u0001\u0000\u0000\u0000"+ - "t\u0250\u0001\u0000\u0000\u0000v\u0264\u0001\u0000\u0000\u0000x\u0268"+ - "\u0001\u0000\u0000\u0000z\u026d\u0001\u0000\u0000\u0000|\u0273\u0001\u0000"+ - "\u0000\u0000~\u0278\u0001\u0000\u0000\u0000\u0080\u027a\u0001\u0000\u0000"+ - "\u0000\u0082\u0283\u0001\u0000\u0000\u0000\u0084\u0285\u0001\u0000\u0000"+ - "\u0000\u0086\u0087\u0003\u0002\u0001\u0000\u0087\u0088\u0005\u0000\u0000"+ - "\u0001\u0088\u0001\u0001\u0000\u0000\u0000\u0089\u008a\u0006\u0001\uffff"+ - "\uffff\u0000\u008a\u008b\u0003\u0004\u0002\u0000\u008b\u0091\u0001\u0000"+ - "\u0000\u0000\u008c\u008d\n\u0001\u0000\u0000\u008d\u008e\u0005\u001d\u0000"+ - "\u0000\u008e\u0090\u0003\u0006\u0003\u0000\u008f\u008c\u0001\u0000\u0000"+ - "\u0000\u0090\u0093\u0001\u0000\u0000\u0000\u0091\u008f\u0001\u0000\u0000"+ - "\u0000\u0091\u0092\u0001\u0000\u0000\u0000\u0092\u0003\u0001\u0000\u0000"+ - "\u0000\u0093\u0091\u0001\u0000\u0000\u0000\u0094\u009b\u0003n7\u0000\u0095"+ - "\u009b\u0003&\u0013\u0000\u0096\u009b\u0003 \u0010\u0000\u0097\u009b\u0003"+ - "r9\u0000\u0098\u0099\u0004\u0002\u0001\u0000\u0099\u009b\u00030\u0018"+ - "\u0000\u009a\u0094\u0001\u0000\u0000\u0000\u009a\u0095\u0001\u0000\u0000"+ - "\u0000\u009a\u0096\u0001\u0000\u0000\u0000\u009a\u0097\u0001\u0000\u0000"+ - "\u0000\u009a\u0098\u0001\u0000\u0000\u0000\u009b\u0005\u0001\u0000\u0000"+ - "\u0000\u009c\u00b0\u00032\u0019\u0000\u009d\u00b0\u0003\b\u0004\u0000"+ - "\u009e\u00b0\u0003P(\u0000\u009f\u00b0\u0003J%\u0000\u00a0\u00b0\u0003"+ - "4\u001a\u0000\u00a1\u00b0\u0003L&\u0000\u00a2\u00b0\u0003R)\u0000\u00a3"+ - "\u00b0\u0003T*\u0000\u00a4\u00b0\u0003X,\u0000\u00a5\u00b0\u0003Z-\u0000"+ - "\u00a6\u00b0\u0003t:\u0000\u00a7\u00b0\u0003\\.\u0000\u00a8\u00b0\u0003"+ - "|>\u0000\u00a9\u00aa\u0004\u0003\u0002\u0000\u00aa\u00b0\u0003z=\u0000"+ - "\u00ab\u00ac\u0004\u0003\u0003\u0000\u00ac\u00b0\u0003x<\u0000\u00ad\u00ae"+ - "\u0004\u0003\u0004\u0000\u00ae\u00b0\u0003\u0084B\u0000\u00af\u009c\u0001"+ - "\u0000\u0000\u0000\u00af\u009d\u0001\u0000\u0000\u0000\u00af\u009e\u0001"+ - "\u0000\u0000\u0000\u00af\u009f\u0001\u0000\u0000\u0000\u00af\u00a0\u0001"+ - "\u0000\u0000\u0000\u00af\u00a1\u0001\u0000\u0000\u0000\u00af\u00a2\u0001"+ - "\u0000\u0000\u0000\u00af\u00a3\u0001\u0000\u0000\u0000\u00af\u00a4\u0001"+ - "\u0000\u0000\u0000\u00af\u00a5\u0001\u0000\u0000\u0000\u00af\u00a6\u0001"+ - "\u0000\u0000\u0000\u00af\u00a7\u0001\u0000\u0000\u0000\u00af\u00a8\u0001"+ - "\u0000\u0000\u0000\u00af\u00a9\u0001\u0000\u0000\u0000\u00af\u00ab\u0001"+ - "\u0000\u0000\u0000\u00af\u00ad\u0001\u0000\u0000\u0000\u00b0\u0007\u0001"+ - "\u0000\u0000\u0000\u00b1\u00b2\u0005\u0010\u0000\u0000\u00b2\u00b3\u0003"+ - "\n\u0005\u0000\u00b3\t\u0001\u0000\u0000\u0000\u00b4\u00b5\u0006\u0005"+ - "\uffff\uffff\u0000\u00b5\u00b6\u00051\u0000\u0000\u00b6\u00d2\u0003\n"+ - "\u0005\b\u00b7\u00d2\u0003\u0010\b\u0000\u00b8\u00d2\u0003\f\u0006\u0000"+ - "\u00b9\u00bb\u0003\u0010\b\u0000\u00ba\u00bc\u00051\u0000\u0000\u00bb"+ - "\u00ba\u0001\u0000\u0000\u0000\u00bb\u00bc\u0001\u0000\u0000\u0000\u00bc"+ - "\u00bd\u0001\u0000\u0000\u0000\u00bd\u00be\u0005,\u0000\u0000\u00be\u00bf"+ - "\u00050\u0000\u0000\u00bf\u00c4\u0003\u0010\b\u0000\u00c0\u00c1\u0005"+ - "\'\u0000\u0000\u00c1\u00c3\u0003\u0010\b\u0000\u00c2\u00c0\u0001\u0000"+ - "\u0000\u0000\u00c3\u00c6\u0001\u0000\u0000\u0000\u00c4\u00c2\u0001\u0000"+ - "\u0000\u0000\u00c4\u00c5\u0001\u0000\u0000\u0000\u00c5\u00c7\u0001\u0000"+ - "\u0000\u0000\u00c6\u00c4\u0001\u0000\u0000\u0000\u00c7\u00c8\u00057\u0000"+ - "\u0000\u00c8\u00d2\u0001\u0000\u0000\u0000\u00c9\u00ca\u0003\u0010\b\u0000"+ - "\u00ca\u00cc\u0005-\u0000\u0000\u00cb\u00cd\u00051\u0000\u0000\u00cc\u00cb"+ - "\u0001\u0000\u0000\u0000\u00cc\u00cd\u0001\u0000\u0000\u0000\u00cd\u00ce"+ - "\u0001\u0000\u0000\u0000\u00ce\u00cf\u00052\u0000\u0000\u00cf\u00d2\u0001"+ - "\u0000\u0000\u0000\u00d0\u00d2\u0003\u000e\u0007\u0000\u00d1\u00b4\u0001"+ - "\u0000\u0000\u0000\u00d1\u00b7\u0001\u0000\u0000\u0000\u00d1\u00b8\u0001"+ - "\u0000\u0000\u0000\u00d1\u00b9\u0001\u0000\u0000\u0000\u00d1\u00c9\u0001"+ - "\u0000\u0000\u0000\u00d1\u00d0\u0001\u0000\u0000\u0000\u00d2\u00db\u0001"+ - "\u0000\u0000\u0000\u00d3\u00d4\n\u0005\u0000\u0000\u00d4\u00d5\u0005\""+ - "\u0000\u0000\u00d5\u00da\u0003\n\u0005\u0006\u00d6\u00d7\n\u0004\u0000"+ - "\u0000\u00d7\u00d8\u00054\u0000\u0000\u00d8\u00da\u0003\n\u0005\u0005"+ - "\u00d9\u00d3\u0001\u0000\u0000\u0000\u00d9\u00d6\u0001\u0000\u0000\u0000"+ - "\u00da\u00dd\u0001\u0000\u0000\u0000\u00db\u00d9\u0001\u0000\u0000\u0000"+ - "\u00db\u00dc\u0001\u0000\u0000\u0000\u00dc\u000b\u0001\u0000\u0000\u0000"+ - "\u00dd\u00db\u0001\u0000\u0000\u0000\u00de\u00e0\u0003\u0010\b\u0000\u00df"+ - "\u00e1\u00051\u0000\u0000\u00e0\u00df\u0001\u0000\u0000\u0000\u00e0\u00e1"+ - "\u0001\u0000\u0000\u0000\u00e1\u00e2\u0001\u0000\u0000\u0000\u00e2\u00e3"+ - "\u0005/\u0000\u0000\u00e3\u00e4\u0003j5\u0000\u00e4\u00ed\u0001\u0000"+ - "\u0000\u0000\u00e5\u00e7\u0003\u0010\b\u0000\u00e6\u00e8\u00051\u0000"+ - "\u0000\u00e7\u00e6\u0001\u0000\u0000\u0000\u00e7\u00e8\u0001\u0000\u0000"+ - "\u0000\u00e8\u00e9\u0001\u0000\u0000\u0000\u00e9\u00ea\u00056\u0000\u0000"+ - "\u00ea\u00eb\u0003j5\u0000\u00eb\u00ed\u0001\u0000\u0000\u0000\u00ec\u00de"+ - "\u0001\u0000\u0000\u0000\u00ec\u00e5\u0001\u0000\u0000\u0000\u00ed\r\u0001"+ - "\u0000\u0000\u0000\u00ee\u00f1\u0003:\u001d\u0000\u00ef\u00f0\u0005%\u0000"+ - "\u0000\u00f0\u00f2\u0003\u001e\u000f\u0000\u00f1\u00ef\u0001\u0000\u0000"+ - "\u0000\u00f1\u00f2\u0001\u0000\u0000\u0000\u00f2\u00f3\u0001\u0000\u0000"+ - "\u0000\u00f3\u00f4\u0005&\u0000\u0000\u00f4\u00f5\u0003D\"\u0000\u00f5"+ - "\u000f\u0001\u0000\u0000\u0000\u00f6\u00fc\u0003\u0012\t\u0000\u00f7\u00f8"+ - "\u0003\u0012\t\u0000\u00f8\u00f9\u0003l6\u0000\u00f9\u00fa\u0003\u0012"+ - "\t\u0000\u00fa\u00fc\u0001\u0000\u0000\u0000\u00fb\u00f6\u0001\u0000\u0000"+ - "\u0000\u00fb\u00f7\u0001\u0000\u0000\u0000\u00fc\u0011\u0001\u0000\u0000"+ - "\u0000\u00fd\u00fe\u0006\t\uffff\uffff\u0000\u00fe\u0102\u0003\u0014\n"+ - "\u0000\u00ff\u0100\u0007\u0000\u0000\u0000\u0100\u0102\u0003\u0012\t\u0003"+ - "\u0101\u00fd\u0001\u0000\u0000\u0000\u0101\u00ff\u0001\u0000\u0000\u0000"+ - "\u0102\u010b\u0001\u0000\u0000\u0000\u0103\u0104\n\u0002\u0000\u0000\u0104"+ - "\u0105\u0007\u0001\u0000\u0000\u0105\u010a\u0003\u0012\t\u0003\u0106\u0107"+ - "\n\u0001\u0000\u0000\u0107\u0108\u0007\u0000\u0000\u0000\u0108\u010a\u0003"+ - "\u0012\t\u0002\u0109\u0103\u0001\u0000\u0000\u0000\u0109\u0106\u0001\u0000"+ - "\u0000\u0000\u010a\u010d\u0001\u0000\u0000\u0000\u010b\u0109\u0001\u0000"+ - "\u0000\u0000\u010b\u010c\u0001\u0000\u0000\u0000\u010c\u0013\u0001\u0000"+ - "\u0000\u0000\u010d\u010b\u0001\u0000\u0000\u0000\u010e\u010f\u0006\n\uffff"+ - "\uffff\u0000\u010f\u0117\u0003D\"\u0000\u0110\u0117\u0003:\u001d\u0000"+ - "\u0111\u0117\u0003\u0016\u000b\u0000\u0112\u0113\u00050\u0000\u0000\u0113"+ - "\u0114\u0003\n\u0005\u0000\u0114\u0115\u00057\u0000\u0000\u0115\u0117"+ - "\u0001\u0000\u0000\u0000\u0116\u010e\u0001\u0000\u0000\u0000\u0116\u0110"+ - "\u0001\u0000\u0000\u0000\u0116\u0111\u0001\u0000\u0000\u0000\u0116\u0112"+ - "\u0001\u0000\u0000\u0000\u0117\u011d\u0001\u0000\u0000\u0000\u0118\u0119"+ - "\n\u0001\u0000\u0000\u0119\u011a\u0005%\u0000\u0000\u011a\u011c\u0003"+ - "\u001e\u000f\u0000\u011b\u0118\u0001\u0000\u0000\u0000\u011c\u011f\u0001"+ - "\u0000\u0000\u0000\u011d\u011b\u0001\u0000\u0000\u0000\u011d\u011e\u0001"+ - "\u0000\u0000\u0000\u011e\u0015\u0001\u0000\u0000\u0000\u011f\u011d\u0001"+ - "\u0000\u0000\u0000\u0120\u0121\u0003\u0018\f\u0000\u0121\u012f\u00050"+ - "\u0000\u0000\u0122\u0130\u0005B\u0000\u0000\u0123\u0128\u0003\n\u0005"+ - "\u0000\u0124\u0125\u0005\'\u0000\u0000\u0125\u0127\u0003\n\u0005\u0000"+ - "\u0126\u0124\u0001\u0000\u0000\u0000\u0127\u012a\u0001\u0000\u0000\u0000"+ - "\u0128\u0126\u0001\u0000\u0000\u0000\u0128\u0129\u0001\u0000\u0000\u0000"+ - "\u0129\u012d\u0001\u0000\u0000\u0000\u012a\u0128\u0001\u0000\u0000\u0000"+ - "\u012b\u012c\u0005\'\u0000\u0000\u012c\u012e\u0003\u001a\r\u0000\u012d"+ - "\u012b\u0001\u0000\u0000\u0000\u012d\u012e\u0001\u0000\u0000\u0000\u012e"+ - "\u0130\u0001\u0000\u0000\u0000\u012f\u0122\u0001\u0000\u0000\u0000\u012f"+ - "\u0123\u0001\u0000\u0000\u0000\u012f\u0130\u0001\u0000\u0000\u0000\u0130"+ - "\u0131\u0001\u0000\u0000\u0000\u0131\u0132\u00057\u0000\u0000\u0132\u0017"+ - "\u0001\u0000\u0000\u0000\u0133\u0134\u0003H$\u0000\u0134\u0019\u0001\u0000"+ - "\u0000\u0000\u0135\u0136\u0005E\u0000\u0000\u0136\u013b\u0003\u001c\u000e"+ - "\u0000\u0137\u0138\u0005\'\u0000\u0000\u0138\u013a\u0003\u001c\u000e\u0000"+ - "\u0139\u0137\u0001\u0000\u0000\u0000\u013a\u013d\u0001\u0000\u0000\u0000"+ - "\u013b\u0139\u0001\u0000\u0000\u0000\u013b\u013c\u0001\u0000\u0000\u0000"+ - "\u013c\u013e\u0001\u0000\u0000\u0000\u013d\u013b\u0001\u0000\u0000\u0000"+ - "\u013e\u013f\u0005F\u0000\u0000\u013f\u001b\u0001\u0000\u0000\u0000\u0140"+ - "\u0141\u0003j5\u0000\u0141\u0142\u0005&\u0000\u0000\u0142\u0143\u0003"+ - "D\"\u0000\u0143\u001d\u0001\u0000\u0000\u0000\u0144\u0145\u0003@ \u0000"+ - "\u0145\u001f\u0001\u0000\u0000\u0000\u0146\u0147\u0005\f\u0000\u0000\u0147"+ - "\u0148\u0003\"\u0011\u0000\u0148!\u0001\u0000\u0000\u0000\u0149\u014e"+ - "\u0003$\u0012\u0000\u014a\u014b\u0005\'\u0000\u0000\u014b\u014d\u0003"+ - "$\u0012\u0000\u014c\u014a\u0001\u0000\u0000\u0000\u014d\u0150\u0001\u0000"+ - "\u0000\u0000\u014e\u014c\u0001\u0000\u0000\u0000\u014e\u014f\u0001\u0000"+ - "\u0000\u0000\u014f#\u0001\u0000\u0000\u0000\u0150\u014e\u0001\u0000\u0000"+ - "\u0000\u0151\u0152\u0003:\u001d\u0000\u0152\u0153\u0005$\u0000\u0000\u0153"+ - "\u0155\u0001\u0000\u0000\u0000\u0154\u0151\u0001\u0000\u0000\u0000\u0154"+ - "\u0155\u0001\u0000\u0000\u0000\u0155\u0156\u0001\u0000\u0000\u0000\u0156"+ - "\u0157\u0003\n\u0005\u0000\u0157%\u0001\u0000\u0000\u0000\u0158\u0159"+ - "\u0005\u0006\u0000\u0000\u0159\u015e\u0003(\u0014\u0000\u015a\u015b\u0005"+ - "\'\u0000\u0000\u015b\u015d\u0003(\u0014\u0000\u015c\u015a\u0001\u0000"+ - "\u0000\u0000\u015d\u0160\u0001\u0000\u0000\u0000\u015e\u015c\u0001\u0000"+ - "\u0000\u0000\u015e\u015f\u0001\u0000\u0000\u0000\u015f\u0162\u0001\u0000"+ - "\u0000\u0000\u0160\u015e\u0001\u0000\u0000\u0000\u0161\u0163\u0003.\u0017"+ - "\u0000\u0162\u0161\u0001\u0000\u0000\u0000\u0162\u0163\u0001\u0000\u0000"+ - "\u0000\u0163\'\u0001\u0000\u0000\u0000\u0164\u0165\u0003*\u0015\u0000"+ - "\u0165\u0166\u0005&\u0000\u0000\u0166\u0168\u0001\u0000\u0000\u0000\u0167"+ - "\u0164\u0001\u0000\u0000\u0000\u0167\u0168\u0001\u0000\u0000\u0000\u0168"+ - "\u0169\u0001\u0000\u0000\u0000\u0169\u016a\u0003,\u0016\u0000\u016a)\u0001"+ - "\u0000\u0000\u0000\u016b\u016c\u0007\u0002\u0000\u0000\u016c+\u0001\u0000"+ - "\u0000\u0000\u016d\u016e\u0007\u0002\u0000\u0000\u016e-\u0001\u0000\u0000"+ - "\u0000\u016f\u0170\u0005R\u0000\u0000\u0170\u0175\u0005S\u0000\u0000\u0171"+ - "\u0172\u0005\'\u0000\u0000\u0172\u0174\u0005S\u0000\u0000\u0173\u0171"+ - "\u0001\u0000\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000\u0175\u0173"+ - "\u0001\u0000\u0000\u0000\u0175\u0176\u0001\u0000\u0000\u0000\u0176/\u0001"+ - "\u0000\u0000\u0000\u0177\u0175\u0001\u0000\u0000\u0000\u0178\u0179\u0005"+ - "\u0015\u0000\u0000\u0179\u017e\u0003(\u0014\u0000\u017a\u017b\u0005\'"+ - "\u0000\u0000\u017b\u017d\u0003(\u0014\u0000\u017c\u017a\u0001\u0000\u0000"+ - "\u0000\u017d\u0180\u0001\u0000\u0000\u0000\u017e\u017c\u0001\u0000\u0000"+ - "\u0000\u017e\u017f\u0001\u0000\u0000\u0000\u017f\u0182\u0001\u0000\u0000"+ - "\u0000\u0180\u017e\u0001\u0000\u0000\u0000\u0181\u0183\u00036\u001b\u0000"+ - "\u0182\u0181\u0001\u0000\u0000\u0000\u0182\u0183\u0001\u0000\u0000\u0000"+ - "\u0183\u0186\u0001\u0000\u0000\u0000\u0184\u0185\u0005!\u0000\u0000\u0185"+ - "\u0187\u0003\"\u0011\u0000\u0186\u0184\u0001\u0000\u0000\u0000\u0186\u0187"+ - "\u0001\u0000\u0000\u0000\u01871\u0001\u0000\u0000\u0000\u0188\u0189\u0005"+ - "\u0004\u0000\u0000\u0189\u018a\u0003\"\u0011\u0000\u018a3\u0001\u0000"+ - "\u0000\u0000\u018b\u018d\u0005\u000f\u0000\u0000\u018c\u018e\u00036\u001b"+ - "\u0000\u018d\u018c\u0001\u0000\u0000\u0000\u018d\u018e\u0001\u0000\u0000"+ - "\u0000\u018e\u0191\u0001\u0000\u0000\u0000\u018f\u0190\u0005!\u0000\u0000"+ - "\u0190\u0192\u0003\"\u0011\u0000\u0191\u018f\u0001\u0000\u0000\u0000\u0191"+ - "\u0192\u0001\u0000\u0000\u0000\u01925\u0001\u0000\u0000\u0000\u0193\u0198"+ - "\u00038\u001c\u0000\u0194\u0195\u0005\'\u0000\u0000\u0195\u0197\u0003"+ - "8\u001c\u0000\u0196\u0194\u0001\u0000\u0000\u0000\u0197\u019a\u0001\u0000"+ - "\u0000\u0000\u0198\u0196\u0001\u0000\u0000\u0000\u0198\u0199\u0001\u0000"+ - "\u0000\u0000\u01997\u0001\u0000\u0000\u0000\u019a\u0198\u0001\u0000\u0000"+ - "\u0000\u019b\u019e\u0003$\u0012\u0000\u019c\u019d\u0005\u0010\u0000\u0000"+ - "\u019d\u019f\u0003\n\u0005\u0000\u019e\u019c\u0001\u0000\u0000\u0000\u019e"+ - "\u019f\u0001\u0000\u0000\u0000\u019f9\u0001\u0000\u0000\u0000\u01a0\u01a5"+ - "\u0003H$\u0000\u01a1\u01a2\u0005)\u0000\u0000\u01a2\u01a4\u0003H$\u0000"+ - "\u01a3\u01a1\u0001\u0000\u0000\u0000\u01a4\u01a7\u0001\u0000\u0000\u0000"+ - "\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a5\u01a6\u0001\u0000\u0000\u0000"+ - "\u01a6;\u0001\u0000\u0000\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a8"+ - "\u01ad\u0003B!\u0000\u01a9\u01aa\u0005)\u0000\u0000\u01aa\u01ac\u0003"+ - "B!\u0000\u01ab\u01a9\u0001\u0000\u0000\u0000\u01ac\u01af\u0001\u0000\u0000"+ - "\u0000\u01ad\u01ab\u0001\u0000\u0000\u0000\u01ad\u01ae\u0001\u0000\u0000"+ - "\u0000\u01ae=\u0001\u0000\u0000\u0000\u01af\u01ad\u0001\u0000\u0000\u0000"+ - "\u01b0\u01b5\u0003<\u001e\u0000\u01b1\u01b2\u0005\'\u0000\u0000\u01b2"+ - "\u01b4\u0003<\u001e\u0000\u01b3\u01b1\u0001\u0000\u0000\u0000\u01b4\u01b7"+ - "\u0001\u0000\u0000\u0000\u01b5\u01b3\u0001\u0000\u0000\u0000\u01b5\u01b6"+ - "\u0001\u0000\u0000\u0000\u01b6?\u0001\u0000\u0000\u0000\u01b7\u01b5\u0001"+ - "\u0000\u0000\u0000\u01b8\u01b9\u0007\u0003\u0000\u0000\u01b9A\u0001\u0000"+ - "\u0000\u0000\u01ba\u01bd\u0005W\u0000\u0000\u01bb\u01bd\u0003F#\u0000"+ - "\u01bc\u01ba\u0001\u0000\u0000\u0000\u01bc\u01bb\u0001\u0000\u0000\u0000"+ - "\u01bdC\u0001\u0000\u0000\u0000\u01be\u01e9\u00052\u0000\u0000\u01bf\u01c0"+ - "\u0003h4\u0000\u01c0\u01c1\u0005J\u0000\u0000\u01c1\u01e9\u0001\u0000"+ - "\u0000\u0000\u01c2\u01e9\u0003f3\u0000\u01c3\u01e9\u0003h4\u0000\u01c4"+ - "\u01e9\u0003b1\u0000\u01c5\u01e9\u0003F#\u0000\u01c6\u01e9\u0003j5\u0000"+ - "\u01c7\u01c8\u0005H\u0000\u0000\u01c8\u01cd\u0003d2\u0000\u01c9\u01ca"+ - "\u0005\'\u0000\u0000\u01ca\u01cc\u0003d2\u0000\u01cb\u01c9\u0001\u0000"+ - "\u0000\u0000\u01cc\u01cf\u0001\u0000\u0000\u0000\u01cd\u01cb\u0001\u0000"+ - "\u0000\u0000\u01cd\u01ce\u0001\u0000\u0000\u0000\u01ce\u01d0\u0001\u0000"+ - "\u0000\u0000\u01cf\u01cd\u0001\u0000\u0000\u0000\u01d0\u01d1\u0005I\u0000"+ - "\u0000\u01d1\u01e9\u0001\u0000\u0000\u0000\u01d2\u01d3\u0005H\u0000\u0000"+ - "\u01d3\u01d8\u0003b1\u0000\u01d4\u01d5\u0005\'\u0000\u0000\u01d5\u01d7"+ - "\u0003b1\u0000\u01d6\u01d4\u0001\u0000\u0000\u0000\u01d7\u01da\u0001\u0000"+ - "\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000\u0000\u01d8\u01d9\u0001\u0000"+ - "\u0000\u0000\u01d9\u01db\u0001\u0000\u0000\u0000\u01da\u01d8\u0001\u0000"+ - "\u0000\u0000\u01db\u01dc\u0005I\u0000\u0000\u01dc\u01e9\u0001\u0000\u0000"+ - "\u0000\u01dd\u01de\u0005H\u0000\u0000\u01de\u01e3\u0003j5\u0000\u01df"+ - "\u01e0\u0005\'\u0000\u0000\u01e0\u01e2\u0003j5\u0000\u01e1\u01df\u0001"+ - "\u0000\u0000\u0000\u01e2\u01e5\u0001\u0000\u0000\u0000\u01e3\u01e1\u0001"+ - "\u0000\u0000\u0000\u01e3\u01e4\u0001\u0000\u0000\u0000\u01e4\u01e6\u0001"+ - "\u0000\u0000\u0000\u01e5\u01e3\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005"+ - "I\u0000\u0000\u01e7\u01e9\u0001\u0000\u0000\u0000\u01e8\u01be\u0001\u0000"+ - "\u0000\u0000\u01e8\u01bf\u0001\u0000\u0000\u0000\u01e8\u01c2\u0001\u0000"+ - "\u0000\u0000\u01e8\u01c3\u0001\u0000\u0000\u0000\u01e8\u01c4\u0001\u0000"+ - "\u0000\u0000\u01e8\u01c5\u0001\u0000\u0000\u0000\u01e8\u01c6\u0001\u0000"+ - "\u0000\u0000\u01e8\u01c7\u0001\u0000\u0000\u0000\u01e8\u01d2\u0001\u0000"+ - "\u0000\u0000\u01e8\u01dd\u0001\u0000\u0000\u0000\u01e9E\u0001\u0000\u0000"+ - "\u0000\u01ea\u01ed\u00055\u0000\u0000\u01eb\u01ed\u0005G\u0000\u0000\u01ec"+ - "\u01ea\u0001\u0000\u0000\u0000\u01ec\u01eb\u0001\u0000\u0000\u0000\u01ed"+ - "G\u0001\u0000\u0000\u0000\u01ee\u01f1\u0003@ \u0000\u01ef\u01f1\u0003"+ - "F#\u0000\u01f0\u01ee\u0001\u0000\u0000\u0000\u01f0\u01ef\u0001\u0000\u0000"+ - "\u0000\u01f1I\u0001\u0000\u0000\u0000\u01f2\u01f3\u0005\t\u0000\u0000"+ - "\u01f3\u01f4\u0005\u001f\u0000\u0000\u01f4K\u0001\u0000\u0000\u0000\u01f5"+ - "\u01f6\u0005\u000e\u0000\u0000\u01f6\u01fb\u0003N\'\u0000\u01f7\u01f8"+ - "\u0005\'\u0000\u0000\u01f8\u01fa\u0003N\'\u0000\u01f9\u01f7\u0001\u0000"+ - "\u0000\u0000\u01fa\u01fd\u0001\u0000\u0000\u0000\u01fb\u01f9\u0001\u0000"+ - "\u0000\u0000\u01fb\u01fc\u0001\u0000\u0000\u0000\u01fcM\u0001\u0000\u0000"+ - "\u0000\u01fd\u01fb\u0001\u0000\u0000\u0000\u01fe\u0200\u0003\n\u0005\u0000"+ - "\u01ff\u0201\u0007\u0004\u0000\u0000\u0200\u01ff\u0001\u0000\u0000\u0000"+ - "\u0200\u0201\u0001\u0000\u0000\u0000\u0201\u0204\u0001\u0000\u0000\u0000"+ - "\u0202\u0203\u00053\u0000\u0000\u0203\u0205\u0007\u0005\u0000\u0000\u0204"+ - "\u0202\u0001\u0000\u0000\u0000\u0204\u0205\u0001\u0000\u0000\u0000\u0205"+ - "O\u0001\u0000\u0000\u0000\u0206\u0207\u0005\b\u0000\u0000\u0207\u0208"+ - "\u0003>\u001f\u0000\u0208Q\u0001\u0000\u0000\u0000\u0209\u020a\u0005\u0002"+ - "\u0000\u0000\u020a\u020b\u0003>\u001f\u0000\u020bS\u0001\u0000\u0000\u0000"+ - "\u020c\u020d\u0005\u000b\u0000\u0000\u020d\u0212\u0003V+\u0000\u020e\u020f"+ - "\u0005\'\u0000\u0000\u020f\u0211\u0003V+\u0000\u0210\u020e\u0001\u0000"+ - "\u0000\u0000\u0211\u0214\u0001\u0000\u0000\u0000\u0212\u0210\u0001\u0000"+ - "\u0000\u0000\u0212\u0213\u0001\u0000\u0000\u0000\u0213U\u0001\u0000\u0000"+ - "\u0000\u0214\u0212\u0001\u0000\u0000\u0000\u0215\u0216\u0003<\u001e\u0000"+ - "\u0216\u0217\u0005[\u0000\u0000\u0217\u0218\u0003<\u001e\u0000\u0218W"+ - "\u0001\u0000\u0000\u0000\u0219\u021a\u0005\u0001\u0000\u0000\u021a\u021b"+ - "\u0003\u0014\n\u0000\u021b\u021d\u0003j5\u0000\u021c\u021e\u0003^/\u0000"+ - "\u021d\u021c\u0001\u0000\u0000\u0000\u021d\u021e\u0001\u0000\u0000\u0000"+ - "\u021eY\u0001\u0000\u0000\u0000\u021f\u0220\u0005\u0007\u0000\u0000\u0220"+ - "\u0221\u0003\u0014\n\u0000\u0221\u0222\u0003j5\u0000\u0222[\u0001\u0000"+ - "\u0000\u0000\u0223\u0224\u0005\n\u0000\u0000\u0224\u0225\u0003:\u001d"+ - "\u0000\u0225]\u0001\u0000\u0000\u0000\u0226\u022b\u0003`0\u0000\u0227"+ - "\u0228\u0005\'\u0000\u0000\u0228\u022a\u0003`0\u0000\u0229\u0227\u0001"+ - "\u0000\u0000\u0000\u022a\u022d\u0001\u0000\u0000\u0000\u022b\u0229\u0001"+ - "\u0000\u0000\u0000\u022b\u022c\u0001\u0000\u0000\u0000\u022c_\u0001\u0000"+ - "\u0000\u0000\u022d\u022b\u0001\u0000\u0000\u0000\u022e\u022f\u0003@ \u0000"+ - "\u022f\u0230\u0005$\u0000\u0000\u0230\u0231\u0003D\"\u0000\u0231a\u0001"+ - "\u0000\u0000\u0000\u0232\u0233\u0007\u0006\u0000\u0000\u0233c\u0001\u0000"+ - "\u0000\u0000\u0234\u0237\u0003f3\u0000\u0235\u0237\u0003h4\u0000\u0236"+ - "\u0234\u0001\u0000\u0000\u0000\u0236\u0235\u0001\u0000\u0000\u0000\u0237"+ - "e\u0001\u0000\u0000\u0000\u0238\u023a\u0007\u0000\u0000\u0000\u0239\u0238"+ - "\u0001\u0000\u0000\u0000\u0239\u023a\u0001\u0000\u0000\u0000\u023a\u023b"+ - "\u0001\u0000\u0000\u0000\u023b\u023c\u0005 \u0000\u0000\u023cg\u0001\u0000"+ - "\u0000\u0000\u023d\u023f\u0007\u0000\u0000\u0000\u023e\u023d\u0001\u0000"+ - "\u0000\u0000\u023e\u023f\u0001\u0000\u0000\u0000\u023f\u0240\u0001\u0000"+ - "\u0000\u0000\u0240\u0241\u0005\u001f\u0000\u0000\u0241i\u0001\u0000\u0000"+ - "\u0000\u0242\u0243\u0005\u001e\u0000\u0000\u0243k\u0001\u0000\u0000\u0000"+ - "\u0244\u0245\u0007\u0007\u0000\u0000\u0245m\u0001\u0000\u0000\u0000\u0246"+ - "\u0247\u0005\u0005\u0000\u0000\u0247\u0248\u0003p8\u0000\u0248o\u0001"+ - "\u0000\u0000\u0000\u0249\u024a\u0005H\u0000\u0000\u024a\u024b\u0003\u0002"+ - "\u0001\u0000\u024b\u024c\u0005I\u0000\u0000\u024cq\u0001\u0000\u0000\u0000"+ - "\u024d\u024e\u0005\r\u0000\u0000\u024e\u024f\u0005k\u0000\u0000\u024f"+ - "s\u0001\u0000\u0000\u0000\u0250\u0251\u0005\u0003\u0000\u0000\u0251\u0254"+ - "\u0005a\u0000\u0000\u0252\u0253\u0005_\u0000\u0000\u0253\u0255\u0003<"+ - "\u001e\u0000\u0254\u0252\u0001\u0000\u0000\u0000\u0254\u0255\u0001\u0000"+ - "\u0000\u0000\u0255\u025f\u0001\u0000\u0000\u0000\u0256\u0257\u0005`\u0000"+ - "\u0000\u0257\u025c\u0003v;\u0000\u0258\u0259\u0005\'\u0000\u0000\u0259"+ - "\u025b\u0003v;\u0000\u025a\u0258\u0001\u0000\u0000\u0000\u025b\u025e\u0001"+ - "\u0000\u0000\u0000\u025c\u025a\u0001\u0000\u0000\u0000\u025c\u025d\u0001"+ - "\u0000\u0000\u0000\u025d\u0260\u0001\u0000\u0000\u0000\u025e\u025c\u0001"+ - "\u0000\u0000\u0000\u025f\u0256\u0001\u0000\u0000\u0000\u025f\u0260\u0001"+ - "\u0000\u0000\u0000\u0260u\u0001\u0000\u0000\u0000\u0261\u0262\u0003<\u001e"+ - "\u0000\u0262\u0263\u0005$\u0000\u0000\u0263\u0265\u0001\u0000\u0000\u0000"+ - "\u0264\u0261\u0001\u0000\u0000\u0000\u0264\u0265\u0001\u0000\u0000\u0000"+ - "\u0265\u0266\u0001\u0000\u0000\u0000\u0266\u0267\u0003<\u001e\u0000\u0267"+ - "w\u0001\u0000\u0000\u0000\u0268\u0269\u0005\u0014\u0000\u0000\u0269\u026a"+ - "\u0003(\u0014\u0000\u026a\u026b\u0005_\u0000\u0000\u026b\u026c\u0003>"+ - "\u001f\u0000\u026cy\u0001\u0000\u0000\u0000\u026d\u026e\u0005\u0013\u0000"+ - "\u0000\u026e\u0271\u00036\u001b\u0000\u026f\u0270\u0005!\u0000\u0000\u0270"+ - "\u0272\u0003\"\u0011\u0000\u0271\u026f\u0001\u0000\u0000\u0000\u0271\u0272"+ - "\u0001\u0000\u0000\u0000\u0272{\u0001\u0000\u0000\u0000\u0273\u0274\u0007"+ - "\b\u0000\u0000\u0274\u0275\u0005y\u0000\u0000\u0275\u0276\u0003~?\u0000"+ - "\u0276\u0277\u0003\u0080@\u0000\u0277}\u0001\u0000\u0000\u0000\u0278\u0279"+ - "\u0003(\u0014\u0000\u0279\u007f\u0001\u0000\u0000\u0000\u027a\u027b\u0005"+ - "_\u0000\u0000\u027b\u0280\u0003\u0082A\u0000\u027c\u027d\u0005\'\u0000"+ - "\u0000\u027d\u027f\u0003\u0082A\u0000\u027e\u027c\u0001\u0000\u0000\u0000"+ - "\u027f\u0282\u0001\u0000\u0000\u0000\u0280\u027e\u0001\u0000\u0000\u0000"+ - "\u0280\u0281\u0001\u0000\u0000\u0000\u0281\u0081\u0001\u0000\u0000\u0000"+ - "\u0282\u0280\u0001\u0000\u0000\u0000\u0283\u0284\u0003\u0010\b\u0000\u0284"+ - "\u0083\u0001\u0000\u0000\u0000\u0285\u0286\u0005\u0012\u0000\u0000\u0286"+ - "\u0289\u0003:\u001d\u0000\u0287\u0288\u0005_\u0000\u0000\u0288\u028a\u0003"+ - ":\u001d\u0000\u0289\u0287\u0001\u0000\u0000\u0000\u0289\u028a\u0001\u0000"+ - "\u0000\u0000\u028a\u0290\u0001\u0000\u0000\u0000\u028b\u028c\u0005[\u0000"+ - "\u0000\u028c\u028d\u0003:\u001d\u0000\u028d\u028e\u0005\'\u0000\u0000"+ - "\u028e\u028f\u0003:\u001d\u0000\u028f\u0291\u0001\u0000\u0000\u0000\u0290"+ - "\u028b\u0001\u0000\u0000\u0000\u0290\u0291\u0001\u0000\u0000\u0000\u0291"+ - "\u0085\u0001\u0000\u0000\u0000?\u0091\u009a\u00af\u00bb\u00c4\u00cc\u00d1"+ - "\u00d9\u00db\u00e0\u00e7\u00ec\u00f1\u00fb\u0101\u0109\u010b\u0116\u011d"+ - "\u0128\u012d\u012f\u013b\u014e\u0154\u015e\u0162\u0167\u0175\u017e\u0182"+ - "\u0186\u018d\u0191\u0198\u019e\u01a5\u01ad\u01b5\u01bc\u01cd\u01d8\u01e3"+ - "\u01e8\u01ec\u01f0\u01fb\u0200\u0204\u0212\u021d\u022b\u0236\u0239\u023e"+ - "\u0254\u025c\u025f\u0264\u0271\u0280\u0289\u0290"; + "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u00b4\b\u0003"+ + "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00c0\b\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005"+ + "\u00c7\b\u0005\n\u0005\f\u0005\u00ca\t\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00d1\b\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0003\u0005\u00d6\b\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00de\b\u0005\n"+ + "\u0005\f\u0005\u00e1\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00e5"+ + "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003"+ + "\u0006\u00ec\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00f1"+ + "\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00f6\b\u0007"+ + "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+ + "\b\u0003\b\u0100\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u0106\b\t"+ + "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u010e\b\t\n\t"+ + "\f\t\u0111\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n"+ + "\u0001\n\u0003\n\u011b\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0120\b\n\n"+ + "\n\f\n\u0123\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+ + "\u000b\u0001\u000b\u0005\u000b\u012b\b\u000b\n\u000b\f\u000b\u012e\t\u000b"+ + "\u0001\u000b\u0001\u000b\u0003\u000b\u0132\b\u000b\u0003\u000b\u0134\b"+ + "\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r"+ + "\u0001\r\u0005\r\u013e\b\r\n\r\f\r\u0141\t\r\u0001\r\u0001\r\u0001\u000e"+ + "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u0010"+ + "\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011"+ + "\u0151\b\u0011\n\u0011\f\u0011\u0154\t\u0011\u0001\u0012\u0001\u0012\u0001"+ + "\u0012\u0003\u0012\u0159\b\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001"+ + "\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u0161\b\u0013\n\u0013\f\u0013"+ + "\u0164\t\u0013\u0001\u0013\u0003\u0013\u0167\b\u0013\u0001\u0014\u0001"+ + "\u0014\u0001\u0014\u0003\u0014\u016c\b\u0014\u0001\u0014\u0001\u0014\u0001"+ + "\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001"+ + "\u0017\u0001\u0017\u0005\u0017\u0178\b\u0017\n\u0017\f\u0017\u017b\t\u0017"+ + "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0005\u0018\u0181\b\u0018"+ + "\n\u0018\f\u0018\u0184\t\u0018\u0001\u0018\u0003\u0018\u0187\b\u0018\u0001"+ + "\u0018\u0001\u0018\u0003\u0018\u018b\b\u0018\u0001\u0019\u0001\u0019\u0001"+ + "\u0019\u0001\u001a\u0001\u001a\u0003\u001a\u0192\b\u001a\u0001\u001a\u0001"+ + "\u001a\u0003\u001a\u0196\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005"+ + "\u001b\u019b\b\u001b\n\u001b\f\u001b\u019e\t\u001b\u0001\u001c\u0001\u001c"+ + "\u0001\u001c\u0003\u001c\u01a3\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d"+ + "\u0005\u001d\u01a8\b\u001d\n\u001d\f\u001d\u01ab\t\u001d\u0001\u001e\u0001"+ + "\u001e\u0001\u001e\u0005\u001e\u01b0\b\u001e\n\u001e\f\u001e\u01b3\t\u001e"+ + "\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01b8\b\u001f\n\u001f"+ + "\f\u001f\u01bb\t\u001f\u0001 \u0001 \u0001!\u0001!\u0003!\u01c1\b!\u0001"+ + "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ + "\"\u0001\"\u0001\"\u0001\"\u0005\"\u01d0\b\"\n\"\f\"\u01d3\t\"\u0001\""+ + "\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01db\b\"\n\"\f\"\u01de"+ + "\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01e6\b\""+ + "\n\"\f\"\u01e9\t\"\u0001\"\u0001\"\u0003\"\u01ed\b\"\u0001#\u0001#\u0003"+ + "#\u01f1\b#\u0001$\u0001$\u0003$\u01f5\b$\u0001%\u0001%\u0001%\u0001&\u0001"+ + "&\u0001&\u0001&\u0005&\u01fe\b&\n&\f&\u0201\t&\u0001\'\u0001\'\u0003\'"+ + "\u0205\b\'\u0001\'\u0001\'\u0003\'\u0209\b\'\u0001(\u0001(\u0001(\u0001"+ + ")\u0001)\u0001)\u0001*\u0001*\u0001*\u0001*\u0005*\u0215\b*\n*\f*\u0218"+ + "\t*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0003,\u0222"+ + "\b,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001/\u0001"+ + "/\u0005/\u022e\b/\n/\f/\u0231\t/\u00010\u00010\u00010\u00010\u00011\u0001"+ + "1\u00012\u00012\u00032\u023b\b2\u00013\u00033\u023e\b3\u00013\u00013\u0001"+ + "4\u00034\u0243\b4\u00014\u00014\u00015\u00015\u00016\u00016\u00017\u0001"+ + "7\u00017\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u0001:\u0001"+ + ":\u0001:\u0001:\u0003:\u0259\b:\u0001:\u0001:\u0001:\u0001:\u0005:\u025f"+ + "\b:\n:\f:\u0262\t:\u0003:\u0264\b:\u0001;\u0001;\u0001;\u0003;\u0269\b"+ + ";\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001"+ + "=\u0001=\u0003=\u0276\b=\u0001>\u0001>\u0001>\u0001>\u0001>\u0001?\u0001"+ + "?\u0001@\u0001@\u0001@\u0001@\u0005@\u0283\b@\n@\f@\u0286\t@\u0001A\u0001"+ + "A\u0001B\u0001B\u0001B\u0001B\u0003B\u028e\bB\u0001B\u0001B\u0001B\u0001"+ + "B\u0001B\u0003B\u0295\bB\u0001C\u0001C\u0001C\u0001C\u0000\u0004\u0002"+ + "\n\u0012\u0014D\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014"+ + "\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfh"+ + "jlnprtvxz|~\u0080\u0082\u0084\u0086\u0000\t\u0001\u0000AB\u0001\u0000"+ + "CE\u0002\u0000\u001f\u001fTT\u0001\u0000KL\u0002\u0000$$))\u0002\u0000"+ + ",,//\u0002\u0000++99\u0002\u0000::<@\u0002\u0000\u0011\u0011\u0018\u0019"+ + "\u02b5\u0000\u0088\u0001\u0000\u0000\u0000\u0002\u008b\u0001\u0000\u0000"+ + "\u0000\u0004\u009c\u0001\u0000\u0000\u0000\u0006\u00b3\u0001\u0000\u0000"+ + "\u0000\b\u00b5\u0001\u0000\u0000\u0000\n\u00d5\u0001\u0000\u0000\u0000"+ + "\f\u00f0\u0001\u0000\u0000\u0000\u000e\u00f2\u0001\u0000\u0000\u0000\u0010"+ + "\u00ff\u0001\u0000\u0000\u0000\u0012\u0105\u0001\u0000\u0000\u0000\u0014"+ + "\u011a\u0001\u0000\u0000\u0000\u0016\u0124\u0001\u0000\u0000\u0000\u0018"+ + "\u0137\u0001\u0000\u0000\u0000\u001a\u0139\u0001\u0000\u0000\u0000\u001c"+ + "\u0144\u0001\u0000\u0000\u0000\u001e\u0148\u0001\u0000\u0000\u0000 \u014a"+ + "\u0001\u0000\u0000\u0000\"\u014d\u0001\u0000\u0000\u0000$\u0158\u0001"+ + "\u0000\u0000\u0000&\u015c\u0001\u0000\u0000\u0000(\u016b\u0001\u0000\u0000"+ + "\u0000*\u016f\u0001\u0000\u0000\u0000,\u0171\u0001\u0000\u0000\u0000."+ + "\u0173\u0001\u0000\u0000\u00000\u017c\u0001\u0000\u0000\u00002\u018c\u0001"+ + "\u0000\u0000\u00004\u018f\u0001\u0000\u0000\u00006\u0197\u0001\u0000\u0000"+ + "\u00008\u019f\u0001\u0000\u0000\u0000:\u01a4\u0001\u0000\u0000\u0000<"+ + "\u01ac\u0001\u0000\u0000\u0000>\u01b4\u0001\u0000\u0000\u0000@\u01bc\u0001"+ + "\u0000\u0000\u0000B\u01c0\u0001\u0000\u0000\u0000D\u01ec\u0001\u0000\u0000"+ + "\u0000F\u01f0\u0001\u0000\u0000\u0000H\u01f4\u0001\u0000\u0000\u0000J"+ + "\u01f6\u0001\u0000\u0000\u0000L\u01f9\u0001\u0000\u0000\u0000N\u0202\u0001"+ + "\u0000\u0000\u0000P\u020a\u0001\u0000\u0000\u0000R\u020d\u0001\u0000\u0000"+ + "\u0000T\u0210\u0001\u0000\u0000\u0000V\u0219\u0001\u0000\u0000\u0000X"+ + "\u021d\u0001\u0000\u0000\u0000Z\u0223\u0001\u0000\u0000\u0000\\\u0227"+ + "\u0001\u0000\u0000\u0000^\u022a\u0001\u0000\u0000\u0000`\u0232\u0001\u0000"+ + "\u0000\u0000b\u0236\u0001\u0000\u0000\u0000d\u023a\u0001\u0000\u0000\u0000"+ + "f\u023d\u0001\u0000\u0000\u0000h\u0242\u0001\u0000\u0000\u0000j\u0246"+ + "\u0001\u0000\u0000\u0000l\u0248\u0001\u0000\u0000\u0000n\u024a\u0001\u0000"+ + "\u0000\u0000p\u024d\u0001\u0000\u0000\u0000r\u0251\u0001\u0000\u0000\u0000"+ + "t\u0254\u0001\u0000\u0000\u0000v\u0268\u0001\u0000\u0000\u0000x\u026c"+ + "\u0001\u0000\u0000\u0000z\u0271\u0001\u0000\u0000\u0000|\u0277\u0001\u0000"+ + "\u0000\u0000~\u027c\u0001\u0000\u0000\u0000\u0080\u027e\u0001\u0000\u0000"+ + "\u0000\u0082\u0287\u0001\u0000\u0000\u0000\u0084\u0289\u0001\u0000\u0000"+ + "\u0000\u0086\u0296\u0001\u0000\u0000\u0000\u0088\u0089\u0003\u0002\u0001"+ + "\u0000\u0089\u008a\u0005\u0000\u0000\u0001\u008a\u0001\u0001\u0000\u0000"+ + "\u0000\u008b\u008c\u0006\u0001\uffff\uffff\u0000\u008c\u008d\u0003\u0004"+ + "\u0002\u0000\u008d\u0093\u0001\u0000\u0000\u0000\u008e\u008f\n\u0001\u0000"+ + "\u0000\u008f\u0090\u0005\u001e\u0000\u0000\u0090\u0092\u0003\u0006\u0003"+ + "\u0000\u0091\u008e\u0001\u0000\u0000\u0000\u0092\u0095\u0001\u0000\u0000"+ + "\u0000\u0093\u0091\u0001\u0000\u0000\u0000\u0093\u0094\u0001\u0000\u0000"+ + "\u0000\u0094\u0003\u0001\u0000\u0000\u0000\u0095\u0093\u0001\u0000\u0000"+ + "\u0000\u0096\u009d\u0003n7\u0000\u0097\u009d\u0003&\u0013\u0000\u0098"+ + "\u009d\u0003 \u0010\u0000\u0099\u009d\u0003r9\u0000\u009a\u009b\u0004"+ + "\u0002\u0001\u0000\u009b\u009d\u00030\u0018\u0000\u009c\u0096\u0001\u0000"+ + "\u0000\u0000\u009c\u0097\u0001\u0000\u0000\u0000\u009c\u0098\u0001\u0000"+ + "\u0000\u0000\u009c\u0099\u0001\u0000\u0000\u0000\u009c\u009a\u0001\u0000"+ + "\u0000\u0000\u009d\u0005\u0001\u0000\u0000\u0000\u009e\u00b4\u00032\u0019"+ + "\u0000\u009f\u00b4\u0003\b\u0004\u0000\u00a0\u00b4\u0003P(\u0000\u00a1"+ + "\u00b4\u0003J%\u0000\u00a2\u00b4\u00034\u001a\u0000\u00a3\u00b4\u0003"+ + "L&\u0000\u00a4\u00b4\u0003R)\u0000\u00a5\u00b4\u0003T*\u0000\u00a6\u00b4"+ + "\u0003X,\u0000\u00a7\u00b4\u0003Z-\u0000\u00a8\u00b4\u0003t:\u0000\u00a9"+ + "\u00b4\u0003\\.\u0000\u00aa\u00b4\u0003|>\u0000\u00ab\u00ac\u0004\u0003"+ + "\u0002\u0000\u00ac\u00b4\u0003z=\u0000\u00ad\u00ae\u0004\u0003\u0003\u0000"+ + "\u00ae\u00b4\u0003x<\u0000\u00af\u00b0\u0004\u0003\u0004\u0000\u00b0\u00b4"+ + "\u0003\u0084B\u0000\u00b1\u00b2\u0004\u0003\u0005\u0000\u00b2\u00b4\u0003"+ + "\u0086C\u0000\u00b3\u009e\u0001\u0000\u0000\u0000\u00b3\u009f\u0001\u0000"+ + "\u0000\u0000\u00b3\u00a0\u0001\u0000\u0000\u0000\u00b3\u00a1\u0001\u0000"+ + "\u0000\u0000\u00b3\u00a2\u0001\u0000\u0000\u0000\u00b3\u00a3\u0001\u0000"+ + "\u0000\u0000\u00b3\u00a4\u0001\u0000\u0000\u0000\u00b3\u00a5\u0001\u0000"+ + "\u0000\u0000\u00b3\u00a6\u0001\u0000\u0000\u0000\u00b3\u00a7\u0001\u0000"+ + "\u0000\u0000\u00b3\u00a8\u0001\u0000\u0000\u0000\u00b3\u00a9\u0001\u0000"+ + "\u0000\u0000\u00b3\u00aa\u0001\u0000\u0000\u0000\u00b3\u00ab\u0001\u0000"+ + "\u0000\u0000\u00b3\u00ad\u0001\u0000\u0000\u0000\u00b3\u00af\u0001\u0000"+ + "\u0000\u0000\u00b3\u00b1\u0001\u0000\u0000\u0000\u00b4\u0007\u0001\u0000"+ + "\u0000\u0000\u00b5\u00b6\u0005\u0010\u0000\u0000\u00b6\u00b7\u0003\n\u0005"+ + "\u0000\u00b7\t\u0001\u0000\u0000\u0000\u00b8\u00b9\u0006\u0005\uffff\uffff"+ + "\u0000\u00b9\u00ba\u00052\u0000\u0000\u00ba\u00d6\u0003\n\u0005\b\u00bb"+ + "\u00d6\u0003\u0010\b\u0000\u00bc\u00d6\u0003\f\u0006\u0000\u00bd\u00bf"+ + "\u0003\u0010\b\u0000\u00be\u00c0\u00052\u0000\u0000\u00bf\u00be\u0001"+ + "\u0000\u0000\u0000\u00bf\u00c0\u0001\u0000\u0000\u0000\u00c0\u00c1\u0001"+ + "\u0000\u0000\u0000\u00c1\u00c2\u0005-\u0000\u0000\u00c2\u00c3\u00051\u0000"+ + "\u0000\u00c3\u00c8\u0003\u0010\b\u0000\u00c4\u00c5\u0005(\u0000\u0000"+ + "\u00c5\u00c7\u0003\u0010\b\u0000\u00c6\u00c4\u0001\u0000\u0000\u0000\u00c7"+ + "\u00ca\u0001\u0000\u0000\u0000\u00c8\u00c6\u0001\u0000\u0000\u0000\u00c8"+ + "\u00c9\u0001\u0000\u0000\u0000\u00c9\u00cb\u0001\u0000\u0000\u0000\u00ca"+ + "\u00c8\u0001\u0000\u0000\u0000\u00cb\u00cc\u00058\u0000\u0000\u00cc\u00d6"+ + "\u0001\u0000\u0000\u0000\u00cd\u00ce\u0003\u0010\b\u0000\u00ce\u00d0\u0005"+ + ".\u0000\u0000\u00cf\u00d1\u00052\u0000\u0000\u00d0\u00cf\u0001\u0000\u0000"+ + "\u0000\u00d0\u00d1\u0001\u0000\u0000\u0000\u00d1\u00d2\u0001\u0000\u0000"+ + "\u0000\u00d2\u00d3\u00053\u0000\u0000\u00d3\u00d6\u0001\u0000\u0000\u0000"+ + "\u00d4\u00d6\u0003\u000e\u0007\u0000\u00d5\u00b8\u0001\u0000\u0000\u0000"+ + "\u00d5\u00bb\u0001\u0000\u0000\u0000\u00d5\u00bc\u0001\u0000\u0000\u0000"+ + "\u00d5\u00bd\u0001\u0000\u0000\u0000\u00d5\u00cd\u0001\u0000\u0000\u0000"+ + "\u00d5\u00d4\u0001\u0000\u0000\u0000\u00d6\u00df\u0001\u0000\u0000\u0000"+ + "\u00d7\u00d8\n\u0005\u0000\u0000\u00d8\u00d9\u0005#\u0000\u0000\u00d9"+ + "\u00de\u0003\n\u0005\u0006\u00da\u00db\n\u0004\u0000\u0000\u00db\u00dc"+ + "\u00055\u0000\u0000\u00dc\u00de\u0003\n\u0005\u0005\u00dd\u00d7\u0001"+ + "\u0000\u0000\u0000\u00dd\u00da\u0001\u0000\u0000\u0000\u00de\u00e1\u0001"+ + "\u0000\u0000\u0000\u00df\u00dd\u0001\u0000\u0000\u0000\u00df\u00e0\u0001"+ + "\u0000\u0000\u0000\u00e0\u000b\u0001\u0000\u0000\u0000\u00e1\u00df\u0001"+ + "\u0000\u0000\u0000\u00e2\u00e4\u0003\u0010\b\u0000\u00e3\u00e5\u00052"+ + "\u0000\u0000\u00e4\u00e3\u0001\u0000\u0000\u0000\u00e4\u00e5\u0001\u0000"+ + "\u0000\u0000\u00e5\u00e6\u0001\u0000\u0000\u0000\u00e6\u00e7\u00050\u0000"+ + "\u0000\u00e7\u00e8\u0003j5\u0000\u00e8\u00f1\u0001\u0000\u0000\u0000\u00e9"+ + "\u00eb\u0003\u0010\b\u0000\u00ea\u00ec\u00052\u0000\u0000\u00eb\u00ea"+ + "\u0001\u0000\u0000\u0000\u00eb\u00ec\u0001\u0000\u0000\u0000\u00ec\u00ed"+ + "\u0001\u0000\u0000\u0000\u00ed\u00ee\u00057\u0000\u0000\u00ee\u00ef\u0003"+ + "j5\u0000\u00ef\u00f1\u0001\u0000\u0000\u0000\u00f0\u00e2\u0001\u0000\u0000"+ + "\u0000\u00f0\u00e9\u0001\u0000\u0000\u0000\u00f1\r\u0001\u0000\u0000\u0000"+ + "\u00f2\u00f5\u0003:\u001d\u0000\u00f3\u00f4\u0005&\u0000\u0000\u00f4\u00f6"+ + "\u0003\u001e\u000f\u0000\u00f5\u00f3\u0001\u0000\u0000\u0000\u00f5\u00f6"+ + "\u0001\u0000\u0000\u0000\u00f6\u00f7\u0001\u0000\u0000\u0000\u00f7\u00f8"+ + "\u0005\'\u0000\u0000\u00f8\u00f9\u0003D\"\u0000\u00f9\u000f\u0001\u0000"+ + "\u0000\u0000\u00fa\u0100\u0003\u0012\t\u0000\u00fb\u00fc\u0003\u0012\t"+ + "\u0000\u00fc\u00fd\u0003l6\u0000\u00fd\u00fe\u0003\u0012\t\u0000\u00fe"+ + "\u0100\u0001\u0000\u0000\u0000\u00ff\u00fa\u0001\u0000\u0000\u0000\u00ff"+ + "\u00fb\u0001\u0000\u0000\u0000\u0100\u0011\u0001\u0000\u0000\u0000\u0101"+ + "\u0102\u0006\t\uffff\uffff\u0000\u0102\u0106\u0003\u0014\n\u0000\u0103"+ + "\u0104\u0007\u0000\u0000\u0000\u0104\u0106\u0003\u0012\t\u0003\u0105\u0101"+ + "\u0001\u0000\u0000\u0000\u0105\u0103\u0001\u0000\u0000\u0000\u0106\u010f"+ + "\u0001\u0000\u0000\u0000\u0107\u0108\n\u0002\u0000\u0000\u0108\u0109\u0007"+ + "\u0001\u0000\u0000\u0109\u010e\u0003\u0012\t\u0003\u010a\u010b\n\u0001"+ + "\u0000\u0000\u010b\u010c\u0007\u0000\u0000\u0000\u010c\u010e\u0003\u0012"+ + "\t\u0002\u010d\u0107\u0001\u0000\u0000\u0000\u010d\u010a\u0001\u0000\u0000"+ + "\u0000\u010e\u0111\u0001\u0000\u0000\u0000\u010f\u010d\u0001\u0000\u0000"+ + "\u0000\u010f\u0110\u0001\u0000\u0000\u0000\u0110\u0013\u0001\u0000\u0000"+ + "\u0000\u0111\u010f\u0001\u0000\u0000\u0000\u0112\u0113\u0006\n\uffff\uffff"+ + "\u0000\u0113\u011b\u0003D\"\u0000\u0114\u011b\u0003:\u001d\u0000\u0115"+ + "\u011b\u0003\u0016\u000b\u0000\u0116\u0117\u00051\u0000\u0000\u0117\u0118"+ + "\u0003\n\u0005\u0000\u0118\u0119\u00058\u0000\u0000\u0119\u011b\u0001"+ + "\u0000\u0000\u0000\u011a\u0112\u0001\u0000\u0000\u0000\u011a\u0114\u0001"+ + "\u0000\u0000\u0000\u011a\u0115\u0001\u0000\u0000\u0000\u011a\u0116\u0001"+ + "\u0000\u0000\u0000\u011b\u0121\u0001\u0000\u0000\u0000\u011c\u011d\n\u0001"+ + "\u0000\u0000\u011d\u011e\u0005&\u0000\u0000\u011e\u0120\u0003\u001e\u000f"+ + "\u0000\u011f\u011c\u0001\u0000\u0000\u0000\u0120\u0123\u0001\u0000\u0000"+ + "\u0000\u0121\u011f\u0001\u0000\u0000\u0000\u0121\u0122\u0001\u0000\u0000"+ + "\u0000\u0122\u0015\u0001\u0000\u0000\u0000\u0123\u0121\u0001\u0000\u0000"+ + "\u0000\u0124\u0125\u0003\u0018\f\u0000\u0125\u0133\u00051\u0000\u0000"+ + "\u0126\u0134\u0005C\u0000\u0000\u0127\u012c\u0003\n\u0005\u0000\u0128"+ + "\u0129\u0005(\u0000\u0000\u0129\u012b\u0003\n\u0005\u0000\u012a\u0128"+ + "\u0001\u0000\u0000\u0000\u012b\u012e\u0001\u0000\u0000\u0000\u012c\u012a"+ + "\u0001\u0000\u0000\u0000\u012c\u012d\u0001\u0000\u0000\u0000\u012d\u0131"+ + "\u0001\u0000\u0000\u0000\u012e\u012c\u0001\u0000\u0000\u0000\u012f\u0130"+ + "\u0005(\u0000\u0000\u0130\u0132\u0003\u001a\r\u0000\u0131\u012f\u0001"+ + "\u0000\u0000\u0000\u0131\u0132\u0001\u0000\u0000\u0000\u0132\u0134\u0001"+ + "\u0000\u0000\u0000\u0133\u0126\u0001\u0000\u0000\u0000\u0133\u0127\u0001"+ + "\u0000\u0000\u0000\u0133\u0134\u0001\u0000\u0000\u0000\u0134\u0135\u0001"+ + "\u0000\u0000\u0000\u0135\u0136\u00058\u0000\u0000\u0136\u0017\u0001\u0000"+ + "\u0000\u0000\u0137\u0138\u0003H$\u0000\u0138\u0019\u0001\u0000\u0000\u0000"+ + "\u0139\u013a\u0005F\u0000\u0000\u013a\u013f\u0003\u001c\u000e\u0000\u013b"+ + "\u013c\u0005(\u0000\u0000\u013c\u013e\u0003\u001c\u000e\u0000\u013d\u013b"+ + "\u0001\u0000\u0000\u0000\u013e\u0141\u0001\u0000\u0000\u0000\u013f\u013d"+ + "\u0001\u0000\u0000\u0000\u013f\u0140\u0001\u0000\u0000\u0000\u0140\u0142"+ + "\u0001\u0000\u0000\u0000\u0141\u013f\u0001\u0000\u0000\u0000\u0142\u0143"+ + "\u0005G\u0000\u0000\u0143\u001b\u0001\u0000\u0000\u0000\u0144\u0145\u0003"+ + "j5\u0000\u0145\u0146\u0005\'\u0000\u0000\u0146\u0147\u0003D\"\u0000\u0147"+ + "\u001d\u0001\u0000\u0000\u0000\u0148\u0149\u0003@ \u0000\u0149\u001f\u0001"+ + "\u0000\u0000\u0000\u014a\u014b\u0005\f\u0000\u0000\u014b\u014c\u0003\""+ + "\u0011\u0000\u014c!\u0001\u0000\u0000\u0000\u014d\u0152\u0003$\u0012\u0000"+ + "\u014e\u014f\u0005(\u0000\u0000\u014f\u0151\u0003$\u0012\u0000\u0150\u014e"+ + "\u0001\u0000\u0000\u0000\u0151\u0154\u0001\u0000\u0000\u0000\u0152\u0150"+ + "\u0001\u0000\u0000\u0000\u0152\u0153\u0001\u0000\u0000\u0000\u0153#\u0001"+ + "\u0000\u0000\u0000\u0154\u0152\u0001\u0000\u0000\u0000\u0155\u0156\u0003"+ + ":\u001d\u0000\u0156\u0157\u0005%\u0000\u0000\u0157\u0159\u0001\u0000\u0000"+ + "\u0000\u0158\u0155\u0001\u0000\u0000\u0000\u0158\u0159\u0001\u0000\u0000"+ + "\u0000\u0159\u015a\u0001\u0000\u0000\u0000\u015a\u015b\u0003\n\u0005\u0000"+ + "\u015b%\u0001\u0000\u0000\u0000\u015c\u015d\u0005\u0006\u0000\u0000\u015d"+ + "\u0162\u0003(\u0014\u0000\u015e\u015f\u0005(\u0000\u0000\u015f\u0161\u0003"+ + "(\u0014\u0000\u0160\u015e\u0001\u0000\u0000\u0000\u0161\u0164\u0001\u0000"+ + "\u0000\u0000\u0162\u0160\u0001\u0000\u0000\u0000\u0162\u0163\u0001\u0000"+ + "\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164\u0162\u0001\u0000"+ + "\u0000\u0000\u0165\u0167\u0003.\u0017\u0000\u0166\u0165\u0001\u0000\u0000"+ + "\u0000\u0166\u0167\u0001\u0000\u0000\u0000\u0167\'\u0001\u0000\u0000\u0000"+ + "\u0168\u0169\u0003*\u0015\u0000\u0169\u016a\u0005\'\u0000\u0000\u016a"+ + "\u016c\u0001\u0000\u0000\u0000\u016b\u0168\u0001\u0000\u0000\u0000\u016b"+ + "\u016c\u0001\u0000\u0000\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d"+ + "\u016e\u0003,\u0016\u0000\u016e)\u0001\u0000\u0000\u0000\u016f\u0170\u0007"+ + "\u0002\u0000\u0000\u0170+\u0001\u0000\u0000\u0000\u0171\u0172\u0007\u0002"+ + "\u0000\u0000\u0172-\u0001\u0000\u0000\u0000\u0173\u0174\u0005S\u0000\u0000"+ + "\u0174\u0179\u0005T\u0000\u0000\u0175\u0176\u0005(\u0000\u0000\u0176\u0178"+ + "\u0005T\u0000\u0000\u0177\u0175\u0001\u0000\u0000\u0000\u0178\u017b\u0001"+ + "\u0000\u0000\u0000\u0179\u0177\u0001\u0000\u0000\u0000\u0179\u017a\u0001"+ + "\u0000\u0000\u0000\u017a/\u0001\u0000\u0000\u0000\u017b\u0179\u0001\u0000"+ + "\u0000\u0000\u017c\u017d\u0005\u0016\u0000\u0000\u017d\u0182\u0003(\u0014"+ + "\u0000\u017e\u017f\u0005(\u0000\u0000\u017f\u0181\u0003(\u0014\u0000\u0180"+ + "\u017e\u0001\u0000\u0000\u0000\u0181\u0184\u0001\u0000\u0000\u0000\u0182"+ + "\u0180\u0001\u0000\u0000\u0000\u0182\u0183\u0001\u0000\u0000\u0000\u0183"+ + "\u0186\u0001\u0000\u0000\u0000\u0184\u0182\u0001\u0000\u0000\u0000\u0185"+ + "\u0187\u00036\u001b\u0000\u0186\u0185\u0001\u0000\u0000\u0000\u0186\u0187"+ + "\u0001\u0000\u0000\u0000\u0187\u018a\u0001\u0000\u0000\u0000\u0188\u0189"+ + "\u0005\"\u0000\u0000\u0189\u018b\u0003\"\u0011\u0000\u018a\u0188\u0001"+ + "\u0000\u0000\u0000\u018a\u018b\u0001\u0000\u0000\u0000\u018b1\u0001\u0000"+ + "\u0000\u0000\u018c\u018d\u0005\u0004\u0000\u0000\u018d\u018e\u0003\"\u0011"+ + "\u0000\u018e3\u0001\u0000\u0000\u0000\u018f\u0191\u0005\u000f\u0000\u0000"+ + "\u0190\u0192\u00036\u001b\u0000\u0191\u0190\u0001\u0000\u0000\u0000\u0191"+ + "\u0192\u0001\u0000\u0000\u0000\u0192\u0195\u0001\u0000\u0000\u0000\u0193"+ + "\u0194\u0005\"\u0000\u0000\u0194\u0196\u0003\"\u0011\u0000\u0195\u0193"+ + "\u0001\u0000\u0000\u0000\u0195\u0196\u0001\u0000\u0000\u0000\u01965\u0001"+ + "\u0000\u0000\u0000\u0197\u019c\u00038\u001c\u0000\u0198\u0199\u0005(\u0000"+ + "\u0000\u0199\u019b\u00038\u001c\u0000\u019a\u0198\u0001\u0000\u0000\u0000"+ + "\u019b\u019e\u0001\u0000\u0000\u0000\u019c\u019a\u0001\u0000\u0000\u0000"+ + "\u019c\u019d\u0001\u0000\u0000\u0000\u019d7\u0001\u0000\u0000\u0000\u019e"+ + "\u019c\u0001\u0000\u0000\u0000\u019f\u01a2\u0003$\u0012\u0000\u01a0\u01a1"+ + "\u0005\u0010\u0000\u0000\u01a1\u01a3\u0003\n\u0005\u0000\u01a2\u01a0\u0001"+ + "\u0000\u0000\u0000\u01a2\u01a3\u0001\u0000\u0000\u0000\u01a39\u0001\u0000"+ + "\u0000\u0000\u01a4\u01a9\u0003H$\u0000\u01a5\u01a6\u0005*\u0000\u0000"+ + "\u01a6\u01a8\u0003H$\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a8\u01ab"+ + "\u0001\u0000\u0000\u0000\u01a9\u01a7\u0001\u0000\u0000\u0000\u01a9\u01aa"+ + "\u0001\u0000\u0000\u0000\u01aa;\u0001\u0000\u0000\u0000\u01ab\u01a9\u0001"+ + "\u0000\u0000\u0000\u01ac\u01b1\u0003B!\u0000\u01ad\u01ae\u0005*\u0000"+ + "\u0000\u01ae\u01b0\u0003B!\u0000\u01af\u01ad\u0001\u0000\u0000\u0000\u01b0"+ + "\u01b3\u0001\u0000\u0000\u0000\u01b1\u01af\u0001\u0000\u0000\u0000\u01b1"+ + "\u01b2\u0001\u0000\u0000\u0000\u01b2=\u0001\u0000\u0000\u0000\u01b3\u01b1"+ + "\u0001\u0000\u0000\u0000\u01b4\u01b9\u0003<\u001e\u0000\u01b5\u01b6\u0005"+ + "(\u0000\u0000\u01b6\u01b8\u0003<\u001e\u0000\u01b7\u01b5\u0001\u0000\u0000"+ + "\u0000\u01b8\u01bb\u0001\u0000\u0000\u0000\u01b9\u01b7\u0001\u0000\u0000"+ + "\u0000\u01b9\u01ba\u0001\u0000\u0000\u0000\u01ba?\u0001\u0000\u0000\u0000"+ + "\u01bb\u01b9\u0001\u0000\u0000\u0000\u01bc\u01bd\u0007\u0003\u0000\u0000"+ + "\u01bdA\u0001\u0000\u0000\u0000\u01be\u01c1\u0005X\u0000\u0000\u01bf\u01c1"+ + "\u0003F#\u0000\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01bf\u0001\u0000"+ + "\u0000\u0000\u01c1C\u0001\u0000\u0000\u0000\u01c2\u01ed\u00053\u0000\u0000"+ + "\u01c3\u01c4\u0003h4\u0000\u01c4\u01c5\u0005K\u0000\u0000\u01c5\u01ed"+ + "\u0001\u0000\u0000\u0000\u01c6\u01ed\u0003f3\u0000\u01c7\u01ed\u0003h"+ + "4\u0000\u01c8\u01ed\u0003b1\u0000\u01c9\u01ed\u0003F#\u0000\u01ca\u01ed"+ + "\u0003j5\u0000\u01cb\u01cc\u0005I\u0000\u0000\u01cc\u01d1\u0003d2\u0000"+ + "\u01cd\u01ce\u0005(\u0000\u0000\u01ce\u01d0\u0003d2\u0000\u01cf\u01cd"+ + "\u0001\u0000\u0000\u0000\u01d0\u01d3\u0001\u0000\u0000\u0000\u01d1\u01cf"+ + "\u0001\u0000\u0000\u0000\u01d1\u01d2\u0001\u0000\u0000\u0000\u01d2\u01d4"+ + "\u0001\u0000\u0000\u0000\u01d3\u01d1\u0001\u0000\u0000\u0000\u01d4\u01d5"+ + "\u0005J\u0000\u0000\u01d5\u01ed\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005"+ + "I\u0000\u0000\u01d7\u01dc\u0003b1\u0000\u01d8\u01d9\u0005(\u0000\u0000"+ + "\u01d9\u01db\u0003b1\u0000\u01da\u01d8\u0001\u0000\u0000\u0000\u01db\u01de"+ + "\u0001\u0000\u0000\u0000\u01dc\u01da\u0001\u0000\u0000\u0000\u01dc\u01dd"+ + "\u0001\u0000\u0000\u0000\u01dd\u01df\u0001\u0000\u0000\u0000\u01de\u01dc"+ + "\u0001\u0000\u0000\u0000\u01df\u01e0\u0005J\u0000\u0000\u01e0\u01ed\u0001"+ + "\u0000\u0000\u0000\u01e1\u01e2\u0005I\u0000\u0000\u01e2\u01e7\u0003j5"+ + "\u0000\u01e3\u01e4\u0005(\u0000\u0000\u01e4\u01e6\u0003j5\u0000\u01e5"+ + "\u01e3\u0001\u0000\u0000\u0000\u01e6\u01e9\u0001\u0000\u0000\u0000\u01e7"+ + "\u01e5\u0001\u0000\u0000\u0000\u01e7\u01e8\u0001\u0000\u0000\u0000\u01e8"+ + "\u01ea\u0001\u0000\u0000\u0000\u01e9\u01e7\u0001\u0000\u0000\u0000\u01ea"+ + "\u01eb\u0005J\u0000\u0000\u01eb\u01ed\u0001\u0000\u0000\u0000\u01ec\u01c2"+ + "\u0001\u0000\u0000\u0000\u01ec\u01c3\u0001\u0000\u0000\u0000\u01ec\u01c6"+ + "\u0001\u0000\u0000\u0000\u01ec\u01c7\u0001\u0000\u0000\u0000\u01ec\u01c8"+ + "\u0001\u0000\u0000\u0000\u01ec\u01c9\u0001\u0000\u0000\u0000\u01ec\u01ca"+ + "\u0001\u0000\u0000\u0000\u01ec\u01cb\u0001\u0000\u0000\u0000\u01ec\u01d6"+ + "\u0001\u0000\u0000\u0000\u01ec\u01e1\u0001\u0000\u0000\u0000\u01edE\u0001"+ + "\u0000\u0000\u0000\u01ee\u01f1\u00056\u0000\u0000\u01ef\u01f1\u0005H\u0000"+ + "\u0000\u01f0\u01ee\u0001\u0000\u0000\u0000\u01f0\u01ef\u0001\u0000\u0000"+ + "\u0000\u01f1G\u0001\u0000\u0000\u0000\u01f2\u01f5\u0003@ \u0000\u01f3"+ + "\u01f5\u0003F#\u0000\u01f4\u01f2\u0001\u0000\u0000\u0000\u01f4\u01f3\u0001"+ + "\u0000\u0000\u0000\u01f5I\u0001\u0000\u0000\u0000\u01f6\u01f7\u0005\t"+ + "\u0000\u0000\u01f7\u01f8\u0005 \u0000\u0000\u01f8K\u0001\u0000\u0000\u0000"+ + "\u01f9\u01fa\u0005\u000e\u0000\u0000\u01fa\u01ff\u0003N\'\u0000\u01fb"+ + "\u01fc\u0005(\u0000\u0000\u01fc\u01fe\u0003N\'\u0000\u01fd\u01fb\u0001"+ + "\u0000\u0000\u0000\u01fe\u0201\u0001\u0000\u0000\u0000\u01ff\u01fd\u0001"+ + "\u0000\u0000\u0000\u01ff\u0200\u0001\u0000\u0000\u0000\u0200M\u0001\u0000"+ + "\u0000\u0000\u0201\u01ff\u0001\u0000\u0000\u0000\u0202\u0204\u0003\n\u0005"+ + "\u0000\u0203\u0205\u0007\u0004\u0000\u0000\u0204\u0203\u0001\u0000\u0000"+ + "\u0000\u0204\u0205\u0001\u0000\u0000\u0000\u0205\u0208\u0001\u0000\u0000"+ + "\u0000\u0206\u0207\u00054\u0000\u0000\u0207\u0209\u0007\u0005\u0000\u0000"+ + "\u0208\u0206\u0001\u0000\u0000\u0000\u0208\u0209\u0001\u0000\u0000\u0000"+ + "\u0209O\u0001\u0000\u0000\u0000\u020a\u020b\u0005\b\u0000\u0000\u020b"+ + "\u020c\u0003>\u001f\u0000\u020cQ\u0001\u0000\u0000\u0000\u020d\u020e\u0005"+ + "\u0002\u0000\u0000\u020e\u020f\u0003>\u001f\u0000\u020fS\u0001\u0000\u0000"+ + "\u0000\u0210\u0211\u0005\u000b\u0000\u0000\u0211\u0216\u0003V+\u0000\u0212"+ + "\u0213\u0005(\u0000\u0000\u0213\u0215\u0003V+\u0000\u0214\u0212\u0001"+ + "\u0000\u0000\u0000\u0215\u0218\u0001\u0000\u0000\u0000\u0216\u0214\u0001"+ + "\u0000\u0000\u0000\u0216\u0217\u0001\u0000\u0000\u0000\u0217U\u0001\u0000"+ + "\u0000\u0000\u0218\u0216\u0001\u0000\u0000\u0000\u0219\u021a\u0003<\u001e"+ + "\u0000\u021a\u021b\u0005\\\u0000\u0000\u021b\u021c\u0003<\u001e\u0000"+ + "\u021cW\u0001\u0000\u0000\u0000\u021d\u021e\u0005\u0001\u0000\u0000\u021e"+ + "\u021f\u0003\u0014\n\u0000\u021f\u0221\u0003j5\u0000\u0220\u0222\u0003"+ + "^/\u0000\u0221\u0220\u0001\u0000\u0000\u0000\u0221\u0222\u0001\u0000\u0000"+ + "\u0000\u0222Y\u0001\u0000\u0000\u0000\u0223\u0224\u0005\u0007\u0000\u0000"+ + "\u0224\u0225\u0003\u0014\n\u0000\u0225\u0226\u0003j5\u0000\u0226[\u0001"+ + "\u0000\u0000\u0000\u0227\u0228\u0005\n\u0000\u0000\u0228\u0229\u0003:"+ + "\u001d\u0000\u0229]\u0001\u0000\u0000\u0000\u022a\u022f\u0003`0\u0000"+ + "\u022b\u022c\u0005(\u0000\u0000\u022c\u022e\u0003`0\u0000\u022d\u022b"+ + "\u0001\u0000\u0000\u0000\u022e\u0231\u0001\u0000\u0000\u0000\u022f\u022d"+ + "\u0001\u0000\u0000\u0000\u022f\u0230\u0001\u0000\u0000\u0000\u0230_\u0001"+ + "\u0000\u0000\u0000\u0231\u022f\u0001\u0000\u0000\u0000\u0232\u0233\u0003"+ + "@ \u0000\u0233\u0234\u0005%\u0000\u0000\u0234\u0235\u0003D\"\u0000\u0235"+ + "a\u0001\u0000\u0000\u0000\u0236\u0237\u0007\u0006\u0000\u0000\u0237c\u0001"+ + "\u0000\u0000\u0000\u0238\u023b\u0003f3\u0000\u0239\u023b\u0003h4\u0000"+ + "\u023a\u0238\u0001\u0000\u0000\u0000\u023a\u0239\u0001\u0000\u0000\u0000"+ + "\u023be\u0001\u0000\u0000\u0000\u023c\u023e\u0007\u0000\u0000\u0000\u023d"+ + "\u023c\u0001\u0000\u0000\u0000\u023d\u023e\u0001\u0000\u0000\u0000\u023e"+ + "\u023f\u0001\u0000\u0000\u0000\u023f\u0240\u0005!\u0000\u0000\u0240g\u0001"+ + "\u0000\u0000\u0000\u0241\u0243\u0007\u0000\u0000\u0000\u0242\u0241\u0001"+ + "\u0000\u0000\u0000\u0242\u0243\u0001\u0000\u0000\u0000\u0243\u0244\u0001"+ + "\u0000\u0000\u0000\u0244\u0245\u0005 \u0000\u0000\u0245i\u0001\u0000\u0000"+ + "\u0000\u0246\u0247\u0005\u001f\u0000\u0000\u0247k\u0001\u0000\u0000\u0000"+ + "\u0248\u0249\u0007\u0007\u0000\u0000\u0249m\u0001\u0000\u0000\u0000\u024a"+ + "\u024b\u0005\u0005\u0000\u0000\u024b\u024c\u0003p8\u0000\u024co\u0001"+ + "\u0000\u0000\u0000\u024d\u024e\u0005I\u0000\u0000\u024e\u024f\u0003\u0002"+ + "\u0001\u0000\u024f\u0250\u0005J\u0000\u0000\u0250q\u0001\u0000\u0000\u0000"+ + "\u0251\u0252\u0005\r\u0000\u0000\u0252\u0253\u0005l\u0000\u0000\u0253"+ + "s\u0001\u0000\u0000\u0000\u0254\u0255\u0005\u0003\u0000\u0000\u0255\u0258"+ + "\u0005b\u0000\u0000\u0256\u0257\u0005`\u0000\u0000\u0257\u0259\u0003<"+ + "\u001e\u0000\u0258\u0256\u0001\u0000\u0000\u0000\u0258\u0259\u0001\u0000"+ + "\u0000\u0000\u0259\u0263\u0001\u0000\u0000\u0000\u025a\u025b\u0005a\u0000"+ + "\u0000\u025b\u0260\u0003v;\u0000\u025c\u025d\u0005(\u0000\u0000\u025d"+ + "\u025f\u0003v;\u0000\u025e\u025c\u0001\u0000\u0000\u0000\u025f\u0262\u0001"+ + "\u0000\u0000\u0000\u0260\u025e\u0001\u0000\u0000\u0000\u0260\u0261\u0001"+ + "\u0000\u0000\u0000\u0261\u0264\u0001\u0000\u0000\u0000\u0262\u0260\u0001"+ + "\u0000\u0000\u0000\u0263\u025a\u0001\u0000\u0000\u0000\u0263\u0264\u0001"+ + "\u0000\u0000\u0000\u0264u\u0001\u0000\u0000\u0000\u0265\u0266\u0003<\u001e"+ + "\u0000\u0266\u0267\u0005%\u0000\u0000\u0267\u0269\u0001\u0000\u0000\u0000"+ + "\u0268\u0265\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000\u0000"+ + "\u0269\u026a\u0001\u0000\u0000\u0000\u026a\u026b\u0003<\u001e\u0000\u026b"+ + "w\u0001\u0000\u0000\u0000\u026c\u026d\u0005\u0015\u0000\u0000\u026d\u026e"+ + "\u0003(\u0014\u0000\u026e\u026f\u0005`\u0000\u0000\u026f\u0270\u0003>"+ + "\u001f\u0000\u0270y\u0001\u0000\u0000\u0000\u0271\u0272\u0005\u0013\u0000"+ + "\u0000\u0272\u0275\u00036\u001b\u0000\u0273\u0274\u0005\"\u0000\u0000"+ + "\u0274\u0276\u0003\"\u0011\u0000\u0275\u0273\u0001\u0000\u0000\u0000\u0275"+ + "\u0276\u0001\u0000\u0000\u0000\u0276{\u0001\u0000\u0000\u0000\u0277\u0278"+ + "\u0007\b\u0000\u0000\u0278\u0279\u0005z\u0000\u0000\u0279\u027a\u0003"+ + "~?\u0000\u027a\u027b\u0003\u0080@\u0000\u027b}\u0001\u0000\u0000\u0000"+ + "\u027c\u027d\u0003(\u0014\u0000\u027d\u007f\u0001\u0000\u0000\u0000\u027e"+ + "\u027f\u0005`\u0000\u0000\u027f\u0284\u0003\u0082A\u0000\u0280\u0281\u0005"+ + "(\u0000\u0000\u0281\u0283\u0003\u0082A\u0000\u0282\u0280\u0001\u0000\u0000"+ + "\u0000\u0283\u0286\u0001\u0000\u0000\u0000\u0284\u0282\u0001\u0000\u0000"+ + "\u0000\u0284\u0285\u0001\u0000\u0000\u0000\u0285\u0081\u0001\u0000\u0000"+ + "\u0000\u0286\u0284\u0001\u0000\u0000\u0000\u0287\u0288\u0003\u0010\b\u0000"+ + "\u0288\u0083\u0001\u0000\u0000\u0000\u0289\u028a\u0005\u0012\u0000\u0000"+ + "\u028a\u028d\u0003:\u001d\u0000\u028b\u028c\u0005`\u0000\u0000\u028c\u028e"+ + "\u0003:\u001d\u0000\u028d\u028b\u0001\u0000\u0000\u0000\u028d\u028e\u0001"+ + "\u0000\u0000\u0000\u028e\u0294\u0001\u0000\u0000\u0000\u028f\u0290\u0005"+ + "\\\u0000\u0000\u0290\u0291\u0003:\u001d\u0000\u0291\u0292\u0005(\u0000"+ + "\u0000\u0292\u0293\u0003:\u001d\u0000\u0293\u0295\u0001\u0000\u0000\u0000"+ + "\u0294\u028f\u0001\u0000\u0000\u0000\u0294\u0295\u0001\u0000\u0000\u0000"+ + "\u0295\u0085\u0001\u0000\u0000\u0000\u0296\u0297\u0005\u0014\u0000\u0000"+ + "\u0297\u0298\u0003>\u001f\u0000\u0298\u0087\u0001\u0000\u0000\u0000?\u0093"+ + "\u009c\u00b3\u00bf\u00c8\u00d0\u00d5\u00dd\u00df\u00e4\u00eb\u00f0\u00f5"+ + "\u00ff\u0105\u010d\u010f\u011a\u0121\u012c\u0131\u0133\u013f\u0152\u0158"+ + "\u0162\u0166\u016b\u0179\u0182\u0186\u018a\u0191\u0195\u019c\u01a2\u01a9"+ + "\u01b1\u01b9\u01c0\u01d1\u01dc\u01e7\u01ec\u01f0\u01f4\u01ff\u0204\u0208"+ + "\u0216\u0221\u022f\u023a\u023d\u0242\u0258\u0260\u0263\u0268\u0275\u0284"+ + "\u028d\u0294"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index ee2c97e1e381..6a2ab4a35384 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -1112,6 +1112,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitChangePointCommand(EsqlBaseParser.ChangePointCommandContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterInsistCommand(EsqlBaseParser.InsistCommandContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitInsistCommand(EsqlBaseParser.InsistCommandContext ctx) { } /** * {@inheritDoc} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index c1e3a4b32795..2a0ba1f1f49a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -657,4 +657,11 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitChangePointCommand(EsqlBaseParser.ChangePointCommandContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitInsistCommand(EsqlBaseParser.InsistCommandContext ctx) { return visitChildren(ctx); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index c4f72ae1444d..9a9ed80f2613 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -991,4 +991,14 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitChangePointCommand(EsqlBaseParser.ChangePointCommandContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#insistCommand}. + * @param ctx the parse tree + */ + void enterInsistCommand(EsqlBaseParser.InsistCommandContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#insistCommand}. + * @param ctx the parse tree + */ + void exitInsistCommand(EsqlBaseParser.InsistCommandContext ctx); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index 4dc2670662f5..0988fea448e6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -597,4 +597,10 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitChangePointCommand(EsqlBaseParser.ChangePointCommandContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#insistCommand}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitInsistCommand(EsqlBaseParser.InsistCommandContext ctx); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 4645a15bc196..775b510868e3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Explain; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Grok; import org.elasticsearch.xpack.esql.plan.logical.InlineStats; +import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -287,6 +288,22 @@ public class LogicalPlanBuilder extends ExpressionBuilder { ); } + @Override + public PlanFactory visitInsistCommand(EsqlBaseParser.InsistCommandContext ctx) { + var source = source(ctx); + List fields = visitQualifiedNamePatterns(ctx.qualifiedNamePatterns(), ne -> { + if (ne instanceof UnresolvedStar || ne instanceof UnresolvedNamePattern) { + Source neSource = ne.source(); + throw new ParsingException(neSource, "INSIST doesn't support wildcards, found [{}]", neSource.text()); + } + }); + return input -> new Insist( + source, + input, + fields.stream().map(ne -> (Attribute) new UnresolvedAttribute(ne.source(), ne.name())).toList() + ); + } + @Override public PlanFactory visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { final Stats stats = stats(source(ctx), ctx.grouping, ctx.stats); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/PlanWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/PlanWritables.java index ab2b6b6e0585..ebe8e04e7c9a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/PlanWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/PlanWritables.java @@ -70,8 +70,8 @@ public class PlanWritables { Eval.ENTRY, Filter.ENTRY, Grok.ENTRY, - InlineStats.ENTRY, InlineJoin.ENTRY, + InlineStats.ENTRY, Join.ENTRY, LocalRelation.ENTRY, Limit.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java index 448085df1e83..e3c562d3d630 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java @@ -226,4 +226,8 @@ public class EsRelation extends LeafPlan { throw new IllegalStateException("not ready to support index mode [" + indexMode + "]"); } } + + public EsRelation withAttributes(List newAttributes) { + return new EsRelation(source(), indexPattern, indexMode, indexNameWithModes, newAttributes); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Insist.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Insist.java new file mode 100644 index 000000000000..78d342ca7e3a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Insist.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.plan.logical; + +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.NamedExpressions; +import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class Insist extends UnaryPlan implements SurrogateLogicalPlan { + private final List insistedAttributes; + private @Nullable List lazyOutput = null; + + public Insist(Source source, LogicalPlan child, List insistedAttributes) { + super(source, child); + this.insistedAttributes = insistedAttributes; + } + + @Override + public List output() { + if (lazyOutput == null) { + lazyOutput = NamedExpressions.mergeOutputAttributes(insistedAttributes, child().output()); + } + return lazyOutput; + } + + public List insistedAttributes() { + return insistedAttributes; + } + + @Override + public Insist replaceChild(LogicalPlan newChild) { + return new Insist(source(), newChild, insistedAttributes); + } + + @Override + public boolean expressionsResolved() { + // Like EsqlProject, we allow unsupported attributes to flow through the engine. + return insistedAttributes().stream().allMatch(a -> a.resolved() || a instanceof UnsupportedAttribute); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create( + this, + (source, insistedAttributes1, child) -> new Insist(source, child, insistedAttributes1), + insistedAttributes, + child() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException("doesn't escape the coordinator node"); + } + + @Override + public String getWriteableName() { + throw new UnsupportedOperationException("doesn't escape the coordinator node"); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), Objects.hashCode(insistedAttributes)); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && Objects.equals(((Insist) obj).insistedAttributes, insistedAttributes); + } + + @Override + public LogicalPlan surrogate() { + return new Project(source(), child(), output()); + } + + public Insist withAttributes(List attributes) { + return new Insist(source(), child(), attributes); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java index 42cf3528f2ae..22ee88f8119e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java @@ -121,8 +121,9 @@ public class EnrichExec extends UnaryExec implements EstimatesRowSize { out.writeMap(concreteIndices(), StreamOutput::writeString, StreamOutput::writeString); } else { if (concreteIndices().keySet().equals(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY))) { - String concreteIndex = concreteIndices().get(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); - new EsIndex(concreteIndex, Map.of(), Map.of(concreteIndex, IndexMode.STANDARD)).writeTo(out); + String enrichIndex = concreteIndices().get(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + EsIndex esIndex = new EsIndex(enrichIndex, Map.of(), Map.of(enrichIndex, IndexMode.STANDARD)); + esIndex.writeTo(out); } else { throw new IllegalStateException("expected a single concrete enrich index; got " + concreteIndices()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 112f101ad842..10c380f2db56 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -26,11 +26,13 @@ import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.OrdinalsGroupingOperator; import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; @@ -49,7 +51,9 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.KeywordEsField; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; @@ -62,6 +66,7 @@ import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.PhysicalOperat import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -110,28 +115,29 @@ public class EsPhysicalOperationProviders extends AbstractPhysicalOperationProvi int docChannel = source.layout.get(sourceAttr.id()).channel(); for (Attribute attr : fieldExtractExec.attributesToExtract()) { layout.append(attr); - var unionTypes = findUnionTypes(attr); DataType dataType = attr.dataType(); MappedFieldType.FieldExtractPreference fieldExtractPreference = fieldExtractExec.fieldExtractPreference(attr); ElementType elementType = PlannerUtils.toElementType(dataType, fieldExtractPreference); - // Do not use the field attribute name, this can deviate from the field name for union types. - String fieldName = attr instanceof FieldAttribute fa ? fa.fieldName() : attr.name(); - boolean isUnsupported = dataType == DataType.UNSUPPORTED; - IntFunction loader = s -> getBlockLoaderFor(s, fieldName, isUnsupported, fieldExtractPreference, unionTypes); - fields.add(new ValuesSourceReaderOperator.FieldInfo(fieldName, elementType, loader)); + IntFunction loader = s -> getBlockLoaderFor(s, attr, fieldExtractPreference); + fields.add(new ValuesSourceReaderOperator.FieldInfo(getFieldName(attr), elementType, loader)); } return source.with(new ValuesSourceReaderOperator.Factory(fields, readers, docChannel), layout.build()); } - private BlockLoader getBlockLoaderFor( - int shardId, - String fieldName, - boolean isUnsupported, - MappedFieldType.FieldExtractPreference fieldExtractPreference, - MultiTypeEsField unionTypes - ) { + private static String getFieldName(Attribute attr) { + // Do not use the field attribute name, this can deviate from the field name for union types. + return attr instanceof FieldAttribute fa ? fa.fieldName() : attr.name(); + } + + private BlockLoader getBlockLoaderFor(int shardId, Attribute attr, MappedFieldType.FieldExtractPreference fieldExtractPreference) { DefaultShardContext shardContext = (DefaultShardContext) shardContexts.get(shardId); - BlockLoader blockLoader = shardContext.blockLoader(fieldName, isUnsupported, fieldExtractPreference); + if (attr instanceof FieldAttribute fa && fa.field() instanceof PotentiallyUnmappedKeywordEsField kf) { + shardContext = new DefaultShardContextForUnmappedField(shardContext, kf); + } + + boolean isUnsupported = attr.dataType() == DataType.UNSUPPORTED; + BlockLoader blockLoader = shardContext.blockLoader(getFieldName(attr), isUnsupported, fieldExtractPreference); + MultiTypeEsField unionTypes = findUnionTypes(attr); if (unionTypes != null) { String indexName = shardContext.ctx.index().getName(); Expression conversion = unionTypes.getConversionExpressionForIndex(indexName); @@ -142,7 +148,25 @@ public class EsPhysicalOperationProviders extends AbstractPhysicalOperationProvi return blockLoader; } - private MultiTypeEsField findUnionTypes(Attribute attr) { + /** A hack to pretend an unmapped field still exists. */ + private static class DefaultShardContextForUnmappedField extends DefaultShardContext { + private final KeywordEsField unmappedEsField; + + DefaultShardContextForUnmappedField(DefaultShardContext ctx, PotentiallyUnmappedKeywordEsField unmappedEsField) { + super(ctx.index, ctx.ctx, ctx.aliasFilter); + this.unmappedEsField = unmappedEsField; + } + + @Override + protected @Nullable MappedFieldType fieldType(String name) { + var superResult = super.fieldType(name); + return superResult == null && name.equals(unmappedEsField.getName()) + ? new KeywordFieldMapper.KeywordFieldType(name, false /* isIndexed */, false /* hasDocValues */, Map.of() /* meta */) + : superResult; + } + } + + private static @Nullable MultiTypeEsField findUnionTypes(Attribute attr) { if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField multiTypeEsField) { return multiTypeEsField; } @@ -237,12 +261,8 @@ public class EsPhysicalOperationProviders extends AbstractPhysicalOperationProvi .toList(); // The grouping-by values are ready, let's group on them directly. // Costin: why are they ready and not already exposed in the layout? - boolean isUnsupported = attrSource.dataType() == DataType.UNSUPPORTED; - var unionTypes = findUnionTypes(attrSource); - // Do not use the field attribute name, this can deviate from the field name for union types. - String fieldName = attrSource instanceof FieldAttribute fa ? fa.fieldName() : attrSource.name(); return new OrdinalsGroupingOperator.OrdinalsGroupingOperatorFactory( - shardIdx -> getBlockLoaderFor(shardIdx, fieldName, isUnsupported, NONE, unionTypes), + shardIdx -> getBlockLoaderFor(shardIdx, attrSource, NONE), vsShardContexts, groupElementType, docChannel, @@ -315,7 +335,7 @@ public class EsPhysicalOperationProviders extends AbstractPhysicalOperationProvi if (asUnsupportedSource) { return BlockLoader.CONSTANT_NULLS; } - MappedFieldType fieldType = ctx.getFieldType(name); + MappedFieldType fieldType = fieldType(name); if (fieldType == null) { // the field does not exist in this context return BlockLoader.CONSTANT_NULLS; @@ -363,6 +383,10 @@ public class EsPhysicalOperationProviders extends AbstractPhysicalOperationProvi return loader; } + + protected @Nullable MappedFieldType fieldType(String name) { + return ctx.getFieldType(name); + } } private static class TypeConvertingBlockLoader implements BlockLoader { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 3e59b5218e7f..d0d35374242e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -92,7 +92,8 @@ public class IndexResolver { } // public for testing only - public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) { + public static IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) { + var numberOfIndices = fieldCapsResponse.getIndexResponses().size(); assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker if (fieldCapsResponse.getIndexResponses().isEmpty()) { return IndexResolution.notFound(indexPattern); @@ -105,6 +106,7 @@ public class IndexResolver { String[] names = fieldsCaps.keySet().toArray(new String[0]); Arrays.sort(names); Map rootFields = new HashMap<>(); + Set partiallyUnmappedFields = new HashSet<>(); for (String name : names) { Map fields = rootFields; String fullName = name; @@ -129,8 +131,9 @@ public class IndexResolver { } // TODO we're careful to make isAlias match IndexResolver - but do we use it? + List fcs = fieldsCaps.get(fullName); EsField field = firstUnsupportedParent == null - ? createField(fieldCapsResponse, name, fullName, fieldsCaps.get(fullName), isAlias) + ? createField(fieldCapsResponse, name, fullName, fcs, isAlias) : new UnsupportedEsField( fullName, firstUnsupportedParent.getOriginalType(), @@ -138,6 +141,11 @@ public class IndexResolver { new HashMap<>() ); fields.put(name, field); + + var isPartiallyUnmapped = fcs.size() < numberOfIndices; + if (isPartiallyUnmapped) { + partiallyUnmappedFields.add(fullName); + } } Map unavailableRemotes = EsqlCCSUtils.determineUnavailableRemoteClusters( @@ -153,11 +161,9 @@ public class IndexResolver { for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { allEmpty &= ir.get().isEmpty(); } - if (allEmpty) { - // If all the mappings are empty we return an empty set of resolved indices to line up with QL - return IndexResolution.valid(new EsIndex(indexPattern, rootFields, Map.of()), concreteIndices.keySet(), unavailableRemotes); - } - return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices), concreteIndices.keySet(), unavailableRemotes); + // If all the mappings are empty we return an empty set of resolved indices to line up with QL + var index = new EsIndex(indexPattern, rootFields, allEmpty ? Map.of() : concreteIndices, partiallyUnmappedFields); + return IndexResolution.valid(index, concreteIndices.keySet(), unavailableRemotes); } private static Map> collectFieldCaps(FieldCapabilitiesResponse fieldCapsResponse) { @@ -179,7 +185,7 @@ public class IndexResolver { return fieldsCaps; } - private EsField createField( + private static EsField createField( FieldCapabilitiesResponse fieldCapsResponse, String name, String fullName, @@ -227,12 +233,12 @@ public class IndexResolver { return new EsField(name, type, new HashMap<>(), aggregatable, isAlias); } - private UnsupportedEsField unsupported(String name, IndexFieldCapabilities fc) { + private static UnsupportedEsField unsupported(String name, IndexFieldCapabilities fc) { String originalType = fc.metricType() == TimeSeriesParams.MetricType.COUNTER ? "counter" : fc.type(); return new UnsupportedEsField(name, originalType); } - private EsField conflictingTypes(String name, String fullName, FieldCapabilitiesResponse fieldCapsResponse) { + private static EsField conflictingTypes(String name, String fullName, FieldCapabilitiesResponse fieldCapsResponse) { Map> typesToIndices = new TreeMap<>(); for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { IndexFieldCapabilities fc = ir.get().get(fullName); @@ -247,7 +253,7 @@ public class IndexResolver { return new InvalidMappedField(name, typesToIndices); } - private EsField conflictingMetricTypes(String name, String fullName, FieldCapabilitiesResponse fieldCapsResponse) { + private static EsField conflictingMetricTypes(String name, String fullName, FieldCapabilitiesResponse fieldCapsResponse) { TreeSet indices = new TreeSet<>(); for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { IndexFieldCapabilities fc = ir.get().get(fullName); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 1e21aa3774af..4faad86c0ba0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -279,6 +279,10 @@ public class CsvTests extends ESTestCase { "CSV tests cannot correctly handle the field caps change", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.SEMANTIC_TEXT_FIELD_CAPS.capabilityName()) ); + assumeFalse( + "CSV tests cannot currently handle the _source field mapping directives", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.SOURCE_FIELD_MAPPING.capabilityName()) + ); if (Build.current().isSnapshot()) { assertThat( "Capability is not included in the enabled list capabilities on a snapshot build. Spelling mistake?", @@ -355,7 +359,10 @@ public class CsvTests extends ESTestCase { .stream() .map(ds -> new MappingPerIndex(ds.indexName(), createMappingForIndex(ds))) .toList(); - return IndexResolution.valid(new EsIndex(datasets.indexPattern(), mergeMappings(mappings), indexModes)); + var mergedMappings = mergeMappings(mappings); + return IndexResolution.valid( + new EsIndex(datasets.indexPattern(), mergedMappings.mapping, indexModes, mergedMappings.partiallyUnmappedFields) + ); } private static Map createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) { @@ -376,7 +383,10 @@ public class CsvTests extends ESTestCase { record MappingPerIndex(String index, Map mapping) {} - private static Map mergeMappings(List mappingsPerIndex) { + record MergedResult(Map mapping, Set partiallyUnmappedFields) {} + + private static MergedResult mergeMappings(List mappingsPerIndex) { + int numberOfIndices = mappingsPerIndex.size(); Map> columnNamesToFieldByIndices = new HashMap<>(); for (var mappingPerIndex : mappingsPerIndex) { for (var entry : mappingPerIndex.mapping().entrySet()) { @@ -386,9 +396,15 @@ public class CsvTests extends ESTestCase { } } - return columnNamesToFieldByIndices.entrySet() + var partiallyUnmappedFields = columnNamesToFieldByIndices.entrySet() + .stream() + .filter(e -> e.getValue().size() < numberOfIndices) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + var mappings = columnNamesToFieldByIndices.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> mergeFields(e.getKey(), e.getValue()))); + return new MergedResult(mappings, partiallyUnmappedFields); } private static EsField mergeFields(String index, Map columnNameToField) { @@ -494,7 +510,8 @@ public class CsvTests extends ESTestCase { var indexPages = new ArrayList(); for (CsvTestsDataLoader.TestDataset dataset : datasets.datasets()) { var testData = loadPageFromCsv(CsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping()); - indexPages.add(new TestPhysicalOperationProviders.IndexPage(dataset.indexName(), testData.v1(), testData.v2())); + Set mappedFields = loadMapping(dataset.mappingFileName()).keySet(); + indexPages.add(new TestPhysicalOperationProviders.IndexPage(dataset.indexName(), testData.v1(), testData.v2(), mappedFields)); } return TestPhysicalOperationProviders.create(foldCtx, indexPages); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 187c83cd5f24..b575ddf4ce92 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.Build; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -29,9 +30,12 @@ import org.elasticsearch.xpack.esql.core.expression.MapExpression; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; +import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.aggregate.Max; import org.elasticsearch.xpack.esql.expression.function.aggregate.Min; @@ -47,6 +51,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; +import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; @@ -85,10 +90,12 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexR import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesRegex; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; @@ -2587,6 +2594,176 @@ public class AnalyzerTests extends ESTestCase { assertEquals(DataType.DOUBLE, ee.dataType()); } + public void testResolveInsist_fieldExists_insistedOutputContainsNoUnmappedFields() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + LogicalPlan plan = analyze("FROM test | INSIST_🐔 emp_no"); + + Attribute last = plan.output().getLast(); + assertThat(last.name(), is("emp_no")); + assertThat(last.dataType(), is(DataType.INTEGER)); + assertThat( + plan.output() + .stream() + .filter(a -> a instanceof FieldAttribute fa && fa.field() instanceof PotentiallyUnmappedKeywordEsField) + .toList(), + is(empty()) + ); + } + + public void testInsist_afterRowThrowsException() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + VerificationException e = expectThrows( + VerificationException.class, + () -> analyze("ROW x = 1 | INSIST_🐔 x", analyzer(TEST_VERIFIER)) + ); + assertThat(e.getMessage(), containsString("[insist] can only be used after [from] or [insist] commands, but was [ROW x = 1]")); + } + + public void testResolveInsist_fieldDoesNotExist_createsUnmappedField() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + LogicalPlan plan = analyze("FROM test | INSIST_🐔 foo"); + + var limit = as(plan, Limit.class); + var insist = as(limit.child(), Insist.class); + assertThat(insist.output(), hasSize(analyze("FROM test").output().size() + 1)); + var expectedAttribute = new FieldAttribute(Source.EMPTY, "foo", new PotentiallyUnmappedKeywordEsField("foo")); + assertThat(insist.insistedAttributes(), is(List.of(expectedAttribute))); + assertThat(insist.output().getLast(), is(expectedAttribute)); + } + + public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordType_createsUnmappedField() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + IndexResolution resolution = IndexResolver.mergedMappings( + "foo, bar", + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("keyword")), + fieldCapabilitiesIndexResponse("bar", Map.of()) + ), + List.of() + ) + ); + + String query = "FROM foo, bar | INSIST_🐔 message"; + var plan = analyze(query, analyzer(resolution, TEST_VERIFIER, configuration(query))); + var limit = as(plan, Limit.class); + var insist = as(limit.child(), Insist.class); + var attribute = (FieldAttribute) EsqlTestUtils.singleValue(insist.output()); + assertThat(attribute.name(), is("message")); + assertThat(attribute.field(), is(new PotentiallyUnmappedKeywordEsField("message"))); + } + + public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeywordAndMissingCast_createsAnInvalidMappedField() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + IndexResolution resolution = IndexResolver.mergedMappings( + "foo, bar", + new FieldCapabilitiesResponse( + List.of(fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), fieldCapabilitiesIndexResponse("bar", Map.of())), + List.of() + ) + ); + var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); + var limit = as(plan, Limit.class); + var insist = as(limit.child(), Insist.class); + var attribute = (UnsupportedAttribute) EsqlTestUtils.singleValue(insist.output()); + assertThat(attribute.name(), is("message")); + + String expected = "Cannot use field [message] due to ambiguities being mapped as [2] incompatible types: " + + "[keyword] enforced by INSIST command, and [long] in index mappings"; + assertThat(attribute.unresolvedMessage(), is(expected)); + } + + public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyword_createsAnInvalidMappedField() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + IndexResolution resolution = IndexResolver.mergedMappings( + "foo, bar", + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", Map.of()) + ), + List.of() + ) + ); + var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); + var limit = as(plan, Limit.class); + var insist = as(limit.child(), Insist.class); + var attr = (UnsupportedAttribute) EsqlTestUtils.singleValue(insist.output()); + + String expected = "Cannot use field [message] due to ambiguities being mapped as [3] incompatible types: " + + "[keyword] enforced by INSIST command, [datetime] in [bar], [long] in [foo]"; + assertThat(attr.unresolvedMessage(), is(expected)); + } + + public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithKeyword_createsAnInvalidMappedField() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + IndexResolution resolution = IndexResolver.mergedMappings( + "foo, bar", + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", messageResponseMap("keyword")), + fieldCapabilitiesIndexResponse("qux", Map.of()) + ), + List.of() + ) + ); + var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); + var limit = as(plan, Limit.class); + var insist = as(limit.child(), Insist.class); + var attr = (UnsupportedAttribute) EsqlTestUtils.singleValue(insist.output()); + + String expected = "Cannot use field [message] due to ambiguities being mapped as [3] incompatible types: " + + "[datetime] in [bar], [keyword] enforced by INSIST command and in [bazz], [long] in [foo]"; + assertThat(attr.unresolvedMessage(), is(expected)); + } + + public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCast_castsAreNotSupported() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + IndexResolution resolution = IndexResolver.mergedMappings( + "foo, bar", + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", Map.of()) + ), + List.of() + ) + ); + VerificationException e = expectThrows( + VerificationException.class, + () -> analyze("FROM multi_index | INSIST_🐔 message | EVAL message = message :: keyword", analyzer(resolution, TEST_VERIFIER)) + ); + // This isn't the most informative error, but it'll do for now. + assertThat( + e.getMessage(), + containsString("EVAL does not support type [unsupported] as the return data type of expression [message]") + ); + } + + // TODO There's too much boilerplate involved here! We need a better way of creating FieldCapabilitiesResponses from a mapping or index. + private static FieldCapabilitiesIndexResponse fieldCapabilitiesIndexResponse( + String indexName, + Map fields + ) { + return new FieldCapabilitiesIndexResponse(indexName, indexName, fields, false, IndexMode.STANDARD); + } + + private static Map messageResponseMap(String date) { + return Map.of("message", new IndexFieldCapabilities("message", date, false, true, true, false, null, null)); + } + private void verifyUnsupported(String query, String errorMessage) { verifyUnsupported(query, errorMessage, "mapping-multi-field-variation.json"); } @@ -2638,7 +2815,7 @@ public class AnalyzerTests extends ESTestCase { new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true, IndexMode.STANDARD) ); FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); - IndexResolution resolution = new IndexResolver(null).mergedMappings("test*", caps); + IndexResolution resolution = IndexResolver.mergedMappings("test*", caps); var analyzer = analyzer(resolution, TEST_VERIFIER, configuration(query)); return analyze(query, analyzer); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 8be008be7a81..71d36ce0ffcf 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -2155,6 +2155,15 @@ public class VerifierTests extends ESTestCase { ); } + public void testInsistNotOnTopOfFrom() { + assumeTrue("requires snapshot builds", Build.current().isSnapshot()); + + assertThat( + error("FROM test | EVAL foo = 42 | INSIST_🐔 bar"), + containsString("1:29: [insist] can only be used after [from] or [insist] commands, but was [EVAL foo = 42]") + ); + } + private void query(String query) { query(query, defaultAnalyzer); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 6fb1cb9ca14e..90c08001f4cf 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.operator.compariso import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.Order; @@ -135,6 +136,7 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; @@ -157,6 +159,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource; import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.singleValue; import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS; @@ -192,23 +195,24 @@ import static org.hamcrest.Matchers.startsWith; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") public class LogicalPlanOptimizerTests extends ESTestCase { - private static EsqlParser parser; - private static Analyzer analyzer; private static LogicalOptimizerContext logicalOptimizerCtx; private static LogicalPlanOptimizer logicalOptimizer; + private static Map mapping; + private static Analyzer analyzer; private static Map mappingAirports; - private static Map mappingTypes; private static Analyzer analyzerAirports; + private static Map mappingTypes; private static Analyzer analyzerTypes; private static Map mappingExtra; private static Analyzer analyzerExtra; - private static EnrichResolution enrichResolution; - private static final LiteralsOnTheRight LITERALS_ON_THE_RIGHT = new LiteralsOnTheRight(); - private static Map metricMapping; private static Analyzer metricsAnalyzer; + private static Analyzer multiIndexAnalyzer; + + private static EnrichResolution enrichResolution; + private static final LiteralsOnTheRight LITERALS_ON_THE_RIGHT = new LiteralsOnTheRight(); private static class SubstitutionOnlyOptimizer extends LogicalPlanOptimizer { static SubstitutionOnlyOptimizer INSTANCE = new SubstitutionOnlyOptimizer(unboundLogicalOptimizerContext()); @@ -279,6 +283,21 @@ public class LogicalPlanOptimizerTests extends ESTestCase { new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), metricsIndex, enrichResolution), TEST_VERIFIER ); + + var multiIndexMapping = loadMapping("mapping-basic.json"); + multiIndexMapping.put("partial_type_keyword", new EsField("partial_type_keyword", KEYWORD, emptyMap(), true)); + var multiIndex = IndexResolution.valid( + new EsIndex( + "multi_index", + multiIndexMapping, + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Set.of("partial_type_keyword") + ) + ); + multiIndexAnalyzer = new Analyzer( + new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), multiIndex, enrichResolution), + TEST_VERIFIER + ); } public void testEmptyProjections() { @@ -2900,6 +2919,45 @@ public class LogicalPlanOptimizerTests extends ESTestCase { ); } + public void testInsist_fieldDoesNotExist_createsUnmappedFieldInRelation() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + LogicalPlan plan = optimizedPlan("FROM test | INSIST_🐔 foo"); + + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + var relation = as(limit.child(), EsRelation.class); + assertPartialTypeKeyword(relation, "foo"); + } + + public void testInsist_multiIndexFieldPartiallyExistsAndIsKeyword_castsAreNotSupported() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + var plan = planMultiIndex("FROM multi_index | INSIST_🐔 partial_type_keyword"); + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + var relation = as(limit.child(), EsRelation.class); + + assertPartialTypeKeyword(relation, "partial_type_keyword"); + } + + public void testInsist_multipleInsistClauses_insistsAreFolded() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + var plan = planMultiIndex("FROM multi_index | INSIST_🐔 partial_type_keyword | INSIST_🐔 foo"); + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + var relation = as(limit.child(), EsRelation.class); + + assertPartialTypeKeyword(relation, "partial_type_keyword"); + assertPartialTypeKeyword(relation, "foo"); + } + + private static void assertPartialTypeKeyword(EsRelation relation, String name) { + var attribute = (FieldAttribute) singleValue(relation.output().stream().filter(attr -> attr.name().equals(name)).toList()); + assertThat(attribute.field(), instanceOf(PotentiallyUnmappedKeywordEsField.class)); + } + public void testSimplifyLikeNoWildcard() { LogicalPlan plan = optimizedPlan(""" from test @@ -5888,6 +5946,10 @@ public class LogicalPlanOptimizerTests extends ESTestCase { return logicalOptimizer.optimize(analyzerTypes.analyze(parser.createStatement(query))); } + private LogicalPlan planMultiIndex(String query) { + return logicalOptimizer.optimize(multiIndexAnalyzer.analyze(parser.createStatement(query))); + } + private EsqlBinaryComparison extractPlannedBinaryComparison(String expression) { LogicalPlan plan = planTypes("FROM types | WHERE " + expression); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 47e1616060bd..4606949bb971 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -2982,4 +2982,10 @@ public class StatementParserTests extends AbstractStatementParserTests { ); } } + + public void testInvalidInsistAsterisk() { + assumeTrue("requires snapshot build", Build.current().isSnapshot()); + expectError("FROM text | EVAL x = 4 | INSIST_🐔 *", "INSIST doesn't support wildcards, found [*]"); + expectError("FROM text | EVAL x = 4 | INSIST_🐔 foo*", "INSIST doesn't support wildcards, found [foo*]"); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 1009eaea9b54..cf2c5735310a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -51,6 +51,7 @@ import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; @@ -64,8 +65,9 @@ import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.OptionalInt; +import java.util.Optional; import java.util.Random; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -91,9 +93,10 @@ public class TestPhysicalOperationProviders extends AbstractPhysicalOperationPro return new TestPhysicalOperationProviders(foldContext, indexPages, createAnalysisRegistry()); } - public record IndexPage(String index, Page page, List columnNames) { - OptionalInt columnIndex(String columnName) { - return IntStream.range(0, columnNames.size()).filter(i -> columnNames.get(i).equals(columnName)).findFirst(); + public record IndexPage(String index, Page page, List columnNames, Set mappedFields) { + Optional columnIndex(String columnName) { + var result = IntStream.range(0, columnNames.size()).filter(i -> columnNames.get(i).equals(columnName)).findFirst(); + return result.isPresent() ? Optional.of(result.getAsInt()) : Optional.empty(); } } @@ -264,43 +267,72 @@ public class TestPhysicalOperationProviders extends AbstractPhysicalOperationPro private Block getBlock(DocBlock docBlock, Attribute attribute, FieldExtractPreference extractPreference) { if (attribute instanceof UnsupportedAttribute) { - return docBlock.blockFactory().newConstantNullBlock(docBlock.getPositionCount()); + return getNullsBlock(docBlock); } - return extractBlockForColumn( - docBlock, - attribute.dataType(), - extractPreference, - attribute instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField multiTypeEsField - ? (indexDoc, blockCopier) -> getBlockForMultiType(indexDoc, multiTypeEsField, blockCopier) - : (indexDoc, blockCopier) -> extractBlockForSingleDoc(indexDoc, attribute.name(), blockCopier) - ); + BiFunction blockExtraction = getBlockExtraction(attribute); + return extractBlockForColumn(docBlock, attribute.dataType(), extractPreference, blockExtraction); + } + + private BiFunction getBlockExtraction(Attribute attribute) { + if (attribute instanceof FieldAttribute fa) { + if (fa.field() instanceof MultiTypeEsField m) { + return (doc, copier) -> getBlockForMultiType(doc, m, copier); + + } + if (fa.field() instanceof PotentiallyUnmappedKeywordEsField k) { + return (doc, copier) -> switch (extractBlockForSingleDoc(doc, k.getName(), copier)) { + case BlockResultMissing unused -> getNullsBlock(doc); + case BlockResultSuccess success -> success.block; + }; + } + } + return (indexDoc, blockCopier) -> switch (extractBlockForSingleDoc(indexDoc, attribute.name(), blockCopier)) { + case BlockResultMissing missing -> throw new EsqlIllegalArgumentException( + "Cannot find column named [{}] in {}", + missing.columnName, + missing.columnNames + ); + case BlockResultSuccess success -> success.block; + }; } private Block getBlockForMultiType(DocBlock indexDoc, MultiTypeEsField multiTypeEsField, TestBlockCopier blockCopier) { - var indexId = indexDoc.asVector().shards().getInt(0); - var indexPage = indexPages.get(indexId); - var conversion = (AbstractConvertFunction) multiTypeEsField.getConversionExpressionForIndex(indexPage.index); - Supplier nulls = () -> indexDoc.blockFactory().newConstantNullBlock(indexDoc.getPositionCount()); + var conversion = (AbstractConvertFunction) multiTypeEsField.getConversionExpressionForIndex(getIndexPage(indexDoc).index); if (conversion == null) { - return nulls.get(); + return getNullsBlock(indexDoc); } - var field = (FieldAttribute) conversion.field(); - return indexPage.columnIndex(field.fieldName()).isEmpty() - ? nulls.get() - : TypeConverter.fromConvertFunction(conversion).convert(extractBlockForSingleDoc(indexDoc, field.fieldName(), blockCopier)); + return switch (extractBlockForSingleDoc(indexDoc, ((FieldAttribute) conversion.field()).fieldName(), blockCopier)) { + case BlockResultMissing unused -> getNullsBlock(indexDoc); + case BlockResultSuccess success -> TypeConverter.fromConvertFunction(conversion).convert(success.block); + }; } - private Block extractBlockForSingleDoc(DocBlock docBlock, String columnName, TestBlockCopier blockCopier) { + private IndexPage getIndexPage(DocBlock indexDoc) { + return indexPages.get(indexDoc.asVector().shards().getInt(0)); + } + + private static Block getNullsBlock(DocBlock indexDoc) { + return indexDoc.blockFactory().newConstantNullBlock(indexDoc.getPositionCount()); + } + + private sealed interface BlockResult {} + + private record BlockResultSuccess(Block block) implements BlockResult {} + + private record BlockResultMissing(String columnName, List columnNames) implements BlockResult {} + + private BlockResult extractBlockForSingleDoc(DocBlock docBlock, String columnName, TestBlockCopier blockCopier) { var indexId = docBlock.asVector().shards().getInt(0); var indexPage = indexPages.get(indexId); if (MetadataAttribute.INDEX.equals(columnName)) { - return docBlock.blockFactory() - .newConstantBytesRefBlockWith(new BytesRef(indexPage.index), blockCopier.docIndices.getPositionCount()); + return new BlockResultSuccess( + docBlock.blockFactory() + .newConstantBytesRefBlockWith(new BytesRef(indexPage.index), blockCopier.docIndices.getPositionCount()) + ); } - int columnIndex = indexPage.columnIndex(columnName) - .orElseThrow(() -> new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", columnName, indexPage.columnNames)); - var originalData = indexPage.page.getBlock(columnIndex); - return blockCopier.copyBlock(originalData); + return indexPage.columnIndex(columnName) + .map(columnIndex -> new BlockResultSuccess(blockCopier.copyBlock(indexPage.page.getBlock(columnIndex)))) + .orElseGet(() -> new BlockResultMissing(columnName, indexPage.columnNames())); } private static void foreachIndexDoc(DocBlock docBlock, Consumer indexDocConsumer) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index e7ea479d199d..6b797710bec3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1575,6 +1575,84 @@ public class IndexResolverFieldNamesTests extends ESTestCase { ); } + public void testInsist_fieldIsMappedToNonKeywordSingleIndex() { + assumeTrue("UNMAPPED_FIELDS available as snapshot only", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + assertFieldNames( + "FROM partial_mapping_sample_data | INSIST_🐔 client_ip | KEEP @timestamp, client_ip", + Set.of("@timestamp", "@timestamp.*", "client_ip", "client_ip.*"), + Set.of() + ); + } + + public void testInsist_fieldIsMappedToKeywordSingleIndex() { + assumeTrue("UNMAPPED_FIELDS available as snapshot only", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + assertFieldNames( + "FROM partial_mapping_sample_data | INSIST_🐔 message | KEEP @timestamp, message", + Set.of("@timestamp", "@timestamp.*", "message", "message.*"), + Set.of() + ); + } + + public void testInsist_fieldDoesNotExistSingleIndex() { + assumeTrue("UNMAPPED_FIELDS available as snapshot only", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + assertFieldNames( + "FROM partial_mapping_sample_data | INSIST_🐔 foo | KEEP @timestamp, foo", + Set.of("@timestamp", "@timestamp.*", "foo", "foo.*"), + Set.of() + ); + } + + public void testInsist_fieldIsUnmappedSingleIndex() { + assumeTrue("UNMAPPED_FIELDS available as snapshot only", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + assertFieldNames( + "FROM partial_mapping_sample_data | INSIST_🐔 unmapped_message | KEEP @timestamp, unmapped_message", + Set.of("@timestamp", "@timestamp.*", "unmapped_message", "unmapped_message.*"), + Set.of() + ); + } + + public void testInsist_multiFieldTestSingleIndex() { + assumeTrue("UNMAPPED_FIELDS available as snapshot only", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + assertFieldNames( + "FROM partial_mapping_sample_data | INSIST_🐔 message, unmapped_message, client_ip, foo | KEEP @timestamp, unmapped_message", + Set.of( + "@timestamp", + "@timestamp.*", + "message", + "message.*", + "unmapped_message", + "unmapped_message.*", + "client_ip", + "client_ip.*", + "foo", + "foo.*" + ), + Set.of() + ); + } + + public void testInsist_fieldIsMappedToDifferentTypesMultiIndex() { + assumeTrue("UNMAPPED_FIELDS available as snapshot only", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + assertFieldNames( + "FROM sample_data_ts_long, sample_data METADATA _index | INSIST_🐔 @timestamp | KEEP _index, @timestamp", + Set.of("@timestamp", "@timestamp.*"), + Set.of() + ); + } + + public void testInsist_multiFieldMappedMultiIndex() { + assumeTrue("UNMAPPED_FIELDS available as snapshot only", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + assertFieldNames( + """ + FROM sample_data_ts_long, sample_data METADATA _index + | INSIST_🐔 @timestamp, unmapped_message + | INSIST_🐔 message, foo + | KEEP _index, @timestamp, message, foo""", + Set.of("@timestamp", "@timestamp.*", "message", "message.*", "unmapped_message", "unmapped_message.*", "foo", "foo.*"), + Set.of() + ); + } + private Set fieldNames(String query, Set enrichPolicyMatchFields) { var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index e4e10a5c6af1..8cf9f08165b7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -54,7 +54,7 @@ public class EsqlDataTypeRegistryTests extends ESTestCase { FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); // IndexResolver uses EsqlDataTypeRegistry directly - IndexResolution resolution = new IndexResolver(null).mergedMappings("idx-*", caps); + IndexResolution resolution = IndexResolver.mergedMappings("idx-*", caps); EsField f = resolution.get().mapping().get(field); assertThat(f.getDataType(), equalTo(expected)); } From 4bba2ca66353ed0f5974ca75537aa53df0b86ca2 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 13 Feb 2025 17:32:50 -0500 Subject: [PATCH 88/93] Unmuting fixed int4 flakiness (#122545) This test was missed back in my "fix int4" PR. https://github.com/elastic/elasticsearch/pull/121437 closes: https://github.com/elastic/elasticsearch/issues/121412 --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index e20d1cff07c8..d7dd55e37274 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -326,9 +326,6 @@ tests: - class: org.elasticsearch.ingest.geoip.FullClusterRestartIT method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/121115 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only} - issue: https://github.com/elastic/elasticsearch/issues/121412 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/cat/health/cat-health-no-timestamp-example} issue: https://github.com/elastic/elasticsearch/issues/121867 From aba25c628a52161a1d2c73c1868a437ad5dac6a2 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Thu, 13 Feb 2025 19:14:30 -0600 Subject: [PATCH 89/93] Add action to copy index metadata when reindexing data stream indices (#122535) When reindexing data stream indices, parts of the index metadata needs to be copied from the source index to destination index, so that ILM and data stream lifecycle function properly. This adds a new CopyLifecycleIndexMetadataTransportAction which copies the following metadata from a source index to a destination index: - creation date setting - rollover info - ILM custom metadata --- .../cluster/metadata/IndexMetadata.java | 6 + .../core/security/user/InternalUsers.java | 1 + x-pack/plugin/migrate/build.gradle | 1 + ...fecycleIndexMetadataTransportActionIT.java | 290 ++++++++++++++++++ .../xpack/migrate/MigratePlugin.java | 3 + .../CopyLifecycleIndexMetadataAction.java | 105 +++++++ ...LifecycleIndexMetadataTransportAction.java | 133 ++++++++ ...ReindexDataStreamIndexTransportAction.java | 19 ++ .../xpack/security/operator/Constants.java | 12 +- .../upgrades/DataStreamsUpgradeIT.java | 34 +- 10 files changed, 590 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportActionIT.java create mode 100644 x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataAction.java create mode 100644 x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportAction.java diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 04f9448a936b..9cfc29c5217a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -2077,6 +2077,12 @@ public class IndexMetadata implements Diffable, ToXContentFragmen return this; } + public Builder putRolloverInfos(Map rolloverInfos) { + this.rolloverInfos.clear(); + this.rolloverInfos.putAllFromMap(rolloverInfos); + return this; + } + public long version() { return this.version; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java index a704b350dba4..9a0b17b22369 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java @@ -207,6 +207,7 @@ public class InternalUsers { TransportDeleteIndexAction.TYPE.name(), "indices:admin/data_stream/index/reindex", "indices:admin/index/create_from_source", + "indices:admin/index/copy_lifecycle_index_metadata", TransportAddIndexBlockAction.TYPE.name(), OpenIndexAction.NAME, TransportCloseIndexAction.NAME, diff --git a/x-pack/plugin/migrate/build.gradle b/x-pack/plugin/migrate/build.gradle index f179a311e0fe..796263846859 100644 --- a/x-pack/plugin/migrate/build.gradle +++ b/x-pack/plugin/migrate/build.gradle @@ -17,6 +17,7 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(xpackModule('ccr')) + testImplementation project(xpackModule('ilm')) testImplementation project(':modules:data-streams') testImplementation project(path: ':modules:reindex') testImplementation project(path: ':modules:ingest-common') diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportActionIT.java new file mode 100644 index 000000000000..f104b66b0e40 --- /dev/null +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportActionIT.java @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.ingest.common.IngestCommonPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.StartILMRequest; +import org.elasticsearch.xpack.core.ilm.StopILMRequest; +import org.elasticsearch.xpack.core.ilm.action.GetStatusAction; +import org.elasticsearch.xpack.core.ilm.action.ILMActions; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleRequest; +import org.elasticsearch.xpack.ilm.IndexLifecycle; +import org.elasticsearch.xpack.migrate.MigratePlugin; + +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +public class CopyLifecycleIndexMetadataTransportActionIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of( + LocalStateCompositeXPackPlugin.class, + MigratePlugin.class, + DataStreamsPlugin.class, + IngestCommonPlugin.class, + IndexLifecycle.class + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s") + // This just generates less churn and makes it easier to read the log file if needed + .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false) + .build(); + } + + public void testCreationDate() { + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); + + // so creation date is different + safeSleep(2); + + var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(destIndex))); + + // verify source and dest date are actually different before copying + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var indexToSettings = settingsResponse.getIndexToSettings(); + var sourceDate = indexToSettings.get(sourceIndex).getAsLong(IndexMetadata.SETTING_CREATION_DATE, 0L); + { + var destDate = indexToSettings.get(destIndex).getAsLong(IndexMetadata.SETTING_CREATION_DATE, 0L); + assertTrue(sourceDate > 0); + assertTrue(destDate > 0); + assertNotEquals(sourceDate, destDate); + } + + // copy over the metadata + copyMetadata(sourceIndex, destIndex); + + var destDate = indicesAdmin().getSettings(new GetSettingsRequest().indices(sourceIndex, destIndex)) + .actionGet() + .getIndexToSettings() + .get(destIndex) + .getAsLong(IndexMetadata.SETTING_CREATION_DATE, 0L); + assertEquals(sourceDate, destDate); + } + + public void testILMState() throws Exception { + + Map phases = Map.of( + "hot", + new Phase( + "hot", + TimeValue.ZERO, + Map.of( + "rollover", + new org.elasticsearch.xpack.core.ilm.RolloverAction(null, null, null, 1L, null, null, null, null, null, null) + ) + ) + ); + + var policyName = "my-policy"; + LifecyclePolicy policy = new LifecyclePolicy(policyName, phases); + PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy); + assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).actionGet()); + + // create data stream with a document and wait for ILM to roll it over + var dataStream = createDataStream(policyName); + createDocument(dataStream); + assertAcked(safeGet(client().execute(ILMActions.START, new StartILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)))); + assertBusy(() -> { + var getIndexResponse = safeGet(indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStream))); + assertTrue(getIndexResponse.indices().length > 1); + }); + // stop ILM so source does not change after copying metadata + assertAcked(safeGet(client().execute(ILMActions.STOP, new StopILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)))); + assertBusy(() -> { + var statusResponse = safeGet( + client().execute(GetStatusAction.INSTANCE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)) + ); + assertEquals(OperationMode.STOPPED, statusResponse.getMode()); + }); + + var getIndexResponse = safeGet(indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStream))); + for (var backingIndex : getIndexResponse.indices()) { + var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(destIndex))); + + IndexMetadata destBefore = getClusterMetadata(destIndex).index(destIndex); + assertNull(destBefore.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY)); + + // copy over the metadata + copyMetadata(backingIndex, destIndex); + + var metadataAfter = getClusterMetadata(backingIndex, destIndex); + IndexMetadata sourceAfter = metadataAfter.index(backingIndex); + IndexMetadata destAfter = metadataAfter.index(destIndex); + assertNotNull(destAfter.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY)); + assertEquals( + sourceAfter.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY), + destAfter.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY) + ); + + } + } + + public void testRolloverInfos() throws Exception { + var dataStream = createDataStream(null); + + // rollover a few times + createDocument(dataStream); + rollover(dataStream); + createDocument(dataStream); + rollover(dataStream); + createDocument(dataStream); + var writeIndex = rollover(dataStream); + + var getIndexResponse = safeGet(indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStream))); + for (var backingIndex : getIndexResponse.indices()) { + + var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(destIndex))); + + var metadataBefore = getClusterMetadata(backingIndex, destIndex); + IndexMetadata source = metadataBefore.index(backingIndex); + IndexMetadata destBefore = metadataBefore.index(destIndex); + + // sanity check not equal before the copy + if (backingIndex.equals(writeIndex)) { + assertTrue(source.getRolloverInfos().isEmpty()); + assertTrue(destBefore.getRolloverInfos().isEmpty()); + } else { + assertNotEquals(source.getRolloverInfos(), destBefore.getRolloverInfos()); + } + + // copy over the metadata + copyMetadata(backingIndex, destIndex); + + // now rollover info should be equal + IndexMetadata destAfter = getClusterMetadata(destIndex).index(destIndex); + assertEquals(source.getRolloverInfos(), destAfter.getRolloverInfos()); + } + } + + private String createDataStream(String ilmPolicy) throws Exception { + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault()); + + Settings settings = ilmPolicy != null ? Settings.builder().put(IndexMetadata.LIFECYCLE_NAME, ilmPolicy).build() : null; + + String mapping = """ + { + "properties": { + "@timestamp": { + "type":"date" + }, + "data":{ + "type":"keyword" + } + } + } + """; + Template idxTemplate = new Template(settings, new CompressedXContent(mapping), null); + + ComposableIndexTemplate template = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName + "*")) + .template(idxTemplate) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build(); + + assertAcked( + client().execute( + TransportPutComposableIndexTemplateAction.TYPE, + new TransportPutComposableIndexTemplateAction.Request(dataStreamName + "_template").indexTemplate(template) + ) + ); + assertAcked( + client().execute( + CreateDataStreamAction.INSTANCE, + new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName) + ) + ); + return dataStreamName; + } + + private long createDocument(String dataStreamName) throws Exception { + // Get some randomized but reasonable timestamps on the data since not all of it is guaranteed to arrive in order. + long timeSeed = System.currentTimeMillis(); + long timestamp = randomLongBetween(timeSeed - TimeUnit.HOURS.toMillis(5), timeSeed); + safeGet( + client().index( + new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE) + .source( + JsonXContent.contentBuilder() + .startObject() + .field("@timestamp", timestamp) + .field("data", randomAlphaOfLength(25)) + .endObject() + ) + ) + ); + safeGet( + indicesAdmin().refresh( + new RefreshRequest(".ds-" + dataStreamName + "*").indicesOptions(IndicesOptions.lenientExpandOpenHidden()) + ) + ); + return timestamp; + } + + private void copyMetadata(String sourceIndex, String destIndex) { + assertAcked( + client().execute( + CopyLifecycleIndexMetadataAction.INSTANCE, + new CopyLifecycleIndexMetadataAction.Request(TEST_REQUEST_TIMEOUT, sourceIndex, destIndex) + ) + ); + } + + private String rollover(String dataStream) { + var rolloverResponse = safeGet(indicesAdmin().rolloverIndex(new RolloverRequest(dataStream, null))); + assertTrue(rolloverResponse.isAcknowledged()); + return rolloverResponse.getNewIndex(); + } + + private Metadata getClusterMetadata(String... indices) { + return safeGet(clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT).indices(indices))).getState().metadata(); + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java index 7811e84ac9f5..0c2f7e561294 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java @@ -36,6 +36,8 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamAction; import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamTransportAction; +import org.elasticsearch.xpack.migrate.action.CopyLifecycleIndexMetadataAction; +import org.elasticsearch.xpack.migrate.action.CopyLifecycleIndexMetadataTransportAction; import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction; import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceTransportAction; import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction; @@ -106,6 +108,7 @@ public class MigratePlugin extends Plugin implements ActionPlugin, PersistentTas actions.add(new ActionHandler<>(CancelReindexDataStreamAction.INSTANCE, CancelReindexDataStreamTransportAction.class)); actions.add(new ActionHandler<>(ReindexDataStreamIndexAction.INSTANCE, ReindexDataStreamIndexTransportAction.class)); actions.add(new ActionHandler<>(CreateIndexFromSourceAction.INSTANCE, CreateIndexFromSourceTransportAction.class)); + actions.add(new ActionHandler<>(CopyLifecycleIndexMetadataAction.INSTANCE, CopyLifecycleIndexMetadataTransportAction.class)); return actions; } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataAction.java new file mode 100644 index 000000000000..d2acca1484b0 --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataAction.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class CopyLifecycleIndexMetadataAction extends ActionType { + + public static final String NAME = "indices:admin/index/copy_lifecycle_index_metadata"; + + public static final ActionType INSTANCE = new CopyLifecycleIndexMetadataAction(); + + private CopyLifecycleIndexMetadataAction() { + super(NAME); + } + + public static class Request extends AcknowledgedRequest implements IndicesRequest { + private final String sourceIndex; + private final String destIndex; + + public Request(TimeValue masterNodeTimeout, String sourceIndex, String destIndex) { + super(masterNodeTimeout, DEFAULT_ACK_TIMEOUT); + this.sourceIndex = sourceIndex; + this.destIndex = destIndex; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.sourceIndex = in.readString(); + this.destIndex = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(sourceIndex); + out.writeString(destIndex); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String sourceIndex() { + return sourceIndex; + } + + public String destIndex() { + return destIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(sourceIndex, request.sourceIndex) && Objects.equals(destIndex, request.destIndex); + } + + @Override + public int hashCode() { + return Objects.hash(sourceIndex, destIndex); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public String getDescription() { + return "copying lifecycle metadata for index " + sourceIndex; + } + + @Override + public String[] indices() { + return new String[] { sourceIndex, destIndex }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportAction.java new file mode 100644 index 000000000000..ef263fdda2db --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportAction.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.AckedBatchedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateAckListener; +import org.elasticsearch.cluster.ClusterStateTaskExecutor; +import org.elasticsearch.cluster.SimpleBatchedAckListenerTaskExecutor; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.HashMap; + +public class CopyLifecycleIndexMetadataTransportAction extends TransportMasterNodeAction< + CopyLifecycleIndexMetadataAction.Request, + AcknowledgedResponse> { + private static final Logger logger = LogManager.getLogger(CopyLifecycleIndexMetadataTransportAction.class); + private final ClusterStateTaskExecutor executor; + private final MasterServiceTaskQueue taskQueue; + + @Inject + public CopyLifecycleIndexMetadataTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters + ) { + super( + CopyLifecycleIndexMetadataAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + CopyLifecycleIndexMetadataAction.Request::new, + AcknowledgedResponse::readFrom, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.executor = new SimpleBatchedAckListenerTaskExecutor<>() { + @Override + public Tuple executeTask(UpdateIndexMetadataTask task, ClusterState clusterState) { + return new Tuple<>(applyUpdate(clusterState, task), task); + } + }; + this.taskQueue = clusterService.createTaskQueue("migrate-copy-index-metadata", Priority.NORMAL, this.executor); + } + + @Override + protected void masterOperation( + Task task, + CopyLifecycleIndexMetadataAction.Request request, + ClusterState state, + ActionListener listener + ) { + taskQueue.submitTask( + "migrate-copy-index-metadata", + new UpdateIndexMetadataTask(request.sourceIndex(), request.destIndex(), request.ackTimeout(), listener), + request.masterNodeTimeout() + ); + } + + @Override + protected ClusterBlockException checkBlock(CopyLifecycleIndexMetadataAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + private static ClusterState applyUpdate(ClusterState state, UpdateIndexMetadataTask updateTask) { + + IndexMetadata sourceMetadata = state.metadata().index(updateTask.sourceIndex); + if (sourceMetadata == null) { + throw new IndexNotFoundException(updateTask.sourceIndex); + } + IndexMetadata destMetadata = state.metadata().index(updateTask.destIndex); + if (destMetadata == null) { + throw new IndexNotFoundException(updateTask.destIndex); + } + + IndexMetadata.Builder newDestMetadata = IndexMetadata.builder(destMetadata); + + var sourceILM = sourceMetadata.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY); + if (sourceILM != null) { + newDestMetadata.putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, sourceILM); + } + + newDestMetadata.putRolloverInfos(sourceMetadata.getRolloverInfos()) + // creation date is required for ILM to function + .creationDate(sourceMetadata.getCreationDate()) + // creation date updates settings so must increment settings version + .settingsVersion(destMetadata.getSettingsVersion() + 1); + + var indices = new HashMap<>(state.metadata().indices()); + indices.put(updateTask.destIndex, newDestMetadata.build()); + + Metadata newMetadata = Metadata.builder(state.metadata()).indices(indices).build(); + return ClusterState.builder(state).metadata(newMetadata).build(); + } + + static class UpdateIndexMetadataTask extends AckedBatchedClusterStateUpdateTask { + private final String sourceIndex; + private final String destIndex; + + UpdateIndexMetadataTask(String sourceIndex, String destIndex, TimeValue ackTimeout, ActionListener listener) { + super(ackTimeout, listener); + this.sourceIndex = sourceIndex; + this.destIndex = destIndex; + } + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index 792ec4ec2b6f..31fdcbe074c1 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -163,6 +163,7 @@ public class ReindexDataStreamIndexTransportAction extends HandledTransportActio .andThen(l -> createIndex(sourceIndex, destIndexName, l, taskId)) .andThen(l -> reindex(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> copyOldSourceSettingsToDest(settingsBefore, destIndexName, l, taskId)) + .andThen(l -> copyIndexMetadataToDest(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> sanityCheck(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> closeIndexIfWasClosed(destIndexName, wasClosed, l, taskId)) .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName)) @@ -334,6 +335,24 @@ public class ReindexDataStreamIndexTransportAction extends HandledTransportActio updateSettings(destIndexName, settings, listener, parentTaskId); } + private void copyIndexMetadataToDest( + String sourceIndexName, + String destIndexName, + ActionListener listener, + TaskId parentTaskId + ) { + logger.debug("Copying index metadata to destination index [{}] from source index [{}]", destIndexName, sourceIndexName); + var request = new CopyLifecycleIndexMetadataAction.Request(TimeValue.MAX_VALUE, sourceIndexName, destIndexName); + request.setParentTask(parentTaskId); + var errorMessage = String.format( + Locale.ROOT, + "Failed to acknowledge copying index metadata from source [%s] to dest [%s]", + sourceIndexName, + destIndexName + ); + client.execute(CopyLifecycleIndexMetadataAction.INSTANCE, request, failIfNotAcknowledged(listener, errorMessage)); + } + private static void copySettingOrUnset(Settings settingsBefore, Settings.Builder builder, String setting) { // if setting was explicitly added to the source index if (settingsBefore.get(setting) != null) { diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 7e571b6db2f9..6ea522a4276a 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.security.operator; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.common.util.FeatureFlag; import java.util.Objects; import java.util.Set; @@ -638,11 +637,12 @@ public class Constants { "internal:gateway/local/started_shards", "internal:admin/indices/prevalidate_shard_path", "internal:index/metadata/migration_version/update", - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/migration/reindex_status" : null, - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/index/reindex" : null, - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex" : null, - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex_cancel" : null, - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/index/create_from_source" : null, + "indices:admin/migration/reindex_status", + "indices:admin/data_stream/index/reindex", + "indices:admin/data_stream/reindex", + "indices:admin/data_stream/reindex_cancel", + "indices:admin/index/create_from_source", + "indices:admin/index/copy_lifecycle_index_metadata", "internal:admin/repository/verify", "internal:admin/repository/verify/coordinate" ).filter(Objects::nonNull).collect(Collectors.toUnmodifiableSet()); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java index ca5fdf94e28f..a156e571b6ce 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -209,6 +209,7 @@ public class DataStreamsUpgradeIT extends AbstractUpgradeTestCase { Map> oldIndicesMetadata, Map> upgradedIndicesMetadata ) { + String oldWriteIndex = getWriteIndexFromDataStreamIndexMetadata(oldIndicesMetadata); for (Map.Entry> upgradedIndexEntry : upgradedIndicesMetadata.entrySet()) { String upgradedIndexName = upgradedIndexEntry.getKey(); if (upgradedIndexName.startsWith(".migrated-")) { @@ -217,18 +218,35 @@ public class DataStreamsUpgradeIT extends AbstractUpgradeTestCase { Map upgradedIndexMetadata = upgradedIndexEntry.getValue(); compareSettings(oldIndexMetadata, upgradedIndexMetadata); assertThat("Mappings did not match", upgradedIndexMetadata.get("mappings"), equalTo(oldIndexMetadata.get("mappings"))); - // TODO: Uncomment the following two checks once we are correctly copying this state over: - // assertThat("ILM states did not match", upgradedIndexMetadata.get("ilm"), equalTo(oldIndexMetadata.get("ilm"))); - // assertThat( - // "Rollover info did not match", - // upgradedIndexMetadata.get("rollover_info"), - // equalTo(oldIndexMetadata.get("rollover_info")) - // ); + assertThat("ILM states did not match", upgradedIndexMetadata.get("ilm"), equalTo(oldIndexMetadata.get("ilm"))); + if (oldIndexName.equals(oldWriteIndex) == false) { // the old write index will have been rolled over by upgrade + assertThat( + "Rollover info did not match", + upgradedIndexMetadata.get("rollover_info"), + equalTo(oldIndexMetadata.get("rollover_info")) + ); + } assertThat(upgradedIndexMetadata.get("system"), equalTo(oldIndexMetadata.get("system"))); } } } + private String getWriteIndexFromDataStreamIndexMetadata(Map> indexMetadataForDataStream) { + return indexMetadataForDataStream.entrySet() + .stream() + .sorted((o1, o2) -> Long.compare(getCreationDate(o2.getValue()), getCreationDate(o1.getValue()))) + .map(Map.Entry::getKey) + .findFirst() + .get(); + } + + @SuppressWarnings("unchecked") + long getCreationDate(Map indexMetadata) { + return Long.parseLong( + (String) ((Map>) indexMetadata.get("settings")).get("index").get("creation_date") + ); + } + private void compareSettings(Map oldIndexMetadata, Map upgradedIndexMetadata) { Map oldIndexSettings = getIndexSettingsFromIndexMetadata(oldIndexMetadata); Map upgradedIndexSettings = getIndexSettingsFromIndexMetadata(upgradedIndexMetadata); @@ -238,7 +256,7 @@ public class DataStreamsUpgradeIT extends AbstractUpgradeTestCase { "routing", "hidden", "number_of_shards", - // "creation_date", TODO: Uncomment this once we are correctly copying over this setting + "creation_date", "number_of_replicas" ); for (String setting : SETTINGS_TO_CHECK) { From b8e4f4ea6a9028c958e14feb091f4afae846c2c0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 13 Feb 2025 17:19:00 -0800 Subject: [PATCH 90/93] Add temp dir access to all modules (#122525) Temp dir access is necessary all over Elasticsearch, and in general is not sensitive. With Security Manager all code in ES, including plugins, are given read/write access to the temp dir. This commit mimicks that behavior with entitlements. --- .../EntitlementInitialization.java | 11 ++- .../runtime/policy/FileAccessTree.java | 11 ++- .../runtime/policy/PolicyManager.java | 56 ++++++++------ .../runtime/policy/FileAccessTreeTests.java | 26 +++++-- .../runtime/policy/PolicyManagerTests.java | 76 +++++++++++++------ 5 files changed, 120 insertions(+), 60 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 12af77de248e..6fed3c2e4b98 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -128,6 +128,7 @@ public class EntitlementInitialization { private static PolicyManager createPolicyManager() { Map pluginPolicies = EntitlementBootstrap.bootstrapArgs().pluginPolicies(); Path[] dataDirs = EntitlementBootstrap.bootstrapArgs().dataDirs(); + Path tempDir = EntitlementBootstrap.bootstrapArgs().tempDir(); // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it var serverPolicy = new Policy( @@ -167,7 +168,15 @@ public class EntitlementInitialization { // this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed List agentEntitlements = List.of(new CreateClassLoaderEntitlement(), new ManageThreadsEntitlement()); var resolver = EntitlementBootstrap.bootstrapArgs().pluginResolver(); - return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, AGENTS_PACKAGE_NAME, ENTITLEMENTS_MODULE); + return new PolicyManager( + serverPolicy, + agentEntitlements, + pluginPolicies, + resolver, + AGENTS_PACKAGE_NAME, + ENTITLEMENTS_MODULE, + tempDir + ); } private static Stream fileSystemProviderChecks() throws ClassNotFoundException, diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java index c69244d7e8a9..700302a42070 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java @@ -20,13 +20,12 @@ import java.util.Objects; import static org.elasticsearch.core.PathUtils.getDefaultFileSystem; public final class FileAccessTree { - public static final FileAccessTree EMPTY = new FileAccessTree(FilesEntitlement.EMPTY); private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator(); private final String[] readPaths; private final String[] writePaths; - private FileAccessTree(FilesEntitlement filesEntitlement) { + private FileAccessTree(FilesEntitlement filesEntitlement, Path tempDir) { List readPaths = new ArrayList<>(); List writePaths = new ArrayList<>(); for (FilesEntitlement.FileData fileData : filesEntitlement.filesData()) { @@ -38,6 +37,10 @@ public final class FileAccessTree { readPaths.add(path); } + // everything has access to the temp dir + readPaths.add(tempDir.toString()); + writePaths.add(tempDir.toString()); + readPaths.sort(String::compareTo); writePaths.sort(String::compareTo); @@ -45,8 +48,8 @@ public final class FileAccessTree { this.writePaths = writePaths.toArray(new String[0]); } - public static FileAccessTree of(FilesEntitlement filesEntitlement) { - return new FileAccessTree(filesEntitlement); + public static FileAccessTree of(FilesEntitlement filesEntitlement, Path tempDir) { + return new FileAccessTree(filesEntitlement, tempDir); } boolean canRead(Path path) { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 008983f099be..ec1ae642329f 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -70,24 +70,6 @@ public class PolicyManager { entitlementsByType = Map.copyOf(entitlementsByType); } - public static ModuleEntitlements none(String componentName) { - return new ModuleEntitlements(componentName, Map.of(), FileAccessTree.EMPTY); - } - - public static ModuleEntitlements from(String componentName, List entitlements) { - FilesEntitlement filesEntitlement = FilesEntitlement.EMPTY; - for (Entitlement entitlement : entitlements) { - if (entitlement instanceof FilesEntitlement) { - filesEntitlement = (FilesEntitlement) entitlement; - } - } - return new ModuleEntitlements( - componentName, - entitlements.stream().collect(groupingBy(Entitlement::getClass)), - FileAccessTree.of(filesEntitlement) - ); - } - public boolean hasEntitlement(Class entitlementClass) { return entitlementsByType.containsKey(entitlementClass); } @@ -101,12 +83,34 @@ public class PolicyManager { } } + // pkg private for testing + ModuleEntitlements defaultEntitlements(String componentName) { + return new ModuleEntitlements(componentName, Map.of(), defaultFileAccess); + } + + // pkg private for testing + ModuleEntitlements policyEntitlements(String componentName, List entitlements) { + FilesEntitlement filesEntitlement = FilesEntitlement.EMPTY; + for (Entitlement entitlement : entitlements) { + if (entitlement instanceof FilesEntitlement) { + filesEntitlement = (FilesEntitlement) entitlement; + } + } + return new ModuleEntitlements( + componentName, + entitlements.stream().collect(groupingBy(Entitlement::getClass)), + FileAccessTree.of(filesEntitlement, tempDir) + ); + } + final Map moduleEntitlementsMap = new ConcurrentHashMap<>(); private final Map> serverEntitlements; private final List apmAgentEntitlements; private final Map>> pluginsEntitlements; private final Function, String> pluginResolver; + private final Path tempDir; + private final FileAccessTree defaultFileAccess; public static final String ALL_UNNAMED = "ALL-UNNAMED"; @@ -141,7 +145,8 @@ public class PolicyManager { Map pluginPolicies, Function, String> pluginResolver, String apmAgentPackageName, - Module entitlementsModule + Module entitlementsModule, + Path tempDir ) { this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy)); this.apmAgentEntitlements = apmAgentEntitlements; @@ -151,6 +156,9 @@ public class PolicyManager { this.pluginResolver = pluginResolver; this.apmAgentPackageName = apmAgentPackageName; this.entitlementsModule = entitlementsModule; + this.defaultFileAccess = FileAccessTree.of(FilesEntitlement.EMPTY, tempDir); + + this.tempDir = tempDir; for (var e : serverEntitlements.entrySet()) { validateEntitlementsPerModule(SERVER_COMPONENT_NAME, e.getKey(), e.getValue()); @@ -425,7 +433,7 @@ public class PolicyManager { if (pluginName != null) { var pluginEntitlements = pluginsEntitlements.get(pluginName); if (pluginEntitlements == null) { - return ModuleEntitlements.none(pluginName); + return defaultEntitlements(pluginName); } else { final String scopeName; if (requestingModule.isNamed() == false) { @@ -439,10 +447,10 @@ public class PolicyManager { if (requestingModule.isNamed() == false && requestingClass.getPackageName().startsWith(apmAgentPackageName)) { // The APM agent is the only thing running non-modular in the system classloader - return ModuleEntitlements.from(APM_AGENT_COMPONENT_NAME, apmAgentEntitlements); + return policyEntitlements(APM_AGENT_COMPONENT_NAME, apmAgentEntitlements); } - return ModuleEntitlements.none(UNKNOWN_COMPONENT_NAME); + return defaultEntitlements(UNKNOWN_COMPONENT_NAME); } private ModuleEntitlements getModuleScopeEntitlements( @@ -452,9 +460,9 @@ public class PolicyManager { ) { var entitlements = scopeEntitlements.get(moduleName); if (entitlements == null) { - return ModuleEntitlements.none(componentName); + return defaultEntitlements(componentName); } - return ModuleEntitlements.from(componentName, entitlements); + return policyEntitlements(componentName, entitlements); } private static boolean isServerModule(Module requestingModule) { diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java index de3e2eafb756..6f3e4795fc29 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java @@ -36,13 +36,13 @@ public class FileAccessTreeTests extends ESTestCase { } public void testEmpty() { - var tree = FileAccessTree.of(FilesEntitlement.EMPTY); + var tree = accessTree(FilesEntitlement.EMPTY); assertThat(tree.canRead(path("path")), is(false)); assertThat(tree.canWrite(path("path")), is(false)); } public void testRead() { - var tree = FileAccessTree.of(entitlement("foo", "read")); + var tree = accessTree(entitlement("foo", "read")); assertThat(tree.canRead(path("foo")), is(true)); assertThat(tree.canRead(path("foo/subdir")), is(true)); assertThat(tree.canRead(path("food")), is(false)); @@ -54,7 +54,7 @@ public class FileAccessTreeTests extends ESTestCase { } public void testWrite() { - var tree = FileAccessTree.of(entitlement("foo", "read_write")); + var tree = accessTree(entitlement("foo", "read_write")); assertThat(tree.canWrite(path("foo")), is(true)); assertThat(tree.canWrite(path("foo/subdir")), is(true)); assertThat(tree.canWrite(path("food")), is(false)); @@ -66,7 +66,7 @@ public class FileAccessTreeTests extends ESTestCase { } public void testTwoPaths() { - var tree = FileAccessTree.of(entitlement("foo", "read", "bar", "read")); + var tree = accessTree(entitlement("foo", "read", "bar", "read")); assertThat(tree.canRead(path("a")), is(false)); assertThat(tree.canRead(path("bar")), is(true)); assertThat(tree.canRead(path("bar/subdir")), is(true)); @@ -77,7 +77,7 @@ public class FileAccessTreeTests extends ESTestCase { } public void testReadWriteUnderRead() { - var tree = FileAccessTree.of(entitlement("foo", "read", "foo/bar", "read_write")); + var tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write")); assertThat(tree.canRead(path("foo")), is(true)); assertThat(tree.canWrite(path("foo")), is(false)); assertThat(tree.canRead(path("foo/bar")), is(true)); @@ -85,7 +85,7 @@ public class FileAccessTreeTests extends ESTestCase { } public void testNormalizePath() { - var tree = FileAccessTree.of(entitlement("foo/../bar", "read")); + var tree = accessTree(entitlement("foo/../bar", "read")); assertThat(tree.canRead(path("foo/../bar")), is(true)); assertThat(tree.canRead(path("foo")), is(false)); assertThat(tree.canRead(path("")), is(false)); @@ -93,7 +93,7 @@ public class FileAccessTreeTests extends ESTestCase { public void testForwardSlashes() { String sep = getDefaultFileSystem().getSeparator(); - var tree = FileAccessTree.of(entitlement("a/b", "read", "m" + sep + "n", "read")); + var tree = accessTree(entitlement("a/b", "read", "m" + sep + "n", "read")); // Native separators work assertThat(tree.canRead(path("a" + sep + "b")), is(true)); @@ -104,6 +104,18 @@ public class FileAccessTreeTests extends ESTestCase { assertThat(tree.canRead(path("m/n")), is(true)); } + public void testTempDirAccess() { + Path tempDir = createTempDir(); + var tree = FileAccessTree.of(FilesEntitlement.EMPTY, tempDir); + + assertThat(tree.canRead(tempDir), is(true)); + assertThat(tree.canWrite(tempDir), is(true)); + } + + FileAccessTree accessTree(FilesEntitlement entitlement) { + return FileAccessTree.of(entitlement, createTempDir()); + } + FilesEntitlement entitlement(String... values) { List filesData = new ArrayList<>(); for (int i = 0; i < values.length; i += 2) { diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java index 34d069c98c7a..d5f2794b292f 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java @@ -71,16 +71,21 @@ public class PolicyManagerTests extends ESTestCase { Map.of("plugin1", createPluginPolicy("plugin.module")), c -> "plugin1", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); // Any class from the current module (unnamed) will do var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - assertEquals("No policy for the unnamed module", ModuleEntitlements.none("plugin1"), policyManager.getEntitlements(callerClass)); + assertEquals( + "No policy for the unnamed module", + policyManager.defaultEntitlements("plugin1"), + policyManager.getEntitlements(callerClass) + ); - assertEquals(Map.of(requestingModule, ModuleEntitlements.none("plugin1")), policyManager.moduleEntitlementsMap); + assertEquals(Map.of(requestingModule, policyManager.defaultEntitlements("plugin1")), policyManager.moduleEntitlementsMap); } public void testGetEntitlementsThrowsOnMissingPolicyForPlugin() { @@ -90,16 +95,17 @@ public class PolicyManagerTests extends ESTestCase { Map.of(), c -> "plugin1", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); // Any class from the current module (unnamed) will do var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - assertEquals("No policy for this plugin", ModuleEntitlements.none("plugin1"), policyManager.getEntitlements(callerClass)); + assertEquals("No policy for this plugin", policyManager.defaultEntitlements("plugin1"), policyManager.getEntitlements(callerClass)); - assertEquals(Map.of(requestingModule, ModuleEntitlements.none("plugin1")), policyManager.moduleEntitlementsMap); + assertEquals(Map.of(requestingModule, policyManager.defaultEntitlements("plugin1")), policyManager.moduleEntitlementsMap); } public void testGetEntitlementsFailureIsCached() { @@ -109,21 +115,22 @@ public class PolicyManagerTests extends ESTestCase { Map.of(), c -> "plugin1", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); // Any class from the current module (unnamed) will do var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - assertEquals(ModuleEntitlements.none("plugin1"), policyManager.getEntitlements(callerClass)); - assertEquals(Map.of(requestingModule, ModuleEntitlements.none("plugin1")), policyManager.moduleEntitlementsMap); + assertEquals(policyManager.defaultEntitlements("plugin1"), policyManager.getEntitlements(callerClass)); + assertEquals(Map.of(requestingModule, policyManager.defaultEntitlements("plugin1")), policyManager.moduleEntitlementsMap); // A second time - assertEquals(ModuleEntitlements.none("plugin1"), policyManager.getEntitlements(callerClass)); + assertEquals(policyManager.defaultEntitlements("plugin1"), policyManager.getEntitlements(callerClass)); // Nothing new in the map - assertEquals(Map.of(requestingModule, ModuleEntitlements.none("plugin1")), policyManager.moduleEntitlementsMap); + assertEquals(Map.of(requestingModule, policyManager.defaultEntitlements("plugin1")), policyManager.moduleEntitlementsMap); } public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() { @@ -133,7 +140,8 @@ public class PolicyManagerTests extends ESTestCase { Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), c -> "plugin2", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); // Any class from the current module (unnamed) will do @@ -150,7 +158,8 @@ public class PolicyManagerTests extends ESTestCase { Map.of(), c -> null, TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); // Tests do not run modular, so we cannot use a server class. @@ -162,11 +171,14 @@ public class PolicyManagerTests extends ESTestCase { assertEquals( "No policy for this module in server", - ModuleEntitlements.none(SERVER_COMPONENT_NAME), + policyManager.defaultEntitlements(SERVER_COMPONENT_NAME), policyManager.getEntitlements(mockServerClass) ); - assertEquals(Map.of(requestingModule, ModuleEntitlements.none(SERVER_COMPONENT_NAME)), policyManager.moduleEntitlementsMap); + assertEquals( + Map.of(requestingModule, policyManager.defaultEntitlements(SERVER_COMPONENT_NAME)), + policyManager.moduleEntitlementsMap + ); } public void testGetEntitlementsReturnsEntitlementsForServerModule() throws ClassNotFoundException { @@ -176,7 +188,8 @@ public class PolicyManagerTests extends ESTestCase { Map.of(), c -> null, TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); // Tests do not run modular, so we cannot use a server class. @@ -201,7 +214,8 @@ public class PolicyManagerTests extends ESTestCase { Map.of("mock-plugin", createPluginPolicy("org.example.plugin")), c -> "mock-plugin", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); var layer = createLayerForJar(jar, "org.example.plugin"); @@ -220,7 +234,8 @@ public class PolicyManagerTests extends ESTestCase { Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), c -> "plugin2", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); // Any class from the current module (unnamed) will do @@ -278,7 +293,8 @@ public class PolicyManagerTests extends ESTestCase { Map.of(), c -> c.getPackageName().startsWith(TEST_AGENTS_PACKAGE_NAME) ? null : "test", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); ModuleEntitlements agentsEntitlements = policyManager.getEntitlements(TestAgent.class); assertThat(agentsEntitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); @@ -305,7 +321,8 @@ public class PolicyManagerTests extends ESTestCase { Map.of(), c -> "test", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ) ); assertEquals( @@ -321,7 +338,8 @@ public class PolicyManagerTests extends ESTestCase { Map.of(), c -> "test", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ) ); assertEquals( @@ -352,7 +370,8 @@ public class PolicyManagerTests extends ESTestCase { ), c -> "plugin1", TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ) ); assertEquals( @@ -371,7 +390,8 @@ public class PolicyManagerTests extends ESTestCase { Map.of(), c -> "test", // Insist that the class is in a plugin TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + createTempDir() ); ModuleEntitlements notAgentsEntitlements = policyManager.getEntitlements(TestAgent.class); assertThat(notAgentsEntitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(false)); @@ -385,7 +405,15 @@ public class PolicyManagerTests extends ESTestCase { } private static PolicyManager policyManager(String agentsPackageName, Module entitlementsModule) { - return new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "test", agentsPackageName, entitlementsModule); + return new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.of(), + c -> "test", + agentsPackageName, + entitlementsModule, + createTempDir() + ); } private static Policy createEmptyTestServerPolicy() { From 542c5d99fb7ef23d6b3dc515942179552ac01daa Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 13 Feb 2025 17:21:08 -0800 Subject: [PATCH 91/93] Fix handling of object arrays FallbackSyntheticSourceBlockLoader (#122528) --- .../FallbackSyntheticSourceBlockLoader.java | 46 +++++++++++++++---- .../index/mapper/BlockLoaderTestCase.java | 29 +++++++++--- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index 28ea37ef73e3..64f4d1bec04c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.Stack; /** * Block loader for fields that use fallback synthetic source implementation. @@ -191,18 +192,45 @@ public abstract class FallbackSyntheticSourceBlockLoader implements BlockLoader .createParser(filterParserConfig, nameValue.value().bytes, nameValue.value().offset + 1, nameValue.value().length - 1) ) { parser.nextToken(); - var fieldNameInParser = new StringBuilder(nameValue.name()); - while (true) { - if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { - fieldNameInParser.append('.').append(parser.currentName()); - if (fieldNameInParser.toString().equals(fieldName)) { - parser.nextToken(); - break; - } + var fieldNames = new Stack() { + { + push(nameValue.name()); } + }; + + while (parser.currentToken() != null) { + // We are descending into an object/array hierarchy of arbitrary depth + // until we find the field that we need. + while (true) { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + fieldNames.push(parser.currentName()); + var nameInParser = String.join(".", fieldNames); + if (nameInParser.equals(fieldName)) { + parser.nextToken(); + break; + } + } else { + assert parser.currentToken() == XContentParser.Token.START_OBJECT + || parser.currentToken() == XContentParser.Token.START_ARRAY; + } + + parser.nextToken(); + } + parseWithReader(parser, blockValues); parser.nextToken(); + + // We are coming back up in object/array hierarchy. + // If arrays are present we will explore all array items by going back down again. + while (parser.currentToken() == XContentParser.Token.END_OBJECT + || parser.currentToken() == XContentParser.Token.END_ARRAY) { + // When exiting an object arrays we'll see END_OBJECT followed by END_ARRAY, but we only need to pop the object name + // once. + if (parser.currentToken() == XContentParser.Token.END_OBJECT) { + fieldNames.pop(); + } + parser.nextToken(); + } } - parseWithReader(parser, blockValues); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java index a7595cf52297..ab6fd109ed37 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.DocumentGenerator; import org.elasticsearch.logsdb.datageneration.FieldType; +import org.elasticsearch.logsdb.datageneration.Mapping; import org.elasticsearch.logsdb.datageneration.MappingGenerator; import org.elasticsearch.logsdb.datageneration.Template; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; @@ -72,9 +73,13 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { public void testBlockLoader() throws IOException { var template = new Template(Map.of(fieldName, new Template.Leaf(fieldName, fieldType))); - runTest(template, fieldName); + var syntheticSource = randomBoolean(); + var mapping = mappingGenerator.generate(template); + + runTest(template, mapping, syntheticSource, fieldName); } + @SuppressWarnings("unchecked") public void testBlockLoaderForFieldInObject() throws IOException { int depth = randomIntBetween(0, 3); @@ -94,14 +99,24 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { fullFieldName.append('.').append(fieldName); currentLevel.put(fieldName, new Template.Leaf(fieldName, fieldType)); var template = new Template(top); - runTest(template, fullFieldName.toString()); - } - - private void runTest(Template template, String fieldName) throws IOException { - var mapping = mappingGenerator.generate(template); - var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); var syntheticSource = randomBoolean(); + + var mapping = mappingGenerator.generate(template); + + if (syntheticSource && randomBoolean()) { + // force fallback synthetic source in the hierarchy + var docMapping = (Map) mapping.raw().get("_doc"); + var topLevelMapping = (Map) ((Map) docMapping.get("properties")).get("top"); + topLevelMapping.put("synthetic_source_keep", "all"); + } + + runTest(template, mapping, syntheticSource, fullFieldName.toString()); + } + + private void runTest(Template template, Mapping mapping, boolean syntheticSource, String fieldName) throws IOException { + var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); + var mapperService = syntheticSource ? createSytheticSourceMapperService(mappingXContent) : createMapperService(mappingXContent); var document = documentGenerator.generate(template, mapping); From 5d48ded537ec9e9335af8acef5ee0caae604d720 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Fri, 14 Feb 2025 09:54:26 +0800 Subject: [PATCH 92/93] Improve SLM Health Indicator to cover missing snapshot (#121370) Currently the SLM health indicator in health report turns YELLOW when snapshots fail for a number of times. However, the SLM health indicator stays GREEN if snapshot is not completed (no success or failure) for a long time. This change adds a new optional setting unhealthy_if_no_snapshot_within to SLM policy, that sets a time threshold. If the SLM policy has not had a successful snapshot for longer than the threshold, the SLM health indicator will turn YELLOW. --- docs/changelog/121370.yaml | 5 + .../org/elasticsearch/core/TimeValue.java | 1 + .../org/elasticsearch/TransportVersions.java | 1 + .../org/elasticsearch/test/ESTestCase.java | 3 +- .../core/slm/SnapshotLifecyclePolicy.java | 68 +++- .../xpack/slm/SLMHealthBlockedSnapshotIT.java | 328 ++++++++++++++++++ .../xpack/slm/SlmHealthIndicatorService.java | 260 ++++++++++---- .../slm/SlmHealthIndicatorServiceTests.java | 222 +++++++++++- .../slm/SnapshotLifecyclePolicyTests.java | 195 +++++++---- 9 files changed, 930 insertions(+), 153 deletions(-) create mode 100644 docs/changelog/121370.yaml create mode 100644 x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java diff --git a/docs/changelog/121370.yaml b/docs/changelog/121370.yaml new file mode 100644 index 000000000000..cfa67bf5b264 --- /dev/null +++ b/docs/changelog/121370.yaml @@ -0,0 +1,5 @@ +pr: 121370 +summary: Improve SLM Health Indicator to cover missing snapshot +area: ILM+SLM +type: enhancement +issues: [] diff --git a/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java b/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java index 89c2494cd128..a95755252883 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java +++ b/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java @@ -23,6 +23,7 @@ public class TimeValue implements Comparable { public static final TimeValue MAX_VALUE = new TimeValue(Long.MAX_VALUE, TimeUnit.NANOSECONDS); public static final TimeValue THIRTY_SECONDS = new TimeValue(30, TimeUnit.SECONDS); public static final TimeValue ONE_MINUTE = new TimeValue(1, TimeUnit.MINUTES); + public static final TimeValue ONE_HOUR = new TimeValue(1, TimeUnit.HOURS); private static final long C0 = 1L; private static final long C1 = C0 * 1000L; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index e4c83dc50fb4..c1c9e721e145 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -189,6 +189,7 @@ public class TransportVersions { public static final TransportVersion ESQL_PROFILE_ASYNC_NANOS = def(9_007_00_0); public static final TransportVersion ESQL_LOOKUP_JOIN_SOURCE_TEXT = def(9_008_0_00); public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR = def(9_009_0_00); + public static final TransportVersion SLM_UNHEALTHY_IF_NO_SNAPSHOT_WITHIN = def(9_010_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 227d7ca3046f..0f69d0d37143 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1459,7 +1459,8 @@ public abstract class ESTestCase extends LuceneTestCase { } /** - * Runs the code block for the provided interval, waiting for no assertions to trip. + * Runs the code block for the provided interval, waiting for no assertions to trip. Retries on AssertionError + * with exponential backoff until provided time runs out */ public static void assertBusy(CheckedRunnable codeBlock, long maxWaitTime, TimeUnit unit) throws Exception { long maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java index 23bf21004040..7290710d6d04 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.slm; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.cluster.SimpleDiffable; @@ -52,12 +53,14 @@ public class SnapshotLifecyclePolicy implements SimpleDiffable configuration; private final SnapshotRetentionConfiguration retentionPolicy; private final boolean isCronSchedule; + private final TimeValue unhealthyIfNoSnapshotWithin; private static final ParseField NAME = new ParseField("name"); private static final ParseField SCHEDULE = new ParseField("schedule"); private static final ParseField REPOSITORY = new ParseField("repository"); private static final ParseField CONFIG = new ParseField("config"); private static final ParseField RETENTION = new ParseField("retention"); + private static final ParseField UNHEALTHY_IF_NO_SNAPSHOT_WITHIN = new ParseField("unhealthy_if_no_snapshot_within"); private static final String METADATA_FIELD_NAME = "metadata"; @SuppressWarnings("unchecked") @@ -70,7 +73,8 @@ public class SnapshotLifecyclePolicy implements SimpleDiffable config = (Map) a[3]; SnapshotRetentionConfiguration retention = (SnapshotRetentionConfiguration) a[4]; - return new SnapshotLifecyclePolicy(id, name, schedule, repo, config, retention); + TimeValue unhealthyIfNoSnapshotWithin = (TimeValue) a[5]; + return new SnapshotLifecyclePolicy(id, name, schedule, repo, config, retention, unhealthyIfNoSnapshotWithin); } ); @@ -80,6 +84,11 @@ public class SnapshotLifecyclePolicy implements SimpleDiffable p.map(), CONFIG); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), SnapshotRetentionConfiguration::parse, RETENTION); + PARSER.declareString( + ConstructingObjectParser.optionalConstructorArg(), + value -> TimeValue.parseTimeValue(value, UNHEALTHY_IF_NO_SNAPSHOT_WITHIN.getPreferredName()), + UNHEALTHY_IF_NO_SNAPSHOT_WITHIN + ); } public SnapshotLifecyclePolicy( @@ -89,6 +98,18 @@ public class SnapshotLifecyclePolicy implements SimpleDiffable configuration, @Nullable final SnapshotRetentionConfiguration retentionPolicy + ) { + this(id, name, schedule, repository, configuration, retentionPolicy, null); + } + + public SnapshotLifecyclePolicy( + final String id, + final String name, + final String schedule, + final String repository, + @Nullable final Map configuration, + @Nullable final SnapshotRetentionConfiguration retentionPolicy, + @Nullable final TimeValue unhealthyIfNoSnapshotWithin ) { this.id = Objects.requireNonNull(id, "policy id is required"); this.name = Objects.requireNonNull(name, "policy snapshot name is required"); @@ -96,6 +117,7 @@ public class SnapshotLifecyclePolicy implements SimpleDiffable 0 + && unhealthyIfNoSnapshotWithin.compareTo(snapshotInterval) < 0) { + err.addValidationError( + "invalid unhealthy_if_no_snapshot_within [" + + unhealthyIfNoSnapshotWithin.getStringRep() + + "]: " + + "time is too short, expecting at least more than the interval between snapshots [" + + snapshotInterval.toHumanReadableString(2) + + "] for schedule [" + + schedule + + "]" + ); + } + } + if (configuration != null && configuration.containsKey(METADATA_FIELD_NAME)) { if (configuration.get(METADATA_FIELD_NAME) instanceof Map == false) { err.addValidationError( @@ -297,7 +350,7 @@ public class SnapshotLifecyclePolicy implements SimpleDiffable addPolicyNameToMetadata(final Map metadata) { @@ -339,6 +392,9 @@ public class SnapshotLifecyclePolicy implements SimpleDiffable> nodePlugins() { + return Arrays.asList( + MockRepository.Plugin.class, + MockTransportService.TestPlugin.class, + LocalStateCompositeXPackPlugin.class, + IndexLifecycle.class, + SnapshotLifecycle.class, + DataStreamsPlugin.class, + TestDelayedRepoPlugin.class + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false) + .put(LifecycleSettings.SLM_MINIMUM_INTERVAL, TimeValue.timeValueSeconds(1L)) // use a small value to allow frequent snapshot + .build(); + } + + public static class TestDelayedRepoPlugin extends Plugin implements RepositoryPlugin { + + // Use static vars since instantiated by plugin system + private static final AtomicBoolean doDelay = new AtomicBoolean(true); + private static final CountDownLatch delayedRepoLatch = new CountDownLatch(1); + + static void enable() { + doDelay.set(true); + } + + static void disable() { + doDelay.set(false); + } + + static void removeDelay() { + delayedRepoLatch.countDown(); + } + + @Override + public Map getRepositories( + Environment env, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings, + RepositoriesMetrics repositoriesMetrics + ) { + return Map.of( + TestDelayedRepo.TYPE, + metadata -> new TestDelayedRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings, () -> { + if (doDelay.get()) { + try { + assertTrue(delayedRepoLatch.await(1, TimeUnit.MINUTES)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }) + ); + } + } + + static class TestDelayedRepo extends FsRepository { + private static final String TYPE = "delayed"; + private final Runnable delayFn; + + protected TestDelayedRepo( + RepositoryMetadata metadata, + Environment env, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings, + Runnable delayFn + ) { + super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + this.delayFn = delayFn; + } + + @Override + protected void snapshotFile(SnapshotShardContext context, BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException { + delayFn.run(); + super.snapshotFile(context, fileInfo); + } + } + + public void testSlmHealthYellowWithBlockedSnapshot() throws Exception { + final String repoName = "test-repo"; + + internalCluster().startMasterOnlyNodes(1); + final String masterNode = internalCluster().getMasterName(); + final String dataNode = internalCluster().startDataOnlyNode(); + ensureStableCluster(2); + + createRepository(repoName, TestDelayedRepo.TYPE); + + String idxName = "test-index"; + String policyHealthy = "policy-health"; + String policyHealthyBelowThreshold = "policy-health-below-threshold"; + String policyUnhealthy = "policy-unhealthy"; + + List policyNames = List.of(policyHealthy, policyHealthyBelowThreshold, policyUnhealthy); + List policyNamesUnhealthy = List.of(policyUnhealthy); + + createRandomIndex(idxName); + putSnapshotPolicy(policyHealthy, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName, null); + // 1hr unhealthyIfNoSnapshotWithin should not be exceeded during test period, so policy is healthy + putSnapshotPolicy(policyHealthyBelowThreshold, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName, TimeValue.ONE_HOUR); + // zero unhealthyIfNoSnapshotWithin will always be exceeded, so policy is always unhealthy + putSnapshotPolicy(policyUnhealthy, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName, TimeValue.ZERO); + + ensureGreen(); + + // allow snapshots to run + TestDelayedRepoPlugin.disable(); + + // create a successful snapshot, so there's baseline time to check against missing snapshot threshold + List firstSnapshots = executePolicies(masterNode, policyNames); + waitForSnapshotsAndClusterState(repoName, firstSnapshots); + + // block snapshot execution, create second set of snapshots, assert YELLOW health + TestDelayedRepoPlugin.enable(); + List secondSnapshots = executePolicies(masterNode, policyNames); + assertSlmYellowMissingSnapshot(policyNamesUnhealthy); + + // resume snapshot execution + TestDelayedRepoPlugin.removeDelay(); + waitForSnapshotsAndClusterState(repoName, secondSnapshots); + + // increase policy unhealthy threshold, assert GREEN health + putSnapshotPolicy(policyUnhealthy, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName, TimeValue.ONE_HOUR); + assertBusy(() -> { + GetHealthAction.Request getHealthRequest = new GetHealthAction.Request(true, 1000); + GetHealthAction.Response health = admin().cluster().execute(GetHealthAction.INSTANCE, getHealthRequest).get(); + assertThat(health.getStatus(), equalTo(HealthStatus.GREEN)); + }); + } + + private void createRandomIndex(String idxName) throws InterruptedException { + createIndex(idxName); + + logger.info("--> indexing some data"); + final int numdocs = randomIntBetween(10, 100); + IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; + for (int i = 0; i < builders.length; i++) { + builders[i] = prepareIndex(idxName).setId(Integer.toString(i)).setSource("field1", "bar " + i); + } + indexRandom(true, builders); + indicesAdmin().refresh(new RefreshRequest(idxName)).actionGet(); + } + + private void putSnapshotPolicy( + String policyName, + String snapshotNamePattern, + String schedule, + String repoId, + String indexPattern, + TimeValue unhealthyIfNoSnapshotWithin + ) throws ExecutionException, InterruptedException { + Map snapConfig = new HashMap<>(); + snapConfig.put("indices", Collections.singletonList(indexPattern)); + snapConfig.put("ignore_unavailable", false); + snapConfig.put("partial", true); + + SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy( + policyName, + snapshotNamePattern, + schedule, + repoId, + snapConfig, + SnapshotRetentionConfiguration.EMPTY, + unhealthyIfNoSnapshotWithin + ); + + PutSnapshotLifecycleAction.Request putLifecycle = new PutSnapshotLifecycleAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + policyName, + policy + ); + + client().execute(PutSnapshotLifecycleAction.INSTANCE, putLifecycle).get(); + } + + private void assertSlmYellowMissingSnapshot(List unhealthyPolicies) throws Exception { + assertBusy(() -> { + GetHealthAction.Request getHealthRequest = new GetHealthAction.Request(true, 1000); + GetHealthAction.Response health = admin().cluster().execute(GetHealthAction.INSTANCE, getHealthRequest).get(); + assertThat(health.getStatus(), equalTo(HealthStatus.YELLOW)); + HealthIndicatorResult slmIndicator = health.findIndicator(SlmHealthIndicatorService.NAME); + assertThat(slmIndicator.status(), equalTo(HealthStatus.YELLOW)); + assertThat(slmIndicator.impacts().size(), equalTo(1)); + assertThat(slmIndicator.impacts().getFirst().id(), equalTo(SlmHealthIndicatorService.MISSING_SNAPSHOT_IMPACT_ID)); + List missingSnapshotPolicies = slmIndicator.impacts() + .stream() + .filter(impact -> SlmHealthIndicatorService.MISSING_SNAPSHOT_IMPACT_ID.equals(impact.id())) + .toList(); + assertThat(missingSnapshotPolicies.size(), equalTo(unhealthyPolicies.size())); + + // validate affected policy names + assertThat(slmIndicator.diagnosisList().size(), equalTo(1)); + Diagnosis diagnosis = slmIndicator.diagnosisList().getFirst(); + List resources = diagnosis.affectedResources(); + assertThat(resources, notNullValue()); + assertThat(resources.size(), equalTo(1)); + assertThat(resources.getFirst().getValues(), equalTo(unhealthyPolicies)); + }); + } + + private List executePolicies(String node, List policies) throws Exception { + List snapshots = new ArrayList<>(); + for (String policyName : policies) { + snapshots.add(executePolicy(node, policyName)); + } + return snapshots; + } + + /** + * Execute the given policy and return the generated snapshot name + */ + private String executePolicy(String node, String policyId) throws ExecutionException, InterruptedException { + ExecuteSnapshotLifecycleAction.Request executeReq = new ExecuteSnapshotLifecycleAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + policyId + ); + ExecuteSnapshotLifecycleAction.Response resp = client(node).execute(ExecuteSnapshotLifecycleAction.INSTANCE, executeReq).get(); + return resp.getSnapshotName(); + } + + private void waitForSnapshotsAndClusterState(String repo, List snapshots) throws Exception { + for (String snapshot : snapshots) { + waitForSnapshot(repo, snapshot); + } + assertBusy(() -> assertTrue(SnapshotsInProgress.get(internalCluster().clusterService().state()).isEmpty())); + } + + private void waitForSnapshot(String repo, String snapshotName) throws Exception { + assertBusy(() -> { + try { + SnapshotsStatusResponse s = getSnapshotStatus(repo, snapshotName); + assertThat("expected a snapshot but none were returned", s.getSnapshots().size(), equalTo(1)); + SnapshotStatus status = s.getSnapshots().get(0); + logger.info("--> waiting for snapshot {} to be completed, got: {}", snapshotName, status.getState()); + assertThat(status.getState(), equalTo(SnapshotsInProgress.State.SUCCESS)); + } catch (SnapshotMissingException e) { + fail("expected a snapshot with name " + snapshotName + " but it does not exist"); + } + }); + } + + private SnapshotsStatusResponse getSnapshotStatus(String repo, String snapshotName) { + return clusterAdmin().prepareSnapshotStatus(TEST_REQUEST_TIMEOUT, repo).setSnapshots(snapshotName).get(); + } + +} diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java index 607621483370..829c9d034b38 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java @@ -9,7 +9,9 @@ package org.elasticsearch.xpack.slm; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.health.Diagnosis; import org.elasticsearch.health.HealthIndicatorDetails; import org.elasticsearch.health.HealthIndicatorImpact; @@ -23,6 +25,7 @@ import org.elasticsearch.xpack.core.slm.SnapshotInvocationRecord; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; +import java.time.Instant; import java.time.ZoneOffset; import java.util.Collection; import java.util.Collections; @@ -65,6 +68,7 @@ public final class SlmHealthIndicatorService implements HealthIndicatorService { public static final String DIAGNOSIS_CHECK_RECENTLY_FAILED_SNAPSHOTS_ID = "check_recent_snapshot_failures"; public static final String DIAGNOSIS_CHECK_RECENTLY_FAILED_SNAPSHOTS_HELP_URL = "https://ela.st/fix-recent-snapshot-failures"; + public static final String DIAGNOSIS_CONTACT_SUPPORT_ID = "contact_support"; // Visible for testing static Diagnosis.Definition checkRecentlyFailedSnapshots(String causeText, String actionText) { @@ -77,8 +81,20 @@ public final class SlmHealthIndicatorService implements HealthIndicatorService { ); } + // Visible for testing + static Diagnosis.Definition contactSupport(String causeText, String actionText) { + return new Diagnosis.Definition( + NAME, + DIAGNOSIS_CONTACT_SUPPORT_ID, + causeText, + actionText, + ReferenceDocs.CONTACT_SUPPORT.toString() + ); + } + public static final String AUTOMATION_DISABLED_IMPACT_ID = "automation_disabled"; public static final String STALE_SNAPSHOTS_IMPACT_ID = "stale_snapshots"; + public static final String MISSING_SNAPSHOT_IMPACT_ID = "missing_snapshot"; private final ClusterService clusterService; private volatile long failedSnapshotWarnThreshold; @@ -130,70 +146,33 @@ public final class SlmHealthIndicatorService implements HealthIndicatorService { verbose ? List.of(SLM_NOT_RUNNING) : List.of() ); } else { - List unhealthyPolicies = slmMetadata.getSnapshotConfigurations() - .values() - .stream() + Collection snapshotConfigs = slmMetadata.getSnapshotConfigurations().values(); + + List failingSnapshotPolicies = snapshotConfigs.stream() .filter(metadata -> snapshotFailuresExceedWarningCount(failedSnapshotWarnThreshold, metadata)) .sorted(Comparator.comparing(SnapshotLifecyclePolicyMetadata::getId)) .toList(); - - if (unhealthyPolicies.size() > 0) { - List impacts = Collections.singletonList( - new HealthIndicatorImpact( - NAME, - STALE_SNAPSHOTS_IMPACT_ID, - 2, - "Some automated snapshots have not had a successful execution recently. Indices restored from affected " - + "snapshots may not contain recent changes.", - List.of(ImpactArea.BACKUP) - ) + if (failingSnapshotPolicies.isEmpty() == false) { + return createFailingSnapshotsIndicator( + failingSnapshotPolicies, + verbose, + maxAffectedResourcesCount, + slmMetadata, + currentMode ); + } - String unhealthyPolicyCauses = unhealthyPolicies.stream() - .map( - policy -> "- [" - + policy.getId() - + "] had [" - + policy.getInvocationsSinceLastSuccess() - + "] repeated failures without successful execution" - + (policy.getLastSuccess() != null && policy.getLastSuccess().getSnapshotStartTimestamp() != null - ? " since [" + FORMATTER.formatMillis(policy.getLastSuccess().getSnapshotStartTimestamp()) + "]" - : "") - ) - .collect(Collectors.joining("\n")); - String cause = (unhealthyPolicies.size() > 1 - ? "Several automated snapshot policies are unhealthy:\n" - : "An automated snapshot policy is unhealthy:\n") + unhealthyPolicyCauses; - - String unhealthyPolicyActions = unhealthyPolicies.stream() - .map(policy -> "- GET /_slm/policy/" + policy.getId() + "?human") - .collect(Collectors.joining("\n")); - String action = "Check the snapshot lifecycle " - + (unhealthyPolicies.size() > 1 ? "policies" : "policy") - + " for detailed failure info:\n" - + unhealthyPolicyActions; - - return createIndicator( - YELLOW, - "Encountered [" + unhealthyPolicies.size() + "] unhealthy snapshot lifecycle management policies.", - createDetails(verbose, unhealthyPolicies, slmMetadata, currentMode), - impacts, - verbose - ? List.of( - new Diagnosis( - checkRecentlyFailedSnapshots(cause, action), - List.of( - new Diagnosis.Resource( - Diagnosis.Resource.Type.SLM_POLICY, - unhealthyPolicies.stream() - .map(SnapshotLifecyclePolicyMetadata::getId) - .limit(Math.min(unhealthyPolicies.size(), maxAffectedResourcesCount)) - .toList() - ) - ) - ) - ) - : List.of() + List missingSnapshotPolicies = snapshotConfigs.stream() + .filter(SlmHealthIndicatorService::missingSnapshotTimeExceeded) + .sorted(Comparator.comparing(SnapshotLifecyclePolicyMetadata::getId)) + .toList(); + if (missingSnapshotPolicies.isEmpty() == false) { + return createMissingSnapshotIndicator( + missingSnapshotPolicies, + verbose, + maxAffectedResourcesCount, + slmMetadata, + currentMode ); } @@ -207,6 +186,21 @@ public final class SlmHealthIndicatorService implements HealthIndicatorService { } } + static boolean missingSnapshotTimeExceeded(SnapshotLifecyclePolicyMetadata policyMetadata) { + TimeValue unhealthyIfNoSnapshotWithin = policyMetadata.getPolicy().getUnhealthyIfNoSnapshotWithin(); + if (unhealthyIfNoSnapshotWithin == null) { + return false; + } + Long startTime = getMissingSnapshotPeriodStartTime(policyMetadata); + if (startTime != null) { + // time since last successful snapshot is exceeded + long now = Instant.now().toEpochMilli(); + long threshold = unhealthyIfNoSnapshotWithin.getMillis(); + return now - startTime > threshold; + } + return false; + } + static boolean snapshotFailuresExceedWarningCount(long failedSnapshotWarnThreshold, SnapshotLifecyclePolicyMetadata policyMetadata) { SnapshotInvocationRecord lastFailure = policyMetadata.getLastFailure(); if (lastFailure == null) { @@ -236,7 +230,7 @@ public final class SlmHealthIndicatorService implements HealthIndicatorService { Map details = new LinkedHashMap<>(); details.put("slm_status", mode); details.put("policies", metadata.getSnapshotConfigurations().size()); - if (unhealthyPolicies.size() > 0) { + if (unhealthyPolicies.isEmpty() == false) { details.put( "unhealthy_policies", Map.of( @@ -255,4 +249,150 @@ public final class SlmHealthIndicatorService implements HealthIndicatorService { } return new SimpleHealthIndicatorDetails(details); } + + private HealthIndicatorResult createFailingSnapshotsIndicator( + List unhealthyPolicies, + boolean verbose, + int maxAffectedResourcesCount, + SnapshotLifecycleMetadata slmMetadata, + OperationMode currentMode + ) { + List impacts = Collections.singletonList( + new HealthIndicatorImpact( + NAME, + STALE_SNAPSHOTS_IMPACT_ID, + 2, + "Some automated snapshots have not had a successful execution recently. Indices restored from affected " + + "snapshots may not contain recent changes.", + List.of(ImpactArea.BACKUP) + ) + ); + + String unhealthyPolicyCauses = unhealthyPolicies.stream() + .map( + policy -> "- [" + + policy.getId() + + "] had [" + + policy.getInvocationsSinceLastSuccess() + + "] repeated failures without successful execution" + + (policy.getLastSuccess() != null && policy.getLastSuccess().getSnapshotStartTimestamp() != null + ? " since [" + FORMATTER.formatMillis(policy.getLastSuccess().getSnapshotStartTimestamp()) + "]" + : "") + ) + .collect(Collectors.joining("\n")); + String cause = (unhealthyPolicies.size() > 1 + ? "Several automated snapshot policies are unhealthy:\n" + : "An automated snapshot policy is unhealthy:\n") + unhealthyPolicyCauses; + + String unhealthyPolicyActions = unhealthyPolicies.stream() + .map(policy -> "- GET /_slm/policy/" + policy.getId() + "?human") + .collect(Collectors.joining("\n")); + String action = "Check the snapshot lifecycle " + + (unhealthyPolicies.size() > 1 ? "policies" : "policy") + + " for detailed failure info:\n" + + unhealthyPolicyActions; + + return createIndicator( + YELLOW, + "Encountered [" + unhealthyPolicies.size() + "] unhealthy snapshot lifecycle management policies", + createDetails(verbose, unhealthyPolicies, slmMetadata, currentMode), + impacts, + verbose + ? List.of( + new Diagnosis( + checkRecentlyFailedSnapshots(cause, action), + List.of( + new Diagnosis.Resource( + Diagnosis.Resource.Type.SLM_POLICY, + unhealthyPolicies.stream() + .map(SnapshotLifecyclePolicyMetadata::getId) + .limit(Math.min(unhealthyPolicies.size(), maxAffectedResourcesCount)) + .toList() + ) + ) + ) + ) + : List.of() + ); + } + + private HealthIndicatorResult createMissingSnapshotIndicator( + List unhealthyPolicies, + boolean verbose, + int maxAffectedResourcesCount, + SnapshotLifecycleMetadata slmMetadata, + OperationMode currentMode + ) { + List impacts = Collections.singletonList( + new HealthIndicatorImpact( + NAME, + MISSING_SNAPSHOT_IMPACT_ID, + 2, + "Some snapshot lifecycle policies have not had a snapshot for some time", + List.of(ImpactArea.BACKUP) + ) + ); + + String unhealthyPolicyCauses = unhealthyPolicies.stream().map(policy -> { + Long missingStartTime = getMissingSnapshotPeriodStartTime(policy); + TimeValue unhealthyIfNoSnapshotWithin = policy.getPolicy().getUnhealthyIfNoSnapshotWithin(); + // missingStartTime and unhealthyIfNoSnapshotWithin should be non-null due to above filtering with missingSnapshotTimeExceeded + assert missingStartTime != null; + assert unhealthyIfNoSnapshotWithin != null; + return "- [" + + policy.getId() + + "] has not had a snapshot for " + + (unhealthyIfNoSnapshotWithin != null ? unhealthyIfNoSnapshotWithin.toHumanReadableString(2) : "some time") + + (missingStartTime != null ? ", since [" + FORMATTER.formatMillis(missingStartTime) + "]" : ""); + }).collect(Collectors.joining("\n")); + String cause = (unhealthyPolicies.size() > 1 + ? "Several automated snapshot policies are unhealthy:\n" + : "An automated snapshot policy is unhealthy:\n") + unhealthyPolicyCauses; + + String unhealthyPolicyActions = unhealthyPolicies.stream() + .map(policy -> "- GET /_slm/policy/" + policy.getId() + "?human") + .collect(Collectors.joining("\n")); + String action = "Check the snapshot lifecycle " + + (unhealthyPolicies.size() > 1 ? "policies" : "policy") + + " for detailed failure info:\n" + + unhealthyPolicyActions; + + return createIndicator( + YELLOW, + "Encountered [" + unhealthyPolicies.size() + "] unhealthy snapshot lifecycle management policies", + createDetails(verbose, unhealthyPolicies, slmMetadata, currentMode), + impacts, + verbose + ? List.of( + new Diagnosis( + contactSupport(cause, action), + List.of( + new Diagnosis.Resource( + Diagnosis.Resource.Type.SLM_POLICY, + unhealthyPolicies.stream() + .map(SnapshotLifecyclePolicyMetadata::getId) + .limit(Math.min(unhealthyPolicies.size(), maxAffectedResourcesCount)) + .toList() + ) + ) + ) + ) + : List.of() + ); + } + + private static Long getMissingSnapshotPeriodStartTime(SnapshotLifecyclePolicyMetadata policy) { + if (policy.getLastSuccess() != null) { + // prefer snapshotStartTimestamp over snapshotFinishTimestamp in case of a very long-running snapshot + // that started a long time ago + SnapshotInvocationRecord lastSuccess = policy.getLastSuccess(); + return lastSuccess.getSnapshotStartTimestamp() != null + ? lastSuccess.getSnapshotStartTimestamp() + : lastSuccess.getSnapshotFinishTimestamp(); + } + // TODO: handle first snapshot (i.e. no prior success of failure), maybe record the policy first trigger timestamp + + // SLM has not been triggered yet + return null; + } } diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java index 9b0d20308cf7..b0946662a24d 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.health.Diagnosis; import org.elasticsearch.health.Diagnosis.Resource.Type; import org.elasticsearch.health.HealthIndicatorDetails; @@ -30,7 +31,9 @@ import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; +import java.time.Instant; import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.List; import java.util.Map; @@ -145,10 +148,11 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { long window = TimeUnit.HOURS.toMillis(24) - 5000L; // Just under 24 hours. var clusterState = createClusterStateWith( new SnapshotLifecycleMetadata( - createSlmPolicyWithInvocations( + createSlmPolicy( snapshotInvocation(randomBoolean() ? null : execTime, execTime + 1000L), snapshotInvocation(null, execTime + window + 1000L), - randomLongBetween(0, 4) + randomLongBetween(0, 4), + null ), RUNNING, null @@ -182,7 +186,9 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { Map.of( "test-policy", SnapshotLifecyclePolicyMetadata.builder() - .setPolicy(new SnapshotLifecyclePolicy("test-policy", "", "", "test-repository", null, null)) + .setPolicy( + new SnapshotLifecyclePolicy("test-policy", "", "", "test-repository", null, null, null) + ) .setVersion(1L) .setModifiedDate(System.currentTimeMillis()) .setLastSuccess(snapshotInvocation(execTime, execTime + 1000L)) @@ -198,6 +204,7 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { "", "test-repository", null, + null, null ) ) @@ -216,6 +223,7 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { "", "test-repository", null, + null, null ) ) @@ -239,7 +247,7 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { new HealthIndicatorResult( NAME, YELLOW, - "Encountered [3] unhealthy snapshot lifecycle management policies.", + "Encountered [3] unhealthy snapshot lifecycle management policies", new SimpleHealthIndicatorDetails( Map.of( "slm_status", @@ -306,9 +314,141 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { ); } + public void testIsYellowWhenPoliciesExceedsUnhealthyIfNoSnapshotWithin() { + long tenMinutesAgo = Instant.now().minus(10, ChronoUnit.MINUTES).toEpochMilli(); + long fiveMinutesAgo = Instant.now().minus(5, ChronoUnit.MINUTES).toEpochMilli(); + + TimeValue threshold = TimeValue.ONE_MINUTE; + + var clusterState = createClusterStateWith( + new SnapshotLifecycleMetadata( + Map.of( + "test-policy-no-time-configured", + SnapshotLifecyclePolicyMetadata.builder() + .setPolicy( + new SnapshotLifecyclePolicy("test-policy-no-time-configured", "test", "", "test-repository", null, null, null) + ) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + .setLastSuccess(snapshotInvocation(tenMinutesAgo, fiveMinutesAgo)) + .build(), + "test-policy-does-not-exceed-time", + SnapshotLifecyclePolicyMetadata.builder() + .setPolicy( + new SnapshotLifecyclePolicy( + "test-policy-does-not-exceeds-time", + "test", + "", + "test-repository", + null, + null, + new TimeValue(1, TimeUnit.HOURS) + ) + ) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + .setLastSuccess(snapshotInvocation(tenMinutesAgo, fiveMinutesAgo)) + .build(), + "test-policy-exceeds-time", + SnapshotLifecyclePolicyMetadata.builder() + .setPolicy( + new SnapshotLifecyclePolicy("test-policy-exceeds-time", "test", "", "test-repository", null, null, threshold) + ) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + .setLastSuccess(snapshotInvocation(tenMinutesAgo, fiveMinutesAgo)) + .build(), + "test-policy-exceeds-time-without-success-start-time", + SnapshotLifecyclePolicyMetadata.builder() + .setPolicy( + new SnapshotLifecyclePolicy( + "test-policy-exceeds-time-without-success-start-time", + "test", + "", + "test-repository", + null, + null, + threshold + ) + ) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + .setLastSuccess(snapshotInvocation(null, fiveMinutesAgo)) + .build() + // TODO: first snapshot + ), + RUNNING, + null + ) + ); + SlmHealthIndicatorService service = createSlmHealthIndicatorService(clusterState); + + HealthIndicatorResult calculate = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); + assertThat( + calculate, + equalTo( + new HealthIndicatorResult( + NAME, + YELLOW, + "Encountered [2] unhealthy snapshot lifecycle management policies", + new SimpleHealthIndicatorDetails( + Map.of( + "slm_status", + RUNNING, + "policies", + 4, + "unhealthy_policies", + Map.of( + "count", + 2, + "invocations_since_last_success", + Map.of("test-policy-exceeds-time", 0L, "test-policy-exceeds-time-without-success-start-time", 0L) + ) + ) + ), + Collections.singletonList( + new HealthIndicatorImpact( + NAME, + SlmHealthIndicatorService.MISSING_SNAPSHOT_IMPACT_ID, + 2, + "Some snapshot lifecycle policies have not had a snapshot for some time", + List.of(ImpactArea.BACKUP) + ) + ), + List.of( + new Diagnosis( + SlmHealthIndicatorService.contactSupport( + "Several automated snapshot policies are unhealthy:\n" + + "- [test-policy-exceeds-time] has not had a snapshot for " + + threshold.toHumanReadableString(2) + + ", since [" + + FORMATTER.formatMillis(tenMinutesAgo) + + "]\n" + + "- [test-policy-exceeds-time-without-success-start-time] has not had a snapshot for " + + threshold.toHumanReadableString(2) + + ", since [" + + FORMATTER.formatMillis(fiveMinutesAgo) + + "]", + "Check the snapshot lifecycle policies for detailed failure info:\n" + + "- GET /_slm/policy/test-policy-exceeds-time?human\n" + + "- GET /_slm/policy/test-policy-exceeds-time-without-success-start-time?human" + ), + List.of( + new Diagnosis.Resource( + Type.SLM_POLICY, + List.of("test-policy-exceeds-time", "test-policy-exceeds-time-without-success-start-time") + ) + ) + ) + ) + ) + ) + ); + } + public void testSnapshotPolicyExceedsWarningThresholdPredicate() { SnapshotLifecyclePolicyMetadata slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() - .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null)) + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, null)) .setVersion(1L) .setModifiedDate(System.currentTimeMillis()) .build(); @@ -318,7 +458,7 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { assertThat(SlmHealthIndicatorService.snapshotFailuresExceedWarningCount(1L, slmPolicyMetadata), is(false)); slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() - .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null)) + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, null)) .setVersion(1L) .setModifiedDate(System.currentTimeMillis()) .setLastSuccess(snapshotInvocation(1000L, 2000L)) @@ -330,7 +470,7 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { assertThat(SlmHealthIndicatorService.snapshotFailuresExceedWarningCount(1L, slmPolicyMetadata), is(false)); slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() - .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null)) + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, null)) .setVersion(1L) .setModifiedDate(System.currentTimeMillis()) .setLastSuccess(snapshotInvocation(1000L, 2000L)) @@ -343,7 +483,7 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { assertThat(SlmHealthIndicatorService.snapshotFailuresExceedWarningCount(1L, slmPolicyMetadata), is(true)); slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() - .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null)) + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, null)) .setVersion(1L) .setModifiedDate(System.currentTimeMillis()) .setLastSuccess(snapshotInvocation(8000L, 9000L)) @@ -356,6 +496,61 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { assertThat(SlmHealthIndicatorService.snapshotFailuresExceedWarningCount(1L, slmPolicyMetadata), is(false)); } + public void testSnapshotPolicyMissingSnapshotTimeExceededPredicate() { + long tenMinutesAgo = Instant.now().minus(10, ChronoUnit.MINUTES).toEpochMilli(); + long fiveMinutesAgo = Instant.now().minus(5, ChronoUnit.MINUTES).toEpochMilli(); + // null unhealthyIfNoSnapshotWithin + { + SnapshotLifecyclePolicyMetadata slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, null)) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + .setLastSuccess(new SnapshotInvocationRecord("test-snapshot", tenMinutesAgo, fiveMinutesAgo, null)) + .build(); + assertThat(SlmHealthIndicatorService.missingSnapshotTimeExceeded(slmPolicyMetadata), is(false)); + } + // does not exceed unhealthyIfNoSnapshotWithin + { + SnapshotLifecyclePolicyMetadata slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, TimeValue.MAX_VALUE)) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + .setLastSuccess(new SnapshotInvocationRecord("test-snapshot", tenMinutesAgo, fiveMinutesAgo, null)) + .build(); + assertThat(SlmHealthIndicatorService.missingSnapshotTimeExceeded(slmPolicyMetadata), is(false)); + } + // exceed unhealthyIfNoSnapshotWithin + { + SnapshotLifecyclePolicyMetadata slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, TimeValue.ONE_MINUTE)) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + .setLastSuccess(new SnapshotInvocationRecord("test-snapshot", tenMinutesAgo, fiveMinutesAgo, null)) + .build(); + assertThat(SlmHealthIndicatorService.missingSnapshotTimeExceeded(slmPolicyMetadata), is(true)); + } + // first snapshot, does not exceed unhealthyIfNoSnapshotWithin + { + SnapshotLifecyclePolicyMetadata slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, TimeValue.MAX_VALUE)) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + // TODO: set first trigger time + .build(); + assertThat(SlmHealthIndicatorService.missingSnapshotTimeExceeded(slmPolicyMetadata), is(false)); + } + // first snapshot, exceed unhealthyIfNoSnapshotWithin + { + SnapshotLifecyclePolicyMetadata slmPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder() + .setPolicy(new SnapshotLifecyclePolicy("id", "test-policy", "", "test-repository", null, null, TimeValue.ONE_MINUTE)) + .setVersion(1L) + .setModifiedDate(System.currentTimeMillis()) + // TODO: set first trigger time + .build(); + assertThat(SlmHealthIndicatorService.missingSnapshotTimeExceeded(slmPolicyMetadata), is(false)); + } + } + public void testSkippingFieldsWhenVerboseIsFalse() { var status = randomFrom(STOPPED, STOPPING); var clusterState = createClusterStateWith(new SnapshotLifecycleMetadata(createSlmPolicy(), status, null)); @@ -404,18 +599,21 @@ public class SlmHealthIndicatorServiceTests extends ESTestCase { } private static Map createSlmPolicy() { - return createSlmPolicyWithInvocations(null, null, 0L); + return createSlmPolicy(null, null, 0L, null); } - private static Map createSlmPolicyWithInvocations( + private static Map createSlmPolicy( SnapshotInvocationRecord lastSuccess, SnapshotInvocationRecord lastFailure, - long invocationsSinceLastSuccess + long invocationsSinceLastSuccess, + TimeValue unhealthyIfNoSnapshotWithin ) { return Map.of( "test-policy", SnapshotLifecyclePolicyMetadata.builder() - .setPolicy(new SnapshotLifecyclePolicy("policy-id", "test-policy", "", "test-repository", null, null)) + .setPolicy( + new SnapshotLifecyclePolicy("policy-id", "test-policy", "", "test-repository", null, null, unhealthyIfNoSnapshotWithin) + ) .setVersion(1L) .setModifiedDate(System.currentTimeMillis()) .setLastSuccess(lastSuccess) diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java index 0ab3e99e1efc..90397dbf6d0d 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests; @@ -47,14 +48,15 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes schedule, "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); CreateSnapshotRequest request = p.toRequest(TEST_REQUEST_TIMEOUT); CreateSnapshotRequest expected = new CreateSnapshotRequest(TEST_REQUEST_TIMEOUT).userMetadata( Collections.singletonMap("policy", "id") ); - p = new SnapshotLifecyclePolicy("id", "name", schedule, "repo", null, null); + p = new SnapshotLifecyclePolicy("id", "name", schedule, "repo", null, null, null); request = p.toRequest(TEST_REQUEST_TIMEOUT); expected.waitForCompletion(true).snapshot(request.snapshot()).repository("repo").uuid(request.uuid()); assertEquals(expected, request); @@ -67,7 +69,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "0 1 2 3 4 ? 2049", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); assertThat(p.calculateNextExecution(-1, Clock.systemUTC()), equalTo(2501028060000L)); } @@ -79,7 +82,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "30m", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); { @@ -144,7 +148,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "30m", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.timeValueMinutes(30))); } @@ -156,7 +161,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes schedule, "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.parseTimeValue(schedule, "schedule"))); } @@ -170,7 +176,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "0 0/5 * * * ?", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.timeValueMinutes(5))); } @@ -182,7 +189,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "0 1 2 3 4 ? 2099", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.MINUS_ONE)); } @@ -194,7 +202,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "* * * 31 FEB ? *", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.MINUS_ONE)); } @@ -208,7 +217,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "* * * * * L", " ", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); @@ -232,7 +242,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes " ", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); @@ -253,7 +264,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "0d", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); @@ -267,7 +279,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "999micros", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); @@ -281,7 +294,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "0 0/30 * * * ?", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); assertThat(e, nullValue()); @@ -294,7 +308,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "30m", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + TimeValue.parseTimeValue("1h", "unhealthyIfNoSnapshotWithin") ); ValidationException e = policy.validate(); assertThat(e, nullValue()); @@ -307,12 +322,68 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "1ms", "repo", Collections.emptyMap(), - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); assertThat(e, nullValue()); } + { + SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy( + "my_policy", + "my_snap", + "15m", + "repo", + Collections.emptyMap(), + SnapshotRetentionConfiguration.EMPTY, + TimeValue.ONE_MINUTE + ); + + ValidationException e = policy.validate(); + assertThat( + e.validationErrors(), + containsInAnyOrder( + "invalid unhealthy_if_no_snapshot_within [1m]: time is too short, " + + "expecting at least more than the interval between snapshots [15m] for schedule [15m]" + ) + ); + } + { + SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy( + "my_policy", + "my_snap", + "0 0 1 * * ?", // every day + "repo", + Collections.emptyMap(), + SnapshotRetentionConfiguration.EMPTY, + TimeValue.parseTimeValue("2d", "unhealthyIfNoSnapshotWithin") + ); + + ValidationException e = policy.validate(); + assertThat(e, nullValue()); + } + { + SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy( + "my_policy", + "my_snap", + "0 0 1 * * ?", // every day + "repo", + Collections.emptyMap(), + SnapshotRetentionConfiguration.EMPTY, + TimeValue.ONE_MINUTE + ); + + ValidationException e = policy.validate(); + assertThat( + e.validationErrors(), + containsInAnyOrder( + "invalid unhealthy_if_no_snapshot_within [1m]: time is too short, " + + "expecting at least more than the interval between snapshots [1d] for schedule [0 0 1 * * ?]" + ) + ); + + } } public void testMetadataValidation() { @@ -327,7 +398,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "1 * * * * ?", "myrepo", configuration, - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); assertThat( @@ -348,7 +420,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "1 * * * * ?", "myrepo", configuration, - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); assertThat( @@ -378,7 +451,8 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes "1 * * * * ?", "myrepo", configuration, - SnapshotRetentionConfiguration.EMPTY + SnapshotRetentionConfiguration.EMPTY, + null ); ValidationException e = policy.validate(); assertThat( @@ -401,68 +475,37 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes @Override protected SnapshotLifecyclePolicy mutateInstance(SnapshotLifecyclePolicy instance) { - switch (between(0, 5)) { - case 0: - return new SnapshotLifecyclePolicy( - instance.getId() + randomAlphaOfLength(2), - instance.getName(), - instance.getSchedule(), - instance.getRepository(), - instance.getConfig(), - instance.getRetentionPolicy() - ); - case 1: - return new SnapshotLifecyclePolicy( - instance.getId(), - instance.getName() + randomAlphaOfLength(2), - instance.getSchedule(), - instance.getRepository(), - instance.getConfig(), - instance.getRetentionPolicy() - ); - case 2: - return new SnapshotLifecyclePolicy( - instance.getId(), - instance.getName(), - randomValueOtherThan(instance.getSchedule(), SnapshotLifecyclePolicyMetadataTests::randomSchedule), - instance.getRepository(), - instance.getConfig(), - instance.getRetentionPolicy() - ); - case 3: - return new SnapshotLifecyclePolicy( - instance.getId(), - instance.getName(), - instance.getSchedule(), - instance.getRepository() + randomAlphaOfLength(2), - instance.getConfig(), - instance.getRetentionPolicy() - ); - case 4: + String id = instance.getId(); + String name = instance.getName(); + String schedule = instance.getSchedule(); + String repository = instance.getRepository(); + Map config = instance.getConfig(); + SnapshotRetentionConfiguration retentionPolicy = instance.getRetentionPolicy(); + TimeValue unhealthyIfNoSnapshotWithin = instance.getUnhealthyIfNoSnapshotWithin(); + + switch (between(0, 6)) { + case 0 -> id += randomAlphaOfLength(2); + case 1 -> name += randomAlphaOfLength(2); + case 2 -> schedule = randomValueOtherThan(instance.getSchedule(), SnapshotLifecyclePolicyMetadataTests::randomSchedule); + case 3 -> repository += randomAlphaOfLength(2); + case 4 -> { Map newConfig = new HashMap<>(); for (int i = 0; i < randomIntBetween(2, 5); i++) { newConfig.put(randomAlphaOfLength(3), randomAlphaOfLength(3)); } - return new SnapshotLifecyclePolicy( - instance.getId(), - instance.getName() + randomAlphaOfLength(2), - instance.getSchedule(), - instance.getRepository(), - newConfig, - instance.getRetentionPolicy() - ); - case 5: - return new SnapshotLifecyclePolicy( - instance.getId(), - instance.getName(), - instance.getSchedule(), - instance.getRepository(), - instance.getConfig(), - randomValueOtherThan(instance.getRetentionPolicy(), SnapshotLifecyclePolicyMetadataTests::randomRetention) - ); - default: - throw new AssertionError("failure, got illegal switch case"); + config = newConfig; + } + case 5 -> retentionPolicy = randomValueOtherThan( + instance.getRetentionPolicy(), + SnapshotLifecyclePolicyMetadataTests::randomRetention + ); + case 6 -> unhealthyIfNoSnapshotWithin = randomValueOtherThan( + instance.getUnhealthyIfNoSnapshotWithin(), + ESTestCase::randomTimeValue + ); + default -> throw new AssertionError("failure, got illegal switch case"); } + return new SnapshotLifecyclePolicy(id, name, schedule, repository, config, retentionPolicy, unhealthyIfNoSnapshotWithin); } @Override From 5c00341c2b9074301a15a2fb2558c4b26a7cb243 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 13 Feb 2025 21:14:04 -0500 Subject: [PATCH 93/93] Handle 404s from the _cat/indices api in FullClusterRestartIT (#122537) --- .../ingest/geoip/FullClusterRestartIT.java | 16 +++++++++------- muted-tests.yml | 3 --- .../upgrades/FullClusterRestartIT.java | 16 +++++++++------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java index 9525eed7f2eb..0ba3b4ebb69f 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java +++ b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; @@ -170,7 +171,7 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas @SuppressWarnings("unchecked") private void testDatabasesLoaded() throws IOException { Request getTaskState = new Request("GET", "/_cluster/state"); - ObjectPath state = ObjectPath.createFromResponse(client().performRequest(getTaskState)); + ObjectPath state = ObjectPath.createFromResponse(assertOK(client().performRequest(getTaskState))); List tasks = state.evaluate("metadata.persistent_tasks.tasks"); // Short-circuit to avoid using steams if the list is empty @@ -196,7 +197,10 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas private void testCatIndices(List indexNames, @Nullable List additionalIndexNames) throws IOException { Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all"); - String response = EntityUtils.toString(client().performRequest(catIndices).getEntity()); + // the cat APIs can sometimes 404, erroneously + // see https://github.com/elastic/elasticsearch/issues/104371 + setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND); + String response = EntityUtils.toString(assertOK(client().performRequest(catIndices)).getEntity()); List indices = List.of(response.trim().split("\\s+")); if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) { @@ -215,7 +219,7 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas assertOK(client().performRequest(putDoc)); Request getDoc = new Request("GET", "/my-index-00001/_doc/my_id"); - ObjectPath doc = ObjectPath.createFromResponse(client().performRequest(getDoc)); + ObjectPath doc = ObjectPath.createFromResponse(assertOK(client().performRequest(getDoc))); assertNull(doc.evaluate("_source.tags")); assertEquals("Sweden", doc.evaluate("_source.geo.country_name")); } @@ -225,8 +229,7 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas getStar.setOptions( RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors ); - Response response = client().performRequest(getStar); - assertOK(response); + Response response = assertOK(client().performRequest(getStar)); if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) { indexNames = new ArrayList<>(indexNames); // recopy into a mutable list @@ -244,8 +247,7 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas .addHeader("X-elastic-product-origin", "kibana") .setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors ); - Response response = client().performRequest(getStar); - assertOK(response); + Response response = assertOK(client().performRequest(getStar)); if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) { indexNames = new ArrayList<>(indexNames); // recopy into a mutable list diff --git a/muted-tests.yml b/muted-tests.yml index d7dd55e37274..d8a68f1a3970 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -323,9 +323,6 @@ tests: - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests method: testSuggestProfileWithData issue: https://github.com/elastic/elasticsearch/issues/121258 -- class: org.elasticsearch.ingest.geoip.FullClusterRestartIT - method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/121115 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/cat/health/cat-health-no-timestamp-example} issue: https://github.com/elastic/elasticsearch/issues/121867 diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index a5e1041dab27..8c3acfb12d20 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.search.SearchFeatures; import org.elasticsearch.test.NotEqualMessageBuilder; @@ -628,13 +629,14 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas ) ); - // assertBusy to work around https://github.com/elastic/elasticsearch/issues/104371 - assertBusy( - () -> assertThat( - EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices?v&error_trace")).getEntity()), - containsString("testrollover-000002") - ) - ); + assertBusy(() -> { + Request catIndices = new Request("GET", "/_cat/indices?v&error_trace"); + // the cat APIs can sometimes 404, erroneously + // see https://github.com/elastic/elasticsearch/issues/104371 + setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND); + Response response = assertOK(client().performRequest(catIndices)); + assertThat(EntityUtils.toString(response.getEntity()), containsString("testrollover-000002")); + }); } Request countRequest = new Request("POST", "/" + index + "-*/_search");