From a393db9a9a549baf8abd0f74d17bf0248a7fe34b Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Tue, 15 Dec 2020 23:10:49 -0800 Subject: [PATCH] Autodetermine heap settings based on node roles and total system memory (#65905) This commit expands our JVM egonomics to also automatically determine appropriate heap size based on the total available system memory as well as the roles assigned to the node. Role determination is done via a naive parsing of elasticsearch.yml. No settings validation is done and only the 'node.roles' setting is taken into consideration. For heap purposes a node falls into one of four (4) categories: 1. A 'master-only' node. This is a node with only the 'master' role. 2. A 'ml-only' node. Similarly, a node with only the 'ml' role. 3. A 'data' node. This is basically the 'other' case. A node with any set of roles other than only master or only ml is considered a 'data' node, to include things like coordinating-only or "tie-breaker" nodes. 4. Unknown. This is the case if legacy settings are used. In this scenario we fallback to the old default heap options of 1GB. In all cases we short-circuit if a user provides explicit heap options so we only ever auto-determine heap if no existing heap options exist. Starting with this commit the default heap settings (1GB) are now removed from the default jvm.options which means we'll start auto- setting heap as the new default. --- distribution/src/config/jvm.options | 18 +- distribution/tools/launchers/build.gradle | 3 +- .../launchers/DefaultSystemMemoryInfo.java | 51 ++++ .../tools/launchers/JvmErgonomics.java | 98 +------ .../tools/launchers/JvmOption.java | 136 ++++++++++ .../tools/launchers/JvmOptionsParser.java | 2 + .../tools/launchers/MachineDependentHeap.java | 250 ++++++++++++++++++ .../tools/launchers/SystemMemoryInfo.java | 42 +++ .../tools/launchers/JvmErgonomicsTests.java | 21 +- .../launchers/MachineDependentHeapTests.java | 127 +++++++++ .../tools/launchers/NodeRoleParserTests.java | 118 +++++++++ .../test/resources/config/elasticsearch.yml | 1 + .../packaging/test/ArchiveTests.java | 12 +- .../packaging/test/DockerTests.java | 43 +++ .../packaging/test/PackageTests.java | 43 ++- .../packaging/test/PackagingTestCase.java | 31 +++ .../packaging/test/RpmPreservationTests.java | 4 + .../packaging/util/DockerRun.java | 11 + 18 files changed, 861 insertions(+), 150 deletions(-) create mode 100644 distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/DefaultSystemMemoryInfo.java create mode 100644 distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOption.java create mode 100644 distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java create mode 100644 distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemMemoryInfo.java create mode 100644 distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/MachineDependentHeapTests.java create mode 100644 distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/NodeRoleParserTests.java create mode 100644 distribution/tools/launchers/src/test/resources/config/elasticsearch.yml diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index 99580b65c593..81f476f566c2 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -20,10 +20,13 @@ ## IMPORTANT: JVM heap size ################################################################ ## -## You must always set the initial and maximum JVM heap size to -## the same value. For example, to set the heap to 4 GB, create -## a new file in the jvm.options.d directory containing these -## lines: +## The heap size is automatically configured by Elasticsearch +## based on the available memory in your system and the roles +## each node is configured to fulfill. If specifying heap is +## required, it should be done through a file in jvm.options.d, +## and the min and max should be set to the same value. For +## example, to set the heap to 4 GB, create a new file in the +## jvm.options.d directory containing these lines: ## ## -Xms4g ## -Xmx4g @@ -33,13 +36,6 @@ ## ################################################################ -# Xms represents the initial size of the JVM heap -# Xmx represents the maximum size of the JVM heap - --Xms${heap.min} --Xmx${heap.max} - - ################################################################ ## Expert settings diff --git a/distribution/tools/launchers/build.gradle b/distribution/tools/launchers/build.gradle index 789eeb3abab7..d45f3a1880ce 100644 --- a/distribution/tools/launchers/build.gradle +++ b/distribution/tools/launchers/build.gradle @@ -22,6 +22,7 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(':distribution:tools:java-version-checker') + compileOnly "org.yaml:snakeyaml:${versions.snakeyaml}" testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" testImplementation "junit:junit:${versions.junit}" testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" @@ -44,4 +45,4 @@ tasks.named("testingConventions").configure { ["javadoc", "loggerUsageCheck", "jarHell"].each { tsk -> tasks.named(tsk).configure { enabled = false } -} \ No newline at end of file +} diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/DefaultSystemMemoryInfo.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/DefaultSystemMemoryInfo.java new file mode 100644 index 000000000000..93897d8cf8fb --- /dev/null +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/DefaultSystemMemoryInfo.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tools.launchers; + +import com.sun.management.OperatingSystemMXBean; +import org.elasticsearch.tools.java_version_checker.JavaVersion; +import org.elasticsearch.tools.java_version_checker.SuppressForbidden; + +import java.lang.management.ManagementFactory; + +/** + * A {@link SystemMemoryInfo} which delegates to {@link OperatingSystemMXBean}. + * + *

Prior to JDK 14 {@link OperatingSystemMXBean} did not take into consideration container memory limits when reporting total system + * memory. Therefore attempts to use this implementation on earlier JDKs will result in an {@link SystemMemoryInfoException}. + */ +@SuppressForbidden(reason = "Using com.sun internals is the only way to query total system memory") +public final class DefaultSystemMemoryInfo implements SystemMemoryInfo { + private final OperatingSystemMXBean operatingSystemMXBean; + + public DefaultSystemMemoryInfo() { + this.operatingSystemMXBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + } + + @Override + @SuppressWarnings("deprecation") + public long availableSystemMemory() throws SystemMemoryInfoException { + if (JavaVersion.majorVersion(JavaVersion.CURRENT) < 14) { + throw new SystemMemoryInfoException("The minimum required Java version is 14 to use " + this.getClass().getName()); + } + + return operatingSystemMXBean.getTotalPhysicalMemorySize(); + } +} diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java index c74cdd525c68..ea10820fb32d 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java @@ -19,22 +19,13 @@ package org.elasticsearch.tools.launchers; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Tunes Elasticsearch JVM settings based on inspection of provided JVM options. @@ -53,9 +44,9 @@ final class JvmErgonomics { */ static List choose(final List userDefinedJvmOptions) throws InterruptedException, IOException { final List ergonomicChoices = new ArrayList<>(); - final Map finalJvmOptions = finalJvmOptions(userDefinedJvmOptions); - final long heapSize = extractHeapSize(finalJvmOptions); - final long maxDirectMemorySize = extractMaxDirectMemorySize(finalJvmOptions); + final Map finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions); + final long heapSize = JvmOption.extractMaxHeapSize(finalJvmOptions); + final long maxDirectMemorySize = JvmOption.extractMaxDirectMemorySize(finalJvmOptions); if (maxDirectMemorySize == 0) { ergonomicChoices.add("-XX:MaxDirectMemorySize=" + heapSize / 2); } @@ -78,89 +69,6 @@ final class JvmErgonomics { return ergonomicChoices; } - private static final Pattern OPTION = Pattern.compile( - "^\\s*\\S+\\s+(?\\S+)\\s+:?=\\s+(?\\S+)?\\s+\\{[^}]+?\\}\\s+\\{(?[^}]+)}" - ); - - private static class JvmOption { - private final String value; - private final String origin; - - JvmOption(String value, String origin) { - this.value = value; - this.origin = origin; - } - - public Optional getValue() { - return Optional.ofNullable(value); - } - - public String getMandatoryValue() { - return value; - } - - public boolean isCommandLineOrigin() { - return "command line".equals(this.origin); - } - } - - static Map finalJvmOptions(final List userDefinedJvmOptions) throws InterruptedException, IOException { - return flagsFinal(userDefinedJvmOptions).stream() - .map(OPTION::matcher) - .filter(Matcher::matches) - .collect(Collectors.toUnmodifiableMap(m -> m.group("flag"), m -> new JvmOption(m.group("value"), m.group("origin")))); - } - - private static List flagsFinal(final List userDefinedJvmOptions) throws InterruptedException, IOException { - /* - * To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM - * options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and - * -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard - * output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is - * lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them - * without having to implement our own JVM option parsing logic. - */ - final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); - final List command = Stream.of( - Stream.of(java), - userDefinedJvmOptions.stream(), - Stream.of("-Xshare:off"), - Stream.of("-XX:+PrintFlagsFinal"), - Stream.of("-version") - ).reduce(Stream::concat).get().collect(Collectors.toUnmodifiableList()); - final Process process = new ProcessBuilder().command(command).start(); - final List output = readLinesFromInputStream(process.getInputStream()); - final List error = readLinesFromInputStream(process.getErrorStream()); - final int status = process.waitFor(); - if (status != 0) { - final String message = String.format( - Locale.ROOT, - "starting java failed with [%d]\noutput:\n%s\nerror:\n%s", - status, - String.join("\n", output), - String.join("\n", error) - ); - throw new RuntimeException(message); - } else { - return output; - } - } - - private static List readLinesFromInputStream(final InputStream is) throws IOException { - try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) { - return br.lines().collect(Collectors.toUnmodifiableList()); - } - } - - // package private for testing - static Long extractHeapSize(final Map finalJvmOptions) { - return Long.parseLong(finalJvmOptions.get("MaxHeapSize").getMandatoryValue()); - } - - static long extractMaxDirectMemorySize(final Map finalJvmOptions) { - return Long.parseLong(finalJvmOptions.get("MaxDirectMemorySize").getMandatoryValue()); - } - // Tune G1GC options for heaps < 8GB static boolean tuneG1GCForSmallHeap(final long heapSize) { return heapSize < 8L << 30; diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOption.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOption.java new file mode 100644 index 000000000000..688309bb1517 --- /dev/null +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOption.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tools.launchers; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +class JvmOption { + private final String value; + private final String origin; + + JvmOption(String value, String origin) { + this.value = value; + this.origin = origin; + } + + public Optional getValue() { + return Optional.ofNullable(value); + } + + public String getMandatoryValue() { + return value; + } + + public boolean isCommandLineOrigin() { + return "command line".equals(this.origin); + } + + private static final Pattern OPTION = Pattern.compile( + "^\\s*\\S+\\s+(?\\S+)\\s+:?=\\s+(?\\S+)?\\s+\\{[^}]+?\\}\\s+\\{(?[^}]+)}" + ); + + public static Long extractMaxHeapSize(final Map finalJvmOptions) { + return Long.parseLong(finalJvmOptions.get("MaxHeapSize").getMandatoryValue()); + } + + public static boolean isMaxHeapSpecified(final Map finalJvmOptions) { + JvmOption maxHeapSize = finalJvmOptions.get("MaxHeapSize"); + return maxHeapSize != null && maxHeapSize.isCommandLineOrigin(); + } + + public static boolean isMinHeapSpecified(final Map finalJvmOptions) { + JvmOption minHeapSize = finalJvmOptions.get("MinHeapSize"); + return minHeapSize != null && minHeapSize.isCommandLineOrigin(); + } + + public static boolean isInitialHeapSpecified(final Map finalJvmOptions) { + JvmOption initialHeapSize = finalJvmOptions.get("InitialHeapSize"); + return initialHeapSize != null && initialHeapSize.isCommandLineOrigin(); + } + + public static long extractMaxDirectMemorySize(final Map finalJvmOptions) { + return Long.parseLong(finalJvmOptions.get("MaxDirectMemorySize").getMandatoryValue()); + } + + /** + * Determine the options present when invoking a JVM with the given user defined options. + */ + public static Map findFinalOptions(final List userDefinedJvmOptions) throws InterruptedException, + IOException { + return flagsFinal(userDefinedJvmOptions).stream() + .map(OPTION::matcher) + .filter(Matcher::matches) + .collect(Collectors.toUnmodifiableMap(m -> m.group("flag"), m -> new JvmOption(m.group("value"), m.group("origin")))); + } + + private static List flagsFinal(final List userDefinedJvmOptions) throws InterruptedException, IOException { + /* + * To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM + * options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and + * -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard + * output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is + * lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them + * without having to implement our own JVM option parsing logic. + */ + final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + final List command = Stream.of( + Stream.of(java), + userDefinedJvmOptions.stream(), + Stream.of("-Xshare:off"), + Stream.of("-XX:+PrintFlagsFinal"), + Stream.of("-version") + ).reduce(Stream::concat).get().collect(Collectors.toUnmodifiableList()); + final Process process = new ProcessBuilder().command(command).start(); + final List output = readLinesFromInputStream(process.getInputStream()); + final List error = readLinesFromInputStream(process.getErrorStream()); + final int status = process.waitFor(); + if (status != 0) { + final String message = String.format( + Locale.ROOT, + "starting java failed with [%d]\noutput:\n%s\nerror:\n%s", + status, + String.join("\n", output), + String.join("\n", error) + ); + throw new RuntimeException(message); + } else { + return output; + } + } + + private static List readLinesFromInputStream(final InputStream is) throws IOException { + try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) { + return br.lines().collect(Collectors.toUnmodifiableList()); + } + } +} diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java index 5f51bc2083b4..c8e26691fe2a 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java @@ -134,6 +134,7 @@ final class JvmOptionsParser { throws InterruptedException, IOException, JvmOptionsFileParserException { final List jvmOptions = readJvmOptionsFiles(config); + final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo()); if (esJavaOpts != null) { jvmOptions.addAll( @@ -142,6 +143,7 @@ final class JvmOptionsParser { } final List substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions)); + substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions)); final List ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions); final List systemJvmOptions = SystemJvmOptions.systemJvmOptions(); final List bootstrapOptions = BootstrapJvmOptions.bootstrapJvmOptions(plugins); diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java new file mode 100644 index 000000000000..ce0febebb780 --- /dev/null +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java @@ -0,0 +1,250 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tools.launchers; + +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.error.YAMLException; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static java.lang.Math.max; +import static java.lang.Math.min; +import static org.elasticsearch.tools.launchers.JvmOption.isInitialHeapSpecified; +import static org.elasticsearch.tools.launchers.JvmOption.isMaxHeapSpecified; +import static org.elasticsearch.tools.launchers.JvmOption.isMinHeapSpecified; + +/** + * Determines optimal default heap settings based on available system memory and assigned node roles. + */ +public final class MachineDependentHeap { + private static final long GB = 1024L * 1024L * 1024L; // 1GB + private static final long MAX_HEAP_SIZE = GB * 31; // 31GB + private static final long MAX_ML_HEAP_SIZE = GB * 2; // 2GB + private static final long MIN_HEAP_SIZE = 1024 * 1024 * 128; // 128MB + private static final int DEFAULT_HEAP_SIZE_MB = 1024; + private static final String ELASTICSEARCH_YML = "elasticsearch.yml"; + + private final SystemMemoryInfo systemMemoryInfo; + + public MachineDependentHeap(SystemMemoryInfo systemMemoryInfo) { + this.systemMemoryInfo = systemMemoryInfo; + } + + /** + * Calculate heap options. + * + * @param configDir path to config directory + * @param userDefinedJvmOptions JVM arguments provided by the user + * @return final heap options, or an empty collection if user provided heap options are to be used + * @throws IOException if unable to load elasticsearch.yml + */ + public List determineHeapSettings(Path configDir, List userDefinedJvmOptions) throws IOException, InterruptedException { + // TODO: this could be more efficient, to only parse final options once + final Map finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions); + if (isMaxHeapSpecified(finalJvmOptions) || isMinHeapSpecified(finalJvmOptions) || isInitialHeapSpecified(finalJvmOptions)) { + // User has explicitly set memory settings so we use those + return Collections.emptyList(); + } + + Path config = configDir.resolve(ELASTICSEARCH_YML); + try (InputStream in = Files.newInputStream(config)) { + return determineHeapSettings(in); + } + } + + List determineHeapSettings(InputStream config) { + MachineNodeRole nodeRole = NodeRoleParser.parse(config); + + try { + long availableSystemMemory = systemMemoryInfo.availableSystemMemory(); + return options(nodeRole.heap(availableSystemMemory)); + } catch (SystemMemoryInfo.SystemMemoryInfoException e) { + // If unable to determine system memory (ex: incompatible jdk version) fallback to defaults + return options(DEFAULT_HEAP_SIZE_MB); + } + } + + private static List options(int heapSize) { + return List.of("-Xms" + heapSize + "m", "-Xmx" + heapSize + "m"); + } + + /** + * Parses role information from elasticsearch.yml and determines machine node role. + */ + static class NodeRoleParser { + private static final Set LEGACY_ROLE_SETTINGS = Set.of( + "node.master", + "node.ingest", + "node.data", + "node.voting_only", + "node.ml", + "node.transform", + "node.remote_cluster_client" + ); + + @SuppressWarnings("unchecked") + public static MachineNodeRole parse(InputStream config) { + Yaml yaml = new Yaml(); + Map root; + try { + root = yaml.load(config); + } catch (YAMLException | ClassCastException ex) { + // Strangely formatted config, so just return defaults and let startup settings validation catch the problem + return MachineNodeRole.UNKNOWN; + } + + if (root != null) { + Map map = flatten(root, null); + + if (hasLegacySettings(map.keySet())) { + // We don't attempt to auto-determine heap if legacy role settings are used + return MachineNodeRole.UNKNOWN; + } else { + List roles = null; + try { + if (map.containsKey("node.roles")) { + roles = (List) map.get("node.roles"); + } + } catch (ClassCastException ex) { + return MachineNodeRole.UNKNOWN; + } + + if (roles == null || roles.isEmpty()) { + // If roles are missing or empty (coordinating node) assume defaults and consider this a data node + return MachineNodeRole.DATA; + } else if (containsOnly(roles, "master")) { + return MachineNodeRole.MASTER_ONLY; + } else if (containsOnly(roles, "ml")) { + return MachineNodeRole.ML_ONLY; + } else { + return MachineNodeRole.DATA; + } + } + } else { // if the config is completely empty, then assume defaults and consider this a data node + return MachineNodeRole.DATA; + } + } + + /** + * Flattens a nested configuration structure. This creates a consistent way of referencing settings from a config file that uses + * a mix of object and flat setting notation. The returned map is a single-level deep structure of dot-notation property names + * to values. + * + *

No attempt is made to deterministically deal with duplicate settings, nor are they explicitly disallowed. + * + * @param config nested configuration map + * @param parentPath parent node path or {@code null} if parsing the root node + * @return flattened configuration map + */ + @SuppressWarnings("unchecked") + private static Map flatten(Map config, String parentPath) { + Map flatMap = new HashMap<>(); + String prefix = parentPath != null ? parentPath + "." : ""; + + for (Map.Entry entry : config.entrySet()) { + if (entry.getValue() instanceof Map) { + flatMap.putAll(flatten((Map) entry.getValue(), prefix + entry.getKey())); + } else { + flatMap.put(prefix + entry.getKey(), entry.getValue()); + } + } + + return flatMap; + } + + @SuppressWarnings("unchecked") + private static boolean containsOnly(Collection collection, T... items) { + return Arrays.asList(items).containsAll(collection); + } + + private static boolean hasLegacySettings(Set keys) { + return LEGACY_ROLE_SETTINGS.stream().anyMatch(keys::contains); + } + } + + enum MachineNodeRole { + /** + * Master-only node. + * + *

Heap is computed as 60% of total system memory up to a maximum of 31 gigabytes. + */ + MASTER_ONLY(m -> mb(min((long) (m * .6), MAX_HEAP_SIZE))), + + /** + * Machine learning only node. + * + *

Heap is computed as: + *

+ */ + ML_ONLY(m -> mb(m < (GB * 2) ? (long) (m * .4) : (long) min(m * .25, MAX_ML_HEAP_SIZE))), + + /** + * Data node. Essentially any node that isn't a master or ML only node. + * + *

Heap is computed as: + *

+ */ + DATA(m -> mb(m < GB ? max((long) (m * .4), MIN_HEAP_SIZE) : min((long) (m * .5), MAX_HEAP_SIZE))), + + /** + * Unknown role node. + * + *

Hard-code heap to a default of 1 gigabyte. + */ + UNKNOWN(m -> DEFAULT_HEAP_SIZE_MB); + + private final Function formula; + + MachineNodeRole(Function formula) { + this.formula = formula; + } + + /** + * Determine the appropriate heap size for the given role and available system memory. + * + * @param systemMemory total available system memory in bytes + * @return recommended heap size in megabytes + */ + public int heap(long systemMemory) { + return formula.apply(systemMemory); + } + + private static int mb(long bytes) { + return (int) (bytes / (1024 * 1024)); + } + } +} diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemMemoryInfo.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemMemoryInfo.java new file mode 100644 index 000000000000..0a386435d356 --- /dev/null +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemMemoryInfo.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tools.launchers; + +/** + * Determines available system memory that could be allocated for Elasticsearch, to include JVM heap and other native processes. + * The "available system memory" is defined as the total system memory which is visible to the Elasticsearch process. For instances + * in which Elasticsearch is running in a containerized environment (i.e. Docker) this is expected to be the limits set for the container, + * not the host system. + */ +public interface SystemMemoryInfo { + + /** + * + * @return total system memory available to heap or native process allocation in bytes + * @throws SystemMemoryInfoException if unable to determine available system memory + */ + long availableSystemMemory() throws SystemMemoryInfoException; + + class SystemMemoryInfoException extends Exception { + public SystemMemoryInfoException(String message) { + super(message); + } + } +} diff --git a/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java index 4eb7bc7d138d..7262c03e1916 100644 --- a/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java +++ b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java @@ -42,23 +42,23 @@ import static org.junit.Assert.fail; public class JvmErgonomicsTests extends LaunchersTestCase { public void testExtractValidHeapSizeUsingXmx() throws InterruptedException, IOException { - assertThat(JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx2g"))), equalTo(2L << 30)); + assertThat(JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx2g"))), equalTo(2L << 30)); } public void testExtractValidHeapSizeUsingMaxHeapSize() throws InterruptedException, IOException { assertThat( - JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-XX:MaxHeapSize=2g"))), + JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-XX:MaxHeapSize=2g"))), equalTo(2L << 30) ); } public void testExtractValidHeapSizeNoOptionPresent() throws InterruptedException, IOException { - assertThat(JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.emptyList())), greaterThan(0L)); + assertThat(JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.emptyList())), greaterThan(0L)); } public void testHeapSizeInvalid() throws InterruptedException, IOException { try { - JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx2Z"))); + JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx2Z"))); fail("expected starting java to fail"); } catch (final RuntimeException e) { assertThat(e, hasToString(containsString(("starting java failed")))); @@ -68,7 +68,7 @@ public class JvmErgonomicsTests extends LaunchersTestCase { public void testHeapSizeTooSmall() throws InterruptedException, IOException { try { - JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx1024"))); + JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx1024"))); fail("expected starting java to fail"); } catch (final RuntimeException e) { assertThat(e, hasToString(containsString(("starting java failed")))); @@ -78,7 +78,7 @@ public class JvmErgonomicsTests extends LaunchersTestCase { public void testHeapSizeWithSpace() throws InterruptedException, IOException { try { - JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx 1024"))); + JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx 1024"))); fail("expected starting java to fail"); } catch (final RuntimeException e) { assertThat(e, hasToString(containsString(("starting java failed")))); @@ -87,17 +87,12 @@ public class JvmErgonomicsTests extends LaunchersTestCase { } public void testMaxDirectMemorySizeUnset() throws InterruptedException, IOException { - assertThat( - JvmErgonomics.extractMaxDirectMemorySize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx1g"))), - equalTo(0L) - ); + assertThat(JvmOption.extractMaxDirectMemorySize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx1g"))), equalTo(0L)); } public void testMaxDirectMemorySizeSet() throws InterruptedException, IOException { assertThat( - JvmErgonomics.extractMaxDirectMemorySize( - JvmErgonomics.finalJvmOptions(Arrays.asList("-Xmx1g", "-XX:MaxDirectMemorySize=512m")) - ), + JvmOption.extractMaxDirectMemorySize(JvmOption.findFinalOptions(Arrays.asList("-Xmx1g", "-XX:MaxDirectMemorySize=512m"))), equalTo(512L << 20) ); } diff --git a/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/MachineDependentHeapTests.java b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/MachineDependentHeapTests.java new file mode 100644 index 000000000000..70def70253fa --- /dev/null +++ b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/MachineDependentHeapTests.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tools.launchers; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.junit.Assert.assertThat; + +public class MachineDependentHeapTests extends LaunchersTestCase { + + public void testDefaultHeapSize() throws Exception { + MachineDependentHeap heap = new MachineDependentHeap(systemMemoryInGigabytes(8)); + List options = heap.determineHeapSettings(configPath(), Collections.emptyList()); + assertThat(options, containsInAnyOrder("-Xmx4096m", "-Xms4096m")); + } + + public void testUserPassedHeapArgs() throws Exception { + MachineDependentHeap heap = new MachineDependentHeap(systemMemoryInGigabytes(8)); + List options = heap.determineHeapSettings(configPath(), List.of("-Xmx4g")); + assertThat(options, empty()); + + options = heap.determineHeapSettings(configPath(), List.of("-Xms4g")); + assertThat(options, empty()); + } + + public void testMasterOnlyOptions() { + List options = calculateHeap(16, "master"); + assertThat(options, containsInAnyOrder("-Xmx9830m", "-Xms9830m")); + + options = calculateHeap(64, "master"); + assertThat(options, containsInAnyOrder("-Xmx31744m", "-Xms31744m")); + } + + public void testMlOnlyOptions() { + List options = calculateHeap(1, "ml"); + assertThat(options, containsInAnyOrder("-Xmx409m", "-Xms409m")); + + options = calculateHeap(4, "ml"); + assertThat(options, containsInAnyOrder("-Xmx1024m", "-Xms1024m")); + + options = calculateHeap(32, "ml"); + assertThat(options, containsInAnyOrder("-Xmx2048m", "-Xms2048m")); + } + + public void testDataNodeOptions() { + List options = calculateHeap(1, "data"); + assertThat(options, containsInAnyOrder("-Xmx512m", "-Xms512m")); + + options = calculateHeap(8, "data"); + assertThat(options, containsInAnyOrder("-Xmx4096m", "-Xms4096m")); + + options = calculateHeap(64, "data"); + assertThat(options, containsInAnyOrder("-Xmx31744m", "-Xms31744m")); + + options = calculateHeap(0.5, "data"); + assertThat(options, containsInAnyOrder("-Xmx204m", "-Xms204m")); + + options = calculateHeap(0.2, "data"); + assertThat(options, containsInAnyOrder("-Xmx128m", "-Xms128m")); + } + + public void testFallbackOptions() throws Exception { + MachineDependentHeap machineDependentHeap = new MachineDependentHeap(errorThrowingMemoryInfo()); + List options = machineDependentHeap.determineHeapSettings(configPath(), Collections.emptyList()); + assertThat(options, containsInAnyOrder("-Xmx1024m", "-Xms1024m")); + } + + private static List calculateHeap(double memoryInGigabytes, String... roles) { + MachineDependentHeap machineDependentHeap = new MachineDependentHeap(systemMemoryInGigabytes(memoryInGigabytes)); + String configYaml = "node.roles: [" + String.join(",", roles) + "]"; + return calculateHeap(machineDependentHeap, configYaml); + } + + private static List calculateHeap(MachineDependentHeap machineDependentHeap, String configYaml) { + try (InputStream in = new ByteArrayInputStream(configYaml.getBytes(StandardCharsets.UTF_8))) { + return machineDependentHeap.determineHeapSettings(in); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static SystemMemoryInfo systemMemoryInGigabytes(double gigabytes) { + return () -> (long) (gigabytes * 1024 * 1024 * 1024); + } + + private static SystemMemoryInfo errorThrowingMemoryInfo() { + return () -> { throw new SystemMemoryInfo.SystemMemoryInfoException("something went wrong"); }; + } + + private static Path configPath() { + URL resource = MachineDependentHeapTests.class.getResource("/config/elasticsearch.yml"); + try { + return Paths.get(resource.toURI()).getParent(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } +} diff --git a/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/NodeRoleParserTests.java b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/NodeRoleParserTests.java new file mode 100644 index 000000000000..0ae805ef9fe7 --- /dev/null +++ b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/NodeRoleParserTests.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tools.launchers; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.function.Consumer; + +import static org.elasticsearch.tools.launchers.MachineDependentHeap.MachineNodeRole.DATA; +import static org.elasticsearch.tools.launchers.MachineDependentHeap.MachineNodeRole.MASTER_ONLY; +import static org.elasticsearch.tools.launchers.MachineDependentHeap.MachineNodeRole.ML_ONLY; +import static org.elasticsearch.tools.launchers.MachineDependentHeap.MachineNodeRole.UNKNOWN; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +public class NodeRoleParserTests extends LaunchersTestCase { + + public void testMasterOnlyNode() throws IOException { + MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: [master]")); + assertThat(nodeRole, equalTo(MASTER_ONLY)); + + nodeRole = parseConfig(sb -> sb.append("node.roles: [master, some_other_role]")); + assertThat(nodeRole, not(equalTo(MASTER_ONLY))); + } + + public void testMlOnlyNode() throws IOException { + MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: [ml]")); + assertThat(nodeRole, equalTo(ML_ONLY)); + + nodeRole = parseConfig(sb -> sb.append("node.roles: [ml, some_other_role]")); + assertThat(nodeRole, not(equalTo(ML_ONLY))); + } + + public void testDataNode() throws IOException { + MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> {}); + assertThat(nodeRole, equalTo(DATA)); + + nodeRole = parseConfig(sb -> sb.append("node.roles: []")); + assertThat(nodeRole, equalTo(DATA)); + + nodeRole = parseConfig(sb -> sb.append("node.roles: [some_unknown_role]")); + assertThat(nodeRole, equalTo(DATA)); + + nodeRole = parseConfig(sb -> sb.append("node.roles: [master, ingest]")); + assertThat(nodeRole, equalTo(DATA)); + + nodeRole = parseConfig(sb -> sb.append("node.roles: [ml, master]")); + assertThat(nodeRole, equalTo(DATA)); + } + + public void testLegacySettings() throws IOException { + MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.ml: true")); + assertThat(nodeRole, equalTo(UNKNOWN)); + + nodeRole = parseConfig(sb -> sb.append("node.master: true")); + assertThat(nodeRole, equalTo(UNKNOWN)); + + nodeRole = parseConfig(sb -> sb.append("node.data: false")); + assertThat(nodeRole, equalTo(UNKNOWN)); + + nodeRole = parseConfig(sb -> sb.append("node.ingest: false")); + assertThat(nodeRole, equalTo(UNKNOWN)); + } + + public void testYamlSyntax() throws IOException { + MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> { + sb.append("node:\n"); + sb.append(" roles:\n"); + sb.append(" - master"); + }); + assertThat(nodeRole, equalTo(MASTER_ONLY)); + + nodeRole = parseConfig(sb -> { + sb.append("node:\n"); + sb.append(" roles: [ml]"); + }); + assertThat(nodeRole, equalTo(ML_ONLY)); + } + + public void testInvalidYaml() throws IOException { + MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("notyaml")); + assertThat(nodeRole, equalTo(UNKNOWN)); + } + + public void testInvalidRoleSyntax() throws IOException { + MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: foo")); + assertThat(nodeRole, equalTo(UNKNOWN)); + } + + private static MachineDependentHeap.MachineNodeRole parseConfig(Consumer action) throws IOException { + StringBuilder sb = new StringBuilder(); + action.accept(sb); + + try (InputStream config = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8))) { + return MachineDependentHeap.NodeRoleParser.parse(config); + } + } +} diff --git a/distribution/tools/launchers/src/test/resources/config/elasticsearch.yml b/distribution/tools/launchers/src/test/resources/config/elasticsearch.yml new file mode 100644 index 000000000000..4f436a154a11 --- /dev/null +++ b/distribution/tools/launchers/src/test/resources/config/elasticsearch.yml @@ -0,0 +1 @@ +node.roles: [] diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index d7dc6294046b..0f3e2d0fb9eb 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -248,7 +248,8 @@ public class ArchiveTests extends PackagingTestCase { public void test70CustomPathConfAndJvmOptions() throws Exception { withCustomConfig(tempConf -> { - final List jvmOptions = List.of("-Xms512m", "-Xmx512m", "-Dlog4j2.disable.jmx=true"); + setHeap("512m", tempConf); + final List jvmOptions = List.of("-Dlog4j2.disable.jmx=true"); Files.write(tempConf.resolve("jvm.options"), jvmOptions, CREATE, APPEND); sh.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops"); @@ -266,6 +267,7 @@ public class ArchiveTests extends PackagingTestCase { public void test71CustomJvmOptionsDirectoryFile() throws Exception { final Path heapOptions = installation.config(Paths.get("jvm.options.d", "heap.options")); try { + setHeap(null); // delete default options append(heapOptions, "-Xms512m\n-Xmx512m\n"); startElasticsearch(); @@ -283,6 +285,7 @@ public class ArchiveTests extends PackagingTestCase { final Path firstOptions = installation.config(Paths.get("jvm.options.d", "first.options")); final Path secondOptions = installation.config(Paths.get("jvm.options.d", "second.options")); try { + setHeap(null); // delete default options /* * We override the heap in the first file, and disable compressed oops, and override the heap in the second file. By doing this, * we can test that both files are processed by the JVM options parser, and also that they are processed in lexicographic order. @@ -306,13 +309,10 @@ public class ArchiveTests extends PackagingTestCase { public void test73CustomJvmOptionsDirectoryFilesWithoutOptionsExtensionIgnored() throws Exception { final Path jvmOptionsIgnored = installation.config(Paths.get("jvm.options.d", "jvm.options.ignored")); try { - append(jvmOptionsIgnored, "-Xms512\n-Xmx512m\n"); + append(jvmOptionsIgnored, "-Xthis_is_not_a_valid_option\n"); startElasticsearch(); - - final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); - assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":1073741824")); - + ServerUtils.runElasticsearchTests(); stopElasticsearch(); } finally { rm(jvmOptionsIgnored); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 45911659c050..4535ee3b11ae 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.packaging.test; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.http.client.fluent.Request; import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.Installation; @@ -35,9 +36,11 @@ import org.junit.BeforeClass; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -68,6 +71,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesPattern; @@ -732,6 +736,45 @@ public class DockerTests extends PackagingTestCase { assertThat("Failed to find [cpuacct] in node OS cgroup stats", cgroupStats.get("cpuacct"), not(nullValue())); } + /** + * Check that when available system memory is constrained by Docker, the machine-dependant heap sizing + * logic sets the correct heap size, based on the container limits. + */ + public void test150MachineDependentHeap() throws Exception { + // Start by ensuring `jvm.options` doesn't define any heap options + final Path jvmOptionsPath = tempDir.resolve("jvm.options"); + final Path containerJvmOptionsPath = installation.config("jvm.options"); + copyFromContainer(containerJvmOptionsPath, jvmOptionsPath); + + final List jvmOptions = Files.readAllLines(jvmOptionsPath) + .stream() + .filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false) + .collect(Collectors.toList()); + + Files.writeString(jvmOptionsPath, String.join("\n", jvmOptions)); + + // Now run the container, being explicit about the available memory + runContainer(distribution(), builder().memory("942m").volumes(Map.of(jvmOptionsPath, containerJvmOptionsPath))); + waitForElasticsearch(installation); + + // Grab the container output and find the line where it print the JVM arguments. This will + // let us see what the automatic heap sizing calculated. + final Optional jvmArgumentsLine = getContainerLogs().stdout.lines() + .filter(line -> line.contains("JVM arguments")) + .findFirst(); + assertThat("Failed to find jvmArguments in container logs", jvmArgumentsLine.isPresent(), is(true)); + + final JsonNode jsonNode = new ObjectMapper().readTree(jvmArgumentsLine.get()); + + final String argsStr = jsonNode.get("message").textValue(); + final List xArgs = Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*")) + .filter(arg -> arg.startsWith("-X")) + .collect(Collectors.toList()); + + // This is roughly 0.4 * 942 + assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m")); + } + /** * Check that the UBI images has the correct license information in the correct place. */ diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java index 91e805f59e8e..df1888095af4 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java @@ -133,31 +133,14 @@ public class PackageTests extends PackagingTestCase { } public void test34CustomJvmOptionsDirectoryFile() throws Exception { - final Path heapOptions = installation.config(Paths.get("jvm.options.d", "heap.options")); - try { - append(heapOptions, "-Xms512m\n-Xmx512m\n"); + setHeap("512m"); - startElasticsearch(); + startElasticsearch(); - final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); - assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); - stopElasticsearch(); - } finally { - rm(heapOptions); - } - } - - public void test42BundledJdkRemoved() throws Exception { - assumeThat(distribution().hasJdk, is(true)); - - Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated"); - try { - mv(installation.bundledJdk, relocatedJdk); - assertRunsWithJavaHome(); - } finally { - mv(relocatedJdk, installation.bundledJdk); - } + stopElasticsearch(); } public void test40StartServer() throws Exception { @@ -177,6 +160,18 @@ public class PackageTests extends PackagingTestCase { stopElasticsearch(); } + public void test42BundledJdkRemoved() throws Exception { + assumeThat(distribution().hasJdk, is(true)); + + Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated"); + try { + mv(installation.bundledJdk, relocatedJdk); + assertRunsWithJavaHome(); + } finally { + mv(relocatedJdk, installation.bundledJdk); + } + } + public void test50Remove() throws Exception { // add fake bin directory as if a plugin was installed Files.createDirectories(installation.bin.resolve("myplugin")); @@ -300,12 +295,12 @@ public class PackageTests extends PackagingTestCase { stopElasticsearch(); withCustomConfig(tempConf -> { - append(installation.envFile, "ES_JAVA_OPTS=-XX:-UseCompressedOops"); + append(installation.envFile, "ES_JAVA_OPTS=\"-Xmx512m -Xms512m -XX:-UseCompressedOops\""); startElasticsearch(); final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); - assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":1073741824")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); stopElasticsearch(); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 9a48d4b6ce03..e1190ebb7562 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -59,10 +59,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermissions; import java.util.Collections; import java.util.List; +import java.util.Locale; import static org.elasticsearch.packaging.util.Cleanup.cleanEverything; import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded; @@ -173,6 +175,9 @@ public abstract class PackagingTestCase extends Assert { Platforms.onLinux(() -> sh.getEnv().put("JAVA_HOME", systemJavaHome)); Platforms.onWindows(() -> sh.getEnv().put("JAVA_HOME", systemJavaHome)); } + if (installation != null && distribution.isDocker() == false) { + setHeap("1g"); + } } @After @@ -224,6 +229,11 @@ public abstract class PackagingTestCase extends Assert { default: throw new IllegalStateException("Unknown Elasticsearch packaging type."); } + + // the purpose of the packaging tests are not to all test auto heap, so we explicitly set heap size to 1g + if (distribution.isDocker() == false) { + setHeap("1g"); + } } protected static void cleanup() throws Exception { @@ -447,4 +457,25 @@ public abstract class PackagingTestCase extends Assert { } IOUtils.rm(tempDir); } + + /** + * Manually set the heap size with a jvm.options.d file. This will be reset before each test. + */ + public static void setHeap(String heapSize) throws IOException { + setHeap(heapSize, installation.config); + } + + public static void setHeap(String heapSize, Path config) throws IOException { + Path heapOptions = config.resolve("jvm.options.d").resolve("heap.options"); + if (heapSize == null) { + FileUtils.rm(heapOptions); + } else { + Files.writeString( + heapOptions, + String.format(Locale.ROOT, "-Xmx%1$s%n-Xms%1$s%n", heapSize), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + } + } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java index d934f0e62c27..bc2968208713 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java @@ -57,6 +57,7 @@ public class RpmPreservationTests extends PackagingTestCase { } public void test20Remove() throws Exception { + setHeap(null); // remove test heap options, so the config directory can be removed remove(distribution()); // config was removed @@ -64,6 +65,9 @@ public class RpmPreservationTests extends PackagingTestCase { // defaults file was removed assertThat(installation.envFile, fileDoesNotExist()); + + // don't perform normal setup/teardown after this since we removed the install + installation = null; } public void test30PreserveConfig() throws Exception { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/DockerRun.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/DockerRun.java index dadf779b34df..947df3d371c8 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/DockerRun.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/DockerRun.java @@ -38,6 +38,7 @@ public class DockerRun { private Integer uid; private Integer gid; private final List extraArgs = new ArrayList<>(); + private String memory = "2g"; // default to 2g memory limit private DockerRun() {} @@ -75,6 +76,13 @@ public class DockerRun { return this; } + public DockerRun memory(String memoryLimit) { + if (memoryLimit != null) { + this.memory = memoryLimit; + } + return this; + } + public DockerRun extraArgs(String... args) { Collections.addAll(this.extraArgs, args); return this; @@ -88,6 +96,9 @@ public class DockerRun { // Run the container in the background cmd.add("--detach"); + // Limit container memory + cmd.add("--memory " + memory); + this.envVars.forEach((key, value) -> cmd.add("--env " + key + "=\"" + value + "\"")); // The container won't run without configuring discovery