github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-24 15:17:30 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge main into multi-project

Browse source
This commit is contained in:
Yang Wang 2024-12-13 12:15:25 +11:00
commit fda1fa19d4
305 changed files with 5959 additions and 2419 deletions
Download patch file Download diff file Expand all files Collapse all files

4
libs/entitlement/qa/build.gradle
View file

@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.internal-test-artifact'
dependencies {
javaRestTestImplementation project(':libs:entitlement:qa:common')
clusterPlugins project(':libs:entitlement:qa:entitlement-allowed')
clusterPlugins project(':libs:entitlement:qa:entitlement-allowed-nonmodular')
clusterModules project(':libs:entitlement:qa:entitlement-allowed')
clusterModules project(':libs:entitlement:qa:entitlement-allowed-nonmodular')
clusterPlugins project(':libs:entitlement:qa:entitlement-denied')
clusterPlugins project(':libs:entitlement:qa:entitlement-denied-nonmodular')
}

4
libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java
View file

@ -28,8 +28,8 @@ public class EntitlementsAllowedIT extends ESRestTestCase {
@ClassRule
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.plugin("entitlement-allowed")
.plugin("entitlement-allowed-nonmodular")
.module("entitlement-allowed")
.module("entitlement-allowed-nonmodular")
.systemProperty("es.entitlements.enabled", "true")
.setting("xpack.security.enabled", "false")
.build();

11
libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java
View file

@ -15,7 +15,6 @@ import com.sun.tools.attach.AttachNotSupportedException;
import com.sun.tools.attach.VirtualMachine;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
import org.elasticsearch.logging.LogManager;
import org.elasticsearch.logging.Logger;
@ -29,7 +28,9 @@ import java.util.function.Function;
public class EntitlementBootstrap {
public record BootstrapArgs(Collection<Tuple<Path, Boolean>> pluginData, Function<Class<?>, String> pluginResolver) {}
public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {}
public record BootstrapArgs(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {}
private static BootstrapArgs bootstrapArgs;
@ -40,11 +41,11 @@ public class EntitlementBootstrap {
/**
* Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid
* policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}.
* @param pluginData a collection of (plugin path, boolean), that holds the paths of all the installed Elasticsearch modules and
* plugins, and whether they are Java modular or not.
* @param pluginData a collection of (plugin path, boolean, boolean), that holds the paths of all the installed Elasticsearch modules
* and plugins, whether they are Java modular or not, and whether they are Elasticsearch modules or external plugins.
* @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
*/
public static void bootstrap(Collection<Tuple<Path, Boolean>> pluginData, Function<Class<?>, String> pluginResolver) {
public static void bootstrap(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {
logger.debug("Loading entitlement agent");
if (EntitlementBootstrap.bootstrapArgs != null) {
throw new IllegalStateException("plugin data is already set");

21
libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java
View file

@ -9,7 +9,6 @@
package org.elasticsearch.entitlement.initialization;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.core.internal.provider.ProviderLocator;
import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
import org.elasticsearch.entitlement.bridge.EntitlementChecker;
@ -96,25 +95,25 @@ public class EntitlementInitialization {
return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver());
}
private static Map<String, Policy> createPluginPolicies(Collection<Tuple<Path, Boolean>> pluginData) throws IOException {
private static Map<String, Policy> createPluginPolicies(Collection<EntitlementBootstrap.PluginData> pluginData) throws IOException {
Map<String, Policy> pluginPolicies = new HashMap<>(pluginData.size());
for (Tuple<Path, Boolean> entry : pluginData) {
Path pluginRoot = entry.v1();
boolean isModular = entry.v2();
for (var entry : pluginData) {
Path pluginRoot = entry.pluginPath();
String pluginName = pluginRoot.getFileName().toString();
final Policy policy = loadPluginPolicy(pluginRoot, isModular, pluginName);
final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin());
pluginPolicies.put(pluginName, policy);
}
return pluginPolicies;
}
private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName) throws IOException {
private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin)
throws IOException {
Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
final Set<String> moduleNames = getModuleNames(pluginRoot, isModular);
final Policy policy = parsePolicyIfExists(pluginName, policyFile);
final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin);
// TODO: should this check actually be part of the parser?
for (Scope scope : policy.scopes) {
@ -125,9 +124,9 @@ public class EntitlementInitialization {
return policy;
}
private static Policy parsePolicyIfExists(String pluginName, Path policyFile) throws IOException {
private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException {
if (Files.exists(policyFile)) {
return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName).parsePolicy();
return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy();
}
return new Policy(pluginName, List.of());
}

8
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java
View file

@ -33,4 +33,12 @@ public @interface ExternalEntitlement {
* have to match the parameter names of the constructor.
*/
String[] parameterNames() default {};
/**
* This flag indicates if this Entitlement can be used in external plugins,
* or if it can be used only in Elasticsearch modules ("internal" plugins).
* Using an entitlement that is not {@code pluginsAccessible} in an external
* plugin policy will throw in exception while parsing.
*/
boolean esModulesOnly() default true;
}

2
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java
View file

@ -26,7 +26,7 @@ public class FileEntitlement implements Entitlement {
private final String path;
private final int actions;
@ExternalEntitlement(parameterNames = { "path", "actions" })
@ExternalEntitlement(parameterNames = { "path", "actions" }, esModulesOnly = false)
public FileEntitlement(String path, List<String> actionsList) {
this.path = path;
int actionsInt = 0;

47
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
View file

@ -18,7 +18,6 @@ import org.elasticsearch.logging.Logger;
import java.lang.module.ModuleFinder;
import java.lang.module.ModuleReference;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.List;
@ -56,8 +55,8 @@ public class PolicyManager {
final Map<Module, ModuleEntitlements> moduleEntitlementsMap = new HashMap<>();
protected final Policy serverPolicy;
protected final Map<String, Policy> pluginPolicies;
protected final Map<String, List<Entitlement>> serverEntitlements;
protected final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
private final Function<Class<?>, String> pluginResolver;
public static final String ALL_UNNAMED = "ALL-UNNAMED";
@ -79,19 +78,16 @@ public class PolicyManager {
}
public PolicyManager(Policy defaultPolicy, Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
this.serverPolicy = Objects.requireNonNull(defaultPolicy);
this.pluginPolicies = Collections.unmodifiableMap(Objects.requireNonNull(pluginPolicies));
this.serverEntitlements = buildScopeEntitlementsMap(Objects.requireNonNull(defaultPolicy));
this.pluginsEntitlements = Objects.requireNonNull(pluginPolicies)
.entrySet()
.stream()
.collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue())));
this.pluginResolver = pluginResolver;
}
private static List<Entitlement> lookupEntitlementsForModule(Policy policy, String moduleName) {
for (int i = 0; i < policy.scopes.size(); ++i) {
var scope = policy.scopes.get(i);
if (scope.name.equals(moduleName)) {
return scope.entitlements;
}
}
return null;
private static Map<String, List<Entitlement>> buildScopeEntitlementsMap(Policy policy) {
return policy.scopes.stream().collect(Collectors.toUnmodifiableMap(scope -> scope.name, scope -> scope.entitlements));
}
public void checkExitVM(Class<?> callerClass) {
@ -141,21 +137,21 @@ public class PolicyManager {
if (isServerModule(requestingModule)) {
var scopeName = requestingModule.getName();
return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverPolicy, scopeName);
return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverEntitlements, scopeName);
}
// plugins
var pluginName = pluginResolver.apply(callerClass);
if (pluginName != null) {
var pluginPolicy = pluginPolicies.get(pluginName);
if (pluginPolicy != null) {
var pluginEntitlements = pluginsEntitlements.get(pluginName);
if (pluginEntitlements != null) {
final String scopeName;
if (requestingModule.isNamed() == false) {
scopeName = ALL_UNNAMED;
} else {
scopeName = requestingModule.getName();
}
return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginPolicy, scopeName);
return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginEntitlements, scopeName);
}
}
@ -167,15 +163,20 @@ public class PolicyManager {
return Strings.format("Missing entitlement policy: caller [%s], module [%s]", callerClass, requestingModule.getName());
}
private ModuleEntitlements getModuleEntitlementsOrThrow(Class<?> callerClass, Module module, Policy policy, String moduleName) {
var entitlements = lookupEntitlementsForModule(policy, moduleName);
private ModuleEntitlements getModuleEntitlementsOrThrow(
Class<?> callerClass,
Module module,
Map<String, List<Entitlement>> scopeEntitlements,
String moduleName
) {
var entitlements = scopeEntitlements.get(moduleName);
if (entitlements == null) {
// Module without entitlements - remember we don't have any
moduleEntitlementsMap.put(module, ModuleEntitlements.NONE);
throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, module));
}
// We have a policy for this module
var classEntitlements = createClassEntitlements(entitlements);
var classEntitlements = new ModuleEntitlements(entitlements);
moduleEntitlementsMap.put(module, classEntitlements);
return classEntitlements;
}
@ -184,10 +185,6 @@ public class PolicyManager {
return requestingModule.isNamed() && requestingModule.getLayer() == ModuleLayer.boot();
}
private ModuleEntitlements createClassEntitlements(List<Entitlement> entitlements) {
return new ModuleEntitlements(entitlements);
}
private static Module requestingModule(Class<?> callerClass) {
if (callerClass != null) {
Module callerModule = callerClass.getModule();
@ -222,6 +219,6 @@ public class PolicyManager {
@Override
public String toString() {
return "PolicyManager{" + "serverPolicy=" + serverPolicy + ", pluginPolicies=" + pluginPolicies + '}';
return "PolicyManager{" + "serverEntitlements=" + serverEntitlements + ", pluginsEntitlements=" + pluginsEntitlements + '}';
}
}

8
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java
View file

@ -39,6 +39,7 @@ public class PolicyParser {
protected final XContentParser policyParser;
protected final String policyName;
private final boolean isExternalPlugin;
static String getEntitlementTypeName(Class<? extends Entitlement> entitlementClass) {
var entitlementClassName = entitlementClass.getSimpleName();
@ -56,9 +57,10 @@ public class PolicyParser {
.collect(Collectors.joining("_"));
}
public PolicyParser(InputStream inputStream, String policyName) throws IOException {
public PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin) throws IOException {
this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream));
this.policyName = policyName;
this.isExternalPlugin = isExternalPlugin;
}
public Policy parsePolicy() {
@ -125,6 +127,10 @@ public class PolicyParser {
throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]");
}
if (entitlementMetadata.esModulesOnly() && isExternalPlugin) {
throw newPolicyParserException("entitlement type [" + entitlementType + "] is allowed only on modules");
}
Class<?>[] parameterTypes = entitlementConstructor.getParameterTypes();
String[] parametersNames = entitlementMetadata.parameterNames();

21
libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java
View file

@ -19,7 +19,7 @@ public class PolicyParserFailureTests extends ESTestCase {
public void testParserSyntaxFailures() {
PolicyParserException ppe = expectThrows(
PolicyParserException.class,
() -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml")
() -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false)
.parsePolicy()
);
assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object <scope name>", ppe.getMessage());
@ -29,7 +29,7 @@ public class PolicyParserFailureTests extends ESTestCase {
PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
entitlement-module-name:
- does_not_exist: {}
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
assertEquals(
"[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name]: "
+ "unknown entitlement type [does_not_exist]",
@ -41,7 +41,7 @@ public class PolicyParserFailureTests extends ESTestCase {
PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
entitlement-module-name:
- file: {}
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
assertEquals(
"[2:12] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
+ "for entitlement type [file]: missing entitlement parameter [path]",
@ -52,7 +52,7 @@ public class PolicyParserFailureTests extends ESTestCase {
entitlement-module-name:
- file:
path: test-path
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
assertEquals(
"[4:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
+ "for entitlement type [file]: missing entitlement parameter [actions]",
@ -68,11 +68,22 @@ public class PolicyParserFailureTests extends ESTestCase {
actions:
- read
extra: test
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
assertEquals(
"[7:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
+ "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}",
ppe.getMessage()
);
}
public void testEntitlementIsNotForExternalPlugins() {
PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
entitlement-module-name:
- create_class_loader
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", true).parsePolicy());
assertEquals(
"[2:5] policy parsing error for [test-failure-policy.yaml]: entitlement type [create_class_loader] is allowed only on modules",
ppe.getMessage()
);
}
}

14
libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java
View file

@ -37,7 +37,17 @@ public class PolicyParserTests extends ESTestCase {
}
public void testPolicyBuilder() throws IOException {
Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml")
Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false)
.parsePolicy();
Policy builtPolicy = new Policy(
"test-policy.yaml",
List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write")))))
);
assertEquals(parsedPolicy, builtPolicy);
}
public void testPolicyBuilderOnExternalPlugin() throws IOException {
Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true)
.parsePolicy();
Policy builtPolicy = new Policy(
"test-policy.yaml",
@ -50,7 +60,7 @@ public class PolicyParserTests extends ESTestCase {
Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream("""
entitlement-module-name:
- create_class_loader
""".getBytes(StandardCharsets.UTF_8)), "test-policy.yaml").parsePolicy();
""".getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
Policy builtPolicy = new Policy(
"test-policy.yaml",
List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement())))