github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 09:28:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

Apply precommit checks for non productive projects (#85533)

Browse source 
Ensure projects with only yaml, java or cluster tests also apply precommit checks.
We only apply testingconventions now for projects with existing src test folder
as the TestingConventionsTask is incompatible with projects with no test sourceSet.
This is a prerequisite to port more projects away from using StandaloneRestTestPlugin
and RestTestPlugin in favor of yaml, java or cluster tests with dedicated sourceSets.

Also we fix deprecation warnings for forbiddenPattern and filePermissions tasks about
implicit non declared dependencies on resources tasks
This commit is contained in:
Rene Groeschke 2022-04-20 08:54:56 +02:00 • committed by GitHub
parent 8d21fe9d9f
commit 718a241449
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 161 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

28
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalPluginBuildPlugin.java
View file

@ -8,6 +8,7 @@
package org.elasticsearch.gradle.internal;
import org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin;
import org.elasticsearch.gradle.internal.precommit.TestingConventionsTasks;
import org.gradle.api.Project;
@ -17,15 +18,22 @@ public class InternalPluginBuildPlugin implements InternalPlugin {
project.getPluginManager().apply(BuildPlugin.class);
project.getPluginManager().apply(BaseInternalPluginBuildPlugin.class);
project.getTasks().withType(TestingConventionsTasks.class).named("testingConventions").configure(t -> {
t.getNaming().clear();
t.getNaming()
.create("Tests", testingConventionRule -> testingConventionRule.baseClass("org.apache.lucene.tests.util.LuceneTestCase"));
t.getNaming().create("IT", testingConventionRule -> {
testingConventionRule.baseClass("org.elasticsearch.test.ESIntegTestCase");
testingConventionRule.baseClass("org.elasticsearch.test.rest.ESRestTestCase");
testingConventionRule.baseClass("org.elasticsearch.test.ESSingleNodeTestCase");
});
});
project.getPlugins()
.withType(
TestingConventionsPrecommitPlugin.class,
plugin -> project.getTasks().withType(TestingConventionsTasks.class).named("testingConventions").configure(t -> {
t.getNaming().clear();
t.getNaming()
.create(
"Tests",
testingConventionRule -> testingConventionRule.baseClass("org.apache.lucene.tests.util.LuceneTestCase")
);
t.getNaming().create("IT", testingConventionRule -> {
testingConventionRule.baseClass("org.elasticsearch.test.ESIntegTestCase");
testingConventionRule.baseClass("org.elasticsearch.test.rest.ESRestTestCase");
testingConventionRule.baseClass("org.elasticsearch.test.ESSingleNodeTestCase");
});
})
);
}
}

21
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java
View file

@ -32,16 +32,19 @@ public class FilePermissionsPrecommitPlugin extends PrecommitPlugin implements I
@Override
public TaskProvider<? extends Task> createTask(Project project) {
return project.getTasks()
.register(
FILEPERMISSIONS_TASK_NAME,
FilePermissionsTask.class,
filePermissionsTask -> filePermissionsTask.getSources()
.addAll(
providerFactory.provider(
() -> GradleUtils.getJavaSourceSets(project).stream().map(s -> s.getAllSource()).collect(Collectors.toList())
)
return project.getTasks().register(FILEPERMISSIONS_TASK_NAME, FilePermissionsTask.class, t -> {
t.getSources()
.addAll(
providerFactory.provider(
() -> GradleUtils.getJavaSourceSets(project).stream().map(s -> s.getAllSource()).collect(Collectors.toList())
)
);
t.dependsOn(
GradleUtils.getJavaSourceSets(project)
.stream()
.map(sourceSet -> sourceSet.getProcessResourcesTaskName())
.collect(Collectors.toList())
);
});
}
}

6
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java
View file

@ -39,6 +39,12 @@ public class ForbiddenPatternsPrecommitPlugin extends PrecommitPlugin implements
() -> GradleUtils.getJavaSourceSets(project).stream().map(s -> s.getAllSource()).collect(Collectors.toList())
)
);
forbiddenPatternsTask.dependsOn(
GradleUtils.getJavaSourceSets(project)
.stream()
.map(sourceSet -> sourceSet.getProcessResourcesTaskName())
.collect(Collectors.toList())
);
forbiddenPatternsTask.getRootDir().set(project.getRootDir());
});
}

1
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java
View file

@ -66,6 +66,7 @@ public abstract class ForbiddenPatternsTask extends DefaultTask {
.exclude("**/*.zip")
.exclude("**/*.jks")
.exclude("**/*.crt")
.exclude("**/*.p12")
.exclude("**/*.keystore")
.exclude("**/*.png")
// vim swap file - included here to stop the build falling over if you happen to have a file open :-|

13
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java
View file

@ -25,13 +25,22 @@ public class InternalPrecommitTasks {
project.getPluginManager().apply(ForbiddenPatternsPrecommitPlugin.class);
project.getPluginManager().apply(LicenseHeadersPrecommitPlugin.class);
project.getPluginManager().apply(FilePermissionsPrecommitPlugin.class);
project.getPluginManager().apply(TestingConventionsPrecommitPlugin.class);
project.getPluginManager().apply(LoggerUsagePrecommitPlugin.class);
project.getPluginManager().apply(JarHellPrecommitPlugin.class);
// TestingConventionsPlugin is incompatible with projects without
// test source sets. We wanna remove this plugin once we moved away from
// StandaloneRestTest plugin and RestTestPlugin. For apply this plugin only
// when tests are available.
// Long Term we want remove the need for most of this plugins functionality
// and not rely on multiple test tasks against a sourceSet.
if (project.file("src/test").exists()) {
project.getPluginManager().apply(TestingConventionsPrecommitPlugin.class);
}
// tasks with just tests don't need certain tasks to run, so this flag makes adding
// the task optional
if (withProductiveCode) {
project.getPluginManager().apply(JarHellPrecommitPlugin.class);
project.getPluginManager().apply(ThirdPartyAuditPrecommitPlugin.class);
project.getPluginManager().apply(DependencyLicensesPrecommitPlugin.class);
project.getPluginManager().apply(SplitPackagesAuditPrecommitPlugin.class);

2
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java
View file

@ -12,6 +12,7 @@ import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin;
import org.elasticsearch.gradle.internal.FixtureStop;
import org.elasticsearch.gradle.internal.InternalTestClustersPlugin;
import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
@ -48,6 +49,7 @@ public class RestTestBasePlugin implements Plugin<Project> {
project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
project.getPluginManager().apply(InternalTestClustersPlugin.class);
InternalPrecommitTasks.create(project, false);
project.getTasks().withType(RestIntegTestTask.class).configureEach(restIntegTestTask -> {
@SuppressWarnings("unchecked")
NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project

1
build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java
View file

@ -138,6 +138,7 @@ public class PluginBuildPlugin implements Plugin<Project> {
});
}
});
final var buildProperties = project.getTasks().register("pluginProperties", Copy.class, copy -> {
copy.dependsOn(copyPluginPropertiesTemplate);
copy.from(templateFile);

5
modules/runtime-fields-common/build.gradle
View file

@ -20,7 +20,4 @@ dependencies {
compileOnly project(':modules:lang-painless:spi')
api project(':libs:elasticsearch-grok')
api project(':libs:elasticsearch-dissect')
}
//this plugin is here only for the painless extension, there are no unit tests
tasks.named("testingConventions").configure { enabled = false }
}

1
rest-api-spec/build.gradle
View file

@ -40,7 +40,6 @@ testClusters.configureEach {
}
tasks.named("test").configure { enabled = false }
tasks.named("jarHell").configure { enabled = false }
tasks.named("yamlRestTestV7CompatTransform").configure { task ->

3
test/x-content/build.gradle
View file

@ -32,9 +32,6 @@ tasks.named("dependencyLicenses").configure { enabled = false }
tasks.named("dependenciesInfo").configure { enabled = false }
tasks.named("dependenciesGraph").configure { enabled = false }
// no tests of the tests, yet
tasks.named("testingConventions").configure { enabled = false }
tasks.named("thirdPartyAudit").configure {
ignoreMissingClasses(
// classes are missing

7
x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
View file

@ -1,3 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.eql;
import org.elasticsearch.common.settings.Settings;

19
x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java
View file

@ -94,7 +94,7 @@ public class SearchableSnapshotActionIT extends ESRestTestCase {
createComposableTemplate(
client(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
dataStream,
new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), null, null)
);
@ -125,7 +125,12 @@ public class SearchableSnapshotActionIT extends ESRestTestCase {
createSnapshotRepo(client(), snapshotRepo, randomBoolean());
createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo, true));
createComposableTemplate(client(), randomAlphaOfLengthBetween(5, 10).toLowerCase(), dataStream, new Template(null, null, null));
createComposableTemplate(
client(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
dataStream,
new Template(null, null, null)
);
for (int i = 0; i < randomIntBetween(5, 10); i++) {
indexDocument(client(), dataStream, true);
@ -199,7 +204,7 @@ public class SearchableSnapshotActionIT extends ESRestTestCase {
createComposableTemplate(
client(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
dataStream,
new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), null, null)
);
@ -289,7 +294,7 @@ public class SearchableSnapshotActionIT extends ESRestTestCase {
createComposableTemplate(
client(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
dataStream,
new Template(
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(LifecycleSettings.LIFECYCLE_NAME, policy).build(),
@ -368,7 +373,7 @@ public class SearchableSnapshotActionIT extends ESRestTestCase {
createComposableTemplate(
client(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
dataStream,
new Template(
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(LifecycleSettings.LIFECYCLE_NAME, policy).build(),
@ -628,7 +633,7 @@ public class SearchableSnapshotActionIT extends ESRestTestCase {
createComposableTemplate(
client(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
dataStream,
new Template(
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(LifecycleSettings.LIFECYCLE_NAME, policy).build(),
@ -675,7 +680,7 @@ public class SearchableSnapshotActionIT extends ESRestTestCase {
createComposableTemplate(
client(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(),
randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
dataStream,
new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), null, null)
);

164
x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java
View file

@ -241,89 +241,89 @@ public class TrainedModelIT extends ESRestTestCase {
private void putRegressionModel(String modelId) throws IOException {
String modelConfig = """
{
"definition": {
"trained_model": {
"ensemble": {
"feature_names": ["field.foo", "field.bar", "animal_cat", "animal_dog"],
"trained_models": [{
"tree": {
"feature_names": ["field.foo", "field.bar", "animal_cat", "animal_dog"],
"tree_structure": [{
"threshold": 0.5,
"split_feature": 0,
"node_index": 0,
"left_child": 1,
"right_child": 2
}, {
"node_index": 1,
"leaf_value": [0.3]
}, {
"threshold": 0.0,
"split_feature": 3,
"node_index": 2,
"left_child": 3,
"right_child": 4
}, {
"node_index": 3,
"leaf_value": [0.1]
}, {
"node_index": 4,
"leaf_value": [0.2]
}]
}
}, {
"tree": {
"feature_names": ["field.foo", "field.bar", "animal_cat", "animal_dog"],
"tree_structure": [{
"threshold": 1.0,
"split_feature": 2,
"node_index": 0,
"left_child": 1,
"right_child": 2
}, {
"node_index": 1,
"leaf_value": [1.5]
}, {
"node_index": 2,
"leaf_value": [0.9]
}]
}
}, {
"tree": {
"feature_names": ["field.foo", "field.bar", "animal_cat", "animal_dog"],
"tree_structure": [{
"threshold": 0.2,
"split_feature": 1,
"node_index": 0,
"left_child": 1,
"right_child": 2
}, {
"node_index": 1,
"leaf_value": [1.5]
}, {
"node_index": 2,
"leaf_value": [0.9]
}]
}
}],
"aggregate_output": {
"weighted_sum": {
"weights": [0.5, 0.5, 0.5]
}
},
"target_type": "regression"
}
},
"preprocessors": []
},
"input": {
"field_names": ["col1", "col2", "col3"]
},
"inference_config": {
"regression": {}
}
{
"definition": {
"trained_model": {
"ensemble": {
"feature_names": ["field.foo", "field.bar", "animal_cat", "animal_dog"],
"trained_models": [{
"tree": {
"feature_names": ["field.foo", "field.bar", "animal_cat", "animal_dog"],
"tree_structure": [{
"threshold": 0.5,
"split_feature": 0,
"node_index": 0,
"left_child": 1,
"right_child": 2
}, {
"node_index": 1,
"leaf_value": [0.3]
}, {
"threshold": 0.0,
"split_feature": 3,
"node_index": 2,
"left_child": 3,
"right_child": 4
}, {
"node_index": 3,
"leaf_value": [0.1]
}, {
"node_index": 4,
"leaf_value": [0.2]
}]
}
}, {
"tree": {
"feature_names": ["field.foo", "field.bar", "animal_cat", "animal_dog"],
"tree_structure": [{
"threshold": 1.0,
"split_feature": 2,
"node_index": 0,
"left_child": 1,
"right_child": 2
}, {
"node_index": 1,
"leaf_value": [1.5]
}, {
"node_index": 2,
"leaf_value": [0.9]
}]
}
}, {
"tree": {
"feature_names": ["field.foo", "field.bar", "animal_cat", "animal_dog"],
"tree_structure": [{
"threshold": 0.2,
"split_feature": 1,
"node_index": 0,
"left_child": 1,
"right_child": 2
}, {
"node_index": 1,
"leaf_value": [1.5]
}, {
"node_index": 2,
"leaf_value": [0.9]
}]
}
}],
"aggregate_output": {
"weighted_sum": {
"weights": [0.5, 0.5, 0.5]
}
},
"target_type": "regression"
}
},
"preprocessors": []
},
"input": {
"field_names": ["col1", "col2", "col3"]
},
"inference_config": {
"regression": {}
}
}
""";
Request model = new Request("PUT", "_ml/trained_models/" + modelId);

2
x-pack/plugin/searchable-snapshots/preallocate/build.gradle
View file

@ -12,5 +12,3 @@ archivesBaseName = 'preallocate'
dependencies {
compileOnly project(":server")
}
tasks.named("testingConventions").configure { enabled = false }

2
x-pack/plugin/sql/qa/jdbc/security/build.gradle
View file

@ -47,6 +47,4 @@ subprojects {
nonInputProperties.systemProperty 'tests.audit.yesterday.logfile',
"${-> testClusters.integTest.singleNode().getAuditLog().getParentFile()}/integTest_audit-${new Date().format('yyyy-MM-dd')}.json"
}
tasks.named("testingConventions").configure { enabled = false }
}

1
x-pack/plugin/sql/qa/server/security/build.gradle
View file

@ -53,5 +53,4 @@ subprojects {
"${-> testClusters.integTest.singleNode().getAuditLog().getParentFile()}/integTest_audit-${new Date().format('yyyy-MM-dd')}-1.json.gz"
}
tasks.named("testingConventions").configure { enabled = false }
}

2
x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/AbstractXPackRestTest.java
View file

@ -42,7 +42,7 @@ import static java.util.Collections.singletonMap;
/** Runs rest tests against external cluster */
// TODO: Remove this timeout increase once this test suite is broken up
@TimeoutSuite(millis = 60 * TimeUnits.MINUTE)
public class AbstractXPackRestTest extends ESClientYamlSuiteTestCase {
public abstract class AbstractXPackRestTest extends ESClientYamlSuiteTestCase {
private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue(
"x_pack_rest_user",
SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING

2
x-pack/plugin/transform/qa/multi-node-tests/build.gradle
View file

@ -11,7 +11,7 @@ File keystoreDir = new File(project.buildDir, 'keystore')
File nodeKey = file("$keystoreDir/testnode.pem")
File nodeCert = file("$keystoreDir/testnode.crt")
// Add key and certs to test classpath: it expects it there
tasks.register("copyKeyCerts", Copy) {
def copyKeyCerts = tasks.register("copyKeyCerts", Copy) {
from(project(':x-pack:plugin:core').file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/')) {
include 'testnode.crt', 'testnode.pem'
}

3
x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/LatestContinuousIT.java
View file

@ -21,6 +21,7 @@ import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
@ -140,7 +141,7 @@ public class LatestContinuousIT extends ContinuousTestCase {
String transformBucketKey = eventFieldValue != null
// The bucket key in source can be either an ordinary field or a runtime field. When it is runtime field, simulate its
// script ("toUpperCase()") here.
? "event".equals(eventField) ? eventFieldValue : eventFieldValue.toUpperCase()
? "event".equals(eventField) ? eventFieldValue : eventFieldValue.toUpperCase(Locale.ROOT)
: MISSING_BUCKET_KEY;
// Verify that the results from the aggregation and the results from the transform are the same.

6
x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java
View file

@ -375,7 +375,8 @@ public class TransformContinuousIT extends TransformRestTestCase {
.startObject("script")
.field(
"source",
"if (doc['metric-timestamp'].size()!=0) {emit(doc['metric-timestamp'].value.minus(5, ChronoUnit.MINUTES).toInstant().toEpochMilli())}"
"if (doc['metric-timestamp'].size()!=0) {emit(doc['metric-timestamp'].value"
+ ".minus(5, ChronoUnit.MINUTES).toInstant().toEpochMilli())}"
)
.endObject()
.endObject()
@ -384,7 +385,8 @@ public class TransformContinuousIT extends TransformRestTestCase {
.startObject("script")
.field(
"source",
"if (doc['some-timestamp'].size()!=0) {emit(doc['some-timestamp'].value.minus(10, ChronoUnit.MINUTES).toInstant().toEpochMilli())}"
"if (doc['some-timestamp'].size()!=0) {emit(doc['some-timestamp'].value"
+ ".minus(10, ChronoUnit.MINUTES).toInstant().toEpochMilli())}"
)
.endObject()
.endObject();