[Connector API] Support hard deletes with new URL param in delete endpoint (#120200)

* [Connector API] Add hard delete support

* Undo accidental change

* undo accidental build gradle change

* Tweak typos

* Update docs/changelog/120200.yaml

* [CI] Auto commit changes from spotless

* Fix yaml test

* Actually skip the feature check since we don't have the feature anyway

---------

Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>
This commit is contained in:
Jedr Blaszyk 2025-01-16 09:45:10 +01:00 committed by GitHub
parent 1ee0be8ba8
commit 0317c1ce36
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 264 additions and 132 deletions

View file

@ -0,0 +1,5 @@
pr: 120200
summary: "[Connector API] Support hard deletes with new URL param in delete endpoint"
area: Extract&Transform
type: feature
issues: []

View file

@ -13,7 +13,7 @@ beta::[]
For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
--
Soft-deletes a connector and removes associated sync jobs.
Deletes a connector and optionally removes associated sync jobs.
Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually.
@ -37,6 +37,9 @@ To get started with Connector APIs, check out <<es-connectors-tutorial-api, our
`<connector_id>`::
(Required, string)
`<hard>`::
(Optional, boolean) If `true`, the connector doc is deleted. If `false`, connector doc is marked as deleted (soft deletion). Defaults to `false`.
`delete_sync_jobs`::
(Optional, boolean) A flag indicating if associated sync jobs should be also removed. Defaults to `false`.

View file

@ -28,6 +28,11 @@
]
},
"params": {
"hard": {
"type": "boolean",
"default": false,
"description": "If true, the connector doc is deleted. If false, connector doc is marked as deleted (soft-deleted)."
},
"delete_sync_jobs": {
"type": "boolean",
"default": false,

View file

@ -280,9 +280,7 @@ setup:
---
"List Connectors - Soft deleted connectors / no deleted":
- requires:
cluster_features: ["connector_soft_deletes"]
reason: Soft deletes were introduced in 9.0 release
- do:
connector.list:
@ -293,9 +291,7 @@ setup:
---
"List Connectors - Single soft deleted connector":
- requires:
cluster_features: ["connector_soft_deletes"]
reason: Soft deletes were introduced in 9.0 release
- do:
connector.delete:
@ -312,11 +308,91 @@ setup:
- match: { count: 3 }
---
"List Connectors - Single hard deleted connector":
- do:
connector.delete:
connector_id: connector-a
hard: true
- do:
connector.list: {}
- match: { count: 2 }
- do:
connector.list:
include_deleted: true
- match: { count: 2 }
---
"List Connectors - All hard deleted connectors":
- do:
connector.delete:
connector_id: connector-a
hard: true
- do:
connector.delete:
connector_id: connector-b
hard: true
- do:
connector.delete:
connector_id: connector-c
hard: true
- do:
connector.list: {}
- match: { count: 0 }
- do:
connector.list:
include_deleted: true
- match: { count: 0 }
---
"List Connectors - 2 hard deleted connectors, 1 soft deleted":
- do:
connector.delete:
connector_id: connector-a
hard: false
- do:
connector.delete:
connector_id: connector-b
hard: true
- do:
connector.delete:
connector_id: connector-c
hard: true
- do:
connector.list: {}
- match: { count: 0 }
- do:
connector.list:
include_deleted: true
- match: { count: 1 }
---
"List Connectors - Soft deleted connectors":
- requires:
cluster_features: ["connector_soft_deletes"]
reason: Soft deletes were introduced in 9.0 release
- do:
connector.delete:
@ -353,9 +429,7 @@ setup:
---
"List Connectors - Soft deleted with from":
- requires:
cluster_features: ["connector_soft_deletes"]
reason: Soft deletes were introduced in 9.0 release
- do:
connector.delete:
@ -387,9 +461,7 @@ setup:
---
"List Connector - Soft deleted with size":
- requires:
cluster_features: ["connector_soft_deletes"]
reason: Soft deletes were introduced in 9.0 release
- do:
connector.delete:

View file

@ -27,6 +27,31 @@ setup:
connector_id: test-connector-to-delete
---
"Delete Connector - Hard Delete":
- do:
connector.put:
connector_id: test-connector-hard-delete
body:
index_name: search-2-test
name: my-hard-delete-connector
language: en
is_native: false
service_type: super-connector
- do:
connector.delete:
connector_id: test-connector-hard-delete
hard: true
- match: { acknowledged: true }
- do:
catch: "missing"
connector.get:
connector_id: test-connector-hard-delete
include_deleted: true
---
"Delete Connector - deletes associated sync jobs":
@ -107,12 +132,9 @@ setup:
connector.delete:
connector_id: test-nonexistent-connector
---
"Delete Connector - Supports soft deletes":
- requires:
cluster_features: ["connector_soft_deletes"]
reason: Soft deletes were introduced in 9.0 release
- do:
connector.delete:

View file

@ -13,6 +13,8 @@ import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.DelegatingActionListener;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
@ -232,16 +234,43 @@ public class ConnectorIndexService {
}
/**
* Soft deletes the {@link Connector} and optionally removes the related instances of {@link ConnectorSyncJob} in the underlying index.
* Deletes the {@link Connector} and optionally removes the related instances of {@link ConnectorSyncJob} in the underlying index.
*
* @param connectorId The id of the {@link Connector}.
* @param hardDelete If set to true, the {@link Connector} is permanently deleted; otherwise, it is soft-deleted.
* @param shouldDeleteSyncJobs The flag indicating if {@link ConnectorSyncJob} should also be deleted.
* @param listener The action listener to invoke on response/failure.
*/
public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, ActionListener<UpdateResponse> listener) {
public void deleteConnector(
String connectorId,
boolean hardDelete,
boolean shouldDeleteSyncJobs,
ActionListener<DocWriteResponse> listener
) {
try {
// ensure that if connector is soft-deleted, deleting it again results in 404
if (hardDelete) {
final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
clientWithOrigin.delete(
deleteRequest,
new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> {
if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
return;
}
if (shouldDeleteSyncJobs) {
new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId(
connectorId,
l.map(r -> deleteResponse)
);
} else {
l.onResponse(deleteResponse);
}
})
);
} else {
getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
WriteRequest.RefreshPolicy.IMMEDIATE
@ -251,7 +280,9 @@ public class ConnectorIndexService {
.id(connectorId)
.source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true))
);
clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
clientWithOrigin.update(
updateRequest,
new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
return;
@ -264,8 +295,10 @@ public class ConnectorIndexService {
} else {
ll.onResponse(updateResponse);
}
})
);
}));
}));
}
} catch (Exception e) {
listener.onFailure(e);
}

View file

@ -9,9 +9,9 @@ package org.elasticsearch.xpack.application.connector.action;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.support.TransportAction;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xcontent.ConstructingObjectParser;
import org.elasticsearch.xcontent.ParseField;
@ -35,19 +35,16 @@ public class DeleteConnectorAction {
public static class Request extends ConnectorActionRequest implements ToXContentObject {
private final String connectorId;
private final boolean hardDelete;
private final boolean deleteSyncJobs;
private static final ParseField CONNECTOR_ID_FIELD = new ParseField("connector_id");
private static final ParseField HARD_DELETE_FIELD = new ParseField("hard");
private static final ParseField DELETE_SYNC_JOB_FIELD = new ParseField("delete_sync_jobs");
public Request(StreamInput in) throws IOException {
super(in);
this.connectorId = in.readString();
this.deleteSyncJobs = in.readBoolean();
}
public Request(String connectorId, boolean deleteSyncJobs) {
public Request(String connectorId, boolean hardDelete, boolean deleteSyncJobs) {
this.connectorId = connectorId;
this.hardDelete = hardDelete;
this.deleteSyncJobs = deleteSyncJobs;
}
@ -66,15 +63,17 @@ public class DeleteConnectorAction {
return connectorId;
}
public boolean isHardDelete() {
return hardDelete;
}
public boolean shouldDeleteSyncJobs() {
return deleteSyncJobs;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(connectorId);
out.writeBoolean(deleteSyncJobs);
TransportAction.localOnly();
}
@Override
@ -82,18 +81,21 @@ public class DeleteConnectorAction {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Request request = (Request) o;
return deleteSyncJobs == request.deleteSyncJobs && Objects.equals(connectorId, request.connectorId);
return hardDelete == request.hardDelete
&& deleteSyncJobs == request.deleteSyncJobs
&& Objects.equals(connectorId, request.connectorId);
}
@Override
public int hashCode() {
return Objects.hash(connectorId, deleteSyncJobs);
return Objects.hash(connectorId, hardDelete, deleteSyncJobs);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(CONNECTOR_ID_FIELD.getPreferredName(), connectorId);
builder.field(HARD_DELETE_FIELD.getPreferredName(), hardDelete);
builder.field(DELETE_SYNC_JOB_FIELD.getPreferredName(), deleteSyncJobs);
builder.endObject();
return builder;
@ -102,10 +104,11 @@ public class DeleteConnectorAction {
private static final ConstructingObjectParser<DeleteConnectorAction.Request, Void> PARSER = new ConstructingObjectParser<>(
"delete_connector_request",
false,
(p) -> new Request((String) p[0], (boolean) p[1])
(p) -> new Request((String) p[0], (boolean) p[1], (boolean) p[2])
);
static {
PARSER.declareString(constructorArg(), CONNECTOR_ID_FIELD);
PARSER.declareBoolean(constructorArg(), HARD_DELETE_FIELD);
PARSER.declareBoolean(constructorArg(), DELETE_SYNC_JOB_FIELD);
}

View file

@ -40,8 +40,9 @@ public class RestDeleteConnectorAction extends BaseRestHandler {
String connectorId = restRequest.param(CONNECTOR_ID_PARAM);
boolean shouldDeleteSyncJobs = restRequest.paramAsBoolean("delete_sync_jobs", false);
boolean hardDelete = restRequest.paramAsBoolean("hard", false);
DeleteConnectorAction.Request request = new DeleteConnectorAction.Request(connectorId, shouldDeleteSyncJobs);
DeleteConnectorAction.Request request = new DeleteConnectorAction.Request(connectorId, hardDelete, shouldDeleteSyncJobs);
return channel -> client.execute(DeleteConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel));
}
}

View file

@ -9,7 +9,7 @@ package org.elasticsearch.xpack.application.connector.action;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.action.support.TransportAction;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.common.util.concurrent.EsExecutors;
@ -18,26 +18,21 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
public class TransportDeleteConnectorAction extends HandledTransportAction<DeleteConnectorAction.Request, AcknowledgedResponse> {
public class TransportDeleteConnectorAction extends TransportAction<DeleteConnectorAction.Request, AcknowledgedResponse> {
protected final ConnectorIndexService connectorIndexService;
@Inject
public TransportDeleteConnectorAction(TransportService transportService, ActionFilters actionFilters, Client client) {
super(
DeleteConnectorAction.NAME,
transportService,
actionFilters,
DeleteConnectorAction.Request::new,
EsExecutors.DIRECT_EXECUTOR_SERVICE
);
super(DeleteConnectorAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
this.connectorIndexService = new ConnectorIndexService(client);
}
@Override
protected void doExecute(Task task, DeleteConnectorAction.Request request, ActionListener<AcknowledgedResponse> listener) {
String connectorId = request.getConnectorId();
boolean hardDelete = request.isHardDelete();
boolean shouldDeleteSyncJobs = request.shouldDeleteSyncJobs();
connectorIndexService.deleteConnector(connectorId, shouldDeleteSyncJobs, listener.map(v -> AcknowledgedResponse.TRUE));
connectorIndexService.deleteConnector(connectorId, hardDelete, shouldDeleteSyncJobs, listener.map(v -> AcknowledgedResponse.TRUE));
}
}

View file

@ -101,7 +101,7 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
assertThat(resp.getId(), equalTo(indexedConnector.getConnectorId()));
}
public void testDeleteConnector() throws Exception {
public void testDeleteConnector_expectSoftDeletionSingle() throws Exception {
int numConnectors = 5;
List<String> connectorIds = new ArrayList<>();
for (int i = 0; i < numConnectors; i++) {
@ -111,11 +111,10 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
}
String connectorIdToDelete = connectorIds.get(0);
UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false);
DocWriteResponse resp = awaitDeleteConnector(connectorIdToDelete, false, false);
assertThat(resp.status(), equalTo(RestStatus.OK));
expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete));
expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false));
expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false, false));
}
public void testDeleteConnector_expectSoftDeletion() throws Exception {
@ -130,13 +129,11 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
}
String connectorIdToDelete = connectorIds.get(0);
UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false);
DocWriteResponse resp = awaitDeleteConnector(connectorIdToDelete, false, false);
assertThat(resp.status(), equalTo(RestStatus.OK));
expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete));
expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false));
Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorIdToDelete);
expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false, false));
Connector softDeletedConnector = awaitGetConnectorIncludeDeleted(connectorIdToDelete);
assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorIdToDelete));
assertThat(softDeletedConnector.getServiceType(), equalTo(connectors.get(0).getServiceType()));
}
@ -150,27 +147,65 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
connectorIds.add(resp.getId());
}
// Delete all of them
for (int i = 0; i < numConnectors; i++) {
String connectorIdToDelete = connectorIds.get(i);
UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false);
DocWriteResponse resp = awaitDeleteConnector(connectorIdToDelete, false, false);
assertThat(resp.status(), equalTo(RestStatus.OK));
}
// Connectors were deleted from main index
for (int i = 0; i < numConnectors; i++) {
String connectorId = connectorIds.get(i);
expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorId));
}
// Soft deleted connectors available in system index
for (int i = 0; i < numConnectors; i++) {
String connectorId = connectorIds.get(i);
Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorId);
Connector softDeletedConnector = awaitGetConnectorIncludeDeleted(connectorId);
assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorId));
}
}
public void testDeleteConnector_expectHardDeletionSingle() throws Exception {
int numConnectors = 3;
List<String> connectorIds = new ArrayList<>();
for (int i = 0; i < numConnectors; i++) {
Connector connector = ConnectorTestUtils.getRandomConnector();
ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector);
connectorIds.add(resp.getId());
}
String connectorIdToDelete = connectorIds.get(0);
DocWriteResponse resp = awaitDeleteConnector(connectorIdToDelete, true, false);
assertThat(resp.status(), equalTo(RestStatus.OK));
expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete));
expectThrows(ResourceNotFoundException.class, () -> awaitGetConnectorIncludeDeleted(connectorIdToDelete));
expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, true, false));
}
public void testDeleteConnector_expectHardDeletionMultipleConnectors() throws Exception {
int numConnectors = 5;
List<String> connectorIds = new ArrayList<>();
for (int i = 0; i < numConnectors; i++) {
Connector connector = ConnectorTestUtils.getRandomConnector();
ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector);
connectorIds.add(resp.getId());
}
for (int i = 0; i < numConnectors; i++) {
String connectorIdToDelete = connectorIds.get(i);
DocWriteResponse resp = awaitDeleteConnector(connectorIdToDelete, true, false);
assertThat(resp.status(), equalTo(RestStatus.OK));
}
for (String connectorId : connectorIds) {
expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorId));
}
for (String connectorId : connectorIds) {
expectThrows(ResourceNotFoundException.class, () -> awaitGetConnectorIncludeDeleted(connectorId));
}
}
public void testUpdateConnectorConfiguration_FullConfiguration() throws Exception {
Connector connector = ConnectorTestUtils.getRandomConnector();
String connectorId = randomUUID();
@ -949,13 +984,14 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
assertThat(updateApiKeyIdRequest.getApiKeySecretId(), equalTo(indexedConnector.getApiKeySecretId()));
}
private UpdateResponse awaitDeleteConnector(String connectorId, boolean deleteConnectorSyncJobs) throws Exception {
private DocWriteResponse awaitDeleteConnector(String connectorId, boolean hardDelete, boolean deleteConnectorSyncJobs)
throws Exception {
CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<UpdateResponse> resp = new AtomicReference<>(null);
final AtomicReference<DocWriteResponse> resp = new AtomicReference<>(null);
final AtomicReference<Exception> exc = new AtomicReference<>(null);
connectorIndexService.deleteConnector(connectorId, deleteConnectorSyncJobs, new ActionListener<>() {
connectorIndexService.deleteConnector(connectorId, hardDelete, deleteConnectorSyncJobs, new ActionListener<>() {
@Override
public void onResponse(UpdateResponse deleteResponse) {
public void onResponse(DocWriteResponse deleteResponse) {
resp.set(deleteResponse);
latch.countDown();
}
@ -1008,7 +1044,7 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
return resp.get();
}
private Connector awaitGetSoftDeletedConnector(String connectorId) throws Exception {
private Connector awaitGetConnectorIncludeDeleted(String connectorId) throws Exception {
return awaitGetConnector(connectorId, true);
}

View file

@ -1,43 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.application.connector.action;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractBWCSerializationTestCase;
import org.elasticsearch.xcontent.XContentParser;
import java.io.IOException;
public class DeleteConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase<DeleteConnectorAction.Request> {
@Override
protected Writeable.Reader<DeleteConnectorAction.Request> instanceReader() {
return DeleteConnectorAction.Request::new;
}
@Override
protected DeleteConnectorAction.Request createTestInstance() {
return new DeleteConnectorAction.Request(randomAlphaOfLengthBetween(1, 10), false);
}
@Override
protected DeleteConnectorAction.Request mutateInstance(DeleteConnectorAction.Request instance) throws IOException {
return randomValueOtherThan(instance, this::createTestInstance);
}
@Override
protected DeleteConnectorAction.Request doParseInstance(XContentParser parser) throws IOException {
return DeleteConnectorAction.Request.parse(parser);
}
@Override
protected DeleteConnectorAction.Request mutateInstanceForVersion(DeleteConnectorAction.Request instance, TransportVersion version) {
return new DeleteConnectorAction.Request(instance.getConnectorId(), instance.shouldDeleteSyncJobs());
}
}