[Connectors] Use Connector API to manage connector filtering (#183148)

Use[ Connector API update filtering
endpoint](https://www.elastic.co/guide/en/elasticsearch/reference/master/update-connector-filtering-api.html)
to manage connector filtering in Kibana.

packages/kbn-search-connectors/lib/update_filtering.test.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+
+import { errors } from '@elastic/elasticsearch';
+
+import { updateFiltering } from './update_filtering';
+import { FilteringRule, FilteringRules, FilteringValidationState } from '../types/connectors';
+
+describe('updateFiltering lib function', () => {
+  const mockClient = {
+    transport: {
+      request: jest.fn(),
+    },
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+    jest.setSystemTime(new Date('2024-05-25T12:00:00.000Z'));
+  });
+
+  it('should activate connector filtering draft', async () => {
+    const filteringRule: FilteringRule = {
+      updated_at: '2024-05-10T12:14:14.291Z',
+      created_at: '2024-05-09T14:37:56.090Z',
+      field: 'name',
+      id: 'my-rule',
+      order: 0,
+      policy: 'exclude',
+      rule: 'regex',
+      value: 'test.*',
+    };
+
+    const draftToActivate: FilteringRules = {
+      advanced_snippet: {
+        created_at: '2024-05-25T12:00:00.000Z',
+        updated_at: '2024-05-25T12:00:00.000Z',
+        value: {},
+      },
+      rules: [
+        {
+          ...filteringRule,
+          updated_at: '2024-05-25T12:00:00.000Z',
+        },
+      ],
+      validation: {
+        errors: [],
+        state: FilteringValidationState.VALID,
+      },
+    };
+
+    mockClient.transport.request.mockImplementationOnce(() => ({ result: 'updated' }));
+    mockClient.transport.request.mockImplementationOnce(() => ({
+      filtering: [{ active: draftToActivate }],
+    }));
+
+    await expect(
+      updateFiltering(mockClient as unknown as ElasticsearchClient, 'connectorId')
+    ).resolves.toEqual(draftToActivate);
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'PUT',
+      path: '/_connector/connectorId/_filtering/activate',
+    });
+  });
+
+  it('should not index document if there is no connector', async () => {
+    mockClient.transport.request.mockImplementationOnce(() => {
+      return Promise.reject(
+        new errors.ResponseError({
+          statusCode: 404,
+          body: {
+            error: {
+              type: `document_missing_exception`,
+            },
+          },
+        } as any)
+      );
+    });
+    await expect(
+      updateFiltering(mockClient as unknown as ElasticsearchClient, 'connectorId')
+    ).rejects.toEqual(
+      new errors.ResponseError({
+        statusCode: 404,
+        body: {
+          error: {
+            type: `document_missing_exception`,
+          },
+        },
+      } as any)
+    );
+  });
+});

packages/kbn-search-connectors/lib/update_filtering.ts

@@ -6,59 +6,24 @@
  * Side Public License, v 1.
  */
 
+import { Result } from '@elastic/elasticsearch/lib/api/types';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 
-import { CONNECTORS_INDEX } from '..';
 import { fetchConnectorById } from './fetch_connectors';
-import {
-  Connector,
-  FilteringRule,
-  FilteringRules,
-  FilteringValidationState,
-} from '../types/connectors';
+import { FilteringRules } from '../types/connectors';
 
 export const updateFiltering = async (
   client: ElasticsearchClient,
-  connectorId: string,
-  {
-    advancedSnippet,
-    filteringRules,
-  }: {
-    advancedSnippet: string;
-    filteringRules: FilteringRule[];
-  }
+  connectorId: string
 ): Promise<FilteringRules | undefined> => {
-  const now = new Date().toISOString();
-  const parsedAdvancedSnippet: Record<string, unknown> = advancedSnippet
-    ? JSON.parse(advancedSnippet)
-    : {};
-  const parsedFilteringRules = filteringRules.map((filteringRule) => ({
-    ...filteringRule,
-    created_at: filteringRule.created_at ? filteringRule.created_at : now,
-    updated_at: now,
-  }));
-  const connector = await fetchConnectorById(client, connectorId);
-  if (!connector) {
-    throw new Error(`Could not find connector with id ${connectorId}`);
-  }
-  const active: FilteringRules = {
-    advanced_snippet: {
-      created_at: connector.filtering[0].active.advanced_snippet.created_at || now,
-      updated_at: now,
-      value: parsedAdvancedSnippet,
-    },
-    rules: parsedFilteringRules,
-    validation: {
-      errors: [],
-      state: FilteringValidationState.VALID,
-    },
-  };
-
-  const result = await client.update<Connector>({
-    doc: { ...connector, filtering: [{ ...connector.filtering[0], active, draft: active }] },
-    id: connectorId,
-    index: CONNECTORS_INDEX,
+  const activateDraftFilteringResult = await client.transport.request<{ result: Result }>({
+    method: 'PUT',
+    path: `/_connector/${connectorId}/_filtering/activate`,
   });
 
-  return result.result === 'updated' ? active : undefined;
+  if (activateDraftFilteringResult.result === 'updated') {
+    const connector = await fetchConnectorById(client, connectorId);
+    return connector?.filtering?.[0]?.active;
+  }
+  return undefined;
 };

packages/kbn-search-connectors/lib/update_filtering_draft.test.ts

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+
+import { errors } from '@elastic/elasticsearch';
+
+import { updateFilteringDraft } from './update_filtering_draft';
+import { FilteringRule, FilteringRules, FilteringValidationState } from '../types/connectors';
+
+describe('updateFilteringDraft lib function', () => {
+  const mockClient = {
+    transport: {
+      request: jest.fn(),
+    },
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+    jest.setSystemTime(new Date('2024-05-25T12:00:00.000Z'));
+  });
+
+  it('should update connector filtering draft', async () => {
+    const filteringRule: FilteringRule = {
+      updated_at: '2024-05-10T12:14:14.291Z',
+      created_at: '2024-05-09T14:37:56.090Z',
+      field: 'name',
+      id: 'my-rule',
+      order: 0,
+      policy: 'exclude',
+      rule: 'regex',
+      value: 'test.*',
+    };
+
+    const draft: FilteringRules = {
+      advanced_snippet: {
+        created_at: '2024-05-25T12:00:00.000Z',
+        updated_at: '2024-05-25T12:00:00.000Z',
+        value: {},
+      },
+      rules: [
+        {
+          ...filteringRule,
+          updated_at: '2024-05-25T12:00:00.000Z',
+        },
+      ],
+      validation: {
+        errors: [],
+        state: FilteringValidationState.EDITED,
+      },
+    };
+
+    mockClient.transport.request.mockImplementationOnce(() => ({ result: 'updated' }));
+    mockClient.transport.request.mockImplementationOnce(() => ({ filtering: [{ draft }] }));
+
+    await expect(
+      updateFilteringDraft(mockClient as unknown as ElasticsearchClient, 'connectorId', {
+        advancedSnippet: '{}',
+        filteringRules: [filteringRule],
+      })
+    ).resolves.toEqual(draft);
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      body: {
+        advanced_snippet: {
+          created_at: '2024-05-25T12:00:00.000Z',
+          updated_at: '2024-05-25T12:00:00.000Z',
+          value: {},
+        },
+        rules: [
+          {
+            ...filteringRule,
+            updated_at: '2024-05-25T12:00:00.000Z',
+          },
+        ],
+      },
+      method: 'PUT',
+      path: '/_connector/connectorId/_filtering',
+    });
+  });
+
+  it('should not index document if there is no connector', async () => {
+    mockClient.transport.request.mockImplementationOnce(() => {
+      return Promise.reject(
+        new errors.ResponseError({
+          statusCode: 404,
+          body: {
+            error: {
+              type: `document_missing_exception`,
+            },
+          },
+        } as any)
+      );
+    });
+    await expect(
+      updateFilteringDraft(mockClient as unknown as ElasticsearchClient, 'connectorId', {
+        advancedSnippet: '{}',
+        filteringRules: [],
+      })
+    ).rejects.toEqual(
+      new errors.ResponseError({
+        statusCode: 404,
+        body: {
+          error: {
+            type: `document_missing_exception`,
+          },
+        },
+      } as any)
+    );
+  });
+});

packages/kbn-search-connectors/lib/update_filtering_draft.ts

@@ -6,16 +6,11 @@
  * Side Public License, v 1.
  */
 
+import { Result } from '@elastic/elasticsearch/lib/api/types';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 
-import { CONNECTORS_INDEX } from '..';
 import { fetchConnectorById } from './fetch_connectors';
-import {
-  Connector,
-  FilteringRule,
-  FilteringRules,
-  FilteringValidationState,
-} from '../types/connectors';
+import { FilteringRule, FilteringRules } from '../types/connectors';
 
 export const updateFilteringDraft = async (
   client: ElasticsearchClient,
@@ -37,28 +32,25 @@ export const updateFilteringDraft = async (
     created_at: filteringRule.created_at ? filteringRule.created_at : now,
     updated_at: now,
   }));
-  const draft: FilteringRules = {
+
+  const draft = {
     advanced_snippet: {
       created_at: now,
       updated_at: now,
       value: parsedAdvancedSnippet,
     },
     rules: parsedFilteringRules,
-    validation: {
-      errors: [],
-      state: FilteringValidationState.EDITED,
-    },
   };
-  const connector = await fetchConnectorById(client, connectorId);
-  if (!connector) {
-    throw new Error(`Could not find connector with id ${connectorId}`);
-  }
 
-  const result = await client.update<Connector>({
-    doc: { ...connector, filtering: [{ ...connector.filtering[0], draft }] },
-    id: connectorId,
-    index: CONNECTORS_INDEX,
+  const updateDraftFilteringResult = await client.transport.request<{ result: Result }>({
+    method: 'PUT',
+    path: `/_connector/${connectorId}/_filtering`,
+    body: draft,
   });
 
-  return result.result === 'updated' ? draft : undefined;
+  if (updateDraftFilteringResult.result === 'updated') {
+    const connector = await fetchConnectorById(client, connectorId);
+    return connector?.filtering?.[0]?.draft;
+  }
+  return undefined;
 };

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/update_connector_filtering_api_logic.ts

@@ -7,32 +7,21 @@
 
 import { i18n } from '@kbn/i18n';
 
-import { FilteringRule, FilteringRules } from '@kbn/search-connectors';
+import { FilteringRules } from '@kbn/search-connectors';
 
 import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
 import { HttpLogic } from '../../../shared/http';
 
 export interface PutConnectorFilteringArgs {
-  advancedSnippet: string;
   connectorId: string;
-  filteringRules: FilteringRule[];
 }
 
 export type PutConnectorFilteringResponse = FilteringRules;
 
-export const putConnectorFiltering = async ({
-  advancedSnippet,
-  connectorId,
-  filteringRules,
-}: PutConnectorFilteringArgs) => {
+export const putConnectorFiltering = async ({ connectorId }: PutConnectorFilteringArgs) => {
   const route = `/internal/enterprise_search/connectors/${connectorId}/filtering`;
 
-  return await HttpLogic.values.http.put(route, {
-    body: JSON.stringify({
-      advanced_snippet: advancedSnippet,
-      filtering_rules: filteringRules,
-    }),
-  });
+  return await HttpLogic.values.http.put(route);
 };
 
 export const ConnectorFilteringApiLogic = createApiLogic(

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/sync_rules/connector_filtering_logic.tsx

@@ -149,9 +149,7 @@ export const ConnectorFilteringLogic = kea<
     applyDraft: () => {
       if (isConnectorIndex(values.index)) {
         actions.makeRequest({
-          advancedSnippet: values.localAdvancedSnippet ?? '',
           connectorId: values.index.connector.id,
-          filteringRules: values.localFilteringRules ?? [],
         });
       }
     },

x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts

@@ -472,21 +472,23 @@ export function registerConnectorRoutes({ router, log }: RouteDependencies) {
     {
       path: '/internal/enterprise_search/connectors/{connectorId}/filtering',
       validate: {
-        body: schema.object({
-          advanced_snippet: schema.string(),
-          filtering_rules: schema.arrayOf(
-            schema.object({
-              created_at: schema.string(),
-              field: schema.string(),
-              id: schema.string(),
-              order: schema.number(),
-              policy: schema.string(),
-              rule: schema.string(),
-              updated_at: schema.string(),
-              value: schema.string(),
-            })
-          ),
-        }),
+        body: schema.maybe(
+          schema.object({
+            advanced_snippet: schema.string(),
+            filtering_rules: schema.arrayOf(
+              schema.object({
+                created_at: schema.string(),
+                field: schema.string(),
+                id: schema.string(),
+                order: schema.number(),
+                policy: schema.string(),
+                rule: schema.string(),
+                updated_at: schema.string(),
+                value: schema.string(),
+              })
+            ),
+          })
+        ),
         params: schema.object({
           connectorId: schema.string(),
         }),
@@ -495,13 +497,7 @@ export function registerConnectorRoutes({ router, log }: RouteDependencies) {
     elasticsearchErrorHandler(log, async (context, request, response) => {
       const { client } = (await context.core).elasticsearch;
       const { connectorId } = request.params;
-      const { advanced_snippet, filtering_rules } = request.body;
-      const result = await updateFiltering(client.asCurrentUser, connectorId, {
-        advancedSnippet: advanced_snippet,
-        // Have to cast here because our API schema validator doesn't know how to deal with enums
-        // We're relying on the schema in the validator above to flag if something goes wrong
-        filteringRules: filtering_rules as FilteringRule[],
-      });
+      const result = await updateFiltering(client.asCurrentUser, connectorId);
       return result ? response.ok({ body: result }) : response.conflict();
     })
   );