[8.6] [Fleet] Run agent policy schema in batches during fleet setup + add xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize config (#150688) (#150781)

# Backport

This will backport the following commits from `main` to `8.6`:
- [[Fleet] Run agent policy schema in batches during fleet setup + add
`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize` config
(#150688)](https://github.com/elastic/kibana/pull/150688)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Mark
Hopkin","email":"mark.hopkin@elastic.co"},"sourceCommit":{"committedDate":"2023-02-09T17:16:35Z","message":"[Fleet]
Run agent policy schema in batches during fleet setup + add
`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize` config
(#150688)\n\n## Summary\r\n\r\nCloses #150538 \r\n\r\nAs part of the
Fleet plugin setup, we check to see if any agent policies\r\nhave an out
of date `schema_version` and upgrade them. We encountered an\r\nerror
when this upgrade happens on a large number of agent policies as\r\nwe
attempted the upgrade in one large batch.\r\n\r\nThis pull request
performs the schema upgrade in batches of 100 by\r\ndefault and also
adds the config
value\r\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize` to make
the batch\r\nsize configurable.\r\n\r\nI have also added more debug
logging to show progress, and reduced the\r\nresponse payload of one of
our requests which was very large.\r\n\r\n### Dev testing\r\n\r\nTo test
this you need an environemnt with lots of agent policies (> 2k)\r\nwhere
`schema_version`\r\nis not set. To create an environment with a large
number of agent\r\npolicies I have added a new param to the agent
creation script, I ran:\r\n\r\n```\r\ncd x-pack/plugins/fleet\r\nnode
scripts/create_agents --count 20 --kibana http://127.0.0.1:5601/mark
--status online --delete --batches 3000 --concurrentBatches
100\r\n```\r\n\r\nTo generate 3000 agent policies each with 20 agents
in.\r\n\r\nI then modified the agent policies so that they require an
upgrade, as\r\n`system_indices_superuser` run:\r\n\r\n```\r\nPOST
/.kibana/_update_by_query\r\n{\r\n \"query\": {\r\n \"bool\": {\r\n
\"filter\": [\r\n {\r\n \"term\": {\r\n \"type\":
\"ingest-agent-policies\"\r\n }\r\n }\r\n ]\r\n }\r\n },\r\n \"script\":
{\r\n \"source\":
\"ctx._source['ingest-agent-policies'].remove('schema_version')\",\r\n
\"lang\": \"painless\"\r\n }\r\n}\r\n```\r\n\r\nrestarting kibana will
run the setup and in
batches.","sha":"6e06452aac11ed22efa923284fbb9ad4da1f7ce1","branchLabelMapping":{"^v8.8.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:enhancement","Team:Fleet","backport:all-open","v8.7.0","v8.8.0"],"number":150688,"url":"https://github.com/elastic/kibana/pull/150688","mergeCommit":{"message":"[Fleet]
Run agent policy schema in batches during fleet setup + add
`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize` config
(#150688)\n\n## Summary\r\n\r\nCloses #150538 \r\n\r\nAs part of the
Fleet plugin setup, we check to see if any agent policies\r\nhave an out
of date `schema_version` and upgrade them. We encountered an\r\nerror
when this upgrade happens on a large number of agent policies as\r\nwe
attempted the upgrade in one large batch.\r\n\r\nThis pull request
performs the schema upgrade in batches of 100 by\r\ndefault and also
adds the config
value\r\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize` to make
the batch\r\nsize configurable.\r\n\r\nI have also added more debug
logging to show progress, and reduced the\r\nresponse payload of one of
our requests which was very large.\r\n\r\n### Dev testing\r\n\r\nTo test
this you need an environemnt with lots of agent policies (> 2k)\r\nwhere
`schema_version`\r\nis not set. To create an environment with a large
number of agent\r\npolicies I have added a new param to the agent
creation script, I ran:\r\n\r\n```\r\ncd x-pack/plugins/fleet\r\nnode
scripts/create_agents --count 20 --kibana http://127.0.0.1:5601/mark
--status online --delete --batches 3000 --concurrentBatches
100\r\n```\r\n\r\nTo generate 3000 agent policies each with 20 agents
in.\r\n\r\nI then modified the agent policies so that they require an
upgrade, as\r\n`system_indices_superuser` run:\r\n\r\n```\r\nPOST
/.kibana/_update_by_query\r\n{\r\n \"query\": {\r\n \"bool\": {\r\n
\"filter\": [\r\n {\r\n \"term\": {\r\n \"type\":
\"ingest-agent-policies\"\r\n }\r\n }\r\n ]\r\n }\r\n },\r\n \"script\":
{\r\n \"source\":
\"ctx._source['ingest-agent-policies'].remove('schema_version')\",\r\n
\"lang\": \"painless\"\r\n }\r\n}\r\n```\r\n\r\nrestarting kibana will
run the setup and in
batches.","sha":"6e06452aac11ed22efa923284fbb9ad4da1f7ce1"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"8.7","label":"v8.7.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/150750","number":150750,"state":"MERGED","mergeCommit":{"sha":"2713d135c6d942c0007315a139c0b4114a9054b7","message":"[8.7]
[Fleet] Run agent policy schema in batches during fleet setup + add
`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize` config (#150688)
(#150750)\n\n# Backport\n\nThis will backport the following commits from
`main` to `8.7`:\n- [[Fleet] Run agent policy schema in batches during
fleet setup + add\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize`
config\n(#150688)](https://github.com/elastic/kibana/pull/150688)\n\n<!---
Backport version: 8.9.7 -->\n\n### Questions ?\nPlease refer to the
[Backport
tool\ndocumentation](https://github.com/sqren/backport)\n\n<!--BACKPORT
[{\"author\":{\"name\":\"Mark\nHopkin\",\"email\":\"mark.hopkin@elastic.co\"},\"sourceCommit\":{\"committedDate\":\"2023-02-09T17:16:35Z\",\"message\":\"[Fleet]\nRun
agent policy schema in batches during fleet setup +
add\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize`
config\n(#150688)\\n\\n## Summary\\r\\n\\r\\nCloses #150538
\\r\\n\\r\\nAs part of the\nFleet plugin setup, we check to see if any
agent policies\\r\\nhave an out\nof date `schema_version` and upgrade
them. We encountered an\\r\\nerror\nwhen this upgrade happens on a large
number of agent policies as\\r\\nwe\nattempted the upgrade in one large
batch.\\r\\n\\r\\nThis pull request\nperforms the schema upgrade in
batches of 100 by\\r\\ndefault and also\nadds the
config\nvalue\\r\\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize`
to make\nthe batch\\r\\nsize configurable.\\r\\n\\r\\nI have also added
more debug\nlogging to show progress, and reduced the\\r\\nresponse
payload of one of\nour requests which was very large.\\r\\n\\r\\n### Dev
testing\\r\\n\\r\\nTo test\nthis you need an environemnt with lots of
agent policies (> 2k)\\r\\nwhere\n`schema_version`\\r\\nis not set. To
create an environment with a large\nnumber of agent\\r\\npolicies I have
added a new param to the agent\ncreation script, I
ran:\\r\\n\\r\\n```\\r\\ncd
x-pack/plugins/fleet\\r\\nnode\nscripts/create_agents --count 20
--kibana http://127.0.0.1:5601/mark\n--status online --delete --batches
3000 --concurrentBatches\n100\\r\\n```\\r\\n\\r\\nTo generate 3000 agent
policies each with 20 agents\nin.\\r\\n\\r\\nI then modified the agent
policies so that they require an\nupgrade,
as\\r\\n`system_indices_superuser`
run:\\r\\n\\r\\n```\\r\\nPOST\n/.kibana/_update_by_query\\r\\n{\\r\\n
\\\"query\\\": {\\r\\n \\\"bool\\\": {\\r\\n\n\\\"filter\\\": [\\r\\n
{\\r\\n \\\"term\\\": {\\r\\n
\\\"type\\\":\n\\\"ingest-agent-policies\\\"\\r\\n }\\r\\n }\\r\\n
]\\r\\n }\\r\\n },\\r\\n \\\"script\\\":\n{\\r\\n
\\\"source\\\":\n\\\"ctx._source['ingest-agent-policies'].remove('schema_version')\\\",\\r\\n\n\\\"lang\\\":
\\\"painless\\\"\\r\\n }\\r\\n}\\r\\n```\\r\\n\\r\\nrestarting kibana
will\nrun the setup and
in\nbatches.\",\"sha\":\"6e06452aac11ed22efa923284fbb9ad4da1f7ce1\",\"branchLabelMapping\":{\"^v8.8.0$\":\"main\",\"^v(\\\\d+).(\\\\d+).\\\\d+$\":\"$1.$2\"}},\"sourcePullRequest\":{\"labels\":[\"release_note:enhancement\",\"Team:Fleet\",\"backport:all-open\",\"v8.8.0\"],\"number\":150688,\"url\":\"https://github.com/elastic/kibana/pull/150688\",\"mergeCommit\":{\"message\":\"[Fleet]\nRun
agent policy schema in batches during fleet setup +
add\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize`
config\n(#150688)\\n\\n## Summary\\r\\n\\r\\nCloses #150538
\\r\\n\\r\\nAs part of the\nFleet plugin setup, we check to see if any
agent policies\\r\\nhave an out\nof date `schema_version` and upgrade
them. We encountered an\\r\\nerror\nwhen this upgrade happens on a large
number of agent policies as\\r\\nwe\nattempted the upgrade in one large
batch.\\r\\n\\r\\nThis pull request\nperforms the schema upgrade in
batches of 100 by\\r\\ndefault and also\nadds the
config\nvalue\\r\\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize`
to make\nthe batch\\r\\nsize configurable.\\r\\n\\r\\nI have also added
more debug\nlogging to show progress, and reduced the\\r\\nresponse
payload of one of\nour requests which was very large.\\r\\n\\r\\n### Dev
testing\\r\\n\\r\\nTo test\nthis you need an environemnt with lots of
agent policies (> 2k)\\r\\nwhere\n`schema_version`\\r\\nis not set. To
create an environment with a large\nnumber of agent\\r\\npolicies I have
added a new param to the agent\ncreation script, I
ran:\\r\\n\\r\\n```\\r\\ncd
x-pack/plugins/fleet\\r\\nnode\nscripts/create_agents --count 20
--kibana http://127.0.0.1:5601/mark\n--status online --delete --batches
3000 --concurrentBatches\n100\\r\\n```\\r\\n\\r\\nTo generate 3000 agent
policies each with 20 agents\nin.\\r\\n\\r\\nI then modified the agent
policies so that they require an\nupgrade,
as\\r\\n`system_indices_superuser`
run:\\r\\n\\r\\n```\\r\\nPOST\n/.kibana/_update_by_query\\r\\n{\\r\\n
\\\"query\\\": {\\r\\n \\\"bool\\\": {\\r\\n\n\\\"filter\\\": [\\r\\n
{\\r\\n \\\"term\\\": {\\r\\n
\\\"type\\\":\n\\\"ingest-agent-policies\\\"\\r\\n }\\r\\n }\\r\\n
]\\r\\n }\\r\\n },\\r\\n \\\"script\\\":\n{\\r\\n
\\\"source\\\":\n\\\"ctx._source['ingest-agent-policies'].remove('schema_version')\\\",\\r\\n\n\\\"lang\\\":
\\\"painless\\\"\\r\\n }\\r\\n}\\r\\n```\\r\\n\\r\\nrestarting kibana
will\nrun the setup and
in\nbatches.\",\"sha\":\"6e06452aac11ed22efa923284fbb9ad4da1f7ce1\"}},\"sourceBranch\":\"main\",\"suggestedTargetBranches\":[],\"targetPullRequestStates\":[{\"branch\":\"main\",\"label\":\"v8.8.0\",\"labelRegex\":\"^v8.8.0$\",\"isSourceBranch\":true,\"state\":\"MERGED\",\"url\":\"https://github.com/elastic/kibana/pull/150688\",\"number\":150688,\"mergeCommit\":{\"message\":\"[Fleet]\nRun
agent policy schema in batches during fleet setup +
add\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize`
config\n(#150688)\\n\\n## Summary\\r\\n\\r\\nCloses #150538
\\r\\n\\r\\nAs part of the\nFleet plugin setup, we check to see if any
agent policies\\r\\nhave an out\nof date `schema_version` and upgrade
them. We encountered an\\r\\nerror\nwhen this upgrade happens on a large
number of agent policies as\\r\\nwe\nattempted the upgrade in one large
batch.\\r\\n\\r\\nThis pull request\nperforms the schema upgrade in
batches of 100 by\\r\\ndefault and also\nadds the
config\nvalue\\r\\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize`
to make\nthe batch\\r\\nsize configurable.\\r\\n\\r\\nI have also added
more debug\nlogging to show progress, and reduced the\\r\\nresponse
payload of one of\nour requests which was very large.\\r\\n\\r\\n### Dev
testing\\r\\n\\r\\nTo test\nthis you need an environemnt with lots of
agent policies (> 2k)\\r\\nwhere\n`schema_version`\\r\\nis not set. To
create an environment with a large\nnumber of agent\\r\\npolicies I have
added a new param to the agent\ncreation script, I
ran:\\r\\n\\r\\n```\\r\\ncd
x-pack/plugins/fleet\\r\\nnode\nscripts/create_agents --count 20
--kibana http://127.0.0.1:5601/mark\n--status online --delete --batches
3000 --concurrentBatches\n100\\r\\n```\\r\\n\\r\\nTo generate 3000 agent
policies each with 20 agents\nin.\\r\\n\\r\\nI then modified the agent
policies so that they require an\nupgrade,
as\\r\\n`system_indices_superuser`
run:\\r\\n\\r\\n```\\r\\nPOST\n/.kibana/_update_by_query\\r\\n{\\r\\n
\\\"query\\\": {\\r\\n \\\"bool\\\": {\\r\\n\n\\\"filter\\\": [\\r\\n
{\\r\\n \\\"term\\\": {\\r\\n
\\\"type\\\":\n\\\"ingest-agent-policies\\\"\\r\\n }\\r\\n }\\r\\n
]\\r\\n }\\r\\n },\\r\\n \\\"script\\\":\n{\\r\\n
\\\"source\\\":\n\\\"ctx._source['ingest-agent-policies'].remove('schema_version')\\\",\\r\\n\n\\\"lang\\\":
\\\"painless\\\"\\r\\n }\\r\\n}\\r\\n```\\r\\n\\r\\nrestarting kibana
will\nrun the setup and
in\nbatches.\",\"sha\":\"6e06452aac11ed22efa923284fbb9ad4da1f7ce1\"}}]}]\nBACKPORT-->\n\nCo-authored-by:
Mark Hopkin
<mark.hopkin@elastic.co>"}},{"branch":"main","label":"v8.8.0","labelRegex":"^v8.8.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/150688","number":150688,"mergeCommit":{"message":"[Fleet]
Run agent policy schema in batches during fleet setup + add
`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize` config
(#150688)\n\n## Summary\r\n\r\nCloses #150538 \r\n\r\nAs part of the
Fleet plugin setup, we check to see if any agent policies\r\nhave an out
of date `schema_version` and upgrade them. We encountered an\r\nerror
when this upgrade happens on a large number of agent policies as\r\nwe
attempted the upgrade in one large batch.\r\n\r\nThis pull request
performs the schema upgrade in batches of 100 by\r\ndefault and also
adds the config
value\r\n`xpack.fleet.setup.agentPolicySchemaUpgradeBatchSize` to make
the batch\r\nsize configurable.\r\n\r\nI have also added more debug
logging to show progress, and reduced the\r\nresponse payload of one of
our requests which was very large.\r\n\r\n### Dev testing\r\n\r\nTo test
this you need an environemnt with lots of agent policies (> 2k)\r\nwhere
`schema_version`\r\nis not set. To create an environment with a large
number of agent\r\npolicies I have added a new param to the agent
creation script, I ran:\r\n\r\n```\r\ncd x-pack/plugins/fleet\r\nnode
scripts/create_agents --count 20 --kibana http://127.0.0.1:5601/mark
--status online --delete --batches 3000 --concurrentBatches
100\r\n```\r\n\r\nTo generate 3000 agent policies each with 20 agents
in.\r\n\r\nI then modified the agent policies so that they require an
upgrade, as\r\n`system_indices_superuser` run:\r\n\r\n```\r\nPOST
/.kibana/_update_by_query\r\n{\r\n \"query\": {\r\n \"bool\": {\r\n
\"filter\": [\r\n {\r\n \"term\": {\r\n \"type\":
\"ingest-agent-policies\"\r\n }\r\n }\r\n ]\r\n }\r\n },\r\n \"script\":
{\r\n \"source\":
\"ctx._source['ingest-agent-policies'].remove('schema_version')\",\r\n
\"lang\": \"painless\"\r\n }\r\n}\r\n```\r\n\r\nrestarting kibana will
run the setup and in
batches.","sha":"6e06452aac11ed22efa923284fbb9ad4da1f7ce1"}}]}]
BACKPORT-->

x-pack/plugins/fleet/common/types/index.ts

@@ -36,6 +36,9 @@ export interface FleetConfigType {
   packageVerification?: {
     gpgKeyPath?: string;
   };
+  setup?: {
+    agentPolicySchemaUpgradeBatchSize?: number;
+  };
   developer?: {
     disableRegistryVersionCheck?: boolean;
     bundledPackageLocation?: string;

x-pack/plugins/fleet/server/config.ts

@@ -118,6 +118,11 @@ export const config: PluginConfigDescriptor = {
     outputs: PreconfiguredOutputsSchema,
     fleetServerHosts: PreconfiguredFleetServerHostsSchema,
     agentIdVerificationEnabled: schema.boolean({ defaultValue: true }),
+    setup: schema.maybe(
+      schema.object({
+        agentPolicySchemaUpgradeBatchSize: schema.maybe(schema.number()),
+      })
+    ),
     developer: schema.object({
       disableRegistryVersionCheck: schema.boolean({ defaultValue: false }),
       allowAgentUpgradeSourceUri: schema.boolean({ defaultValue: false }),

x-pack/plugins/fleet/server/services/agent_policy.ts

@@ -316,8 +316,14 @@ class AgentPolicyService {
     soClient: SavedObjectsClientContract,
     options: ListWithKuery & {
       withPackagePolicies?: boolean;
+      fields?: string[];
     }
-  ): Promise<{ items: AgentPolicy[]; total: number; page: number; perPage: number }> {
+  ): Promise<{
+    items: AgentPolicy[];
+    total: number;
+    page: number;
+    perPage: number;
+  }> {
     const {
       page = 1,
       perPage = 20,
@@ -325,6 +331,7 @@ class AgentPolicyService {
       sortOrder = 'desc',
       kuery,
       withPackagePolicies = false,
+      fields,
     } = options;
 
     const baseFindParams = {
@@ -333,6 +340,7 @@ class AgentPolicyService {
       sortOrder,
       page,
       perPage,
+      ...(fields ? { fields } : {}),
     };
     const filter = kuery ? normalizeKuery(SAVED_OBJECT_TYPE, kuery) : undefined;
     let agentPoliciesSO;

x-pack/plugins/fleet/server/services/setup/upgrade_agent_policy_schema_version.ts

@@ -10,14 +10,16 @@ import type { SavedObjectsClientContract } from '@kbn/core/server';
 import {
   AGENT_POLICY_SAVED_OBJECT_TYPE,
   FLEET_AGENT_POLICIES_SCHEMA_VERSION,
-  SO_SEARCH_LIMIT,
 } from '../../constants';
 import { agentPolicyService } from '../agent_policy';
+import { appContextService } from '../app_context';
 
-function getOutdatedAgentPoliciesBatch(soClient: SavedObjectsClientContract) {
+const DEFAULT_BATCH_SIZE = 100;
+function getOutdatedAgentPoliciesBatch(soClient: SavedObjectsClientContract, batchSize: number) {
   return agentPolicyService.list(soClient, {
-    perPage: SO_SEARCH_LIMIT,
+    perPage: batchSize,
     kuery: `NOT ${AGENT_POLICY_SAVED_OBJECT_TYPE}.schema_version:${FLEET_AGENT_POLICIES_SCHEMA_VERSION}`,
+    fields: ['id'], // we only need the ID of the agent policy
   });
 }
 
@@ -26,13 +28,23 @@ function getOutdatedAgentPoliciesBatch(soClient: SavedObjectsClientContract) {
 // deploy outdated policies to .fleet-policies index
 // bump oudated SOs schema_version
 export async function upgradeAgentPolicySchemaVersion(soClient: SavedObjectsClientContract) {
-  let outdatedAgentPolicies = await getOutdatedAgentPoliciesBatch(soClient);
+  const config = appContextService.getConfig();
+  const logger = appContextService.getLogger();
 
+  const batchSize = config?.setup?.agentPolicySchemaUpgradeBatchSize ?? DEFAULT_BATCH_SIZE;
+  let outdatedAgentPolicies = await getOutdatedAgentPoliciesBatch(soClient, batchSize);
+  logger.debug(`Found ${outdatedAgentPolicies.total} outdated agent policies`);
   while (outdatedAgentPolicies.total > 0) {
+    const start = Date.now();
     const outdatedAgentPolicyIds = outdatedAgentPolicies.items.map(
       (outdatedAgentPolicy) => outdatedAgentPolicy.id
     );
     await agentPolicyService.deployPolicies(soClient, outdatedAgentPolicyIds);
-    outdatedAgentPolicies = await getOutdatedAgentPoliciesBatch(soClient);
+    outdatedAgentPolicies = await getOutdatedAgentPoliciesBatch(soClient, batchSize);
+    logger.debug(
+      `Upgraded ${outdatedAgentPolicyIds.length} agent policies in ${Date.now() - start}ms, ${
+        outdatedAgentPolicies.total
+      } remaining`
+    );
   }
 }