[SIEM][Detection Engine][Lists] Adds version and immutability data structures (#72730)

###  Summary

The intent is to get the data structures in similar to rules so that we can have eventually immutable and versioned lists in later releases without too much hassle of upgrading the list and list item data structures.

* Adds version and immutability data structures to the exception lists and the value lists.
* Adds an optional version number to the update route of each so that you can modify the number either direction or you can omit it and it works like the detection rules where it will auto-increment the number.
* Does _not_ add a version and immutability to the exception list items and value list items.
* Does _not_ update the version number when you add a new exception list item or value list item. 

**Examples:**

❯ ./post_list.sh
```json
{
  "_version": "WzAsMV0=",
  "id": "ip_list",
  "created_at": "2020-07-21T20:31:11.679Z",
  "created_by": "yo",
  "description": "This list describes bad internet ip",
  "immutable": false,
  "name": "Simple list with an ip",
  "tie_breaker_id": "d6bd7552-84d1-4f95-88c4-cc504517b4e5",
  "type": "ip",
  "updated_at": "2020-07-21T20:31:11.679Z",
  "updated_by": "yo",
  "version": 1
}
```
❯ ./post_exception_list.sh
```json
{
  "_tags": [
    "endpoint",
    "process",
    "malware",
    "os:linux"
  ],
  "_version": "WzMzOTgsMV0=",
  "created_at": "2020-07-21T20:31:35.933Z",
  "created_by": "yo",
  "description": "This is a sample endpoint type exception",
  "id": "2c24b100-cb91-11ea-a872-adfddf68361e",
  "immutable": false,
  "list_id": "simple_list",
  "name": "Sample Endpoint Exception List",
  "namespace_type": "single",
  "tags": [
    "user added string for a tag",
    "malware"
  ],
  "tie_breaker_id": "c11c4d53-d0be-4904-870e-d33ec7ca387f",
  "type": "detection",
  "updated_at": "2020-07-21T20:31:35.952Z",
  "updated_by": "yo",
  "version": 1
}
```

```json
❯ ./update_list.sh
{
  "_version": "WzEsMV0=",
  "created_at": "2020-07-21T20:31:11.679Z",
  "created_by": "yo",
  "description": "Some other description here for you",
  "id": "ip_list",
  "immutable": false,
  "name": "Changed the name here to something else",
  "tie_breaker_id": "d6bd7552-84d1-4f95-88c4-cc504517b4e5",
  "type": "ip",
  "updated_at": "2020-07-21T20:31:47.089Z",
  "updated_by": "yo",
  "version": 2
}
```

```json
❯ ./update_exception_list.sh
{
  "_tags": [
    "endpoint",
    "process",
    "malware",
    "os:linux"
  ],
  "_version": "WzMzOTksMV0=",
  "created_at": "2020-07-21T20:31:35.933Z",
  "created_by": "yo",
  "description": "Different description",
  "id": "2c24b100-cb91-11ea-a872-adfddf68361e",
  "immutable": false,
  "list_id": "simple_list",
  "name": "Sample Endpoint Exception List",
  "namespace_type": "single",
  "tags": [
    "user added string for a tag",
    "malware"
  ],
  "tie_breaker_id": "c11c4d53-d0be-4904-870e-d33ec7ca387f",
  "type": "endpoint",
  "updated_at": "2020-07-21T20:31:56.628Z",
  "updated_by": "yo",
  "version": 2
}
```

### Checklist

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

x-pack/plugins/lists/common/constants.mock.ts

@@ -61,3 +61,5 @@ export const COMMENTS = [];
 export const FILTER = 'name:Nicolas Bourbaki';
 export const CURSOR = 'c29tZXN0cmluZ2ZvcnlvdQ==';
 export const _VERSION = 'WzI5NywxXQ==';
+export const VERSION = 1;
+export const IMMUTABLE = false;

x-pack/plugins/lists/common/schemas/common/schemas.ts

@@ -311,3 +311,15 @@ export type DeserializerOrUndefined = t.TypeOf<typeof deserializerOrUndefined>;
 export const _version = t.string;
 export const _versionOrUndefined = t.union([_version, t.undefined]);
 export type _VersionOrUndefined = t.TypeOf<typeof _versionOrUndefined>;
+
+export const version = t.number;
+export type Version = t.TypeOf<typeof version>;
+
+export const versionOrUndefined = t.union([version, t.undefined]);
+export type VersionOrUndefined = t.TypeOf<typeof versionOrUndefined>;
+
+export const immutable = t.boolean;
+export type Immutable = t.TypeOf<typeof immutable>;
+
+export const immutableOrUndefined = t.union([immutable, t.undefined]);
+export type ImmutableOrUndefined = t.TypeOf<typeof immutableOrUndefined>;

x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.mock.ts

@@ -8,11 +8,13 @@ import { IndexEsListSchema } from '../../../common/schemas';
 import {
   DATE_NOW,
   DESCRIPTION,
+  IMMUTABLE,
   META,
   NAME,
   TIE_BREAKER,
   TYPE,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 
 export const getIndexESListMock = (): IndexEsListSchema => ({
@@ -20,6 +22,7 @@ export const getIndexESListMock = (): IndexEsListSchema => ({
   created_by: USER,
   description: DESCRIPTION,
   deserializer: undefined,
+  immutable: IMMUTABLE,
   meta: META,
   name: NAME,
   serializer: undefined,
@@ -27,4 +30,5 @@ export const getIndexESListMock = (): IndexEsListSchema => ({
   type: TYPE,
   updated_at: DATE_NOW,
   updated_by: USER,
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts

@@ -13,6 +13,7 @@ import {
   created_by,
   description,
   deserializerOrUndefined,
+  immutable,
   metaOrUndefined,
   name,
   serializerOrUndefined,
@@ -20,6 +21,7 @@ import {
   type,
   updated_at,
   updated_by,
+  version,
 } from '../common/schemas';
 
 export const indexEsListSchema = t.exact(
@@ -28,6 +30,7 @@ export const indexEsListSchema = t.exact(
     created_by,
     description,
     deserializer: deserializerOrUndefined,
+    immutable,
     meta: metaOrUndefined,
     name,
     serializer: serializerOrUndefined,
@@ -35,6 +38,7 @@ export const indexEsListSchema = t.exact(
     type,
     updated_at,
     updated_by,
+    version,
   })
 );
 

x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.mock.ts

@@ -10,6 +10,7 @@ import { SearchEsListSchema } from '../../../common/schemas';
 import {
   DATE_NOW,
   DESCRIPTION,
+  IMMUTABLE,
   LIST_ID,
   LIST_INDEX,
   META,
@@ -17,6 +18,7 @@ import {
   TIE_BREAKER,
   TYPE,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 import { getShardMock } from '../../get_shard.mock';
 
@@ -25,6 +27,7 @@ export const getSearchEsListMock = (): SearchEsListSchema => ({
   created_by: USER,
   description: DESCRIPTION,
   deserializer: undefined,
+  immutable: IMMUTABLE,
   meta: META,
   name: NAME,
   serializer: undefined,
@@ -32,6 +35,7 @@ export const getSearchEsListMock = (): SearchEsListSchema => ({
   type: TYPE,
   updated_at: DATE_NOW,
   updated_by: USER,
+  version: VERSION,
 });
 
 export const getSearchListMock = (): SearchResponse<SearchEsListSchema> => ({

x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts

@@ -13,6 +13,7 @@ import {
   created_by,
   description,
   deserializerOrUndefined,
+  immutable,
   metaOrUndefined,
   name,
   serializerOrUndefined,
@@ -20,6 +21,7 @@ import {
   type,
   updated_at,
   updated_by,
+  version,
 } from '../common/schemas';
 
 export const searchEsListSchema = t.exact(
@@ -28,6 +30,7 @@ export const searchEsListSchema = t.exact(
     created_by,
     description,
     deserializer: deserializerOrUndefined,
+    immutable,
     meta: metaOrUndefined,
     name,
     serializer: serializerOrUndefined,
@@ -35,6 +38,7 @@ export const searchEsListSchema = t.exact(
     type,
     updated_at,
     updated_by,
+    version,
   })
 );
 

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.mock.ts

@@ -4,7 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { DESCRIPTION, ENDPOINT_TYPE, META, NAME, NAMESPACE_TYPE } from '../../constants.mock';
+import {
+  DESCRIPTION,
+  ENDPOINT_TYPE,
+  META,
+  NAME,
+  NAMESPACE_TYPE,
+  VERSION,
+} from '../../constants.mock';
 
 import { CreateExceptionListSchema } from './create_exception_list_schema';
 
@@ -17,4 +24,5 @@ export const getCreateExceptionListSchemaMock = (): CreateExceptionListSchema =>
   namespace_type: NAMESPACE_TYPE,
   tags: [],
   type: ENDPOINT_TYPE,
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.ts

@@ -21,7 +21,11 @@ import {
   tags,
 } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
-import { DefaultUuid } from '../../siem_common_deps';
+import {
+  DefaultUuid,
+  DefaultVersionNumber,
+  DefaultVersionNumberDecoded,
+} from '../../siem_common_deps';
 import { NamespaceType } from '../types';
 
 export const createExceptionListSchema = t.intersection([
@@ -39,6 +43,7 @@ export const createExceptionListSchema = t.intersection([
       meta, // defaults to undefined if not set during decode
       namespace_type, // defaults to 'single' if not set during decode
       tags, // defaults to empty array if not set during decode
+      version: DefaultVersionNumber, // defaults to numerical 1 if not set during decode
     })
   ),
 ]);
@@ -54,4 +59,5 @@ export type CreateExceptionListSchemaDecoded = Omit<
   tags: Tags;
   list_id: ListId;
   namespace_type: NamespaceType;
+  version: DefaultVersionNumberDecoded;
 };

x-pack/plugins/lists/common/schemas/request/create_list_schema.mock.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { DESCRIPTION, LIST_ID, META, NAME, TYPE } from '../../constants.mock';
+import { DESCRIPTION, LIST_ID, META, NAME, TYPE, VERSION } from '../../constants.mock';
 
 import { CreateListSchema } from './create_list_schema';
 
@@ -16,4 +16,5 @@ export const getCreateListSchemaMock = (): CreateListSchema => ({
   name: NAME,
   serializer: undefined,
   type: TYPE,
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/request/create_list_schema.ts

@@ -8,6 +8,7 @@ import * as t from 'io-ts';
 
 import { description, deserializer, id, meta, name, serializer, type } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
+import { DefaultVersionNumber, DefaultVersionNumberDecoded } from '../../siem_common_deps';
 
 export const createListSchema = t.intersection([
   t.exact(
@@ -17,8 +18,18 @@ export const createListSchema = t.intersection([
       type,
     })
   ),
-  t.exact(t.partial({ deserializer, id, meta, serializer })),
+  t.exact(
+    t.partial({
+      deserializer, // defaults to undefined if not set during decode
+      id, // defaults to undefined if not set during decode
+      meta, // defaults to undefined if not set during decode
+      serializer, // defaults to undefined if not set during decode
+      version: DefaultVersionNumber, // defaults to a numerical 1 if not set during decode
+    })
+  ),
 ]);
 
 export type CreateListSchema = t.OutputOf<typeof createListSchema>;
-export type CreateListSchemaDecoded = RequiredKeepUndefined<t.TypeOf<typeof createListSchema>>;
+export type CreateListSchemaDecoded = RequiredKeepUndefined<
+  Omit<t.TypeOf<typeof createListSchema>, 'version'>
+> & { version: DefaultVersionNumberDecoded };

x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { _version, description, id, meta, name } from '../common/schemas';
+import { _version, description, id, meta, name, version } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
 
 export const patchListSchema = t.intersection([
@@ -17,7 +17,15 @@ export const patchListSchema = t.intersection([
       id,
     })
   ),
-  t.exact(t.partial({ _version, description, meta, name })),
+  t.exact(
+    t.partial({
+      _version, // is undefined if not set during decode
+      description, // is undefined if not set during decode
+      meta, // is undefined if not set during decode
+      name, // is undefined if not set during decode
+      version, // is undefined if not set during decode
+    })
+  ),
 ]);
 
 export type PatchListSchema = t.OutputOf<typeof patchListSchema>;

x-pack/plugins/lists/common/schemas/request/update_exception_list_schema.ts

@@ -21,6 +21,7 @@ import {
   name,
   namespace_type,
   tags,
+  version,
 } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
 import { NamespaceType } from '../types';
@@ -42,6 +43,7 @@ export const updateExceptionListSchema = t.intersection([
       meta, // defaults to undefined if not set during decode
       namespace_type, // defaults to 'single' if not set during decode
       tags, // defaults to empty array if not set during decode
+      version, // defaults to undefined if not set during decode
     })
   ),
 ]);

x-pack/plugins/lists/common/schemas/request/update_list_schema.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { _version, description, id, meta, name } from '../common/schemas';
+import { _version, description, id, meta, name, version } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
 
 export const updateListSchema = t.intersection([
@@ -23,6 +23,7 @@ export const updateListSchema = t.intersection([
     t.partial({
       _version, // defaults to undefined if not set during decode
       meta, // defaults to undefined if not set during decode
+      version, // defaults to undefined if not set during decode
     })
   ),
 ]);

x-pack/plugins/lists/common/schemas/response/create_endpoint_list_schema.test.ts

@@ -41,7 +41,7 @@ describe('create_endpoint_list_schema', () => {
     const message = pipe(checked, foldLeftRight);
 
     expect(getPaths(left(message.errors))).toEqual([
-      'invalid keys "_tags,["endpoint","process","malware","os:linux"],_version,created_at,created_by,description,id,meta,{},name,namespace_type,tags,["user added string for a tag","malware"],tie_breaker_id,type,updated_at,updated_by"',
+      'invalid keys "_tags,["endpoint","process","malware","os:linux"],_version,created_at,created_by,description,id,immutable,meta,{},name,namespace_type,tags,["user added string for a tag","malware"],tie_breaker_id,type,updated_at,updated_by,version"',
     ]);
     expect(message.schema).toEqual({});
   });

x-pack/plugins/lists/common/schemas/response/exception_list_schema.mock.ts

@@ -8,9 +8,11 @@ import {
   DATE_NOW,
   DESCRIPTION,
   ENDPOINT_TYPE,
+  IMMUTABLE,
   META,
   TIE_BREAKER,
   USER,
+  VERSION,
   _VERSION,
 } from '../../constants.mock';
 import { ENDPOINT_LIST_ID } from '../..';
@@ -23,6 +25,7 @@ export const getExceptionListSchemaMock = (): ExceptionListSchema => ({
   created_by: USER,
   description: DESCRIPTION,
   id: '1',
+  immutable: IMMUTABLE,
   list_id: ENDPOINT_LIST_ID,
   meta: META,
   name: 'Sample Endpoint Exception List',
@@ -32,4 +35,5 @@ export const getExceptionListSchemaMock = (): ExceptionListSchema => ({
   type: ENDPOINT_TYPE,
   updated_at: DATE_NOW,
   updated_by: 'user_name',
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/response/exception_list_schema.ts

@@ -16,6 +16,7 @@ import {
   description,
   exceptionListType,
   id,
+  immutable,
   list_id,
   metaOrUndefined,
   name,
@@ -24,6 +25,7 @@ import {
   tie_breaker_id,
   updated_at,
   updated_by,
+  version,
 } from '../common/schemas';
 
 export const exceptionListSchema = t.exact(
@@ -34,6 +36,7 @@ export const exceptionListSchema = t.exact(
     created_by,
     description,
     id,
+    immutable,
     list_id,
     meta: metaOrUndefined,
     name,
@@ -43,6 +46,7 @@ export const exceptionListSchema = t.exact(
     type: exceptionListType,
     updated_at,
     updated_by,
+    version,
   })
 );
 

x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts

@@ -8,12 +8,14 @@ import { ListSchema } from '../../../common/schemas';
 import {
   DATE_NOW,
   DESCRIPTION,
+  IMMUTABLE,
   LIST_ID,
   META,
   NAME,
   TIE_BREAKER,
   TYPE,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 
 export const getListResponseMock = (): ListSchema => ({
@@ -23,6 +25,7 @@ export const getListResponseMock = (): ListSchema => ({
   description: DESCRIPTION,
   deserializer: undefined,
   id: LIST_ID,
+  immutable: IMMUTABLE,
   meta: META,
   name: NAME,
   serializer: undefined,
@@ -30,4 +33,5 @@ export const getListResponseMock = (): ListSchema => ({
   type: TYPE,
   updated_at: DATE_NOW,
   updated_by: USER,
+  version: VERSION,
 });

x-pack/plugins/lists/common/schemas/response/list_schema.ts

@@ -15,6 +15,7 @@ import {
   description,
   deserializerOrUndefined,
   id,
+  immutable,
   metaOrUndefined,
   name,
   serializerOrUndefined,
@@ -22,6 +23,7 @@ import {
   type,
   updated_at,
   updated_by,
+  version,
 } from '../common/schemas';
 
 export const listSchema = t.exact(
@@ -32,6 +34,7 @@ export const listSchema = t.exact(
     description,
     deserializer: deserializerOrUndefined,
     id,
+    immutable,
     meta: metaOrUndefined,
     name,
     serializer: serializerOrUndefined,
@@ -39,6 +42,7 @@ export const listSchema = t.exact(
     type,
     updated_at,
     updated_by,
+    version,
   })
 );
 

x-pack/plugins/lists/common/schemas/saved_objects/exceptions_list_so_schema.ts

@@ -16,6 +16,7 @@ import {
   description,
   exceptionListItemType,
   exceptionListType,
+  immutableOrUndefined,
   itemIdOrUndefined,
   list_id,
   list_type,
@@ -24,8 +25,12 @@ import {
   tags,
   tie_breaker_id,
   updated_by,
+  versionOrUndefined,
 } from '../common/schemas';
 
+/**
+ * Superset saved object of both lists and list items since they share the same saved object type.
+ */
 export const exceptionListSoSchema = t.exact(
   t.type({
     _tags,
@@ -34,6 +39,7 @@ export const exceptionListSoSchema = t.exact(
     created_by,
     description,
     entries: entriesArrayOrUndefined,
+    immutable: immutableOrUndefined,
     item_id: itemIdOrUndefined,
     list_id,
     list_type,
@@ -43,6 +49,7 @@ export const exceptionListSoSchema = t.exact(
     tie_breaker_id,
     type: t.union([exceptionListType, exceptionListItemType]),
     updated_by,
+    version: versionOrUndefined,
   })
 );
 

x-pack/plugins/lists/common/shared_imports.ts

@@ -8,6 +8,8 @@ export {
   NonEmptyString,
   DefaultUuid,
   DefaultStringArray,
+  DefaultVersionNumber,
+  DefaultVersionNumberDecoded,
   exactCheck,
   getPaths,
   foldLeftRight,

x-pack/plugins/lists/server/routes/create_exception_list_route.ts

@@ -43,6 +43,7 @@ export const createExceptionListRoute = (router: IRouter): void => {
           description,
           list_id: listId,
           type,
+          version,
         } = request.body;
         const exceptionLists = getExceptionListClient(context);
         const exceptionList = await exceptionLists.getExceptionList({
@@ -59,12 +60,14 @@ export const createExceptionListRoute = (router: IRouter): void => {
           const createdList = await exceptionLists.createExceptionList({
             _tags,
             description,
+            immutable: false,
             listId,
             meta,
             name,
             namespaceType,
             tags,
             type,
+            version,
           });
           const [validated, errors] = validate(createdList, exceptionListSchema);
           if (errors != null) {

x-pack/plugins/lists/server/routes/create_list_route.ts

@@ -9,7 +9,7 @@ import { IRouter } from 'kibana/server';
 import { LIST_URL } from '../../common/constants';
 import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps';
 import { validate } from '../../common/siem_common_deps';
-import { createListSchema, listSchema } from '../../common/schemas';
+import { CreateListSchemaDecoded, createListSchema, listSchema } from '../../common/schemas';
 
 import { getListClient } from '.';
 
@@ -21,13 +21,24 @@ export const createListRoute = (router: IRouter): void => {
       },
       path: LIST_URL,
       validate: {
-        body: buildRouteValidation(createListSchema),
+        body: buildRouteValidation<typeof createListSchema, CreateListSchemaDecoded>(
+          createListSchema
+        ),
       },
     },
     async (context, request, response) => {
       const siemResponse = buildSiemResponse(response);
       try {
-        const { name, description, deserializer, id, serializer, type, meta } = request.body;
+        const {
+          name,
+          description,
+          deserializer,
+          id,
+          serializer,
+          type,
+          meta,
+          version,
+        } = request.body;
         const lists = getListClient(context);
         const listExists = await lists.getListIndexExists();
         if (!listExists) {
@@ -49,10 +60,12 @@ export const createListRoute = (router: IRouter): void => {
             description,
             deserializer,
             id,
+            immutable: false,
             meta,
             name,
             serializer,
             type,
+            version,
           });
           const [validated, errors] = validate(list, listSchema);
           if (errors != null) {

x-pack/plugins/lists/server/routes/import_list_item_route.ts

@@ -55,6 +55,7 @@ export const importListItemRoute = (router: IRouter, config: ConfigType): void =
             serializer: list.serializer,
             stream,
             type: list.type,
+            version: 1,
           });
 
           const [validated, errors] = validate(list, listSchema);
@@ -71,6 +72,7 @@ export const importListItemRoute = (router: IRouter, config: ConfigType): void =
             serializer,
             stream,
             type,
+            version: 1,
           });
           if (importedList == null) {
             return siemResponse.error({

x-pack/plugins/lists/server/routes/patch_list_route.ts

@@ -27,9 +27,9 @@ export const patchListRoute = (router: IRouter): void => {
     async (context, request, response) => {
       const siemResponse = buildSiemResponse(response);
       try {
-        const { name, description, id, meta, _version } = request.body;
+        const { name, description, id, meta, _version, version } = request.body;
         const lists = getListClient(context);
-        const list = await lists.updateList({ _version, description, id, meta, name });
+        const list = await lists.updateList({ _version, description, id, meta, name, version });
         if (list == null) {
           return siemResponse.error({
             body: `list id: "${id}" found found`,

x-pack/plugins/lists/server/routes/update_exception_list_route.ts

@@ -45,6 +45,7 @@ export const updateExceptionListRoute = (router: IRouter): void => {
           meta,
           namespace_type: namespaceType,
           type,
+          version,
         } = request.body;
         const exceptionLists = getExceptionListClient(context);
         if (id == null && listId == null) {
@@ -64,6 +65,7 @@ export const updateExceptionListRoute = (router: IRouter): void => {
             namespaceType,
             tags,
             type,
+            version,
           });
           if (list == null) {
             return siemResponse.error({

x-pack/plugins/lists/server/routes/update_list_route.ts

@@ -27,9 +27,9 @@ export const updateListRoute = (router: IRouter): void => {
     async (context, request, response) => {
       const siemResponse = buildSiemResponse(response);
       try {
-        const { name, description, id, meta, _version } = request.body;
+        const { name, description, id, meta, _version, version } = request.body;
         const lists = getListClient(context);
-        const list = await lists.updateList({ _version, description, id, meta, name });
+        const list = await lists.updateList({ _version, description, id, meta, name, version });
         if (list == null) {
           return siemResponse.error({
             body: `list id: "${id}" found found`,

x-pack/plugins/lists/server/saved_objects/exception_list.ts

@@ -30,6 +30,9 @@ export const commonMapping: SavedObjectsType['mappings'] = {
     description: {
       type: 'keyword',
     },
+    immutable: {
+      type: 'boolean',
+    },
     list_id: {
       type: 'keyword',
     },
@@ -54,6 +57,9 @@ export const commonMapping: SavedObjectsType['mappings'] = {
     updated_by: {
       type: 'keyword',
     },
+    version: {
+      type: 'keyword',
+    },
   },
 };
 

x-pack/plugins/lists/server/services/exception_lists/create_endpoint_list.ts

@@ -12,7 +12,7 @@ import {
   ENDPOINT_LIST_ID,
   ENDPOINT_LIST_NAME,
 } from '../../../common/constants';
-import { ExceptionListSchema, ExceptionListSoSchema } from '../../../common/schemas';
+import { ExceptionListSchema, ExceptionListSoSchema, Version } from '../../../common/schemas';
 
 import { getSavedObjectType, transformSavedObjectToExceptionList } from './utils';
 
@@ -20,12 +20,14 @@ interface CreateEndpointListOptions {
   savedObjectsClient: SavedObjectsClientContract;
   user: string;
   tieBreaker?: string;
+  version: Version;
 }
 
 export const createEndpointList = async ({
   savedObjectsClient,
   user,
   tieBreaker,
+  version,
 }: CreateEndpointListOptions): Promise<ExceptionListSchema | null> => {
   const savedObjectType = getSavedObjectType({ namespaceType: 'agnostic' });
   const dateNow = new Date().toISOString();
@@ -39,6 +41,7 @@ export const createEndpointList = async ({
         created_by: user,
         description: ENDPOINT_LIST_DESCRIPTION,
         entries: undefined,
+        immutable: false,
         item_id: undefined,
         list_id: ENDPOINT_LIST_ID,
         list_type: 'list',
@@ -48,6 +51,7 @@ export const createEndpointList = async ({
         tie_breaker_id: tieBreaker ?? uuid.v4(),
         type: 'endpoint',
         updated_by: user,
+        version,
       },
       {
         // We intentionally hard coding the id so that there can only be one exception list within the space

x-pack/plugins/lists/server/services/exception_lists/create_exception_list.ts

@@ -12,11 +12,13 @@ import {
   ExceptionListSchema,
   ExceptionListSoSchema,
   ExceptionListType,
+  Immutable,
   ListId,
   MetaOrUndefined,
   Name,
   NamespaceType,
   Tags,
+  Version,
   _Tags,
 } from '../../../common/schemas';
 
@@ -29,16 +31,19 @@ interface CreateExceptionListOptions {
   namespaceType: NamespaceType;
   name: Name;
   description: Description;
+  immutable: Immutable;
   meta: MetaOrUndefined;
   user: string;
   tags: Tags;
   tieBreaker?: string;
   type: ExceptionListType;
+  version: Version;
 }
 
 export const createExceptionList = async ({
   _tags,
   listId,
+  immutable,
   savedObjectsClient,
   namespaceType,
   name,
@@ -48,6 +53,7 @@ export const createExceptionList = async ({
   tags,
   tieBreaker,
   type,
+  version,
 }: CreateExceptionListOptions): Promise<ExceptionListSchema> => {
   const savedObjectType = getSavedObjectType({ namespaceType });
   const dateNow = new Date().toISOString();
@@ -58,6 +64,7 @@ export const createExceptionList = async ({
     created_by: user,
     description,
     entries: undefined,
+    immutable,
     item_id: undefined,
     list_id: listId,
     list_type: 'list',
@@ -67,6 +74,7 @@ export const createExceptionList = async ({
     tie_breaker_id: tieBreaker ?? uuid.v4(),
     type,
     updated_by: user,
+    version,
   });
   return transformSavedObjectToExceptionList({ savedObject });
 };

x-pack/plugins/lists/server/services/exception_lists/create_exception_list_item.ts

@@ -72,6 +72,7 @@ export const createExceptionListItem = async ({
     created_by: user,
     description,
     entries,
+    immutable: undefined,
     item_id: itemId,
     list_id: listId,
     list_type: 'item',
@@ -81,6 +82,7 @@ export const createExceptionListItem = async ({
     tie_breaker_id: tieBreaker ?? uuid.v4(),
     type,
     updated_by: user,
+    version: undefined,
   });
   return transformSavedObjectToExceptionListItem({ savedObject });
 };

x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts

@@ -85,6 +85,7 @@ export class ExceptionListClient {
     return createEndpointList({
       savedObjectsClient,
       user,
+      version: 1,
     });
   };
 
@@ -176,17 +177,20 @@ export class ExceptionListClient {
   public createExceptionList = async ({
     _tags,
     description,
+    immutable,
     listId,
     meta,
     name,
     namespaceType,
     tags,
     type,
+    version,
   }: CreateExceptionListOptions): Promise<ExceptionListSchema> => {
     const { savedObjectsClient, user } = this;
     return createExceptionList({
       _tags,
       description,
+      immutable,
       listId,
       meta,
       name,
@@ -195,6 +199,7 @@ export class ExceptionListClient {
       tags,
       type,
       user,
+      version,
     });
   };
 
@@ -209,6 +214,7 @@ export class ExceptionListClient {
     namespaceType,
     tags,
     type,
+    version,
   }: UpdateExceptionListOptions): Promise<ExceptionListSchema | null> => {
     const { savedObjectsClient, user } = this;
     return updateExceptionList({
@@ -224,6 +230,7 @@ export class ExceptionListClient {
       tags,
       type,
       user,
+      version,
     });
   };
 

x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts

@@ -21,6 +21,7 @@ import {
   ExceptionListTypeOrUndefined,
   FilterOrUndefined,
   IdOrUndefined,
+  Immutable,
   ItemId,
   ItemIdOrUndefined,
   ListId,
@@ -36,6 +37,8 @@ import {
   Tags,
   TagsOrUndefined,
   UpdateCommentsArray,
+  Version,
+  VersionOrUndefined,
   _Tags,
   _TagsOrUndefined,
   _VersionOrUndefined,
@@ -61,6 +64,8 @@ export interface CreateExceptionListOptions {
   meta: MetaOrUndefined;
   tags: Tags;
   type: ExceptionListType;
+  immutable: Immutable;
+  version: Version;
 }
 
 export interface UpdateExceptionListOptions {
@@ -74,6 +79,7 @@ export interface UpdateExceptionListOptions {
   meta: MetaOrUndefined;
   tags: TagsOrUndefined;
   type: ExceptionListTypeOrUndefined;
+  version: VersionOrUndefined;
 }
 
 export interface DeleteExceptionListOptions {

x-pack/plugins/lists/server/services/exception_lists/update_exception_list.ts

@@ -17,6 +17,7 @@ import {
   NameOrUndefined,
   NamespaceType,
   TagsOrUndefined,
+  VersionOrUndefined,
   _TagsOrUndefined,
   _VersionOrUndefined,
 } from '../../../common/schemas';
@@ -38,6 +39,7 @@ interface UpdateExceptionListOptions {
   tags: TagsOrUndefined;
   tieBreaker?: string;
   type: ExceptionListTypeOrUndefined;
+  version: VersionOrUndefined;
 }
 
 export const updateExceptionList = async ({
@@ -53,12 +55,14 @@ export const updateExceptionList = async ({
   user,
   tags,
   type,
+  version,
 }: UpdateExceptionListOptions): Promise<ExceptionListSchema | null> => {
   const savedObjectType = getSavedObjectType({ namespaceType });
   const exceptionList = await getExceptionList({ id, listId, namespaceType, savedObjectsClient });
   if (exceptionList == null) {
     return null;
   } else {
+    const calculatedVersion = version == null ? exceptionList.version + 1 : version;
     const savedObject = await savedObjectsClient.update<ExceptionListSoSchema>(
       savedObjectType,
       exceptionList.id,
@@ -70,6 +74,7 @@ export const updateExceptionList = async ({
         tags,
         type,
         updated_by: user,
+        version: calculatedVersion,
       },
       {
         version: _version,

x-pack/plugins/lists/server/services/exception_lists/utils.ts

@@ -78,6 +78,7 @@ export const transformSavedObjectToExceptionList = ({
       created_at,
       created_by,
       description,
+      immutable,
       list_id,
       meta,
       name,
@@ -85,6 +86,7 @@ export const transformSavedObjectToExceptionList = ({
       tie_breaker_id,
       type,
       updated_by,
+      version,
     },
     id,
     updated_at: updatedAt,
@@ -99,6 +101,7 @@ export const transformSavedObjectToExceptionList = ({
     created_by,
     description,
     id,
+    immutable: immutable ?? false, // This should never be undefined for a list (only a list item)
     list_id,
     meta,
     name,
@@ -108,6 +111,7 @@ export const transformSavedObjectToExceptionList = ({
     type: exceptionListType.is(type) ? type : 'detection',
     updated_at: updatedAt ?? dateNow,
     updated_by,
+    version: version ?? 1, // This should never be undefined for a list (only a list item)
   };
 };
 
@@ -121,7 +125,17 @@ export const transformSavedObjectUpdateToExceptionList = ({
   const dateNow = new Date().toISOString();
   const {
     version: _version,
-    attributes: { _tags, description, meta, name, tags, type, updated_by: updatedBy },
+    attributes: {
+      _tags,
+      description,
+      immutable,
+      meta,
+      name,
+      tags,
+      type,
+      updated_by: updatedBy,
+      version,
+    },
     id,
     updated_at: updatedAt,
   } = savedObject;
@@ -135,6 +149,7 @@ export const transformSavedObjectUpdateToExceptionList = ({
     created_by: exceptionList.created_by,
     description: description ?? exceptionList.description,
     id,
+    immutable: immutable ?? exceptionList.immutable,
     list_id: exceptionList.list_id,
     meta: meta ?? exceptionList.meta,
     name: name ?? exceptionList.name,
@@ -144,6 +159,7 @@ export const transformSavedObjectUpdateToExceptionList = ({
     type: exceptionListType.is(type) ? type : exceptionList.type,
     updated_at: updatedAt ?? dateNow,
     updated_by: updatedBy ?? exceptionList.updated_by,
+    version: version ?? exceptionList.version,
   };
 };
 

x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.mock.ts

@@ -12,6 +12,7 @@ import {
   META,
   TYPE,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 import { getConfigMockDecoded } from '../../config.mock';
 
@@ -29,6 +30,7 @@ export const getImportListItemsToStreamOptionsMock = (): ImportListItemsToStream
   stream: new TestReadable(),
   type: TYPE,
   user: USER,
+  version: VERSION,
 });
 
 export const getWriteBufferToItemsOptionsMock = (): WriteBufferToItemsOptions => ({

x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts

@@ -16,6 +16,7 @@ import {
   MetaOrUndefined,
   SerializerOrUndefined,
   Type,
+  Version,
 } from '../../../common/schemas';
 import { ConfigType } from '../../config';
 
@@ -34,6 +35,7 @@ export interface ImportListItemsToStreamOptions {
   type: Type;
   user: string;
   meta: MetaOrUndefined;
+  version: Version;
 }
 
 export const importListItemsToStream = ({
@@ -48,6 +50,7 @@ export const importListItemsToStream = ({
   type,
   user,
   meta,
+  version,
 }: ImportListItemsToStreamOptions): Promise<ListSchema | null> => {
   return new Promise<ListSchema | null>((resolve) => {
     const readBuffer = new BufferLines({ bufferSize: config.importBufferSize, input: stream });
@@ -62,12 +65,14 @@ export const importListItemsToStream = ({
           description: `File uploaded from file system of ${fileNameEmitted}`,
           deserializer,
           id: fileNameEmitted,
+          immutable: false,
           listIndex,
           meta,
           name: fileNameEmitted,
           serializer,
           type,
           user,
+          version,
         });
       }
       readBuffer.resume();

x-pack/plugins/lists/server/services/lists/create_list.mock.ts

@@ -9,6 +9,7 @@ import { CreateListOptions } from '../lists';
 import {
   DATE_NOW,
   DESCRIPTION,
+  IMMUTABLE,
   LIST_ID,
   LIST_INDEX,
   META,
@@ -16,6 +17,7 @@ import {
   TIE_BREAKER,
   TYPE,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 
 export const getCreateListOptionsMock = (): CreateListOptions => ({
@@ -24,6 +26,7 @@ export const getCreateListOptionsMock = (): CreateListOptions => ({
   description: DESCRIPTION,
   deserializer: undefined,
   id: LIST_ID,
+  immutable: IMMUTABLE,
   listIndex: LIST_INDEX,
   meta: META,
   name: NAME,
@@ -31,4 +34,5 @@ export const getCreateListOptionsMock = (): CreateListOptions => ({
   tieBreaker: TIE_BREAKER,
   type: TYPE,
   user: USER,
+  version: VERSION,
 });

x-pack/plugins/lists/server/services/lists/create_list.ts

@@ -13,12 +13,14 @@ import {
   Description,
   DeserializerOrUndefined,
   IdOrUndefined,
+  Immutable,
   IndexEsListSchema,
   ListSchema,
   MetaOrUndefined,
   Name,
   SerializerOrUndefined,
   Type,
+  Version,
 } from '../../../common/schemas';
 
 export interface CreateListOptions {
@@ -34,6 +36,8 @@ export interface CreateListOptions {
   meta: MetaOrUndefined;
   dateNow?: string;
   tieBreaker?: string;
+  immutable: Immutable;
+  version: Version;
 }
 
 export const createList = async ({
@@ -49,6 +53,8 @@ export const createList = async ({
   meta,
   dateNow,
   tieBreaker,
+  immutable,
+  version,
 }: CreateListOptions): Promise<ListSchema> => {
   const createdAt = dateNow ?? new Date().toISOString();
   const body: IndexEsListSchema = {
@@ -56,6 +62,7 @@ export const createList = async ({
     created_by: user,
     description,
     deserializer,
+    immutable,
     meta,
     name,
     serializer,
@@ -63,6 +70,7 @@ export const createList = async ({
     type,
     updated_at: createdAt,
     updated_by: user,
+    version,
   };
   const response = await callCluster<CreateDocumentResponse>('index', {
     body,

x-pack/plugins/lists/server/services/lists/create_list_if_it_does_not_exist.ts

@@ -10,11 +10,13 @@ import {
   Description,
   DeserializerOrUndefined,
   Id,
+  Immutable,
   ListSchema,
   MetaOrUndefined,
   Name,
   SerializerOrUndefined,
   Type,
+  Version,
 } from '../../../common/schemas';
 
 import { getList } from './get_list';
@@ -27,12 +29,14 @@ export interface CreateListIfItDoesNotExistOptions {
   deserializer: DeserializerOrUndefined;
   serializer: SerializerOrUndefined;
   description: Description;
+  immutable: Immutable;
   callCluster: LegacyAPICaller;
   listIndex: string;
   user: string;
   meta: MetaOrUndefined;
   dateNow?: string;
   tieBreaker?: string;
+  version: Version;
 }
 
 export const createListIfItDoesNotExist = async ({
@@ -48,6 +52,8 @@ export const createListIfItDoesNotExist = async ({
   serializer,
   dateNow,
   tieBreaker,
+  version,
+  immutable,
 }: CreateListIfItDoesNotExistOptions): Promise<ListSchema> => {
   const list = await getList({ callCluster, id, listIndex });
   if (list == null) {
@@ -57,6 +63,7 @@ export const createListIfItDoesNotExist = async ({
       description,
       deserializer,
       id,
+      immutable,
       listIndex,
       meta,
       name,
@@ -64,6 +71,7 @@ export const createListIfItDoesNotExist = async ({
       tieBreaker,
       type,
       user,
+      version,
     });
   } else {
     return list;

x-pack/plugins/lists/server/services/lists/list_client.ts

@@ -110,11 +110,13 @@ export class ListClient {
   public createList = async ({
     id,
     deserializer,
+    immutable,
     serializer,
     name,
     description,
     type,
     meta,
+    version,
   }: CreateListOptions): Promise<ListSchema> => {
     const { callCluster, user } = this;
     const listIndex = this.getListIndex();
@@ -123,12 +125,14 @@ export class ListClient {
       description,
       deserializer,
       id,
+      immutable,
       listIndex,
       meta,
       name,
       serializer,
       type,
       user,
+      version,
     });
   };
 
@@ -138,8 +142,10 @@ export class ListClient {
     serializer,
     name,
     description,
+    immutable,
     type,
     meta,
+    version,
   }: CreateListIfItDoesNotExistOptions): Promise<ListSchema> => {
     const { callCluster, user } = this;
     const listIndex = this.getListIndex();
@@ -148,12 +154,14 @@ export class ListClient {
       description,
       deserializer,
       id,
+      immutable,
       listIndex,
       meta,
       name,
       serializer,
       type,
       user,
+      version,
     });
   };
 
@@ -334,6 +342,7 @@ export class ListClient {
     listId,
     stream,
     meta,
+    version,
   }: ImportListItemsToStreamOptions): Promise<ListSchema | null> => {
     const { callCluster, user, config } = this;
     const listItemIndex = this.getListItemIndex();
@@ -350,6 +359,7 @@ export class ListClient {
       stream,
       type,
       user,
+      version,
     });
   };
 
@@ -419,6 +429,7 @@ export class ListClient {
     name,
     description,
     meta,
+    version,
   }: UpdateListOptions): Promise<ListSchema | null> => {
     const { callCluster, user } = this;
     const listIndex = this.getListIndex();
@@ -431,6 +442,7 @@ export class ListClient {
       meta,
       name,
       user,
+      version,
     });
   };
 

x-pack/plugins/lists/server/services/lists/list_client_types.ts

@@ -15,6 +15,7 @@ import {
   Filter,
   Id,
   IdOrUndefined,
+  Immutable,
   ListId,
   ListIdOrUndefined,
   MetaOrUndefined,
@@ -26,6 +27,8 @@ import {
   SortFieldOrUndefined,
   SortOrderOrUndefined,
   Type,
+  Version,
+  VersionOrUndefined,
   _VersionOrUndefined,
 } from '../../../common/schemas';
 import { ConfigType } from '../../config';
@@ -52,11 +55,13 @@ export interface DeleteListItemOptions {
 export interface CreateListOptions {
   id: IdOrUndefined;
   deserializer: DeserializerOrUndefined;
+  immutable: Immutable;
   serializer: SerializerOrUndefined;
   name: Name;
   description: Description;
   type: Type;
   meta: MetaOrUndefined;
+  version: Version;
 }
 
 export interface CreateListIfItDoesNotExistOptions {
@@ -67,6 +72,8 @@ export interface CreateListIfItDoesNotExistOptions {
   description: Description;
   type: Type;
   meta: MetaOrUndefined;
+  version: Version;
+  immutable: Immutable;
 }
 
 export interface DeleteListItemByValueOptions {
@@ -94,6 +101,7 @@ export interface ImportListItemsToStreamOptions {
   type: Type;
   stream: Readable;
   meta: MetaOrUndefined;
+  version: Version;
 }
 
 export interface CreateListItemOptions {
@@ -119,6 +127,7 @@ export interface UpdateListOptions {
   name: NameOrUndefined;
   description: DescriptionOrUndefined;
   meta: MetaOrUndefined;
+  version: VersionOrUndefined;
 }
 
 export interface GetListItemOptions {

x-pack/plugins/lists/server/services/lists/list_mappings.json

@@ -34,6 +34,12 @@
     },
     "updated_by": {
       "type": "keyword"
+    },
+    "version": {
+      "type": "keyword"
+    },
+    "immutable": {
+      "type": "boolean"
     }
   }
 }

x-pack/plugins/lists/server/services/lists/update_list.mock.ts

@@ -13,6 +13,7 @@ import {
   META,
   NAME,
   USER,
+  VERSION,
 } from '../../../common/constants.mock';
 
 export const getUpdateListOptionsMock = (): UpdateListOptions => ({
@@ -25,4 +26,5 @@ export const getUpdateListOptionsMock = (): UpdateListOptions => ({
   meta: META,
   name: NAME,
   user: USER,
+  version: VERSION,
 });

x-pack/plugins/lists/server/services/lists/update_list.ts

@@ -16,6 +16,7 @@ import {
   MetaOrUndefined,
   NameOrUndefined,
   UpdateEsListSchema,
+  VersionOrUndefined,
   _VersionOrUndefined,
 } from '../../../common/schemas';
 
@@ -31,6 +32,7 @@ export interface UpdateListOptions {
   description: DescriptionOrUndefined;
   meta: MetaOrUndefined;
   dateNow?: string;
+  version: VersionOrUndefined;
 }
 
 export const updateList = async ({
@@ -43,12 +45,14 @@ export const updateList = async ({
   user,
   meta,
   dateNow,
+  version,
 }: UpdateListOptions): Promise<ListSchema | null> => {
   const updatedAt = dateNow ?? new Date().toISOString();
   const list = await getList({ callCluster, id, listIndex });
   if (list == null) {
     return null;
   } else {
+    const calculatedVersion = version == null ? list.version + 1 : version;
     const doc: UpdateEsListSchema = {
       description,
       meta,
@@ -70,6 +74,7 @@ export const updateList = async ({
       description: description ?? list.description,
       deserializer: list.deserializer,
       id: response._id,
+      immutable: list.immutable,
       meta,
       name: name ?? list.name,
       serializer: list.serializer,
@@ -77,6 +82,7 @@ export const updateList = async ({
       type: list.type,
       updated_at: updatedAt,
       updated_by: user,
+      version: calculatedVersion,
     };
   }
 };

x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_version_number.ts

@@ -19,3 +19,5 @@ export const DefaultVersionNumber = new t.Type<Version, Version | undefined, unk
     input == null ? t.success(1) : version.validate(input, context),
   t.identity
 );
+
+export type DefaultVersionNumberDecoded = t.TypeOf<typeof DefaultVersionNumber>;

x-pack/plugins/security_solution/common/shared_exports.ts

@@ -7,6 +7,10 @@
 export { NonEmptyString } from './detection_engine/schemas/types/non_empty_string';
 export { DefaultUuid } from './detection_engine/schemas/types/default_uuid';
 export { DefaultStringArray } from './detection_engine/schemas/types/default_string_array';
+export {
+  DefaultVersionNumber,
+  DefaultVersionNumberDecoded,
+} from './detection_engine/schemas/types/default_version_number';
 export { exactCheck } from './exact_check';
 export { getPaths, foldLeftRight } from './test_utils';
 export { validate, validateEither } from './validate';

x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.test.tsx

@@ -15,7 +15,7 @@ import { getField } from '../../../../../../../src/plugins/data/common/index_pat
 import { ListSchema } from '../../../lists_plugin_deps';
 import { getFoundListSchemaMock } from '../../../../../lists/common/schemas/response/found_list_schema.mock';
 import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock';
-import { DATE_NOW } from '../../../../../lists/common/constants.mock';
+import { DATE_NOW, VERSION, IMMUTABLE } from '../../../../../lists/common/constants.mock';
 
 import { AutocompleteFieldListsComponent } from './field_value_lists';
 
@@ -221,6 +221,8 @@ describe('AutocompleteFieldListsComponent', () => {
       type: 'ip',
       updated_at: DATE_NOW,
       updated_by: 'some user',
+      version: VERSION,
+      immutable: IMMUTABLE,
     });
   });
 });