[SIEM][Detections] Adds large list support using REST endpoints

## Summary
* Adds large list support using REST endpoints.

Status: 
---

* Currently ready to be merged behind the feature flag of it being disabled with ongoing work happening after it is merged. 
* REST Endpoints shouldn't have large refactoring at this point
* Team meeting occurred where the pieces were discussed in person.

What is left?
---

- [ ] Add other data types. At the moment `ip` and `keyword` are the two types of lists. See: https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html
- [x] Unit tests
- [x] Lots of misc TODO's in the code base still
- [ ] Import loads everything into memory first when it should attempt streaming
- [ ] Add end to end backend tests
- [x] Add transform and io-ts validation for returns 

Testing
---

Ensure you set this in your ENV before starting Kibana:
```ts
export ELASTIC_XPACK_SIEM_LISTS_FEATURE=true
```

Download or create a large list file such as this one filled with IP's:
https://cinsscore.com/list/ci-badguys.txt

Go to your REST endpoint folder of scripts:
```ts
cd kibana/x-pack/plugins/lists/server/scripts
```

Do a hard reset:
```ts
./hard_reset
```

Then import it as either a data type of `ip`:
```ts
./import_list_items_by_filename.sh ip ~/Downloads/ci-badguys-smaller.txt
```

Or as a `keyword`
```ts
./import_list_items_by_filename.sh keyword ~/Downloads/ci-badguys-smaller.txt
```

Then you can export it through:
```ts
./export_list_items.sh ci-badgusy-smaller.txt
```

For all the other endpoints and testing of the CRUD operations you have access to:

```ts
delete_all_lists.sh
delete_list.sh
delete_list_index.sh
delete_list_item.sh
delete_list_item_by_id.sh
delete_list_item_by_value.sh
export_list_items.sh
export_list_items_to_file.sh
get_list.sh
get_list_item_by_id.sh
get_list_item_by_value.sh
import_list_items.sh
import_list_items_by_filename.sh
lists_index_exists.sh
patch_list.sh
patch_list_item.sh
post_list.sh
post_list_index.sh
post_list_item.sh
```

### Checklist

Delete any items that are not applicable to this PR.

- [ ] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

### For maintainers

- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

.eslintrc.js

@@ -742,6 +742,101 @@ module.exports = {
       },
     },
 
+    /**
+     * Lists overrides
+     */
+    {
+      // typescript and javascript for front and back end
+      files: ['x-pack/plugins/lists/**/*.{js,ts,tsx}'],
+      plugins: ['eslint-plugin-node'],
+      env: {
+        mocha: true,
+        jest: true,
+      },
+      rules: {
+        'accessor-pairs': 'error',
+        'array-callback-return': 'error',
+        'no-array-constructor': 'error',
+        complexity: 'error',
+        'consistent-return': 'error',
+        'func-style': ['error', 'expression'],
+        'import/order': [
+          'error',
+          {
+            groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
+            'newlines-between': 'always',
+          },
+        ],
+        'sort-imports': [
+          'error',
+          {
+            ignoreDeclarationSort: true,
+          },
+        ],
+        'node/no-deprecated-api': 'error',
+        'no-bitwise': 'error',
+        'no-continue': 'error',
+        'no-dupe-keys': 'error',
+        'no-duplicate-case': 'error',
+        'no-duplicate-imports': 'error',
+        'no-empty-character-class': 'error',
+        'no-empty-pattern': 'error',
+        'no-ex-assign': 'error',
+        'no-extend-native': 'error',
+        'no-extra-bind': 'error',
+        'no-extra-boolean-cast': 'error',
+        'no-extra-label': 'error',
+        'no-func-assign': 'error',
+        'no-implicit-globals': 'error',
+        'no-implied-eval': 'error',
+        'no-invalid-regexp': 'error',
+        'no-inner-declarations': 'error',
+        'no-lone-blocks': 'error',
+        'no-multi-assign': 'error',
+        'no-misleading-character-class': 'error',
+        'no-new-symbol': 'error',
+        'no-obj-calls': 'error',
+        'no-param-reassign': ['error', { props: true }],
+        'no-process-exit': 'error',
+        'no-prototype-builtins': 'error',
+        'no-return-await': 'error',
+        'no-self-compare': 'error',
+        'no-shadow-restricted-names': 'error',
+        'no-sparse-arrays': 'error',
+        'no-this-before-super': 'error',
+        'no-undef': 'error',
+        'no-unreachable': 'error',
+        'no-unsafe-finally': 'error',
+        'no-useless-call': 'error',
+        'no-useless-catch': 'error',
+        'no-useless-concat': 'error',
+        'no-useless-computed-key': 'error',
+        'no-useless-escape': 'error',
+        'no-useless-rename': 'error',
+        'no-useless-return': 'error',
+        'no-void': 'error',
+        'one-var-declaration-per-line': 'error',
+        'prefer-object-spread': 'error',
+        'prefer-promise-reject-errors': 'error',
+        'prefer-rest-params': 'error',
+        'prefer-spread': 'error',
+        'prefer-template': 'error',
+        'require-atomic-updates': 'error',
+        'symbol-description': 'error',
+        'vars-on-top': 'error',
+        '@typescript-eslint/explicit-member-accessibility': 'error',
+        '@typescript-eslint/no-this-alias': 'error',
+        '@typescript-eslint/no-explicit-any': 'error',
+        '@typescript-eslint/no-useless-constructor': 'error',
+        '@typescript-eslint/unified-signatures': 'error',
+        '@typescript-eslint/explicit-function-return-type': 'error',
+        '@typescript-eslint/no-non-null-assertion': 'error',
+        '@typescript-eslint/no-unused-vars': 'error',
+        'no-template-curly-in-string': 'error',
+        'sort-keys': 'error',
+        'prefer-destructuring': 'error',
+      },
+    },
     /**
      * Alerting Services overrides
      */

.github/CODEOWNERS

@@ -224,7 +224,7 @@
 /x-pack/test/detection_engine_api_integration @elastic/siem
 /x-pack/test/api_integration/apis/siem @elastic/siem
 /x-pack/plugins/case @elastic/siem
+/x-pack/plugins/lists @elastic/siem
 
 # Security Intelligence And Analytics
-/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics
 /x-pack/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics

x-pack/plugins/lists/common/constants.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/**
+ * Lists routes
+ */
+export const LIST_URL = `/api/lists`;
+export const LIST_INDEX = `${LIST_URL}/index`;
+export const LIST_ITEM_URL = `${LIST_URL}/items`;

x-pack/plugins/lists/common/schemas/common/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './schemas';

x-pack/plugins/lists/common/schemas/common/schemas.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { NonEmptyString } from '../types/non_empty_string';
+
+export const name = t.string;
+export type Name = t.TypeOf<typeof name>;
+export const nameOrUndefined = t.union([name, t.undefined]);
+export type NameOrUndefined = t.TypeOf<typeof nameOrUndefined>;
+
+export const description = t.string;
+export type Description = t.TypeOf<typeof description>;
+export const descriptionOrUndefined = t.union([description, t.undefined]);
+export type DescriptionOrUndefined = t.TypeOf<typeof descriptionOrUndefined>;
+
+export const list_id = NonEmptyString;
+export const list_idOrUndefined = t.union([list_id, t.undefined]);
+export type List_idOrUndefined = t.TypeOf<typeof list_idOrUndefined>;
+
+export const item = t.string;
+export const created_at = t.string; // TODO: Make this into an ISO Date string check
+export const updated_at = t.string; // TODO: Make this into an ISO Date string check
+export const updated_by = t.string;
+export const created_by = t.string;
+export const file = t.object;
+
+export const id = NonEmptyString;
+export type Id = t.TypeOf<typeof id>;
+export const idOrUndefined = t.union([id, t.undefined]);
+export type IdOrUndefined = t.TypeOf<typeof idOrUndefined>;
+
+export const ip = t.string;
+export const ipOrUndefined = t.union([ip, t.undefined]);
+
+export const keyword = t.string;
+export const keywordOrUndefined = t.union([keyword, t.undefined]);
+
+export const value = t.string;
+export const valueOrUndefined = t.union([value, t.undefined]);
+
+export const tie_breaker_id = t.string; // TODO: Use UUID for this instead of a string for validation
+export const _index = t.string;
+
+export const type = t.keyof({ ip: null, keyword: null }); // TODO: Add the other data types here
+
+export const typeOrUndefined = t.union([type, t.undefined]);
+export type Type = t.TypeOf<typeof type>;
+
+export const meta = t.object;
+export type Meta = t.TypeOf<typeof meta>;
+export const metaOrUndefined = t.union([meta, t.undefined]);
+export type MetaOrUndefined = t.TypeOf<typeof metaOrUndefined>;
+
+export const esDataTypeUnion = t.union([t.type({ ip }), t.type({ keyword })]);
+export type EsDataTypeUnion = t.TypeOf<typeof esDataTypeUnion>;

x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+import { _index } from '../common/schemas';
+
+export const createEsBulkTypeSchema = t.exact(
+  t.type({
+    create: t.exact(t.type({ _index })),
+  })
+);
+
+export type CreateEsBulkTypeSchema = t.TypeOf<typeof createEsBulkTypeSchema>;

x-pack/plugins/lists/common/schemas/elastic_query/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+export * from './update_es_list_schema';
+export * from './index_es_list_schema';
+export * from './update_es_list_item_schema';
+export * from './index_es_list_item_schema';
+export * from './create_es_bulk_type';

x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  created_at,
+  created_by,
+  esDataTypeUnion,
+  list_id,
+  metaOrUndefined,
+  tie_breaker_id,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const indexEsListItemSchema = t.intersection([
+  t.exact(
+    t.type({
+      created_at,
+      created_by,
+      list_id,
+      meta: metaOrUndefined,
+      tie_breaker_id,
+      updated_at,
+      updated_by,
+    })
+  ),
+  esDataTypeUnion,
+]);
+
+export type IndexEsListItemSchema = t.TypeOf<typeof indexEsListItemSchema>;

x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  created_at,
+  created_by,
+  description,
+  metaOrUndefined,
+  name,
+  tie_breaker_id,
+  type,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const indexEsListSchema = t.exact(
+  t.type({
+    created_at,
+    created_by,
+    description,
+    meta: metaOrUndefined,
+    name,
+    tie_breaker_id,
+    type,
+    updated_at,
+    updated_by,
+  })
+);
+
+export type IndexEsListSchema = t.TypeOf<typeof indexEsListSchema>;

x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_item_schema.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { esDataTypeUnion, metaOrUndefined, updated_at, updated_by } from '../common/schemas';
+
+export const updateEsListItemSchema = t.intersection([
+  t.exact(
+    t.type({
+      meta: metaOrUndefined,
+      updated_at,
+      updated_by,
+    })
+  ),
+  esDataTypeUnion,
+]);
+
+export type UpdateEsListItemSchema = t.TypeOf<typeof updateEsListItemSchema>;

x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  descriptionOrUndefined,
+  metaOrUndefined,
+  nameOrUndefined,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const updateEsListSchema = t.exact(
+  t.type({
+    description: descriptionOrUndefined,
+    meta: metaOrUndefined,
+    name: nameOrUndefined,
+    updated_at,
+    updated_by,
+  })
+);
+
+export type UpdateEsListSchema = t.TypeOf<typeof updateEsListSchema>;

x-pack/plugins/lists/common/schemas/elastic_response/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './search_es_list_item_schema';
+export * from './search_es_list_schema';

x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  created_at,
+  created_by,
+  ipOrUndefined,
+  keywordOrUndefined,
+  list_id,
+  metaOrUndefined,
+  tie_breaker_id,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const searchEsListItemSchema = t.exact(
+  t.type({
+    created_at,
+    created_by,
+    ip: ipOrUndefined,
+    keyword: keywordOrUndefined,
+    list_id,
+    meta: metaOrUndefined,
+    tie_breaker_id,
+    updated_at,
+    updated_by,
+  })
+);
+
+export type SearchEsListItemSchema = t.TypeOf<typeof searchEsListItemSchema>;

x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  created_at,
+  created_by,
+  description,
+  metaOrUndefined,
+  name,
+  tie_breaker_id,
+  type,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const searchEsListSchema = t.exact(
+  t.type({
+    created_at,
+    created_by,
+    description,
+    meta: metaOrUndefined,
+    name,
+    tie_breaker_id,
+    type,
+    updated_at,
+    updated_by,
+  })
+);
+
+export type SearchEsListSchema = t.TypeOf<typeof searchEsListSchema>;

x-pack/plugins/lists/common/schemas/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './common';
+export * from './request';
+export * from './response';
+export * from './elastic_query';
+export * from './elastic_response';

x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { idOrUndefined, list_id, metaOrUndefined, value } from '../common/schemas';
+
+export const createListItemSchema = t.exact(
+  t.type({
+    id: idOrUndefined,
+    list_id,
+    meta: metaOrUndefined,
+    value,
+  })
+);
+
+export type CreateListItemSchema = t.TypeOf<typeof createListItemSchema>;

x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { left } from 'fp-ts/lib/Either';
+import { pipe } from 'fp-ts/lib/pipeable';
+
+import { exactCheck, foldLeftRight, getPaths } from '../../siem_common_deps';
+
+import { getListRequest } from './mocks/utils';
+import { createListSchema } from './create_list_schema';
+
+describe('create_list_schema', () => {
+  // TODO: Finish the tests for this
+  test('it should validate a typical lists request', () => {
+    const payload = getListRequest();
+    const decoded = createListSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual({
+      description: 'Description of a list item',
+      id: 'some-list-id',
+      name: 'Name of a list item',
+      type: 'ip',
+    });
+  });
+});

x-pack/plugins/lists/common/schemas/request/create_list_schema.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { description, idOrUndefined, metaOrUndefined, name, type } from '../common/schemas';
+
+export const createListSchema = t.exact(
+  t.type({
+    description,
+    id: idOrUndefined,
+    meta: metaOrUndefined,
+    name,
+    type,
+  })
+);
+
+export type CreateListSchema = t.TypeOf<typeof createListSchema>;

x-pack/plugins/lists/common/schemas/request/delete_list_item_schema.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { idOrUndefined, list_idOrUndefined, valueOrUndefined } from '../common/schemas';
+
+export const deleteListItemSchema = t.exact(
+  t.type({
+    id: idOrUndefined,
+    list_id: list_idOrUndefined,
+    value: valueOrUndefined,
+  })
+);
+
+export type DeleteListItemSchema = t.TypeOf<typeof deleteListItemSchema>;

x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { id } from '../common/schemas';
+
+export const deleteListSchema = t.exact(
+  t.type({
+    id,
+  })
+);
+
+export type DeleteListSchema = t.TypeOf<typeof deleteListSchema>;

x-pack/plugins/lists/common/schemas/request/export_list_item_query_schema.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { list_id } from '../common/schemas';
+
+export const exportListItemQuerySchema = t.exact(
+  t.type({
+    list_id,
+    // TODO: Add file_name here with a default value
+  })
+);
+
+export type ExportListItemQuerySchema = t.TypeOf<typeof exportListItemQuerySchema>;

x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { list_idOrUndefined, typeOrUndefined } from '../common/schemas';
+
+export const importListItemQuerySchema = t.exact(
+  t.type({ list_id: list_idOrUndefined, type: typeOrUndefined })
+);
+
+export type ImportListItemQuerySchema = t.TypeOf<typeof importListItemQuerySchema>;

x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import { Readable } from 'stream';
+
+import * as t from 'io-ts';
+
+import { file } from '../common/schemas';
+
+export const importListItemSchema = t.exact(
+  t.type({
+    file,
+  })
+);
+
+export interface HapiReadableStream extends Readable {
+  hapi: {
+    filename: string;
+  };
+}
+
+/**
+ * Special interface since we are streaming in a file through a reader
+ */
+export interface ImportListItemSchema {
+  file: HapiReadableStream;
+}

x-pack/plugins/lists/common/schemas/request/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './create_list_item_schema';
+export * from './create_list_schema';
+export * from './delete_list_item_schema';
+export * from './delete_list_schema';
+export * from './export_list_item_query_schema';
+export * from './import_list_item_schema';
+export * from './patch_list_item_schema';
+export * from './patch_list_schema';
+export * from './read_list_item_schema';
+export * from './read_list_schema';
+export * from './import_list_item_query_schema';
+export * from './update_list_schema';
+export * from './update_list_item_schema';

x-pack/plugins/lists/common/schemas/request/mocks/utils.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CreateListSchema } from '../create_list_schema';
+
+export const getListRequest = (): CreateListSchema => ({
+  description: 'Description of a list item',
+  id: 'some-list-id',
+  meta: undefined,
+  name: 'Name of a list item',
+  type: 'ip',
+});

x-pack/plugins/lists/common/schemas/request/patch_list_item_schema.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { id, metaOrUndefined, valueOrUndefined } from '../common/schemas';
+
+export const patchListItemSchema = t.exact(
+  t.type({
+    id,
+    meta: metaOrUndefined,
+    value: valueOrUndefined,
+  })
+);
+
+export type PatchListItemSchema = t.TypeOf<typeof patchListItemSchema>;

x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { descriptionOrUndefined, id, metaOrUndefined, nameOrUndefined } from '../common/schemas';
+
+export const patchListSchema = t.exact(
+  t.type({
+    description: descriptionOrUndefined,
+    id,
+    meta: metaOrUndefined,
+    name: nameOrUndefined,
+  })
+);
+
+export type PatchListSchema = t.TypeOf<typeof patchListSchema>;

x-pack/plugins/lists/common/schemas/request/read_list_item_schema.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { idOrUndefined, list_idOrUndefined, valueOrUndefined } from '../common/schemas';
+
+export const readListItemSchema = t.exact(
+  t.type({ id: idOrUndefined, list_id: list_idOrUndefined, value: valueOrUndefined })
+);
+
+export type ReadListItemSchema = t.TypeOf<typeof readListItemSchema>;

x-pack/plugins/lists/common/schemas/request/read_list_schema.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { id } from '../common/schemas';
+
+export const readListSchema = t.exact(
+  t.type({
+    id,
+  })
+);
+
+export type ReadListSchema = t.TypeOf<typeof readListSchema>;

x-pack/plugins/lists/common/schemas/request/update_list_item_schema.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { id, metaOrUndefined, value } from '../common/schemas';
+
+export const updateListItemSchema = t.exact(
+  t.type({
+    id,
+    meta: metaOrUndefined,
+    value,
+  })
+);
+
+export type UpdateListItemSchema = t.TypeOf<typeof updateListItemSchema>;

x-pack/plugins/lists/common/schemas/request/update_list_schema.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { description, id, metaOrUndefined, name } from '../common/schemas';
+
+export const updateListSchema = t.exact(
+  t.type({
+    description,
+    id,
+    meta: metaOrUndefined,
+    name,
+  })
+);
+
+export type UpdateListSchema = t.TypeOf<typeof updateListSchema>;

x-pack/plugins/lists/common/schemas/response/acknowledge_schema.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+export const acknowledgeSchema = t.type({ acknowledged: t.boolean });
+
+export type AcknowledgeSchema = t.TypeOf<typeof acknowledgeSchema>;

x-pack/plugins/lists/common/schemas/response/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './list_item_schema';
+export * from './list_schema';
+export * from './acknowledge_schema';
+export * from './list_item_index_exist_schema';

x-pack/plugins/lists/common/schemas/response/list_item_index_exist_schema.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+export const listItemIndexExistSchema = t.type({
+  list_index: t.boolean,
+  list_item_index: t.boolean,
+});
+
+export type ListItemIndexExistSchema = t.TypeOf<typeof listItemIndexExistSchema>;

x-pack/plugins/lists/common/schemas/response/list_item_schema.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import {
+  created_at,
+  created_by,
+  id,
+  list_id,
+  metaOrUndefined,
+  tie_breaker_id,
+  type,
+  updated_at,
+  updated_by,
+  value,
+} from '../common/schemas';
+
+export const listItemSchema = t.exact(
+  t.type({
+    created_at,
+    created_by,
+    id,
+    list_id,
+    meta: metaOrUndefined,
+    tie_breaker_id,
+    type,
+    updated_at,
+    updated_by,
+    value,
+  })
+);
+
+export type ListItemSchema = t.TypeOf<typeof listItemSchema>;
+
+export const listItemArraySchema = t.array(listItemSchema);
+export type ListItemArraySchema = t.TypeOf<typeof listItemArraySchema>;

x-pack/plugins/lists/common/schemas/response/list_schema.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  created_at,
+  created_by,
+  description,
+  id,
+  metaOrUndefined,
+  name,
+  tie_breaker_id,
+  type,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const listSchema = t.exact(
+  t.type({
+    created_at,
+    created_by,
+    description,
+    id,
+    meta: metaOrUndefined,
+    name,
+    tie_breaker_id,
+    type,
+    updated_at,
+    updated_by,
+  })
+);
+
+export type ListSchema = t.TypeOf<typeof listSchema>;

x-pack/plugins/lists/common/schemas/types/non_empty_string.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+import { Either } from 'fp-ts/lib/Either';
+
+export type NonEmptyStringC = t.Type<string, string, unknown>;
+
+/**
+ * Types the NonEmptyString as:
+ *   - A string that is not empty
+ */
+export const NonEmptyString: NonEmptyStringC = new t.Type<string, string, unknown>(
+  'NonEmptyString',
+  t.string.is,
+  (input, context): Either<t.Errors, string> => {
+    if (typeof input === 'string' && input.trim() !== '') {
+      return t.success(input);
+    } else {
+      return t.failure(input, context);
+    }
+  },
+  t.identity
+);

x-pack/plugins/lists/common/siem_common_deps.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { getPaths, foldLeftRight } from '../../siem/server/utils/build_validation/__mocks__/utils';
+export { exactCheck } from '../../siem/server/utils/build_validation/exact_check';

x-pack/plugins/lists/kibana.json

@@ -0,0 +1,10 @@
+{
+  "configPath": ["xpack", "lists"],
+  "id": "lists",
+  "kibanaVersion": "kibana",
+  "requiredPlugins": [],
+  "optionalPlugins": ["spaces", "security"],
+  "server": true,
+  "ui": false,
+  "version": "8.0.0"
+}

x-pack/plugins/lists/server/config.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TypeOf, schema } from '@kbn/config-schema';
+
+export const ConfigSchema = schema.object({
+  enabled: schema.boolean({ defaultValue: false }),
+  listIndex: schema.string({ defaultValue: '.lists' }),
+  listItemIndex: schema.string({ defaultValue: '.items' }),
+});
+
+export type ConfigType = TypeOf<typeof ConfigSchema>;

x-pack/plugins/lists/server/create_config.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { map } from 'rxjs/operators';
+import { PluginInitializerContext } from 'kibana/server';
+import { Observable } from 'rxjs';
+
+import { ConfigType } from './config';
+
+export const createConfig$ = (
+  context: PluginInitializerContext
+): Observable<Readonly<{
+  enabled: boolean;
+  listIndex: string;
+  listItemIndex: string;
+}>> => {
+  return context.config.create<ConfigType>().pipe(map(config => config));
+};

x-pack/plugins/lists/server/error_with_status_code.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export class ErrorWithStatusCode extends Error {
+  private readonly statusCode: number;
+
+  constructor(message: string, statusCode: number) {
+    super(message);
+    this.statusCode = statusCode;
+  }
+
+  public getStatusCode = (): number => this.statusCode;
+}

x-pack/plugins/lists/server/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { PluginInitializerContext } from '../../../../src/core/server';
+
+import { ConfigSchema } from './config';
+import { ListPlugin } from './plugin';
+
+export const config = { schema: ConfigSchema };
+export const plugin = (initializerContext: PluginInitializerContext): ListPlugin =>
+  new ListPlugin(initializerContext);

x-pack/plugins/lists/server/plugin.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { first } from 'rxjs/operators';
+import { ElasticsearchServiceSetup, Logger, PluginInitializerContext } from 'kibana/server';
+import { CoreSetup } from 'src/core/server';
+
+import { SecurityPluginSetup } from '../../security/server';
+import { SpacesServiceSetup } from '../../spaces/server';
+
+import { ConfigType } from './config';
+import { initRoutes } from './routes/init_routes';
+import { ListClient } from './services/lists/client';
+import { ContextProvider, ContextProviderReturn, PluginsSetup } from './types';
+import { createConfig$ } from './create_config';
+
+export class ListPlugin {
+  private readonly logger: Logger;
+  private spaces: SpacesServiceSetup | undefined | null;
+  private config: ConfigType | undefined | null;
+  private elasticsearch: ElasticsearchServiceSetup | undefined | null;
+  private security: SecurityPluginSetup | undefined | null;
+
+  constructor(private readonly initializerContext: PluginInitializerContext) {
+    this.logger = this.initializerContext.logger.get();
+  }
+
+  public async setup(core: CoreSetup, plugins: PluginsSetup): Promise<void> {
+    const config = await createConfig$(this.initializerContext)
+      .pipe(first())
+      .toPromise();
+
+    this.logger.error(
+      'You have activated the lists values feature flag which is NOT currently supported for Elastic Security! You should turn this feature flag off immediately by un-setting "xpack.lists.enabled: true" in kibana.yml and restarting Kibana'
+    );
+    this.spaces = plugins.spaces?.spacesService;
+    this.config = config;
+    this.elasticsearch = core.elasticsearch;
+    this.security = plugins.security;
+
+    core.http.registerRouteHandlerContext('lists', this.createRouteHandlerContext());
+    const router = core.http.createRouter();
+    initRoutes(router);
+  }
+
+  public start(): void {
+    this.logger.debug('Starting plugin');
+  }
+
+  public stop(): void {
+    this.logger.debug('Stopping plugin');
+  }
+
+  private createRouteHandlerContext = (): ContextProvider => {
+    return async (context, request): ContextProviderReturn => {
+      const { spaces, config, security, elasticsearch } = this;
+      const {
+        core: {
+          elasticsearch: { dataClient },
+        },
+      } = context;
+      if (config == null) {
+        throw new TypeError('Configuration is required for this plugin to operate');
+      } else if (elasticsearch == null) {
+        throw new TypeError('Elastic Search is required for this plugin to operate');
+      } else if (security == null) {
+        // TODO: This might be null, test authentication being turned off.
+        throw new TypeError('Security plugin is required for this plugin to operate');
+      } else {
+        return {
+          getListClient: (): ListClient =>
+            new ListClient({
+              config,
+              dataClient,
+              request,
+              security,
+              spaces,
+            }),
+        };
+      }
+    };
+  };
+}

x-pack/plugins/lists/server/routes/create_list_index_route.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { buildSiemResponse, transformError, validate } from '../siem_server_deps';
+import { LIST_INDEX } from '../../common/constants';
+import { acknowledgeSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const createListIndexRoute = (router: IRouter): void => {
+  router.post(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_INDEX,
+      validate: false,
+    },
+    async (context, _, response) => {
+      const siemResponse = buildSiemResponse(response);
+
+      try {
+        const lists = getListClient(context);
+        const listIndexExists = await lists.getListIndexExists();
+        const listItemIndexExists = await lists.getListItemIndexExists();
+
+        if (listIndexExists && listItemIndexExists) {
+          return siemResponse.error({
+            body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`,
+            statusCode: 409,
+          });
+        } else {
+          const policyExists = await lists.getListPolicyExists();
+          const policyListItemExists = await lists.getListItemPolicyExists();
+
+          if (!policyExists) {
+            await lists.setListPolicy();
+          }
+          if (!policyListItemExists) {
+            await lists.setListItemPolicy();
+          }
+
+          const templateExists = await lists.getListTemplateExists();
+          const templateListItemsExists = await lists.getListItemTemplateExists();
+
+          if (!templateExists) {
+            await lists.setListTemplate();
+          }
+
+          if (!templateListItemsExists) {
+            await lists.setListItemTemplate();
+          }
+
+          if (!listIndexExists) {
+            await lists.createListBootStrapIndex();
+          }
+          if (!listItemIndexExists) {
+            await lists.createListItemBootStrapIndex();
+          }
+
+          const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/create_list_item_route.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_ITEM_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { createListItemSchema, listItemSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const createListItemRoute = (router: IRouter): void => {
+  router.post(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_ITEM_URL,
+      validate: {
+        body: buildRouteValidation(createListItemSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { id, list_id: listId, value, meta } = request.body;
+        const lists = getListClient(context);
+        const list = await lists.getList({ id: listId });
+        if (list == null) {
+          return siemResponse.error({
+            body: `list id: "${listId}" does not exist`,
+            statusCode: 404,
+          });
+        } else {
+          const listItem = await lists.getListItemByValue({ listId, type: list.type, value });
+          if (listItem.length !== 0) {
+            return siemResponse.error({
+              body: `list_id: "${listId}" already contains the given value: ${value}`,
+              statusCode: 409,
+            });
+          } else {
+            const createdListItem = await lists.createListItem({
+              id,
+              listId,
+              meta,
+              type: list.type,
+              value,
+            });
+            const [validated, errors] = validate(createdListItem, listItemSchema);
+            if (errors != null) {
+              return siemResponse.error({ body: errors, statusCode: 500 });
+            } else {
+              return response.ok({ body: validated ?? {} });
+            }
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/create_list_route.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { createListSchema, listSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const createListRoute = (router: IRouter): void => {
+  router.post(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_URL,
+      validate: {
+        body: buildRouteValidation(createListSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { name, description, id, type, meta } = request.body;
+        const lists = getListClient(context);
+        const listExists = await lists.getListIndexExists();
+        if (!listExists) {
+          return siemResponse.error({
+            body: `To create a list, the index must exist first. Index "${lists.getListIndex()}" does not exist`,
+            statusCode: 400,
+          });
+        } else {
+          if (id != null) {
+            const list = await lists.getList({ id });
+            if (list != null) {
+              return siemResponse.error({
+                body: `list id: "${id}" already exists`,
+                statusCode: 409,
+              });
+            }
+          }
+          const list = await lists.createList({ description, id, meta, name, type });
+          const [validated, errors] = validate(list, listSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/delete_list_index_route.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_INDEX } from '../../common/constants';
+import { buildSiemResponse, transformError, validate } from '../siem_server_deps';
+import { acknowledgeSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+/**
+ * Deletes all of the indexes, template, ilm policies, and aliases. You can check
+ * this by looking at each of these settings from ES after a deletion:
+ *
+ * GET /_template/.lists-default
+ * GET /.lists-default-000001/
+ * GET /_ilm/policy/.lists-default
+ * GET /_alias/.lists-default
+ *
+ * GET /_template/.items-default
+ * GET /.items-default-000001/
+ * GET /_ilm/policy/.items-default
+ * GET /_alias/.items-default
+ *
+ * And ensuring they're all gone
+ */
+export const deleteListIndexRoute = (router: IRouter): void => {
+  router.delete(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_INDEX,
+      validate: false,
+    },
+    async (context, _, response) => {
+      const siemResponse = buildSiemResponse(response);
+
+      try {
+        const lists = getListClient(context);
+        const listIndexExists = await lists.getListIndexExists();
+        const listItemIndexExists = await lists.getListItemIndexExists();
+
+        if (!listIndexExists && !listItemIndexExists) {
+          return siemResponse.error({
+            body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" does not exist`,
+            statusCode: 404,
+          });
+        } else {
+          if (listIndexExists) {
+            await lists.deleteListIndex();
+          }
+          if (listItemIndexExists) {
+            await lists.deleteListItemIndex();
+          }
+
+          const listsPolicyExists = await lists.getListPolicyExists();
+          const listItemPolicyExists = await lists.getListItemPolicyExists();
+
+          if (listsPolicyExists) {
+            await lists.deleteListPolicy();
+          }
+          if (listItemPolicyExists) {
+            await lists.deleteListItemPolicy();
+          }
+
+          const listsTemplateExists = await lists.getListTemplateExists();
+          const listItemTemplateExists = await lists.getListItemTemplateExists();
+
+          if (listsTemplateExists) {
+            await lists.deleteListTemplate();
+          }
+          if (listItemTemplateExists) {
+            await lists.deleteListItemTemplate();
+          }
+
+          const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/delete_list_item_route.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_ITEM_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { deleteListItemSchema, listItemArraySchema, listItemSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const deleteListItemRoute = (router: IRouter): void => {
+  router.delete(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_ITEM_URL,
+      validate: {
+        query: buildRouteValidation(deleteListItemSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { id, list_id: listId, value } = request.query;
+        const lists = getListClient(context);
+        if (id != null) {
+          const deleted = await lists.deleteListItem({ id });
+          if (deleted == null) {
+            return siemResponse.error({
+              body: `list item with id: "${id}" item not found`,
+              statusCode: 404,
+            });
+          } else {
+            const [validated, errors] = validate(deleted, listItemSchema);
+            if (errors != null) {
+              return siemResponse.error({ body: errors, statusCode: 500 });
+            } else {
+              return response.ok({ body: validated ?? {} });
+            }
+          }
+        } else if (listId != null && value != null) {
+          const list = await lists.getList({ id: listId });
+          if (list == null) {
+            return siemResponse.error({
+              body: `list_id: "${listId}" does not exist`,
+              statusCode: 404,
+            });
+          } else {
+            const deleted = await lists.deleteListItemByValue({ listId, type: list.type, value });
+            if (deleted == null || deleted.length === 0) {
+              return siemResponse.error({
+                body: `list_id: "${listId}" with ${value} was not found`,
+                statusCode: 404,
+              });
+            } else {
+              const [validated, errors] = validate(deleted, listItemArraySchema);
+              if (errors != null) {
+                return siemResponse.error({ body: errors, statusCode: 500 });
+              } else {
+                return response.ok({ body: validated ?? {} });
+              }
+            }
+          }
+        } else {
+          return siemResponse.error({
+            body: `Either "list_id" or "id" needs to be defined in the request`,
+            statusCode: 400,
+          });
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/delete_list_route.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { deleteListSchema, listSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const deleteListRoute = (router: IRouter): void => {
+  router.delete(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_URL,
+      validate: {
+        query: buildRouteValidation(deleteListSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const lists = getListClient(context);
+        const { id } = request.query;
+        const deleted = await lists.deleteList({ id });
+        if (deleted == null) {
+          return siemResponse.error({
+            body: `list id: "${id}" was not found`,
+            statusCode: 404,
+          });
+        } else {
+          const [validated, errors] = validate(deleted, listSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/export_list_item_route.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Stream } from 'stream';
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_ITEM_URL } from '../../common/constants';
+import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps';
+import { exportListItemQuerySchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const exportListItemRoute = (router: IRouter): void => {
+  router.post(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: `${LIST_ITEM_URL}/_export`,
+      validate: {
+        query: buildRouteValidation(exportListItemQuerySchema),
+        // TODO: Do we want to add a body here like export_rules_route and allow a size limit?
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { list_id: listId } = request.query;
+        const lists = getListClient(context);
+        const list = await lists.getList({ id: listId });
+        if (list == null) {
+          return siemResponse.error({
+            body: `list_id: ${listId} does not exist`,
+            statusCode: 400,
+          });
+        } else {
+          // TODO: Allow the API to override the name of the file to export
+          const fileName = list.name;
+
+          const stream = new Stream.PassThrough();
+          lists.exportListItemsToStream({ listId, stream, stringToAppend: '\n' });
+          return response.ok({
+            body: stream,
+            headers: {
+              'Content-Disposition': `attachment; filename="${fileName}"`,
+              'Content-Type': 'text/plain',
+            },
+          });
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/import_list_item_route.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_ITEM_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import {
+  ImportListItemSchema,
+  importListItemQuerySchema,
+  importListItemSchema,
+  listSchema,
+} from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const importListItemRoute = (router: IRouter): void => {
+  router.post(
+    {
+      options: {
+        body: {
+          output: 'stream',
+        },
+        tags: ['access:lists'],
+      },
+      path: `${LIST_ITEM_URL}/_import`,
+      validate: {
+        body: buildRouteValidation<typeof importListItemSchema, ImportListItemSchema>(
+          importListItemSchema
+        ),
+        query: buildRouteValidation(importListItemQuerySchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { list_id: listId, type } = request.query;
+        const lists = getListClient(context);
+        if (listId != null) {
+          const list = await lists.getList({ id: listId });
+          if (list == null) {
+            return siemResponse.error({
+              body: `list id: "${listId}" does not exist`,
+              statusCode: 409,
+            });
+          }
+          await lists.importListItemsToStream({
+            listId,
+            meta: undefined,
+            stream: request.body.file,
+            type: list.type,
+          });
+
+          const [validated, errors] = validate(list, listSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        } else if (type != null) {
+          const { filename } = request.body.file.hapi;
+          // TODO: Should we prevent the same file from being uploaded multiple times?
+          const list = await lists.createListIfItDoesNotExist({
+            description: `File uploaded from file system of ${filename}`,
+            id: filename,
+            meta: undefined,
+            name: filename,
+            type,
+          });
+          await lists.importListItemsToStream({
+            listId: list.id,
+            meta: undefined,
+            stream: request.body.file,
+            type: list.type,
+          });
+          const [validated, errors] = validate(list, listSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        } else {
+          return siemResponse.error({
+            body: 'Either type or list_id need to be defined in the query',
+            statusCode: 400,
+          });
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './create_list_index_route';
+export * from './create_list_item_route';
+export * from './create_list_route';
+export * from './delete_list_index_route';
+export * from './delete_list_item_route';
+export * from './delete_list_route';
+export * from './export_list_item_route';
+export * from './import_list_item_route';
+export * from './init_routes';
+export * from './patch_list_item_route';
+export * from './patch_list_route';
+export * from './read_list_index_route';
+export * from './read_list_item_route';
+export * from './read_list_route';
+export * from './utils';

x-pack/plugins/lists/server/routes/init_routes.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { updateListRoute } from './update_list_route';
+import { updateListItemRoute } from './update_list_item_route';
+
+import {
+  createListIndexRoute,
+  createListItemRoute,
+  createListRoute,
+  deleteListIndexRoute,
+  deleteListItemRoute,
+  deleteListRoute,
+  exportListItemRoute,
+  importListItemRoute,
+  patchListItemRoute,
+  patchListRoute,
+  readListIndexRoute,
+  readListItemRoute,
+  readListRoute,
+} from '.';
+
+export const initRoutes = (router: IRouter): void => {
+  // lists
+  createListRoute(router);
+  readListRoute(router);
+  updateListRoute(router);
+  deleteListRoute(router);
+  patchListRoute(router);
+
+  // lists items
+  createListItemRoute(router);
+  readListItemRoute(router);
+  updateListItemRoute(router);
+  deleteListItemRoute(router);
+  patchListItemRoute(router);
+  exportListItemRoute(router);
+  importListItemRoute(router);
+
+  // indexes of lists
+  createListIndexRoute(router);
+  readListIndexRoute(router);
+  deleteListIndexRoute(router);
+};

x-pack/plugins/lists/server/routes/patch_list_item_route.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_ITEM_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { listItemSchema, patchListItemSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const patchListItemRoute = (router: IRouter): void => {
+  router.patch(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_ITEM_URL,
+      validate: {
+        body: buildRouteValidation(patchListItemSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { value, id, meta } = request.body;
+        const lists = getListClient(context);
+        const listItem = await lists.updateListItem({
+          id,
+          meta,
+          value,
+        });
+        if (listItem == null) {
+          return siemResponse.error({
+            body: `list item id: "${id}" not found`,
+            statusCode: 404,
+          });
+        } else {
+          const [validated, errors] = validate(listItem, listItemSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/patch_list_route.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { listSchema, patchListSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const patchListRoute = (router: IRouter): void => {
+  router.patch(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_URL,
+      validate: {
+        body: buildRouteValidation(patchListSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { name, description, id, meta } = request.body;
+        const lists = getListClient(context);
+        const list = await lists.updateList({ description, id, meta, name });
+        if (list == null) {
+          return siemResponse.error({
+            body: `list id: "${id}" found found`,
+            statusCode: 404,
+          });
+        } else {
+          const [validated, errors] = validate(list, listSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/read_list_index_route.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_INDEX } from '../../common/constants';
+import { buildSiemResponse, transformError, validate } from '../siem_server_deps';
+import { listItemIndexExistSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const readListIndexRoute = (router: IRouter): void => {
+  router.get(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_INDEX,
+      validate: false,
+    },
+    async (context, _, response) => {
+      const siemResponse = buildSiemResponse(response);
+
+      try {
+        const lists = getListClient(context);
+        const listIndexExists = await lists.getListIndexExists();
+        const listItemIndexExists = await lists.getListItemIndexExists();
+
+        if (listIndexExists || listItemIndexExists) {
+          const [validated, errors] = validate(
+            { list_index: listIndexExists, lists_item_index: listItemIndexExists },
+            listItemIndexExistSchema
+          );
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        } else if (!listIndexExists && listItemIndexExists) {
+          return siemResponse.error({
+            body: `index ${lists.getListIndex()} does not exist`,
+            statusCode: 404,
+          });
+        } else if (!listItemIndexExists && listIndexExists) {
+          return siemResponse.error({
+            body: `index ${lists.getListItemIndex()} does not exist`,
+            statusCode: 404,
+          });
+        } else {
+          return siemResponse.error({
+            body: `index ${lists.getListIndex()} and index ${lists.getListItemIndex()} does not exist`,
+            statusCode: 404,
+          });
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/read_list_item_route.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_ITEM_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { listItemArraySchema, listItemSchema, readListItemSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const readListItemRoute = (router: IRouter): void => {
+  router.get(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_ITEM_URL,
+      validate: {
+        query: buildRouteValidation(readListItemSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { id, list_id: listId, value } = request.query;
+        const lists = getListClient(context);
+        if (id != null) {
+          const listItem = await lists.getListItem({ id });
+          if (listItem == null) {
+            return siemResponse.error({
+              body: `list item id: "${id}" does not exist`,
+              statusCode: 404,
+            });
+          } else {
+            const [validated, errors] = validate(listItem, listItemSchema);
+            if (errors != null) {
+              return siemResponse.error({ body: errors, statusCode: 500 });
+            } else {
+              return response.ok({ body: validated ?? {} });
+            }
+          }
+        } else if (listId != null && value != null) {
+          const list = await lists.getList({ id: listId });
+          if (list == null) {
+            return siemResponse.error({
+              body: `list id: "${listId}" does not exist`,
+              statusCode: 404,
+            });
+          } else {
+            const listItem = await lists.getListItemByValue({
+              listId,
+              type: list.type,
+              value,
+            });
+            if (listItem.length === 0) {
+              return siemResponse.error({
+                body: `list_id: "${listId}" item of ${value} does not exist`,
+                statusCode: 404,
+              });
+            } else {
+              const [validated, errors] = validate(listItem, listItemArraySchema);
+              if (errors != null) {
+                return siemResponse.error({ body: errors, statusCode: 500 });
+              } else {
+                return response.ok({ body: validated ?? {} });
+              }
+            }
+          }
+        } else {
+          return siemResponse.error({
+            body: `Either "list_id" or "id" needs to be defined in the request`,
+            statusCode: 400,
+          });
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/read_list_route.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { listSchema, readListSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const readListRoute = (router: IRouter): void => {
+  router.get(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_URL,
+      validate: {
+        query: buildRouteValidation(readListSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { id } = request.query;
+        const lists = getListClient(context);
+        const list = await lists.getList({ id });
+        if (list == null) {
+          return siemResponse.error({
+            body: `list id: "${id}" does not exist`,
+            statusCode: 404,
+          });
+        } else {
+          const [validated, errors] = validate(list, listSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/update_list_item_route.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_ITEM_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { listItemSchema, updateListItemSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const updateListItemRoute = (router: IRouter): void => {
+  router.put(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_ITEM_URL,
+      validate: {
+        body: buildRouteValidation(updateListItemSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { value, id, meta } = request.body;
+        const lists = getListClient(context);
+        const listItem = await lists.updateListItem({
+          id,
+          meta,
+          value,
+        });
+        if (listItem == null) {
+          return siemResponse.error({
+            body: `list item id: "${id}" not found`,
+            statusCode: 404,
+          });
+        } else {
+          const [validated, errors] = validate(listItem, listItemSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/update_list_route.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+
+import { LIST_URL } from '../../common/constants';
+import {
+  buildRouteValidation,
+  buildSiemResponse,
+  transformError,
+  validate,
+} from '../siem_server_deps';
+import { listSchema, updateListSchema } from '../../common/schemas';
+
+import { getListClient } from '.';
+
+export const updateListRoute = (router: IRouter): void => {
+  router.put(
+    {
+      options: {
+        tags: ['access:lists'],
+      },
+      path: LIST_URL,
+      validate: {
+        body: buildRouteValidation(updateListSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const { name, description, id, meta } = request.body;
+        const lists = getListClient(context);
+        const list = await lists.updateList({ description, id, meta, name });
+        if (list == null) {
+          return siemResponse.error({
+            body: `list id: "${id}" found found`,
+            statusCode: 404,
+          });
+        } else {
+          const [validated, errors] = validate(list, listSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/utils/get_list_client.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { RequestHandlerContext } from 'kibana/server';
+
+import { ListClient } from '../../services/lists/client';
+import { ErrorWithStatusCode } from '../../error_with_status_code';
+
+export const getListClient = (context: RequestHandlerContext): ListClient => {
+  const lists = context.lists?.getListClient();
+  if (lists == null) {
+    throw new ErrorWithStatusCode('Lists is not found as a plugin', 404);
+  } else {
+    return lists;
+  }
+};

x-pack/plugins/lists/server/routes/utils/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+export * from './get_list_client';

x-pack/plugins/lists/server/scripts/check_env_variables.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# Add this to the start of any scripts to detect if env variables are set
+
+set -e 
+
+if [ -z "${ELASTICSEARCH_USERNAME}" ]; then
+  echo "Set ELASTICSEARCH_USERNAME in your environment"
+  exit 1
+fi
+
+if [ -z "${ELASTICSEARCH_PASSWORD}" ]; then
+  echo "Set ELASTICSEARCH_PASSWORD in your environment"
+  exit 1
+fi
+
+if [ -z "${ELASTICSEARCH_URL}" ]; then
+  echo "Set ELASTICSEARCH_URL in your environment"
+  exit 1
+fi
+
+if [ -z "${KIBANA_URL}" ]; then
+  echo "Set KIBANA_URL in your environment"
+  exit 1
+fi
+
+if [ -z "${TASK_MANAGER_INDEX}" ]; then
+  echo "Set TASK_MANAGER_INDEX in your environment"
+  exit 1
+fi
+
+if [ -z "${KIBANA_INDEX}" ]; then
+  echo "Set KIBANA_INDEX in your environment"
+  exit 1
+fi

x-pack/plugins/lists/server/scripts/delete_all_lists.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./delete_all_lists.sh
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+
+
+# Delete all the main lists that have children items
+curl -s -k \
+  -H "Content-Type: application/json" \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST ${ELASTICSEARCH_URL}/${KIBANA_INDEX}*/_delete_by_query \
+  --data '{
+    "query": {
+      "exists": { "field": "siem_list" }
+    }
+  }' \
+  | jq .
+
+# Delete all the list children items as well
+curl -s -k \
+  -H "Content-Type: application/json" \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST ${ELASTICSEARCH_URL}/${KIBANA_INDEX}*/_delete_by_query \
+  --data '{
+    "query": {
+      "exists": { "field": "siem_list_item" }
+    }
+  }' \
+  | jq .

x-pack/plugins/lists/server/scripts/delete_list.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./delete_list_by_list_id.sh ${list_id}
+curl -s -k \
+ -H 'kbn-xsrf: 123' \
+ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+ -X DELETE ${KIBANA_URL}${SPACE_URL}/api/lists?id="$1" | jq .

x-pack/plugins/lists/server/scripts/delete_list_index.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./delete_signal_index.sh
+curl -s -k \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X DELETE ${KIBANA_URL}${SPACE_URL}/api/lists/index | jq .

x-pack/plugins/lists/server/scripts/delete_list_item_by_id.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./delete_list_item_by_id.sh?id={id}
+curl -s -k \
+ -H 'kbn-xsrf: 123' \
+ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+ -X DELETE ${KIBANA_URL}${SPACE_URL}/api/lists/items?id=$1 | jq .

x-pack/plugins/lists/server/scripts/delete_list_item_by_value.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./delete_list_item_by_value.sh?list_id=${some_id}&value=${some_ip}
+curl -s -k \
+ -H 'kbn-xsrf: 123' \
+ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+ -X DELETE "${KIBANA_URL}${SPACE_URL}/api/lists/items?list_id=$1&value=$2" | jq .

x-pack/plugins/lists/server/scripts/export_list_items.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a defaults if no argument is specified
+LIST_ID=${1:-ips.txt}
+
+# Example to export
+# ./export_list_items.sh > /tmp/ips.txt
+
+curl -s -k \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST "${KIBANA_URL}${SPACE_URL}/api/lists/items/_export?list_id=${LIST_ID}"

x-pack/plugins/lists/server/scripts/export_list_items_to_file.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a defaults if no argument is specified
+FOLDER=${1:-/tmp}
+
+# Example to export
+# ./export_list_items_to_file.sh
+
+# Change current working directory as exports cause Kibana to restart
+pushd ${FOLDER} > /dev/null
+
+curl -s -k -OJ \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST "${KIBANA_URL}${SPACE_URL}/api/lists/items/_export?list_id=list-ip"
+
+popd > /dev/null

x-pack/plugins/lists/server/scripts/get_list.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./get_list.sh {list_id}
+curl -s -k \
+ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+ -X GET ${KIBANA_URL}${SPACE_URL}/api/lists?id="$1" | jq .

x-pack/plugins/lists/server/scripts/get_list_item_by_id.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./get_list_item_by_id ${id}
+curl -s -k \
+ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+ -X GET "${KIBANA_URL}${SPACE_URL}/api/lists/items?id=$1" | jq .

x-pack/plugins/lists/server/scripts/get_list_item_by_value.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./get_list_item_by_value.sh ${list_id} ${value}
+curl -s -k \
+ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+ -X GET "${KIBANA_URL}${SPACE_URL}/api/lists/items?list_id=$1&value=$2" | jq .

x-pack/plugins/lists/server/scripts/hard_reset.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# re-create the list and list item indexes
+./delete_list_index.sh
+./post_list_index.sh

x-pack/plugins/lists/server/scripts/import_list_items.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a defaults if no argument is specified
+LIST_ID=${1:-list-ip}
+FILE=${2:-./lists/files/ips.txt}
+
+# ./import_list_items.sh list-ip ./lists/files/ips.txt
+curl -s -k \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST "${KIBANA_URL}${SPACE_URL}/api/lists/items/_import?list_id=${LIST_ID}" \
+  --form file=@${FILE} \
+  | jq .;

x-pack/plugins/lists/server/scripts/import_list_items_by_filename.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a defaults if no argument is specified
+TYPE=${1:-ip}
+FILE=${2:-./lists/files/ips.txt}
+
+# Example to import ips from ./lists/files/ips.txt
+# ./import_list_items_by_filename.sh ip ./lists/files/ips.txt
+
+curl -s -k \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST "${KIBANA_URL}${SPACE_URL}/api/lists/items/_import?type=${TYPE}" \
+  --form file=@${FILE} \
+  | jq .;

x-pack/plugins/lists/server/scripts/lists/files/hosts.txt

@@ -0,0 +1,2 @@
+kibana
+rock01

x-pack/plugins/lists/server/scripts/lists/files/ips.txt

@@ -0,0 +1,9 @@
+127.0.0.1
+127.0.0.2
+127.0.0.3
+127.0.0.4
+127.0.0.5
+127.0.0.6
+127.0.0.7
+127.0.0.8
+127.0.0.9

x-pack/plugins/lists/server/scripts/lists/new/list_everything.json

@@ -0,0 +1,13 @@
+{
+  "id": "list-ip-everything",
+  "name": "Simple list with an ip",
+  "description": "This list describes bad internet ip",
+  "type": "ip",
+  "meta": {
+    "level_1_meta": {
+      "level_2_meta": {
+        "level_3_key": "some_value_ui"
+      }
+    }
+  }
+}

x-pack/plugins/lists/server/scripts/lists/new/list_ip.json

@@ -0,0 +1,6 @@
+{
+  "id": "list-ip",
+  "name": "Simple list with an ip",
+  "description": "This list describes bad internet ip",
+  "type": "ip"
+}

x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json

@@ -0,0 +1,5 @@
+{
+  "id": "hand_inserted_item_id",
+  "list_id": "list-ip",
+  "value": "127.0.0.1"
+}

x-pack/plugins/lists/server/scripts/lists/new/list_ip_item_everything.json

@@ -0,0 +1,12 @@
+{
+  "id": "hand_inserted_item_id_everything",
+  "list_id": "list-ip",
+  "value": "127.0.0.2",
+  "meta": {
+    "level_1_meta": {
+      "level_2_meta": {
+        "level_3_key": "some_value_ui"
+      }
+    }
+  }
+}

x-pack/plugins/lists/server/scripts/lists/new/list_ip_no_id.json

@@ -0,0 +1,5 @@
+{
+  "name": "Simple list with an ip",
+  "description": "This list describes bad internet ip",
+  "type": "ip"
+}

x-pack/plugins/lists/server/scripts/lists/new/list_keyword.json

@@ -0,0 +1,6 @@
+{
+  "id": "list-keyword",
+  "name": "Simple list with a keyword",
+  "description": "This list describes bad host names",
+  "type": "keyword"
+}

x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json

@@ -0,0 +1,4 @@
+{
+  "list_id": "list-keyword",
+  "value": "kibana"
+}

x-pack/plugins/lists/server/scripts/lists/patches/list_ip_item.json

@@ -0,0 +1,4 @@
+{
+  "id": "hand_inserted_item_id",
+  "value": "255.255.255.255"
+}

x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json

@@ -0,0 +1,4 @@
+{
+  "id": "list-ip",
+  "name": "Changed the name here to something else"
+}

x-pack/plugins/lists/server/scripts/lists/updates/list_ip_item.json

@@ -0,0 +1,4 @@
+{
+  "id": "hand_inserted_item_id",
+  "value": "255.255.255.255"
+}

x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json

@@ -0,0 +1,5 @@
+{
+  "id": "list-ip",
+  "name": "Changed the name here to something else",
+  "description": "Some other description here for you"
+}

x-pack/plugins/lists/server/scripts/lists_index_exists.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./lists_index_exists.sh
+curl -s -k -f \
+ -H 'Content-Type: application/json' \
+ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+ ${KIBANA_URL}${SPACE_URL}/api/lists/index | jq .

x-pack/plugins/lists/server/scripts/patch_list.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a default if no argument is specified
+LISTS=(${@:-./lists/patches/simplest_updated_name.json})
+
+# Example: ./patch_list.sh
+# Example: ./patch_list.sh ./lists/patches/simplest_updated_name.json
+for LIST in "${LISTS[@]}"
+do {
+  [ -e "$LIST" ] || continue
+  curl -s -k \
+  -H 'Content-Type: application/json' \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X PATCH ${KIBANA_URL}${SPACE_URL}/api/lists \
+  -d @${LIST} \
+  | jq .;
+} &
+done
+
+wait

x-pack/plugins/lists/server/scripts/patch_list_item.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a default if no argument is specified
+LISTS=(${@:-./lists/patches/list_ip_item.json})
+
+# Example: ./patch_list.sh
+# Example: ./patch_list.sh ./lists/patches/list_ip_item.json
+for LIST in "${LISTS[@]}"
+do {
+  [ -e "$LIST" ] || continue
+  curl -s -k \
+  -H 'Content-Type: application/json' \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X PATCH ${KIBANA_URL}${SPACE_URL}/api/lists/items \
+  -d @${LIST} \
+  | jq .;
+} &
+done
+
+wait

x-pack/plugins/lists/server/scripts/post_list.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a default if no argument is specified
+LISTS=(${@:-./lists/new/list_ip.json})
+
+# Example: ./post_list.sh
+# Example: ./post_list.sh ./lists/new/list_ip.json
+for LIST in "${LISTS[@]}"
+do {
+  [ -e "$LIST" ] || continue
+  curl -s -k \
+  -H 'Content-Type: application/json' \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST ${KIBANA_URL}${SPACE_URL}/api/lists \
+   -d @${LIST} \
+  | jq .;
+} &
+done
+
+wait

x-pack/plugins/lists/server/scripts/post_list_index.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./post_signal_index.sh
+curl -s -k \
+ -H 'Content-Type: application/json' \
+ -H 'kbn-xsrf: 123' \
+ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+ -X POST ${KIBANA_URL}${SPACE_URL}/api/lists/index | jq .

x-pack/plugins/lists/server/scripts/post_list_item.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a default if no argument is specified
+LISTS=(${@:-./lists/new/list_ip_item.json})
+
+# Example: ./post_list.sh
+# Example: ./post_list.sh ./lists/new/list_ip_item.json
+for LIST in "${LISTS[@]}"
+do {
+  [ -e "$LIST" ] || continue
+  curl -s -k \
+  -H 'Content-Type: application/json' \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST ${KIBANA_URL}${SPACE_URL}/api/lists/items \
+   -d @${LIST} \
+  | jq .;
+} &
+done
+
+wait

x-pack/plugins/lists/server/scripts/update_list.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a default if no argument is specified
+LISTS=(${@:-./lists/updates/simple_update.json})
+
+# Example: ./update_list.sh
+# Example: ./update_list.sh ./lists/updates/simple_update.json
+for LIST in "${LISTS[@]}"
+do {
+  [ -e "$LIST" ] || continue
+  curl -s -k \
+  -H 'Content-Type: application/json' \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X PUT ${KIBANA_URL}${SPACE_URL}/api/lists \
+  -d @${LIST} \
+  | jq .;
+} &
+done
+
+wait

x-pack/plugins/lists/server/scripts/update_list_item.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Uses a default if no argument is specified
+LISTS=(${@:-./lists/updates/list_ip_item.json})
+
+# Example: ./patch_list.sh
+# Example: ./patch_list.sh ./lists/updates/list_ip_item.json
+for LIST in "${LISTS[@]}"
+do {
+  [ -e "$LIST" ] || continue
+  curl -s -k \
+  -H 'Content-Type: application/json' \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X PATCH ${KIBANA_URL}${SPACE_URL}/api/lists/items \
+  -d @${LIST} \
+  | jq .;
+} &
+done
+
+wait

x-pack/plugins/lists/server/services/items/buffer_lines.test.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TestReadable } from '../mocks/test_readable';
+
+import { BufferLines } from './buffer_lines';
+
+describe('buffer_lines', () => {
+  test('it can read a single line', done => {
+    const input = new TestReadable();
+    input.push('line one\n');
+    input.push(null);
+    const bufferedLine = new BufferLines({ input });
+    let linesToTest: string[] = [];
+    bufferedLine.on('lines', (lines: string[]) => {
+      linesToTest = [...linesToTest, ...lines];
+    });
+    bufferedLine.on('close', () => {
+      expect(linesToTest).toEqual(['line one']);
+      done();
+    });
+  });
+
+  test('it can read two lines', done => {
+    const input = new TestReadable();
+    input.push('line one\n');
+    input.push('line two\n');
+    input.push(null);
+    const bufferedLine = new BufferLines({ input });
+    let linesToTest: string[] = [];
+    bufferedLine.on('lines', (lines: string[]) => {
+      linesToTest = [...linesToTest, ...lines];
+    });
+    bufferedLine.on('close', () => {
+      expect(linesToTest).toEqual(['line one', 'line two']);
+      done();
+    });
+  });
+
+  test('two identical lines are collapsed into just one line without duplicates', done => {
+    const input = new TestReadable();
+    input.push('line one\n');
+    input.push('line one\n');
+    input.push(null);
+    const bufferedLine = new BufferLines({ input });
+    let linesToTest: string[] = [];
+    bufferedLine.on('lines', (lines: string[]) => {
+      linesToTest = [...linesToTest, ...lines];
+    });
+    bufferedLine.on('close', () => {
+      expect(linesToTest).toEqual(['line one']);
+      done();
+    });
+  });
+
+  test('it can close out without writing any lines', done => {
+    const input = new TestReadable();
+    input.push(null);
+    const bufferedLine = new BufferLines({ input });
+    let linesToTest: string[] = [];
+    bufferedLine.on('lines', (lines: string[]) => {
+      linesToTest = [...linesToTest, ...lines];
+    });
+    bufferedLine.on('close', () => {
+      expect(linesToTest).toEqual([]);
+      done();
+    });
+  });
+
+  test('it can read 200 lines', done => {
+    const input = new TestReadable();
+    const bufferedLine = new BufferLines({ input });
+    let linesToTest: string[] = [];
+    const size200: string[] = new Array(200).fill(null).map((_, index) => `${index}\n`);
+    size200.forEach(element => input.push(element));
+    input.push(null);
+    bufferedLine.on('lines', (lines: string[]) => {
+      linesToTest = [...linesToTest, ...lines];
+    });
+    bufferedLine.on('close', () => {
+      expect(linesToTest.length).toEqual(200);
+      done();
+    });
+  });
+});

x-pack/plugins/lists/server/services/items/buffer_lines.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import readLine from 'readline';
+import { Readable } from 'stream';
+
+const BUFFER_SIZE = 100;
+
+export class BufferLines extends Readable {
+  private set = new Set<string>();
+  constructor({ input }: { input: NodeJS.ReadableStream }) {
+    super({ encoding: 'utf-8' });
+    const readline = readLine.createInterface({
+      input,
+    });
+
+    readline.on('line', line => {
+      this.push(line);
+    });
+
+    readline.on('close', () => {
+      this.push(null);
+    });
+  }
+
+  public _read(): void {
+    // No operation but this is required to be implemented
+  }
+
+  public push(line: string | null): boolean {
+    if (line == null) {
+      this.emit('lines', Array.from(this.set));
+      this.set.clear();
+      this.emit('close');
+      return true;
+    } else {
+      this.set.add(line);
+      if (this.set.size > BUFFER_SIZE) {
+        this.emit('lines', Array.from(this.set));
+        this.set.clear();
+        return true;
+      } else {
+        return true;
+      }
+    }
+  }
+}