[index patterns] Fleep app - Keep saved object field list until field caps provides fields (#85370)

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.allownoindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPattern](./kibana-plugin-plugins-data-public.indexpattern.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-public.indexpattern.allownoindex.md)
+
+## IndexPattern.allowNoIndex property
+
+prevents errors when index pattern exists before indices
+
+<b>Signature:</b>
+
+```typescript
+readonly allowNoIndex: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.getassavedobjectbody.md

@@ -19,6 +19,7 @@ getAsSavedObjectBody(): {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     };
 ```
 <b>Returns:</b>
@@ -33,5 +34,6 @@ getAsSavedObjectBody(): {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     }`
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.md

@@ -20,6 +20,7 @@ export declare class IndexPattern implements IIndexPattern
 
 |  Property | Modifiers | Type | Description |
 |  --- | --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-public.indexpattern.allownoindex.md) |  | <code>boolean</code> | prevents errors when index pattern exists before indices |
 |  [deleteFieldFormat](./kibana-plugin-plugins-data-public.indexpattern.deletefieldformat.md) |  | <code>(fieldName: string) =&gt; void</code> |  |
 |  [fieldAttrs](./kibana-plugin-plugins-data-public.indexpattern.fieldattrs.md) |  | <code>FieldAttrs</code> |  |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-public.indexpattern.fieldformatmap.md) |  | <code>Record&lt;string, any&gt;</code> |  |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternAttributes](./kibana-plugin-plugins-data-public.indexpatternattributes.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md)
+
+## IndexPatternAttributes.allowNoIndex property
+
+prevents errors when index pattern exists before indices
+
+<b>Signature:</b>
+
+```typescript
+allowNoIndex?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.md

@@ -14,6 +14,7 @@ export interface IndexPatternAttributes
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md) | <code>boolean</code> | prevents errors when index pattern exists before indices |
 |  [fieldAttrs](./kibana-plugin-plugins-data-public.indexpatternattributes.fieldattrs.md) | <code>string</code> |  |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-public.indexpatternattributes.fieldformatmap.md) | <code>string</code> |  |
 |  [fields](./kibana-plugin-plugins-data-public.indexpatternattributes.fields.md) | <code>string</code> |  |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternSpec](./kibana-plugin-plugins-data-public.indexpatternspec.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md)
+
+## IndexPatternSpec.allowNoIndex property
+
+<b>Signature:</b>
+
+```typescript
+allowNoIndex?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.md

@@ -14,6 +14,7 @@ export interface IndexPatternSpec
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md) | <code>boolean</code> |  |
 |  [fieldAttrs](./kibana-plugin-plugins-data-public.indexpatternspec.fieldattrs.md) | <code>FieldAttrs</code> |  |
 |  [fieldFormats](./kibana-plugin-plugins-data-public.indexpatternspec.fieldformats.md) | <code>Record&lt;string, SerializedFieldFormat&gt;</code> |  |
 |  [fields](./kibana-plugin-plugins-data-public.indexpatternspec.fields.md) | <code>IndexPatternFieldMap</code> |  |

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.allownoindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [IndexPattern](./kibana-plugin-plugins-data-server.indexpattern.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-server.indexpattern.allownoindex.md)
+
+## IndexPattern.allowNoIndex property
+
+prevents errors when index pattern exists before indices
+
+<b>Signature:</b>
+
+```typescript
+readonly allowNoIndex: boolean;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.getassavedobjectbody.md

@@ -19,6 +19,7 @@ getAsSavedObjectBody(): {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     };
 ```
 <b>Returns:</b>
@@ -33,5 +34,6 @@ getAsSavedObjectBody(): {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     }`
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.md

@@ -20,6 +20,7 @@ export declare class IndexPattern implements IIndexPattern
 
 |  Property | Modifiers | Type | Description |
 |  --- | --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-server.indexpattern.allownoindex.md) |  | <code>boolean</code> | prevents errors when index pattern exists before indices |
 |  [deleteFieldFormat](./kibana-plugin-plugins-data-server.indexpattern.deletefieldformat.md) |  | <code>(fieldName: string) =&gt; void</code> |  |
 |  [fieldAttrs](./kibana-plugin-plugins-data-server.indexpattern.fieldattrs.md) |  | <code>FieldAttrs</code> |  |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-server.indexpattern.fieldformatmap.md) |  | <code>Record&lt;string, any&gt;</code> |  |

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [IndexPatternAttributes](./kibana-plugin-plugins-data-server.indexpatternattributes.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md)
+
+## IndexPatternAttributes.allowNoIndex property
+
+prevents errors when index pattern exists before indices
+
+<b>Signature:</b>
+
+```typescript
+allowNoIndex?: boolean;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.md

@@ -14,6 +14,7 @@ export interface IndexPatternAttributes
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md) | <code>boolean</code> | prevents errors when index pattern exists before indices |
 |  [fieldAttrs](./kibana-plugin-plugins-data-server.indexpatternattributes.fieldattrs.md) | <code>string</code> |  |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-server.indexpatternattributes.fieldformatmap.md) | <code>string</code> |  |
 |  [fields](./kibana-plugin-plugins-data-server.indexpatternattributes.fields.md) | <code>string</code> |  |

src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_pattern.test.ts.snap

@@ -2,6 +2,7 @@
 
 exports[`IndexPattern toSpec should match snapshot 1`] = `
 Object {
+  "allowNoIndex": false,
   "fieldAttrs": Object {},
   "fieldFormats": Object {},
   "fields": Object {

src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_patterns.test.ts.snap

@@ -2,6 +2,7 @@
 
 exports[`IndexPatterns savedObjectToSpec 1`] = `
 Object {
+  "allowNoIndex": undefined,
   "fieldAttrs": Object {},
   "fieldFormats": Object {
     "field": Object {},

src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts

@@ -74,6 +74,10 @@ export class IndexPattern implements IIndexPattern {
   private fieldFormats: FieldFormatsStartCommon;
   // make private once manual field refresh is removed
   public fieldAttrs: FieldAttrs;
+  /**
+   * prevents errors when index pattern exists before indices
+   */
+  public readonly allowNoIndex: boolean = false;
 
   constructor({
     spec = {},
@@ -110,6 +114,7 @@ export class IndexPattern implements IIndexPattern {
     this.typeMeta = spec.typeMeta;
     this.fieldAttrs = spec.fieldAttrs || {};
     this.intervalName = spec.intervalName;
+    this.allowNoIndex = spec.allowNoIndex || false;
   }
 
   /**
@@ -204,6 +209,7 @@ export class IndexPattern implements IIndexPattern {
       fieldFormats: this.fieldFormatMap,
       fieldAttrs: this.fieldAttrs,
       intervalName: this.intervalName,
+      allowNoIndex: this.allowNoIndex,
     };
   }
 
@@ -309,6 +315,7 @@ export class IndexPattern implements IIndexPattern {
       fieldFormatMap,
       type: this.type,
       typeMeta: this.typeMeta ? JSON.stringify(this.typeMeta) : undefined,
+      allowNoIndex: this.allowNoIndex ? this.allowNoIndex : undefined,
     };
   }
 

src/plugins/data/common/index_patterns/index_patterns/index_patterns.test.ts

@@ -114,6 +114,21 @@ describe('IndexPatterns', () => {
     SOClientGetDelay = 0;
   });
 
+  test('allowNoIndex flag preserves existing fields when index is missing', async () => {
+    const id = '2';
+    setDocsourcePayload(id, {
+      id: 'foo',
+      version: 'foo',
+      attributes: {
+        title: 'something',
+        allowNoIndex: true,
+        fields: '[{"name":"field"}]',
+      },
+    });
+
+    expect((await indexPatterns.get(id)).fields.length).toBe(1);
+  });
+
   test('savedObjectCache pre-fetches only title', async () => {
     expect(await indexPatterns.getIds()).toEqual(['id']);
     expect(savedObjectsClient.find).toHaveBeenCalledWith({

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -222,6 +222,7 @@ export class IndexPatternsService {
       metaFields,
       type: options.type,
       rollupIndex: options.rollupIndex,
+      allowNoIndex: options.allowNoIndex,
     });
   };
 
@@ -281,10 +282,21 @@ export class IndexPatternsService {
     options: GetFieldsOptions,
     fieldAttrs: FieldAttrs = {}
   ) => {
-    const scriptedFields = Object.values(fields).filter((field) => field.scripted);
+    const fieldsAsArr = Object.values(fields);
+    const scriptedFields = fieldsAsArr.filter((field) => field.scripted);
     try {
+      let updatedFieldList: FieldSpec[];
       const newFields = (await this.getFieldsForWildcard(options)) as FieldSpec[];
-      return this.fieldArrayToMap([...newFields, ...scriptedFields], fieldAttrs);
+
+      // If allowNoIndex, only update field list if field caps finds fields. To support
+      // beats creating index pattern and dashboard before docs
+      if (!options.allowNoIndex || (newFields && newFields.length > 5)) {
+        updatedFieldList = [...newFields, ...scriptedFields];
+      } else {
+        updatedFieldList = fieldsAsArr;
+      }
+
+      return this.fieldArrayToMap(updatedFieldList, fieldAttrs);
     } catch (err) {
       if (err instanceof IndexPatternMissingIndices) {
         this.onNotification({ title: (err as any).message, color: 'danger', iconType: 'alert' });
@@ -334,6 +346,7 @@ export class IndexPatternsService {
         typeMeta,
         type,
         fieldAttrs,
+        allowNoIndex,
       },
     } = savedObject;
 
@@ -355,6 +368,7 @@ export class IndexPatternsService {
       type,
       fieldFormats: parsedFieldFormatMap,
       fieldAttrs: parsedFieldAttrs,
+      allowNoIndex,
     };
   };
 
@@ -384,6 +398,7 @@ export class IndexPatternsService {
           metaFields: await this.config.get(UI_SETTINGS.META_FIELDS),
           type,
           rollupIndex: typeMeta?.params?.rollup_index,
+          allowNoIndex: spec.allowNoIndex,
         },
         spec.fieldAttrs
       );

src/plugins/data/common/index_patterns/types.ts

@@ -49,6 +49,10 @@ export interface IndexPatternAttributes {
   sourceFilters?: string;
   fieldFormatMap?: string;
   fieldAttrs?: string;
+  /**
+   * prevents errors when index pattern exists before indices
+   */
+  allowNoIndex?: boolean;
 }
 
 export interface FieldAttrs {
@@ -101,6 +105,7 @@ export interface GetFieldsOptions {
   lookBack?: boolean;
   metaFields?: string[];
   rollupIndex?: string;
+  allowNoIndex?: boolean;
 }
 
 export interface GetFieldsOptionsTimePattern {
@@ -193,6 +198,7 @@ export interface IndexPatternSpec {
   type?: string;
   fieldFormats?: Record<string, SerializedFieldFormat>;
   fieldAttrs?: FieldAttrs;
+  allowNoIndex?: boolean;
 }
 
 export interface SourceFilter {

src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts

@@ -64,12 +64,13 @@ export class IndexPatternsApiClient implements IIndexPatternsApiClient {
     }).then((resp: any) => resp.fields);
   }
 
-  getFieldsForWildcard({ pattern, metaFields, type, rollupIndex }: GetFieldsOptions) {
+  getFieldsForWildcard({ pattern, metaFields, type, rollupIndex, allowNoIndex }: GetFieldsOptions) {
     return this._request(this._getUrl(['_fields_for_wildcard']), {
       pattern,
       meta_fields: metaFields,
       type,
       rollup_index: rollupIndex,
-    }).then((resp: any) => resp.fields);
+      allow_no_index: allowNoIndex,
+    }).then((resp: any) => resp.fields || []);
   }
 }

src/plugins/data/public/public.api.md

@@ -1256,6 +1256,7 @@ export class IndexPattern implements IIndexPattern {
     // Warning: (ae-forgotten-export) The symbol "IndexPatternDeps" needs to be exported by the entry point index.d.ts
     constructor({ spec, fieldFormats, shortDotsEnable, metaFields, }: IndexPatternDeps);
     addScriptedField(name: string, script: string, fieldType?: string): Promise<void>;
+    readonly allowNoIndex: boolean;
     // (undocumented)
     readonly deleteFieldFormat: (fieldName: string) => void;
     // Warning: (ae-forgotten-export) The symbol "FieldAttrs" needs to be exported by the entry point index.d.ts
@@ -1296,6 +1297,7 @@ export class IndexPattern implements IIndexPattern {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     };
     // (undocumented)
     getComputedFields(): {
@@ -1388,6 +1390,7 @@ export type IndexPatternAggRestrictions = Record<string, {
 //
 // @public (undocumented)
 export interface IndexPatternAttributes {
+    allowNoIndex?: boolean;
     // (undocumented)
     fieldAttrs?: string;
     // (undocumented)
@@ -1518,6 +1521,8 @@ export type IndexPatternSelectProps = Required<Omit<EuiComboBoxProps<any>, 'isLo
 //
 // @public (undocumented)
 export interface IndexPatternSpec {
+    // (undocumented)
+    allowNoIndex?: boolean;
     // (undocumented)
     fieldAttrs?: FieldAttrs;
     // (undocumented)
@@ -2564,7 +2569,7 @@ export const UI_SETTINGS: {
 // src/plugins/data/common/es_query/filters/phrase_filter.ts:33:3 - (ae-forgotten-export) The symbol "PhraseFilterMeta" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/es_query/filters/phrases_filter.ts:31:3 - (ae-forgotten-export) The symbol "PhrasesFilterMeta" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:64:5 - (ae-forgotten-export) The symbol "FormatFieldFn" needs to be exported by the entry point index.d.ts
-// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:128:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
+// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:133:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/search/aggs/types.ts:150:51 - (ae-forgotten-export) The symbol "AggTypesRegistryStart" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/search/search_source/search_source.ts:197:7 - (ae-forgotten-export) The symbol "SearchFieldValue" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/field_formats/field_formats_service.ts:67:3 - (ae-forgotten-export) The symbol "FormatFactory" needs to be exported by the entry point index.d.ts

src/plugins/data/server/index_patterns/index_patterns_api_client.ts

@@ -30,8 +30,14 @@ export class IndexPatternsApiServer implements IIndexPatternsApiClient {
   constructor(elasticsearchClient: ElasticsearchClient) {
     this.esClient = elasticsearchClient;
   }
-  async getFieldsForWildcard({ pattern, metaFields, type, rollupIndex }: GetFieldsOptions) {
+  async getFieldsForWildcard({
-    const indexPatterns = new IndexPatternsFetcher(this.esClient);
+    pattern,
+    metaFields,
+    type,
+    rollupIndex,
+    allowNoIndex,
+  }: GetFieldsOptions) {
+    const indexPatterns = new IndexPatternsFetcher(this.esClient, allowNoIndex);
     return await indexPatterns.getFieldsForWildcard({
       pattern,
       metaFields,

src/plugins/data/server/index_patterns/routes.ts

@@ -75,13 +75,20 @@ export function registerRoutes(
           }),
           type: schema.maybe(schema.string()),
           rollup_index: schema.maybe(schema.string()),
+          allow_no_index: schema.maybe(schema.boolean()),
         }),
       },
     },
     async (context, request, response) => {
       const { asCurrentUser } = context.core.elasticsearch.client;
       const indexPatterns = new IndexPatternsFetcher(asCurrentUser);
-      const { pattern, meta_fields: metaFields, type, rollup_index: rollupIndex } = request.query;
+      const {
+        pattern,
+        meta_fields: metaFields,
+        type,
+        rollup_index: rollupIndex,
+        allow_no_index: allowNoIndex,
+      } = request.query;
 
       let parsedFields: string[] = [];
       try {
@@ -96,6 +103,9 @@ export function registerRoutes(
           metaFields: parsedFields,
           type,
           rollupIndex,
+          fieldCapsOptions: {
+            allow_no_indices: allowNoIndex || false,
+          },
         });
 
         return response.ok({

src/plugins/data/server/index_patterns/routes/create_index_pattern.ts

@@ -50,6 +50,7 @@ const indexPatternSpecSchema = schema.object({
       })
     )
   ),
+  allowNoIndex: schema.maybe(schema.boolean()),
 });
 
 export const registerCreateIndexPatternRoute = (

src/plugins/data/server/index_patterns/routes/update_index_pattern.ts

@@ -38,6 +38,7 @@ const indexPatternUpdateSchema = schema.object({
   ),
   fieldFormats: schema.maybe(schema.recordOf(schema.string(), serializedFieldFormatSchema)),
   fields: schema.maybe(schema.recordOf(schema.string(), fieldSpecSchema)),
+  allowNoIndex: schema.maybe(schema.boolean()),
 });
 
 export const registerUpdateIndexPatternRoute = (

src/plugins/data/server/saved_objects/index_pattern_migrations.test.ts

@@ -94,4 +94,55 @@ Object {
       expect(migrationFn(input, savedObjectMigrationContext)).toEqual(expected);
     });
   });
+
+  describe('7.11.0', () => {
+    const migrationFn = indexPatternSavedObjectTypeMigrations['7.11.0'];
+
+    test('should set allowNoIndex', () => {
+      const input = {
+        type: 'index-pattern',
+        id: 'logs-*',
+        attributes: {},
+      };
+      const expected = {
+        type: 'index-pattern',
+        id: 'logs-*',
+        attributes: {
+          allowNoIndex: true,
+        },
+      };
+
+      expect(migrationFn(input, savedObjectMigrationContext)).toEqual(expected);
+
+      const input2 = {
+        type: 'index-pattern',
+        id: 'metrics-*',
+        attributes: {},
+      };
+      const expected2 = {
+        type: 'index-pattern',
+        id: 'metrics-*',
+        attributes: {
+          allowNoIndex: true,
+        },
+      };
+
+      expect(migrationFn(input2, savedObjectMigrationContext)).toEqual(expected2);
+
+      const input3 = {
+        type: 'index-pattern',
+        id: 'xxx',
+        attributes: {},
+      };
+      const expected3 = {
+        type: 'index-pattern',
+        id: 'xxx',
+        attributes: {
+          allowNoIndex: undefined,
+        },
+      };
+
+      expect(migrationFn(input3, savedObjectMigrationContext)).toEqual(expected3);
+    });
+  });
 });

src/plugins/data/server/saved_objects/index_pattern_migrations.ts

@@ -54,7 +54,16 @@ const migrateSubTypeAndParentFieldProperties: SavedObjectMigrationFn<any, any> =
   };
 };
 
+const addAllowNoIndex: SavedObjectMigrationFn<any, any> = (doc) => ({
+  ...doc,
+  attributes: {
+    ...doc.attributes,
+    allowNoIndex: doc.id === 'logs-*' || doc.id === 'metrics-*' || undefined,
+  },
+});
+
 export const indexPatternSavedObjectTypeMigrations = {
   '6.5.0': flow(migrateAttributeTypeAndAttributeTypeMeta),
   '7.6.0': flow(migrateSubTypeAndParentFieldProperties),
+  '7.11.0': flow(addAllowNoIndex),
 };

src/plugins/data/server/server.api.md

@@ -689,6 +689,7 @@ export class IndexPattern implements IIndexPattern {
     // Warning: (ae-forgotten-export) The symbol "IndexPatternDeps" needs to be exported by the entry point index.d.ts
     constructor({ spec, fieldFormats, shortDotsEnable, metaFields, }: IndexPatternDeps);
     addScriptedField(name: string, script: string, fieldType?: string): Promise<void>;
+    readonly allowNoIndex: boolean;
     // (undocumented)
     readonly deleteFieldFormat: (fieldName: string) => void;
     // Warning: (ae-forgotten-export) The symbol "FieldAttrs" needs to be exported by the entry point index.d.ts
@@ -731,6 +732,7 @@ export class IndexPattern implements IIndexPattern {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     };
     // (undocumented)
     getComputedFields(): {
@@ -819,6 +821,7 @@ export class IndexPattern implements IIndexPattern {
 //
 // @public (undocumented)
 export interface IndexPatternAttributes {
+    allowNoIndex?: boolean;
     // (undocumented)
     fieldAttrs?: string;
     // (undocumented)
@@ -1388,7 +1391,7 @@ export function usageProvider(core: CoreSetup_2): SearchUsage;
 // src/plugins/data/common/es_query/filters/meta_filter.ts:54:3 - (ae-forgotten-export) The symbol "FilterMeta" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:58:45 - (ae-forgotten-export) The symbol "IndexPatternFieldMap" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:64:5 - (ae-forgotten-export) The symbol "FormatFieldFn" needs to be exported by the entry point index.d.ts
-// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:128:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
+// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:133:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/index.ts:40:23 - (ae-forgotten-export) The symbol "buildCustomFilter" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/index.ts:40:23 - (ae-forgotten-export) The symbol "buildFilter" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/index.ts:57:23 - (ae-forgotten-export) The symbol "datatableToCSV" needs to be exported by the entry point index.d.ts

x-pack/plugins/fleet/common/constants/epm.ts

@@ -6,7 +6,6 @@
 
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
-export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;
 
 export const requiredPackages = {

x-pack/plugins/fleet/server/constants/index.ts

@@ -14,7 +14,6 @@ export {
   AGENT_POLICY_ROLLOUT_RATE_LIMIT_REQUEST_PER_INTERVAL,
   AGENT_POLICY_ROLLOUT_RATE_LIMIT_INTERVAL_MS,
   AGENT_UPDATE_ACTIONS_INTERVAL_MS,
-  INDEX_PATTERN_PLACEHOLDER_SUFFIX,
   MAX_TIME_COMPLETE_INSTALL,
   // Routes
   LIMITED_CONCURRENCY_ROUTE_TAG,

x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts

@@ -5,15 +5,11 @@
  */
 
 import { SavedObjectsClientContract } from 'src/core/server';
-import {
+import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../../constants';
-  INDEX_PATTERN_SAVED_OBJECT_TYPE,
-  INDEX_PATTERN_PLACEHOLDER_SUFFIX,
-} from '../../../../constants';
 import { loadFieldsFromYaml, Fields, Field } from '../../fields/field';
 import { dataTypes, installationStatuses } from '../../../../../common/constants';
 import { ArchivePackage, InstallSource, ValueOf } from '../../../../../common/types';
-import { RegistryPackage, CallESAsCurrentUser, DataType } from '../../../../types';
+import { RegistryPackage, DataType } from '../../../../types';
-import { appContextService } from '../../../../services';
 import { getPackageFromSource, getPackageSavedObjects } from '../../packages/get';
 
 interface FieldFormatMap {
@@ -172,6 +168,7 @@ export const createIndexPattern = (indexPatternType: string, fields: Fields) =>
     timeFieldName: '@timestamp',
     fields: JSON.stringify(indexPatternFields),
     fieldFormatMap: JSON.stringify(fieldFormatMap),
+    allowNoIndex: true,
   };
 };
 
@@ -382,31 +379,3 @@ const getFieldFormatParams = (field: Field): FieldFormatParams => {
   if (field.open_link_in_current_tab) params.openLinkInCurrentTab = field.open_link_in_current_tab;
   return params;
 };
-
-export const ensureDefaultIndices = async (callCluster: CallESAsCurrentUser) => {
-  // create placeholder indices to supress errors in the kibana Dashboards app
-  // that no matching indices exist https://github.com/elastic/kibana/issues/62343
-  const logger = appContextService.getLogger();
-  return Promise.all(
-    Object.values(dataTypes).map(async (indexPattern) => {
-      const defaultIndexPatternName = indexPattern + INDEX_PATTERN_PLACEHOLDER_SUFFIX;
-      const indexExists = await callCluster('indices.exists', { index: defaultIndexPatternName });
-      if (!indexExists) {
-        try {
-          await callCluster('indices.create', {
-            index: defaultIndexPatternName,
-            body: {
-              mappings: {
-                properties: {
-                  '@timestamp': { type: 'date' },
-                },
-              },
-            },
-          });
-        } catch (putErr) {
-          logger.error(`${defaultIndexPatternName} could not be created`);
-        }
-      }
-    })
-  );
-};

x-pack/plugins/fleet/server/services/setup.ts

@@ -13,7 +13,6 @@ import {
   ensureInstalledDefaultPackages,
   ensurePackagesCompletedInstall,
 } from './epm/packages/install';
-import { ensureDefaultIndices } from './epm/kibana/index_pattern/install';
 import {
   packageToPackagePolicy,
   PackagePolicy,
@@ -58,7 +57,6 @@ async function createSetupSideEffects(
     ensureInstalledDefaultPackages(soClient, callCluster),
     outputService.ensureDefaultOutput(soClient),
     agentPolicyService.ensureDefaultAgentPolicy(soClient),
-    ensureDefaultIndices(callCluster),
     settingsService.getSettings(soClient).catch((e: any) => {
       if (e.isBoom && e.output.statusCode === 404) {
         const defaultSettings = createDefaultSettings();

x-pack/test/fleet_api_integration/apis/index.js

@@ -7,8 +7,6 @@
 export default function ({ loadTestFile }) {
   describe('Fleet Endpoints', function () {
     this.tags('ciGroup10');
-    // Fleet setup
-    loadTestFile(require.resolve('./setup'));
     // Agent setup
     loadTestFile(require.resolve('./agents_setup'));
     // Agents

x-pack/test/fleet_api_integration/apis/setup.ts

@@ -1,32 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import expect from '@kbn/expect';
-import { FtrProviderContext } from '../../api_integration/ftr_provider_context';
-
-export default function (providerContext: FtrProviderContext) {
-  const { getService } = providerContext;
-  const supertest = getService('supertest');
-  const es = getService('es');
-  describe('Fleet setup', async () => {
-    before(async () => {
-      await supertest.post(`/api/fleet/setup`).set('kbn-xsrf', 'xxx').send();
-    });
-
-    it('should have installed placeholder indices', async function () {
-      const resLogsIndexPatternPlaceholder = await es.transport.request({
-        method: 'GET',
-        path: `/logs-index_pattern_placeholder`,
-      });
-      expect(resLogsIndexPatternPlaceholder.statusCode).equal(200);
-      const resMetricsIndexPatternPlaceholder = await es.transport.request({
-        method: 'GET',
-        path: `/metrics-index_pattern_placeholder`,
-      });
-      expect(resMetricsIndexPatternPlaceholder.statusCode).equal(200);
-    });
-  });
-}

x-pack/test/security_solution_endpoint/apps/endpoint/resolver.ts

@@ -13,19 +13,32 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const browser = getService('browser');
 
-  describe('Endpoint Event Resolver', function () {
+  /**
+   * Navigating to the hosts page must be done after data is loaded into ES otherwise
+   * the hosts page will display the empty default page and if we load data after that
+   * we'd have to set the source filter on the page.
+   */
+  const navigateToHostsAndSetDate = async () => {
+    await pageObjects.hosts.navigateToSecurityHostsPage();
+    await pageObjects.common.dismissBanner();
+    const fromTime = 'Jan 1, 2018 @ 00:00:00.000';
+    const toTime = 'now';
+    await pageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
+  };
+
+  describe.skip('Endpoint Event Resolver', function () {
     before(async () => {
-      await pageObjects.hosts.navigateToSecurityHostsPage();
-      await pageObjects.common.dismissBanner();
-      const fromTime = 'Jan 1, 2018 @ 00:00:00.000';
-      const toTime = 'now';
-      await pageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
       await browser.setWindowSize(1800, 1200);
     });
-    describe.skip('Endpoint Resolver Tree', function () {
+    after(async () => {
+      await pageObjects.hosts.deleteDataStreams();
+    });
+
+    describe('Endpoint Resolver Tree', function () {
       before(async () => {
         await esArchiver.load('empty_kibana');
         await esArchiver.load('endpoint/resolver_tree/functions', { useCreate: true });
+        await navigateToHostsAndSetDate();
         await pageObjects.hosts.executeQueryAndOpenResolver('event.dataset : endpoint.events.file');
       });
       after(async () => {
@@ -213,6 +226,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       before(async () => {
         await esArchiver.load('empty_kibana');
         await esArchiver.load('endpoint/resolver_tree/alert_events', { useCreate: true });
+        await navigateToHostsAndSetDate();
       });
       after(async () => {
         await pageObjects.hosts.deleteDataStreams();

x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts

@@ -277,7 +277,7 @@ export default function ({ getService }: FtrProviderContext) {
         .set('kbn-xsrf', 'xxx')
         .send({
           filter: entityIDFilter,
-          indexPatterns: ['metrics-*'],
+          indexPatterns: ['doesnotexist-*'],
           timeRange: {
             from: tree.startTime,
             to: tree.endTime,

x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts

@@ -281,7 +281,7 @@ export default function ({ getService }: FtrProviderContext) {
               from: tree.startTime.toISOString(),
               to: tree.endTime.toISOString(),
             },
-            indexPatterns: ['metrics-*'],
+            indexPatterns: ['doesnotexist-*'],
           })
           .expect(200);
         expect(body).to.be.empty();