github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

Remove field_stats pre-flight option for index patterns (#12814)

Browse source 
* Remove field_stats pre-flight option for index patterns

This feature was originally added to make searches against Elasticsearch
more performant when searching across many indices, such as when dealing
with daily indices that have accumulated for years. Elasticsearch now
automatically optimizes index access once a certain amount of shards are
in play, so Kibana doesn't need to hack its own solution.

This option can be removed entirely as existing index patterns that had
this option configured will degrade gracefully.

* test: fix parse error test to account for multiple shards
This commit is contained in:
Court Ewing 2017-07-12 20:14:26 -04:00 • committed by GitHub
parent 10d243de15
commit 60c636ba4d
10 changed files with 1 additions and 459 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/core_plugins/kibana/public/management/sections/indices/create_index_pattern/__tests__/create_index_pattern.js
View file

@ -108,13 +108,6 @@ describe('createIndexPattern UI', () => {
}
});
});
it('displays the option (off) to expand wildcards', () => {
const { $view } = setup();
const $enableExpand = $view.findTestSubject('createIndexPatternEnableExpand');
expect($enableExpand).to.have.length(1);
expect($enableExpand.is(':checked')).to.be(false);
});
});
describe('cross cluster pattern', () => {
@ -127,13 +120,5 @@ describe('createIndexPattern UI', () => {
expect(classes).to.contain('ng-valid');
expect(classes).to.not.contain('ng-invalid');
});
it('removes the option to expand wildcards', () => {
const { $view, setNameTo } = setup();
setNameTo('cluster2:logstash-*');
const $enableExpand = $view.findTestSubject('createIndexPatternEnableExpand');
expect($enableExpand).to.have.length(0);
});
});
});

37
src/core_plugins/kibana/public/management/sections/indices/create_index_pattern/create_index_pattern.html
View file

@ -131,43 +131,6 @@
</div>
</div>
<!-- Expand index pattern checkbox -->
<div class="kuiVerticalRhythm" ng-if="controller.canEnableExpandWildcard()">
<label class="kuiCheckBoxLabel kuiVerticalRhythm">
<input
class="kuiCheckBox"
type="checkbox"
data-test-subj="createIndexPatternEnableExpand"
ng-model="controller.formValues.expandWildcard"
>
<span class="kuiCheckBoxLabel__text">
<span translate="KIBANA-EXPAND_INDEX_PATTERN"></span>
<span translate="KIBANA-DEPRECATED"></span>
</span>
</label>
<!-- Checkbox help text -->
<div class="kuiVerticalRhythm">
<p
class="kuiSubText kuiVerticalRhythmSmall"
translate="KIBANA-WILDCARD_DEFAULT_EXPANDED_TO_CURRENT_TIME_RANGE"
></p>
<p class="kuiSubText kuiVerticalRhythmSmall">
<span translate="KIBANA-SEARCH_AGAINST_INDEX_PATTERN"></span>
<em translate="KIBANA-LOGSTASH_WILDCARD"></em>
<span translate="KIBANA-ACTUALLY_QUERY"></span>
<em translate="KIBANA-EXAMPLE_TIME_RANGE"></em>
<span translate="KIBANA-FALL_WITHIN_CURRENT_TIME_RANGE"></span>
</p>
<p
class="kuiSubText kuiVerticalRhythmSmall"
translate="KIBANA-EXPAND_INDEX_PATTERN_DEPRECATION"
></p>
</div>
</div>
<!-- Form actions -->
<button
data-test-subj="createIndexPatternCreateButton"

22
src/core_plugins/kibana/public/management/sections/indices/create_index_pattern/create_index_pattern.js
View file

@ -1,4 +1,3 @@
import _ from 'lodash';
import { IndexPatternMissingIndices } from 'ui/errors';
import 'ui/directives/validate_index_name';
import 'ui/directives/auto_select_if_only_one';
@ -33,7 +32,6 @@ uiModules.get('apps/management')
this.formValues = {
id: $routeParams.id ? decodeURIComponent($routeParams.id) : undefined,
name: config.get('indexPattern:placeholder'),
expandWildcard: false,
timeFieldOption: null,
};
@ -134,21 +132,6 @@ uiModules.get('apps/management')
return Boolean(this.formValues.timeFieldOption.fieldName);
};
this.canEnableExpandWildcard = () => {
return (
this.isTimeBased() &&
!this.isCrossClusterName() &&
_.includes(this.formValues.name, '*')
);
};
this.isExpandWildcardEnabled = () => {
return (
this.canEnableExpandWildcard() &&
!!this.formValues.expandWildcard
);
};
this.isCrossClusterName = () => {
return (
this.formValues.name &&
@ -217,16 +200,11 @@ uiModules.get('apps/management')
? timeFieldOption.fieldName
: undefined;
const notExpandable = this.isExpandWildcardEnabled()
? undefined
: true;
loadingCount += 1;
sendCreateIndexPatternRequest(indexPatterns, {
id,
name,
timeFieldName,
notExpandable,
}).then(createdId => {
if (!createdId) {
return;

2
src/core_plugins/kibana/public/management/sections/indices/create_index_pattern/send_create_index_pattern_request.js
View file

@ -2,7 +2,6 @@ export function sendCreateIndexPatternRequest(indexPatterns, {
id,
name,
timeFieldName,
notExpandable,
}) {
// get an empty indexPattern to start
return indexPatterns.get()
@ -11,7 +10,6 @@ export function sendCreateIndexPatternRequest(indexPatterns, {
id,
title: name,
timeFieldName,
notExpandable,
});
// fetch the fields

9
src/core_plugins/kibana/translations/en.json
View file

@ -4,7 +4,6 @@
"KIBANA-CONFIGURE_INDEX_PATTERN": "Configure an index pattern",
"KIBANA-MUST_CONFIGURE_INDEX_PATTERN": "In order to use Kibana you must configure at least one index pattern. Index patterns are used to identify the Elasticsearch index to run search and analytics against. They are also used to configure fields.",
"KIBANA-INDEX_NAME_CREATED_BY_EVENT_TIMES": "Use event times to create index names ",
"KIBANA-DEPRECATED": "[DEPRECATED]",
"KIBANA-ALERT_INDEX_PATTERN_DEPRECATED": "Time-interval based index patterns are deprecated!",
"KIBANA-WE": " We",
"KIBANA-STRONGLY_RECOMMEND": " strongly recommend",
@ -18,14 +17,6 @@
"KIBANA-STATIC_TEXT_IN_DYNAMIC_INDEX_PATTERNS": "Patterns allow you to define dynamic index names. Static text in an index name is denoted using brackets. Example: [logstash-]YYYY.MM.DD. Please note that weeks are setup to use ISO weeks which start on Monday.",
"KIBANA-NOTE_COLON": "Note:",
"KIBANA-WEEKLY_ISO_NOTICE": "I noticed you are using weekly indices. Kibana requires ISO weeks be used in your index creation.",
"KIBANA-EXPAND_INDEX_PATTERN": "Expand index pattern when searching",
"KIBANA-WILDCARD_DEFAULT_EXPANDED_TO_CURRENT_TIME_RANGE": "With this option selected, searches against any time-based index pattern that contains a wildcard will automatically be expanded to query only the indices that contain data within the currently selected time range.",
"KIBANA-SEARCH_AGAINST_INDEX_PATTERN": "Searching against the index pattern ",
"KIBANA-LOGSTASH_WILDCARD": "logstash-*",
"KIBANA-ACTUALLY_QUERY": " will actually query Elasticsearch for the specific matching indices (e.g. ",
"KIBANA-EXAMPLE_TIME_RANGE": "logstash-2015.12.21",
"KIBANA-FALL_WITHIN_CURRENT_TIME_RANGE": ") that fall within the current time range.",
"KIBANA-EXPAND_INDEX_PATTERN_DEPRECATION": "With recent changes to Elasticsearch, this option should no longer be necessary and will likely be removed in future versions of Kibana.",
"KIBANA-SAMPLE_ALERT": "Attempted to match the following indices and aliases:",
"KIBANA-EXPAND_SEARCH": "Expand Search",
"KIBANA-EXISTING_MATCH_PERCENT": "Pattern matches {{indexExistingMatchPercent}} of existing indices and aliases",

78
src/ui/public/index_patterns/__tests__/_index_pattern.js
View file

@ -16,7 +16,6 @@ import { Notifier } from 'ui/notify';
import { FieldsFetcherProvider } from '../fields_fetcher_provider';
import { StubIndexPatternsApiClientModule } from './stub_index_patterns_api_client';
import { IndexPatternsApiClientProvider } from '../index_patterns_api_client_provider';
import { IndexPatternsCalculateIndicesProvider } from '../_calculate_indices';
import { IsUserAwareOfUnsupportedTimePatternProvider } from '../unsupported_time_patterns';
import { SavedObjectsClientProvider } from 'ui/saved_objects';
@ -33,25 +32,12 @@ describe('index pattern', function () {
let savedObjectsResponse;
const indexPatternId = 'test-pattern';
let indexPattern;
let calculateIndices;
let intervals;
let indexPatternsApiClient;
let defaultTimeField;
let isUserAwareOfUnsupportedTimePattern;
beforeEach(ngMock.module('kibana', StubIndexPatternsApiClientModule, (PrivateProvider) => {
PrivateProvider.swap(IndexPatternsCalculateIndicesProvider, () => {
// stub calculateIndices
calculateIndices = sinon.spy(function () {
return Promise.resolve([
{ index: 'foo', max: Infinity, min: -Infinity },
{ index: 'bar', max: Infinity, min: -Infinity }
]);
});
return calculateIndices;
});
isUserAwareOfUnsupportedTimePattern = sinon.stub().returns(false);
PrivateProvider.swap(IsUserAwareOfUnsupportedTimePatternProvider, () => {
return isUserAwareOfUnsupportedTimePattern;
@ -317,33 +303,6 @@ describe('index pattern', function () {
});
describe('when index pattern is a time-base wildcard', function () {
beforeEach(function () {
indexPattern.id = 'randomID';
indexPattern.title = 'logstash-*';
indexPattern.timeFieldName = defaultTimeField.name;
indexPattern.intervalName = null;
indexPattern.notExpandable = false;
});
it('invokes calculateIndices with given start/stop times and sortOrder', async function () {
await indexPattern.toDetailedIndexList(1, 2, 'sortOrder');
const { title, timeFieldName } = indexPattern;
sinon.assert.calledOnce(calculateIndices);
expect(calculateIndices.getCall(0).args).to.eql([
title, timeFieldName, 1, 2, 'sortOrder'
]);
});
it('is fulfilled by the result of calculateIndices', async function () {
const indexList = await indexPattern.toDetailedIndexList();
expect(indexList[0].index).to.equal('foo');
expect(indexList[1].index).to.equal('bar');
});
});
describe('when index pattern is a time-base wildcard that is configured not to expand', function () {
beforeEach(function () {
indexPattern.id = 'randomID';
indexPattern.title = 'logstash-*';
@ -411,28 +370,6 @@ describe('index pattern', function () {
});
describe('when index pattern is a time-base wildcard', function () {
beforeEach(function () {
indexPattern.id = 'randomID';
indexPattern.title = 'logstash-*';
indexPattern.timeFieldName = defaultTimeField.name;
indexPattern.intervalName = null;
indexPattern.notExpandable = false;
});
it('invokes calculateIndices with given start/stop times and sortOrder', async function () {
await indexPattern.toIndexList(1, 2, 'sortOrder');
const { title, timeFieldName } = indexPattern;
expect(calculateIndices.calledWith(title, timeFieldName, 1, 2, 'sortOrder')).to.be(true);
});
it('is fulfilled by the result of calculateIndices', async function () {
const indexList = await indexPattern.toIndexList();
expect(indexList[0]).to.equal('foo');
expect(indexList[1]).to.equal('bar');
});
});
describe('when index pattern is a time-base wildcard that is configured not to expand', function () {
beforeEach(function () {
indexPattern.id = 'randomID';
indexPattern.title = 'logstash-*';
@ -463,21 +400,6 @@ describe('index pattern', function () {
});
});
describe('#isIndexExpansionEnabled()', function () {
it('returns true if notExpandable is false', function () {
indexPattern.notExpandable = false;
expect(indexPattern.isIndexExpansionEnabled()).to.be(true);
});
it('returns true if notExpandable is not defined', function () {
delete indexPattern.notExpandable;
expect(indexPattern.isIndexExpansionEnabled()).to.be(true);
});
it('returns false if notExpandable is true', function () {
indexPattern.notExpandable = true;
expect(indexPattern.isIndexExpansionEnabled()).to.be(false);
});
});
describe('#isTimeBased()', function () {
beforeEach(function () {
// for the sake of these tests, it doesn't much matter what type of field

201
src/ui/public/index_patterns/__tests__/calculate_indices.js
View file

@ -1,201 +0,0 @@
import { pluck, first, size, includes } from 'lodash';
import sinon from 'sinon';
import expect from 'expect.js';
import ngMock from 'ng_mock';
import moment from 'moment';
import { IndexPatternsCalculateIndicesProvider } from 'ui/index_patterns/_calculate_indices';
describe('IndexPatternsCalculateIndicesProvider', () => {
let $rootScope;
let calculateIndices;
let es;
let response;
let config;
let constraints;
let indices;
beforeEach(ngMock.module('kibana', ($provide) => {
response = {
indices: {
'mock-*': { fields: { '@something': {} } },
'ignore-*': { fields: {} }
}
};
$provide.service('es', function (Promise) {
return {
fieldStats: sinon.spy(function () {
return Promise.resolve(response);
})
};
});
}));
beforeEach(ngMock.inject((Private, $injector) => {
$rootScope = $injector.get('$rootScope');
es = $injector.get('es');
calculateIndices = Private(IndexPatternsCalculateIndicesProvider);
}));
function run({ start = undefined, stop = undefined } = {}) {
calculateIndices('wat-*-no', '@something', start, stop).then(value => {
indices = pluck(value, 'index');
});
$rootScope.$apply();
config = first(es.fieldStats.lastCall.args);
constraints = config.body.index_constraints;
}
describe('transport configuration', () => {
it('uses pattern path for indec', () => {
run();
expect(config.index).to.equal('wat-*-no');
});
it('has level indices', () => {
run();
expect(config.level).to.equal('indices');
});
it('includes time field', () => {
run();
expect(includes(config.body.fields, '@something')).to.be(true);
});
it('no constraints by default', () => {
run();
expect(size(constraints['@something'])).to.equal(0);
});
describe('when given start', () => {
beforeEach(() => run({ start: '1234567890' }));
it('includes max_value', () => {
expect(constraints['@something']).to.have.property('max_value');
});
it('max_value is gte', () => {
expect(constraints['@something'].max_value).to.have.property('gte');
});
it('max_value is set to original if not a moment object', () => {
expect(constraints['@something'].max_value.gte).to.equal('1234567890');
});
it('max_value format is set to epoch_millis', () => {
expect(constraints['@something'].max_value.format).to.equal('epoch_millis');
});
it('max_value is set to moment.valueOf if given a moment object', () => {
const start = moment();
run({ start });
expect(constraints['@something'].max_value.gte).to.equal(start.valueOf());
});
});
describe('when given stop', () => {
beforeEach(() => run({ stop: '1234567890' }));
it('includes min_value', () => {
expect(constraints['@something']).to.have.property('min_value');
});
it('min_value is lte', () => {
expect(constraints['@something'].min_value).to.have.property('lte');
});
it('min_value is set to original if not a moment object', () => {
expect(constraints['@something'].min_value.lte).to.equal('1234567890');
});
it('min_value format is set to epoch_millis', () => {
expect(constraints['@something'].min_value.format).to.equal('epoch_millis');
});
it('max_value is set to moment.valueOf if given a moment object', () => {
const stop = moment();
run({ stop });
expect(constraints['@something'].min_value.lte).to.equal(stop.valueOf());
});
});
});
describe('response filtering', () => {
it('filters out any indices that have empty fields', () => {
run();
expect(includes(indices, 'mock-*')).to.be(true);
expect(includes(indices, 'ignore-*')).to.be(false);
});
});
describe('response sorting', function () {
require('test_utils/no_digest_promises').activateForSuite();
describe('when no sorting direction given', function () {
it('returns the indices in the order that elasticsearch sends them', function () {
response = {
indices: {
c: { fields: { time: {} } },
a: { fields: { time: {} } },
b: { fields: { time: {} } },
}
};
return calculateIndices('*', 'time', null, null).then(function (resp) {
expect(pluck(resp, 'index')).to.eql(['c', 'a', 'b']);
});
});
});
describe('when sorting asc', function () {
it('resolves to an array of objects, each with index, start, and end properties', function () {
response = {
indices: {
c: { fields: { time: { max_value: 10, min_value: 9 } } },
a: { fields: { time: { max_value: 15, min_value: 5 } } },
b: { fields: { time: { max_value: 1, min_value: 0 } } },
}
};
return calculateIndices('*', 'time', null, null, 'asc').then(function (resp) {
expect(resp).to.eql([
{
index: 'b',
min: 0,
max: 1
},
{
index: 'a',
min: 5,
max: 15
},
{
index: 'c',
min: 9,
max: 10
}
]);
});
});
});
describe('when sorting desc', function () {
it('resolves to an array of objects, each with index, min, and max properties', function () {
response = {
indices: {
c: { fields: { time: { max_value: 10, min_value: 9 } } },
a: { fields: { time: { max_value: 15, min_value: 5 } } },
b: { fields: { time: { max_value: 1, min_value: 0 } } },
}
};
return calculateIndices('*', 'time', null, null, 'desc').then(function (resp) {
expect(resp).to.eql([
{
index: 'a',
max: 15,
min: 5
},
{
index: 'c',
max: 10,
min: 9
},
{
index: 'b',
max: 1,
min: 0
},
]);
});
});
});
});
});

81
src/ui/public/index_patterns/_calculate_indices.js
View file

@ -1,81 +0,0 @@
import _ from 'lodash';
import moment from 'moment';
// gets parsed value if given arg is a moment object
function timeValue(val) {
return moment.isMoment(val) ? val.valueOf() : val;
}
// returns a properly formatted millisecond timestamp index constraint
function msConstraint(comparison, value) {
return {
[comparison]: timeValue(value),
format: 'epoch_millis'
};
}
// returns a new object with any indexes removed that do not include the
// time field
//
// fixme: this really seems like a bug that needs to be fixed in
// elasticsearch itself, but this workaround will do for now
function omitIndicesWithoutTimeField(indices, timeFieldName) {
return _.pick(indices, index => index.fields[timeFieldName]);
}
export function IndexPatternsCalculateIndicesProvider(es) {
// Uses the field stats api to determine the names of indices that need to
// be queried against that match the given pattern and fall within the
// given time range
function calculateIndices(pattern, timeFieldName, start, stop, sortDirection) {
return getFieldStats(pattern, timeFieldName, start, stop)
.then(resp => omitIndicesWithoutTimeField(resp.indices, timeFieldName))
.then(indices => sortIndexStats(indices, timeFieldName, sortDirection));
}
// creates the configuration hash that must be passed to the elasticsearch
// client
function getFieldStats(pattern, timeFieldName, start, stop) {
const constraints = {};
if (start) {
constraints.max_value = msConstraint('gte', start);
}
if (stop) {
constraints.min_value = msConstraint('lte', stop);
}
return es.fieldStats({
index: pattern,
level: 'indices',
body: {
fields: [ timeFieldName ],
index_constraints: {
[timeFieldName]: constraints
}
}
});
}
function sortIndexStats(indices, timeFieldName, sortDirection) {
const desc = sortDirection === 'desc';
const leader = desc ? 'max' : 'min';
let indexDetails = _(indices).map((stats, index) => {
const field = stats.fields[timeFieldName];
return {
index,
min: field.min_value,
max: field.max_value
};
});
if (sortDirection) {
indexDetails = indexDetails.sortByOrder([leader], [sortDirection]);
}
return indexDetails.value();
}
return calculateIndices;
}

13
src/ui/public/index_patterns/_index_pattern.js
View file

@ -11,7 +11,6 @@ import { IndexPatternsGetIdsProvider } from './_get_ids';
import { IndexPatternsIntervalsProvider } from './_intervals';
import { IndexPatternsFieldListProvider } from './_field_list';
import { IndexPatternsFlattenHitProvider } from './_flatten_hit';
import { IndexPatternsCalculateIndicesProvider } from './_calculate_indices';
import { IndexPatternsPatternCacheProvider } from './_pattern_cache';
import { FieldsFetcherProvider } from './fields_fetcher_provider';
import { IsUserAwareOfUnsupportedTimePatternProvider } from './unsupported_time_patterns';
@ -36,7 +35,6 @@ export function IndexPatternProvider(Private, $http, config, kbnIndex, Promise,
const mappingSetup = Private(UtilsMappingSetupProvider);
const FieldList = Private(IndexPatternsFieldListProvider);
const flattenHit = Private(IndexPatternsFlattenHitProvider);
const calculateIndices = Private(IndexPatternsCalculateIndicesProvider);
const patternCache = Private(IndexPatternsPatternCacheProvider);
const isUserAwareOfUnsupportedTimePattern = Private(IsUserAwareOfUnsupportedTimePatternProvider);
const savedObjectsClient = Private(SavedObjectsClientProvider);
@ -48,7 +46,6 @@ export function IndexPatternProvider(Private, $http, config, kbnIndex, Promise,
const mapping = mappingSetup.expandShorthand({
title: 'text',
timeFieldName: 'keyword',
notExpandable: 'boolean',
intervalName: 'keyword',
fields: 'json',
sourceFilters: 'json',
@ -310,12 +307,6 @@ export function IndexPatternProvider(Private, $http, config, kbnIndex, Promise,
);
}
if (this.isTimeBasedWildcard() && this.isIndexExpansionEnabled()) {
return calculateIndices(
this.title, this.timeFieldName, start, stop, sortDirection
);
}
return [
{
index: this.title,
@ -326,10 +317,6 @@ export function IndexPatternProvider(Private, $http, config, kbnIndex, Promise,
});
}
isIndexExpansionEnabled() {
return !this.notExpandable;
}
isTimeBased() {
return !!this.timeFieldName && (!this.fields || !!this.getTimeField());
}

2
test/functional/apps/discover/_field_data.js
View file

@ -234,7 +234,7 @@ export default function ({ getService, getPageObjects }) {
})
.then(function (toastMessage) {
screenshots.take('Discover-syntax-error-toast');
expect(toastMessage).to.be(expectedError);
expect(toastMessage).to.contain(expectedError);
})
.then(function () {
return PageObjects.header.clickToastOK();