[Index Management] Added docs count and size for serverless (#191985)

## Summary

Fixes https://github.com/elastic/kibana/issues/190131

This PR adds size and documents count to indices and data streams tables
in Index Management on serverless.

### Screenrecording



https://github.com/user-attachments/assets/51a933e2-e4ef-42a0-9c82-39bf6e194ee0



### Screenshots 
<img width="1047" alt="Screenshot 2024-09-06 at 19 20 59"
src="https://github.com/user-attachments/assets/8c6d0378-116f-44e3-aaca-0c95f135b1bf">
<img width="1045" alt="Screenshot 2024-09-06 at 19 21 06"
src="https://github.com/user-attachments/assets/37477f7b-e229-4400-9a28-8382f7d1155e">
<img width="1036" alt="Screenshot 2024-09-06 at 19 27 59"
src="https://github.com/user-attachments/assets/0fd2ef1b-3b0c-4d3e-8b56-984281436898">
<img width="506" alt="Screenshot 2024-09-06 at 19 28 12"
src="https://github.com/user-attachments/assets/1823a964-6f6f-464b-9910-e586ccd2e9bb">



### Checklist

Delete any items that are not applicable to this PR.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [ ] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [ ] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [ ] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)


### Risk Matrix

Delete this section if it is not applicable to this PR.

Before closing this PR, invite QA, stakeholders, and other developers to
identify risks that should be tested prior to the change/feature
release.

When forming the risk matrix, consider some of the following examples
and how they may potentially impact the change:

| Risk | Probability | Severity | Mitigation/Notes |

|---------------------------|-------------|----------|-------------------------|
| Multiple Spaces&mdash;unexpected behavior in non-default Kibana Space.
| Low | High | Integration tests will verify that all features are still
supported in non-default Kibana Space and when user switches between
spaces. |
| Multiple nodes&mdash;Elasticsearch polling might have race conditions
when multiple Kibana nodes are polling for the same tasks. | High | Low
| Tasks are idempotent, so executing them multiple times will not result
in logical error, but will degrade performance. To test for this case we
add plenty of unit tests around this logic and document manual testing
procedure. |
| Code should gracefully handle cases when feature X or plugin Y are
disabled. | Medium | High | Unit tests will verify that any feature flag
or plugin combination still results in our service operational. |
| [See more potential risk
examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) |


### For maintainers

- [ ] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Yulia Čech 2024-09-16 13:44:41 +02:00 committed by GitHub
parent f168fb4791
commit 3de252e688
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 531 additions and 215 deletions

View file

@ -295,6 +295,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
'xpack.index_management.enableLegacyTemplates (boolean?|never)',
'xpack.index_management.enableIndexStats (boolean?|never)',
'xpack.index_management.enableDataStreamStats (boolean?|never)',
'xpack.index_management.enableSizeAndDocCount (boolean?|never)',
'xpack.index_management.editableIndexSettings (all?|limited?|never)',
'xpack.index_management.enableMappingsSourceFieldSection (boolean?|never)',
'xpack.index_management.dev.enableSemanticText (boolean?)',

View file

@ -24,7 +24,7 @@ export interface DataStreamsTabTestBed extends TestBed<TestSubjects> {
actions: {
goToDataStreamsList: () => void;
clickEmptyPromptIndexTemplateLink: () => void;
clickIncludeStatsSwitch: () => void;
clickIncludeStatsSwitch: () => Promise<void>;
toggleViewFilterAt: (index: number) => void;
sortTableOnStorageSize: () => void;
sortTableOnName: () => void;
@ -90,9 +90,13 @@ export const setup = async (
component.update();
};
const clickIncludeStatsSwitch = () => {
const { find } = testBed;
find('includeStatsSwitch').simulate('click');
const clickIncludeStatsSwitch = async () => {
const { find, component } = testBed;
await act(async () => {
find('includeStatsSwitch').simulate('click');
});
component.update();
};
const toggleViewFilterAt = (index: number) => {

View file

@ -156,6 +156,10 @@ describe('Data Streams tab', () => {
name: 'dataStream1',
storageSize: '5b',
storageSizeBytes: 5,
// metering API mock
meteringStorageSize: '156kb',
meteringStorageSizeBytes: 156000,
meteringDocsCount: 10000,
});
setLoadDataStreamsResponse([
@ -164,6 +168,10 @@ describe('Data Streams tab', () => {
name: 'dataStream2',
storageSize: '1kb',
storageSizeBytes: 1000,
// metering API mock
meteringStorageSize: '156kb',
meteringStorageSizeBytes: 156000,
meteringDocsCount: 10000,
lifecycle: {
enabled: true,
data_retention: '7d',
@ -224,15 +232,12 @@ describe('Data Streams tab', () => {
});
test('has a switch that will reload the data streams with additional stats when clicked', async () => {
const { exists, actions, table, component } = testBed;
const { exists, actions, table } = testBed;
expect(exists('includeStatsSwitch')).toBe(true);
// Changing the switch will automatically reload the data streams.
await act(async () => {
actions.clickIncludeStatsSwitch();
});
component.update();
await actions.clickIncludeStatsSwitch();
expect(httpSetup.get).toHaveBeenLastCalledWith(
`${API_BASE_PATH}/data_streams`,
@ -267,12 +272,9 @@ describe('Data Streams tab', () => {
test('sorting on stats sorts by bytes value instead of human readable value', async () => {
// Guards against regression of #86122.
const { actions, table, component } = testBed;
const { actions, table } = testBed;
await act(async () => {
actions.clickIncludeStatsSwitch();
});
component.update();
await actions.clickIncludeStatsSwitch();
actions.sortTableOnStorageSize();
@ -306,7 +308,7 @@ describe('Data Streams tab', () => {
actions.sortTableOnName();
});
test('hides stats toggle if enableDataStreamStats===false', async () => {
test(`doesn't hide stats toggle if enableDataStreamStats===false`, async () => {
testBed = await setup(httpSetup, {
config: {
enableDataStreamStats: false,
@ -321,14 +323,82 @@ describe('Data Streams tab', () => {
component.update();
expect(exists('includeStatsSwitch')).toBeFalsy();
expect(exists('includeStatsSwitch')).toBeTruthy();
});
test('shows storage size and documents count if enableSizeAndDocCount===true, enableDataStreamStats==false', async () => {
testBed = await setup(httpSetup, {
config: {
enableSizeAndDocCount: true,
enableDataStreamStats: false,
},
});
const { actions, component, table } = testBed;
await act(async () => {
actions.goToDataStreamsList();
});
component.update();
await actions.clickIncludeStatsSwitch();
const { tableCellsValues } = table.getMetaData('dataStreamTable');
expect(tableCellsValues).toEqual([
['', 'dataStream1', 'green', '156kb', '10000', '1', '7 days', 'Delete'],
['', 'dataStream2', 'green', '156kb', '10000', '1', '5 days ', 'Delete'],
]);
});
test('shows last updated and storage size if enableDataStreamStats===true, enableSizeAndDocCount===false', async () => {
testBed = await setup(httpSetup, {
config: {
enableDataStreamStats: true,
enableSizeAndDocCount: false,
},
});
const { actions, component, table } = testBed;
await act(async () => {
actions.goToDataStreamsList();
});
component.update();
await actions.clickIncludeStatsSwitch();
const { tableCellsValues } = table.getMetaData('dataStreamTable');
expect(tableCellsValues).toEqual([
[
'',
'dataStream1',
'green',
'December 31st, 1969 7:00:00 PM',
'5b',
'1',
'7 days',
'Delete',
],
[
'',
'dataStream2',
'green',
'December 31st, 1969 7:00:00 PM',
'1kb',
'1',
'5 days ',
'Delete',
],
]);
});
test('clicking the indices count navigates to the backing indices', async () => {
const { table, actions } = testBed;
await actions.clickIndicesAt(0);
expect(table.getMetaData('indexTable').tableCellsValues).toEqual([
['', 'data-stream-index', '', '', '', '', '', '', 'dataStream1'],
['', 'data-stream-index', '', '', '', '', '0', '', 'dataStream1'],
]);
});
@ -707,7 +777,7 @@ describe('Data Streams tab', () => {
const { table, actions } = testBed;
await actions.clickIndicesAt(0);
expect(table.getMetaData('indexTable').tableCellsValues).toEqual([
['', 'data-stream-index', '', '', '', '', '', '', '%dataStream'],
['', 'data-stream-index', '', '', '', '', '0', '', '%dataStream'],
]);
});
});

View file

@ -394,7 +394,7 @@ describe('<IndexManagementHome />', () => {
component.update();
});
test('renders the table column with index stats by default', () => {
test('renders the table column with all index stats when enableIndexStats is true', () => {
const { table } = testBed;
const { tableCellsValues } = table.getMetaData('indexTable');
@ -403,7 +403,7 @@ describe('<IndexManagementHome />', () => {
]);
});
describe('Disabled', () => {
describe('renders only size and docs count when enableIndexStats is false, enableSizeAndDocCount is true', () => {
beforeEach(async () => {
await act(async () => {
testBed = await setup(httpSetup, {
@ -411,6 +411,7 @@ describe('<IndexManagementHome />', () => {
enableLegacyTemplates: true,
enableIndexActions: true,
enableIndexStats: false,
enableSizeAndDocCount: true,
},
});
});
@ -420,7 +421,33 @@ describe('<IndexManagementHome />', () => {
component.update();
});
test('hides index stats information from table', async () => {
test('hides some index stats information from table', async () => {
const { table } = testBed;
const { tableCellsValues } = table.getMetaData('indexTable');
expect(tableCellsValues).toEqual([['', 'test', '10,000', '156kb', '']]);
});
});
describe('renders no index stats when enableIndexStats is false, enableSizeAndDocCount is false', () => {
beforeEach(async () => {
await act(async () => {
testBed = await setup(httpSetup, {
config: {
enableLegacyTemplates: true,
enableIndexActions: true,
enableIndexStats: false,
enableSizeAndDocCount: false,
},
});
});
const { component } = testBed;
component.update();
});
test('hides all index stats information from table', async () => {
const { table } = testBed;
const { tableCellsValues } = table.getMetaData('indexTable');

View file

@ -5,9 +5,9 @@
* 2.0.
*/
import { splitSizeAndUnits } from './data_stream_serialization';
import { splitSizeAndUnits } from './data_stream_utils';
describe('Data stream serialization', () => {
describe('Data stream utils', () => {
test('can split size and units from lifecycle string', () => {
expect(splitSizeAndUnits('1h')).toEqual({ size: '1', unit: 'h' });
expect(splitSizeAndUnits('20micron')).toEqual({ size: '20', unit: 'micron' });

View file

@ -0,0 +1,64 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { DataStream, DataRetention } from '../types';
export const splitSizeAndUnits = (field: string): { size: string; unit: string } => {
let size = '';
let unit = '';
const result = /(\d+)(\w+)/.exec(field);
if (result) {
size = result[1];
unit = result[2];
}
return {
size,
unit,
};
};
export const serializeAsESLifecycle = (lifecycle?: DataRetention): DataStream['lifecycle'] => {
if (!lifecycle || !lifecycle?.enabled) {
return undefined;
}
const { infiniteDataRetention, value, unit } = lifecycle;
if (infiniteDataRetention) {
return {
enabled: true,
};
}
return {
enabled: true,
data_retention: `${value}${unit}`,
};
};
export const deserializeESLifecycle = (lifecycle?: DataStream['lifecycle']): DataRetention => {
if (!lifecycle || !lifecycle?.enabled) {
return { enabled: false };
}
if (!lifecycle.data_retention) {
return {
enabled: true,
infiniteDataRetention: true,
};
}
const { size, unit } = splitSizeAndUnits(lifecycle.data_retention as string);
return {
enabled: true,
value: Number(size),
unit,
};
};

View file

@ -6,10 +6,10 @@
*/
export {
deserializeDataStream,
deserializeDataStreamList,
splitSizeAndUnits,
} from './data_stream_serialization';
serializeAsESLifecycle,
deserializeESLifecycle,
} from './data_stream_utils';
export {
deserializeTemplate,

View file

@ -12,7 +12,7 @@ import {
TemplateListItem,
TemplateType,
} from '../types';
import { deserializeESLifecycle } from './data_stream_serialization';
import { deserializeESLifecycle } from './data_stream_utils';
import { allowAutoCreateRadioValues, allowAutoCreateRadioIds } from '../constants';
const hasEntries = (data: object = {}) => Object.entries(data).length > 0;

View file

@ -37,6 +37,8 @@ export interface EnhancedDataStreamFromEs extends IndicesDataStream {
store_size?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size'];
store_size_bytes?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size_bytes'];
maximum_timestamp?: IndicesDataStreamsStatsDataStreamsStatsItem['maximum_timestamp'];
metering_size_in_bytes?: number;
metering_doc_count?: number;
indices: DataStreamIndexFromEs[];
privileges: {
delete_index: boolean;
@ -55,6 +57,9 @@ export interface DataStream {
storageSize?: ByteSize;
storageSizeBytes?: number;
maxTimeStamp?: number;
meteringStorageSizeBytes?: number;
meteringStorageSize?: string;
meteringDocsCount?: number;
_meta?: Metadata;
privileges: Privileges;
hidden: boolean;

View file

@ -12,7 +12,7 @@ import { breadcrumbService, IndexManagementBreadcrumb } from '../../../../servic
import { setupEnvironment } from './helpers';
import { API_BASE_PATH } from './helpers/constants';
import { setup, ComponentTemplateCreateTestBed } from './helpers/component_template_create.helpers';
import { serializeAsESLifecycle } from '../../../../../../common/lib/data_stream_serialization';
import { serializeAsESLifecycle } from '../../../../../../common/lib';
jest.mock('@kbn/code-editor', () => {
const original = jest.requireActual('@kbn/code-editor');

View file

@ -19,10 +19,7 @@ import {
StepMappingsContainer,
StepAliasesContainer,
} from '../../shared_imports';
import {
serializeAsESLifecycle,
deserializeESLifecycle,
} from '../../../../../../common/lib/data_stream_serialization';
import { serializeAsESLifecycle, deserializeESLifecycle } from '../../../../../../common/lib';
import { useComponentTemplatesContext } from '../../component_templates_context';
import { StepLogisticsContainer, StepReviewContainer } from './steps';

View file

@ -22,7 +22,7 @@ import {
} from '../shared';
import { documentationService } from '../../services/documentation';
import { SectionError } from '../section_error';
import { serializeAsESLifecycle } from '../../../../common/lib/data_stream_serialization';
import { serializeAsESLifecycle } from '../../../../common/lib';
import {
SimulateTemplateFlyoutContent,
SimulateTemplateProps,

View file

@ -52,13 +52,14 @@ import { useAppContext } from '../../../../app_context';
import { DataStreamsBadges } from '../data_stream_badges';
import { useIlmLocator } from '../../../../services/use_ilm_locator';
interface Detail {
name: string;
toolTip: string;
content: any;
dataTestSubj: string;
}
interface DetailsListProps {
details: Array<{
name: string;
toolTip: string;
content: any;
dataTestSubj: string;
}>;
details: Detail[];
}
const DetailsList: React.FunctionComponent<DetailsListProps> = ({ details }) => {
@ -162,6 +163,8 @@ export const DataStreamDetailPanel: React.FunctionComponent<Props> = ({
ilmPolicyName,
storageSize,
maxTimeStamp,
meteringStorageSize,
meteringDocsCount,
lifecycle,
} = dataStream;
@ -222,7 +225,7 @@ export const DataStreamDetailPanel: React.FunctionComponent<Props> = ({
</EuiLink>
);
const defaultDetails = [
const defaultDetails: Detail[] = [
{
name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.healthTitle', {
defaultMessage: 'Health',
@ -233,34 +236,67 @@ export const DataStreamDetailPanel: React.FunctionComponent<Props> = ({
content: <DataHealth health={health} />,
dataTestSubj: 'healthDetail',
},
{
name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampTitle', {
defaultMessage: 'Last updated',
}),
toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampToolTip', {
defaultMessage: 'The most recent document to be added to the data stream.',
}),
content: maxTimeStamp ? (
humanizeTimeStamp(maxTimeStamp)
) : (
<em>
{i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampNoneMessage', {
defaultMessage: `Never`,
})}
</em>
),
dataTestSubj: 'lastUpdatedDetail',
},
{
name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeTitle', {
defaultMessage: 'Storage size',
}),
toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeToolTip', {
defaultMessage: `The total size of all shards in the data streams backing indices.`,
}),
content: storageSize,
dataTestSubj: 'storageSizeDetail',
},
];
// add either documents count and size or last updated and size
if (config.enableSizeAndDocCount) {
defaultDetails.push(
{
name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.meteringDocsCountTitle', {
defaultMessage: 'Documents count',
}),
toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.meteringDocsCountToolTip', {
defaultMessage: 'The number of documents in this data stream.',
}),
content: meteringDocsCount,
dataTestSubj: 'docsCountDetail',
},
{
name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeTitle', {
defaultMessage: 'Storage size',
}),
toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeToolTip', {
defaultMessage: `The total size of all shards in the data streams backing indices.`,
}),
content: meteringStorageSize,
dataTestSubj: 'meteringStorageSizeDetail',
}
);
}
if (config.enableDataStreamStats) {
defaultDetails.push(
{
name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampTitle', {
defaultMessage: 'Last updated',
}),
toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampToolTip', {
defaultMessage: 'The most recent document to be added to the data stream.',
}),
content: maxTimeStamp ? (
humanizeTimeStamp(maxTimeStamp)
) : (
<em>
{i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampNoneMessage', {
defaultMessage: `Never`,
})}
</em>
),
dataTestSubj: 'lastUpdatedDetail',
},
{
name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeTitle', {
defaultMessage: 'Storage size',
}),
toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeToolTip', {
defaultMessage: `The total size of all shards in the data streams backing indices.`,
}),
content: storageSize,
dataTestSubj: 'storageSizeDetail',
}
);
}
defaultDetails.push(
{
name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.indicesTitle', {
defaultMessage: 'Indices',
@ -328,8 +364,8 @@ export const DataStreamDetailPanel: React.FunctionComponent<Props> = ({
</ConditionalWrap>
),
dataTestSubj: 'dataRetentionDetail',
},
];
}
);
// If both rentention types are available, we wanna surface to the user both
if (lifecycle?.effective_retention && lifecycle?.data_retention) {

View file

@ -60,11 +60,8 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
const {
core: { getUrlForApp, executionContext },
plugins: { isFleetEnabled },
config,
} = useAppContext();
const { enableDataStreamStats: isDataStreamStatsEnabled } = config;
useExecutionContext(executionContext, {
type: 'application',
page: 'indexManagementDataStreamsTab',
@ -149,37 +146,35 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
</EuiText>
</EuiFlexItem>
{isDataStreamStatsEnabled && (
<EuiFlexItem grow={false}>
<EuiFlexGroup gutterSize="s">
<EuiFlexItem grow={false}>
<EuiSwitch
label={i18n.translate(
'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchLabel',
{
defaultMessage: 'Include stats',
}
)}
checked={isIncludeStatsChecked}
onChange={(e) => setIsIncludeStatsChecked(e.target.checked)}
data-test-subj="includeStatsSwitch"
/>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiFlexGroup gutterSize="s">
<EuiFlexItem grow={false}>
<EuiSwitch
label={i18n.translate(
'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchLabel',
{
defaultMessage: 'Include stats',
}
)}
checked={isIncludeStatsChecked}
onChange={(e) => setIsIncludeStatsChecked(e.target.checked)}
data-test-subj="includeStatsSwitch"
/>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiIconTip
content={i18n.translate(
'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchToolTip',
{
defaultMessage: 'Including stats can increase reload times',
}
)}
position="top"
/>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
)}
<EuiFlexItem grow={false}>
<EuiIconTip
content={i18n.translate(
'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchToolTip',
{
defaultMessage: 'Including stats can increase reload times',
}
)}
position="top"
/>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<FilterListButton<DataStreamFilterName> filters={filters} onChange={setFilters} />
</EuiFlexItem>

View file

@ -104,31 +104,55 @@ export const DataStreamTable: React.FunctionComponent<Props> = ({
});
if (includeStats) {
columns.push({
field: 'maxTimeStamp',
name: i18n.translate('xpack.idxMgmt.dataStreamList.table.maxTimeStampColumnTitle', {
defaultMessage: 'Last updated',
}),
truncateText: true,
sortable: true,
render: (maxTimeStamp: DataStream['maxTimeStamp']) =>
maxTimeStamp
? humanizeTimeStamp(maxTimeStamp)
: i18n.translate('xpack.idxMgmt.dataStreamList.table.maxTimeStampColumnNoneMessage', {
defaultMessage: 'Never',
}),
});
columns.push({
field: 'storageSizeBytes',
name: i18n.translate('xpack.idxMgmt.dataStreamList.table.storageSizeColumnTitle', {
defaultMessage: 'Storage size',
}),
truncateText: true,
sortable: true,
render: (storageSizeBytes: DataStream['storageSizeBytes'], dataStream: DataStream) =>
dataStream.storageSize,
});
if (config.enableSizeAndDocCount) {
// datastreams stats from metering API on serverless
columns.push({
field: 'meteringStorageSizeBytes',
name: i18n.translate('xpack.idxMgmt.dataStreamList.table.storageSizeColumnTitle', {
defaultMessage: 'Storage size',
}),
truncateText: true,
sortable: true,
render: (
meteringStorageSizeBytes: DataStream['meteringStorageSizeBytes'],
dataStream: DataStream
) => dataStream.meteringStorageSize,
});
columns.push({
field: 'meteringDocsCount',
name: i18n.translate('xpack.idxMgmt.dataStreamList.table.docsCountColumnTitle', {
defaultMessage: 'Documents count',
}),
truncateText: true,
sortable: true,
});
}
if (config.enableDataStreamStats) {
columns.push({
field: 'maxTimeStamp',
name: i18n.translate('xpack.idxMgmt.dataStreamList.table.maxTimeStampColumnTitle', {
defaultMessage: 'Last updated',
}),
truncateText: true,
sortable: true,
render: (maxTimeStamp: DataStream['maxTimeStamp']) =>
maxTimeStamp
? humanizeTimeStamp(maxTimeStamp)
: i18n.translate('xpack.idxMgmt.dataStreamList.table.maxTimeStampColumnNoneMessage', {
defaultMessage: 'Never',
}),
});
columns.push({
field: 'storageSizeBytes',
name: i18n.translate('xpack.idxMgmt.dataStreamList.table.storageSizeColumnTitle', {
defaultMessage: 'Storage size',
}),
truncateText: true,
sortable: true,
render: (storageSizeBytes: DataStream['storageSizeBytes'], dataStream: DataStream) =>
dataStream.storageSize,
});
}
}
columns.push({

View file

@ -26,14 +26,15 @@ import {
getLanguageDefinitionCodeSnippet,
getConsoleRequest,
} from '@kbn/search-api-panels';
import { StatusDetails } from './status_details';
import type { Index } from '../../../../../../../common';
import { useAppContext } from '../../../../../app_context';
import { documentationService } from '../../../../../services';
import { languageDefinitions, curlDefinition } from './languages';
import { StatusDetails } from './status_details';
import { DataStreamDetails } from './data_stream_details';
import { StorageDetails } from './storage_details';
import { AliasesDetails } from './aliases_details';
import { SizeDocCountDetails } from './size_doc_count_details';
interface Props {
indexDetails: Index;
@ -85,6 +86,8 @@ export const DetailsPageOverview: React.FunctionComponent<Props> = ({ indexDetai
health={health}
/>
<SizeDocCountDetails size={size} documents={documents} />
<AliasesDetails aliases={aliases} />
{dataStream && <DataStreamDetails dataStreamName={dataStream} />}

View file

@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { FunctionComponent } from 'react';
import { css } from '@emotion/react';
import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText, EuiTextColor } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { euiThemeVars } from '@kbn/ui-theme';
import type { Index } from '../../../../../../../common';
import { useAppContext } from '../../../../../app_context';
import { OverviewCard } from './overview_card';
export const SizeDocCountDetails: FunctionComponent<{
size: Index['size'];
documents: Index['documents'];
}> = ({ size, documents }) => {
const { config } = useAppContext();
if (!config.enableSizeAndDocCount) {
return null;
}
return (
<OverviewCard
data-test-subj="indexDetailsStorage"
title={i18n.translate('xpack.idxMgmt.indexDetails.overviewTab.storage.cardTitle', {
defaultMessage: 'Storage',
})}
content={{
left: (
<EuiFlexGroup gutterSize="xs" alignItems="baseline">
<EuiFlexItem grow={false}>
<EuiText
css={css`
font-size: ${euiThemeVars.euiFontSizeL};
`}
>
{size}
</EuiText>
</EuiFlexItem>
<EuiFlexItem>
<EuiTextColor color="subdued">
{i18n.translate('xpack.idxMgmt.indexDetails.overviewTab.storage.totalSizeLabel', {
defaultMessage: 'Total',
})}
</EuiTextColor>
</EuiFlexItem>
</EuiFlexGroup>
),
right: null,
}}
footer={{
left: (
<EuiFlexGroup gutterSize="xs">
<EuiFlexItem grow={false}>
<EuiIcon type="documents" />
</EuiFlexItem>
<EuiFlexItem grow={false}>{documents}</EuiFlexItem>
<EuiFlexItem>
<EuiTextColor color="subdued">
{i18n.translate(
'xpack.idxMgmt.indexDetails.overviewTab.status.meteringDocumentsLabel',
{
defaultMessage: '{documents, plural, one {Document} other {Documents}}',
values: {
documents,
},
}
)}
</EuiTextColor>
</EuiFlexItem>
</EuiFlexGroup>
),
}}
/>
);
};

View file

@ -52,6 +52,7 @@ import { IndexTablePagination, PAGE_SIZE_OPTIONS } from './index_table_paginatio
const getColumnConfigs = ({
showIndexStats,
showSizeAndDocCount,
history,
filterChanged,
extensionsService,
@ -100,6 +101,28 @@ const getColumnConfigs = ({
},
];
// size and docs count enabled by either "enableIndexStats" or "enableSizeAndDocCount" configs
if (showIndexStats || showSizeAndDocCount) {
columns.push(
{
fieldName: 'documents',
label: i18n.translate('xpack.idxMgmt.indexTable.headers.documentsHeader', {
defaultMessage: 'Documents count',
}),
order: 60,
render: (index) => {
return Number(index.documents ?? 0).toLocaleString();
},
},
{
fieldName: 'size',
label: i18n.translate('xpack.idxMgmt.indexTable.headers.storageSizeHeader', {
defaultMessage: 'Storage size',
}),
order: 70,
}
);
}
if (showIndexStats) {
columns.push(
{
@ -130,25 +153,6 @@ const getColumnConfigs = ({
defaultMessage: 'Replicas',
}),
order: 50,
},
{
fieldName: 'documents',
label: i18n.translate('xpack.idxMgmt.indexTable.headers.documentsHeader', {
defaultMessage: 'Docs count',
}),
order: 60,
render: (index) => {
if (index.documents) {
return Number(index.documents).toLocaleString();
}
},
},
{
fieldName: 'size',
label: i18n.translate('xpack.idxMgmt.indexTable.headers.storageSizeHeader', {
defaultMessage: 'Storage size',
}),
order: 70,
}
);
}
@ -533,6 +537,7 @@ export class IndexTable extends Component {
const { extensionsService } = services;
const columnConfigs = getColumnConfigs({
showIndexStats: config.enableIndexStats,
showSizeAndDocCount: config.enableSizeAndDocCount,
extensionsService,
filterChanged,
history,

View file

@ -22,7 +22,7 @@ import {
} from '@elastic/eui';
import { reactRouterNavigate } from '../../../../../../shared_imports';
import { useAppContext } from '../../../../../app_context';
import { serializeAsESLifecycle } from '../../../../../../../common/lib/data_stream_serialization';
import { serializeAsESLifecycle } from '../../../../../../../common/lib';
import { getLifecycleValue } from '../../../../../lib/data_streams';
import { TemplateDeserialized } from '../../../../../../../common';
import { ILM_PAGES_POLICY_EDIT } from '../../../../../constants';

View file

@ -44,8 +44,8 @@ export class IndexMgmtUIPlugin
enableIndexActions: boolean;
enableLegacyTemplates: boolean;
enableIndexStats: boolean;
enableSizeAndDocCount: boolean;
enableDataStreamStats: boolean;
enableSizeAndDocCount: boolean;
editableIndexSettings: 'all' | 'limited';
isIndexManagementUiEnabled: boolean;
enableMappingsSourceFieldSection: boolean;
@ -63,8 +63,8 @@ export class IndexMgmtUIPlugin
enableIndexActions,
enableLegacyTemplates,
enableIndexStats,
enableSizeAndDocCount,
enableDataStreamStats,
enableSizeAndDocCount,
editableIndexSettings,
enableMappingsSourceFieldSection,
enableTogglingDataRetention,
@ -75,8 +75,8 @@ export class IndexMgmtUIPlugin
enableIndexActions: enableIndexActions ?? true,
enableLegacyTemplates: enableLegacyTemplates ?? true,
enableIndexStats: enableIndexStats ?? true,
enableSizeAndDocCount: enableSizeAndDocCount ?? true,
enableDataStreamStats: enableDataStreamStats ?? true,
enableSizeAndDocCount: enableSizeAndDocCount ?? false,
editableIndexSettings: editableIndexSettings ?? 'all',
enableMappingsSourceFieldSection: enableMappingsSourceFieldSection ?? true,
enableTogglingDataRetention: enableTogglingDataRetention ?? true,

View file

@ -83,6 +83,7 @@ const configLatest: PluginConfigDescriptor<IndexManagementConfig> = {
enableLegacyTemplates: true,
enableIndexStats: true,
enableDataStreamStats: true,
enableSizeAndDocCount: true,
editableIndexSettings: true,
enableMappingsSourceFieldSection: true,
enableTogglingDataRetention: true,

View file

@ -5,7 +5,8 @@
* 2.0.
*/
import { DataStream, EnhancedDataStreamFromEs, Health, DataRetention } from '../types';
import { ByteSizeValue } from '@kbn/config-schema';
import type { DataStream, EnhancedDataStreamFromEs, Health } from '../../common';
export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs): DataStream {
const {
@ -19,12 +20,18 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs
store_size: storageSize,
store_size_bytes: storageSizeBytes,
maximum_timestamp: maxTimeStamp,
metering_size_in_bytes: meteringStorageSizeBytes,
metering_doc_count: meteringDocsCount,
_meta,
privileges,
hidden,
lifecycle,
next_generation_managed_by: nextGenerationManagedBy,
} = dataStreamFromEs;
const meteringStorageSize =
meteringStorageSizeBytes !== undefined
? new ByteSizeValue(meteringStorageSizeBytes).toString()
: undefined;
return {
name,
@ -54,6 +61,9 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs
storageSize,
storageSizeBytes,
maxTimeStamp,
meteringStorageSize,
meteringStorageSizeBytes,
meteringDocsCount,
_meta,
privileges,
hidden,
@ -67,59 +77,3 @@ export function deserializeDataStreamList(
): DataStream[] {
return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream));
}
export const splitSizeAndUnits = (field: string): { size: string; unit: string } => {
let size = '';
let unit = '';
const result = /(\d+)(\w+)/.exec(field);
if (result) {
size = result[1];
unit = result[2];
}
return {
size,
unit,
};
};
export const serializeAsESLifecycle = (lifecycle?: DataRetention): DataStream['lifecycle'] => {
if (!lifecycle || !lifecycle?.enabled) {
return undefined;
}
const { infiniteDataRetention, value, unit } = lifecycle;
if (infiniteDataRetention) {
return {
enabled: true,
};
}
return {
enabled: true,
data_retention: `${value}${unit}`,
};
};
export const deserializeESLifecycle = (lifecycle?: DataStream['lifecycle']): DataRetention => {
if (!lifecycle || !lifecycle?.enabled) {
return { enabled: false };
}
if (!lifecycle.data_retention) {
return {
enabled: true,
infiniteDataRetention: true,
};
}
const { size, unit } = splitSizeAndUnits(lifecycle.data_retention as string);
return {
enabled: true,
value: Number(size),
unit,
};
};

View file

@ -10,13 +10,10 @@ import { IScopedClusterClient } from '@kbn/core/server';
import { IndexDataEnricher } from '../services';
import { Index } from '..';
import { RouteDependencies } from '../types';
import type { MeteringStats } from './types';
interface MeteringStatsResponse {
indices: Array<{
name: string;
num_docs: number;
size_in_bytes: number;
}>;
indices: MeteringStats[];
}
async function fetchIndicesCall(

View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface MeteringStats {
name: string;
num_docs: number;
size_in_bytes: number;
}

View file

@ -13,18 +13,27 @@ import {
IndicesDataStreamsStatsDataStreamsStatsItem,
SecurityHasPrivilegesResponse,
} from '@elastic/elasticsearch/lib/api/types';
import { deserializeDataStream, deserializeDataStreamList } from '../../../../common/lib';
import type { MeteringStats } from '../../../lib/types';
import {
deserializeDataStream,
deserializeDataStreamList,
} from '../../../lib/data_stream_serialization';
import { EnhancedDataStreamFromEs } from '../../../../common/types';
import { RouteDependencies } from '../../../types';
import { addBasePath } from '..';
interface MeteringStatsResponse {
datastreams: MeteringStats[];
}
const enhanceDataStreams = ({
dataStreams,
dataStreamsStats,
meteringStats,
dataStreamsPrivileges,
}: {
dataStreams: IndicesDataStream[];
dataStreamsStats?: IndicesDataStreamsStatsDataStreamsStatsItem[];
meteringStats?: MeteringStats[];
dataStreamsPrivileges?: SecurityHasPrivilegesResponse;
}): EnhancedDataStreamFromEs[] => {
return dataStreams.map((dataStream) => {
@ -51,6 +60,14 @@ const enhanceDataStreams = ({
}
}
if (meteringStats) {
const datastreamMeteringStats = meteringStats.find((s) => s.name === dataStream.name);
if (datastreamMeteringStats) {
enhancedDataStream.metering_size_in_bytes = datastreamMeteringStats.size_in_bytes;
enhancedDataStream.metering_doc_count = datastreamMeteringStats.num_docs;
}
}
return enhancedDataStream;
});
};
@ -70,6 +87,17 @@ const getDataStreamsStats = (client: IScopedClusterClient, name = '*') => {
});
};
const getMeteringStats = (client: IScopedClusterClient, name?: string) => {
let path = `/_metering/stats`;
if (name) {
path = `${path}/${name}`;
}
return client.asSecondaryAuthUser.transport.request<MeteringStatsResponse>({
method: 'GET',
path,
});
};
const getDataStreamsPrivileges = (client: IScopedClusterClient, names: string[]) => {
return client.asCurrentUser.security.hasPrivileges({
body: {
@ -99,10 +127,14 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }:
let dataStreamsStats;
let dataStreamsPrivileges;
let meteringStats;
if (includeStats && config.isDataStreamStatsEnabled !== false) {
({ data_streams: dataStreamsStats } = await getDataStreamsStats(client));
}
if (includeStats && config.isSizeAndDocCountEnabled !== false) {
({ datastreams: meteringStats } = await getMeteringStats(client));
}
if (config.isSecurityEnabled() && dataStreams.length > 0) {
dataStreamsPrivileges = await getDataStreamsPrivileges(
@ -114,6 +146,7 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }:
const enhancedDataStreams = enhanceDataStreams({
dataStreams,
dataStreamsStats,
meteringStats,
dataStreamsPrivileges,
});
@ -138,6 +171,7 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }:
const { name } = request.params as TypeOf<typeof paramsSchema>;
const { client } = (await context.core).elasticsearch;
let dataStreamsStats;
let meteringStats;
try {
const { data_streams: dataStreams } = await getDataStreams(client, name);
@ -146,6 +180,10 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }:
({ data_streams: dataStreamsStats } = await getDataStreamsStats(client, name));
}
if (config.isSizeAndDocCountEnabled !== false) {
({ datastreams: meteringStats } = await getMeteringStats(client, name));
}
if (dataStreams[0]) {
let dataStreamsPrivileges;
@ -156,6 +194,7 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }:
const enhancedDataStreams = enhanceDataStreams({
dataStreams,
dataStreamsStats,
meteringStats,
dataStreamsPrivileges,
});
const body = deserializeDataStream(enhancedDataStreams[0]);

View file

@ -118,6 +118,9 @@ export default function ({ getService }: FtrProviderContext) {
lifecycle: {
enabled: true,
},
meteringDocsCount: 0,
meteringStorageSize: '0b',
meteringStorageSizeBytes: 0,
});
});
});