github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Log Stream] Support date nanos (#170308)

Browse source 
closes #88290 

## 📝  Summary

As described in #88290 we need to add `date_nanos` support to the Stream
UI page. In this PR the necessary changes have been made all over the
Stream UI and the usages of it.

## ✅  Testing

⚠️ Testing the Stream UI with old timestamp indices is important to make
sure that the behavior is still as before and not affected at all. This
can be done by running local env from the PR and simulating all
interactions on edge-lit cluster for example, to make sure that the
behavior is not changed.

For testing the new changes with support of `date_nano`:

1. You can use [the steps
here](https://github.com/elastic/kibana/issues/88290#issuecomment-1713400873)
to create and ingest documents with nano precision.
2. Navigate to the stream UI and the documents should be displayed
properly.
3. Sync with the URL state should be changed from timestamp to ISO
string date.
4. Changing time ranges should behave as before, as well as Text
Highlights.
5. Open the logs flyout and you should see the timestamp in nano
seconds.
6. Play around with the minimap, it should behave exactly as before.

### Stream UI:

<img width="2556" alt="Screenshot 2023-11-02 at 14 15 49"
src="596966cd-0ee0-44ee-ba15-f387f3725f66">

- The stream UI has been affected in many areas:
- The logPosition key in the URL should now be in ISO string, but still
backward compatible incase the user has bookmarks with timestamps.
- The minimap should still behave as before in terms of navigation
onClick and also highlighting displayed areas

### Stream UI Flyout:

<img width="2556" alt="Screenshot 2023-11-02 at 14 15 23"
src="6081533c-3bed-43e1-872d-c83fe78ab436">

- The logs flyout should now display the date in nanos format if the
document is ingested using a nano format.

### Anomalies:

<img width="1717" alt="Screenshot 2023-11-01 at 10 37 22"
src="b6170d76-40a4-44db-85de-d8ae852bc17e">

-Anomalies tab under logs as a navigation to stream UI which should
still behave as before passing the filtration and time.

### Categories:

<img width="1705" alt="Screenshot 2023-11-01 at 10 38 19"
src="c4c19202-d27f-410f-b94d-80507542c775">

-Categories tab under logs as a navigation to stream UI which should
still behave as before passing the filtration and time.

### External Links To Stream:
- All links to the Stream UI should still work as before:
   - APM Links for traces, containers, hosts
   - Infra links in Inventory and Hosts views

## 🎥 Demo


9a39bc5a-ba37-49e0-b7f2-e73260fb01f0
This commit is contained in:
mohamedhamed-ahmed 2023-11-13 13:51:33 +00:00 • committed by GitHub
parent 19e43c8508
commit ddc07c53a5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
60 changed files with 850 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

8
packages/kbn-io-ts-utils/index.ts
View file

@ -32,3 +32,11 @@ export {
export { datemathStringRt } from './src/datemath_string_rt';
export { createPlainError, decodeOrThrow, formatErrors, throwErrors } from './src/decode_or_throw';
export {
DateFromStringOrNumber,
minimalTimeKeyRT,
type MinimalTimeKey,
type TimeKey,
type UniqueTimeKey,
} from './src/time_key_rt';

53
packages/kbn-io-ts-utils/src/time_key_rt/index.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import * as rt from 'io-ts';
import moment from 'moment';
import { pipe } from 'fp-ts/lib/pipeable';
import { chain } from 'fp-ts/lib/Either';
const NANO_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3,9}Z$/;
export const DateFromStringOrNumber = new rt.Type<string, number | string>(
'DateFromStringOrNumber',
(input): input is string => typeof input === 'string',
(input, context) => {
if (typeof input === 'string') {
return NANO_DATE_PATTERN.test(input) ? rt.success(input) : rt.failure(input, context);
}
return pipe(
rt.number.validate(input, context),
chain((timestamp) => {
const momentValue = moment(timestamp);
return momentValue.isValid()
? rt.success(momentValue.toISOString())
: rt.failure(timestamp, context);
})
);
},
String
);
export const minimalTimeKeyRT = rt.type({
time: DateFromStringOrNumber,
tiebreaker: rt.number,
});
export type MinimalTimeKey = rt.TypeOf<typeof minimalTimeKeyRT>;
const timeKeyRT = rt.intersection([
minimalTimeKeyRT,
rt.partial({
gid: rt.string,
fromAutoReload: rt.boolean,
}),
]);
export type TimeKey = rt.TypeOf<typeof timeKeyRT>;
export interface UniqueTimeKey extends TimeKey {
gid: string;
}

2
x-pack/plugins/infra/common/locators/locators.test.ts
View file

@ -233,7 +233,7 @@ const constructLogView = (logView?: LogViewReference) => {
};
const constructLogPosition = (time: number = 1550671089404) => {
return `(position:(tiebreaker:0,time:${time}))`;
return `(position:(tiebreaker:0,time:'${moment(time).toISOString()}'))`;
};
const constructLogFilter = ({

3
x-pack/plugins/infra/common/time/time_key.ts
View file

@ -5,12 +5,13 @@
* 2.0.
*/
import { DateFromStringOrNumber } from '@kbn/io-ts-utils';
import { ascending, bisector } from 'd3-array';
import * as rt from 'io-ts';
import { pick } from 'lodash';
export const minimalTimeKeyRT = rt.type({
time: rt.number,
time: DateFromStringOrNumber,
tiebreaker: rt.number,
});
export type MinimalTimeKey = rt.TypeOf<typeof minimalTimeKeyRT>;

6
x-pack/plugins/infra/common/url_state_storage_service.ts
View file

@ -24,18 +24,18 @@ export const defaultLogViewKey = 'logView';
const encodeRisonUrlState = (state: any) => encode(state);
// Used by linkTo components
// Used by Locator components
export const replaceLogPositionInQueryString = (time?: number) =>
Number.isNaN(time) || time == null
? (value: string) => value
: replaceStateKeyInQueryString<PositionStateInUrl>(defaultPositionStateKey, {
position: {
time,
time: moment(time).toISOString(),
tiebreaker: 0,
},
});
// NOTE: Used by link-to components
// NOTE: Used by Locator components
export const replaceLogViewInQueryString = (logViewReference: LogViewReference) =>
replaceStateKeyInQueryString(defaultLogViewKey, logViewReference);

7
x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx
View file

@ -12,6 +12,7 @@ import {
LogEntryTime,
} from '@kbn/logs-shared-plugin/common';
import { scaleLinear } from 'd3-scale';
import moment from 'moment';
import * as React from 'react';
import { DensityChart } from './density_chart';
import { HighlightedInterval } from './highlighted_interval';
@ -67,7 +68,7 @@ export class LogMinimap extends React.Component<LogMinimapProps, LogMinimapState
this.props.jumpToTarget({
tiebreaker: 0,
time: clickedTime,
time: moment(clickedTime).toISOString(),
});
};
@ -142,9 +143,9 @@ export class LogMinimap extends React.Component<LogMinimapProps, LogMinimapState
{highlightedInterval ? (
<HighlightedInterval
end={highlightedInterval.end}
end={moment(highlightedInterval.end).valueOf()}
getPositionOfTime={this.getPositionOfTime}
start={highlightedInterval.start}
start={moment(highlightedInterval.start).valueOf()}
targetWidth={TIMERULER_WIDTH}
width={width}
target={target}

2
x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts
View file

@ -146,7 +146,7 @@ const validateSetupIndices = async (
fields: [
{
name: timestampField,
validTypes: ['date'],
validTypes: ['date', 'date_nanos'],
},
{
name: partitionField,

2
x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts
View file

@ -138,7 +138,7 @@ const validateSetupIndices = async (
fields: [
{
name: timestampField,
validTypes: ['date'],
validTypes: ['date', 'date_nanos'],
},
{
name: partitionField,

2
x-pack/plugins/infra/public/observability_logs/log_stream_page/state/src/types.ts
View file

@ -109,7 +109,7 @@ export type LogStreamPageContext = LogStreamPageTypestate['context'];
export interface LogStreamPageCallbacks {
updateTimeRange: (timeRange: Partial<TimeRange>) => void;
jumpToTargetPosition: (targetPosition: TimeKey | null) => void;
jumpToTargetPositionTime: (time: number) => void;
jumpToTargetPositionTime: (time: string) => void;
reportVisiblePositions: (visiblePositions: VisiblePositions) => void;
startLiveStreaming: () => void;
stopLiveStreaming: () => void;

16
x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts
View file

@ -7,6 +7,8 @@
import { IToasts } from '@kbn/core-notifications-browser';
import { IKbnUrlStateStorage } from '@kbn/kibana-utils-plugin/public';
import { convertISODateToNanoPrecision } from '@kbn/logs-shared-plugin/common';
import moment from 'moment';
import { actions, ActorRefFrom, createMachine, EmittedFrom, SpecialTargets } from 'xstate';
import { isSameTimeKey } from '../../../../common/time';
import { OmitDeprecatedState, sendIfDefined } from '../../xstate_helpers';
@ -159,11 +161,19 @@ export const createPureLogStreamPositionStateMachine = (initialContext: LogStrea
updatePositionsFromTimeChange: actions.assign((_context, event) => {
if (!('timeRange' in event)) return {};
const {
timestamps: { startTimestamp, endTimestamp },
} = event;
// Reset the target position if it doesn't fall within the new range.
const targetPositionNanoTime =
_context.targetPosition && convertISODateToNanoPrecision(_context.targetPosition.time);
const startNanoDate = convertISODateToNanoPrecision(moment(startTimestamp).toISOString());
const endNanoDate = convertISODateToNanoPrecision(moment(endTimestamp).toISOString());
const targetPositionShouldReset =
_context.targetPosition &&
(event.timestamps.startTimestamp > _context.targetPosition.time ||
event.timestamps.endTimestamp < _context.targetPosition.time);
targetPositionNanoTime &&
(startNanoDate > targetPositionNanoTime || endNanoDate < targetPositionNanoTime);
return {
targetPosition: targetPositionShouldReset ? null : _context.targetPosition,

11
x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts
View file

@ -10,7 +10,6 @@ import { IKbnUrlStateStorage, withNotifyOnErrors } from '@kbn/kibana-utils-plugi
import * as Either from 'fp-ts/lib/Either';
import { pipe } from 'fp-ts/lib/function';
import { InvokeCreator } from 'xstate';
import { replaceStateKeyInQueryString } from '../../../../common/url_state_storage_service';
import { minimalTimeKeyRT, pickTimeKey } from '../../../../common/time';
import { createPlainError, formatErrors } from '../../../../common/runtime_types';
import type { LogStreamPositionContext, LogStreamPositionEvent } from './types';
@ -98,13 +97,3 @@ export type PositionStateInUrl = rt.TypeOf<typeof positionStateInUrlRT>;
const decodePositionQueryValueFromUrl = (queryValueFromUrl: unknown) => {
return positionStateInUrlRT.decode(queryValueFromUrl);
};
export const replaceLogPositionInQueryString = (time?: number) =>
Number.isNaN(time) || time == null
? (value: string) => value
: replaceStateKeyInQueryString<PositionStateInUrl>(defaultPositionStateKey, {
position: {
time,
tiebreaker: 0,
},
});

5
x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts
View file

@ -17,6 +17,7 @@ import {
defaultPositionStateKey,
DEFAULT_REFRESH_INTERVAL,
} from '@kbn/logs-shared-plugin/common';
import moment from 'moment';
import {
getTimeRangeEndFromTime,
getTimeRangeStartFromTime,
@ -159,8 +160,8 @@ export const initializeFromUrl =
Either.chain(({ position }) =>
position && position.time
? Either.right({
from: getTimeRangeStartFromTime(position.time),
to: getTimeRangeEndFromTime(position.time),
from: getTimeRangeStartFromTime(moment(position.time).valueOf()),
to: getTimeRangeEndFromTime(moment(position.time).valueOf()),
})
: Either.left(null)
)

7
x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx
View file

@ -58,7 +58,7 @@ export const CategoryExampleMessage: React.FunctionComponent<{
search: {
logPosition: encode({
end: moment(timeRange.endTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
position: { tiebreaker, time: timestamp },
position: { tiebreaker, time: moment(timestamp).toISOString() },
start: moment(timeRange.startTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
streamLive: false,
}),
@ -128,7 +128,10 @@ export const CategoryExampleMessage: React.FunctionComponent<{
id,
index: '', // TODO: use real index when loading via async search
context,
cursor: { time: timestamp, tiebreaker },
cursor: {
time: moment(timestamp).toISOString(),
tiebreaker,
},
columns: [],
};
trackMetric({ metric: 'view_in_context__categories' });

2
x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx
View file

@ -102,7 +102,7 @@ export const LogEntryExampleMessage: React.FunctionComponent<Props> = ({
search: {
logPosition: encode({
end: moment(timeRange.endTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
position: { tiebreaker, time: timestamp },
position: { tiebreaker, time: moment(timestamp).toISOString() },
start: moment(timeRange.startTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
streamLive: false,
}),

2
x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx
View file

@ -38,7 +38,7 @@ export const ConnectedStreamPageContent: React.FC = () => {
jumpToTargetPosition: (targetPosition: TimeKey | null) => {
logStreamPageSend({ type: 'JUMP_TO_TARGET_POSITION', targetPosition });
},
jumpToTargetPositionTime: (time: number) => {
jumpToTargetPositionTime: (time: string) => {
logStreamPageSend({ type: 'JUMP_TO_TARGET_POSITION', targetPosition: { time } });
},
reportVisiblePositions: (visiblePositions: VisiblePositions) => {

10
x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx
View file

@ -8,7 +8,7 @@
import { EuiSpacer } from '@elastic/eui';
import type { Query } from '@kbn/es-query';
import { euiStyled } from '@kbn/kibana-react-plugin/common';
import { LogEntry } from '@kbn/logs-shared-plugin/common';
import { LogEntry, convertISODateToNanoPrecision } from '@kbn/logs-shared-plugin/common';
import {
LogEntryFlyout,
LogEntryStreamItem,
@ -117,8 +117,12 @@ export const StreamPageLogsContent = React.memo<{
const isCenterPointOutsideLoadedRange =
targetPosition != null &&
((topCursor != null && targetPosition.time < topCursor.time) ||
(bottomCursor != null && targetPosition.time > bottomCursor.time));
((topCursor != null &&
convertISODateToNanoPrecision(targetPosition.time) <
convertISODateToNanoPrecision(topCursor.time)) ||
(bottomCursor != null &&
convertISODateToNanoPrecision(targetPosition.time) >
convertISODateToNanoPrecision(bottomCursor.time)));
const hasQueryChanged = filterQuery !== prevFilterQuery;

6
x-pack/plugins/infra/public/test_utils/entries.ts
View file

@ -18,14 +18,16 @@ export function generateFakeEntries(
const timestampStep = Math.floor((endTimestamp - startTimestamp) / count);
for (let i = 0; i < count; i++) {
const timestamp = i === count - 1 ? endTimestamp : startTimestamp + timestampStep * i;
const date = new Date(timestamp).toISOString();
entries.push({
id: `entry-${i}`,
index: 'logs-fake',
context: {},
cursor: { time: timestamp, tiebreaker: i },
cursor: { time: date, tiebreaker: i },
columns: columns.map((column) => {
if ('timestampColumn' in column) {
return { columnId: column.timestampColumn.id, timestamp };
return { columnId: column.timestampColumn.id, time: date };
} else if ('messageColumn' in column) {
return {
columnId: column.messageColumn.id,

2
x-pack/plugins/logs_shared/common/index.ts
View file

@ -44,6 +44,8 @@ export {
// eslint-disable-next-line @kbn/eslint/no_export_all
export * from './log_entry';
export { convertISODateToNanoPrecision } from './utils';
// Http types
export type { LogEntriesSummaryBucket, LogEntriesSummaryHighlightsBucket } from './http_api';

4
x-pack/plugins/logs_shared/common/log_entry/log_entry.ts
View file

@ -5,8 +5,8 @@
* 2.0.
*/
import { TimeKey } from '@kbn/io-ts-utils';
import * as rt from 'io-ts';
import { TimeKey } from '../time';
import { jsonArrayRT } from '../typed_json';
import { logEntryCursorRT } from './log_entry_cursor';
@ -34,7 +34,7 @@ export type LogMessagePart = rt.TypeOf<typeof logMessagePartRT>;
* columns
*/
export const logTimestampColumnRT = rt.type({ columnId: rt.string, timestamp: rt.number });
export const logTimestampColumnRT = rt.type({ columnId: rt.string, time: rt.string });
export type LogTimestampColumn = rt.TypeOf<typeof logTimestampColumnRT>;
export const logFieldColumnRT = rt.type({

4
x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts
View file

@ -9,7 +9,7 @@ import * as rt from 'io-ts';
import { decodeOrThrow } from '../runtime_types';
export const logEntryCursorRT = rt.type({
time: rt.number,
time: rt.string,
tiebreaker: rt.number,
});
export type LogEntryCursor = rt.TypeOf<typeof logEntryCursorRT>;
@ -29,7 +29,7 @@ export const logEntryAroundCursorRT = rt.type({
});
export type LogEntryAroundCursor = rt.TypeOf<typeof logEntryAroundCursorRT>;
export const getLogEntryCursorFromHit = (hit: { sort: [number, number] }) =>
export const getLogEntryCursorFromHit = (hit: { sort: [string, number] }) =>
decodeOrThrow(logEntryCursorRT)({
time: hit.sort[0],
tiebreaker: hit.sort[1],

20
x-pack/plugins/logs_shared/common/time/time_key.ts
View file

@ -5,26 +5,8 @@
* 2.0.
*/
import type { TimeKey } from '@kbn/io-ts-utils';
import { ascending, bisector } from 'd3-array';
import * as rt from 'io-ts';
export const minimalTimeKeyRT = rt.type({
time: rt.number,
tiebreaker: rt.number,
});
export const timeKeyRT = rt.intersection([
minimalTimeKeyRT,
rt.partial({
gid: rt.string,
fromAutoReload: rt.boolean,
}),
]);
export type TimeKey = rt.TypeOf<typeof timeKeyRT>;
export interface UniqueTimeKey extends TimeKey {
gid: string;
}
export type Comparator = (firstValue: any, secondValue: any) => number;

45
x-pack/plugins/logs_shared/common/utils/date_helpers.test.ts Normal file
View file

@ -0,0 +1,45 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { subtractMillisecondsFromDate } from './date_helpers';
describe('Date Helpers', function () {
describe('subtractMillisecondsFromDate', function () {
it('should subtract milliseconds from the nano date correctly', () => {
const inputDate = '2023-10-30T12:00:00.001000000Z';
const millisecondsToSubtract = 1;
const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract);
const expectedDate = '2023-10-30T12:00:00.000000000Z';
expect(result).toBe(expectedDate);
});
it('should subtract seconds from the date if no milliseconds available', () => {
const inputDate = '2023-10-30T12:00:00.000000000Z';
const millisecondsToSubtract = 1;
const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract);
const expectedDate = '2023-10-30T11:59:59.999000000Z';
expect(result).toBe(expectedDate);
});
it('should convert date to nano and subtract milliseconds properly', () => {
const inputDate = '2023-10-30T12:00:00.000Z';
const millisecondsToSubtract = 1;
const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract);
const expectedDate = '2023-10-30T11:59:59.999000000Z';
expect(result).toBe(expectedDate);
});
});
});

29
x-pack/plugins/logs_shared/common/utils/date_helpers.ts Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import dateMath from '@kbn/datemath';
export function convertISODateToNanoPrecision(date: string): string {
const dateParts = date.split('.');
const fractionSeconds = dateParts.length === 2 ? dateParts[1].replace('Z', '') : '';
const fractionSecondsInNanos =
fractionSeconds.length !== 9 ? fractionSeconds.padEnd(9, '0') : fractionSeconds;
return `${dateParts[0]}.${fractionSecondsInNanos}Z`;
}
export function subtractMillisecondsFromDate(date: string, milliseconds: number): string {
const dateInNano = convertISODateToNanoPrecision(date);
const dateParts = dateInNano.split('.');
const nanoPart = dateParts[1].substring(3, dateParts[1].length); // given 123456789Z => 456789Z
const isoDate = dateMath.parse(date)?.subtract(milliseconds, 'ms').toISOString();
return `${isoDate?.replace('Z', nanoPart)}`;
}

8
x-pack/plugins/logs_shared/common/utils/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export * from './date_helpers';

4
x-pack/plugins/logs_shared/public/components/formatted_time.tsx
View file

@ -11,7 +11,7 @@ import { useMemo } from 'react';
import { useKibanaUiSetting } from '../utils/use_kibana_ui_setting';
const getFormattedTime = (
time: number,
time: string,
userFormat: string | undefined,
fallbackFormat: string = 'Y-MM-DD HH:mm:ss.SSS'
) => {
@ -26,7 +26,7 @@ interface UseFormattedTimeOptions {
}
export const useFormattedTime = (
time: number,
time: string,
{ format = 'dateTime', fallbackFormat }: UseFormattedTimeOptions = {}
) => {
// `dateFormat:scaled` is an array of `[key, format]` tuples.

2
x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx
View file

@ -38,7 +38,7 @@ export const BasicDateRange = LogStreamStoryTemplate.bind({});
export const CenteredOnLogEntry = LogStreamStoryTemplate.bind({});
CenteredOnLogEntry.args = {
center: { time: 1595146275000, tiebreaker: 150 },
center: { time: '2020-07-19T08:11:15.000Z', tiebreaker: 150 },
};
export const HighlightedLogEntry = LogStreamStoryTemplate.bind({});

2
x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx
View file

@ -40,7 +40,7 @@ interface CommonColumnDefinition {
interface TimestampColumnDefinition extends CommonColumnDefinition {
type: 'timestamp';
/** Timestamp renderer. Takes a epoch_millis and returns a valid `ReactNode` */
render?: (timestamp: number) => React.ReactNode;
render?: (timestamp: string) => React.ReactNode;
}
interface MessageColumnDefinition extends CommonColumnDefinition {

17
x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx
View file

@ -24,6 +24,7 @@ const ProviderWrapper: React.FC = ({ children }) => {
};
describe('LogEntryActionsMenu component', () => {
const time = new Date().toISOString();
describe('uptime link', () => {
it('renders with a host ip filter when present in log entry', () => {
const elementWrapper = mount(
@ -34,7 +35,7 @@ describe('LogEntryActionsMenu component', () => {
id: 'ITEM_ID',
index: 'INDEX',
cursor: {
time: 0,
time,
tiebreaker: 0,
},
}}
@ -64,7 +65,7 @@ describe('LogEntryActionsMenu component', () => {
id: 'ITEM_ID',
index: 'INDEX',
cursor: {
time: 0,
time,
tiebreaker: 0,
},
}}
@ -94,7 +95,7 @@ describe('LogEntryActionsMenu component', () => {
id: 'ITEM_ID',
index: 'INDEX',
cursor: {
time: 0,
time,
tiebreaker: 0,
},
}}
@ -128,7 +129,7 @@ describe('LogEntryActionsMenu component', () => {
id: 'ITEM_ID',
index: 'INDEX',
cursor: {
time: 0,
time,
tiebreaker: 0,
},
}}
@ -160,7 +161,7 @@ describe('LogEntryActionsMenu component', () => {
id: 'ITEM_ID',
index: 'INDEX',
cursor: {
time: 0,
time,
tiebreaker: 0,
},
}}
@ -194,7 +195,7 @@ describe('LogEntryActionsMenu component', () => {
id: 'ITEM_ID',
index: 'INDEX',
cursor: {
time: 0,
time,
tiebreaker: 0,
},
}}
@ -228,7 +229,7 @@ describe('LogEntryActionsMenu component', () => {
id: 'ITEM_ID',
index: 'INDEX',
cursor: {
time: 0,
time,
tiebreaker: 0,
},
}}
@ -258,7 +259,7 @@ describe('LogEntryActionsMenu component', () => {
id: 'ITEM_ID',
index: 'INDEX',
cursor: {
time: 0,
time,
tiebreaker: 0,
},
}}

2
x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx
View file

@ -10,9 +10,9 @@ import { EuiBasicTableColumn, EuiInMemoryTable } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import * as rt from 'io-ts';
import React, { useMemo } from 'react';
import { TimeKey } from '@kbn/io-ts-utils';
import { LogEntryField } from '../../../../common/log_entry';
import { LogEntry } from '../../../../common/search_strategies/log_entries/log_entry';
import { TimeKey } from '../../../../common/time';
import { JsonScalar, jsonScalarRT } from '../../../../common/typed_json';
import { FieldValue } from '../log_text_stream/field_value';

2
x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx
View file

@ -21,9 +21,9 @@ import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import { createKibanaReactContext } from '@kbn/kibana-react-plugin/public';
import React, { useCallback, useEffect, useRef } from 'react';
import { TimeKey } from '@kbn/io-ts-utils';
import { useKibanaContextForPlugin } from '../../../hooks/use_kibana';
import { LogViewReference } from '../../../../common/log_views';
import { TimeKey } from '../../../../common/time';
import { useLogEntry } from '../../../containers/logs/log_entry';
import { CenteredEuiFlyoutBody } from '../../centered_flyout_body';
import { DataSearchErrorCallout } from '../../data_search_error_callout';

2
x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx
View file

@ -41,7 +41,7 @@ export const LogColumnHeaders: React.FunctionComponent<{
columnHeader = columnConfiguration.timestampColumn.header;
} else {
columnHeader = firstVisiblePosition
? localizedDate(firstVisiblePosition.time)
? localizedDate(new Date(firstVisiblePosition.time))
: i18n.translate('xpack.logsShared.logs.stream.timestampColumnTitle', {
defaultMessage: 'Timestamp',
});

9
x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts
View file

@ -5,9 +5,10 @@
* 2.0.
*/
import type { TimeKey } from '@kbn/io-ts-utils';
import { bisector } from 'd3-array';
import { compareToTimeKey, TimeKey } from '../../../../common/time';
import { LogEntry } from '../../../../common/log_entry';
import { compareToTimeKey } from '../../../../common/time';
export type StreamItem = LogEntryStreamItem;
@ -27,17 +28,17 @@ export function getStreamItemTimeKey(item: StreamItem) {
export function getStreamItemId(item: StreamItem) {
switch (item.kind) {
case 'logEntry':
return `${item.logEntry.cursor.time}:${item.logEntry.cursor.tiebreaker}:${item.logEntry.id}`;
return `${item.logEntry.cursor.time}/${item.logEntry.cursor.tiebreaker}/${item.logEntry.id}`;
}
}
export function parseStreamItemId(id: string) {
const idFragments = id.split(':');
const idFragments = id.split('/');
return {
gid: idFragments.slice(2).join(':'),
tiebreaker: parseInt(idFragments[1], 10),
time: parseInt(idFragments[0], 10),
time: idFragments[0],
};
}

6
x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx
View file

@ -10,14 +10,14 @@ import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule, EuiTitle } from '@elastic
import { localizedDate } from '../../../../common/formatters/datetime';
interface LogDateRowProps {
timestamp: number;
time: string;
}
/**
* Show a row with the date in the log stream
*/
export const LogDateRow: React.FC<LogDateRowProps> = ({ timestamp }) => {
const formattedDate = localizedDate(timestamp);
export const LogDateRow: React.FC<LogDateRowProps> = ({ time }) => {
const formattedDate = localizedDate(new Date(time));
return (
<EuiFlexGroup alignItems="center" gutterSize="s">

3
x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx
View file

@ -8,6 +8,7 @@
import { useMemo } from 'react';
import { euiStyled } from '@kbn/kibana-react-plugin/common';
import moment from 'moment';
import { TextScale } from '../../../../common/log_text_scale';
import {
LogColumnRenderConfiguration,
@ -142,7 +143,7 @@ export const useColumnWidths = ({
timeFormat?: TimeFormat;
}) => {
const { CharacterDimensionsProbe, dimensions } = useMeasuredCharacterDimensions(scale);
const referenceTime = useMemo(() => Date.now(), []);
const referenceTime = useMemo(() => moment().toISOString(), []);
const formattedCurrentDate = useFormattedTime(referenceTime, { format: timeFormat });
const columnWidths = useMemo(
() => getColumnWidths(columnConfigurations, dimensions.width, formattedCurrentDate.length),

2
x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx
View file

@ -189,7 +189,7 @@ export const LogEntryRow = memo(
>
{isTimestampColumn(column) ? (
<LogEntryTimestampColumn
time={column.timestamp}
time={column.time}
render={columnConfiguration.timestampColumn.render}
/>
) : null}

4
x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx
View file

@ -13,8 +13,8 @@ import { LogEntryColumnContent } from './log_entry_column';
interface LogEntryTimestampColumnProps {
format?: TimeFormat;
time: number;
render?: (timestamp: number) => React.ReactNode;
time: string;
render?: (time: string) => React.ReactNode;
}
export const LogEntryTimestampColumn = memo<LogEntryTimestampColumnProps>(

17
x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx
View file

@ -9,10 +9,9 @@ import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import React, { Fragment, GetDerivedStateFromProps } from 'react';
import moment from 'moment';
import { euiStyled } from '@kbn/kibana-react-plugin/common';
import { TimeKey, UniqueTimeKey } from '@kbn/io-ts-utils';
import { TextScale } from '../../../../common/log_text_scale';
import { TimeKey, UniqueTimeKey } from '../../../../common/time';
import { callWithoutRepeats } from '../../../utils/handlers';
import { AutoSizer } from '../../auto_sizer';
import { NoData } from '../../empty_states';
@ -216,7 +215,7 @@ export class ScrollableLogTextStreamView extends React.PureComponent<
position="start"
isLoading={isLoadingMore}
hasMore={hasMoreBeforeStart}
timestamp={items[0].logEntry.cursor.time}
timestamp={moment(items[0].logEntry.cursor.time).valueOf()}
isStreaming={false}
startDateExpression={startDateExpression}
endDateExpression={endDateExpression}
@ -225,17 +224,17 @@ export class ScrollableLogTextStreamView extends React.PureComponent<
}
/>
{items.map((item, idx) => {
const currentTimestamp = item.logEntry.cursor.time;
const currentTime = item.logEntry.cursor.time;
let showDate = false;
if (idx > 0) {
const prevTimestamp = items[idx - 1].logEntry.cursor.time;
showDate = !moment(currentTimestamp).isSame(prevTimestamp, 'day');
const prevTime = items[idx - 1].logEntry.cursor.time;
showDate = !moment(currentTime).isSame(prevTime, 'day');
}
return (
<Fragment key={getStreamItemId(item)}>
{showDate && <LogDateRow timestamp={currentTimestamp} />}
{showDate && <LogDateRow time={currentTime} />}
<MeasurableItemView
register={registerChild}
registrationKey={getStreamItemId(item)}
@ -279,8 +278,8 @@ export class ScrollableLogTextStreamView extends React.PureComponent<
isStreaming={isStreaming}
timestamp={
isStreaming && lastLoadedTime
? lastLoadedTime.valueOf()
: items[items.length - 1].logEntry.cursor.time
? lastLoadedTime.getTime()
: moment(items[items.length - 1].logEntry.cursor.time).valueOf()
}
startDateExpression={startDateExpression}
endDateExpression={endDateExpression}

2
x-pack/plugins/logs_shared/public/containers/logs/log_highlights/log_entry_highlights.tsx
View file

@ -5,11 +5,11 @@
* 2.0.
*/
import { TimeKey } from '@kbn/io-ts-utils';
import { useEffect, useMemo, useState } from 'react';
import { LogViewReference } from '../../../../common';
import { LogEntriesHighlightsResponse } from '../../../../common/http_api';
import { LogEntry } from '../../../../common/log_entry';
import { TimeKey } from '../../../../common/time';
import { useKibanaContextForPlugin } from '../../../hooks/use_kibana';
import { useTrackedPromise } from '../../../utils/use_tracked_promise';
import { fetchLogEntriesHighlights } from './api/fetch_log_entries_highlights';

3
x-pack/plugins/logs_shared/public/containers/logs/log_highlights/log_highlights.tsx
View file

@ -8,12 +8,11 @@
import createContainer from 'constate';
import { useState } from 'react';
import useThrottle from 'react-use/lib/useThrottle';
import { TimeKey } from '@kbn/io-ts-utils';
import { LogViewReference } from '../../../../common';
import { useLogEntryHighlights } from './log_entry_highlights';
import { useLogSummaryHighlights } from './log_summary_highlights';
import { useNextAndPrevious } from './next_and_previous';
import { TimeKey } from '../../../../common/time';
import { useLogPositionStateContext } from '../log_position';
const FETCH_THROTTLE_INTERVAL = 3000;

3
x-pack/plugins/logs_shared/public/containers/logs/log_highlights/next_and_previous.tsx
View file

@ -7,8 +7,7 @@
import { isNumber } from 'lodash';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { TimeKey, UniqueTimeKey } from '../../../../common/time';
import { TimeKey, UniqueTimeKey } from '@kbn/io-ts-utils';
import {
getLogEntryIndexAtTime,
getLogEntryIndexBeforeTime,

6
x-pack/plugins/logs_shared/public/containers/logs/log_position/use_log_position.ts
View file

@ -6,10 +6,10 @@
*/
import { TimeRange } from '@kbn/es-query';
import { TimeKey } from '@kbn/io-ts-utils';
import createContainer from 'constate';
import { useMemo } from 'react';
import { ActorRefWithDeprecatedState } from 'xstate';
import { TimeKey } from '../../../../common/time';
import {
MatchedStateFromActor,
OmitDeprecatedState,
@ -52,7 +52,7 @@ export type LogPositionStateParams = DateRange & {
export interface LogPositionCallbacks {
jumpToTargetPosition: (pos: TimeKeyOrNull) => void;
jumpToTargetPositionTime: (time: number) => void;
jumpToTargetPositionTime: (time: string) => void;
reportVisiblePositions: (visPos: VisiblePositions) => void;
startLiveStreaming: () => void;
stopLiveStreaming: () => void;
@ -62,7 +62,7 @@ export interface LogPositionCallbacks {
export interface LogStreamPageCallbacks {
updateTimeRange: (timeRange: Partial<TimeRange>) => void;
jumpToTargetPosition: (targetPosition: TimeKey | null) => void;
jumpToTargetPositionTime: (time: number) => void;
jumpToTargetPositionTime: (time: string) => void;
reportVisiblePositions: (visiblePositions: VisiblePositions) => void;
startLiveStreaming: () => void;
stopLiveStreaming: () => void;

3
x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts
View file

@ -8,6 +8,7 @@
import { useCallback } from 'react';
import { combineLatest, Observable, ReplaySubject } from 'rxjs';
import { last, map, startWith, switchMap } from 'rxjs/operators';
import { subtractMillisecondsFromDate } from '../../../../common/utils';
import { LogEntryCursor } from '../../../../common/log_entry';
import { LogViewColumnConfiguration, LogViewReference } from '../../../../common/log_views';
import { LogEntriesSearchRequestQuery } from '../../../../common/search_strategies/log_entries/log_entries';
@ -73,7 +74,7 @@ export const useFetchLogEntriesAround = ({
last(), // in the future we could start earlier if we receive partial results already
map((lastBeforeSearchResponse) => {
const cursorAfter = lastBeforeSearchResponse.response.data?.bottomCursor ?? {
time: cursor.time - 1,
time: subtractMillisecondsFromDate(cursor.time, 1),
tiebreaker: 0,
};

6
x-pack/plugins/logs_shared/public/test_utils/entries.ts
View file

@ -27,14 +27,16 @@ export function generateFakeEntries(
const timestampStep = Math.floor((endTimestamp - startTimestamp) / count);
for (let i = 0; i < count; i++) {
const timestamp = i === count - 1 ? endTimestamp : startTimestamp + timestampStep * i;
const date = new Date(timestamp).toISOString();
entries.push({
id: `entry-${i}`,
index: 'logs-fake',
context: {},
cursor: { time: timestamp, tiebreaker: i },
cursor: { time: date, tiebreaker: i },
columns: columns.map((column) => {
if ('timestampColumn' in column) {
return { columnId: column.timestampColumn.id, timestamp };
return { columnId: column.timestampColumn.id, time: date };
} else if ('messageColumn' in column) {
return {
columnId: column.messageColumn.id,

2
x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx
View file

@ -19,7 +19,7 @@ interface CommonRenderConfiguration {
interface TimestampColumnRenderConfiguration {
timestampColumn: CommonRenderConfiguration & {
render?: (timestamp: number) => ReactNode;
render?: (time: string) => ReactNode;
};
}

5
x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts
View file

@ -5,8 +5,9 @@
* 2.0.
*/
import type { TimeKey, UniqueTimeKey } from '@kbn/io-ts-utils';
import { bisector } from 'd3-array';
import { compareToTimeKey, getIndexAtTimeKey, TimeKey, UniqueTimeKey } from '../../../common/time';
import { compareToTimeKey, getIndexAtTimeKey } from '../../../common/time';
import {
LogEntry,
LogColumn,
@ -38,7 +39,7 @@ export const getLogEntryAtTime = (entries: LogEntry[], time: TimeKey) => {
};
export const isTimestampColumn = (column: LogColumn): column is LogTimestampColumn =>
column != null && 'timestamp' in column;
column != null && 'time' in column;
export const isMessageColumn = (column: LogColumn): column is LogMessageColumn =>
column != null && 'message' in column;

21
x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts
View file

@ -62,7 +62,11 @@ export class LogsSharedKibanaLogEntriesAdapter implements LogEntriesAdapter {
: {};
const sort = {
[TIMESTAMP_FIELD]: sortDirection,
[TIMESTAMP_FIELD]: {
order: sortDirection,
format: 'strict_date_optional_time_nanos',
numeric_type: 'date_nanos',
},
[TIEBREAKER_FIELD]: sortDirection,
};
@ -155,7 +159,16 @@ export class LogsSharedKibanaLogEntriesAdapter implements LogEntriesAdapter {
top_hits_by_key: {
top_hits: {
size: 1,
sort: [{ [TIMESTAMP_FIELD]: 'asc' }, { [TIEBREAKER_FIELD]: 'asc' }],
sort: [
{
[TIMESTAMP_FIELD]: {
order: 'asc',
format: 'strict_date_optional_time_nanos',
numeric_type: 'date_nanos',
},
},
{ [TIEBREAKER_FIELD]: 'asc' },
],
_source: false,
},
},
@ -265,7 +278,7 @@ const createQueryFilterClauses = (filterQuery: LogEntryQuery | undefined) =>
function processCursor(cursor: LogEntriesParams['cursor']): {
sortDirection: 'asc' | 'desc';
searchAfterClause: { search_after?: readonly [number, number] };
searchAfterClause: { search_after?: readonly [string, number] };
} {
if (cursor) {
if ('before' in cursor) {
@ -295,7 +308,7 @@ const LogSummaryDateRangeBucketRuntimeType = runtimeTypes.intersection([
hits: runtimeTypes.type({
hits: runtimeTypes.array(
runtimeTypes.type({
sort: runtimeTypes.tuple([runtimeTypes.number, runtimeTypes.number]),
sort: runtimeTypes.tuple([runtimeTypes.string, runtimeTypes.number]),
})
),
}),

5
x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts
View file

@ -7,6 +7,7 @@
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { JsonObject } from '@kbn/utility-types';
import { subtractMillisecondsFromDate } from '../../../../common/utils';
import {
LogEntriesSummaryBucket,
LogEntriesSummaryHighlightsBucket,
@ -147,7 +148,7 @@ export class LogsSharedLogEntriesDomain implements ILogsSharedLogEntriesDomain {
const cursorAfter =
entriesBefore.length > 0
? entriesBefore[entriesBefore.length - 1].cursor
: { time: center.time - 1, tiebreaker: 0 };
: { time: subtractMillisecondsFromDate(center.time, 1), tiebreaker: 0 };
const { entries: entriesAfter, hasMoreAfter } = await this.getLogEntries(
requestContext,
@ -200,7 +201,7 @@ export class LogsSharedLogEntriesDomain implements ILogsSharedLogEntriesDomain {
if ('timestampColumn' in column) {
return {
columnId: column.timestampColumn.id,
timestamp: doc.cursor.time,
time: doc.cursor.time,
};
} else if ('messageColumn' in column) {
return {

9
x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts
View file

@ -95,6 +95,7 @@ describe('LogEntries search strategy', () => {
});
it('handles subsequent polling requests', async () => {
const date = new Date(1605116827143).toISOString();
const esSearchStrategyMock = createEsSearchStrategyMock({
id: 'ASYNC_REQUEST_ID',
isRunning: false,
@ -112,12 +113,12 @@ describe('LogEntries search strategy', () => {
_score: 0,
_source: null,
fields: {
'@timestamp': [1605116827143],
'@timestamp': [date],
'event.dataset': ['HIT_DATASET'],
message: ['HIT_MESSAGE'],
'container.id': ['HIT_CONTAINER_ID'],
},
sort: [1605116827143 as any, 1 as any], // incorrectly typed as string upstream
sort: [date as any, 1 as any], // incorrectly typed as string upstream
},
],
},
@ -164,13 +165,13 @@ describe('LogEntries search strategy', () => {
id: 'HIT_ID',
index: 'HIT_INDEX',
cursor: {
time: 1605116827143,
time: date,
tiebreaker: 1,
},
columns: [
{
columnId: 'TIMESTAMP_COLUMN_ID',
timestamp: 1605116827143,
time: date,
},
{
columnId: 'DATASET_COLUMN_ID',

2
x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts
View file

@ -190,7 +190,7 @@ const getLogEntryFromHit =
if ('timestampColumn' in column) {
return {
columnId: column.timestampColumn.id,
timestamp: cursor.time,
time: cursor.time,
};
} else if ('messageColumn' in column) {
return {

9
x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts
View file

@ -99,6 +99,7 @@ describe('LogEntry search strategy', () => {
});
it('handles subsequent polling requests', async () => {
const date = new Date(1605116827143).toISOString();
const esSearchStrategyMock = createEsSearchStrategyMock({
id: 'ASYNC_REQUEST_ID',
isRunning: false,
@ -116,10 +117,10 @@ describe('LogEntry search strategy', () => {
_score: 0,
_source: null,
fields: {
'@timestamp': [1605116827143],
'@timestamp': [date],
message: ['HIT_MESSAGE'],
},
sort: [1605116827143 as any, 1 as any], // incorrectly typed as string upstream
sort: [date as any, 1 as any], // incorrectly typed as string upstream
},
],
},
@ -163,11 +164,11 @@ describe('LogEntry search strategy', () => {
id: 'HIT_ID',
index: 'HIT_INDEX',
cursor: {
time: 1605116827143,
time: date,
tiebreaker: 1,
},
fields: [
{ field: '@timestamp', value: [1605116827143] },
{ field: '@timestamp', value: [date] },
{ field: 'message', value: ['HIT_MESSAGE'] },
],
});

6
x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts
View file

@ -11,7 +11,11 @@ export const createSortClause = (
tiebreakerField: string
) => ({
sort: {
[timestampField]: sortDirection,
[timestampField]: {
order: sortDirection,
format: 'strict_date_optional_time_nanos',
numeric_type: 'date_nanos',
},
[tiebreakerField]: sortDirection,
},
});

4
x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts
View file

@ -68,7 +68,7 @@ export const getSortDirection = (
const createSearchAfterClause = (
cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined
): { search_after?: [number, number] } => {
): { search_after?: [string, number] } => {
if (logEntryBeforeCursorRT.is(cursor) && cursor.before !== 'last') {
return {
search_after: [cursor.before.time, cursor.before.tiebreaker],
@ -122,7 +122,7 @@ const createHighlightQuery = (
export const logEntryHitRT = rt.intersection([
commonHitFieldsRT,
rt.type({
sort: rt.tuple([rt.number, rt.number]),
sort: rt.tuple([rt.string, rt.number]),
}),
rt.partial({
fields: rt.record(rt.string, jsonArrayRT),

13
x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts
View file

@ -33,7 +33,16 @@ export const createGetLogEntryQuery = (
},
fields: ['*'],
runtime_mappings: runtimeMappings,
sort: [{ [timestampField]: 'desc' }, { [tiebreakerField]: 'desc' }],
sort: [
{
[timestampField]: {
order: 'desc',
format: 'strict_date_optional_time_nanos',
numeric_type: 'date_nanos',
},
},
{ [tiebreakerField]: 'desc' },
],
_source: false,
},
});
@ -41,7 +50,7 @@ export const createGetLogEntryQuery = (
export const logEntryHitRT = rt.intersection([
commonHitFieldsRT,
rt.type({
sort: rt.tuple([rt.number, rt.number]),
sort: rt.tuple([rt.string, rt.number]),
}),
rt.partial({
fields: rt.record(rt.string, jsonArrayRT),

5
x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts
View file

@ -19,6 +19,7 @@ import {
logEntriesHighlightsResponseRT,
} from '@kbn/logs-shared-plugin/common';
import moment from 'moment';
import { FtrProviderContext } from '../../ftr_provider_context';
const KEY_BEFORE_START = {
@ -112,8 +113,8 @@ export default function ({ getService }: FtrProviderContext) {
// Entries fall within range
// @kbn/expect doesn't have a `lessOrEqualThan` or `moreOrEqualThan` comparators
expect(firstEntry.cursor.time >= KEY_BEFORE_START.time).to.be(true);
expect(lastEntry.cursor.time <= KEY_AFTER_END.time).to.be(true);
expect(firstEntry.cursor.time >= moment(KEY_BEFORE_START.time).toISOString()).to.be(true);
expect(lastEntry.cursor.time <= moment(KEY_AFTER_END.time).toISOString()).to.be(true);
// All entries contain the highlights
entries.forEach((entry) => {

1
x-pack/test/functional/apps/infra/index.ts
View file

@ -26,6 +26,7 @@ export default ({ loadTestFile }: FtrProviderContext) => {
loadTestFile(require.resolve('./log_entry_categories_tab'));
loadTestFile(require.resolve('./log_entry_rate_tab'));
loadTestFile(require.resolve('./logs_source_configuration'));
loadTestFile(require.resolve('./log_stream_date_nano'));
loadTestFile(require.resolve('./link_to'));
loadTestFile(require.resolve('./log_stream'));
});

7
x-pack/test/functional/apps/infra/link_to.ts
View file

@ -18,7 +18,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const retry = getService('retry');
const browser = getService('browser');
const timestamp = Date.now();
const date = new Date();
const timestamp = date.getTime();
const startDate = new Date(timestamp - ONE_HOUR).toISOString();
const endDate = new Date(timestamp + ONE_HOUR).toISOString();
@ -52,7 +53,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
`(query:(language:kuery,query:\'trace.id:${traceId}'),refreshInterval:(pause:!t,value:5000),timeRange:(from:'${startDate}',to:'${endDate}'))`
);
expect(parsedUrl.searchParams.get('logPosition')).to.be(
`(position:(tiebreaker:0,time:${timestamp}))`
`(position:(tiebreaker:0,time:'${date.toISOString()}'))`
);
expect(parsedUrl.searchParams.get('logView')).to.be(LOG_VIEW_REFERENCE);
expect(documentTitle).to.contain('Stream - Logs - Observability - Elastic');
@ -87,7 +88,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
`(query:(language:kuery,query:\'(kubernetes.pod.uid: 1234) and (trace.id:${traceId})\'),refreshInterval:(pause:!t,value:5000),timeRange:(from:'${startDate}',to:'${endDate}'))`
);
expect(parsedUrl.searchParams.get('logPosition')).to.be(
`(position:(tiebreaker:0,time:${timestamp}))`
`(position:(tiebreaker:0,time:'${date.toISOString()}'))`
);
expect(parsedUrl.searchParams.get('logView')).to.be(LOG_VIEW_REFERENCE);
expect(documentTitle).to.contain('Stream - Logs - Observability - Elastic');

98
x-pack/test/functional/apps/infra/log_stream_date_nano.ts Normal file
View file

@ -0,0 +1,98 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { URL } from 'url';
import { FtrProviderContext } from '../../ftr_provider_context';
import { DATES } from './constants';
export default ({ getPageObjects, getService }: FtrProviderContext) => {
const retry = getService('retry');
const browser = getService('browser');
const esArchiver = getService('esArchiver');
const logsUi = getService('logsUi');
const find = getService('find');
const logFilter = {
timeRange: {
from: DATES.metricsAndLogs.stream.startWithData,
to: DATES.metricsAndLogs.stream.endWithData,
},
};
describe('Log stream supports nano precision', function () {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/infra/logs_with_nano_date');
});
after(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/infra/logs_with_nano_date');
});
it('should display logs entries containing date_nano timestamps properly ', async () => {
await logsUi.logStreamPage.navigateTo({ logFilter });
const logStreamEntries = await logsUi.logStreamPage.getStreamEntries();
expect(logStreamEntries.length).to.be(4);
});
it('should render timestamp column properly', async () => {
await logsUi.logStreamPage.navigateTo({ logFilter });
await retry.try(async () => {
const columnHeaderLabels = await logsUi.logStreamPage.getColumnHeaderLabels();
expect(columnHeaderLabels[0]).to.eql('Oct 17, 2018');
});
});
it('should render timestamp column values properly', async () => {
await logsUi.logStreamPage.navigateTo({ logFilter });
const logStreamEntries = await logsUi.logStreamPage.getStreamEntries();
const firstLogStreamEntry = logStreamEntries[0];
const entryTimestamp = await logsUi.logStreamPage.getLogEntryColumnValueByName(
firstLogStreamEntry,
'timestampLogColumn'
);
expect(entryTimestamp).to.be('19:43:22.111');
});
it('should properly sync logPosition in url', async () => {
const currentUrl = await browser.getCurrentUrl();
const parsedUrl = new URL(currentUrl);
expect(parsedUrl.searchParams.get('logPosition')).to.be(
`(position:(tiebreaker:3,time:\'2018-10-17T19:46:22.333333333Z\'))`
);
});
it('should properly render timestamp in flyout with nano precision', async () => {
await logsUi.logStreamPage.navigateTo({ logFilter });
const logStreamEntries = await logsUi.logStreamPage.getStreamEntries();
const firstLogStreamEntry = logStreamEntries[0];
await logsUi.logStreamPage.openLogEntryDetailsFlyout(firstLogStreamEntry);
const cells = await find.allByCssSelector('.euiTableCellContent');
let isFound = false;
for (const cell of cells) {
const cellText = await cell.getVisibleText();
if (cellText === '2018-10-17T19:43:22.111111111Z') {
isFound = true;
return;
}
}
expect(isFound).to.be(true);
});
});
};

BIN
x-pack/test/functional/es_archives/infra/logs_with_nano_date/data.json.gz Normal file
View file

Binary file not shown.

419
x-pack/test/functional/es_archives/infra/logs_with_nano_date/mappings.json Normal file
View file

@ -0,0 +1,419 @@
{
"type": "data_stream",
"value": {
"data_stream": "logs-gaming-activity",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-gaming-activity"
],
"name": "logs-gaming-activity",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-gaming-events",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-gaming-events"
],
"name": "logs-gaming-events",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-gaming-scores",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-gaming-scores"
],
"name": "logs-gaming-scores",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-manufacturing-downtime",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-manufacturing-downtime"
],
"name": "logs-manufacturing-downtime",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-manufacturing-output",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-manufacturing-output"
],
"name": "logs-manufacturing-output",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-manufacturing-quality",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-manufacturing-quality"
],
"name": "logs-manufacturing-quality",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-retail-customers",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-retail-customers"
],
"name": "logs-retail-customers",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-retail-inventory",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-retail-inventory"
],
"name": "logs-retail-inventory",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-retail-promotions",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-retail-promotions"
],
"name": "logs-retail-promotions",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}
{
"type": "data_stream",
"value": {
"data_stream": "logs-retail-sales",
"template": {
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
},
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"index_patterns": [
"logs-retail-sales"
],
"name": "logs-retail-sales",
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time_nanos",
"type": "date_nanos"
},
"data_stream": {
"properties": {
"namespace": {
"type": "constant_keyword"
}
}
},
"message": {
"type": "wildcard"
}
}
}
}
}
}
}

26
x-pack/test/functional/services/logs_ui/log_stream.ts
View file

@ -12,6 +12,7 @@ import { TabsParams } from '../../page_objects/infra_logs_page';
export function LogStreamPageProvider({ getPageObjects, getService }: FtrProviderContext) {
const pageObjects = getPageObjects(['infraLogs']);
const retry = getService('retry');
const find = getService('find');
const testSubjects = getService('testSubjects');
return {
@ -43,6 +44,31 @@ export function LogStreamPageProvider({ getPageObjects, getService }: FtrProvide
return await testSubjects.findAllDescendant('~logColumn', entryElement);
},
async getLogEntryColumnValueByName(
entryElement: WebElementWrapper,
column: string
): Promise<string> {
const columnElement = await testSubjects.findDescendant(`~${column}`, entryElement);
const contentElement = await columnElement.findByCssSelector(
`[data-test-subj='LogEntryColumnContent']`
);
return await contentElement.getVisibleText();
},
async openLogEntryDetailsFlyout(entryElement: WebElementWrapper) {
await entryElement.click();
const menuButton = await testSubjects.findDescendant(
`~infraLogEntryContextMenuButton`,
entryElement
);
await menuButton.click();
await find.clickByButtonText('View details');
},
async getNoLogsIndicesPrompt() {
return await testSubjects.find('noLogsIndicesPrompt');
},