mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
[ML] Show better file structure finder explanations (#62316)
* [ML] Show better file structure finder explanations * more typescript changes * changing function format * fixing some types * fixing translation id * fix boom error reporting * changes based on review Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
James Gowdy
GitHub
parent
8429a8ede9
commit
64f27ca34e
43 changed files with 608 additions and 266 deletions
|
|
@ -9,6 +9,7 @@ export interface ErrorResponse {
|
|||
statusCode: number;
|
||||
error: string;
|
||||
message: string;
|
||||
attributes?: any;
|
||||
};
|
||||
name: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,21 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export interface InputOverrides {
|
||||
[key: string]: string;
|
||||
}
|
||||
|
||||
export type FormattedOverrides = InputOverrides & {
|
||||
column_names: string[];
|
||||
has_header_row: boolean;
|
||||
should_trim_fields: boolean;
|
||||
};
|
||||
|
||||
export interface AnalysisResult {
|
||||
results: FindFileStructureResponse;
|
||||
overrides?: FormattedOverrides;
|
||||
}
|
||||
|
||||
export interface FindFileStructureResponse {
|
||||
charset: string;
|
||||
has_header_row: boolean;
|
||||
|
|
@ -28,4 +43,54 @@ export interface FindFileStructureResponse {
|
|||
need_client_timezone: boolean;
|
||||
num_lines_analyzed: number;
|
||||
column_names: string[];
|
||||
explanation?: string[];
|
||||
grok_pattern?: string;
|
||||
multiline_start_pattern?: string;
|
||||
exclude_lines_pattern?: string;
|
||||
java_timestamp_formats?: string[];
|
||||
joda_timestamp_formats?: string[];
|
||||
timestamp_field?: string;
|
||||
should_trim_fields?: boolean;
|
||||
}
|
||||
|
||||
export interface ImportResponse {
|
||||
success: boolean;
|
||||
id: string;
|
||||
index?: string;
|
||||
pipelineId?: string;
|
||||
docCount: number;
|
||||
failures: ImportFailure[];
|
||||
error?: any;
|
||||
ingestError?: boolean;
|
||||
}
|
||||
|
||||
export interface ImportFailure {
|
||||
item: number;
|
||||
reason: string;
|
||||
doc: Doc;
|
||||
}
|
||||
|
||||
export interface Doc {
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface Settings {
|
||||
pipeline?: string;
|
||||
index: string;
|
||||
body: any[];
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface Mappings {
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface IngestPipelineWrapper {
|
||||
id: string;
|
||||
pipeline: IngestPipeline;
|
||||
}
|
||||
|
||||
export interface IngestPipeline {
|
||||
description: string;
|
||||
processors: any[];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import {
|
||||
EuiFlexGroup,
|
||||
|
|
@ -23,7 +23,11 @@ import {
|
|||
|
||||
import { WelcomeContent } from './welcome_content';
|
||||
|
||||
export function AboutPanel({ onFilePickerChange }) {
|
||||
interface Props {
|
||||
onFilePickerChange(files: FileList | null): void;
|
||||
}
|
||||
|
||||
export const AboutPanel: FC<Props> = ({ onFilePickerChange }) => {
|
||||
return (
|
||||
<EuiPage restrictWidth={1000} data-test-subj="mlPageFileDataVisualizerUpload">
|
||||
<EuiPageBody>
|
||||
|
|
@ -54,9 +58,9 @@ export function AboutPanel({ onFilePickerChange }) {
|
|||
</EuiPageBody>
|
||||
</EuiPage>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export function LoadingPanel() {
|
||||
export const LoadingPanel: FC = () => {
|
||||
return (
|
||||
<EuiPage restrictWidth={400} data-test-subj="mlPageFileDataVisLoading">
|
||||
<EuiPageBody>
|
||||
|
|
@ -79,4 +83,4 @@ export function LoadingPanel() {
|
|||
</EuiPageBody>
|
||||
</EuiPage>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
@ -5,7 +5,8 @@
|
|||
*/
|
||||
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
|
||||
import {
|
||||
EuiFlexGroup,
|
||||
|
|
@ -19,7 +20,14 @@ import {
|
|||
|
||||
import { ExperimentalBadge } from '../experimental_badge';
|
||||
|
||||
export function WelcomeContent() {
|
||||
export const WelcomeContent: FC = () => {
|
||||
const toolTipContent = i18n.translate(
|
||||
'xpack.ml.fileDatavisualizer.welcomeContent.experimentalFeatureTooltip',
|
||||
{
|
||||
defaultMessage: "Experimental feature. We'd love to hear your feedback.",
|
||||
}
|
||||
);
|
||||
|
||||
return (
|
||||
<EuiFlexGroup gutterSize="xl" alignItems="center">
|
||||
<EuiFlexItem grow={false}>
|
||||
|
|
@ -32,16 +40,7 @@ export function WelcomeContent() {
|
|||
id="xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileTitle"
|
||||
defaultMessage="Visualize data from a log file {experimentalBadge}"
|
||||
values={{
|
||||
experimentalBadge: (
|
||||
<ExperimentalBadge
|
||||
tooltipContent={
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.welcomeContent.experimentalFeatureTooltip"
|
||||
defaultMessage="Experimental feature. We'd love to hear your feedback."
|
||||
/>
|
||||
}
|
||||
/>
|
||||
),
|
||||
experimentalBadge: <ExperimentalBadge tooltipContent={toolTipContent} />,
|
||||
}}
|
||||
/>
|
||||
</h1>
|
||||
|
|
@ -144,4 +143,4 @@ export function WelcomeContent() {
|
|||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
@ -5,11 +5,12 @@
|
|||
*/
|
||||
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import { EuiTitle, EuiSpacer, EuiDescriptionList } from '@elastic/eui';
|
||||
import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
|
||||
|
||||
export function AnalysisSummary({ results }) {
|
||||
export const AnalysisSummary: FC<{ results: FindFileStructureResponse }> = ({ results }) => {
|
||||
const items = createDisplayItems(results);
|
||||
|
||||
return (
|
||||
|
|
@ -28,10 +29,10 @@ export function AnalysisSummary({ results }) {
|
|||
<EuiDescriptionList type="column" listItems={items} className="analysis-summary-list" />
|
||||
</React.Fragment>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function createDisplayItems(results) {
|
||||
const items = [
|
||||
function createDisplayItems(results: FindFileStructureResponse) {
|
||||
const items: Array<{ title: any; description: string | number }> = [
|
||||
{
|
||||
title: (
|
||||
<FormattedMessage
|
||||
|
|
@ -5,11 +5,11 @@
|
|||
*/
|
||||
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import { EuiBetaBadge } from '@elastic/eui';
|
||||
|
||||
export function ExperimentalBadge({ tooltipContent }) {
|
||||
export const ExperimentalBadge: FC<{ tooltipContent: string }> = ({ tooltipContent }) => {
|
||||
return (
|
||||
<span>
|
||||
<EuiBetaBadge
|
||||
|
|
@ -24,4 +24,4 @@ export function ExperimentalBadge({ tooltipContent }) {
|
|||
/>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import React, { FC } from 'react';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import {
|
||||
EuiFlyout,
|
||||
EuiFlyoutFooter,
|
||||
EuiFlexGroup,
|
||||
EuiFlexItem,
|
||||
EuiFlyoutHeader,
|
||||
EuiButtonEmpty,
|
||||
EuiTitle,
|
||||
EuiFlyoutBody,
|
||||
EuiSpacer,
|
||||
EuiText,
|
||||
EuiSubSteps,
|
||||
} from '@elastic/eui';
|
||||
import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
|
||||
|
||||
interface Props {
|
||||
results: FindFileStructureResponse;
|
||||
closeFlyout(): void;
|
||||
}
|
||||
export const ExplanationFlyout: FC<Props> = ({ results, closeFlyout }) => {
|
||||
const explanation = results.explanation!;
|
||||
return (
|
||||
<EuiFlyout onClose={closeFlyout} hideCloseButton size={'m'}>
|
||||
<EuiFlyoutHeader hasBorder>
|
||||
<EuiTitle size="m">
|
||||
<h2>
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.explanationFlyout.title"
|
||||
defaultMessage="Analysis explanation"
|
||||
/>
|
||||
</h2>
|
||||
</EuiTitle>
|
||||
</EuiFlyoutHeader>
|
||||
<EuiFlyoutBody>
|
||||
<Content explanation={explanation} />
|
||||
</EuiFlyoutBody>
|
||||
<EuiFlyoutFooter>
|
||||
<EuiFlexGroup justifyContent="spaceBetween">
|
||||
<EuiFlexItem grow={false}>
|
||||
<EuiButtonEmpty iconType="cross" onClick={closeFlyout} flush="left">
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.explanationFlyout.closeButton"
|
||||
defaultMessage="Close"
|
||||
/>
|
||||
</EuiButtonEmpty>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
</EuiFlyoutFooter>
|
||||
</EuiFlyout>
|
||||
);
|
||||
};
|
||||
|
||||
const Content: FC<{ explanation: string[] }> = ({ explanation }) => (
|
||||
<>
|
||||
<EuiText size={'s'}>
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.explanationFlyout.content"
|
||||
defaultMessage="The logical steps that have produced the analysis results."
|
||||
/>
|
||||
|
||||
<EuiSpacer size="l" />
|
||||
<EuiSubSteps>
|
||||
<ul style={{ wordBreak: 'break-word' }}>
|
||||
{explanation.map((e, i) => (
|
||||
<li key={i}>
|
||||
{e}
|
||||
<EuiSpacer size="s" />
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</EuiSubSteps>
|
||||
</EuiText>
|
||||
</>
|
||||
);
|
||||
|
|
@ -4,4 +4,4 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export { ImportProgress, IMPORT_STATUS } from './import_progress';
|
||||
export { ExplanationFlyout } from './explanation_flyout';
|
||||
|
|
@ -5,13 +5,19 @@
|
|||
*/
|
||||
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import { EuiTitle, EuiSpacer } from '@elastic/eui';
|
||||
|
||||
import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor';
|
||||
|
||||
export function FileContents({ data, format, numberOfLines }) {
|
||||
interface Props {
|
||||
data: string;
|
||||
format: string;
|
||||
numberOfLines: number;
|
||||
}
|
||||
|
||||
export const FileContents: FC<Props> = ({ data, format, numberOfLines }) => {
|
||||
let mode = ML_EDITOR_MODE.TEXT;
|
||||
if (format === ML_EDITOR_MODE.JSON) {
|
||||
mode = ML_EDITOR_MODE.JSON;
|
||||
|
|
@ -35,7 +41,7 @@ export function FileContents({ data, format, numberOfLines }) {
|
|||
id="xpack.ml.fileDatavisualizer.fileContents.firstLinesDescription"
|
||||
defaultMessage="First {numberOfLines, plural, zero {# line} one {# line} other {# lines}}"
|
||||
values={{
|
||||
numberOfLines: numberOfLines,
|
||||
numberOfLines,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
|
@ -51,9 +57,9 @@ export function FileContents({ data, format, numberOfLines }) {
|
|||
/>
|
||||
</React.Fragment>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function limitByNumberOfLines(data, numberOfLines) {
|
||||
function limitByNumberOfLines(data: string, numberOfLines: number) {
|
||||
return data
|
||||
.split('\n')
|
||||
.slice(0, numberOfLines)
|
||||
|
|
@ -17,9 +17,9 @@ import { BottomBar } from '../bottom_bar';
|
|||
import { ResultsView } from '../results_view';
|
||||
import { FileCouldNotBeRead, FileTooLarge } from './file_error_callouts';
|
||||
import { EditFlyout } from '../edit_flyout';
|
||||
import { ExplanationFlyout } from '../explanation_flyout';
|
||||
import { ImportView } from '../import_view';
|
||||
import { MAX_BYTES } from '../../../../../../common/constants/file_datavisualizer';
|
||||
import { isErrorResponse } from '../../../../../../common/types/errors';
|
||||
import {
|
||||
readFile,
|
||||
createUrlOverrides,
|
||||
|
|
@ -42,12 +42,14 @@ export class FileDataVisualizerView extends Component {
|
|||
fileSize: 0,
|
||||
fileTooLarge: false,
|
||||
fileCouldNotBeRead: false,
|
||||
serverErrorMessage: '',
|
||||
serverError: null,
|
||||
loading: false,
|
||||
loaded: false,
|
||||
results: undefined,
|
||||
explanation: undefined,
|
||||
mode: MODE.READ,
|
||||
isEditFlyoutVisible: false,
|
||||
isExplanationFlyoutVisible: false,
|
||||
bottomBarVisible: false,
|
||||
hasPermissionToImport: false,
|
||||
};
|
||||
|
|
@ -78,8 +80,9 @@ export class FileDataVisualizerView extends Component {
|
|||
fileSize: 0,
|
||||
fileTooLarge: false,
|
||||
fileCouldNotBeRead: false,
|
||||
serverErrorMessage: '',
|
||||
serverError: null,
|
||||
results: undefined,
|
||||
explanation: undefined,
|
||||
},
|
||||
() => {
|
||||
if (files.length) {
|
||||
|
|
@ -128,7 +131,7 @@ export class FileDataVisualizerView extends Component {
|
|||
console.log('overrides', overrides);
|
||||
const { analyzeFile } = ml.fileDatavisualizer;
|
||||
const resp = await analyzeFile(lessData, overrides);
|
||||
const serverSettings = processResults(resp.results);
|
||||
const serverSettings = processResults(resp);
|
||||
const serverOverrides = resp.overrides;
|
||||
|
||||
this.previousOverrides = this.overrides;
|
||||
|
|
@ -172,6 +175,7 @@ export class FileDataVisualizerView extends Component {
|
|||
|
||||
this.setState({
|
||||
results: resp.results,
|
||||
explanation: resp.explanation,
|
||||
loaded: true,
|
||||
loading: false,
|
||||
fileCouldNotBeRead: isRetry,
|
||||
|
|
@ -179,19 +183,13 @@ export class FileDataVisualizerView extends Component {
|
|||
} catch (error) {
|
||||
console.error(error);
|
||||
|
||||
let serverErrorMsg;
|
||||
if (isErrorResponse(error) === true) {
|
||||
serverErrorMsg = `${error.body.error}: ${error.body.message}`;
|
||||
} else {
|
||||
serverErrorMsg = JSON.stringify(error, null, 2);
|
||||
}
|
||||
|
||||
this.setState({
|
||||
results: undefined,
|
||||
explanation: undefined,
|
||||
loaded: false,
|
||||
loading: false,
|
||||
fileCouldNotBeRead: true,
|
||||
serverErrorMessage: serverErrorMsg,
|
||||
serverError: error,
|
||||
});
|
||||
|
||||
// as long as the previous overrides are different to the current overrides,
|
||||
|
|
@ -216,6 +214,16 @@ export class FileDataVisualizerView extends Component {
|
|||
this.hideBottomBar();
|
||||
};
|
||||
|
||||
closeExplanationFlyout = () => {
|
||||
this.setState({ isExplanationFlyoutVisible: false });
|
||||
this.showBottomBar();
|
||||
};
|
||||
|
||||
showExplanationFlyout = () => {
|
||||
this.setState({ isExplanationFlyoutVisible: true });
|
||||
this.hideBottomBar();
|
||||
};
|
||||
|
||||
showBottomBar = () => {
|
||||
this.setState({ bottomBarVisible: true });
|
||||
};
|
||||
|
|
@ -252,14 +260,16 @@ export class FileDataVisualizerView extends Component {
|
|||
loading,
|
||||
loaded,
|
||||
results,
|
||||
explanation,
|
||||
fileContents,
|
||||
fileName,
|
||||
fileSize,
|
||||
fileTooLarge,
|
||||
fileCouldNotBeRead,
|
||||
serverErrorMessage,
|
||||
serverError,
|
||||
mode,
|
||||
isEditFlyoutVisible,
|
||||
isExplanationFlyoutVisible,
|
||||
bottomBarVisible,
|
||||
hasPermissionToImport,
|
||||
} = this.state;
|
||||
|
|
@ -281,7 +291,7 @@ export class FileDataVisualizerView extends Component {
|
|||
|
||||
{fileCouldNotBeRead && loading === false && (
|
||||
<React.Fragment>
|
||||
<FileCouldNotBeRead error={serverErrorMessage} loaded={loaded} />
|
||||
<FileCouldNotBeRead error={serverError} loaded={loaded} />
|
||||
<EuiSpacer size="l" />
|
||||
</React.Fragment>
|
||||
)}
|
||||
|
|
@ -289,9 +299,12 @@ export class FileDataVisualizerView extends Component {
|
|||
{loaded && (
|
||||
<ResultsView
|
||||
results={results}
|
||||
explanation={explanation}
|
||||
fileName={fileName}
|
||||
data={fileContents}
|
||||
showEditFlyout={() => this.showEditFlyout()}
|
||||
showExplanationFlyout={() => this.showExplanationFlyout()}
|
||||
disableButtons={isEditFlyoutVisible || isExplanationFlyoutVisible}
|
||||
/>
|
||||
)}
|
||||
<EditFlyout
|
||||
|
|
@ -303,6 +316,10 @@ export class FileDataVisualizerView extends Component {
|
|||
fields={fields}
|
||||
/>
|
||||
|
||||
{isExplanationFlyoutVisible && (
|
||||
<ExplanationFlyout results={results} closeFlyout={this.closeExplanationFlyout} />
|
||||
)}
|
||||
|
||||
{bottomBarVisible && loaded && (
|
||||
<BottomBar
|
||||
mode={MODE.READ}
|
||||
|
|
|
|||
|
|
@ -5,15 +5,21 @@
|
|||
*/
|
||||
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import { EuiCallOut } from '@elastic/eui';
|
||||
import { EuiCallOut, EuiSpacer } from '@elastic/eui';
|
||||
|
||||
import numeral from '@elastic/numeral';
|
||||
import { ErrorResponse } from '../../../../../../common/types/errors';
|
||||
|
||||
const FILE_SIZE_DISPLAY_FORMAT = '0,0.[0] b';
|
||||
|
||||
export function FileTooLarge({ fileSize, maxFileSize }) {
|
||||
interface FileTooLargeProps {
|
||||
fileSize: number;
|
||||
maxFileSize: number;
|
||||
}
|
||||
|
||||
export const FileTooLarge: FC<FileTooLargeProps> = ({ fileSize, maxFileSize }) => {
|
||||
const fileSizeFormatted = numeral(fileSize).format(FILE_SIZE_DISPLAY_FORMAT);
|
||||
const maxFileSizeFormatted = numeral(maxFileSize).format(FILE_SIZE_DISPLAY_FORMAT);
|
||||
|
||||
|
|
@ -67,9 +73,15 @@ export function FileTooLarge({ fileSize, maxFileSize }) {
|
|||
{errorText}
|
||||
</EuiCallOut>
|
||||
);
|
||||
};
|
||||
|
||||
interface FileCouldNotBeReadProps {
|
||||
error: ErrorResponse;
|
||||
loaded: boolean;
|
||||
}
|
||||
|
||||
export function FileCouldNotBeRead({ error, loaded }) {
|
||||
export const FileCouldNotBeRead: FC<FileCouldNotBeReadProps> = ({ error, loaded }) => {
|
||||
const message = error.body.message;
|
||||
return (
|
||||
<EuiCallOut
|
||||
title={
|
||||
|
|
@ -82,15 +94,32 @@ export function FileCouldNotBeRead({ error, loaded }) {
|
|||
iconType="cross"
|
||||
data-test-subj="mlFileUploadErrorCallout fileCouldNotBeRead"
|
||||
>
|
||||
{error !== undefined && <p>{error}</p>}
|
||||
{message}
|
||||
<Explanation error={error} />
|
||||
{loaded && (
|
||||
<p>
|
||||
<>
|
||||
<EuiSpacer size="s" />
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.fileErrorCallouts.revertingToPreviousSettingsDescription"
|
||||
defaultMessage="Reverting to previous settings"
|
||||
/>
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</EuiCallOut>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const Explanation: FC<{ error: ErrorResponse }> = ({ error }) => {
|
||||
if (!error.body.attributes?.body?.error?.suppressed?.length) {
|
||||
return null;
|
||||
}
|
||||
const reason: string = error.body.attributes.body.error.suppressed[0].reason;
|
||||
return (
|
||||
<>
|
||||
<EuiSpacer size="s" />
|
||||
{reason.split('\n').map((m, i) => (
|
||||
<div key={i}>{m}</div>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
@ -5,13 +5,24 @@
|
|||
*/
|
||||
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
|
||||
import { EuiCallOut, EuiAccordion } from '@elastic/eui';
|
||||
|
||||
import { IMPORT_STATUS } from '../import_progress';
|
||||
import { IMPORT_STATUS, Statuses } from '../import_progress';
|
||||
|
||||
export function ImportErrors({ errors, statuses }) {
|
||||
interface ImportError {
|
||||
msg: string;
|
||||
more?: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
errors: any[];
|
||||
statuses: Statuses;
|
||||
}
|
||||
|
||||
export const ImportErrors: FC<Props> = ({ errors, statuses }) => {
|
||||
return (
|
||||
<EuiCallOut title={title(statuses)} color="danger" iconType="cross">
|
||||
{errors.map((e, i) => (
|
||||
|
|
@ -19,9 +30,9 @@ export function ImportErrors({ errors, statuses }) {
|
|||
))}
|
||||
</EuiCallOut>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function title(statuses) {
|
||||
function title(statuses: Statuses) {
|
||||
switch (IMPORT_STATUS.FAILED) {
|
||||
case statuses.readStatus:
|
||||
return (
|
||||
|
|
@ -82,7 +93,7 @@ function title(statuses) {
|
|||
}
|
||||
}
|
||||
|
||||
function ImportError(error, key) {
|
||||
function ImportError(error: any, key: number) {
|
||||
const errorObj = toString(error);
|
||||
return (
|
||||
<React.Fragment>
|
||||
|
|
@ -106,7 +117,7 @@ function ImportError(error, key) {
|
|||
);
|
||||
}
|
||||
|
||||
function toString(error) {
|
||||
function toString(error: any): ImportError {
|
||||
if (typeof error === 'string') {
|
||||
return { msg: error };
|
||||
}
|
||||
|
|
@ -118,7 +129,7 @@ function toString(error) {
|
|||
if (typeof error.error === 'object') {
|
||||
if (error.error.msg !== undefined) {
|
||||
// this will catch a bulk ingest failure
|
||||
const errorObj = { msg: error.error.msg };
|
||||
const errorObj: ImportError = { msg: error.error.msg };
|
||||
if (error.error.body !== undefined) {
|
||||
errorObj.more = error.error.response;
|
||||
}
|
||||
|
|
@ -139,11 +150,8 @@ function toString(error) {
|
|||
}
|
||||
|
||||
return {
|
||||
msg: (
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.importErrors.unknownErrorMessage"
|
||||
defaultMessage="Unknown error"
|
||||
/>
|
||||
),
|
||||
msg: i18n.translate('xpack.ml.fileDatavisualizer.importErrors.unknownErrorMessage', {
|
||||
defaultMessage: 'Unknown error',
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
|
@ -6,17 +6,31 @@
|
|||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import { EuiStepsHorizontal, EuiProgress, EuiSpacer } from '@elastic/eui';
|
||||
|
||||
export const IMPORT_STATUS = {
|
||||
INCOMPLETE: 'incomplete',
|
||||
COMPLETE: 'complete',
|
||||
FAILED: 'danger',
|
||||
};
|
||||
export enum IMPORT_STATUS {
|
||||
INCOMPLETE = 'incomplete',
|
||||
COMPLETE = 'complete',
|
||||
FAILED = 'danger',
|
||||
}
|
||||
|
||||
export function ImportProgress({ statuses }) {
|
||||
export interface Statuses {
|
||||
reading: boolean;
|
||||
readStatus: IMPORT_STATUS;
|
||||
parseJSONStatus: IMPORT_STATUS;
|
||||
indexCreatedStatus: IMPORT_STATUS;
|
||||
ingestPipelineCreatedStatus: IMPORT_STATUS;
|
||||
indexPatternCreatedStatus: IMPORT_STATUS;
|
||||
uploadProgress: number;
|
||||
uploadStatus: IMPORT_STATUS;
|
||||
createIndexPattern: boolean;
|
||||
createPipeline: boolean;
|
||||
permissionCheckStatus: IMPORT_STATUS;
|
||||
}
|
||||
|
||||
export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
|
||||
const {
|
||||
reading,
|
||||
readStatus,
|
||||
|
|
@ -271,9 +285,9 @@ export function ImportProgress({ statuses }) {
|
|||
)}
|
||||
</React.Fragment>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function UploadFunctionProgress({ progress }) {
|
||||
const UploadFunctionProgress: FC<{ progress: number }> = ({ progress }) => {
|
||||
return (
|
||||
<React.Fragment>
|
||||
<p>
|
||||
|
|
@ -290,4 +304,4 @@ function UploadFunctionProgress({ progress }) {
|
|||
)}
|
||||
</React.Fragment>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export { ImportProgress, IMPORT_STATUS, Statuses } from './import_progress';
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import {
|
||||
EuiFieldText,
|
||||
|
|
@ -20,7 +20,25 @@ import {
|
|||
import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor';
|
||||
const EDITOR_HEIGHT = '300px';
|
||||
|
||||
export function AdvancedSettings({
|
||||
interface Props {
|
||||
index: string;
|
||||
indexPattern: string;
|
||||
initialized: boolean;
|
||||
onIndexChange(): void;
|
||||
createIndexPattern: boolean;
|
||||
onCreateIndexPatternChange(): void;
|
||||
onIndexPatternChange(): void;
|
||||
indexSettingsString: string;
|
||||
mappingsString: string;
|
||||
pipelineString: string;
|
||||
onIndexSettingsStringChange(): void;
|
||||
onMappingsStringChange(): void;
|
||||
onPipelineStringChange(): void;
|
||||
indexNameError: string;
|
||||
indexPatternNameError: string;
|
||||
}
|
||||
|
||||
export const AdvancedSettings: FC<Props> = ({
|
||||
index,
|
||||
indexPattern,
|
||||
initialized,
|
||||
|
|
@ -36,7 +54,7 @@ export function AdvancedSettings({
|
|||
onPipelineStringChange,
|
||||
indexNameError,
|
||||
indexPatternNameError,
|
||||
}) {
|
||||
}) => {
|
||||
return (
|
||||
<React.Fragment>
|
||||
<EuiFormRow
|
||||
|
|
@ -93,7 +111,6 @@ export function AdvancedSettings({
|
|||
defaultMessage="Index pattern name"
|
||||
/>
|
||||
}
|
||||
disabled={createIndexPattern === false || initialized === true}
|
||||
isInvalid={indexPatternNameError !== ''}
|
||||
error={[indexPatternNameError]}
|
||||
>
|
||||
|
|
@ -133,9 +150,15 @@ export function AdvancedSettings({
|
|||
</EuiFlexGroup>
|
||||
</React.Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
interface JsonEditorProps {
|
||||
initialized: boolean;
|
||||
data: string;
|
||||
onChange(): void;
|
||||
}
|
||||
|
||||
function IndexSettings({ initialized, data, onChange }) {
|
||||
const IndexSettings: FC<JsonEditorProps> = ({ initialized, data, onChange }) => {
|
||||
return (
|
||||
<React.Fragment>
|
||||
<EuiFormRow
|
||||
|
|
@ -145,7 +168,6 @@ function IndexSettings({ initialized, data, onChange }) {
|
|||
defaultMessage="Index settings"
|
||||
/>
|
||||
}
|
||||
disabled={initialized === true}
|
||||
fullWidth
|
||||
>
|
||||
<MLJobEditor
|
||||
|
|
@ -159,9 +181,9 @@ function IndexSettings({ initialized, data, onChange }) {
|
|||
</EuiFormRow>
|
||||
</React.Fragment>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function Mappings({ initialized, data, onChange }) {
|
||||
const Mappings: FC<JsonEditorProps> = ({ initialized, data, onChange }) => {
|
||||
return (
|
||||
<React.Fragment>
|
||||
<EuiFormRow
|
||||
|
|
@ -171,7 +193,6 @@ function Mappings({ initialized, data, onChange }) {
|
|||
defaultMessage="Mappings"
|
||||
/>
|
||||
}
|
||||
disabled={initialized === true}
|
||||
fullWidth
|
||||
>
|
||||
<MLJobEditor
|
||||
|
|
@ -185,9 +206,9 @@ function Mappings({ initialized, data, onChange }) {
|
|||
</EuiFormRow>
|
||||
</React.Fragment>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function IngestPipeline({ initialized, data, onChange }) {
|
||||
const IngestPipeline: FC<JsonEditorProps> = ({ initialized, data, onChange }) => {
|
||||
return (
|
||||
<React.Fragment>
|
||||
<EuiFormRow
|
||||
|
|
@ -197,7 +218,6 @@ function IngestPipeline({ initialized, data, onChange }) {
|
|||
defaultMessage="Ingest pipeline"
|
||||
/>
|
||||
}
|
||||
disabled={initialized === true}
|
||||
fullWidth
|
||||
>
|
||||
<MLJobEditor
|
||||
|
|
@ -211,4 +231,4 @@ function IngestPipeline({ initialized, data, onChange }) {
|
|||
</EuiFormRow>
|
||||
</React.Fragment>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
@ -5,14 +5,32 @@
|
|||
*/
|
||||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import { EuiTabbedContent, EuiSpacer } from '@elastic/eui';
|
||||
|
||||
import { SimpleSettings } from './simple';
|
||||
import { AdvancedSettings } from './advanced';
|
||||
|
||||
export const ImportSettings = ({
|
||||
interface Props {
|
||||
index: string;
|
||||
indexPattern: string;
|
||||
initialized: boolean;
|
||||
onIndexChange(): void;
|
||||
createIndexPattern: boolean;
|
||||
onCreateIndexPatternChange(): void;
|
||||
onIndexPatternChange(): void;
|
||||
indexSettingsString: string;
|
||||
mappingsString: string;
|
||||
pipelineString: string;
|
||||
onIndexSettingsStringChange(): void;
|
||||
onMappingsStringChange(): void;
|
||||
onPipelineStringChange(): void;
|
||||
indexNameError: string;
|
||||
indexPatternNameError: string;
|
||||
}
|
||||
|
||||
export const ImportSettings: FC<Props> = ({
|
||||
index,
|
||||
indexPattern,
|
||||
initialized,
|
||||
|
|
@ -6,11 +6,20 @@
|
|||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import { EuiFieldText, EuiFormRow, EuiCheckbox, EuiSpacer } from '@elastic/eui';
|
||||
|
||||
export const SimpleSettings = ({
|
||||
interface Props {
|
||||
index: string;
|
||||
initialized: boolean;
|
||||
onIndexChange(): void;
|
||||
createIndexPattern: boolean;
|
||||
onCreateIndexPatternChange(): void;
|
||||
indexNameError: string;
|
||||
}
|
||||
|
||||
export const SimpleSettings: FC<Props> = ({
|
||||
index,
|
||||
initialized,
|
||||
onIndexChange,
|
||||
|
|
@ -5,11 +5,29 @@
|
|||
*/
|
||||
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import React, { FC } from 'react';
|
||||
|
||||
import { EuiSpacer, EuiDescriptionList, EuiCallOut, EuiAccordion } from '@elastic/eui';
|
||||
|
||||
export function ImportSummary({
|
||||
interface Props {
|
||||
index: string;
|
||||
indexPattern: string;
|
||||
ingestPipelineId: string;
|
||||
docCount: number;
|
||||
importFailures: DocFailure[];
|
||||
createIndexPattern: boolean;
|
||||
createPipeline: boolean;
|
||||
}
|
||||
|
||||
interface DocFailure {
|
||||
item: number;
|
||||
reason: string;
|
||||
doc: {
|
||||
message: string;
|
||||
};
|
||||
}
|
||||
|
||||
export const ImportSummary: FC<Props> = ({
|
||||
index,
|
||||
indexPattern,
|
||||
ingestPipelineId,
|
||||
|
|
@ -17,7 +35,7 @@ export function ImportSummary({
|
|||
importFailures,
|
||||
createIndexPattern,
|
||||
createPipeline,
|
||||
}) {
|
||||
}) => {
|
||||
const items = createDisplayItems(
|
||||
index,
|
||||
indexPattern,
|
||||
|
|
@ -75,9 +93,13 @@ export function ImportSummary({
|
|||
)}
|
||||
</React.Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
interface FailuresProps {
|
||||
failedDocs: DocFailure[];
|
||||
}
|
||||
|
||||
function Failures({ failedDocs }) {
|
||||
const Failures: FC<FailuresProps> = ({ failedDocs }) => {
|
||||
return (
|
||||
<EuiAccordion
|
||||
id="failureList"
|
||||
|
|
@ -101,16 +123,16 @@ function Failures({ failedDocs }) {
|
|||
</div>
|
||||
</EuiAccordion>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function createDisplayItems(
|
||||
index,
|
||||
indexPattern,
|
||||
ingestPipelineId,
|
||||
docCount,
|
||||
importFailures,
|
||||
createIndexPattern,
|
||||
createPipeline
|
||||
index: string,
|
||||
indexPattern: string,
|
||||
ingestPipelineId: string,
|
||||
docCount: number,
|
||||
importFailures: DocFailure[],
|
||||
createIndexPattern: boolean,
|
||||
createPipeline: boolean
|
||||
) {
|
||||
const items = [
|
||||
{
|
||||
|
|
@ -623,7 +623,6 @@ async function createKibanaIndexPattern(
|
|||
id,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
return {
|
||||
success: false,
|
||||
error,
|
||||
|
|
|
|||
|
|
@ -4,30 +4,53 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { ml } from '../../../../../services/ml_api_service';
|
||||
import { chunk } from 'lodash';
|
||||
import moment from 'moment';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { ml } from '../../../../../services/ml_api_service';
|
||||
import {
|
||||
Doc,
|
||||
ImportFailure,
|
||||
ImportResponse,
|
||||
Mappings,
|
||||
Settings,
|
||||
IngestPipeline,
|
||||
} from '../../../../../../../common/types/file_datavisualizer';
|
||||
|
||||
const CHUNK_SIZE = 5000;
|
||||
const MAX_CHUNK_CHAR_COUNT = 1000000;
|
||||
const IMPORT_RETRIES = 5;
|
||||
|
||||
export class Importer {
|
||||
constructor({ settings, mappings, pipeline }) {
|
||||
this.settings = settings;
|
||||
this.mappings = mappings;
|
||||
this.pipeline = pipeline;
|
||||
export interface ImportConfig {
|
||||
settings: Settings;
|
||||
mappings: Mappings;
|
||||
pipeline: IngestPipeline;
|
||||
}
|
||||
|
||||
this.data = [];
|
||||
this.docArray = [];
|
||||
this.docSizeArray = [];
|
||||
export interface ImportResults {
|
||||
success: boolean;
|
||||
failures?: any[];
|
||||
docCount?: number;
|
||||
error?: any;
|
||||
}
|
||||
|
||||
export class Importer {
|
||||
private _settings: Settings;
|
||||
private _mappings: Mappings;
|
||||
private _pipeline: IngestPipeline;
|
||||
|
||||
protected _docArray: Doc[] = [];
|
||||
|
||||
constructor({ settings, mappings, pipeline }: ImportConfig) {
|
||||
this._settings = settings;
|
||||
this._mappings = mappings;
|
||||
this._pipeline = pipeline;
|
||||
}
|
||||
|
||||
async initializeImport(index) {
|
||||
const settings = this.settings;
|
||||
const mappings = this.mappings;
|
||||
const pipeline = this.pipeline;
|
||||
async initializeImport(index: string) {
|
||||
const settings = this._settings;
|
||||
const mappings = this._mappings;
|
||||
const pipeline = this._pipeline;
|
||||
updatePipelineTimezone(pipeline);
|
||||
|
||||
// if no pipeline has been supplied,
|
||||
|
|
@ -52,7 +75,12 @@ export class Importer {
|
|||
return createIndexResp;
|
||||
}
|
||||
|
||||
async import(id, index, pipelineId, setImportProgress) {
|
||||
async import(
|
||||
id: string,
|
||||
index: string,
|
||||
pipelineId: string,
|
||||
setImportProgress: (progress: number) => void
|
||||
): Promise<ImportResults> {
|
||||
if (!id || !index) {
|
||||
return {
|
||||
success: false,
|
||||
|
|
@ -65,14 +93,14 @@ export class Importer {
|
|||
};
|
||||
}
|
||||
|
||||
const chunks = createDocumentChunks(this.docArray);
|
||||
const chunks = createDocumentChunks(this._docArray);
|
||||
|
||||
const ingestPipeline = {
|
||||
id: pipelineId,
|
||||
};
|
||||
|
||||
let success = true;
|
||||
const failures = [];
|
||||
const failures: ImportFailure[] = [];
|
||||
let error;
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
|
|
@ -86,10 +114,13 @@ export class Importer {
|
|||
};
|
||||
|
||||
let retries = IMPORT_RETRIES;
|
||||
let resp = {
|
||||
let resp: ImportResponse = {
|
||||
success: false,
|
||||
failures: [],
|
||||
docCount: 0,
|
||||
id: '',
|
||||
index: '',
|
||||
pipelineId: '',
|
||||
};
|
||||
|
||||
while (resp.success === false && retries > 0) {
|
||||
|
|
@ -97,12 +128,14 @@ export class Importer {
|
|||
resp = await ml.fileDatavisualizer.import(aggs);
|
||||
|
||||
if (retries < IMPORT_RETRIES) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Retrying import ${IMPORT_RETRIES - retries}`);
|
||||
}
|
||||
|
||||
retries--;
|
||||
} catch (err) {
|
||||
resp = { success: false, error: err };
|
||||
resp.success = false;
|
||||
resp.error = err;
|
||||
retries = 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -110,6 +143,7 @@ export class Importer {
|
|||
if (resp.success) {
|
||||
setImportProgress(((i + 1) / chunks.length) * 100);
|
||||
} else {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(resp);
|
||||
success = false;
|
||||
error = resp.error;
|
||||
|
|
@ -120,10 +154,10 @@ export class Importer {
|
|||
populateFailures(resp, failures, i);
|
||||
}
|
||||
|
||||
const result = {
|
||||
const result: ImportResults = {
|
||||
success,
|
||||
failures,
|
||||
docCount: this.docArray.length,
|
||||
docCount: this._docArray.length,
|
||||
};
|
||||
|
||||
if (success) {
|
||||
|
|
@ -136,7 +170,7 @@ export class Importer {
|
|||
}
|
||||
}
|
||||
|
||||
function populateFailures(error, failures, chunkCount) {
|
||||
function populateFailures(error: ImportResponse, failures: ImportFailure[], chunkCount: number) {
|
||||
if (error.failures && error.failures.length) {
|
||||
// update the item value to include the chunk count
|
||||
// e.g. item 3 in chunk 2 is actually item 20003
|
||||
|
|
@ -155,10 +189,10 @@ function populateFailures(error, failures, chunkCount) {
|
|||
// But it's not sending every single field that Filebeat would add, so the ingest pipeline
|
||||
// cannot look for a event.timezone variable in each input record.
|
||||
// Therefore we need to replace {{ event.timezone }} with the actual browser timezone
|
||||
function updatePipelineTimezone(ingestPipeline) {
|
||||
function updatePipelineTimezone(ingestPipeline: IngestPipeline) {
|
||||
if (ingestPipeline !== undefined && ingestPipeline.processors && ingestPipeline.processors) {
|
||||
const dateProcessor = ingestPipeline.processors.find(
|
||||
p => p.date !== undefined && p.date.timezone === '{{ event.timezone }}'
|
||||
(p: any) => p.date !== undefined && p.date.timezone === '{{ event.timezone }}'
|
||||
);
|
||||
|
||||
if (dateProcessor) {
|
||||
|
|
@ -167,8 +201,8 @@ function updatePipelineTimezone(ingestPipeline) {
|
|||
}
|
||||
}
|
||||
|
||||
function createDocumentChunks(docArray) {
|
||||
const chunks = [];
|
||||
function createDocumentChunks(docArray: Doc[]) {
|
||||
const chunks: Doc[][] = [];
|
||||
// chop docArray into 5000 doc chunks
|
||||
const tempChunks = chunk(docArray, CHUNK_SIZE);
|
||||
|
||||
|
|
@ -6,8 +6,14 @@
|
|||
|
||||
import { MessageImporter } from './message_importer';
|
||||
import { NdjsonImporter } from './ndjson_importer';
|
||||
import { ImportConfig } from './importer';
|
||||
import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer';
|
||||
|
||||
export function importerFactory(format, results, settings) {
|
||||
export function importerFactory(
|
||||
format: string,
|
||||
results: FindFileStructureResponse,
|
||||
settings: ImportConfig
|
||||
) {
|
||||
switch (format) {
|
||||
// delimited and semi-structured text are both handled by splitting the
|
||||
// file into messages, then sending these to ES for further processing
|
||||
|
|
@ -4,17 +4,24 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { Importer } from './importer';
|
||||
import { Importer, ImportConfig } from './importer';
|
||||
import {
|
||||
Doc,
|
||||
FindFileStructureResponse,
|
||||
} from '../../../../../../../common/types/file_datavisualizer';
|
||||
|
||||
export class MessageImporter extends Importer {
|
||||
constructor(results, settings) {
|
||||
private _excludeLinesRegex: RegExp | null;
|
||||
private _multilineStartRegex: RegExp | null;
|
||||
|
||||
constructor(results: FindFileStructureResponse, settings: ImportConfig) {
|
||||
super(settings);
|
||||
|
||||
this.excludeLinesRegex =
|
||||
this._excludeLinesRegex =
|
||||
results.exclude_lines_pattern === undefined
|
||||
? null
|
||||
: new RegExp(results.exclude_lines_pattern);
|
||||
this.multilineStartRegex =
|
||||
this._multilineStartRegex =
|
||||
results.multiline_start_pattern === undefined
|
||||
? null
|
||||
: new RegExp(results.multiline_start_pattern);
|
||||
|
|
@ -26,9 +33,9 @@ export class MessageImporter extends Importer {
|
|||
// multiline_start_pattern regex
|
||||
// if it does, it is a legitimate end of line and can be pushed into the list,
|
||||
// if not, it must be a newline char inside a field value, so keep looking.
|
||||
read(text) {
|
||||
read(text: string) {
|
||||
try {
|
||||
const data = [];
|
||||
const data: Doc[] = [];
|
||||
|
||||
let message = '';
|
||||
let line = '';
|
||||
|
|
@ -57,14 +64,12 @@ export class MessageImporter extends Importer {
|
|||
data.shift();
|
||||
}
|
||||
|
||||
this.data = data;
|
||||
this.docArray = this.data;
|
||||
this._docArray = data;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
return {
|
||||
success: false,
|
||||
error,
|
||||
|
|
@ -72,9 +77,9 @@ export class MessageImporter extends Importer {
|
|||
}
|
||||
}
|
||||
|
||||
processLine(data, message, line) {
|
||||
if (this.excludeLinesRegex === null || line.match(this.excludeLinesRegex) === null) {
|
||||
if (this.multilineStartRegex === null || line.match(this.multilineStartRegex) !== null) {
|
||||
processLine(data: Doc[], message: string, line: string) {
|
||||
if (this._excludeLinesRegex === null || line.match(this._excludeLinesRegex) === null) {
|
||||
if (this._multilineStartRegex === null || line.match(this._multilineStartRegex) !== null) {
|
||||
this.addMessage(data, message);
|
||||
message = '';
|
||||
} else if (data.length === 0) {
|
||||
|
|
@ -90,7 +95,7 @@ export class MessageImporter extends Importer {
|
|||
return message;
|
||||
}
|
||||
|
||||
addMessage(data, message) {
|
||||
addMessage(data: Doc[], message: string) {
|
||||
// if the message ended \r\n (Windows line endings)
|
||||
// then omit the \r as well as the \n for consistency
|
||||
message = message.replace(/\r$/, '');
|
||||
|
|
@ -4,18 +4,19 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { Importer } from './importer';
|
||||
import { Importer, ImportConfig } from './importer';
|
||||
import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer';
|
||||
|
||||
export class NdjsonImporter extends Importer {
|
||||
constructor(results, settings) {
|
||||
constructor(results: FindFileStructureResponse, settings: ImportConfig) {
|
||||
super(settings);
|
||||
}
|
||||
|
||||
read(json) {
|
||||
read(json: string) {
|
||||
try {
|
||||
const splitJson = json.split(/}\s*\n/);
|
||||
|
||||
const ndjson = [];
|
||||
const ndjson: any[] = [];
|
||||
for (let i = 0; i < splitJson.length; i++) {
|
||||
if (splitJson[i] !== '') {
|
||||
// note the extra } at the end of the line, adding back
|
||||
|
|
@ -24,7 +25,7 @@ export class NdjsonImporter extends Importer {
|
|||
}
|
||||
}
|
||||
|
||||
this.docArray = ndjson;
|
||||
this._docArray = ndjson;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
|
|
@ -7,10 +7,10 @@
|
|||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
|
||||
import React from 'react';
|
||||
|
||||
import React, { FC } from 'react';
|
||||
import {
|
||||
EuiButton,
|
||||
EuiButtonEmpty,
|
||||
EuiPage,
|
||||
EuiPageBody,
|
||||
EuiPageContentHeader,
|
||||
|
|
@ -18,13 +18,33 @@ import {
|
|||
EuiTabbedContent,
|
||||
EuiSpacer,
|
||||
EuiTitle,
|
||||
EuiFlexGroup,
|
||||
EuiFlexItem,
|
||||
} from '@elastic/eui';
|
||||
import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
|
||||
|
||||
import { FileContents } from '../file_contents';
|
||||
import { AnalysisSummary } from '../analysis_summary';
|
||||
// @ts-ignore
|
||||
import { FieldsStats } from '../fields_stats';
|
||||
|
||||
export const ResultsView = ({ data, fileName, results, showEditFlyout }) => {
|
||||
interface Props {
|
||||
data: string;
|
||||
fileName: string;
|
||||
results: FindFileStructureResponse;
|
||||
showEditFlyout(): void;
|
||||
showExplanationFlyout(): void;
|
||||
disableButtons: boolean;
|
||||
}
|
||||
|
||||
export const ResultsView: FC<Props> = ({
|
||||
data,
|
||||
fileName,
|
||||
results,
|
||||
showEditFlyout,
|
||||
showExplanationFlyout,
|
||||
disableButtons,
|
||||
}) => {
|
||||
const tabs = [
|
||||
{
|
||||
id: 'file-stats',
|
||||
|
|
@ -60,12 +80,24 @@ export const ResultsView = ({ data, fileName, results, showEditFlyout }) => {
|
|||
|
||||
<EuiSpacer size="m" />
|
||||
|
||||
<EuiButton onClick={() => showEditFlyout()}>
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.resultsView.overrideSettingsButtonLabel"
|
||||
defaultMessage="Override settings"
|
||||
/>
|
||||
</EuiButton>
|
||||
<EuiFlexGroup gutterSize="s" alignItems="center">
|
||||
<EuiFlexItem grow={false}>
|
||||
<EuiButton onClick={() => showEditFlyout()} disabled={disableButtons}>
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.resultsView.overrideSettingsButtonLabel"
|
||||
defaultMessage="Override settings"
|
||||
/>
|
||||
</EuiButton>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem grow={false}>
|
||||
<EuiButtonEmpty onClick={() => showExplanationFlyout()} disabled={disableButtons}>
|
||||
<FormattedMessage
|
||||
id="xpack.ml.fileDatavisualizer.resultsView.analysisExplanationButtonLabel"
|
||||
defaultMessage="Analysis explanation"
|
||||
/>
|
||||
</EuiButtonEmpty>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
</EuiPanel>
|
||||
|
||||
<EuiSpacer size="m" />
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export const DEFAULT_LINES_TO_SAMPLE = 1000;
|
||||
|
||||
export const overrideDefaults = {
|
||||
timestampFormat: undefined,
|
||||
timestampField: undefined,
|
||||
format: undefined,
|
||||
delimiter: undefined,
|
||||
quote: undefined,
|
||||
hasHeaderRow: undefined,
|
||||
charset: undefined,
|
||||
columnNames: undefined,
|
||||
shouldTrimFields: undefined,
|
||||
grokPattern: undefined,
|
||||
linesToSample: undefined,
|
||||
};
|
||||
|
|
@ -4,11 +4,27 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { overrideDefaults, DEFAULT_LINES_TO_SAMPLE } from './overrides';
|
||||
import { isEqual } from 'lodash';
|
||||
import { ml } from '../../../../services/ml_api_service';
|
||||
import { AnalysisResult, InputOverrides } from '../../../../../../common/types/file_datavisualizer';
|
||||
|
||||
export function readFile(file) {
|
||||
const DEFAULT_LINES_TO_SAMPLE = 1000;
|
||||
|
||||
const overrideDefaults = {
|
||||
timestampFormat: undefined,
|
||||
timestampField: undefined,
|
||||
format: undefined,
|
||||
delimiter: undefined,
|
||||
quote: undefined,
|
||||
hasHeaderRow: undefined,
|
||||
charset: undefined,
|
||||
columnNames: undefined,
|
||||
shouldTrimFields: undefined,
|
||||
grokPattern: undefined,
|
||||
linesToSample: undefined,
|
||||
};
|
||||
|
||||
export function readFile(file: File) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (file && file.size) {
|
||||
const reader = new FileReader();
|
||||
|
|
@ -23,14 +39,14 @@ export function readFile(file) {
|
|||
resolve({ data });
|
||||
}
|
||||
};
|
||||
})(file);
|
||||
})();
|
||||
} else {
|
||||
reject();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function reduceData(data, mb) {
|
||||
export function reduceData(data: string, mb: number) {
|
||||
// assuming ascii characters in the file where 1 char is 1 byte
|
||||
// TODO - change this when other non UTF-8 formats are
|
||||
// supported for the read data
|
||||
|
|
@ -38,8 +54,8 @@ export function reduceData(data, mb) {
|
|||
return data.length >= size ? data.slice(0, size) : data;
|
||||
}
|
||||
|
||||
export function createUrlOverrides(overrides, originalSettings) {
|
||||
const formattedOverrides = {};
|
||||
export function createUrlOverrides(overrides: InputOverrides, originalSettings: InputOverrides) {
|
||||
const formattedOverrides: InputOverrides = {};
|
||||
for (const o in overrideDefaults) {
|
||||
if (overrideDefaults.hasOwnProperty(o)) {
|
||||
let value = overrides[o];
|
||||
|
|
@ -93,15 +109,15 @@ export function createUrlOverrides(overrides, originalSettings) {
|
|||
return formattedOverrides;
|
||||
}
|
||||
|
||||
export function processResults(results) {
|
||||
export function processResults({ results, overrides }: AnalysisResult) {
|
||||
const timestampFormat =
|
||||
results.java_timestamp_formats !== undefined && results.java_timestamp_formats.length
|
||||
? results.java_timestamp_formats[0]
|
||||
: undefined;
|
||||
|
||||
const linesToSample =
|
||||
results.overrides !== undefined && results.overrides.lines_to_sample !== undefined
|
||||
? results.overrides.lines_to_sample
|
||||
overrides !== undefined && overrides.lines_to_sample !== undefined
|
||||
? overrides.lines_to_sample
|
||||
: DEFAULT_LINES_TO_SAMPLE;
|
||||
|
||||
return {
|
||||
|
|
@ -125,8 +141,8 @@ export function processResults(results) {
|
|||
* @param {string} indexName
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
export async function hasImportPermission(indexName) {
|
||||
const priv = {
|
||||
export async function hasImportPermission(indexName: string) {
|
||||
const priv: { cluster: string[]; index?: any } = {
|
||||
cluster: ['cluster:monitor/nodes/info', 'cluster:admin/ingest/pipeline/put'],
|
||||
};
|
||||
|
||||
|
|
@ -7,6 +7,7 @@
|
|||
import { http } from '../http_service';
|
||||
|
||||
import { basePath } from './index';
|
||||
import { ImportResponse } from '../../../../common/types/file_datavisualizer';
|
||||
|
||||
export const fileDatavisualizer = {
|
||||
analyzeFile(file: string, params: Record<string, string> = {}) {
|
||||
|
|
@ -27,7 +28,7 @@ export const fileDatavisualizer = {
|
|||
mappings,
|
||||
ingestPipeline,
|
||||
}: {
|
||||
id: string;
|
||||
id: string | undefined;
|
||||
index: string;
|
||||
data: any;
|
||||
settings: any;
|
||||
|
|
@ -43,7 +44,7 @@ export const fileDatavisualizer = {
|
|||
ingestPipeline,
|
||||
});
|
||||
|
||||
return http<any>({
|
||||
return http<ImportResponse>({
|
||||
path: `${basePath()}/file_data_visualizer/import`,
|
||||
method: 'POST',
|
||||
query,
|
||||
|
|
|
|||
|
|
@ -740,7 +740,7 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any)
|
|||
urls: [
|
||||
{
|
||||
fmt:
|
||||
'/_ml/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>',
|
||||
'/_ml/find_file_structure?&explain=true&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>',
|
||||
req: {
|
||||
charset: {
|
||||
type: 'string',
|
||||
|
|
@ -778,7 +778,7 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any)
|
|||
},
|
||||
},
|
||||
{
|
||||
fmt: '/_ml/find_file_structure',
|
||||
fmt: '/_ml/find_file_structure?&explain=true',
|
||||
},
|
||||
],
|
||||
needBody: true,
|
||||
|
|
|
|||
|
|
@ -9,9 +9,13 @@ import { ResponseError, CustomHttpResponseOptions } from 'kibana/server';
|
|||
|
||||
export function wrapError(error: any): CustomHttpResponseOptions<ResponseError> {
|
||||
const boom = isBoom(error) ? error : boomify(error, { statusCode: error.status });
|
||||
const statusCode = boom.output.statusCode;
|
||||
return {
|
||||
body: boom,
|
||||
body: {
|
||||
message: boom,
|
||||
...(statusCode !== 500 && error.body ? { attributes: { body: error.body } } : {}),
|
||||
},
|
||||
headers: boom.output.headers,
|
||||
statusCode: boom.output.statusCode,
|
||||
statusCode,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,40 +4,21 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import Boom from 'boom';
|
||||
import { APICaller } from 'kibana/server';
|
||||
import { FindFileStructureResponse } from '../../../common/types/file_datavisualizer';
|
||||
import {
|
||||
AnalysisResult,
|
||||
FormattedOverrides,
|
||||
InputOverrides,
|
||||
} from '../../../common/types/file_datavisualizer';
|
||||
|
||||
export type InputData = any[];
|
||||
|
||||
export interface InputOverrides {
|
||||
[key: string]: string;
|
||||
}
|
||||
|
||||
export type FormattedOverrides = InputOverrides & {
|
||||
column_names: string[];
|
||||
has_header_row: boolean;
|
||||
should_trim_fields: boolean;
|
||||
};
|
||||
|
||||
export interface AnalysisResult {
|
||||
results: FindFileStructureResponse;
|
||||
overrides?: FormattedOverrides;
|
||||
}
|
||||
|
||||
export function fileDataVisualizerProvider(callAsCurrentUser: APICaller) {
|
||||
async function analyzeFile(data: any, overrides: any): Promise<AnalysisResult> {
|
||||
let results = [];
|
||||
|
||||
try {
|
||||
results = await callAsCurrentUser('ml.fileStructure', {
|
||||
body: data,
|
||||
...overrides,
|
||||
});
|
||||
} catch (error) {
|
||||
const err = error.message !== undefined ? error.message : error;
|
||||
throw Boom.badRequest(err);
|
||||
}
|
||||
const results = await callAsCurrentUser('ml.fileStructure', {
|
||||
body: data,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const { hasOverrides, reducedOverrides } = formatOverrides(overrides);
|
||||
|
||||
|
|
|
|||
|
|
@ -6,39 +6,24 @@
|
|||
|
||||
import { APICaller } from 'kibana/server';
|
||||
import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer';
|
||||
import {
|
||||
ImportResponse,
|
||||
ImportFailure,
|
||||
Settings,
|
||||
Mappings,
|
||||
IngestPipelineWrapper,
|
||||
} from '../../../common/types/file_datavisualizer';
|
||||
import { InputData } from './file_data_visualizer';
|
||||
|
||||
export interface Settings {
|
||||
pipeline?: string;
|
||||
index: string;
|
||||
body: any[];
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface Mappings {
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface InjectPipeline {
|
||||
id: string;
|
||||
pipeline: any;
|
||||
}
|
||||
|
||||
interface Failure {
|
||||
item: number;
|
||||
reason: string;
|
||||
doc: any;
|
||||
}
|
||||
|
||||
export function importDataProvider(callAsCurrentUser: APICaller) {
|
||||
async function importData(
|
||||
id: string,
|
||||
index: string,
|
||||
settings: Settings,
|
||||
mappings: Mappings,
|
||||
ingestPipeline: InjectPipeline,
|
||||
ingestPipeline: IngestPipelineWrapper,
|
||||
data: InputData
|
||||
) {
|
||||
): Promise<ImportResponse> {
|
||||
let createdIndex;
|
||||
let createdPipelineId;
|
||||
const docCount = data.length;
|
||||
|
|
@ -66,7 +51,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) {
|
|||
createdPipelineId = pipelineId;
|
||||
}
|
||||
|
||||
let failures: Failure[] = [];
|
||||
let failures: ImportFailure[] = [];
|
||||
if (data.length) {
|
||||
const resp = await indexData(index, createdPipelineId, data);
|
||||
if (resp.success === false) {
|
||||
|
|
@ -144,7 +129,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) {
|
|||
};
|
||||
}
|
||||
} catch (error) {
|
||||
let failures: Failure[] = [];
|
||||
let failures: ImportFailure[] = [];
|
||||
let ingestError = false;
|
||||
if (error.errors !== undefined && Array.isArray(error.items)) {
|
||||
// an expected error where some or all of the bulk request
|
||||
|
|
@ -169,7 +154,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) {
|
|||
return await callAsCurrentUser('ingest.putPipeline', { id, body: pipeline });
|
||||
}
|
||||
|
||||
function getFailures(items: any[], data: InputData): Failure[] {
|
||||
function getFailures(items: any[], data: InputData): ImportFailure[] {
|
||||
const failures = [];
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const item = items[i];
|
||||
|
|
|
|||
|
|
@ -4,11 +4,6 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export {
|
||||
fileDataVisualizerProvider,
|
||||
InputOverrides,
|
||||
InputData,
|
||||
AnalysisResult,
|
||||
} from './file_data_visualizer';
|
||||
export { fileDataVisualizerProvider, InputData } from './file_data_visualizer';
|
||||
|
||||
export { importDataProvider, Settings, InjectPipeline, Mappings } from './import_data';
|
||||
export { importDataProvider } from './import_data';
|
||||
|
|
|
|||
|
|
@ -7,15 +7,17 @@
|
|||
import { schema } from '@kbn/config-schema';
|
||||
import { RequestHandlerContext } from 'kibana/server';
|
||||
import { MAX_BYTES } from '../../common/constants/file_datavisualizer';
|
||||
import { wrapError } from '../client/error_wrapper';
|
||||
import {
|
||||
InputOverrides,
|
||||
Settings,
|
||||
IngestPipelineWrapper,
|
||||
Mappings,
|
||||
} from '../../common/types/file_datavisualizer';
|
||||
import { wrapError } from '../client/error_wrapper';
|
||||
import {
|
||||
InputData,
|
||||
fileDataVisualizerProvider,
|
||||
importDataProvider,
|
||||
Settings,
|
||||
InjectPipeline,
|
||||
Mappings,
|
||||
} from '../models/file_data_visualizer';
|
||||
|
||||
import { RouteInitialization } from '../types';
|
||||
|
|
@ -32,7 +34,7 @@ function importData(
|
|||
index: string,
|
||||
settings: Settings,
|
||||
mappings: Mappings,
|
||||
ingestPipeline: InjectPipeline,
|
||||
ingestPipeline: IngestPipelineWrapper,
|
||||
data: InputData
|
||||
) {
|
||||
const { importData: importDataFunc } = importDataProvider(context.ml!.mlClient.callAsCurrentUser);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue